Skip to content

train

birdnet_stm32.cli.train

CLI entry point for training.

AdaptiveLoaderTuner

Bases: Callback

Tune loader in-flight queue online using throughput and free RAM.

Source code in birdnet_stm32/cli/train.py
class AdaptiveLoaderTuner(tf.keras.callbacks.Callback):
    """Tune loader in-flight queue online using throughput and free RAM."""

    def __init__(
        self,
        control: dict,
        batch_size: int,
        adjust_every: int = 200,
        min_inflight: int = 128,
        max_inflight: int = 4096,
        target_free_gb: float = 8.0,
    ):
        super().__init__()
        self.control = control
        self.batch_size = batch_size
        self.adjust_every = adjust_every
        self.min_inflight = min_inflight
        self.max_inflight = max_inflight
        self.target_free_gb = target_free_gb
        self._t0 = 0.0
        self._steps = 0
        self._durations: list[float] = []
        self._prev_throughput: float | None = None
        self._direction = 1
        self._step_size = max(32, min(512, min_inflight // 2))

    def on_train_batch_begin(self, batch, logs=None):
        self._t0 = time.perf_counter()

    def on_train_batch_end(self, batch, logs=None):
        dt = time.perf_counter() - self._t0
        self._durations.append(dt)
        self._steps += 1
        if self._steps % self.adjust_every != 0:
            return

        step_sec = float(np.mean(self._durations)) if self._durations else 0.0
        self._durations.clear()
        throughput = self.batch_size / max(step_sec, 1e-6)
        _total_gb, avail_gb = _read_meminfo_gb()

        cur = int(self.control.get("max_inflight_files", self.min_inflight))
        new = cur
        reason = "hold"

        if avail_gb > 0 and avail_gb < self.target_free_gb:
            new = max(self.min_inflight, int(cur * 0.8))
            self._direction = -1
            self._step_size = max(16, self._step_size // 2)
            reason = "memory-pressure"
        elif self._prev_throughput is None:
            new = min(self.max_inflight, cur + self._step_size)
            self._direction = 1
            reason = "initial-probe"
        elif throughput >= self._prev_throughput * 1.01:
            if self._direction >= 0:
                new = min(self.max_inflight, cur + self._step_size)
                reason = "throughput-up"
            else:
                new = max(self.min_inflight, cur - self._step_size)
                reason = "throughput-up-down-dir"
        else:
            self._step_size = max(16, self._step_size // 2)
            if self._direction >= 0:
                new = max(self.min_inflight, cur - self._step_size)
                self._direction = -1
                reason = "reverse-down"
            else:
                new = min(self.max_inflight, cur + self._step_size)
                self._direction = 1
                reason = "reverse-up"

        self._prev_throughput = throughput
        if new != cur:
            self.control["max_inflight_files"] = int(new)
        self.control["last_tuning_event"] = {
            "step": int(self._steps),
            "throughput": float(throughput),
            "available_gb": float(avail_gb),
            "previous_inflight": int(cur),
            "current_inflight": int(self.control.get("max_inflight_files", cur)),
            "reason": reason,
        }

HostMemoryGuard

Bases: Callback

Abort training before host memory pressure makes the machine unusable.

Source code in birdnet_stm32/cli/train.py
class HostMemoryGuard(tf.keras.callbacks.Callback):
    """Abort training before host memory pressure makes the machine unusable."""

    def __init__(
        self,
        check_every: int = 25,
        reserve_gb: float = 12.0,
        reserve_fraction: float = 0.20,
    ):
        super().__init__()
        self.check_every = max(1, int(check_every))
        self.reserve_gb = float(reserve_gb)
        self.reserve_fraction = float(reserve_fraction)
        self._steps = 0

    def _memory_state(self) -> tuple[float, float, float]:
        total_gb, available_gb = _read_meminfo_gb()
        adaptive_reserve_gb = min(self.reserve_gb, max(2.0, total_gb * self.reserve_fraction))
        return total_gb, available_gb, adaptive_reserve_gb

    def on_train_batch_end(self, batch, logs=None):
        self._steps += 1
        if self._steps % self.check_every != 0:
            return

        _total_gb, available_gb, adaptive_reserve_gb = self._memory_state()
        if available_gb > 0 and available_gb < adaptive_reserve_gb:
            raise MemoryError(
                "Host memory guard stopped training: "
                f"{available_gb:.1f} GiB available is below the "
                f"{adaptive_reserve_gb:.1f} GiB reserve. "
                "The last completed-epoch checkpoint is still usable."
            )

    def on_epoch_end(self, epoch, logs=None):
        _total_gb, available_gb, adaptive_reserve_gb = self._memory_state()
        if available_gb > 0:
            print(f"[memory] epoch={epoch + 1} available={available_gb:.1f} GiB reserve={adaptive_reserve_gb:.1f} GiB")

filterbank_design(audio_frontend)

(raw_magnitude, raw_bank) for a model: the raw filterbank's release design for the raw frontend, and the neutral values the other frontends' configs accept (they have no raw filterbank).

Source code in birdnet_stm32/cli/train.py
def filterbank_design(audio_frontend: str) -> tuple[str, str]:
    """``(raw_magnitude, raw_bank)`` for a model: the raw filterbank's release design for the raw frontend, and the
    neutral values the other frontends' configs accept (they have no raw filterbank)."""
    if audio_frontend == "raw":
        return RELEASE_RAW_MAGNITUDE, RELEASE_RAW_BANK
    return "alpha_max", "pair"

get_args()

Parse command-line arguments for training.

Sensible defaults are chosen so that most users only need to specify --data_path_train. SpecAugment, deterministic seeding, and gradient clipping are enabled by default.

Source code in birdnet_stm32/cli/train.py
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
def get_args() -> argparse.Namespace:
    """Parse command-line arguments for training.

    Sensible defaults are chosen so that most users only need to specify
    ``--data_path_train``. SpecAugment, deterministic seeding, and gradient
    clipping are enabled by default.
    """
    parser = argparse.ArgumentParser(description="Train STM32N6 audio classifier")

    # -- Data -----------------------------------------------------------------
    parser.add_argument("--data_path_train", type=str, required=True, help="Path to train dataset")
    parser.add_argument(
        "--data_path_val",
        type=str,
        default=None,
        help="Separate validation dataset root; disables the random --val_split",
    )
    parser.add_argument(
        "--classes_file",
        type=str,
        default=None,
        help="Ordered one-label-per-line output schema; noise remains an all-zero folder",
    )
    parser.add_argument(
        "--model_config",
        type=str,
        default="",
        help=(
            "Architecture config for --qat and --linear_probe. Defaults to the "
            "checkpoint's sibling _model_config.json; pass it explicitly when fine-tuning a "
            "checkpoint that inherited its architecture from an earlier step, such as a QAT "
            "checkpoint, which writes no config of its own."
        ),
    )
    parser.add_argument("--max_classes", type=int, default=None, help="Use top N classes by sample count")
    parser.add_argument("--max_samples", type=int, default=None, help="Max samples per class")

    # -- Audio ----------------------------------------------------------------
    parser.add_argument("--sample_rate", type=int, default=24000, help="Audio sample rate (Hz)")
    parser.add_argument("--num_mels", type=int, default=128, help="Number of mel bins (2.2: 128; up to 2.1: 64)")
    parser.add_argument(
        "--spec_width",
        type=int,
        default=None,
        help=f"Frames per chunk (default per frontend: {', '.join(f'{k} {v}' for k, v in SPEC_WIDTH.items())})",
    )
    parser.add_argument("--fft_length", type=int, default=512, help="FFT length")
    parser.add_argument("--chunk_duration", type=float, default=2.5, help="Audio chunk duration (seconds)")
    parser.add_argument(
        "--max_duration",
        type=int,
        default=60,
        help=(
            "Maximum seconds to read per file. The loader still reads only the bytes it needs "
            "for the candidate chunks (smart-crop bounded by --max_chunks_per_file)."
        ),
    )
    parser.add_argument(
        "--audio_frontend",
        type=str,
        default="raw",
        choices=["raw", "hybrid", "librosa"],
        help="Audio frontend mode",
    )
    parser.add_argument(
        "--mag_scale",
        type=str,
        default="pwl",
        choices=["pwl", "none"],
        help="Magnitude scaling: learned hinge sum (pwl) or none",
    )
    parser.add_argument(
        "--input_compression",
        type=str,
        default="none",
        choices=["none", "sqrt", "log"],
        help="Compress a librosa/hybrid spectrogram input before it is quantized (computed on host/M55)",
    )

    # -- Model architecture ---------------------------------------------------
    parser.add_argument(
        "--embeddings_size",
        type=int,
        default=1280,
        help="Embeddings layer size (2.2: 1280, N6-B0's top conv; up to 2.1: 1024)",
    )
    parser.add_argument("--alpha", type=float, default=1.5, help="Width multiplier")
    parser.add_argument("--depth_multiplier", type=int, default=1, help="Depth multiplier")
    parser.add_argument(
        "--dw_kernel_size",
        type=int,
        default=5,
        choices=list(DW_KERNEL_SIZES),
        help="Depthwise kernel size in stages 2-4 (stage 1 stays 3x3)",
    )
    parser.add_argument(
        "--stage_widths",
        type=int,
        nargs=4,
        default=list(STAGE_WIDTHS),
        metavar="W",
        help="Base output channels of the four stages, before --alpha",
    )
    parser.add_argument(
        "--backbone",
        type=str,
        default="efficientnet_b0_n6w",
        choices=list(BACKBONES),
        help=(
            "Backbone behind the frontend: efficientnet_b0_n6w (2.3: N6-B0 with wider bottlenecks), efficientnet_b0_n6 "
            "(2.2: EfficientNet-B0 with ReLU6, no squeeze-and-excite and a bounded residual stream) or dscnn "
            "(releases up to 2.1; --alpha, --depth_multiplier, --dw_kernel_size and --stage_widths apply to it only)"
        ),
    )
    parser.add_argument(
        "--b0_blocks",
        default=None,
        help=(
            "Stage table for the EfficientNet backbones instead of their own: 'e,k,r,c,s/e,k,r,c,s/...' (expand ratio, "
            "depthwise kernel, repeats, output channels, first stride per stage), e.g. N6-B0 at half width or depth "
            "for the 2.5 size study"
        ),
    )
    parser.add_argument(
        "--attention_stages",
        type=int,
        nargs="*",
        default=[],
        help="Stages of the EfficientNet backbone (1-based) followed by a channel-attention block (EdgeNeXt's "
        "transposed attention); none by default",
    )
    parser.add_argument(
        "--raw_exposure_gain",
        type=float,
        default=None,
        help=(
            f"Gain of the raw filterbank's second, clamped exposure, mixed per band into a knee compressor "
            f"(default {RAW_EXPOSURE_GAIN:g} for the raw frontend; 1 = one exposure)"
        ),
    )

    # -- Augmentation ---------------------------------------------------------
    parser.add_argument("--no_spec_augment", action="store_true", default=False, help="Disable SpecAugment")
    parser.add_argument("--freq_mask_max", type=int, default=8, help="Max frequency mask width (bins)")
    parser.add_argument("--time_mask_max", type=int, default=25, help="Max time mask width (frames)")
    parser.add_argument(
        "--crop_policy",
        type=str,
        default=None,
        choices=["energy", "teacher"],
        help=(
            "How training chunks are chosen from a recording. 'energy' ranks candidates by "
            "short-time energy and keeps the loudest; 'teacher' (needs --teacher_cache) keeps "
            "the chunk where the teacher hears the labelled species most, and falls back to "
            "energy where it has no score. Default: teacher with --teacher_cache, otherwise "
            "energy. Evaluation is unaffected: it always scores whole files with overlapping windows."
        ),
    )
    parser.add_argument(
        "--label_sidecar",
        type=str,
        default=None,
        help=(
            "CSV of per-file label additions for training (sample_id, positives, negatives; "
            "classes separated by ';'): positives join the folder's class in the hard label "
            "(multi-species recordings); negatives are confirmed absent, held at 0 and never "
            "raised by the teacher's soft target. See birdnet_stm32.data.sidecar."
        ),
    )
    parser.add_argument(
        "--teacher_cache",
        type=str,
        default=None,
        help=(
            "Directory of cached per-window teacher scores (see birdnet_stm32.data.teacher). "
            "Each training chunk's label is blended with the teacher window nearest it. "
            "Validation and checkpoint selection keep hard labels."
        ),
    )
    parser.add_argument(
        "--teacher_weight",
        type=float,
        default=None,
        help=(
            f"Teacher share of the training target in [0, 1] (default {TEACHER_WEIGHT:g} with "
            "--teacher_cache, otherwise 0 = hard labels only)"
        ),
    )
    parser.add_argument(
        "--teacher_embedding_weight",
        type=float,
        default=None,
        help=(
            "Weight of a cosine loss between the student's embedding (through a training-only "
            "projector) and the teacher's embedding of the same window. Default: "
            f"{TEACHER_EMBEDDING_WEIGHT} when --teacher_cache holds emb.npy, otherwise off. "
            "0 disables it. The deployed model is unchanged."
        ),
    )
    parser.add_argument(
        "--aux_table",
        default="",
        help=(
            "Folder of location/date targets per recording (python -m birdnet_stm32.data.aux_targets): a "
            "training-only linear head on the embedding predicts where and when each recording was made "
            "(needs teacher embeddings; the deployed model is unchanged)"
        ),
    )
    parser.add_argument(
        "--aux_weight",
        type=float,
        default=AUX_WEIGHT,
        help=f"Weight of the location/date head's loss with --aux_table (default {AUX_WEIGHT})",
    )
    parser.add_argument(
        "--distill_from",
        default=None,
        help=(
            "A trained .keras model to distil from online (2.5): run frozen on every training batch, its class "
            "probabilities add a Bernoulli KL term to the student's loss. It must take the student's input and "
            "score the same classes in the same order. Float training only."
        ),
    )
    parser.add_argument(
        "--distill_weight",
        type=float,
        default=None,
        help="Weight of the --distill_from KL term against the supervised loss (default 1.0)",
    )
    parser.add_argument("--mixup_alpha", type=float, default=0.2, help="Mixup alpha")
    parser.add_argument("--mixup_probability", type=float, default=0.25, help="Mixup batch fraction")
    parser.add_argument(
        "--mixup_cooccurrence",
        default="",
        help="Optional [C, C] .npy (1 = the two classes can occur together, in labels-file order): mixup partners "
        "only among batch samples whose primary classes can co-occur",
    )

    # -- Training -------------------------------------------------------------
    parser.add_argument("--batch_size", type=int, default=32, help="Batch size")
    parser.add_argument("--num_workers", type=int, default=8, help="Parallel data loading workers (0 = sequential)")
    parser.add_argument(
        "--max_chunks_per_file",
        type=int,
        default=1,
        help="Max salient chunks to extract per file open (reduces redundant I/O for long recordings)",
    )
    parser.add_argument(
        "--prefetch_batches",
        type=int,
        default=2,
        help="Loader prefetch queue depth in batches (higher = faster, but more RAM)",
    )
    parser.add_argument(
        "--epochs",
        type=int,
        default=None,
        help=f"Number of epochs (default: {TRAIN_EPOCHS}; {QAT_EPOCHS} with --qat)",
    )
    parser.add_argument(
        "--class_cap_per_epoch",
        type=int,
        default=CLASS_CAP_PER_EPOCH,
        help=(
            f"At most this many training files per class (folder) in each epoch (default {CLASS_CAP_PER_EPOCH}; "
            "0 = off). A larger "
            "class contributes a different subset each epoch, rotating through all its files, so the draw is "
            "more balanced without repeating minority files or discarding majority ones. With "
            "--steps_per_epoch 0 an epoch is one such capped pass."
        ),
    )
    parser.add_argument(
        "--warmup_epochs",
        type=float,
        default=TRAIN_WARMUP_EPOCHS,
        help=(
            f"Linear learning-rate warm-up before the cosine decay, in epochs (default {TRAIN_WARMUP_EPOCHS:g}; "
            "may be fractional, e.g. 0.5 when one epoch is a pass over a large dataset)."
        ),
    )
    parser.add_argument(
        "--steps_per_epoch",
        type=int,
        default=0,
        help=(
            "Training steps per epoch (default 0: one pass over the training files). Fixing it keeps a "
            "schedule (warm-up, cosine decay, validation cadence) the same when the dataset grows."
        ),
    )
    parser.add_argument(
        "--learning_rate",
        type=float,
        default=None,
        help=(
            f"Initial learning rate (default: {TRAIN_LEARNING_RATE:g}; {QAT_LEARNING_RATE:g} with --qat, "
            f"{PROBE_LEARNING_RATE:g} with --linear_probe)"
        ),
    )
    parser.add_argument("--dropout", type=float, default=0.5, help="Dropout rate before classifier head")
    parser.add_argument("--optimizer", type=str, default="adam", choices=["adam", "sgd", "adamw"], help="Optimizer")
    parser.add_argument("--weight_decay", type=float, default=0.0, help="Weight decay (adamw only)")
    parser.add_argument("--val_split", type=float, default=0.2, help="Validation split ratio")
    parser.add_argument(
        "--checkpoint_path", type=str, default="checkpoints/best_model.keras", help="Output checkpoint path (.keras)"
    )
    parser.add_argument("--grad_clip", type=float, default=1.0, help="Max gradient norm for clipping (0 = disabled)")
    parser.add_argument(
        "--field_table",
        type=str,
        default=None,
        help="CSV sample_id,site,recording of field clips: the class cap draws them for diversity (see --field_share)",
    )
    parser.add_argument(
        "--field_share", type=float, default=0.5, help="Max share of a class's per-epoch cap filled by field clips"
    )
    parser.add_argument(
        "--field_per_recording", type=int, default=2, help="Max segments of one field recording per class and epoch"
    )
    parser.add_argument("--resume", action="store_true", default=False, help="Resume training from checkpoint")
    parser.add_argument(
        "--init_checkpoint",
        type=str,
        default="",
        help="Start from this model's weights at epoch 0; its frontend must match the architecture flags",
    )
    parser.add_argument("--seed", type=int, default=42, help="Random seed for deterministic training")

    parser.add_argument(
        "--validation_overlap",
        type=float,
        default=None,
        help="File validation overlap in seconds (default: half the model chunk duration).",
    )
    parser.add_argument(
        "--validation_pooling",
        choices=["max", "avg", "lme"],
        default="max",
        help="File validation pooling for exact cMAP checkpoint selection.",
    )
    parser.add_argument(
        "--validation_subset",
        type=int,
        default=0,
        help=(
            "Score checkpoint selection (standard training and QAT) on a fixed stratified subset of "
            "this many validation files instead of all of them. Classes are drawn round-robin with "
            "seed 1234, so the subset is as class-balanced as the folders allow and identical across "
            "runs and epochs. Subset cMAP is biased upward and not comparable to full-manifest "
            "numbers. 0 uses all."
        ),
    )

    # -- QAT ------------------------------------------------------------------
    parser.add_argument(
        "--qat",
        action="store_true",
        default=False,
        help="Quantization-aware fine-tuning (requires pretrained --checkpoint_path)",
    )
    parser.add_argument(
        "--qat_calibration_samples",
        type=int,
        default=1024,
        help="Exact stratified samples used for QAT ranges and final INT8 calibration",
    )
    parser.add_argument(
        "--qat_distillation_weight",
        type=float,
        default=1.0,
        help="QAT teacher Bernoulli-KL loss weight",
    )
    parser.add_argument(
        "--qat_cosine_weight",
        type=float,
        default=0.10,
        help="QAT mean teacher/student cosine-loss weight",
    )
    parser.add_argument(
        "--qat_cosine_tail_weight",
        type=float,
        default=0.75,
        help="QAT worst-sample teacher/student cosine-loss weight",
    )
    parser.add_argument(
        "--qat_cosine_tail_fraction",
        type=float,
        default=0.10,
        help="Fraction of each QAT batch included in the worst-sample loss",
    )
    parser.add_argument(
        "--qat_range_refresh",
        action=argparse.BooleanOptionalAction,
        default=True,
        help="Recalibrate QAT activation ranges on the current weights after every epoch",
    )
    # -- Linear probing -------------------------------------------------------
    parser.add_argument(
        "--linear_probe",
        action="store_true",
        default=False,
        help="Freeze backbone and train only the classifier head (requires pretrained --checkpoint_path)",
    )

    args = parser.parse_args()

    # Derive positive flags from --no_* flags
    args.spec_augment = not args.no_spec_augment
    frontend = normalize_frontend_name(args.audio_frontend)
    if args.spec_width is None:
        args.spec_width = SPEC_WIDTH[frontend]
    if args.raw_exposure_gain is None:
        args.raw_exposure_gain = RAW_EXPOSURE_GAIN if frontend == "raw" else 1.0
    if args.teacher_weight is None:
        args.teacher_weight = TEACHER_WEIGHT if args.teacher_cache else 0.0
    if args.crop_policy is None:
        args.crop_policy = "teacher" if args.teacher_cache else "energy"
    if not 0.0 <= args.teacher_weight <= 1.0:
        parser.error(f"--teacher_weight must be in [0, 1], got {args.teacher_weight}")
    if args.teacher_weight > 0 and not args.teacher_cache:
        parser.error("--teacher_weight > 0 needs --teacher_cache")
    has_embeddings = bool(args.teacher_cache) and os.path.isfile(os.path.join(args.teacher_cache, "emb.npy"))
    if args.teacher_embedding_weight is None:
        # On wherever the cache can feed it: float training with teacher embeddings.
        float_training = not (args.qat or args.linear_probe)
        args.teacher_embedding_weight = TEACHER_EMBEDDING_WEIGHT if has_embeddings and float_training else 0.0
    if args.teacher_embedding_weight < 0:
        parser.error(f"--teacher_embedding_weight must be >= 0, got {args.teacher_embedding_weight}")
    if args.teacher_embedding_weight > 0 and not has_embeddings:
        parser.error("--teacher_embedding_weight > 0 needs a --teacher_cache that holds emb.npy")
    if args.field_table and args.class_cap_per_epoch <= 0:
        parser.error("--field_table shapes the per-epoch class cap: it needs --class_cap_per_epoch > 0")
    if args.teacher_embedding_weight > 0 and (args.qat or args.linear_probe):
        parser.error("--teacher_embedding_weight applies to float training only (not --qat or --linear_probe)")
    if args.aux_table:
        if args.teacher_embedding_weight <= 0:
            parser.error("--aux_table trains in the teacher-embedding wrapper: it needs teacher embeddings")
        if not os.path.isfile(os.path.join(args.aux_table, "keys.npy")):
            parser.error(
                f"--aux_table {args.aux_table} holds no keys.npy (build it with birdnet_stm32.data.aux_targets)"
            )
        if args.aux_weight <= 0:
            parser.error(f"--aux_weight must be > 0 with --aux_table, got {args.aux_weight}")
    if (args.b0_blocks or args.attention_stages) and args.backbone == "dscnn":
        parser.error("--b0_blocks and --attention_stages apply to the EfficientNet backbones only")
    if args.b0_blocks:
        from birdnet_stm32.models.efficientnet import parse_blocks

        try:
            args.b0_blocks = [list(b) for b in parse_blocks(args.b0_blocks)]
        except ValueError as exc:
            parser.error(f"--b0_blocks: {exc}")
    else:
        args.b0_blocks = []
    if args.distill_weight is not None and not args.distill_from:
        parser.error("--distill_weight needs --distill_from")
    if args.distill_from:
        if args.qat or args.linear_probe:
            parser.error("--distill_from applies to float training only (QAT distils from its own float model)")
        if not os.path.isfile(args.distill_from):
            parser.error(f"--distill_from: no such file {args.distill_from}")
        if args.distill_weight is None:
            from birdnet_stm32.training.distillation import DEFAULT_ONLINE_DISTILLATION_WEIGHT

            args.distill_weight = DEFAULT_ONLINE_DISTILLATION_WEIGHT
        if args.distill_weight <= 0:
            parser.error(f"--distill_weight must be > 0, got {args.distill_weight}")
    if args.crop_policy == "teacher" and not args.teacher_cache:
        parser.error("--crop_policy teacher needs --teacher_cache")
    args.deterministic = True  # always deterministic
    # Float training on a GPU runs XLA-compiled in mixed precision (~3x faster on
    # the raw 448 model). QAT simulates the INT8 grid in float32, uncompiled.
    float_training = not (args.qat or args.linear_probe)
    has_gpu = bool(tf.config.list_physical_devices("GPU"))
    args.mixed_precision = args.jit_compile = float_training and has_gpu

    if args.validation_subset < 0:
        parser.error("--validation_subset must be non-negative")

    # The compression steps run one at a time against a converged checkpoint;
    # the documented order is QAT, then convert.
    exclusive = [name for name in ("qat", "linear_probe") if getattr(args, name)]
    if len(exclusive) > 1:
        raise SystemExit(f"Options are mutually exclusive, run them as separate steps: {exclusive}")

    # Each step has its own best schedule: fine-tuning at the training rate
    # destroys a converged model (measured for QAT at 2e-4).
    if args.epochs is None:
        args.epochs = QAT_EPOCHS if args.qat else TRAIN_EPOCHS
    if args.learning_rate is None:
        args.learning_rate = (
            QAT_LEARNING_RATE if args.qat else PROBE_LEARNING_RATE if args.linear_probe else TRAIN_LEARNING_RATE
        )

    return args

main()

Train a DS-CNN model on a class-structured audio dataset.

Source code in birdnet_stm32/cli/train.py
 709
 710
 711
 712
 713
 714
 715
 716
 717
 718
 719
 720
 721
 722
 723
 724
 725
 726
 727
 728
 729
 730
 731
 732
 733
 734
 735
 736
 737
 738
 739
 740
 741
 742
 743
 744
 745
 746
 747
 748
 749
 750
 751
 752
 753
 754
 755
 756
 757
 758
 759
 760
 761
 762
 763
 764
 765
 766
 767
 768
 769
 770
 771
 772
 773
 774
 775
 776
 777
 778
 779
 780
 781
 782
 783
 784
 785
 786
 787
 788
 789
 790
 791
 792
 793
 794
 795
 796
 797
 798
 799
 800
 801
 802
 803
 804
 805
 806
 807
 808
 809
 810
 811
 812
 813
 814
 815
 816
 817
 818
 819
 820
 821
 822
 823
 824
 825
 826
 827
 828
 829
 830
 831
 832
 833
 834
 835
 836
 837
 838
 839
 840
 841
 842
 843
 844
 845
 846
 847
 848
 849
 850
 851
 852
 853
 854
 855
 856
 857
 858
 859
 860
 861
 862
 863
 864
 865
 866
 867
 868
 869
 870
 871
 872
 873
 874
 875
 876
 877
 878
 879
 880
 881
 882
 883
 884
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
def main():
    """Train a DS-CNN model on a class-structured audio dataset."""
    args = get_args()
    args.audio_frontend = normalize_frontend_name(args.audio_frontend)

    # Enable dynamic GPU memory growth so we don't reserve all VRAM
    for gpu in tf.config.list_physical_devices("GPU"):
        tf.config.experimental.set_memory_growth(gpu, True)

    # Deterministic mode: seed all RNGs and enable TF deterministic ops
    if args.deterministic:
        seed = args.seed
        random.seed(seed)
        np.random.seed(seed)
        tf.random.set_seed(seed)
        os.environ["PYTHONHASHSEED"] = str(seed)
        if args.jit_compile:
            # Keras turns XLA off whenever TF op determinism is on, so XLA keeps the
            # seeds but not the deterministic kernels (GPU sums may differ in the last bits).
            print(f"Seeded (seed={seed}); deterministic TF ops off: they would disable XLA.")
        else:
            os.environ["TF_DETERMINISTIC_OPS"] = "1"
            print(f"Deterministic mode enabled (seed={seed}).")

    # Early warning for STM32N6 raw frontend constraint
    if args.audio_frontend == "raw":
        T = int(args.sample_rate * args.chunk_duration)
        if T >= (1 << 16):
            print(f"[WARN] STM32N6 constraint: raw input length {T} >= 65536.")
            print("       Use --sample_rate 16000 or --chunk_duration 2, or --audio_frontend hybrid.")

    # Mixed precision
    if args.mixed_precision:
        tf.keras.mixed_precision.set_global_policy("mixed_float16")
        print("Mixed precision enabled (float16 compute, float32 accumulation).")

    # Quantization-aware fine-tuning
    if args.qat:
        from birdnet_stm32.training.qat import run_qat

        run_qat(args)
        return

    # Linear probing: freeze backbone, retrain head only
    if args.linear_probe:
        from birdnet_stm32.training.linear_probe import run_linear_probe

        run_linear_probe(args)
        return

    hop_length = compute_hop_length(args.sample_rate, args.chunk_duration, args.spec_width)

    # Load file paths
    top_classes = load_classes_file(args.classes_file) if args.classes_file else None
    if top_classes is not None and args.max_classes is not None:
        raise ValueError("--classes_file and --max_classes are mutually exclusive")
    if args.max_classes is not None:
        top_classes = get_classes_with_most_samples(args.data_path_train, n_classes=args.max_classes)
        print(f"Selected top {len(top_classes)} classes by sample count.")
    file_paths, classes = load_file_paths_from_directory(
        args.data_path_train, classes=top_classes, max_samples=args.max_samples
    )
    if top_classes is not None and classes != top_classes:
        missing = [class_name for class_name in top_classes if class_name not in classes]
        raise ValueError(f"Training dataset is missing configured classes: {missing}")

    # Train/val split
    if args.data_path_val:
        train_paths = file_paths
        val_paths, val_classes = load_file_paths_from_directory(
            args.data_path_val, classes=classes, max_samples=args.max_samples
        )
        if val_classes != classes:
            missing = [class_name for class_name in classes if class_name not in val_classes]
            raise ValueError(f"Validation dataset is missing configured classes: {missing}")
        print("Using separate validation root; --val_split is ignored.")
    else:
        split_idx = int(len(file_paths) * (1 - args.val_split))
        train_paths = file_paths[:split_idx]
        val_paths = file_paths[split_idx:]
    print(f"Training on {len(train_paths)} files, validating on {len(val_paths)} files.")

    # Datasets
    common_kwargs = dict(
        sample_rate=args.sample_rate,
        max_duration=args.max_duration,
        chunk_duration=args.chunk_duration,
        spec_width=args.spec_width,
        mel_bins=args.num_mels,
        fft_length=args.fft_length,
        mag_scale=args.mag_scale,
        input_compression=args.input_compression,
        prefetch_batches=args.prefetch_batches,
    )

    initial_inflight = max(256, args.num_workers * 64)
    common_kwargs["max_inflight_files"] = initial_inflight

    train_loader_control: dict | None = {"max_inflight_files": int(initial_inflight)} if args.num_workers > 0 else None
    extra_callbacks: list[tf.keras.callbacks.Callback] = [HostMemoryGuard()]
    if train_loader_control is not None:
        extra_callbacks.append(
            AdaptiveLoaderTuner(
                control=train_loader_control,
                batch_size=args.batch_size,
                adjust_every=_LOADER_TUNE_ADJUST_EVERY,
                min_inflight=max(64, args.num_workers * 8),
                max_inflight=max(512, args.num_workers * 256),
                target_free_gb=_LOADER_TARGET_FREE_GB,
            )
        )
        print(f"Loader auto-tuning enabled (initial max_inflight_files={initial_inflight}).")

    train_kwargs = dict(common_kwargs)
    field_kwargs: dict = {}
    if args.field_table:
        from birdnet_stm32.data.sidecar import load_field_table

        field_kwargs = dict(
            field=load_field_table(args.field_table),
            field_share=args.field_share,
            per_recording=args.field_per_recording,
        )
        print(
            f"Field table: {len(field_kwargs['field'])} field clips, share {args.field_share}, {args.field_per_recording} per recording"
        )
    if args.class_cap_per_epoch > 0:
        train_kwargs["class_cap_per_epoch"] = args.class_cap_per_epoch
        if field_kwargs:
            train_kwargs.update(
                field_table=field_kwargs["field"],
                field_share=args.field_share,
                field_per_recording=args.field_per_recording,
            )
        print(
            f"Per-epoch class cap {args.class_cap_per_epoch}: {len(ClassCappedPasses(train_paths, args.class_cap_per_epoch, **field_kwargs))} "
            f"of {len(train_paths)} training files per epoch"
        )
    if train_loader_control is not None:
        train_kwargs["loader_control"] = train_loader_control

    val_kwargs = dict(common_kwargs)
    embed = args.teacher_embedding_weight > 0
    if args.teacher_cache and (args.teacher_weight > 0 or args.crop_policy == "teacher" or embed):
        # Fail here, in the main process, rather than inside every loader worker.
        from pathlib import Path

        from birdnet_stm32.data.teacher import TeacherTargets

        teacher = TeacherTargets(args.teacher_cache, classes, embeddings=embed)
        covered = sum(Path(p).stem in teacher for p in train_paths)
        print(
            f"Teacher targets: {covered}/{len(train_paths)} training files covered, "
            f"{int(teacher.mask.sum())}/{len(classes)} classes, weight {args.teacher_weight}, "
            f"crop policy {args.crop_policy}"
            + (f", embeddings {teacher.embedding_dim}-d at weight {args.teacher_embedding_weight}" if embed else "")
        )
    # How a training chunk is drawn and labelled; shared with the projector fit below.
    chunk_kwargs = dict(
        audio_frontend=args.audio_frontend,
        num_workers=args.num_workers,
        max_chunks_per_file=args.max_chunks_per_file,
        random_offset=True,
        snr_threshold=0.1,
        spec_augment=args.spec_augment,
        freq_mask_max=args.freq_mask_max,
        time_mask_max=args.time_mask_max,
        crop_policy=args.crop_policy,
        teacher_cache=args.teacher_cache,
        teacher_weight=args.teacher_weight,
        teacher_background=args.teacher_weight > 0,  # the background-bird rule (2.3) wherever teacher targets are used
        teacher_embeddings=embed,
        aux_table=args.aux_table or None,
    )
    if args.aux_table:
        from pathlib import Path

        from birdnet_stm32.data.aux_targets import AuxTargets

        aux_targets = AuxTargets(args.aux_table)
        covered = sum(Path(p).stem in aux_targets for p in train_paths)
        print(
            f"Location/date head: {covered}/{len(train_paths)} training files in {args.aux_table}, weight {args.aux_weight}"
        )
    if args.label_sidecar:
        from birdnet_stm32.data.sidecar import load_label_sidecar

        chunk_kwargs["label_sidecar"] = load_label_sidecar(args.label_sidecar, classes)
        n_pos = sum(bool(p) for p, _ in chunk_kwargs["label_sidecar"].values())
        n_neg = sum(bool(n) for _, n in chunk_kwargs["label_sidecar"].values())
        print(f"Label sidecar: {n_pos} files with extra positives, {n_neg} with confirmed absences")
    train_dataset = load_dataset(
        train_paths,
        classes,
        batch_size=args.batch_size,
        mixup_alpha=args.mixup_alpha,
        mixup_probability=args.mixup_probability,
        mixup_cooccurrence=np.load(args.mixup_cooccurrence) if args.mixup_cooccurrence else None,
        **chunk_kwargs,
        **train_kwargs,
    )
    from birdnet_stm32.training.validation import VALIDATION_SUBSET_SEED, FileCmap, stratified_validation_subset

    selection_paths = stratified_validation_subset(val_paths, args.validation_subset)
    if len(selection_paths) < len(val_paths):
        print(
            f"Selecting checkpoints on a fixed stratified subset of {len(selection_paths)} "
            f"of {len(val_paths)} validation files (seed {VALIDATION_SUBSET_SEED})"
        )
    # Keras' own per-epoch validation pass (val_loss and the chunk metrics in the
    # history) reads the same files as selection. Over the full manifest it
    # re-decoded every validation file each epoch for numbers that select nothing.
    val_dataset = load_dataset(
        selection_paths,
        classes,
        audio_frontend=args.audio_frontend,
        batch_size=args.batch_size,
        num_workers=args.num_workers,
        max_chunks_per_file=1,
        mixup_alpha=0.0,
        mixup_probability=0.0,
        random_offset=False,
        snr_threshold=0.5,
        spec_augment=False,
        **val_kwargs,
    )

    epoch_files = (
        len(ClassCappedPasses(train_paths, args.class_cap_per_epoch, **field_kwargs))
        if args.class_cap_per_epoch > 0
        else len(train_paths)
    )
    steps_per_epoch = args.steps_per_epoch or max(
        1, math.ceil(estimate_samples_per_epoch(epoch_files, args.max_chunks_per_file) / float(args.batch_size))
    )
    val_steps = max(1, math.ceil(len(selection_paths) / float(args.batch_size)))

    raw_magnitude, raw_bank = filterbank_design(args.audio_frontend)

    # Build model
    print("Building model...")
    model = build_dscnn_model(
        num_mels=args.num_mels,
        spec_width=args.spec_width,
        sample_rate=args.sample_rate,
        chunk_duration=args.chunk_duration,
        audio_frontend=args.audio_frontend,
        num_classes=len(classes),
        alpha=args.alpha,
        depth_multiplier=args.depth_multiplier,
        dw_kernel_size=args.dw_kernel_size,
        stage_widths=args.stage_widths,
        backbone=args.backbone,
        b0_blocks=args.b0_blocks,
        attention_stages=args.attention_stages,
        embeddings_size=args.embeddings_size,
        fft_length=args.fft_length,
        mag_scale=args.mag_scale,
        raw_magnitude=raw_magnitude,
        raw_bank=raw_bank,
        raw_exposure_gain=args.raw_exposure_gain,
        raw_exposure_mode="compress",
        dropout_rate=args.dropout,
    )
    # Per-layer MACs and N6 compatibility, rather than a plain Keras summary:
    # on this target the MAC budget and op support decide whether the model is
    # deployable at all.
    print_profile(model)

    # Save model config
    cfg = ModelConfig(
        sample_rate=args.sample_rate,
        num_mels=args.num_mels,
        spec_width=args.spec_width,
        fft_length=args.fft_length,
        chunk_duration=args.chunk_duration,
        hop_length=hop_length,
        audio_frontend=args.audio_frontend,
        mag_scale=args.mag_scale,
        input_compression=args.input_compression,
        raw_magnitude=raw_magnitude,
        raw_bank=raw_bank,
        embeddings_size=args.embeddings_size,
        alpha=args.alpha,
        depth_multiplier=args.depth_multiplier,
        dw_kernel_size=args.dw_kernel_size,
        stage_widths=list(args.stage_widths),
        backbone=args.backbone,
        b0_blocks=args.b0_blocks,
        attention_stages=list(args.attention_stages),
        num_classes=len(classes),
        class_names=classes,
        raw_exposure_gain=args.raw_exposure_gain,
        raw_exposure_mode="compress",
        dropout_rate=args.dropout,
    )
    if args.init_checkpoint:
        if args.resume:
            raise SystemExit("--init_checkpoint starts a new run; it cannot be combined with --resume")
        from birdnet_stm32.models.runners import load_keras_model

        start = load_keras_model(args.init_checkpoint)
        got, want = start.get_layer("audio_frontend"), model.get_layer("audio_frontend")
        for key in ("mode", "raw_exposure_gain", "raw_exposure_mode", "mel_bins", "spec_width"):
            if getattr(got, key) != getattr(want, key):
                raise SystemExit(
                    f"--init_checkpoint {key}={getattr(got, key)!r} but the flags say {getattr(want, key)!r}"
                )
        if start.count_params() != model.count_params():
            raise SystemExit("--init_checkpoint does not match the architecture flags")
        model = start
        print(f"Starting from {args.init_checkpoint} (epoch 0)")
    elif args.raw_exposure_gain > 1.0 and not args.resume:
        # The low exposure must span +-1 per band before training: the clamps
        # are in the graph from the first step, and the filterbank is frozen, so
        # this normalization holds for the whole run.
        from birdnet_stm32.conversion.equalize import normalize_exposure_bank
        from birdnet_stm32.conversion.quantize import representative_data_gen, stratified_sample_paths

        norm_paths = stratified_sample_paths(train_paths, _EXPOSURE_NORM_SAMPLES, seed=42)
        tensors = [item[0] for item in representative_data_gen(norm_paths, cfg.to_dict(), num_samples=len(norm_paths))]
        print(f"[exposure] normalized on {len(tensors)} windows: {normalize_exposure_bank(model, tensors)}")
    cfg_path = os.path.splitext(args.checkpoint_path)[0] + "_model_config.json"
    cfg.save(cfg_path)
    print(f"Saved model config to '{cfg_path}'")

    # Write the labels before training starts. The class list is already fixed
    # here, and the checkpoint callback can leave a usable .keras behind at any
    # epoch, so the contract files must not depend on the run reaching its end:
    # an interrupt or a crash would otherwise strand a checkpoint with no labels.
    labels_file = args.checkpoint_path.replace(".keras", "_labels.txt")
    with open(labels_file, "w") as f:
        for cls in classes:
            f.write(f"{cls}\n")
    print(f"Saved labels to '{labels_file}'")

    extra_callbacks.append(
        FileCmap(
            selection_paths,
            classes,
            cfg.to_dict(),
            overlap=args.validation_overlap,
            pooling=args.validation_pooling,
            batch_size=args.batch_size,
        )
    )

    training_wrapper = None
    distill_teacher = None
    if args.distill_from:
        from birdnet_stm32.training.trainer import load_for_resume

        # Its saved dtype policy, so the teacher runs in mixed precision beside the student.
        distill_teacher = load_for_resume(args.distill_from)
        teacher_labels = os.path.splitext(args.distill_from)[0] + "_labels.txt"
        if os.path.isfile(teacher_labels):
            with open(teacher_labels) as f:
                if [line.strip() for line in f if line.strip()] != list(classes):
                    raise SystemExit(f"--distill_from scores other classes than this run ({teacher_labels})")
        print(
            f"Distilling online from {args.distill_from} ({distill_teacher.count_params():,} parameters), "
            f"KL weight {args.distill_weight}"
        )

        def training_wrapper(student):
            from birdnet_stm32.training.distillation import OnlineDistilledModel

            return OnlineDistilledModel(student, distill_teacher, args.distill_weight)

    if embed:
        from birdnet_stm32.training.embedding_distillation import (
            EmbeddingDistilledModel,
            fit_aux_head,
            fit_projector,
        )

        # From trained weights, a random projector would open with the largest
        # possible loss and push its gradients into a converged backbone; fit it
        # to the student as it is instead. From scratch there is nothing to fit.
        from_trained = bool(args.init_checkpoint) or (args.resume and os.path.isfile(args.checkpoint_path))

        def training_wrapper(student):
            wrapped = EmbeddingDistilledModel(
                student,
                teacher.embedding_dim,
                args.teacher_embedding_weight,
                output_teacher=distill_teacher,
                output_weight=args.distill_weight or 0.0,
                aux_weight=args.aux_weight if args.aux_table else 0.0,
            )
            if from_trained:
                rng = random.Random(args.seed)
                files = rng.sample(train_paths, min(_PROJECTOR_FIT_FILES, len(train_paths)))
                chunks = sample_chunks(files, classes, **chunk_kwargs, **common_kwargs)
                batches = [
                    (
                        np.stack([c[0] for c in chunks[i : i + 64]]),
                        (
                            None,
                            np.stack([c[2] for c in chunks[i : i + 64]]),
                            np.stack([c[3] for c in chunks[i : i + 64]]),
                        ),
                    )
                    for i in range(0, len(chunks), 64)
                ]
                fit = fit_projector(wrapped, batches)
                print(f"[teacher embeddings] projector fitted by ridge regression on {len(files)} files: {fit}")
                if args.aux_table:
                    aux_batches = [
                        (
                            np.stack([c[0] for c in chunks[i : i + 64]]),
                            np.stack([c[4] for c in chunks[i : i + 64]]),
                            np.stack([c[5] for c in chunks[i : i + 64]]),
                        )
                        for i in range(0, len(chunks), 64)
                    ]
                    print(f"[location/date head] fitted by ridge regression: {fit_aux_head(wrapped, aux_batches)}")
            return wrapped

    def _arm_raw_spec_augment(m: tf.keras.Model) -> None:
        # The raw frontend computes the spectrogram in the model, out of the loader's reach (which
        # applies SpecAugment for librosa/hybrid only), so float training masks the frontend output in
        # the graph instead. Not in the saved config: finetune-head, QAT and conversion run without it.
        if args.audio_frontend == "raw" and args.spec_augment and not (args.qat or args.linear_probe):
            m.get_layer("audio_frontend").train_spec_augment = (args.freq_mask_max, args.time_mask_max)
            print(
                f"SpecAugment on the raw frontend output (freq <= {args.freq_mask_max}, time <= {args.time_mask_max})"
            )

    _arm_raw_spec_augment(model)

    # Train
    print("Starting training...")
    try:
        train_model(
            model,
            train_dataset,
            val_dataset,
            epochs=args.epochs,
            learning_rate=args.learning_rate,
            batch_size=args.batch_size,
            checkpoint_path=args.checkpoint_path,
            steps_per_epoch=steps_per_epoch,
            val_steps=val_steps,
            optimizer=args.optimizer,
            weight_decay=args.weight_decay,
            gradient_clip_norm=args.grad_clip,
            resume=args.resume,
            prepare_model=_arm_raw_spec_augment,
            extra_callbacks=extra_callbacks,
            # A run from --init_checkpoint starts near a trained model: its first
            # epoch, still at warm-up learning rates, validates high, and the
            # full-rate epochs after it take a while to pass it. Selecting and
            # early-stopping from epoch 1 kept that barely adapted checkpoint once
            # (a 20-epoch fine-tune stopped at epoch 11 and shipped epoch 1).
            checkpoint_start_epoch=_WARMUP_EPOCHS if args.init_checkpoint else 0,
            training_wrapper=training_wrapper,
            warmup_epochs=args.warmup_epochs,
            jit_compile=args.jit_compile,
        )
        print(f"Training complete. Best model saved to '{args.checkpoint_path}'.")
    except KeyboardInterrupt:
        print(f"\nTraining interrupted. Best checkpoint so far: '{args.checkpoint_path}'")