diff --git a/scripts/training/zhizhi_v2/ds_config.json b/scripts/training/zhizhi_v2/ds_config.json index 058a027..6b50896 100644 --- a/scripts/training/zhizhi_v2/ds_config.json +++ b/scripts/training/zhizhi_v2/ds_config.json @@ -1,56 +1 @@ -{ - "train_batch_size": "auto", - "train_micro_batch_size_per_gpu": 1, - "gradient_accumulation_steps": 8, - "optimizer": { - "type": "AdamW", - "params": { - "lr": 5e-6, - "betas": [0.9, 0.999], - "eps": 1e-8, - "weight_decay": 0.01 - } - }, - "scheduler": { - "type": "WarmupLR", - "params": { - "warmup_min_lr": 0, - "warmup_max_lr": 5e-6, - "warmup_num_steps": 100 - } - }, - "zero_optimization": { - "stage": 2, - "offload_optimizer": { - "device": "cpu", - "pin_memory": true - }, - "allgather_partitions": true, - "allgather_bucket_size": 2e8, - "overlap_comm": true, - "reduce_scatter": true, - "reduce_bucket_size": 2e8, - "contiguous_gradients": true - }, - "gradient_clipping": 1.0, - "fp16": { - "enabled": "auto", - "loss_scale": 0, - "loss_scale_window": 1000, - "initial_scale_power": 16, - "hysteresis": 2, - "min_loss_scale": 1 - }, - "bf16": { - "enabled": "auto" - }, - "activation_checkpointing": { - "partition_activations": false, - "cpu_checkpointing": false, - "contiguous_memory_optimization": false, - "number_checkpoints": null, - "synchronize_checkpoint_boundary": false, - "profile": false - }, - "wall_clock_breakdown": false -} +# 此文件已迁移至 zhizhi 仓库 \ No newline at end of file