| { |
| "model": { |
| "vocab_size": 50257, |
| "hidden_size": 2048, |
| "num_hidden_layers": 24, |
| "num_attention_heads": 32, |
| "intermediate_size": 8192, |
| "hidden_act": "silu", |
| "max_position_embeddings": 2048, |
| "initializer_range": 0.02, |
| "layer_norm_eps": 1e-05, |
| "use_cache": false, |
| "rope_theta": 10000.0, |
| "attention_dropout": 0.0, |
| "hidden_dropout": 0.0 |
| }, |
| "training": { |
| "batch_size_per_device": 4, |
| "gradient_accumulation_steps": 8, |
| "learning_rate": 0.0006, |
| "min_learning_rate": 6e-05, |
| "weight_decay": 0.1, |
| "adam_beta1": 0.9, |
| "adam_beta2": 0.95, |
| "adam_epsilon": 1e-08, |
| "max_grad_norm": 1.0, |
| "warmup_steps": 2000, |
| "num_training_steps": 100000, |
| "logging_steps": 10, |
| "save_steps": 1000, |
| "eval_steps": 500, |
| "save_total_limit": 5, |
| "fp16": true, |
| "gradient_checkpointing": true |
| }, |
| "data": { |
| "dataset_name": "tiiuae/falcon-refinedweb", |
| "dataset_config": "default", |
| "text_column": "content", |
| "max_seq_length": 2048, |
| "num_workers": 8, |
| "preprocessing_num_workers": 16 |
| }, |
| "tokenizer": { |
| "tokenizer_name": "gpt2", |
| "add_special_tokens": true |
| }, |
| "infrastructure": { |
| "num_gpus": 4, |
| "seed": 42, |
| "output_dir": "./checkpoints", |
| "logging_dir": "./logs", |
| "resume_from_checkpoint": "./checkpoints/checkpoint-8000" |
| }, |
| "distributed": { |
| "backend": "nccl", |
| "find_unused_parameters": false |
| } |
| } |
