max_batch_size: 1024
max_num_tokens: 3200
tensor_parallel_size: 8
moe_expert_parallel_size: 8
trust_remote_code: true
enable_attention_dp: true
cuda_graph_config:
  enable_padding: true
kv_cache_config:
  dtype: fp8
  free_gpu_memory_fraction: 0.8
stream_interval: 10
speculative_config:
  decoding_type: MTP
  num_nextn_predict_layers: 1
moe_config:
  backend: DEEPGEMM
  max_num_tokens: 3200
