hydra:
  searchpath:
  - file:///opt/NeMo/examples/nlp/language_modeling/conf

run:
  name: bsc_40b
  results_dir: /results
  time_limit: "90:00:00:00"
  dependency: singleton

trainer:
  num_nodes: 256
  devices: 4
  accelerator: gpu
  precision: bf16
  logger: false # logger provided by exp_manager
  enable_checkpointing: false
  use_distributed_sampler: false
  max_time: "89:00:00:00" # should be lower than run.time_limit
  log_every_n_steps: 5
  limit_val_batches: 32
  limit_test_batches: 32
  gradient_clip_val: 1.0
  val_check_interval: 2000
  max_epochs: null
  max_steps: 2194040 # max_steps = (total_tokens/seq_length)/GBS
  accumulate_grad_batches: 1 # grad acc is automatic for training megatron models

exp_manager:
  seconds_to_sleep: 180
  name: ${run.name} # The name of the experiment
  exp_dir: null # The base directory to create the logging directory. Defaults to None, which logs to ./nemo_experiments.
  explicit_log_dir: ${run.results_dir} # Can be used to override exp_dir/name/version folder creation.
  log_global_rank_0_only: true # Reduces amount of log files in exp_dir. Cannot be used with log_local_rank_0_only
  log_local_rank_0_only: false # Reduces amount of log files in exp_dir. Cannot be used with log_global_rank_0_only
  create_tensorboard_logger: true
  create_wandb_logger: true
  wandb_logger_kwargs:
    project: ${run.name}
    name: 40b_${model.encoder_seq_length}_${trainer.num_nodes}nodes
  create_neptune_logger: false
  neptune_logger_kwargs:
    project: null
    name: null
    prefix: train
    log_model_checkpoints: false
    tags: null
    description: null
  resume_if_exists: true # whether this experiment is resuming from a previous run
  resume_past_end: false # exp_manager errors out if resume_if_exists is True and there's a *end.ckpt checkpoint
  resume_ignore_no_checkpoint: false # exp_manager errors out if resume_if_exists is True and no checkpoint could be found
  create_checkpoint_callback: true # whether to create a ModelCheckpoint callback and attach it to the pytorch lightning trainer
  checkpoint_callback_params:
    monitor: val_loss
    save_top_k: 200
    mode: min
    always_save_nemo: false # saves nemo file during validation, not implemented for model parallel
    save_nemo_on_train_end: true # save compressed nemo file when training is completed
    filename: megatron_gpt40b--{val_loss:.2f}-{step}-{consumed_samples}
    model_parallel_size: ${multiply:${model.tensor_model_parallel_size}, ${model.pipeline_model_parallel_size}}
    every_n_epochs: null
    every_n_train_steps: ${trainer.val_check_interval}
  log_step_timing: true
  step_timing_kwargs:
    sync_cuda: true
    buffer_size: 5

model:
  ## PARALLELISM
  micro_batch_size: 1 # limited by VRAM
  global_batch_size: 1024 # will use more micro batches to reach global batch size
  rampup_batch_size: null # [<start_batch_size>, <batch_size_increment>, <rampup_samples>]
  context_parallel_size: 1
  tensor_model_parallel_size: 4 # intra-layer model parallelism
  pipeline_model_parallel_size: 2 # inter-layer model parallelism
  virtual_pipeline_model_parallel_size: null # interleaved pipeline
  sequence_parallel: true # Makes TP more memory efficient for LLMs (20B+) by parallelizing layer norms and dropout sequentially

  ## ARCHITECTURE
  use_flash_attention: true # this config does nothing when transformer_engine=True
  encoder_seq_length: 4096
  max_position_embeddings: 4096
  num_layers: 48
  hidden_size: 8192
  ffn_hidden_size: 24576
  num_attention_heads: 64
  init_method_std: 0.007
  use_scaled_init_method: true
  hidden_dropout: 0.0
  attention_dropout: 0.0
  ffn_dropout: 0.0
  kv_channels: null
  apply_query_key_layer_scaling: true
  normalization: rmsnorm
  layernorm_zero_centered_gamma: true
  layernorm_epsilon: 1.0e-05
  do_layer_norm_weight_decay: false
  make_vocab_size_divisible_by: 128
  pre_process: true # add embedding
  post_process: true # add pooler
  persist_layer_norm: true
  bias: false
  activation: fast-swiglu
  headscale: false
  transformer_block_type: pre_ln
  openai_gelu: false
  normalize_attention_scores: true
  position_embedding_type: rope
  rotary_percentage: 1.0
  rotary_base: 10000
  attention_type: multihead
  num_query_groups: 8
  share_embeddings_and_output_weights: false
  overlap_p2p_comm: false
  batch_p2p_comm: true

  ## MEGATRON O2-STYLE HALF-PRECISION
  megatron_amp_O2: true # Enable O2-level automatic mixed precision using main parameters
  grad_allreduce_chunk_size_mb: 125

  # MISCELLANEOUS
  seed: 1234
  resume_from_checkpoint: null # Manually set the checkpoint file to load from
  use_cpu_initialization: false # Init weights on the CPU (slow for large models)
  onnx_safe: false # Use work-arounds for known problems with Torch ONNX exporter
  apex_transformer_log_level: 30 # Python logging level displays logs with severity greater than or equal to this
  gradient_as_bucket_view: true # PyTorch DDP argument. Allocate gradients in a contiguous bucket to save memory (less fragmentation and buffer memory)
  sync_batch_comm: false # Enable stream synchronization after each p2p communication between pipeline stages
  nccl_communicator_config_path: null # Path to the yaml file with NCCL communicator options (min_ctas, max_ctas, and cga_cluster_size)

  ## ACTIVATION CHECKPOINTING
  activations_checkpoint_granularity: selective
  activations_checkpoint_method: uniform
  activations_checkpoint_num_layers: null
  num_micro_batches_with_partial_activation_checkpoints: null
  activations_checkpoint_layers_per_pipeline: null

  ## MEGATRON CORE
  mcore_gpt: true # use GPTModel from megatron.core

  ## NETWORK
  sharp: false # Enable the use of SHARP (if supported) for NCCL data-parallel communications.

  ## MIXED PRECISION
  native_amp_init_scale: 4294967296 # 2**32
  native_amp_growth_interval: 1000
  hysteresis: 2 # Gradient scale hysteresis
  fp32_residual_connection: false # Move residual connections to fp32
  fp16_lm_cross_entropy: false # Move the cross entropy unreduced loss calculation for lm head to fp16

  ## FUSION
  grad_div_ar_fusion: true # Fuse grad division into torch.distributed.all_reduce. Only used with O2 and no PP.
  gradient_accumulation_fusion: false # Fuse weight gradient accumulation to GEMMs. Only used with O2 and PP.
  bias_activation_fusion: false # Use a kernel that fuses the bias addition from weight matrices with the subsequent activation function.
  bias_dropout_add_fusion: false # Use a kernel that fuses the bias addition, dropout and residual connection addition.
  masked_softmax_fusion: true # Use a kernel that fuses the attention softmax with it's mask.
  apply_rope_fusion: true # Use a kernel to add rotary positional embeddings.
  # get_attention_mask_from_fusion: true # When using fused softmax it will create the attention mask so we won't copy it to the pipeline stages.

  ## TRANSFORMER ENGINE (FP8 TRAINING)
  transformer_engine: false # set to true for FP8 training
  fp8: false # set to true for FP8 training
  fp8_e4m3: false # sets fp8_format = recipe.Format.E4M3
  fp8_hybrid: true # sets fp8_format = recipe.Format.HYBRID
  fp8_margin: 0 # scaling margin
  fp8_interval: 1 # scaling update interval
  fp8_amax_history_len: 1024 # Number of steps for which amax history is recorded per tensor
  fp8_amax_compute_algo: max # 'most_recent' or 'max'. Algorithm for computing amax from history
  reduce_amax: true # Perform reduction to sync amax tensors across GPUs after every iteration
  use_emha: false # Use fused multi-head attention for large sequence-length. Set to False because it is not supported yet.
  ub_tp_comm_overlap: false # Use userbuffer backend to overlap TP communications with computes. This feature is only available with TE and squence parallelism enabled.
  ub_tp_comm_overlap_cfg: null # A yaml file with userbuffer communicator configurations. If the configuration file is not provided a default setting is used for all communicators.

  ## TOKENIZER
  tokenizer:
    library: sentencepiece
    type: /tokenizer
    model: /tokenizer/alia.model
    vocab_file: /tokenizer/alia.vocab
    merge_file: null
    delimiter: null
    sentencepiece_legacy: false
    eod: 2

  ## OPTIMIZER
  optim:
    name: distributed_fused_adam
    lr: 5.0e-05
    weight_decay: 0.1
    betas: [0.9,0.95]
    sched:
      name: CosineAnnealing
      warmup_steps: 2000
      constant_steps: 0
      min_lr: 9.0e-06
    grad_sync_dtype: bf16
    overlap_grad_sync: true
    overlap_param_sync: true
    contiguous_grad_buffer: true
    contiguous_param_buffer: true
    bucket_cap_mb: 220

  ## TRAINING DATA
  data:
    data_impl: mmap
    num_workers: 0
    pin_memory: true
    skip_warmup: true
    eod_mask_loss: false
    dataloader_type: single
    reset_position_ids: false
    reset_attention_mask: true
    splits_string: "98,1,1"
    seq_length: ${model.encoder_seq_length}
    exchange_indices_distributed: true # Set to True to exchange indices via torch.distributed instead of filesystem
    shuffle_documents: true # Set to False to disable documents shuffling. Sample index will still be shuffled
    index_mapping_dir: null # path to save index mapping .npy files, by default will save in the same location as data_prefix
    data_prefix:
      train:
      - 0.004288603698278739
      - /data/train/fi/fi-train-optimized-0.jsonl_text_document
      - 0.003567467373668187
      - /data/train/no/no-train-optimized-0.jsonl_text_document
      - 5.02740508513718e-05
      - /data/train/sh/sh-train-optimized-0.jsonl_text_document
      - 0.004265288614312686
      - /data/train/sk/sk-train-optimized-0.jsonl_text_document
      - 0.0018155074696101221
      - /data/train/sk/sk-train-optimized-1.jsonl_text_document
      - 0.004884997657212562
      - /data/train/sv/sv-train-optimized-0.jsonl_text_document
      - 0.00034741050962021157
      - /data/train/sv/sv-train-optimized-1.jsonl_text_document
      - 0.004902762213995981
      - /data/train/pl/pl-train-optimized-1.jsonl_text_document
      - 0.004914162265296719
      - /data/train/pl/pl-train-optimized-0.jsonl_text_document
      - 0.001986356839883793
      - /data/train/pl/pl-train-optimized-2.jsonl_text_document
      - 0.005078384381323338
      - /data/train/pt/pt-train-optimized-0.jsonl_text_document
      - 0.005095373112658907
      - /data/train/pt/pt-train-optimized-2.jsonl_text_document
      - 0.005091960771609911
      - /data/train/pt/pt-train-optimized-1.jsonl_text_document
      - 0.001540433376401606
      - /data/train/pt/pt-train-optimized-4.jsonl_text_document
      - 0.005083915377049959
      - /data/train/pt/pt-train-optimized-3.jsonl_text_document
      - 0.0013469038354801445
      - /data/train/el/el-train-optimized-6.jsonl_text_document
      - 0.0013495657895445708
      - /data/train/el/el-train-optimized-3.jsonl_text_document
      - 0.0013507556847655534
      - /data/train/el/el-train-optimized-4.jsonl_text_document
      - 0.0013457411641979952
      - /data/train/el/el-train-optimized-5.jsonl_text_document
      - 0.0013446041757504935
      - /data/train/el/el-train-optimized-2.jsonl_text_document
      - 0.0011704251282579695
      - /data/train/el/el-train-optimized-7.jsonl_text_document
      - 0.001347133863439993
      - /data/train/el/el-train-optimized-1.jsonl_text_document
      - 0.0013452370316555553
      - /data/train/el/el-train-optimized-0.jsonl_text_document
      - 0.001377069072416479
      - /data/train/bg/bg-train-optimized-1.jsonl_text_document
      - 0.0013705478991838306
      - /data/train/bg/bg-train-optimized-0.jsonl_text_document
      - 0.001378701496244368
      - /data/train/bg/bg-train-optimized-2.jsonl_text_document
      - 0.0013825309930000395
      - /data/train/bg/bg-train-optimized-3.jsonl_text_document
      - 0.0010580943604719983
      - /data/train/bg/bg-train-optimized-4.jsonl_text_document
      - 0.002252984748256927
      - /data/train/et/et-train-optimized-0.jsonl_text_document
      - 0.0038322427423809
      - /data/train/code/code-train-optimized-7.jsonl_text_document
      - 0.0003566691379842504
      - /data/train/code/code-train-optimized-15.jsonl_text_document
      - 0.003826056959164061
      - /data/train/code/code-train-optimized-0.jsonl_text_document
      - 0.003828106413160286
      - /data/train/code/code-train-optimized-6.jsonl_text_document
      - 0.003836046734923982
      - /data/train/code/code-train-optimized-11.jsonl_text_document
      - 0.0038346209991690266
      - /data/train/code/code-train-optimized-14.jsonl_text_document
      - 0.003830441757442319
      - /data/train/code/code-train-optimized-12.jsonl_text_document
      - 0.003831317366685406
      - /data/train/code/code-train-optimized-13.jsonl_text_document
      - 0.0038311184977511824
      - /data/train/code/code-train-optimized-2.jsonl_text_document
      - 0.0038280983002916625
      - /data/train/code/code-train-optimized-5.jsonl_text_document
      - 0.0038309460425306045
      - /data/train/code/code-train-optimized-9.jsonl_text_document
      - 0.003828690589245985
      - /data/train/code/code-train-optimized-3.jsonl_text_document
      - 0.003832988118634894
      - /data/train/code/code-train-optimized-4.jsonl_text_document
      - 0.003831352179919758
      - /data/train/code/code-train-optimized-1.jsonl_text_document
      - 0.0038316329153107844
      - /data/train/code/code-train-optimized-8.jsonl_text_document
      - 0.0038330868271588625
      - /data/train/code/code-train-optimized-10.jsonl_text_document
      - 0.0034975907394618074
      - /data/train/da/da-train-optimized-0.jsonl_text_document
      - 0.001768591465034702
      - /data/train/lt/lt-train-optimized-0.jsonl_text_document
      - 2.291285614625527e-05
      - /data/train/nn/nn-train-optimized-0.jsonl_text_document
      - 0.010856014306530748
      - /data/train/ca/ca-train-optimized-0.jsonl_text_document
      - 0.006165389468145078
      - /data/train/ca/ca-train-optimized-1.jsonl_text_document
      - 0.005687489287112264
      - /data/train/nl/nl-train-optimized-1.jsonl_text_document
      - 0.00567596166874269
      - /data/train/nl/nl-train-optimized-0.jsonl_text_document
      - 0.0008261117616001808
      - /data/train/nl/nl-train-optimized-2.jsonl_text_document
      - 9.429679715815217e-05
      - /data/train/sr/sr-train-optimized-1.jsonl_text_document
      - 0.0028832031010018236
      - /data/train/sr/sr-train-optimized-0.jsonl_text_document
      - 6.852005573455437e-05
      - /data/train/cy/cy-train-optimized-0.jsonl_text_document
      - 0.0025839388813692814
      - /data/train/hr/hr-train-optimized-0.jsonl_text_document
      - 0.0031993455018651485
      - /data/train/hu/hu-train-optimized-6.jsonl_text_document
      - 0.0032031706986533053
      - /data/train/hu/hu-train-optimized-11.jsonl_text_document
      - 0.003198784007987022
      - /data/train/hu/hu-train-optimized-3.jsonl_text_document
      - 0.0031994153954287776
      - /data/train/hu/hu-train-optimized-5.jsonl_text_document
      - 0.003201376904416703
      - /data/train/hu/hu-train-optimized-4.jsonl_text_document
      - 0.0031992978177846033
      - /data/train/hu/hu-train-optimized-13.jsonl_text_document
      - 0.003201356420433882
      - /data/train/hu/hu-train-optimized-8.jsonl_text_document
      - 0.0031994529116488603
      - /data/train/hu/hu-train-optimized-10.jsonl_text_document
      - 0.0031339007700080935
      - /data/train/hu/hu-train-optimized-0.jsonl_text_document
      - 0.0032034083114384653
      - /data/train/hu/hu-train-optimized-7.jsonl_text_document
      - 0.003201261369285037
      - /data/train/hu/hu-train-optimized-9.jsonl_text_document
      - 0.0013391949641337131
      - /data/train/hu/hu-train-optimized-14.jsonl_text_document
      - 0.0031990999313648324
      - /data/train/hu/hu-train-optimized-1.jsonl_text_document
      - 0.003200597087073167
      - /data/train/hu/hu-train-optimized-2.jsonl_text_document
      - 0.0032008625265301967
      - /data/train/hu/hu-train-optimized-12.jsonl_text_document
      - 0.010364828165738362
      - /data/train/es/es-train-optimized-4.jsonl_text_document
      - 0.010362127746652218
      - /data/train/es/es-train-optimized-13.jsonl_text_document
      - 0.010354484378752707
      - /data/train/es/es-train-optimized-2.jsonl_text_document
      - 0.0036289100710364046
      - /data/train/es/es-train-optimized-15.jsonl_text_document
      - 0.010341941069165323
      - /data/train/es/es-train-optimized-0.jsonl_text_document
      - 0.01035980302023158
      - /data/train/es/es-train-optimized-11.jsonl_text_document
      - 0.010349079249180805
      - /data/train/es/es-train-optimized-9.jsonl_text_document
      - 0.010368465866732571
      - /data/train/es/es-train-optimized-14.jsonl_text_document
      - 0.010362293802956397
      - /data/train/es/es-train-optimized-12.jsonl_text_document
      - 0.01035950340156469
      - /data/train/es/es-train-optimized-10.jsonl_text_document
      - 0.010348379456754663
      - /data/train/es/es-train-optimized-7.jsonl_text_document
      - 0.010346814971891983
      - /data/train/es/es-train-optimized-3.jsonl_text_document
      - 0.010362376910082162
      - /data/train/es/es-train-optimized-8.jsonl_text_document
      - 0.0103617465802171
      - /data/train/es/es-train-optimized-1.jsonl_text_document
      - 0.010363386502073763
      - /data/train/es/es-train-optimized-6.jsonl_text_document
      - 0.010362244869825253
      - /data/train/es/es-train-optimized-5.jsonl_text_document
      - 0.0022645014615542474
      - /data/train/ro/ro-train-optimized-1.jsonl_text_document
      - 0.005099860638178066
      - /data/train/ro/ro-train-optimized-0.jsonl_text_document
      - 0.006478268063656108
      - /data/train/en/en-train-optimized-27.jsonl_text_document
      - 0.00647960195992036
      - /data/train/en/en-train-optimized-16.jsonl_text_document
      - 0.00647953390249464
      - /data/train/en/en-train-optimized-37.jsonl_text_document
      - 0.006474760565663346
      - /data/train/en/en-train-optimized-42.jsonl_text_document
      - 0.006478094598354796
      - /data/train/en/en-train-optimized-6.jsonl_text_document
      - 0.006479921234091534
      - /data/train/en/en-train-optimized-18.jsonl_text_document
      - 0.0064784424229990065
      - /data/train/en/en-train-optimized-24.jsonl_text_document
      - 0.00647170022592284
      - /data/train/en/en-train-optimized-25.jsonl_text_document
      - 0.006478324435958289
      - /data/train/en/en-train-optimized-2.jsonl_text_document
      - 0.0064787319423486225
      - /data/train/en/en-train-optimized-38.jsonl_text_document
      - 0.006476690729180723
      - /data/train/en/en-train-optimized-7.jsonl_text_document
      - 0.0064798161029963166
      - /data/train/en/en-train-optimized-4.jsonl_text_document
      - 0.006483138733304879
      - /data/train/en/en-train-optimized-26.jsonl_text_document
      - 0.006479818687894054
      - /data/train/en/en-train-optimized-41.jsonl_text_document
      - 0.0064797992659581345
      - /data/train/en/en-train-optimized-51.jsonl_text_document
      - 0.006480559476597156
      - /data/train/en/en-train-optimized-53.jsonl_text_document
      - 0.006480840809878493
      - /data/train/en/en-train-optimized-58.jsonl_text_document
      - 0.006476332374216689
      - /data/train/en/en-train-optimized-33.jsonl_text_document
      - 0.006479802187239019
      - /data/train/en/en-train-optimized-44.jsonl_text_document
      - 0.006476762214138376
      - /data/train/en/en-train-optimized-21.jsonl_text_document
      - 0.006479699019682636
      - /data/train/en/en-train-optimized-23.jsonl_text_document
      - 0.0064820102918949464
      - /data/train/en/en-train-optimized-50.jsonl_text_document
      - 0.006482185494989593
      - /data/train/en/en-train-optimized-52.jsonl_text_document
      - 0.0064693590547247656
      - /data/train/en/en-train-optimized-59.jsonl_text_document
      - 0.006482629354973429
      - /data/train/en/en-train-optimized-17.jsonl_text_document
      - 0.006477864842705113
      - /data/train/en/en-train-optimized-43.jsonl_text_document
      - 0.006475667336539679
      - /data/train/en/en-train-optimized-9.jsonl_text_document
      - 0.006480127193706833
      - /data/train/en/en-train-optimized-13.jsonl_text_document
      - 0.00647838301095543
      - /data/train/en/en-train-optimized-11.jsonl_text_document
      - 0.006479825556741061
      - /data/train/en/en-train-optimized-49.jsonl_text_document
      - 0.006478445289147327
      - /data/train/en/en-train-optimized-8.jsonl_text_document
      - 0.006477854994837036
      - /data/train/en/en-train-optimized-28.jsonl_text_document
      - 0.006479717500639786
      - /data/train/en/en-train-optimized-60.jsonl_text_document
      - 0.006479683920065271
      - /data/train/en/en-train-optimized-47.jsonl_text_document
      - 0.006479151994750038
      - /data/train/en/en-train-optimized-56.jsonl_text_document
      - 0.006477291366807138
      - /data/train/en/en-train-optimized-55.jsonl_text_document
      - 0.006474731531662817
      - /data/train/en/en-train-optimized-1.jsonl_text_document
      - 0.006476404654498838
      - /data/train/en/en-train-optimized-34.jsonl_text_document
      - 0.006480401300152011
      - /data/train/en/en-train-optimized-5.jsonl_text_document
      - 0.006482953396993882
      - /data/train/en/en-train-optimized-12.jsonl_text_document
      - 0.00647574289385676
      - /data/train/en/en-train-optimized-57.jsonl_text_document
      - 0.006479495125675919
      - /data/train/en/en-train-optimized-10.jsonl_text_document
      - 0.0064806461058742534
      - /data/train/en/en-train-optimized-45.jsonl_text_document
      - 0.006476361393316525
      - /data/train/en/en-train-optimized-14.jsonl_text_document
      - 0.006479509673595122
      - /data/train/en/en-train-optimized-54.jsonl_text_document
      - 0.00647767543105455
      - /data/train/en/en-train-optimized-19.jsonl_text_document
      - 0.006478385862203057
      - /data/train/en/en-train-optimized-36.jsonl_text_document
      - 0.006469138090111503
      - /data/train/en/en-train-optimized-31.jsonl_text_document
      - 0.006470366086404773
      - /data/train/en/en-train-optimized-15.jsonl_text_document
      - 0.006479777816037882
      - /data/train/en/en-train-optimized-46.jsonl_text_document
      - 0.006476243205861485
      - /data/train/en/en-train-optimized-39.jsonl_text_document
      - 0.006475910126198144
      - /data/train/en/en-train-optimized-35.jsonl_text_document
      - 0.006480261362527753
      - /data/train/en/en-train-optimized-30.jsonl_text_document
      - 0.006477910923843597
      - /data/train/en/en-train-optimized-40.jsonl_text_document
      - 0.006477667284473109
      - /data/train/en/en-train-optimized-32.jsonl_text_document
      - 0.0064679404435499625
      - /data/train/en/en-train-optimized-0.jsonl_text_document
      - 0.000264617869690329
      - /data/train/en/en-train-optimized-61.jsonl_text_document
      - 0.006475778302373802
      - /data/train/en/en-train-optimized-29.jsonl_text_document
      - 0.006481204666178424
      - /data/train/en/en-train-optimized-3.jsonl_text_document
      - 0.006476904206195697
      - /data/train/en/en-train-optimized-22.jsonl_text_document
      - 0.006478704862573995
      - /data/train/en/en-train-optimized-20.jsonl_text_document
      - 0.006479715375428431
      - /data/train/en/en-train-optimized-48.jsonl_text_document
      - 0.0008634209548056815
      - /data/train/mt/mt-train-optimized-0.jsonl_text_document
      - 0.0012420705421623893
      - /data/train/ru/ru-train-optimized-29.jsonl_text_document
      - 0.0012418569176305533
      - /data/train/ru/ru-train-optimized-15.jsonl_text_document
      - 0.0012430086994861765
      - /data/train/ru/ru-train-optimized-17.jsonl_text_document
      - 0.0012438020596971643
      - /data/train/ru/ru-train-optimized-39.jsonl_text_document
      - 0.00124227912653478
      - /data/train/ru/ru-train-optimized-12.jsonl_text_document
      - 0.0012424448773747509
      - /data/train/ru/ru-train-optimized-9.jsonl_text_document
      - 0.0012438319776813121
      - /data/train/ru/ru-train-optimized-42.jsonl_text_document
      - 0.0012425922418769736
      - /data/train/ru/ru-train-optimized-14.jsonl_text_document
      - 0.0012419330542120788
      - /data/train/ru/ru-train-optimized-8.jsonl_text_document
      - 0.0012429440412810725
      - /data/train/ru/ru-train-optimized-28.jsonl_text_document
      - 0.0012421564390704205
      - /data/train/ru/ru-train-optimized-40.jsonl_text_document
      - 0.0012423180560856048
      - /data/train/ru/ru-train-optimized-34.jsonl_text_document
      - 0.0012426048489808984
      - /data/train/ru/ru-train-optimized-27.jsonl_text_document
      - 0.0012419258426491298
      - /data/train/ru/ru-train-optimized-30.jsonl_text_document
      - 0.0012422833635468665
      - /data/train/ru/ru-train-optimized-32.jsonl_text_document
      - 0.0012420425210364747
      - /data/train/ru/ru-train-optimized-44.jsonl_text_document
      - 0.0012427187483889778
      - /data/train/ru/ru-train-optimized-19.jsonl_text_document
      - 0.001242070961244383
      - /data/train/ru/ru-train-optimized-22.jsonl_text_document
      - 0.0012431380721465008
      - /data/train/ru/ru-train-optimized-31.jsonl_text_document
      - 0.0012415695372754337
      - /data/train/ru/ru-train-optimized-26.jsonl_text_document
      - 0.0012417704783372926
      - /data/train/ru/ru-train-optimized-5.jsonl_text_document
      - 0.001242513952890328
      - /data/train/ru/ru-train-optimized-2.jsonl_text_document
      - 0.0012425449638403154
      - /data/train/ru/ru-train-optimized-24.jsonl_text_document
      - 0.0012423418521199873
      - /data/train/ru/ru-train-optimized-1.jsonl_text_document
      - 0.0012414990648198774
      - /data/train/ru/ru-train-optimized-25.jsonl_text_document
      - 0.001242039745037348
      - /data/train/ru/ru-train-optimized-16.jsonl_text_document
      - 0.0012425108405080543
      - /data/train/ru/ru-train-optimized-36.jsonl_text_document
      - 0.0012423964218108198
      - /data/train/ru/ru-train-optimized-33.jsonl_text_document
      - 0.00010423631567408616
      - /data/train/ru/ru-train-optimized-45.jsonl_text_document
      - 0.001242989517451416
      - /data/train/ru/ru-train-optimized-23.jsonl_text_document
      - 0.0012425124557431876
      - /data/train/ru/ru-train-optimized-6.jsonl_text_document
      - 0.001242301689909325
      - /data/train/ru/ru-train-optimized-43.jsonl_text_document
      - 0.0012422132822345045
      - /data/train/ru/ru-train-optimized-11.jsonl_text_document
      - 0.0012418288633505965
      - /data/train/ru/ru-train-optimized-3.jsonl_text_document
      - 0.0012434180282312772
      - /data/train/ru/ru-train-optimized-41.jsonl_text_document
      - 0.0012416452488146882
      - /data/train/ru/ru-train-optimized-21.jsonl_text_document
      - 0.001243219188167147
      - /data/train/ru/ru-train-optimized-18.jsonl_text_document
      - 0.0012426527178300382
      - /data/train/ru/ru-train-optimized-13.jsonl_text_document
      - 0.0012421367723906165
      - /data/train/ru/ru-train-optimized-7.jsonl_text_document
      - 0.0012434340099696747
      - /data/train/ru/ru-train-optimized-10.jsonl_text_document
      - 0.0012425607220683161
      - /data/train/ru/ru-train-optimized-37.jsonl_text_document
      - 0.001241882858247192
      - /data/train/ru/ru-train-optimized-20.jsonl_text_document
      - 0.0012239436020125534
      - /data/train/ru/ru-train-optimized-0.jsonl_text_document
      - 0.001242434091880559
      - /data/train/ru/ru-train-optimized-4.jsonl_text_document
      - 0.0012411615093608417
      - /data/train/ru/ru-train-optimized-35.jsonl_text_document
      - 0.00124245131521921
      - /data/train/ru/ru-train-optimized-38.jsonl_text_document
      - 3.25197792713731e-05
      - /data/train/oc/oc-train-optimized-0.jsonl_text_document
      - 0.0010176471941698436
      - /data/train/lv/lv-train-optimized-0.jsonl_text_document
      - 0.00413609178663726
      - /data/train/it/it-train-optimized-3.jsonl_text_document
      - 0.005544602466265981
      - /data/train/it/it-train-optimized-1.jsonl_text_document
      - 0.005527289192619188
      - /data/train/it/it-train-optimized-2.jsonl_text_document
      - 0.00553129982629802
      - /data/train/it/it-train-optimized-0.jsonl_text_document
      - 0.006533103856281339
      - /data/train/fr/fr-train-optimized-8.jsonl_text_document
      - 0.006524852039067049
      - /data/train/fr/fr-train-optimized-0.jsonl_text_document
      - 0.006522002956883279
      - /data/train/fr/fr-train-optimized-4.jsonl_text_document
      - 0.006530650858734115
      - /data/train/fr/fr-train-optimized-2.jsonl_text_document
      - 0.00557134808502732
      - /data/train/fr/fr-train-optimized-9.jsonl_text_document
      - 0.006548094057484428
      - /data/train/fr/fr-train-optimized-7.jsonl_text_document
      - 0.006534514648317432
      - /data/train/fr/fr-train-optimized-1.jsonl_text_document
      - 0.006545308393768095
      - /data/train/fr/fr-train-optimized-3.jsonl_text_document
      - 0.006531195805020023
      - /data/train/fr/fr-train-optimized-6.jsonl_text_document
      - 0.006533940146134218
      - /data/train/fr/fr-train-optimized-5.jsonl_text_document
      - 0.0035061480760174594
      - /data/train/sl/sl-train-optimized-0.jsonl_text_document
      - 0.004216691499384702
      - /data/train/cs/cs-train-optimized-1.jsonl_text_document
      - 0.004227848831805268
      - /data/train/cs/cs-train-optimized-0.jsonl_text_document
      - 0.0014988128904094866
      - /data/train/cs/cs-train-optimized-2.jsonl_text_document
      - 0.0012900530787479996
      - /data/train/uk/uk-train-optimized-6.jsonl_text_document
      - 0.0012880900014621242
      - /data/train/uk/uk-train-optimized-3.jsonl_text_document
      - 0.0012881187661326252
      - /data/train/uk/uk-train-optimized-7.jsonl_text_document
      - 0.0004966754484773691
      - /data/train/uk/uk-train-optimized-8.jsonl_text_document
      - 0.0012862894281551907
      - /data/train/uk/uk-train-optimized-1.jsonl_text_document
      - 0.0012802331990146147
      - /data/train/uk/uk-train-optimized-0.jsonl_text_document
      - 0.0012898861678890701
      - /data/train/uk/uk-train-optimized-5.jsonl_text_document
      - 0.0012891494545255345
      - /data/train/uk/uk-train-optimized-4.jsonl_text_document
      - 0.0012883877240158156
      - /data/train/uk/uk-train-optimized-2.jsonl_text_document
      - 0.0024452532345798973
      - /data/train/eu/eu-train-optimized-0.jsonl_text_document
      - 6.883195145501655e-05
      - /data/train/ga/ga-train-optimized-0.jsonl_text_document
      - 0.0031458957787910318
      - /data/train/gl/gl-train-optimized-0.jsonl_text_document
      - 0.005427589021935817
      - /data/train/de/de-train-optimized-8.jsonl_text_document
      - 0.005424632628685426
      - /data/train/de/de-train-optimized-2.jsonl_text_document
      - 0.005427128102893471
      - /data/train/de/de-train-optimized-3.jsonl_text_document
      - 0.005412179217587209
      - /data/train/de/de-train-optimized-0.jsonl_text_document
      - 0.005419718587961731
      - /data/train/de/de-train-optimized-4.jsonl_text_document
      - 0.005426006107895897
      - /data/train/de/de-train-optimized-5.jsonl_text_document
      - 0.005414930734875207
      - /data/train/de/de-train-optimized-1.jsonl_text_document
      - 0.00012411566189533623
      - /data/train/de/de-train-optimized-9.jsonl_text_document
      - 0.005424739253575128
      - /data/train/de/de-train-optimized-6.jsonl_text_document
      - 0.005417295817588036
      - /data/train/de/de-train-optimized-7.jsonl_text_document
      validation:
      - 0.03251875122473439
      - /data/validation/fi/fi-train-optimized-0-valid.jsonl_text_document
      - 0.011689920565309729
      - /data/validation/no/no-train-optimized-0-valid.jsonl_text_document
      - 0.002660288337954411
      - /data/validation/sh/sh-train-optimized-0-valid.jsonl_text_document
      - 0.038416096156957694
      - /data/validation/sk/sk-train-optimized-0-valid.jsonl_text_document
      - 0.022860022473320314
      - /data/validation/sv/sv-train-optimized-0-valid.jsonl_text_document
      - 0.03104823411839893
      - /data/validation/pl/pl-train-optimized-0-valid.jsonl_text_document
      - 0.021984001255799294
      - /data/validation/pt/pt-train-optimized-0-valid.jsonl_text_document
      - 0.011454479959188421
      - /data/validation/el/el-train-optimized-0-valid.jsonl_text_document
      - 0.02259081014368524
      - /data/validation/bg/bg-train-optimized-0-valid.jsonl_text_document
      - 0.016460903045836944
      - /data/validation/et/et-train-optimized-0-valid.jsonl_text_document
      - 0.011710180434122598
      - /data/validation/code/code-train-optimized-0-valid.jsonl_text_document
      - 0.025640554448254995
      - /data/validation/da/da-train-optimized-0-valid.jsonl_text_document
      - 0.03257868609296549
      - /data/validation/lt/lt-train-optimized-0-valid.jsonl_text_document
      - 0.0012483074349490895
      - /data/validation/nn/nn-train-optimized-0-valid.jsonl_text_document
      - 0.03400977746689477
      - /data/validation/ca/ca-train-optimized-0-valid.jsonl_text_document
      - 0.02079081744008678
      - /data/validation/nl/nl-train-optimized-0-valid.jsonl_text_document
      - 0.011359465704785097
      - /data/validation/sr/sr-train-optimized-0-valid.jsonl_text_document
      - 0.003637035189750131
      - /data/validation/cy/cy-train-optimized-0-valid.jsonl_text_document
      - 0.009020069078654859
      - /data/validation/hr/hr-train-optimized-0-valid.jsonl_text_document
      - 0.1700043377801533
      - /data/validation/hu/hu-train-optimized-0-valid.jsonl_text_document
      - 0.04908741675216245
      - /data/validation/es/es-train-optimized-0-valid.jsonl_text_document
      - 0.06127231631918941
      - /data/validation/ro/ro-train-optimized-0-valid.jsonl_text_document
      - 0.026674866153357703
      - /data/validation/en/en-train-optimized-0-valid.jsonl_text_document
      - 0.044815425285260724
      - /data/validation/mt/mt-train-optimized-0-valid.jsonl_text_document
      - 0.04420079218282982
      - /data/validation/ru/ru-train-optimized-0-valid.jsonl_text_document
      - 0.0015726315139630211
      - /data/validation/oc/oc-train-optimized-0-valid.jsonl_text_document
      - 0.04014130084370217
      - /data/validation/lv/lv-train-optimized-0-valid.jsonl_text_document
      - 0.029621677030117442
      - /data/validation/it/it-train-optimized-0-valid.jsonl_text_document
      - 0.05344896827384665
      - /data/validation/fr/fr-train-optimized-0-valid.jsonl_text_document
      - 0.02089617034056139
      - /data/validation/sl/sl-train-optimized-0-valid.jsonl_text_document
      - 0.006059823501269379
      - /data/validation/cs/cs-train-optimized-0-valid.jsonl_text_document
      - 0.019309750008550747
      - /data/validation/uk/uk-train-optimized-0-valid.jsonl_text_document
      - 0.021632143063501745
      - /data/validation/eu/eu-train-optimized-0-valid.jsonl_text_document
      - 0.002310093904046828
      - /data/validation/ga/ga-train-optimized-0-valid.jsonl_text_document
      - 0.01741796373860944
      - /data/validation/gl/gl-train-optimized-0-valid.jsonl_text_document
      - 0.029855922737228598
      - /data/validation/de/de-train-optimized-0-valid.jsonl_text_document
      test:
      - 0.0299924135588579
      - /data/test/fi/fi-train-optimized-0-test.jsonl_text_document
      - 0.009999538955355092
      - /data/test/no/no-train-optimized-0-test.jsonl_text_document
      - 0.002487871132492304
      - /data/test/sh/sh-train-optimized-0-test.jsonl_text_document
      - 0.036818775211956485
      - /data/test/sk/sk-train-optimized-0-test.jsonl_text_document
      - 0.021723478563722212
      - /data/test/sv/sv-train-optimized-0-test.jsonl_text_document
      - 0.026892969094046256
      - /data/test/pl/pl-train-optimized-0-test.jsonl_text_document
      - 0.020826558754655788
      - /data/test/pt/pt-train-optimized-0-test.jsonl_text_document
      - 0.010783811530958321
      - /data/test/el/el-train-optimized-0-test.jsonl_text_document
      - 0.023986038264911073
      - /data/test/bg/bg-train-optimized-0-test.jsonl_text_document
      - 0.016770040684694735
      - /data/test/et/et-train-optimized-0-test.jsonl_text_document
      - 0.011265044727018335
      - /data/test/code/code-train-optimized-0-test.jsonl_text_document
      - 0.024647164542588904
      - /data/test/da/da-train-optimized-0-test.jsonl_text_document
      - 0.03490102024413871
      - /data/test/lt/lt-train-optimized-0-test.jsonl_text_document
      - 0.0011928671365380027
      - /data/test/nn/nn-train-optimized-0-test.jsonl_text_document
      - 0.03204444157101407
      - /data/test/ca/ca-train-optimized-0-test.jsonl_text_document
      - 0.01940369801874897
      - /data/test/nl/nl-train-optimized-0-test.jsonl_text_document
      - 0.010674685348198066
      - /data/test/sr/sr-train-optimized-0-test.jsonl_text_document
      - 0.003582565419050993
      - /data/test/cy/cy-train-optimized-0-test.jsonl_text_document
      - 0.010621347461826991
      - /data/test/hr/hr-train-optimized-0-test.jsonl_text_document
      - 0.15799307551639005
      - /data/test/hu/hu-train-optimized-0-test.jsonl_text_document
      - 0.04643272895644684
      - /data/test/es/es-train-optimized-0-test.jsonl_text_document
      - 0.056538097350622436
      - /data/test/ro/ro-train-optimized-0-test.jsonl_text_document
      - 0.02495785768357714
      - /data/test/en/en-train-optimized-0-test.jsonl_text_document
      - 0.08847567465581604
      - /data/test/mt/mt-train-optimized-0-test.jsonl_text_document
      - 0.0453578848759656
      - /data/test/ru/ru-train-optimized-0-test.jsonl_text_document
      - 0.0015517127193128393
      - /data/test/oc/oc-train-optimized-0-test.jsonl_text_document
      - 0.039803644159113756
      - /data/test/lv/lv-train-optimized-0-test.jsonl_text_document
      - 0.028798079946170597
      - /data/test/it/it-train-optimized-0-test.jsonl_text_document
      - 0.04629976593900274
      - /data/test/fr/fr-train-optimized-0-test.jsonl_text_document
      - 0.020185064031084568
      - /data/test/sl/sl-train-optimized-0-test.jsonl_text_document
      - 0.006296173977203736
      - /data/test/cs/cs-train-optimized-0-test.jsonl_text_document
      - 0.01908332010339686
      - /data/test/uk/uk-train-optimized-0-test.jsonl_text_document
      - 0.021289237634551768
      - /data/test/eu/eu-train-optimized-0-test.jsonl_text_document
      - 0.0031088841981737098
      - /data/test/ga/ga-train-optimized-0-test.jsonl_text_document
      - 0.016070855769574125
      - /data/test/gl/gl-train-optimized-0-test.jsonl_text_document
      - 0.029143612262823977
      - /data/test/de/de-train-optimized-0-test.jsonl_text_document