Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+2024-09-23/09-33-58/wandb/run-20240923_093407-jnzzkcth/run-jnzzkcth.wandb filter=lfs diff=lfs merge=lfs -text
+2024-09-23/15-02-55/wandb/run-20240923_150304-bbl5fd2u/run-bbl5fd2u.wandb filter=lfs diff=lfs merge=lfs -text
+2024-09-23/15-28-03/wandb/run-20240923_152812-jp82yqcj/run-jp82yqcj.wandb filter=lfs diff=lfs merge=lfs -text

2024-09-23/06-36-18/.hydra/config.yaml

@@ -0,0 +1,74 @@
+experimental:
+  model:
+    core_model_type: pass_through
+    hidden_dim: 384
+    byte_hidden: 128
+    max_chunk_length: 12
+    max_num_chunks: 1024
+    num_delimiter_layers: 3
+    num_byte_decoder_layers: 5
+    target_chunk_len: 8.0
+    chunk_len_loss_weight: 0.1
+    chunk_len_penalty: 0.1
+    context_window: 8192
+    embedding_model_type: byte_level
+    tokenizer_type: bpe
+    tokenizer_dataset_name: simple_en_wiki
+    tokenizer_simplify_data: true
+    vocab_size: 259
+    lm_head_type: byte_level
+    lm_head_normalization: rms_norm
+    lm_head_bias: false
+    lm_head_dropout: 0.0
+    model_shell_type: byte_autoencoder_shell
+    embedding_weight_tying: true
+    ffn_weight_tying: false
+    cproj_weight_tying: false
+    positional_encoding_type: rope
+  trainer:
+    trainer_type: base_trainer
+    dataset: fineweb_edu_10B
+    batch_size: 6
+    gradient_accumulation_steps: 8
+    max_iters: 10000
+    eval_interval: 50000000
+    log_interval: 1
+    checkpoint_interval: 1000
+    eval_iters: 1000
+    run_eval: false
+    eval:
+      mcq_benchmarks: null
+      mcq_num_samples: 1000
+      eval_byte_metrics: false
+      text_modeling_eval: false
+      text_generation_eval: false
+    optimizer:
+      optimizer_name: adamW
+      lr: 0.0005
+      min_lr: 5.0e-05
+      weight_decay: 0.01
+      beta1: 0.9
+      beta2: 0.95
+      grad_clip: 1.0
+    lr_scheduler:
+      name: cosine
+      warmup_iters: 100
+    dataloader:
+      name: autoencoder
+    datasampling:
+      name: standard
+    loss_fn:
+      name: pass_through
+  general:
+    logging:
+      wandb_log: true
+      wandb_project: SuperTinyLanguageModels
+      wandb_run_name: null
+      group_name: experimental_byte_level
+    paths:
+      output_dir: outputs
+      data_dir: data
+      checkpoint_dir: checkpoints
+      eval_dir: evals
+    seed: 489
+    device: cuda

2024-09-23/06-36-18/.hydra/hydra.yaml

@@ -0,0 +1,154 @@
+hydra:
+  run:
+    dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task: []
+  job:
+    name: train
+    chdir: null
+    override_dirname: ''
+    id: ???
+    num: ???
+    config_name: experimental/byte_autoencoder_1
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /root/SuperTinyLanguageModels
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /root/SuperTinyLanguageModels/configs/train
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /root/SuperTinyLanguageModels/outputs/2024-09-23/06-36-18
+    choices:
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

2024-09-23/06-36-18/.hydra/overrides.yaml

@@ -0,0 +1 @@
+[]

2024-09-23/07-06-14/.hydra/config.yaml

@@ -0,0 +1,74 @@
+experimental:
+  model:
+    core_model_type: pass_through
+    hidden_dim: 384
+    byte_hidden: 128
+    max_chunk_length: 12
+    max_num_chunks: 1024
+    num_delimiter_layers: 3
+    num_byte_decoder_layers: 5
+    target_chunk_len: 8.0
+    chunk_len_loss_weight: 0.1
+    chunk_len_penalty: 0.1
+    context_window: 8192
+    embedding_model_type: byte_level
+    tokenizer_type: bpe
+    tokenizer_dataset_name: simple_en_wiki
+    tokenizer_simplify_data: true
+    vocab_size: 259
+    lm_head_type: byte_level
+    lm_head_normalization: rms_norm
+    lm_head_bias: false
+    lm_head_dropout: 0.0
+    model_shell_type: byte_autoencoder_shell
+    embedding_weight_tying: true
+    ffn_weight_tying: false
+    cproj_weight_tying: false
+    positional_encoding_type: rope
+  trainer:
+    trainer_type: base_trainer
+    dataset: fineweb_edu_10B
+    batch_size: 6
+    gradient_accumulation_steps: 8
+    max_iters: 10000
+    eval_interval: 50000000
+    log_interval: 1
+    checkpoint_interval: 1000
+    eval_iters: 1000
+    run_eval: false
+    eval:
+      mcq_benchmarks: null
+      mcq_num_samples: 1000
+      eval_byte_metrics: false
+      text_modeling_eval: false
+      text_generation_eval: false
+    optimizer:
+      optimizer_name: adamW
+      lr: 0.0005
+      min_lr: 5.0e-05
+      weight_decay: 0.01
+      beta1: 0.9
+      beta2: 0.95
+      grad_clip: 1.0
+    lr_scheduler:
+      name: cosine
+      warmup_iters: 100
+    dataloader:
+      name: autoencoder
+    datasampling:
+      name: standard
+    loss_fn:
+      name: pass_through
+  general:
+    logging:
+      wandb_log: true
+      wandb_project: SuperTinyLanguageModels
+      wandb_run_name: null
+      group_name: experimental_byte_level
+    paths:
+      output_dir: outputs
+      data_dir: data
+      checkpoint_dir: checkpoints
+      eval_dir: evals
+    seed: 489
+    device: cuda

2024-09-23/07-06-14/.hydra/hydra.yaml

@@ -0,0 +1,154 @@
+hydra:
+  run:
+    dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task: []
+  job:
+    name: train
+    chdir: null
+    override_dirname: ''
+    id: ???
+    num: ???
+    config_name: experimental/byte_autoencoder_1
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /root/SuperTinyLanguageModels
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /root/SuperTinyLanguageModels/configs/train
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /root/SuperTinyLanguageModels/outputs/2024-09-23/07-06-14
+    choices:
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

2024-09-23/07-06-14/.hydra/overrides.yaml

@@ -0,0 +1 @@
+[]

2024-09-23/08-39-13/.hydra/config.yaml

@@ -0,0 +1,74 @@
+experimental:
+  model:
+    core_model_type: pass_through
+    hidden_dim: 384
+    byte_hidden: 128
+    max_chunk_length: 12
+    max_num_chunks: 1024
+    num_delimiter_layers: 3
+    num_byte_decoder_layers: 5
+    target_chunk_len: 8.0
+    chunk_len_loss_weight: 0.1
+    chunk_len_penalty: 0.1
+    context_window: 8192
+    embedding_model_type: byte_level
+    tokenizer_type: bpe
+    tokenizer_dataset_name: simple_en_wiki
+    tokenizer_simplify_data: true
+    vocab_size: 259
+    lm_head_type: byte_level
+    lm_head_normalization: rms_norm
+    lm_head_bias: false
+    lm_head_dropout: 0.0
+    model_shell_type: byte_autoencoder_shell
+    embedding_weight_tying: true
+    ffn_weight_tying: false
+    cproj_weight_tying: false
+    positional_encoding_type: rope
+  trainer:
+    trainer_type: base_trainer
+    dataset: fineweb_edu_10B
+    batch_size: 6
+    gradient_accumulation_steps: 8
+    max_iters: 10000
+    eval_interval: 50000000
+    log_interval: 1
+    checkpoint_interval: 1000
+    eval_iters: 1000
+    run_eval: false
+    eval:
+      mcq_benchmarks: null
+      mcq_num_samples: 1000
+      eval_byte_metrics: false
+      text_modeling_eval: false
+      text_generation_eval: false
+    optimizer:
+      optimizer_name: adamW
+      lr: 0.0005
+      min_lr: 5.0e-05
+      weight_decay: 0.01
+      beta1: 0.9
+      beta2: 0.95
+      grad_clip: 1.0
+    lr_scheduler:
+      name: cosine
+      warmup_iters: 100
+    dataloader:
+      name: autoencoder
+    datasampling:
+      name: standard
+    loss_fn:
+      name: pass_through
+  general:
+    logging:
+      wandb_log: true
+      wandb_project: SuperTinyLanguageModels
+      wandb_run_name: null
+      group_name: experimental_byte_level
+    paths:
+      output_dir: outputs
+      data_dir: data
+      checkpoint_dir: checkpoints
+      eval_dir: evals
+    seed: 489
+    device: cuda

2024-09-23/08-39-13/.hydra/hydra.yaml

@@ -0,0 +1,154 @@
+hydra:
+  run:
+    dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task: []
+  job:
+    name: train
+    chdir: null
+    override_dirname: ''
+    id: ???
+    num: ???
+    config_name: experimental/byte_autoencoder_1
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /root/SuperTinyLanguageModels
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /root/SuperTinyLanguageModels/configs/train
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /root/SuperTinyLanguageModels/outputs/2024-09-23/08-39-13
+    choices:
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

2024-09-23/08-39-13/.hydra/overrides.yaml

@@ -0,0 +1 @@
+[]

2024-09-23/08-40-08/.hydra/config.yaml

@@ -0,0 +1,74 @@
+experimental:
+  model:
+    core_model_type: pass_through
+    hidden_dim: 384
+    byte_hidden: 128
+    max_chunk_length: 12
+    max_num_chunks: 1024
+    num_delimiter_layers: 3
+    num_byte_decoder_layers: 5
+    target_chunk_len: 8.0
+    chunk_len_loss_weight: 0.1
+    chunk_len_penalty: 0.1
+    context_window: 8192
+    embedding_model_type: byte_level
+    tokenizer_type: bpe
+    tokenizer_dataset_name: simple_en_wiki
+    tokenizer_simplify_data: true
+    vocab_size: 259
+    lm_head_type: byte_level
+    lm_head_normalization: rms_norm
+    lm_head_bias: false
+    lm_head_dropout: 0.0
+    model_shell_type: byte_autoencoder_shell
+    embedding_weight_tying: true
+    ffn_weight_tying: false
+    cproj_weight_tying: false
+    positional_encoding_type: rope
+  trainer:
+    trainer_type: base_trainer
+    dataset: fineweb_edu_10B
+    batch_size: 6
+    gradient_accumulation_steps: 8
+    max_iters: 10000
+    eval_interval: 50000000
+    log_interval: 1
+    checkpoint_interval: 1000
+    eval_iters: 1000
+    run_eval: false
+    eval:
+      mcq_benchmarks: null
+      mcq_num_samples: 1000
+      eval_byte_metrics: false
+      text_modeling_eval: false
+      text_generation_eval: false
+    optimizer:
+      optimizer_name: adamW
+      lr: 0.0005
+      min_lr: 5.0e-05
+      weight_decay: 0.01
+      beta1: 0.9
+      beta2: 0.95
+      grad_clip: 1.0
+    lr_scheduler:
+      name: cosine
+      warmup_iters: 100
+    dataloader:
+      name: autoencoder
+    datasampling:
+      name: standard
+    loss_fn:
+      name: pass_through
+  general:
+    logging:
+      wandb_log: true
+      wandb_project: SuperTinyLanguageModels
+      wandb_run_name: null
+      group_name: experimental_byte_level
+    paths:
+      output_dir: outputs
+      data_dir: data
+      checkpoint_dir: checkpoints
+      eval_dir: evals
+    seed: 489
+    device: cuda

2024-09-23/08-40-08/.hydra/hydra.yaml

@@ -0,0 +1,154 @@
+hydra:
+  run:
+    dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task: []
+  job:
+    name: train
+    chdir: null
+    override_dirname: ''
+    id: ???
+    num: ???
+    config_name: experimental/byte_autoencoder_1
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /root/SuperTinyLanguageModels
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /root/SuperTinyLanguageModels/configs/train
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /root/SuperTinyLanguageModels/outputs/2024-09-23/08-40-08
+    choices:
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

2024-09-23/08-40-08/.hydra/overrides.yaml

@@ -0,0 +1 @@
+[]

2024-09-23/08-40-08/wandb/debug-internal.log

@@ -0,0 +1,14 @@
+{"time":"2024-09-23T09:14:22.59580271Z","level":"INFO","msg":"using version","core version":"0.18.1"}
+{"time":"2024-09-23T09:14:22.59581747Z","level":"INFO","msg":"created symlink","path":"/root/SuperTinyLanguageModels/outputs/2024-09-23/08-40-08/wandb/run-20240923_091422-a2kxhd8v/logs/debug-core.log"}
+{"time":"2024-09-23T09:14:22.595881422Z","level":"INFO","msg":"using version","core version":"0.18.1"}
+{"time":"2024-09-23T09:14:22.595887882Z","level":"INFO","msg":"created symlink","path":"/root/SuperTinyLanguageModels/outputs/2024-09-23/08-40-08/wandb/run-20240923_091422-a2kxhd8v/logs/debug-core.log"}
+{"time":"2024-09-23T09:14:22.59917443Z","level":"INFO","msg":"created new stream","id":"a2kxhd8v"}
+{"time":"2024-09-23T09:14:22.59919309Z","level":"INFO","msg":"stream: started","id":"a2kxhd8v"}
+{"time":"2024-09-23T09:14:22.59921417Z","level":"INFO","msg":"sender: started","stream_id":{"value":"a2kxhd8v"}}
+{"time":"2024-09-23T09:14:22.599226691Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"a2kxhd8v"}}
+{"time":"2024-09-23T09:14:22.599236461Z","level":"INFO","msg":"handler: started","stream_id":{"value":"a2kxhd8v"}}
+{"time":"2024-09-23T09:14:22.982350736Z","level":"INFO","msg":"wandb-core","!BADKEY":null}
+{"time":"2024-09-23T09:14:22.985015444Z","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-09-23T09:14:27.10372121Z","level":"INFO","msg":"stream: closing","id":"a2kxhd8v"}
+{"time":"2024-09-23T09:14:27.103806442Z","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-09-23T09:14:27.104964992Z","level":"INFO","msg":"Stopped system monitor"}

2024-09-23/08-40-08/wandb/debug.log

@@ -0,0 +1,26 @@
+2024-09-23 09:14:22,582 INFO    MainThread:78108 [wandb_setup.py:_flush():77] Current SDK version is 0.18.1
+2024-09-23 09:14:22,582 INFO    MainThread:78108 [wandb_setup.py:_flush():77] Configure stats pid to 78108
+2024-09-23 09:14:22,582 INFO    MainThread:78108 [wandb_setup.py:_flush():77] Loading settings from /root/.config/wandb/settings
+2024-09-23 09:14:22,583 INFO    MainThread:78108 [wandb_setup.py:_flush():77] Loading settings from /root/SuperTinyLanguageModels/outputs/2024-09-23/08-40-08/wandb/settings
+2024-09-23 09:14:22,583 INFO    MainThread:78108 [wandb_setup.py:_flush():77] Loading settings from environment variables: {}
+2024-09-23 09:14:22,583 INFO    MainThread:78108 [wandb_setup.py:_flush():77] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-09-23 09:14:22,583 INFO    MainThread:78108 [wandb_setup.py:_flush():77] Inferring run settings from compute environment: {'program_relpath': 'train.py', 'program_abspath': '/root/SuperTinyLanguageModels/train.py', 'program': '/root/SuperTinyLanguageModels/train.py'}
+2024-09-23 09:14:22,583 INFO    MainThread:78108 [wandb_setup.py:_flush():77] Applying login settings: {}
+2024-09-23 09:14:22,583 INFO    MainThread:78108 [wandb_init.py:_log_setup():532] Logging user logs to /root/SuperTinyLanguageModels/outputs/2024-09-23/08-40-08/wandb/run-20240923_091422-a2kxhd8v/logs/debug.log
+2024-09-23 09:14:22,583 INFO    MainThread:78108 [wandb_init.py:_log_setup():533] Logging internal logs to /root/SuperTinyLanguageModels/outputs/2024-09-23/08-40-08/wandb/run-20240923_091422-a2kxhd8v/logs/debug-internal.log
+2024-09-23 09:14:22,583 INFO    MainThread:78108 [wandb_init.py:init():616] calling init triggers
+2024-09-23 09:14:22,583 INFO    MainThread:78108 [wandb_init.py:init():623] wandb.init called with sweep_config: {}
+config: {'model': {'core_model_type': 'pass_through', 'hidden_dim': 384, 'byte_hidden': 128, 'max_chunk_length': 12, 'max_num_chunks': 1024, 'num_delimiter_layers': 3, 'num_byte_decoder_layers': 5, 'target_chunk_len': 8.0, 'chunk_len_loss_weight': 0.1, 'chunk_len_penalty': 0.1, 'context_window': 8192, 'embedding_model_type': 'byte_level', 'tokenizer_type': 'bpe', 'tokenizer_dataset_name': 'simple_en_wiki', 'tokenizer_simplify_data': True, 'vocab_size': 259, 'lm_head_type': 'byte_level', 'lm_head_normalization': 'rms_norm', 'lm_head_bias': False, 'lm_head_dropout': 0.0, 'model_shell_type': 'byte_autoencoder_shell', 'embedding_weight_tying': True, 'ffn_weight_tying': False, 'cproj_weight_tying': False, 'positional_encoding_type': 'rope'}, 'trainer': {'trainer_type': 'base_trainer', 'dataset': 'fineweb_edu_10B', 'batch_size': 6, 'gradient_accumulation_steps': 8, 'max_iters': 10000, 'eval_interval': 50000000, 'log_interval': 1, 'checkpoint_interval': 1000, 'eval_iters': 1000, 'run_eval': False, 'eval': {'mcq_benchmarks': None, 'mcq_num_samples': 1000, 'eval_byte_metrics': False, 'text_modeling_eval': False, 'text_generation_eval': False}, 'optimizer': {'optimizer_name': 'adamW', 'lr': 0.0005, 'min_lr': 5e-05, 'weight_decay': 0.01, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 1.0}, 'lr_scheduler': {'name': 'cosine', 'warmup_iters': 100}, 'dataloader': {'name': 'autoencoder'}, 'datasampling': {'name': 'standard'}, 'loss_fn': {'name': 'pass_through'}}, 'general': {'logging': {'wandb_log': True, 'wandb_project': 'SuperTinyLanguageModels', 'wandb_run_name': None, 'group_name': 'experimental_byte_level'}, 'paths': {'output_dir': 'outputs', 'data_dir': '/root/SuperTinyLanguageModels/data', 'checkpoint_dir': 'checkpoints', 'eval_dir': '/root/SuperTinyLanguageModels/evals'}, 'seed': 489, 'device': 'cuda'}}
+2024-09-23 09:14:22,583 INFO    MainThread:78108 [wandb_init.py:init():666] starting backend
+2024-09-23 09:14:22,583 INFO    MainThread:78108 [wandb_init.py:init():670] setting up manager
+2024-09-23 09:14:22,584 INFO    MainThread:78108 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-09-23 09:14:22,586 INFO    MainThread:78108 [wandb_init.py:init():678] backend started and connected
+2024-09-23 09:14:22,588 INFO    MainThread:78108 [wandb_init.py:init():773] updated telemetry
+2024-09-23 09:14:22,598 INFO    MainThread:78108 [wandb_init.py:init():806] communicating run to backend with 90.0 second timeout
+2024-09-23 09:14:22,974 INFO    MainThread:78108 [wandb_init.py:init():857] starting run threads in backend
+2024-09-23 09:14:23,128 INFO    MainThread:78108 [wandb_run.py:_console_start():2459] atexit reg
+2024-09-23 09:14:23,128 INFO    MainThread:78108 [wandb_run.py:_redirect():2307] redirect: wrap_raw
+2024-09-23 09:14:23,129 INFO    MainThread:78108 [wandb_run.py:_redirect():2372] Wrapping output streams.
+2024-09-23 09:14:23,129 INFO    MainThread:78108 [wandb_run.py:_redirect():2397] Redirects installed.
+2024-09-23 09:14:23,135 INFO    MainThread:78108 [wandb_init.py:init():900] run started, returning control to user process
+2024-09-23 09:14:27,104 WARNING MsgRouterThr:78108 [router.py:message_loop():77] message_loop has been closed

2024-09-23/08-40-08/wandb/run-20240923_091422-a2kxhd8v/files/config.yaml

@@ -0,0 +1,114 @@
+_wandb:
+    value:
+        cli_version: 0.18.1
+        m: []
+        python_version: 3.10.14
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 50
+                - 51
+                - 53
+                - 55
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 50
+                - 51
+                - 53
+                - 55
+            "3":
+                - 13
+                - 15
+                - 16
+                - 23
+                - 55
+            "4": 3.10.14
+            "5": 0.18.1
+            "6": 4.44.2
+            "8":
+                - 5
+                - 9
+            "12": 0.18.1
+            "13": linux-x86_64
+general:
+    value:
+        device: cuda
+        logging:
+            group_name: experimental_byte_level
+            wandb_log: true
+            wandb_project: SuperTinyLanguageModels
+            wandb_run_name: null
+        paths:
+            checkpoint_dir: checkpoints
+            data_dir: /root/SuperTinyLanguageModels/data
+            eval_dir: /root/SuperTinyLanguageModels/evals
+            output_dir: outputs
+        seed: 489
+model:
+    value:
+        byte_hidden: 128
+        chunk_len_loss_weight: 0.1
+        chunk_len_penalty: 0.1
+        context_window: 8192
+        core_model_type: pass_through
+        cproj_weight_tying: false
+        embedding_model_type: byte_level
+        embedding_weight_tying: true
+        ffn_weight_tying: false
+        hidden_dim: 384
+        lm_head_bias: false
+        lm_head_dropout: 0
+        lm_head_normalization: rms_norm
+        lm_head_type: byte_level
+        max_chunk_length: 12
+        max_num_chunks: 1024
+        model_shell_type: byte_autoencoder_shell
+        num_byte_decoder_layers: 5
+        num_delimiter_layers: 3
+        positional_encoding_type: rope
+        target_chunk_len: 8
+        tokenizer_dataset_name: simple_en_wiki
+        tokenizer_simplify_data: true
+        tokenizer_type: bpe
+        vocab_size: 259
+trainer:
+    value:
+        batch_size: 6
+        checkpoint_interval: 1000
+        dataloader:
+            name: autoencoder
+        datasampling:
+            name: standard
+        dataset: fineweb_edu_10B
+        eval:
+            eval_byte_metrics: false
+            mcq_benchmarks: null
+            mcq_num_samples: 1000
+            text_generation_eval: false
+            text_modeling_eval: false
+        eval_interval: 50000000
+        eval_iters: 1000
+        gradient_accumulation_steps: 8
+        log_interval: 1
+        loss_fn:
+            name: pass_through
+        lr_scheduler:
+            name: cosine
+            warmup_iters: 100
+        max_iters: 10000
+        optimizer:
+            beta1: 0.9
+            beta2: 0.95
+            grad_clip: 1
+            lr: 0.0005
+            min_lr: 5e-05
+            optimizer_name: adamW
+            weight_decay: 0.01
+        run_eval: false
+        trainer_type: base_trainer

2024-09-23/08-40-08/wandb/run-20240923_091422-a2kxhd8v/files/output.log

@@ -0,0 +1,3 @@
+Weight and Biases Initialized
+Rank0 Trainer built
+Training loop is starting

2024-09-23/08-40-08/wandb/run-20240923_091422-a2kxhd8v/files/requirements.txt

@@ -0,0 +1,121 @@
+setuptools==75.1.0
+wheel==0.44.0
+pip==24.2
+wcwidth==0.2.13
+sentencepiece==0.2.0
+pytz==2024.2
+mpmath==1.3.0
+distlib==0.3.8
+antlr4-python3-runtime==4.9.3
+xxhash==3.5.0
+urllib3==2.2.3
+tzdata==2024.1
+typing_extensions==4.12.2
+tqdm==4.66.5
+threadpoolctl==3.5.0
+sympy==1.13.3
+smmap==5.0.1
+six==1.16.0
+setproctitle==1.3.3
+safetensors==0.4.5
+regex==2024.9.11
+rapidfuzz==3.9.7
+PyYAML==6.0.2
+pytrec-eval-terrier==0.5.6
+pyphen==0.16.0
+Pygments==2.18.0
+psutil==6.0.0
+protobuf==5.28.2
+prettytable==3.11.0
+polars==1.7.1
+platformdirs==4.3.6
+pillow==10.4.0
+packaging==24.1
+nvidia-nvtx-cu12==12.1.105
+nvidia-nvjitlink-cu12==12.6.68
+nvidia-nccl-cu12==2.20.5
+nvidia-curand-cu12==10.3.2.106
+nvidia-cufft-cu12==11.0.2.54
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cublas-cu12==12.1.3.1
+numpy==1.26.4
+nodeenv==1.9.1
+networkx==3.3
+mdurl==0.1.2
+MarkupSafe==2.1.5
+joblib==1.4.2
+idna==3.10
+identify==2.6.1
+fsspec==2024.6.1
+frozenlist==1.4.1
+filelock==3.16.1
+eval_type_backport==0.2.0
+dill==0.3.8
+click==8.1.7
+charset-normalizer==3.3.2
+cfgv==3.4.0
+certifi==2024.8.30
+attrs==24.2.0
+async-timeout==4.0.3
+annotated-types==0.7.0
+aiohappyeyeballs==2.4.0
+virtualenv==20.26.5
+triton==3.0.0
+textstat==0.7.4
+sentry-sdk==2.14.0
+scipy==1.14.1
+requests==2.32.3
+python-dateutil==2.9.0.post0
+pydantic_core==2.23.4
+pyarrow==17.0.0
+omegaconf==2.3.0
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-cudnn-cu12==9.1.0.70
+nltk==3.9.1
+multiprocess==0.70.16
+multidict==6.1.0
+markdown-it-py==3.0.0
+Levenshtein==0.26.0
+Jinja2==3.1.4
+gitdb==4.0.11
+docker-pycreds==0.4.0
+aiosignal==1.3.1
+yarl==1.11.1
+tiktoken==0.7.0
+scikit-learn==1.5.2
+rich==13.8.1
+pydantic==2.9.2
+pre-commit==3.8.0
+pandas==2.2.3
+nvidia-cusolver-cu12==11.4.5.107
+language_tool_python==2.8.1
+hydra-core==1.3.2
+huggingface-hub==0.25.0
+GitPython==3.1.43
+wandb==0.18.1
+torch==2.4.1
+tokenizers==0.19.1
+aiohttp==3.10.5
+transformers==4.44.2
+sentence-transformers==3.1.1
+datasets==3.0.0
+mteb==1.14.21
+autocommand==2.2.2
+backports.tarfile==1.2.0
+importlib_metadata==8.0.0
+importlib_resources==6.4.0
+inflect==7.3.1
+jaraco.collections==5.1.0
+jaraco.context==5.3.0
+jaraco.functools==4.0.1
+jaraco.text==3.12.1
+more-itertools==10.3.0
+packaging==24.1
+platformdirs==4.2.2
+tomli==2.0.1
+typeguard==4.3.0
+typing_extensions==4.12.2
+wheel==0.43.0
+zipp==3.19.2

2024-09-23/08-40-08/wandb/run-20240923_091422-a2kxhd8v/files/wandb-metadata.json

@@ -0,0 +1,88 @@
+{
+  "os": "Linux-5.15.0-117-generic-x86_64-with-glibc2.31",
+  "python": "3.10.14",
+  "startedAt": "2024-09-23T09:14:22.586171Z",
+  "args": [
+    "--config-name",
+    "experimental/byte_autoencoder_1"
+  ],
+  "program": "/root/SuperTinyLanguageModels/train.py",
+  "codePath": "train.py",
+  "git": {
+    "remote": "https://github.com/LeonGuertler/SuperTinyLanguageModels.git",
+    "commit": "ebdf9039e89c5d337997d0c2b11bf4e992886243"
+  },
+  "email": "[email protected]",
+  "root": "/root/SuperTinyLanguageModels/outputs/2024-09-23/08-40-08",
+  "host": "11c6e13f6a55",
+  "username": "root",
+  "executable": "/root/SuperTinyLanguageModels/.conda/bin/python3",
+  "cpu_count": 128,
+  "cpu_count_logical": 256,
+  "gpu": "[NVIDIA GeForce RTX 4090, NVIDIA GeForce RTX 4090, NVIDIA GeForce RTX 4090, NVIDIA GeForce RTX 4090, NVIDIA GeForce RTX 4090, NVIDIA GeForce RTX 4090, NVIDIA GeForce RTX 4090, NVIDIA GeForce RTX 4090]",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1123133947904",
+      "used": "551794225152"
+    }
+  },
+  "memory": {
+    "total": "540812599296"
+  },
+  "cpu": {
+    "count": 128,
+    "countLogical": 256
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA GeForce RTX 4090",
+      "memoryTotal": "25757220864",
+      "cudaCores": 16384,
+      "architecture": "Ada"
+    },
+    {
+      "name": "NVIDIA GeForce RTX 4090",
+      "memoryTotal": "25757220864",
+      "cudaCores": 16384,
+      "architecture": "Ada"
+    },
+    {
+      "name": "NVIDIA GeForce RTX 4090",
+      "memoryTotal": "25757220864",
+      "cudaCores": 16384,
+      "architecture": "Ada"
+    },
+    {
+      "name": "NVIDIA GeForce RTX 4090",
+      "memoryTotal": "25757220864",
+      "cudaCores": 16384,
+      "architecture": "Ada"
+    },
+    {
+      "name": "NVIDIA GeForce RTX 4090",
+      "memoryTotal": "25757220864",
+      "cudaCores": 16384,
+      "architecture": "Ada"
+    },
+    {
+      "name": "NVIDIA GeForce RTX 4090",
+      "memoryTotal": "25757220864",
+      "cudaCores": 16384,
+      "architecture": "Ada"
+    },
+    {
+      "name": "NVIDIA GeForce RTX 4090",
+      "memoryTotal": "25757220864",
+      "cudaCores": 16384,
+      "architecture": "Ada"
+    },
+    {
+      "name": "NVIDIA GeForce RTX 4090",
+      "memoryTotal": "25757220864",
+      "cudaCores": 16384,
+      "architecture": "Ada"
+    }
+  ],
+  "cudaVersion": "12.5"
+}

2024-09-23/08-40-08/wandb/run-20240923_091422-a2kxhd8v/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":4}}

2024-09-23/08-40-08/wandb/run-20240923_091422-a2kxhd8v/logs/debug-core.log

@@ -0,0 +1,12 @@
+{"time":"2024-09-23T09:14:21.933081362Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmp9hgpve6u/port-78108.txt","pid":78108,"debug":false,"disable-analytics":false}
+{"time":"2024-09-23T09:14:21.933136193Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false}
+{"time":"2024-09-23T09:14:21.935284221Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":78108}
+{"time":"2024-09-23T09:14:21.935348272Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":43153,"Zone":""}}
+{"time":"2024-09-23T09:14:22.076126266Z","level":"INFO","msg":"created new connection","id":"127.0.0.1:57616"}
+{"time":"2024-09-23T09:14:22.595626377Z","level":"INFO","msg":"connection init received","streamId":"a2kxhd8v","id":"127.0.0.1:57616"}
+{"time":"2024-09-23T09:14:22.595853241Z","level":"ERROR","msg":"error creating symlink","error":"symlink /root/.cache/wandb/logs/core-debug-20240923_091421.log /root/SuperTinyLanguageModels/outputs/2024-09-23/08-40-08/wandb/run-20240923_091422-a2kxhd8v/logs/debug-core.log: file exists"}
+{"time":"2024-09-23T09:14:22.59919809Z","level":"INFO","msg":"connection init completed","streamId":"a2kxhd8v","id":"127.0.0.1:57616"}
+{"time":"2024-09-23T09:14:27.103590738Z","level":"INFO","msg":"connection: teardown","id":"127.0.0.1:57616"}
+{"time":"2024-09-23T09:14:27.103797162Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2024-09-23T09:14:27.104072727Z","level":"INFO","msg":"closed connection","id":"127.0.0.1:57616"}
+{"time":"2024-09-23T09:14:28.465863147Z","level":"INFO","msg":"Parent process exited, terminating service process."}

2024-09-23/08-40-08/wandb/run-20240923_091422-a2kxhd8v/logs/debug-internal.log

@@ -0,0 +1,14 @@
+{"time":"2024-09-23T09:14:22.59580271Z","level":"INFO","msg":"using version","core version":"0.18.1"}
+{"time":"2024-09-23T09:14:22.59581747Z","level":"INFO","msg":"created symlink","path":"/root/SuperTinyLanguageModels/outputs/2024-09-23/08-40-08/wandb/run-20240923_091422-a2kxhd8v/logs/debug-core.log"}
+{"time":"2024-09-23T09:14:22.595881422Z","level":"INFO","msg":"using version","core version":"0.18.1"}
+{"time":"2024-09-23T09:14:22.595887882Z","level":"INFO","msg":"created symlink","path":"/root/SuperTinyLanguageModels/outputs/2024-09-23/08-40-08/wandb/run-20240923_091422-a2kxhd8v/logs/debug-core.log"}
+{"time":"2024-09-23T09:14:22.59917443Z","level":"INFO","msg":"created new stream","id":"a2kxhd8v"}
+{"time":"2024-09-23T09:14:22.59919309Z","level":"INFO","msg":"stream: started","id":"a2kxhd8v"}
+{"time":"2024-09-23T09:14:22.59921417Z","level":"INFO","msg":"sender: started","stream_id":{"value":"a2kxhd8v"}}
+{"time":"2024-09-23T09:14:22.599226691Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"a2kxhd8v"}}
+{"time":"2024-09-23T09:14:22.599236461Z","level":"INFO","msg":"handler: started","stream_id":{"value":"a2kxhd8v"}}
+{"time":"2024-09-23T09:14:22.982350736Z","level":"INFO","msg":"wandb-core","!BADKEY":null}
+{"time":"2024-09-23T09:14:22.985015444Z","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-09-23T09:14:27.10372121Z","level":"INFO","msg":"stream: closing","id":"a2kxhd8v"}
+{"time":"2024-09-23T09:14:27.103806442Z","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-09-23T09:14:27.104964992Z","level":"INFO","msg":"Stopped system monitor"}

2024-09-23/08-40-08/wandb/run-20240923_091422-a2kxhd8v/logs/debug.log

@@ -0,0 +1,26 @@
+2024-09-23 09:14:22,582 INFO    MainThread:78108 [wandb_setup.py:_flush():77] Current SDK version is 0.18.1
+2024-09-23 09:14:22,582 INFO    MainThread:78108 [wandb_setup.py:_flush():77] Configure stats pid to 78108
+2024-09-23 09:14:22,582 INFO    MainThread:78108 [wandb_setup.py:_flush():77] Loading settings from /root/.config/wandb/settings
+2024-09-23 09:14:22,583 INFO    MainThread:78108 [wandb_setup.py:_flush():77] Loading settings from /root/SuperTinyLanguageModels/outputs/2024-09-23/08-40-08/wandb/settings
+2024-09-23 09:14:22,583 INFO    MainThread:78108 [wandb_setup.py:_flush():77] Loading settings from environment variables: {}
+2024-09-23 09:14:22,583 INFO    MainThread:78108 [wandb_setup.py:_flush():77] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-09-23 09:14:22,583 INFO    MainThread:78108 [wandb_setup.py:_flush():77] Inferring run settings from compute environment: {'program_relpath': 'train.py', 'program_abspath': '/root/SuperTinyLanguageModels/train.py', 'program': '/root/SuperTinyLanguageModels/train.py'}
+2024-09-23 09:14:22,583 INFO    MainThread:78108 [wandb_setup.py:_flush():77] Applying login settings: {}
+2024-09-23 09:14:22,583 INFO    MainThread:78108 [wandb_init.py:_log_setup():532] Logging user logs to /root/SuperTinyLanguageModels/outputs/2024-09-23/08-40-08/wandb/run-20240923_091422-a2kxhd8v/logs/debug.log
+2024-09-23 09:14:22,583 INFO    MainThread:78108 [wandb_init.py:_log_setup():533] Logging internal logs to /root/SuperTinyLanguageModels/outputs/2024-09-23/08-40-08/wandb/run-20240923_091422-a2kxhd8v/logs/debug-internal.log
+2024-09-23 09:14:22,583 INFO    MainThread:78108 [wandb_init.py:init():616] calling init triggers
+2024-09-23 09:14:22,583 INFO    MainThread:78108 [wandb_init.py:init():623] wandb.init called with sweep_config: {}
+config: {'model': {'core_model_type': 'pass_through', 'hidden_dim': 384, 'byte_hidden': 128, 'max_chunk_length': 12, 'max_num_chunks': 1024, 'num_delimiter_layers': 3, 'num_byte_decoder_layers': 5, 'target_chunk_len': 8.0, 'chunk_len_loss_weight': 0.1, 'chunk_len_penalty': 0.1, 'context_window': 8192, 'embedding_model_type': 'byte_level', 'tokenizer_type': 'bpe', 'tokenizer_dataset_name': 'simple_en_wiki', 'tokenizer_simplify_data': True, 'vocab_size': 259, 'lm_head_type': 'byte_level', 'lm_head_normalization': 'rms_norm', 'lm_head_bias': False, 'lm_head_dropout': 0.0, 'model_shell_type': 'byte_autoencoder_shell', 'embedding_weight_tying': True, 'ffn_weight_tying': False, 'cproj_weight_tying': False, 'positional_encoding_type': 'rope'}, 'trainer': {'trainer_type': 'base_trainer', 'dataset': 'fineweb_edu_10B', 'batch_size': 6, 'gradient_accumulation_steps': 8, 'max_iters': 10000, 'eval_interval': 50000000, 'log_interval': 1, 'checkpoint_interval': 1000, 'eval_iters': 1000, 'run_eval': False, 'eval': {'mcq_benchmarks': None, 'mcq_num_samples': 1000, 'eval_byte_metrics': False, 'text_modeling_eval': False, 'text_generation_eval': False}, 'optimizer': {'optimizer_name': 'adamW', 'lr': 0.0005, 'min_lr': 5e-05, 'weight_decay': 0.01, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 1.0}, 'lr_scheduler': {'name': 'cosine', 'warmup_iters': 100}, 'dataloader': {'name': 'autoencoder'}, 'datasampling': {'name': 'standard'}, 'loss_fn': {'name': 'pass_through'}}, 'general': {'logging': {'wandb_log': True, 'wandb_project': 'SuperTinyLanguageModels', 'wandb_run_name': None, 'group_name': 'experimental_byte_level'}, 'paths': {'output_dir': 'outputs', 'data_dir': '/root/SuperTinyLanguageModels/data', 'checkpoint_dir': 'checkpoints', 'eval_dir': '/root/SuperTinyLanguageModels/evals'}, 'seed': 489, 'device': 'cuda'}}
+2024-09-23 09:14:22,583 INFO    MainThread:78108 [wandb_init.py:init():666] starting backend
+2024-09-23 09:14:22,583 INFO    MainThread:78108 [wandb_init.py:init():670] setting up manager
+2024-09-23 09:14:22,584 INFO    MainThread:78108 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-09-23 09:14:22,586 INFO    MainThread:78108 [wandb_init.py:init():678] backend started and connected
+2024-09-23 09:14:22,588 INFO    MainThread:78108 [wandb_init.py:init():773] updated telemetry
+2024-09-23 09:14:22,598 INFO    MainThread:78108 [wandb_init.py:init():806] communicating run to backend with 90.0 second timeout
+2024-09-23 09:14:22,974 INFO    MainThread:78108 [wandb_init.py:init():857] starting run threads in backend
+2024-09-23 09:14:23,128 INFO    MainThread:78108 [wandb_run.py:_console_start():2459] atexit reg
+2024-09-23 09:14:23,128 INFO    MainThread:78108 [wandb_run.py:_redirect():2307] redirect: wrap_raw
+2024-09-23 09:14:23,129 INFO    MainThread:78108 [wandb_run.py:_redirect():2372] Wrapping output streams.
+2024-09-23 09:14:23,129 INFO    MainThread:78108 [wandb_run.py:_redirect():2397] Redirects installed.
+2024-09-23 09:14:23,135 INFO    MainThread:78108 [wandb_init.py:init():900] run started, returning control to user process
+2024-09-23 09:14:27,104 WARNING MsgRouterThr:78108 [router.py:message_loop():77] message_loop has been closed

2024-09-23/09-32-28/.hydra/config.yaml

@@ -0,0 +1,74 @@
+experimental:
+  model:
+    core_model_type: pass_through
+    hidden_dim: 384
+    byte_hidden: 128
+    max_chunk_length: 12
+    max_num_chunks: 1024
+    num_delimiter_layers: 3
+    num_byte_decoder_layers: 5
+    target_chunk_len: 8.0
+    chunk_len_loss_weight: 0.1
+    chunk_len_penalty: 0.1
+    context_window: 8192
+    embedding_model_type: byte_level
+    tokenizer_type: bpe
+    tokenizer_dataset_name: simple_en_wiki
+    tokenizer_simplify_data: true
+    vocab_size: 259
+    lm_head_type: byte_level
+    lm_head_normalization: rms_norm
+    lm_head_bias: false
+    lm_head_dropout: 0.0
+    model_shell_type: byte_autoencoder_shell
+    embedding_weight_tying: true
+    ffn_weight_tying: false
+    cproj_weight_tying: false
+    positional_encoding_type: rope
+  trainer:
+    trainer_type: base_trainer
+    dataset: fineweb_edu_10B
+    batch_size: 6
+    gradient_accumulation_steps: 8
+    max_iters: 10000
+    eval_interval: 50000000
+    log_interval: 1
+    checkpoint_interval: 1000
+    eval_iters: 1000
+    run_eval: false
+    eval:
+      mcq_benchmarks: null
+      mcq_num_samples: 1000
+      eval_byte_metrics: false
+      text_modeling_eval: false
+      text_generation_eval: false
+    optimizer:
+      optimizer_name: adamW
+      lr: 0.0005
+      min_lr: 5.0e-05
+      weight_decay: 0.01
+      beta1: 0.9
+      beta2: 0.95
+      grad_clip: 1.0
+    lr_scheduler:
+      name: cosine
+      warmup_iters: 100
+    dataloader:
+      name: autoencoder
+    datasampling:
+      name: standard
+    loss_fn:
+      name: pass_through
+  general:
+    logging:
+      wandb_log: true
+      wandb_project: SuperTinyLanguageModels
+      wandb_run_name: null
+      group_name: experimental_byte_level
+    paths:
+      output_dir: outputs
+      data_dir: data
+      checkpoint_dir: checkpoints
+      eval_dir: evals
+    seed: 489
+    device: cuda

2024-09-23/09-32-28/.hydra/hydra.yaml

@@ -0,0 +1,154 @@
+hydra:
+  run:
+    dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task: []
+  job:
+    name: train
+    chdir: null
+    override_dirname: ''
+    id: ???
+    num: ???
+    config_name: experimental/byte_autoencoder_1
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /root/SuperTinyLanguageModels
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /root/SuperTinyLanguageModels/configs/train
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /root/SuperTinyLanguageModels/outputs/2024-09-23/09-32-28
+    choices:
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

2024-09-23/09-32-28/.hydra/overrides.yaml

@@ -0,0 +1 @@
+[]

2024-09-23/09-32-28/wandb/debug-internal.log

@@ -0,0 +1,18 @@
+{"time":"2024-09-23T09:32:37.2270228Z","level":"INFO","msg":"using version","core version":"0.18.1"}
+{"time":"2024-09-23T09:32:37.227060611Z","level":"INFO","msg":"created symlink","path":"/root/SuperTinyLanguageModels/outputs/2024-09-23/09-32-28/wandb/run-20240923_093237-tkkvzfon/logs/debug-core.log"}
+{"time":"2024-09-23T09:32:37.227169702Z","level":"INFO","msg":"using version","core version":"0.18.1"}
+{"time":"2024-09-23T09:32:37.227182172Z","level":"INFO","msg":"created symlink","path":"/root/SuperTinyLanguageModels/outputs/2024-09-23/09-32-28/wandb/run-20240923_093237-tkkvzfon/logs/debug-core.log"}
+{"time":"2024-09-23T09:32:37.230824708Z","level":"INFO","msg":"created new stream","id":"tkkvzfon"}
+{"time":"2024-09-23T09:32:37.230859859Z","level":"INFO","msg":"stream: started","id":"tkkvzfon"}
+{"time":"2024-09-23T09:32:37.230903499Z","level":"INFO","msg":"sender: started","stream_id":{"value":"tkkvzfon"}}
+{"time":"2024-09-23T09:32:37.23092371Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"tkkvzfon"}}
+{"time":"2024-09-23T09:32:37.23097304Z","level":"INFO","msg":"handler: started","stream_id":{"value":"tkkvzfon"}}
+{"time":"2024-09-23T09:32:37.634282756Z","level":"INFO","msg":"wandb-core","!BADKEY":null}
+{"time":"2024-09-23T09:32:37.636527894Z","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-09-23T09:33:46.746283667Z","level":"INFO","msg":"stream: closing","id":"tkkvzfon"}
+{"time":"2024-09-23T09:33:46.746349498Z","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-09-23T09:33:46.747359311Z","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-09-23T09:33:49.926631346Z","level":"INFO","msg":"handler: closed","stream_id":{"value":"tkkvzfon"}}
+{"time":"2024-09-23T09:33:49.926725448Z","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"tkkvzfon"}}
+{"time":"2024-09-23T09:33:49.926795918Z","level":"INFO","msg":"sender: closed","stream_id":{"value":"tkkvzfon"}}
+{"time":"2024-09-23T09:33:49.927056922Z","level":"INFO","msg":"stream: closed","id":"tkkvzfon"}

2024-09-23/09-32-28/wandb/debug.log

@@ -0,0 +1,26 @@
+2024-09-23 09:32:37,221 INFO    MainThread:81916 [wandb_setup.py:_flush():77] Current SDK version is 0.18.1
+2024-09-23 09:32:37,221 INFO    MainThread:81916 [wandb_setup.py:_flush():77] Configure stats pid to 81916
+2024-09-23 09:32:37,221 INFO    MainThread:81916 [wandb_setup.py:_flush():77] Loading settings from /root/.config/wandb/settings
+2024-09-23 09:32:37,221 INFO    MainThread:81916 [wandb_setup.py:_flush():77] Loading settings from /root/SuperTinyLanguageModels/outputs/2024-09-23/09-32-28/wandb/settings
+2024-09-23 09:32:37,221 INFO    MainThread:81916 [wandb_setup.py:_flush():77] Loading settings from environment variables: {}
+2024-09-23 09:32:37,221 INFO    MainThread:81916 [wandb_setup.py:_flush():77] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-09-23 09:32:37,221 INFO    MainThread:81916 [wandb_setup.py:_flush():77] Inferring run settings from compute environment: {'program_relpath': 'train.py', 'program_abspath': '/root/SuperTinyLanguageModels/train.py', 'program': '/root/SuperTinyLanguageModels/train.py'}
+2024-09-23 09:32:37,221 INFO    MainThread:81916 [wandb_setup.py:_flush():77] Applying login settings: {}
+2024-09-23 09:32:37,221 INFO    MainThread:81916 [wandb_init.py:_log_setup():532] Logging user logs to /root/SuperTinyLanguageModels/outputs/2024-09-23/09-32-28/wandb/run-20240923_093237-tkkvzfon/logs/debug.log
+2024-09-23 09:32:37,221 INFO    MainThread:81916 [wandb_init.py:_log_setup():533] Logging internal logs to /root/SuperTinyLanguageModels/outputs/2024-09-23/09-32-28/wandb/run-20240923_093237-tkkvzfon/logs/debug-internal.log
+2024-09-23 09:32:37,222 INFO    MainThread:81916 [wandb_init.py:init():616] calling init triggers
+2024-09-23 09:32:37,222 INFO    MainThread:81916 [wandb_init.py:init():623] wandb.init called with sweep_config: {}
+config: {'model': {'core_model_type': 'pass_through', 'hidden_dim': 384, 'byte_hidden': 128, 'max_chunk_length': 12, 'max_num_chunks': 1024, 'num_delimiter_layers': 3, 'num_byte_decoder_layers': 5, 'target_chunk_len': 8.0, 'chunk_len_loss_weight': 0.1, 'chunk_len_penalty': 0.1, 'context_window': 8192, 'embedding_model_type': 'byte_level', 'tokenizer_type': 'bpe', 'tokenizer_dataset_name': 'simple_en_wiki', 'tokenizer_simplify_data': True, 'vocab_size': 259, 'lm_head_type': 'byte_level', 'lm_head_normalization': 'rms_norm', 'lm_head_bias': False, 'lm_head_dropout': 0.0, 'model_shell_type': 'byte_autoencoder_shell', 'embedding_weight_tying': True, 'ffn_weight_tying': False, 'cproj_weight_tying': False, 'positional_encoding_type': 'rope'}, 'trainer': {'trainer_type': 'base_trainer', 'dataset': 'fineweb_edu_10B', 'batch_size': 6, 'gradient_accumulation_steps': 8, 'max_iters': 10000, 'eval_interval': 50000000, 'log_interval': 1, 'checkpoint_interval': 1000, 'eval_iters': 1000, 'run_eval': False, 'eval': {'mcq_benchmarks': None, 'mcq_num_samples': 1000, 'eval_byte_metrics': False, 'text_modeling_eval': False, 'text_generation_eval': False}, 'optimizer': {'optimizer_name': 'adamW', 'lr': 0.0005, 'min_lr': 5e-05, 'weight_decay': 0.01, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 1.0}, 'lr_scheduler': {'name': 'cosine', 'warmup_iters': 100}, 'dataloader': {'name': 'autoencoder'}, 'datasampling': {'name': 'standard'}, 'loss_fn': {'name': 'pass_through'}}, 'general': {'logging': {'wandb_log': True, 'wandb_project': 'SuperTinyLanguageModels', 'wandb_run_name': None, 'group_name': 'experimental_byte_level'}, 'paths': {'output_dir': 'outputs', 'data_dir': '/root/SuperTinyLanguageModels/data', 'checkpoint_dir': 'checkpoints', 'eval_dir': '/root/SuperTinyLanguageModels/evals'}, 'seed': 489, 'device': 'cuda'}}
+2024-09-23 09:32:37,222 INFO    MainThread:81916 [wandb_init.py:init():666] starting backend
+2024-09-23 09:32:37,222 INFO    MainThread:81916 [wandb_init.py:init():670] setting up manager
+2024-09-23 09:32:37,223 INFO    MainThread:81916 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-09-23 09:32:37,224 INFO    MainThread:81916 [wandb_init.py:init():678] backend started and connected
+2024-09-23 09:32:37,227 INFO    MainThread:81916 [wandb_init.py:init():773] updated telemetry
+2024-09-23 09:32:37,236 INFO    MainThread:81916 [wandb_init.py:init():806] communicating run to backend with 90.0 second timeout
+2024-09-23 09:32:37,631 INFO    MainThread:81916 [wandb_init.py:init():857] starting run threads in backend
+2024-09-23 09:32:37,802 INFO    MainThread:81916 [wandb_run.py:_console_start():2459] atexit reg
+2024-09-23 09:32:37,802 INFO    MainThread:81916 [wandb_run.py:_redirect():2307] redirect: wrap_raw
+2024-09-23 09:32:37,803 INFO    MainThread:81916 [wandb_run.py:_redirect():2372] Wrapping output streams.
+2024-09-23 09:32:37,803 INFO    MainThread:81916 [wandb_run.py:_redirect():2397] Redirects installed.
+2024-09-23 09:32:37,806 INFO    MainThread:81916 [wandb_init.py:init():900] run started, returning control to user process
+2024-09-23 09:33:46,746 WARNING MsgRouterThr:81916 [router.py:message_loop():77] message_loop has been closed

2024-09-23/09-32-28/wandb/run-20240923_093237-tkkvzfon/files/config.yaml

@@ -0,0 +1,115 @@
+_wandb:
+    value:
+        cli_version: 0.18.1
+        m: []
+        python_version: 3.10.14
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 50
+                - 51
+                - 53
+                - 55
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 50
+                - 51
+                - 53
+                - 55
+            "3":
+                - 13
+                - 15
+                - 16
+                - 23
+                - 55
+                - 61
+            "4": 3.10.14
+            "5": 0.18.1
+            "6": 4.44.2
+            "8":
+                - 5
+                - 9
+            "12": 0.18.1
+            "13": linux-x86_64
+general:
+    value:
+        device: cuda
+        logging:
+            group_name: experimental_byte_level
+            wandb_log: true
+            wandb_project: SuperTinyLanguageModels
+            wandb_run_name: null
+        paths:
+            checkpoint_dir: checkpoints
+            data_dir: /root/SuperTinyLanguageModels/data
+            eval_dir: /root/SuperTinyLanguageModels/evals
+            output_dir: outputs
+        seed: 489
+model:
+    value:
+        byte_hidden: 128
+        chunk_len_loss_weight: 0.1
+        chunk_len_penalty: 0.1
+        context_window: 8192
+        core_model_type: pass_through
+        cproj_weight_tying: false
+        embedding_model_type: byte_level
+        embedding_weight_tying: true
+        ffn_weight_tying: false
+        hidden_dim: 384
+        lm_head_bias: false
+        lm_head_dropout: 0
+        lm_head_normalization: rms_norm
+        lm_head_type: byte_level
+        max_chunk_length: 12
+        max_num_chunks: 1024
+        model_shell_type: byte_autoencoder_shell
+        num_byte_decoder_layers: 5
+        num_delimiter_layers: 3
+        positional_encoding_type: rope
+        target_chunk_len: 8
+        tokenizer_dataset_name: simple_en_wiki
+        tokenizer_simplify_data: true
+        tokenizer_type: bpe
+        vocab_size: 259
+trainer:
+    value:
+        batch_size: 6
+        checkpoint_interval: 1000
+        dataloader:
+            name: autoencoder
+        datasampling:
+            name: standard
+        dataset: fineweb_edu_10B
+        eval:
+            eval_byte_metrics: false
+            mcq_benchmarks: null
+            mcq_num_samples: 1000
+            text_generation_eval: false
+            text_modeling_eval: false
+        eval_interval: 50000000
+        eval_iters: 1000
+        gradient_accumulation_steps: 8
+        log_interval: 1
+        loss_fn:
+            name: pass_through
+        lr_scheduler:
+            name: cosine
+            warmup_iters: 100
+        max_iters: 10000
+        optimizer:
+            beta1: 0.9
+            beta2: 0.95
+            grad_clip: 1
+            lr: 0.0005
+            min_lr: 5e-05
+            optimizer_name: adamW
+            weight_decay: 0.01
+        run_eval: false
+        trainer_type: base_trainer

2024-09-23/09-32-28/wandb/run-20240923_093237-tkkvzfon/files/output.log

@@ -0,0 +1,33 @@
+Weight and Biases Initialized
+Rank0 Trainer built
+Training loop is starting
+All GPU(s): step 1: loss 10.4062, lr 5.0e-06, dt 2.1s
+All GPU(s): step 2: loss 10.4297, lr 1.0e-05, dt 2.1s
+All GPU(s): step 3: loss 10.3672, lr 1.5e-05, dt 2.1s
+All GPU(s): step 4: loss 10.3203, lr 2.0e-05, dt 2.1s
+All GPU(s): step 5: loss 10.2344, lr 2.5e-05, dt 2.1s
+All GPU(s): step 6: loss 10.1406, lr 3.0e-05, dt 2.1s
+All GPU(s): step 7: loss 10.0234, lr 3.5e-05, dt 2.1s
+All GPU(s): step 8: loss 9.9688, lr 4.0e-05, dt 2.1s
+All GPU(s): step 9: loss 9.8594, lr 4.5e-05, dt 2.2s
+All GPU(s): step 10: loss 9.6328, lr 5.0e-05, dt 2.1s
+All GPU(s): step 11: loss 9.5312, lr 5.5e-05, dt 2.1s
+All GPU(s): step 12: loss 9.3750, lr 6.0e-05, dt 2.1s
+All GPU(s): step 13: loss 9.2109, lr 6.5e-05, dt 2.1s
+All GPU(s): step 14: loss 9.0078, lr 7.0e-05, dt 2.1s
+All GPU(s): step 15: loss 8.8203, lr 7.5e-05, dt 2.1s
+All GPU(s): step 16: loss 8.6562, lr 8.0e-05, dt 2.0s
+All GPU(s): step 17: loss 8.4922, lr 8.5e-05, dt 2.1s
+All GPU(s): step 18: loss 8.2891, lr 9.0e-05, dt 2.1s
+All GPU(s): step 19: loss 8.1328, lr 9.5e-05, dt 2.1s
+All GPU(s): step 20: loss 7.9414, lr 1.0e-04, dt 2.0s
+All GPU(s): step 21: loss 7.7852, lr 1.1e-04, dt 2.1s
+All GPU(s): step 22: loss 7.5977, lr 1.1e-04, dt 2.1s
+All GPU(s): step 23: loss 7.4453, lr 1.2e-04, dt 2.1s
+All GPU(s): step 24: loss 7.3164, lr 1.2e-04, dt 2.1s
+All GPU(s): step 25: loss 7.1836, lr 1.3e-04, dt 2.1s
+All GPU(s): step 26: loss 7.1406, lr 1.3e-04, dt 2.1s
+All GPU(s): step 27: loss 6.9414, lr 1.4e-04, dt 2.1s
+All GPU(s): step 28: loss 6.8633, lr 1.4e-04, dt 2.2s
+All GPU(s): step 29: loss 6.7461, lr 1.5e-04, dt 2.1s
+All GPU(s): step 30: loss 6.5742, lr 1.5e-04, dt 2.1s

2024-09-23/09-32-28/wandb/run-20240923_093237-tkkvzfon/files/wandb-metadata.json

@@ -0,0 +1,88 @@
+{
+  "os": "Linux-5.15.0-117-generic-x86_64-with-glibc2.31",
+  "python": "3.10.14",
+  "startedAt": "2024-09-23T09:32:37.224689Z",
+  "args": [
+    "--config-name",
+    "experimental/byte_autoencoder_1"
+  ],
+  "program": "/root/SuperTinyLanguageModels/train.py",
+  "codePath": "train.py",
+  "git": {
+    "remote": "https://github.com/LeonGuertler/SuperTinyLanguageModels.git",
+    "commit": "c36bf6b78927d4d365c52a835f0e178edacbab29"
+  },
+  "email": "[email protected]",
+  "root": "/root/SuperTinyLanguageModels/outputs/2024-09-23/09-32-28",
+  "host": "11c6e13f6a55",
+  "username": "root",
+  "executable": "/root/SuperTinyLanguageModels/.conda/bin/python3",
+  "cpu_count": 128,
+  "cpu_count_logical": 256,
+  "gpu": "[NVIDIA GeForce RTX 4090, NVIDIA GeForce RTX 4090, NVIDIA GeForce RTX 4090, NVIDIA GeForce RTX 4090, NVIDIA GeForce RTX 4090, NVIDIA GeForce RTX 4090, NVIDIA GeForce RTX 4090, NVIDIA GeForce RTX 4090]",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1123133947904",
+      "used": "551794495488"
+    }
+  },
+  "memory": {
+    "total": "540812599296"
+  },
+  "cpu": {
+    "count": 128,
+    "countLogical": 256
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA GeForce RTX 4090",
+      "memoryTotal": "25757220864",
+      "cudaCores": 16384,
+      "architecture": "Ada"
+    },
+    {
+      "name": "NVIDIA GeForce RTX 4090",
+      "memoryTotal": "25757220864",
+      "cudaCores": 16384,
+      "architecture": "Ada"
+    },
+    {
+      "name": "NVIDIA GeForce RTX 4090",
+      "memoryTotal": "25757220864",
+      "cudaCores": 16384,
+      "architecture": "Ada"
+    },
+    {
+      "name": "NVIDIA GeForce RTX 4090",
+      "memoryTotal": "25757220864",
+      "cudaCores": 16384,
+      "architecture": "Ada"
+    },
+    {
+      "name": "NVIDIA GeForce RTX 4090",
+      "memoryTotal": "25757220864",
+      "cudaCores": 16384,
+      "architecture": "Ada"
+    },
+    {
+      "name": "NVIDIA GeForce RTX 4090",
+      "memoryTotal": "25757220864",
+      "cudaCores": 16384,
+      "architecture": "Ada"
+    },
+    {
+      "name": "NVIDIA GeForce RTX 4090",
+      "memoryTotal": "25757220864",
+      "cudaCores": 16384,
+      "architecture": "Ada"
+    },
+    {
+      "name": "NVIDIA GeForce RTX 4090",
+      "memoryTotal": "25757220864",
+      "cudaCores": 16384,
+      "architecture": "Ada"
+    }
+  ],
+  "cudaVersion": "12.5"
+}

2024-09-23/09-32-28/wandb/run-20240923_093237-tkkvzfon/files/wandb-summary.json

@@ -0,0 +1 @@
+{"additional_info/chunk_len_penalty_loss":0,"additional_info/total-loss":6.543508529663086,"_step":1474560,"additional_info/chunk_len_loss":2.0561606884002686,"iter":30,"token_num":1474560,"additional_info/BCE-loss":4.487347602844238,"loss":6.57421875,"lr":0.00015,"_timestamp":1.7270840240730202e+09,"_runtime":69.521643938,"additional_info/average_chunk_length":3.4655094146728516,"_wandb":{"runtime":69}}

2024-09-23/09-32-28/wandb/run-20240923_093237-tkkvzfon/logs/debug-core.log

@@ -0,0 +1,13 @@
+{"time":"2024-09-23T09:32:36.53490736Z","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmppr55fcxh/port-81916.txt","pid":81916,"debug":false,"disable-analytics":false}
+{"time":"2024-09-23T09:32:36.534984841Z","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false}
+{"time":"2024-09-23T09:32:36.551541231Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":81916}
+{"time":"2024-09-23T09:32:36.55148544Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":44587,"Zone":""}}
+{"time":"2024-09-23T09:32:36.722786198Z","level":"INFO","msg":"created new connection","id":"127.0.0.1:60908"}
+{"time":"2024-09-23T09:32:37.226730857Z","level":"INFO","msg":"connection init received","streamId":"tkkvzfon","id":"127.0.0.1:60908"}
+{"time":"2024-09-23T09:32:37.227116001Z","level":"ERROR","msg":"error creating symlink","error":"symlink /root/.cache/wandb/logs/core-debug-20240923_093236.log /root/SuperTinyLanguageModels/outputs/2024-09-23/09-32-28/wandb/run-20240923_093237-tkkvzfon/logs/debug-core.log: file exists"}
+{"time":"2024-09-23T09:32:37.230871019Z","level":"INFO","msg":"connection init completed","streamId":"tkkvzfon","id":"127.0.0.1:60908"}
+{"time":"2024-09-23T09:33:46.746114105Z","level":"INFO","msg":"connection: teardown","id":"127.0.0.1:60908"}
+{"time":"2024-09-23T09:33:46.746363968Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2024-09-23T09:33:46.746627582Z","level":"INFO","msg":"closed connection","id":"127.0.0.1:60908"}
+{"time":"2024-09-23T09:33:49.927260015Z","level":"INFO","msg":"connection closed","id":"127.0.0.1:60908"}
+{"time":"2024-09-23T09:33:49.927297555Z","level":"INFO","msg":"server is closed"}

2024-09-23/09-32-28/wandb/run-20240923_093237-tkkvzfon/logs/debug-internal.log

@@ -0,0 +1,18 @@
+{"time":"2024-09-23T09:32:37.2270228Z","level":"INFO","msg":"using version","core version":"0.18.1"}
+{"time":"2024-09-23T09:32:37.227060611Z","level":"INFO","msg":"created symlink","path":"/root/SuperTinyLanguageModels/outputs/2024-09-23/09-32-28/wandb/run-20240923_093237-tkkvzfon/logs/debug-core.log"}
+{"time":"2024-09-23T09:32:37.227169702Z","level":"INFO","msg":"using version","core version":"0.18.1"}
+{"time":"2024-09-23T09:32:37.227182172Z","level":"INFO","msg":"created symlink","path":"/root/SuperTinyLanguageModels/outputs/2024-09-23/09-32-28/wandb/run-20240923_093237-tkkvzfon/logs/debug-core.log"}
+{"time":"2024-09-23T09:32:37.230824708Z","level":"INFO","msg":"created new stream","id":"tkkvzfon"}
+{"time":"2024-09-23T09:32:37.230859859Z","level":"INFO","msg":"stream: started","id":"tkkvzfon"}
+{"time":"2024-09-23T09:32:37.230903499Z","level":"INFO","msg":"sender: started","stream_id":{"value":"tkkvzfon"}}
+{"time":"2024-09-23T09:32:37.23092371Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"tkkvzfon"}}
+{"time":"2024-09-23T09:32:37.23097304Z","level":"INFO","msg":"handler: started","stream_id":{"value":"tkkvzfon"}}
+{"time":"2024-09-23T09:32:37.634282756Z","level":"INFO","msg":"wandb-core","!BADKEY":null}
+{"time":"2024-09-23T09:32:37.636527894Z","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-09-23T09:33:46.746283667Z","level":"INFO","msg":"stream: closing","id":"tkkvzfon"}
+{"time":"2024-09-23T09:33:46.746349498Z","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-09-23T09:33:46.747359311Z","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-09-23T09:33:49.926631346Z","level":"INFO","msg":"handler: closed","stream_id":{"value":"tkkvzfon"}}
+{"time":"2024-09-23T09:33:49.926725448Z","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"tkkvzfon"}}
+{"time":"2024-09-23T09:33:49.926795918Z","level":"INFO","msg":"sender: closed","stream_id":{"value":"tkkvzfon"}}
+{"time":"2024-09-23T09:33:49.927056922Z","level":"INFO","msg":"stream: closed","id":"tkkvzfon"}

2024-09-23/09-32-28/wandb/run-20240923_093237-tkkvzfon/logs/debug.log

@@ -0,0 +1,26 @@
+2024-09-23 09:32:37,221 INFO    MainThread:81916 [wandb_setup.py:_flush():77] Current SDK version is 0.18.1
+2024-09-23 09:32:37,221 INFO    MainThread:81916 [wandb_setup.py:_flush():77] Configure stats pid to 81916
+2024-09-23 09:32:37,221 INFO    MainThread:81916 [wandb_setup.py:_flush():77] Loading settings from /root/.config/wandb/settings
+2024-09-23 09:32:37,221 INFO    MainThread:81916 [wandb_setup.py:_flush():77] Loading settings from /root/SuperTinyLanguageModels/outputs/2024-09-23/09-32-28/wandb/settings
+2024-09-23 09:32:37,221 INFO    MainThread:81916 [wandb_setup.py:_flush():77] Loading settings from environment variables: {}
+2024-09-23 09:32:37,221 INFO    MainThread:81916 [wandb_setup.py:_flush():77] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-09-23 09:32:37,221 INFO    MainThread:81916 [wandb_setup.py:_flush():77] Inferring run settings from compute environment: {'program_relpath': 'train.py', 'program_abspath': '/root/SuperTinyLanguageModels/train.py', 'program': '/root/SuperTinyLanguageModels/train.py'}
+2024-09-23 09:32:37,221 INFO    MainThread:81916 [wandb_setup.py:_flush():77] Applying login settings: {}
+2024-09-23 09:32:37,221 INFO    MainThread:81916 [wandb_init.py:_log_setup():532] Logging user logs to /root/SuperTinyLanguageModels/outputs/2024-09-23/09-32-28/wandb/run-20240923_093237-tkkvzfon/logs/debug.log
+2024-09-23 09:32:37,221 INFO    MainThread:81916 [wandb_init.py:_log_setup():533] Logging internal logs to /root/SuperTinyLanguageModels/outputs/2024-09-23/09-32-28/wandb/run-20240923_093237-tkkvzfon/logs/debug-internal.log
+2024-09-23 09:32:37,222 INFO    MainThread:81916 [wandb_init.py:init():616] calling init triggers
+2024-09-23 09:32:37,222 INFO    MainThread:81916 [wandb_init.py:init():623] wandb.init called with sweep_config: {}
+config: {'model': {'core_model_type': 'pass_through', 'hidden_dim': 384, 'byte_hidden': 128, 'max_chunk_length': 12, 'max_num_chunks': 1024, 'num_delimiter_layers': 3, 'num_byte_decoder_layers': 5, 'target_chunk_len': 8.0, 'chunk_len_loss_weight': 0.1, 'chunk_len_penalty': 0.1, 'context_window': 8192, 'embedding_model_type': 'byte_level', 'tokenizer_type': 'bpe', 'tokenizer_dataset_name': 'simple_en_wiki', 'tokenizer_simplify_data': True, 'vocab_size': 259, 'lm_head_type': 'byte_level', 'lm_head_normalization': 'rms_norm', 'lm_head_bias': False, 'lm_head_dropout': 0.0, 'model_shell_type': 'byte_autoencoder_shell', 'embedding_weight_tying': True, 'ffn_weight_tying': False, 'cproj_weight_tying': False, 'positional_encoding_type': 'rope'}, 'trainer': {'trainer_type': 'base_trainer', 'dataset': 'fineweb_edu_10B', 'batch_size': 6, 'gradient_accumulation_steps': 8, 'max_iters': 10000, 'eval_interval': 50000000, 'log_interval': 1, 'checkpoint_interval': 1000, 'eval_iters': 1000, 'run_eval': False, 'eval': {'mcq_benchmarks': None, 'mcq_num_samples': 1000, 'eval_byte_metrics': False, 'text_modeling_eval': False, 'text_generation_eval': False}, 'optimizer': {'optimizer_name': 'adamW', 'lr': 0.0005, 'min_lr': 5e-05, 'weight_decay': 0.01, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 1.0}, 'lr_scheduler': {'name': 'cosine', 'warmup_iters': 100}, 'dataloader': {'name': 'autoencoder'}, 'datasampling': {'name': 'standard'}, 'loss_fn': {'name': 'pass_through'}}, 'general': {'logging': {'wandb_log': True, 'wandb_project': 'SuperTinyLanguageModels', 'wandb_run_name': None, 'group_name': 'experimental_byte_level'}, 'paths': {'output_dir': 'outputs', 'data_dir': '/root/SuperTinyLanguageModels/data', 'checkpoint_dir': 'checkpoints', 'eval_dir': '/root/SuperTinyLanguageModels/evals'}, 'seed': 489, 'device': 'cuda'}}
+2024-09-23 09:32:37,222 INFO    MainThread:81916 [wandb_init.py:init():666] starting backend
+2024-09-23 09:32:37,222 INFO    MainThread:81916 [wandb_init.py:init():670] setting up manager
+2024-09-23 09:32:37,223 INFO    MainThread:81916 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-09-23 09:32:37,224 INFO    MainThread:81916 [wandb_init.py:init():678] backend started and connected
+2024-09-23 09:32:37,227 INFO    MainThread:81916 [wandb_init.py:init():773] updated telemetry
+2024-09-23 09:32:37,236 INFO    MainThread:81916 [wandb_init.py:init():806] communicating run to backend with 90.0 second timeout
+2024-09-23 09:32:37,631 INFO    MainThread:81916 [wandb_init.py:init():857] starting run threads in backend
+2024-09-23 09:32:37,802 INFO    MainThread:81916 [wandb_run.py:_console_start():2459] atexit reg
+2024-09-23 09:32:37,802 INFO    MainThread:81916 [wandb_run.py:_redirect():2307] redirect: wrap_raw
+2024-09-23 09:32:37,803 INFO    MainThread:81916 [wandb_run.py:_redirect():2372] Wrapping output streams.
+2024-09-23 09:32:37,803 INFO    MainThread:81916 [wandb_run.py:_redirect():2397] Redirects installed.
+2024-09-23 09:32:37,806 INFO    MainThread:81916 [wandb_init.py:init():900] run started, returning control to user process
+2024-09-23 09:33:46,746 WARNING MsgRouterThr:81916 [router.py:message_loop():77] message_loop has been closed

2024-09-23/09-33-58/.hydra/config.yaml

@@ -0,0 +1,74 @@
+experimental:
+  model:
+    core_model_type: pass_through
+    hidden_dim: 384
+    byte_hidden: 128
+    max_chunk_length: 12
+    max_num_chunks: 1024
+    num_delimiter_layers: 3
+    num_byte_decoder_layers: 5
+    target_chunk_len: 8.0
+    chunk_len_loss_weight: 0.1
+    chunk_len_penalty: 0.1
+    context_window: 8192
+    embedding_model_type: byte_level
+    tokenizer_type: bpe
+    tokenizer_dataset_name: simple_en_wiki
+    tokenizer_simplify_data: true
+    vocab_size: 259
+    lm_head_type: byte_level
+    lm_head_normalization: rms_norm
+    lm_head_bias: false
+    lm_head_dropout: 0.0
+    model_shell_type: byte_autoencoder_shell
+    embedding_weight_tying: true
+    ffn_weight_tying: false
+    cproj_weight_tying: false
+    positional_encoding_type: rope
+  trainer:
+    trainer_type: base_trainer
+    dataset: fineweb_edu_10B
+    batch_size: 6
+    gradient_accumulation_steps: 8
+    max_iters: 10000
+    eval_interval: 50000000
+    log_interval: 1
+    checkpoint_interval: 1000
+    eval_iters: 1000
+    run_eval: false
+    eval:
+      mcq_benchmarks: null
+      mcq_num_samples: 1000
+      eval_byte_metrics: false
+      text_modeling_eval: false
+      text_generation_eval: false
+    optimizer:
+      optimizer_name: adamW
+      lr: 0.0005
+      min_lr: 5.0e-05
+      weight_decay: 0.01
+      beta1: 0.9
+      beta2: 0.95
+      grad_clip: 1.0
+    lr_scheduler:
+      name: cosine
+      warmup_iters: 100
+    dataloader:
+      name: autoencoder
+    datasampling:
+      name: standard
+    loss_fn:
+      name: pass_through
+  general:
+    logging:
+      wandb_log: true
+      wandb_project: SuperTinyLanguageModels
+      wandb_run_name: null
+      group_name: experimental_byte_level
+    paths:
+      output_dir: outputs
+      data_dir: data
+      checkpoint_dir: checkpoints
+      eval_dir: evals
+    seed: 489
+    device: cuda

2024-09-23/09-33-58/.hydra/hydra.yaml

@@ -0,0 +1,154 @@
+hydra:
+  run:
+    dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task: []
+  job:
+    name: train
+    chdir: null
+    override_dirname: ''
+    id: ???
+    num: ???
+    config_name: experimental/byte_autoencoder_1
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /root/SuperTinyLanguageModels
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /root/SuperTinyLanguageModels/configs/train
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /root/SuperTinyLanguageModels/outputs/2024-09-23/09-33-58
+    choices:
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

2024-09-23/09-33-58/.hydra/overrides.yaml

@@ -0,0 +1 @@
+[]

2024-09-23/09-33-58/checkpoints/ckpt_1000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9e847e5371dfd2f5ac68ee97e737d4ab63d42fdde1c885d6ab4915a9b3ccf83
+size 69377274

2024-09-23/09-33-58/checkpoints/ckpt_2000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:261a3f997548dd7b6a92a1a7a51b37b1d559a7b64547c95b98a336bdc2685da0
+size 69377274

2024-09-23/09-33-58/checkpoints/ckpt_3000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51018e44f695f15948d2cbcd014d62113a7a82a67ca7ca25dc767a77c12ae563
+size 69377274

2024-09-23/09-33-58/checkpoints/ckpt_4000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf165a859555ddeb74ad0c7b6e10f17fa5f91c1b060a14bd77dd7fedbde5503c
+size 69377274

2024-09-23/09-33-58/checkpoints/ckpt_5000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99bdcae468dc981532ae56ecd8616824b1cf86801d364510be19a57467a81dbb
+size 69377274