# pytorch_lightning==1.8.1
seed_everything: true
trainer:
  logger:
    class_path: pytorch_lightning.loggers.TensorBoardLogger
    init_args:
      save_dir: logs
      name: clm
      version: null
      log_graph: false
      default_hp_metric: true
      prefix: ''
      sub_dir: null
      comment: ''
      purge_step: null
      max_queue: 10
      flush_secs: 120
      filename_suffix: ''
  enable_checkpointing: true
  callbacks:
  - class_path: pytorch_lightning.callbacks.LearningRateMonitor
    init_args:
      logging_interval: step
      log_momentum: false
  - class_path: pytorch_lightning.callbacks.ModelCheckpoint
    init_args:
      dirpath: null
      filename: '{epoch:03d}-{val_loss:.3f}'
      monitor: val_loss
      verbose: false
      save_last: null
      save_top_k: 1
      save_weights_only: true
      mode: min
      auto_insert_metric_name: true
      every_n_train_steps: null
      train_time_interval: null
      every_n_epochs: null
      save_on_train_epoch_end: null
  default_root_dir: logs
  gradient_clip_val: 0.5
  gradient_clip_algorithm: null
  num_nodes: 1
  num_processes: null
  devices: '2'
  gpus: null
  auto_select_gpus: false
  tpu_cores: null
  ipus: null
  enable_progress_bar: true
  overfit_batches: 0.0
  track_grad_norm: -1
  check_val_every_n_epoch: 1
  fast_dev_run: false
  accumulate_grad_batches: 2
  max_epochs: 8
  min_epochs: null
  max_steps: -1
  min_steps: null
  max_time: null
  limit_train_batches: null
  limit_val_batches: null
  limit_test_batches: null
  limit_predict_batches: null
  val_check_interval: 0.5
  log_every_n_steps: 50
  accelerator: gpu
  strategy: ddp_find_unused_parameters_false
  sync_batchnorm: false
  precision: 32
  enable_model_summary: true
  num_sanity_val_steps: 2
  resume_from_checkpoint: null
  profiler: null
  benchmark: null
  deterministic: null
  reload_dataloaders_every_n_epochs: 0
  auto_lr_find: false
  replace_sampler_ddp: true
  detect_anomaly: false
  auto_scale_batch_size: false
  plugins: null
  amp_backend: native
  amp_level: null
  move_metrics_to_cpu: false
  multiple_trainloader_mode: max_size_cycle
  inference_mode: true
model:
  num_channels: 512
  num_latents: 512
  num_heads: 8
  num_self_attention_layers: 8
  self_attention_widening_factor: 4
  cross_attention_widening_factor: 4
  cross_attention_dropout: 0.5
  post_attention_dropout: 0.0
  random_truncation: false
  random_min_seq_len: 16
  init_scale: 0.02
  activation_checkpointing: false
  activation_offloading: false
  validation_sample_prompt: The city was crowded with peo
  validation_sample_record: -1
lr_scheduler:
  warmup_steps: 200
  last_epoch: -1
ckpt_path: null
data:
  class_path: perceiver.data.text.WikiTextDataModule
  init_args:
    dataset_dir: .cache/wikitext
    tokenizer: deepmind/language-perceiver
    max_seq_len: 4096
    task: clm
    mask_prob: 0.15
    mask_words: true
    static_masking: false
    add_special_tokens: false
    random_train_shift: false
    random_valid_shift: false
    preproc_batch_size: 1000
    preproc_workers: null
    batch_size: 24
    num_workers: 3
    pin_memory: true
optimizer:
  class_path: torch.optim.Adam
  init_args:
    lr: 0.0002
    betas:
    - 0.9
    - 0.999
    eps: 1.0e-08
    weight_decay: 0
    amsgrad: false
    foreach: null
    maximize: false
    capturable: false
