activation_checkpointing: true
activation_offloading: false
add_special_tokens: true
batch_size: 16
dataset_dir: .cache/imdb
decoder: !!python/object:perceiver.model.text.mlm.TextDecoderConfig
  cross_attention_residual: false
  cross_attention_widening_factor: 1
  dropout: 0.1
  freeze: false
  init_scale: 0.02
  max_seq_len: 2048
  num_cross_attention_heads: 8
  num_cross_attention_qk_channels: 256
  num_cross_attention_v_channels: 768
  num_output_query_channels: null
  vocab_size: 262
encoder: !!python/object:perceiver.model.text.common.TextEncoderConfig
  cross_attention_widening_factor: 1
  dropout: 0.1
  first_cross_attention_layer_shared: false
  first_self_attention_block_shared: true
  freeze: false
  init_scale: 0.02
  max_seq_len: 2048
  num_cross_attention_heads: 8
  num_cross_attention_layers: 1
  num_cross_attention_qk_channels: 256
  num_cross_attention_v_channels: 1280
  num_input_channels: 768
  num_self_attention_blocks: 1
  num_self_attention_heads: 8
  num_self_attention_layers_per_block: 26
  num_self_attention_qk_channels: 256
  num_self_attention_v_channels: 1280
  params: null
  self_attention_widening_factor: 1
  vocab_size: 262
mask_prob: 0.15
mask_words: true
masked_samples:
- I watched this<mask><mask><mask><mask> and it was awesome.
- I watched this<mask><mask><mask><mask><mask> and it was awesome.
- I watched this<mask><mask><mask><mask><mask><mask> and it was awesome.
- I watched this<mask><mask><mask><mask><mask><mask><mask> and it was awesome.
- I watched this<mask><mask><mask><mask><mask><mask><mask><mask> and it was awesome.
- I watched this<mask><mask><mask><mask><mask><mask><mask><mask><mask> and it was
  awesome.
- I watched this<mask><mask><mask><mask><mask><mask><mask><mask><mask><mask> and it
  was awesome.
max_seq_len: 2048
num_latent_channels: 1280
num_latents: 256
num_predictions: 5
num_workers: 3
params: deepmind/language-perceiver
pin_memory: true
preproc_batch_size: 1000
preproc_workers: 1
random_train_shift: false
random_valid_shift: false
static_masking: false
task: mlm
tokenizer: deepmind/language-perceiver
