activation_checkpointing: false
activation_offloading: false
add_special_tokens: false
batch_size: 24
cross_attention_dropout: 0.5
cross_attention_widening_factor: 4
dataset_dir: .cache/wikitext
init_scale: 0.02
mask_prob: 0.15
mask_words: true
max_heads_parallel: null
max_seq_len: 4096
num_channels: 512
num_heads: 8
num_latents: 512
num_self_attention_layers: 8
num_workers: 3
padding_side: left
pin_memory: true
post_attention_dropout: 0.0
preproc_batch_size: 1000
preproc_workers: null
random_min_seq_len: 16
random_train_shift: true
random_train_truncation: false
random_valid_shift: false
random_valid_truncation: false
self_attention_widening_factor: 4
static_masking: false
task: clm
tokenizer: deepmind/language-perceiver
valid_batch_size: null
validation_sample_prompt: null
validation_sample_record: null
vocab_size: 262
