llama_vs_gemma_tokenizer

Created Diff never expires
13 removals
50 lines
226 additions
262 lines
normalizer_spec {
normalizer_spec {
name: "identity"
name: "identity"
precompiled_charsmap: ""
precompiled_charsmap: ""
add_dummy_prefix: true
add_dummy_prefix: false
remove_extra_whitespaces: false
remove_extra_whitespaces: false
normalization_rule_tsv: ""
normalization_rule_tsv: ""
}
}


trainer_spec {
trainer_spec {
input: "/large_experiments/theorem/datasets/MERGED/all.test1.merged"
model_prefix: "/cns/mf-d/home/gemini-data-access/tokenizers/final_v1_51GB_run1/bpe_coverage_0_999995_v5/255969"
model_prefix: "spm_model_32k_200M_charcov099995_allowWSO__v2"
model_type: BPE
model_type: BPE
vocab_size: 32000
vocab_size: 256000
self_test_sample_size: 0
self_test_sample_size: 0
input_format: "text"
input_format: ""
character_coverage: 0.99995
character_coverage: 0.999995
input_sentence_size: 200000000
input_sentence_size: 2000000000
seed_sentencepiece_size: 1000000
seed_sentencepiece_size: 1000000
shrinking_factor: 0.75
shrinking_factor: 0.75
num_threads: 80
num_threads: 16
num_sub_iterations: 2
num_sub_iterations: 2
max_sentence_length: 4192
max_sentence_length: 4192
shuffle_input_sentence: true
shuffle_input_sentence: true
max_sentencepiece_length: 16
max_sentencepiece_length: 16
split_by_unicode_script: true
split_by_unicode_script: true
split_by_whitespace: true
split_by_whitespace: true
split_by_number: true
split_by_number: true
treat_whitespace_as_suffix: false
treat_whitespace_as_suffix: false
split_digits: true
split_digits: true
allow_whitespace_only_pieces: true
allow_whitespace_only_pieces: true
user_defined_symbols: "<mask>"
user_defined_symbols: "<2mass>"
user_defined_symbols: "[@BOS@]"
user_defined_symbols: "<unused0>"
user_defined_symbols: "<unused1>"
user_defined_symbols: "<unused2>"
user_defined_symbols: "<unused3>"
user_defined_symbols: "<unused4>"
user_defined_symbols: "<unused5>"
user_defined_symbols: "<unused6>"
user_defined_symbols: "<unused7>"
user_defined_symbols: "<unused8>"
user_defined_symbols: "<unused9>"
user_defined_symbols: "<unused10>"
user_defined_symbols: "<unused11>"
user_defined_symbols: "<unused12>"
user_defined_symbols: "<unused13>"
user_defined_symbols: "<unused14>"
user_defined_symbols: "<unused15>"
user_defined_symbols: "<unused16>"
user_defined_symbols: "<unused17>"
user_defined_symbols: "<unused18>"
user_defined_symbols: "<unused19>"
user_defined_symbols: "<unused20>"
user_defined_symbols: "<unused21>"
user_defined_symbols: "<unused22>"
user_defined_symbols: "<unused23>"
user_defined_symbols: "<unused24>"
user_defined_symbols: "<unused25>"
user_defined_symbols: "<unused26>"
user_defined_symbols: "<unused27>"
user_defined_symbols: "<unused28>"
user_defined_symbols: "<unused29>"
user_defined_symbols: "<unused30>"
user_defined_symbols: "<unused31>"
user_defined_symbols: "<unused32>"
user_defined_symbols: "<unused33>"
user_defined_symbols: "<unused34>"
user_defined_symbols: "<unused35>"
user_defined_symbols: "<unused36>"
user_defined_symbols: "<unused37>"
user_defined_symbols: "<unused38>"
user_defined_symbols: "<unused39>"
user_defined_symbols: "<unused40>"
user_defined_symbols: "<unused41>"
user_defined_symbols: "<unused42>"
user_defined_symbols: "<unused43>"
user_defined_symbols: "<unused44>"
user_defined_symbols: "<unused45>"
user_defined_symbols: "<unused46>"
user_defined_symbols: "<unused47>"
user_defined_symbols: "<unused48>"
user_defined_symbols: "<unused49>"
user_defined_symbols: "<unused50>"
user_defined_symbols: "<unused51>"
user_defined_symbols: "<unused52>"
user_defined_symbols: "<unused53>"
user_defined_symbols: "<unused54>"
user_defined_symbols: "<unused55>"
user_defined_symbols: "<unused56>"
user_defined_symbols: "<unused57>"
user_defined_symbols: "<unused58>"
user_defined_symbols: "<unused59>"
user_defined_symbols: "<unused60>"
user_defined_symbols: "<unused61>"
user_defined_symbols: "<unused62>"
user_defined_symbols: "<unused63>"
user_defined_symbols: "<unused64>"
user_defined_symbols: "<unused65>"
user_defined_symbols: "<unused66>"