L ivDdZddlmZddlZddlmZmZmZmZm Z ddl m Z m Z ddl mZddlmZmZmZmZmZdd lmZdd lmZej2eZid d d ddddddddddddd dddddddddddd dddddddddddd dddddddddddddd d!ddddddddddd"d# d$ddddddddddd d%dddddddddddddd& d'ddddddddddd d(d)d*d+d,d-d.ddddddddddd d/d0dd1dd2d3d4d5d6d7d8d9d:dd2d;ddddddddd?dd@ dAd0dBdCdDd1d2dEdFdddddddGdHdIdddd2ddJdKdLddM dddddddddNdd dddddddddddOddP dddddddddddOddP d5d6d7d8d9d:dd2d;de#dAe#dFe#d4e$dIe#dle#dme%dne%doe$e e dpZ&dqe'dre(ee)ffdsZ*y)tz Integration with GGML / The file is copied and adapted from https://github.com/99991/pygguf with extra methods beings exposed )arrayN) Tokenizerdecoders normalizerspre_tokenizers processors)BPEUnigram) AddedToken)GemmaConverter GPT2ConverterLlamaConverterQwen2Converter T5Converter)logging)tqdmgeneral model_type_model_name_or_path) architecturenamellamamax_position_embeddingsnum_hidden_layersintermediate_size hidden_sizehead_dim rope_thetanum_attention_headsnum_key_value_heads rms_norm_eps vocab_size) context_length block_countfeed_forward_lengthembedding_lengthrope.dimension_countrope.freq_baseattention.head_countattention.head_count_kv attention.layer_norm_rms_epsilonr#mistralqwen2qwen2moe num_expertsnum_experts_per_tok) r$r%r&r'r(r)r*r+r,r# expert_countexpert_used_countlfm2 conv_L_cache) r$r%r&r'r(r)r*r+r,r#zshortconv.l_cacheqwen3 qwen3_moe) r$r%r&r'r(r)attention.key_lengthr*r+r,r#r2r3falcon tokenizer bos_token_id eos_token_id unk_token_id pad_token_id)ggml.bos_token_idggml.eos_token_idggml.unknown_token_idggml.padding_token_idphi3bloomn_layern_headlayer_norm_epsilon)r%r'r*r#attention.layer_norm_epsilont5 n_positions num_layersd_ffd_modeld_kv num_headsrelative_attention_num_bucketsdecoder_start_token_id) r$r%r&r'r8r*r+rHz attention.relative_buckets_countrQr#stablelmlayer_norm_eps) r$r%r&r'r(r*r+rHr#gpt2n_ctxn_embdr&)r%r$r'r&r*rH starcoder2 norm_epsilon)r%r$r'r&r*r+rHmamba conv_kernel state_sizetime_step_rank) r#r$r'r,r%zssm.conv_kernelzssm.state_sizezssm.time_step_rankzssm.inner_sizenorm_epssliding_window) r$r%r&r'r(r)r8r*r+r,zattention.sliding_windowr#)nemotrongemma2gemma3umt5decitokenizer_typetokensscores token_typemergesadd_prefix_space) ggml.modelz ggml.tokensz ggml.scoreszggml.token_typez ggml.mergesr?r@rArBzggml.add_space_prefix chat_template)rkrjr?r@rArB)r:tokenizer_configct|ts|g}t|dk(r|d}d}n|ddk7r td|\}}|dvrt |d}|S|dvrt |d}|S|dk(rt |d}|S|dk(r3td t|jj}|S|dk(r t||}|S) Nr zPReceived multiple types, therefore expected the first type to indicate an array.)rrnr  ) B) isinstancelistlen ValueErrorintfloatboolrtobytesdecode_gguf_parse_value)_value data_typearray_data_types d/mnt/ssd/data/python-lab/Trading/venv/lib/python3.12/site-packages/transformers/integrations/ggml.pyrr=s i &K  9~aL  Q<1 op p%." ?..VAY M g vay! M afQi M asDL)113::< M a"6?; MceZdZdZy)GGUFTokenizerSkeletoncp|jD]\}}t|||t|ds)t|dr t|ds td|j}|j }t |Dcic] \}}||| c}}tjdg}tjD]k\} } g} tdt| D]*} | d| | | d}} | |vs||vs| j| || f,t| fdd } |j| mt|d d }|Dcgc] }|d |df}}||_nt|jD cgc]} t!| j#d c} |_t|ds1tt|jDcgc]}dc}|_t|d sg|_t|dsd|_t|dr|j&|j(|_yyycc}}wcc}wcc} wcc}w)Nrhrerfz\tokens and scores need to be passed for a LLaMa tokenizer without merges to be instantiated.z:Merges were not in checkpoint, building merges on the fly.rnc$|d|dfS)Nrrn)xvocabs rz0GGUFTokenizerSkeleton.__init__..ksU1Q4[%!+4NrT)keyreversec |dS)Nr r)vals rrz0GGUFTokenizerSkeleton.__init__..ms CFrr added_tokensr=unknown_token_id)itemssetattrhasattrr}rerf enumerateloggerwarningrranger|appendsortedextendrhtuplesplitrr=r)selfdict_kvrerfitrhmerge piece_scorelocalindexpiece_lpiece_rr_rs @r__init__zGGUFTokenizerSkeleton.__init__Vs KKM DAq D!Q  tX&4*'$2I r[[F[[F.7.?@daQq \@E NNW XF&*5;;=&9 %"{"1c%j1FE',Ve}eEFmWG&(W-> gw %DEFu*NX\] e$ %F(:DIF2893s1vs1v&9F9 DK@D Lu5S!12LDK4*-23t{{3C-DEtE t^, "D t^, $D  4+ ,1B1B1J $ 5 5D 2K ,7A:MEs=H#H)+!H. H3N)__name__ __module__ __qualname__rrrrrrUs'6rrc0eZdZdZdZdZdZdZdZy)GGUFLlamaConverterct||_|j|_i|_t |jdddk7|_y)Nrdr)rprotooriginal_tokenizeradditional_kwargsgetattris_llama_3_tokenizerrtokenizer_dicts rrzGGUFLlamaConverter.__init__s>*>: "&**!#$+DJJ8H'$RV]$]!rcTtt|j|jSNr{ziprerfrrs rrzGGUFLlamaConverter.vocabC ell344rc|jSrrhrs rrhzGGUFLlamaConverter.merges ||rc |j|j}|j|j}t|Dcic] \}\}}|| }}}}|j|j |jnd}t |dd|j |jnd} t |dd|j |jnd} tt|||dd} g} t|jds^|| jt|dd| | jt| dd| | jt| ddn}tjtj|jj dk(d } | D]6}| jt|jj |dd8t#| d k7r| j%| t#|jj&d k7r?| j)|jj&Dcgc]}t|ddc}||j*d <| |j*d <| |j*d <|j,r>d|j*d <d|j*d<d|j*d<d|j._| Scc}}}wcc}w)Nr;r<T) unk_tokenfuse_unk byte_fallbackrgF normalizedspecialrprr eos_token bos_tokenriclean_up_tokenization_spaceslegacy)rrrhrr=rerr;rr rrr npwhererrgr|add_special_tokensr add_tokensrrrr)rr vocab_scoresrhrword_score bpe_vocabrrrr:special_tokensspecial_tokens_idxidx added_tokens rr:zGGUFLlamaConverter.tokenizerszz$**- TZZ(6? 6MNN!2NT6T1WN N8=8J8J8VELL!3!34\` 8?~W[8\8hELL!3!34nr 8?~W[8\8hELL!3!34nr  #"   tzz<0$%%juVZ&[\$%%juVZ&[\$%%juVZ&[\"$"((4::3H3H*IQ*N!OPQ!R ) j%%j1B1B31GTYcg&hi j ~ ! #  ( ( 8 tzz&& '1 ,  ]a]g]g]t]tukKE5Iu /8{+.7{+.7{+  $ $9=D " "#5 6EID " "#A B/4D " "8 ,-2D # # *kONvs KKc0tjtjtjddg}|jr|tj dddgz }|r|tj ddgz }tj|S)N▁rFTri trim_offsets use_regexrncontentleft)r ByteFallbackFuseReplacer ByteLevelStripSequencer replacementrisequences rdecoderzGGUFLlamaConverter.decoders  ! ! # MMO   UC (   $ $ ++UQVbfgh hH  !<= =H  **rc|j|j}|j|j}|||_d}d}t|jdr|jj }|j ||}|||_|j|||_|j}|r||_|jr7tjddd|_tjg|_|S)NrTriFr)r:r normalizerrrri pre_tokenizerrpost_processorrrrrr)rr:rrrirrs r convertedzGGUFLlamaConverter.convertedsNN4::. __TZZ0  !#-I   4**,> ?#66GG **;8HI  $&3I # LL6FG ,,. '5I $  $ $&4&>&>!&Ud'I # $/#7#7#;I rN) rrrrrrhr:rrrrrrrs"^ 58t +!rrc*eZdZdZdeffd ZxZS)GGUFQwen2Converterc2t||_i|_yrrrrrs rrzGGUFQwen2Converter.__init__"7"G!#rreturnc 4t|jjDcic]\}}|| }}}|jj}t|||}|j tdddtdddtdddg|Scc}}w)N <|endoftext|>FTrz <|im_start|>z <|im_end|>)rrrerhsuperrrr rrrrrhr: __class__s rrzGGUFQwen2Converter.converteds(1$2I2I2P2P(QRWQqRR((//G%eV4 $$?udK>eTJ: "&**!#rcTtt|j|jSrrrs rrzGGUFPhi3Converter.vocabrrc|jSrrrs rrhzGGUFPhi3Converter.mergesrrc|j|j}|j|j}t|Dcic] \}\}}|| }}}}t t ||}|j tdddddtdddtddddtd dddtd dddtd dddtd dddtd dddtddddtddddtddddtddddg |j|j|jnd|jd<|j|j|jnd|jd<|j|j|jnd|jd<|j|j|jnd|jd<|Scc}}}w)NTF)rstriplstriprrrrz <|assistant|>)r rrz<|placeholder1|>z<|placeholder2|>z<|placeholder3|>z<|placeholder4|>z <|system|>z<|end|>z<|placeholder5|>z<|placeholder6|>z<|user|>rrr pad_token)rrrhrrr rr r=rerr<r;r>) rrrrhrrrrr:s rr:zGGUFPhi3Converter.tokenizerszz$**- TZZ(6? 6MNN!2NT6T1WN Nc)V45 $$6$uX\]?udK?4ESWX-duVZ[-duVZ[-duVZ[-duVZ[<PTU9TeTR-duVZ[-duVZ[:dudS  $160B0B0NELL++ ,TX {+160B0B0NELL++ ,TX {+160B0B0NELL++ ,TX {+160B0B0NELL++ ,TX {+EOsG:ctjtjtj|dg}|r|tjddgz }tj |S)Nrrnr)rrrrrrrs rrzGGUFPhi3Converter.decoder@s\  ! ! # MMO   [# .   !<= =H  **rrc|j|j}d}d}t|jdr|jj}|j |||_|S)NrTri)r:rrrrir)rr:rris rrzGGUFPhi3Converter.convertedKs\NN4::.   4**,> ?#66GG  LL6FG rN) rrrrrrhr:rrrrrrrr s($ 5%N + 9 rrc*eZdZdZdeffd ZxZS)GGUFGPTConverterc2t||_i|_yrrrs rrzGGUFGPTConverter.__init__Yrrrct|jjDcic]\}}|| }}}|jj}t|||}|Scc}}wr)rrrerhrrrs rrzGGUFGPTConverter.converted]s](1$2I2I2P2P(QRWQqRR((//G%eV4 Ss Arrs@rrrXs$9rrc0eZdZdZdZdZdZdefdZy)GGUFT5Convertercdg|d<t||_t|jjDcic]\}}|| c}}|_|j|_i|_ycc}}wNz dummy textrh)rrrretoken2idrr)rrrrs rrzGGUFT5Converter.__init__es]$0>x *>: *3DJJ4E4E*FG$!QAG "&**!#Hs A(cTtt|j|jSrrrs rrzGGUFT5Converter.vocabnrrct|jddrcg}t|jddr|tjdgz }|tjddgz }tj |Sy)NrTrir)prependr)patternr)rrrPrependrr)rrrs rrzGGUFT5Converter.normalizerqsr 4**Hd ;Ht..0BDI[00?@@ ,,S%HI IH''1 1rcZtjddggdd|jdfgS)N$Ar )r r z$Br )singlepairr)rTemplateProcessingr)rs rrzGGUFT5Converter.post_processorzs5,,&>-v./  rrc|j|j}tt||jjd}|j |j}|||_d}d}t |jdr|jj}|j||}|||_ |j|||_ |j}|r||_ |S)NFunk_idrrTri) rrrr r=rrrrirrr)rrr:rrrirrs rrzGGUFT5Converter.convertedszz$**-  zz..#  __TZZ0  !#-I   4**,> ?#66GG **;8HI  $&3I # LL6FG ,,. '5I $rN) rrrrrrrrrrrrrrds"$5 9rrc0eZdZdZdZdZdZdefdZy)GGUFGemmaConverterc`dg|d<t||_|j|_i|_yrrrs rrzGGUFGemmaConverter.__init__s.$0>x *>: "&**!#rcLtt|j|j}g}|D]s\}}|dk(r|j d|fd|vr>t |j dk(r"dt |z}|j ||fa|j ||fu|S)Nz<0x09> rrr)r{rrerfrr|strip)rroriginal_vocab updated_vocabtokenscore underscoress rrzGGUFGemmaConverter.vocabsc%,, => * 5LE5 $$dE]3#ekkm"4"9#c%j0 $$k5%9:$$eU^4 5rc.tjddS)Nrr)rrrs rrzGGUFGemmaConverter.normalizers""3..rctjddtjtjg}|r|tjddgz }tj |S)Nrrrnr)rrrrrrrs rrzGGUFGemmaConverter.decoders\   UC (  ! ! # MMO   !<= =H  **rrc|j|j}tt||jj|j }|j |j}|||_d}d}t|jdr|jj}|j|||_ |j||}|||_ |S)Nr%rTri) rrrr r=handle_byte_fallbackrrrrirr)rrr:rrrirs rrzGGUFGemmaConverter.convertedszz$**-  zz.."77  __TZZ0  !#-I   4**,> ?#66GG  LL6FG **;8HI  $&3I #rN) rrrrrrrrrrrrr(r(s"$ / +9rr( qwen2_moer_r` gemma3_textrb)rcdecilmrrc^|}t||}|j}||jfS)a6 Utilities to convert a slow tokenizer instance in a fast tokenizer instance. Args: architecture (`str`): The model architecture derived from gguf file. transformer_tokenizer ([`~tokenization_utils_base.PreTrainedTokenizer`]): Instance of a slow tokenizer to convert in the backend tokenizer for [`~tokenization_utils_base.PreTrainedTokenizerFast`]. Return: A instance of [`~tokenizers.Tokenizer`] to be used as the backend tokenizer of a [`~tokenization_utils_base.PreTrainedTokenizerFast`] )GGUF_TO_FAST_CONVERTERSrr)rrtokenizer_class_name converterfast_tokenizers rconvert_gguf_tokenizerr>s9('(<=nMI((*N 966 66r)+__doc__rnumpyr tokenizersrrrrrtokenizers.modelsr r r convert_slow_tokenizerr rrrrutilsr utils.loggingr get_loggerrrGGUF_CONFIG_MAPPINGGGUF_TOKENIZER_MAPPINGrrrrrrrr(r:strrdictr>rrrrLs  SS*oo    H % $%  3*2) *& 5#8,:"  $3*2) *& 5#8,:" %> 3*2) $& 5#8,:" ?V3*2) $& 5#8,:"%2 Wr 3*2) $& 5#8,:"+ sL 3*2) $& 5#8,:" Md3*2) $& * 5#8,:"%2eB 3*2) $& 5#8,:" CZ++!/!/ [f 3*2) $& 5#8,:" g~  ) ("(< L '#%% & +#8(<,L":" Mf3*2) $ 5#8(8" g|  !$4 ((<  }L*3)2 5#8(6M^ "3),@*(&.- _v4*2) $& 5#8,6" 4*2) $&!+ 5#8,:$4""4*2) $&!+ 5#8,:$4""(#%% & +#8(<,L":" 4*2) $& 5#8,:" gF''++!/!/!3 )"++!/!/ 00(6(6Vvvr(HHV } <k<~>>B  #   #        " /     %!" O#$  '.77yRVAW7r