pyiSSKJr SSKrSSKJr SSKJr SSKrSSKJ r SSKJ r SSK J r SSKJr SS KJr SS KJr SS KJrJrJr S S KJr S SKJrJr \R:"\5r\R@r!\RD"\!5RFr$\RD"\!5RJr&Sr'Sr(\"SS9"SS55r)\RTS,Sj5r+\"SS9"SS55r,\RTS-Sj5r-S.Sjr.\R^4S/Sjjr0"SS\ Rb5r2S0Sjr3S0Sjr4S1Sjr5S2S jr6S3S!jr7S0S"jr8"S#S$\ Rr5r:"S%S&\5r;\;"5r<S4S5S'jjr="S(S)\5r>"S*S+\5r?g)6) annotationsN)Callable) dataclass) functional)ACT2FN) ConversionOps)should_convert_module)logging)get_cuda_runtime_versionis_kernels_availableresolve_internal_import)lazy_load_kernel)ExpertsInterfaceuse_experts_implementationcUH n[X5(dM[X5s $ [[U5RSU35e)Nz has none of: )hasattrgetattrAttributeErrortype__name__)objnamesnames f/home/mande/repo/quber/.venv/lib/python3.13/site-packages/transformers/integrations/finegrained_fp8.py _first_attrr/sE 3  3% % DI../~eWE FFT)frozencB\rSrSr%SrS\S'S\S'S\S'S\S'Srg ) FineGrainedFP86zNEntry points exposed by the `kernels-community/finegrained-fp8` Triton kernel.r fp8_matmul fp8_act_quantbatched_fp8_matmulgrouped_fp8_matmulNr __module__ __qualname____firstlineno____doc____annotations____static_attributes__r(rrr"r"6sX    rr"c[5(d [S5e[S5nUc [S5e[USS5n[USS5n[USS5n[USS5nSU4SU4SU4SU44VVs/sH upVUbM UPM nnnU(a[S S R U5S 35e[ UUUUS 9$s snnf) z Load the finegrained-fp8 Triton kernel once and return its entry points. Raises `ImportError` if the `kernels` package is missing, or the kernel or required symbols cannot be found. z`finegrained-fp8 kernel requires the `kernels` package. Install it with `pip install -U kernels`.zfinegrained-fp8NuFailed to load the finegrained-fp8 kernel — check that `kernels-community/finegrained-fp8` has a build matching the current torch/CUDA.w8a8_fp8_matmulr%w8a8_fp8_matmul_batchedw8a8_fp8_matmul_groupedz4finegrained-fp8 kernel is missing required symbols: , A. Please update the `kernels` package (`pip install -U kernels`).)r$r%r&r')r ImportErrorrrjoinr")kernelr$r%r&r'rattrmissings r_load_finegrained_fp8_kernelr;@s! ! ! n  / 0F ~ ;  !2D9JFOT:M )BDI )BDI  + m , &(: ; &(: ;    JD    B499WCUBVWN N  #--  ! s 9 CCc8\rSrSr%SrS\S'S\S'S\S'Srg) DeepGEMMqzAEntry points exposed by the `kernels-community/deep-gemm` kernel.rr$r'per_token_cast_to_fp8r(Nr)r(rrr=r=qsK  ##rr=c[5(d [S5e[RR 5(d [S5e[RR 5SnUS:a[SUS35e[ 5upUS:d US:XaUS:a[S US US 35e[S 5nUc [S5e[USS 5n[USS 5n[USS9nSU4SU4SU44VVs/sH upxUbM UPM n nnU (a[SSRU 5S35e[UUUS9$s snnf)z Load DeepGEMM once and return its entry points. Raises `ImportError` if CUDA/hardware requirements are not met, or the kernel or required symbols are not found. zYDeepGEMM kernel requires the `kernels` package. Install it with `pip install -U kernels`.zcDeepGEMM kernel requires CUDA, but CUDA is not available. Use a different `experts_implementation`.r z_DeepGEMM requires a Hopper (SM90+) or newer GPU, but the current device has compute capability z-.x. Use a different `experts_implementation`. z0DeepGEMM requires CUDA runtime 12.3+, but found .zO. Please upgrade your CUDA toolkit or use a different `experts_implementation`.z deep-gemmNu|Failed to load the DeepGEMM kernel — check that `kernels-community/deep-gemm` has a build matching the current torch/CUDA. fp8_gemm_nt m_grouped_fp8_gemm_nt_contiguouszutils.per_token_cast_to_fp8) chained_pathz-DeepGEMM kernel is missing required symbols: r4r5)r$r'r?) r r6torchcuda is_availableget_device_capabilityr rrrr7r=) major cuda_major cuda_minorr8r$r'r?rr9r:s r_load_deepgemm_kernelrOzs ! !uvv :: " " $ $ q  JJ , , .q 1E qy&&+W,Y [  67JB:+ Q>zl!J<X\ \  k *F ~ ;  5J )KTR3FIfg J ' /1C D *,A B  JD    ;DIIgf=f)uLFP8 matmul: C = dequant(A, As) @ dequant(B, Bs)^T. Supports both per-tensor and block-wise quantization: - block_size=None or block_size=[N, K]: per-tensor mode (As is scalar/per-row, Bs is scalar) - block_size=[block_n, block_k]: block-wise mode (As and Bs are per-block scale grids) Dispatch order: 1. DeepGEMM (Hopper+, block_size 128x128) if available 2. Triton finegrained-fp8 kernel (universal fallback) Args: A: (M, K) float8_e4m3fn — quantized activations B: (N, K) float8_e4m3fn — quantized weights As: block-wise: (M, K//block_k) float32; per-tensor: (M,) per-row scales Bs: block-wise: (N//block_n, K//block_k) float32; per-tensor: scalar or (1,) single weight scale block_size: [block_n, block_k] for block-wise quantization, or None/[N, K] for per-tensor output_dtype: desired output dtype NrrrdevicedtypeaDeepGEMM kernel is not available or compatible, falling back to Triton finegrained-fp8 kernel. To use DeepGEMM FP8 matmul, ensure you have a Hopper (SM90+) or newer GPU with CUDA runtime 12.3+, and that the `kernels` package is installed and up to date (`pip install -U kernels`).) rOviewshaperHemptyrWr$floatr6logger warning_oncer;) ABAsBs block_size output_dtypedeepgemmA_2dAs_2doutputfinegrained_fp8s rr1r1s4*Q-:a="GC"G =,.H66"aggbk*DGGB -E[[A 188S_`F   {{} 588: O;;qwws|qwwqzm;< <24O  % %aBJ MM    i  s DD=<D=cT^\rSrSrSSS\4SU4SjjjrS SjrSrU=r$) FP8LinearNdynamicFc>[T U]X5 XPlX0lX@l[ R R[ R"X!US95Ul URc=[ R"[ R"S[ RS95Ul OX RS-S- URS-nXRS-S- URS-n[ R"[ R"Xx[ RS95Ul URS:Xa=[ R"[ R"S[ RS95Ul OURSS5 UR(a:[ R"[ R"UR55UlgURSS5 g)NrX?rrstaticactivation_scalebias)super__init__has_biasrcactivation_schemerHnn Parameterr[weighttensorfloat32weight_scale_invrrregister_parameter out_featuresrs) self in_featuresrrcrwrvrXscale_out_featuresscale_in_features __class__s rruFP8Linear.__init__sU 3 $!2hh((\V[)\] ?? "$&LLc1W$XD !".1C"Ca"GDOO\]L^!^ !,q/A!AA!E$//Z[J\ \ $&LL .W%D !  ! !X -$&LLc1W$XD !  # #$6 = == U[[1B1B%CDDI  # #FD 1rc URR5S:a+[R"XRUR5$URnUR n[ U[RRR5(a UR5nUR5nURS:XaG[5nURXRbURSOUR S5upVOURS:XaWUR"R%[R&5nX- R)[*[,S9R%[.5nO[1SUR35e[3UUUUURUR4S9nURbUR7UR5 UR%UR4S9$) NrrmrUrqminmaxzUnsupported activation scheme: rdro)rz element_sizeFlinearrsr} isinstancerH distributedr{DTensorto_localrwr;r%rcrZrrtor|clamp_FP8_MIN_FP8_MAX _FP8_DTYPENotImplementedErrorr1rXadd_)rinputrz scale_invriqinputscalerhs rforwardFP8Linear.forwards ;; # # % )88E;; : :)) fe//66>> ? ?__&F!**,I  ! !Y .:?%+ +1++*>,$$&LDTDTDVK K6F6FASWScScfgSghll,!5;;GG"%5%55@@DM!% 4  4<T>TE ^^F&&G$))008899}} """$11   +?? 2H }}##H-;;x(11+?? 2H.11(..AKKBOOLmS1%H\\$))A,v>HTN)L'++J:NRRWXRY  ! !-"5"5 66rcURnURS5n[R"UR 5USUS- S9R 5nXb-S- U-U-nU[ XQ5US- --nXv- n [RRRU RS5S5n [R"XTS9X-n U(a URS5R 5n OP[R"U4SU[RS9n [R"X:UR 5S5X'XU4$)aBuild the TMA-aligned layout DeepGEMM's grouped GEMM expects. Returns `(sorted_to_padded, grouped_layout, total_padded_rows)`. `grouped_layout` encodes expert boundaries as a cumsum of aligned counts on Blackwell (`use_psum_layout=True`) or per-row expert ids with -1 for padding on Hopper. Accepts EP sentinels: values in `expert_ids_sorted` equal to `num_experts` (unclamped sentinels) are routed past the last aligned expert block and marked `-1` in the Hopper layout (and excluded from the Blackwell cumsum), so DeepGEMM skips them. rrr)rrrrUrV)rWrrHrrlongrrxrpadrrfullrwhere) expert_ids_sortedr alignmentuse_psum_layoutrWrraligned_tokens_per_experttotal_padded_rowspadding_per_expertcumulative_paddingsorted_to_paddedgrouped_layouts r!_build_deepgemm_contiguous_layoutrs= % %F"''*J $5$9$9$;+STZehiZijooq"3"?!"C !QU^ ^"S%AYQR]%SS 3F,,001C1J1J11MvV||J>ASAff 399!<@@B%6$8"VSXS^S^_+0;;7H7VXiXmXmXoqs+t( -> >>rc[R"X0RSURURS9nXU'[R"X1RSUR[R S9nXU'XE4$)zKPad sorted hidden states and scales into the TMA-aligned contiguous layout.rrV)rHzerosrZrWrXr|)rscalesrr hidden_padded scales_paddeds r"_pad_to_deepgemm_contiguous_layoutr svKK..q1-:N:NVcViViM'4"#KK 1<<?=K_K_glgtgtuM&,"#  ''rc X$)z;Remove padding rows from the TMA-aligned contiguous layout.r()hidden_states_paddedrs r&_unpad_from_deepgemm_contiguous_layoutrs  11rcURS:Xa [S5eURc [S5eURSS:wdURSS:wa[SUR35e[ 5nUR nUR S5nUR S5nUR S5nURS5n URS5n [R"U 5upXU-n Xn[RRU5SS :n[XR[US 9unnnXR:RS5nUR (a UR"O UR$nUR (a UR&O UR(nUR*nUR,n[/U[R0R2R45(a@UR75nUR75nUR75nUR75nUR9U S S 9unn[;UUUU5unn[R<"UUR>SU[R@S 9nURCUU4UURE54UUUS9 UR (aURGU5nOURIU5nUR9US S 9unn[R<"UX[R@S 9nURCUU4UURE54UUUS9 [KUU5nUURMURN5RS5-nURQUS5 [RR"U 5n[RT"U R S5US9UU 'UUnURWXvU5RYSS9nURMURN5$)NrqzDeepGEMM experts dispatch does not support activation_scheme='static'. Use the default eager dispatch or switch to activation_scheme='dynamic'.zuDeepGEMM requires block-wise quantization (block_size=[128, 128]), but got per-tensor quantization (block_size=None).rrrz-DeepGEMM requires block_size=(128, 128), got rU )rrF) use_ue8m0rV)rrrr)-rwrrc ValueErrorrOrWrrrHrrIrKrr_DEEPGEMM_M_ALIGNMENTrrrrrrrrrrr{rrr?rr[rZbfloat16r'r\rrrrrXrrrrYr) rrrrrerWrrrrrrrrrrrrrrrrrract_fp8 act_scalesrproj_fp8 proj_scalesrrrs rfp8_deepgemm_experts_forwardr s  )! W    A   qS DOOA$6#$=HHYZ[[$&H  ! !F  $I##A&J##B'J#**2.N$$R(JJ/L,Y->?%+jj66v>qARGO:[&&2GYh;7n&7"%5%55@@DM!% 4  4<T>TE ^^F&&G$))008899}} """$#889Q]b8cGZHTN)L'++J:NRRWXRY  ! !-"5"5 66rc^\rSrSrSSSS\4S U4SjjjrS SjrS SjrSSS jjrS r U=r $) FP8ExpertsiNrmFTc H>[TU]5 USLdS5eXlX@lXPlX lUR UlX0l[USS5Ul [USS5Ul [[USS5Ul UR(aS UR-URp[R"[ R""URXxUS 95UlUR b['XpR S 5OS n UR b['XR S 5OS n [R"[ R""URX[ R(S 95UlUR-S S5 OURURp[R"[ R""URXUS 95UlUR b['XR S 5OS n UR b['XR S 5OS n[R"[ R""URX[ R(S 95UlUR-SS5 URURnn[R"[ R""URUUUS 95UlUR b['XR S 5OS nUR b['UUR S 5OS n[R"[ R""URUU[ R(S 95UlUR-SS5 URS:Xa[R"[ R6"UR[ R(S 95Ul[R"[ R6"UR[ R(S 95Ulgg)NFzWFP8Experts does not support bias for now, please open an issue if you want this featurenum_local_expertsrmoe_intermediate_sizeintermediate_sizehidden_activation hidden_actrrorrgate_up_proj_bias up_proj_biasdown_proj_biasrq)rtruconfigrvrrc hidden_sizerrwrrintermediate_dimrrrxryrHr[rrSr|rr~rrrronesgate_up_proj_activation_scaledown_proj_activation_scale)rrrcrwrvrrX gu_proj_out gu_proj_in gu_scale_out gu_scale_in u_proj_out u_proj_in u_scale_out u_scale_in d_proj_out d_proj_in d_scale_out d_scale_inrs rruFP8Experts.__init__s 5  e     $ ,,!2&v/BMR +F4KM` a[1DlST ==&'$*?*?&? " U[[9I9I;jo-p qD EI__E`5ooa.@AfgLCG??C^% OOA,>?deK*,,, D,,lu}}]+D '  # #$7 >$($9$94?? << D4D4Djch(ijDLCG??C^% OOA,>?deKAEA\y//!*<=bcJ%'\\ D,,kU]][&D "  # #ND 9 $1F1FI ekk$2B2BJPYaf&gh?C?ZeJ(:;`a =A__=XU9dooa&89^_ #%<< KK((+z W$    0$7  ! !X -13ejjIYIYafanan>o1pD ..0ll5::dFVFV^c^k^k;l.mD + .rcNURSSS9up#URU5U-$)NrrUr)chunkr)rgate_upgateups rrFP8Experts._apply_gates*===+{{4 2%%rc[R"U[RS9n[R"5 [RR R X RS9nURSSS5n[R"URSS9S5RSS 9RS 5nSSS5 WGHnXpR:XaM[R"WU5upXn URS :XaURUOSn UR!U UR"(aUR$UOUR&UUR"(aUR(UOUR*UU S 9n UR"(aUR-U 5OUR/U 5n URS :XaUR0UOSn UR!U UR2UUR4UU S 9n X9US4nXR7U R85-nUR;SXR7UR855 GM UR7UR85$!,(df  GN=f) Nro) num_classesrrr)rUrF)as_tuplerUrq)rr)rH zeros_liker|no_gradrxrone_hotrpermutegreaterrnonzerorYrrwrrrrrrrrrrrrrrX index_add_)rrrrr expert_mask expert_hit expert_idx top_k_pos token_idx current_stategate_up_act_scalerdown_act_scalerouting_weightsrs rrFP8Experts.forwards7 $..}EMMR ]]_((--55kO_O_5`K%--aA6K{8'DaHPPZ_P`eefhiJ %J---#(;;{:/F#G I)4MBFBXBX\dBd22:>jn {{15!!*-DLLQ[D\;?==++J7dNdNdeoNp!2 #H 6:]]t''1 T\H]H?C?U?UYa?a// ;gk {{z*((4!/ #H ,y$,FGO#&8&8&HHL  * *1iI\IbIb9c d7%8#%%m&9&9::C_s BI88 Jc UR5S:a[R"XS5$URS:XaPUbMUR [ R 5nX- R[[S9R [5nOF[5nURXRbURSOURS5upe[UUUUURUR S9nUR UR S9$)NrrqrrUrro)rrrrwrrHr|rrrrr;r%rcrZr1rX) rrrzr}rrrrrirhs rrFP8Experts.linears    1 $88E40 0  ! !X -2B2N$'' 6Em**xX*FII*UF:< 33E:[[_ekk"od >Q $.A"5<  CR( _ _ XXemm, %%_}_j_'_:_W^_,,.%%(%3{{7Q;9QR L(Wq[&%//&2IJ!++B/99"=,,KKH(CFFzR %%n5 Fl&&u}}5 CF<&>F')=)=i)PQQ#/ /@B$**,JC!S(,>#,E//4I*#s )%66eUGG*F)&$77VfXF7|s6{* EcUKG ~\#f+iICFgBVWBV$!4//5BVWFK!-" Xs*Ec,[UR5$r@)r_rbrs rrFp8Dequantize.reverse_ops 4,,--rra)rrrr)rrrr)rrrrrrr@)r,dict[str, list[torch.Tensor] | torch.Tensor]rz str | Nonerrr)rr*r+r,r-rurrrrrrrr/r(rrrrs_") 2mMJ=>'+&@&$& 6 &P..rr)rr")rr=)rQrrRrrr)r_rr`rrarrbrrcz list[int]rdz torch.dtyperr) rztorch.nn.Modulerrrrrrrr) rrrrrrrrrri) rrrrrrrrrz!tuple[torch.Tensor, torch.Tensor])rrrrrr)NNF)rTzlist[str] | None)@ __future__r functoolscollections.abcr dataclassesrrHtorch.nnrxrr activationsrcore_model_loadingr quantizers.quantizers_utilsr utilsr utils.import_utilsr r r hub_kernelsrmoerr get_loggerrr] float8_e4m3fnrfinforrrrrrr"cacher;r=rOrSr|r1rPrkrrrrrrModulerrBrNr]r_rr(rrrs#$! $ .?hh)=   H % ;;z " & & ;;z " & & G $!!! --` $$$$ ==@!& ,N,N,N ,N  ,N  ,N  ,N,N^F, F,R>7 >7>7>7 >7  >7Bb7 b7b7b7 b7  b7J(?#(?25(?BE(?X\(? (?V ( (  (# ( ( ' ( 2&2:F22g7 g7g7g7 g7  g7TC,C,L*01ejA#3AHA0-A0Hx.Mx.r