ó
    pyüi³4  ã            	       óH  • S SK r S SKrS SKJr  S SKrS SKJrJr  SSKJr  SSK	J
r
  SSKJr  \
R                  " \5      r\" S5       " S	 S
\R                   5      5       r\r\" S5       " S S\R                   5      5       r\" S5       " S S\R                   5      5       r\" S5       " S S\R                   5      5       r\" S5       " S S\R                   5      5       r\" S5       " S S\R                   5      5       r " S S\R                   5      r " S S\R                   5      r " S S\R                   5      r " S  S!\R                   5      r " S" S#\R                   5      r " S$ S%\R                   5      r " S& S'\R                   5      r " S( S)\5      r " S* S+\R                   5      r 0 S,\_S-\S.S/S0.4_S1\_S2\_S3\S4S504_S6\_S7\S8S504_S9\_S:\RB                  _S;\_S<\RD                  _S=\_S>\_S?\_S@\RF                  _SA\_SB\RH                  _\RJ                  \\\RL                  \RN                  \RP                  \ SC.Er)\" \)5      r*SD r+\+" S35      r,\+" S25      r-\+" S,5      r.\+" S15      r/\+" S65      r0\+" S?5      r1\+" SE5      r2\+" S>5      r3\+" S=5      r4g)Fé    N)ÚOrderedDict)ÚTensorÚnné   )Úuse_kernel_forward_from_hub)Úlogging)Úis_torchdynamo_compilingÚGeluTanhc                   ó\   ^ • \ rS rSrSrS
S\4U 4S jjjrS\S\4S jrS\S\4S jr	S	r
U =r$ )ÚGELUTanhé   a  
A fast C implementation of the tanh approximation of the GeLU activation function. See
https://huggingface.co/papers/1606.08415.

This implementation is equivalent to NewGELU and FastGELU but much faster. However, it is not an exact numerical
match due to rounding errors.
Úuse_gelu_tanh_pythonc                 ó¸   >• [         TU ]  5         U(       a  U R                  U l        g [        R
                  " [        R                  R                  SS9U l        g )NÚtanh)Úapproximate)	ÚsuperÚ__init__Ú_gelu_tanh_pythonÚactÚ	functoolsÚpartialr   Ú
functionalÚgelu)Úselfr   Ú	__class__s     €ÚU/home/mande/repo/quber/.venv/lib/python3.13/site-packages/transformers/activations.pyr   ÚGELUTanh.__init__(   s<   ø€ Ü‰ÑÔÞØ×-Ñ-ˆD�Hä ×(Ò(¬¯©×);Ñ);ÈÑPˆD�Hó    ÚinputÚreturnc                 óÆ   • US-  S[         R                  " [        R                  " S[        R                  -  5      US[         R
                  " US5      -  -   -  5      -   -  $ ©Nç      à?ç      ð?ç       @ç÷Hmâä¦?g      @©Útorchr   ÚmathÚsqrtÚpiÚpow©r   r   s     r   r   ÚGELUTanh._gelu_tanh_python/   sP   € Ø�s‰{˜c¤E§J¢J¬t¯yªy¸¼t¿w¹w¹Ó/GÈ5ÐS[Ô^c×^gÒ^gÐhmÐorÓ^sÑSsÑKsÑ/tÓ$uÑuÑvÐvr   c                 ó$   • U R                  U5      $ ©N©r   r-   s     r   ÚforwardÚGELUTanh.forward2   ó   € Ø�x‰x˜‹Ðr   r1   ©F)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Úboolr   r   r   r2   Ú__static_attributes__Ú__classcell__©r   s   @r   r   r      sJ   ø† ññQ¨T÷ Qð Qðw vð w°&ô wð˜Vð ¨÷ ò r   r   ÚNewGELUc                   ó*   • \ rS rSrSrS\S\4S jrSrg)ÚNewGELUActivationé:   zÂ
Implementation of the GELU activation function currently in Google BERT repo (identical to OpenAI GPT). Also see
the Gaussian Error Linear Units paper: https://huggingface.co/papers/1606.08415
r   r    c                 óÆ   • SU-  S[         R                  " [        R                  " S[        R                  -  5      US[         R
                  " US5      -  -   -  5      -   -  $ r"   r'   r-   s     r   r2   ÚNewGELUActivation.forwardA   sP   € Ø�U‰{˜c¤E§J¢J¬t¯yªy¸¼t¿w¹w¹Ó/GÈ5ÐS[Ô^c×^gÒ^gÐhmÐorÓ^sÑSsÑKsÑ/tÓ$uÑuÑvÐvr   © N©r6   r7   r8   r9   r:   r   r2   r<   rE   r   r   rA   rA   :   s   † ñð
w˜Vð w¨÷ wr   rA   ÚGeLUc                   ó\   ^ • \ rS rSrSrS
S\4U 4S jjjrS\S\4S jrS\S\4S jr	S	r
U =r$ )ÚGELUActivationéE   aŸ  
Original Implementation of the GELU activation function in Google BERT repo when initially created. For
information: OpenAI GPT's GELU is slightly different (and gives slightly different results): 0.5 * x * (1 +
torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) This is now written in C in nn.functional
Also see the Gaussian Error Linear Units paper: https://huggingface.co/papers/1606.08415
Úuse_gelu_pythonc                 ó’   >• [         TU ]  5         U(       a  U R                  U l        g [        R
                  R                  U l        g r0   )r   r   Ú_gelu_pythonr   r   r   r   )r   rK   r   s     €r   r   ÚGELUActivation.__init__N   s/   ø€ Ü‰ÑÔÞØ×(Ñ(ˆD�Hä—}‘}×)Ñ)ˆD�Hr   r   r    c                 ón   • US-  S[         R                  " U[        R                  " S5      -  5      -   -  $ )Nr#   r$   r%   )r(   Úerfr)   r*   r-   s     r   rM   ÚGELUActivation._gelu_pythonU   s,   € Ø�s‰{˜c¤E§I¢I¨e´d·i²iÀ³nÑ.DÓ$EÑEÑFÐFr   c                 ó$   • U R                  U5      $ r0   r1   r-   s     r   r2   ÚGELUActivation.forwardX   r4   r   r1   r5   )r6   r7   r8   r9   r:   r;   r   r   rM   r2   r<   r=   r>   s   @r   rI   rI   E   sG   ø† ññ*¨÷ *ð *ðG &ð G¨Vô Gð˜Vð ¨÷ ò r   rI   ÚSiLUc                   ó*   • \ rS rSrSrS\S\4S jrSrg)ÚSiLUActivationé\   aÐ  
See Gaussian Error Linear Units (Hendrycks et al., https://arxiv.org/abs/1606.08415) where the SiLU (Sigmoid Linear
Unit) was originally introduced and coined, and see Sigmoid-Weighted Linear Units for Neural Network Function
Approximation in Reinforcement Learning (Elfwing et al., https://arxiv.org/abs/1702.03118) and Swish: a Self-Gated
Activation Function (Ramachandran et al., https://arxiv.org/abs/1710.05941v1) where the SiLU was experimented with
later.
r   r    c                 ó@   • [         R                  R                  U5      $ r0   )r   r   Úsilur-   s     r   r2   ÚSiLUActivation.forwardf   s   € Ü�}‰}×!Ñ! %Ó(Ð(r   rE   NrF   rE   r   r   rV   rV   \   s   † ñð)˜Vð )¨÷ )r   rV   ÚFastGELUc                   ó*   • \ rS rSrSrS\S\4S jrSrg)ÚFastGELUActivationéj   zu
Applies GELU approximation that is slower than QuickGELU but more accurate. See: https://github.com/hendrycks/GELUs
r   r    c                 ó^   • SU-  S[         R                  " US-  SSU-  U-  -   -  5      -   -  $ )Nr#   r$   g€ÑÓ3Eˆé?r&   )r(   r   r-   s     r   r2   ÚFastGELUActivation.forwardp   s:   € Ø�U‰{˜c¤E§J¢J¨u°|Ñ/CÀsÈXÐX]ÑM]Ð`eÑMeÑGeÑ/fÓ$gÑgÑhÐhr   rE   NrF   rE   r   r   r]   r]   j   s   † ñði˜Vð i¨÷ ir   r]   Ú	QuickGELUc                   ó*   • \ rS rSrSrS\S\4S jrSrg)ÚQuickGELUActivationét   zj
Applies GELU approximation that is fast but somewhat inaccurate. See: https://github.com/hendrycks/GELUs
r   r    c                 ó:   • U[         R                  " SU-  5      -  $ )Ng¬Zd;û?)r(   Úsigmoidr-   s     r   r2   ÚQuickGELUActivation.forwardz   s   € Ø”u—}’} U¨U¡]Ó3Ñ3Ð3r   rE   NrF   rE   r   r   rc   rc   t   s   † ñð4˜Vð 4¨÷ 4r   rc   c                   óJ   ^ • \ rS rSrSrS\S\4U 4S jjrS\S\4S jrS	r	U =r
$ )
ÚClippedGELUActivationé~   ar  
Clip the range of possible GeLU outputs between [min, max]. This is especially useful for quantization purpose, as
it allows mapping negatives values in the GeLU spectrum. For more information on this trick, please refer to
https://huggingface.co/papers/2004.09602.

Gaussian Error Linear Unit. Original Implementation of the gelu activation function in Google Bert repo when
initially created.

For information: OpenAI GPT's gelu is slightly different (and gives slightly different results): 0.5 * x * (1 +
torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))). See https://huggingface.co/papers/1606.08415
ÚminÚmaxc                 óh   >• X:”  a  [        SU SU S35      e[        TU ]	  5         Xl        X l        g )Nzmin should be < max (got min: z, max: Ú))Ú
ValueErrorr   r   rk   rl   )r   rk   rl   r   s      €r   r   ÚClippedGELUActivation.__init__‹   s8   ø€ Ø‹9ÜÐ=¸c¸UÀ'È#ÈÈaÐPÓQÐQä‰ÑÔØŒØ�r   Úxr    c                 ól   • [         R                  " [        U5      U R                  U R                  5      $ r0   )r(   Úclipr   rk   rl   )r   rq   s     r   r2   ÚClippedGELUActivation.forward“   s!   € Ü�zŠzœ$˜q›' 4§8¡8¨T¯X©XÓ6Ð6r   )rl   rk   )r6   r7   r8   r9   r:   Úfloatr   r   r2   r<   r=   r>   s   @r   ri   ri   ~   s3   ø† ñ
ð˜Eð ¨÷ ð7˜ð 7 F÷ 7ò 7r   ri   c                   ó>   ^ • \ rS rSrSrU 4S jrS\S\4S jrSrU =r	$ )ÚAccurateGELUActivationé—   zÉ
Applies GELU approximation that is faster than default and more accurate than QuickGELU. See:
https://github.com/hendrycks/GELUs

Implemented along with MEGA (Moving Average Equipped Gated Attention)
c                 óz   >• [         TU ]  5         [        R                  " S[        R                  -  5      U l        g )Né   )r   r   r)   r*   r+   Úprecomputed_constant©r   r   s    €r   r   ÚAccurateGELUActivation.__init__Ÿ   s'   ø€ Ü‰ÑÔÜ$(§I¢I¨a´$·'±'©kÓ$:ˆÕ!r   r   r    c                 ó�   • SU-  S[         R                  " U R                  US[         R                  " US5      -  -   -  5      -   -  $ )Nr#   r   r&   é   )r(   r   r{   r,   r-   s     r   r2   ÚAccurateGELUActivation.forward£   sE   € Ø�U‰{˜a¤%§*¢*¨T×-FÑ-FÈ%ÐRZÔ]b×]fÒ]fÐglÐnoÓ]pÑRpÑJpÑ-qÓ"rÑrÑsÐsr   )r{   )
r6   r7   r8   r9   r:   r   r   r2   r<   r=   r>   s   @r   rw   rw   —   s)   ø† ñõ;ðt˜Vð t¨÷ tò tr   rw   c                   óP   ^ • \ rS rSrSrU 4S jrS\S\4S jrS\S\4S jrSr	U =r
$ )	ÚMishActivationé§   zÍ
See Mish: A Self-Regularized Non-Monotonic Activation Function (Misra., https://huggingface.co/papers/1908.08681). Also
visit the official repository for the paper: https://github.com/digantamisra98/Mish
c                 ó`   >• [         TU ]  5         [        R                  R                  U l        g r0   )r   r   r   r   Úmishr   r|   s    €r   r   ÚMishActivation.__init__­   s   ø€ Ü‰ÑÔÜ—=‘=×%Ñ%ˆ�r   r   r    c                 ón   • U[         R                  " [        R                  R	                  U5      5      -  $ r0   )r(   r   r   r   Úsoftplusr-   s     r   Ú_mish_pythonÚMishActivation._mish_python±   s%   € Ø”u—z’z¤"§-¡-×"8Ñ"8¸Ó"?Ó@Ñ@Ð@r   c                 ó$   • U R                  U5      $ r0   r1   r-   s     r   r2   ÚMishActivation.forward´   r4   r   r1   )r6   r7   r8   r9   r:   r   r   r‰   r2   r<   r=   r>   s   @r   r‚   r‚   §   s;   ø† ñõ
&ðA &ð A¨Vô Að˜Vð ¨÷ ò r   r‚   c                   ó*   • \ rS rSrSrS\S\4S jrSrg)ÚLinearActivationé¸   zS
Applies the linear activation function, i.e. forwarding input directly to output.
r   r    c                 ó   • U$ r0   rE   r-   s     r   r2   ÚLinearActivation.forward½   s   € Øˆr   rE   NrF   rE   r   r   rŽ   rŽ   ¸   s   † ñð˜Vð ¨÷ r   rŽ   c                   ó"   • \ rS rSrSrSS jrSrg)ÚLaplaceActivationéÁ   zë
Applies elementwise activation based on Laplace function, introduced in MEGA as an attention activation. See
https://huggingface.co/papers/2209.10655

Inspired by squared relu, but with bounded range and gradient for better stability
c                 óŽ   • X-
  R                  U[        R                  " S5      -  5      nSS[        R                  " U5      -   -  $ )Nr%   r#   r$   )Údivr)   r*   r(   rP   )r   r   ÚmuÚsigmas       r   r2   ÚLaplaceActivation.forwardÉ   s:   € Ø‘× Ñ  ¬¯ª°3«Ñ!7Ó8ˆØ�cœEŸIšI eÓ,Ñ,Ñ-Ð-r   rE   N)g»¹øÛž æ?g ^×/ØÒ?©r6   r7   r8   r9   r:   r2   r<   rE   r   r   r“   r“   Á   s   † ñ÷.r   r“   c                   ó   • \ rS rSrSrS rSrg)ÚReLUSquaredActivationéÎ   zV
Applies the relu^2 activation introduced in https://huggingface.co/papers/2109.08668
c                 óp   • [         R                  R                  U5      n[        R                  " U5      nU$ r0   )r   r   Úrelur(   Úsquare)r   r   Úrelu_appliedÚsquareds       r   r2   ÚReLUSquaredActivation.forwardÓ   s)   € Ü—}‘}×)Ñ)¨%Ó0ˆÜ—,’,˜|Ó,ˆØˆr   rE   Nrš   rE   r   r   rœ   rœ   Î   s   † ñõr   rœ   c                   ó   • \ rS rSrSrS rSrg)ÚSqrtSoftplusActivationéÙ   uF   sqrt(softplus(x)) â€” the router scoring function used by DeepSeek V4.c                 ó\   • [         R                  R                  U5      R                  5       $ r0   )r   r   rˆ   r*   r-   s     r   r2   ÚSqrtSoftplusActivation.forwardÜ   s    € Ü�}‰}×%Ñ% eÓ,×1Ñ1Ó3Ð3r   rE   Nrš   rE   r   r   r¥   r¥   Ù   s
   † ÙPõ4r   r¥   c                   ó(   ^ • \ rS rSrU 4S jrSrU =r$ )ÚClassInstantieréà   c                 ól   >• [         TU ]  U5      n[        U[        5      (       a  UOU0 4u  p4U" S0 UD6$ )NrE   )r   Ú__getitem__Ú
isinstanceÚtuple)r   ÚkeyÚcontentÚclsÚkwargsr   s        €r   r­   ÚClassInstantier.__getitem__á   s7   ø€ Ü‘'Ñ% cÓ*ˆÜ!+¨G´U×!;Ñ!;‘gÀ'È2À‰ˆÙ‰}�V‰}Ðr   rE   )r6   r7   r8   r9   r­   r<   r=   r>   s   @r   rª   rª   à   s   ø† ÷ó r   rª   c                   ó†   ^ • \ rS rSrSrSSSS\R                  S4U 4S jjrS\S	\4S
 jr	S\S	\4S jr
S\S	\4S jrSrU =r$ )ÚXIELUActivationéç   zÞ
Applies the xIELU activation function introduced in https://arxiv.org/abs/2411.13010

If the user has installed the nickjbrowning/XIELU wheel, we import xIELU CUDA
Otherwise, we emit a single warning and use xIELU Python
gš™™™™™é?r#   g�íµ ÷Æ°¾Fc                 ó<  >• [         TU ]  5         [        R                  " [        R
                  " [        R                  " [        R                  " XS95      5      R                  S5      5      U l	        [        R                  " [        R
                  " [        R                  " [        R                  " X#-
  US95      5      R                  S5      5      U l
        U R                  S[        R                  " X5S95        U R                  S[        R                  " XES95        X`l        [        U5      U l        [        U5      U l        S U l         SS Kn[        R$                  R&                  R)                  5       U l        Sn SSKJn	  U	" U R.                  5      U l        US-  n[4        R7                  U5        g ! [2         a$  n
USU
 S	3-  nU R.                  U l         S n
A
N?S n
A
ff = f! [2         a#  n
[4        R7                  S
U
 S35         S n
A
g S n
A
ff = f)N)Údtyper   ÚbetaÚepszUsing experimental xIELU CUDA.)Úallow_in_graphz& Enabled torch._dynamo for xIELU CUDA.z+ Could not enable torch._dynamo for xIELU (z*) - this may result in slower performance.z CUDA-fused xIELU not available (u   ) â€“ falling back to a Python version.
For CUDA xIELU (experimental), `pip install git+https://github.com/nickjbrowning/XIELU`)r   r   r   Ú	Parameterr(   ÚlogÚexpm1ÚtensorÚ	unsqueezeÚalpha_pÚalpha_nÚregister_bufferÚwith_vector_loadsru   Ú_beta_scalarÚ_eps_scalarÚ_xielu_cuda_objÚ	xielu.opsÚclassesÚxieluÚXIELUÚtorch.compilerr¼   Ú_xielu_cudaÚ_xielu_cuda_fnÚ	ExceptionÚloggerÚwarning_once)r   Úalpha_p_initÚalpha_n_initrº   r»   r¹   rÅ   rË   Úmsgr¼   Úerrr   s              €r   r   ÚXIELUActivation.__init__ï   s­  ø€ ô 	‰ÑÔÜ—|’|¤E§I¢I¬e¯kªk¼%¿,º,À|Ñ:aÓ.bÓ$c×$mÑ$mÐnoÓ$pÓqˆŒÜ—|’|Ü�IŠI”e—k’k¤%§,¢,¨|Ñ/BÈ%Ñ"PÓQÓR×\Ñ\Ð]^Ó_ó
ˆŒð 	×Ñ˜V¤U§\¢\°$Ñ%DÔEØ×Ñ˜U¤E§L¢L°Ñ$BÔCØ!2Ôä! $›KˆÔÜ  ›:ˆÔà#ˆÔð	Ûä#(§=¡=×#6Ñ#6×#<Ñ#<Ó#>ˆDÔ Ø2ˆCð7Ý9á&4°T×5EÑ5EÓ&F�Ô#ØÐ?Ñ?�ô ×Ñ Õ$øô ó 7ØÐDÀSÀEÐIsÐtÑt�Ø&*×&6Ñ&6�×#Ñ#ûð7ûô ó 	Ü×ÑØ2°3°%ð 8jð j÷ñ ûð	úsB   Å3G. Æ"F= Æ'G. Æ=
G+ÇG&Ç!G. Ç&G+Ç+G. Ç.
HÇ8HÈHrq   r    c           
      ó°  • [         R                  R                  U R                  5      nU R                  [         R                  R                  U R
                  5      -   n[        R                  " US:„  X!-  U-  U R                  U-  -   [        R                  " [        R                  " XR                  5      5      U-
  U-  U R                  U-  -   5      $ )Nr   )r   r   rˆ   rÂ   rº   rÃ   r(   Úwherer¿   rk   r»   )r   rq   rÂ   rÃ   s       r   Ú_xielu_pythonÚXIELUActivation._xielu_python  s™   € Ü—-‘-×(Ñ(¨¯©Ó6ˆØ—)‘)œbŸm™m×4Ñ4°T·\±\ÓBÑBˆÜ�{Š{Ø�‰EØ‰K˜!‰O˜dŸi™i¨!™mÑ+Ü�[Š[œŸš 1§h¡hÓ/Ó0°1Ñ4¸Ñ?À$Ç)Á)ÈaÁ-ÑOó
ð 	
r   c                 ó†  • UR                   nUR                  5       S:  a'  UR                  S5      nUR                  5       S:  a  M'  UR                  5       S:”  a"  UR                  SSUR	                  S5      5      nX!R                   :w  a!  [
        R                  SUUR                   5        U R                  R                  UU R                  R                  UR                  5      U R                  R                  UR                  5      U R                  U R                  U R                  5      nUR                  U5      $ )zDFirewall function to prevent torch.compile from seeing .item() callsr   r   éÿÿÿÿr   z_Warning: xIELU input tensor expects 3 dimensions but got (shape: %s). Reshaping to (shape: %s).)ÚshapeÚdimrÁ   ÚviewÚsizerÑ   rÒ   rÈ   r2   rÂ   Útor¹   rÃ   rÆ   rÇ   rÅ   )r   rq   Úoriginal_shapeÚresults       r   rÎ   ÚXIELUActivation._xielu_cuda"  sï   € àŸ™ˆà�e‰e‹g˜‹kØ—‘˜A“ˆAð �e‰e‹g˜�kà�5‰5‹7�Q‹;Ø—‘�r˜1˜aŸf™f R›jÓ)ˆAØŸW™WÓ$Ü×ÑØqØØ—‘ôð
 ×%Ñ%×-Ñ-ØØ�L‰L�O‰O˜AŸG™GÓ$Ø�L‰L�O‰O˜AŸG™GÓ$à×ÑØ×ÑØ×"Ñ"ó
ˆð �{‰{˜>Ó*Ð*r   r   c                 óÊ   • U R                   bF  UR                  (       a5  [        5       (       d  U R                  U5      $ [        R                  S5        U R                  U5      $ )Nz:torch._dynamo is compiling, using Python version of xIELU.)rÈ   Úis_cudar	   rÏ   rÑ   rÒ   rÚ   r-   s     r   r2   ÚXIELUActivation.forward;  sN   € Ø×ÑÑ+°··Ü+×-Ñ-Ø×*Ñ*¨5Ó1Ð1ä×#Ñ#Ð$`ÔaØ×!Ñ! %Ó(Ð(r   )rÆ   rÇ   rÏ   rÈ   rÃ   rÂ   rÅ   )r6   r7   r8   r9   r:   r(   Úbfloat16r   r   rÚ   rÎ   r2   r<   r=   r>   s   @r   r¶   r¶   ç   sd   ø† ñð ØØØØ�n‰nØ÷(ðT
˜vð 
¨&ô 
ð+˜Vð +¨ô +ð2)˜Vð )¨÷ )ò )r   r¶   r   Úgelu_10iöÿÿÿé
   )rk   rl   Ú	gelu_fastÚgelu_newÚgelu_pythonrK   TÚgelu_pytorch_tanhÚgelu_python_tanhr   Úgelu_accurateÚ	hardswishÚlaplaceÚ
leaky_reluÚlinearr…   Ú
quick_gelurŸ   Úrelu2Úrelu6)rf   rY   ÚsqrtsoftplusÚswishr   ÚprelurË   c           	      ó€   • U [         ;   a	  [         U    $ [        SU  S[        [         R                  5       5       35      e)Nz	function z not found in ACT2FN mapping )ÚACT2FNÚKeyErrorÚlistÚkeys)Úactivation_strings    r   Úget_activationr  a  sB   € ØœFÓ"ÜÐ'Ñ(Ð(ä˜Ð#4Ð"5Ð5RÔSWÔX^×XcÑXcÓXeÓSfÐRgÐhÓiÐir   rY   )5r   r)   Úcollectionsr   r(   r   r   Úintegrations.hub_kernelsr   Úutilsr   Úutils.import_utilsr	   Ú
get_loggerr6   rÑ   ÚModuler   ÚPytorchGELUTanhrA   rI   rV   r]   rc   ri   rw   r‚   rŽ   r“   rœ   r¥   rª   r¶   Ú	HardswishÚ	LeakyReLUÚReLUÚReLU6ÚSigmoidrT   ÚTanhÚPReLUÚACT2CLSrý   r  rî   rí   r   rì   rï   rö   rY   r…   Ú
linear_actrE   r   r   Ú<module>r     sJ  ðó Û Ý #ã ß å AÝ Ý 8ð 
×	Ò	˜HÓ	%€ñ ˜ZÓ(ôˆr�y‰yó ó )ðð0 €ñ ˜YÓ'ôw˜Ÿ	™	ó wó (ðwñ ˜VÓ$ô�R—Y‘Yó ó %ðñ, ˜VÓ$ô
)�R—Y‘Yó 
)ó %ð
)ñ ˜ZÓ(ôi˜Ÿ™ó ió )ðiñ ˜[Ó)ô4˜"Ÿ)™)ó 4ó *ð4ô7˜BŸI™Iô 7ô2t˜RŸY™Yô tô �R—Y‘Yô ô"�r—y‘yô ô
.˜Ÿ	™	ô 
.ô˜BŸI™Iô ô4˜RŸY™Yô 4ô�kô ôZ)�b—i‘iô Z)ðzØ
ˆNðàÐ%¨s¸2Ñ'>Ð?ðð Ð#ðð Ð!ð	ð
 �NÐ%6¸Ð$=Ð>ðð ˜ðð ˜Ð$:¸DÐ#AÐBðð Ð+ðð �—‘ðð Ð ðð �"—,‘,ðð Ððð ˆNðð Ð%ðð ˆB�G‰Gðð  Ð"ð!ð" ˆR�X‰Xð#ð$ �z‰zØØ*Ø�W‰WØ�G‰GØ�X‰XØò1€ñ4 
˜Ó	!€òjñ ˜]Ó+€Ù˜*Ó%€Ù�fÓ€Ù˜;Ó'€	Ù"Ð#6Ó7Ð Ù˜LÓ)€
Ù�fÓ€Ù�fÓ€Ù˜HÓ%�
r   