ó
    qyüiD¢  ã            
       óÐ  • S r SSKrSSKrSSKJr  SSKJrJrJrJr  SSKJ	r
  SSKJrJrJr  SSKJr  SS	KJr  SS
KJr  SSKJrJrJrJrJr  SSKJr  SSKJrJr  SSK J!r!  \RD                  " \#5      r$S\RJ                  S\&S\RN                  S\RJ                  4S jr(S\RJ                  S\RJ                  S\)S\*S\RJ                  4
S jr+S\RJ                  S\RJ                  4S jr,S\RJ                  S\RJ                  S\RJ                  4S jr- " S S\R\                  R^                  5      r0 " S S \Rb                  5      r2 " S! S"\Rb                  5      r3 " S# S$\Rb                  5      r4 " S% S&\5      r5\ " S' S(\5      5       r6\ " S) S*\65      5       r7\" S+S,9 " S- S.\6\5      5       r8\" S/S,9 " S0 S1\65      5       r9\ " S2 S3\65      5       r:\ " S4 S5\65      5       r;/ S6Qr<g)7zPyTorch BLOOM model.é    N)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚ	LayerNormÚMSELoss)Ú
functionalé   )ÚCacheÚDynamicCacheÚStaticCache)ÚGenerationMixin)Úcreate_causal_mask)ÚGradientCheckpointingLayer)Ú)BaseModelOutputWithPastAndCrossAttentionsÚ!CausalLMOutputWithCrossAttentionsÚQuestionAnsweringModelOutputÚ SequenceClassifierOutputWithPastÚTokenClassifierOutput)ÚPreTrainedModel)Úauto_docstringÚloggingé   )ÚBloomConfigÚattention_maskÚ	num_headsÚdtypeÚreturnc                 óÀ  • U R                   u  p4S[        R                  " [        R                  " U5      5      -  n[        R
                  " SS[        R                  " U5      S-
  * -  * -  U R                  [        R                  S9n[        R                  " SSU-   U R                  [        R                  S9n[        R                  " Xg5      nXQ:w  a¿  [        R
                  " SS[        R                  " SU-  5      S-
  * -  * -  U R                  [        R                  S9n	[        XQU-
  5      n
[        R                  " SSSU
-  -   SU R                  [        R                  S9n[        R                  " U[        R                  " X›5      /SS9nU R                  SS9S-
  U -  SS2SSS24   nUS	   U-  nUR                  X1-  SU5      R                  U5      $ )
aV  
Link to paper: https://huggingface.co/papers/2108.12409 Alibi tensor is not causal as the original paper mentions, it
relies on a translation invariance of softmax for quick implementation: with l being a tensor, and a fixed value
`softmax(l+a) = softmax(l)`. Based on
https://github.com/ofirpress/attention_with_linear_biases/blob/a35aaca144e0eb6b789dfcb46784c4b8e31b7983/fairseq/models/transformer.py#L742
TODO @thomasw21 this doesn't work as nicely due to the masking strategy, and so masking varies slightly.

Args:
Returns tensor shaped (batch_size * num_heads, 1, max_seq_len)
    attention_mask (`torch.Tensor`):
        Token-wise attention mask, this should be of shape (batch_size, max_seq_len).
    num_heads (`int`):
        number of heads
    dtype (`torch.dtype`, *optional*, default=`torch.bfloat16`):
        dtype of the output tensor
é   r	   ©Údevicer   r   r   ©ÚdiméÿÿÿÿN).N)ÚshapeÚmathÚfloorÚlog2ÚtorchÚtensorr!   Úfloat32ÚarangeÚint32ÚpowÚminÚcatÚcumsumÚreshapeÚto)r   r   r   Ú
batch_sizeÚ
seq_lengthÚclosest_power_of_2ÚbaseÚpowersÚslopesÚ
extra_baseÚnum_remaining_headsÚextra_powersÚarange_tensorÚalibis                 Úe/home/mande/repo/quber/.venv/lib/python3.13/site-packages/transformers/models/bloom/modeling_bloom.pyÚbuild_alibi_tensorr@   -   s®  € ð" ,×1Ñ1Ñ€JØœdŸjšj¬¯ª°9Ó)=Ó>Ñ>ÐÜ�<Š<Ø	�”t—y’yÐ!3Ó4°qÑ8Ð9Ñ9Ð:Ñ;ÀN×DYÑDYÔaf×anÑanñ€Dô �\Š\˜!˜QÐ!3Ñ3¸N×<QÑ<QÔY^×YdÑYdÑe€FÜ�YŠY�tÓ$€FàÓ&Ü—\’\Ø�Aœ4Ÿ9š9 QÐ);Ñ%;Ó<¸qÑ@ÐAÑAÐBÑCÈN×LaÑLaÔin×ivÑivñ
ˆ
ô "Ð"4ÐBTÑ6TÓUÐÜ—|’| A q¨1Ð/BÑ+BÑ'BÀAÈn×NcÑNcÔkp×kvÑkvÑwˆÜ—’˜F¤E§I¢I¨jÓ$GÐHÈaÑPˆð %×+Ñ+°Ð+Ð3°aÑ7¸>ÑIÊ1ÈdÒTUÈ:ÑV€MØ�9Ñ Ñ-€EØ�=‰=˜Ñ/°°JÓ?×BÑBÀ5ÓIÐIó    ÚxÚresidualÚprobÚtrainingc                 ó8   • [         R                  " XUS9nX-   nU$ )zÞ
Dropout add function

Args:
    x (`torch.tensor`):
        input tensor
    residual (`torch.tensor`):
        residual tensor
    prob (`float`):
        dropout probability
    training (`bool`):
        training mode
)ÚprE   )ÚFÚdropout)rB   rC   rD   rE   Úouts        r?   Údropout_addrK   Y   s    € ô �)Š)�A¨Ñ
1€CØ
‰.€CØ€JrA   c                 ó^   • U S-  S[         R                  " SU -  SSU -  U -  -   -  5      -   -  $ )zÈ
Custom bias GELU function. Adapted from Megatron-DeepSpeed code. Here we use a simple implementation (inference) to
make the model jitable.

Args:
    x (`torch.tensor`):
        input hidden states
ç      à?ç      ð?ç Þe3Eˆé?r   ç÷Hmâä¦?©r)   Útanh)rB   s    r?   Úbloom_gelu_forwardrS   l   s8   € ð ˆs‰7�cœEŸJšJ z°A¡~¸¸XÈ¹\ÈAÑ=MÑ9MÑ'NÓOÑOÑPÐPrA   Úgc                 ó¢   • US   n[         R                  " SU-  SSU-  U-  -   -  5      nSU-  SX"-  -
  SSU-  U-  -   -  -  SSU-   -  -   nX0-  $ )a   
gradient of tanh approximation of gelu gradient of actual gelu is: 0.5 * (1. + torch.erf(x * 0.70710678)) +
0.3989423 * x * torch.exp(-0.5 * x * x)

Args:
    g (`torch.tensor`):
        gradient output tensor
    x (`torch.tensor`):
        input tensor
r   rO   r   rP   rM   g6”ü¾vf»?rQ   )rT   rB   Útanh_outÚffs       r?   Úbloom_gelu_backrX   x   sv   € ð 	
ˆ!‰€AÜ�zŠz˜* q™.¨A°¸1±¸qÑ0@Ñ,@ÑAÓB€Hà	ˆq‰�Q˜Ñ,Ñ,°¸lÈQÑ>NÐQRÑ>RÑ1RÑSÑ	TÐWZÐ^_ÐbjÑ^jÑWkÑ	k€BØ‰6€MrA   c                   óœ   • \ rS rSr\S\R                  S\R                  4S j5       r\S\R                  S\R                  4S j5       rSr	g)	ÚGeLUFunctionéŠ   Úinputr   c                 ó:   • U R                  U5        [        U5      $ ©N)Úsave_for_backwardrS   )Úctxr\   s     r?   ÚforwardÚGeLUFunction.forward‹   s   € à×Ñ˜eÔ$Ü! %Ó(Ð(rA   Úgrad_outputc                 ó4   • U R                   n[        X5      nU$ r^   )Úsaved_tensorsrX   )r`   rc   r\   Útmps       r?   ÚbackwardÚGeLUFunction.backward�   s   € à×!Ñ!ˆÜ˜kÓ1ˆØˆ
rA   © N)
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ústaticmethodr)   ÚTensorra   rg   Ú__static_attributes__ri   rA   r?   rZ   rZ   Š   sT   † Øð)˜EŸL™Lð )¨U¯\©\ó )ó ð)ð ð 5§<¡<ð °E·L±Ló ó órA   rZ   c                   óf   ^ • \ rS rSrSrU 4S jrS\R                  S\R                  4S jrSr	U =r
$ )Ú	BloomGelué—   zF
Partly copied from Megatron-DeepSpeed code and adapted for our needs
c                 ó"   >• [         TU ]  5         g r^   )ÚsuperÚ__init__)ÚselfÚ	__class__s    €r?   rv   ÚBloomGelu.__init__œ   s   ø€ Ü‰ÑÕrA   rB   r   c                 ó,   • [         R                  U5      $ r^   )rZ   Úapply)rw   rB   s     r?   ra   ÚBloomGelu.forwardŸ   s   € Ü×!Ñ! !Ó$Ð$rA   ri   )rj   rk   rl   rm   Ú__doc__rv   r)   ro   ra   rp   Ú__classcell__©rx   s   @r?   rr   rr   —   s-   ø† ñõð%˜Ÿ™ð %¨%¯,©,÷ %ò %rA   rr   c                   ól  ^ • \ rS rSrSS\S\S-  4U 4S jjjrS\R                  S\	\R                  \R                  \R                  4   4S jr
S	\R                  S\R                  4S
 jr   SS\R                  S\R                  S\R                  S\R                  S\S-  S\S\4S jjrSrU =r$ )ÚBloomAttentioné£   NÚconfigÚ	layer_idxc                 óœ  >• [         TU ]  5         UR                  U l        UR                  U l        UR                  U l        UR
                  U l        U R                  U R                  -  U l        U R                  U l        UR                  U l	        U R                  U R                  -  U R                  :w  a&  [        SU R                   SU R                   S35      eS[        R                  " U R                  5      -  U l        SU l        X l        Uc-  [         R#                  SU R$                  R&                   S35        [(        R*                  " U R                  SU R                  -  SS	9U l        [(        R*                  " U R                  U R                  5      U l        [(        R0                  " UR2                  5      U l        g )
NzA`hidden_size` must be divisible by num_heads (got `hidden_size`: z and `num_heads`: z).rN   zInstantiating z¹ without passing a `layer_idx` is not recommended and will lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` when creating this class.r	   T©Úbias)ru   rv   Úpretraining_tpÚslow_but_exactÚhidden_sizeÚn_headr   Úhead_dimÚ
split_sizeÚhidden_dropoutÚ
ValueErrorr&   ÚsqrtÚinv_norm_factorÚbetar„   ÚloggerÚwarning_oncerx   rj   r   ÚLinearÚquery_key_valueÚdenseÚDropoutÚattention_dropout)rw   rƒ   r„   rx   s      €r?   rv   ÚBloomAttention.__init__¤   sx  ø€ Ü‰ÑÔà$×3Ñ3ˆÔØ$×3Ñ3ˆÔà!×-Ñ-ˆÔØŸ™ˆŒØ×(Ñ(¨D¯N©NÑ:ˆŒØ×*Ñ*ˆŒØ$×3Ñ3ˆÔà�=‰=˜4Ÿ>™>Ñ)¨T×-=Ñ-=Ó=ÜØSÐTX×TdÑTdÐSeð fØ—N‘NÐ# 2ð'óð ð  #¤T§Y¢Y¨t¯}©}Ó%=Ñ=ˆÔØˆŒ	Ø"ŒØÑÜ×ÑØ  §¡×!8Ñ!8Ð 9ð :,ð ,ôô  "Ÿyšy¨×)9Ñ)9¸1¸t×?OÑ?OÑ;OÐVZÑ[ˆÔÜ—Y’Y˜t×/Ñ/°×1AÑ1AÓBˆŒ
Ü!#§¢¨F×,DÑ,DÓ!EˆÕrA   Ú	fused_qkvr   c                 ó  • UR                   u  p#nUR                  X#U R                  SU R                  5      nUSSSS24   R	                  SS5      nUSSSS24   R	                  SS5      nUSSSS24   R	                  SS5      nXVU4$ )aº  
Split the last dimension into (num_heads, head_dim) and reshapes to (bs, heads, len, dim) shape
without making any copies, results share same memory storage as `fused_qkv`

Args:
    fused_qkv (`torch.tensor`): [batch_size, seq_length, num_heads * 3 * head_dim]

Returns:
    query: [batch_size, num_heads, seq_length, head_dim]
    key: [batch_size, num_heads, seq_length, head_dim]
    value: [batch_size, num_heads, seq_length, head_dim]
r	   .r   Nr   r   )r%   Úviewr   rŒ   Ú	transpose)rw   r›   r4   r5   Úthree_times_hidden_sizeÚquery_layerÚ	key_layerÚvalue_layers           r?   Ú_reshapeÚBloomAttention._reshapeÅ   s’   € ð ;D¿/¹/Ñ7ˆ
Ð 7Ø—N‘N :¸4¿>¹>È1ÈdÏmÉmÓ\ˆ	Ø  Qª 	Ñ*×4Ñ4°Q¸Ó:ˆØ˜c 1¢a˜iÑ(×2Ñ2°1°aÓ8ˆ	Ø  Qª 	Ñ*×4Ñ4°Q¸Ó:ˆØ {Ð2Ð2rA   rB   c                 ó  • UR                   u  p#nX R                  -  nUR                  XPR                  X0R                  5      nUR	                  SSSS5      nUR                  XSU R                  U R                  -  5      $ )zÇ
Merge heads together over the last dimension

Args:
    x (`torch.tensor`): [batch_size * num_heads, seq_length, head_dim]

Returns:
    torch.tensor: [batch_size, seq_length, num_heads * head_dim]
r   r   r   r	   )r%   r   r�   rŒ   Úpermuter2   )rw   rB   Úbatch_size_and_num_headsr5   Ú_r4   s         r?   Ú_merge_headsÚBloomAttention._merge_headsÙ   so   € ð 34·'±'Ñ/Ð ¨aØ-·±Ñ?ˆ
ð �F‰F�:Ÿ~™~¨z¿=¹=ÓIˆð �I‰I�a˜˜A˜qÓ!ˆð �y‰y˜°·±À$Ç-Á-Ñ1OÓPÐPrA   Úhidden_statesrC   r>   r   Ú
layer_pastÚ	use_cacheÚoutput_attentionsc                 óº  • UR                   u  pšnU R                  U5      nU R                  U5      u  pÞnUb  UR                  XïU R                  5      u  pïUR                  X�R                  -  SU R                  5      nUR                  X�R                  -  SU R                  5      R                  SS5      nUR                  X�R                  -  SU R                  5      nUR                  UUU R                  U R                  S9nUR                  X�R                  U
S5      nUb  UU-   n[        R                  " US[        R                   S9R#                  UR$                  5      nU R'                  U5      nUR                  X�R                  -  U
S5      n[        R(                  " UU5      nU R+                  U5      nU R,                  S:”  aÖ  U R.                  (       aÅ  U R0                  U R,                  -  n[        R2                  " U5      n[5        U R,                  5       H|  nU[        R6                  " US S 2S S 2[9        UU-  5      [9        US-   U-  5      24   U R:                  R<                  S S 2[9        UU-  5      [9        US-   U-  5      24   5      -   nM~     OU R;                  U5      n[?        UX R@                  U RB                  5      nUU4$ )Nr$   éþÿÿÿ)Úbatch1Úbatch2r’   Úalpha)r#   r   r   )"r%   r–   r£   Úupdater„   r2   r   rŒ   rž   Úbaddbmmr’   r‘   r�   rH   Úsoftmaxr)   r+   r3   r   r™   Úbmmr©   rˆ   r‰   rŠ   Ú
zeros_likeÚrangeÚlinearÚintr—   ÚweightrK   rŽ   rE   )rw   r«   rC   r>   r   r¬   r­   r®   Úkwargsr4   Úq_lengthr¨   r›   r    r¡   r¢   Úattention_scoresÚattn_weightsÚattention_probsÚattention_probs_reshapedÚcontext_layerÚslicesÚoutput_tensorÚis                           r?   ra   ÚBloomAttention.forwardò   s–  € ð #0×"5Ñ"5Ñˆ
˜aØ×(Ñ(¨Ó7ˆ	à.2¯m©m¸IÓ.FÑ+ˆ àÑ!Ø%/×%6Ñ%6°yÈtÏ~É~Ó%^Ñ"ˆIð "×)Ñ)¨*·~±~Ñ*EÀrÈ4Ï=É=ÓYˆØ×%Ñ% j·>±>Ñ&AÀ2ÀtÇ}Á}ÓU×_Ñ_Ð`bÐdfÓgˆ	Ø!×)Ñ)¨*·~±~Ñ*EÀrÈ4Ï=É=ÓYˆð !Ÿ=™=ØØØ—‘Ø×&Ñ&ð	 )ð 
Ðð (×,Ñ,¨Z¿¹ÈÐSUÓVˆØÑ%Ø'¨.Ñ8ˆLô Ÿ)š) L°bÄÇÁÑN×QÑQÐR]×RcÑRcÓdˆð ×0Ñ0°ÓAˆð $3×#7Ñ#7¸
Ç^Á^Ñ8SÐU]Ð_aÓ#bÐ ô Ÿ	š	Ð":¸KÓHˆð ×)Ñ)¨-Ó8ˆð ×Ñ Ó" t×':×':Ø×%Ñ%¨×(;Ñ(;Ñ;ˆFÜ!×,Ò,¨]Ó;ˆMÜ˜4×.Ñ.Ö/�Ø -´·²Ø!¢!¢Q¬¨A°©J«¼#¸qÀ1¹uÈÑ>NÓ:OÐ(OÐ"OÑPØ—J‘J×%Ñ%¢a¬¨Q°©Z«¼3ÀÀAÁÈÑ?OÓ;PÐ)PÐ&PÑQó1ñ !’ò 0ð !ŸJ™J }Ó5ˆMä# M°8×=PÑ=PÐRV×R_ÑR_Ó`ˆØ˜oÐ-Ð-rA   )r™   r’   r—   rŒ   rŽ   rŠ   r‘   r„   r   rˆ   r–   r‰   r�   r^   ©NFF)rj   rk   rl   rm   r   r»   rv   r)   ro   Útupler£   r©   r
   Úboolra   rp   r~   r   s   @r?   r�   r�   £   sê   ø† ñF˜{ð F°s¸T±z÷ Fð FðB3 %§,¡,ð 3°5¸¿¹ÀuÇ|Á|ÐUZ×UaÑUaÐ9aÑ3bô 3ð(Q˜eŸl™lð Q¨u¯|©|ô Qð> $(ØØ"'ñA.à—|‘|ðA.ð —,‘,ðA.ð �|‰|ð	A.ð
 Ÿ™ðA.ð ˜D‘LðA.ð ðA.ð  ÷A.ó A.rA   r�   c                   ó‚   ^ • \ rS rSrS\4U 4S jjrS\R                  S\R                  S\R                  4S jrSr	U =r
$ )	ÚBloomMLPi6  rƒ   c                 ó:  >• [         TU ]  5         UR                  nUR                  U l        UR                  U l        [
        R                  " USU-  5      U l        [        5       U l	        [
        R                  " SU-  U5      U l
        UR                  U l        g )Né   )ru   rv   rŠ   rˆ   r‰   r   r•   Údense_h_to_4hrr   Ú	gelu_implÚdense_4h_to_hrŽ   )rw   rƒ   rŠ   rx   s      €r?   rv   ÚBloomMLP.__init__7  sz   ø€ Ü‰ÑÔØ×(Ñ(ˆà$×3Ñ3ˆÔØ$×3Ñ3ˆÔÜŸYšY {°A¸±OÓDˆÔÜ"›ˆŒÜŸYšY q¨;¡¸ÓDˆÔØ$×3Ñ3ˆÕrA   r«   rC   r   c                 ó   • U R                  U R                  U5      5      nU R                  S:”  aë  U R                  (       aÚ  [        R
                  " U5      nU R                  R                  R                  S   U R                  -  n[        U R                  5       Hz  nU[        R                  " US S 2S S 2[        XT-  5      [        US-   U-  5      24   U R                  R                  S S 2[        XT-  5      [        US-   U-  5      24   5      -   nM|     OU R                  U5      n[        X2U R                  U R                  5      nU$ )Nr   r$   )rÐ   rÏ   rˆ   r‰   r)   r¸   rÑ   r¼   r%   r¹   rH   rº   r»   rK   rŽ   rE   )rw   r«   rC   Úintermediate_outputrÄ   rÆ   Úoutputs          r?   ra   ÚBloomMLP.forwardB  s#  € ØŸ™ t×'9Ñ'9¸-Ó'HÓIˆà×Ñ Ó" t×':×':Ü"'×"2Ò"2°8Ó"<ÐØ×'Ñ'×.Ñ.×4Ñ4°RÑ8¸4×;NÑ;NÑNˆFÜ˜4×.Ñ.Ö/�Ø&9¼A¿HºHØ!¢!¢Q¬¨A©J«¼#¸qÀ1¹uÈÑ>NÓ:OÐ(OÐ"OÑPØ×&Ñ&×-Ñ-ªa´°Q±Z³Ä3ÈÈAÉÐQWÑGWÓCXÐ1XÐ.XÑYó=ñ 'Ò#ò 0ð #'×"4Ñ"4°]Ó"CÐäÐ0¸D×<OÑ<OÐQU×Q^ÑQ^Ó_ˆàˆrA   )rÑ   rÏ   rÐ   rŽ   rˆ   r‰   )rj   rk   rl   rm   r   rv   r)   ro   ra   rp   r~   r   s   @r?   rÌ   rÌ   6  s:   ø† ð	4˜{÷ 	4ð U§\¡\ð ¸U¿\¹\ð ÈeÏlÉl÷ ò rA   rÌ   c                   ó¬   ^ • \ rS rSrSS\S\S-  4U 4S jjjr   SS\R                  S\R                  S\R                  S	\	S-  S
\
S\
4S jjrSrU =r$ )Ú
BloomBlockiU  Nrƒ   r„   c                 ó@  >• [         TU ]  5         UR                  n[        X1R                  S9U l        UR                  U l        [        X5      U l	        [        X1R                  S9U l
        [        U5      U l        UR                  U l        UR                  U l        g )N©Úeps)ru   rv   rŠ   r   Úlayer_norm_epsilonÚinput_layernormr‹   r   r�   Úself_attentionÚpost_attention_layernormrÌ   ÚmlpÚ(apply_residual_connection_post_layernormrŽ   )rw   rƒ   r„   rŠ   rx   s       €r?   rv   ÚBloomBlock.__init__V  s   ø€ Ü‰ÑÔØ×(Ñ(ˆä(¨×:SÑ:SÑTˆÔØŸ™ˆŒÜ,¨VÓ?ˆÔÜ(1°+×C\ÑC\Ñ(]ˆÔ%ä˜FÓ#ˆŒà8>×8gÑ8gˆÔ5Ø$×3Ñ3ˆÕrA   r«   r>   r   r¬   r­   r®   c           
      óô   • U R                  U5      nU R                  (       a  Un	OUn	U R                  UU	UUUUUS9u  p«U R                  U
5      nU R                  (       a  Un	OU
n	U R	                  X‰5      nXË4$ )N)r¬   r   r>   r­   r®   )rÝ   rá   rÞ   rß   rà   )rw   r«   r>   r   r¬   r­   r®   r½   Úlayernorm_outputrC   Úattention_outputrÀ   rÕ   s                r?   ra   ÚBloomBlock.forwardd  s�   € ð  ×/Ñ/°Ó>Ðð ×8×8Ø'‰Hà$ˆHð *.×)<Ñ)<ØØØ!Ø)ØØØ/ð *=ð *
Ñ&Ðð  ×8Ñ8Ð9IÓJÐð ×8×8Ø'‰Hà'ˆHð —‘Ð*Ó5ˆàÐ#Ð#rA   )rá   rŽ   rÝ   rà   r   rß   rÞ   r^   rÈ   )rj   rk   rl   rm   r   r»   rv   r)   ro   r
   rÊ   ra   rp   r~   r   s   @r?   rØ   rØ   U  s~   ø† ñ4˜{ð 4°s¸T±z÷ 4ð 4ð& $(ØØ"'ñ+$à—|‘|ð+$ð �|‰|ð+$ð Ÿ™ð	+$ð
 ˜D‘Lð+$ð ð+$ð  ÷+$ó +$rA   rØ   c                   ó6   • \ rS rSr% \\S'   SrSrS/rSr	Sr
Srg)	ÚBloomPreTrainedModeli’  rƒ   ÚtransformerTrØ   Úpast_key_valuesri   N)rj   rk   rl   rm   r   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_can_compile_fullgraphrp   ri   rA   r?   rè   rè   ’  s(   ‡ àÓØ%ÐØ&*Ð#Ø%˜ÐØ"3ÐØ!ÓrA   rè   c                   óŠ  ^ • \ rS rSrS\4U 4S jjrS\R                  S\S\R                  S\R                  4S jr
S	 rS
\R                  4S jr\        SS\R                  S-  S\S-  S\R                  S-  S\R                  S-  S\S-  S\S-  S\S-  S\S-  S\\R                  S4   \-  4S jj5       rSrU =r$ )Ú
BloomModeliœ  rƒ   c           
      ó  >• [         TU ]  U5        UR                  U l        UR                  U l        [        R                  " UR                  U R                  5      U l	        [        U R                  UR                  S9U l        [        R                  " [        UR                  5       Vs/ s H  n[!        XS9PM     sn5      U l        [        U R                  UR                  S9U l        SU l        U R)                  5         g s  snf )NrÚ   )r„   F)ru   rv   rŠ   Ú	embed_dimr‹   r   r   Ú	EmbeddingÚ
vocab_sizeÚword_embeddingsr   rÜ   Úword_embeddings_layernormÚ
ModuleListr¹   Únum_hidden_layersrØ   ÚhÚln_fÚgradient_checkpointingÚ	post_init)rw   rƒ   rÆ   rx   s      €r?   rv   ÚBloomModel.__init__ž  sÇ   ø€ Ü‰Ñ˜Ô à×+Ñ+ˆŒØŸ™ˆŒô  "Ÿ|š|¨F×,=Ñ,=¸t¿~¹~ÓNˆÔÜ)2°4·>±>Àv×G`ÑG`Ñ)aˆÔ&ô —’ÌÈv×OgÑOgÔIhÓiÒIhÀA¤
¨6Ô ?ÑIhÑiÓjˆŒô ˜dŸn™n°&×2KÑ2KÑLˆŒ	à&+ˆÔ#ð 	�‰Õùò  js   Â-Dr   r   r   r   c                 ó   • [        XU5      $ r^   )r@   )rw   r   r   r   s       r?   r@   ÚBloomModel.build_alibi_tensor³  s   € Ü! .¸UÓCÐCrA   c                 ó   • U R                   $ r^   ©r÷   )rw   s    r?   Úget_input_embeddingsÚBloomModel.get_input_embeddings¶  s   € Ø×#Ñ#Ð#rA   Únew_embeddingsc                 ó   • Xl         g r^   r  ©rw   r  s     r?   Úset_input_embeddingsÚBloomModel.set_input_embeddings¹  s   € Ø-ÕrA   NÚ	input_idsrê   Úinputs_embedsr­   r®   Úoutput_hidden_statesÚreturn_dict.c	           
      ó¦  • Ub  UOU R                   R                  nUb  UOU R                   R                  nUb  UOU R                   R                  nUb  UOU R                   R                  nUSL USL-  (       a  [        S5      eU R                  (       a/  U R                  (       a  U(       a  [        R                  S5        SnUc  U R                  U5      nU(       a  Uc  [        U R                   S9nUR                  u  p«nUb  UR                  5       OSnX½-   nU R                  U5      nU(       a  SOSnU(       a  SOSnUc!  [        R                   " X®4UR"                  S9nOUR%                  UR"                  5      nU R'                  X0R(                  UR*                  S	9n[-        U R                   UUUS
9n[/        U R0                  5       H5  u  nnU(       a  UU4-   nU" UUUUUUS9nUS   nU(       d  M,  UUS   4-   nM7     U R3                  U5      nU(       a  UU4-   nU(       d  [5        S XòUU4 5       5      $ [7        UUUUS9$ )áj  
input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`):
    `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values.get_seq_length()`
    (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary.

    If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as
    `input_ids`.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are input IDs?](../glossary#input-ids)
Nz:You must specify exactly one of input_ids or inputs_embedszZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...F)rƒ   r   ri   ©r!   )r   )rƒ   r  r   rê   )r¬   r   r­   r®   r>   r   c              3   ó.   #   • U  H  oc  M  Uv •  M     g 7fr^   ri   )Ú.0Úvs     r?   Ú	<genexpr>Ú%BloomModel.forward.<locals>.<genexpr>  s   é € ð Úc�a—‘Òcùs   ‚Œ	)Úlast_hidden_staterê   r«   Ú
attentions)rƒ   r®   r  r­   r  r�   rý   rE   r“   r”   r÷   r   r%   Úget_seq_lengthrø   r)   Úonesr!   r3   r@   r   r   r   Ú	enumeraterû   rü   rÉ   r   )rw   r  rê   r   r  r­   r®   r  r  r½   r4   r5   r¨   Úpast_lengthÚseq_length_with_pastr«   Úall_self_attentionsÚall_hidden_statesr>   Úcausal_maskrÆ   ÚblockÚoutputss                          r?   ra   ÚBloomModel.forward¼  si  € ð4 2CÑ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ñ$DÑ È$Ï+É+×JjÑJjð 	ð "+Ñ!6‘I¸D¿K¹K×<QÑ<Qˆ	Ø%0Ñ%<‘kÀ$Ç+Á+×BYÑBYˆà˜Ð -°tÐ";×<ÜÐYÓZÐZà×&×&¨4¯=¯=¾YÜ×ÑØlôð ˆIàÑ Ø ×0Ñ0°Ó;ˆMæ˜Ñ0Ü*°$·+±+Ñ>ˆOà$1×$7Ñ$7Ñ!ˆ
 Ø:IÑ:U�o×4Ñ4Ô6Ð[\ˆØ)Ñ7Ðà×6Ñ6°}ÓEˆæ$5™b¸4ÐÞ"6™B¸DÐð Ñ!Ü"ŸZšZ¨Ð(JÐS`×SgÑSgÑh‰Nà+×.Ñ.¨}×/CÑ/CÓDˆNà×'Ñ'¨¿¹Èm×NaÑNaÐ'ÐbˆÜ(Ø—;‘;Ø'Ø)Ø+ñ	
ˆô " $§&¡&Ö)‰HˆAˆuÞ#Ø$5¸Ð8HÑ$HÐ!áØØ*Ø*Ø#Ø"3ØñˆGð $ A™JˆMß Ð Ø&9¸WÀQ¹Z¸MÑ&IÒ#ñ *ð$ Ÿ	™	 -Ó0ˆæØ 1°]Ð4DÑ DÐæÜñ Ø)Ð<MÐObÑcóó ð ô 9Ø+Ø+Ø+Ø*ñ	
ð 	
rA   )rô   rý   rû   rü   r   r÷   rø   ©NNNNNNNN)rj   rk   rl   rm   r   rv   r)   ro   r»   r   r@   r  r	  r   Ú
LongTensorr
   rÊ   rÉ   r   ra   rp   r~   r   s   @r?   rò   rò   œ  s4  ø† ð˜{÷ ð*D°·±ð DÈ#ð DÐV[×VaÑVað DÐfk×frÑfrô Dò$ð.°5·<±<ô .ð ð .2Ø(,Ø.2Ø15Ø!%Ø)-Ø,0Ø#'ñg
à×#Ñ# dÑ*ðg
ð  ™ðg
ð Ÿ™ tÑ+ð	g
ð
 ×'Ñ'¨$Ñ.ðg
ð ˜$‘;ðg
ð   $™;ðg
ð # T™kðg
ð ˜D‘[ðg
ð 
ˆu�|‰|˜SÐ Ñ	!Ð$MÑ	Môg
ó ög
rA   rò   zˆ
    The Bloom Model transformer with a language modeling head on top (linear layer with weights tied to the input
    embeddings).
    )Úcustom_introc                   óŒ  ^ • \ rS rSrSS0rS\4U 4S jjrS\R                  4S jr	     SU 4S	 jjr
\          SS
\R                  S-  S\S-  S\R                  S-  S\R                  S-  S\R                  S-  S\S-  S\S-  S\S-  S\S-  S\\R                  -  S\\R                     \-  4S jj5       rSrU =r$ )ÚBloomForCausalLMi'  zlm_head.weightz"transformer.word_embeddings.weightrƒ   c                 óÂ   >• [         TU ]  U5        [        U5      U l        [        R
                  " UR                  UR                  SS9U l        U R                  5         g ©NFr†   )
ru   rv   rò   ré   r   r•   rŠ   rö   Úlm_headrþ   ©rw   rƒ   rx   s     €r?   rv   ÚBloomForCausalLM.__init__0  sI   ø€ Ü‰Ñ˜Ô Ü% fÓ-ˆÔÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÑSˆŒð 	�‰ÕrA   r  c                 ó   • Xl         g r^   )r+  r  s     r?   Úset_output_embeddingsÚ&BloomForCausalLM.set_output_embeddings8  s   € Ø%�rA   Nc           	      ó2  >• [         TU ]  " U4UUUUUS.UD6n[        U[        5      (       ai  Ubf  UR	                  5       n	UR
                  u  p«X›-
  n[        R                  " X¬UR                  UR                  S9n[        R                  " X=/SS9nX8S'   U$ )N)rê   r   r  r­   Úis_first_iterationr    r$   r"   r   )ru   Úprepare_inputs_for_generationÚ
isinstancer   Úget_max_cache_shaper%   r)   Úzerosr!   r   r0   )rw   r  rê   r   r  r­   r2  r½   Úmodel_inputsÚtarget_lengthr4   r5   ÚdiffÚnew_attn_maskrx   s                 €r?   r3  Ú.BloomForCausalLM.prepare_inputs_for_generation;  s®   ø€ ô ‘wÒ<Øð
à+Ø)Ø'ØØ1ñ
ð ñ
ˆô �o¤{×3Ñ3¸Ñ8RØ+×?Ñ?ÓAˆMØ%3×%9Ñ%9Ñ"ˆJØ Ñ-ˆDä!ŸKšK¨
À×AVÑAVÐ^l×^rÑ^rÑsˆMÜ"ŸYšY¨Ð'FÈBÑOˆNØ-;Ð)Ñ*àÐrA   r  rê   r   r  Úlabelsr­   r®   r  r  Úlogits_to_keepr   c                 óì  • U	b  U	OU R                   R                  n	U R                  UUUUUUUU	S9nUS   n[        U
[        5      (       a  [        U
* S5      OU
nU R                  USS2USS24   5      nSnUb5  U R                  UUU R                   R                  UR                  S5      S9nU	(       d  U4USS -   nUb  U4U-   $ U$ [        UUUR                  UR                  UR                  S9$ )aô  
input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`):
    `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values.get_seq_length()`
    (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary.

    If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as
    `input_ids`.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are input IDs?](../glossary#input-ids)
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
    `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
    are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
N©rê   r   r  r­   r®   r  r  r   Únum_items_in_batch)rö   r@  r   ©ÚlossÚlogitsrê   r«   r  )rƒ   r  ré   r4  r»   Úslicer+  Úloss_functionrö   Úgetr   rê   r«   r  )rw   r  rê   r   r  r<  r­   r®   r  r  r=  r½   Útransformer_outputsr«   Úslice_indicesrC  rB  rÕ   s                     r?   ra   ÚBloomForCausalLM.forward^  s+  € ð@ &1Ñ%<‘kÀ$Ç+Á+×BYÑBYˆà"×.Ñ.ØØ+Ø)Ø'ØØ/Ø!5Ø#ð /ð 	
Ðð ,¨AÑ.ˆä8BÀ>ÔSV×8WÑ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆàˆØÑØ×%Ñ%ØØØŸ;™;×1Ñ1Ø#)§:¡:Ð.BÓ#Cð	 &ð ˆDö Ø�YÐ!4°Q°RÐ!8Ñ8ˆFØ)-Ñ)9�T�G˜fÑ$ÐE¸vÐEä0ØØØ/×?Ñ?Ø-×;Ñ;Ø*×5Ñ5ñ
ð 	
rA   )r+  ré   )NNNTF)
NNNNNNNNNr   )rj   rk   rl   rm   Ú_tied_weights_keysr   rv   r)   ro   r/  r3  r   r%  r
   rÊ   r»   rÉ   r   ra   rp   r~   r   s   @r?   r(  r(  '  sC  ø† ð +Ð,PÐQÐð˜{÷ ð&°E·L±Lô &ð ØØØØ ÷!ðF ð .2Ø(,Ø.2Ø-1Ø&*Ø!%Ø)-Ø,0Ø#'Ø-.ñD
à×#Ñ# dÑ*ðD
ð  ™ðD
ð Ÿ™ tÑ+ð	D
ð
 —|‘| dÑ*ðD
ð —‘˜tÑ#ðD
ð ˜$‘;ðD
ð   $™;ðD
ð # T™kðD
ð ˜D‘[ðD
ð ˜eŸl™lÑ*ðD
ð 
ˆu�|‰|Ñ	Ð@Ñ	@ôD
ó öD
rA   r(  aÖ  
    The Bloom Model transformer with a sequence classification head on top (linear layer).

    [`BloomForSequenceClassification`] uses the last token in order to do the classification, as other causal models
    (e.g. GPT-1) do.

    Since it does classification on the last token, it requires to know the position of the last token. If a
    `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
    no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
    padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
    each row of the batch).
    c                   ó(  ^ • \ rS rSrS\4U 4S jjr\         SS\R                  S-  S\	S-  S\R                  S-  S\R                  S-  S	\R                  S-  S
\S-  S\S-  S\S-  S\S-  S\\R                     \-  4S jj5       rSrU =r$ )ÚBloomForSequenceClassificationi¦  rƒ   c                 óä   >• [         TU ]  U5        UR                  U l        [        U5      U l        [
        R                  " UR                  UR                  SS9U l        U R                  5         g r*  )
ru   rv   Ú
num_labelsrò   ré   r   r•   rŠ   Úscorerþ   r,  s     €r?   rv   Ú'BloomForSequenceClassification.__init__µ  sV   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒÜ% fÓ-ˆÔÜ—Y’Y˜v×1Ñ1°6×3DÑ3DÈ5ÑQˆŒ
ð 	�‰ÕrA   Nr  rê   r   r  r<  r­   r®   r  r  r   c
                 ó†  • U	b  U	OU R                   R                  n	U R                  UUUUUUUU	S9nUS   nU R                  U5      nUb  UR                  S   nOUR                  S   nU R                   R
                  c  US:w  a  [        S5      eU R                   R
                  c  SnOÁUb�  XR                   R
                  :g  R                  UR                  [        R                  5      n[        R                  " UR                  S   UR                  [        R                  S9nUU-  R                  S5      nO.Sn[        R                  U R                  R                    S35        U[        R                  " XíR                  S	9U4   nSnUGbg  U R                   R"                  c‘  U R$                  S:X  a  S
U R                   l        OoU R$                  S:”  aN  UR&                  [        R(                  :X  d  UR&                  [        R*                  :X  a  SU R                   l        OSU R                   l        U R                   R"                  S
:X  aJ  [-        5       nU R$                  S:X  a&  U" UR/                  5       UR/                  5       5      nOeU" UU5      nO[U R                   R"                  S:X  a  [1        5       nU" UU5      nO-U R                   R"                  S:X  a  [3        5       nU" UU5      nU	(       d  U4USS -   nUb  U4U-   $ U$ [5        UUUR6                  UR8                  UR:                  S9$ )áÎ  
input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`):
    `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values.get_seq_length()`
    (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary.

    If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as
    `input_ids`.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are input IDs?](../glossary#input-ids)
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
Nr?  r   r   z=Cannot handle batch sizes > 1 if no padding token is defined.r$   r    zŠ will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`r  Ú
regressionÚsingle_label_classificationÚmulti_label_classificationrA  )rƒ   r  ré   rO  r%   Úpad_token_idr�   r3   r!   r)   r-   r,   Úargmaxr“   r”   rx   rj   Úproblem_typerN  r   Úlongr»   r   Úsqueezer   r   r   rê   r«   r  )rw   r  rê   r   r  r<  r­   r®   r  r  r½   rG  r«   rC  r4   Úlast_non_pad_tokenÚnon_pad_maskÚtoken_indicesÚpooled_logitsrB  Úloss_fctrÕ   s                         r?   ra   Ú&BloomForSequenceClassification.forward¾  sé  € ð> &1Ñ%<‘kÀ$Ç+Á+×BYÑBYˆà"×.Ñ.ØØ+Ø)Ø'ØØ/Ø!5Ø#ð /ð 	
Ðð ,¨AÑ.ˆØ—‘˜MÓ*ˆàÑ Ø"Ÿ™¨Ñ+‰Jà&×,Ñ,¨QÑ/ˆJà�;‰;×#Ñ#Ñ+°
¸a³ÜÐ\Ó]Ð]Ø�;‰;×#Ñ#Ñ+Ø!#ÑØÑ"à%¯©×)AÑ)AÑA×EÑEÀfÇmÁmÔUZ×U`ÑU`ÓaˆLÜ!ŸLšL¨¯©¸Ñ)<ÀVÇ]Á]ÔZ_×ZeÑZeÑfˆMØ"/°,Ñ">×!FÑ!FÀrÓ!JÑà!#ÐÜ×ÑØ—>‘>×*Ñ*Ð+ð ,Zð Zôð
 œuŸ|š|¨J¿}¹}ÑMÐOaÐaÑbˆàˆØÒØ�{‰{×'Ñ'Ñ/Ø—?‘? aÓ'Ø/;�D—K‘KÕ,Ø—_‘_ qÓ(¨f¯l©l¼e¿j¹jÓ.HÈFÏLÉLÔ\a×\eÑ\eÓLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ó7Ü"›9�Ø—?‘? aÓ'Ù# M×$9Ñ$9Ó$;¸V¿^¹^Ó=MÓN‘Dá# M°6Ó:‘DØ—‘×)Ñ)Ð-JÓJÜ+Ó-�Ù ¨vÓ6‘Ø—‘×)Ñ)Ð-IÓIÜ,Ó.�Ù ¨vÓ6�ÞØ#Ð%Ð(;¸A¸BÐ(?Ñ?ˆFØ)-Ñ)9�T�G˜fÑ$ÐE¸vÐEä/ØØ Ø/×?Ñ?Ø-×;Ñ;Ø*×5Ñ5ñ
ð 	
rA   )rN  rO  ré   ©	NNNNNNNNN)rj   rk   rl   rm   r   rv   r   r)   r%  r
   ro   rÊ   rÉ   r   ra   rp   r~   r   s   @r?   rL  rL  ¦  s÷   ø† ð˜{÷ ð ð .2Ø(,Ø.2Ø-1Ø&*Ø!%Ø)-Ø,0Ø#'ñe
à×#Ñ# dÑ*ðe
ð  ™ðe
ð Ÿ™ tÑ+ð	e
ð
 —|‘| dÑ*ðe
ð —‘˜tÑ#ðe
ð ˜$‘;ðe
ð   $™;ðe
ð # T™kðe
ð ˜D‘[ðe
ð 
ˆu�|‰|Ñ	Ð?Ñ	?ôe
ó öe
rA   rL  c                   ó(  ^ • \ rS rSrS\4U 4S jjr\         SS\R                  S-  S\	S-  S\R                  S-  S\R                  S-  S	\R                  S-  S
\S-  S\S-  S\S-  S\S-  S\\R                     \-  4S jj5       rSrU =r$ )ÚBloomForTokenClassificationi'  rƒ   c                 óÌ  >• [         TU ]  U5        UR                  U l        [        U5      U l        [        US5      (       a  UR                  b  UR                  nO-[        US5      (       a  UR                  b  UR                  nOSn[        R                  " U5      U l
        [        R                  " UR                  UR                  5      U l        U R                  5         g )NÚclassifier_dropoutrŽ   gš™™™™™¹?)ru   rv   rN  rò   ré   Úhasattrre  rŽ   r   r˜   rI   r•   rŠ   Ú
classifierrþ   )rw   rƒ   re  rx   s      €r?   rv   Ú$BloomForTokenClassification.__init__)  sµ   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä% fÓ-ˆÔÜ�6Ð/×0Ñ0°V×5NÑ5NÑ5ZØ!'×!:Ñ!:ÑÜ�VÐ-×.Ñ.°6×3HÑ3HÑ3TØ!'×!6Ñ!6Ñà!$ÐÜ—z’zÐ"4Ó5ˆŒÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰ÕrA   Nr  rê   r   r  r<  r­   r®   r  r  r   c
                 ó
  • U	b  U	OU R                   R                  n	U R                  UUUUUUUU	S9nUS   nU R                  U5      nU R	                  U5      nSnUbl  UR                  UR                  5      nUR                  u  nn[        5       nU" UR                  UU-  U R                  5      UR                  UU-  5      5      nU	(       d  U4USS -   nUb  U4U-   $ U$ [        UUUR                  UR                  S9$ )rR  Nr?  r   r   )rB  rC  r«   r  )rƒ   r  ré   rI   rg  r3   r!   r%   r   r�   rN  r   r«   r  )rw   r  rê   r   r  r<  r­   r®   r  r  r½   rG  r«   rC  rB  r4   r5   r_  rÕ   s                      r?   ra   Ú#BloomForTokenClassification.forward:  s+  € ð> &1Ñ%<‘kÀ$Ç+Á+×BYÑBYˆà"×.Ñ.ØØ+Ø)Ø'ØØ/Ø!5Ø#ð /ð 	
Ðð ,¨AÑ.ˆØŸ™ ]Ó3ˆØ—‘ Ó/ˆàˆØÑà—Y‘Y˜vŸ}™}Ó-ˆFØ%+§\¡\Ñ"ˆJ˜
Ü'Ó)ˆHÙØ—‘˜J¨Ñ3°T·_±_ÓEÀvÇ{Á{ÐS]Ð`jÑSjÓGkóˆDö Ø�YÐ!4°Q°RÐ!8Ñ8ˆFØ)-Ñ)9�T�G˜fÑ$ÐE¸vÐEä$ØØØ-×;Ñ;Ø*×5Ñ5ñ	
ð 	
rA   )rg  rI   rN  ré   ra  )rj   rk   rl   rm   r   rv   r   r)   r%  r
   ro   rÊ   rÉ   r   ra   rp   r~   r   s   @r?   rc  rc  '  s÷   ø† ð˜{÷ ð" ð .2Ø(,Ø.2Ø-1Ø&*Ø!%Ø)-Ø,0Ø#'ñB
à×#Ñ# dÑ*ðB
ð  ™ðB
ð Ÿ™ tÑ+ð	B
ð
 —|‘| dÑ*ðB
ð —‘˜tÑ#ðB
ð ˜$‘;ðB
ð   $™;ðB
ð # T™kðB
ð ˜D‘[ðB
ð 
ˆu�|‰|Ñ	Ð4Ñ	4ôB
ó öB
rA   rc  c                   ó  ^ • \ rS rSrU 4S jr\        SS\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S	\	S-  S
\	S-  S\	S-  S\
\-  4S jj5       rSrU =r$ )ÚBloomForQuestionAnsweringi€  c                 ó°   >• [         TU ]  U5        [        U5      U l        [        R
                  " UR                  S5      U l        U R                  5         g )Nr   )	ru   rv   rò   ré   r   r•   rŠ   Ú
qa_outputsrþ   r,  s     €r?   rv   Ú"BloomForQuestionAnswering.__init__‚  sA   ø€ Ü‰Ñ˜Ô Ü% fÓ-ˆÔÜŸ)š) F×$6Ñ$6¸Ó:ˆŒð 	�‰ÕrA   Nr  r   r  Ústart_positionsÚend_positionsr®   r  r  r   c	           	      ó  • Ub  UOU R                   R                  nU R                  UUUUUUS9n
U
S   nU R                  U5      nUR	                  SSS9u  pÞUR                  S5      R                  5       nUR                  S5      R                  5       nSnUbµ  Ub²  [        UR                  5       5      S:”  a  UR                  S5      n[        UR                  5       5      S:”  a  UR                  S5      nUR                  S5      nUR                  SU5      nUR                  SU5      n[        US9nU" XÔ5      nU" Xå5      nUU-   S-  nU(       d  XÞ4U
SS -   nUb  U4U-   $ U$ [        UUUU
R                  U
R                  S	9$ )
r  N)r   r  r®   r  r  r   r   r$   r"   )Úignore_indexr   )rB  Ústart_logitsÚ
end_logitsr«   r  )rƒ   r  ré   rn  ÚsplitrZ  Ú
contiguousÚlenÚsizeÚclampr   r   r«   r  )rw   r  r   r  rp  rq  r®   r  r  r½   r"  Úsequence_outputrC  rt  ru  Ú
total_lossÚignored_indexr_  Ú
start_lossÚend_lossrÕ   s                        r?   ra   Ú!BloomForQuestionAnswering.forwardŠ  s³  € ð4 &1Ñ%<‘kÀ$Ç+Á+×BYÑBYˆà×"Ñ"ØØ)Ø'Ø/Ø!5Ø#ð #ð 
ˆð " !™*ˆà—‘ Ó1ˆØ#)§<¡<°°r <Ð#:Ñ ˆØ#×+Ñ+¨BÓ/×:Ñ:Ó<ˆØ×'Ñ'¨Ó+×6Ñ6Ó8ˆ
àˆ
ØÑ&¨=Ñ+Dä�?×'Ñ'Ó)Ó*¨QÓ.Ø"1×"9Ñ"9¸"Ó"=�Ü�=×%Ñ%Ó'Ó(¨1Ó,Ø -× 5Ñ 5°bÓ 9�à(×-Ñ-¨aÓ0ˆMØ-×3Ñ3°A°}ÓEˆOØ)×/Ñ/°°=ÓAˆMä'°]ÑCˆHÙ! ,Ó@ˆJÙ 
Ó:ˆHØ$ xÑ/°1Ñ4ˆJæØ"Ð/°'¸!¸"°+Ñ=ˆFØ/9Ñ/E�Z�M FÑ*ÐQÈ6ÐQä+ØØ%Ø!Ø!×/Ñ/Ø×)Ñ)ñ
ð 	
rA   )rn  ré   r$  )rj   rk   rl   rm   rv   r   r)   r%  ÚFloatTensorrÊ   rÉ   r   ra   rp   r~   r   s   @r?   rl  rl  €  sâ   ø† õð ð .2Ø37Ø26Ø37Ø15Ø)-Ø,0Ø#'ñF
à×#Ñ# dÑ*ðF
ð ×)Ñ)¨DÑ0ðF
ð ×(Ñ(¨4Ñ/ð	F
ð
 ×)Ñ)¨DÑ0ðF
ð ×'Ñ'¨$Ñ.ðF
ð   $™;ðF
ð # T™kðF
ð ˜D‘[ðF
ð 
Ð-Ñ	-ôF
ó öF
rA   rl  )r(  rò   rè   rL  rc  rl  )=r}   r&   r)   r   Útorch.nnr   r   r   r   r   rH   Úcache_utilsr
   r   r   Ú
generationr   Úmasking_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   Úmodeling_utilsr   Úutilsr   r   Úconfiguration_bloomr   Ú
get_loggerrj   r“   ro   r»   r   r@   ÚfloatrÊ   rK   rS   rX   ÚautogradÚFunctionrZ   ÚModulerr   r�   rÌ   rØ   rè   rò   r(  rL  rc  rl  Ú__all__ri   rA   r?   Ú<module>r‘     s7  ðñ ã ã Ý ß LÓ LÝ $ç ;Ñ ;Ý )Ý /Ý 9÷õ õ .÷õ -ð 
×	Ò	˜HÓ	%€ð)J u§|¡|ð )JÀð )JÈEÏKÉKð )JÐ\a×\hÑ\hô )JðX�5—<‘<ð ¨5¯<©<ð ¸uð ÐPTð ÐY^×YeÑYeô ð&	Q˜%Ÿ,™,ð 	Q¨5¯<©<ô 	Qð�u—|‘|ð ¨¯©ð ¸¿¹ô ô$
�5—>‘>×*Ñ*ô 
ô	%�—	‘	ô 	%ôP.�R—Y‘Yô P.ôfˆr�y‰yô ô>:$Ð+ô :$ðz ô"˜?ó "ó ð"ð ôG
Ð%ó G
ó ðG
ñT ðñôv
Ð+¨_ó v
óðv
ñr ðñôp
Ð%9ó p
óðp
ðf ôU
Ð"6ó U
ó ðU
ðp ôP
Ð 4ó P
ó ðP
òf�rA   