ó
    pyüißC  ã                   óø  • S SK r S SKJr  S SKJrJr  S SKJr  S SKrSSK	J
r
Jr  SSKJr  SSKJr  \
" 5       (       a  S SKrS	r\R$                  " S
5      r\R(                  (       aJ  \R*                  " 5       r\R/                  \R0                  " S5      5        \R3                  \5        S\l        S\\R6                  \\\4   4S jr " S S\5      r\ " S S5      5       r\ " S S5      5       r  " S S5      r!g)é    N)Údeepcopy)Ú	dataclassÚfield)ÚIntEnumé   )Úis_psutil_availableÚis_torch_xpu_available)Úlogging)ÚtracedéÿÿÿÿÚContinuousBatchingLoggerz4%(asctime)s - %(name)s - %(levelname)s - %(message)sFÚreturnc                  ó˜  • [         R                  R                  5       (       a˜  [         R                  " S5      n [         R                  R	                  5         [         R                  R                  5         [         R                  R                  U 5      u  p[         R                  R                  U 5      nX!-
  nGO[        5       (       a»  [         R                  " S5      n [         R                  R	                  5         [         R                  R                  5         [         R                  R                  U 5      R                  n[         R                  R                  U 5      n[         R                  R                  U 5      nGOB[         R                  R                  R                  5       (       a†  [         R                  R                  R                  5       (       aY  [         R                  " S5      n [         R                  R!                  5       nU[#        [         R                  S5      " 5       -
  nSnO�[         R                  " S5      n [%        5       (       aO  [&        R(                  " 5       R*                  n[&        R,                  " 5       R/                  5       R0                  nUnO[2        R5                  S5        SnSnSnXX44$ )NÚcudaÚxpuÚmpsÚrecommended_max_memoryr   ÚcpuzŽCannot get memory breakdown on CPU without psutil: returning 0 for all memory values. Please install psutil to get an actual memory breakdown.)Útorchr   Úis_availableÚdeviceÚempty_cacheÚsynchronizeÚmem_get_infoÚmemory_reservedr	   r   Úget_device_propertiesÚtotal_memoryÚmemory_allocatedÚbackendsr   Úis_builtÚdriver_allocated_memoryÚgetattrr   ÚpsutilÚvirtual_memoryÚtotalÚProcessÚmemory_infoÚrssÚloggerÚerror)r   Úfree_memoryr   Úreserved_memoryÚallocated_memorys        Úq/home/mande/repo/quber/.venv/lib/python3.13/site-packages/transformers/generation/continuous_batching/requests.pyÚget_device_and_memory_breakdownr/   ,   sÝ  € Ü‡z�z×Ñ× Ñ Ü—’˜fÓ%ˆÜ�
‰
×ÑÔ Ü�
‰
×ÑÔ ô %*§J¡J×$;Ñ$;¸FÓ$CÑ!ˆÜŸ*™*×4Ñ4°VÓ<ˆØ'Ñ5ÒÜ	×	!Ñ	!Ü—’˜eÓ$ˆÜ�	‰	×ÑÔÜ�	‰	×ÑÔÜ—y‘y×6Ñ6°vÓ>×KÑKˆÜŸ)™)×3Ñ3°FÓ;ˆÜ Ÿ9™9×5Ñ5°fÓ=ÒÜ	�‰×	Ñ	×	(Ñ	(×	*Ñ	*¬u¯~©~×/AÑ/A×/JÑ/J×/LÑ/LÜ—’˜eÓ$ˆä—y‘y×8Ñ8Ó:ˆØ'¬'´%·)±)Ð=UÔ*VÓ*XÑXÐØ‰ä—’˜eÓ$ˆÜ× Ñ Ü!×0Ò0Ó2×8Ñ8ˆLÜ%Ÿ~š~Ó/×;Ñ;Ó=×AÑAÐØ.‰Oä�L‰Lð<ôð ˆLØˆOØ Ðà ÐBÐBó    c                   ó,   • \ rS rSrSrSrSrSrSrSr	Sr
g	)
ÚRequestStatuséU   z5Status of a generation request through its lifecycle.r   é   é   r   é   © N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__ÚPENDINGÚ
PREFILLINGÚDECODINGÚFINISHEDÚFAILEDÚ__static_attributes__r7   r0   r.   r2   r2   U   s   † Ù?à€GØ€JØ€HØ€HØƒFr0   r2   c                   ó  • \ rS rSr% Sr\\S'   \" \S9r	\\
   \S'   \" \S9r\\
   \S'   \" \S9r\\   \S'   Sr\S-  \S	'   \R                   r\\S
'   \" \R&                  S9r\\S'   Sr\\\4   \S'   Sr\\   S-  \S'   S\4S jrSrg)ÚGenerationOutputé_   aƒ  Tracks the output of a generation request.

Attributes:
    request_id (str): The ID of the generation request.
    prompt_ids (list[int]): The IDs of the prompt tokens.
    generated_tokens (list[int]): The generated tokens.
    logprobs (list[float]): The log probabilities of the generated tokens.
    error (Optional[str]): Any error message associated with the request. When None, the request was successful.
    status (RequestStatus): The status of the request.
    created_time (float): The time the request was created.
    lifespan (tuple[float, float]): The time the request was no longer pending and the time the request finished.
Ú
request_id©Údefault_factoryÚ
prompt_idsÚgenerated_tokensÚlogprobsNr*   ÚstatusÚcreated_time©r   r   ÚlifespanÚ
timestampsr   c                 ó<   • U R                   [        R                  :H  $ ©N)rL   r2   r@   ©Úselfs    r.   Úis_finishedÚGenerationOutput.is_finishedx   s   € Ø�{‰{œm×4Ñ4Ñ4Ð4r0   r7   )r8   r9   r:   r;   r<   ÚstrÚ__annotations__r   ÚlistrI   ÚintrJ   rK   Úfloatr*   r2   r=   rL   ÚtimeÚperf_counterrM   rO   ÚtuplerP   ÚboolrU   rB   r7   r0   r.   rD   rD   _   s­   ‡ ñð ƒOÙ!°$Ñ7€J��S‘	Ó7Ù"'¸Ñ"=Ð�d˜3‘iÓ=Ù!°$Ñ7€Hˆd�5‰kÓ7Ø€Eˆ3�‰:ÓØ)×1Ñ1€FˆMÓ1Ù°×0AÑ0AÑB€L�%ÓBØ$,€Hˆe�E˜5�LÑ!Ó,Ø%)€J��U‘˜dÑ"Ó)ð5˜T÷ 5r0   rD   c                   óú  • \ rS rSr% Sr\\S'   \\   \S'   Sr	\
\S'   Sr\
\S'   Sr\S	-  \S
'   S	r\\\   -  S	-  \S'   Sr\\S'   \" \S9r\\S'   \" \S9r\\   \S'   \" \S9r\\   \S'   \" \S9r\\   \S'   Sr\\S'   Sr\\S'   \R2                  r\\S'   \" \S9r\\   \S'   \" \R<                  S9r\\S'   S	r \S	-  \S'   Sr!\"\\4   \S'   \" \S9r#\\   \S'   Sr$\\S'   Sr%\\S'   \" \S9r&\\   \S'   Sr'\
\S '   S! r(\)S"\4S# j5       r*\*RV                  S$\4S% j5       r*\)S"\\   S	-  4S& j5       r,S' r-S"\4S( jr.S"\4S) jr/\0S*\S+\S	-  S"\
4S, j5       r1S- r2S. r3S/\S"S 4S0 jr4S"\4S1 jr5S4S2 jr6S3r7g	)5ÚRequestStateé|   aw  Tracks the state of a generation request through its lifecycle.

Attributes:
    request_id (str): The ID of the generation request.
    initial_tokens (list[int]): The initial prompt tokens.
    num_children (int): The number of children requests
    full_prompt_ids (list[int] | None): The tokens IDs of the full prompt.
    prompt_ids (list[int] | None): The tokens IDs currently being processed.
    remaining_prompt_ids (list[int]): The initial tokens IDs remaining to be processed.
    static_outputs (list[int]): The generated tokens.
    allocated_blocks (int): The number of blocks allocated to the request.
    position_offset (int): The current position in the sequence for position_ids.
    status (RequestStatus): The status of the request: can be one of PENDING, PREFILLING, PREFILLING_SPLIT,
                            SPLIT_PENDING_REMAINDER, DECODING, FINISHED, FAILED
    max_new_tokens (int | None): The maximum number of new tokens to generate.
    eos_token_id (None | int | list[int]): The ID(s) of the end-of-sequence tokens. Only used in post-init.
    _eos_token_ids (set[int]): The IDs of the end-of-sequence tokens, formatted as a set.
    streaming (bool): Whether to stream tokens as they're generated
    created_time (float): The time the request was created.
    error (Optional[str]): Any error message associated with the request. When None, has had no error yet.
rF   Úinitial_tokensFÚ	streamingÚrecord_timestampsé   NÚmax_new_tokensÚeos_token_idr   Únum_childrenrG   Úlogit_processor_kwargsÚtokens_to_processrJ   rK   Úposition_offsetÚallocated_blocksÚ_statusÚ_eos_token_idsrM   r*   rN   rO   Ú_timestampsÚ_true_initial_tokenséÿÿÿÚ_new_tokens_limitÚremaining_prefill_tokensÚis_cpu_offloadedc                 ó   • U R                   c  SOU R                   U l        U R                  S S  U l        U R                  c  g [        U R                  [        5      (       a7  U R                  S:¼  a&  U R                  R                  U R                  5        g g U R                   H&  nUS:¼  d  M  U R                  R                  U5        M(     g )Nrr   r   )	rg   rs   rc   rt   rh   Ú
isinstancerZ   ro   Úadd)rT   Útoken_ids     r.   Ú__post_init__ÚRequestState.__post_init__¹   s­   € à/3×/BÑ/BÑ/J¡ÐPT×PcÑPcˆÔà(,×(;Ñ(;¹AÐ(>ˆÔ%à×ÑÑ$Øä˜×)Ñ)¬3×/Ñ/Ø× Ñ  AÓ%Ø×#Ñ#×'Ñ'¨×(9Ñ(9Õ:ð &ð !×-Ô-�Ø˜q•=Ø×'Ñ'×+Ñ+¨HÖ5ò .r0   r   c                 ó   • U R                   $ rR   )rn   rS   s    r.   rL   ÚRequestState.statusË   s   € à�|‰|Ðr0   Úvaluec                 ó   • U R                   [        R                  :X  a  [        R                  " 5       S4U l        OMU[        R                  :X  a9  U R
                  S   [        R                  " 5       4U l        U R                  5         Xl         g )Nr   r   )rn   r2   r=   r\   r]   rO   r@   Úlog_end_of_request)rT   r~   s     r.   rL   r}   Ï   sf   € à�<‰<œ=×0Ñ0Ó0Ü!×.Ò.Ó0°"Ð5ˆD�MØ”m×,Ñ,Ó,Ø!Ÿ]™]¨1Ñ-¬t×/@Ò/@Ó/BÐCˆDŒMØ×#Ñ#Ô%Ø�r0   c                 ó@   • U R                   (       a  U R                  $ S $ rR   )re   rp   rS   s    r.   rP   ÚRequestState.timestampsØ   s   € à#'×#9×#9ˆt×ÑÐC¸tÐCr0   c                 ó"  • [        U R                  5      nU R                  5       nU R                  S   U R                  -
  nU R                  S   U R                  -
  n[
        R                  SU R                   SU< SU< SU< SU< 3
5        g )Nr   r4   úRequest z finished: prefill_len = z decode_len = z start_time = z end_time = )Úlenrc   Úgenerated_lenrO   rM   r)   ÚinforF   )rT   Úprefill_lenÚ
decode_lenÚ
start_timeÚend_times        r.   r€   ÚRequestState.log_end_of_requestÜ   s�   € Ü˜$×-Ñ-Ó.ˆØ×'Ñ'Ó)ˆ
Ø—]‘] 1Ñ%¨×(9Ñ(9Ñ9ˆ
Ø—=‘= Ñ# d×&7Ñ&7Ñ7ˆÜ�‰Ø�t—‘Ð'Ð'A°;Ñ2BÀ/ÀJÁ?ÐRaÐT^ÑSbÐboÐdlÑcpÐqõ	
r0   c                 ó   • U R                   $ )zCGet the current length of the sequence (prompt + generated tokens).)rl   rS   s    r.   Úcurrent_lenÚRequestState.current_lenå   s   € à×#Ñ#Ð#r0   c                 ó,   • [        U R                  5      $ )z*Get the number of tokens generated so far.)r…   rJ   rS   s    r.   r†   ÚRequestState.generated_lené   s   € ä�4×(Ñ(Ó)Ð)r0   ry   Úlogprobc                 óX  • U R                   [        R                  :w  a  gU R                  (       a.  U R                  R                  [        R                  " 5       5        XR                  ;   nU R                  5       nU(       d  X@R                  :  aG  U R                  R                  U5        U/U l        US-  nUb  U R                  R                  U5        O%[        R                  SU R                    SU 35        U(       d  X@R                  :¼  a  [        R"                  U l         gg)z³Update the request with a newly generated token (and optional log probability of the token) and check for
completion. Returns True if the request is now complete, False otherwise.Fr4   r„   z generated a useless token: T)rL   r2   r?   re   rp   Úappendr\   r]   ro   r†   rs   rJ   rk   rK   r)   ÚwarningrF   r@   )rT   ry   r’   Úis_eosrŽ   s        r.   Úupdate_and_check_completionÚ(RequestState.update_and_check_completionî   sò   € ð
 �;‰;œ-×0Ñ0Ó0Øð ×!×!Ø×Ñ×#Ñ#¤D×$5Ò$5Ó$7Ô8ð ×0Ñ0Ñ0ˆØ×(Ñ(Ó*ˆö �k×$:Ñ$:Ó:Ø×!Ñ!×(Ñ(¨Ô2Ø&. ZˆDÔ"Ø˜1ÑˆKØÑ"Ø—‘×$Ñ$ WÔ-øä�N‰N˜X d§o¡oÐ%6Ð6RÐS[ÐR\Ð]Ô^æ�[×$:Ñ$:Ó:Ü'×0Ñ0ˆDŒKØØr0   c                 óŠ  • SU R                    3SU R                   3SU R                  5        3S[        U R                  5       3S[        U R
                  5       3SU R                   3S[        U R                  5       3SU R                   3S	U R                   3S
U R                   3/
nSSR                  U5      -   S-   $ )Nzrequest_id=zstatus=zout_tokens=zquery_length=zremaining_tokens=z
kv_length=zfull_prompt_length=zallocated_blocks=zgenerated_tokens=zlogit_processor_kwargs=zRequestState(
	z,
	z
))rF   rn   r†   r…   rk   rt   rl   rc   rm   rJ   rj   Újoin)rT   Úmsgs     r.   Ú__repr__ÚRequestState.__repr__  sÛ   € à˜$Ÿ/™/Ð*Ð+Ø�d—l‘l�^Ð$Ø˜$×,Ñ,Ó.Ð/Ð0ØœC × 6Ñ 6Ó7Ð8Ð9Ø¤ D×$AÑ$AÓ BÐCÐDØ˜×-Ñ-Ð.Ð/Ø!¤# d×&9Ñ&9Ó":Ð!;Ð<Ø × 5Ñ 5Ð6Ð7Ø × 5Ñ 5Ð6Ð7Ø% d×&AÑ&AÐ%BÐCð
ˆð # W§\¡\°#Ó%6Ñ6¸Ñ>Ð>r0   c                 óŒ  • U R                   (       aI  U R                  U R                   S U R                  -   U l        U R                  SU R                    U l        [        U R                  U R                  U R                  U R
                  U R                  U R                  U R                  U R                  U R                  S9	$ )z7Convert the request state to a GenerationOutput object.N)	rF   rI   rJ   rK   r*   rL   rM   rO   rP   )rq   rc   rJ   rD   rF   rK   r*   rL   rM   rO   rP   rS   s    r.   Úto_generation_outputÚ!RequestState.to_generation_output  s¡   € à×$×$Ø$(×$7Ñ$7¸×8QÑ8QÐ8SÐ$TÐW[×WlÑWlÑ$lˆDÔ!Ø"&×"5Ñ"5Ð6Q¸×8QÑ8QÐ"RˆDÔÜØ—‘Ø×*Ñ*Ø!×2Ñ2Ø—]‘]Ø—*‘*Ø—;‘;Ø×*Ñ*Ø—]‘]Ø—‘ñ

ð 
	
r0   Únew_request_idc                 ó¸   • [        U 5      nXl        [        R                  " 5       Ul        UR                  S4Ul        / Ul        U R                  SS Ul        U$ )ziFork the request into a new request with the same state except for request_id, created_time and lifespan.r   N)r   rF   r\   r]   rM   rO   rp   rt   )rT   r¡   Únew_requests      r.   ÚforkÚRequestState.fork.  sV   € ä˜t“nˆà!/ÔÜ#'×#4Ò#4Ó#6ˆÔ Ø +× 8Ñ 8¸"Ð=ˆÔØ"$ˆÔà/3×/LÑ/LÉQÐ/OˆÔ,ØÐr0   c                 óž   • U R                   U R                  U R                  U R                  U R                  [        U R                  5      S.$ )zXGet all the fields necessary to create a request that would have the same configuration.)rd   re   rg   rh   ri   rj   )rd   re   rg   rh   ri   r   rj   rS   s    r.   Úget_request_configÚRequestState.get_request_config:  sG   € ð Ÿ™Ø!%×!7Ñ!7Ø"×1Ñ1Ø ×-Ñ-Ø ×-Ñ-Ü&.¨t×/JÑ/JÓ&Kñ
ð 	
r0   c           
      óÂ  • U R                  5       nU R                  b%  U R                  [        U R                  5      -
  US'   [	        SU R
                  U R                  U R                  -   U R                  SS U R                  [        U R                  5      -   S.UD6nU R                  (       a  U R                  Ul        U$ [        U R                  5      Ul        U$ )aD  Creates an equivalent new request by removing the generated tokens and adding them to the initial prompt. The
created request has THE SAME request_id. Notably, we can retrieve the original request from the created one with
the _true_initial_tokens attribute. The logprobs of the generated tokens are kept in the new request.Nrg   )rF   rc   rK   rq   r7   )	r§   rg   r…   rJ   ra   rF   rc   rK   rq   )rT   Úrequest_configÚ	new_states      r.   Ú!create_equivalent_initial_requestÚ.RequestState.create_equivalent_initial_requestE  sÔ   € ð
 ×0Ñ0Ó2ˆà×ÑÑ*Ø/3×/BÑ/BÄSÈ×I^ÑI^ÓE_Ñ/_ˆNÐ+Ñ,ä ð 
Ø—‘Ø×.Ñ.°×1FÑ1FÑFØ—]‘]¡1Ð%Ø!%×!:Ñ!:¼SÀ×ATÑATÓ=UÑ!Uñ	
ð
 ñ
ˆ	ð ×$×$Ø-1×-FÑ-FˆIÔ*ð Ðô .1°×1DÑ1DÓ-EˆIÔ*ØÐr0   )rs   rn   rJ   rc   rO   rt   rL   rk   )r   ra   )8r8   r9   r:   r;   r<   rW   rX   rY   rZ   rd   r_   re   rg   rh   ri   r   Údictrj   rk   rJ   rK   r[   rl   rm   r2   r=   rn   Úsetro   r\   r]   rM   r*   rO   r^   rp   rq   rs   rt   ru   rz   ÚpropertyrL   ÚsetterrP   r€   rŽ   r†   r   r—   rœ   rŸ   r¤   r§   r¬   rB   r7   r0   r.   ra   ra   |   sF  ‡ ñð. ƒOØ˜‘IÓð €IˆtÓØ#Ð�tÓ#ð "$€N�C˜$‘JÓ#Ø+/€L�#˜˜S™	‘/ DÑ(Ó/Ø€L�#ÓÙ#(¸Ñ#>Ð˜DÓ>ñ $)¸Ñ#>Ð�t˜C‘yÓ>Ù"'¸Ñ"=Ð�d˜3‘iÓ=Ù!°$Ñ7€Hˆd�5‰kÓ7Ø€O�SÓØÐ�cÓà*×2Ñ2€Gˆ]Ó2Ù$°SÑ9€N�C˜‘HÓ9ñ  °×0AÑ0AÑB€L�%ÓBØ€Eˆ3�‰:ÓØ$,€Hˆe�E˜5�LÑ!Ó,Ù$°TÑ:€K��e‘Ó:Ø !Ð˜#Ó!ð (Ð�sÓ'Ù*/ÀÑ*EÐ˜d 3™iÓEØ"Ð�dÓ"ò6ð$ ð˜ó ó ðð ‡]�]ð˜Mó ó ðð ðD˜D ™K¨$Ñ.ó Dó ðDò
ð$˜Sô $ð*˜sô *ð
 ð°Cð À%È$Á,ð ÐSWó ó ðò>?ò
ð"
 3ð 
¨>ô 
ð	
 Dô 	
÷r0   ra   c            
       ó:   • \ rS rSrSrSrS\S\S\S\SS	4
S
 jr	Sr
g	)ÚFutureRequestStatei_  zPTracks the current state of a request and the relevant information to update it.©ÚstateÚhas_new_tokenÚcomplete_blocksÚquery_lengthrµ   r¶   r·   r¸   r   Nc                 ó4   • Xl         X l        X0l        X@l        g rR   r´   )rT   rµ   r¶   r·   r¸   s        r.   Ú__init__ÚFutureRequestState.__init__e  s   € ØŒ
Ø*ÔØ.ÔØ(Õr0   )r·   r¶   r¸   rµ   )r8   r9   r:   r;   r<   Ú	__slots__ra   r_   rZ   rº   rB   r7   r0   r.   r³   r³   _  s8   † ÙZð N€Ið)˜lð )¸4ð )ÐRUð )Ðehð )Ðmq÷ )r0   r³   )"r\   Úcopyr   Údataclassesr   r   Úenumr   r   Úutilsr   r	   Úutils.loggingr
   Úutils.metricsr   r#   ÚTMP_TOKEN_IDÚ	getLoggerr)   Ú	propagateÚStreamHandlerÚhandlerÚsetFormatterÚ	FormatterÚ
addHandlerr^   r   rZ   r/   r2   rD   ra   r³   r7   r0   r.   Ú<module>rË      sû   ðó Ý ß (Ý ã ç @Ý $Ý #ñ ×ÑÛð €ð 
×	Ò	Ð5Ó	6€à	××Ø×#Ò#Ó%€GØ×Ñ˜×*Ò*Ð+aÓbÔcØ
×Ñ�gÔØ€FÔð&C¨¨u¯|©|¸SÀ#ÀsÐ/JÑ)Kô &CôR�Gô ð ÷5ð 5ó ð5ð8 ÷_ð _ó ð_÷D
)ò 
)r0   