ó
    9°Mj¹  ã                  ó¸   • S r SSKJr  SSKrSSKJs  Jr  SSK	r	SSK
r
SSKrSSKrSSKJr  SSKrSSKJrJrJr  SS jrSS jrSS jrSS	 jrSS
 jrS rSS jrg)aA  Unit tests for the S3 ingestion cache startup sweep (QUE-223).

The sweep is pure filesystem and keyed off atime, so these tests need no S3 /
moto: we create files, set their atime with ``os.utime``, and assert which
ones the sweep reclaims. Source-change (ETag) behavior is cloudpathlib's own
and is not re-tested here.
é    )ÚannotationsN)ÚPath)Úevict_stale_s3_cacheÚresolve_artifactsÚresolve_documentc                ó’  • Sn[         R                   " 5       nU S-  nUR                  S5        [        R                  " X2U45        U S-  S-  nUR                  R                  SS9  UR                  S5        X!-
  S	-
  n[        R                  " XEU45        [        X5      nS
ovU:H  oˆ(       d¥  [        R                  " SU4SXg45      S[        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OS[        R                  " U5      S.-  n	SSU	0-  n
[        [        R                  " U
5      5      eS =p‡UR                  oˆ" 5       o»(       dš  SS[        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OS[        R                  " U5      [        R                  " U5      S.-  n[        [        R                  " U5      5      eS =p‹UR                  oˆ" 5       o»(       + oÝ(       dš  SS[        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OS[        R                  " U5      [        R                  " U5      S.-  n
[        [        R                  " U
5      5      eS =n=p½g )Nç     @�@z	fresh.pdfs   freshÚnestedz	stale.pdfT)Úparentss   staleéd   é   ©ú==©z%(py0)s == %(py3)sÚevicted©Úpy0Úpy3úassert %(py5)sÚpy5zAassert %(py4)s
{%(py4)s = %(py2)s
{%(py2)s = %(py0)s.exists
}()
}Úfresh)r   Úpy2Úpy4zEassert not %(py4)s
{%(py4)s = %(py2)s
{%(py2)s = %(py0)s.exists
}()
}Ústale)ÚtimeÚwrite_bytesÚosÚutimeÚparentÚmkdirr   Ú
@pytest_arÚ_call_reprcompareÚ@py_builtinsÚlocalsÚ_should_repr_global_nameÚ	_safereprÚAssertionErrorÚ_format_explanationÚexists)Útmp_pathÚttlÚnowr   r   Úoldr   Ú@py_assert2Ú@py_assert1Ú@py_format4Ú@py_format6Ú@py_assert3Ú@py_format5Ú@py_assert5s                 Ú-/home/mande/repo/quber/tests/test_s3_cache.pyÚ#test_evict_removes_only_stale_filesr6      s¬  € Ø
€CÜ
�)Š)‹+€Cà�{Ñ"€EØ	×Ñ�hÔÜ‡H‚HˆU˜#�JÔà�xÑ +Ñ-€EØ	‡L�L×Ñ˜tÐÑ$Ø	×Ñ�hÔØ
‰)�c‰/€CÜ‡H‚HˆU˜#�JÔä" 8Ó1€GàÐ�a‰<×××Òˆ7×××××Õˆ7×××Ñˆ7××Õ�a××××××ÑØ�<‰<Ðˆ<‹>Ð�>×××××Ñˆ5×××Ñˆ5××Õˆ<××Óˆ>×××××ÒØ�|‰|Ðˆ|‹~ÐÔÐÕ×××××Ñˆu×××Ñˆu××Õˆ|××Óˆ~×××××Õó    c           	     óÈ  • SoU-  nSn[        X#5      nSoTU:H  of(       Gd7  [        R                  " SU4SXE45      S[        R                  " 5       ;   d  [        R
                  " [         5      (       a  [        R                  " [         5      OSS[        R                  " 5       ;   d  [        R
                  " U 5      (       a  [        R                  " U 5      OS[        R                  " U5      [        R                  " U5      [        R                  " U5      [        R                  " U5      S.-  nS	S
U0-  n[        [        R                  " U5      5      eS =n=n=n=n=peg )Nzdoes-not-existr	   r   r   )zF%(py8)s
{%(py8)s = %(py0)s((%(py1)s / %(py3)s), %(py6)s)
} == %(py11)sr   r*   )r   Úpy1r   Úpy6Úpy8Úpy11zassert %(py13)sÚpy13)	r   r!   r"   r#   r$   r%   r&   r'   r(   )	r*   r.   Ú@py_assert4r4   Ú@py_assert7Ú@py_assert10Ú@py_assert9Ú@py_format12Ú@py_format14s	            r5   Útest_evict_missing_dir_is_nooprD   +   sÇ   € Ø+;ÐIÐ+;Ñ ;ÐI¸VÐIÔÐ ;ÓDÐIÈÐIÈÑI×I×I×IÓIÐD×I×I×I×I×IÕIÔ×I×I×IÑIÔ×I×I×I×I×IÖI ×I×I×IÑI ×I×IÕIÐ+;×I×IÓI¸V×I×IÓIÐD×I×IÓIÈ×I×I×I×I×I×I×IÒIr7   c                ó  • U S-  nUR                  S5        [        [        U5      5      nX!:H  o3(       dÛ  [        R                  " SU4SX!45      S[
        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OSS[
        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OSS.-  nSS	U0-  n[        [        R                  " U5      5      eS
nUR                  o3" 5       nSovU:H  oˆ(       dÏ  [        R                  " SU4SXg45      S[
        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OS[        R                  " U5      [        R                  " U5      [        R                  " U5      S.-  n	SSU	0-  n
[        [        R                  " U
5      5      eS
=n=n=p‡g
)zEA local path resolves to the same filesystem Path, untouched (no S3).zdoc.pdfs   %PDF-1.4r   )z%(py0)s == %(py2)sÚresolvedÚlocal)r   r   zassert %(py4)sr   N)zI%(py4)s
{%(py4)s = %(py2)s
{%(py2)s = %(py0)s.read_bytes
}()
} == %(py7)s©r   r   r   Úpy7zassert %(py9)sÚpy9)r   r   Ústrr!   r"   r#   r$   r%   r&   r'   r(   Ú
read_bytes)r*   rG   rF   r/   Ú@py_format3r3   r2   Ú@py_assert6r4   Ú@py_format8Ú@py_format10s              r5   Ú'test_resolve_document_local_passthroughrQ   /   s  € à�yÑ €EØ	×Ñ�kÔ"Ü¤ E£
Ó+€HØÑ×××Òˆ8×××××Õˆ8×××Ñˆ8×××××Ö�u×××Ñ�u××××××ÒØ×ÑÐ/ÐÓ Ð/ KÐ/ KÑ/×/×/×/Ò/Ð ×/×/×/×/×/Õ/ˆ8×/×/×/Ñ/ˆ8×/×/Õ/Ð×/×/Ó/Ð ×/×/Ó/ K×/×/×/×/×/×/Ö/r7   c           
     ó  • U S-  R                  S5        [        U 5      nS/n[        X5      o3U :H  oD(       GdÀ  [        R                  " SU4SX045      S[
        R                  " 5       ;   d  [        R                  " [        5      (       a  [        R                  " [        5      OSS[
        R                  " 5       ;   d  [        R                  " [        5      (       a  [        R                  " [        5      OSS[
        R                  " 5       ;   d  [        R                  " U 5      (       a  [        R                  " U 5      OS[        R                  " U5      [        R                  " U5      [        R                  " U5      S[
        R                  " 5       ;   d  [        R                  " U 5      (       a  [        R                  " U 5      OSS.-  nS	S
U0-  n[        [        R                  " U5      5      eS=n=n=p4g)zBA local artifacts directory resolves to itself, untouched (no S3).zdoc.docling.jsonz{}r   )zX%(py8)s
{%(py8)s = %(py0)s(%(py4)s
{%(py4)s = %(py1)s(%(py2)s)
}, %(py6)s)
} == %(py10)sr   rK   r*   )r   r9   r   r   r:   r;   Úpy10zassert %(py12)sÚpy12N)Ú
write_textrK   r   r!   r"   r#   r$   r%   r&   r'   r(   )r*   r2   r4   r?   rA   Ú@py_format11Ú@py_format13s          r5   Ú(test_resolve_artifacts_local_passthroughrX   8   s  € àÐ"Ñ"×.Ñ.¨tÔ4Ü  ›]ÐMÐ-?Ð,@ÐMÔ˜]ÓAÐMÀXÑM×M×M×MÓMÐA×M×M×M×M×MÕMÔ×M×M×MÑMÔ×M×M×M×M×MÖMœS×M×M×MÑMœS×M×M×M×M×MÖM ×M×M×MÑM ×M×MÕM˜]×M×MÓMÐ,@×M×MÓMÐA×M×M×M×M×MÔMÀX×M×M×MÑMÀX×M×M×M×M×M×M×MÐMr7   c                ó2  ^ ^• SSK Js  Jn  / mSSUU 4S jjjnUR                  USU5        [	        SSS/5      nSS	/nTU:H  of(       d¦  [
        R                  " S
U4STU45      S[        R                  " 5       ;   d  [
        R                  " T5      (       a  [
        R                  " T5      OS[
        R                  " U5      S.-  nSSU0-  n[        [
        R                  " U5      5      eS=peSn	T U	-  n
SoºU-  oÄU:H  of(       Gd  [
        R                  " S
U4SXL45      S[        R                  " 5       ;   d  [
        R                  " U5      (       a  [
        R                  " U5      OSS[        R                  " 5       ;   d  [
        R                  " T 5      (       a  [
        R                  " T 5      OS[
        R                  " U	5      [
        R                  " U5      S.-  nSSU0-  n[        [
        R                  " U5      5      eS=n=n	=n
=p¼g)zqAn s3:// prefix resolves each named file through resolve_document and
returns their shared local cache directory.r   Nc                ób   >• TR                  U 5        TS-  S-  U R                  SS5      S   -  $ )NÚbucketÚprefixÚ/r   éÿÿÿÿ)ÚappendÚrsplit)ÚrawÚ	cache_dirrF   r*   s     €€r5   Úfake_resolveÚLtest_resolve_artifacts_s3_materializes_each_named_file.<locals>.fake_resolveE   s4   ø€ Ø�‰˜ÔØ˜(Ñ" XÑ-°·
±
¸3ÀÓ0BÀ2Ñ0FÑFÐFr7   r   ús3://bucket/prefixza.jsonzb.jsonzs3://bucket/prefix/a.jsonzs3://bucket/prefix/b.jsonr   r   rF   r   r   r   r[   r\   )z*%(py0)s == ((%(py2)s / %(py4)s) / %(py7)s)Úoutr*   rH   zassert %(py10)srS   )N)ra   rK   Úreturnr   )Úquber.files.cacheÚfilesÚcacheÚsetattrr   r!   r"   r#   r$   r%   r&   r'   r(   )r*   Úmonkeypatchrj   rc   rf   r.   r/   r0   r1   r2   r4   rN   Ú@py_assert8Ú@py_format9rV   rF   s   `              @r5   Ú6test_resolve_artifacts_s3_materializes_each_named_filero   >   sN  ù€ ÷ &Ð%à€H÷Gò Gð ×Ñ˜Ð1°<Ô@Ü
Ð0°8¸XÐ2FÓ
G€CØ3Ð5PÐQÐQˆ8ÐQÑQ×Q×Q×QÒQˆ8ÐQ×Q×Q×Q×Q×QÕQˆ8×Q×Q×QÑQˆ8×Q×QÕQÐQ×Q×Q×Q×Q×Q×QÑQØ%Ð0�(˜XÑ%Ð0¨Ð0¨Ñ0Ð0Ð0Ñ0×0×0×0Ó0ˆ3×0×0×0×0×0Õ0ˆ3×0×0×0Ñ0ˆ3×0×0×0×0×0Ö0�(×0×0×0Ñ0�(×0×0Õ0˜X×0×0Ó0¨×0×0×0×0×0×0×0Ð0r7   c                 ó„   • [         R                  " [        5         [        S/ 5        S S S 5        g ! , (       d  f       g = f)Nre   )ÚpytestÚraisesÚ
ValueErrorr   © r7   r5   Ú)test_resolve_artifacts_requires_filenamesru   O   s%   € Ü	�Š”zÕ	"ÜÐ.°Ô3÷ 
#×	"Ö	"ús   ›1±
?c                óR  • [         R                  R                  SS5      [        U 5      S.n[        R
                  " [        R                  SS/UU SSS9nUR                  nSoCU:H  oU(       dÞ  [        R                  " S	U4S
X445      S[        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OS[        R                  " U5      [        R                  " U5      S.-  n[        R                  " SUR                    35      S-   SU0-  n[#        [        R$                  " U5      5      eS=n=pTUR&                  o3R(                  oU" 5       oˆ(       d¯  SS[        R                  " 5       ;   d  [        R                  " U5      (       a  [        R                  " U5      OS[        R                  " U5      [        R                  " U5      [        R                  " U5      S.-  n	[#        [        R$                  " U	5      5      eS=n=pXg)a!  `import quber` runs init_s3_cache() (which reads get_settings().s3_cache);
it must succeed on a host with no AWS credentials and no .env (QUE-228).

Runs in a subprocess with a scrubbed environment so no ambient AWS creds or
`.env` can mask a regression, and cwd set to an empty temp dir.
ÚPATHÚ )rw   ÚHOMEz-cz&import quber; print(quber.__version__)T)ÚenvÚcwdÚcapture_outputÚtextr   r   )z2%(py2)s
{%(py2)s = %(py0)s.returncode
} == %(py5)sÚresult)r   r   r   zimport quber failed:
z
>assert %(py7)srI   Nz\assert %(py6)s
{%(py6)s = %(py4)s
{%(py4)s = %(py2)s
{%(py2)s = %(py0)s.stdout
}.strip
}()
})r   r   r   r:   )r   ÚenvironÚgetrK   Ú
subprocessÚrunÚsysÚ
executableÚ
returncoder!   r"   r#   r$   r%   r&   Ú_format_assertmsgÚstderrr'   r(   ÚstdoutÚstrip)
r*   Úscrubbedr~   r/   r>   r2   r1   rO   r4   Ú@py_format7s
             r5   Ú$test_import_quber_is_credential_freerŒ   T   sT  € ô —
‘
—‘˜v rÓ*Ü�H“ñ€Hô �^Š^Ü	�‰˜ÐGÐHØØØØñ€Fð ×ÑÐK ÐK Ñ!×K×K×KÒKÐ×K×K×K×K×KÕKˆ6×K×K×KÑKˆ6×K×KÕKÐ×K×KÓK ×K×K×KÐKÐ%;¸F¿M¹M¸?Ð#K×K×K×K×K×K×KÐKØ�=‰=Ð ×ÑÐ ÐÓ Ð Õ × × × × × Ñ ˆ6× × × Ñ ˆ6× × Õ ˆ=× × Ó Ð× × Ó Ð × × × × × Õ r7   )r*   r   )Ú__doc__Ú
__future__r   Úbuiltinsr#   Ú_pytest.assertion.rewriteÚ	assertionÚrewriter!   r   r�   rƒ   r   Úpathlibr   rq   rh   r   r   r   r6   rD   rQ   rX   ro   ru   rŒ   rt   r7   r5   Ú<module>r”      sP   ðñõ #ç  „ ƒ	Û Û 
Û Ý ã ç WÑ Wôô*Jô0ôNô1ò"4õ
!r7   