ó
    6ljE&  ã                   ó&  • S r SSKrSSKJr  SSKJr  SSKJr  SSKJr  SSK	J
r
  SSKJr  SS	KJrJr  SS
KJrJr     SS\\-  S\
S-  S\S-  S\S\\\4   4
S jjr   SS\\-  S\S\
S-  S\S\\\   \4   4
S jjrSS\
S-  S\\\4   4S jjrg)z;Import utility for loading JSON analyses into the database.é    N)Údatetime)ÚPath)ÚAny)Úlogger)ÚSession)Úget_session)ÚEmbeddingServiceÚget_embedding_service)ÚDocumentÚExtractedTableÚ	json_pathÚsessionÚembedding_serviceÚgenerate_embeddingsÚreturnc                 óØ
  • [        U 5      n U R                  5       (       d  [        SU  35      e[        R                  " SU R
                   S35        [        U 5       n[        R                  " U5      nSSS5        WR                  SS5      n [        R                  " U5      n[        UR                  S	U R                   5      UUR                  S
5      UR                  S5      UR                  S5      UR                  S5      UR                  S5      S9nU(       a#  Uc   [        R                  " S5        [#        5       nUR                  S/ 5      n	U	 GH  n
U
R                  SS5      nU
R                  SS5      nSnSnU(       aD  U(       a=  U(       a6  Uc   eUR%                  X¼5      nUR'                  5       nUR'                  5       n[)        U
R                  S5      U
R                  S5      U
R                  S5      UUU
R                  S5      SU
R                  S/ 5      0U
R                  S0 5      UUS9
nUR*                  R-                  U5        GM     [/        U	5      nUGc7  [1        5        nUR3                  [        5      R5                  UR6                  S9R9                  5       nU(       ax  [        R                  " SUR6                   SUR:                   S35        UR3                  [(        5      R5                  UR:                  S9R=                  5       nUU4sSSS5        $ UR?                  U5        URA                  5         URC                  U5        [        RD                  " S UR6                   S!U S"URF                   S#35        UU4sSSS5        $  UR3                  [        5      R5                  UR6                  S9R9                  5       nU(       ao  [        R                  " SUR6                   SUR:                   S35        UR3                  [(        5      R5                  UR:                  S9R=                  5       nUU4$ UR?                  U5        URA                  5         URC                  U5        [        RD                  " S UR6                   S!U S"URF                   S#35        UU4$ ! , (       d  f       GNR= f! [         a3    [        R                  " SU S35        [        R                  " 5       n GNhf = f! , (       d  f       g= f! [H         a<  nURK                  5         [        RL                  " S$U R
                   S!U 35        e SnAff = f)%a  
Import a single JSON analysis file into the database.

Args:
    json_path: Path to the JSON file.
    session: Optional database session. If None, creates a new session.
    embedding_service: Optional embedding service. If None and generate_embeddings=True, creates one.
    generate_embeddings: Whether to generate embeddings during import.

Returns:
    Tuple of (Document instance, number of tables imported).

Raises:
    FileNotFoundError: If the JSON file doesn't exist.
    ValueError: If the JSON structure is invalid.
zJSON file not found: z
Importing z...NÚextraction_dateÚ z Invalid extraction_date format: z, using current timeÚdocumentÚtotal_pagesÚtotal_tablesÚexecutive_summaryÚmodel_providerÚmodel)Úfilenamer   r   r   r   r   r   z!Initializing embedding service...ÚtablesÚ	llm_titleÚllm_descriptionÚtable_idÚpageÚprocedural_titleÚtable_markdownÚheadersÚmetadata)
r   Úpage_numberr!   r   r   r"   r#   Útable_metadataÚtitle_embeddingÚdescription_embedding)r   z	Document z already exists (id=z), skipping)Údocument_idz	Imported ú: z tables from z pagesúFailed to import )'r   ÚexistsÚFileNotFoundErrorr   ÚinfoÚnameÚopenÚjsonÚloadÚgetr   ÚfromisoformatÚ
ValueErrorÚwarningÚnowr   Ústemr
   Úembed_table_metadataÚtolistr   r   ÚappendÚlenr   ÚqueryÚ	filter_byr   ÚfirstÚidÚcountÚaddÚcommitÚrefreshÚsuccessr   Ú	ExceptionÚrollbackÚerror)r   r   r   r   ÚfÚdataÚextraction_date_strr   r   Útables_dataÚ
table_datar   r   Ú	title_embÚdesc_embÚcombined_embÚtableÚtable_countÚ
db_sessionÚexistingÚexisting_countÚes                         Ú//home/mande/repo/quber/src/quber/db/importer.pyÚimport_json_filerX      s°  € ô, �Y“€IØ×Ñ×ÑÜÐ"7¸	°{Ð CÓDÐDä
‡K‚K�*˜YŸ^™^Ð,¨CÐ0Ô1ä	ˆiŒ˜AÜ#Ÿyšy¨›|ˆ÷ 
ð Ÿ(™(Ð#4°bÓ9Ðð)Ü"×0Ò0Ð1DÓEˆô Ø—‘˜* i§n¡nÓ5Ø'Ø—H‘H˜]Ó+Ø—X‘X˜nÓ-ØŸ(™(Ð#6Ó7Ø—x‘xÐ 0Ó1Ø�h‰h�wÓñ€Hö Ð0Ñ8Ü�ŠÐ7Ô8Ü1Ó3Ðð —(‘(˜8 RÓ(€KÜ!ˆ
Ø—N‘N ;°Ó3ˆ	Ø$Ÿ.™.Ð):¸BÓ?ˆð ˆ	ØˆÞ¦9¶Ø$Ñ0Ð0Ð0à,×AÑAÀ)Ó]ˆLà$×+Ñ+Ó-ˆIØ#×*Ñ*Ó,ˆHäØ—^‘^ JÓ/Ø"Ÿ™ vÓ.Ø'Ÿ^™^Ð,>Ó?ØØ+Ø%Ÿ>™>Ð*:Ó;Ø 
§¡¨y¸"Ó =Ð>Ø%Ÿ>™>¨*°bÓ9Ø%Ø"*ñ
ˆð 	�‰×Ñ˜u×%ñ7 "ô< �kÓ"€Kð ‚äŒ]˜jà!×'Ñ'¬Ó1×;Ñ;ÀX×EVÑEVÐ;ÐW×]Ñ]Ó_ˆHÞÜ—’ ¨8×+<Ñ+<Ð*=Ð=QÐRZ×R]ÑR]ÐQ^Ð^iÐjÔkØ!+×!1Ñ!1´.Ó!A×!KÑ!KÐX`×XcÑXcÐ!KÐ!d×!jÑ!jÓ!l�Ø  .Ð1÷ ‰]ð �N‰N˜8Ô$Ø×ÑÔØ×Ñ˜xÔ(ä�NŠNØ˜H×-Ñ-Ð.¨b°°¸]È8×K_ÑK_ÐJ`Ð`fÐgôð ˜kÐ*÷ ‰]ð$	Ø—}‘}¤XÓ.×8Ñ8À(×BSÑBSÐ8ÐT×ZÑZÓ\ˆHÞÜ—’ ¨8×+<Ñ+<Ð*=Ð=QÐRZ×R]ÑR]ÐQ^Ð^iÐjÔkØ!(§¡¬~Ó!>×!HÑ!HÐU]×U`ÑU`Ð!HÐ!a×!gÑ!gÓ!i�Ø  .Ð1Ð1à�K‰K˜Ô!Ø�N‰NÔØ�O‰O˜HÔ%ä�NŠNØ˜H×-Ñ-Ð.¨b°°¸]È8×K_ÑK_ÐJ`Ð`fÐgôð ˜kÐ*Ð*÷A 
Žûô ó )Ü�ŠÐ9Ð:MÐ9NÐNbÐcÔdÜ"Ÿ,š,›.‹ð)ú÷r �]ûôD ó 	Ø×ÑÔÜ�LŠLÐ,¨Y¯^©^Ð,<¸B¸q¸cÐBÔCØûð	úsR   ÁS ÂS É8B0TÌ2A)TÎ&B/T# ÑA)T# Ó 
SÓ9TÔTÔ
T Ô#
U)Ô-7U$Õ$U)Ú	directoryÚpatternc                 ó˜  • [        U 5      n U R                  5       (       d  [        SU  35      e[        U R	                  U5      5      nU(       d   [
        R                  " SU SU  35        / S4$ [
        R                  " S[        U5       S35        SnU(       a   [
        R                  " S5        [        5       n/ nSnUc.  U H'  n [        USUUS	9u  pšUR                  U	5        Xz-  nM)     O-U H'  n [        UUUUS	9u  pšUR                  U	5        Xz-  nM)     [
        R                  " S[        U5       S[        U5       S35        Xg4$ ! [         a1  n[
        R                  " S
UR                   SU 35         SnAMÁ  SnAff = f! [         a1  n[
        R                  " S
UR                   SU 35         SnAMÑ  SnAff = f)a�  
Import all JSON files matching a pattern from a directory.

Args:
    directory: Path to directory containing JSON files.
    pattern: Glob pattern for matching files (default: "*_analysis.json").
    session: Optional database session. If None, creates a new session.
    generate_embeddings: Whether to generate embeddings during import.

Returns:
    Tuple of (list of Document instances, total table count).
zDirectory not found: zNo files matching 'z' found in r   zFound z JSON files to importNz2Initializing embedding service for batch import...)r   r   r   r+   r*   zSuccessfully imported Ú/z files)r   r,   r-   ÚlistÚglobr   r6   r.   r<   r
   rX   r;   rF   rH   r/   rE   )rY   rZ   r   r   Ú
json_filesr   Ú	documentsr   Ú	json_fileÚdocrR   rV   s               rW   Úimport_directoryrc   ”   sÍ  € ô$ �Y“€IØ×Ñ×ÑÜÐ"7¸	°{Ð CÓDÐDä�i—n‘n WÓ-Ó.€JÞÜ�ŠÐ,¨W¨I°[ÀÀÐLÔMØ�Aˆwˆä
‡K‚K�&œ˜Z›Ð)Ð)>Ð?Ô@ð ÐÞÜ�ŠÐHÔIÜ1Ó3Ðà€IØ€Là�ã#ˆIðÜ#3ØØ Ø&7Ø(;ñ	$Ñ �ð × Ñ  Ô%ØÑ+’ò $ó $ˆIðÜ#3ØØ#Ø&7Ø(;ñ	$Ñ �ð × Ñ  Ô%ØÑ+’ñ $ô ‡N‚NÐ+¬C°	«NÐ+;¸1¼SÀ»_Ð<MÈVÐTÔUØÐ$Ð$øô) ó Ü—’Ð0°·±Ð0@ÀÀ1À#ÐFÔGÝûðûô ó Ü—’Ð0°·±Ð0@ÀÀ1À#ÐFÔGÝûðús0   Ã	#EÃ7#FÅ
FÅ&FÆFÆ
G	Æ&GÇG	c                 ón  • U cÖ  [        5        nUR                  [        5      R                  5       nUR                  [        5      R                  5       nUR                  [        5      R                  5       nUUUS:”  a  [        X2-  S5      OSU Vs/ s H0  nUR                  [        UR                  5      UR                  S.PM2     snS.nUsSSS5        $ U R                  [        5      R                  5       nU R                  [        5      R                  5       nU R                  [        5      R                  5       nUUUS:”  a  [        X2-  S5      OSU Vs/ s H0  nUR                  [        UR                  5      UR                  S.PM2     snS.nU$ s  snf ! , (       d  f       g= fs  snf )z¥
Get statistics about imported data.

Args:
    session: Optional database session. If None, creates a new session.

Returns:
    Dictionary with import statistics.
Nr   é   )r   r   Úpages)Útotal_documentsr   Úavg_tables_per_docr`   )r   r=   r   rA   r   ÚallÚroundr   r<   r   r   )r   rS   Ú	doc_countrR   r`   rb   Ústatss          rW   Úget_import_statsrm   Ý   s�  € ð �ÜŒ]˜jØ"×(Ñ(¬Ó2×8Ñ8Ó:ˆIØ$×*Ñ*¬>Ó:×@Ñ@ÓBˆKð #×(Ñ(¬Ó2×6Ñ6Ó8ˆIð $-Ø +ØKTÐWXË=¤e¨KÑ,CÀQÔ&GÐ^_ñ  )óâ(˜ð "%§¡¼¸S¿Z¹Z»ÐSV×SbÑSbÔcÙ(ññ	ˆEð ÷# ‰]ð& —M‘M¤(Ó+×1Ñ1Ó3ˆ	Ø—m‘m¤NÓ3×9Ñ9Ó;ˆð —M‘M¤(Ó+×/Ñ/Ó1ˆ	ð  )Ø'ØGPÐSTÃ}¤%¨Ñ(?ÀÔ"CÐZ[ñ %óâ$�Cð !Ÿ\™\´S¸¿¹³_ÈsÏÉÔ_Ù$ññ	
ˆð ˆùò1÷ �]üò:s$   ŽBF!Â7FÃ
F!Å7F2
ÆF!Æ!
F/)NNF)z*_analysis.jsonNF)N)Ú__doc__r1   r   Úpathlibr   Útypingr   Úlogurur   Úsqlalchemy.ormr   Úquber.db.connectionr   Úquber.db.embeddingsr	   r
   Úquber.db.modelsr   r   ÚstrÚboolÚtupleÚintrX   r]   rc   Údictrm   © ó    rW   Ú<module>r}      sø   ðÙ Aã Ý Ý Ý å Ý "å +ß Gß 4ð
 #Ø15Ø %ñ	AØ�c‰zðAà�t‰^ðAð (¨$Ñ.ðAð ð	Að
 ˆ8�Sˆ=ÑõAðL %Ø"Ø %ñ	F%Ø�c‰zðF%àðF%ð �t‰^ðF%ð ð	F%ð
 ˆ4�‰>˜3ÐÑõF%ñR.˜g¨™nð .¸¸SÀ#¸X¹ö .r|   