h:X dZddlZddlZddlmZddlmZddlZddlm Z ddl m Z m Z ddl mZdd lmZej"ej$d d Zej)ej*d ejdej,ddddej,ddej.e Dcgc]}|j0c}dej,dddej,dded !ej,d"d#dd$%d ed&ed'eed(eed)ed*ef d+Zej)ej*d,ejdd-d.ej,dddd/ej,ddej.e Dcgc]}|j0c}d0ej,ddd1ej,dded2!ej,d3d4d5d,ed&ed'eed(eed)ed6ef d7Zej)ej,ddej.e Dcgc]}|j0c}d8ej,d9d:dd;%d'eedZej)ej,d?dd@%ej,d"d#dd$%dAed*efdBZ ej)ej*dCejdej,dDddE%ej,d"d#dd$%dCedFed*efdGZ!ej)ej*d,ejdd-d.ej,d3dHdIej,dDddE%ej,d"d#dd$%d,ed6edFed*efdJZ"ej)ej,d"d#dd$%d*efdKZ#ej)ej,dej.dLdMgdNej,dOej.dPdQgdRej,dSdTedU!ej,d"d#dd$%d'eedVeedWed*efdXZ$dYZ%e&dZk(re%yycc}wcc}wcc}w)[z3 Command-line interface for Quber table inference. N)Path)Optional)logger) ModelConfig ModelProvider)TableInferenceProcessor)Configz0.2.5)versioncy)z7Quber - Advanced table extraction and inference system.Nr '/home/mande/repo/quber/src/quber/cli.pyclirs rpdf_pathT)exists)typez --output-dirz-ooutputz"Output directory (default: output))defaulthelpz --providerz-pz6Model provider to use (auto-detected if not specified))rrz--modelz-mz>Specific model to use (uses provider default if not specified))rz--max-concurrentz.Maximum concurrent table analyses (default: 5))rrrz --verbosez-vzEnable verbose logging)is_flagr output_dirprovidermodelmax_concurrentverbosec|rdnd}tjtjtj|t j }||_||_|r t|nd} t|||tjd fd } tj | \} } } tjd tjd | ddtjd| tjd| y#t$r9} tjd| dtjd Yd} ~ d} ~ wwxYw)z Analyze tables in a PDF document with LLM inference. Example: quber analyze document.pdf quber analyze document.pdf --provider anthropic --model claude-3-opus-20240229 DEBUGINFOlevelNrrconfigError: Terrrz Analyzing: cK jd{}t}|jddtj}||dz }||dz }j |t |j |t ||||fS7#t$r}tjd|d}~wwxYww)NTparentsexist_ok_analysis.json _summary.mdzAnalysis failed: ) process_documentrmkdirstem save_jsonstr save_summary Exceptionrerror) results output_path base_name json_file summary_fileerr processors r run_analysiszanalyze..run_analysisMs %66x@@Gz*K   dT  :X++I# >&BBI&I;k)BBL   Y 8  " "7C ,= >I|3 3A   LL,QC0 1  s9CBBBBCB C'CCCu ✓ Analysis completez - Found total_tables tablesz - JSON output: z - Summary: )rremoveaddsysstderrr load log_levelmax_concurrent_tablesrr ValueErrorclickechoexitasynciorun)rrrrrrrEr$ provider_enumr;r=r6r9r:r<s`` @ranalyzerNs"<#I MMO JJszz+[[]F F#1F 08M(+TM+]%X^_  JJxj)*,(/{{<>'B$GY  JJ() JJGN34G<= JJ"9+./ JJ|n-.G  WQC=d+  s5D"" E$+/EE$ directoryF)r file_okaydir_okayz.Output directory for results (default: output)zModel provider to usezSpecific model to usez;Maximum concurrent table analyses per document (default: 5)z --patternz*.pdfz&File pattern to match (default: *.pdf)patternctj}||_t|}t |j |s/t jd|dtjdt jdtdt|jdd|r t|nd} t||| fd } tj | } t| } t#d | D} t jd t jd| dtdt jd| dt jd|dy#t$r9} t jd | dtjdYd} ~ d} ~ wwxYw)z Process multiple PDF documents in a directory. Example: quber batch documents/ quber batch documents/ --pattern "*.pdf" --output-dir results/ z!No files found matching pattern: Tr&rFound z PDF files to processr)Nr#r%cFKg}D]}tjd|j jt |d{}|j }|dz }|dz } j |t | j|t |tjd|dd|j||S7#t$r@}tjd|d tjd |d |Yd}~d}~wwxYww) Nz Processing: r,r-u ✓ r>z tables analyzedu ✗ Failed: Tr&zFailed to process : ) rHrInamer.r2r0r1r3appendr4rr5) r6pdf_fileresultr8 json_path summary_pathr;r7 pdf_filesr<s r process_batchzbatch..process_batchs!H JJ 7 8 C(99#h-HH%MM 'YK~*FF * {+-FF ##FC N;&&vs$:#;;KLMv&"(#I C ^A3/T: 1(2aSABB CsA*D!C C BCD!C D5DD!DD!c3&K|] }|d yw)r>Nr ).0rs r zbatch..s:'Qq('su ✓ Batch processing complete!z - Processed /z documentsz - Analyzed total tablesz - Results saved to: )r rDrFrlistglobrHrIrBrJlenr/rr rGrKrLsum)rOrrrrrRr$dir_pathrMr;r^r6total_documentsr>r7r]r<s @@@rbatchrklsx@[[]F#1F IHX]]7+,I  6wi@dK   JJI''<=>z"KdT208M(+TM+]%X^_ 0kk-/*G'lO:'::L JJ12 JJ0#i.1ALM JJ|nM:; JJ' |156M  WQC=d+  sE>> G/F;;Gz&Show models for specific provider onlyz--allz-az"Show all models (can be very long)allcVddlm}|d}|j}|r t|g}nt t}t j dt j dt j dt j d|D]'}|j|}|j|v}|rd nd } t j d |jjd | t j d ||j} |s"t| dkDr| dd} t| dz } n| } d} | D][} | tjj|k(rdnd}t| dkDr | dddz}n| }t j d||]| dkDst j d| d*|sit j dtD]J}tjj|}|s%t j d|d|jdLyy)z(List available models for each provider.r) AgentFactoryF)enable_logfirez%Model Providers and Available Models:2==================================================z? Note: You can use any model name directly, even if not listed.z?The list below shows commonly used models from known_models.py u✓ (API key found)u✗ (no API key)  z(---------------------------------------- Nrz (default)<9z... • z ... and z# more models (use --all to see all)u9 ⚠ No API keys found. Please set environment variables:z (for ))agentsrnget_available_providersrrerHrIlist_available_modelsvalueupperrgrDEFAULT_MODELSget API_KEY_VARS)rrlrnfactory availableproviders_to_showpmodels is_availablestatus model_listdisplay_models remainingrr model_displaykey_vars rrrs%%0G//1I*845 / JJ67 JJx JJQR JJQR ..q1ww)+ *6& rcy)zDatabase management commands.Nr r rrdbrs rz--dropz9Drop existing tables before creating (WARNING: data loss)dropcddlm}m}|rdnd}tjtj t j|tjd |}|||tjdy #t$r9}tjd |d t jdYd }~y d }~wwxYw) zInitialize the database schema.r) get_engineinit_dbrr r!zInitializing database...)drop_allu%✓ Database initialized successfullyu#✗ Failed to initialize database: Tr&N) db.connectionrrrr@rArBrCrHrIr4rJ)rrrrrEenginer;s rinitrs 3"I MMO JJszz+ JJ)*& :;  8<$G  s&B C/CCr[z--generate-embeddingsz!Generate embeddings during importgenerate_embeddingscddlm}|rdnd}tjtjt j | |||\}}|rdnd}tjd |jd |d |y#t$r9}tjd |d t jdYd}~yd}~wwxYw)z5Import a single JSON analysis file into the database.r)import_json_filerr r!)r with embeddingsrt ✓ Imported rVr?✗ Import failed: Tr&N) db.importerrrr@rArBrCrHrIfilenamer4rJ) r[rrrrEdoc table_countemb_msgr;s r import_jsonr)s ."I MMO JJszz++IK^_[(;$ ]3<<.;-wwiPQ  (,$7  ;B C /CCz*_analysis.jsonz0File pattern to match (default: *_analysis.json)cddlm}|rdnd}tjtjt j | ||||\}}|rdnd}tjd t|d |d |y#t$r9} tjd | d t jdYd} ~ yd} ~ wwxYw)z0Import all JSON analysis files from a directory.r)import_directoryrr r!)rRrrrtrz documents with rdrTr&N) rrrr@rArBrCrHrIrgr4rJ) rOrRrrrrEdocsr>rr;s r import_dirr>s."I MMO JJszz+-i^qrl(;$ ]3t9+-=l^=Y`Xabc  (,$7  rc ddlm}|rdnd}tjtjt j | |}tjdtjdtjd|d tjd |d tjd |d |drHtjd|dD]*}tjd|dd|dd|dd,yy#t$r9}tjd|dt jdYd}~yd}~wwxYw)zShow database statistics.r)get_import_statsrr r!z Database StatisticsrpzTotal documents: rjzTotal tables: r>zAvg tables/doc: avg_tables_per_doc documentsz Documents:rwrrVtablesz tables, pagesz pagesu✗ Failed to retrieve stats: Tr&N) rrrr@rArBrCrHrIr4rJ)rrrE stats_datarr;s rstatsrTs-."I MMO JJszz+%'  *+ 8 &z2C'D&EFG ^J~$>#?@A %j1E&F%GHI k " JJ~ &!+. VC O#4Bs8}oYsSZ|n\bcd/ #  3A37TB  sCD E /EElocalopenaiz&Embedding provider (default: from env)z--devicecudacpuz&Device for local model (default: cuda)z --batch-size z'Batch size for processing (default: 32)device batch_sizec ddlm}m}m}|rdnd}t j t j tj| tjd|||}tjd|5} | j|j} t| } | d k(rtjd  d d d y tjd | d d } t| dD]\} }|js|j s |j#|j|j }|j%|_|j%|_| dz } | |zd k(s~| j+tjd| d| d| j+tjd| d| dd d d y #1swYy xYw#t,$rN}tjd|dt j.|tj0dYd }~y d }~wwxYw)z3Generate embeddings for all tables in the database.r)get_embedding_service get_sessionExtractedTablerr r!z!Initializing embedding service...)rrzLoading tables from database...rzNo tables found in databaseNrTz tables to processz Processed rcz tables...u✓ Generated embeddings for r?u#✗ Failed to generate embeddings: Tr&)rrrrrr@rArBrCrHrIqueryrlrg enumerate llm_titlellm_descriptionembed_table_metadatatolisttitle_embeddingdescription_embeddingcommitr4 exceptionrJ)rrrrrrrrEembedding_servicesessionrtotalupdateditable embeddingr;s rrrqsGF"I MMO JJszz+( 6718FS 46 ]g]]>2668FKEz 89 ] JJw&89 :G%fa05??u'<'< 1 F F)>)>!I -6,<,<,>E)2;2B2B2DE/qLG:~*( \!AeWJ#GH1" NN  JJ6wiqwO P=]]@  8<$G  sX ;G)AG G)9G GAG:AGG)G&"G)&G)) I2AH;;Icty)zMain entry point for the CLI.N)rr rrmainrsEr__main__)'__doc__rKrBpathlibrtypingrrHlogururagents.factoryrr processorsr utilsr groupversion_optionrcommandargumentoptionChoicer|intr2boolrNrkrrrrrrrr__name__)rs0rrs  6/ g& '  4!89 D(1U  6 1qww 6 7 A  i$de  !#