Coverage for src / quber / settings / __init__.py: 99%
102 statements
« prev ^ index » next coverage.py v7.14.0, created at 2026-09-23 22:14 -0400
1"""Centralized configuration: one Pydantic ``Settings`` tree (QUE-228).
3Replaces the fragmented config layers (the dead ``utils.config.Config`` plus
4~dozen scattered ``os.environ`` reads) with a single ``BaseSettings`` object
5composed of nested domain groups. Modules read config via ``get_settings()``
6(a cached singleton) instead of touching the environment directly.
8Design notes:
10- Each group is a ``BaseSettings`` subclass (not a plain ``BaseModel``) so it
11 loads the *flat* legacy env names — ``ANTHROPIC_API_KEY``, ``POSTGRES_*``,
12 etc. — natively via per-field ``validation_alias``. A plain nested
13 ``BaseModel`` would only populate through an ``env_nested_delimiter`` prefix
14 (``LLM__ANTHROPIC_API_KEY``), which would break every existing ``.env``.
15- Renamed vars keep their old name as an ``AliasChoices`` fallback so a
16 transition window exists (e.g. ``LOGFIRE_TOKEN`` -> ``PYDANTIC_LOGFIRE_TOKEN``).
17- ``Settings()`` construction performs no network or credential calls, so it is
18 safe to call at package import (``files.cache.init_s3_cache`` does, via
19 ``quber/__init__.py``).
20- AWS credentials are intentionally absent: they stay on the boto3 default
21 credential chain (env / ``~/.aws`` / IAM role). Only non-secret AWS knobs
22 (S3 cache dir, TTL, region) live here.
23"""
25from __future__ import annotations
27from functools import lru_cache
28from pathlib import Path
29from typing import Annotated, Any, Literal, Optional
31from pydantic import AliasChoices, BeforeValidator, Field
32from pydantic_settings import BaseSettings, SettingsConfigDict
34# Shared config: read `.env` (relative to CWD), ignore unknown env vars,
35# case-insensitive env matching. Each group reuses this so they all load the
36# same `.env` and flat environment.
37BASE_CONFIG = SettingsConfigDict(env_file=".env", extra="ignore", case_sensitive=False)
39# Truthy token set matching the legacy `langsmith_tracer.truthy()` helper this
40# package replaces. Pydantic's native bool parser would raise on an empty or
41# unrecognized value (e.g. `TRACE_TO_LANGSMITH=`), whereas the old code treated
42# anything outside this set as False and never errored. `LegacyBool` preserves
43# that exact truth table so no `.env` that worked before silently breaks.
44TRUTHY_TOKENS = {"1", "true", "yes", "on"}
47def coerce_legacy_bool(value: Any) -> Any:
48 if isinstance(value, str):
49 return value.strip().lower() in TRUTHY_TOKENS
50 if value is None:
51 return False
52 return value
55LegacyBool = Annotated[bool, BeforeValidator(coerce_legacy_bool)]
57# The model the extraction agents fall back to when neither an explicit model
58# nor QUBER_LLM_MODEL/ANTHROPIC_MODEL is set. One project-wide default so the
59# pinned model lives in a single place rather than being repeated in every
60# agent. A dated id (not a rolling alias) keeps extraction reproducible run to
61# run, the same reason the structural agents pin temperature 0.
62DEFAULT_LLM_MODEL = "claude-haiku-4-5-20251001"
65class LLMSettings(BaseSettings):
66 """LLM model selection, Anthropic/OpenAI credentials, per-agent backends."""
68 model_config = BASE_CONFIG
70 model: Optional[str] = Field(
71 default=None,
72 validation_alias=AliasChoices("QUBER_LLM_MODEL", "ANTHROPIC_MODEL"),
73 )
74 anthropic_api_key: Optional[str] = Field(default=None, validation_alias="ANTHROPIC_API_KEY")
75 anthropic_auth_token: Optional[str] = Field(default=None, validation_alias="ANTHROPIC_AUTH_TOKEN")
76 # Shared with the embeddings layer; defined here once (single source of truth).
77 openai_api_key: Optional[str] = Field(default=None, validation_alias="OPENAI_API_KEY")
79 # Per-agent backend selectors. Kept as separate fields (not collapsed to one)
80 # so each agent retains its independent override, matching today's behavior.
81 llm_backend: str = Field(default="api", validation_alias="QUBER_LLM_BACKEND")
82 unifier_backend: str = Field(default="api", validation_alias="QUBER_UNIFIER_BACKEND")
83 classifier_backend: str = Field(default="api", validation_alias="QUBER_CLASSIFIER_BACKEND")
84 detector_backend: str = Field(default="api", validation_alias="QUBER_DETECTOR_BACKEND")
85 grid_locator_backend: str = Field(default="api", validation_alias="QUBER_GRID_LOCATOR_BACKEND")
86 cell_reader_backend: str = Field(default="api", validation_alias="QUBER_CELL_READER_BACKEND")
87 completeness_backend: str = Field(default="text", validation_alias="QUBER_COMPLETENESS_BACKEND")
88 # The capture advisor recommends a Camelot retry adjustment when a
89 # capture drop is detected (values in the region text layer missing from
90 # the grid). `off` disables the repair loop entirely.
91 capture_advisor_backend: str = Field(default="api", validation_alias="QUBER_CAPTURE_ADVISOR_BACKEND")
92 # The status inspector verifies each unboxed cell's proposed condition
93 # against the table image and downgrades whatever it cannot positively
94 # confirm to `unverified`. `off` disables inspection; the proposed
95 # statuses are then recorded as-is.
96 status_inspector_backend: str = Field(default="api", validation_alias="QUBER_STATUS_INSPECTOR_BACKEND")
97 # The figure corrector names the text left on a page that belongs to a figure
98 # the scan has already read — axis ticks, gridline labels — so it is removed
99 # rather than indexed beside the reading that supersedes it. `off` removes
100 # nothing and keeps every fragment.
101 figure_correction_backend: str = Field(default="api", validation_alias="QUBER_FIGURE_CORRECTION_BACKEND")
102 # The figure-value readers give every value printed in a figure a second,
103 # independent read against the page image and the scan's prose, so the two
104 # can be tied cell by cell. `off` skips the figure-value reconciliation
105 # pass entirely.
106 figure_values_backend: str = Field(default="api", validation_alias="QUBER_FIGURE_VALUES_BACKEND")
108 # The grid locator reads a labelled grid off the page image — a vision task
109 # the smaller model resolves at too coarse a fidelity (it under-flags the
110 # row-label stub column and wobbles a row at stacked boundaries), so it runs
111 # on a stronger model than the rest of the extraction agents. Overridable;
112 # falls back to the project-wide model when unset.
113 grid_locator_model: Optional[str] = Field(
114 default="claude-sonnet-5", validation_alias="QUBER_GRID_LOCATOR_MODEL"
115 )
117 # The structure-correction (vet) agent's own model. That agent carries the
118 # most compliance-sensitive output in the pipeline — footnote marker
119 # catalogues, quoted carrying cells, footnote/section judgements — and the
120 # smaller model intermittently omits whole marks lists on real filings.
121 # Overridable per the grid-locator pattern; falls back to the project-wide
122 # model when unset. Validated on a real filing: the smaller model emitted
123 # no marks list at all on some tables; the stronger model emitted them
124 # complete with identical results whether scoped here or run pipeline-wide.
125 vet_model: Optional[str] = Field(default="claude-sonnet-5", validation_alias="QUBER_VET_MODEL")
127 # The figure corrector's own model, pinned for the same reason as the two
128 # above. Asked which text on a page belongs to a figure the scan has read,
129 # the smaller model deleted a four-line annotation about rate sensitivity
130 # while its own stated reason called it an annotation, and missed seven of a
131 # page's forty axis tick labels. The stronger model returned all forty ticks
132 # and left the annotation alone, identically across three runs.
133 figure_correction_model: Optional[str] = Field(
134 default="claude-sonnet-5", validation_alias="QUBER_FIGURE_CORRECTION_MODEL"
135 )
137 # The local figure-value reader's own model. Reading every number printed
138 # inside a chart off the page image is a vision task, pinned to the
139 # stronger model for the same reason as the figure corrector. Overridable;
140 # falls back to the project-wide model when unset.
141 figure_values_model: Optional[str] = Field(
142 default="claude-sonnet-5", validation_alias="QUBER_FIGURE_VALUES_MODEL"
143 )
146class DBSettings(BaseSettings):
147 """Postgres connection parameters for the RAG store."""
149 model_config = BASE_CONFIG
151 user: str = Field(default="quber", validation_alias="POSTGRES_USER")
152 password: str = Field(default="quber_dev", validation_alias="POSTGRES_PASSWORD")
153 host: str = Field(default="localhost", validation_alias="POSTGRES_HOST")
154 port: int = Field(default=5432, validation_alias="POSTGRES_PORT")
155 db: str = Field(default="quber_rag", validation_alias="POSTGRES_DB")
158class S3CacheSettings(BaseSettings):
159 """Non-secret S3 ingestion-cache knobs. Credentials stay on the boto3 chain."""
161 model_config = BASE_CONFIG
163 cache_dir: Path = Field(default=Path(".cache/s3"), validation_alias="QUBER_S3_CACHE_DIR")
164 ttl_seconds: float = Field(default=5 * 86400, validation_alias="QUBER_S3_CACHE_TTL")
165 region: Optional[str] = Field(
166 default=None,
167 validation_alias=AliasChoices("QUBER_S3_REGION", "AWS_REGION"),
168 )
171class EmbeddingSettings(BaseSettings):
172 """Embedding provider/device selection. OpenAI key lives in ``LLMSettings``."""
174 model_config = BASE_CONFIG
176 provider: str = Field(default="local", validation_alias="EMBEDDING_PROVIDER")
177 device: str = Field(default="cuda", validation_alias="EMBEDDING_DEVICE")
180class LandingSettings(BaseSettings):
181 """Landing.AI ADE parse credentials for the standalone extraction path."""
183 model_config = BASE_CONFIG
185 # ADE_API_KEY is our .env name; VISION_AGENT_API_KEY is the name the
186 # landingai-ade SDK itself reads, accepted so an SDK-style environment
187 # also works.
188 api_key: Optional[str] = Field(
189 default=None,
190 validation_alias=AliasChoices("ADE_API_KEY", "VISION_AGENT_API_KEY"),
191 )
194class TypeSafeSettings(BaseSettings):
195 """TypeSafe scoring API: credential, pinned model, the chunk grade cut, and
196 how hard the playground's ranker drives it."""
198 model_config = BASE_CONFIG
200 # TYPESAFE_API_KEY is the SDK's own variable name. The key is still passed
201 # explicitly from here into the SDK client, so a missing key is reported by
202 # settings the way a missing Anthropic key is, not by the SDK at the first
203 # request.
204 api_key: Optional[str] = Field(default=None, validation_alias="TYPESAFE_API_KEY")
205 model: str = Field(default="jev-1.13.0", validation_alias="QUBER_TYPESAFE_MODEL")
206 # Chunks whose position on the 0 to 3 rubric is below this do not reach the
207 # answer model. 1.5 is the midpoint between "same topic" and "supports";
208 # the default sits just under it so a chunk Jev is nearly sure supports the
209 # item is kept.
210 grade_cut: float = Field(default=1.4, validation_alias="QUBER_TYPESAFE_GRADE_CUT")
211 # Calls in flight at once. 32 is the highest value measured, with no
212 # throttling from the API.
213 concurrency: int = Field(default=32, validation_alias="QUBER_TYPESAFE_CONCURRENCY")
214 # The SDK's own default, made explicit: retries after the first attempt on
215 # 429, 5xx, connection and timeout errors.
216 max_retries: int = Field(default=2, validation_alias="QUBER_TYPESAFE_MAX_RETRIES")
217 timeout_seconds: float = Field(default=60.0, validation_alias="QUBER_TYPESAFE_TIMEOUT_SECONDS")
220class PlaygroundSettings(BaseSettings):
221 """The answering playground: where its local corpus lives, how many
222 ingests may run at once, and which step ranks retrieved chunks."""
224 model_config = BASE_CONFIG
226 # Staged PDFs, parse artifacts and uploads. Relative paths resolve against
227 # the server's working directory, the same convention as the S3 cache dir.
228 data_dir: Path = Field(default=Path("data/playground"), validation_alias="QUBER_PLAYGROUND_DATA_DIR")
229 # How many uploads run their pipelines at once. Every stage is a
230 # subprocess, so two concurrent extractions are two `quber fuse` processes
231 # sharing one CUDA device, each also making model calls. Two overlaps the
232 # non-GPU parts of a run without saturating the device; one is for a
233 # machine that cannot spare the contention.
234 upload_concurrency: int = Field(default=2, validation_alias="QUBER_PLAYGROUND_UPLOAD_CONCURRENCY")
235 # The durable home of every document's files when the playground is
236 # hosted: an s3:// prefix holding one <doc_key>/ prefix per document, to
237 # which each stage's outputs are copied and from which a served PDF is
238 # fetched when the local copy is gone. Unset, the local data_dir is the
239 # only copy, which is the developer host's mode.
240 artifacts_uri: Optional[str] = Field(default=None, validation_alias="QUBER_PLAYGROUND_ARTIFACTS_URI")
241 # The RunPod serverless endpoint the hosted playground sends its GPU
242 # stages to, the docling parse and the chunk embedding, with the key that
243 # authorizes the submission. Unset, both run on this host.
244 runpod_endpoint_id: Optional[str] = Field(default=None, validation_alias="RUNPOD_ENDPOINT_ID")
245 runpod_api_key: Optional[str] = Field(default=None, validation_alias="RUNPOD_API_KEY")
246 # Where the hosted playground reports signed-in activity: a CloudWatch
247 # namespace for one metric, the count of requests that carried a valid
248 # session. The idle alarm watches it, because the public hostname's
249 # load-balancer request count never reaches zero. Unset, nothing is sent.
250 activity_namespace: Optional[str] = Field(
251 default=None, validation_alias="QUBER_PLAYGROUND_ACTIVITY_NAMESPACE"
252 )
253 # The sign-in the hosted playground asks for: the WorkOS environment's API
254 # key and client ID, the one organization whose members may enter, and the
255 # secret that signs the session cookie. The four are set together; all
256 # unset disables the sign-in, which is the developer host's mode, and a
257 # partial set is refused at startup rather than silently running open.
258 workos_api_key: Optional[str] = Field(default=None, validation_alias="WORKOS_API_KEY")
259 workos_client_id: Optional[str] = Field(default=None, validation_alias="WORKOS_CLIENT_ID")
260 workos_organization_id: Optional[str] = Field(default=None, validation_alias="WORKOS_ORGANIZATION_ID")
261 session_secret: Optional[str] = Field(default=None, validation_alias="PLAYGROUND_SESSION_SECRET")
262 # Which step orders retrieved chunks before the answer model reads them:
263 # the Haiku selection agent over a window of the fused ranking, or Jev
264 # over the whole of it. A backend setting only; it never reaches a client
265 # payload or the UI.
266 ranker: Literal["haiku", "jev"] = Field(default="haiku", validation_alias="QUBER_PLAYGROUND_RANKER")
267 # Questions a batch run answers at once. Each one holds an answer-model
268 # call, and the provider returns 529 under load (observed 2026-07-29
269 # during a sweep), so the default is deliberate.
270 question_concurrency: int = Field(default=5, validation_alias="QUBER_PLAYGROUND_QUESTION_CONCURRENCY")
273class ObservabilitySettings(BaseSettings):
274 """Logfire + LangSmith tracing configuration."""
276 model_config = BASE_CONFIG
278 logfire_token: Optional[str] = Field(
279 default=None,
280 validation_alias=AliasChoices("PYDANTIC_LOGFIRE_TOKEN", "LOGFIRE_TOKEN"),
281 )
282 enable_logfire: LegacyBool = Field(default=True, validation_alias="QUBER_ENABLE_LOGFIRE")
283 trace_to_langsmith: LegacyBool = Field(default=False, validation_alias="TRACE_TO_LANGSMITH")
284 langsmith_api_key: Optional[str] = Field(default=None, validation_alias="CC_LANGSMITH_API_KEY")
285 langsmith_project: str = Field(default="default", validation_alias="CC_LANGSMITH_PROJECT")
286 langsmith_debug: LegacyBool = Field(default=False, validation_alias="CC_LANGSMITH_DEBUG")
289class Settings(BaseSettings):
290 """Root settings object. Access via :func:`get_settings`."""
292 model_config = BASE_CONFIG
294 log_level: str = Field(default="INFO", validation_alias="QUBER_LOG_LEVEL")
295 max_concurrent_tables: int = Field(default=5, validation_alias="QUBER_MAX_CONCURRENT_TABLES")
297 llm: LLMSettings = Field(default_factory=LLMSettings)
298 db: DBSettings = Field(default_factory=DBSettings)
299 s3_cache: S3CacheSettings = Field(default_factory=S3CacheSettings)
300 embeddings: EmbeddingSettings = Field(default_factory=EmbeddingSettings)
301 landing: LandingSettings = Field(default_factory=LandingSettings)
302 typesafe: TypeSafeSettings = Field(default_factory=TypeSafeSettings)
303 playground: PlaygroundSettings = Field(default_factory=PlaygroundSettings)
304 obs: ObservabilitySettings = Field(default_factory=ObservabilitySettings)
307@lru_cache
308def get_settings() -> Settings:
309 """Return the process-wide cached :class:`Settings` singleton.
311 Construction reads ``.env`` and the environment only — no network or
312 credential resolution — so this is safe to call at import time.
313 """
314 return Settings()
317__all__ = [
318 "DEFAULT_LLM_MODEL",
319 "DBSettings",
320 "EmbeddingSettings",
321 "LLMSettings",
322 "LandingSettings",
323 "ObservabilitySettings",
324 "PlaygroundSettings",
325 "S3CacheSettings",
326 "Settings",
327 "TypeSafeSettings",
328 "get_settings",
329]