"""General helper utilities to parse the pdf tables."""
from __future__ import annotations
import atexit
import math
import os
import random
import re
import shutil
import string
import tempfile
import warnings
from bisect import bisect_left
from collections.abc import Callable
from itertools import groupby
from operator import itemgetter
from pathlib import Path
from typing import Any
from urllib.parse import urlparse as parse_url
from urllib.parse import uses_netloc
from urllib.parse import uses_params
from urllib.parse import uses_relative
from urllib.request import Request
from urllib.request import urlopen
import numpy as np
import playa.miner as pm
from playa.miner import LAParams
from playa.miner import LTAnno
from playa.miner import LTChar
from playa.miner import LTContainer
from playa.miner import LTImage
from playa.miner import LTItem
from playa.miner import LTTextLine
from playa.miner import LTTextLineHorizontal
from playa.miner import LTTextLineVertical
_VALID_URLS = set(uses_relative + uses_netloc + uses_params)
_VALID_URLS.discard("")
# https://github.com/pandas-dev/pandas/blob/master/pandas/io/common.py
def is_url(url):
"""Check to see if a URL has a valid protocol.
Parameters
----------
url : str or unicode
Returns
-------
isurl : bool
If url has a valid protocol return True otherwise False.
"""
try:
return parse_url(url).scheme in _VALID_URLS
except Exception:
return False
_RANDOM_STRING_ALPHABET = (
string.digits + string.ascii_lowercase + string.ascii_uppercase
)
def random_string(length):
"""Generate a random string.
Parameters
----------
length : int
The length of the string to return.
Returns
-------
string
returns a random string
"""
# Single random.choices + str.join is several times faster than the old
# per-char loop with string concatenation and rebuilds the alphabet
# once at import time instead of on every call.
return "".join(random.choices(_RANDOM_STRING_ALPHABET, k=length)) # noqa S311
def download_url(url: str) -> str | Path:
"""Download file from specified URL.
Parameters
----------
url : str
Returns
-------
filepath : Union[str, Path]
Temporary filepath.
"""
filename = f"{random_string(6)}.pdf"
with tempfile.NamedTemporaryFile("wb", delete=False) as f: # noqa S310
# Valid url checking has been done in function is_url
headers = {
"User-Agent": "Mozilla/5.0",
"Accept-Encoding": "gzip;q=1.0, deflate;q=0.9, br;q=0.8, compress;q=0.7, *;q=0.1",
}
request = Request(url, None, headers) # noqa: S310 - documented URL feature
obj = urlopen(request) # noqa: S310 - user-supplied PDF URL; documented feature
content_type = obj.info().get_content_type()
if content_type != "application/pdf":
raise NotImplementedError("File format not supported")
f.write(obj.read())
filepath = os.path.join(os.path.dirname(f.name), filename)
shutil.move(f.name, filepath)
return filepath
common_kwargs = [
"flag_size",
"margins",
"replace_text",
"split_text",
"strip_text",
"table_areas",
"table_regions",
"backend",
]
text_kwargs = common_kwargs + ["columns", "edge_tol", "row_tol", "column_tol"]
lattice_kwargs = common_kwargs + [
"process_background",
"line_scale",
"copy_text",
"shift_text",
"line_tol",
"joint_tol",
"threshold_blocksize",
"threshold_constant",
"iterations",
"erode_iterations",
"resolution",
"use_fallback",
"engine",
]
# The ML parser (flavor='ml', optional [ml] extra) gets its structure from a
# neural model, so it shares the common text-handling kwargs (split/strip/flag/
# regions/areas) + the spanning-text controls, plus a few model-specific knobs.
ml_kwargs = common_kwargs + [
"copy_text",
"shift_text",
"resolution",
"use_fallback",
"device",
"structure_model",
"detection_model",
"detection_threshold",
"structure_threshold",
"crop_padding",
"ocr",
]
flavor_to_kwargs = {
"stream": text_kwargs,
"network": text_kwargs,
"lattice": lattice_kwargs,
"hybrid": text_kwargs + lattice_kwargs,
"ml": ml_kwargs,
}
def validate_input(kwargs, flavor="lattice"):
"""Validates input keyword arguments.
Parameters
----------
kwargs : [type]
[description]
flavor : str, optional
[description], by default "lattice"
Raises
------
ValueError
[description]
"""
parser_kwargs = flavor_to_kwargs[flavor]
# s.difference(t): new set with elements in s but not in t
isec = set(kwargs.keys()).difference(set(parser_kwargs))
if isec:
raise ValueError(
"{} cannot be used with flavor='{}'".format(",".join(sorted(isec)), flavor)
)
def remove_extra(kwargs, flavor="lattice"):
"""Remove extra key - value pairs from a kwargs dictionary.
Parameters
----------
kwargs : [type]
[description]
flavor : str, optional
[description], by default "lattice"
Returns
-------
[type]
[description]
"""
parser_kwargs = flavor_to_kwargs[flavor]
# Avoid "dictionary changed size during iteration"
kwargs_keys = list(kwargs.keys())
for key in kwargs_keys:
if key not in parser_kwargs:
kwargs.pop(key)
return kwargs
# https://stackoverflow.com/a/22726782
# and https://stackoverflow.com/questions/10965479
class TemporaryDirectory:
"""A class method that will be used to create temporary directories."""
def __enter__(self):
"""Enter the temporary directory .
Returns
-------
[type]
[description]
"""
self.name = tempfile.mkdtemp()
# Only delete the temporary directory upon
# program exit.
atexit.register(shutil.rmtree, self.name)
return self.name
def __exit__(self, exc_type, exc_value, traceback):
"""Called when the client exits.
Parameters
----------
exc_type : [type]
[description]
exc_value : [type]
[description]
traceback : [type]
[description]
"""
def build_file_path_in_temp_dir(filename, extension=None):
"""Generate a new path within a temporary directory.
Parameters
----------
filename : str
extension : str
Returns
-------
file_path_in_temporary_dir : str
"""
with TemporaryDirectory() as temp_dir:
if extension:
filename = filename + extension
path = os.path.join(temp_dir, filename)
return path
def translate(x1: float, x2: float) -> float:
"""Translate x2 by x1.
Parameters
----------
x1 : float
The offset to apply.
x2 : float
The original y-coordinate.
Returns
-------
float
The translated y-coordinate.
"""
return x2 + x1
def scale(value: float, factor: float) -> float:
"""Scale a given value by a factor.
Parameters
----------
value : float
The value to scale.
factor : float
The scaling factor.
Returns
-------
float
The scaled value.
"""
return value * factor
def scale_pdf(k, factors):
"""Translate and scale pdf coordinate space to image coordinate space.
Parameters
----------
k : tuple
Tuple (x1, y1, x2, y2) representing table bounding box where
(x1, y1) -> lt and (x2, y2) -> rb in PDFMiner coordinate
space.
factors : tuple
Tuple (scaling_factor_x, scaling_factor_y, pdf_y) where the
first two elements are scaling factors and pdf_y is height of
pdf.
Returns
-------
knew : tuple
Tuple (x1, y1, x2, y2) representing table bounding box where
(x1, y1) -> lt and (x2, y2) -> rb in OpenCV coordinate
space.
"""
x1, y1, x2, y2 = k
scaling_factor_x, scaling_factor_y, pdf_y = factors
x1 = scale(x1, scaling_factor_x)
y1 = scale(abs(translate(-pdf_y, y1)), scaling_factor_y)
x2 = scale(x2, scaling_factor_x)
y2 = scale(abs(translate(-pdf_y, y2)), scaling_factor_y)
knew = (int(x1), int(y1), int(x2), int(y2))
return knew
def scale_image(
tables: dict[tuple[float, float, float, float], list[tuple[float, float]]],
v_segments: list[tuple[float, float, float, float]],
h_segments: list[tuple[float, float, float, float]],
factors: tuple[float, float, float],
) -> tuple[
dict[tuple[float, float, float, float], dict[str, list[tuple[float, float]]]],
list[tuple[float, float, float, float]],
list[tuple[float, float, float, float]],
]:
"""Translate and scale image coordinate space to PDF coordinate space.
Parameters
----------
tables : dict
A dictionary with table boundaries as keys (tuples of four floats)
and a list of intersections (list of tuples of two floats) as values.
v_segments : list
A list of vertical line segments, where each segment is a tuple
of four floats (x1, y1, x2, y2).
h_segments : list
A list of horizontal line segments, where each segment is a tuple
of four floats (x1, y1, x2, y2).
factors : tuple
A tuple (scaling_factor_x, scaling_factor_y, img_y) where the
first two elements are scaling factors and img_y is the height of
the image.
Returns
-------
Tuple[Dict[Tuple[float, float, float, float], Dict[str, List[Tuple[float, float]]]],
List[Tuple[float, float, float, float]],
List[Tuple[float, float, float, float]]]
A tuple containing:
- tables_new: A new dictionary with scaled table boundaries and joints.
- v_segments_new: A new list of scaled vertical segments.
- h_segments_new: A new list of scaled horizontal segments.
"""
scaling_factor_x, scaling_factor_y, img_y = factors
tables_new = {}
for k in tables.keys():
x1, y1, x2, y2 = k
x1 = scale(x1, scaling_factor_x)
y1 = scale(abs(translate(-img_y, y1)), scaling_factor_y)
x2 = scale(x2, scaling_factor_x)
y2 = scale(abs(translate(-img_y, y2)), scaling_factor_y)
# j_x and j_y are tuples of floats
j_x, j_y = zip(*tables[k]) # noqa B905
j_x_scaled = [scale(j, scaling_factor_x) for j in j_x]
j_y_scaled = [scale(abs(translate(-img_y, j)), scaling_factor_y) for j in j_y]
tables_new[(x1, y1, x2, y2)] = {
"joints": list(zip(j_x_scaled, j_y_scaled)) # noqa B905
}
# Scale vertical segments
v_segments_new = []
for v in v_segments:
x1, x2 = scale(v[0], scaling_factor_x), scale(v[2], scaling_factor_x)
y1, y2 = (
scale(abs(translate(-img_y, v[1])), scaling_factor_y),
scale(abs(translate(-img_y, v[3])), scaling_factor_y),
)
v_segments_new.append((x1, y1, x2, y2))
# Scale horizontal segments
h_segments_new = []
for h in h_segments:
x1, x2 = scale(h[0], scaling_factor_x), scale(h[2], scaling_factor_x)
y1, y2 = (
scale(abs(translate(-img_y, h[1])), scaling_factor_y),
scale(abs(translate(-img_y, h[3])), scaling_factor_y),
)
h_segments_new.append((x1, y1, x2, y2))
return tables_new, v_segments_new, h_segments_new
def get_rotation(chars, horizontal_text, vertical_text):
"""Get text rotation.
Detects if text in table is rotated or not using the current
transformation matrix (CTM) and returns its orientation.
Parameters
----------
horizontal_text : list
List of PDFMiner LTTextLineHorizontal objects.
vertical_text : list
List of PDFMiner LTTextLineVertical objects.
ltchar : list
List of PDFMiner LTChar objects.
Returns
-------
rotation : string
'' if text in table is upright, 'anticlockwise' if
rotated 90 degree anticlockwise and 'clockwise' if
rotated 90 degree clockwise.
"""
rotation = ""
hlen = len([t for t in horizontal_text if t.get_text().strip()])
vlen = len([t for t in vertical_text if t.get_text().strip()])
if hlen < vlen:
clockwise = sum(t.matrix[1] < 0 and t.matrix[2] > 0 for t in chars)
anticlockwise = sum(t.matrix[1] > 0 and t.matrix[2] < 0 for t in chars)
rotation = "anticlockwise" if clockwise < anticlockwise else "clockwise"
return rotation
def segments_in_bbox(bbox, v_segments, h_segments):
"""Return all line segments present inside a bounding box.
Parameters
----------
bbox : tuple
Tuple (x1, y1, x2, y2) representing a bounding box where
(x1, y1) -> lb and (x2, y2) -> rt in PDFMiner coordinate
space.
v_segments : list
List of vertical line segments.
h_segments : list
List of vertical horizontal segments.
Returns
-------
v_s : list
List of vertical line segments that lie inside table.
h_s : list
List of horizontal line segments that lie inside table.
"""
lb = (bbox[0], bbox[1])
rt = (bbox[2], bbox[3])
v_s = [
v
for v in v_segments
if v[1] > lb[1] - 2 and v[3] < rt[1] + 2 and lb[0] - 2 <= v[0] <= rt[0] + 2
]
h_s = [
h
for h in h_segments
if h[0] > lb[0] - 2 and h[2] < rt[0] + 2 and lb[1] - 2 <= h[1] <= rt[1] + 2
]
return v_s, h_s
def get_textline_coords(textline):
"""Calculate the coordinates of each alignment for a given textline."""
return {
"left": textline.x0,
"right": textline.x1,
"middle": (textline.x0 + textline.x1) / 2.0,
"bottom": textline.y0,
"top": textline.y1,
"center": (textline.y0 + textline.y1) / 2.0,
}
def bbox_from_str(bbox_str):
"""Deserialize bbox from string ("x1,y1,x2,y2") to tuple (x1, y1, x2, y2).
Parameters
----------
bbox_str : str
Serialized bbox with comma separated coordinates, "x1,y1,x2,y2".
Returns
-------
bbox : tuple
Tuple (x1, y1, x2, y2).
Raises
------
ValueError
If the string isn't four numbers, or describes a zero-area box.
The latter is the usual cause of the otherwise cryptic
``ZeroDivisionError`` downstream (#63), commonly because the
coordinates came from a top-left-origin tool — PDF space has its
origin at the bottom-left, so ``y1`` must be greater than ``y2``.
"""
parts = bbox_str.split(",")
if len(parts) != 4:
raise ValueError(
f"Invalid table area/region {bbox_str!r}: expected four "
"comma-separated numbers in the form 'x1,y1,x2,y2'."
)
try:
x1, y1, x2, y2 = (float(p) for p in parts)
except ValueError:
raise ValueError(
f"Invalid table area/region {bbox_str!r}: coordinates must be "
"numbers, in the form 'x1,y1,x2,y2'."
) from None
if x1 == x2 or y1 == y2:
raise ValueError(
f"Invalid table area/region {bbox_str!r}: it has zero width or "
"height. Use 'x1,y1,x2,y2' where (x1, y1) is the top-left and "
"(x2, y2) the bottom-right corner in PDF coordinate space — the "
"origin is the bottom-left of the page, so y1 must be greater "
"than y2."
)
return (min(x1, x2), min(y1, y2), max(x1, x2), max(y1, y2))
def bboxes_overlap(bbox1, bbox2):
"""Check if boundingboxes overlap.
Parameters
----------
bbox1 : tuple
Tuple (x1, y1, x2, y2) representing a bounding box where
(x1, y1) -> lb and (x2, y2) -> rt in the PDF coordinate
space.
bbox2 : tuple
Tuple (x1, y1, x2, y2) representing a bounding box where
(x1, y1) -> lb and (x2, y2) -> rt in the PDF coordinate
space.
Returns
-------
bool
Returns True if two bounding boxes overlap
"""
left1, bottom1, right1, top1 = bbox1
left2, bottom2, right2, top2 = bbox2
return ((left1 < left2 < right1) or (left1 < right2 < right1)) and (
(bottom1 < bottom2 < top1) or (bottom1 < top2 < top1)
)
def textlines_overlapping_bbox(bbox, textlines):
"""Return all text objects which overlap or are within a bounding box.
Parameters
----------
bbox : tuple
Tuple (x1, y1, x2, y2) representing a bounding box where
(x1, y1) -> lb and (x2, y2) -> rt in the PDF coordinate
space.
textlines : List of PDFMiner text objects.
Returns
-------
t_bbox : list
List of PDFMiner text objects.
"""
t_bbox = [t for t in textlines if bboxes_overlap(bbox, (t.x0, t.y0, t.x1, t.y1))]
return t_bbox
# Threshold above which the NumPy NxN intersection matrix in text_in_bbox
# would use too much memory. n=600 needs ~3MB (float64); n=2000 ~32MB.
# For larger inputs we fall back to the Python loop.
_TEXT_IN_BBOX_NUMPY_MAX = 1500
def _normalised_textline_text(t) -> str:
"""Whitespace-stripped text of a PDFMiner-style textline (#288, #625).
Returns ``""`` if the textline has no get_text or no content. Used by
the text_in_bbox dedup pass to confirm overlapping bboxes are *actual*
duplicates before discarding the shorter one.
"""
getter = getattr(t, "get_text", None)
if getter is None:
return ""
s = getter()
return s.strip() if s else ""
def _is_textual_duplicate(a_text: str, b_text: str) -> bool:
"""Return True if textline A's content can be considered a duplicate of B's.
The bbox-overlap dedup pass introduced in #206 (to fix #15) was meant
to drop *literal* duplicate textlines emitted by PDF font-render
quirks — typically two textlines with identical content placed at
nearly the same coordinates. Skipping the content check made it also
discard legitimate adjacent-cell text whenever a wide neighbour's
bbox happened to cover it (#288, #625). Restricting the discard to
*equal* stripped text matches #206's original intent without false
positives (a single-char "B" textline is not a duplicate of a wider
sibling that merely contains the letter B).
"""
if not a_text:
return True
return a_text == b_text
def _text_in_bbox_loop(t_bbox):
"""Reference Python implementation of the >80%-contained discard pass.
Used as a fallback when ``len(t_bbox)`` is large enough that the
NumPy NxN intersection matrix would exceed comfortable memory.
"""
n = len(t_bbox)
discarded: set[int] = set()
texts = [_normalised_textline_text(t) for t in t_bbox]
for i in range(n):
if i in discarded:
continue
ba = t_bbox[i]
ba_area = bbox_area(ba)
ai_text = texts[i]
for j in range(n):
if i == j or j in discarded:
continue
bb = t_bbox[j]
if not bbox_intersect(ba, bb):
continue
if ba_area == 0 or (bbox_intersection_area(ba, bb) / ba_area) > 0.8:
if bbox_longer(bb, ba) and _is_textual_duplicate(ai_text, texts[j]):
discarded.add(i)
break
return [t_bbox[i] for i in range(n) if i not in discarded]
def text_in_bbox(bbox, text):
"""Return all text objects in a bounding box.
Keeps every text object whose centre lies inside the bbox (with a
small pad), then discards each one whose bbox is >80% contained in a
longer sibling *and* whose stripped text equals that sibling's
stripped text. The content equality check (#288, #625) prevents
adjacent-cell text from being dropped when a wide neighbour's bbox
happens to cover it — the geometry-only rule introduced in #206 was
meant to deduplicate PDF font-render duplicates (#15), not to
discard legitimate non-duplicate content.
Parameters
----------
bbox : tuple
Tuple (x1, y1, x2, y2) representing a bounding box where
(x1, y1) -> lb and (x2, y2) -> rt in the PDF coordinate
space.
text : List of PDFMiner text objects.
Returns
-------
t_bbox : list
List of PDFMiner text objects that lie inside table, with
duplicate-content overlapping siblings discarded.
"""
if not text:
return []
# Pull coordinates into parallel float arrays in one pass.
n_in = len(text)
x0 = np.empty(n_in, dtype=np.float64)
y0 = np.empty(n_in, dtype=np.float64)
x1 = np.empty(n_in, dtype=np.float64)
y1 = np.empty(n_in, dtype=np.float64)
for i, t in enumerate(text):
x0[i] = t.x0
y0[i] = t.y0
x1[i] = t.x1
y1[i] = t.y1
# Filter: centre inside bbox (+/- 2 pad), vectorised.
x_lo, y_lo, x_hi, y_hi = bbox[0] - 2, bbox[1] - 2, bbox[2] + 2, bbox[3] + 2
xm = (x0 + x1) * 0.5
ym = (y0 + y1) * 0.5
in_bbox = (xm >= x_lo) & (xm <= x_hi) & (ym >= y_lo) & (ym <= y_hi)
idx_in = np.flatnonzero(in_bbox)
n = idx_in.size
if n == 0:
return []
if n == 1:
return [text[int(idx_in[0])]]
t_bbox = [text[i] for i in idx_in.tolist()]
# For very large bbox populations the NxN pairwise matrix would blow the
# memory budget — fall back to the side-set Python loop.
if n > _TEXT_IN_BBOX_NUMPY_MAX:
return _text_in_bbox_loop(t_bbox)
# Vectorised >80%-contained-in-longer-sibling discard.
sx0 = x0[idx_in]
sy0 = y0[idx_in]
sx1 = x1[idx_in]
sy1 = y1[idx_in]
# Pairwise AABB intersection area (axis-aligned), broadcasting nxn.
x_left = np.maximum(sx0[:, None], sx0[None, :])
y_bot = np.maximum(sy0[:, None], sy0[None, :])
x_right = np.minimum(sx1[:, None], sx1[None, :])
y_top = np.minimum(sy1[:, None], sy1[None, :])
inter_w = np.clip(x_right - x_left, 0.0, None)
inter_h = np.clip(y_top - y_bot, 0.0, None)
inter_area = inter_w * inter_h
# Per-row "ba" area. inter_area / ba_area is the fraction of A
# contained in B. Treat degenerate (zero-area) boxes as fully
# contained, matching the original behaviour.
widths = sx1 - sx0
heights = sy1 - sy0
ba_areas = widths * heights
with np.errstate(divide="ignore", invalid="ignore"):
frac = np.where(ba_areas[:, None] > 0, inter_area / ba_areas[:, None], 1.0)
np.fill_diagonal(frac, 0.0)
# B is longer than A: widths[j] > widths[i] (rows = i, cols = j).
b_longer = widths[None, :] > widths[:, None]
# Bbox-level discard candidates: A is >80%-contained in a longer B.
discard_candidate = (frac > 0.8) & b_longer
# Content check (#288, #625): only discard A when the offending B
# actually contains A's text — otherwise we drop adjacent-cell
# content that just happened to be physically overlapped. The check
# runs per candidate row and bails on the first confirmed duplicate.
keep = np.ones(n, dtype=bool)
candidate_rows = np.flatnonzero(discard_candidate.any(axis=1)).tolist()
if candidate_rows:
texts = [_normalised_textline_text(t) for t in t_bbox]
for i in candidate_rows:
ai_text = texts[i]
for j in np.flatnonzero(discard_candidate[i]).tolist():
if _is_textual_duplicate(ai_text, texts[j]):
keep[i] = False
break
# keep is a boolean mask of length n derived from idx_in/t_bbox, so the
# two iterables are equal-length by construction.
return [t for t, k in zip(t_bbox, keep.tolist(), strict=True) if k]
def text_in_bbox_per_axis(bbox, horizontal_text, vertical_text):
"""Return all text objects present inside a bounding box.
split between horizontal and vertical text.
Parameters
----------
bbox : tuple
Tuple (x1, y1, x2, y2) representing a bounding box where
(x1, y1) -> lb and (x2, y2) -> rt in the PDF coordinate
space.
horizontal_text : List of PDFMiner text objects.
vertical_text : List of PDFMiner text objects.
Returns
-------
t_bbox : dict
Dict of lists of PDFMiner text objects that lie inside table, with one
key each for "horizontal" and "vertical"
"""
t_bbox = {}
t_bbox["horizontal"] = text_in_bbox(bbox, horizontal_text)
t_bbox["vertical"] = text_in_bbox(bbox, vertical_text)
t_bbox["horizontal"].sort(key=lambda x: (-x.y0, x.x0))
t_bbox["vertical"].sort(key=lambda x: (x.x0, -x.y0))
return t_bbox
def expand_bbox_with_textline(bbox, textline):
"""Expand (if needed) a bbox so that it fits the parameter textline."""
return (
min(bbox[0], textline.x0),
min(bbox[1], textline.y0),
max(bbox[2], textline.x1),
max(bbox[3], textline.y1),
)
def bbox_from_textlines(textlines):
"""Return the smallest bbox containing all the text objects passed as a parameters.
Parameters
----------
textlines : List of PDFMiner text objects.
Returns
-------
bbox : tuple
Tuple (x1, y1, x2, y2) representing a bounding box where
(x1, y1) -> lb and (x2, y2) -> rt in the PDF coordinate
space.
"""
if len(textlines) == 0:
return None
bbox = (textlines[0].x0, textlines[0].y0, textlines[0].x1, textlines[0].y1)
for tl in textlines[1:]:
bbox = expand_bbox_with_textline(bbox, tl)
return bbox
def bbox_intersection_area(ba, bb) -> float:
"""Return area of the intersection of the bounding boxes of two PDFMiner objects.
Parameters
----------
ba : PDFMiner text object
bb : PDFMiner text object
Returns
-------
intersection_area : float
Area of the intersection of the bounding boxes of both objects
"""
x_left = max(ba.x0, bb.x0)
y_top = min(ba.y1, bb.y1)
x_right = min(ba.x1, bb.x1)
y_bottom = max(ba.y0, bb.y0)
if x_right < x_left or y_bottom > y_top:
return 0.0
intersection_area = (x_right - x_left) * (y_top - y_bottom)
return intersection_area
def bbox_area(bb) -> float:
"""Return area of the bounding box of a PDFMiner object.
Parameters
----------
bb : PDFMiner text object
Returns
-------
area : float
Area of the bounding box of the object
"""
return (bb.x1 - bb.x0) * (bb.y1 - bb.y0)
def bbox_intersect(ba, bb) -> bool:
"""Return True if the bounding boxes of two PDFMiner objects intersect.
Parameters
----------
ba : PDFMiner text object
bb : PDFMiner text object
Returns
-------
overlaps : bool
True if the bounding boxes intersect
"""
return ba.x1 >= bb.x0 and bb.x1 >= ba.x0 and ba.y1 >= bb.y0 and bb.y1 >= ba.y0
def find_columns_boundaries(tls, min_gap=1.0):
"""Make a list of disjunct cols boundaries for a list of text objects.
Parameters
----------
tls : list of PDFMiner text object.
min_gap : minimum distance between columns. Any elements closer than
this threshold are merged together. This is to prevent spaces between
words to be misinterpreted as boundaries.
Returns
-------
boundaries : list
List x-coordinates for cols.
[(1st col left, 1st col right), (2nd col left, 2nd col right), ...]
"""
cols_bounds = []
tls.sort(key=lambda tl: tl.x0)
for tl in tls:
if (not cols_bounds) or cols_bounds[-1][1] + min_gap < tl.x0:
cols_bounds.append([tl.x0, tl.x1])
else:
cols_bounds[-1][1] = max(cols_bounds[-1][1], tl.x1)
return cols_bounds
def boundaries_to_split_lines(boundaries):
"""Find split lines given a list of boundaries between rows or cols.
Boundaries: [ a ] [b] [ c ] [d]
Splits: | | | | |
Parameters
----------
boundaries : list
List of tuples of x- (for columns) or y- (for rows) coord boundaries.
These are the (left, right most) or (bottom, top most) coordinates.
Returns
-------
anchors : list
List of coordinates representing the split points, each half way
between boundaries
"""
# Check if boundaries list is empty
if not boundaries:
return []
# Check if boundaries have at least one tuple
if len(boundaries) < 1:
return []
# From the row boundaries, identify splits by getting the mid points
# between the boundaries.
anchors = list(
map(
lambda idx: (boundaries[idx - 1][1] + boundaries[idx][0]) / 2.0,
range(1, len(boundaries)),
)
)
# Insert the first boundary's left coordinate
anchors.insert(0, boundaries[0][0])
# Append the last boundary's right coordinate
anchors.append(boundaries[-1][1])
return anchors
def get_index_closest_point(
point: Any, sorted_list: list[Any], fn: Callable[[Any], Any] = lambda x: x
) -> int | None:
"""Find the index of the closest point in sorted_list.
Parameters
----------
point : Any
The reference sortable element to search.
sorted_list : List[Any]
A sorted list of elements.
fn : Callable[[Any], Any], optional
Optional accessor function, by default lambda x: x
Returns
-------
Optional[int]
The index of the closest point, or None if the list is empty.
"""
n = len(sorted_list)
# If the list is empty, return None
if n == 0:
return None
# Edge cases for points outside the range of the sorted list
if point <= fn(sorted_list[0]):
return 0
if point >= fn(sorted_list[-1]):
return n - 1
# Binary search
left, right = 0, n - 1
while left < right:
mid = (left + right) // 2
mid_val = fn(sorted_list[mid])
if mid_val < point:
left = mid + 1
else:
right = mid
# After the loop, left is the first index greater than or equal to the point
# We need to check which of the closest points is closer to the reference point
if left == 0:
return 0
if left == n:
return n - 1
# Compare the closest two points
if abs(fn(sorted_list[left]) - point) < abs(fn(sorted_list[left - 1]) - point):
return left
else:
return left - 1
def bbox_longer(ba, bb) -> bool:
"""Return True if the bounding box of the first PDFMiner object is longer or equal to the second.
Parameters
----------
ba : PDFMiner text object
bb : PDFMiner text object
Returns
-------
longer : bool
True if the bounding box of the first object is longer or equal
"""
return (ba.x1 - ba.x0) >= (bb.x1 - bb.x0)
def merge_close_lines(ar, line_tol=2):
"""Merge lines which are within a tolerance.
By calculating a moving mean, based on their x or y axis projections.
Parameters
----------
ar : list
line_tol : int, optional (default: 2)
Returns
-------
ret : list
"""
ret = []
for a in ar:
if not ret:
ret.append(a)
else:
temp = ret[-1]
if math.isclose(temp, a, abs_tol=line_tol):
temp = (temp + a) / 2.0
ret[-1] = temp
else:
ret.append(a)
return ret
def text_replace(text, replacements):
"""Apply key→value substring substitutions to `text`.
Parameters
----------
text : str
Text to process.
replacements : dict[str, str] or sequence of (str, str), optional
Mapping of pattern → replacement. Each key is matched as a
literal substring (regex metacharacters are escaped) and every
occurrence is replaced with the corresponding value. Empty
keys are ignored. Falsy `replacements` returns the text
unchanged. Order: when several keys could match at the same
position, the longest one wins (so ``{"abc": "X", "ab": "Y"}``
replaces ``"abc"`` with ``"X"`` rather than producing ``"Yc"``).
Returns
-------
str
"""
if not replacements:
return text
if hasattr(replacements, "items"):
items = list(replacements.items())
else:
items = list(replacements)
# Drop empty patterns and prefer longer keys (re's alternation is
# left-first, so we sort longest-first to make {"abc":"X","ab":"Y"}
# behave the obvious way).
items = [(k, v) for k, v in items if k]
if not items:
return text
items.sort(key=lambda kv: len(kv[0]), reverse=True)
pattern = "|".join(re.escape(k) for k, _ in items)
table = dict(items)
return re.sub(pattern, lambda m: table[m.group(0)], text, flags=re.UNICODE)
def text_strip(text, strip=""):
"""Strip characters (or whole substrings) from `text`.
Parameters
----------
text : str
Text to process and strip.
strip : str or sequence of str, optional (default: '')
If a ``str``: every occurrence of *any character* in `strip` is
removed from `text` (the long-standing behaviour).
If a list/tuple of ``str``: every occurrence of *each substring*
is removed from `text`. This is the request from #484 — the
whole-substring mode is opt-in by passing a sequence, so
existing callers passing a single string keep their per-character
semantics.
Returns
-------
stripped : str
"""
if not strip:
return text
if isinstance(strip, (list, tuple)):
# Substring-strip mode: build an alternation of escaped pieces.
# Drop empties so users passing ["", "foo"] don't get "match the
# empty string everywhere" behaviour.
pieces = [s for s in strip if s]
if not pieces:
return text
pattern = "|".join(re.escape(s) for s in pieces)
return re.sub(pattern, "", text, flags=re.UNICODE)
# Backward-compatible character-class strip.
stripped = re.sub(
rf"[{''.join(map(re.escape, strip))}]", "", text, flags=re.UNICODE
)
return stripped
# TODO: combine the following functions into a TextProcessor class which
# applies corresponding transformations sequentially
# (inspired from sklearn.pipeline.Pipeline)
def flag_font_size(
textline: list[LTChar | LTAnno], direction: str, strip_text: str = ""
) -> str:
"""Flag super/subscripts.
Flag super/subscripts in text by enclosing them with {combined_chars}")
else:
flist.append(combined_chars)
fstring = "".join(flist)
else:
fstring = "".join(text for text, _ in d)
return text_strip(fstring, strip_text)
def split_textline(
table: Any,
textline: LTTextLine,
direction: str,
flag_size: bool = False,
strip_text: str = "",
) -> list[tuple[int, int, str]]:
"""Split textline into substrings if it spans across multiple rows/columns.
Parameters
----------
table : camelot.core.Table
The table structure containing rows and columns.
textline : LTTextLine
PDFMiner LTTextLine object.
direction : str
Direction of the PDFMiner LTTextLine object, either "horizontal" or "vertical".
flag_size : bool
Whether to highlight a substring using