fix(pdf): short dates without year + diagnostic for "0 rows" runs
User uploaded a real Chase statement and got "0 rows detected."
Two bugs the rewrite shipped with, plus a diagnostic:
**1. Short dates without year weren't recognized.** Most bank
statements (Chase, Wells, BofA, …) display transaction dates as
``01/13`` or ``Jan 13`` because the year is implied by the
statement period. The original regex required ``\d{2,4}`` after
the second slash, so ``01/13`` failed to match and rows with no
detected date got dropped.
Split ``_DATE_RES`` into ``_FULL`` (with year) and ``_SHORT``
(no year), with a two-pass detector: pass 1 tries full-year
patterns across the whole row; pass 2 only tries short patterns
if pass 1 found nothing. This prevents a stray ``Page 1/2`` from
shadowing the real dated transaction on the same line.
Short patterns:
- ``\d{1,2}/\d{1,2}`` — Chase, etc.
- ``\d{1,2}-\d{1,2}``
- ``[A-Z][a-z]{2}\s+\d{1,2}`` — "Jan 13"
When parsing, short dates pass through ``parse_date`` and
return None (no year to bind to), so the scanner falls back to
the raw text — the user sees ``01/13`` in the date column and
can correct in the editor.
**2. Multi-word dates leaked the day token into the description.**
A pre-existing bug: ``_find_dates_in_words`` returned only the
START word index, and ``_description_from_row`` only excluded
that single word. For "Jan 13 Coffee $4.50", the description
became "13 Coffee" instead of "Coffee". Fixed by returning
``(start, end, text)`` with ``end`` exclusive (computed from
``len(m.group(1).split())`` so window-overrun doesn't
over-consume), and the description builder now skips the full
range.
**3. New diagnostic: ``diagnose_pdf_lines(pdf_bytes)``.** Returns
every clustered text line the scanner saw with ``has_date`` /
``has_amount`` flags. When the page's scan returns 0 rows, an
auto-expanded "what the scanner saw" expander now renders a
table of all extracted lines so the user can:
- Spot scanned-PDF cases (empty result → enable OCR)
- See which lines have a date but no amount (or vice versa)
- Eyeball the date / amount format the scanner missed
Without leaving the app or asking the developer for help.
Eight new tests cover: short US date (``01/13``), short month-
name date with two-word consumption (``Jan 13``), the
``Page 1/2 ... 01/13/2026`` shadowing case, and the multi-word-
date description fix.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -98,7 +98,7 @@ class Page:
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
_DATE_RES = [
|
||||
_DATE_RES_FULL = [
|
||||
re.compile(r"\b(\d{1,2}/\d{1,2}/\d{2,4})\b"),
|
||||
re.compile(r"\b(\d{1,2}-\d{1,2}-\d{2,4})\b"),
|
||||
re.compile(r"\b(\d{4}-\d{2}-\d{2})\b"),
|
||||
@@ -106,6 +106,19 @@ _DATE_RES = [
|
||||
re.compile(r"\b(\d{1,2}\s+[A-Z][a-z]{2}\s+\d{2,4})\b"),
|
||||
]
|
||||
|
||||
# Short-date patterns (no year). Many bank statements show dates as
|
||||
# ``MM/DD`` or ``Jan 13`` because the year is implied by the
|
||||
# statement period. Tried only after the full-year patterns fail
|
||||
# so a string like "1/2 cup" in a memo can't claim to be a date
|
||||
# when a real dated transaction was already matched on the same row.
|
||||
_DATE_RES_SHORT = [
|
||||
re.compile(r"\b(\d{1,2}/\d{1,2})(?!\d)"),
|
||||
re.compile(r"\b(\d{1,2}-\d{1,2})(?!\d)"),
|
||||
re.compile(r"\b([A-Z][a-z]{2}\s+\d{1,2})(?!\d)"),
|
||||
]
|
||||
|
||||
_DATE_RES = _DATE_RES_FULL + _DATE_RES_SHORT
|
||||
|
||||
_DATE_FORMATS_FALLBACK = [
|
||||
"%m/%d/%Y", "%m/%d/%y", "%Y-%m-%d", "%d/%m/%Y", "%d/%m/%y",
|
||||
"%b %d %Y", "%b %d, %Y", "%d %b %Y", "%d-%b-%Y",
|
||||
@@ -427,21 +440,45 @@ def extract_pages_auto(
|
||||
|
||||
def _find_dates_in_words(
|
||||
row_words: list[WordBox],
|
||||
) -> list[tuple[int, str]]:
|
||||
"""Return ``[(word_index, date_text)]`` for the first date-like
|
||||
substring on this row, or ``[]`` if none. The index lets the
|
||||
caller exclude the date words from the description text.
|
||||
) -> list[tuple[int, int, str]]:
|
||||
"""Return ``[(start_idx, end_idx, date_text)]`` for the first
|
||||
date-like substring on this row, or ``[]`` if none.
|
||||
|
||||
Multi-word formats like ``Jan 15, 2026`` are handled by stitching
|
||||
up to three adjacent words before matching.
|
||||
Two-pass search:
|
||||
|
||||
- **Pass 1** — full-year patterns (``01/15/2026``,
|
||||
``Jan 13, 2026``). Tries the longest window first within
|
||||
this pass so a multi-word ``Jan 15, 2026`` isn't truncated
|
||||
to ``Jan 15``.
|
||||
- **Pass 2** — short patterns (``01/13``, ``Jan 13``). Only
|
||||
runs if pass 1 found nothing — otherwise a stray
|
||||
``Page 1/2`` on the same line could shadow the real dated
|
||||
transaction.
|
||||
|
||||
``end_idx`` is exclusive — caller uses ``range(start, end)``
|
||||
to exclude all words the date consumed from the description
|
||||
(the previous single-index return mis-attributed the day
|
||||
token of multi-word dates like ``Jan 13`` to the description).
|
||||
"""
|
||||
for i, w in enumerate(row_words):
|
||||
for window in (3, 2, 1):
|
||||
chunk = " ".join(x.text for x in row_words[i : i + window])
|
||||
for rx in _DATE_RES:
|
||||
m = rx.search(chunk)
|
||||
if m:
|
||||
return [(i, m.group(1))]
|
||||
for patterns, window_order in (
|
||||
(_DATE_RES_FULL, (3, 2, 1)),
|
||||
(_DATE_RES_SHORT, (2, 1)),
|
||||
):
|
||||
for i in range(len(row_words)):
|
||||
for window in window_order:
|
||||
end = i + window
|
||||
if end > len(row_words):
|
||||
continue
|
||||
chunk = " ".join(x.text for x in row_words[i:end])
|
||||
for rx in patterns:
|
||||
m = rx.search(chunk)
|
||||
if m:
|
||||
# Count whitespace-separated tokens in the
|
||||
# MATCH, not in the window — the window may
|
||||
# have included extra trailing words the
|
||||
# regex didn't actually consume.
|
||||
consumed = max(1, len(m.group(1).split()))
|
||||
return [(i, i + consumed, m.group(1))]
|
||||
return []
|
||||
|
||||
|
||||
@@ -469,18 +506,23 @@ def _find_amount_tokens(
|
||||
|
||||
def _description_from_row(
|
||||
row_words: list[WordBox],
|
||||
date_idx: int,
|
||||
date_range: tuple[int, int],
|
||||
amount_idxs: set[int],
|
||||
) -> str:
|
||||
"""Stitch the description from the row's non-date, non-amount
|
||||
tokens. Keeps tokens before the first amount and after the last
|
||||
amount (trailing check numbers and memos); drops words between
|
||||
tokens. ``date_range`` is ``(start, end)`` exclusive — every
|
||||
word in that range is excluded so multi-word dates like
|
||||
``Jan 13`` don't leak the day token into the description.
|
||||
|
||||
Keeps tokens before the first amount and after the last
|
||||
amount (trailing check numbers, memos); drops words between
|
||||
amount tokens (usually whitespace artifacts in column gaps)."""
|
||||
date_start, date_end = date_range
|
||||
keep: list[str] = []
|
||||
seen_first_amount = False
|
||||
last_amount_idx = max(amount_idxs) if amount_idxs else -1
|
||||
for i, w in enumerate(row_words):
|
||||
if i == date_idx:
|
||||
if date_start <= i < date_end:
|
||||
continue
|
||||
if i in amount_idxs:
|
||||
seen_first_amount = True
|
||||
@@ -552,9 +594,11 @@ def scan_pdf_for_transactions(
|
||||
)
|
||||
continue
|
||||
|
||||
date_idx, date_text = dates[0]
|
||||
date_start, date_end, date_text = dates[0]
|
||||
amount_idxs = {idx for idx, _, _ in amount_tokens}
|
||||
desc = _description_from_row(row_words, date_idx, amount_idxs)
|
||||
desc = _description_from_row(
|
||||
row_words, (date_start, date_end), amount_idxs,
|
||||
)
|
||||
|
||||
record: dict[str, Any] = {
|
||||
"date": parse_date(date_text, date_formats) or date_text,
|
||||
@@ -578,11 +622,58 @@ def scan_pdf_for_transactions(
|
||||
return out_rows, warnings
|
||||
|
||||
|
||||
def diagnose_pdf_lines(
|
||||
pdf_bytes: bytes,
|
||||
*,
|
||||
allow_ocr: bool = True,
|
||||
max_lines: int = 200,
|
||||
) -> tuple[list[dict[str, Any]], list[str]]:
|
||||
"""Dump every clustered text line from a PDF for diagnosis.
|
||||
|
||||
Surfaces what the scanner actually saw — including lines the
|
||||
detector dropped because they lacked a date or amount. Use
|
||||
when ``scan_pdf_for_transactions`` returns 0 rows so the user
|
||||
can spot what's wrong (no extractable text → scanned PDF /
|
||||
weird date format / amounts in a column the regex misses).
|
||||
|
||||
Returns ``(lines, warnings)`` where each line is::
|
||||
|
||||
{"page": int, "text": str,
|
||||
"has_date": bool, "has_amount": bool}
|
||||
|
||||
Capped at *max_lines* across all pages so a 100-page statement
|
||||
doesn't dump 10,000 rows into the UI.
|
||||
"""
|
||||
pages, warnings = extract_pages_auto(pdf_bytes, allow_ocr=allow_ocr)
|
||||
out: list[dict[str, Any]] = []
|
||||
for page in pages:
|
||||
rows = cluster_rows(page.words)
|
||||
for row_words in rows:
|
||||
text = " ".join(w.text for w in row_words).strip()
|
||||
if not text:
|
||||
continue
|
||||
out.append({
|
||||
"page": page.page_no,
|
||||
"text": text,
|
||||
"has_date": bool(_find_dates_in_words(row_words)),
|
||||
"has_amount": bool(_find_amount_tokens(row_words)),
|
||||
})
|
||||
if len(out) >= max_lines:
|
||||
warnings.append(
|
||||
f"Diagnostic capped at {max_lines} lines. "
|
||||
"Larger PDFs aren't fully shown here — the full "
|
||||
"scan still runs in Scan mode."
|
||||
)
|
||||
return out, warnings
|
||||
return out, warnings
|
||||
|
||||
|
||||
__all__ = [
|
||||
"PdfDependencyMissing",
|
||||
"Page",
|
||||
"WordBox",
|
||||
"cluster_rows",
|
||||
"diagnose_pdf_lines",
|
||||
"extract_pages",
|
||||
"extract_pages_auto",
|
||||
"ocr_available",
|
||||
|
||||
Reference in New Issue
Block a user