fix: cross-tool audit findings + alignment with format standardizer
Closes 12 bugs and 8 gaps surfaced by parallel audits across all core modules, plus aligns the dedup-side normalizers with the new format_standardize behavior where they had silently diverged. Bugs (data integrity / correctness): - dedup: NaN/None values matched as duplicates because str(None)='None'. Two rows with missing email silently merged. - dedup: removed_df had 0 columns when nothing was removed; downstream code expecting matching schema broke. Now preserves column shape. - dedup: ColumnMatchStrategy threshold accepted any value; out-of-range silently broke matching. Validated to [0, 100] in __post_init__. - dedup: strategy referencing a missing column was silently skipped. Now raises ValueError listing available columns. - fixes: replace_null_sentinels crashed on non-string sentinels (int/None from JSON payload). Coerced to str. - fixes: _vectorized_regex_sub raised raw re.error on bad patterns. Now wraps as ValueError with clear message. - io: detect_header_row mis-identified all-empty and metadata-only rows as headers (all([]) is True). Now requires ≥2 non-empty cells. - config: from_dict crashed when JSON had unknown fields, breaking forward compat. Now filters to known fields. - analyze: mixed-case email detector flagged all-None columns because str(None)='None' contains both N and one. Now drops NaN before stringify. New features and gap closures: - io: _detect_excel_header_row mirrors detect_header_row for Excel via openpyxl read-only; _read_excel uses it when header_row=None. - io: write_file gains delimiter + encoding params; .tsv extension defaults to tab. - normalizers: normalize_phone preserves extensions as ;ext=N suffix. - normalizers: normalize_address folds spelled-out US state names to 2-letter codes (California ≡ CA). - normalizers: normalize_name drops surname particles (van, de, von) so "Charles de Gaulle" ≡ "Charles Gaulle" for matching. - analyze: new _detect_inconsistent_date_format detector flags columns with mixed ISO/US/EU date shapes; routes to format standardizer. - analyze: _NULL_LIKE recognizes "<na>" (pd.NA repr). - analyze: duplicate-row finding renamed count → n_extra (rows that would actually be removed) with clarified description. - dedup: group_confidence no longer falsely 100.0 when transitive group members lack a recorded direct pair; falls back to 100.0 only when truly no pairs were observed. - dedup: MatchResult / DeduplicationResult docstrings clarify that row_indices refer to the input frame's positional index (output index is reset). - text_clean: visualize_hidden_html(None) now returns None (matches visualize_hidden_text); strip_bom strips at most one BOM per call; sentence_case dead elif branch removed. Tests: - tests/test_audit_fixes.py — 28 regression tests, one or more per numbered finding, named after BUG/GAP/NIT tags so future readers can trace each test back to its audit. - tests/test_fixes_unit.py — 26 isolated unit tests for previously integration-only fix functions (trim_whitespace, strip_nbsp, strip_zero_width, normalize_line_endings, clean_headers, repair_mojibake — last skipped if ftfy unavailable). - tests/test_io.py — adds CSV / TSV / semicolon / UTF-8-BOM round-trip tests + Excel auto-header-detection tests. - tests/test_normalizers.py — adds 8 tests for the alignment work above (phone extension, state names, particles). Adds .claude/ to .gitignore (agent worktrees + local settings). Full project suite: 1197 passed, 4 skipped, 17 xfailed. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -109,8 +109,18 @@ def detect_header_row(path: Path, encoding: str = "utf-8", delimiter: str = ",",
|
||||
break
|
||||
if not row:
|
||||
continue
|
||||
# All cells must be non-empty, non-numeric strings
|
||||
if all(_looks_like_header(cell) for cell in row if cell.strip()):
|
||||
# Header heuristic:
|
||||
# - every non-empty cell looks like a header;
|
||||
# - at least 2 non-empty cells (or just 1 in a single-column
|
||||
# file). Without the count check, blank rows match
|
||||
# vacuously (``all([])`` is True) and metadata banners
|
||||
# like ``["Report 2024", "", ""]`` claim row 0 falsely.
|
||||
non_empty = [cell for cell in row if cell.strip()]
|
||||
min_required = 1 if len(row) <= 1 else 2
|
||||
if (
|
||||
len(non_empty) >= min_required
|
||||
and all(_looks_like_header(cell) for cell in non_empty)
|
||||
):
|
||||
return idx
|
||||
return 0
|
||||
|
||||
@@ -263,7 +273,11 @@ def _read_excel(
|
||||
header_row: Optional[int] = None,
|
||||
sheet_name: Optional[str | int] = 0,
|
||||
) -> pd.DataFrame:
|
||||
hdr = header_row if header_row is not None else 0
|
||||
hdr = (
|
||||
header_row
|
||||
if header_row is not None
|
||||
else _detect_excel_header_row(path, sheet_name)
|
||||
)
|
||||
logger.debug("Reading Excel {} (sheet={}, header_row={})", path.name, sheet_name, hdr)
|
||||
return pd.read_excel(
|
||||
path,
|
||||
@@ -275,6 +289,52 @@ def _read_excel(
|
||||
)
|
||||
|
||||
|
||||
def _detect_excel_header_row(
|
||||
path: Path,
|
||||
sheet_name: Optional[str | int] = 0,
|
||||
max_scan: int = 20,
|
||||
) -> int:
|
||||
"""Mirror of :func:`detect_header_row` for Excel workbooks.
|
||||
|
||||
Scans the first *max_scan* rows of *sheet_name* in read-only mode
|
||||
(so a 100 MB workbook doesn't get fully materialized) and returns
|
||||
the index of the first row where every non-empty cell looks like a
|
||||
column header. Falls back to 0.
|
||||
"""
|
||||
try:
|
||||
from openpyxl import load_workbook
|
||||
except ImportError:
|
||||
return 0
|
||||
|
||||
try:
|
||||
wb = load_workbook(path, read_only=True, data_only=True)
|
||||
except Exception:
|
||||
return 0
|
||||
try:
|
||||
if isinstance(sheet_name, int):
|
||||
names = wb.sheetnames
|
||||
target = names[sheet_name] if 0 <= sheet_name < len(names) else names[0]
|
||||
elif isinstance(sheet_name, str):
|
||||
target = sheet_name if sheet_name in wb.sheetnames else wb.sheetnames[0]
|
||||
else:
|
||||
target = wb.sheetnames[0]
|
||||
ws = wb[target]
|
||||
for idx, row in enumerate(ws.iter_rows(values_only=True)):
|
||||
if idx >= max_scan:
|
||||
break
|
||||
cells = ["" if v is None else str(v) for v in row]
|
||||
non_empty = [c for c in cells if c.strip()]
|
||||
min_required = 1 if len(cells) <= 1 else 2
|
||||
if (
|
||||
len(non_empty) >= min_required
|
||||
and all(_looks_like_header(c) for c in non_empty)
|
||||
):
|
||||
return idx
|
||||
return 0
|
||||
finally:
|
||||
wb.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Writing
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -285,6 +345,7 @@ def write_file(
|
||||
*,
|
||||
file_format: Optional[str] = None,
|
||||
encoding: str = "utf-8-sig",
|
||||
delimiter: Optional[str] = None,
|
||||
) -> Path:
|
||||
"""Write a DataFrame to CSV or Excel.
|
||||
|
||||
@@ -292,8 +353,12 @@ def write_file(
|
||||
----------
|
||||
df : DataFrame to write
|
||||
path : output file path
|
||||
file_format : ``"csv"`` or ``"xlsx"``; auto-detected from *path* suffix if *None*
|
||||
file_format : ``"csv"``, ``"tsv"``, or ``"xlsx"``; auto-detected from
|
||||
*path* suffix if *None*
|
||||
encoding : output encoding (default ``utf-8-sig`` for Windows Excel compat)
|
||||
delimiter : field separator for delimited output. Defaults to ``,``
|
||||
for ``.csv``, ``\\t`` for ``.tsv``, and the explicit value
|
||||
otherwise. Ignored for Excel formats.
|
||||
|
||||
Returns the resolved output Path.
|
||||
"""
|
||||
@@ -302,7 +367,10 @@ def write_file(
|
||||
if fmt in ("xlsx", "xls"):
|
||||
df.to_excel(out, index=False, engine="openpyxl")
|
||||
else:
|
||||
df.to_csv(out, index=False, encoding=encoding)
|
||||
sep = delimiter if delimiter is not None else (
|
||||
"\t" if fmt == "tsv" else ","
|
||||
)
|
||||
df.to_csv(out, index=False, encoding=encoding, sep=sep)
|
||||
logger.info("Wrote {} rows to {}", len(df), out)
|
||||
return out
|
||||
|
||||
|
||||
Reference in New Issue
Block a user