fix: cross-tool audit findings + alignment with format standardizer
Closes 12 bugs and 8 gaps surfaced by parallel audits across all core modules, plus aligns the dedup-side normalizers with the new format_standardize behavior where they had silently diverged. Bugs (data integrity / correctness): - dedup: NaN/None values matched as duplicates because str(None)='None'. Two rows with missing email silently merged. - dedup: removed_df had 0 columns when nothing was removed; downstream code expecting matching schema broke. Now preserves column shape. - dedup: ColumnMatchStrategy threshold accepted any value; out-of-range silently broke matching. Validated to [0, 100] in __post_init__. - dedup: strategy referencing a missing column was silently skipped. Now raises ValueError listing available columns. - fixes: replace_null_sentinels crashed on non-string sentinels (int/None from JSON payload). Coerced to str. - fixes: _vectorized_regex_sub raised raw re.error on bad patterns. Now wraps as ValueError with clear message. - io: detect_header_row mis-identified all-empty and metadata-only rows as headers (all([]) is True). Now requires ≥2 non-empty cells. - config: from_dict crashed when JSON had unknown fields, breaking forward compat. Now filters to known fields. - analyze: mixed-case email detector flagged all-None columns because str(None)='None' contains both N and one. Now drops NaN before stringify. New features and gap closures: - io: _detect_excel_header_row mirrors detect_header_row for Excel via openpyxl read-only; _read_excel uses it when header_row=None. - io: write_file gains delimiter + encoding params; .tsv extension defaults to tab. - normalizers: normalize_phone preserves extensions as ;ext=N suffix. - normalizers: normalize_address folds spelled-out US state names to 2-letter codes (California ≡ CA). - normalizers: normalize_name drops surname particles (van, de, von) so "Charles de Gaulle" ≡ "Charles Gaulle" for matching. - analyze: new _detect_inconsistent_date_format detector flags columns with mixed ISO/US/EU date shapes; routes to format standardizer. - analyze: _NULL_LIKE recognizes "<na>" (pd.NA repr). - analyze: duplicate-row finding renamed count → n_extra (rows that would actually be removed) with clarified description. - dedup: group_confidence no longer falsely 100.0 when transitive group members lack a recorded direct pair; falls back to 100.0 only when truly no pairs were observed. - dedup: MatchResult / DeduplicationResult docstrings clarify that row_indices refer to the input frame's positional index (output index is reset). - text_clean: visualize_hidden_html(None) now returns None (matches visualize_hidden_text); strip_bom strips at most one BOM per call; sentence_case dead elif branch removed. Tests: - tests/test_audit_fixes.py — 28 regression tests, one or more per numbered finding, named after BUG/GAP/NIT tags so future readers can trace each test back to its audit. - tests/test_fixes_unit.py — 26 isolated unit tests for previously integration-only fix functions (trim_whitespace, strip_nbsp, strip_zero_width, normalize_line_endings, clean_headers, repair_mojibake — last skipped if ftfy unavailable). - tests/test_io.py — adds CSV / TSV / semicolon / UTF-8-BOM round-trip tests + Excel auto-header-detection tests. - tests/test_normalizers.py — adds 8 tests for the alignment work above (phone extension, state names, particles). Adds .claude/ to .gitignore (agent worktrees + local settings). Full project suite: 1197 passed, 4 skipped, 17 xfailed. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
238
tests/test_fixes_unit.py
Normal file
238
tests/test_fixes_unit.py
Normal file
@@ -0,0 +1,238 @@
|
||||
"""Isolated unit tests for individual fix functions in src.core.fixes.
|
||||
|
||||
The integration tests at tests/test_normalize.py exercise these
|
||||
functions through the full analyze→fix pipeline. These tests pin each
|
||||
function's behavior in isolation so a regression surfaces close to the
|
||||
broken function rather than at the pipeline output.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pandas as pd
|
||||
import pytest
|
||||
|
||||
from src.core.fixes import (
|
||||
clean_headers,
|
||||
normalize_line_endings,
|
||||
repair_mojibake,
|
||||
strip_nbsp,
|
||||
strip_zero_width,
|
||||
trim_whitespace,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# trim_whitespace
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestTrimWhitespace:
|
||||
def test_strips_leading_trailing(self):
|
||||
df = pd.DataFrame({"x": [" hello ", " world "]})
|
||||
out, changed = trim_whitespace(df)
|
||||
assert list(out["x"]) == ["hello", "world"]
|
||||
assert changed == 2
|
||||
|
||||
def test_collapses_internal_runs(self):
|
||||
df = pd.DataFrame({"x": ["a b c"]})
|
||||
out, _ = trim_whitespace(df)
|
||||
assert out.loc[0, "x"] == "a b c"
|
||||
|
||||
def test_preserves_internal_in_structured(self):
|
||||
# Phone-shaped strings keep internal spacing (often semantic).
|
||||
df = pd.DataFrame({"x": ["(555) 123-4567"]})
|
||||
out, changed = trim_whitespace(df)
|
||||
assert out.loc[0, "x"] == "(555) 123-4567"
|
||||
assert changed == 0
|
||||
|
||||
def test_empty_df(self):
|
||||
df = pd.DataFrame({"x": []})
|
||||
out, changed = trim_whitespace(df)
|
||||
assert len(out) == 0
|
||||
assert changed == 0
|
||||
|
||||
def test_no_string_columns(self):
|
||||
df = pd.DataFrame({"n": [1, 2, 3]})
|
||||
out, changed = trim_whitespace(df)
|
||||
assert changed == 0
|
||||
assert list(out["n"]) == [1, 2, 3]
|
||||
|
||||
def test_nan_preserved(self):
|
||||
df = pd.DataFrame({"x": [" ok ", None]})
|
||||
out, _ = trim_whitespace(df)
|
||||
assert out.loc[0, "x"] == "ok"
|
||||
# NaN/None passes through (becomes empty string after strip OR stays)
|
||||
assert out.loc[1, "x"] is None or out.loc[1, "x"] == ""
|
||||
|
||||
def test_idempotent(self):
|
||||
df = pd.DataFrame({"x": [" hello world "]})
|
||||
out1, _ = trim_whitespace(df)
|
||||
out2, changed2 = trim_whitespace(out1)
|
||||
assert changed2 == 0
|
||||
assert list(out2["x"]) == list(out1["x"])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# strip_nbsp
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestStripNbsp:
|
||||
def test_replaces_nbsp_with_ascii_space(self):
|
||||
df = pd.DataFrame({"x": ["a b"]})
|
||||
out, changed = strip_nbsp(df)
|
||||
assert out.loc[0, "x"] == "a b"
|
||||
assert changed == 1
|
||||
|
||||
def test_no_change_when_clean(self):
|
||||
df = pd.DataFrame({"x": ["a b c"]})
|
||||
out, changed = strip_nbsp(df)
|
||||
assert changed == 0
|
||||
|
||||
def test_other_unicode_spaces(self):
|
||||
# Em space (U+2003), thin space (U+2009)
|
||||
df = pd.DataFrame({"x": ["a b c"]})
|
||||
out, _ = strip_nbsp(df)
|
||||
assert out.loc[0, "x"] == "a b c"
|
||||
|
||||
def test_idempotent(self):
|
||||
df = pd.DataFrame({"x": ["a b"]})
|
||||
out1, _ = strip_nbsp(df)
|
||||
out2, changed2 = strip_nbsp(out1)
|
||||
assert changed2 == 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# strip_zero_width
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestStripZeroWidth:
|
||||
def test_removes_zero_width_space(self):
|
||||
df = pd.DataFrame({"x": ["ab"]})
|
||||
out, changed = strip_zero_width(df)
|
||||
assert out.loc[0, "x"] == "ab"
|
||||
assert changed == 1
|
||||
|
||||
def test_removes_zero_width_joiner(self):
|
||||
df = pd.DataFrame({"x": ["ab"]})
|
||||
out, _ = strip_zero_width(df)
|
||||
assert out.loc[0, "x"] == "ab"
|
||||
|
||||
def test_clean_passthrough(self):
|
||||
df = pd.DataFrame({"x": ["clean"]})
|
||||
out, changed = strip_zero_width(df)
|
||||
assert changed == 0
|
||||
|
||||
def test_idempotent(self):
|
||||
df = pd.DataFrame({"x": ["abc"]})
|
||||
out1, _ = strip_zero_width(df)
|
||||
out2, changed2 = strip_zero_width(out1)
|
||||
assert changed2 == 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# normalize_line_endings
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestNormalizeLineEndings:
|
||||
def test_crlf_to_lf(self):
|
||||
df = pd.DataFrame({"x": ["line1\r\nline2"]})
|
||||
out, changed = normalize_line_endings(df)
|
||||
assert out.loc[0, "x"] == "line1\nline2"
|
||||
assert changed == 1
|
||||
|
||||
def test_bare_cr_to_lf(self):
|
||||
df = pd.DataFrame({"x": ["line1\rline2"]})
|
||||
out, _ = normalize_line_endings(df)
|
||||
assert out.loc[0, "x"] == "line1\nline2"
|
||||
|
||||
def test_already_lf_unchanged(self):
|
||||
df = pd.DataFrame({"x": ["line1\nline2"]})
|
||||
out, changed = normalize_line_endings(df)
|
||||
assert changed == 0
|
||||
|
||||
def test_idempotent(self):
|
||||
df = pd.DataFrame({"x": ["a\r\nb\rc"]})
|
||||
out1, _ = normalize_line_endings(df)
|
||||
out2, changed2 = normalize_line_endings(out1)
|
||||
assert changed2 == 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# clean_headers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestCleanHeaders:
|
||||
def test_strips_bom_from_header(self):
|
||||
df = pd.DataFrame({"name": [1], "email": [2]})
|
||||
out, changed = clean_headers(df)
|
||||
assert "name" in out.columns
|
||||
assert "name" not in out.columns
|
||||
assert changed >= 1
|
||||
|
||||
def test_strips_nbsp_from_header(self):
|
||||
df = pd.DataFrame({"first name": [1]})
|
||||
out, _ = clean_headers(df)
|
||||
assert "first name" in out.columns
|
||||
|
||||
def test_strips_trailing_whitespace_from_header(self):
|
||||
df = pd.DataFrame({"Email ": [1]})
|
||||
out, _ = clean_headers(df)
|
||||
assert "Email" in out.columns
|
||||
assert "Email " not in out.columns
|
||||
|
||||
def test_non_string_label_preserved(self):
|
||||
df = pd.DataFrame({0: [1], 1: [2]})
|
||||
out, changed = clean_headers(df)
|
||||
assert list(out.columns) == [0, 1]
|
||||
assert changed == 0
|
||||
|
||||
def test_clean_headers_idempotent(self):
|
||||
df = pd.DataFrame({"name": [1]})
|
||||
out1, _ = clean_headers(df)
|
||||
out2, changed2 = clean_headers(out1)
|
||||
assert changed2 == 0
|
||||
assert list(out2.columns) == list(out1.columns)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# repair_mojibake
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_HAS_FTFY = True
|
||||
try:
|
||||
import ftfy # noqa: F401
|
||||
except ImportError:
|
||||
_HAS_FTFY = False
|
||||
|
||||
|
||||
@pytest.mark.skipif(not _HAS_FTFY, reason="ftfy library not installed — fix is a no-op")
|
||||
class TestRepairMojibake:
|
||||
def test_classic_cafe_repair(self):
|
||||
df = pd.DataFrame({"x": ["café"]}) # café miscoded
|
||||
out, changed = repair_mojibake(df)
|
||||
assert out.loc[0, "x"] == "café"
|
||||
assert changed == 1
|
||||
|
||||
def test_clean_text_unchanged(self):
|
||||
df = pd.DataFrame({"x": ["café"]})
|
||||
out, changed = repair_mojibake(df)
|
||||
assert changed == 0
|
||||
|
||||
def test_no_string_columns(self):
|
||||
df = pd.DataFrame({"n": [1, 2]})
|
||||
out, changed = repair_mojibake(df)
|
||||
assert changed == 0
|
||||
|
||||
def test_idempotent(self):
|
||||
df = pd.DataFrame({"x": ["café"]})
|
||||
out1, _ = repair_mojibake(df)
|
||||
out2, changed2 = repair_mojibake(out1)
|
||||
assert changed2 == 0
|
||||
|
||||
|
||||
class TestRepairMojibakeNoFtfy:
|
||||
@pytest.mark.skipif(_HAS_FTFY, reason="ftfy installed — exercises the no-op path")
|
||||
def test_returns_input_unchanged_without_ftfy(self):
|
||||
df = pd.DataFrame({"x": ["café"]})
|
||||
out, changed = repair_mojibake(df)
|
||||
assert changed == 0
|
||||
assert out.loc[0, "x"] == "café"
|
||||
Reference in New Issue
Block a user