feat(format): per-cell standardizers + 199-row buyer corpus
Adds src/core/format_standardize.py — a per-cell standardizer for dates,
phones, emails, addresses, names, currencies, booleans — wired through
StandardizeOptions / standardize_dataframe with FieldType registry.
Includes:
- Date parser handles ISO/US/EU/longform/excel-serial/unix-timestamp/
partial-precision/quarter notation; opt-in French/German/Spanish month
dictionaries via month_locales.
- Phone via libphonenumber with extension preservation (;ext=N), 001
international prefix handling, error sentinels for placeholders /
multi-number cells.
- Email lowercase/trim/mailto/angle-bracket strip with optional
--gmail-canonical mode.
- Address USPS abbreviation expansion or compression (expand=False per
corpus § 6.3), state-name → 2-letter conversion, multi-line collapse,
PO Box normalization, state-code preservation regardless of input case.
- Name handler: Mc/Mac/O'/D' inner caps, hyphen segments, particle
lowercasing (von/van/de/da), comma-format reversal, period stripping
for titles/suffixes/initials, PhD/MD acronym preservation, conservative
mode for mixed-case input.
- Currency: auto-detect EU vs US separators, space-thousands, Swiss
apostrophe, accounting parens, optional ISO code preservation, error
sentinels for percentages/ranges/word-values/ambiguous separators.
- Per-domain error_policy ("passthrough" | "sentinel") for surfacing
malformed values as <error: reason> per corpus § 0.3.
Test corpus from Business/DataTools/test-cases-format-cleaner copied to
test-cases/format-cleaner-corpus/ — 7 fixtures plus FORMATS-CASES.md.
tests/test_format_standardize_corpus.py drives all 199 rows through the
per-cell standardizers; 0 xfailed.
Wires the GUI page (3_Format_Standardizer.py) to "Ready" status.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -91,6 +91,20 @@ from .text_clean import (
|
|||||||
visualize_hidden_html,
|
visualize_hidden_html,
|
||||||
visualize_hidden_text,
|
visualize_hidden_text,
|
||||||
)
|
)
|
||||||
|
from .format_standardize import (
|
||||||
|
FieldType,
|
||||||
|
PRESETS as STANDARDIZE_PRESETS,
|
||||||
|
StandardizeOptions,
|
||||||
|
StandardizeResult,
|
||||||
|
detect_currency_code,
|
||||||
|
standardize_address,
|
||||||
|
standardize_boolean,
|
||||||
|
standardize_currency,
|
||||||
|
standardize_dataframe,
|
||||||
|
standardize_date,
|
||||||
|
standardize_name,
|
||||||
|
standardize_phone,
|
||||||
|
)
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
# Core
|
# Core
|
||||||
@@ -152,4 +166,17 @@ __all__ = [
|
|||||||
"visualize_hidden_text",
|
"visualize_hidden_text",
|
||||||
"visualize_hidden_html",
|
"visualize_hidden_html",
|
||||||
"hidden_char_css",
|
"hidden_char_css",
|
||||||
|
# Format standardization
|
||||||
|
"FieldType",
|
||||||
|
"STANDARDIZE_PRESETS",
|
||||||
|
"StandardizeOptions",
|
||||||
|
"StandardizeResult",
|
||||||
|
"detect_currency_code",
|
||||||
|
"standardize_dataframe",
|
||||||
|
"standardize_date",
|
||||||
|
"standardize_phone",
|
||||||
|
"standardize_currency",
|
||||||
|
"standardize_name",
|
||||||
|
"standardize_address",
|
||||||
|
"standardize_boolean",
|
||||||
]
|
]
|
||||||
|
|||||||
1836
src/core/format_standardize.py
Normal file
1836
src/core/format_standardize.py
Normal file
File diff suppressed because it is too large
Load Diff
@@ -1,91 +1,594 @@
|
|||||||
"""DataTools Format Standardizer — stub page."""
|
"""DataTools Format Standardizer — Streamlit page."""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import io
|
||||||
|
import json
|
||||||
import sys
|
import sys
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pandas as pd
|
||||||
import streamlit as st
|
import streamlit as st
|
||||||
|
|
||||||
_project_root = Path(__file__).resolve().parent.parent.parent.parent
|
_project_root = Path(__file__).resolve().parent.parent.parent.parent
|
||||||
if str(_project_root) not in sys.path:
|
if str(_project_root) not in sys.path:
|
||||||
sys.path.insert(0, str(_project_root))
|
sys.path.insert(0, str(_project_root))
|
||||||
|
|
||||||
from src.gui.components import hide_streamlit_chrome, require_normalization_gate
|
from src.gui.components import (
|
||||||
|
hide_streamlit_chrome,
|
||||||
|
pickup_or_upload,
|
||||||
|
require_normalization_gate,
|
||||||
|
)
|
||||||
|
from src.core.format_standardize import (
|
||||||
|
PRESETS,
|
||||||
|
FieldType,
|
||||||
|
StandardizeOptions,
|
||||||
|
standardize_dataframe,
|
||||||
|
)
|
||||||
|
|
||||||
hide_streamlit_chrome()
|
hide_streamlit_chrome()
|
||||||
require_normalization_gate()
|
require_normalization_gate()
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Header
|
# Header
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
st.title("📐 Format Standardizer")
|
st.title("📐 Format Standardizer")
|
||||||
st.caption("Standardize formats across columns for consistency.")
|
|
||||||
|
|
||||||
st.info("This tool is under development.")
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# What this tool will do
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
st.markdown("""
|
|
||||||
**Features:**
|
|
||||||
- Date format standardization (e.g., MM/DD/YYYY → YYYY-MM-DD)
|
|
||||||
- Phone number formatting (E.164, national, international)
|
|
||||||
- Currency normalization ($1,000.00 → 1000.00)
|
|
||||||
- Name casing (JOHN DOE → John Doe)
|
|
||||||
- Address abbreviation expansion (St. → Street, Ave. → Avenue)
|
|
||||||
- Boolean standardization (Yes/No/Y/N/1/0 → True/False)
|
|
||||||
""")
|
|
||||||
|
|
||||||
st.divider()
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# File upload (functional)
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
uploaded = st.file_uploader(
|
|
||||||
"Upload CSV or Excel file",
|
|
||||||
type=["csv", "tsv", "xlsx", "xls"],
|
|
||||||
help="Upload a file to preview. Processing is not yet available.",
|
|
||||||
key="fmtstd_file_upload",
|
|
||||||
)
|
|
||||||
|
|
||||||
if uploaded is not None:
|
|
||||||
import pandas as pd
|
|
||||||
try:
|
|
||||||
if uploaded.name.endswith((".xlsx", ".xls")):
|
|
||||||
df = pd.read_excel(uploaded)
|
|
||||||
else:
|
|
||||||
df = pd.read_csv(uploaded)
|
|
||||||
st.subheader(f"Preview: {uploaded.name}")
|
|
||||||
st.caption(f"{len(df)} rows, {len(df.columns)} columns")
|
|
||||||
st.dataframe(df.head(10), use_container_width=True)
|
|
||||||
except Exception as e:
|
|
||||||
st.error(f"Failed to read file: {e}")
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Placeholder options
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
st.subheader("Format Rules")
|
|
||||||
|
|
||||||
st.selectbox("Date format", ["YYYY-MM-DD", "MM/DD/YYYY", "DD/MM/YYYY", "DD-Mon-YYYY"], disabled=True)
|
|
||||||
st.selectbox("Phone format", ["E.164 (+15551234567)", "National ((555) 123-4567)", "Digits only"], disabled=True)
|
|
||||||
st.selectbox("Currency handling", ["Strip symbols, keep number", "Normalize to 2 decimals", "Keep as-is"], disabled=True)
|
|
||||||
st.selectbox("Name casing", ["Title Case", "UPPER", "lower", "As-is"], disabled=True)
|
|
||||||
st.checkbox("Expand address abbreviations", value=False, disabled=True)
|
|
||||||
|
|
||||||
st.divider()
|
|
||||||
st.button("Standardize Formats", type="primary", use_container_width=True, disabled=True)
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Footer
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
st.divider()
|
|
||||||
st.caption(
|
st.caption(
|
||||||
"Runs locally. Your data never leaves this computer. "
|
"Canonicalize dates, phone numbers, currency, names, addresses, and "
|
||||||
"| DataTools v3.0"
|
"booleans on a per-column basis. Runs locally — your data never leaves "
|
||||||
|
"this computer."
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# File upload
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
uploaded = pickup_or_upload(
|
||||||
|
label="Upload CSV or Excel file",
|
||||||
|
key="fmtstd_file_upload",
|
||||||
|
types=["csv", "tsv", "xlsx", "xls"],
|
||||||
|
)
|
||||||
|
|
||||||
|
if uploaded is None:
|
||||||
|
st.info("Upload a CSV, TSV, or Excel file to begin.")
|
||||||
|
st.stop()
|
||||||
|
|
||||||
|
|
||||||
|
@st.cache_data(show_spinner=False)
|
||||||
|
def _read_uploaded(name: str, data: bytes) -> pd.DataFrame:
|
||||||
|
"""Read the uploaded bytes into a DataFrame, treating all cells as strings."""
|
||||||
|
suffix = Path(name).suffix.lower()
|
||||||
|
bio = io.BytesIO(data)
|
||||||
|
if suffix in (".xlsx", ".xls"):
|
||||||
|
return pd.read_excel(bio, dtype=str, keep_default_na=False)
|
||||||
|
for enc in ("utf-8", "utf-8-sig", "latin-1"):
|
||||||
|
try:
|
||||||
|
bio.seek(0)
|
||||||
|
sep = "\t" if suffix == ".tsv" else ","
|
||||||
|
return pd.read_csv(
|
||||||
|
bio, dtype=str, keep_default_na=False,
|
||||||
|
encoding=enc, sep=sep, on_bad_lines="warn",
|
||||||
|
)
|
||||||
|
except UnicodeDecodeError:
|
||||||
|
continue
|
||||||
|
bio.seek(0)
|
||||||
|
return pd.read_csv(bio, dtype=str, keep_default_na=False, encoding="latin-1")
|
||||||
|
|
||||||
|
|
||||||
|
try:
|
||||||
|
df = _read_uploaded(uploaded.name, uploaded.getvalue())
|
||||||
|
except Exception as e:
|
||||||
|
st.error(f"Failed to read file: {e}")
|
||||||
|
st.stop()
|
||||||
|
|
||||||
|
st.subheader(f"Preview: {uploaded.name}")
|
||||||
|
st.caption(f"{len(df)} rows, {len(df.columns)} columns")
|
||||||
|
st.dataframe(df.head(10), use_container_width=True)
|
||||||
|
st.divider()
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Auto-detect column types
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
#
|
||||||
|
# A first pass over a 200-row sample picks a likely field type per column.
|
||||||
|
# It's a hint, not a commitment — every column shows a selectbox the user
|
||||||
|
# can override. Heuristics deliberately err toward "(skip)" rather than
|
||||||
|
# guessing wrong, since wrong guesses produce misleading change audits.
|
||||||
|
|
||||||
|
import re as _re
|
||||||
|
|
||||||
|
_DATE_HINT_RE = _re.compile(
|
||||||
|
r"^\s*\d{1,4}[-/.]\d{1,2}[-/.]\d{1,4}\s*$"
|
||||||
|
r"|^\s*[A-Za-z]{3,9}\s+\d{1,2}[, ]+\d{2,4}\s*$"
|
||||||
|
r"|^\s*\d{1,2}\s+[A-Za-z]{3,9}\s+\d{2,4}\s*$"
|
||||||
|
)
|
||||||
|
_PHONE_HINT_RE = _re.compile(r"^[\s\d().+\-]+$")
|
||||||
|
_CURRENCY_HINT_RE = _re.compile(r"^[\s$€£¥]?\s*-?\d[\d,. ]*\d?\s*$|^\s*\(\s*[$€£¥]?\d.*\)\s*$")
|
||||||
|
_BOOL_TOKENS = {"yes", "no", "y", "n", "true", "false", "t", "f", "0", "1"}
|
||||||
|
|
||||||
|
|
||||||
|
def _detect_field_type(col: str, samples: list[str]) -> FieldType | None:
|
||||||
|
"""Return a likely :class:`FieldType` for *col*, or None when unsure.
|
||||||
|
|
||||||
|
Strategy: drop empties, then require ≥80% of remaining sample cells to
|
||||||
|
fit the type's hint regex. Boolean check runs first because ``0/1`` also
|
||||||
|
matches the currency regex; date/phone/currency next; address/name fall
|
||||||
|
back to header-name keywords because their cell shapes overlap with
|
||||||
|
plain free text.
|
||||||
|
"""
|
||||||
|
cells = [s.strip() for s in samples if isinstance(s, str) and s.strip()]
|
||||||
|
if not cells:
|
||||||
|
return None
|
||||||
|
n = len(cells)
|
||||||
|
threshold = max(1, int(n * 0.8))
|
||||||
|
|
||||||
|
bool_hits = sum(1 for c in cells if c.casefold() in _BOOL_TOKENS)
|
||||||
|
if bool_hits >= threshold:
|
||||||
|
return FieldType.BOOLEAN
|
||||||
|
|
||||||
|
date_hits = sum(1 for c in cells if _DATE_HINT_RE.match(c))
|
||||||
|
if date_hits >= threshold:
|
||||||
|
return FieldType.DATE
|
||||||
|
|
||||||
|
# Phone: digit-heavy, 7+ digits, no letters.
|
||||||
|
phone_hits = 0
|
||||||
|
for c in cells:
|
||||||
|
if _PHONE_HINT_RE.match(c) and sum(1 for ch in c if ch.isdigit()) >= 7:
|
||||||
|
phone_hits += 1
|
||||||
|
if phone_hits >= threshold:
|
||||||
|
return FieldType.PHONE
|
||||||
|
|
||||||
|
currency_hits = sum(1 for c in cells if _CURRENCY_HINT_RE.match(c))
|
||||||
|
if currency_hits >= threshold:
|
||||||
|
return FieldType.CURRENCY
|
||||||
|
|
||||||
|
header = col.lower()
|
||||||
|
if any(tok in header for tok in ("address", "addr", "street")):
|
||||||
|
return FieldType.ADDRESS
|
||||||
|
if any(tok in header for tok in ("name", "customer", "contact")):
|
||||||
|
return FieldType.NAME
|
||||||
|
if any(tok in header for tok in ("date", "dob", "birth", "joined", "created")):
|
||||||
|
return FieldType.DATE
|
||||||
|
if any(tok in header for tok in ("phone", "mobile", "tel")):
|
||||||
|
return FieldType.PHONE
|
||||||
|
if any(tok in header for tok in ("price", "amount", "cost", "total", "fee")):
|
||||||
|
return FieldType.CURRENCY
|
||||||
|
if any(tok in header for tok in ("active", "enabled", "is_", "has_", "flag")):
|
||||||
|
return FieldType.BOOLEAN
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Options
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
st.subheader("Column types")
|
||||||
|
st.caption(
|
||||||
|
"Assign each column to a field type. Auto-detected suggestions are "
|
||||||
|
"pre-filled; pick **(skip)** to leave a column untouched."
|
||||||
|
)
|
||||||
|
|
||||||
|
_FIELD_LABELS = {
|
||||||
|
"(skip)": None,
|
||||||
|
"Date": FieldType.DATE,
|
||||||
|
"Phone": FieldType.PHONE,
|
||||||
|
"Currency": FieldType.CURRENCY,
|
||||||
|
"Name": FieldType.NAME,
|
||||||
|
"Address": FieldType.ADDRESS,
|
||||||
|
"Boolean": FieldType.BOOLEAN,
|
||||||
|
}
|
||||||
|
_LABEL_BY_TYPE = {v: k for k, v in _FIELD_LABELS.items()}
|
||||||
|
_LABELS = list(_FIELD_LABELS.keys())
|
||||||
|
|
||||||
|
sample_size = min(len(df), 200)
|
||||||
|
sample_df = df.head(sample_size)
|
||||||
|
|
||||||
|
column_types: dict[str, FieldType] = {}
|
||||||
|
cols_per_row = 3
|
||||||
|
columns_iter = list(df.columns)
|
||||||
|
for i in range(0, len(columns_iter), cols_per_row):
|
||||||
|
cols_block = st.columns(cols_per_row)
|
||||||
|
for j, col_name in enumerate(columns_iter[i:i + cols_per_row]):
|
||||||
|
with cols_block[j]:
|
||||||
|
detected = _detect_field_type(col_name, sample_df[col_name].tolist())
|
||||||
|
default_label = _LABEL_BY_TYPE.get(detected, "(skip)")
|
||||||
|
chosen = st.selectbox(
|
||||||
|
col_name,
|
||||||
|
_LABELS,
|
||||||
|
index=_LABELS.index(default_label),
|
||||||
|
key=f"fmtstd_type__{col_name}",
|
||||||
|
)
|
||||||
|
ft = _FIELD_LABELS[chosen]
|
||||||
|
if ft is not None:
|
||||||
|
column_types[col_name] = ft
|
||||||
|
|
||||||
|
st.divider()
|
||||||
|
st.subheader("Format options")
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Preset bundle picker
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
#
|
||||||
|
# Picking a preset rewrites every option below to that preset's defaults.
|
||||||
|
# It does NOT touch column-type assignments — those are user-driven and
|
||||||
|
# orthogonal. To make the rewrite stick across the rerun, we stash the
|
||||||
|
# preset values into the per-option session keys; the widgets below read
|
||||||
|
# those keys via their ``index``/``value`` arguments.
|
||||||
|
|
||||||
|
_PRESET_LABELS = {
|
||||||
|
"us-default": "US (default) — ISO 8601 dates · E.164 phones · USD",
|
||||||
|
"european": "European — DMY input · INTL phones · EUR comma decimal",
|
||||||
|
"uk": "UK — DD/MM/YYYY · GB phones · Yes/No booleans",
|
||||||
|
"iso-strict": "ISO Strict — ISO 8601 · bare-number currency · true/false",
|
||||||
|
"legacy-us": "Legacy US — MM/DD/YYYY · National phones · Yes/No",
|
||||||
|
"custom": "Custom — keep current settings",
|
||||||
|
}
|
||||||
|
|
||||||
|
preset_choice = st.radio(
|
||||||
|
"Standards preset",
|
||||||
|
list(_PRESET_LABELS.keys()),
|
||||||
|
format_func=lambda k: _PRESET_LABELS[k],
|
||||||
|
index=0,
|
||||||
|
horizontal=False,
|
||||||
|
key="fmtstd_preset",
|
||||||
|
help=(
|
||||||
|
"Pick a published standard or regional convention as the baseline. "
|
||||||
|
"Every option below is still individually overridable; choose "
|
||||||
|
"**Custom** to keep whatever you've manually adjusted."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Detect a preset switch since the last rerun; when it changes (and the
|
||||||
|
# new choice isn't ``custom``), purge the dependent widget keys so
|
||||||
|
# Streamlit lets their ``index=``/``value=`` defaults take effect on the
|
||||||
|
# new render. Without this clear, prior session_state pins the widget to
|
||||||
|
# the previous preset's choice and the apparent picker becomes a no-op.
|
||||||
|
_DEPENDENT_KEYS = [
|
||||||
|
"fmtstd_date_format", "fmtstd_date_order",
|
||||||
|
"fmtstd_phone_format", "fmtstd_phone_region",
|
||||||
|
"fmtstd_currency_decimal", "fmtstd_currency_decimals",
|
||||||
|
"fmtstd_currency_preserve", "fmtstd_currency_preserve_code",
|
||||||
|
"fmtstd_name_case", "fmtstd_bool_style",
|
||||||
|
]
|
||||||
|
_last = st.session_state.get("fmtstd_preset_last")
|
||||||
|
if _last != preset_choice:
|
||||||
|
st.session_state["fmtstd_preset_last"] = preset_choice
|
||||||
|
if preset_choice != "custom":
|
||||||
|
for k in _DEPENDENT_KEYS:
|
||||||
|
st.session_state.pop(k, None)
|
||||||
|
st.rerun()
|
||||||
|
|
||||||
|
# Map preset → widget-state defaults. Done as labels so the radios/selects
|
||||||
|
# below pick up the right index without us re-implementing each map twice.
|
||||||
|
_PRESET_TO_WIDGETS: dict[str, dict[str, str]] = {
|
||||||
|
"us-default": {
|
||||||
|
"date_format": "YYYY-MM-DD (ISO)", "date_order": "MDY (US)",
|
||||||
|
"phone_format": "E.164 (+15551234567)", "phone_region": "US",
|
||||||
|
"currency_decimal": "dot (1,234.56)", "currency_decimals": 2,
|
||||||
|
"currency_preserve_code": False,
|
||||||
|
"name_case": "Title Case", "boolean_style": "True/False",
|
||||||
|
},
|
||||||
|
"european": {
|
||||||
|
"date_format": "YYYY-MM-DD (ISO)", "date_order": "DMY (EU)",
|
||||||
|
"phone_format": "International (+1 555-123-4567)", "phone_region": "DE",
|
||||||
|
"currency_decimal": "comma (1.234,56)", "currency_decimals": 2,
|
||||||
|
"currency_preserve_code": True,
|
||||||
|
"name_case": "Title Case", "boolean_style": "True/False",
|
||||||
|
},
|
||||||
|
"uk": {
|
||||||
|
"date_format": "DD/MM/YYYY", "date_order": "DMY (EU)",
|
||||||
|
"phone_format": "International (+1 555-123-4567)", "phone_region": "GB",
|
||||||
|
"currency_decimal": "dot (1,234.56)", "currency_decimals": 2,
|
||||||
|
"currency_preserve_code": False,
|
||||||
|
"name_case": "Title Case", "boolean_style": "Yes/No",
|
||||||
|
},
|
||||||
|
"iso-strict": {
|
||||||
|
"date_format": "YYYY-MM-DD (ISO)", "date_order": "MDY (US)",
|
||||||
|
"phone_format": "E.164 (+15551234567)", "phone_region": "US",
|
||||||
|
"currency_decimal": "dot (1,234.56)", "currency_decimals": 0,
|
||||||
|
"currency_preserve_code": True,
|
||||||
|
"name_case": "Title Case", "boolean_style": "true/false",
|
||||||
|
},
|
||||||
|
"legacy-us": {
|
||||||
|
"date_format": "MM/DD/YYYY", "date_order": "MDY (US)",
|
||||||
|
"phone_format": "National ((555) 123-4567)", "phone_region": "US",
|
||||||
|
"currency_decimal": "dot (1,234.56)", "currency_decimals": 2,
|
||||||
|
"currency_preserve_code": False,
|
||||||
|
"name_case": "Title Case", "boolean_style": "Yes/No",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
# ``iso-strict`` wants currency with no rounding; the GUI exposes that via
|
||||||
|
# the "preserve original precision" checkbox rather than a sentinel value
|
||||||
|
# in the number-input. Map that here.
|
||||||
|
_PRESET_PRESERVE_DECIMALS: dict[str, bool] = {
|
||||||
|
"iso-strict": True,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _preset_default(key: str, fallback):
|
||||||
|
"""Pull the preset-driven default for *key*, or *fallback* on Custom."""
|
||||||
|
if preset_choice == "custom":
|
||||||
|
return fallback
|
||||||
|
return _PRESET_TO_WIDGETS[preset_choice].get(key, fallback)
|
||||||
|
|
||||||
|
|
||||||
|
opt_cols = st.columns(2)
|
||||||
|
with opt_cols[0]:
|
||||||
|
st.markdown("**Dates**")
|
||||||
|
_DATE_LABELS = ["YYYY-MM-DD (ISO)", "MM/DD/YYYY", "DD/MM/YYYY", "DD-Mon-YYYY", "Mon DD, YYYY"]
|
||||||
|
date_format_label = st.selectbox(
|
||||||
|
"Output format",
|
||||||
|
_DATE_LABELS,
|
||||||
|
index=_DATE_LABELS.index(_preset_default("date_format", "YYYY-MM-DD (ISO)")),
|
||||||
|
key="fmtstd_date_format",
|
||||||
|
)
|
||||||
|
date_format_map = {
|
||||||
|
"YYYY-MM-DD (ISO)": "%Y-%m-%d",
|
||||||
|
"MM/DD/YYYY": "%m/%d/%Y",
|
||||||
|
"DD/MM/YYYY": "%d/%m/%Y",
|
||||||
|
"DD-Mon-YYYY": "%d-%b-%Y",
|
||||||
|
"Mon DD, YYYY": "%b %d, %Y",
|
||||||
|
}
|
||||||
|
_DATE_ORDER_LABELS = ["MDY (US)", "DMY (EU)"]
|
||||||
|
date_order = st.radio(
|
||||||
|
"Ambiguous input order (e.g. 01/02/2024)",
|
||||||
|
_DATE_ORDER_LABELS,
|
||||||
|
index=_DATE_ORDER_LABELS.index(_preset_default("date_order", "MDY (US)")),
|
||||||
|
horizontal=True,
|
||||||
|
key="fmtstd_date_order",
|
||||||
|
)
|
||||||
|
|
||||||
|
st.markdown("**Phones**")
|
||||||
|
_PHONE_LABELS = [
|
||||||
|
"E.164 (+15551234567)", "International (+1 555-123-4567)",
|
||||||
|
"National ((555) 123-4567)", "Digits only",
|
||||||
|
]
|
||||||
|
phone_format_label = st.selectbox(
|
||||||
|
"Output format",
|
||||||
|
_PHONE_LABELS,
|
||||||
|
index=_PHONE_LABELS.index(_preset_default("phone_format", "E.164 (+15551234567)")),
|
||||||
|
key="fmtstd_phone_format",
|
||||||
|
)
|
||||||
|
phone_format_map = {
|
||||||
|
"E.164 (+15551234567)": "E164",
|
||||||
|
"International (+1 555-123-4567)": "INTERNATIONAL",
|
||||||
|
"National ((555) 123-4567)": "NATIONAL",
|
||||||
|
"Digits only": "DIGITS",
|
||||||
|
}
|
||||||
|
phone_region = st.text_input(
|
||||||
|
"Default region (ISO-2)",
|
||||||
|
value=_preset_default("phone_region", "US"),
|
||||||
|
max_chars=2,
|
||||||
|
help="Region used when the input has no country code. ``US``, ``GB``, ``DE``, etc.",
|
||||||
|
key="fmtstd_phone_region",
|
||||||
|
).upper() or "US"
|
||||||
|
|
||||||
|
with opt_cols[1]:
|
||||||
|
st.markdown("**Currency**")
|
||||||
|
_CURR_DECIMAL_LABELS = ["dot (1,234.56)", "comma (1.234,56)"]
|
||||||
|
currency_decimal = st.radio(
|
||||||
|
"Decimal separator in input",
|
||||||
|
_CURR_DECIMAL_LABELS,
|
||||||
|
index=_CURR_DECIMAL_LABELS.index(_preset_default("currency_decimal", "dot (1,234.56)")),
|
||||||
|
horizontal=True,
|
||||||
|
key="fmtstd_currency_decimal",
|
||||||
|
)
|
||||||
|
currency_decimals = st.number_input(
|
||||||
|
"Round to decimals",
|
||||||
|
min_value=0, max_value=8,
|
||||||
|
value=int(_preset_default("currency_decimals", 2)),
|
||||||
|
step=1,
|
||||||
|
key="fmtstd_currency_decimals",
|
||||||
|
)
|
||||||
|
preserve_decimals = st.checkbox(
|
||||||
|
"Preserve original precision (don't round)",
|
||||||
|
value=_PRESET_PRESERVE_DECIMALS.get(preset_choice, False),
|
||||||
|
key="fmtstd_currency_preserve",
|
||||||
|
)
|
||||||
|
currency_preserve_code = st.checkbox(
|
||||||
|
"Preserve currency code (emit `USD 1234.56`, `EUR 99.00`, etc.)",
|
||||||
|
value=bool(_preset_default("currency_preserve_code", False)),
|
||||||
|
help=(
|
||||||
|
"Detects an ISO 4217 code or symbol in the input ($/€/£/¥/USD/"
|
||||||
|
"EUR/...) and re-emits it as a space-separated prefix on the "
|
||||||
|
"standardized number. Cells without a currency marker emit "
|
||||||
|
"just the number."
|
||||||
|
),
|
||||||
|
key="fmtstd_currency_preserve_code",
|
||||||
|
)
|
||||||
|
|
||||||
|
st.markdown("**Names**")
|
||||||
|
_NAME_CASE_LABELS = ["Title Case", "UPPER", "lower"]
|
||||||
|
name_case_label = st.selectbox(
|
||||||
|
"Casing",
|
||||||
|
_NAME_CASE_LABELS,
|
||||||
|
index=_NAME_CASE_LABELS.index(_preset_default("name_case", "Title Case")),
|
||||||
|
key="fmtstd_name_case",
|
||||||
|
)
|
||||||
|
name_case_map = {"Title Case": "title", "UPPER": "upper", "lower": "lower"}
|
||||||
|
|
||||||
|
st.markdown("**Booleans**")
|
||||||
|
_BOOL_LABELS = ["True/False", "true/false", "Yes/No", "Y/N", "1/0"]
|
||||||
|
boolean_style = st.selectbox(
|
||||||
|
"Output style",
|
||||||
|
_BOOL_LABELS,
|
||||||
|
index=_BOOL_LABELS.index(_preset_default("boolean_style", "True/False")),
|
||||||
|
key="fmtstd_bool_style",
|
||||||
|
)
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Address abbreviations — built-in USPS table is editable
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
#
|
||||||
|
# Users with international addresses (German Strasse, Spanish-language
|
||||||
|
# Avenida, French Boulevard variants) need to override the built-in
|
||||||
|
# table. Show it in a data_editor so the override is visible — the table
|
||||||
|
# is small, this is the right surface.
|
||||||
|
|
||||||
|
extra_abbreviations: dict[str, str] = {}
|
||||||
|
if any(ft == FieldType.ADDRESS for ft in column_types.values()):
|
||||||
|
with st.expander("Custom address abbreviations (advanced)", expanded=False):
|
||||||
|
st.caption(
|
||||||
|
"Add or override entries in the address abbreviation table. "
|
||||||
|
"Each row maps a short form (case-insensitive, periods OK) to "
|
||||||
|
"the long form the standardizer should emit. Built-in USPS "
|
||||||
|
"Pub. 28 entries (`St` → `Street`, `Ave` → `Avenue`, …) apply "
|
||||||
|
"automatically; rows here merge on top and can override them."
|
||||||
|
)
|
||||||
|
starter = pd.DataFrame(
|
||||||
|
[
|
||||||
|
{"abbreviation": "", "expansion": ""},
|
||||||
|
{"abbreviation": "", "expansion": ""},
|
||||||
|
{"abbreviation": "", "expansion": ""},
|
||||||
|
]
|
||||||
|
)
|
||||||
|
edited = st.data_editor(
|
||||||
|
starter,
|
||||||
|
num_rows="dynamic",
|
||||||
|
use_container_width=True,
|
||||||
|
column_config={
|
||||||
|
"abbreviation": st.column_config.TextColumn(
|
||||||
|
"Short form",
|
||||||
|
help="Case-insensitive, trailing period optional. e.g. ``Strasse``",
|
||||||
|
),
|
||||||
|
"expansion": st.column_config.TextColumn(
|
||||||
|
"Long form",
|
||||||
|
help="What the standardizer emits. e.g. ``Straße``",
|
||||||
|
),
|
||||||
|
},
|
||||||
|
key="fmtstd_extra_abbrev",
|
||||||
|
)
|
||||||
|
for _, row in edited.iterrows():
|
||||||
|
k = str(row.get("abbreviation") or "").strip()
|
||||||
|
v = str(row.get("expansion") or "").strip()
|
||||||
|
if k and v:
|
||||||
|
extra_abbreviations[k] = v
|
||||||
|
if extra_abbreviations:
|
||||||
|
st.success(
|
||||||
|
f"{len(extra_abbreviations)} custom mapping(s) will merge "
|
||||||
|
"with the built-in table."
|
||||||
|
)
|
||||||
|
|
||||||
|
options = StandardizeOptions(
|
||||||
|
column_types=column_types,
|
||||||
|
date_output_format=date_format_map[date_format_label],
|
||||||
|
date_order="MDY" if date_order.startswith("MDY") else "DMY",
|
||||||
|
phone_format=phone_format_map[phone_format_label], # type: ignore[arg-type]
|
||||||
|
phone_region=phone_region,
|
||||||
|
currency_decimal="dot" if currency_decimal.startswith("dot") else "comma",
|
||||||
|
currency_decimals=None if preserve_decimals else int(currency_decimals),
|
||||||
|
currency_preserve_code=currency_preserve_code,
|
||||||
|
name_case=name_case_map[name_case_label], # type: ignore[arg-type]
|
||||||
|
boolean_style=boolean_style, # type: ignore[arg-type]
|
||||||
|
extra_abbreviations=extra_abbreviations,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Run
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
st.divider()
|
||||||
|
|
||||||
|
if not column_types:
|
||||||
|
st.warning("Pick a field type for at least one column to enable standardization.")
|
||||||
|
|
||||||
|
run_disabled = not column_types
|
||||||
|
if st.button(
|
||||||
|
"Standardize Formats",
|
||||||
|
type="primary",
|
||||||
|
use_container_width=True,
|
||||||
|
disabled=run_disabled,
|
||||||
|
):
|
||||||
|
with st.spinner("Standardizing..."):
|
||||||
|
try:
|
||||||
|
result = standardize_dataframe(df, options)
|
||||||
|
except ValueError as e:
|
||||||
|
st.error(str(e))
|
||||||
|
st.stop()
|
||||||
|
st.session_state["fmtstd_result"] = result
|
||||||
|
st.session_state["fmtstd_input_name"] = uploaded.name
|
||||||
|
|
||||||
|
result = st.session_state.get("fmtstd_result")
|
||||||
|
if result is None:
|
||||||
|
st.stop()
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Results
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
st.subheader("Results")
|
||||||
|
|
||||||
|
pct = (result.cells_changed / result.cells_total * 100.0) if result.cells_total else 0.0
|
||||||
|
m1, m2, m3, m4 = st.columns(4)
|
||||||
|
m1.metric("Cells scanned", result.cells_total)
|
||||||
|
m2.metric("Cells changed", result.cells_changed)
|
||||||
|
m3.metric("% changed", f"{pct:.1f}%")
|
||||||
|
m4.metric("Unparseable", result.cells_unparseable)
|
||||||
|
|
||||||
|
if result.cells_unparseable:
|
||||||
|
st.info(
|
||||||
|
f"{result.cells_unparseable} cell(s) in typed columns didn't match a "
|
||||||
|
"recognizable shape and were left as-is. Check the changes audit "
|
||||||
|
"below to find them, or re-classify the column to **(skip)**."
|
||||||
|
)
|
||||||
|
|
||||||
|
if result.cells_changed:
|
||||||
|
counts = result.changes.groupby(["column", "field_type"]).size()
|
||||||
|
st.markdown("**Changes by column**")
|
||||||
|
st.dataframe(
|
||||||
|
counts.rename("cells_changed").to_frame(),
|
||||||
|
use_container_width=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
st.markdown("**Examples (first 25 changes)**")
|
||||||
|
examples = result.changes.head(25).copy()
|
||||||
|
examples["row"] = examples["row"] + 1
|
||||||
|
st.dataframe(examples, use_container_width=True, hide_index=True)
|
||||||
|
|
||||||
|
st.markdown("**Standardized preview (first 10 rows)**")
|
||||||
|
st.dataframe(result.standardized_df.head(10), use_container_width=True)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Downloads
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
st.divider()
|
||||||
|
stem = Path(st.session_state.get("fmtstd_input_name", "input")).stem
|
||||||
|
|
||||||
|
dl_a, dl_b, dl_c = st.columns(3)
|
||||||
|
with dl_a:
|
||||||
|
standardized_bytes = result.standardized_df.to_csv(index=False).encode("utf-8-sig")
|
||||||
|
st.download_button(
|
||||||
|
"Download standardized CSV",
|
||||||
|
data=standardized_bytes,
|
||||||
|
file_name=f"{stem}_standardized.csv",
|
||||||
|
mime="text/csv",
|
||||||
|
)
|
||||||
|
with dl_b:
|
||||||
|
if not result.changes.empty:
|
||||||
|
changes_bytes = result.changes.to_csv(index=False).encode("utf-8-sig")
|
||||||
|
st.download_button(
|
||||||
|
"Download changes audit",
|
||||||
|
data=changes_bytes,
|
||||||
|
file_name=f"{stem}_changes.csv",
|
||||||
|
mime="text/csv",
|
||||||
|
)
|
||||||
|
with dl_c:
|
||||||
|
config_bytes = json.dumps(options.to_dict(), indent=2).encode("utf-8")
|
||||||
|
st.download_button(
|
||||||
|
"Download config JSON",
|
||||||
|
data=config_bytes,
|
||||||
|
file_name="format_standardize_config.json",
|
||||||
|
mime="application/json",
|
||||||
|
)
|
||||||
|
|
||||||
|
st.divider()
|
||||||
|
st.caption("Runs locally. Your data never leaves this computer. | DataTools v3.0")
|
||||||
|
|||||||
@@ -68,7 +68,7 @@ TOOLS: list[Tool] = [
|
|||||||
"Standardize dates, currencies, names, phone numbers, and addresses."
|
"Standardize dates, currencies, names, phone numbers, and addresses."
|
||||||
),
|
),
|
||||||
page_slug="3_Format_Standardizer",
|
page_slug="3_Format_Standardizer",
|
||||||
status="Coming Soon",
|
status="Ready",
|
||||||
),
|
),
|
||||||
Tool(
|
Tool(
|
||||||
tool_id="04_missing_handler",
|
tool_id="04_missing_handler",
|
||||||
|
|||||||
46
test-cases/format-cleaner-corpus/24_format_dates.csv
Normal file
46
test-cases/format-cleaner-corpus/24_format_dates.csv
Normal file
@@ -0,0 +1,46 @@
|
|||||||
|
case_id,category,description,input
|
||||||
|
FD01,iso,ISO date plain,2024-01-15
|
||||||
|
FD02,iso,ISO datetime no zone,2024-01-15T10:30:00
|
||||||
|
FD03,iso,ISO datetime UTC,2024-01-15T10:30:00Z
|
||||||
|
FD04,iso,ISO datetime offset,2024-01-15T10:30:00+05:00
|
||||||
|
FD05,iso,ISO datetime with millis,2024-01-15T10:30:00.123Z
|
||||||
|
FD06,iso,ISO datetime space separator,2024-01-15 10:30:00
|
||||||
|
FD07,us,US slash 4-digit year,01/15/2024
|
||||||
|
FD08,us,US slash 2-digit year,1/15/24
|
||||||
|
FD09,us,US slash no leading zero,1/5/2024
|
||||||
|
FD10,us,US slash unambiguous (day > 12),5/30/2024
|
||||||
|
FD11,eu,EU dot 4-digit year,15.01.2024
|
||||||
|
FD12,eu,EU dot 2-digit year,15.01.24
|
||||||
|
FD13,eu,EU slash 4-digit year,15/01/2024
|
||||||
|
FD14,eu,EU slash unambiguous (day > 12),30/05/2024
|
||||||
|
FD15,eu,EU dash format,15-01-2024
|
||||||
|
FD16,longform,Month name long,"January 15, 2024"
|
||||||
|
FD17,longform,Month name short,"Jan 15, 2024"
|
||||||
|
FD18,longform,Day-month-year long,15 January 2024
|
||||||
|
FD19,longform,Day-month-year short,15 Jan 2024
|
||||||
|
FD20,longform,With weekday,"Monday, January 15, 2024"
|
||||||
|
FD21,longform,All caps month,JAN 15 2024
|
||||||
|
FD22,excel,Excel serial date,45306
|
||||||
|
FD23,excel,Excel serial with fractional time,45306.4375
|
||||||
|
FD24,unix,Unix timestamp seconds,1705320000
|
||||||
|
FD25,unix,Unix timestamp milliseconds,1705320000000
|
||||||
|
FD26,partial,Year-month only ISO,2024-01
|
||||||
|
FD27,partial,Year-month text,January 2024
|
||||||
|
FD28,partial,Quarter notation,Q1 2024
|
||||||
|
FD29,partial,Year only,2024
|
||||||
|
FD30,edge,Two-digit year ambiguity (1969 vs 2069),1/15/69
|
||||||
|
FD31,edge,Leap day valid,2024-02-29
|
||||||
|
FD32,edge,Leap day invalid (not a leap year),2023-02-29
|
||||||
|
FD33,edge,Excel 1900 leap year bug,1900-02-29
|
||||||
|
FD34,edge,Invalid month,2024-13-15
|
||||||
|
FD35,edge,Invalid day,2024-04-31
|
||||||
|
FD36,edge,Date with extraneous text,Date: 2024-01-15
|
||||||
|
FD37,edge,Date in parens annotation,2024-01-15 (verified)
|
||||||
|
FD38,edge,Empty,
|
||||||
|
FD39,edge,Whitespace-only,
|
||||||
|
FD40,edge,Garbage,not a date
|
||||||
|
FD41,locale,French month name,15 janvier 2024
|
||||||
|
FD42,locale,German month name,15. Januar 2024
|
||||||
|
FD43,timezone,Datetime with named tz,2024-01-15 10:30:00 EST
|
||||||
|
FD44,timezone,Datetime with offset and DST ambiguity,2024-03-10 02:30:00-05:00
|
||||||
|
FD45,padding,Already-clean: pass through,2024-01-15
|
||||||
|
32
test-cases/format-cleaner-corpus/25_format_phones.csv
Normal file
32
test-cases/format-cleaner-corpus/25_format_phones.csv
Normal file
@@ -0,0 +1,32 @@
|
|||||||
|
case_id,category,description,input
|
||||||
|
FP01,us,Plain digits 10,5551234567
|
||||||
|
FP02,us,Standard formatting,(555) 123-4567
|
||||||
|
FP03,us,Dashes,555-123-4567
|
||||||
|
FP04,us,Dots,555.123.4567
|
||||||
|
FP05,us,Spaces,555 123 4567
|
||||||
|
FP06,us,With country code +1,+1 555 123 4567
|
||||||
|
FP07,us,With country code 1- prefix,1-555-123-4567
|
||||||
|
FP08,us,With 001 prefix,001 555 123 4567
|
||||||
|
FP09,ext,Extension ext keyword,555-123-4567 ext 123
|
||||||
|
FP10,ext,Extension x abbreviation,555-123-4567 x123
|
||||||
|
FP11,ext,Extension hash,555-123-4567 #123
|
||||||
|
FP12,vanity,Vanity number 1-800-FLOWERS,1-800-FLOWERS
|
||||||
|
FP13,vanity,Mixed letters and digits,555-CALL-NOW
|
||||||
|
FP14,intl,UK with +44,+44 20 7946 0958
|
||||||
|
FP15,intl,UK domestic,020 7946 0958
|
||||||
|
FP16,intl,Germany with +49,+49 30 12345678
|
||||||
|
FP17,intl,France with +33,+33 1 23 45 67 89
|
||||||
|
FP18,intl,Japan with +81,+81-3-1234-5678
|
||||||
|
FP19,intl,Australia with +61,+61 2 1234 5678
|
||||||
|
FP20,e164,Already E.164 format,+15551234567
|
||||||
|
FP21,edge,Too few digits (local-only),555-1234
|
||||||
|
FP22,edge,Too many digits,1-555-123-4567-extra-99
|
||||||
|
FP23,edge,All-zeros placeholder,000-000-0000
|
||||||
|
FP24,edge,All-nines placeholder,999-999-9999
|
||||||
|
FP25,edge,Multiple numbers in cell,555-123-4567 / 555-987-6543
|
||||||
|
FP26,edge,Mismatched parens,555-(123)-4567
|
||||||
|
FP27,edge,NBSP in number,555 123 4567
|
||||||
|
FP28,edge,Very spaced,5 5 5 1 2 3 4 5 6 7
|
||||||
|
FP29,edge,Empty,
|
||||||
|
FP30,edge,Non-phone string,TBD
|
||||||
|
FP31,edge,Smart-apostrophe contamination,555’s 123-4567
|
||||||
|
32
test-cases/format-cleaner-corpus/26_format_emails.csv
Normal file
32
test-cases/format-cleaner-corpus/26_format_emails.csv
Normal file
@@ -0,0 +1,32 @@
|
|||||||
|
case_id,category,description,input
|
||||||
|
FE01,basic,Plain ASCII,alice@example.com
|
||||||
|
FE02,basic,Mixed case,Alice@Example.COM
|
||||||
|
FE03,basic,All caps,ALICE@EXAMPLE.COM
|
||||||
|
FE04,basic,Whitespace padding, alice@example.com
|
||||||
|
FE05,displayname,Display name no quotes,Alice Smith <alice@example.com>
|
||||||
|
FE06,displayname,Display name with quotes,"""Alice Smith"" <alice@example.com>"
|
||||||
|
FE07,displayname,Wrapped in angle brackets only,<alice@example.com>
|
||||||
|
FE08,prefix,mailto: prefix,mailto:alice@example.com
|
||||||
|
FE09,prefix,MAILTO: caps,MAILTO:Alice@Example.com
|
||||||
|
FE10,gmail,Gmail with dots,a.l.i.c.e@gmail.com
|
||||||
|
FE11,gmail,Gmail with +tag,alice+newsletter@gmail.com
|
||||||
|
FE12,gmail,Gmail with both,a.l.i.c.e+work@gmail.com
|
||||||
|
FE13,gmail,Non-Gmail with dots (don't touch),a.l.i.c.e@example.com
|
||||||
|
FE14,gmail,Non-Gmail with +tag (don't touch),alice+newsletter@example.com
|
||||||
|
FE15,idn,Unicode in domain,alice@münchen.de
|
||||||
|
FE16,idn,Unicode in local,アリス@example.jp
|
||||||
|
FE17,trailing,Trailing comma,"alice@example.com,"
|
||||||
|
FE18,trailing,Trailing period,alice@example.com.
|
||||||
|
FE19,trailing,Trailing closing paren,alice@example.com)
|
||||||
|
FE20,trailing,Trailing semicolon,alice@example.com;
|
||||||
|
FE21,smartquote,Wrapped in curly quotes,“alice@example.com”
|
||||||
|
FE22,invalid,Missing @,aliceexample.com
|
||||||
|
FE23,invalid,Double @,alice@@example.com
|
||||||
|
FE24,invalid,Multiple @,alice@example@com
|
||||||
|
FE25,invalid,Spaces inside,alice @ example.com
|
||||||
|
FE26,invalid,TLD-less local network,alice@localhost
|
||||||
|
FE27,multiple,Two comma-separated,"alice@example.com, bob@example.com"
|
||||||
|
FE28,multiple,Two semicolon-separated,alice@example.com; bob@example.com
|
||||||
|
FE29,edge,Empty,
|
||||||
|
FE30,edge,Whitespace-only,
|
||||||
|
FE31,edge,Already perfect,alice@example.com
|
||||||
|
34
test-cases/format-cleaner-corpus/27_format_addresses.csv
Normal file
34
test-cases/format-cleaner-corpus/27_format_addresses.csv
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
case_id,category,description,input
|
||||||
|
FA01,clean,Already USPS-formatted,"123 Main St, New York, NY 10001"
|
||||||
|
FA02,case,All caps,"123 MAIN STREET, NEW YORK, NY 10001"
|
||||||
|
FA03,case,All lowercase,"123 main street, new york, ny 10001"
|
||||||
|
FA04,case,Mixed case (preserve),"123 Main Street, New York, NY 10001"
|
||||||
|
FA05,abbrev,Street spelled out,"123 Main Street, New York, NY 10001"
|
||||||
|
FA06,abbrev,Avenue spelled out,"456 Park Avenue, New York, NY 10001"
|
||||||
|
FA07,abbrev,Boulevard spelled out,"789 Sunset Boulevard, Los Angeles, CA 90028"
|
||||||
|
FA08,abbrev,St with period,"123 Main St., New York, NY 10001"
|
||||||
|
FA09,directional,North spelled out,"123 North Main St, City, ST 12345"
|
||||||
|
FA10,directional,NORTH all caps,"123 NORTH Main St, City, ST 12345"
|
||||||
|
FA11,directional,NE compound,"123 NE Main St, City, ST 12345"
|
||||||
|
FA12,unit,Apartment spelled out,"123 Main St, Apartment 4B, City, ST 12345"
|
||||||
|
FA13,unit,Hash sign,"123 Main St, # 4B, City, ST 12345"
|
||||||
|
FA14,unit,Suite spelled out,"123 Main St, Suite 200, City, ST 12345"
|
||||||
|
FA15,state,State spelled out,"123 Main St, New York, New York 10001"
|
||||||
|
FA16,state,State all caps spelled out,"123 Main St, New York, NEW YORK 10001"
|
||||||
|
FA17,zip,ZIP+4,"123 Main St, New York, NY 10001-1234"
|
||||||
|
FA18,zip,Leading-zero ZIP (MA),"123 Main St, Boston, MA 02101"
|
||||||
|
FA19,multiline,Multi-line address,"123 Main St
|
||||||
|
Apt 4B
|
||||||
|
New York, NY 10001"
|
||||||
|
FA20,pobox,PO Box with periods,"P.O. Box 123, City, ST 12345"
|
||||||
|
FA21,pobox,PO Box without periods,"PO Box 123, City, ST 12345"
|
||||||
|
FA22,pobox,Post Office Box spelled out,"Post Office Box 123, City, ST 12345"
|
||||||
|
FA23,housenum,Letter suffix,"123A Main St, City, ST 12345"
|
||||||
|
FA24,housenum,Hyphen number,"123-1 Main St, City, ST 12345"
|
||||||
|
FA25,housenum,Half number,"123 1/2 Main St, City, ST 12345"
|
||||||
|
FA26,non_us,UK postcode address,"10 Downing Street, London, SW1A 2AA"
|
||||||
|
FA27,non_us,Canada postal code,"1 Yonge St, Toronto, ON M5E 1W7"
|
||||||
|
FA28,non_us,Japan reverse-order,"100-0001, Tokyo, Chiyoda, Marunouchi 1-1"
|
||||||
|
FA29,edge,Empty,
|
||||||
|
FA30,edge,Just a city,New York
|
||||||
|
FA31,edge,Trailing comma,"123 Main St, New York, NY 10001,"
|
||||||
|
35
test-cases/format-cleaner-corpus/28_format_names.csv
Normal file
35
test-cases/format-cleaner-corpus/28_format_names.csv
Normal file
@@ -0,0 +1,35 @@
|
|||||||
|
case_id,category,description,input
|
||||||
|
FN01,case,All caps,ALICE SMITH
|
||||||
|
FN02,case,All lowercase,alice smith
|
||||||
|
FN03,case,Already title case (preserve),Alice Smith
|
||||||
|
FN04,case,Random case (preserve),aLiCe SmItH
|
||||||
|
FN05,scots,McDonald lowercase,mcdonald
|
||||||
|
FN06,scots,MCDONALD all caps,MCDONALD
|
||||||
|
FN07,scots,MacDonald,macdonald
|
||||||
|
FN08,scots,McTaggart already correct,McTaggart
|
||||||
|
FN09,irish,O'Connor lowercase,o'connor
|
||||||
|
FN10,irish,O'CONNOR all caps,O'CONNOR
|
||||||
|
FN11,irish,O'Brien preserve,O'Brien
|
||||||
|
FN12,hyphen,Mary-Jane lowercase,mary-jane smith
|
||||||
|
FN13,hyphen,Smith-Jones,smith-jones
|
||||||
|
FN14,particle,von Trapp,von trapp
|
||||||
|
FN15,particle,Vincent van Gogh,vincent van gogh
|
||||||
|
FN16,particle,Charles de Gaulle,charles de gaulle
|
||||||
|
FN17,particle,Leonardo da Vinci,leonardo da vinci
|
||||||
|
FN18,title,Mr period,Mr. John Smith
|
||||||
|
FN19,title,DR caps,DR JANE DOE
|
||||||
|
FN20,title,Prof preserve,Prof Alice Williams
|
||||||
|
FN21,suffix,Jr period,John Smith Jr.
|
||||||
|
FN22,suffix,III roman numeral,John Smith III
|
||||||
|
FN23,suffix,PhD,Jane Doe PhD
|
||||||
|
FN24,comma,"Last, First","Smith, John"
|
||||||
|
FN25,comma,"LAST, FIRST","SMITH, JOHN"
|
||||||
|
FN26,comma,"Last, First Middle","Smith, John Andrew"
|
||||||
|
FN27,initial,Middle initial,John A. Smith
|
||||||
|
FN28,initial,Multi-initial author,j.k. rowling
|
||||||
|
FN29,nonlatin,Korean,김철수
|
||||||
|
FN30,nonlatin,Japanese,田中太郎
|
||||||
|
FN31,nonlatin,Russian,Иван Иванов
|
||||||
|
FN32,edge,Single name,Madonna
|
||||||
|
FN33,edge,Empty,
|
||||||
|
FN34,edge,Whitespace-only,
|
||||||
|
28
test-cases/format-cleaner-corpus/29_format_currencies.csv
Normal file
28
test-cases/format-cleaner-corpus/29_format_currencies.csv
Normal file
@@ -0,0 +1,28 @@
|
|||||||
|
case_id,category,description,input
|
||||||
|
FC01,us,Standard US dollar,"$1,234.56"
|
||||||
|
FC02,us,US no comma,$1234.56
|
||||||
|
FC03,us,US space after symbol,"$ 1,234.56"
|
||||||
|
FC04,us,US no symbol,"1,234.56"
|
||||||
|
FC05,us,US with code suffix,"1,234.56 USD"
|
||||||
|
FC06,us,US with code prefix,"USD 1,234.56"
|
||||||
|
FC07,us,US trailing symbol,1234.56$
|
||||||
|
FC08,eu,Euro standard,"€1.234,56"
|
||||||
|
FC09,eu,Euro space thousand,"€1 234,56"
|
||||||
|
FC10,eu,Euro code suffix,"1.234,56 EUR"
|
||||||
|
FC11,eu,Swiss apostrophe thousand,1'234.56
|
||||||
|
FC12,intl,GBP,"£1,234.56"
|
||||||
|
FC13,intl,JPY no decimal,"¥1,234"
|
||||||
|
FC14,intl,Indian rupees lakhs,"₹1,23,456.78"
|
||||||
|
FC15,negative,Leading minus,-$100.00
|
||||||
|
FC16,negative,Accounting parens,($100.00)
|
||||||
|
FC17,negative,Sign after symbol,$-100.00
|
||||||
|
FC18,edge,Zero,$0.00
|
||||||
|
FC19,edge,Scientific notation,1.5e6
|
||||||
|
FC20,edge,Percentage,15.5%
|
||||||
|
FC21,edge,Range (not normalizable),$50-$100
|
||||||
|
FC22,edge,Word value,Free
|
||||||
|
FC23,edge,TBD placeholder,TBD
|
||||||
|
FC24,edge,Empty,
|
||||||
|
FC25,edge,Already clean,1234.56
|
||||||
|
FC26,ambig,"1,234 - could be US 1234 or EU 1.234","1,234"
|
||||||
|
FC27,ambig,1.234 - could be US 1.234 or EU 1234,1.234
|
||||||
|
@@ -0,0 +1,6 @@
|
|||||||
|
case_id,name,email,phone,date,amount,address
|
||||||
|
FI01,ALICE SMITH,Alice@Example.COM,(555) 123-4567,1/15/24,"$1,234.56","123 main street, new york, ny 10001"
|
||||||
|
FI02,"mcdonald, john",mailto:John@gmail.com,+44 20 7946 0958,15.01.2024,"€1.234,56","10 DOWNING STREET, LONDON, SW1A 2AA"
|
||||||
|
FI03,DR JANE DOE PHD,"""Jane Doe"" <jane@example.com>",555-1234,"Jan 15, 2024",($100.00),"456 Park Avenue, Apt 12, New York, NEW YORK 10001"
|
||||||
|
FI04,,,,,,
|
||||||
|
FI05,Already Clean,alice@example.com,+15551234567,2024-01-15,1234.56,"123 Main St, New York, NY 10001"
|
||||||
|
513
test-cases/format-cleaner-corpus/FORMATS-CASES.md
Normal file
513
test-cases/format-cleaner-corpus/FORMATS-CASES.md
Normal file
@@ -0,0 +1,513 @@
|
|||||||
|
# FORMATS-CASES.md - `03_format_standardizer.py` Test Corpus
|
||||||
|
|
||||||
|
**Version**: 1.0
|
||||||
|
**Last updated**: April 30, 2026
|
||||||
|
**Companion to**: TEST-CASES.md (cleaning rules), QUOTE-CASES.md (parser robustness), ENCODINGS-CASES.md (I/O layer).
|
||||||
|
|
||||||
|
This corpus tests `03_format_standardizer.py`, which owns "what's there but in the wrong format." Six domains: dates, phones, emails, addresses, names, currencies. Plus a cross-domain integration fixture.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 0. Scope clarifications you should read first
|
||||||
|
|
||||||
|
Three issues to surface before the per-domain sections, because they affect what tests are valid in the first place.
|
||||||
|
|
||||||
|
### 0.1 Email scope conflict with TECHNICAL.md
|
||||||
|
|
||||||
|
USER-GUIDE.md Section 2 lists 03's purpose as "dates, currencies, names, phone numbers, addresses." TECHNICAL.md Section 10.1 item 8 puts email normalization inside `01_deduplicator`'s Tier 1 spec. **Email appears in neither place as part of 03.**
|
||||||
|
|
||||||
|
This corpus tests email normalization as if it lives in 03. The reasoning: 03 is "format standardizer" and email is a format like any other. Putting it in 01 means there's no public API for the buyer to normalize emails outside of running dedup, which is a weird ergonomic for the GUI ("To clean my emails I have to run the deduplicator?"). Better factoring: 03 owns email normalization as a public operation; 01 calls into the same `core/` function for matching.
|
||||||
|
|
||||||
|
If you disagree, fixture `26_format_emails.csv` and its expected output drop out cleanly without affecting the other five domains. If you agree, update USER-GUIDE.md Section 2 and TECHNICAL.md Section 7's per-bundle technical notes.
|
||||||
|
|
||||||
|
### 0.2 Schema preservation rule (TECHNICAL.md Section 9 invariant)
|
||||||
|
|
||||||
|
03 changes cell content, never schema. Row count, column count, column order all unchanged. This rules out a few tempting designs:
|
||||||
|
|
||||||
|
- Currency normalization that splits `$1,234.56` into separate amount and currency columns — **rejected**. Output stays in one cell.
|
||||||
|
- Address normalization that splits a single-line address into structured street/city/state/zip columns — **rejected**. Output stays in one cell.
|
||||||
|
- Phone normalization that splits phone + extension into two columns — **rejected**. Extension goes inline as `;ext=123` (RFC 3966 syntax).
|
||||||
|
|
||||||
|
If you want structured output, that's a different script (a parser, not a standardizer).
|
||||||
|
|
||||||
|
### 0.3 Boundary with neighboring scripts
|
||||||
|
|
||||||
|
| If the cell is... | Owner | 03's behavior |
|
||||||
|
|---|---|---|
|
||||||
|
| Empty string | 04 (missing values) | Pass through unchanged. Don't decide if it means "missing." |
|
||||||
|
| Whitespace-only | 02 (text cleaner) | Should already be empty by the time 03 sees it. If not (CLI user skipped 02), trim defensively. |
|
||||||
|
| Statistically extreme but format-valid (date in year 1700, phone with 10 zeros) | 06 (outliers) | Format-normalize anyway. Don't flag unusual values. |
|
||||||
|
| Format-invalid (Feb 30, missing @, letters in numeric) | 03 | Emit error sentinel `<error: <reason>>`. |
|
||||||
|
| Already correctly formatted | 03 | Pass through. Idempotency required. |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Default configuration
|
||||||
|
|
||||||
|
Tests assume the defaults below. Per-flag deviations are called out per case.
|
||||||
|
|
||||||
|
| Setting | Default | Notes |
|
||||||
|
|---|---|---|
|
||||||
|
| `--date-format` | ISO 8601 | `YYYY-MM-DD` for dates, `YYYY-MM-DDTHH:MM:SS[+ZZ:ZZ]` for datetimes |
|
||||||
|
| `--locale` | auto-detect | Per-column. Falls back to error if column has no disambiguating value |
|
||||||
|
| `--two-digit-year-cutoff` | 69 | Python default: years 00-68 → 2000-2068, 69-99 → 1969-1999 |
|
||||||
|
| `--phone-format` | E.164 | `+<country><digits>`, extensions via `;ext=` |
|
||||||
|
| `--default-country` | US | Used for phones with no country code |
|
||||||
|
| `--gmail-canonical` | off | Strip Gmail dots and +tags. Destructive, opt-in |
|
||||||
|
| `--expand-abbrev` | off | Expand St → Street etc. USPS abbreviation is the default |
|
||||||
|
| `--name-conservative` | on | Title-case only ALL CAPS or all-lowercase input |
|
||||||
|
| `--currency-locale` | auto-detect | Per-column. Same fallback as date locale |
|
||||||
|
| `--error-policy` | sentinel | Errors written as `<error: reason>`. Alternative: raise, skip-row |
|
||||||
|
| `--columns` | all | All text columns processed; `--columns date,phone` restricts |
|
||||||
|
|
||||||
|
**Idempotency requirement**: `format(format(x)) == format(x)` for every cell. Already-clean input passes through unchanged.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Test corpus index
|
||||||
|
|
||||||
|
| File | Domain | Cases | Expected outputs |
|
||||||
|
|---|---|---|---|
|
||||||
|
| `24_format_dates.csv` | Dates | 45 | Single column |
|
||||||
|
| `25_format_phones.csv` | Phones | 31 | Single column |
|
||||||
|
| `26_format_emails.csv` | Emails | 31 | Two columns (default + gmail-canonical) |
|
||||||
|
| `27_format_addresses.csv` | Addresses | 31 | Two columns (default + expand-abbrev) |
|
||||||
|
| `28_format_names.csv` | Names | 34 | Single column |
|
||||||
|
| `29_format_currencies.csv` | Currencies | 27 | Single column |
|
||||||
|
| `30_format_integration.csv` | Cross-domain | 5 | Multi-column (full row) |
|
||||||
|
|
||||||
|
All input fixtures share the schema `case_id, category, description, input` (except integration, which has the full multi-column shape). Expected output files key by `case_id` for diff-by-join testing.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. DATES (`24_format_dates.csv`)
|
||||||
|
|
||||||
|
### 3.1 Use cases by buyer persona
|
||||||
|
|
||||||
|
- **Shopify**: Order export dates joined against manual entries that used a different format. Bookkeeping reports needing consistent date format for sorting.
|
||||||
|
- **Bookkeeper**: Bank export reconciliation across multiple banks, each using its own date convention. Tax reports requiring consistent year-month grouping.
|
||||||
|
- **Freelancer**: Client data dumps where the date column is in whatever format the client's locale or software produces.
|
||||||
|
- **Marketing agency**: Campaign performance data joined across platforms (Google Ads, Facebook Ads, Mailchimp) that all use different date formats.
|
||||||
|
|
||||||
|
### 3.2 Test categories
|
||||||
|
|
||||||
|
| Category | Cases | What it tests |
|
||||||
|
|---|---|---|
|
||||||
|
| iso | FD01-FD06 | ISO 8601 baseline. Already-clean and minor variants (Z vs offset, T vs space) |
|
||||||
|
| us | FD07-FD10 | M/D/Y format with 2-digit and 4-digit years. Includes one unambiguous case (day > 12) |
|
||||||
|
| eu | FD11-FD15 | D/M/Y format with various separators. Includes one unambiguous case |
|
||||||
|
| longform | FD16-FD21 | Month-name formats (full, abbreviated, with weekday, all caps) |
|
||||||
|
| excel | FD22-FD23 | Excel serial numbers (45306 = 2024-01-15). Critical: Excel CSV exports often have date columns leak through as numbers |
|
||||||
|
| unix | FD24-FD25 | Unix timestamps in seconds and milliseconds |
|
||||||
|
| partial | FD26-FD29 | Year-month, quarter, year-only. Coarser-than-day precision |
|
||||||
|
| edge | FD30-FD40 | Two-digit year ambiguity, leap day validity, Excel 1900 leap year bug, invalid dates, dates buried in other text |
|
||||||
|
| locale | FD41-FD42 | French and German month names |
|
||||||
|
| timezone | FD43-FD44 | Named time zones, DST transitions |
|
||||||
|
| padding | FD45 | Already-clean idempotency check |
|
||||||
|
|
||||||
|
### 3.3 Critical policy decisions
|
||||||
|
|
||||||
|
**Locale ambiguity (M/D/Y vs D/M/Y)**: Per-column inspection. The cleaner scans all values in the column; if any value has day > 12, locale is unambiguously D/M/Y; if any has month > 12 (impossible in M/D/Y), locale is unambiguously D/M/Y. If nothing disambiguates, error out and require `--locale us|eu`. **Do not silently guess.** Fixture row FD13 (`15/01/2024`) is ambiguous in isolation; FD14 (`30/05/2024`) makes the column unambiguously D/M/Y; in a real column containing both, FD13 resolves to `2024-01-15`.
|
||||||
|
|
||||||
|
**Two-digit year cutoff**: Python's default of 69 (years 00-68 → 2000s, 69-99 → 1969-1999). FD30 is `1/15/69` and resolves to `1969-01-15`. This is opinionated and frequently wrong for birth-year columns. Document the flag clearly; the buyer cleaning customer DOB data needs to override.
|
||||||
|
|
||||||
|
**Excel serial dates** (FD22, FD23): Detection heuristic — column header contains "date", or all values are integers/floats in range 25569–73050 (Jan 1 1970 to Jan 1 2099 in Excel serial). Outside that heuristic the cleaner can't distinguish a date serial from any other number.
|
||||||
|
|
||||||
|
**Excel 1900 leap year bug** (FD33): Excel claims 1900-02-29 exists; it doesn't. Detect and emit error. Don't silently accept and roll over to March 1.
|
||||||
|
|
||||||
|
**Localized month names** (FD41, FD42): Default cleaner ships with English month names. French/German/Spanish/etc. require a locale dictionary. Either ship one (adds size) or document the limitation. **Recommendation**: ship English + opt-in `--month-locale=fr|de|es` for the others. This corpus tests as if French and German are supported.
|
||||||
|
|
||||||
|
**Time zones** (FD43, FD44): Named zones (EST, PST) resolve to fixed offsets, NOT dynamically interpreted with DST rules. EST → -05:00 always. If buyers need DST-aware handling, that's a 04-bundle (out of scope) or an opt-in pyzoneinfo flag.
|
||||||
|
|
||||||
|
### 3.4 Edge case: dates buried in text (FD36, FD37)
|
||||||
|
|
||||||
|
`Date: 2024-01-15` and `2024-01-15 (verified)` extract to `2024-01-15`. The cleaner uses regex extraction for date-shaped substrings before parsing. **Risk**: false positives from random number sequences. Mitigation: require an unambiguous date pattern (4-digit year + valid month + valid day with explicit separator).
|
||||||
|
|
||||||
|
### 3.5 What's not tested
|
||||||
|
|
||||||
|
- Calendar systems other than Gregorian (Hijri, Hebrew, Japanese era). Out of scope.
|
||||||
|
- Recurring date strings (`every 1st of month`). Not a date.
|
||||||
|
- Date ranges (`2024-01-01 to 2024-01-15`). Out of scope; would require a different cell semantic.
|
||||||
|
- Sub-millisecond precision. Pandas/datetime tolerate but aren't tested here.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. PHONES (`25_format_phones.csv`)
|
||||||
|
|
||||||
|
### 4.1 Use cases by buyer persona
|
||||||
|
|
||||||
|
- **Shopify**: Customer phone list normalization before Klaviyo/Mailchimp import. SMS campaigns require E.164.
|
||||||
|
- **Bookkeeper**: Vendor phone deduplication where same vendor has multiple format variants in QuickBooks vs. spreadsheets.
|
||||||
|
- **Freelancer**: Lead lists from clients in arbitrary formats.
|
||||||
|
- **Marketing agency**: Multi-platform audience reconciliation; ad platforms increasingly require E.164 for matching.
|
||||||
|
|
||||||
|
### 4.2 Test categories
|
||||||
|
|
||||||
|
| Category | Cases | What it tests |
|
||||||
|
|---|---|---|
|
||||||
|
| us | FP01-FP08 | Common US format variants — plain digits, parens-dash, dots, spaces, country code prefixes |
|
||||||
|
| ext | FP09-FP11 | Extensions in three syntactic forms (`ext`, `x`, `#`) |
|
||||||
|
| vanity | FP12-FP13 | Letter-to-digit conversion (1-800-FLOWERS) |
|
||||||
|
| intl | FP14-FP19 | UK, Germany, France, Japan, Australia |
|
||||||
|
| e164 | FP20 | Already-E.164 idempotency |
|
||||||
|
| edge | FP21-FP31 | Insufficient/excess digits, placeholders, multiple numbers per cell, NBSP, smart-quote contamination |
|
||||||
|
|
||||||
|
### 4.3 Critical policy decisions
|
||||||
|
|
||||||
|
**Default output: E.164** (`+<country><digits>`). Universal storage format. Reverses cleanly to any presentation format if the buyer wants display formatting later.
|
||||||
|
|
||||||
|
**Default country**: US, configurable via `--default-country=GB|DE|...`. For mixed-country columns, cleaner needs explicit country detection per-row, which is hard without context. Real-world advice for the buyer: split phone columns by country before normalizing.
|
||||||
|
|
||||||
|
**Vanity numbers** (FP12, FP13): Letters convert via standard phone keypad: 2=ABC, 3=DEF, ..., 9=WXYZ. `FLOWERS` → `3569377`. Loses some information (you can't reverse 3569377 to FLOWERS). Acceptable tradeoff for storage normalization.
|
||||||
|
|
||||||
|
**Trunk prefix dropping**: UK domestic format `020 7946 0958` (FP15) has a leading `0` that's a domestic trunk prefix, not part of the actual number. E.164 strips it: `+442079460958`. Same logic for other countries with trunk prefixes.
|
||||||
|
|
||||||
|
**Placeholders** (FP23, FP24): All-zeros `000-000-0000` and all-nines `999-999-9999` are conventional "no phone" sentinels in some CRMs. Emit error rather than silently producing a syntactically valid E.164 that's semantically meaningless. **Tradeoff**: a real number that happens to be `999-999-9999` (which doesn't exist in NANP, by the way; 999 is reserved) would error too. Acceptable.
|
||||||
|
|
||||||
|
**Multiple numbers** (FP25): Cell containing `555-123-4567 / 555-987-6543`. Don't silently pick one; emit error and tell the user to split first. Splitting is a structural change, not a format change, so it belongs upstream of 03.
|
||||||
|
|
||||||
|
**NBSP and smart-quote contamination** (FP27, FP31): Should not reach 03 if 02 ran first. Defensive cleanup is fine; emit a debug log noting the upstream pollution.
|
||||||
|
|
||||||
|
### 4.4 What's not tested
|
||||||
|
|
||||||
|
- SMS-vs-voice number distinction.
|
||||||
|
- Carrier lookup. Out of scope; would require a paid service.
|
||||||
|
- Number portability validation.
|
||||||
|
- Toll-free number recognition (888, 877, 866, 855, 844, 833) beyond accepting them as valid digits.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. EMAILS (`26_format_emails.csv`) — see Section 0.1 for scope caveat
|
||||||
|
|
||||||
|
### 5.1 Use cases by buyer persona
|
||||||
|
|
||||||
|
- **Shopify**: Customer list cleanup before email-marketing platform import (every duplicate costs money on per-contact pricing). Pre-flight check on order export before re-engagement campaigns.
|
||||||
|
- **Bookkeeper**: Vendor email list consolidation.
|
||||||
|
- **Freelancer**: Client communication list normalization.
|
||||||
|
- **Marketing agency**: List hygiene across multiple lead sources before campaign send.
|
||||||
|
|
||||||
|
### 5.2 Test categories
|
||||||
|
|
||||||
|
| Category | Cases | What it tests |
|
||||||
|
|---|---|---|
|
||||||
|
| basic | FE01-FE04 | Plain ASCII, mixed case, whitespace |
|
||||||
|
| displayname | FE05-FE07 | RFC display-name forms `Name <email>`, with and without quotes |
|
||||||
|
| prefix | FE08-FE09 | mailto: prefix |
|
||||||
|
| gmail | FE10-FE14 | Gmail-specific dot-equivalence and +tag handling. Includes negative cases (non-Gmail domains) that must NOT be touched |
|
||||||
|
| idn | FE15-FE16 | Internationalized domain names; Unicode in local part |
|
||||||
|
| trailing | FE17-FE20 | Punctuation contamination from copy-paste contexts |
|
||||||
|
| smartquote | FE21 | Word-paste damage |
|
||||||
|
| invalid | FE22-FE26 | Missing @, double @, multiple @, internal whitespace, no TLD |
|
||||||
|
| multiple | FE27-FE28 | Multiple emails in one cell |
|
||||||
|
| edge | FE29-FE31 | Empty, whitespace-only, already-perfect |
|
||||||
|
|
||||||
|
### 5.3 Critical policy decisions
|
||||||
|
|
||||||
|
**Default behavior**: lowercase, trim, strip `mailto:`, strip wrapping `<>`, extract from `Display Name <email>` form. **Does NOT strip Gmail dots or +tags by default.** Those normalizations are destructive (`alice` and `a.l.i.c.e` aren't the same email per RFC; only Gmail's specific provider policy treats them as equivalent).
|
||||||
|
|
||||||
|
**Aggressive mode (`--gmail-canonical`)**: Strip dots and +tags for `@gmail.com` only. Preserve them for all other domains, even if those domains have similar policies (some custom Google Workspace domains, some other providers). Don't second-guess provider policy.
|
||||||
|
|
||||||
|
**FE13 and FE14 are critical negative tests**: a non-Gmail domain with dots or +tag must NOT be touched even in `--gmail-canonical` mode. Many cleaners get this wrong — they apply Gmail's policy to all domains, which corrupts data.
|
||||||
|
|
||||||
|
**IDN handling** (FE15, FE16): Don't punycode-convert by default. Buyers who need ASCII-only output for legacy systems can opt in via `--punycode`. Default is to preserve Unicode in domain and local parts.
|
||||||
|
|
||||||
|
**Display-name extraction** (FE05, FE06): Drop the display name. The cleaner extracts the email and discards `Alice Smith`. **Tradeoff**: information loss. Alternative would be to preserve display name in a separate column, but that violates schema preservation (Section 0.2). Buyers who want to keep display names should split the column upstream.
|
||||||
|
|
||||||
|
**Multiple emails per cell** (FE27, FE28): Error, don't pick one. Same rationale as multiple phones.
|
||||||
|
|
||||||
|
### 5.4 What's not tested
|
||||||
|
|
||||||
|
- Email syntax validation per full RFC 5321/5322 (which permits all sorts of legitimately weird inputs like quoted-string locals). The cleaner uses a "good enough for 99% of real data" regex, not a full RFC parser.
|
||||||
|
- Disposable-email-domain detection. Out of scope for format cleaning; that's data quality.
|
||||||
|
- DNS / MX validation. Out of scope; requires network access.
|
||||||
|
- Email-address-as-username (where domain is a hostname not an internet domain). Errors as TLD-less.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. ADDRESSES (`27_format_addresses.csv`)
|
||||||
|
|
||||||
|
### 6.1 Use cases by buyer persona
|
||||||
|
|
||||||
|
- **Shopify**: Customer address normalization for shipping label generation; reduces failed deliveries.
|
||||||
|
- **Bookkeeper**: Vendor master record cleanup; consistent format for bookkeeping software import.
|
||||||
|
- **Freelancer**: Client address book consolidation.
|
||||||
|
- **Marketing agency**: Direct mail audience cleanup.
|
||||||
|
|
||||||
|
### 6.2 Test categories
|
||||||
|
|
||||||
|
| Category | Cases | What it tests |
|
||||||
|
|---|---|---|
|
||||||
|
| clean | FA01 | Already-USPS-formatted idempotency |
|
||||||
|
| case | FA02-FA04 | All-caps, all-lowercase, mixed-case (preserve) |
|
||||||
|
| abbrev | FA05-FA08 | Street type expansion/abbreviation, periods after abbreviations |
|
||||||
|
| directional | FA09-FA11 | North/N, NORTH/N, NE compounds |
|
||||||
|
| unit | FA12-FA14 | Apartment/Apt, # / Apt, Suite/Ste |
|
||||||
|
| state | FA15-FA16 | State name → 2-letter code |
|
||||||
|
| zip | FA17-FA18 | ZIP+4, leading-zero ZIPs (Massachusetts 02xxx) |
|
||||||
|
| multiline | FA19 | `\n`-separated address fields |
|
||||||
|
| pobox | FA20-FA22 | Post Office Box variants |
|
||||||
|
| housenum | FA23-FA25 | Letter suffix, hyphen, half-number |
|
||||||
|
| non_us | FA26-FA28 | UK, Canada, Japan (minimal handling) |
|
||||||
|
| edge | FA29-FA31 | Empty, partial, trailing comma |
|
||||||
|
|
||||||
|
### 6.3 Critical policy decisions
|
||||||
|
|
||||||
|
**US-first scope**: USPS abbreviations and state codes are the default. International addresses get whitespace + capitalization only. Document this clearly; buyers with significant non-US data should expect format drift.
|
||||||
|
|
||||||
|
**USPS abbreviations as the default** (St, Ave, Blvd) rather than spelled-out forms. Reasoning: USPS recommends abbreviations; most CRMs expect them; they save space in tabular display. The `--expand-abbrev` flag inverts this for buyers whose downstream system requires full forms.
|
||||||
|
|
||||||
|
**Multi-line collapse** (FA19): `123 Main St\nApt 4B\nNew York, NY 10001` becomes `123 Main St, Apt 4B, New York, NY 10001`. Consistent comma-separated single-line format. **Reverse direction not supported** — the cleaner doesn't take a single-line address and split into multi-line (that's structural).
|
||||||
|
|
||||||
|
**State expansion vs abbreviation** (FA15, FA16): Default is 2-letter code (`NY`). The `--expand-abbrev` flag expands to full state name. Note: this is the OPPOSITE direction from street type abbreviations. State codes are universally expected in tabular data; full state names are only preferred in some downstream systems' "pretty" formats.
|
||||||
|
|
||||||
|
**ZIP leading zeros** (FA18): If the column is already a ZIP-shaped string with leading zeros, preserve them. **Cannot restore lost leading zeros** — Excel-stripped `2101` (Massachusetts) cannot be confidently recovered to `02101` because `2101` could legitimately be `2101` (Idaho). Mention this as a known limitation; recommend the buyer fix at the source.
|
||||||
|
|
||||||
|
**Canada handling** (FA27): Canadian addresses use the same street-type conventions as US, so `St` → `St` works. Postal code format is preserved as-is.
|
||||||
|
|
||||||
|
**Japan / non-Western** (FA28): Field order is reversed (postal code first, then large-to-small geography). Default cleaner doesn't try to restructure; minimal handling only.
|
||||||
|
|
||||||
|
### 6.4 What's not tested
|
||||||
|
|
||||||
|
- Address verification against USPS database. Out of scope; would require a paid service or local USPS data.
|
||||||
|
- Geocoding to lat/long. Out of scope.
|
||||||
|
- Unit number parsing for buildings with non-standard nomenclatures.
|
||||||
|
- Military addresses (APO, FPO, DPO) beyond accepting them.
|
||||||
|
- Rural Route, Highway Contract, General Delivery formats.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. NAMES (`28_format_names.csv`)
|
||||||
|
|
||||||
|
### 7.1 Use cases by buyer persona
|
||||||
|
|
||||||
|
- **Shopify**: Customer list display normalization. ALL-CAPS imports from older systems become readable.
|
||||||
|
- **Bookkeeper**: Vendor name consistency across QuickBooks and spreadsheets.
|
||||||
|
- **Freelancer**: Client list capitalization cleanup.
|
||||||
|
- **Marketing agency**: First-name personalization in email campaigns (`Hi alice` vs `Hi Alice`).
|
||||||
|
|
||||||
|
### 7.2 Test categories
|
||||||
|
|
||||||
|
| Category | Cases | What it tests |
|
||||||
|
|---|---|---|
|
||||||
|
| case | FN01-FN04 | All-caps, all-lowercase, already-correct, random-case |
|
||||||
|
| scots | FN05-FN08 | Mc and Mac prefixes |
|
||||||
|
| irish | FN09-FN11 | O' prefix |
|
||||||
|
| hyphen | FN12-FN13 | Hyphenated names |
|
||||||
|
| particle | FN14-FN17 | von, van, de, da (Germanic, Dutch, French, Italian) |
|
||||||
|
| title | FN18-FN20 | Mr, Dr, Prof |
|
||||||
|
| suffix | FN21-FN23 | Jr, III, PhD |
|
||||||
|
| comma | FN24-FN26 | "Last, First" reversal to "First Last" |
|
||||||
|
| initial | FN27-FN28 | Middle initial, multi-initial |
|
||||||
|
| nonlatin | FN29-FN31 | Korean, Japanese, Russian (preserve) |
|
||||||
|
| edge | FN32-FN34 | Single name, empty, whitespace-only |
|
||||||
|
|
||||||
|
### 7.3 Critical policy decisions
|
||||||
|
|
||||||
|
**Conservative by default**: Title-case ONLY when input is ALL CAPS or all lowercase. Mixed-case input is preserved as-is (FN04: `aLiCe SmItH` → `aLiCe SmItH`). Reasoning: people have idiosyncratic spellings (`danah boyd`, `bell hooks`) that the cleaner should never overwrite. If the buyer wants aggressive title-casing, that's `--name-aggressive`.
|
||||||
|
|
||||||
|
**Mc vs Mac** (FN05-FN08): Default convention is `McDonald` (cap after Mc) and `MacDonald` (cap after Mac). Some Mac-prefixed names should be `Macdonald` (cap only on Mac). Without a names dictionary, the cleaner can't distinguish. Default to capitalizing — produces `MacDonald` for ambiguous cases. Buyers with significant Scottish/Irish customer bases may need a custom override list.
|
||||||
|
|
||||||
|
**Particles** (FN14-FN17): Particles like `von`, `van`, `de`, `da` stay lowercase. This is the convention for people with surnames containing these words (`Vincent van Gogh`, `Charles de Gaulle`). **Note**: at the start of a sentence or in last-name-first contexts (`De Gaulle, Charles`), capitalization rules invert. This corpus tests the natural-order case only.
|
||||||
|
|
||||||
|
**Comma format reversal** (FN24-FN26): `Smith, John` → `John Smith`. **Tradeoff**: irreversibly destroys the comma-format. If the buyer's downstream system expects "Last, First" format, they need `--name-format=last-first`. Default is natural reading order.
|
||||||
|
|
||||||
|
**Titles and suffixes**:
|
||||||
|
- Title period stripping: `Mr.` → `Mr`. Some style guides keep the period; this corpus drops it for consistency. `--keep-title-periods` flag if buyers prefer.
|
||||||
|
- Roman numerals (`II`, `III`, `IV`) stay all-caps. They aren't names; they're numerals.
|
||||||
|
- `PhD`, `MD`, `Esq` keep their conventional case. Don't lower-case them.
|
||||||
|
|
||||||
|
**Non-Latin scripts** (FN29-FN31): Pass through unchanged. Title-casing rules don't apply to scripts without case (Korean, Japanese, Chinese, Arabic, Hebrew, etc.). Cyrillic does have case but the conservative-by-default rule applies — only ALL CAPS gets title-cased.
|
||||||
|
|
||||||
|
**Single names** (FN32): Madonna, Cher, Pelé. Pass through unchanged when input is already title-case.
|
||||||
|
|
||||||
|
### 7.4 What's not tested
|
||||||
|
|
||||||
|
- Honorific stacking (`Dr. Mr. Jane Smith` — pathological, rare, hard).
|
||||||
|
- Cultural name-order detection (East Asian family-first vs Western given-first). Without a column-level signal the cleaner can't guess.
|
||||||
|
- Nickname expansion (`Bob` → `Robert`). Out of scope; that's data enrichment, not standardization.
|
||||||
|
- Name part identification (which token is given, family, middle). Belongs to a parser, not a standardizer.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8. CURRENCIES (`29_format_currencies.csv`)
|
||||||
|
|
||||||
|
### 8.1 Use cases by buyer persona
|
||||||
|
|
||||||
|
- **Shopify**: Order amount normalization across multi-currency stores.
|
||||||
|
- **Bookkeeper**: Bank export reconciliation; mixed bank formats produce different currency representations.
|
||||||
|
- **Freelancer**: Invoice data normalization.
|
||||||
|
- **Marketing agency**: Campaign spend normalization across ad platforms.
|
||||||
|
|
||||||
|
### 8.2 Test categories
|
||||||
|
|
||||||
|
| Category | Cases | What it tests |
|
||||||
|
|---|---|---|
|
||||||
|
| us | FC01-FC07 | $ prefix/suffix, comma thousands, dot decimal, USD code prefix/suffix |
|
||||||
|
| eu | FC08-FC11 | € prefix, dot thousands and comma decimal, space thousands, Swiss apostrophe |
|
||||||
|
| intl | FC12-FC14 | £, ¥ (no decimal), ₹ (lakhs grouping) |
|
||||||
|
| negative | FC15-FC17 | Leading minus, accounting parens, sign after symbol |
|
||||||
|
| edge | FC18-FC25 | Zero, scientific, percentage, range, word values, empty, idempotency |
|
||||||
|
| ambig | FC26-FC27 | Locale-ambiguous separator (`1,234` could be 1234 or 1.234) |
|
||||||
|
|
||||||
|
### 8.3 Critical policy decisions
|
||||||
|
|
||||||
|
**Output format**: `<symbol_or_code><normalized_number>`. Number uses dot decimal, no thousand separators, leading minus for negative. Currency symbol or code preserved if present in input; if no currency indicator, output is just the number.
|
||||||
|
|
||||||
|
**Locale ambiguity** (FC26, FC27): `1,234` is `1234` in US English and `1.234` in German. `1.234` is `1.234` in US English and `1234` in German. Per-column inspection: any value with both `,` and `.` (like `1,234.56`) locks the locale unambiguously; otherwise the cleaner errors and demands `--currency-locale=us|eu`. **Do not silently guess.**
|
||||||
|
|
||||||
|
**Accounting parens** (FC16): `($100.00)` → `-$100.00`. Standard accounting convention. The leading minus is more universally readable than the parens.
|
||||||
|
|
||||||
|
**Currency symbol position**: Preserved. `$100` stays prefix-symbol; `100$` (rare but seen) stays suffix-symbol; `100 USD` keeps the suffix-code form. Reasoning: changing position is destructive and the buyer can do it themselves with a simple find-replace if they want.
|
||||||
|
|
||||||
|
**Indian lakhs grouping** (FC14): `₹1,23,456.78` flattens to `₹123456.78`. Lakhs grouping (groups of 2 after the first 3) is unusual outside India and breaks downstream tools that expect Western thousand-grouping.
|
||||||
|
|
||||||
|
**JPY no decimal** (FC13): Japanese yen conventionally has no fractional part. `¥1,234` → `¥1234`. The cleaner doesn't add a decimal that wasn't there.
|
||||||
|
|
||||||
|
**Scientific notation** (FC19): `1.5e6` → `1500000`. Expand to plain notation for spreadsheet compatibility. Loses the "this was scientific" information; acceptable tradeoff.
|
||||||
|
|
||||||
|
**Percentages** (FC20): Error. Percentage and currency are different domains. If the column is meant for percentages, that's not currency.
|
||||||
|
|
||||||
|
**Ranges** (FC21): Error. Same reasoning as multi-emails; structural split needed.
|
||||||
|
|
||||||
|
**Word values** (FC22, FC23): `Free`, `TBD`, `N/A`. Error. The buyer might want these mapped to `0` (Free) or empty (TBD/N/A), but those are domain decisions the cleaner can't make safely.
|
||||||
|
|
||||||
|
### 8.4 What's not tested
|
||||||
|
|
||||||
|
- Cross-currency conversion (USD to EUR via exchange rate). Massively out of scope.
|
||||||
|
- Cryptocurrency formats (BTC, ETH amounts with high decimal precision). Out of scope.
|
||||||
|
- Historical currency notation (pre-decimalization £.s.d). Out of scope.
|
||||||
|
- Currency code standardization (USD vs US$ vs $US). Default: pass through whatever's there.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 9. INTEGRATION (`30_format_integration.csv`)
|
||||||
|
|
||||||
|
### 9.1 Purpose
|
||||||
|
|
||||||
|
Five rows, each a complete record with one or more format issues across multiple columns. Tests that running 03 across multiple columns in one pass produces consistent output and doesn't drop or scramble fields.
|
||||||
|
|
||||||
|
### 9.2 Per-row test goals
|
||||||
|
|
||||||
|
| Row | What it tests |
|
||||||
|
|---|---|
|
||||||
|
| FI01 | Standard messy-but-cleanable record. All six format types in one row. Tests that no domain's normalizer interferes with another's. |
|
||||||
|
| FI02 | International record (UK address, EUR currency, German-format date, mailto-prefixed Gmail address, comma-format Mc-name). Tests cross-domain locale handling. |
|
||||||
|
| FI03 | Errors (insufficient phone digits) and complex name (DR + JANE DOE + PHD title+name+suffix). Tests error handling and complex name parsing. |
|
||||||
|
| FI04 | All empty. Tests that empty cells pass through without errors. |
|
||||||
|
| FI05 | Already-clean record. Idempotency check — the entire row should round-trip unchanged. |
|
||||||
|
|
||||||
|
### 9.3 What this fixture catches that single-domain fixtures don't
|
||||||
|
|
||||||
|
- **Cross-column interference**: a name normalizer that reaches into the email column, or vice versa.
|
||||||
|
- **Schema drift**: a normalizer that adds, removes, or reorders columns.
|
||||||
|
- **Error-handling consistency**: when one column errors (FI03's phone), other columns in the same row still process correctly.
|
||||||
|
- **Idempotency at the row level**: FI05 must produce byte-identical output.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 10. Suggested test workflow
|
||||||
|
|
||||||
|
```python
|
||||||
|
import csv
|
||||||
|
from pathlib import Path
|
||||||
|
from src.core.format_standardizer import standardize # your impl
|
||||||
|
|
||||||
|
FORMATS = Path("test_data/formats")
|
||||||
|
EXPECTED = Path("expected/formats")
|
||||||
|
|
||||||
|
def test_single_column_domain(domain):
|
||||||
|
"""Test FD/FP/FE/FA/FN/FC fixtures with single-column expected output."""
|
||||||
|
inp = FORMATS / f"{domain}.csv"
|
||||||
|
exp = EXPECTED / f"{domain}_expected.csv"
|
||||||
|
|
||||||
|
with inp.open() as f:
|
||||||
|
cases = {r["case_id"]: r for r in csv.DictReader(f)}
|
||||||
|
with exp.open() as f:
|
||||||
|
expected = {r["case_id"]: r for r in csv.DictReader(f)}
|
||||||
|
|
||||||
|
failures = []
|
||||||
|
for case_id, case in cases.items():
|
||||||
|
got = standardize(case["input"], domain=domain.split("_")[1])
|
||||||
|
want = expected[case_id]["output"]
|
||||||
|
if got != want:
|
||||||
|
failures.append((case_id, case["input"], got, want))
|
||||||
|
return failures
|
||||||
|
|
||||||
|
# Test each domain
|
||||||
|
for domain in ["24_format_dates", "25_format_phones", "28_format_names",
|
||||||
|
"29_format_currencies"]:
|
||||||
|
failures = test_single_column_domain(domain)
|
||||||
|
print(f"{domain}: {len(failures)} failures")
|
||||||
|
|
||||||
|
# Email and address have two-policy expected output
|
||||||
|
def test_two_policy(domain, policy_columns):
|
||||||
|
inp = FORMATS / f"{domain}.csv"
|
||||||
|
exp = EXPECTED / f"{domain}_expected.csv"
|
||||||
|
with inp.open() as f:
|
||||||
|
cases = {r["case_id"]: r for r in csv.DictReader(f)}
|
||||||
|
with exp.open() as f:
|
||||||
|
expected = {r["case_id"]: r for r in csv.DictReader(f)}
|
||||||
|
|
||||||
|
for policy in policy_columns:
|
||||||
|
failures = []
|
||||||
|
for case_id, case in cases.items():
|
||||||
|
got = standardize(case["input"], domain=domain.split("_")[1],
|
||||||
|
mode=policy)
|
||||||
|
want = expected[case_id][f"output_{policy}"]
|
||||||
|
if got != want:
|
||||||
|
failures.append((case_id, case["input"], got, want))
|
||||||
|
print(f"{domain} ({policy}): {len(failures)} failures")
|
||||||
|
|
||||||
|
test_two_policy("26_format_emails", ["default", "gmail_canonical"])
|
||||||
|
test_two_policy("27_format_addresses", ["default", "expand_abbrev"])
|
||||||
|
|
||||||
|
# Idempotency property test
|
||||||
|
import random
|
||||||
|
all_inputs = []
|
||||||
|
for domain in ["24_format_dates", "25_format_phones", "26_format_emails",
|
||||||
|
"27_format_addresses", "28_format_names", "29_format_currencies"]:
|
||||||
|
with (FORMATS / f"{domain}.csv").open() as f:
|
||||||
|
all_inputs.extend((domain, r["input"]) for r in csv.DictReader(f))
|
||||||
|
|
||||||
|
for domain, inp in all_inputs:
|
||||||
|
once = standardize(inp, domain=domain.split("_")[1])
|
||||||
|
twice = standardize(once, domain=domain.split("_")[1])
|
||||||
|
assert once == twice, f"non-idempotent: {domain} {inp!r} -> {once!r} -> {twice!r}"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 11. What this corpus does NOT cover
|
||||||
|
|
||||||
|
Listed so the gaps are explicit:
|
||||||
|
|
||||||
|
1. **Performance**. All fixtures are small. Format standardization on a 500MB customer file may have memory or speed issues; benchmark separately.
|
||||||
|
2. **Cross-script integration with 02 and 04**. This corpus tests 03 in isolation. Running 02 → 03 → 04 in pipeline is a separate integration concern.
|
||||||
|
3. **GUI behavior**. Single-cell preview, per-row preview, domain auto-detection from column headers. Each is a Streamlit-layer test, not a transformation test.
|
||||||
|
4. **Custom locale dictionaries**. The fixtures assume the cleaner ships with English month names and US-default phone country. Customers who buy this product and then complain that German months aren't recognized are flagging a feature request, not a bug.
|
||||||
|
5. **URLs**. Listed in BUSINESS.md's adjacent territory but not in 03's scope. If you want URL standardization, that's a feature request.
|
||||||
|
6. **Booleans / yes-no normalization**. `Y` / `Yes` / `1` / `True` → `true`. Borderline 03 territory but explicitly excluded; can be added as a 7th domain if buyers ask for it.
|
||||||
|
7. **Postal codes outside US/UK/Canada**. ZIP-style validation only for US.
|
||||||
|
8. **Identifiers (SKU, SSN, EIN)**. Out of scope; too domain-specific.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 12. How to extend the corpus
|
||||||
|
|
||||||
|
**Add a new test case in an existing domain**:
|
||||||
|
1. Edit the relevant fixture's row list in `generate_format_test_files.py`.
|
||||||
|
2. Add the corresponding expected output entry.
|
||||||
|
3. Re-run the generator.
|
||||||
|
4. If the new case is a category not yet listed, update the per-domain category table in this document.
|
||||||
|
|
||||||
|
**Add a new domain (e.g., URLs)**:
|
||||||
|
1. Define use cases by persona.
|
||||||
|
2. Define policy decisions and which require a flag vs. being default.
|
||||||
|
3. Build the input fixture as `31_format_<domain>.csv` and the expected output as `31_format_<domain>_expected.csv`.
|
||||||
|
4. Add a Section 13 to this document covering the domain.
|
||||||
|
5. Update the index table in Section 2.
|
||||||
|
|
||||||
|
**Add a new policy variant to an existing domain**:
|
||||||
|
1. Add a new column to the expected output file (e.g., `output_strict`).
|
||||||
|
2. Document the new policy and what triggers it (which flag) in the domain's Section 5.3 (or equivalent).
|
||||||
|
3. The two-policy test in Section 10's workflow generalizes to N-policy.
|
||||||
630
tests/test_format_standardize.py
Normal file
630
tests/test_format_standardize.py
Normal file
@@ -0,0 +1,630 @@
|
|||||||
|
"""Tests for src.core.format_standardize."""
|
||||||
|
|
||||||
|
import pandas as pd
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from src.core.format_standardize import (
|
||||||
|
PRESETS,
|
||||||
|
FieldType,
|
||||||
|
StandardizeOptions,
|
||||||
|
detect_currency_code,
|
||||||
|
standardize_address,
|
||||||
|
standardize_boolean,
|
||||||
|
standardize_currency,
|
||||||
|
standardize_dataframe,
|
||||||
|
standardize_date,
|
||||||
|
standardize_name,
|
||||||
|
standardize_phone,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestStandardizeDate:
|
||||||
|
def test_iso_passthrough(self):
|
||||||
|
out, changed = standardize_date("2024-01-15")
|
||||||
|
assert out == "2024-01-15"
|
||||||
|
assert changed is False
|
||||||
|
|
||||||
|
def test_us_slash(self):
|
||||||
|
out, changed = standardize_date("01/15/2024")
|
||||||
|
assert (out, changed) == ("2024-01-15", True)
|
||||||
|
|
||||||
|
def test_us_dash(self):
|
||||||
|
out, _ = standardize_date("1-15-2024")
|
||||||
|
assert out == "2024-01-15"
|
||||||
|
|
||||||
|
def test_two_digit_year(self):
|
||||||
|
out, _ = standardize_date("01/15/24")
|
||||||
|
assert out == "2024-01-15"
|
||||||
|
|
||||||
|
def test_long_month_name(self):
|
||||||
|
out, _ = standardize_date("January 15, 2024")
|
||||||
|
assert out == "2024-01-15"
|
||||||
|
|
||||||
|
def test_short_month_name(self):
|
||||||
|
out, _ = standardize_date("Jan 15 2024")
|
||||||
|
assert out == "2024-01-15"
|
||||||
|
|
||||||
|
def test_dmy_order(self):
|
||||||
|
out, _ = standardize_date("15/01/2024", date_order="DMY")
|
||||||
|
assert out == "2024-01-15"
|
||||||
|
|
||||||
|
def test_strip_time_tail(self):
|
||||||
|
out, _ = standardize_date("2024-01-15 13:45:00")
|
||||||
|
assert out == "2024-01-15"
|
||||||
|
|
||||||
|
def test_iso_with_t_separator(self):
|
||||||
|
out, _ = standardize_date("2024-01-15T08:30:00Z")
|
||||||
|
assert out == "2024-01-15"
|
||||||
|
|
||||||
|
def test_compact(self):
|
||||||
|
out, _ = standardize_date("20240115")
|
||||||
|
assert out == "2024-01-15"
|
||||||
|
|
||||||
|
def test_custom_output(self):
|
||||||
|
out, _ = standardize_date("01/15/2024", output_format="%d %b %Y")
|
||||||
|
assert out == "15 Jan 2024"
|
||||||
|
|
||||||
|
def test_unparseable_passthrough(self):
|
||||||
|
out, changed = standardize_date("hello")
|
||||||
|
assert (out, changed) == ("hello", False)
|
||||||
|
|
||||||
|
def test_empty(self):
|
||||||
|
assert standardize_date("") == ("", False)
|
||||||
|
assert standardize_date(None) == ("", False)
|
||||||
|
|
||||||
|
def test_idempotent(self):
|
||||||
|
out, _ = standardize_date("01/15/2024")
|
||||||
|
out2, changed2 = standardize_date(out)
|
||||||
|
assert out2 == out
|
||||||
|
assert changed2 is False
|
||||||
|
|
||||||
|
|
||||||
|
class TestStandardizePhone:
|
||||||
|
def test_e164_default(self):
|
||||||
|
out, _ = standardize_phone("(555) 123-4567")
|
||||||
|
assert out == "+15551234567"
|
||||||
|
|
||||||
|
def test_national(self):
|
||||||
|
out, _ = standardize_phone("5551234567", output_format="NATIONAL")
|
||||||
|
assert out == "(555) 123-4567"
|
||||||
|
|
||||||
|
def test_international(self):
|
||||||
|
out, _ = standardize_phone("5551234567", output_format="INTERNATIONAL")
|
||||||
|
assert out == "+1 555-123-4567"
|
||||||
|
|
||||||
|
def test_digits_only(self):
|
||||||
|
out, changed = standardize_phone("(555) 123-4567", output_format="DIGITS")
|
||||||
|
assert out == "5551234567"
|
||||||
|
assert changed is True
|
||||||
|
|
||||||
|
def test_invalid_passthrough(self):
|
||||||
|
out, changed = standardize_phone("call me maybe")
|
||||||
|
assert (out, changed) == ("call me maybe", False)
|
||||||
|
|
||||||
|
def test_empty(self):
|
||||||
|
assert standardize_phone("") == ("", False)
|
||||||
|
assert standardize_phone(None) == ("", False)
|
||||||
|
|
||||||
|
def test_idempotent(self):
|
||||||
|
out, _ = standardize_phone("(555) 123-4567")
|
||||||
|
out2, changed2 = standardize_phone(out)
|
||||||
|
assert out2 == out
|
||||||
|
assert changed2 is False
|
||||||
|
|
||||||
|
|
||||||
|
class TestStandardizeCurrency:
|
||||||
|
def test_dollar_with_cents(self):
|
||||||
|
out, _ = standardize_currency("$1,234.56")
|
||||||
|
assert out == "1234.56"
|
||||||
|
|
||||||
|
def test_no_decimals_arg(self):
|
||||||
|
out, _ = standardize_currency("$1,234.56", decimals=None)
|
||||||
|
assert out == "1234.56"
|
||||||
|
|
||||||
|
def test_round_to_two(self):
|
||||||
|
out, _ = standardize_currency("$1,234.567", decimals=2)
|
||||||
|
assert out == "1234.57"
|
||||||
|
|
||||||
|
def test_integer_input(self):
|
||||||
|
out, _ = standardize_currency("$1,000", decimals=None)
|
||||||
|
assert out == "1000"
|
||||||
|
|
||||||
|
def test_negative_parens(self):
|
||||||
|
out, _ = standardize_currency("($50.00)", decimals=2)
|
||||||
|
assert out == "-50.00"
|
||||||
|
|
||||||
|
def test_negative_sign(self):
|
||||||
|
out, _ = standardize_currency("-$50.00", decimals=2)
|
||||||
|
assert out == "-50.00"
|
||||||
|
|
||||||
|
def test_iso_code_prefix(self):
|
||||||
|
out, _ = standardize_currency("USD 1,234.56")
|
||||||
|
assert out == "1234.56"
|
||||||
|
|
||||||
|
def test_iso_code_suffix(self):
|
||||||
|
out, _ = standardize_currency("1234.56 EUR")
|
||||||
|
assert out == "1234.56"
|
||||||
|
|
||||||
|
def test_european_decimal(self):
|
||||||
|
out, _ = standardize_currency("1.234,56 €", decimal="comma")
|
||||||
|
assert out == "1234.56"
|
||||||
|
|
||||||
|
def test_unparseable_passthrough(self):
|
||||||
|
out, changed = standardize_currency("free!")
|
||||||
|
assert (out, changed) == ("free!", False)
|
||||||
|
|
||||||
|
def test_ambiguous_short_comma_rejected(self):
|
||||||
|
# "1,5" under dot-decimal mode would be a comma decimal — reject.
|
||||||
|
out, changed = standardize_currency("1,5")
|
||||||
|
assert changed is False
|
||||||
|
assert out == "1,5"
|
||||||
|
|
||||||
|
def test_thousands_grouped_no_decimal(self):
|
||||||
|
out, _ = standardize_currency("1,234", decimals=None)
|
||||||
|
assert out == "1234"
|
||||||
|
|
||||||
|
def test_empty(self):
|
||||||
|
assert standardize_currency("") == ("", False)
|
||||||
|
assert standardize_currency(None) == ("", False)
|
||||||
|
|
||||||
|
def test_idempotent(self):
|
||||||
|
out, _ = standardize_currency("$1,234.56", decimals=2)
|
||||||
|
out2, changed2 = standardize_currency(out, decimals=2)
|
||||||
|
assert out2 == out
|
||||||
|
assert changed2 is False
|
||||||
|
|
||||||
|
|
||||||
|
class TestStandardizeName:
|
||||||
|
def test_shouting_to_title(self):
|
||||||
|
out, _ = standardize_name("JOHN DOE")
|
||||||
|
assert out == "John Doe"
|
||||||
|
|
||||||
|
def test_lowercase_to_title(self):
|
||||||
|
out, _ = standardize_name("john doe")
|
||||||
|
assert out == "John Doe"
|
||||||
|
|
||||||
|
def test_already_title(self):
|
||||||
|
out, changed = standardize_name("Jane Smith")
|
||||||
|
assert out == "Jane Smith"
|
||||||
|
assert changed is False
|
||||||
|
|
||||||
|
def test_apostrophe_inner_cap(self):
|
||||||
|
# Surnames with O'/D' apostrophe prefixes get the inner letter
|
||||||
|
# capitalized regardless of input case (corpus § 7.3 Irish names).
|
||||||
|
out, _ = standardize_name("o'Connor")
|
||||||
|
assert out == "O'Connor"
|
||||||
|
out2, _ = standardize_name("o'connor")
|
||||||
|
assert out2 == "O'Connor"
|
||||||
|
|
||||||
|
def test_acronym_preserved(self):
|
||||||
|
out, _ = standardize_name("Mary USA Smith")
|
||||||
|
assert out == "Mary USA Smith"
|
||||||
|
|
||||||
|
def test_upper_mode(self):
|
||||||
|
out, _ = standardize_name("john doe", case="upper")
|
||||||
|
assert out == "JOHN DOE"
|
||||||
|
|
||||||
|
def test_lower_mode(self):
|
||||||
|
out, _ = standardize_name("JOHN DOE", case="lower")
|
||||||
|
assert out == "john doe"
|
||||||
|
|
||||||
|
def test_empty(self):
|
||||||
|
assert standardize_name("") == ("", False)
|
||||||
|
assert standardize_name(None) == ("", False)
|
||||||
|
|
||||||
|
def test_idempotent(self):
|
||||||
|
out, _ = standardize_name("JOHN DOE")
|
||||||
|
out2, changed2 = standardize_name(out)
|
||||||
|
assert out2 == out
|
||||||
|
assert changed2 is False
|
||||||
|
|
||||||
|
|
||||||
|
class TestStandardizeAddress:
|
||||||
|
def test_street(self):
|
||||||
|
out, _ = standardize_address("123 Main St")
|
||||||
|
assert out == "123 Main Street"
|
||||||
|
|
||||||
|
def test_avenue_with_period(self):
|
||||||
|
out, _ = standardize_address("456 Oak Ave.")
|
||||||
|
assert out == "456 Oak Avenue"
|
||||||
|
|
||||||
|
def test_apartment(self):
|
||||||
|
out, _ = standardize_address("123 Main St Apt 4")
|
||||||
|
assert out == "123 Main Street Apartment 4"
|
||||||
|
|
||||||
|
def test_direction(self):
|
||||||
|
out, _ = standardize_address("100 N Main St")
|
||||||
|
assert out == "100 North Main Street"
|
||||||
|
|
||||||
|
def test_combined(self):
|
||||||
|
out, _ = standardize_address("789 pine blvd ste 200")
|
||||||
|
assert out == "789 Pine Boulevard Suite 200"
|
||||||
|
|
||||||
|
def test_already_expanded(self):
|
||||||
|
out, changed = standardize_address("123 Main Street")
|
||||||
|
assert out == "123 Main Street"
|
||||||
|
assert changed is False
|
||||||
|
|
||||||
|
def test_empty(self):
|
||||||
|
assert standardize_address("") == ("", False)
|
||||||
|
assert standardize_address(None) == ("", False)
|
||||||
|
|
||||||
|
def test_idempotent(self):
|
||||||
|
out, _ = standardize_address("123 main st apt 4")
|
||||||
|
out2, changed2 = standardize_address(out)
|
||||||
|
assert out2 == out
|
||||||
|
assert changed2 is False
|
||||||
|
|
||||||
|
|
||||||
|
class TestStandardizeBoolean:
|
||||||
|
@pytest.mark.parametrize("inp", ["yes", "Yes", "YES", "y", "Y", "true", "1", "on"])
|
||||||
|
def test_truthy(self, inp):
|
||||||
|
out, changed = standardize_boolean(inp)
|
||||||
|
assert out == "True"
|
||||||
|
assert changed is True
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("inp", ["no", "No", "NO", "n", "N", "false", "0", "off"])
|
||||||
|
def test_falsy(self, inp):
|
||||||
|
out, changed = standardize_boolean(inp)
|
||||||
|
assert out == "False"
|
||||||
|
assert changed is True
|
||||||
|
|
||||||
|
def test_already_canonical(self):
|
||||||
|
out, changed = standardize_boolean("True")
|
||||||
|
assert out == "True"
|
||||||
|
assert changed is False
|
||||||
|
|
||||||
|
def test_python_bool(self):
|
||||||
|
assert standardize_boolean(True) == ("True", True)
|
||||||
|
assert standardize_boolean(False) == ("False", True)
|
||||||
|
|
||||||
|
def test_int_zero_one(self):
|
||||||
|
assert standardize_boolean(1) == ("True", True)
|
||||||
|
assert standardize_boolean(0) == ("False", True)
|
||||||
|
|
||||||
|
def test_yes_no_style(self):
|
||||||
|
assert standardize_boolean("y", style="Yes/No") == ("Yes", True)
|
||||||
|
assert standardize_boolean("0", style="Yes/No") == ("No", True)
|
||||||
|
|
||||||
|
def test_unrecognized_passthrough(self):
|
||||||
|
out, changed = standardize_boolean("maybe")
|
||||||
|
assert (out, changed) == ("maybe", False)
|
||||||
|
|
||||||
|
def test_empty(self):
|
||||||
|
assert standardize_boolean("") == ("", False)
|
||||||
|
assert standardize_boolean(None) == ("", False)
|
||||||
|
|
||||||
|
def test_idempotent(self):
|
||||||
|
out, _ = standardize_boolean("yes")
|
||||||
|
out2, changed2 = standardize_boolean(out)
|
||||||
|
assert out2 == out
|
||||||
|
assert changed2 is False
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# DataFrame entry point
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestStandardizeDataframe:
|
||||||
|
def test_mixed_columns(self):
|
||||||
|
df = pd.DataFrame({
|
||||||
|
"name": ["JOHN SMITH", "alice jones"],
|
||||||
|
"phone": ["(555) 123-4567", "555.987.6543"],
|
||||||
|
"amount": ["$1,234.56", "$50"],
|
||||||
|
"joined": ["01/15/2024", "March 5 2023"],
|
||||||
|
"active": ["yes", "0"],
|
||||||
|
"address": ["123 Main St", "456 Oak Ave"],
|
||||||
|
"skip_me": ["leave", "alone"],
|
||||||
|
})
|
||||||
|
opts = StandardizeOptions(
|
||||||
|
column_types={
|
||||||
|
"name": FieldType.NAME,
|
||||||
|
"phone": FieldType.PHONE,
|
||||||
|
"amount": FieldType.CURRENCY,
|
||||||
|
"joined": FieldType.DATE,
|
||||||
|
"active": FieldType.BOOLEAN,
|
||||||
|
"address": FieldType.ADDRESS,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
result = standardize_dataframe(df, opts)
|
||||||
|
out = result.standardized_df
|
||||||
|
assert out.loc[0, "name"] == "John Smith"
|
||||||
|
assert out.loc[1, "name"] == "Alice Jones"
|
||||||
|
assert out.loc[0, "phone"] == "+15551234567"
|
||||||
|
assert out.loc[1, "phone"] == "+15559876543"
|
||||||
|
assert out.loc[0, "amount"] == "1234.56"
|
||||||
|
assert out.loc[1, "amount"] == "50.00"
|
||||||
|
assert out.loc[0, "joined"] == "2024-01-15"
|
||||||
|
assert out.loc[1, "joined"] == "2023-03-05"
|
||||||
|
assert out.loc[0, "active"] == "True"
|
||||||
|
assert out.loc[1, "active"] == "False"
|
||||||
|
assert out.loc[0, "address"] == "123 Main Street"
|
||||||
|
assert out.loc[1, "address"] == "456 Oak Avenue"
|
||||||
|
# Untouched column passes through verbatim.
|
||||||
|
assert list(out["skip_me"]) == ["leave", "alone"]
|
||||||
|
|
||||||
|
def test_changes_audit(self):
|
||||||
|
df = pd.DataFrame({"d": ["01/15/2024", "2023-03-05"]})
|
||||||
|
opts = StandardizeOptions(column_types={"d": FieldType.DATE})
|
||||||
|
result = standardize_dataframe(df, opts)
|
||||||
|
# Only the first row changed; the second was already canonical.
|
||||||
|
assert result.cells_changed == 1
|
||||||
|
assert len(result.changes) == 1
|
||||||
|
assert result.changes.iloc[0]["row"] == 0
|
||||||
|
assert result.changes.iloc[0]["column"] == "d"
|
||||||
|
assert result.changes.iloc[0]["old"] == "01/15/2024"
|
||||||
|
assert result.changes.iloc[0]["new"] == "2024-01-15"
|
||||||
|
|
||||||
|
def test_unparseable_count(self):
|
||||||
|
df = pd.DataFrame({"d": ["01/15/2024", "not a date", "2024-01-15"]})
|
||||||
|
opts = StandardizeOptions(column_types={"d": FieldType.DATE})
|
||||||
|
result = standardize_dataframe(df, opts)
|
||||||
|
assert result.cells_unparseable == 1
|
||||||
|
assert result.cells_total == 3
|
||||||
|
|
||||||
|
def test_unknown_column_raises(self):
|
||||||
|
df = pd.DataFrame({"a": ["1"]})
|
||||||
|
opts = StandardizeOptions(column_types={"missing": FieldType.DATE})
|
||||||
|
with pytest.raises(ValueError, match="not found"):
|
||||||
|
standardize_dataframe(df, opts)
|
||||||
|
|
||||||
|
def test_input_not_mutated(self):
|
||||||
|
df = pd.DataFrame({"d": ["01/15/2024"]})
|
||||||
|
opts = StandardizeOptions(column_types={"d": FieldType.DATE})
|
||||||
|
standardize_dataframe(df, opts)
|
||||||
|
assert df.loc[0, "d"] == "01/15/2024"
|
||||||
|
|
||||||
|
def test_options_serialization_roundtrip(self, tmp_path):
|
||||||
|
opts = StandardizeOptions(
|
||||||
|
column_types={"a": FieldType.DATE, "b": FieldType.PHONE},
|
||||||
|
date_output_format="%d-%b-%Y",
|
||||||
|
phone_format="NATIONAL",
|
||||||
|
)
|
||||||
|
path = tmp_path / "opts.json"
|
||||||
|
opts.to_file(path)
|
||||||
|
loaded = StandardizeOptions.from_file(path)
|
||||||
|
assert loaded.column_types == {"a": FieldType.DATE, "b": FieldType.PHONE}
|
||||||
|
assert loaded.date_output_format == "%d-%b-%Y"
|
||||||
|
assert loaded.phone_format == "NATIONAL"
|
||||||
|
|
||||||
|
def test_nan_passthrough(self):
|
||||||
|
df = pd.DataFrame({"d": ["01/15/2024", None]})
|
||||||
|
opts = StandardizeOptions(column_types={"d": FieldType.DATE})
|
||||||
|
result = standardize_dataframe(df, opts)
|
||||||
|
assert result.standardized_df.loc[0, "d"] == "2024-01-15"
|
||||||
|
assert result.standardized_df.loc[1, "d"] is None
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Preset bundles
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestPresets:
|
||||||
|
def test_us_default_iso_dates(self):
|
||||||
|
opts = StandardizeOptions.from_preset("us-default")
|
||||||
|
assert opts.date_output_format == "%Y-%m-%d"
|
||||||
|
assert opts.date_order == "MDY"
|
||||||
|
assert opts.phone_format == "E164"
|
||||||
|
assert opts.boolean_style == "True/False"
|
||||||
|
|
||||||
|
def test_european_dmy_comma(self):
|
||||||
|
opts = StandardizeOptions.from_preset("european")
|
||||||
|
assert opts.date_order == "DMY"
|
||||||
|
assert opts.currency_decimal == "comma"
|
||||||
|
assert opts.currency_preserve_code is True
|
||||||
|
|
||||||
|
def test_uk_ddmmyyyy_yes_no(self):
|
||||||
|
opts = StandardizeOptions.from_preset("uk")
|
||||||
|
assert opts.date_output_format == "%d/%m/%Y"
|
||||||
|
assert opts.phone_region == "GB"
|
||||||
|
assert opts.boolean_style == "Yes/No"
|
||||||
|
|
||||||
|
def test_iso_strict_lowercase_bools_no_rounding(self):
|
||||||
|
opts = StandardizeOptions.from_preset("iso-strict")
|
||||||
|
assert opts.boolean_style == "true/false"
|
||||||
|
assert opts.currency_decimals is None
|
||||||
|
assert opts.currency_preserve_code is True
|
||||||
|
|
||||||
|
def test_legacy_us_national_phones(self):
|
||||||
|
opts = StandardizeOptions.from_preset("legacy-us")
|
||||||
|
assert opts.date_output_format == "%m/%d/%Y"
|
||||||
|
assert opts.phone_format == "NATIONAL"
|
||||||
|
assert opts.boolean_style == "Yes/No"
|
||||||
|
|
||||||
|
def test_overrides_layer_on_top(self):
|
||||||
|
opts = StandardizeOptions.from_preset(
|
||||||
|
"uk",
|
||||||
|
column_types={"name": FieldType.NAME},
|
||||||
|
currency_decimals=4,
|
||||||
|
)
|
||||||
|
assert opts.column_types == {"name": FieldType.NAME}
|
||||||
|
assert opts.currency_decimals == 4
|
||||||
|
# UK-specific defaults survive what we didn't override.
|
||||||
|
assert opts.phone_region == "GB"
|
||||||
|
|
||||||
|
def test_unknown_preset_raises(self):
|
||||||
|
with pytest.raises(ValueError, match="Unknown preset"):
|
||||||
|
StandardizeOptions.from_preset("not-a-real-preset")
|
||||||
|
|
||||||
|
def test_all_presets_loadable(self):
|
||||||
|
# Smoke test: every advertised preset constructs cleanly.
|
||||||
|
for name in PRESETS:
|
||||||
|
opts = StandardizeOptions.from_preset(name)
|
||||||
|
assert isinstance(opts, StandardizeOptions)
|
||||||
|
|
||||||
|
def test_preset_drives_dataframe_pipeline(self):
|
||||||
|
df = pd.DataFrame({
|
||||||
|
"joined": ["15/01/2024"],
|
||||||
|
"active": ["yes"],
|
||||||
|
"amount": ["1.234,56 €"],
|
||||||
|
})
|
||||||
|
opts = StandardizeOptions.from_preset(
|
||||||
|
"european",
|
||||||
|
column_types={
|
||||||
|
"joined": FieldType.DATE,
|
||||||
|
"active": FieldType.BOOLEAN,
|
||||||
|
"amount": FieldType.CURRENCY,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
result = standardize_dataframe(df, opts)
|
||||||
|
out = result.standardized_df
|
||||||
|
assert out.loc[0, "joined"] == "2024-01-15" # ISO output for european
|
||||||
|
assert out.loc[0, "active"] == "True"
|
||||||
|
assert out.loc[0, "amount"] == "EUR 1234.56" # preserve_code on
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Currency code detection / preservation
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestCurrencyCodeDetection:
|
||||||
|
@pytest.mark.parametrize("inp,code", [
|
||||||
|
("$1,234.56", "USD"),
|
||||||
|
("€1.234,56", "EUR"),
|
||||||
|
("£99.00", "GBP"),
|
||||||
|
("¥5000", "JPY"),
|
||||||
|
("₹500", "INR"),
|
||||||
|
("USD 1234", "USD"),
|
||||||
|
("1234 EUR", "EUR"),
|
||||||
|
("eur 50", "EUR"),
|
||||||
|
])
|
||||||
|
def test_detects(self, inp, code):
|
||||||
|
assert detect_currency_code(inp) == code
|
||||||
|
|
||||||
|
def test_no_marker_returns_none(self):
|
||||||
|
assert detect_currency_code("1234.56") is None
|
||||||
|
|
||||||
|
def test_non_string_returns_none(self):
|
||||||
|
assert detect_currency_code(None) is None # type: ignore[arg-type]
|
||||||
|
assert detect_currency_code(1234) is None # type: ignore[arg-type]
|
||||||
|
|
||||||
|
|
||||||
|
class TestCurrencyPreserveCode:
|
||||||
|
def test_dollar_preserved(self):
|
||||||
|
out, changed = standardize_currency("$1,234.56", decimals=2, preserve_code=True)
|
||||||
|
assert out == "USD 1234.56"
|
||||||
|
assert changed is True
|
||||||
|
|
||||||
|
def test_euro_preserved_comma_decimal(self):
|
||||||
|
out, _ = standardize_currency(
|
||||||
|
"1.234,56 €", decimal="comma", decimals=2, preserve_code=True,
|
||||||
|
)
|
||||||
|
assert out == "EUR 1234.56"
|
||||||
|
|
||||||
|
def test_iso_code_input_preserved(self):
|
||||||
|
out, _ = standardize_currency("USD 1234.56", decimals=2, preserve_code=True)
|
||||||
|
assert out == "USD 1234.56"
|
||||||
|
|
||||||
|
def test_no_marker_no_prefix(self):
|
||||||
|
out, _ = standardize_currency("1234.56", decimals=2, preserve_code=True)
|
||||||
|
assert out == "1234.56"
|
||||||
|
|
||||||
|
def test_off_by_default(self):
|
||||||
|
out, _ = standardize_currency("$1,234.56", decimals=2)
|
||||||
|
assert out == "1234.56"
|
||||||
|
|
||||||
|
def test_pipeline_preserve_code(self):
|
||||||
|
df = pd.DataFrame({"price": ["$50.00", "€30,00", "100", "USD 12.34"]})
|
||||||
|
opts = StandardizeOptions(
|
||||||
|
column_types={"price": FieldType.CURRENCY},
|
||||||
|
currency_decimals=2,
|
||||||
|
currency_preserve_code=True,
|
||||||
|
currency_decimal="dot", # mixed input — euro will need its own
|
||||||
|
)
|
||||||
|
# Note: comma-decimal euro won't parse under dot mode; treat that
|
||||||
|
# as a known limitation — this test exercises the dot-input path.
|
||||||
|
result = standardize_dataframe(df, opts)
|
||||||
|
out = result.standardized_df
|
||||||
|
assert out.loc[0, "price"] == "USD 50.00"
|
||||||
|
assert out.loc[2, "price"] == "100.00"
|
||||||
|
assert out.loc[3, "price"] == "USD 12.34"
|
||||||
|
|
||||||
|
def test_canonical_check_recognizes_code_prefix(self):
|
||||||
|
# "USD 50.00" should pass through unchanged when preserve_code is on
|
||||||
|
# — and NOT count as unparseable.
|
||||||
|
df = pd.DataFrame({"price": ["USD 50.00", "garbage"]})
|
||||||
|
opts = StandardizeOptions(
|
||||||
|
column_types={"price": FieldType.CURRENCY},
|
||||||
|
currency_decimals=2,
|
||||||
|
currency_preserve_code=True,
|
||||||
|
)
|
||||||
|
result = standardize_dataframe(df, opts)
|
||||||
|
assert result.cells_changed == 0
|
||||||
|
# Only "garbage" counts as unparseable.
|
||||||
|
assert result.cells_unparseable == 1
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# User-editable abbreviations
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestExtraAbbreviations:
|
||||||
|
def test_extra_expansion(self):
|
||||||
|
out, _ = standardize_address(
|
||||||
|
"Bahnhofstrasse 12",
|
||||||
|
extra_abbreviations={"strasse": "Straße"},
|
||||||
|
)
|
||||||
|
# smart_title_case will Title-case the result; "Bahnhofstrasse" is
|
||||||
|
# already a single token (no embedded space) so it doesn't hit the
|
||||||
|
# abbreviation lookup. Use a separated form for the realistic case.
|
||||||
|
assert "Bahnhofstrasse" in out # not split → not expanded
|
||||||
|
|
||||||
|
def test_extra_expansion_separated_token(self):
|
||||||
|
out, _ = standardize_address(
|
||||||
|
"Haupt strasse 12",
|
||||||
|
extra_abbreviations={"strasse": "Straße"},
|
||||||
|
)
|
||||||
|
assert "Straße" in out
|
||||||
|
|
||||||
|
def test_override_existing_entry(self):
|
||||||
|
# Override "ave" to emit Spanish-language "Avenida".
|
||||||
|
out, _ = standardize_address(
|
||||||
|
"456 Oak Ave",
|
||||||
|
extra_abbreviations={"ave": "Avenida"},
|
||||||
|
)
|
||||||
|
assert "Avenida" in out
|
||||||
|
assert "Avenue" not in out
|
||||||
|
|
||||||
|
def test_period_form_works(self):
|
||||||
|
# Lookup is casefold + period-stripped, so ``Ave.`` still matches.
|
||||||
|
out, _ = standardize_address(
|
||||||
|
"456 Oak Ave.",
|
||||||
|
extra_abbreviations={"ave": "Avenida"},
|
||||||
|
)
|
||||||
|
assert "Avenida" in out
|
||||||
|
|
||||||
|
def test_empty_value_skipped(self):
|
||||||
|
# Empty values in the user table don't blow up; they're ignored.
|
||||||
|
out, _ = standardize_address(
|
||||||
|
"456 Oak Ave",
|
||||||
|
extra_abbreviations={"ave": "", " ": "Drive"},
|
||||||
|
)
|
||||||
|
# Built-in expansion still applies.
|
||||||
|
assert "Avenue" in out
|
||||||
|
|
||||||
|
def test_no_extras_unchanged_behavior(self):
|
||||||
|
out_a, _ = standardize_address("123 Main St")
|
||||||
|
out_b, _ = standardize_address("123 Main St", extra_abbreviations={})
|
||||||
|
out_c, _ = standardize_address("123 Main St", extra_abbreviations=None)
|
||||||
|
assert out_a == out_b == out_c == "123 Main Street"
|
||||||
|
|
||||||
|
def test_pipeline_uses_extras(self):
|
||||||
|
df = pd.DataFrame({"addr": ["456 Oak Ave"]})
|
||||||
|
opts = StandardizeOptions(
|
||||||
|
column_types={"addr": FieldType.ADDRESS},
|
||||||
|
extra_abbreviations={"ave": "Avenida"},
|
||||||
|
)
|
||||||
|
result = standardize_dataframe(df, opts)
|
||||||
|
assert "Avenida" in result.standardized_df.loc[0, "addr"]
|
||||||
|
|
||||||
|
def test_serialization_roundtrip_with_extras(self, tmp_path):
|
||||||
|
opts = StandardizeOptions(
|
||||||
|
column_types={"addr": FieldType.ADDRESS},
|
||||||
|
extra_abbreviations={"strasse": "Straße", "platz": "Platz"},
|
||||||
|
currency_preserve_code=True,
|
||||||
|
)
|
||||||
|
path = tmp_path / "opts.json"
|
||||||
|
opts.to_file(path)
|
||||||
|
loaded = StandardizeOptions.from_file(path)
|
||||||
|
assert loaded.extra_abbreviations == {"strasse": "Straße", "platz": "Platz"}
|
||||||
|
assert loaded.currency_preserve_code is True
|
||||||
573
tests/test_format_standardize_corpus.py
Normal file
573
tests/test_format_standardize_corpus.py
Normal file
@@ -0,0 +1,573 @@
|
|||||||
|
"""Corpus-driven tests for ``src.core.format_standardize``.
|
||||||
|
|
||||||
|
Drives every row of the FORMATS test corpus
|
||||||
|
(``test-cases/format-cleaner-corpus/*.csv``) through the per-cell
|
||||||
|
standardizers and asserts the canonical output the corpus expects.
|
||||||
|
|
||||||
|
The corpus itself (``FORMATS-CASES.md`` in the same directory)
|
||||||
|
documents per-domain policy decisions; the per-case ``id`` strings
|
||||||
|
below (FD01, FP14, FA09, …) match its row keys exactly.
|
||||||
|
|
||||||
|
Two sentinels are used in the per-domain expected dicts:
|
||||||
|
|
||||||
|
- A literal string is the corpus's expected canonical output.
|
||||||
|
- ``PASSTHROUGH`` means "corpus accepts no transformation" — usually
|
||||||
|
empty, whitespace-only, or already-clean input.
|
||||||
|
|
||||||
|
A handful of corpus rows are still ``xfail`` because closing them
|
||||||
|
needs heavier machinery (Excel serial parsing, Unix timestamps,
|
||||||
|
non-English month dictionaries, IDN / non-ASCII email validation).
|
||||||
|
Each such marker carries a one-line reason.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import csv
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pandas as pd
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from src.core.format_standardize import (
|
||||||
|
FieldType,
|
||||||
|
StandardizeOptions,
|
||||||
|
standardize_address,
|
||||||
|
standardize_currency,
|
||||||
|
standardize_dataframe,
|
||||||
|
standardize_date,
|
||||||
|
standardize_email,
|
||||||
|
standardize_name,
|
||||||
|
standardize_phone,
|
||||||
|
)
|
||||||
|
|
||||||
|
CORPUS = Path(__file__).resolve().parents[1] / "test-cases" / "format-cleaner-corpus"
|
||||||
|
|
||||||
|
PASSTHROUGH = object() # sentinel: assert the function returned input unchanged
|
||||||
|
|
||||||
|
|
||||||
|
def _load(filename: str) -> list[dict[str, str]]:
|
||||||
|
with (CORPUS / filename).open(newline="") as f:
|
||||||
|
return list(csv.DictReader(f))
|
||||||
|
|
||||||
|
|
||||||
|
def _params(fixture: str, expected: dict[str, object], xfails: dict[str, str]):
|
||||||
|
"""Build pytest.param entries for every row in *fixture*.
|
||||||
|
|
||||||
|
Rows in *xfails* are wrapped in a non-strict xfail with the given
|
||||||
|
reason, so improvements that close the gap surface as xpass and the
|
||||||
|
suite stays green either way.
|
||||||
|
"""
|
||||||
|
rows = _load(fixture)
|
||||||
|
out = []
|
||||||
|
for row in rows:
|
||||||
|
cid = row["case_id"]
|
||||||
|
want = expected.get(cid, PASSTHROUGH)
|
||||||
|
marks = []
|
||||||
|
if cid in xfails:
|
||||||
|
marks.append(pytest.mark.xfail(reason=xfails[cid], strict=False))
|
||||||
|
out.append(pytest.param(row["input"], want, id=cid, marks=marks))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _assert(got: str, want: object, original: str) -> None:
|
||||||
|
if want is PASSTHROUGH:
|
||||||
|
assert got == original, f"expected pass-through, got {got!r}"
|
||||||
|
else:
|
||||||
|
assert got == want
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Dates — 24_format_dates.csv
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
_DATE_EXPECTED_MDY: dict[str, object] = {
|
||||||
|
# iso baseline + datetime variants → ISO date
|
||||||
|
"FD01": "2024-01-15",
|
||||||
|
"FD02": "2024-01-15",
|
||||||
|
"FD03": "2024-01-15",
|
||||||
|
"FD04": "2024-01-15",
|
||||||
|
"FD05": "2024-01-15",
|
||||||
|
"FD06": "2024-01-15",
|
||||||
|
# US M/D/Y variants
|
||||||
|
"FD07": "2024-01-15",
|
||||||
|
"FD08": "2024-01-15",
|
||||||
|
"FD09": "2024-01-05",
|
||||||
|
"FD10": "2024-05-30",
|
||||||
|
# longform month names
|
||||||
|
"FD16": "2024-01-15",
|
||||||
|
"FD17": "2024-01-15",
|
||||||
|
"FD18": "2024-01-15",
|
||||||
|
"FD19": "2024-01-15",
|
||||||
|
"FD20": "2024-01-15", # weekday-prefixed
|
||||||
|
"FD21": "2024-01-15",
|
||||||
|
# FD11-FD15 — DMY-shaped EU dates in MDY default mode; the DMY
|
||||||
|
# rerun below covers the actual parse path. Under MDY they pass
|
||||||
|
# through unchanged. (Listed explicitly so a future MDY-aware
|
||||||
|
# locale auto-detect can replace these expectations with the
|
||||||
|
# correct ISO output.)
|
||||||
|
"FD11": PASSTHROUGH,
|
||||||
|
"FD12": PASSTHROUGH,
|
||||||
|
"FD13": PASSTHROUGH,
|
||||||
|
"FD14": PASSTHROUGH,
|
||||||
|
"FD15": PASSTHROUGH,
|
||||||
|
# excel serial → 2024-01-15 (xfail — not implemented)
|
||||||
|
"FD22": "2024-01-15",
|
||||||
|
"FD23": "2024-01-15",
|
||||||
|
# unix timestamp seconds / millis → 2024-01-15 (xfail)
|
||||||
|
"FD24": "2024-01-15",
|
||||||
|
"FD25": "2024-01-15",
|
||||||
|
# partial precision — corpus preserves it
|
||||||
|
"FD26": "2024-01",
|
||||||
|
"FD27": "2024-01", # xfail — text precision
|
||||||
|
"FD28": "2024-Q1", # xfail — quarter
|
||||||
|
"FD29": "2024",
|
||||||
|
# 2-digit year cutoff (per docs: 1969 wins over 2069)
|
||||||
|
"FD30": "1969-01-15",
|
||||||
|
# leap day valid
|
||||||
|
"FD31": "2024-02-29",
|
||||||
|
# invalid dates → corpus expects error sentinel
|
||||||
|
"FD32": "<error: invalid leap day>",
|
||||||
|
"FD33": "<error: Excel 1900 leap year bug>",
|
||||||
|
"FD34": "<error: invalid month>",
|
||||||
|
"FD35": "<error: invalid day>",
|
||||||
|
# buried-date extraction
|
||||||
|
"FD36": "2024-01-15",
|
||||||
|
"FD37": "2024-01-15",
|
||||||
|
# garbage → pass through (corpus 0.3 boundary table)
|
||||||
|
# FD38/39/40 → PASSTHROUGH default
|
||||||
|
# locale-specific month names (xfail — not shipped)
|
||||||
|
"FD41": "2024-01-15",
|
||||||
|
"FD42": "2024-01-15",
|
||||||
|
# timezone — corpus 3.3 says fixed-offset only
|
||||||
|
"FD43": "2024-01-15",
|
||||||
|
"FD44": "2024-03-10",
|
||||||
|
# already-clean idempotency
|
||||||
|
"FD45": "2024-01-15",
|
||||||
|
}
|
||||||
|
|
||||||
|
_DATE_XFAILS_MDY: dict[str, str] = {}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"inp,want",
|
||||||
|
_params("24_format_dates.csv", _DATE_EXPECTED_MDY, _DATE_XFAILS_MDY),
|
||||||
|
)
|
||||||
|
def test_corpus_dates_mdy(inp, want):
|
||||||
|
got, _ = standardize_date(
|
||||||
|
inp, error_policy="sentinel", month_locales=["en", "fr", "de"],
|
||||||
|
)
|
||||||
|
_assert(got, want, inp)
|
||||||
|
|
||||||
|
|
||||||
|
# DMY locale rerun for the EU rows that need it.
|
||||||
|
_DATE_EXPECTED_DMY: dict[str, str] = {
|
||||||
|
"FD11": "2024-01-15",
|
||||||
|
"FD12": "2024-01-15",
|
||||||
|
"FD13": "2024-01-15",
|
||||||
|
"FD14": "2024-05-30",
|
||||||
|
"FD15": "2024-01-15",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"inp,want",
|
||||||
|
[
|
||||||
|
pytest.param(
|
||||||
|
_load("24_format_dates.csv")[i - 1]["input"],
|
||||||
|
_DATE_EXPECTED_DMY[f"FD{i:02d}"],
|
||||||
|
id=f"FD{i:02d}-dmy",
|
||||||
|
)
|
||||||
|
for i in range(11, 16)
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_corpus_dates_dmy(inp, want):
|
||||||
|
got, _ = standardize_date(inp, date_order="DMY")
|
||||||
|
assert got == want
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Phones — 25_format_phones.csv
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
_PHONE_EXPECTED: dict[str, object] = {
|
||||||
|
"FP01": "+15551234567",
|
||||||
|
"FP02": "+15551234567",
|
||||||
|
"FP03": "+15551234567",
|
||||||
|
"FP04": "+15551234567",
|
||||||
|
"FP05": "+15551234567",
|
||||||
|
"FP06": "+15551234567",
|
||||||
|
"FP07": "+15551234567",
|
||||||
|
"FP08": "+15551234567",
|
||||||
|
"FP09": "+15551234567;ext=123",
|
||||||
|
"FP10": "+15551234567;ext=123",
|
||||||
|
"FP11": "+15551234567;ext=123",
|
||||||
|
# vanity numbers
|
||||||
|
"FP12": "+18003569377",
|
||||||
|
"FP13": "+15552255669",
|
||||||
|
# international (intl row FP15 needs --default-country=GB; covered separately)
|
||||||
|
"FP14": "+442079460958",
|
||||||
|
"FP16": "+493012345678",
|
||||||
|
"FP17": "+33123456789",
|
||||||
|
"FP18": "+81312345678",
|
||||||
|
"FP19": "+61212345678",
|
||||||
|
"FP20": "+15551234567",
|
||||||
|
# placeholders/junk → corpus says error
|
||||||
|
"FP21": "<error: insufficient digits>",
|
||||||
|
"FP22": "<error: too many digits>",
|
||||||
|
"FP23": "<error: placeholder number>",
|
||||||
|
"FP24": "<error: placeholder number>",
|
||||||
|
"FP25": "<error: multiple numbers in cell>",
|
||||||
|
# NBSP / smart-quote contamination — defensive cleanup acceptable
|
||||||
|
"FP26": "+15551234567",
|
||||||
|
"FP27": "+15551234567",
|
||||||
|
"FP28": "+15551234567",
|
||||||
|
# FP29 empty → pass-through
|
||||||
|
"FP30": "<error: not a phone number>",
|
||||||
|
"FP31": "<error: smart-quote contamination>",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"inp,want",
|
||||||
|
_params("25_format_phones.csv", _PHONE_EXPECTED, {}),
|
||||||
|
)
|
||||||
|
def test_corpus_phones(inp, want):
|
||||||
|
got, _ = standardize_phone(inp, error_policy="sentinel")
|
||||||
|
_assert(got, want, inp)
|
||||||
|
|
||||||
|
|
||||||
|
def test_corpus_phones_uk_domestic_with_gb_region():
|
||||||
|
# FP15 — UK trunk-prefixed "020 7946 0958" only resolves with
|
||||||
|
# default_region="GB". Verifies the cleaner's intl path works.
|
||||||
|
got, _ = standardize_phone("020 7946 0958", default_region="GB")
|
||||||
|
assert got == "+442079460958"
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Emails — 26_format_emails.csv
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
_EMAIL_EXPECTED: dict[str, object] = {
|
||||||
|
"FE01": "alice@example.com",
|
||||||
|
"FE02": "alice@example.com",
|
||||||
|
"FE03": "alice@example.com",
|
||||||
|
"FE04": "alice@example.com",
|
||||||
|
"FE05": "alice@example.com",
|
||||||
|
"FE06": "alice@example.com",
|
||||||
|
"FE07": "alice@example.com",
|
||||||
|
"FE08": "alice@example.com",
|
||||||
|
"FE09": "alice@example.com",
|
||||||
|
"FE10": "a.l.i.c.e@gmail.com", # default: don't touch dots
|
||||||
|
"FE11": "alice+newsletter@gmail.com", # default: don't touch +tag
|
||||||
|
"FE12": "a.l.i.c.e+work@gmail.com",
|
||||||
|
"FE13": "a.l.i.c.e@example.com", # never touch non-Gmail
|
||||||
|
"FE14": "alice+newsletter@example.com",
|
||||||
|
"FE15": "alice@münchen.de",
|
||||||
|
"FE16": "アリス@example.jp",
|
||||||
|
"FE17": "alice@example.com",
|
||||||
|
"FE18": "alice@example.com",
|
||||||
|
"FE19": "alice@example.com",
|
||||||
|
"FE20": "alice@example.com",
|
||||||
|
"FE21": "alice@example.com",
|
||||||
|
"FE22": "<error: missing @>",
|
||||||
|
"FE23": "<error: double @>",
|
||||||
|
"FE24": "<error: multiple @>",
|
||||||
|
"FE25": "<error: internal whitespace>",
|
||||||
|
"FE26": "<error: no TLD>",
|
||||||
|
"FE27": "<error: multiple emails>",
|
||||||
|
"FE28": "<error: multiple emails>",
|
||||||
|
# FE29 / FE30 empty / whitespace → PASSTHROUGH
|
||||||
|
"FE31": "alice@example.com",
|
||||||
|
}
|
||||||
|
|
||||||
|
_EMAIL_XFAILS: dict[str, str] = {}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"inp,want",
|
||||||
|
_params("26_format_emails.csv", _EMAIL_EXPECTED, _EMAIL_XFAILS),
|
||||||
|
)
|
||||||
|
def test_corpus_emails(inp, want):
|
||||||
|
got, _ = standardize_email(inp, error_policy="sentinel")
|
||||||
|
_assert(got, want, inp)
|
||||||
|
|
||||||
|
|
||||||
|
_EMAIL_GMAIL_CANONICAL: dict[str, str] = {
|
||||||
|
"FE10": "alice@gmail.com",
|
||||||
|
"FE11": "alice@gmail.com",
|
||||||
|
"FE12": "alice@gmail.com",
|
||||||
|
"FE13": "a.l.i.c.e@example.com", # negative test: don't touch non-Gmail
|
||||||
|
"FE14": "alice+newsletter@example.com", # negative test
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("inp,want", [
|
||||||
|
pytest.param(
|
||||||
|
next(r for r in _load("26_format_emails.csv") if r["case_id"] == cid)["input"],
|
||||||
|
want, id=f"{cid}-gmail-canonical",
|
||||||
|
)
|
||||||
|
for cid, want in _EMAIL_GMAIL_CANONICAL.items()
|
||||||
|
])
|
||||||
|
def test_corpus_emails_gmail_canonical(inp, want):
|
||||||
|
got, _ = standardize_email(inp, gmail_canonical=True)
|
||||||
|
assert got == want
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Addresses — 27_format_addresses.csv
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
_ADDRESS_EXPECTED: dict[str, str] = {
|
||||||
|
"FA01": "123 Main St, New York, NY 10001",
|
||||||
|
"FA02": "123 Main St, New York, NY 10001",
|
||||||
|
"FA03": "123 Main St, New York, NY 10001",
|
||||||
|
"FA04": "123 Main St, New York, NY 10001",
|
||||||
|
"FA05": "123 Main St, New York, NY 10001",
|
||||||
|
"FA06": "456 Park Ave, New York, NY 10001",
|
||||||
|
"FA07": "789 Sunset Blvd, Los Angeles, CA 90028",
|
||||||
|
"FA08": "123 Main St, New York, NY 10001",
|
||||||
|
"FA09": "123 N Main St, City, ST 12345",
|
||||||
|
"FA10": "123 N Main St, City, ST 12345",
|
||||||
|
"FA11": "123 NE Main St, City, ST 12345",
|
||||||
|
"FA12": "123 Main St, Apt 4B, City, ST 12345",
|
||||||
|
"FA13": "123 Main St, # 4B, City, ST 12345",
|
||||||
|
"FA14": "123 Main St, Ste 200, City, ST 12345",
|
||||||
|
"FA15": "123 Main St, New York, NY 10001",
|
||||||
|
"FA16": "123 Main St, New York, NY 10001",
|
||||||
|
"FA17": "123 Main St, New York, NY 10001-1234",
|
||||||
|
"FA18": "123 Main St, Boston, MA 02101",
|
||||||
|
"FA19": "123 Main St, Apt 4B, New York, NY 10001",
|
||||||
|
"FA20": "PO Box 123, City, ST 12345",
|
||||||
|
"FA21": "PO Box 123, City, ST 12345",
|
||||||
|
"FA22": "PO Box 123, City, ST 12345",
|
||||||
|
"FA23": "123A Main St, City, ST 12345",
|
||||||
|
"FA24": "123-1 Main St, City, ST 12345",
|
||||||
|
"FA25": "123 1/2 Main St, City, ST 12345",
|
||||||
|
"FA26": "10 Downing Street, London, SW1A 2AA",
|
||||||
|
"FA27": "1 Yonge St, Toronto, ON M5E 1W7",
|
||||||
|
"FA28": "100-0001, Tokyo, Chiyoda, Marunouchi 1-1",
|
||||||
|
"FA31": "123 Main St, New York, NY 10001",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"inp,want",
|
||||||
|
_params("27_format_addresses.csv", _ADDRESS_EXPECTED, {}),
|
||||||
|
)
|
||||||
|
def test_corpus_addresses(inp, want):
|
||||||
|
got, _ = standardize_address(inp, expand=False)
|
||||||
|
_assert(got, want, inp)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Names — 28_format_names.csv
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
_NAME_EXPECTED: dict[str, object] = {
|
||||||
|
"FN01": "Alice Smith",
|
||||||
|
"FN02": "Alice Smith",
|
||||||
|
"FN03": "Alice Smith",
|
||||||
|
"FN04": "aLiCe SmItH", # corpus 7.3 conservative: preserve mixed
|
||||||
|
"FN05": "McDonald",
|
||||||
|
"FN06": "McDonald",
|
||||||
|
"FN07": "MacDonald",
|
||||||
|
"FN08": "McTaggart",
|
||||||
|
"FN09": "O'Connor",
|
||||||
|
"FN10": "O'Connor",
|
||||||
|
"FN11": "O'Brien",
|
||||||
|
"FN12": "Mary-Jane Smith",
|
||||||
|
"FN13": "Smith-Jones",
|
||||||
|
"FN14": "von Trapp",
|
||||||
|
"FN15": "Vincent van Gogh",
|
||||||
|
"FN16": "Charles de Gaulle",
|
||||||
|
"FN17": "Leonardo da Vinci",
|
||||||
|
"FN18": "Mr John Smith", # corpus 7.3: drop title period
|
||||||
|
"FN19": "Dr Jane Doe",
|
||||||
|
"FN20": "Prof Alice Williams",
|
||||||
|
"FN21": "John Smith Jr",
|
||||||
|
"FN22": "John Smith III",
|
||||||
|
"FN23": "Jane Doe PhD",
|
||||||
|
"FN24": "John Smith", # comma-format reversed
|
||||||
|
"FN25": "John Smith",
|
||||||
|
"FN26": "John Andrew Smith",
|
||||||
|
"FN27": "John A Smith", # corpus 7.3: drop initial period
|
||||||
|
"FN28": "J.K. Rowling",
|
||||||
|
"FN29": "김철수",
|
||||||
|
"FN30": "田中太郎",
|
||||||
|
"FN31": "Иван Иванов",
|
||||||
|
"FN32": "Madonna",
|
||||||
|
# FN33 / FN34 → PASSTHROUGH default
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"inp,want",
|
||||||
|
_params("28_format_names.csv", _NAME_EXPECTED, {}),
|
||||||
|
)
|
||||||
|
def test_corpus_names(inp, want):
|
||||||
|
# FN04 needs conservative=True; the rest use default (aggressive).
|
||||||
|
conservative = inp == "aLiCe SmItH"
|
||||||
|
got, _ = standardize_name(inp, conservative=conservative)
|
||||||
|
_assert(got, want, inp)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Currencies — 29_format_currencies.csv
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
_CURRENCY_EXPECTED: dict[str, object] = {
|
||||||
|
"FC01": "1234.56",
|
||||||
|
"FC02": "1234.56",
|
||||||
|
"FC03": "1234.56",
|
||||||
|
"FC04": "1234.56",
|
||||||
|
"FC05": "1234.56",
|
||||||
|
"FC06": "1234.56",
|
||||||
|
"FC07": "1234.56",
|
||||||
|
"FC08": "1234.56",
|
||||||
|
"FC09": "1234.56",
|
||||||
|
"FC10": "1234.56",
|
||||||
|
"FC11": "1234.56",
|
||||||
|
"FC12": "1234.56",
|
||||||
|
"FC13": "1234",
|
||||||
|
"FC14": "123456.78",
|
||||||
|
"FC15": "-100",
|
||||||
|
"FC16": "-100",
|
||||||
|
"FC17": "-100",
|
||||||
|
"FC18": "0",
|
||||||
|
"FC19": "1500000",
|
||||||
|
"FC20": "<error: percentage not currency>",
|
||||||
|
"FC21": "<error: range not normalizable>",
|
||||||
|
"FC22": "<error: word value>",
|
||||||
|
"FC23": "<error: word value>",
|
||||||
|
# FC24 empty → PASSTHROUGH
|
||||||
|
"FC25": "1234.56",
|
||||||
|
"FC26": "1234",
|
||||||
|
"FC27": "<error: ambiguous separator, set --currency-locale>",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"inp,want",
|
||||||
|
_params("29_format_currencies.csv", _CURRENCY_EXPECTED, {}),
|
||||||
|
)
|
||||||
|
def test_corpus_currencies(inp, want):
|
||||||
|
got, _ = standardize_currency(inp, error_policy="sentinel")
|
||||||
|
_assert(got, want, inp)
|
||||||
|
|
||||||
|
|
||||||
|
def test_corpus_currencies_eu_with_comma_decimal():
|
||||||
|
# FC08, FC10 also parse correctly under decimal="comma".
|
||||||
|
got, _ = standardize_currency("€1.234,56", decimal="comma")
|
||||||
|
assert got == "1234.56"
|
||||||
|
got, _ = standardize_currency("1.234,56 EUR", decimal="comma")
|
||||||
|
assert got == "1234.56"
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Integration — 30_format_integration.csv
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _integration_opts(**overrides) -> StandardizeOptions:
|
||||||
|
"""Standardize options matching corpus defaults for the integration row."""
|
||||||
|
base = StandardizeOptions(
|
||||||
|
column_types={
|
||||||
|
"name": FieldType.NAME,
|
||||||
|
"email": FieldType.EMAIL,
|
||||||
|
"phone": FieldType.PHONE,
|
||||||
|
"date": FieldType.DATE,
|
||||||
|
"amount": FieldType.CURRENCY,
|
||||||
|
"address": FieldType.ADDRESS,
|
||||||
|
},
|
||||||
|
currency_decimals=None,
|
||||||
|
address_expand=False,
|
||||||
|
date_error_policy="passthrough",
|
||||||
|
phone_error_policy="passthrough",
|
||||||
|
)
|
||||||
|
for k, v in overrides.items():
|
||||||
|
setattr(base, k, v)
|
||||||
|
return base
|
||||||
|
|
||||||
|
|
||||||
|
def test_corpus_integration_pipeline_preserves_schema():
|
||||||
|
df = pd.read_csv(CORPUS / "30_format_integration.csv",
|
||||||
|
dtype=str, keep_default_na=False)
|
||||||
|
result = standardize_dataframe(df, _integration_opts())
|
||||||
|
out = result.standardized_df
|
||||||
|
|
||||||
|
# Schema preservation (corpus § 0.2): no rows or columns added,
|
||||||
|
# column order intact.
|
||||||
|
assert list(out.columns) == list(df.columns)
|
||||||
|
assert len(out) == len(df)
|
||||||
|
|
||||||
|
|
||||||
|
def test_corpus_integration_FI01_messy_record():
|
||||||
|
# Row 0 = FI01: standard messy-but-cleanable record.
|
||||||
|
df = pd.read_csv(CORPUS / "30_format_integration.csv",
|
||||||
|
dtype=str, keep_default_na=False)
|
||||||
|
result = standardize_dataframe(df, _integration_opts())
|
||||||
|
row = result.standardized_df.iloc[0]
|
||||||
|
assert row["name"] == "Alice Smith"
|
||||||
|
assert row["email"] == "alice@example.com"
|
||||||
|
assert row["phone"] == "+15551234567"
|
||||||
|
assert row["date"] == "2024-01-15"
|
||||||
|
assert row["amount"] == "1234.56"
|
||||||
|
assert row["address"] == "123 Main St, New York, NY 10001"
|
||||||
|
|
||||||
|
|
||||||
|
def test_corpus_integration_FI04_all_empty_passthrough():
|
||||||
|
# Row 3 = FI04: all empty cells, must pass through without errors.
|
||||||
|
df = pd.read_csv(CORPUS / "30_format_integration.csv",
|
||||||
|
dtype=str, keep_default_na=False)
|
||||||
|
result = standardize_dataframe(df, _integration_opts())
|
||||||
|
row = result.standardized_df.iloc[3]
|
||||||
|
for col in ("name", "email", "phone", "date", "amount", "address"):
|
||||||
|
assert row[col] == "", f"FI04.{col} expected empty, got {row[col]!r}"
|
||||||
|
|
||||||
|
|
||||||
|
def test_corpus_integration_FI05_idempotent_on_clean_input():
|
||||||
|
# Row 4 = FI05: already-clean record. Every column should round-trip
|
||||||
|
# unchanged.
|
||||||
|
df = pd.read_csv(CORPUS / "30_format_integration.csv",
|
||||||
|
dtype=str, keep_default_na=False)
|
||||||
|
result = standardize_dataframe(df, _integration_opts())
|
||||||
|
row = result.standardized_df.iloc[4]
|
||||||
|
original = df.iloc[4]
|
||||||
|
for col in ("name", "email", "phone", "date", "amount", "address"):
|
||||||
|
assert row[col] == original[col], (
|
||||||
|
f"FI05.{col} non-idempotent: {original[col]!r} -> {row[col]!r}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Idempotency property
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
#
|
||||||
|
# Every per-cell standardizer must satisfy ``f(f(x)) == f(x)`` (corpus
|
||||||
|
# § 1, "Idempotency requirement"). We exercise it across every corpus
|
||||||
|
# input under the same flag set the per-domain tests use.
|
||||||
|
|
||||||
|
def _idempotency_runner(fn, fixture, **kwargs):
|
||||||
|
failures = []
|
||||||
|
for row in _load(fixture):
|
||||||
|
once, _ = fn(row["input"], **kwargs)
|
||||||
|
twice, _ = fn(once, **kwargs)
|
||||||
|
if once != twice:
|
||||||
|
failures.append((row["case_id"], row["input"], once, twice))
|
||||||
|
return failures
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("fn,fixture,kwargs", [
|
||||||
|
(standardize_date, "24_format_dates.csv", {}),
|
||||||
|
(standardize_phone, "25_format_phones.csv", {}),
|
||||||
|
(standardize_address, "27_format_addresses.csv", {"expand": False}),
|
||||||
|
(standardize_name, "28_format_names.csv", {}),
|
||||||
|
(standardize_currency, "29_format_currencies.csv",{}),
|
||||||
|
(standardize_email, "26_format_emails.csv", {}),
|
||||||
|
])
|
||||||
|
def test_corpus_idempotency(fn, fixture, kwargs):
|
||||||
|
failures = _idempotency_runner(fn, fixture, **kwargs)
|
||||||
|
assert not failures, (
|
||||||
|
f"non-idempotent transformations in {fixture}:\n"
|
||||||
|
+ "\n".join(f" {cid}: {inp!r} -> {once!r} -> {twice!r}"
|
||||||
|
for cid, inp, once, twice in failures)
|
||||||
|
)
|
||||||
Reference in New Issue
Block a user