Michael
f2fdc10af7
Convert single-page deduplicator into a multi-page suite. Home page shows tool card grid. Deduplicator extracted to its own page (fully working). 8 stub pages added for Text Cleaner, Format Standardizer, Missing Values, Column Mapper, Outlier Detector, Multi-File Merger, Validator & Reporter, and Pipeline Runner — each with functional file upload and coming-soon UI. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
356 lines
14 KiB
Python
356 lines
14 KiB
Python
"""DataTools Deduplicator — full working tool page."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
import pandas as pd
|
|
import streamlit as st
|
|
|
|
# Ensure project root is on sys.path so `src.core` imports work
|
|
_project_root = Path(__file__).resolve().parent.parent.parent.parent
|
|
if str(_project_root) not in sys.path:
|
|
sys.path.insert(0, str(_project_root))
|
|
|
|
from src.core.dedup import deduplicate, DeduplicationResult
|
|
from src.core.io import read_file, list_sheets, detect_encoding, detect_delimiter
|
|
from src.gui.components import (
|
|
apply_review_decisions,
|
|
config_panel,
|
|
match_group_card,
|
|
results_summary,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Session state defaults
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_DEFAULTS = {
|
|
"df": None,
|
|
"result": None,
|
|
"review_decisions": {},
|
|
"config": None,
|
|
"file_name": "",
|
|
"sheet_names": [],
|
|
"detected_delimiter": ",",
|
|
}
|
|
for key, default in _DEFAULTS.items():
|
|
if key not in st.session_state:
|
|
st.session_state[key] = default
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Header
|
|
# ---------------------------------------------------------------------------
|
|
|
|
st.title("🔍 Deduplicator")
|
|
st.caption("Find and remove duplicate rows in CSV, delimited text, and Excel files.")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# File upload
|
|
# ---------------------------------------------------------------------------
|
|
|
|
uploaded = st.file_uploader(
|
|
"Upload CSV or Excel file",
|
|
type=["csv", "tsv", "xlsx", "xls"],
|
|
help="Supports CSV, TSV, and Excel files. Encoding and delimiters are auto-detected.",
|
|
key="dedup_file_upload",
|
|
)
|
|
|
|
if uploaded is not None:
|
|
# Detect if file changed
|
|
if uploaded.name != st.session_state["file_name"]:
|
|
st.session_state["file_name"] = uploaded.name
|
|
st.session_state["result"] = None
|
|
st.session_state["review_decisions"] = {}
|
|
|
|
# Read the file
|
|
try:
|
|
suffix = Path(uploaded.name).suffix
|
|
with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp:
|
|
tmp.write(uploaded.getvalue())
|
|
tmp_path = Path(tmp.name)
|
|
|
|
# Check for Excel sheets / detect delimiter
|
|
if suffix.lower() in (".xlsx", ".xls"):
|
|
st.session_state["sheet_names"] = list_sheets(tmp_path)
|
|
st.session_state["detected_delimiter"] = ","
|
|
else:
|
|
st.session_state["sheet_names"] = []
|
|
enc = detect_encoding(tmp_path)
|
|
st.session_state["detected_delimiter"] = detect_delimiter(tmp_path, enc)
|
|
|
|
df = read_file(tmp_path)
|
|
if not isinstance(df, pd.DataFrame):
|
|
df = pd.concat(list(df), ignore_index=True)
|
|
|
|
st.session_state["df"] = df
|
|
tmp_path.unlink(missing_ok=True)
|
|
|
|
except Exception as e:
|
|
st.error(f"Failed to read file: {e}")
|
|
st.session_state["df"] = None
|
|
|
|
df = st.session_state["df"]
|
|
|
|
if df is not None:
|
|
# Sheet selector for Excel files
|
|
if st.session_state["sheet_names"] and len(st.session_state["sheet_names"]) > 1:
|
|
sheet = st.selectbox(
|
|
"Select sheet",
|
|
st.session_state["sheet_names"],
|
|
)
|
|
if sheet != st.session_state.get("_current_sheet"):
|
|
st.session_state["_current_sheet"] = sheet
|
|
suffix = Path(uploaded.name).suffix
|
|
with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp:
|
|
tmp.write(uploaded.getvalue())
|
|
tmp_path = Path(tmp.name)
|
|
df = read_file(tmp_path, sheet_name=sheet)
|
|
if not isinstance(df, pd.DataFrame):
|
|
df = pd.concat(list(df), ignore_index=True)
|
|
st.session_state["df"] = df
|
|
st.session_state["result"] = None
|
|
st.session_state["review_decisions"] = {}
|
|
tmp_path.unlink(missing_ok=True)
|
|
|
|
# Delimiter selector for CSV/TSV files
|
|
is_csv = Path(uploaded.name).suffix.lower() not in (".xlsx", ".xls")
|
|
if is_csv:
|
|
_DELIMITERS = {
|
|
"Comma (,)": ",",
|
|
"Tab (\\t)": "\t",
|
|
"Semicolon (;)": ";",
|
|
"Pipe (|)": "|",
|
|
"Other": None,
|
|
}
|
|
_DELIM_LABELS = list(_DELIMITERS.keys())
|
|
_DELIM_VALUES = list(_DELIMITERS.values())
|
|
detected = st.session_state.get("detected_delimiter", ",")
|
|
default_idx = _DELIM_VALUES.index(detected) if detected in _DELIM_VALUES else 0
|
|
chosen_label = st.selectbox(
|
|
"Delimiter",
|
|
_DELIM_LABELS,
|
|
index=default_idx,
|
|
help="Auto-detected on upload. Change if the preview looks wrong.",
|
|
)
|
|
if chosen_label == "Other":
|
|
custom_delim = st.text_input(
|
|
"Enter delimiter character",
|
|
max_chars=5,
|
|
help="Enter the character(s) used to separate fields.",
|
|
)
|
|
chosen_delim = custom_delim if custom_delim else ","
|
|
else:
|
|
chosen_delim = _DELIMITERS[chosen_label]
|
|
if chosen_delim != st.session_state.get("_current_delimiter"):
|
|
st.session_state["_current_delimiter"] = chosen_delim
|
|
suffix = Path(uploaded.name).suffix
|
|
with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp:
|
|
tmp.write(uploaded.getvalue())
|
|
tmp_path = Path(tmp.name)
|
|
df = read_file(tmp_path, delimiter=chosen_delim)
|
|
if not isinstance(df, pd.DataFrame):
|
|
df = pd.concat(list(df), ignore_index=True)
|
|
st.session_state["df"] = df
|
|
st.session_state["result"] = None
|
|
st.session_state["review_decisions"] = {}
|
|
tmp_path.unlink(missing_ok=True)
|
|
|
|
# Preview
|
|
st.subheader(f"Preview: {uploaded.name}")
|
|
st.caption(f"{len(df)} rows, {len(df.columns)} columns")
|
|
st.dataframe(df.head(10), use_container_width=True)
|
|
|
|
# Advanced options
|
|
settings = config_panel(df)
|
|
|
|
# Apply loaded config if present
|
|
loaded_cfg = st.session_state.get("loaded_config")
|
|
if loaded_cfg is not None:
|
|
settings["strategies"] = loaded_cfg.to_strategies()
|
|
settings["survivor_rule"] = loaded_cfg.to_survivor_rule()
|
|
settings["date_column"] = loaded_cfg.date_column
|
|
settings["merge"] = loaded_cfg.merge
|
|
del st.session_state["loaded_config"]
|
|
|
|
# -------------------------------------------------------------------
|
|
# Find Duplicates button
|
|
# -------------------------------------------------------------------
|
|
|
|
st.divider()
|
|
|
|
if st.button("Find Duplicates", type="primary", use_container_width=True):
|
|
progress_bar = st.progress(0, text="Comparing rows...")
|
|
|
|
def _gui_progress(current: int, total: int) -> None:
|
|
if total > 0:
|
|
pct = min(current / total, 1.0)
|
|
progress_bar.progress(pct, text=f"Comparing rows... {current:,}/{total:,}")
|
|
|
|
with st.spinner("Running deduplication..."):
|
|
result = deduplicate(
|
|
df,
|
|
strategies=settings["strategies"],
|
|
survivor_rule=settings["survivor_rule"],
|
|
date_column=settings["date_column"],
|
|
merge=settings["merge"],
|
|
preview=False,
|
|
progress_callback=_gui_progress,
|
|
)
|
|
|
|
progress_bar.empty()
|
|
st.session_state["result"] = result
|
|
st.session_state["review_decisions"] = {}
|
|
|
|
# -------------------------------------------------------------------
|
|
# Results
|
|
# -------------------------------------------------------------------
|
|
|
|
result: DeduplicationResult | None = st.session_state["result"]
|
|
|
|
if result is not None:
|
|
st.divider()
|
|
st.subheader("Results")
|
|
|
|
# Summary + download buttons
|
|
results_summary(result, df)
|
|
|
|
# Match group review
|
|
if result.match_groups:
|
|
st.divider()
|
|
st.subheader("Match Groups")
|
|
|
|
# Batch actions
|
|
def _accept_all():
|
|
for g in result.match_groups:
|
|
st.session_state["review_decisions"][g.group_id] = {
|
|
"keep_indices": [g.survivor_index],
|
|
"overrides": {},
|
|
}
|
|
|
|
def _reject_all():
|
|
for g in result.match_groups:
|
|
st.session_state["review_decisions"][g.group_id] = {
|
|
"keep_indices": list(g.row_indices),
|
|
"overrides": {},
|
|
}
|
|
|
|
def _clear_all():
|
|
st.session_state["review_decisions"] = {}
|
|
for k in list(st.session_state):
|
|
if k.startswith("editor_"):
|
|
del st.session_state[k]
|
|
|
|
action_left, action_mid, action_right = st.columns(3)
|
|
with action_left:
|
|
st.button("Accept All", on_click=_accept_all)
|
|
with action_mid:
|
|
st.button("Reject All", on_click=_reject_all)
|
|
with action_right:
|
|
st.button("Clear Decisions", on_click=_clear_all)
|
|
|
|
# Individual group cards
|
|
decisions = st.session_state["review_decisions"]
|
|
for i, group in enumerate(result.match_groups):
|
|
match_group_card(group, df, group_num=i + 1)
|
|
|
|
# Show decision summary
|
|
if decisions:
|
|
st.divider()
|
|
merged = 0
|
|
customized = 0
|
|
split = 0
|
|
kept_all = 0
|
|
for v in decisions.values():
|
|
if not isinstance(v, dict):
|
|
continue
|
|
ki = v.get("keep_indices", [])
|
|
gid_for_v = next(
|
|
(gid for gid, d in decisions.items() if d is v),
|
|
None,
|
|
)
|
|
group_size = next(
|
|
(len(g.row_indices) for g in result.match_groups
|
|
if g.group_id == gid_for_v),
|
|
0,
|
|
)
|
|
if len(ki) == group_size:
|
|
kept_all += 1
|
|
elif len(ki) == 1:
|
|
if v.get("overrides"):
|
|
customized += 1
|
|
else:
|
|
merged += 1
|
|
else:
|
|
split += 1
|
|
|
|
pending = len(result.match_groups) - len(decisions)
|
|
parts = []
|
|
if merged:
|
|
parts.append(f"{merged} merged")
|
|
if customized:
|
|
parts.append(f"{customized} customized")
|
|
if split:
|
|
parts.append(f"{split} split")
|
|
if kept_all:
|
|
parts.append(f"{kept_all} kept all")
|
|
parts.append(f"{pending} pending")
|
|
st.caption("Decisions: " + ", ".join(parts))
|
|
|
|
# Apply decisions and offer download
|
|
if st.button(
|
|
"Apply Review Decisions & Download",
|
|
type="primary",
|
|
use_container_width=True,
|
|
):
|
|
reviewed_df, reviewed_removed = apply_review_decisions(
|
|
df, result.match_groups, decisions,
|
|
)
|
|
|
|
csv_bytes = reviewed_df.to_csv(
|
|
index=False
|
|
).encode("utf-8-sig")
|
|
st.download_button(
|
|
"Download Reviewed & Deduplicated CSV",
|
|
data=csv_bytes,
|
|
file_name="deduplicated_reviewed.csv",
|
|
mime="text/csv",
|
|
key="reviewed_download",
|
|
)
|
|
if not reviewed_removed.empty:
|
|
removed_bytes = reviewed_removed.to_csv(
|
|
index=False
|
|
).encode("utf-8-sig")
|
|
st.download_button(
|
|
"Download Reviewed Removed Rows",
|
|
data=removed_bytes,
|
|
file_name="removed_reviewed.csv",
|
|
mime="text/csv",
|
|
key="reviewed_removed_download",
|
|
)
|
|
|
|
# Log entries
|
|
if result.log_entries:
|
|
with st.expander("Processing Log"):
|
|
st.code("\n".join(result.log_entries))
|
|
|
|
else:
|
|
# No file uploaded — show placeholder
|
|
st.info("Upload a file to get started.")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Footer
|
|
# ---------------------------------------------------------------------------
|
|
|
|
st.divider()
|
|
st.caption(
|
|
"Runs locally. Your data never leaves this computer. "
|
|
"| DataTools Deduplicator v3.0"
|
|
)
|