Michael
27f0648093
Only "Download cleaned CSV" was working; "Download changes audit" and
"Download config JSON" did nothing on click.
The symptom is the classic Streamlit footgun for multiple
``st.download_button`` widgets in adjacent columns: without an explicit
``key`` argument the auto-derived widget IDs can collide, especially
when one button is conditionally rendered, and only the first button
in source order actually fires on click. Same goes for unstable
``data`` bytes recomputed inside the ``with col:`` block — the widget
identity can drift between renders.
Robustness pattern applied:
- Compute all three byte buffers up front, outside the columns, so the
``data`` parameter is the same object across reruns.
- Pass an explicit unique ``key`` ("textclean_dl_cleaned" /
"textclean_dl_changes" / "textclean_dl_config") to each button.
- Render the changes button unconditionally with ``disabled=True`` and
a help tooltip when ``result.changes.empty`` — instead of hiding it.
Layout stays steady and the empty case is self-explanatory.
- ``use_container_width=True`` so the three buttons size identically
inside their columns.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
411 lines
15 KiB
Python
411 lines
15 KiB
Python
"""DataTools Clean Text — Streamlit page."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import io
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pandas as pd
|
|
import streamlit as st
|
|
|
|
_project_root = Path(__file__).resolve().parent.parent.parent.parent
|
|
if str(_project_root) not in sys.path:
|
|
sys.path.insert(0, str(_project_root))
|
|
|
|
from src.gui.components import (
|
|
back_to_home_link,
|
|
hide_streamlit_chrome,
|
|
pickup_or_upload,
|
|
render_hidden_aware_preview,
|
|
require_feature_or_render_upgrade,
|
|
)
|
|
from src.license import FeatureFlag
|
|
from src.core.text_clean import (
|
|
PRESETS,
|
|
CleanOptions,
|
|
clean_dataframe,
|
|
hidden_char_css,
|
|
visualize_hidden_html,
|
|
)
|
|
|
|
hide_streamlit_chrome()
|
|
back_to_home_link()
|
|
require_feature_or_render_upgrade(FeatureFlag.TEXT_CLEANER)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Header
|
|
# ---------------------------------------------------------------------------
|
|
|
|
st.title("✂️ Clean Text")
|
|
st.caption(
|
|
"Trim whitespace, fold smart quotes, strip invisible characters, and "
|
|
"normalize line endings. Runs locally — your data never leaves this computer."
|
|
)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# File upload
|
|
# ---------------------------------------------------------------------------
|
|
|
|
uploaded = pickup_or_upload(
|
|
label="Upload CSV or Excel file",
|
|
key="textclean_file_upload",
|
|
types=["csv", "tsv", "xlsx", "xls"],
|
|
)
|
|
|
|
if uploaded is None:
|
|
st.info("Upload a CSV, TSV, or Excel file to begin.")
|
|
st.stop()
|
|
|
|
|
|
@st.cache_data(show_spinner=False)
|
|
def _read_uploaded(name: str, data: bytes) -> pd.DataFrame:
|
|
"""Read the uploaded bytes into a DataFrame, treating all cells as strings."""
|
|
suffix = Path(name).suffix.lower()
|
|
bio = io.BytesIO(data)
|
|
if suffix in (".xlsx", ".xls"):
|
|
return pd.read_excel(bio, dtype=str, keep_default_na=False)
|
|
# CSV / TSV — try utf-8 then utf-8-sig then latin-1 as a fallback
|
|
for enc in ("utf-8", "utf-8-sig", "latin-1"):
|
|
try:
|
|
bio.seek(0)
|
|
sep = "\t" if suffix == ".tsv" else ","
|
|
return pd.read_csv(
|
|
bio, dtype=str, keep_default_na=False,
|
|
encoding=enc, sep=sep, on_bad_lines="warn",
|
|
)
|
|
except UnicodeDecodeError:
|
|
continue
|
|
bio.seek(0)
|
|
return pd.read_csv(bio, dtype=str, keep_default_na=False, encoding="latin-1")
|
|
|
|
|
|
try:
|
|
df = _read_uploaded(uploaded.name, uploaded.getvalue())
|
|
except UnicodeDecodeError as e:
|
|
st.error(
|
|
f"**Could not decode `{uploaded.name}`**\n\n"
|
|
f"The file isn't UTF-8, UTF-8-with-BOM, or Latin-1.\n\n"
|
|
f"_Underlying error: {e}_\n\n"
|
|
f"Try re-saving the file as UTF-8 from the source application, "
|
|
f"or convert it with `iconv -f <source-encoding> -t utf-8`."
|
|
)
|
|
st.stop()
|
|
except Exception as e:
|
|
from src.core.errors import format_for_user
|
|
st.error(
|
|
f"**Could not read `{uploaded.name}`**\n\n"
|
|
f"```\n{format_for_user(e)}\n```"
|
|
)
|
|
st.stop()
|
|
|
|
# Collapse the input preview once the user has clicked Clean Text so
|
|
# the Results section below is the primary visual focus. The user can
|
|
# re-expand the expander to re-inspect the source rows.
|
|
_has_result = st.session_state.get("textclean_result") is not None
|
|
with st.expander(f"Preview: {uploaded.name}", expanded=not _has_result):
|
|
st.caption(f"{len(df)} rows, {len(df.columns)} columns")
|
|
preview_show_hidden = st.toggle(
|
|
"Show hidden characters in preview",
|
|
value=True,
|
|
help="Highlights NBSP, zero-width chars, smart quotes, and leading/trailing whitespace.",
|
|
key="textclean_preview_show_hidden",
|
|
)
|
|
if preview_show_hidden:
|
|
render_hidden_aware_preview(df, n_rows=10)
|
|
else:
|
|
st.dataframe(df.head(10), use_container_width=True)
|
|
|
|
st.divider()
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Options
|
|
# ---------------------------------------------------------------------------
|
|
#
|
|
# Wrapped in an outer expander whose default state mirrors the preview
|
|
# expander above: open before a result exists, folded once the user has
|
|
# clicked Clean Text. Together they push the Results section to the top
|
|
# of the visible area after a run.
|
|
|
|
with st.expander("Options", expanded=not _has_result):
|
|
preset_label = st.radio(
|
|
"Preset",
|
|
["excel-hygiene (recommended)", "minimal", "paranoid"],
|
|
index=0,
|
|
horizontal=True,
|
|
help=(
|
|
"excel-hygiene: trim, collapse whitespace, fold smart quotes, strip "
|
|
"invisible chars, normalize line endings, NFC. "
|
|
"minimal: only trim and collapse. "
|
|
"paranoid: everything including NFKC compat fold (lossy)."
|
|
),
|
|
)
|
|
preset_key = preset_label.split(" ", 1)[0]
|
|
options = CleanOptions.from_preset(preset_key)
|
|
|
|
with st.expander("Advanced options"):
|
|
col_a, col_b = st.columns(2)
|
|
with col_a:
|
|
options.trim = st.checkbox("Trim leading/trailing whitespace", value=options.trim)
|
|
options.collapse_whitespace = st.checkbox(
|
|
"Collapse internal whitespace", value=options.collapse_whitespace,
|
|
)
|
|
options.normalize_line_endings = st.checkbox(
|
|
"Normalize line endings (\\r\\n → \\n)", value=options.normalize_line_endings,
|
|
)
|
|
options.strip_control = st.checkbox(
|
|
"Strip control characters", value=options.strip_control,
|
|
)
|
|
options.strip_bom = st.checkbox("Strip BOM", value=options.strip_bom)
|
|
with col_b:
|
|
options.fold_smart_chars = st.checkbox(
|
|
"Fold smart characters (curly quotes, em-dash, NBSP)",
|
|
value=options.fold_smart_chars,
|
|
)
|
|
options.strip_zero_width = st.checkbox(
|
|
"Strip zero-width / invisible characters", value=options.strip_zero_width,
|
|
)
|
|
options.nfc = st.checkbox("Unicode NFC normalization", value=options.nfc)
|
|
options.nfkc = st.checkbox(
|
|
"Unicode NFKC compat fold (lossy: ① → 1, fi → fi)",
|
|
value=options.nfkc,
|
|
)
|
|
|
|
st.markdown("**Scope**")
|
|
string_cols = [
|
|
c for c in df.columns
|
|
if pd.api.types.is_object_dtype(df[c]) or pd.api.types.is_string_dtype(df[c])
|
|
]
|
|
selected_cols = st.multiselect(
|
|
"Columns to clean (default: all string columns)",
|
|
options=list(df.columns),
|
|
default=string_cols,
|
|
)
|
|
skip_cols = st.multiselect(
|
|
"Columns to skip even if they look like text",
|
|
options=list(df.columns),
|
|
default=[],
|
|
)
|
|
options.columns = selected_cols if selected_cols else None
|
|
options.skip_columns = list(skip_cols)
|
|
|
|
st.markdown("**Case conversion**")
|
|
case_global = st.selectbox(
|
|
"Apply case conversion to selected columns",
|
|
["None", "UPPER", "lower", "Title", "Sentence"],
|
|
index=0,
|
|
)
|
|
case_map = {
|
|
"UPPER": "upper", "lower": "lower",
|
|
"Title": "title", "Sentence": "sentence",
|
|
}
|
|
if case_global != "None":
|
|
options.case = case_map[case_global] # type: ignore[assignment]
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Run
|
|
# ---------------------------------------------------------------------------
|
|
|
|
st.divider()
|
|
|
|
if st.button("Clean Text", type="primary", use_container_width=True):
|
|
with st.spinner("Cleaning..."):
|
|
try:
|
|
result = clean_dataframe(df, options)
|
|
except ValueError as e:
|
|
st.error(str(e))
|
|
st.stop()
|
|
st.session_state["textclean_result"] = result
|
|
st.session_state["textclean_input_name"] = uploaded.name
|
|
# One-shot flag picked up on the next pass to scroll the parent
|
|
# document to the Results anchor (see scroll snippet below).
|
|
st.session_state["_textclean_scroll_to_results"] = True
|
|
# Force a second rerun so the preview and options expanders see
|
|
# the new result on the NEXT script pass and collapse themselves.
|
|
# Without this they stay expanded until the user touches any
|
|
# other widget.
|
|
st.rerun()
|
|
|
|
result = st.session_state.get("textclean_result")
|
|
if result is None:
|
|
st.stop()
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Results
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# Anchor target for the auto-scroll snippet at the end of this block.
|
|
# A bare ``<div id="...">`` survives Streamlit's HTML sanitizer (only
|
|
# ``<script>`` is stripped), and a 1px-tall div doesn't visually shift
|
|
# anything. Placed before the subheader so the scrolled-to viewport
|
|
# starts a few pixels above the section heading rather than below it.
|
|
st.markdown(
|
|
'<div id="textclean-results-anchor" style="height:1px"></div>',
|
|
unsafe_allow_html=True,
|
|
)
|
|
|
|
st.subheader("Results")
|
|
|
|
pct = (result.cells_changed / result.cells_total * 100.0) if result.cells_total else 0.0
|
|
m1, m2, m3, m4 = st.columns(4)
|
|
m1.metric("Cells scanned", result.cells_total)
|
|
m2.metric("Cells changed", result.cells_changed)
|
|
m3.metric("% changed", f"{pct:.1f}%")
|
|
m4.metric("Columns processed", len(result.columns_processed))
|
|
|
|
if result.cells_changed:
|
|
counts = result.changes["column"].value_counts()
|
|
st.markdown("**Changes by column**")
|
|
st.dataframe(
|
|
counts.rename("cells_changed").to_frame(),
|
|
use_container_width=True,
|
|
)
|
|
|
|
st.markdown("**Examples (first 25 changes)**")
|
|
show_hidden = st.toggle(
|
|
"Show hidden characters (NBSP, ZWSP, smart quotes, control chars…)",
|
|
value=True,
|
|
help=(
|
|
"Highlights characters the cleaner is removing or replacing. "
|
|
"Hover any badge to see the codepoint and label."
|
|
),
|
|
key="textclean_show_hidden",
|
|
)
|
|
examples = result.changes.head(25).copy()
|
|
examples["row"] = examples["row"] + 1
|
|
if show_hidden:
|
|
# Inject the badge CSS once, then render an HTML table so the
|
|
# invisibles in old/new are actually visible to the user.
|
|
# ``mark_outer_whitespace=True`` matches the input preview's
|
|
# rendering so leading/trailing spaces show up as badges in the
|
|
# Before/After columns — without it, the examples table missed
|
|
# exactly the whitespace the cleaner is removing.
|
|
st.markdown(hidden_char_css(), unsafe_allow_html=True)
|
|
rows_html = []
|
|
for _, row in examples.iterrows():
|
|
rows_html.append(
|
|
"<tr>"
|
|
f"<td>{row['row']}</td>"
|
|
f"<td><code>{visualize_hidden_html(str(row['column']), mark_outer_whitespace=True)}</code></td>"
|
|
f"<td>{visualize_hidden_html(str(row['old']), mark_outer_whitespace=True)}</td>"
|
|
f"<td>{visualize_hidden_html(str(row['new']), mark_outer_whitespace=True)}</td>"
|
|
f"<td><code>{row['ops_applied']}</code></td>"
|
|
"</tr>"
|
|
)
|
|
st.markdown(
|
|
"<table class='hidden-char-table'>"
|
|
"<thead><tr>"
|
|
"<th style='text-align:left'>Row</th>"
|
|
"<th style='text-align:left'>Column</th>"
|
|
"<th style='text-align:left'>Before</th>"
|
|
"<th style='text-align:left'>After</th>"
|
|
"<th style='text-align:left'>Ops applied</th>"
|
|
"</tr></thead>"
|
|
f"<tbody>{''.join(rows_html)}</tbody>"
|
|
"</table>"
|
|
"<style>"
|
|
".hidden-char-table { width: 100%; border-collapse: collapse; }"
|
|
".hidden-char-table th, .hidden-char-table td { "
|
|
" padding: 4px 8px; border-bottom: 1px solid #eee; "
|
|
" vertical-align: top; }"
|
|
".hidden-char-table tbody tr:hover { background: #fafafa; }"
|
|
"</style>",
|
|
unsafe_allow_html=True,
|
|
)
|
|
else:
|
|
st.dataframe(examples, use_container_width=True, hide_index=True)
|
|
|
|
st.markdown("**Cleaned preview (first 10 rows)**")
|
|
# Reuse the same toggle the Examples table uses so the user controls both
|
|
# the changes audit and the cleaned preview with one switch.
|
|
if show_hidden:
|
|
render_hidden_aware_preview(result.cleaned_df, n_rows=10)
|
|
else:
|
|
st.dataframe(result.cleaned_df.head(10), use_container_width=True)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Downloads
|
|
# ---------------------------------------------------------------------------
|
|
#
|
|
# All three byte buffers are prepared up front (outside the columns) so
|
|
# each ``st.download_button`` sees stable ``data`` across reruns and an
|
|
# explicit ``key`` — without those, Streamlit auto-derived widget IDs
|
|
# can collide for multiple download_buttons in adjacent columns and
|
|
# only the first one actually fires on click. The empty-changes case
|
|
# now renders a disabled button (rather than vanishing) so the layout
|
|
# stays steady and the user understands why nothing's available.
|
|
|
|
st.divider()
|
|
stem = Path(st.session_state.get("textclean_input_name", "input")).stem
|
|
|
|
cleaned_bytes = result.cleaned_df.to_csv(index=False).encode("utf-8-sig")
|
|
changes_bytes = (
|
|
result.changes.to_csv(index=False).encode("utf-8-sig")
|
|
if not result.changes.empty
|
|
else b""
|
|
)
|
|
config_bytes = json.dumps(options.to_dict(), indent=2).encode("utf-8")
|
|
|
|
dl_a, dl_b, dl_c = st.columns(3)
|
|
with dl_a:
|
|
st.download_button(
|
|
"Download cleaned CSV",
|
|
data=cleaned_bytes,
|
|
file_name=f"{stem}_cleaned.csv",
|
|
mime="text/csv",
|
|
key="textclean_dl_cleaned",
|
|
use_container_width=True,
|
|
)
|
|
with dl_b:
|
|
st.download_button(
|
|
"Download changes audit",
|
|
data=changes_bytes,
|
|
file_name=f"{stem}_changes.csv",
|
|
mime="text/csv",
|
|
key="textclean_dl_changes",
|
|
disabled=result.changes.empty,
|
|
help="No changes to audit." if result.changes.empty else None,
|
|
use_container_width=True,
|
|
)
|
|
with dl_c:
|
|
st.download_button(
|
|
"Download config JSON",
|
|
data=config_bytes,
|
|
file_name="text_clean_config.json",
|
|
mime="application/json",
|
|
key="textclean_dl_config",
|
|
use_container_width=True,
|
|
)
|
|
|
|
st.divider()
|
|
st.caption("Runs locally. Your data never leaves this computer. | DataTools v3.0")
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Post-run auto-scroll
|
|
# ---------------------------------------------------------------------------
|
|
#
|
|
# When the user clicks Clean Text, the preview + options collapse but
|
|
# Streamlit by itself doesn't scroll — the Results section is at the
|
|
# bottom of a tall script so the user has to find it. Inject a tiny
|
|
# component-html iframe that calls ``scrollIntoView`` on the parent's
|
|
# Results anchor. Streamlit's main page is same-origin with component
|
|
# iframes so ``window.parent.document`` access is allowed.
|
|
#
|
|
# The flag is one-shot (``pop`` removes it) so re-renders triggered by
|
|
# unrelated widgets in the Results section (e.g., the Show-hidden
|
|
# toggle) don't yank the viewport back to the top of Results.
|
|
if st.session_state.pop("_textclean_scroll_to_results", False):
|
|
from streamlit.components.v1 import html as _components_html
|
|
_components_html(
|
|
"""
|
|
<script>
|
|
const doc = window.parent.document;
|
|
const target = doc.getElementById('textclean-results-anchor');
|
|
if (target) target.scrollIntoView({behavior: 'smooth', block: 'start'});
|
|
</script>
|
|
""",
|
|
height=0,
|
|
)
|