Compare commits
174 Commits
b2c7b94fe9
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| 41ab2166ef | |||
| 9943e6e537 | |||
| e7ec79b9b5 | |||
| 6df726e69e | |||
| 38616d69e2 | |||
| 00d3f28865 | |||
| 837f4b88b5 | |||
| fd9606c67b | |||
| 28ab51a869 | |||
| 1895074b8f | |||
| d807d3c11b | |||
| 09ec01e98b | |||
| 48251b625f | |||
| dd0942d710 | |||
| cf31d9ef14 | |||
| 563d845b70 | |||
| be1e263223 | |||
| 7ebfd0f153 | |||
| 2592604067 | |||
| 58d0009849 | |||
| b6c39d7a09 | |||
| b2fa8503e6 | |||
| b703911df3 | |||
| 93ccada974 | |||
| 17faf84aed | |||
| 4d8513b1a3 | |||
| ac94208d8f | |||
| 4955fb239b | |||
| 4a8961d58a | |||
| fe4b5dc755 | |||
| 209b5fb1aa | |||
| 904356f4e8 | |||
| 7203a81af7 | |||
| dd3b9bd59d | |||
| 2bd94c4441 | |||
| 9c426194b1 | |||
| 6627895a10 | |||
| ea99e292d2 | |||
| 0be59c0f03 | |||
| 3a3a9a895b | |||
| d090f8cb5e | |||
| e44af3a45e | |||
| 450d4fc9a8 | |||
| a0042d4aba | |||
| a18b126885 | |||
| 981a1a9cba | |||
| dbcf4d4048 | |||
| 34b56b404a | |||
| ad7c22d7fb | |||
| 6f2ad57490 | |||
| a1824b8dc4 | |||
| 155dd30746 | |||
| 3cf935c999 | |||
| 263af3c7c2 | |||
| bece2b4030 | |||
| 60969c0770 | |||
| 48cd9e8249 | |||
| d80befd05a | |||
| 10015c40e1 | |||
| e6ee2e3481 | |||
| 538e23d219 | |||
| 2d927bc95f | |||
| 967d3f6a11 | |||
| b86828d791 | |||
| 5a8e2ec9e1 | |||
| 2f349e8191 | |||
| aea520d2f7 | |||
| b8aff862ed | |||
| c16e2a5e29 | |||
| 7c9139f199 | |||
| b3ae913bb9 | |||
| ba07dcb6c7 | |||
| 76c9f5a679 | |||
| a8ff8f4bd0 | |||
| 4451f74895 | |||
| a022059b1e | |||
| 69240fc922 | |||
| 9a7d861903 | |||
| 1016a4d2c4 | |||
| 6c3939d21b | |||
| d436e34a45 | |||
| 0bb72ecd7e | |||
| 74d0ee270f | |||
| 06f1ea6cf7 | |||
| 784695e3a7 | |||
| 4816da1ad6 | |||
| 6703e2c15c | |||
| a9788ba712 | |||
| da7d86f457 | |||
| 2501119ac2 | |||
| 444dffbc63 | |||
| 3c4b80895e | |||
| b0ee65e922 | |||
| 65b663be97 | |||
| c942b8aa19 | |||
| 61e63913cb | |||
| e011c0b6e6 | |||
| 2fe324279e | |||
| 04dc326020 | |||
| d487a44170 | |||
| f106275643 | |||
| 8232ab1ca7 | |||
| 4c8e1199a4 | |||
| e282f061dc | |||
| 5daae9e5fa | |||
| 48cb802dfb | |||
| d022167ba2 | |||
| 24ee021314 | |||
| add3b866ee | |||
| b568773a1f | |||
| 4a7f99f0ec | |||
| b2449d3139 | |||
| d840230e48 | |||
| 9e8b4b2ca9 | |||
| dd231f5a38 | |||
| 143c775cdf | |||
| d1b9f642e2 | |||
| 65c85107b6 | |||
| d9e32e578b | |||
| 7cb1bc922d | |||
| be7191a5d1 | |||
| 2d2ff43754 | |||
| 36510eee7b | |||
| 1caedbbbc7 | |||
| c0bfd4dbc9 | |||
| 59c6d0f914 | |||
| ee0b1f6f6b | |||
| c73d716d06 | |||
| f0885aeb1e | |||
| 229e1afd45 | |||
| 7ad19ac7f4 | |||
| 84e4665ab0 | |||
| 4685bb4289 | |||
| e96d5901f4 | |||
| ecfc52499f | |||
| 21fd8a4cd7 | |||
| 42f8d78dd5 | |||
| 0f89d7ba66 | |||
| b9147f3b66 | |||
| 5128d35961 | |||
| 696996c119 | |||
| ae9d4a2db5 | |||
| ef9f8b5de4 | |||
| aeead05e4c | |||
| 6415be8bf4 | |||
| d1aaf3c2b9 | |||
| 27f0648093 | |||
| 0a61d52200 | |||
| ca14ce2952 | |||
| 502a72cd46 | |||
| 604debb9a9 | |||
| c575efd26e | |||
| 175389219f | |||
| c568aec8a7 | |||
| ff2eaeb6c4 | |||
| dad744f17f | |||
| fc6c22c6a7 | |||
| db5ec084da | |||
| 93e43fc0d9 | |||
| 624f99653e | |||
| 86ad21db79 | |||
| 2bbaba954b | |||
| b5cd74d474 | |||
| 1cf69dd23b | |||
| 673b902377 | |||
| bab2c9468c | |||
| 4179cb5156 | |||
| 52e04f63a9 | |||
| 23c51fd759 | |||
| 65e17e0a70 | |||
| e534fb4989 | |||
| d32b58e61a | |||
| e612c751a8 | |||
| e435103113 |
145
.github/workflows/build.yml
vendored
145
.github/workflows/build.yml
vendored
@@ -1,8 +1,17 @@
|
||||
name: Build installers
|
||||
|
||||
# Triggers:
|
||||
# * Tag push (v*) → produces installers, attaches to a GitHub Release.
|
||||
# * Manual dispatch → produces installers as workflow artifacts only.
|
||||
# * Tag push (v*) → produces installers, attaches them to a GitHub Release.
|
||||
# * Manual dispatch → uploads the installers as workflow artifacts only.
|
||||
#
|
||||
# Outputs per platform (downloadable by buyers):
|
||||
# * macOS: .dmg installer
|
||||
# * Windows: .exe installer
|
||||
# * Linux: .AppImage (already portable; no separate installer step)
|
||||
#
|
||||
# Self-contained: every artifact ships its own Python interpreter + every
|
||||
# runtime dep (including bundled Tesseract OCR) through PyInstaller. No
|
||||
# pre/post install steps on the buyer's machine.
|
||||
#
|
||||
# What this workflow doesn't do (yet):
|
||||
# * Code signing (Mac Developer ID, Windows code-signing cert).
|
||||
@@ -29,12 +38,15 @@ jobs:
|
||||
matrix:
|
||||
include:
|
||||
- os: macos-latest
|
||||
platform: mac
|
||||
artifact_name: DataTools-mac.dmg
|
||||
artifact_path: dist/DataTools-*-mac.dmg
|
||||
- os: windows-latest
|
||||
platform: win
|
||||
artifact_name: DataTools-win.exe
|
||||
artifact_path: dist/DataTools-*-win-setup.exe
|
||||
- os: ubuntu-latest
|
||||
platform: linux
|
||||
artifact_name: DataTools-linux.AppImage
|
||||
artifact_path: dist/DataTools-*-linux-x86_64.AppImage
|
||||
runs-on: ${{ matrix.os }}
|
||||
@@ -50,7 +62,31 @@ jobs:
|
||||
run: |
|
||||
pip install --upgrade pip
|
||||
pip install -r requirements.txt
|
||||
pip install pyinstaller
|
||||
pip install pyinstaller pillow
|
||||
|
||||
# ---- Tesseract bundling cache --------------------------------
|
||||
# The fetch logic inside build/tesseract.py downloads:
|
||||
# * build/vendor/tessdata/eng.traineddata (~16 MB, shared)
|
||||
# * build/_tesseract/<platform>/ (binary + libs, 30-120 MB)
|
||||
# Cache both so iterative CI runs don't re-download. The
|
||||
# cache key bakes in the pinned Tesseract version + tessdata
|
||||
# URL so a version bump invalidates automatically.
|
||||
- name: Cache Tesseract bundle inputs
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
build/_tesseract
|
||||
build/vendor/tessdata
|
||||
key: tesseract-${{ runner.os }}-5.5.0-tessdata_best-v1
|
||||
|
||||
# ---- Linux: install patchelf so tesseract.py can rewrite
|
||||
# RPATH on the bundled tesseract binary. apt-get install
|
||||
# tesseract-ocr is handled inside tesseract.py itself. --------
|
||||
- name: Install Linux build prereqs for Tesseract bundling
|
||||
if: matrix.os == 'ubuntu-latest'
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y patchelf
|
||||
|
||||
- name: Read version
|
||||
id: version
|
||||
@@ -59,12 +95,109 @@ jobs:
|
||||
VER=$(python -c "import re; print(re.search(r'__version__\s*=\s*\"([^\"]+)\"', open('src/__init__.py').read()).group(1))")
|
||||
echo "version=$VER" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Generate platform icons
|
||||
run: python build/generate_icons.py
|
||||
|
||||
# Stage Tesseract before PyInstaller. The tesseract.py helpers
|
||||
# handle the per-platform fetch (UB-Mannheim on Win, brew on
|
||||
# Mac, apt on Linux) and stage the binary + libs into
|
||||
# build/_tesseract/<platform>/ where the spec picks them up.
|
||||
# We invoke a tiny inline Python so the workflow doesn't have
|
||||
# to know the per-platform target string.
|
||||
- name: Stage Tesseract binary + tessdata
|
||||
shell: bash
|
||||
env:
|
||||
DATATOOLS_PLATFORM: ${{ matrix.platform }}
|
||||
run: |
|
||||
python - <<'PY'
|
||||
import os, sys
|
||||
sys.path.insert(0, "build")
|
||||
from tesseract import fetch_tessdata, fetch_tesseract_for_platform
|
||||
target = os.environ["DATATOOLS_PLATFORM"]
|
||||
fetch_tessdata()
|
||||
fetch_tesseract_for_platform(target)
|
||||
PY
|
||||
|
||||
- name: Build PyInstaller bundle
|
||||
shell: bash
|
||||
env:
|
||||
# The spec reads this to find the per-platform staging dir;
|
||||
# see build/datatools.spec for the contract.
|
||||
DATATOOLS_TESS_STAGING: build/_tesseract/${{ matrix.platform }}
|
||||
run: pyinstaller build/datatools.spec --clean --noconfirm
|
||||
|
||||
# ---- Per-platform packaging ----------------------------------
|
||||
# ---- macOS code signing + notarization (before DMG packaging) -
|
||||
# Signs dist/DataTools.app with the Developer ID, notarizes it,
|
||||
# and staples the ticket so Gatekeeper passes offline. Wrapped in
|
||||
# a guard: if the cert secret is absent the step prints a warning
|
||||
# and exits 0, so dry-run dispatches still produce an (unsigned)
|
||||
# build. Secret names match build/README.md "Signing".
|
||||
- name: Sign & notarize macOS app
|
||||
if: matrix.os == 'macos-latest'
|
||||
env:
|
||||
CERT_P12_BASE64: ${{ secrets.MACOS_DEVELOPER_ID_CERT_P12_BASE64 }}
|
||||
CERT_PASSWORD: ${{ secrets.MACOS_DEVELOPER_ID_CERT_PASSWORD }}
|
||||
NOTARY_APPLE_ID: ${{ secrets.MACOS_NOTARY_APPLE_ID }}
|
||||
NOTARY_TEAM_ID: ${{ secrets.MACOS_NOTARY_TEAM_ID }}
|
||||
NOTARY_PASSWORD: ${{ secrets.MACOS_NOTARY_PASSWORD }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${CERT_P12_BASE64:-}" ]; then
|
||||
echo "::warning::MACOS_DEVELOPER_ID_CERT_P12_BASE64 not set — shipping an UNSIGNED build (Gatekeeper will warn buyers)."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
- name: Package macOS DMG
|
||||
APP="dist/DataTools.app"
|
||||
|
||||
# 1. Import the Developer ID cert into an ephemeral keychain.
|
||||
KEYCHAIN="$RUNNER_TEMP/build.keychain-db"
|
||||
KEYCHAIN_PW="$(uuidgen)"
|
||||
security create-keychain -p "$KEYCHAIN_PW" "$KEYCHAIN"
|
||||
security set-keychain-settings -lut 3600 "$KEYCHAIN"
|
||||
security unlock-keychain -p "$KEYCHAIN_PW" "$KEYCHAIN"
|
||||
echo "$CERT_P12_BASE64" | base64 --decode > "$RUNNER_TEMP/cert.p12"
|
||||
security import "$RUNNER_TEMP/cert.p12" -k "$KEYCHAIN" -P "$CERT_PASSWORD" \
|
||||
-T /usr/bin/codesign
|
||||
security set-key-partition-list -S apple-tool:,apple: -s -k "$KEYCHAIN_PW" "$KEYCHAIN" >/dev/null
|
||||
# Make the ephemeral keychain searchable (preserve the login keychain).
|
||||
security list-keychains -d user -s "$KEYCHAIN" \
|
||||
$(security list-keychains -d user | sed 's/"//g')
|
||||
|
||||
IDENTITY="$(security find-identity -v -p codesigning "$KEYCHAIN" \
|
||||
| grep 'Developer ID Application' | head -1 | awk -F'"' '{print $2}')"
|
||||
if [ -z "$IDENTITY" ]; then
|
||||
echo "::error::No 'Developer ID Application' identity found in the imported cert."
|
||||
exit 1
|
||||
fi
|
||||
echo "Signing with: $IDENTITY"
|
||||
|
||||
# 2. Sign the bundle (hardened runtime + secure timestamp + entitlements).
|
||||
# --deep signs the nested dylibs/.so the PyInstaller bundle carries.
|
||||
codesign --deep --force --options runtime --timestamp \
|
||||
--entitlements build/macos/entitlements.plist \
|
||||
--sign "$IDENTITY" "$APP"
|
||||
codesign --verify --strict --verbose=2 "$APP"
|
||||
|
||||
# 3. Notarize the .app (notarytool needs a zip/dmg/pkg, not a bare .app),
|
||||
# then staple so Gatekeeper validates offline.
|
||||
if [ -n "${NOTARY_APPLE_ID:-}" ]; then
|
||||
ditto -c -k --keepParent "$APP" "$RUNNER_TEMP/DataTools.zip"
|
||||
xcrun notarytool submit "$RUNNER_TEMP/DataTools.zip" \
|
||||
--apple-id "$NOTARY_APPLE_ID" \
|
||||
--team-id "$NOTARY_TEAM_ID" \
|
||||
--password "$NOTARY_PASSWORD" \
|
||||
--wait
|
||||
xcrun stapler staple "$APP"
|
||||
xcrun stapler validate "$APP"
|
||||
else
|
||||
echo "::warning::Notary credentials not set — app is signed but NOT notarized (Gatekeeper will still warn)."
|
||||
fi
|
||||
|
||||
rm -f "$RUNNER_TEMP/cert.p12"
|
||||
|
||||
# ---- Per-platform installer packaging ------------------------
|
||||
|
||||
- name: Package macOS DMG (installer)
|
||||
if: matrix.os == 'macos-latest'
|
||||
run: bash build/macos/build_dmg.sh "${{ steps.version.outputs.version }}"
|
||||
|
||||
@@ -92,7 +225,7 @@ jobs:
|
||||
|
||||
# ---- Upload + release ----------------------------------------
|
||||
|
||||
- name: Upload artifact
|
||||
- name: Upload installer artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ${{ matrix.artifact_name }}
|
||||
|
||||
13
.gitignore
vendored
13
.gitignore
vendored
@@ -11,6 +11,19 @@ dist/
|
||||
build/build/
|
||||
build/__pycache__/
|
||||
build/dist/
|
||||
# Generated by build/generate_icons.py from src/gui/assets/datatools_icon_256.png.
|
||||
# Build artifacts, not source — regenerated each CI run.
|
||||
build/icon.ico
|
||||
build/icon.icns
|
||||
build/icon.png
|
||||
|
||||
# Tesseract bundling — fetched at build time, not committed. See
|
||||
# build/vendor/README.md for the canonical URLs and rationale.
|
||||
# - build/_tesseract/ : per-platform binary + DLLs/dylibs staging dir
|
||||
# - build/vendor/tessdata/eng.traineddata : ~16 MB language data
|
||||
build/_tesseract/
|
||||
build/vendor/tessdata/*.traineddata
|
||||
|
||||
.pytest_cache/
|
||||
|
||||
# Claude Code agent worktrees + local settings
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
[client]
|
||||
toolbarMode = "minimal"
|
||||
# ``viewer`` is the most aggressive — hides Streamlit's running
|
||||
# indicator, deploy button, and status icons. Keeps the main content
|
||||
# area's top-right corner clean.
|
||||
toolbarMode = "viewer"
|
||||
|
||||
[browser]
|
||||
gatherUsageStats = false
|
||||
@@ -9,3 +12,17 @@ gatherUsageStats = false
|
||||
# reads "Limit 1024MB per file" — matches the analyzer + gate's stated
|
||||
# 1 GB efficiency target. See docs/REQUIREMENTS.md §1.1.
|
||||
maxUploadSize = 1024
|
||||
|
||||
# Warm, editorial palette inspired by the
|
||||
# ``datatools_layout_redesign.html`` mockup — cream paper background,
|
||||
# stone ink, burnt-orange accent. Streamlit reads these on startup and
|
||||
# threads them through its widget chrome (file uploader, focus rings,
|
||||
# primary buttons, links). Heavier visual restyling rides on the CSS
|
||||
# in ``_legacy.py:_DESIGN_TOKENS_CSS``.
|
||||
[theme]
|
||||
base = "light"
|
||||
primaryColor = "#c2410c"
|
||||
backgroundColor = "#fafaf7"
|
||||
secondaryBackgroundColor = "#f5f4ef"
|
||||
textColor = "#1c1917"
|
||||
font = "sans serif"
|
||||
|
||||
33
DECISIONS.md
Normal file
33
DECISIONS.md
Normal file
@@ -0,0 +1,33 @@
|
||||
# Product & architecture decisions
|
||||
|
||||
A running log of decisions that aren't obvious from the code and would
|
||||
otherwise be re-litigated. Newest first.
|
||||
|
||||
## 2026-06-08 — PDF to CSV and Reconcile stay in the bundle, under a "Finance" group
|
||||
|
||||
**Decision:** `10_pdf_extractor` (PDF to CSV) and `11_reconciler` (Reconcile
|
||||
Two Files) remain part of the DataTools suite. In the sidebar they are
|
||||
segregated into their own **Finance** section, distinct from the
|
||||
file-cleaning tools.
|
||||
|
||||
**Context / why this needed deciding:**
|
||||
- Both tools sit outside the documented 9-script cleaning architecture
|
||||
(TECHNICAL.md / USER-GUIDE.md stop at the orchestrator).
|
||||
- They occupy the "reconciliation / manual data-entry" territory the
|
||||
product's honest-positioning note explicitly placed outside a
|
||||
file-cleaning tool's scope.
|
||||
- A journey-level UX review flagged that every extra tool in the main
|
||||
sidebar raises the "which tool do I need?" load for a non-technical
|
||||
buyer, so tools serving a different job should live in a clearly
|
||||
different place.
|
||||
|
||||
**Resolution:** Keep them in-bundle (they're built, useful, and ship
|
||||
today) but group them under "Finance" so the cleaning flow stays
|
||||
uncluttered. Revisit only if a separate finance-focused product emerges.
|
||||
|
||||
**Implications:**
|
||||
- `tools_registry.py`: Reconcile + PDF to CSV carry a `finance` section.
|
||||
- Sidebar order: Start here → Data Cleaners → Transformations →
|
||||
Automations → Finance → Coming soon.
|
||||
- This is the source-of-truth realization of the `layout-review/`
|
||||
mockups (see `layout-review/shell.js`).
|
||||
220
LICENSE_TESSERACT.txt
Normal file
220
LICENSE_TESSERACT.txt
Normal file
@@ -0,0 +1,220 @@
|
||||
This license applies to the bundled Tesseract OCR binary distributed
|
||||
inside DataTools installer artifacts (Windows .exe, macOS .dmg, Linux
|
||||
.AppImage) and the corresponding portable .zip downloads.
|
||||
|
||||
Tesseract OCR upstream: https://github.com/tesseract-ocr/tesseract
|
||||
Copyright (C) 2006-2024 Google Inc. and the Tesseract OCR contributors
|
||||
|
||||
The Tesseract OCR binary is distributed under the Apache License,
|
||||
Version 2.0, the full text of which is reproduced verbatim below.
|
||||
|
||||
The bundled `eng.traineddata` data file is the "best" English model
|
||||
from https://github.com/tesseract-ocr/tessdata_best and is licensed
|
||||
under the Apache License, Version 2.0 as well.
|
||||
|
||||
DataTools itself is proprietary and is NOT covered by this license;
|
||||
see LICENSE.txt at the repository root for DataTools' own license.
|
||||
|
||||
================================================================================
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for describing the origin of the Work and
|
||||
reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may accept and charge a
|
||||
fee for, acceptance of support, warranty, indemnity, or other
|
||||
liability obligations and/or rights consistent with this License.
|
||||
However, in accepting such obligations, You may act only on Your
|
||||
own behalf and on Your sole responsibility, not on behalf of any
|
||||
other Contributor, and only if You agree to indemnify, defend,
|
||||
and hold each Contributor harmless for any liability incurred by,
|
||||
or claims asserted against, such Contributor by reason of your
|
||||
accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied. See the License for the specific language governing
|
||||
permissions and limitations under the License.
|
||||
42
README.es.md
42
README.es.md
@@ -8,27 +8,37 @@ Limpieza local de CSV / Excel. CLI + GUI en el navegador, sin nube, sin ceremoni
|
||||
|
||||
| # | Herramienta | Estado |
|
||||
|---|------|--------|
|
||||
| 01 | **Eliminador de duplicados** — coincidencia exacta + difusa, 5 normalizadores, reglas de superviviente, auditoría | Listo |
|
||||
| 02 | **Limpiador de texto** — espacios, caracteres tipográficos, BOM, finales de línea, mayúsculas/minúsculas | Listo |
|
||||
| 03 | **Estandarizador de formatos** — fechas, teléfonos, correos, direcciones, nombres, monedas, booleanos | Listo |
|
||||
| 04 | **Gestor de valores faltantes** — detección de nulos disfrazados, perfil, media/mediana/moda/ffill/bfill/interpolación, estrategias de descarte | Listo |
|
||||
| 05 | **Mapeador de columnas** — autodetección difusa de renombrados, esquema objetivo con coerción de tipos, campos requeridos con valores por defecto, descartar/reordenar | Listo |
|
||||
| 06 | Detector de valores atípicos | Próximamente |
|
||||
| 07 | Combinador de varios archivos | Próximamente |
|
||||
| 08 | Validador e informes | Próximamente |
|
||||
| 09 | **Ejecutor de canalizaciones** — encadena herramientas en un orden recomendado (no forzado), guarda/carga JSON, automatiza limpiezas semanales | Listo |
|
||||
| 01 | **Buscar duplicados** — coincidencia exacta + difusa, 5 normalizadores, reglas de superviviente, auditoría | Listo |
|
||||
| 02 | **Limpiar texto** — espacios, caracteres tipográficos, BOM, finales de línea, mayúsculas/minúsculas | Listo |
|
||||
| 03 | **Estandarizar formatos** — fechas, teléfonos, correos, direcciones, nombres, monedas, booleanos | Listo |
|
||||
| 04 | **Corregir valores faltantes** — detección de nulos disfrazados, perfil, media/mediana/moda/ffill/bfill/interpolación, estrategias de descarte | Listo |
|
||||
| 05 | **Mapear columnas** — autodetección difusa de renombrados, esquema objetivo con coerción de tipos, campos requeridos con valores por defecto, descartar/reordenar | Listo |
|
||||
| 06 | Detectar valores atípicos | Próximamente |
|
||||
| 07 | Combinar archivos | Próximamente |
|
||||
| 08 | Verificación de calidad | Próximamente |
|
||||
| 09 | **Flujos automatizados** — encadena herramientas en un orden recomendado (no forzado), guarda/carga JSON, automatiza limpiezas semanales | Listo |
|
||||
|
||||
Cada página de herramienta incluye una ventana emergente de **Help** (a la derecha del título) con una guía compacta de Cuándo usarla / Pasos / Ejemplos / Consejo. El texto vive en los paquetes de idioma (`tools.<id>.help_md`).
|
||||
|
||||
## Descarga (usuarios no técnicos)
|
||||
|
||||
Instaladores precompilados — no se requiere Python:
|
||||
Paquetes precompilados — sin instalar Python, sin permisos de administrador, sin internet en ejecución. Cada versión ofrece un **instalador** por sistema operativo que crea accesos directos en el escritorio + menú Inicio / Launchpad.
|
||||
|
||||
| Plataforma | Descarga | Nota de primer arranque |
|
||||
|---|---|---|
|
||||
| **macOS** | `DataTools-X.Y.Z-mac.dmg` | Arrastra DataTools.app a /Applications y haz doble clic. |
|
||||
| **Windows** | `DataTools-X.Y.Z-win-setup.exe` | Ejecuta el instalador; se inicia desde el menú Inicio. |
|
||||
| **Linux** | `DataTools-X.Y.Z-linux-x86_64.AppImage` | `chmod +x` al archivo y luego doble clic. |
|
||||
| Plataforma | Instalador |
|
||||
|---|---|
|
||||
| **macOS** | `DataTools-X.Y.Z-mac.dmg` — ábrelo, arrastra DataTools.app a /Applications, ejecútalo desde Launchpad. |
|
||||
| **Windows** | `DataTools-X.Y.Z-win-setup.exe` — ejecuta el instalador (por usuario, sin admin). Crea acceso directo en el escritorio + entrada en el menú Inicio. |
|
||||
| **Linux** | `DataTools-X.Y.Z-linux-x86_64.AppImage` — `chmod +x` y doble clic. El AppImage ya es portable. |
|
||||
|
||||
Última versión: consulta [GitHub Releases](https://git.invixiom.com/giteadmin/datatools-dev/releases) (o el listado de Gumroad). Los instaladores ocupan ~150–200 MB; el lanzador arranca un servidor local en http://127.0.0.1:8501 y abre tu navegador. Nada se envía a la nube.
|
||||
Última versión: consulta [GitHub Releases](https://git.invixiom.com/giteadmin/datatools-dev/releases) (o el listado de Gumroad). Cada paquete ocupa ~300 MB descomprimido; al primer arranque la app levanta un servidor local en http://127.0.0.1:8501 y abre tu navegador predeterminado. Nada sale de tu equipo.
|
||||
|
||||
**Tesseract OCR viene incluido.** El soporte para PDFs escaneados del Extractor de PDF funciona sin configuración adicional en las tres plataformas — no hace falta instalar Tesseract por separado. Atribución de licencia: ver [`LICENSE_TESSERACT.txt`](LICENSE_TESSERACT.txt).
|
||||
|
||||
**Avisos del primer arranque (una sola vez):**
|
||||
- **macOS** sin firma: clic derecho → **Abrir** → confirma. (Las compilaciones firmadas se lo saltan.)
|
||||
- **Windows** SmartScreen: pulsa **Más información** → **Ejecutar de todas formas**.
|
||||
|
||||
Guía detallada de instalación y resolución de problemas: [Guía del usuario §1](docs/USER-GUIDE.es.md#1-instalaci%C3%B3n).
|
||||
|
||||
## Instalar desde el código (desarrolladores)
|
||||
|
||||
|
||||
42
README.md
42
README.md
@@ -8,27 +8,37 @@ Local CSV / Excel cleaning. CLI + browser GUI, no cloud, no install ceremony. GU
|
||||
|
||||
| # | Tool | Status |
|
||||
|---|------|--------|
|
||||
| 01 | **Deduplicator** — exact + fuzzy match, 5 normalizers, survivor rules, audit | Ready |
|
||||
| 02 | **Text Cleaner** — whitespace, smart chars, BOM, line endings, case ops | Ready |
|
||||
| 03 | **Format Standardizer** — dates, phones, emails, addresses, names, currencies, booleans | Ready |
|
||||
| 04 | **Missing Value Handler** — disguised-null detection, profile, mean/median/mode/ffill/bfill/interpolate, drop strategies | Ready |
|
||||
| 05 | **Column Mapper** — fuzzy auto-rename, target schema with type coercion, required fields with defaults, drop/reorder | Ready |
|
||||
| 06 | Outlier Detector | Coming Soon |
|
||||
| 07 | Multi-File Merger | Coming Soon |
|
||||
| 08 | Validator & Reporter | Coming Soon |
|
||||
| 09 | **Pipeline Runner** — chain tools with recommended (not forced) order, save/load JSON, automate weekly cleanups | Ready |
|
||||
| 01 | **Find Duplicates** — exact + fuzzy match, 5 normalizers, survivor rules, audit | Ready |
|
||||
| 02 | **Clean Text** — whitespace, smart chars, BOM, line endings, case ops | Ready |
|
||||
| 03 | **Standardize Formats** — dates, phones, emails, addresses, names, currencies, booleans | Ready |
|
||||
| 04 | **Fix Missing Values** — disguised-null detection, profile, mean/median/mode/ffill/bfill/interpolate, drop strategies | Ready |
|
||||
| 05 | **Map Columns** — fuzzy auto-rename, target schema with type coercion, required fields with defaults, drop/reorder | Ready |
|
||||
| 06 | Find Unusual Values | Coming Soon |
|
||||
| 07 | Combine Files | Coming Soon |
|
||||
| 08 | Quality Check | Coming Soon |
|
||||
| 09 | **Automated Workflows** — chain tools with recommended (not forced) order, save/load JSON, automate weekly cleanups | Ready |
|
||||
|
||||
Every tool page has an in-tool **Help** popover (right of the title) with a compact When-to-use / Steps / Examples / Tip card. Copy lives in the language packs (`tools.<id>.help_md`).
|
||||
|
||||
## Download (non-technical users)
|
||||
|
||||
Pre-built installers — no Python required:
|
||||
Pre-built bundles — no Python install, no admin rights, no internet at runtime. Each release ships an **installer** per OS that wires up Desktop + Start Menu / Launchpad shortcuts.
|
||||
|
||||
| Platform | Download | First-launch note |
|
||||
|---|---|---|
|
||||
| **macOS** | `DataTools-X.Y.Z-mac.dmg` | Drag DataTools.app into /Applications, then double-click. |
|
||||
| **Windows** | `DataTools-X.Y.Z-win-setup.exe` | Run the installer; launches from Start Menu. |
|
||||
| **Linux** | `DataTools-X.Y.Z-linux-x86_64.AppImage` | `chmod +x` the file, then double-click. |
|
||||
| Platform | Installer |
|
||||
|---|---|
|
||||
| **macOS** | `DataTools-X.Y.Z-mac.dmg` — open, drag DataTools.app into /Applications, launch from Launchpad. |
|
||||
| **Windows** | `DataTools-X.Y.Z-win-setup.exe` — run installer (per-user, no admin). Desktop shortcut + Start Menu entry created. |
|
||||
| **Linux** | `DataTools-X.Y.Z-linux-x86_64.AppImage` — `chmod +x`, double-click. The AppImage is already portable. |
|
||||
|
||||
Latest release: see [GitHub Releases](https://git.invixiom.com/giteadmin/datatools-dev/releases) (or the Gumroad listing). The installers are ~150–200 MB; the launcher boots a local server at http://127.0.0.1:8501 and opens your browser. Nothing is sent to the cloud.
|
||||
Latest release: see [GitHub Releases](https://git.invixiom.com/giteadmin/datatools-dev/releases) (or the Gumroad listing). Each bundle is ~300 MB unpacked; on first launch the app starts a local server at http://127.0.0.1:8501 and opens your default browser. Nothing leaves your machine.
|
||||
|
||||
**Tesseract OCR is bundled.** Scanned-PDF support in the PDF Extractor works out of the box on all three platforms — no separate Tesseract install required. License attribution: see [`LICENSE_TESSERACT.txt`](LICENSE_TESSERACT.txt).
|
||||
|
||||
**First-launch warnings (one-time):**
|
||||
- **macOS** unsigned builds: right-click → **Open** → confirm. (Signed builds skip this.)
|
||||
- **Windows** SmartScreen: click **More info** → **Run anyway**.
|
||||
|
||||
Detailed install + troubleshooting walkthrough: [User Guide §1](docs/USER-GUIDE.md#1-install).
|
||||
|
||||
## Install from source (developers)
|
||||
|
||||
|
||||
142
build/README.md
142
build/README.md
@@ -19,23 +19,52 @@ build/
|
||||
│ Mac .app bundle config. Reads the version
|
||||
│ from src/__init__.py.
|
||||
├── installer.iss Inno Setup script — Windows .exe installer.
|
||||
│ Adds Start Menu + Desktop + App Paths entries.
|
||||
├── generate_icons.py Builds icon.ico / icon.icns / icon.png from
|
||||
│ src/gui/assets/datatools_icon_256.png. Run
|
||||
│ once before pyinstaller (CI does this).
|
||||
├── tesseract.py Fetches the per-platform Tesseract binary +
|
||||
│ eng.traineddata at build time. CI imports
|
||||
│ fetch_tessdata + fetch_tesseract_for_platform.
|
||||
├── macos/
|
||||
│ └── build_dmg.sh Wraps dist/DataTools.app into a .dmg with a
|
||||
│ drag-to-/Applications layout.
|
||||
│ drag-to-/Applications layout (installer).
|
||||
├── appimage/
|
||||
│ ├── AppRun Entry point invoked when the AppImage runs.
|
||||
│ ├── datatools.desktop Linux desktop-entry metadata.
|
||||
│ └── build.sh Wraps dist/DataTools/ into an .AppImage.
|
||||
├── hooks/ PyInstaller hooks for libs the static analyser
|
||||
│ └── hook-streamlit.py misses (Streamlit's dynamic imports).
|
||||
├── icon.icns macOS app icon (TODO: produce from a 1024×1024
|
||||
│ PNG. Optional — bundle still builds without).
|
||||
├── icon.ico Windows app icon (TODO).
|
||||
├── icon.png Linux AppImage icon (TODO — build.sh generates
|
||||
│ a placeholder if missing).
|
||||
├── icon.{ico,icns,png} Generated by generate_icons.py — gitignored.
|
||||
└── README.md this file
|
||||
```
|
||||
|
||||
## Distribution outputs per platform
|
||||
|
||||
Each CI run produces one installer per platform:
|
||||
|
||||
| Platform | Installer |
|
||||
|----------|----------------------------------------|
|
||||
| macOS | `DataTools-<ver>-mac.dmg` |
|
||||
| Windows | `DataTools-<ver>-win-setup.exe` |
|
||||
| Linux | `DataTools-<ver>-linux-x86_64.AppImage` (already portable) |
|
||||
|
||||
All three outputs are self-contained: every dependency (Python, pandas,
|
||||
streamlit, pdfplumber, **Tesseract OCR + `eng.traineddata`**, the lot)
|
||||
is frozen into the bundle. The buyer does not need to install Python,
|
||||
pip, Tesseract, or anything else first. With Tesseract bundled, each
|
||||
artifact is roughly **250–300 MB** on disk (up from ~120 MB pre-OCR);
|
||||
unpacked installs run ~300–400 MB once scratch space is counted.
|
||||
|
||||
## Easy-launch surface
|
||||
|
||||
| Affordance | Windows | macOS |
|
||||
|------------------|--------------------------------------------------|------------------------------------------------------|
|
||||
| Desktop shortcut | Inno Setup `desktopicon` task (checked default) | The .app bundle in /Applications is the icon |
|
||||
| App menu | Start Menu → DataTools (always installed) | Launchpad + Spotlight (auto from /Applications) |
|
||||
| Taskbar / Dock | User pins manually (OS forbids programmatic pin) | User pins manually after first launch |
|
||||
| Run from terminal| `DataTools` (registered via App Paths) | `open -a DataTools` (auto from .app bundle) |
|
||||
|
||||
CI: `.github/workflows/build.yml` runs the full pipeline on tag push
|
||||
(matrix: macos-latest, windows-latest, ubuntu-latest) and attaches
|
||||
the resulting installers to a GitHub Release. Manual
|
||||
@@ -43,21 +72,55 @@ the resulting installers to a GitHub Release. Manual
|
||||
|
||||
## Releasing
|
||||
|
||||
### CI build (push tag → GitHub Release) — the release process
|
||||
|
||||
Releases are built by GitHub Actions (`.github/workflows/build.yml`),
|
||||
not on a developer's machine. The matrix runs on
|
||||
macos-latest / windows-latest / ubuntu-latest, stages Tesseract
|
||||
(`build/tesseract.py`), runs PyInstaller, packages the per-platform
|
||||
installer, and attaches it to a GitHub Release on tag push:
|
||||
|
||||
1. Bump `__version__` in `src/__init__.py`.
|
||||
2. `git commit -am "release: vX.Y.Z" && git tag vX.Y.Z`.
|
||||
3. `git push && git push --tags`.
|
||||
4. CI builds all three platforms and creates a GitHub Release with
|
||||
the installers attached.
|
||||
5. Mirror the GitHub Release assets to Gumroad (manual until v2).
|
||||
4. CI builds all three platforms and creates a Release with the
|
||||
installers attached.
|
||||
5. Mirror the Release assets to Gumroad (manual until v2).
|
||||
|
||||
A manual `workflow_dispatch` run does the same build but uploads the
|
||||
installers as workflow artifacts instead of creating a Release —
|
||||
useful for smoke-testing a build without cutting a tag.
|
||||
|
||||
### Local build (single platform, for testing)
|
||||
|
||||
PyInstaller can't cross-compile, so a local build produces only the
|
||||
current OS's installer. This mirrors what CI does, by hand — use it to
|
||||
debug the bundle before tagging. See the per-platform recipes below for
|
||||
the exact commands; the short version is:
|
||||
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
pip install pyinstaller pillow
|
||||
python build/generate_icons.py
|
||||
python -c "import sys; sys.path.insert(0,'build'); \
|
||||
from tesseract import fetch_tessdata, fetch_tesseract_for_platform; \
|
||||
fetch_tessdata(); fetch_tesseract_for_platform('mac')" # win / mac / linux
|
||||
pyinstaller build/datatools.spec --clean --noconfirm
|
||||
# then run the matching packager: build/macos/build_dmg.sh,
|
||||
# build/installer.iss (iscc), or build/appimage/build.sh
|
||||
```
|
||||
|
||||
## Signing (Phase 2 — needs accounts/credentials)
|
||||
|
||||
Both code-signing steps are intentionally not in CI yet because they
|
||||
require credentials the owner sets up first.
|
||||
**macOS signing + notarization is now wired into `build.yml`** (the
|
||||
"Sign & notarize macOS app" step, with `build/macos/entitlements.plist`).
|
||||
It is guarded: if `MACOS_DEVELOPER_ID_CERT_P12_BASE64` is absent the step
|
||||
warns and exits 0, so dry-run dispatches still produce an unsigned build.
|
||||
To activate it, just add the secrets below — no code change needed.
|
||||
**Windows** code-signing is still not wired (accepted v1 friction).
|
||||
|
||||
**macOS** — Apple Developer Program enrollment ($99/yr). Once enrolled,
|
||||
add these GitHub Secrets and uncomment the `codesign` + `notarytool`
|
||||
steps in `build.yml`:
|
||||
add these GitHub Secrets to activate the signing step in `build.yml`:
|
||||
|
||||
| Secret | Value |
|
||||
|---|---|
|
||||
@@ -223,6 +286,57 @@ Mac code-signing in CI requires the cert + private key as a GitHub
|
||||
secret (encoded with `base64`). Detailed walkthrough belongs in a
|
||||
later doc — for v1, sign locally and upload to GitHub Releases.
|
||||
|
||||
## Tesseract bundling (PDF Extractor OCR)
|
||||
|
||||
Frozen artifacts ship a per-platform Tesseract binary plus the English
|
||||
`eng.traineddata` model so scanned-PDF support in the PDF Extractor
|
||||
works out of the box — no separate user install. Source / pip
|
||||
developer setups still need system Tesseract on `PATH`.
|
||||
|
||||
**Layout inside the bundle**:
|
||||
|
||||
```
|
||||
DataTools/ (or DataTools.app/Contents/MacOS/)
|
||||
└── tesseract/
|
||||
├── tesseract (Linux/macOS binary; tesseract.exe on Windows)
|
||||
└── tessdata/
|
||||
└── eng.traineddata
|
||||
```
|
||||
|
||||
The runtime resolver (in `src/`, owned by the runtime team) walks:
|
||||
|
||||
1. `DATATOOLS_TESSERACT_BIN` env var override.
|
||||
2. `Path(sys._MEIPASS) / "tesseract" / "tesseract[.exe]"` — frozen
|
||||
bundles only.
|
||||
3. `tesseract` on `PATH`.
|
||||
4. Windows well-known paths.
|
||||
|
||||
**Where the bytes come from**:
|
||||
|
||||
- **Tessdata** — vendored in-repo at `build/vendor/tessdata/eng.traineddata`
|
||||
(sourced from [tessdata_best](https://github.com/tesseract-ocr/tessdata_best)).
|
||||
`datatools.spec` copies it into `tesseract/tessdata/`.
|
||||
- **Binary** — fetched per-platform at build time by
|
||||
`build/tesseract.py` from pinned upstream URLs. Current pin:
|
||||
**Tesseract 5.5.0**. CI imports `fetch_tessdata` +
|
||||
`fetch_tesseract_for_platform` from this module before PyInstaller.
|
||||
|
||||
**Updating Tesseract**:
|
||||
|
||||
1. Bump the version pin and the per-platform fetch URLs in
|
||||
`build/tesseract.py`.
|
||||
2. If the model schema changed upstream, refresh
|
||||
`build/vendor/tessdata/eng.traineddata` from `tessdata_best` at the
|
||||
matching tag.
|
||||
3. Push a `v*` tag so CI rebuilds all three platforms, then
|
||||
smoke-test a scanned PDF through the PDF Extractor.
|
||||
4. Update `LICENSE_TESSERACT.txt` at the repo root if upstream license
|
||||
terms change (Apache-2.0 today).
|
||||
|
||||
License attribution for the bundled binary lives at
|
||||
`LICENSE_TESSERACT.txt` at the repo root — it must ship alongside any
|
||||
binary that contains Tesseract.
|
||||
|
||||
## Common pitfalls
|
||||
|
||||
| Symptom | Fix |
|
||||
@@ -246,7 +360,7 @@ much state to trust:
|
||||
4. Double-click the app icon.
|
||||
5. Browser should open to http://127.0.0.1:850x within 5 seconds.
|
||||
6. Drop samples/demo/shopify_pet_customers.csv into the
|
||||
Pipeline Runner page; click Run; AFTER preview should appear.
|
||||
Automated Workflows page; click Run; AFTER preview should appear.
|
||||
7. Confirm in the network tab: zero outbound calls except to
|
||||
127.0.0.1 and the Streamlit static asset paths (also local).
|
||||
```
|
||||
|
||||
@@ -9,6 +9,11 @@
|
||||
# latest release from https://github.com/AppImage/AppImageKit/releases).
|
||||
#
|
||||
# Output: dist/DataTools-<version>-linux-x86_64.AppImage
|
||||
#
|
||||
# Tesseract bundling: no-op here. The PyInstaller bundle in
|
||||
# dist/DataTools/ already contains tesseract/{tesseract, *.so,
|
||||
# tessdata/eng.traineddata} from the spec's datas; ``cp -R``
|
||||
# below carries it along into the AppDir.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
|
||||
@@ -24,6 +24,7 @@
|
||||
|
||||
# -*- mode: python ; coding: utf-8 -*-
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from PyInstaller.utils.hooks import (
|
||||
collect_all,
|
||||
@@ -58,6 +59,15 @@ hidden_imports += collect_submodules("charset_normalizer")
|
||||
hidden_imports += collect_submodules("openpyxl")
|
||||
hidden_imports += collect_submodules("loguru")
|
||||
|
||||
# PDF Extractor stack. ``pypdfium2`` has its own PyInstaller hook
|
||||
# under ``build/hooks/`` that pulls in the native PDFium binary —
|
||||
# keep the ``collect_submodules`` calls here for belt-and-braces.
|
||||
hidden_imports += collect_submodules("pdfplumber")
|
||||
hidden_imports += collect_submodules("pdfminer")
|
||||
hidden_imports += collect_submodules("pypdfium2")
|
||||
hidden_imports += collect_submodules("PIL")
|
||||
hidden_imports += collect_submodules("pytesseract")
|
||||
|
||||
# Our own engine + GUI modules. Even though we import them directly
|
||||
# at the top of ``launcher.py`` / ``app.py``, the Streamlit
|
||||
# session-state and per-page page discovery layers re-import via
|
||||
@@ -77,6 +87,14 @@ datas += collect_data_files("streamlit", include_py_files=False)
|
||||
# phonenumbers ships its country/area-code metadata as resources.
|
||||
datas += collect_data_files("phonenumbers", include_py_files=False)
|
||||
|
||||
# PDF Extractor data files. ``pypdfium2`` ships a native PDFium
|
||||
# shared library (``.dll`` / ``.so`` / ``.dylib``) under its package
|
||||
# dir; ``pdfminer`` ships the Adobe CMap tables it uses for
|
||||
# character mapping. The drawable-canvas frontend bundle is gone
|
||||
# now that the visual picker was removed.
|
||||
datas += collect_data_files("pypdfium2", include_py_files=False)
|
||||
datas += collect_data_files("pdfminer", include_py_files=False)
|
||||
|
||||
# Our application files. PyInstaller's bundler treats source as code
|
||||
# (.pyc) by default; we add it again as data so the launcher's
|
||||
# ``Path(sys._MEIPASS) / "src" / "gui" / "app.py"`` resolution works.
|
||||
@@ -86,6 +104,78 @@ datas += [
|
||||
(str(REPO / ".streamlit" / "config.toml"),".streamlit"),
|
||||
]
|
||||
|
||||
# ----- Tesseract OCR bundle ----------------------------------------
|
||||
# ``build/tesseract.py`` stages the per-platform Tesseract binary
|
||||
# + its runtime libs (DLLs/dylibs/sos) into
|
||||
# ``build/_tesseract/<target>/`` and the shared eng.traineddata into
|
||||
# ``build/vendor/tessdata/``. We add both to ``datas`` so PyInstaller
|
||||
# drops them at the path the runtime expects:
|
||||
#
|
||||
# <bundle>/tesseract/tesseract[.exe]
|
||||
# <bundle>/tesseract/<all dll/dylib/so deps>
|
||||
# <bundle>/tesseract/tessdata/eng.traineddata
|
||||
#
|
||||
# The runtime discovery code in src/pdf_extract.py reads this layout
|
||||
# from ``Path(sys._MEIPASS) / "tesseract" / ...``. Keep the two ends
|
||||
# in sync — if you rename "tesseract" here, update pdf_extract.py too.
|
||||
#
|
||||
# CI (.github/workflows/build.yml) sets DATATOOLS_TESS_STAGING to the
|
||||
# right per-platform dir before invoking PyInstaller. For ad-hoc
|
||||
# `pyinstaller build/datatools.spec` runs without that env var, fall
|
||||
# back to the canonical staging path.
|
||||
_tess_staging_env = os.environ.get("DATATOOLS_TESS_STAGING")
|
||||
if _tess_staging_env:
|
||||
_tess_staging = Path(_tess_staging_env)
|
||||
else:
|
||||
# Pick the obvious per-host staging dir as a fallback so spec-only
|
||||
# builds (without the CI env var) still work in dev.
|
||||
import sys as _sys_for_target
|
||||
_target_guess = (
|
||||
"win" if _sys_for_target.platform.startswith("win")
|
||||
else "mac" if _sys_for_target.platform == "darwin"
|
||||
else "linux"
|
||||
)
|
||||
_tess_staging = REPO / "build" / "_tesseract" / _target_guess
|
||||
|
||||
_tessdata = REPO / "build" / "vendor" / "tessdata"
|
||||
|
||||
if _tess_staging.is_dir() and any(_tess_staging.iterdir()):
|
||||
# Drop every file in the staging dir directly under
|
||||
# ``<bundle>/tesseract/`` (binary + DLL/dylib/so siblings).
|
||||
datas += [(str(_tess_staging), "tesseract")]
|
||||
else:
|
||||
# Don't hard-fail spec parse — useful for first-time devs running
|
||||
# PyInstaller before fetching binaries. Surface a loud warning
|
||||
# though, since the OCR feature will silently fail at runtime.
|
||||
print(
|
||||
f"WARNING: {_tess_staging} is empty or missing — OCR will be "
|
||||
"disabled in the bundle. Run build/tesseract.py's "
|
||||
"fetch_tesseract_for_platform before pyinstaller, or "
|
||||
"pre-stage the binary manually."
|
||||
)
|
||||
|
||||
if (_tessdata / "eng.traineddata").exists():
|
||||
datas += [(str(_tessdata), "tesseract/tessdata")]
|
||||
else:
|
||||
print(
|
||||
f"WARNING: {_tessdata}/eng.traineddata is missing — OCR will "
|
||||
"have no language data at runtime. Run build/tesseract.py's "
|
||||
"fetch_tessdata or fetch manually per build/vendor/README.md."
|
||||
)
|
||||
|
||||
# Bundle the Apache-2.0 LICENSE text alongside the binary. The docs
|
||||
# agent maintains LICENSE_TESSERACT.txt at the repo root; PyInstaller
|
||||
# drops it at the bundle root next to DataTools[.exe].
|
||||
_tess_license = REPO / "LICENSE_TESSERACT.txt"
|
||||
if _tess_license.exists():
|
||||
datas += [(str(_tess_license), ".")]
|
||||
else:
|
||||
print(
|
||||
"WARNING: LICENSE_TESSERACT.txt missing at repo root. Required "
|
||||
"by Apache-2.0 for redistribution; the docs agent should "
|
||||
"create it. Continuing without it for now."
|
||||
)
|
||||
|
||||
# ----- Analysis ------------------------------------------------------
|
||||
|
||||
a = Analysis(
|
||||
@@ -141,6 +231,13 @@ coll = COLLECT(
|
||||
|
||||
# macOS .app bundle wrapper. PyInstaller produces it only on Mac;
|
||||
# this block is a no-op on Win/Linux.
|
||||
#
|
||||
# Tesseract bundling note: ``BUNDLE(coll, ...)`` carries the entire
|
||||
# COLLECT output (binaries + datas) into the .app's
|
||||
# Contents/Resources tree, so the ``tesseract/`` subdir we built up
|
||||
# in ``datas`` lands at ``DataTools.app/Contents/Resources/tesseract/``
|
||||
# and the runtime ``sys._MEIPASS`` resolves there. No extra plumbing
|
||||
# needed.
|
||||
import sys as _sys
|
||||
if _sys.platform == "darwin":
|
||||
app = BUNDLE(
|
||||
|
||||
78
build/generate_icons.py
Normal file
78
build/generate_icons.py
Normal file
@@ -0,0 +1,78 @@
|
||||
"""Generate platform-specific app icons from the source PNG asset.
|
||||
|
||||
Outputs:
|
||||
build/icon.ico Windows multi-resolution icon (16..256 px sizes).
|
||||
build/icon.icns macOS icon bundle (16..1024 px scaled tiers).
|
||||
build/icon.png Plain 256x256 PNG used by the Linux AppImage.
|
||||
|
||||
Source: ``src/gui/assets/datatools_icon_256.png`` (the same icon
|
||||
``st.set_page_config`` uses, so the installer / Dock / Taskbar match
|
||||
the in-app tab favicon).
|
||||
|
||||
Run manually:
|
||||
python build/generate_icons.py
|
||||
|
||||
CI runs this automatically before invoking PyInstaller (see
|
||||
``.github/workflows/build.yml``). Both files are .gitignored — they
|
||||
are build artifacts derived from the committed PNG.
|
||||
|
||||
Self-contained: pulls only Pillow (already a transitive dep of
|
||||
``pdfplumber``) so no extra installs are required.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image
|
||||
|
||||
# Repo layout: this script lives at <REPO>/build/. The source PNG is at
|
||||
# <REPO>/src/gui/assets/datatools_icon_256.png.
|
||||
BUILD_DIR = Path(__file__).resolve().parent
|
||||
REPO = BUILD_DIR.parent
|
||||
SOURCE_PNG = REPO / "src" / "gui" / "assets" / "datatools_icon_256.png"
|
||||
|
||||
# Windows ICO needs every size the OS might render at: taskbar (16/24),
|
||||
# Start Menu (32/48), tile (64/128), shell properties dialog (256).
|
||||
ICO_SIZES = [(16, 16), (24, 24), (32, 32), (48, 48), (64, 64),
|
||||
(128, 128), (256, 256)]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if not SOURCE_PNG.exists():
|
||||
sys.stderr.write(
|
||||
f"Source icon not found at {SOURCE_PNG}.\n"
|
||||
"Add a 256x256 (or larger) RGBA PNG there and re-run.\n"
|
||||
)
|
||||
return 1
|
||||
|
||||
src = Image.open(SOURCE_PNG).convert("RGBA")
|
||||
if src.size[0] < 256 or src.size[1] < 256:
|
||||
sys.stderr.write(
|
||||
f"Source icon is {src.size}; recommend 256x256 or larger "
|
||||
"so downscaled tiers look crisp.\n"
|
||||
)
|
||||
|
||||
ico_path = BUILD_DIR / "icon.ico"
|
||||
src.save(ico_path, format="ICO", sizes=ICO_SIZES)
|
||||
print(f"wrote {ico_path} ({ico_path.stat().st_size:,} bytes)")
|
||||
|
||||
icns_path = BUILD_DIR / "icon.icns"
|
||||
# Pillow's ICNS writer derives the per-tier sizes from the source
|
||||
# image; passing a 256x256 source yields ic07..ic12 entries which
|
||||
# cover Finder, Dock, and the Get Info panel.
|
||||
src.save(icns_path, format="ICNS")
|
||||
print(f"wrote {icns_path} ({icns_path.stat().st_size:,} bytes)")
|
||||
|
||||
# AppImage uses a plain PNG for its desktop entry. Copy the source
|
||||
# so the AppImage build script doesn't have to know the asset path.
|
||||
png_path = BUILD_DIR / "icon.png"
|
||||
src.save(png_path, format="PNG")
|
||||
print(f"wrote {png_path} ({png_path.stat().st_size:,} bytes)")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
31
build/hooks/hook-pypdfium2.py
Normal file
31
build/hooks/hook-pypdfium2.py
Normal file
@@ -0,0 +1,31 @@
|
||||
"""PyInstaller hook for pypdfium2.
|
||||
|
||||
``pypdfium2`` ships the native PDFium shared library as a data file
|
||||
inside its package directory (``pdfium``-prefixed ``.dll`` on
|
||||
Windows, ``.so`` on Linux, ``.dylib`` on macOS). PyInstaller's
|
||||
default discovery picks up Python ``.py``/``.pyc`` but can miss
|
||||
the binary if the package is wheel-installed and the shared lib
|
||||
isn't on the ``__init__``'s module-level path it scans.
|
||||
|
||||
This hook is belt-and-braces — the main spec already calls
|
||||
``collect_data_files("pypdfium2")`` and ``collect_submodules``,
|
||||
but PyInstaller's hook-discovery-by-name is the documented
|
||||
escape hatch for native-bundled libraries. Without this, the
|
||||
visual picker (which renders PDF pages via
|
||||
``pypdfium2.PdfDocument(...).render(...)``) silently fails on
|
||||
installed builds with a ``FileNotFoundError`` for the PDFium
|
||||
shared library.
|
||||
"""
|
||||
|
||||
from PyInstaller.utils.hooks import (
|
||||
collect_all,
|
||||
collect_data_files,
|
||||
collect_dynamic_libs,
|
||||
)
|
||||
|
||||
datas, binaries, hiddenimports = collect_all("pypdfium2")
|
||||
# Make absolutely sure the bundled PDFium .dll/.so/.dylib is
|
||||
# carried over — PyInstaller treats it as a dynamic lib, not data.
|
||||
binaries += collect_dynamic_libs("pypdfium2")
|
||||
# And its raw data files (the type stubs + metadata file).
|
||||
datas += collect_data_files("pypdfium2", include_py_files=False)
|
||||
@@ -1,11 +1,26 @@
|
||||
; Inno Setup script for DataTools — Windows installer.
|
||||
;
|
||||
; Compile from the repo root:
|
||||
; iscc /DAppVersion=1.0.0 build\installer.iss
|
||||
; iscc /DAppVersion=3.0 build\installer.iss
|
||||
;
|
||||
; CI passes the version via /DAppVersion to keep src/__init__.py the
|
||||
; single source of truth. Local manual builds: pass /DAppVersion or
|
||||
; let the default kick in.
|
||||
;
|
||||
; What this installer wires up (covers the "easy launch" surface):
|
||||
; * Start Menu group: Start → DataTools → DataTools / Uninstall
|
||||
; * Desktop shortcut: optional, checked by default during install
|
||||
; * Quick Launch: optional, off by default (legacy Win 7 + power
|
||||
; users who keep the bar enabled). Windows 10/11
|
||||
; users pin to taskbar manually via right-click —
|
||||
; OS security policy forbids programmatic pinning.
|
||||
; * App Paths entry: so ``DataTools`` typed into Win+R / cmd works.
|
||||
;
|
||||
; Self-contained: the installer contains a frozen PyInstaller bundle
|
||||
; (Python + every runtime dep). No pre-install or post-install steps
|
||||
; on the buyer's machine. UAC is NOT required because we install
|
||||
; per-user by default; the prompt only fires if the buyer asks for an
|
||||
; all-users install.
|
||||
|
||||
#ifndef AppVersion
|
||||
#define AppVersion "0.0.0-dev"
|
||||
@@ -18,11 +33,15 @@ AppVersion={#AppVersion}
|
||||
AppVerName=DataTools {#AppVersion}
|
||||
AppPublisher=DataTools
|
||||
AppPublisherURL=https://datatools.app
|
||||
AppSupportURL=https://datatools.app/support
|
||||
AppUpdatesURL=https://datatools.app/releases
|
||||
DefaultDirName={autopf}\DataTools
|
||||
DefaultGroupName=DataTools
|
||||
DisableProgramGroupPage=yes
|
||||
OutputDir=..\dist
|
||||
OutputBaseFilename=DataTools-{#AppVersion}-win-setup
|
||||
SetupIconFile=icon.ico
|
||||
UninstallDisplayIcon={app}\DataTools.exe
|
||||
Compression=lzma2/max
|
||||
SolidCompression=yes
|
||||
WizardStyle=modern
|
||||
@@ -30,20 +49,45 @@ ArchitecturesInstallIn64BitMode=x64
|
||||
PrivilegesRequired=lowest
|
||||
PrivilegesRequiredOverridesAllowed=dialog
|
||||
; Allow per-user install (no UAC prompt) when admin isn't available.
|
||||
; Buyers without admin rights can still install without IT involvement.
|
||||
|
||||
ChangesAssociations=no
|
||||
CloseApplications=force
|
||||
RestartApplications=no
|
||||
|
||||
[Languages]
|
||||
Name: "english"; MessagesFile: "compiler:Default.isl"
|
||||
|
||||
[Tasks]
|
||||
Name: "desktopicon"; Description: "Create a &desktop shortcut"; GroupDescription: "Additional shortcuts:"
|
||||
Name: "quicklaunchicon"; Description: "Create a &Quick Launch shortcut"; GroupDescription: "Additional shortcuts:"; Flags: unchecked; OnlyBelowVersion: 6.1
|
||||
|
||||
[Files]
|
||||
; PyInstaller's dist/DataTools/ tree includes:
|
||||
; * DataTools.exe + frozen Python runtime
|
||||
; * tesseract/tesseract.exe + DLLs + tessdata/eng.traineddata
|
||||
; (bundled via build/datatools.spec datas; runtime discovery in
|
||||
; src/pdf_extract.py reads sys._MEIPASS / "tesseract" / ...).
|
||||
; * LICENSE_TESSERACT.txt at the bundle root (Apache-2.0).
|
||||
; The recursesubdirs flag below picks all of those up — no separate
|
||||
; Files: entry needed for tesseract/.
|
||||
Source: "..\dist\DataTools\*"; DestDir: "{app}"; Flags: recursesubdirs ignoreversion
|
||||
|
||||
[Icons]
|
||||
Name: "{group}\DataTools"; Filename: "{app}\DataTools.exe"
|
||||
; Start Menu entries — created unconditionally so the app is always
|
||||
; discoverable via Start search.
|
||||
Name: "{group}\DataTools"; Filename: "{app}\DataTools.exe"; IconFilename: "{app}\DataTools.exe"
|
||||
Name: "{group}\Uninstall DataTools"; Filename: "{uninstallexe}"
|
||||
Name: "{autodesktop}\DataTools"; Filename: "{app}\DataTools.exe"; Tasks: desktopicon
|
||||
; Desktop shortcut — opt-in via the Tasks page.
|
||||
Name: "{autodesktop}\DataTools"; Filename: "{app}\DataTools.exe"; IconFilename: "{app}\DataTools.exe"; Tasks: desktopicon
|
||||
; Quick Launch (legacy) — only relevant on Win 7 and older.
|
||||
Name: "{userappdata}\Microsoft\Internet Explorer\Quick Launch\DataTools"; Filename: "{app}\DataTools.exe"; IconFilename: "{app}\DataTools.exe"; Tasks: quicklaunchicon
|
||||
|
||||
[Registry]
|
||||
; App Paths — lets the buyer launch from Win+R or cmd with just
|
||||
; "DataTools" instead of a full path. Per-user hive so the per-user
|
||||
; install path doesn't need admin to register.
|
||||
Root: HKCU; Subkey: "Software\Microsoft\Windows\CurrentVersion\App Paths\DataTools.exe"; ValueType: string; ValueName: ""; ValueData: "{app}\DataTools.exe"; Flags: uninsdeletekey
|
||||
|
||||
[Run]
|
||||
Filename: "{app}\DataTools.exe"; Description: "Launch DataTools"; Flags: nowait postinstall skipifsilent
|
||||
|
||||
@@ -10,6 +10,11 @@
|
||||
#
|
||||
# Code signing + notarization happen separately (see build/README.md
|
||||
# "Signing"). This script only handles the packaging step.
|
||||
#
|
||||
# Tesseract bundling: no-op here. The .app already contains
|
||||
# Contents/Resources/tesseract/{tesseract, *.dylib, tessdata/} thanks
|
||||
# to PyInstaller's BUNDLE() carrying the spec's datas through. This
|
||||
# script just wraps the finished .app — no extra steps for OCR.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
|
||||
28
build/macos/entitlements.plist
Normal file
28
build/macos/entitlements.plist
Normal file
@@ -0,0 +1,28 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<!--
|
||||
Hardened-runtime entitlements for the notarized DataTools.app.
|
||||
|
||||
PyInstaller freezes a CPython interpreter that maps writable+executable
|
||||
memory and loads many unsigned .so/.dylib modules at runtime. Without
|
||||
these entitlements the hardened runtime kills the process on launch
|
||||
(or notarization rejects the bundle). Keep this list minimal — the app
|
||||
is a local-only Streamlit server, so no network-server/device/camera
|
||||
entitlements are needed.
|
||||
-->
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<!-- CPython JIT-style writable/executable memory + ctypes trampolines -->
|
||||
<key>com.apple.security.cs.allow-jit</key>
|
||||
<true/>
|
||||
<key>com.apple.security.cs.allow-unsigned-executable-memory</key>
|
||||
<true/>
|
||||
<!-- Load the bundled C-extension .so / .dylib modules (pandas, pdfplumber,
|
||||
Pillow, the bundled Tesseract dylibs) that aren't Team-ID signed -->
|
||||
<key>com.apple.security.cs.disable-library-validation</key>
|
||||
<true/>
|
||||
<!-- Launcher sets DATATOOLS_*/TESSDATA_PREFIX/PYTHON* before exec -->
|
||||
<key>com.apple.security.cs.allow-dyld-environment-variables</key>
|
||||
<true/>
|
||||
</dict>
|
||||
</plist>
|
||||
453
build/tesseract.py
Normal file
453
build/tesseract.py
Normal file
@@ -0,0 +1,453 @@
|
||||
"""Tesseract bundling helpers for the release build.
|
||||
|
||||
PDF Extractor OCR ships a per-platform Tesseract binary plus the English
|
||||
``eng.traineddata`` model inside the frozen PyInstaller bundle so scanned
|
||||
PDFs work without a separate user install. These helpers fetch the binary
|
||||
and tessdata at build time; the GitHub Actions workflow
|
||||
(``.github/workflows/build.yml``) imports ``fetch_tessdata`` and
|
||||
``fetch_tesseract_for_platform`` and runs them before PyInstaller.
|
||||
|
||||
Everything is staged under ``build/_tesseract/<platform>/`` (gitignored).
|
||||
The PyInstaller spec (``build/datatools.spec``) reads that staging dir plus
|
||||
``build/vendor/tessdata/`` and bundles them under ``<bundle>/tesseract/``,
|
||||
where the runtime discovery code in ``src/pdf_extract.py`` expects:
|
||||
Path(sys._MEIPASS) / "tesseract" / "tesseract[.exe]"
|
||||
Path(sys._MEIPASS) / "tesseract" / "tessdata" / "eng.traineddata"
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent
|
||||
BUILD = REPO / "build"
|
||||
|
||||
# Tesseract bundling. The runtime discovery code in
|
||||
# ``src/pdf_extract.py`` looks for the binary at
|
||||
# ``Path(sys._MEIPASS) / "tesseract" / "tesseract[.exe]"`` and tessdata
|
||||
# at ``... / "tesseract" / "tessdata" / "eng.traineddata"``. We stage
|
||||
# everything under ``build/_tesseract/<platform>/`` (gitignored) and
|
||||
# the PyInstaller spec adds that staging dir to ``datas=`` so it lands
|
||||
# at the right place inside the frozen bundle.
|
||||
TESSERACT_VERSION = "5.5.0"
|
||||
TESSDATA_DIR = BUILD / "vendor" / "tessdata"
|
||||
TESSDATA_URL = (
|
||||
"https://github.com/tesseract-ocr/tessdata_best/raw/main/eng.traineddata"
|
||||
)
|
||||
TESSERACT_STAGING = BUILD / "_tesseract"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Output helpers — colourless so logs stay readable in any terminal/CI tail.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _step(msg: str) -> None:
|
||||
print(f"\n==> {msg}", flush=True)
|
||||
|
||||
|
||||
def _ok(msg: str) -> None:
|
||||
print(f" ok: {msg}", flush=True)
|
||||
|
||||
|
||||
def _warn(msg: str) -> None:
|
||||
print(f" warn: {msg}", flush=True)
|
||||
|
||||
|
||||
def _err(msg: str) -> None:
|
||||
print(f" ERROR: {msg}", file=sys.stderr, flush=True)
|
||||
|
||||
|
||||
def _run(cmd: list[str], cwd: Path | None = None, env: dict | None = None) -> None:
|
||||
"""Run *cmd*, stream output, exit on failure with a useful banner."""
|
||||
printable = " ".join(map(str, cmd))
|
||||
print(f" $ {printable}", flush=True)
|
||||
try:
|
||||
subprocess.run(cmd, check=True, cwd=cwd or REPO, env=env)
|
||||
except subprocess.CalledProcessError as e:
|
||||
_err(f"command failed (exit {e.returncode}): {printable}")
|
||||
sys.exit(e.returncode)
|
||||
except FileNotFoundError:
|
||||
_err(f"command not found: {cmd[0]}")
|
||||
sys.exit(127)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tesseract bundling — fetch the binary + tessdata at build time.
|
||||
#
|
||||
# We download (not vendor) because:
|
||||
# * Binaries are large (5-40 MB per platform) and license-encumbered
|
||||
# to keep current in git.
|
||||
# * tessdata is Apache-2.0 and ~16 MB — fine to redistribute but
|
||||
# bloats clones for contributors who don't touch OCR.
|
||||
#
|
||||
# Caching layout:
|
||||
# build/_tesseract/win/tesseract.exe + DLLs
|
||||
# build/_tesseract/mac/tesseract + dylibs
|
||||
# build/_tesseract/linux/tesseract + libs
|
||||
# build/vendor/tessdata/eng.traineddata (shared across platforms)
|
||||
#
|
||||
# The PyInstaller spec reads ``build/_tesseract/<platform>/`` and the
|
||||
# tessdata dir, then bundles them under ``<bundle>/tesseract/``.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _download(url: str, dest: Path, *, expected_min_bytes: int = 1024) -> None:
|
||||
"""Download *url* to *dest* atomically. Sanity-check the size."""
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = dest.with_suffix(dest.suffix + ".part")
|
||||
print(f" GET {url}", flush=True)
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=120) as r, open(tmp, "wb") as f:
|
||||
shutil.copyfileobj(r, f)
|
||||
except Exception as e: # noqa: BLE001 — bubble any network error up
|
||||
if tmp.exists():
|
||||
tmp.unlink()
|
||||
_err(f"download failed: {url}\n {e}")
|
||||
raise
|
||||
size = tmp.stat().st_size
|
||||
if size < expected_min_bytes:
|
||||
tmp.unlink()
|
||||
raise RuntimeError(
|
||||
f"downloaded file too small ({size} bytes < {expected_min_bytes}); "
|
||||
f"the URL probably 404'd into an HTML error page."
|
||||
)
|
||||
tmp.replace(dest)
|
||||
_ok(f"downloaded {dest.name} ({size / (1024 * 1024):.1f} MB)")
|
||||
|
||||
|
||||
def fetch_tessdata() -> Path:
|
||||
"""Ensure ``build/vendor/tessdata/eng.traineddata`` exists; return its path.
|
||||
|
||||
Shared across platforms. Downloaded once and cached. The
|
||||
runtime expects this file at ``<bundle>/tesseract/tessdata/eng.traineddata``;
|
||||
the PyInstaller spec handles the placement.
|
||||
"""
|
||||
_step("fetch tessdata (eng.traineddata)")
|
||||
TESSDATA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
target = TESSDATA_DIR / "eng.traineddata"
|
||||
if target.exists() and target.stat().st_size > 1_000_000:
|
||||
_ok(f"already cached: {target.relative_to(REPO)} "
|
||||
f"({target.stat().st_size / (1024 * 1024):.1f} MB)")
|
||||
return target
|
||||
# ~16 MB on disk for the "best" model. Allow some slack on the
|
||||
# min-bytes check (3 MB) so we still catch HTML 404 pages.
|
||||
_download(TESSDATA_URL, target, expected_min_bytes=3 * 1024 * 1024)
|
||||
return target
|
||||
|
||||
|
||||
def _fetch_tesseract_windows(staging: Path) -> None:
|
||||
"""Stage tesseract.exe + DLLs into *staging*.
|
||||
|
||||
Strategy (no easy stand-alone Windows tarball exists — UB-Mannheim
|
||||
ships the canonical Windows builds as Inno Setup installers):
|
||||
|
||||
1. Download the installer .exe from the UB-Mannheim mirror.
|
||||
2. Extract it with 7-Zip (which can read Inno Setup archives via
|
||||
the {app} group). 7-Zip is preinstalled on
|
||||
``windows-latest`` GitHub Actions runners (`C:\\Program Files\\7-Zip\\7z.exe`).
|
||||
3. Copy tesseract.exe + every DLL + the tessdata dir from the
|
||||
extraction into ``staging/``.
|
||||
|
||||
The DLL set tesseract.exe needs at runtime (per UB-Mannheim's
|
||||
Inno Setup script):
|
||||
libtesseract-5.dll, libleptonica-6.dll, libgomp-1.dll,
|
||||
libstdc++-6.dll, libwinpthread-1.dll, libgcc_s_seh-1.dll,
|
||||
liblz4.dll, libjpeg-8.dll, libpng16-16.dll, libtiff-6.dll,
|
||||
libwebp-7.dll, libwebpmux-3.dll, libopenjp2-7.dll, zlib1.dll
|
||||
The whole {app} tree from the installer is ~120 MB; we copy
|
||||
just the .exe + .dll files (~50 MB) since the runtime only
|
||||
needs the binary and its direct deps.
|
||||
"""
|
||||
# UB-Mannheim posts builds under a versioned filename; the exact
|
||||
# build revision changes (5.5.0.20241111 at time of writing).
|
||||
# We pin a specific rev so reproducible builds don't drift.
|
||||
rev = "20241111" # patch rev for tesseract 5.5.0 on the UB-Mannheim mirror
|
||||
fname = f"tesseract-ocr-w64-setup-{TESSERACT_VERSION}.{rev}.exe"
|
||||
url = f"https://digi.bib.uni-mannheim.de/tesseract/{fname}"
|
||||
|
||||
cache = TESSERACT_STAGING / fname
|
||||
if not cache.exists():
|
||||
_download(url, cache, expected_min_bytes=20 * 1024 * 1024)
|
||||
|
||||
# 7-Zip is preinstalled on windows-latest runners; on a dev box
|
||||
# the user installs it (choco install 7zip) or substitutes
|
||||
# innoextract. Locate it.
|
||||
sevenz = (
|
||||
shutil.which("7z")
|
||||
or shutil.which("7z.exe")
|
||||
or r"C:\Program Files\7-Zip\7z.exe"
|
||||
)
|
||||
if not Path(sevenz).exists() and not shutil.which("7z"):
|
||||
_err(
|
||||
"7-Zip not found. On Windows CI runners it's preinstalled; "
|
||||
"on a dev box install via ``choco install 7zip`` or extract "
|
||||
f"{cache} manually into {staging}/ and re-run with "
|
||||
"TESSERACT_SKIP_FETCH=1."
|
||||
)
|
||||
raise FileNotFoundError("7z")
|
||||
|
||||
extract = TESSERACT_STAGING / "win_extract"
|
||||
if extract.exists():
|
||||
shutil.rmtree(extract)
|
||||
extract.mkdir(parents=True)
|
||||
_run([str(sevenz), "x", "-y", f"-o{extract}", str(cache)])
|
||||
|
||||
staging.mkdir(parents=True, exist_ok=True)
|
||||
# The Inno Setup payload lands under ``{app}/`` inside the
|
||||
# extraction. Recursively grab tesseract.exe + DLLs.
|
||||
found_exe = False
|
||||
for root, _dirs, files in os.walk(extract):
|
||||
for f in files:
|
||||
src = Path(root) / f
|
||||
if f.lower() == "tesseract.exe":
|
||||
shutil.copy2(src, staging / "tesseract.exe")
|
||||
found_exe = True
|
||||
elif f.lower().endswith(".dll"):
|
||||
shutil.copy2(src, staging / f)
|
||||
if not found_exe:
|
||||
raise RuntimeError(
|
||||
f"tesseract.exe not found inside extracted installer at {extract}"
|
||||
)
|
||||
_ok(f"staged Windows tesseract into {staging.relative_to(REPO)}")
|
||||
|
||||
|
||||
def _fetch_tesseract_macos(staging: Path) -> None:
|
||||
"""Stage tesseract + dylibs into *staging* on macOS.
|
||||
|
||||
Strategy: use Homebrew. ``brew install tesseract`` is the
|
||||
sanctioned macOS path and the binary it installs is the same one
|
||||
every guide on the internet points at. We copy the binary +
|
||||
every dylib it links against into the staging dir, then run
|
||||
``install_name_tool`` to rewrite the load paths so the binary
|
||||
works after relocation into the .app bundle.
|
||||
|
||||
Caveat: ``brew`` must be on PATH (it is on ``macos-latest``
|
||||
runners). If it isn't, we surface a helpful error rather than
|
||||
fail mysteriously.
|
||||
"""
|
||||
if not shutil.which("brew"):
|
||||
_err(
|
||||
"Homebrew not found. On macos-latest GitHub runners it's "
|
||||
"preinstalled; on a dev Mac install from https://brew.sh and "
|
||||
"re-run. Alternatively pre-stage tesseract into "
|
||||
f"{staging}/ and set TESSERACT_SKIP_FETCH=1."
|
||||
)
|
||||
raise FileNotFoundError("brew")
|
||||
|
||||
# ``brew install`` is idempotent — fine to run on every build. We
|
||||
# don't pin the version through brew because brew tracks its own
|
||||
# taps; instead we assert the version matches TESSERACT_VERSION
|
||||
# after install.
|
||||
_run(["brew", "install", "tesseract"])
|
||||
|
||||
# Find the binary brew just installed.
|
||||
tess_path = shutil.which("tesseract")
|
||||
if not tess_path:
|
||||
raise RuntimeError("brew install tesseract succeeded but tesseract not on PATH")
|
||||
|
||||
staging.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(tess_path, staging / "tesseract")
|
||||
|
||||
# Copy every non-system dylib the binary links against. The
|
||||
# ``otool -L`` output lists absolute paths under /opt/homebrew/
|
||||
# (Apple Silicon) or /usr/local/ (Intel). We skip /usr/lib/* and
|
||||
# /System/* (Apple-shipped, present on every Mac).
|
||||
try:
|
||||
otool = subprocess.run(
|
||||
["otool", "-L", str(staging / "tesseract")],
|
||||
check=True, capture_output=True, text=True,
|
||||
)
|
||||
except subprocess.CalledProcessError as e:
|
||||
raise RuntimeError(f"otool failed: {e.stderr}") from e
|
||||
|
||||
deps = []
|
||||
for line in otool.stdout.splitlines()[1:]:
|
||||
path = line.strip().split(" ", 1)[0]
|
||||
if path.startswith(("/opt/homebrew/", "/usr/local/")):
|
||||
deps.append(path)
|
||||
|
||||
# Copy each dep and its transitive deps. One level of recursion
|
||||
# is usually enough for the tesseract dep tree (libtesseract →
|
||||
# libleptonica → libpng/libjpeg/libtiff/libwebp).
|
||||
copied: set[str] = set()
|
||||
|
||||
def _copy_with_deps(libpath: str) -> None:
|
||||
if libpath in copied or not Path(libpath).exists():
|
||||
return
|
||||
copied.add(libpath)
|
||||
dest = staging / Path(libpath).name
|
||||
shutil.copy2(libpath, dest)
|
||||
# Rewrite the dest's own load path to @loader_path so the
|
||||
# bundle is relocatable.
|
||||
try:
|
||||
subprocess.run(
|
||||
["install_name_tool", "-id", f"@loader_path/{Path(libpath).name}", str(dest)],
|
||||
check=True, capture_output=True,
|
||||
)
|
||||
except subprocess.CalledProcessError:
|
||||
# Not fatal — install_name_tool refuses on already-relative
|
||||
# IDs. The dyld loader will still find them via
|
||||
# @loader_path rewrites on the consumer side.
|
||||
pass
|
||||
# Walk this lib's own deps.
|
||||
try:
|
||||
sub = subprocess.run(
|
||||
["otool", "-L", libpath], check=True, capture_output=True, text=True,
|
||||
)
|
||||
for sub_line in sub.stdout.splitlines()[1:]:
|
||||
sub_path = sub_line.strip().split(" ", 1)[0]
|
||||
if sub_path.startswith(("/opt/homebrew/", "/usr/local/")):
|
||||
_copy_with_deps(sub_path)
|
||||
except subprocess.CalledProcessError:
|
||||
pass
|
||||
|
||||
for dep in deps:
|
||||
_copy_with_deps(dep)
|
||||
|
||||
# Rewrite the tesseract binary's references to point at
|
||||
# @loader_path/<dyname> so it can find its deps inside the bundle.
|
||||
bin_path = staging / "tesseract"
|
||||
for dep in deps:
|
||||
try:
|
||||
subprocess.run(
|
||||
["install_name_tool", "-change", dep,
|
||||
f"@loader_path/{Path(dep).name}", str(bin_path)],
|
||||
check=True, capture_output=True,
|
||||
)
|
||||
except subprocess.CalledProcessError:
|
||||
pass
|
||||
|
||||
_ok(f"staged macOS tesseract + {len(copied)} dylibs into {staging.relative_to(REPO)}")
|
||||
|
||||
|
||||
def _fetch_tesseract_linux(staging: Path) -> None:
|
||||
"""Stage tesseract + .so files into *staging* on Linux.
|
||||
|
||||
Strategy: ``apt-get install tesseract-ocr libtesseract5``
|
||||
(preinstalled on most ubuntu-latest images; we run install
|
||||
anyway because the package is idempotent). Then copy the
|
||||
binary + every .so it links against into staging. ``patchelf``
|
||||
rewrites RPATH so the bundle is relocatable.
|
||||
"""
|
||||
if not shutil.which("apt-get") and not shutil.which("tesseract"):
|
||||
_err(
|
||||
"Neither apt-get nor a pre-installed tesseract found. On "
|
||||
"ubuntu-latest runners both are present. On other distros "
|
||||
"install tesseract-ocr via your package manager and re-run "
|
||||
"with TESSERACT_SKIP_FETCH=1 after pre-staging the binary."
|
||||
)
|
||||
raise FileNotFoundError("tesseract")
|
||||
|
||||
if shutil.which("apt-get") and not shutil.which("tesseract"):
|
||||
_run(["sudo", "apt-get", "update"])
|
||||
_run(["sudo", "apt-get", "install", "-y", "tesseract-ocr", "libtesseract5"])
|
||||
|
||||
tess_path = shutil.which("tesseract")
|
||||
if not tess_path:
|
||||
raise RuntimeError("apt-get install succeeded but tesseract not on PATH")
|
||||
|
||||
staging.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(tess_path, staging / "tesseract")
|
||||
|
||||
# Collect .so dependencies via ldd. Skip the dynamic linker and
|
||||
# libc/libpthread/libdl/libm/libstdc++/libgcc_s — those are
|
||||
# guaranteed to exist on every Linux target and shipping them can
|
||||
# cause GLIBC mismatch errors on older distros. The interesting
|
||||
# tesseract-specific deps are libtesseract, libleptonica, and the
|
||||
# image format libs (libpng, libjpeg, libtiff, libwebp, libgif).
|
||||
SKIP_PREFIXES = (
|
||||
"linux-vdso", "/lib64/ld-linux", "/lib/ld-linux",
|
||||
"libc.so", "libdl.so", "libpthread.so", "libm.so",
|
||||
"librt.so", "libnsl.so", "libutil.so",
|
||||
)
|
||||
try:
|
||||
ldd = subprocess.run(
|
||||
["ldd", str(staging / "tesseract")],
|
||||
check=True, capture_output=True, text=True,
|
||||
)
|
||||
except subprocess.CalledProcessError as e:
|
||||
raise RuntimeError(f"ldd failed: {e.stderr}") from e
|
||||
|
||||
copied = 0
|
||||
for line in ldd.stdout.splitlines():
|
||||
# Format: " libfoo.so.N => /path/to/libfoo.so.N (0x...)"
|
||||
parts = line.split("=>")
|
||||
if len(parts) != 2:
|
||||
continue
|
||||
soname = parts[0].strip()
|
||||
if soname.startswith(SKIP_PREFIXES):
|
||||
continue
|
||||
path_part = parts[1].strip().split(" ", 1)[0]
|
||||
if not path_part or not Path(path_part).exists():
|
||||
continue
|
||||
shutil.copy2(path_part, staging / Path(path_part).name)
|
||||
copied += 1
|
||||
|
||||
# patchelf is optional — if present, rewrite RPATH to $ORIGIN so
|
||||
# the binary finds its bundled .so files. If absent, the
|
||||
# PyInstaller LD_LIBRARY_PATH that the launcher sets will cover
|
||||
# it (we already chdir into _MEIPASS for the runtime).
|
||||
if shutil.which("patchelf"):
|
||||
try:
|
||||
_run(["patchelf", "--set-rpath", "$ORIGIN", str(staging / "tesseract")])
|
||||
except SystemExit:
|
||||
_warn("patchelf rpath rewrite failed — relying on LD_LIBRARY_PATH at runtime")
|
||||
|
||||
_ok(f"staged Linux tesseract + {copied} .so files into {staging.relative_to(REPO)}")
|
||||
|
||||
|
||||
def fetch_tesseract_for_platform(target: str) -> Path:
|
||||
"""Stage the per-platform Tesseract binary + libs into ``build/_tesseract/<target>/``.
|
||||
|
||||
Returns the staging dir path. The PyInstaller spec adds this dir
|
||||
(plus tessdata) to its ``datas=`` so the bundle ends up with
|
||||
everything under ``<bundle>/tesseract/`` where the runtime
|
||||
discovery code expects it.
|
||||
|
||||
Honours ``TESSERACT_SKIP_FETCH=1`` — set this when you've
|
||||
pre-staged the binary by hand (offline build, behind a proxy,
|
||||
custom build of tesseract, etc.). The script still verifies the
|
||||
binary is present and surfaces a helpful error if not.
|
||||
"""
|
||||
_step(f"fetch tesseract binary ({target})")
|
||||
staging = TESSERACT_STAGING / target
|
||||
exe_name = "tesseract.exe" if target == "win" else "tesseract"
|
||||
exe_path = staging / exe_name
|
||||
|
||||
if os.environ.get("TESSERACT_SKIP_FETCH") == "1":
|
||||
if not exe_path.exists():
|
||||
_err(
|
||||
f"TESSERACT_SKIP_FETCH=1 but {exe_path} is missing. "
|
||||
"Pre-stage the binary + its libs into that dir, then re-run."
|
||||
)
|
||||
sys.exit(1)
|
||||
_ok(f"skipping fetch (TESSERACT_SKIP_FETCH=1); using {exe_path.relative_to(REPO)}")
|
||||
return staging
|
||||
|
||||
if exe_path.exists():
|
||||
_ok(f"already staged: {exe_path.relative_to(REPO)}")
|
||||
return staging
|
||||
|
||||
if target == "win":
|
||||
_fetch_tesseract_windows(staging)
|
||||
elif target == "mac":
|
||||
_fetch_tesseract_macos(staging)
|
||||
elif target == "linux":
|
||||
_fetch_tesseract_linux(staging)
|
||||
else:
|
||||
_err(f"unknown target {target!r} for tesseract fetch")
|
||||
sys.exit(2)
|
||||
|
||||
if not exe_path.exists():
|
||||
_err(
|
||||
f"fetch step finished but {exe_path.relative_to(REPO)} is missing. "
|
||||
"Inspect the logs above; you may need to pre-stage the binary manually."
|
||||
)
|
||||
sys.exit(1)
|
||||
return staging
|
||||
63
build/vendor/README.md
vendored
Normal file
63
build/vendor/README.md
vendored
Normal file
@@ -0,0 +1,63 @@
|
||||
# build/vendor/ — third-party bundle inputs (fetched at build time)
|
||||
|
||||
This tree holds the third-party assets that get bundled into the
|
||||
PyInstaller artifacts but that we deliberately do **not** keep in git
|
||||
(too large / license-encumbered / re-fetchable on demand).
|
||||
|
||||
The build's Tesseract helper (`build/tesseract.py`) populates
|
||||
everything in here before the PyInstaller step — CI
|
||||
(`.github/workflows/build.yml`) calls it ahead of the build. The
|
||||
contents are git-ignored except for this README.
|
||||
|
||||
## tessdata/
|
||||
|
||||
Holds the Tesseract language data file(s) used by the PDF Extractor
|
||||
OCR fallback. Only English is bundled today.
|
||||
|
||||
### Canonical source
|
||||
|
||||
We use the **"best" model** from `tesseract-ocr/tessdata_best` (LSTM,
|
||||
slower but higher accuracy than the legacy `tessdata` set, and only
|
||||
~12 MB compressed → ~16 MB uncompressed):
|
||||
|
||||
```
|
||||
https://github.com/tesseract-ocr/tessdata_best/raw/main/eng.traineddata
|
||||
```
|
||||
|
||||
There is also `tessdata_fast/` (~4 MB, lower accuracy) if you ever
|
||||
want to optimise for bundle size over recognition quality. For bank
|
||||
statements (the only OCR use case so far), the extra accuracy of the
|
||||
`_best` model is worth the 10 MB.
|
||||
|
||||
### Why we don't vendor it in git
|
||||
|
||||
* ~16 MB binary file — bloats clone times for everyone, including
|
||||
contributors who never touch the OCR code path.
|
||||
* Apache-2.0-licensed and stable; the file rarely changes upstream
|
||||
(last touched 2021), so a build-time fetch is safe.
|
||||
* The Tesseract project explicitly distributes these via GitHub
|
||||
raw URLs — they're meant to be downloaded, not redistributed
|
||||
through other repos.
|
||||
|
||||
### How it gets populated
|
||||
|
||||
`build/tesseract.py::fetch_tessdata()` checks for
|
||||
`build/vendor/tessdata/eng.traineddata` on every run. If it's
|
||||
missing, it downloads the file from the canonical URL above and
|
||||
caches it here. Subsequent builds reuse the cached file.
|
||||
|
||||
On CI, the directory is restored from the GitHub Actions cache so we
|
||||
don't pay the download cost on every run (`.github/workflows/build.yml`
|
||||
caches `build/vendor/tessdata/` keyed on the URL above).
|
||||
|
||||
## Manual one-time fetch (if you're offline or behind a proxy)
|
||||
|
||||
```bash
|
||||
mkdir -p build/vendor/tessdata
|
||||
curl -L -o build/vendor/tessdata/eng.traineddata \
|
||||
https://github.com/tesseract-ocr/tessdata_best/raw/main/eng.traineddata
|
||||
```
|
||||
|
||||
Verify the file is non-empty and starts with the magic bytes
|
||||
`b"\x00\x00\x00\x00"` followed by a header that `pytesseract` can
|
||||
read; the script does a basic sanity check after download.
|
||||
0
build/vendor/tessdata/.gitkeep
vendored
Normal file
0
build/vendor/tessdata/.gitkeep
vendored
Normal file
481
docs/ADMIN.md
Normal file
481
docs/ADMIN.md
Normal file
@@ -0,0 +1,481 @@
|
||||
# ADMIN — Internal license operations
|
||||
|
||||
Creator/operator-only reference. End users should read `USER-GUIDE.md` instead.
|
||||
|
||||
This doc covers everything the creator does that buyers never see: minting
|
||||
through the live server, where state lives on the box, how to rotate secrets,
|
||||
generating the signing keypair, the dev vs. production key story, and how to
|
||||
recover from key loss.
|
||||
|
||||
For the end-to-end system + tech stack diagrams, see `ARCHITECTURE.md`.
|
||||
|
||||
---
|
||||
|
||||
## Live deployment (PR 1)
|
||||
|
||||
The license server is running at:
|
||||
|
||||
| URL | What it serves |
|
||||
|---|---|
|
||||
| `https://datatools.unalogix.com/` | Marketing site (placeholder — "DataTools — coming soon") |
|
||||
| `https://licenses.datatools.unalogix.com/health` | Liveness + DB reachability probe |
|
||||
| `https://licenses.datatools.unalogix.com/internal/*` | nginx-blocked on the public side — accessible only via SSH tunnel |
|
||||
| Postgres @ `127.0.0.1:5433` (localhost) | DB containing the authoritative `licenses` table |
|
||||
|
||||
**Host**: `46.225.166.142` (Ubuntu 24.04), nginx 1.24, Postgres 16-alpine + FastAPI in Docker.
|
||||
|
||||
**Cert**: Let's Encrypt, covers both subdomains, expires 2026-08-12, auto-renews via `certbot.timer`.
|
||||
|
||||
### On-box state
|
||||
|
||||
| Path | Contents |
|
||||
|---|---|
|
||||
| `/srv/datatools-license/` | Deploy root, mode 750, owned by `datatools-api` |
|
||||
| `/srv/datatools-license/compose.yml` | Production docker-compose definition |
|
||||
| `/srv/datatools-license/app/` | Git clone of this repo (re-clone or `git pull` to update) |
|
||||
| `/srv/datatools-license/secrets/` | Mode 750 dir holding `pg_password`, `admin_token`. Files are mode 400, owned UID 10001 (container app user) |
|
||||
| `/srv/datatools-license/backups/` | Postgres dumps land here (cron not yet wired — see §"Backups" below) |
|
||||
| `/etc/nginx/sites-available/unalogix` | nginx config for both subdomains |
|
||||
| `/etc/letsencrypt/live/datatools.unalogix.com/` | TLS cert + key |
|
||||
|
||||
Container names: `datatools-api`, `datatools-postgres`. Both use
|
||||
`restart: unless-stopped`.
|
||||
|
||||
### Get the admin token
|
||||
|
||||
```bash
|
||||
ssh michael@46.225.166.142 'sudo cat /srv/datatools-license/secrets/admin_token'
|
||||
```
|
||||
|
||||
The token is **never** in git, in environment-variable dumps, or in
|
||||
`docker inspect`. It lives on disk under mode 400 / UID 10001 (so only
|
||||
root and the container app user can read it).
|
||||
|
||||
### Rotate the admin token
|
||||
|
||||
Any time it's been shown somewhere it shouldn't, or as routine hygiene:
|
||||
|
||||
```bash
|
||||
cd /srv/datatools-license
|
||||
openssl rand -hex 32 > secrets/admin_token
|
||||
chown 10001:10001 secrets/admin_token
|
||||
chmod 400 secrets/admin_token
|
||||
docker compose restart api # ~3 seconds; old token stops working immediately
|
||||
```
|
||||
|
||||
### Mint a license from your laptop
|
||||
|
||||
```bash
|
||||
# 1. Open the SSH tunnel (leave running in a background terminal)
|
||||
ssh -L 8090:127.0.0.1:8090 michael@46.225.166.142 -N &
|
||||
|
||||
# 2. Set the auth env
|
||||
export DATATOOLS_ADMIN_TOKEN="$(ssh michael@46.225.166.142 'sudo cat /srv/datatools-license/secrets/admin_token')"
|
||||
export DATATOOLS_ADMIN_URL=http://127.0.0.1:8090
|
||||
|
||||
# 3. Mint
|
||||
python3 -m src.admin_cli mint \
|
||||
--name "Buyer Name" \
|
||||
--email buyer@example.com \
|
||||
--tier core
|
||||
|
||||
# 4. (optional) List or revoke
|
||||
python3 -m src.admin_cli list --email buyer@example.com
|
||||
python3 -m src.admin_cli revoke DT1-CORE-xxxx-yyyy --reason "refund"
|
||||
```
|
||||
|
||||
The blob lands in the response (and in the `licenses` table). Deliver it
|
||||
to the buyer however suits — copy-paste into email, attach as `.dtlic`.
|
||||
|
||||
### Inspect / debug
|
||||
|
||||
```bash
|
||||
# Container status + recent logs
|
||||
ssh michael@46.225.166.142 'cd /srv/datatools-license && docker compose ps && docker compose logs api --tail 30'
|
||||
|
||||
# Query the licenses table directly
|
||||
ssh michael@46.225.166.142 'cd /srv/datatools-license && docker compose exec -T postgres \
|
||||
psql -U datatools_api -d datatools_licenses -c "SELECT license_key, email, tier, source, expires_at FROM licenses ORDER BY created_at DESC LIMIT 20;"'
|
||||
|
||||
# Public-side health
|
||||
curl https://licenses.datatools.unalogix.com/health
|
||||
```
|
||||
|
||||
### Bring it down / back up / rebuild
|
||||
|
||||
```bash
|
||||
cd /srv/datatools-license
|
||||
|
||||
# Restart just the API (e.g. after rotating a secret)
|
||||
docker compose restart api
|
||||
|
||||
# Restart everything
|
||||
docker compose restart
|
||||
|
||||
# Bring down (DB volume PRESERVED)
|
||||
docker compose down
|
||||
|
||||
# Bring up
|
||||
docker compose up -d
|
||||
|
||||
# Rebuild the image after a git pull
|
||||
cd app && git pull
|
||||
cd ..
|
||||
docker compose build && docker compose up -d
|
||||
docker compose exec api alembic upgrade head # if new migrations
|
||||
```
|
||||
|
||||
### Backups (not yet automated)
|
||||
|
||||
Postgres state is the system of record for the customer list — once PR 2
|
||||
auto-mints from Gumroad webhooks, losing the DB would mean losing every
|
||||
buyer record. Schedule a daily dump:
|
||||
|
||||
```bash
|
||||
# /etc/cron.daily/datatools-license-backup — see SETUP-LICENSE-SERVER.md §9
|
||||
```
|
||||
|
||||
Until that's in place, dump manually before any risky operation:
|
||||
|
||||
```bash
|
||||
docker compose exec -T postgres \
|
||||
pg_dump -U datatools_api datatools_licenses \
|
||||
| gzip > backups/db-$(date -u +%Y%m%dT%H%M%SZ).sql.gz
|
||||
```
|
||||
|
||||
### Production signing key (not yet rotated)
|
||||
|
||||
The server currently signs with the in-tree dev keypair (no
|
||||
`DATATOOLS_LICENSE_PRIVKEY_FILE` configured → falls back to
|
||||
`src/license/_dev_keypair.py`). That matches what the desktop currently
|
||||
verifies against, so existing buyers continue to work.
|
||||
|
||||
**Before shipping v1.0 to paying buyers**, rotate to a production keypair:
|
||||
|
||||
1. `python scripts/generate_keypair.py` (on a trusted machine).
|
||||
2. Save the private hex to `/srv/datatools-license/secrets/license_privkey`,
|
||||
chmod 400, chown 10001:10001.
|
||||
3. Bake the public hex into the PyInstaller build's
|
||||
`DATATOOLS_LICENSE_PUBKEY` env.
|
||||
4. Wire `DATATOOLS_LICENSE_PRIVKEY_FILE` + `DATATOOLS_LICENSE_PUBKEY`
|
||||
into compose.yml's `api.environment` and add `license_privkey` to
|
||||
the secrets block.
|
||||
5. `docker compose restart api`.
|
||||
|
||||
### What's deployed (PR 1) vs queued (PR 2 / 3)
|
||||
|
||||
| Capability | Status |
|
||||
|---|---|
|
||||
| Mint API + Postgres + auth | **Live** |
|
||||
| `datatools-admin` CLI (manual mints) | **Live** |
|
||||
| `licenses.datatools.unalogix.com/health` public | **Live** |
|
||||
| Gumroad webhook receiver | **PR 2 — code merged, deploy pending** |
|
||||
| Postmark transactional email | **PR 2 — code merged, deploy pending** |
|
||||
| Buyer renewal / re-delivery portal | **PR 3** |
|
||||
| Cloudflare in front (DDoS / WAF) | Deferred (DNS at supercp/cPanel) |
|
||||
| Production signing keypair | Deferred (still using dev key) |
|
||||
| Automated DB backups | **Pending** — see §"Backups" |
|
||||
|
||||
### Running a Gumroad webhook (PR 2)
|
||||
|
||||
Once PR 2 is deployed, sales fire `POST` to
|
||||
`https://licenses.datatools.unalogix.com/webhooks/gumroad?secret=<gumroad_secret>`.
|
||||
Auth is the URL secret (Gumroad's recommended pattern). The handler
|
||||
audit-logs the raw payload, mints idempotently keyed on `sale_id`,
|
||||
sends the buyer their blob via Postmark, and returns 200 (always —
|
||||
non-2xx would trigger 3-day retry storms).
|
||||
|
||||
**Adding a new SKU:**
|
||||
|
||||
1. Create the product in Gumroad and copy its `product_id`.
|
||||
2. Edit `/srv/datatools-license/app/server/config/products.yaml`,
|
||||
add a row under `gumroad:` with that ID + the tier you sold.
|
||||
3. `cd /srv/datatools-license && docker compose restart api` — the
|
||||
config is read at startup and cached.
|
||||
|
||||
**Inspecting webhook activity:**
|
||||
|
||||
```bash
|
||||
# Recent webhook deliveries (all storefronts share this table)
|
||||
ssh michael@46.225.166.142 'cd /srv/datatools-license && docker compose exec -T postgres \
|
||||
psql -U datatools_api -d datatools_licenses -c \
|
||||
"SELECT received_at, order_id, processed, error FROM gumroad_events ORDER BY received_at DESC LIMIT 20;"'
|
||||
|
||||
# Failures only (replay candidates)
|
||||
ssh michael@46.225.166.142 'cd /srv/datatools-license && docker compose exec -T postgres \
|
||||
psql -U datatools_api -d datatools_licenses -c \
|
||||
"SELECT id, received_at, order_id, error FROM gumroad_events WHERE processed=false ORDER BY received_at DESC;"'
|
||||
```
|
||||
|
||||
**Replaying a failed webhook** (after fixing the products.yaml mapping
|
||||
or whatever surfaced the error): the safest path is to ask the buyer
|
||||
to re-trigger via Gumroad's "Send Test Ping" button in their order
|
||||
record, *or* mint manually via `datatools-admin mint --source manual`
|
||||
and add a note linking to the original `gumroad_events.id`.
|
||||
|
||||
**Testing without buyers:** Gumroad's seller dashboard has a "Send
|
||||
Test Ping" button. It sets `test=true` in the payload; the adapter
|
||||
tags the resulting license with `notes='gumroad test ping'` so it's
|
||||
trivially filterable later.
|
||||
|
||||
---
|
||||
|
||||
## TL;DR — I just need a license for my dev machine
|
||||
|
||||
You're running from source, so the repo's embedded dev keypair signs and
|
||||
verifies. No env vars needed.
|
||||
|
||||
```bash
|
||||
python scripts/generate_license.py \
|
||||
--name "Michael Dombaugh" \
|
||||
--email michael.dombaugh@gmail.com \
|
||||
--tier core
|
||||
```
|
||||
|
||||
Copy the `DTLIC2:…` blob from stdout, then activate:
|
||||
|
||||
```bash
|
||||
python -m src.license_cli activate "DTLIC2:..." \
|
||||
--name "Michael Dombaugh" \
|
||||
--email michael.dombaugh@gmail.com
|
||||
```
|
||||
|
||||
Verify:
|
||||
|
||||
```bash
|
||||
python -m src.license_cli status
|
||||
```
|
||||
|
||||
License lands at `~/.datatools/license.json`, valid 1 year.
|
||||
|
||||
> The `--name` / `--email` you pass to `activate` **must** match the values
|
||||
> the blob was minted with — they're part of the signed payload.
|
||||
|
||||
---
|
||||
|
||||
## Key model (Ed25519, asymmetric)
|
||||
|
||||
| Key | Lives where | Used for |
|
||||
|-----|------------|---------|
|
||||
| **Private** (32 bytes hex) | Creator's password manager / KMS only | Signing license blobs |
|
||||
| **Public** (32 bytes hex) | Baked into the shipped binary | Verifying blobs at activation |
|
||||
|
||||
The split is the whole point: an attacker with a copy of the binary still
|
||||
can't mint blobs — they'd need the private key, which never ships.
|
||||
|
||||
There's also an in-tree **dev keypair** (`src/license/_dev_keypair.py`)
|
||||
derived deterministically from a seed. It's used when no env vars are set,
|
||||
so devs/tests can sign and verify locally without juggling secrets. Frozen
|
||||
builds that still use it are rejected at startup by
|
||||
`assert_production_safe` — see `src/license/crypto.py:84`.
|
||||
|
||||
Blob format prefix: `DTLIC2:` (v1 was HMAC; v2 is Ed25519).
|
||||
|
||||
---
|
||||
|
||||
## One-time setup — generating the production keypair
|
||||
|
||||
Run once, before the first paid release.
|
||||
|
||||
```bash
|
||||
python scripts/generate_keypair.py --output keypair.env
|
||||
```
|
||||
|
||||
You'll get:
|
||||
|
||||
```
|
||||
DATATOOLS_LICENSE_PRIVKEY=<64 hex chars> # KEEP SECRET
|
||||
DATATOOLS_LICENSE_PUBKEY=<64 hex chars> # BAKE INTO BUILD
|
||||
```
|
||||
|
||||
Then:
|
||||
|
||||
1. **Stash the private key** in a password manager / KMS / hardware token.
|
||||
Losing it means no more renewals — see "Recovery" below.
|
||||
2. **Delete `keypair.env`** from disk once stored.
|
||||
3. **Set the public key** as `DATATOOLS_LICENSE_PUBKEY` in the PyInstaller
|
||||
build environment. The shipped binary embeds it via the env at freeze time.
|
||||
|
||||
---
|
||||
|
||||
## Minting a buyer license (production)
|
||||
|
||||
With the production private key loaded:
|
||||
|
||||
```bash
|
||||
export DATATOOLS_LICENSE_PRIVKEY=<your-private-hex>
|
||||
|
||||
python scripts/generate_license.py \
|
||||
--name "Buyer Name" \
|
||||
--email buyer@example.com \
|
||||
--tier core \
|
||||
--years 1 \
|
||||
--output buyer.dtlic
|
||||
```
|
||||
|
||||
Flags:
|
||||
|
||||
| Flag | Default | Notes |
|
||||
|------|---------|-------|
|
||||
| `--name` | required | Buyer's full name. Goes into signed payload. |
|
||||
| `--email` | required | Buyer's email. Goes into signed payload. |
|
||||
| `--tier` | `core` | One of: `lite`, `core`, `pro` |
|
||||
| `--years` | `1` | Lifetime in years |
|
||||
| `--key` | random | Override the auto-generated license key |
|
||||
| `--output` / `-o` | stdout | Write blob to file instead of printing |
|
||||
|
||||
Deliver the blob to the buyer either inline in the purchase email or as
|
||||
the attached `.dtlic` file.
|
||||
|
||||
---
|
||||
|
||||
## Tiers
|
||||
|
||||
| Tier | Features |
|
||||
|------|---------|
|
||||
| **lite** | Find Duplicates, Clean Text, Standardize Formats |
|
||||
| **core** | All 9 tools |
|
||||
| **pro** | All 9 tools + future Pro-only features |
|
||||
|
||||
Source of truth: `src/license/features.py::all_features_for_tier`.
|
||||
|
||||
---
|
||||
|
||||
## Useful one-liners
|
||||
|
||||
Mint a free internal/team license (dev key, no env needed):
|
||||
|
||||
```bash
|
||||
python scripts/generate_license.py --name "QA Bot" --email qa@datatools.app --tier core --years 5
|
||||
```
|
||||
|
||||
Mint with a stable, human-readable key:
|
||||
|
||||
```bash
|
||||
python scripts/generate_license.py --name "Acme Corp" --email ops@acme.com \
|
||||
--tier pro --key "DT1-PRO-ACME-2026"
|
||||
```
|
||||
|
||||
Renew an existing buyer (just re-mint with the same email; they paste the
|
||||
new blob):
|
||||
|
||||
```bash
|
||||
python -m src.license_cli renew "DTLIC2:..."
|
||||
```
|
||||
|
||||
Check what's active locally:
|
||||
|
||||
```bash
|
||||
python -m src.license_cli status
|
||||
```
|
||||
|
||||
Wipe a local license (move to a new machine, debug a buyer issue):
|
||||
|
||||
```bash
|
||||
python -m src.license_cli deactivate
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Customer record-keeping — the issuance log
|
||||
|
||||
Every successful `scripts/generate_license.py` run appends one JSON
|
||||
line to a local **issuance log**. This is the creator-side system of
|
||||
record for "who has a license" until the server-side flow in
|
||||
`docs/LICENSE-SERVER.md` lands.
|
||||
|
||||
**Path:** `~/.datatools-creator/issued.jsonl` (override with
|
||||
`$DATATOOLS_ISSUANCE_LOG`). Mode 600. Outside the buyer-facing
|
||||
`~/.datatools/` dir so it never gets bundled into a shipped install.
|
||||
|
||||
**Format** — one record per line:
|
||||
|
||||
```json
|
||||
{
|
||||
"license_key": "DT1-CORE-5dd8e1db-d90c4656",
|
||||
"name": "Michael Dombaugh",
|
||||
"email": "michael.dombaugh@gmail.com",
|
||||
"tier": "core",
|
||||
"issued_at": "2026-05-13T22:10:27Z",
|
||||
"expires_at": "2031-05-13T22:10:27Z",
|
||||
"blob": "DTLIC2:..."
|
||||
}
|
||||
```
|
||||
|
||||
The full blob is stored so you can re-deliver to a buyer who lost
|
||||
their email without re-minting (the re-minted blob would have a
|
||||
different signature and would invalidate any device they'd already
|
||||
activated against the old one).
|
||||
|
||||
**Useful operations:**
|
||||
|
||||
```bash
|
||||
# Full list of issued licenses
|
||||
cat ~/.datatools-creator/issued.jsonl | jq
|
||||
|
||||
# Find by buyer email
|
||||
jq -r 'select(.email == "buyer@example.com")' ~/.datatools-creator/issued.jsonl
|
||||
|
||||
# Count by tier
|
||||
jq -r .tier ~/.datatools-creator/issued.jsonl | sort | uniq -c
|
||||
|
||||
# Licenses expiring in the next 30 days
|
||||
jq -r 'select(.expires_at < "'"$(date -u -d '+30 days' +%Y-%m-%dT%H:%M:%SZ)"'") | .email' \
|
||||
~/.datatools-creator/issued.jsonl
|
||||
|
||||
# Re-deliver a buyer's blob
|
||||
jq -r 'select(.email == "buyer@example.com") | .blob' \
|
||||
~/.datatools-creator/issued.jsonl
|
||||
```
|
||||
|
||||
**Skipping the log** for test mints: pass `--no-log`. Never use this
|
||||
for real buyer fulfillment — an unlogged mint is invisible to every
|
||||
future query and to the eventual server-side migration.
|
||||
|
||||
**Backup:** treat this file like a small business ledger. Copy it
|
||||
into your password manager / encrypted cloud sync alongside the
|
||||
private key. Losing it doesn't break anything cryptographically (you
|
||||
can still mint new licenses) but it does lose the customer list.
|
||||
|
||||
**Migrating to the server:** the JSONL schema is intentionally close
|
||||
to the planned `licenses` table in `docs/LICENSE-SERVER.md`. Once the
|
||||
server is up, a one-shot import script will read the JSONL and
|
||||
insert each row.
|
||||
|
||||
---
|
||||
|
||||
## Recovery — what if the private key is lost?
|
||||
|
||||
Existing licenses keep working until they expire (the public key in the
|
||||
shipped binary still verifies them). What breaks:
|
||||
|
||||
- **Renewals** — you can't mint a new blob for an existing buyer.
|
||||
- **New sales** — you can't mint anything.
|
||||
|
||||
Path forward:
|
||||
|
||||
1. Generate a new keypair (`scripts/generate_keypair.py`).
|
||||
2. Ship a new build with the new public key.
|
||||
3. Re-issue every active buyer a new blob signed by the new private key.
|
||||
4. Communicate the upgrade path to buyers.
|
||||
|
||||
Treat the private key like a code-signing cert — back it up to two
|
||||
independent secure locations.
|
||||
|
||||
---
|
||||
|
||||
## Files & code pointers
|
||||
|
||||
| Path | Purpose |
|
||||
|------|---------|
|
||||
| `scripts/generate_keypair.py` | One-time keypair generation |
|
||||
| `scripts/generate_license.py` | Mint a signed blob |
|
||||
| `src/license/crypto.py` | Sign / verify / dev-key detection |
|
||||
| `src/license/_dev_keypair.py` | In-tree dev keypair (never ships in prod) |
|
||||
| `src/license/manager.py` | `assert_production_safe` startup check |
|
||||
| `src/license/features.py` | Tier → features mapping |
|
||||
| `src/license_cli.py` | End-user `activate` / `status` / `renew` / `deactivate` |
|
||||
| `~/.datatools/license.json` | Where activated licenses are stored on each machine |
|
||||
| `~/.datatools-creator/issued.jsonl` | Creator-side issuance log (one JSON line per mint) |
|
||||
| `docs/LICENSE-SERVER.md` | Design for the future online issuance + record-keeping system |
|
||||
| `docs/SETUP-LICENSE-SERVER.md` | Self-hosted server install runbook (DNS, Docker, nginx, TLS, backups) |
|
||||
241
docs/ARCHITECTURE.md
Normal file
241
docs/ARCHITECTURE.md
Normal file
@@ -0,0 +1,241 @@
|
||||
# ARCHITECTURE — end-to-end view
|
||||
|
||||
Stitches the desktop app (`TECHNICAL.md`) and the license server
|
||||
(`LICENSE-SERVER.md`) into a single picture. Read this first for "how
|
||||
does it all fit together"; drill into the per-component docs for
|
||||
detail.
|
||||
|
||||
---
|
||||
|
||||
## 1. System diagram
|
||||
|
||||
```
|
||||
┌────────────────────────────────────────────────────────────────────────┐
|
||||
│ OPERATOR / DEVELOPER LAPTOP │
|
||||
│ │
|
||||
│ git clone / push ←─── code lives in git.invixiom.com │
|
||||
│ datatools-admin CLI ─── manual mints, list, revoke ─────┐ │
|
||||
│ ssh -L 8090:127.0.0.1:8090 ───── tunnel for /internal/* ─────┤ │
|
||||
└────────────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
┌─────────────────────────────────────────────────────────┘
|
||||
│
|
||||
│ internal Bearer-auth API (over SSH tunnel only)
|
||||
▼
|
||||
┌────────────────────────────────────────────────────────────────────────┐
|
||||
│ LICENSE SERVER — 46.225.166.142 │
|
||||
│ ───────────────────────────────────────────────────────────────── │
|
||||
│ │
|
||||
│ ┌──────────────────────────────────────────────────────────────────┐ │
|
||||
│ │ nginx 1.24 (TLS termination, public reverse proxy) │ │
|
||||
│ │ │ │
|
||||
│ │ datatools.unalogix.com → static placeholder │ │
|
||||
│ │ licenses.datatools.unalogix.com → 127.0.0.1:8090 (FastAPI) │ │
|
||||
│ │ /internal/* on public surface → blocked (404) │ │
|
||||
│ └────────────────────────────┬─────────────────────────────────────┘ │
|
||||
│ │ │
|
||||
│ ┌────────────────────────────▼─────────────────────────────────────┐ │
|
||||
│ │ FastAPI app — datatools-api (Docker container, UID 10001) │ │
|
||||
│ │ │ │
|
||||
│ │ ┌──────────────────┐ ┌──────────────────┐ ┌───────────────┐ │ │
|
||||
│ │ │ /webhooks/* │ │ /internal/* │ │ /health │ │ │
|
||||
│ │ │ (storefronts) │ │ (Bearer-auth) │ │ (public) │ │ │
|
||||
│ │ └────────┬─────────┘ └────────┬─────────┘ └───────────────┘ │ │
|
||||
│ │ │ │ │ │
|
||||
│ │ ▼ ▼ │ │
|
||||
│ │ ┌────────────────────────────────────────┐ │ │
|
||||
│ │ │ SourceAdapter (Protocol) — normalized │ │ │
|
||||
│ │ │ • ManualAdapter • GumroadAdapter │ │ │
|
||||
│ │ │ • (LemonSqueezy, Stripe — future) │ │ │
|
||||
│ │ └────────────────┬───────────────────────┘ │ │
|
||||
│ │ │ SaleEvent / RefundEvent │ │
|
||||
│ │ ▼ │ │
|
||||
│ │ ┌────────────────────────────────────────┐ │ │
|
||||
│ │ │ mint_from_sale() │ │ │
|
||||
│ │ │ • Ed25519 sign via PyCA cryptography │ │ │
|
||||
│ │ │ • idempotent on (source, order_id) │ │ │
|
||||
│ │ └────────────────┬───────────────────────┘ │ │
|
||||
│ └────────────────────┼─────────────────────────────────────────────┘ │
|
||||
│ │ SQL │
|
||||
│ ┌────────────────────▼─────────────────────────────────────────────┐ │
|
||||
│ │ Postgres 16 — datatools-postgres (container, vol pg_data) │ │
|
||||
│ │ • licenses — authoritative customer record │ │
|
||||
│ │ • gumroad_events — webhook audit log (idempotency, replay) │ │
|
||||
│ └──────────────────────────────────────────────────────────────────┘ │
|
||||
└───────────────────────┬────────────────────────────────┬───────────────┘
|
||||
│ │
|
||||
┌───────────┘ └──────────┐
|
||||
│ POST /email (httpx) Gumroad Ping│
|
||||
▼ POST │
|
||||
┌───────────────────┐ ┌─────────────▼──┐
|
||||
│ Postmark │ │ Gumroad │
|
||||
│ (transactional │ │ (storefront, │
|
||||
│ email) │ │ payments) │
|
||||
└───────┬───────────┘ └────────────────┘
|
||||
│ DKIM-signed email with license blob ▲
|
||||
▼ │
|
||||
┌────────────────────────────────────────────────────────────────┴───────┐
|
||||
│ BUYER'S MACHINE │
|
||||
│ │
|
||||
│ Receives email ──► copies DTLIC2: blob ──► pastes into desktop app │
|
||||
│ │
|
||||
│ ┌──────────────────────────────────────────────────────────────────┐ │
|
||||
│ │ DataTools desktop (Python 3.12 + Streamlit + Typer CLIs) │ │
|
||||
│ │ │ │
|
||||
│ │ ┌────────────────────────────────────────────────────────────┐ │ │
|
||||
│ │ │ Activate screen — verifies blob signature │ │ │
|
||||
│ │ │ against EMBEDDED Ed25519 public key │ │ │
|
||||
│ │ │ (NO network call to the license server, ever) │ │ │
|
||||
│ │ └─────────────────────────┬──────────────────────────────────┘ │ │
|
||||
│ │ ▼ │ │
|
||||
│ │ ~/.datatools/license.json (signed blob, mode 644, on disk) │ │
|
||||
│ └──────────────────────────────────────────────────────────────────┘ │
|
||||
│ │
|
||||
│ Pays via web browser ─────► Gumroad ────► (kicks off the Ping) │
|
||||
└────────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
**Three primary flows**, distinguishable by where the green arrows
|
||||
start in the diagram:
|
||||
|
||||
1. **Sale → fulfillment** (the automated path)
|
||||
Buyer pays at Gumroad → Gumroad fires Ping to
|
||||
`licenses.datatools.unalogix.com/webhooks/gumroad?secret=…` → nginx
|
||||
→ FastAPI → audit-log row → adapter normalizes payload → `mint_from_sale`
|
||||
writes the `licenses` row + Ed25519-signs the blob → Postmark emails
|
||||
the buyer their blob. End-to-end latency: a few hundred milliseconds.
|
||||
|
||||
2. **Manual mint** (operator path — comps, support replacements)
|
||||
Operator opens SSH tunnel → `datatools-admin mint` → `/internal/mint`
|
||||
(Bearer-authed, never publicly reachable) → same `mint_from_sale`
|
||||
path → blob returned in HTTP response. Operator delivers to buyer
|
||||
out-of-band.
|
||||
|
||||
3. **Activation** (buyer path — fully offline)
|
||||
Buyer pastes blob into desktop's Activate screen → desktop verifies
|
||||
the Ed25519 signature against the public key **embedded in the
|
||||
shipped binary** → license written to `~/.datatools/license.json`.
|
||||
The desktop app makes **no network calls** to the license server at
|
||||
any point. This preserves the "your data never leaves your computer"
|
||||
promise (`DECISIONS.md §9b`).
|
||||
|
||||
---
|
||||
|
||||
## 2. Tech stack
|
||||
|
||||
Layered view of what technology lives where. "External SaaS" entries
|
||||
are services we depend on but don't operate.
|
||||
|
||||
```
|
||||
┌────────────────────────────────────────────────────────────────────────┐
|
||||
│ DESKTOP APP (shipped binary, runs on buyer's box) │
|
||||
├──────────────────┬─────────────────────────────────────────────────────┤
|
||||
│ GUI │ Streamlit 1.35 — local web server, browser opens │
|
||||
│ CLI │ Typer 0.12 — per-tool entry points │
|
||||
│ Core logic │ pandas 2.x, numpy, rapidfuzz, charset-normalizer │
|
||||
│ Crypto (verify) │ PyCA cryptography — Ed25519 public-key verify only │
|
||||
│ Storage │ ~/.datatools/license.json (file, mode 644) │
|
||||
│ Internationalization │ i18n via JSON catalogs in src/i18n/ │
|
||||
│ Build │ PyInstaller — one-file binary, per OS │
|
||||
│ Runtimes │ Python 3.12 (bundled into installer) │
|
||||
│ Platforms │ Windows · macOS · Linux │
|
||||
└──────────────────┴─────────────────────────────────────────────────────┘
|
||||
|
||||
┌────────────────────────────────────────────────────────────────────────┐
|
||||
│ LICENSE SERVER (this box; non-buyer-facing) │
|
||||
├──────────────────┬─────────────────────────────────────────────────────┤
|
||||
│ Edge │ nginx 1.24 + Let's Encrypt (auto-renew via timer) │
|
||||
│ HTTP framework │ FastAPI 0.119 + Starlette + Pydantic v2 │
|
||||
│ ASGI server │ uvicorn 0.39 (+uvloop, +httptools, +watchfiles) │
|
||||
│ Form parsing │ python-multipart (for Gumroad form-encoded Pings) │
|
||||
│ ORM │ SQLAlchemy 2.0 │
|
||||
│ Migrations │ Alembic 1.18 (one initial migration so far) │
|
||||
│ Database │ Postgres 16-alpine (containerized, single node) │
|
||||
│ Database driver │ psycopg 3.3 (with binary wheel) │
|
||||
│ Crypto (sign) │ PyCA cryptography — Ed25519 private-key sign │
|
||||
│ HTTP client │ httpx 0.28 (Postmark calls, test mocking) │
|
||||
│ Config │ Pydantic Settings + YAML (products.yaml) │
|
||||
│ Container │ Docker + Docker Compose v2 plugin │
|
||||
│ Image base │ python:3.12-slim │
|
||||
│ Process user │ UID 10001 (non-root `app` user defined in image) │
|
||||
│ Logging │ stdlib `logging` to container stdout → docker logs │
|
||||
│ Host OS │ Ubuntu 24.04 LTS │
|
||||
└──────────────────┴─────────────────────────────────────────────────────┘
|
||||
|
||||
┌────────────────────────────────────────────────────────────────────────┐
|
||||
│ OPERATOR / DEVELOPER MACHINE │
|
||||
├──────────────────┬─────────────────────────────────────────────────────┤
|
||||
│ Source control │ git → self-hosted Gitea (git.invixiom.com) │
|
||||
│ Admin CLI │ Typer (src/admin_cli.py) │
|
||||
│ Server access │ SSH tunnel for /internal/* (no public exposure) │
|
||||
│ Break-glass │ scripts/generate_license.py (offline-only mints, │
|
||||
│ │ used when the license server is unreachable) │
|
||||
│ Test runner │ pytest 8.3 + SQLite in-memory (no docker required) │
|
||||
│ Smoke test │ bash + docker compose (server/scripts/smoke.sh) │
|
||||
└──────────────────┴─────────────────────────────────────────────────────┘
|
||||
|
||||
┌────────────────────────────────────────────────────────────────────────┐
|
||||
│ EXTERNAL SaaS / dependencies │
|
||||
├──────────────────┬─────────────────────────────────────────────────────┤
|
||||
│ Storefront │ Gumroad — Ping webhook to /webhooks/gumroad │
|
||||
│ Transactional │ Postmark — HTTP API for license-delivery emails │
|
||||
│ email │ (LoggingEmailService fallback when token unset) │
|
||||
│ TLS CA │ Let's Encrypt — ACME HTTP-01 challenge via certbot │
|
||||
│ Authoritative │ supercp / cPanel (your DNS host for unalogix.com) │
|
||||
│ DNS │ — Cloudflare front-door deferred │
|
||||
│ Source hosting │ Self-hosted Gitea (git.invixiom.com) — not on the │
|
||||
│ │ datatools box; shares the same physical host │
|
||||
└──────────────────┴─────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3. Trust + isolation boundaries
|
||||
|
||||
Worth tracing explicitly because the threat model differs at each
|
||||
boundary:
|
||||
|
||||
| Boundary | What crosses it | Trust model |
|
||||
|---|---|---|
|
||||
| Buyer ↔ Gumroad | Payment, buyer details | Out of scope — Gumroad's problem |
|
||||
| Gumroad → license server (webhook) | Signed-by-shared-secret POST | URL secret check; non-matching = 404 (no info leak); audit-log everything regardless |
|
||||
| License server → Postmark | DKIM-signed transactional mail | Postmark verified-sender domain; HTTP API auth via server token |
|
||||
| License server → Postgres | SQL over local docker bridge | Same compose project; password from on-disk secret file |
|
||||
| Operator → license server (`/internal/*`) | Bearer token over SSH tunnel | Token only on disk + in the operator's env; nginx blocks `/internal/*` publicly as defense-in-depth |
|
||||
| License server → buyer (email) | Plaintext blob in inbox | Buyer's email account hygiene; we deliberately don't encrypt — blob is self-protecting (signature) |
|
||||
| Buyer → desktop app (activation) | Signed blob pasted in | Verified against pubkey **embedded in the shipped binary**; no network call |
|
||||
|
||||
The single most important property to preserve: **the desktop app
|
||||
never talks to the license server.** All trust in the desktop comes
|
||||
from the embedded public key + the signed blob. This is what makes
|
||||
the offline activation guarantee real, and what keeps a license-server
|
||||
outage from breaking buyers who've already activated.
|
||||
|
||||
---
|
||||
|
||||
## 4. Where things are stored
|
||||
|
||||
| Lives on… | Path / location | Contents |
|
||||
|---|---|---|
|
||||
| Buyer's machine | `~/.datatools/license.json` | Activated license blob |
|
||||
| Buyer's machine | Postmark email | Delivery copy of the blob |
|
||||
| License server | `licenses` table (Postgres) | Authoritative customer record — name, email, tier, blob, source, order ID, promotion, amount paid |
|
||||
| License server | `gumroad_events` table | Append-only webhook delivery audit log |
|
||||
| License server | `/srv/datatools-license/secrets/` | Postgres password, admin Bearer token, (PR 2) Postmark token + Gumroad secret |
|
||||
| License server | `/etc/letsencrypt/live/datatools.unalogix.com/` | TLS cert + key |
|
||||
| Operator's laptop | `~/.datatools-creator/issued.jsonl` | Creator-side issuance log (pre-server era, kept as a break-glass backup) |
|
||||
| Operator's laptop | Git clone of this repo | Source code, including `server/config/products.yaml` |
|
||||
| Gitea | This repo's commits | Everything except secrets |
|
||||
|
||||
---
|
||||
|
||||
## 5. Related docs
|
||||
|
||||
| Doc | Scope |
|
||||
|---|---|
|
||||
| `TECHNICAL.md` | Desktop app internals (core libs, GUI, CLIs) |
|
||||
| `LICENSE-SERVER.md` | Server architecture rationale + DB schema |
|
||||
| `SETUP-LICENSE-SERVER.md` | Server install runbook (DNS, packages, nginx, TLS, Postgres) |
|
||||
| `ADMIN.md` | Day-2 operations (minting, rotation, inspection) |
|
||||
| `DECISIONS.md` | Architecture decision records — `§9b` = no online activation check |
|
||||
| `USER-GUIDE.md` | Buyer-facing documentation |
|
||||
@@ -47,7 +47,7 @@ Sell niche Python automation tools as one-time downloadable digital products. Ta
|
||||
|
||||
**Surface**: desktop install per OS (PyInstaller) with Streamlit GUI + CLI. Constrained demo on Streamlit Community Cloud.
|
||||
|
||||
## 4a. Lead bundle — Deduplicator
|
||||
## 4a. Lead bundle — Find Duplicates
|
||||
|
||||
Highest pain density across all 4 personas. Feeds landing copy, demo design, feature priority. Tech spec: TECHNICAL.md §11.1.
|
||||
|
||||
@@ -208,7 +208,7 @@ Headroom enables optional ad spend ($100-200/mo) once a bundle has proven conver
|
||||
|
||||
## 13. Honest status (2026-05-01)
|
||||
|
||||
- 3 of 9 tools shipped (Dedup, Text Cleaner, Format Standardizer).
|
||||
- 3 of 9 tools shipped (Find Duplicates, Clean Text, Standardize Formats).
|
||||
- Cross-platform build pipeline designed, not yet built.
|
||||
- macOS code signing not yet set up.
|
||||
- Streamlit GUI shipped for the 3 ready tools.
|
||||
|
||||
@@ -8,15 +8,15 @@ Tres módulos de CLI, uno por cada herramienta Lista:
|
||||
|
||||
| Módulo | Comando | Propósito |
|
||||
|--------|---------|---------|
|
||||
| `src.cli` | `python -m src.cli FILE` | Eliminador de duplicados |
|
||||
| `src.cli_text_clean` | `python -m src.cli_text_clean FILE` | Limpiador de texto |
|
||||
| `src.cli` | `python -m src.cli FILE` | Buscar duplicados |
|
||||
| `src.cli_text_clean` | `python -m src.cli_text_clean FILE` | Limpiar texto |
|
||||
| `src.cli_analyze` | `python -m src.cli_analyze FILE` | Analizador (escaneo de solo lectura) |
|
||||
|
||||
Cada comando es **previsualización por defecto** — añade `--apply` para escribir la salida.
|
||||
|
||||
---
|
||||
|
||||
# Eliminador de duplicados
|
||||
# Buscar duplicados
|
||||
|
||||
```
|
||||
python -m src.cli ARCHIVO_ENTRADA [OPCIONES]
|
||||
@@ -125,7 +125,7 @@ Registro: `logs/dedup_YYYYMMDD_HHMMSS.log`.
|
||||
|
||||
---
|
||||
|
||||
# Limpiador de texto
|
||||
# Limpiar texto
|
||||
|
||||
```
|
||||
python -m src.cli_text_clean ARCHIVO_ENTRADA [OPCIONES]
|
||||
@@ -156,7 +156,7 @@ Higiene a nivel de carácter. Ver [TECHNICAL.md §10.2](TECHNICAL.md) (solo en i
|
||||
- `--config RUTA` / `--save-config RUTA`.
|
||||
|
||||
### Archivo
|
||||
- `--sheet`, `--encoding`, `--header-row` — iguales que en el Eliminador de duplicados.
|
||||
- `--sheet`, `--encoding`, `--header-row` — iguales que en Buscar duplicados.
|
||||
|
||||
## Presets
|
||||
|
||||
|
||||
@@ -6,15 +6,15 @@ Three CLI modules, one per Ready tool:
|
||||
|
||||
| Module | Command | Purpose |
|
||||
|--------|---------|---------|
|
||||
| `src.cli` | `python -m src.cli FILE` | Deduplicator |
|
||||
| `src.cli_text_clean` | `python -m src.cli_text_clean FILE` | Text Cleaner |
|
||||
| `src.cli` | `python -m src.cli FILE` | Find Duplicates |
|
||||
| `src.cli_text_clean` | `python -m src.cli_text_clean FILE` | Clean Text |
|
||||
| `src.cli_analyze` | `python -m src.cli_analyze FILE` | Analyzer (read-only scan) |
|
||||
|
||||
Every command is **preview-only by default** — add `--apply` to write output.
|
||||
|
||||
---
|
||||
|
||||
# Deduplicator
|
||||
# Find Duplicates
|
||||
|
||||
```
|
||||
python -m src.cli INPUT_FILE [OPTIONS]
|
||||
@@ -123,7 +123,7 @@ Log: `logs/dedup_YYYYMMDD_HHMMSS.log`.
|
||||
|
||||
---
|
||||
|
||||
# Text Cleaner
|
||||
# Clean Text
|
||||
|
||||
```
|
||||
python -m src.cli_text_clean INPUT_FILE [OPTIONS]
|
||||
@@ -154,7 +154,7 @@ Character-level hygiene. See [TECHNICAL.md §10.2](TECHNICAL.md) for the spec.
|
||||
- `--config PATH` / `--save-config PATH`.
|
||||
|
||||
### File
|
||||
- `--sheet`, `--encoding`, `--header-row` — same as Deduplicator.
|
||||
- `--sheet`, `--encoding`, `--header-row` — same as Find Duplicates.
|
||||
|
||||
## Presets
|
||||
|
||||
|
||||
@@ -67,7 +67,7 @@ Each candidate scored 1-5 on 6 dimensions. Total /30 → verdict.
|
||||
|
||||
**v1.2 rationale**:
|
||||
- Buyer persona ("hates Excel work but can't code") won't learn a CLI. Refunds at this price.
|
||||
- Deduplicator needs interactive review — not viable in pure CLI.
|
||||
- Find Duplicates needs interactive review — not viable in pure CLI.
|
||||
- Dual interface keeps CLI for automation without sacrificing primary buyer surface.
|
||||
|
||||
## 4a. Functional scope principle (v1.2)
|
||||
@@ -170,11 +170,60 @@ $49-79/bundle · $149 full suite (when 3+ exist).
|
||||
| Apr 28 (v1.3) | Add hosted browser demo as conversion lever | Direct consequence of Streamlit choice. See §5. |
|
||||
| Apr 28 (v1.4) | Re-apply 04/06 boundary work (silent-drift recovery) | Stream B v1.2 content overwritten in parallel v1.3 work. Restored per no-silent-drift rule. |
|
||||
| Apr 28 (v1.5) | Add `02_text_cleaner.py`; renumber 02-08 → 03-09 | Character-level hygiene had no clear owner. See TECHNICAL §10. |
|
||||
| Apr 29 (v1.7) | Adopt Text Cleaner Tier 1/2/3 spec; lock `excel-hygiene` default | Promotes from stub to buildable v1 target. Full spec in TECHNICAL §11.2. |
|
||||
| Apr 29 (v1.7) | Adopt Clean Text Tier 1/2/3 spec; lock `excel-hygiene` default | Promotes from stub to buildable v1 target. Full spec in TECHNICAL §11.2. |
|
||||
| Apr 28 (v1.6) | Fold conversation-history content into docs (deduplicator spec, lead bundle use cases, full GUI matrix, 04/06 examples, Streamlit-to-SaaS reasoning) | No new decisions; promote at-risk analysis from chat history per no-silent-drift rule. |
|
||||
| May 1 (v1.6) | Mark Format Standardizer **Ready** | 199-row buyer corpus passing; Tier 1 + most Tier 2 built. |
|
||||
| May 1 (v1.6) | Mark Standardize Formats **Ready** | 199-row buyer corpus passing; Tier 1 + most Tier 2 built. |
|
||||
| May 1 (v1.6) | Add `src/core/errors.py` structured hierarchy | Uniform helpful messages across CLI + GUI. See TECHNICAL §7. |
|
||||
| May 13 (v1.6) | Ship in-house JSON i18n + EN/ES packs | Expand addressable market (Spanish-first buyers, LatAm bookkeepers) without a `gettext` build step. JSON packs editable by non-devs; parity test prevents drift. See TECHNICAL §10b. |
|
||||
| May 13 (v1.6) | Ship licensing: 1-year HMAC-signed blobs, name+email registration, offline verification, tier-scaffolded for future SKUs | Unlock the lifetime-update business model without recurring infra. Honor-system DRM (HMAC + 30-day refund) — sufficient at $49. See §9b below. |
|
||||
| May 13 (v1.6) | Add Lite SKU (Find Duplicates + Clean Text + Standardize Formats) | Lower-priced entry point for buyers who only need the three universal tools. Per-tool feature gating + lock badges on the home grid surface the upgrade path. See §9b. |
|
||||
| May 13 (v1.6) | Remove user-facing free trial | A 1-year all-features trial undercut the paid Lite SKU. Paid-only keeps tier economics clean. Internal ``_mint`` API still exists for tests and the seller's key generator. See §9b. |
|
||||
| May 13 (v1.6) | Upgrade license crypto: HMAC → Ed25519 (asymmetric) | HMAC's symmetric secret was extractable from the shipped binary — anyone with the binary could mint blobs. Ed25519 splits sign (seller) from verify (binary), so binary compromise doesn't let an attacker forge licenses. Blob prefix bumped DTLIC1 → DTLIC2. See §9b. |
|
||||
| May 13 (v1.6) | Add ``assert_production_safe`` tripwire | A shipped build with ``DATATOOLS_DEV_MODE=1`` or the in-source dev pubkey would silently defeat licensing. The tripwire refuses to boot such a build. No-op in source / pytest runs. See §9b. |
|
||||
|
||||
## 9b. Licensing model
|
||||
|
||||
**Decision (v1.6)**: offline HMAC-signed license blobs, 1-year lifetime, name + email registration required. Tier-scaffolded so future SKUs (PRO, ENTERPRISE) can carve per-tool feature sets without code changes.
|
||||
|
||||
| Option | Verdict |
|
||||
|---|---|
|
||||
| **Offline HMAC blob (chosen)** | **CHOSEN.** No server, no internet, fits the no-touch constraint. Honor-system at this price point. |
|
||||
| Online activation check | Rejected. Conflicts with the "your data never leaves your computer" promise; introduces support load (server downtime, network issues). |
|
||||
| No license at all | Rejected. The lifetime-update value prop requires *some* gating to make renewal meaningful. |
|
||||
| Time-bombed binary (PyInstaller --no-license) | Rejected. Can't deliver renewals without re-shipping the installer. |
|
||||
| Hardware-locked license | Rejected. Friction on legitimate device-swaps; doesn't match the buyer persona's tolerance. |
|
||||
|
||||
**Threat model** (v1.6 — Ed25519): the binary ships only the public key. A motivated reverse engineer who pulls everything out of the binary has the verification key but not the signing key — they can't mint new licenses. The earlier HMAC scheme had this hole; the asymmetric upgrade closes it. The remaining attack surface is:
|
||||
|
||||
- Re-signing with a forked binary that ships an attacker-controlled pubkey + auto-grants licenses. Costs more effort than the price of a legitimate copy and the result is per-fork, not shareable.
|
||||
- Hooking the verification call to always return True. Defeats DRM entirely but only on the attacker's own machine — they could just write down "I unlocked DataTools" and skip the work.
|
||||
- Setting ``DATATOOLS_DEV_MODE=1`` to bypass checks. **Refused in shipped builds** by ``assert_production_safe``; works in source/test runs only.
|
||||
|
||||
The 30-day refund window covers casual blob sharing from a different angle (anyone who shares their blob is implicitly authorizing the buyer to issue them a refund-on-demand).
|
||||
|
||||
**What's enforced**:
|
||||
- License blob signature must match (HMAC-SHA256 with the build secret).
|
||||
- Buyer-entered name + email must match the values embedded in the blob.
|
||||
- Expiry date must be in the future.
|
||||
- Tier must include the requested feature.
|
||||
|
||||
**What's NOT enforced**:
|
||||
- Number of devices the same blob is used on (no concurrent-use detection).
|
||||
- Reverse-engineered re-signing of expired blobs (would require RSA / online check).
|
||||
|
||||
**Future SKUs**: the ``FEATURES_BY_TIER`` table in ``src/license/features.py`` is the single source of truth for "which tools each tier unlocks". Adding a PRO SKU that excludes Automated Workflows is a 1-line edit there + a 1-line edit at the gate site. No consumer-code churn.
|
||||
|
||||
**v1.6 SKU lineup**:
|
||||
|
||||
| Tier | Tools unlocked | Notes |
|
||||
|---|---|---|
|
||||
| LITE | Find Duplicates, Clean Text, Standardize Formats | Entry SKU. Three universal tools that handle the most common bookkeeping / RevOps / Klaviyo prep workflows. |
|
||||
| CORE | All 9 tools | Full v1 suite. |
|
||||
| PRO | All 9 tools (scaffolded) | Reserved for future per-feature carve-outs (e.g., scheduled pipelines, API access). |
|
||||
| ENTERPRISE | All 9 tools (scaffolded) | Reserved for future bulk / multi-seat SKUs. |
|
||||
| TRIAL | Same as LITE | Deprecated — no longer issuable. Mapping kept for any legacy on-disk trial licenses to load without error. |
|
||||
|
||||
**Trial removed (v1.6)**: a 1-year free trial that unlocked every tool would undercut the paid Lite SKU (why pay for Lite when trial gives more for longer?). Paid-only keeps the funnel clean. The internal ``LicenseManager._mint`` API still exists for tests and for the seller's ``scripts/generate_license.py`` key generator; there's no user-facing way to self-issue a license.
|
||||
|
||||
## 8. Re-lock triggers
|
||||
|
||||
|
||||
@@ -32,17 +32,22 @@ rebuilds it from a stale headline.
|
||||
| Friction kills conversion | BUSINESS.md §7 | Demo dataset preloaded; no "select a file" first-step |
|
||||
| < $1,200/mo recurring | BUSINESS.md §9 | Migration plan to $5/mo VPS only after rate-limit signal |
|
||||
|
||||
## 3. The three personas (per PLAN.md §2.3)
|
||||
## 3. The three personas — one audience: accounting (per PLAN.md §2.3)
|
||||
|
||||
We niche to **accounting** and enter through the three workflows where a
|
||||
messy export costs real money. Same engine, three landing pages — each
|
||||
is the same buyer at a different desk (bookkeeping, payables, receivables).
|
||||
|
||||
| Tag | Persona | Top-of-funnel keyword | Demo dataset | Pre-saved pipeline |
|
||||
|---|---|---|---|---|
|
||||
| `shopify-pet` | Shopify operator (priority: pet supplies) | "shopify customer cleanup" | `samples/demo/shopify_pet_customers.csv` | `shopify_pet_pipeline.json` |
|
||||
| `bookkeeper` | Bookkeeper / freelance accountant | "reconcile bank export csv" | `samples/demo/bookkeeper_bank_reconcile.csv` | `bookkeeper_bank_pipeline.json` |
|
||||
| `revops` | Marketing / RevOps agency | "dedupe lead list across vendors" | `samples/demo/agency_combined_leads.csv` | `agency_leads_pipeline.json` |
|
||||
| `bookkeeper` | Bookkeeper — bank reconciliation | "reconcile bank export csv duplicates" | `samples/demo/bank_reconciliation.csv` | `bank_reconciliation_pipeline.json` |
|
||||
| `ap-1099` | Accounts payable — 1099 vendor prep | "clean 1099 vendor list missing EIN" | `samples/demo/vendor_1099.csv` | `vendor_1099_pipeline.json` |
|
||||
| `ar-aging` | Accounts receivable — open invoices | "remove duplicate invoices aging report" | `samples/demo/ar_open_invoices.csv` | `ar_open_invoices_pipeline.json` |
|
||||
|
||||
Each persona gets its **own landing page URL**, its **own demo dataset
|
||||
loaded by default**, and its **own H1 + below-the-fold copy.** The
|
||||
engine is identical; only positioning differs.
|
||||
Each persona gets its **own landing page URL** (`?p=<tag>`), its **own
|
||||
demo dataset loaded by default**, and its **own H1 + below-the-fold
|
||||
copy** — wired in `src/gui/app_demo.py::PERSONAS`. The engine is
|
||||
identical; only positioning differs.
|
||||
|
||||
## 4. Demo dataset specifications
|
||||
|
||||
@@ -53,114 +58,77 @@ persona's tooling. Each contains every kind of pollution the bundle's
|
||||
five tools fix, so a single demo run shows every tool earning its
|
||||
keep.
|
||||
|
||||
### 4.0 Pain-point coverage map
|
||||
### 4.0 Value-proof map
|
||||
|
||||
Each demo dataset is engineered so the buyer sees their **own top
|
||||
pain** demonstrated in the AFTER preview. The mapping below pairs
|
||||
each pain from PLAN.md §2.3a with the rows / columns that exercise
|
||||
it. Refresh the dataset only when this coverage drops.
|
||||
Each demo dataset is engineered so the buyer sees their **own top pain**
|
||||
fixed in the AFTER preview, with one unmistakable headline number. All
|
||||
three run the same saved 4-step pipeline (Clean Text → Standardize
|
||||
Formats → Fix Missing Values → Find Duplicates). The numbers below are
|
||||
**validated against the live engine** (`tests/test_demo_pipelines.py`
|
||||
pins them) — refresh the dataset only if a number stops landing.
|
||||
|
||||
| Persona | Pain (from PLAN §2.3a) | Demo coverage |
|
||||
| Persona | Headline proof | What the visitor watches happen |
|
||||
|---|---|---|
|
||||
| Shopify pet | S1 — Klaviyo per-contact dupes | 5 dup pairs across rows 1–15 (case + format + address-twin variants) |
|
||||
| Shopify pet | S2 — feed-rejection chars | smart-quote / NBSP / BOM in rows 1–6, 9, 11 |
|
||||
| Shopify pet | S3 — multi-channel | partner-style customer IDs (`SHOP-`); demonstration of column-level mapping covered in RevOps demo |
|
||||
| Shopify pet | S4 — subscription identity | rows 1+2, 7+8, 9+10 — same person, different format |
|
||||
| Shopify pet | S5 — VAT-MOSS country drift | rows 16–18 (`United Kingdom` / `U.K.` / `UK`) + rows 19–20 (`Germany`/`Italia`) |
|
||||
| Bookkeeper | B1 — month-overlap re-import | 7 dup pairs spanning Jan↔Feb and Mar boundaries |
|
||||
| Bookkeeper | B2 — 1099 vendor consolidation | Amazon × 3 spellings, Verizon × 2, Acme Realty × 2, Adobe × 2, Costco × 2, Zoom × 2, Stripe × 4 |
|
||||
| Bookkeeper | B3 — audit trail | every cell change in the run logged with old/new/rule — surface in the demo's audit tab |
|
||||
| Bookkeeper | B4 — per-license economics | demonstrated by pricing copy, not data |
|
||||
| Bookkeeper | B5 — multi-currency | rows 26 (EUR), 27 (GBP), 28 (BRL with comma decimal), 29 (parens-negative) |
|
||||
| RevOps | R1 — per-contact tier | 6 cross-source dup pairs (HubSpot × LinkedIn × Manual Scrape) |
|
||||
| RevOps | R2 — deliverability | rows 26–27 (`uma at uniform dot com`, `victor@@victorco.com` invalid emails) |
|
||||
| RevOps | R3 — GDPR / privacy | demonstrated by the network-tab moat panel + zero-upload claim |
|
||||
| RevOps | R4 — vendor unification | 3 source values (HubSpot / LinkedIn / Manual Scrape), 13 country codes, mixed-shape headers |
|
||||
| RevOps | R5 — suppression list | rows 29–30 (`Suppressed`, `Opted Out` tags) |
|
||||
| Bookkeeper | **26 → 20 rows · 6 phantom duplicates removed** | The same payment posted twice (different date + amount format) collapses to one; dates go ISO, parens-negatives become real negatives |
|
||||
| AP / 1099 | **24 records → 8 vendors · 7 missing EINs recovered** | Each vendor's scattered records merge into one complete row; `merge=true` backfills the EIN/address/phone that any single record was missing |
|
||||
| AR aging | **26 → 21 rows · 5 double-entered invoices removed** | Duplicate invoice numbers collapse; a blank status is backfilled from its twin; invoice + due dates go ISO, amounts numeric |
|
||||
|
||||
### 4.1 `shopify_pet_customers.csv` (20 rows)
|
||||
### 4.1 `bank_reconciliation.csv` (26 rows) — Bookkeeper
|
||||
|
||||
**Looks like**: a Shopify customer export filtered for "Pet Supplies"
|
||||
sales channel, 12 months activity.
|
||||
**Looks like**: two months (Jan + Feb 2025) of business-checking activity
|
||||
from a bank portal, where the Feb re-export overlaps Jan so the same
|
||||
transaction posts twice. Columns: `Date, Description, Vendor, Category,
|
||||
Amount, Account`.
|
||||
|
||||
**Pollution included**:
|
||||
- Whitespace padding (" Alice ", "Sydney Opera House Drive ")
|
||||
- Mixed phone formats: `(415) 555-1234`, `415.555.1234`, `5559876543`,
|
||||
`+1 555-111-1111`
|
||||
- International phones: GB, ES, DE, AU, JP (15 demo rows span 6
|
||||
countries)
|
||||
- Currency variants: `$1,240.50`, `£890.25`, `€2.410,75` (EU comma
|
||||
decimal), `A$ 1,299.00`, `¥75000`
|
||||
- Date formats: `2025-12-04`, `12/15/2025`, `?`, `(blank)`, `(none)`,
|
||||
`#N/A`
|
||||
- Disguised nulls: `N/A`, blank, `(blank)`, `?`, `#N/A`, `(none)`,
|
||||
`unknown`
|
||||
- Name casing: `EVE MARTINEZ`, `henry`, `O'NEIL`, `noah`, mixed Title /
|
||||
ALL CAPS / lower
|
||||
- Email case variants that *should* dedup: `Bob@PetShop.com` vs
|
||||
`alice@petshop.com`
|
||||
- 4 fuzzy duplicates (Alice/Bob same address, Grace/Henry same phone,
|
||||
Carlos/Olivia same address, Ivy/Jack same address)
|
||||
- Mixed date formats: `01/15/2025`, `2025-01-15`, `Jan 18 2025`, `1/27/25`, `Feb 5 2025`.
|
||||
- Currency formats incl. negatives: `-$129.99`, `($89.50)` parens-negative, `+$3,450.00`, `- $599.88`, bare `-129.99`, `(50.00)`.
|
||||
- Whitespace + NBSP padding; smart quotes and an em-dash inside descriptions.
|
||||
- Vendor casing variety on *non-duplicate* rows: `Amazon` / `amazon.com` / `AMAZON.COM`, `Verizon` / `verizon`.
|
||||
- Disguised nulls in Category: `—`, `(blank)`, `?`, `unknown`, `TBD`.
|
||||
- **6 duplicate transactions** — each pair shares the same vendor + real value but a different date *and* amount format, so they collapse only after standardization.
|
||||
|
||||
**After running the pipeline**: 20 rows → 15, ~29 cells canonicalized,
|
||||
~45 sentinels standardised, 5 cross-row duplicates merged. The
|
||||
customer table is now Klaviyo-import-ready and the country column
|
||||
(previously `UK` / `U.K.` / `United Kingdom` / `Germany` / `Italia`)
|
||||
is GB / DE / IT — VAT MOSS report won't break.
|
||||
**After running the pipeline** (validated): **26 → 20 rows, 6 duplicates
|
||||
removed**, 36 date/amount cells standardized (0 unparseable), all dates
|
||||
ISO, parens-negatives resolved (`($89.50)` → `-89.50`), disguised-null
|
||||
categories flagged. The reconciliation ties out.
|
||||
|
||||
### 4.2 `bookkeeper_bank_reconcile.csv` (30 rows)
|
||||
### 4.2 `vendor_1099.csv` (24 rows) — Accounts payable / 1099
|
||||
|
||||
**Looks like**: two months of business checking + credit-card activity
|
||||
exported from a bank portal, with the Feb export accidentally
|
||||
overlapping the Jan export at the month boundary.
|
||||
**Looks like**: a 1099-NEC vendor master list where the same vendor was
|
||||
entered 2–3 times across the year by different staff, each record holding
|
||||
only *part* of the vendor's details. Columns: `Vendor, Contact, Email,
|
||||
Phone, EIN, Address, Total_Paid`.
|
||||
|
||||
**Pollution included**:
|
||||
- Mixed date formats: `01/15/2025`, `2025-01-15`, `Jan 18 2025`,
|
||||
`1/27/25`, `Feb 5 2025`
|
||||
- Currency formats: `-$129.99`, `($89.50)` parens-negative,
|
||||
`+$3,450.00`, `- $599.88` space, bare `-129.99`, `(50.00)`
|
||||
- Header trailing whitespace: `"Date "`
|
||||
- Smart quotes around descriptions: `"autopay"`
|
||||
- Em-dash sentinels in Vendor: `—`
|
||||
- Smart-em-dash inside descriptions: `STAPLES #4422 — paper, toner`
|
||||
- Vendor casing inconsistency: `Amazon` / `amazon.com` / `AMAZON.COM`,
|
||||
`Verizon` / `verizon`
|
||||
- 6 duplicate transactions (same date+amount+vendor recorded twice
|
||||
with different formats)
|
||||
- The duplicate records for a vendor share one email differing only by case/whitespace (the reliable dedup key, matched with the `email` normalizer).
|
||||
- EIN / Phone / Address scattered across the duplicate set so no single record is complete but the union is — gaps marked `—`, `(blank)`, `TBD`, `unknown`, `N/A`.
|
||||
- Vendor name casing/spelling variants, phone formats, EIN formats (`12-3456789` vs `123456789`), `Total_Paid` currency variants.
|
||||
|
||||
**After running the pipeline**: 30 rows → 23, ~84 cells normalized, 7
|
||||
duplicates removed (month-overlap + VAT-MOSS dups). All dates
|
||||
ISO-formatted, all amounts numeric (including EUR/GBP/BRL with comma
|
||||
decimal), vendor casing canonical, parens-negative resolved.
|
||||
**After running the pipeline** (validated): **24 records → 8 vendors, 16
|
||||
duplicates removed, 7 missing EINs recovered** by `merge=true` +
|
||||
`most_complete` survivor, 35 disguised nulls caught, phones/emails/amounts
|
||||
standardized (0 unparseable). One vendor genuinely has no EIN in any
|
||||
record — it survives with a blank EIN as the realistic "flag for
|
||||
follow-up" case.
|
||||
|
||||
### 4.3 `agency_combined_leads.csv` (30 rows)
|
||||
### 4.3 `ar_open_invoices.csv` (26 rows) — Accounts receivable
|
||||
|
||||
**Looks like**: a marketing-ops worksheet combining lead exports from
|
||||
HubSpot + LinkedIn Sales Navigator + manual scraping, ready for
|
||||
campaign targeting.
|
||||
**Looks like**: an open-invoices (unpaid AR) export where some invoices
|
||||
were double-entered in different formats and client contacts are messy.
|
||||
Columns: `Invoice, Client, Email, Invoice_Date, Due_Date, Amount, Status`.
|
||||
|
||||
**Pollution included**:
|
||||
- Phone formats per region: US, UK, Spain, Germany, China, India,
|
||||
Australia, Mexico, Israel, Singapore, Hong Kong, Italy, South
|
||||
Korea — 13 country codes
|
||||
- Country column inconsistent: `USA` / `US` / `United States`
|
||||
- Disguised nulls: `N/A`, `unknown`, `(unknown)`, `(blank)`, `(none)`,
|
||||
`?`, `—`, `#N/A`, `TBD`
|
||||
- Source column tags origin (`HubSpot` / `LinkedIn` / `Manual Scrape`)
|
||||
- Email duplicates across sources with case variants: `alice@acme.com`
|
||||
+ `Alice.Johnson@acme.com`, `bob@beta.com` + `Bob@Beta.com`,
|
||||
`diana@delta.com` from two sources, `carlos@gamma.io` from two
|
||||
sources, `Frank@Foxtrot.de` + `frank@foxtrot.de`
|
||||
- Name casing: `DIANA LEE`, `henry`, `IVY CHEN`, mixed
|
||||
- 6 fuzzy / cross-source duplicates designed to survive the dedup
|
||||
- Score column with sentinel pollution that needs coercion to integer
|
||||
- Two date columns with mixed formats; currency variants incl. a credit memo `($300.00)` → `-300.00`.
|
||||
- Client name casing variety; email case variants (`AP@Acme.com` vs `ap@acme.com`).
|
||||
- Status disguised nulls: `—`, `?`, `(blank)`, `TBD`, `unknown`, `(none)`.
|
||||
- **5 double-entered invoices** — same invoice number twice, dates/amount in different formats, one copy with a blank status the other fills.
|
||||
|
||||
**After running the pipeline**: 30 rows → 24, ~43 cells canonicalized,
|
||||
14 sentinels resolved, 6 cross-source duplicates merged with `merge=true`
|
||||
so each survivor inherits the most-complete picture. Invalid-email
|
||||
rows (deliverability stress) and `Suppressed`/`Opted Out` tags
|
||||
(suppression-list use case) survive as flagged rows the operator
|
||||
manually reviews.
|
||||
**After running the pipeline** (validated): **26 → 21 rows, 5 duplicate
|
||||
invoices removed**, both date columns ISO + amounts numeric + emails
|
||||
lowercased (0 unparseable), 7 disguised-null statuses caught, and a blank
|
||||
status backfilled from its twin via `merge=true`. The aging report stops
|
||||
double-counting.
|
||||
|
||||
## 5. UX flow (per persona)
|
||||
|
||||
@@ -174,26 +142,26 @@ dedicated `app_demo.py` for the cloud build).
|
||||
│ "{Persona-specific H1}" │
|
||||
├──────────────────────────────────────────────────────────┤
|
||||
│ │
|
||||
│ Sample dataset preloaded: shopify_pet_customers.csv │
|
||||
│ Sample dataset preloaded: bank_reconciliation.csv │
|
||||
│ [Replace with your own file (capped 100 rows)] │
|
||||
│ │
|
||||
│ ┌─ BEFORE preview (15 rows) ─────────────────────────┐ │
|
||||
│ │ Alice | (415) 555-1234 | $1,240.50 | … │ │
|
||||
│ │ Bob | 415.555.1234 | $1,240.50 | … │ │
|
||||
│ ┌─ BEFORE preview (26 rows) ─────────────────────────┐ │
|
||||
│ │ 01/15/2025 | Stripe | +$3,450.00 | … │ │
|
||||
│ │ 2025-01-15 | Stripe | 3450.00 | … (dup) │ │
|
||||
│ │ ... │ │
|
||||
│ └──────────────────────────────────────────────────┘ │
|
||||
│ │
|
||||
│ Pipeline (saved): │
|
||||
│ 1. Text Clean → 2. Format Standardize → │
|
||||
│ 3. Missing → 4. Deduplicate │
|
||||
│ 1. Clean Text → 2. Standardize Formats → │
|
||||
│ 3. Fix Missing → 4. Find Duplicates │
|
||||
│ │
|
||||
│ [▶ Run pipeline] │
|
||||
│ │
|
||||
│ ┌─ AFTER preview ───────────────────────────────────┐ │
|
||||
│ │ 15 rows → 11 (4 duplicates merged) │ │
|
||||
│ │ 27 cells canonicalized · 33 sentinels resolved │ │
|
||||
│ │ 26 rows → 20 (6 duplicate transactions removed) │ │
|
||||
│ │ 36 cells standardized · 4 disguised nulls flagged │ │
|
||||
│ │ │ │
|
||||
│ │ Alice Johnson | +14155551234 | 1240.50 | … │ │
|
||||
│ │ 2025-01-15 | Stripe | 3450.00 | … │ │
|
||||
│ │ ... │ │
|
||||
│ └──────────────────────────────────────────────────┘ │
|
||||
│ │
|
||||
@@ -244,27 +212,35 @@ not "demo crippled" data.
|
||||
|
||||
## 7. CTA copy (per persona)
|
||||
|
||||
### 7.1 Shopify pet operator
|
||||
Copy lives in `src/gui/app_demo.py::PERSONAS` (H1 / sub / CTA per tag);
|
||||
keep this section in sync with that dict.
|
||||
|
||||
- **H1**: *Clean your customer / vendor / subscriber exports — locally.*
|
||||
- **Sub**: *Klaviyo-import-ready in 30 seconds. Catches duplicates Excel
|
||||
misses. Your data never leaves your computer.*
|
||||
- **CTA**: *Get DataTools for Shopify — $49 →*
|
||||
### 7.1 Bookkeeper — bank reconciliation (`?p=bookkeeper`)
|
||||
|
||||
### 7.2 Bookkeeper / freelance accountant
|
||||
|
||||
- **H1**: *Reconcile messy bank exports. Hand your client an audit
|
||||
trail.*
|
||||
- **Sub**: *Catches the duplicate transaction Quickbooks imported twice.
|
||||
Standardizes dates, amounts, vendor casing. Every change auditable.*
|
||||
- **H1**: *Catch the transactions your bank export posted twice. Locally.*
|
||||
- **Sub**: *When the Jan and Feb exports overlap, the same payment posts
|
||||
twice in two formats. DataTools standardizes every date and amount, then
|
||||
dedups on the real transaction so your reconciliation ties out — 26 rows
|
||||
→ 20, six phantom duplicates gone.*
|
||||
- **CTA**: *Get DataTools for Bookkeepers — $49 →*
|
||||
|
||||
### 7.3 Marketing / RevOps agency
|
||||
### 7.2 Accounts payable — 1099 prep (`?p=ap-1099`)
|
||||
|
||||
- **H1**: *Dedupe leads across HubSpot, LinkedIn, and manual scrapes.*
|
||||
- **Sub**: *International phones, country normalization, fuzzy dedup
|
||||
with merge — one tool, one schema, no upload.*
|
||||
- **CTA**: *Get DataTools for RevOps — $49 →*
|
||||
- **H1**: *Build a clean 1099 vendor list — with the missing EINs filled in.*
|
||||
- **Sub**: *The same vendor entered three times, each record holding only
|
||||
part of the details. DataTools consolidates to one row and backfills the
|
||||
gaps from the duplicates — 24 records → 8 vendors, 7 missing EINs
|
||||
recovered.*
|
||||
- **CTA**: *Get DataTools for Accounting — $49 →*
|
||||
|
||||
### 7.3 Accounts receivable — open invoices (`?p=ar-aging`)
|
||||
|
||||
- **H1**: *Stop chasing the invoices your aging report counted twice. Locally.*
|
||||
- **Sub**: *Double-entered invoices inflate your AR aging and your
|
||||
follow-ups. DataTools standardizes dates and amounts, lowercases client
|
||||
emails, and removes the duplicate invoice numbers — 26 rows → 21, five
|
||||
phantom invoices off the books.*
|
||||
- **CTA**: *Get DataTools for Accounting — $49 →*
|
||||
|
||||
## 8. Telemetry / conversion tracking
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ CLI (src/cli*.py) GUI (src/gui/app.py + pages/)
|
||||
| `core.errors` | `DataToolsError` hierarchy, `ensure_dataframe()`, `ensure_choice()`, `wrap_file_read/write()`, `format_for_user()` |
|
||||
| `core._constants` | `US_STATE_NAMES`, `US_STATE_CODES`, `USPS_EXPANSIONS`, `USPS_COMPRESSIONS` |
|
||||
|
||||
## Data flow — Deduplicator
|
||||
## Data flow — Find Duplicates
|
||||
|
||||
```
|
||||
read_file() # auto-detect encoding, delimiter, header
|
||||
@@ -96,6 +96,36 @@ DeduplicationResult # deduplicated_df, removed_df, match_groups, l
|
||||
|
||||
No other call sites change. Gate auto-discovers it via the registry.
|
||||
|
||||
### Tool page header — `render_tool_header(tool_id)`
|
||||
|
||||
Every tool page renders its title block via `render_tool_header(tool_id)` in `src/gui/components/_legacy.py` — do not call `st.title()` + `st.caption()` directly. The helper renders:
|
||||
|
||||
- `tools.<id>.page_title` as the page title (left column).
|
||||
- A **Help** popover button right of the title (icon `:material/help_outline:`, label from `help.button_label`). Clicking opens an `st.popover` containing the markdown body.
|
||||
- `tools.<id>.page_caption` as the caption below.
|
||||
|
||||
All copy is i18n-driven; editors can tweak help text without touching Python. If a tool is missing its `help_md` key, the popover falls back to `help.missing_body`.
|
||||
|
||||
**`help_md` structure** (markdown, stored as a single string with `\n` line breaks in JSON):
|
||||
|
||||
```
|
||||
**When to use**
|
||||
- bullet 1
|
||||
- bullet 2
|
||||
|
||||
**Steps**
|
||||
1. numbered step
|
||||
2. numbered step
|
||||
|
||||
**Examples**
|
||||
- example 1
|
||||
- example 2
|
||||
|
||||
**Tip** one-sentence pro tip.
|
||||
```
|
||||
|
||||
Keep it short — the popover is intentionally compact. Mirror the structure across every tool so the muscle memory transfers.
|
||||
|
||||
### i18n — language packs
|
||||
|
||||
The GUI's user-facing strings live in `src/i18n/packs/<code>.json`, keyed by ISO-639-1 code. English (`en.json`) is canonical; missing keys in other packs fall back to English, and missing keys in English fall back to the literal dotted key so a typo is visible rather than silent.
|
||||
@@ -120,12 +150,123 @@ st.warning(t("gate.warning", name=filename)) # {name} interpolated via str.for
|
||||
3. Use the dotted key at the call site: `t("section.subsection.key")` or `t("section.key", name=value)` for placeholder interpolation.
|
||||
|
||||
**Authoring rules:**
|
||||
- Keys live under semantic sections (`home.*`, `upload.*`, `findings.*`, `tools.<id>.name`). Don't nest by language or by tool unless the string is genuinely tool-specific.
|
||||
- Keys live under semantic sections (`home.*`, `upload.*`, `findings.*`, `help.*`, `tools.<id>.name`). Don't nest by language or by tool unless the string is genuinely tool-specific.
|
||||
- Per-tool header copy lives under `tools.<id>.{page_title, page_caption, help_md}`. `page_caption` is the one-line subtitle under the title; `help_md` is the popover body (see *Tool page header* above). Top-level `help.button_label` / `help.missing_body` are shared across every tool.
|
||||
- Use `{named}` placeholders (not positional `{0}`) so translators see what's being interpolated.
|
||||
- Strings can contain Streamlit markdown (`**bold**`) — pass through `st.markdown` / `st.caption` as usual.
|
||||
- Do **not** put strings inside the farewell-overlay JS payload without going through `_js_html_safe()` in `src/gui/components/_legacy.py`; the helper escapes both the JS string terminator and HTML special chars. The test `TestFarewellEscape` pins that contract.
|
||||
- The sidebar picker is mounted by `hide_streamlit_chrome()`, so every page that calls that helper automatically gets the picker. Pages that don't call it (rare) can call `render_language_selector()` directly.
|
||||
|
||||
### Licensing
|
||||
|
||||
The license layer lives at ``src/license/``. The public API:
|
||||
|
||||
```python
|
||||
from src.license import (
|
||||
get_manager, require_feature, current_state,
|
||||
FeatureFlag, Tier, License,
|
||||
)
|
||||
|
||||
mgr = get_manager()
|
||||
if not mgr.is_valid():
|
||||
raise RuntimeError("Not licensed")
|
||||
require_feature(FeatureFlag.DEDUPLICATOR)
|
||||
```
|
||||
|
||||
**Storage**: ``~/.datatools/license.json`` (override via
|
||||
``DATATOOLS_LICENSE_PATH``). Signed with Ed25519 (asymmetric) — the
|
||||
seller's private key signs; the buyer's binary verifies with the
|
||||
embedded public key.
|
||||
|
||||
**Key material**:
|
||||
|
||||
| Variable | Who has it | Where it's used |
|
||||
|---|---|---|
|
||||
| ``DATATOOLS_LICENSE_PRIVKEY`` | Seller only | ``scripts/generate_license.py`` (mint a buyer's blob), ``scripts/generate_keypair.py`` writes a fresh one |
|
||||
| ``DATATOOLS_LICENSE_PUBKEY`` | Every shipped binary | Verification at activation time; set at build time via PyInstaller env |
|
||||
|
||||
If neither env var is set, ``src.license.crypto`` falls back to the
|
||||
deterministic dev keypair in ``src/license/_dev_keypair.py``. The
|
||||
dev key is in source on purpose (so tests work without secrets),
|
||||
but a frozen build that's still using it is a build-config bug —
|
||||
:func:`assert_production_safe` refuses to start such a binary.
|
||||
|
||||
**First-time setup for shipped builds**:
|
||||
|
||||
1. ``python scripts/generate_keypair.py --output prod-keys.env`` —
|
||||
creates a fresh keypair.
|
||||
2. Stash ``DATATOOLS_LICENSE_PRIVKEY`` somewhere safe (password
|
||||
manager / KMS). Lose it and you can't issue renewals without
|
||||
reshipping a new build with a new public key.
|
||||
3. Configure the PyInstaller build env with
|
||||
``DATATOOLS_LICENSE_PUBKEY=<hex>`` so the shipped binary
|
||||
verifies against the production key.
|
||||
4. Mint buyer licenses with
|
||||
``DATATOOLS_LICENSE_PRIVKEY=<hex> python scripts/generate_license.py ...``.
|
||||
|
||||
**Dev bypass**: ``DATATOOLS_DEV_MODE=1`` short-circuits every check.
|
||||
The test suite's autouse fixture sets this so existing tests don't
|
||||
need their own license fixtures. Tests that need the real check
|
||||
explicitly use ``isolated_license_path`` /
|
||||
``activated_license_manager`` / ``unactivated_license_manager``.
|
||||
|
||||
**Adding a feature flag**:
|
||||
|
||||
1. Add the enum value to ``FeatureFlag`` in ``src/license/schema.py``.
|
||||
2. Add it to the relevant tier's set in
|
||||
``FEATURES_BY_TIER`` in ``src/license/features.py``.
|
||||
3. Gate at the call site: ``require_feature(FeatureFlag.YOUR_FLAG)``.
|
||||
|
||||
**Adding a new tier**:
|
||||
|
||||
1. Add the enum value to ``Tier``.
|
||||
2. Add a row to ``FEATURES_BY_TIER`` listing the unlocked flags.
|
||||
3. Add ``license.tier_<name>`` translation keys to every i18n pack.
|
||||
4. The activation flow, sidebar status badge, feature gate, and home
|
||||
grid lock badge all pick up the new tier automatically.
|
||||
|
||||
**Worked example — the Lite tier**:
|
||||
|
||||
```python
|
||||
# src/license/schema.py
|
||||
class Tier(str, Enum):
|
||||
LITE = "lite" # new
|
||||
CORE = "core"
|
||||
...
|
||||
|
||||
# src/license/features.py
|
||||
FEATURES_BY_TIER = {
|
||||
...
|
||||
Tier.LITE: frozenset({
|
||||
FeatureFlag.DEDUPLICATOR,
|
||||
FeatureFlag.TEXT_CLEANER,
|
||||
FeatureFlag.FORMAT_STANDARDIZER,
|
||||
}),
|
||||
Tier.CORE: _all(),
|
||||
...
|
||||
}
|
||||
```
|
||||
|
||||
Then in en.json/es.json add ``license.tier_lite``. That's it — the
|
||||
existing ``require_feature_or_render_upgrade`` (GUI) and
|
||||
``guard(feature=...)`` (CLI) calls in every tool page/CLI route a
|
||||
Lite user into the upgrade prompt for any tool the tier doesn't
|
||||
unlock. The home grid's lock badge fires off the same feature
|
||||
lookup.
|
||||
|
||||
**Minting a license** (creator-only):
|
||||
|
||||
```bash
|
||||
DATATOOLS_LICENSE_SECRET=<shipping-secret> \
|
||||
python scripts/generate_license.py \
|
||||
--name "Jane Doe" --email jane@example.com \
|
||||
--tier core --years 1
|
||||
```
|
||||
|
||||
The script prints a ``DTLIC1:`` blob to stdout — deliver this in the
|
||||
Gumroad / purchase email. The buyer pastes it into the activation
|
||||
page or runs ``python -m src.license_cli activate <blob> --name ...``.
|
||||
|
||||
### Add a format-standardizer field type
|
||||
|
||||
1. Add value to `FieldType` enum in `core/format_standardize.py`.
|
||||
@@ -155,6 +296,37 @@ GUI / CLI handlers: use `format_for_user(exc, context="...")` to render.
|
||||
|
||||
All `DataToolsError` subclasses extend stdlib `ValueError` or `OSError` so existing handlers still catch them.
|
||||
|
||||
## PDF Extractor — bundled Tesseract
|
||||
|
||||
Frozen builds (installer / AppImage) ship Tesseract OCR inside the bundle so scanned PDFs work without a separate system install. Source / `pip` developer environments still resolve Tesseract from `PATH`.
|
||||
|
||||
**Runtime layout (frozen bundles)**:
|
||||
|
||||
| Resource | Path |
|
||||
|---|---|
|
||||
| Tesseract binary | `Path(sys._MEIPASS) / "tesseract" / "tesseract"` (Linux/macOS), `…/tesseract/tesseract.exe` (Windows) |
|
||||
| Tessdata directory | `Path(sys._MEIPASS) / "tesseract" / "tessdata"` |
|
||||
| English model | `Path(sys._MEIPASS) / "tesseract" / "tessdata" / "eng.traineddata"` |
|
||||
|
||||
**Discovery order** (PDF Extractor runtime):
|
||||
|
||||
1. `DATATOOLS_TESSERACT_BIN` env var (override — explicit path to a `tesseract` binary).
|
||||
2. Bundled path under `sys._MEIPASS` (frozen bundles only — falls through to step 3 otherwise).
|
||||
3. `tesseract` on `PATH` (developer setups, source checkouts).
|
||||
4. Windows well-known locations (`C:\Program Files\Tesseract-OCR\tesseract.exe`, etc.).
|
||||
|
||||
**Where the bytes come from**:
|
||||
|
||||
- **Tessdata** is vendored at `build/vendor/tessdata/eng.traineddata` — the "best" English model from [tessdata_best](https://github.com/tesseract-ocr/tessdata_best). PyInstaller's spec copies it into `tesseract/tessdata/` inside the bundle.
|
||||
- **Tesseract binary** is fetched at build time by `build/tesseract.py` — per-platform download URLs are pinned in that module. The current pin is **Tesseract 5.5.0**. CI (`.github/workflows/build.yml`) imports `fetch_tessdata` + `fetch_tesseract_for_platform` and runs them before PyInstaller.
|
||||
|
||||
**To update Tesseract**:
|
||||
|
||||
1. Bump the version pin + the per-platform fetch URLs in `build/tesseract.py`.
|
||||
2. If upstream changed the `eng.traineddata` schema, refresh `build/vendor/tessdata/eng.traineddata` from `tessdata_best` at the matching tag.
|
||||
3. Push a `v*` tag so CI rebuilds all three platforms, then smoke-test a scanned-PDF run through the PDF Extractor before publishing the release.
|
||||
4. Update `LICENSE_TESSERACT.txt` at the repo root if the upstream license terms change (Tesseract is Apache-2.0 today).
|
||||
|
||||
## Tests
|
||||
|
||||
```bash
|
||||
@@ -180,6 +352,8 @@ tests/
|
||||
├── test_analyze.py · test_normalize.py · test_text_clean.py
|
||||
├── test_format_standardize.py
|
||||
├── test_format_standardize_corpus.py # 199-row buyer corpus
|
||||
├── test_pipeline.py # pipeline engine: adapters, run, validate, serialize
|
||||
├── test_cli_pipeline.py # pipeline CLI: recommend/apply/strict/audit
|
||||
├── test_audit_fixes.py · test_errors.py · test_fixes_unit.py
|
||||
├── test_corpus.py · test_encodings_corpus.py · test_fixtures_sweep.py
|
||||
├── test_cli.py · test_cli_*.py · test_e2e.py · test_install.py
|
||||
@@ -193,10 +367,27 @@ tests/
|
||||
├── test_workflows.py # happy path per Ready tool
|
||||
├── test_dedup_review.py # match-group card interactions
|
||||
├── test_advanced_panels.py # config_panel widgets
|
||||
├── test_pipeline_builder.py # module-card builder: cards, reorder, JSON, run
|
||||
├── test_pipeline_phrasing.py # step_phrase/step_status + name bridge (pure fns)
|
||||
├── test_errors.py # malformed-upload error paths
|
||||
└── test_findings_panel.py # analyzer findings rendering
|
||||
```
|
||||
|
||||
### Pipeline (Automated Workflows) coverage
|
||||
|
||||
The pipeline feature is pinned end to end across four files (~115 tests):
|
||||
`test_pipeline.py` (core engine — every adapter's summary numbers, step
|
||||
data-flow, error stop/continue, empty/single-column/all-disabled edges,
|
||||
dict + file serialization round-trips, `recommended_pipeline(include=…)`,
|
||||
soft-dependency validation), `test_cli_pipeline.py` (CLI — `--recommend`,
|
||||
dry-run-by-default, `--apply` output + audit JSON, `--steps`, `--strict`,
|
||||
`--continue-on-error`, arg validation, save→load round-trip),
|
||||
`test_pipeline_builder.py` (the visual builder via AppTest — card seeding,
|
||||
toggle, reorder ▲/▼, add/remove, restore-recommended, Advanced JSON
|
||||
import/export, per-tool Configure panels emitting the right option dicts),
|
||||
and `test_pipeline_phrasing.py` (the plain-English `step_phrase`/`step_status`
|
||||
helpers and the adapter-key→friendly-name bridge as pure functions).
|
||||
|
||||
### GUI test layer
|
||||
|
||||
GUI tests drive pages with `streamlit.testing.v1.AppTest` —
|
||||
|
||||
244
docs/FUTURE-TOOLS.md
Normal file
244
docs/FUTURE-TOOLS.md
Normal file
@@ -0,0 +1,244 @@
|
||||
# Future tools — design notes
|
||||
|
||||
> Creator-only. Specs for tools the strategic plan refuses to build right now
|
||||
> but that surface repeatedly enough to be worth documenting once instead of
|
||||
> re-thinking from scratch every time a customer asks.
|
||||
> **Status of these tools**: post-launch, post-revenue. See `PLAN.md` §2.1 —
|
||||
> new-tool development is frozen until DataTools has a paying customer and a
|
||||
> repeated demand signal for the same idea. This file is the resting place
|
||||
> for those ideas in the meantime; nothing here ships unless a future
|
||||
> decision says it does.
|
||||
|
||||
Each entry follows the same shape: **What it does**, **Why someone would
|
||||
want it**, **Can we ship it now?**, **Approach**, **GUI sketch**, **Effort**,
|
||||
**Risks/unknowns**, **Ship criteria** (the signal that overrides the freeze).
|
||||
|
||||
---
|
||||
|
||||
## 10. PDF → CSV extractor (bank statements + similar)
|
||||
|
||||
### What it does
|
||||
|
||||
Takes a PDF (typically a bank statement, expense report, paystub, invoice,
|
||||
or any document where humans-but-not-computers can read a table) and turns
|
||||
the tabular content into a CSV that the rest of DataTools can consume.
|
||||
|
||||
The user shows the tool **where** the data lives by drawing rectangles on
|
||||
a rendered preview of the first page; the tool then applies those region
|
||||
templates to every page of the document (and remembers the template so the
|
||||
same template can be re-applied to next month's statement without
|
||||
re-clicking).
|
||||
|
||||
### Why someone would want it
|
||||
|
||||
Bookkeepers, accountants, and any small-business operator who:
|
||||
|
||||
- Gets bank/credit-card statements only as PDFs (most US banks; many
|
||||
European ones).
|
||||
- Wants to import transactions into QuickBooks / Xero / a spreadsheet
|
||||
without paying $10–$30/month for a SaaS converter (Docparser,
|
||||
Rossum, Hubdoc) or relying on a Python script they can't maintain.
|
||||
- Has 12 months × N accounts of statements to back-fill into a
|
||||
ledger.
|
||||
|
||||
This is the most-requested DataTools adjacency in the casual feedback we
|
||||
have so far. It maps tightly onto the **bookkeeper niche** identified in
|
||||
`PLAN.md` §2.3 — that persona is exactly who needs PDF extraction and is
|
||||
exactly the kind of operator who'd pay for a one-time desktop tool over a
|
||||
recurring SaaS subscription.
|
||||
|
||||
### Can we ship it now?
|
||||
|
||||
**No.** Current state, verified 2026-05-17:
|
||||
|
||||
- No PDF dependency in `requirements.txt` or `requirements-dev.txt`.
|
||||
- No PDF-touching code anywhere under `src/`. The single
|
||||
string-mention of "PDF" in the codebase is in the **output** copy for
|
||||
the Quality Check tool ("generate PDF/Excel quality reports"),
|
||||
unrelated to extraction.
|
||||
- No region-selection / canvas component in the Streamlit GUI today.
|
||||
|
||||
Building this requires net-new infrastructure on three axes (libraries,
|
||||
extraction core, region-picker UI). Estimates below.
|
||||
|
||||
### Approach (technical)
|
||||
|
||||
PDFs split cleanly into two populations and the strategy differs:
|
||||
|
||||
1. **Native / text-layer PDFs** — text is stored as text, just laid out
|
||||
visually. Most modern bank statements are this. Solvable with
|
||||
coordinate-aware text extraction:
|
||||
|
||||
- **`pdfplumber`** (BSD-3, on top of `pdfminer.six`) — gives `(x0, y0,
|
||||
x1, y1, text)` per character/word/line for each page. Mature, well
|
||||
tested, single dependency, no native compiler. **First-choice.**
|
||||
- **`pypdf`** (BSD-3) — text-only, no positions. Too coarse for
|
||||
statement parsing; useful only for "the whole document as one big
|
||||
string."
|
||||
- **`camelot-py`** (MIT) — purpose-built for table extraction.
|
||||
Heavier (needs `ghostscript` and `tk`/`opencv` for some modes),
|
||||
and assumes the table grid is already visible. Worth evaluating
|
||||
as a fallback for documents with explicit ruled tables.
|
||||
|
||||
2. **Scanned / image-only PDFs** — pixels of a scanner; no text layer.
|
||||
Less common from major banks today but still happens with old PDFs
|
||||
and receipts. Needs OCR:
|
||||
|
||||
- **`pytesseract`** wrapping the **Tesseract** binary (Apache-2). The
|
||||
OCR is good for English on clean scans, mediocre on receipts.
|
||||
Detect with `pdfplumber`: a page where every character is in a
|
||||
glyph "image" object means the page is image-only → OCR fallback.
|
||||
|
||||
The extraction core would be a state machine:
|
||||
|
||||
1. Render page to an image (`pdfplumber.Page.to_image()` returns a PIL
|
||||
image at a chosen DPI).
|
||||
2. User draws a header region and per-row regions (or marks a single
|
||||
table bounding box + column dividers) on the preview.
|
||||
3. For each PDF page, crop the corresponding pixel region (or pdf
|
||||
coordinate region), pull the text in that crop, and apply per-region
|
||||
parsing (date, amount, description).
|
||||
4. Emit one CSV row per detected statement row.
|
||||
|
||||
Bank-statement-specific niceties — implementable as templates on top of
|
||||
the generic engine:
|
||||
|
||||
- Recurring-template store: save "Chase visa October layout" once, the
|
||||
next month's PDF lands on the same template automatically. JSON file
|
||||
in `~/.datatools/templates/` keyed by a layout fingerprint (page
|
||||
size + header text hash).
|
||||
- Multi-page row stitching: a row that wraps across pages gets merged
|
||||
back together based on date-column continuity.
|
||||
- Currency / sign inference: a column that mixes `$1,234.56` and
|
||||
`($45.00)` — already handled by the (now-existing) Standardize
|
||||
Formats analyzer rules.
|
||||
|
||||
### GUI sketch
|
||||
|
||||
The hardest part of the whole project. Streamlit doesn't ship a native
|
||||
"draw rectangles on an image" widget. Options:
|
||||
|
||||
- **`streamlit-drawable-canvas`** — community component (MIT-licensed).
|
||||
Lets the user draw freehand rectangles on top of a background image.
|
||||
Returns the rectangle coordinates as JSON. Active maintenance.
|
||||
**First-choice for the region picker.**
|
||||
- **`streamlit-cropper`** — single-rectangle crop tool. Good if we only
|
||||
needed the table bbox; too limited for "header region + column
|
||||
dividers + repeating-row template."
|
||||
- **Custom React component** — fully tailored UX but adds a build
|
||||
toolchain DataTools doesn't have today. Last resort.
|
||||
|
||||
Sketch of the proposed page (under "Transformations" in the sidebar
|
||||
section):
|
||||
|
||||
```
|
||||
🧾 PDF → CSV (Beta)
|
||||
─────────────────────────────────────────────────────────────────────
|
||||
Upload a PDF [ Browse… ]
|
||||
(statement / invoice / form — text-based PDFs work best)
|
||||
|
||||
[ ▸ Preview: October-statement.pdf · 3 pages ]
|
||||
┌────────────────────────────────────────────────┐
|
||||
│ CHASE BANK │
|
||||
│ Statement period Oct 1–31, 2025 │
|
||||
│ ┌─[1: header strip — drawn in red]──────────┐ │
|
||||
│ │ Date Description Amount │ │
|
||||
│ └────────────────────────────────────────────┘ │
|
||||
│ ┌─[2: row template — drawn in green]────────┐ │
|
||||
│ │ 10/03 AMAZON.COM #42… -45.67 │ │
|
||||
│ └────────────────────────────────────────────┘ │
|
||||
│ ⋮ (more transactions) │
|
||||
└────────────────────────────────────────────────┘
|
||||
|
||||
Columns: [Date] [Description] [Amount] [+ Add column]
|
||||
|
||||
Apply template to: ( ) Only this page
|
||||
(•) All pages with this layout
|
||||
( ) All pages (force)
|
||||
|
||||
[ Save template as… Chase Visa Oct 2025 ]
|
||||
|
||||
[ Run extraction → CSV ]
|
||||
```
|
||||
|
||||
After "Run extraction": the standard tool-page result layout (preview
|
||||
table, "Saved to ~/Downloads/<name>_extracted.csv", "Open Downloads
|
||||
folder" — matching the other Ready tools).
|
||||
|
||||
The **template save/recall** is what makes this a one-time setup
|
||||
instead of a per-document chore — bookkeepers don't want to re-draw
|
||||
rectangles every month.
|
||||
|
||||
### Effort estimate
|
||||
|
||||
| Phase | Scope | Estimate | Risk |
|
||||
|---|---|---|---|
|
||||
| **A. Backend, native PDFs only** | pdfplumber-based extraction, hard-coded region passed via a JSON config (no GUI) | **1–2 weeks** | Low — straightforward use of pdfplumber. |
|
||||
| **B. Region-picker GUI** | streamlit-drawable-canvas, multi-region drawing, per-region role assignment (date / amount / description) | **2–3 weeks** | Medium — the canvas component has quirks; persisting region state across reruns is non-trivial. |
|
||||
| **C. Multi-page application + template persistence** | Apply one page's template to N pages, save/load templates, layout fingerprint | **1–2 weeks** | Medium — "is the next page the same layout?" is a real perception problem; we'll need a heuristic. |
|
||||
| **D. Scanned-PDF OCR fallback** | Detect image-only pages, run Tesseract, merge OCR text into the extraction path | **2–3 weeks** | High — OCR accuracy is variable; we'd want a quality threshold + a "fail this page noisily" path. Bundling Tesseract with the PyInstaller build is its own packaging headache. |
|
||||
| **E. Bank-statement specifics** | Cross-page row stitching, currency-sign inference, multi-account splits | **1–2 weeks** | Medium — every bank's idea of a "statement" differs. Templates absorb most of the variance. |
|
||||
|
||||
**Realistic total for a polished v1**: 6–10 calendar weeks of focused work
|
||||
(text-PDFs + GUI + templates + statement-specific niceties). Add another
|
||||
2–3 weeks if scanned PDFs are required at launch.
|
||||
|
||||
**Minimum viable extract** (just text PDFs, single-region drawing, no
|
||||
template recall, no OCR): **3–4 weeks**. Worth scoping a beta at that
|
||||
level before committing to the full surface.
|
||||
|
||||
### Difficulty rating
|
||||
|
||||
**Medium-hard.** Not because any single piece is novel — pdfplumber +
|
||||
streamlit-drawable-canvas are well-trodden libraries — but because the
|
||||
*combination* (point-and-click region selection that persists across
|
||||
multiple PDF pages and across documents with similar layouts) is where
|
||||
most of the engineering goes. The "every bank does it slightly
|
||||
differently" reality makes templates a hard requirement rather than a
|
||||
nice-to-have, and templates raise the design effort.
|
||||
|
||||
### Risks / unknowns
|
||||
|
||||
- **Scanned-PDF coverage**: if a meaningful slice of the addressable
|
||||
market sends image-only PDFs (older statements, scanned receipts),
|
||||
shipping text-only extraction limits the audience. Decide via the
|
||||
first 10–20 user requests.
|
||||
- **PyInstaller packaging of Tesseract**: bundling the OCR binary into
|
||||
the desktop build is non-trivial. May force a "Tesseract not found —
|
||||
install it separately" path on first launch, which hurts the "one-
|
||||
click install" story.
|
||||
- **Bank layout drift**: a template captured today can stop working
|
||||
next month if the bank redesigns its statement. Layout-fingerprint
|
||||
detection has to fail loudly rather than silently produce garbage.
|
||||
- **PII surface**: bank statements are some of the most sensitive
|
||||
documents the user might touch. The "runs locally — your data never
|
||||
leaves this computer" guarantee is even more load-bearing here than
|
||||
for CSVs. No telemetry, no cloud OCR services, hard line.
|
||||
|
||||
### Ship criteria
|
||||
|
||||
Before this tool re-enters active development, all of these need to be
|
||||
true:
|
||||
|
||||
- DataTools has shipped to **≥1 paying customer** (the `PLAN.md` §2.1
|
||||
freeze condition).
|
||||
- **At least 3 paying customers OR 5 demo-traffic emails** have
|
||||
explicitly asked for PDF extraction. Below that signal, build
|
||||
something else.
|
||||
- The bookkeeper niche (per `PLAN.md` §2.3) has at least one converted
|
||||
customer — that's the persona who actually needs this tool, and
|
||||
confirming they pay before building a tool aimed squarely at them
|
||||
is the discipline the freeze exists to enforce.
|
||||
|
||||
If those three trip, the **Phase A minimum-viable beta (3–4 weeks)**
|
||||
goes first — text PDFs + single-region drawing — so we can see real
|
||||
user behaviour before committing to the full template surface.
|
||||
|
||||
---
|
||||
|
||||
## (placeholder for additional future-tool entries)
|
||||
|
||||
Add new entries above this line. Keep the same shape:
|
||||
What / Why / Can we ship now / Approach / GUI / Effort / Risks /
|
||||
Ship criteria. The shape is what makes "is this idea ready" a
|
||||
factual question instead of an opinion.
|
||||
259
docs/LICENSE-SERVER.md
Normal file
259
docs/LICENSE-SERVER.md
Normal file
@@ -0,0 +1,259 @@
|
||||
# LICENSE-SERVER — online issuance & record-keeping
|
||||
|
||||
**Status:** **deployed (PR 1 + PR 2 code merged)**. Live at
|
||||
`licenses.datatools.unalogix.com`. See `ADMIN.md §"Live deployment"`
|
||||
for day-2 operations, and `ARCHITECTURE.md` for the end-to-end
|
||||
diagram including the desktop and storefronts.
|
||||
|
||||
This doc describes the smallest useful server we could build to
|
||||
replace the manual mint-and-paste workflow, without compromising the
|
||||
"your data never leaves your computer" promise to buyers (see
|
||||
`DECISIONS.md §9b`).
|
||||
|
||||
---
|
||||
|
||||
## Goals
|
||||
|
||||
1. **Automate fulfillment.** Gumroad sale → buyer gets a blob in
|
||||
their inbox within seconds. No creator intervention.
|
||||
2. **Authoritative customer list.** A queryable record of who has
|
||||
what tier, when it expires, what they paid. Replaces the JSONL
|
||||
log as the system of record.
|
||||
3. **Self-service renewal & re-delivery.** Buyer enters their email
|
||||
→ gets a fresh blob or a copy of their existing one. Cuts support
|
||||
load.
|
||||
4. **Move the private key off the founder's laptop.** Today the prod
|
||||
private key has to be loaded as an env var to mint anything;
|
||||
that's a security hazard. Server-side, it lives in a KMS and the
|
||||
laptop never touches it.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- **No phone-home from the desktop app.** Activation stays offline.
|
||||
The shipped binary still verifies blobs against the embedded
|
||||
pubkey with no network call. `DECISIONS.md §9b` stands.
|
||||
- **No per-machine activation limits enforced server-side.** v1
|
||||
treats one license = one buyer, used on as many of their machines
|
||||
as they want. Revisit only if abuse appears.
|
||||
- **No telemetry.** The server only knows what the buyer or Gumroad
|
||||
tells it (purchase events, renewal requests). It does not learn
|
||||
anything from desktop installations.
|
||||
|
||||
---
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
┌─────────────────┐
|
||||
│ Gumroad │
|
||||
└────────┬────────┘
|
||||
│ webhook (sale, refund)
|
||||
▼
|
||||
┌──────────────┐ ┌───────────────┐ ┌──────────────┐
|
||||
│ Buyer email │◄──────│ Mint API │──────►│ licenses │
|
||||
│ (SMTP send) │ │ (Python web) │ │ (Postgres) │
|
||||
└──────────────┘ └───────┬───────┘ └──────────────┘
|
||||
│ sign() via
|
||||
▼
|
||||
┌─────────────────┐
|
||||
│ KMS / HSM │
|
||||
│ (private key) │
|
||||
└─────────────────┘
|
||||
|
||||
┌─────────────────────────────────────────┐
|
||||
│ Renewal / re-delivery portal │
|
||||
│ - buyer enters email │
|
||||
│ - signed magic link │
|
||||
│ - sees current license + "resend" │
|
||||
└─────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Components
|
||||
|
||||
### 1. Mint API
|
||||
|
||||
Thin Python web service (FastAPI or Flask — Streamlit isn't appropriate
|
||||
here). Two internal endpoints:
|
||||
|
||||
- `POST /internal/mint` — name, email, tier, years → blob + DB row.
|
||||
Auth: shared HMAC header from the webhook receiver only.
|
||||
- `POST /internal/revoke` — license_key → sets `revoked_at`. Auth: same.
|
||||
|
||||
The mint endpoint is the **only** place that calls `crypto.sign()`.
|
||||
It pulls the private key from the KMS at request time; the key
|
||||
material never lives in the API process's environment.
|
||||
|
||||
### 2. Webhook receiver
|
||||
|
||||
Public endpoint `POST /webhooks/gumroad`. Verifies Gumroad's
|
||||
signature, maps the payload to a `mint` call, returns 200. Stores
|
||||
the raw payload to a `gumroad_events` table for audit.
|
||||
|
||||
Refunds: webhook → `POST /internal/revoke` keyed on
|
||||
`gumroad_order_id`. The desktop app doesn't currently honor
|
||||
revocations (no online check), but future buyers won't be able to
|
||||
renew a revoked license, and the row remains as evidence if a
|
||||
dispute escalates.
|
||||
|
||||
### 3. Renewal portal
|
||||
|
||||
Single-page form, public. Buyer enters email → server emails a
|
||||
signed magic link → click → page shows their license (tier, expiry,
|
||||
"resend blob" button, "renew" button).
|
||||
|
||||
Renew flow: button → `POST /internal/mint` with the same name/email
|
||||
and a fresh expiry → buyer gets the new blob → pastes into desktop
|
||||
app via existing `license_cli.py renew`. No code change in the
|
||||
desktop app.
|
||||
|
||||
### 4. Database
|
||||
|
||||
Postgres (small — a few thousand rows for the foreseeable future).
|
||||
Single source of truth for the customer list.
|
||||
|
||||
---
|
||||
|
||||
## Schema
|
||||
|
||||
```sql
|
||||
CREATE TABLE licenses (
|
||||
license_key text PRIMARY KEY, -- DT1-{TIER}-xxxx-xxxx
|
||||
name text NOT NULL,
|
||||
email text NOT NULL,
|
||||
tier text NOT NULL, -- lite | core | pro | enterprise
|
||||
issued_at timestamptz NOT NULL,
|
||||
expires_at timestamptz NOT NULL,
|
||||
blob text NOT NULL, -- DTLIC2:...
|
||||
gumroad_order_id text UNIQUE, -- null for manual mints
|
||||
revoked_at timestamptz, -- null = active
|
||||
notes text -- free-form support notes
|
||||
);
|
||||
|
||||
CREATE INDEX idx_licenses_email ON licenses (lower(email));
|
||||
CREATE INDEX idx_licenses_expires ON licenses (expires_at) WHERE revoked_at IS NULL;
|
||||
CREATE INDEX idx_licenses_gumroad ON licenses (gumroad_order_id);
|
||||
|
||||
CREATE TABLE gumroad_events (
|
||||
id bigserial PRIMARY KEY,
|
||||
received_at timestamptz NOT NULL DEFAULT now(),
|
||||
event_type text NOT NULL, -- sale | refund | dispute | ...
|
||||
order_id text,
|
||||
raw_payload jsonb NOT NULL,
|
||||
processed boolean NOT NULL DEFAULT false,
|
||||
error text -- non-null if processing failed
|
||||
);
|
||||
```
|
||||
|
||||
The `licenses` schema is the JSONL log fields plus
|
||||
`gumroad_order_id`, `revoked_at`, `notes`. The migration script from
|
||||
JSONL → Postgres is therefore a flat insert.
|
||||
|
||||
---
|
||||
|
||||
## Security
|
||||
|
||||
- **Private key**: AWS KMS, GCP KMS, or HashiCorp Vault. Mint API
|
||||
has IAM permission to *use* the key (sign operation), not to
|
||||
*export* it. Rotating to a new key still requires a new desktop
|
||||
build (the pubkey is embedded); plan a 90-day overlap window where
|
||||
both keys are accepted.
|
||||
- **Webhook secret**: Gumroad's HMAC signature, verified before
|
||||
touching the body.
|
||||
- **Internal endpoints**: not reachable from the public internet —
|
||||
bind to localhost or a private subnet, fronted by the webhook
|
||||
receiver and the renewal portal.
|
||||
- **PII**: name + email + Gumroad order ID. Standard customer-data
|
||||
hygiene — DB backups encrypted at rest, no PII in application
|
||||
logs, GDPR delete-on-request supported via a `DELETE FROM
|
||||
licenses WHERE email = ?` (the desktop activation still works
|
||||
until the license expires; the buyer just won't appear in our
|
||||
records anymore).
|
||||
- **Mint API access**: short-lived signed tokens for any creator
|
||||
CLI that talks to it. The CLI is a thin wrapper around the same
|
||||
`POST /internal/mint`; the days of running
|
||||
`scripts/generate_license.py` against the prod private key on a
|
||||
laptop are over once the server exists.
|
||||
|
||||
---
|
||||
|
||||
## Migration plan
|
||||
|
||||
Three phases, each independently revertable.
|
||||
|
||||
### Phase 0 (done)
|
||||
|
||||
- Ed25519 signing with prod key on creator's laptop.
|
||||
- Local JSONL issuance log at `~/.datatools-creator/issued.jsonl`.
|
||||
|
||||
### Phase 1 — server stands up, no behavior change
|
||||
|
||||
1. Stand up Postgres + Mint API in a small VPS / Fly.io / Render box.
|
||||
2. Provision a KMS-held keypair; **the public key must match the one
|
||||
already embedded in the shipped binary** — i.e., import the
|
||||
existing prod private key into KMS, do not generate a new one. If
|
||||
the existing key is laptop-only and can't be imported, plan a
|
||||
build-with-new-pubkey + buyer-side rotation cycle (see
|
||||
`ADMIN.md` Recovery).
|
||||
3. Run a one-shot script: read `~/.datatools-creator/issued.jsonl`,
|
||||
`INSERT … ON CONFLICT (license_key) DO NOTHING` each row.
|
||||
4. Add a creator-only CLI command `datatools-admin mint` that calls
|
||||
`POST /internal/mint` instead of running the local script. Local
|
||||
script stays as a fallback.
|
||||
|
||||
At this point: nothing buyer-facing has changed. The creator now has
|
||||
two ways to mint (server or local) and a real DB.
|
||||
|
||||
### Phase 2 — automation
|
||||
|
||||
5. Wire the Gumroad webhook. New buyers get automated fulfillment.
|
||||
6. Manual mints (friends, comps, support replacements) still go
|
||||
through `datatools-admin mint`, which writes to the same DB.
|
||||
7. Old local script is deprecated but kept (read-only) as a break-glass
|
||||
tool if the server is down.
|
||||
|
||||
### Phase 3 — self-service
|
||||
|
||||
8. Ship the renewal portal.
|
||||
9. Replace "email support to lose-my-blob" with a self-service form.
|
||||
|
||||
Each phase ships independently. The desktop app sees no change
|
||||
across any of them — that's the whole point.
|
||||
|
||||
---
|
||||
|
||||
## Open questions
|
||||
|
||||
- **Hosting choice.** *Decided: self-hosted* on the existing
|
||||
`46.225.166.142` box alongside the `*.invixiom.com` services.
|
||||
Runbook in `SETUP-LICENSE-SERVER.md`. Operator owns uptime,
|
||||
backups, TLS renewal, and key custody — see that doc's
|
||||
"Operational concerns" section.
|
||||
- **Per-seat or per-device limits?** v1 says no. Revisit if/when
|
||||
abuse is observable.
|
||||
- **Email delivery.** Postmark or SES — both fine. Pick whichever the
|
||||
rest of the stack uses. Avoid Gmail SMTP for transactional mail.
|
||||
- **Audit log retention.** `gumroad_events` rows are unbounded growth
|
||||
but trivially small. Default to forever; partition by year if it
|
||||
ever exceeds a few GB.
|
||||
- **Existing Gumroad customers.** Before any of this lands, every
|
||||
buyer is already in Gumroad's records. A one-shot import from
|
||||
Gumroad's CSV export → `licenses` table would catch anyone whose
|
||||
blob the JSONL log doesn't have (e.g., if the creator's laptop
|
||||
was lost before this design lands).
|
||||
|
||||
---
|
||||
|
||||
## Code pointers (current state, for the future implementer)
|
||||
|
||||
| File | What it does now | What changes |
|
||||
|------|------------------|--------------|
|
||||
| `scripts/generate_license.py` | Sign locally, append JSONL | Becomes a CLI client of the Mint API |
|
||||
| `src/license/crypto.py` | `sign()` reads `$DATATOOLS_LICENSE_PRIVKEY` | `sign()` calls KMS; the env var stays as a fallback for local dev |
|
||||
| `src/license_cli.py` | Activate / status / renew — already buyer-facing | **No change.** Still verifies offline against embedded pubkey |
|
||||
| `src/license/manager.py` | Verify, persist | **No change.** |
|
||||
|
||||
The desktop app is deliberately decoupled from any of this. The
|
||||
server is a fulfillment + record-keeping layer wrapped around the
|
||||
existing, frozen, offline activation flow.
|
||||
@@ -30,7 +30,7 @@ Status legend:
|
||||
| ✓ | Item | Where it lives |
|
||||
|---|------|----------------|
|
||||
| 🟢 | 6 of 9 tools shipped (Dedup, Text, Format, Missing, Column-Map, Pipeline) | `src/core/`, `src/cli_*.py`, `src/gui/pages/` |
|
||||
| 🟢 | Pipeline Runner (the retention multiplier per `PLAN.md` §2.6) | `src/core/pipeline.py`, `src/cli_pipeline.py`, `src/gui/pages/9_Pipeline_Runner.py` |
|
||||
| 🟢 | Automated Workflows (the retention multiplier per `PLAN.md` §2.6) | `src/core/pipeline.py`, `src/cli_pipeline.py`, `src/gui/pages/9_Pipeline_Runner.py` |
|
||||
| 🟢 | 1,729 passing tests · 0 skipped · 0 xfailed | `tests/` |
|
||||
| 🟢 | 3 niche demo datasets + pre-tuned pipeline JSONs | `samples/demo/` |
|
||||
| 🟢 | Streamlit demo app + Cloud entry shim | `streamlit_app.py`, `src/gui/app_demo.py` |
|
||||
@@ -269,6 +269,7 @@ moves until $5k/mo MRR:
|
||||
| | Why locked |
|
||||
|---|---|
|
||||
| ❌ More tools (06–08) | `PLAN.md` §2.1 distribution-gate. Tool 09 was the exception; no others until first paid customer + one external review. |
|
||||
| ❌ Tool #10 PDF → CSV (the most-asked-for adjacency) | Parked in `docs/FUTURE-TOOLS.md` with full design + 3–4 wk MVP / 6–10 wk polished estimate. Ship trigger: paying customer + ≥3 paid or ≥5 demo emails asking for PDF + the bookkeeper niche converting first. None have fired yet. |
|
||||
| ❌ SaaS pivot | `DECISIONS.md` §4 — recurring infra conflicts with the lifestyle constraint. |
|
||||
| ❌ Live chat / sales calls | `DECISIONS.md` §1 #8 — no-touch is locked until $5k/mo. |
|
||||
| ❌ Custom integrations / one-off consulting | Breaks "build once, sell many." |
|
||||
|
||||
32
docs/PLAN.md
32
docs/PLAN.md
@@ -29,8 +29,8 @@ win.
|
||||
|
||||
| Asset | State |
|
||||
|---|---|
|
||||
| Tools 1–5 (Dedup, Text Clean, Format Standardize, Missing, Column Mapper) | Ready · 1,691 tests passing · 0 xfailed |
|
||||
| Tools 6–9 (Outlier, Multi-File Merge, Validator, Pipeline) | Coming Soon |
|
||||
| Tools 1–5 (Find Duplicates, Clean Text, Standardize Formats, Fix Missing Values, Map Columns) | Ready · 1,691 tests passing · 0 xfailed |
|
||||
| Tools 6–9 (Find Unusual Values, Combine Files, Quality Check, Automated Workflows) | Coming Soon |
|
||||
| PyInstaller installer pipeline | Not started |
|
||||
| macOS code signing (Apple Dev Program) | Not started |
|
||||
| Hosted browser demo (Streamlit Cloud) | Not deployed |
|
||||
@@ -52,12 +52,20 @@ Tools 6–8 are blocked behind a **distribution gate**: no work on them
|
||||
until the existing 5 tools have a paying customer + one external review
|
||||
(BUSINESS.md §4 sequence rule, applied recursively inside the bundle).
|
||||
|
||||
**Exception granted 2026-05-01**: Tool 09 Pipeline Runner is built
|
||||
**Exception granted 2026-05-01**: Tool 09 Automated Workflows is built
|
||||
*now*. Rationale: the pipeline transforms the bundle from "5 tools you
|
||||
buy" into "an automatable workflow you depend on." That conversion is
|
||||
what produces retention and word-of-mouth — the only marketing channel
|
||||
that scales under the no-network/no-touch constraint.
|
||||
|
||||
**Parked behind the freeze**: post-launch tool ideas are captured in
|
||||
`docs/FUTURE-TOOLS.md` with feasibility, GUI sketch, effort estimate,
|
||||
and ship criteria for each. Currently parked: **#10 PDF → CSV
|
||||
extractor** (bank statements et al.) — gated on a paying customer +
|
||||
≥3 paying customers or ≥5 demo emails explicitly asking for PDF
|
||||
extraction, with the bookkeeper niche converting at least one customer
|
||||
first. None of those triggers have fired yet.
|
||||
|
||||
### 2.2 The demo *is* the product. Make it embarrassingly good.
|
||||
|
||||
- Three persona-tagged sample datasets, not one generic CSV: Shopify
|
||||
@@ -104,10 +112,10 @@ demo dataset.
|
||||
| # | Pain | $ / time impact | Tools that fix it |
|
||||
|---|------|-----------------|---|
|
||||
| S1 | **Klaviyo / Mailchimp / Omnisend per-contact billing.** Subscriber list with 10–18 % duplicate rate (case drift, plus signs in Gmail addresses, multiple devices) → recurring overpay forever. | $30–300/mo per percent of dupes on a 50 k list — recurring | Dedup + Format Standardize (email canonicalization) + Pipeline (re-run weekly) |
|
||||
| S2 | **Product feed rejected by Google Merchant Center / Meta Catalog.** Smart quotes in titles, NBSP in SKU, inconsistent attributes; campaign launch delayed 24–72 h while feed gets fixed. | 1–3 days delayed launch × campaign value | Text Cleaner + Format Standardize |
|
||||
| S3 | **Multi-channel order consolidation.** Shopify + Etsy + Amazon + Faire + wholesale spreadsheet, each with a different column for "customer email" / "order total" / "ship country". | 4–8 hr / month manually merging | Column Mapper + Dedup + Pipeline |
|
||||
| S2 | **Product feed rejected by Google Merchant Center / Meta Catalog.** Smart quotes in titles, NBSP in SKU, inconsistent attributes; campaign launch delayed 24–72 h while feed gets fixed. | 1–3 days delayed launch × campaign value | Clean Text + Standardize Formats |
|
||||
| S3 | **Multi-channel order consolidation.** Shopify + Etsy + Amazon + Faire + wholesale spreadsheet, each with a different column for "customer email" / "order total" / "ship country". | 4–8 hr / month manually merging | Map Columns + Find Duplicates + Automated Workflows |
|
||||
| S4 | **Subscription identity fragmentation.** Pet-box subscribers cancel and re-sub under a different email; cohort analysis says churn is 20 % when it's actually 12 % — pricing decisions wrong. | Mis-priced LTV → over- or under-paid acquisition | Dedup with `merge=true` survivor |
|
||||
| S5 | **International tax / VAT MOSS compliance.** Country column is `UK` / `U.K.` / `United Kingdom` / `GB` in the same export; VAT report breaks. Phone formats per region break call-center routing. | Compliance penalty risk + ops friction | Format Standardize (per-row country) + Column Mapper |
|
||||
| S5 | **International tax / VAT MOSS compliance.** Country column is `UK` / `U.K.` / `United Kingdom` / `GB` in the same export; VAT report breaks. Phone formats per region break call-center routing. | Compliance penalty risk + ops friction | Standardize Formats (per-row country) + Map Columns |
|
||||
|
||||
#### Bookkeeper / freelance accountant
|
||||
|
||||
@@ -126,7 +134,7 @@ demo dataset.
|
||||
| R1 | **HubSpot / Marketo / Iterable per-contact tier pricing.** 10 k contacts → enterprise tier at $4–8 k/mo. Every duplicate is a recurring tax. | $200–800 / month per 1 k duplicate contacts — recurring | Dedup with cross-source merge + Pipeline |
|
||||
| R2 | **Email-deliverability / sender reputation.** Sending to invalid or duplicate addresses tanks reputation; recovery takes weeks. | Catastrophic — entire email programme degraded | Format Standardize (email canonicalization) + Missing (sentinel detection) |
|
||||
| R3 | **GDPR / contact-data privacy.** Uploading lead data to a third-party cleaning SaaS is itself a GDPR concern; legal review blocks adoption. | Compliance risk + 4–8 wk legal-review delay | Local-only desktop app, zero outbound calls |
|
||||
| R4 | **Multi-vendor lead-source unification.** Apollo, ZoomInfo, LinkedIn Sales Nav, manual scrapes — each export has different headers, scoring, country format. | 1–3 days per campaign of manual unification | Column Mapper (alias matching) + Format Standardize (per-row country) + Dedup |
|
||||
| R4 | **Multi-vendor lead-source unification.** Apollo, ZoomInfo, LinkedIn Sales Nav, manual scrapes — each export has different headers, scoring, country format. | 1–3 days per campaign of manual unification | Map Columns (alias matching) + Standardize Formats (per-row country) + Find Duplicates |
|
||||
| R5 | **Suppression-list management across 5+ platforms.** Each platform has its own format; un-deduped suppression lists let opt-outs slip through, triggering CAN-SPAM / GDPR exposure. | Compliance risk + churn-back cost | Pipeline saved as JSON, re-run on each new suppression batch |
|
||||
|
||||
### 2.4 Operationalize the moat the docs already name.
|
||||
@@ -154,7 +162,7 @@ right after "runs locally."
|
||||
Copy seed: *"Every change auditable. Hand the audit CSV to your client
|
||||
with the cleaned file."*
|
||||
|
||||
### 2.6 The Pipeline Runner is the retention multiplier.
|
||||
### 2.6 Automated Workflows is the retention multiplier.
|
||||
|
||||
A buyer with a saved pipeline isn't a one-off purchase — they're a
|
||||
recurring user who recommends the product. This is exactly the
|
||||
@@ -172,8 +180,8 @@ trigger DECISIONS.md §8 already names).
|
||||
### 2.8 Dependency-aware pipeline UX.
|
||||
|
||||
Tools have soft execution-order preferences (Text Clean before Format
|
||||
Standardize, Format before Dedup, Missing before Dedup). The Pipeline
|
||||
Runner *recommends* the order, *warns* on reversals, and **never
|
||||
Standardize, Format before Dedup, Missing before Dedup). Automated
|
||||
Workflows *recommends* the order, *warns* on reversals, and **never
|
||||
forces** — the user owns their workflow. Implementation: see
|
||||
`src/core/pipeline.py` `SOFT_DEPENDENCIES`.
|
||||
|
||||
@@ -184,7 +192,7 @@ forces** — the user owns their workflow. Implementation: see
|
||||
| 1 | PyInstaller pipeline · Mac/Win unsigned installers · Apple Dev Program enrollment (1–2 wk lead) | `dist/datatools-mac.dmg` and `dist/datatools-win.exe` install on a clean machine |
|
||||
| 2 | Demo deployed to Streamlit Cloud · landing page v1 with embedded demo · 3 persona datasets in the demo | Public URL serves a working pipeline run on a sample dataset in < 30 s |
|
||||
| 3 | Gumroad listing live · share value-first in 3 niche communities (no pitch) · 1 long-tail SEO post for the lead persona | First listing impression captured · post not removed for self-promotion |
|
||||
| 4 | Pipeline Runner v1.0 shipped (this week, 2026-05-01 — exception per §2.1) · v1.1 patch announced with Tool 09 + intl improvements | Pipeline saves/loads JSON · 3 demo pipelines preloaded |
|
||||
| 4 | Automated Workflows v1.0 shipped (this week, 2026-05-01 — exception per §2.1) · v1.1 patch announced with Tool 09 + intl improvements | Pipeline saves/loads JSON · 3 demo pipelines preloaded |
|
||||
| 5–8 | Bookkeeper landing page · agency landing page · second tool's promo cycle · priority-support tier added (defer purchase until §2.7 trigger) | Three live landing pages with distinct H1, demo dataset, conversion target |
|
||||
| 9–13 | Tool 06–08 only **if** revenue trajectory supports continued investment · otherwise more market work on the existing 5 + 09 | Decision made on 13 Aug 2026 with revenue data, not feature ambition |
|
||||
|
||||
@@ -202,7 +210,7 @@ These flip the plan, not the underlying criteria:
|
||||
|
||||
## 5. Anti-temptations (things the plan refuses)
|
||||
|
||||
- **More tools before more buyers.** Locked. Exception only for Pipeline Runner per §2.1.
|
||||
- **More tools before more buyers.** Locked. Exception only for Automated Workflows per §2.1.
|
||||
- **SaaS pivot.** Recurring infra conflicts with the lifestyle constraint (DECISIONS.md §4).
|
||||
- **Live chat / sales calls.** Conflicts with no-touch (DECISIONS.md §1 #8).
|
||||
- **Custom integrations / one-off consulting.** $300/hr looks tempting; breaks the "build once, sell many" model that justifies the entire strategy.
|
||||
|
||||
@@ -144,7 +144,7 @@ Reading PLAN.md §3 + this doc together, the rough script:
|
||||
| **M1** (June) | Installers · demo · 3 landing pages · Gumroad live | Whether the funnel mechanically works. Numbers will be noisy; just look for one purchase. |
|
||||
| **M2** (July) | M1 + community posts in 3 niches + 1 SEO post | Which persona converts. Re-allocate effort to the highest-converting niche. |
|
||||
| **M3** (August) | M2 + landing-page changes from M2 review | Whether intent-rate moved on the change. Decide tools 06–08 go/no-go. |
|
||||
| **M4** (September) | M3 + first repeat-buyer signals | Whether the Pipeline Runner is producing retention as designed. |
|
||||
| **M4** (September) | M3 + first repeat-buyer signals | Whether Automated Workflows is producing retention as designed. |
|
||||
|
||||
By end of M4, the data tells you whether the plan is producing
|
||||
$1k–3k/mo (BUSINESS.md §6 6-month target) — extrapolated from the
|
||||
|
||||
@@ -6,11 +6,13 @@
|
||||
|
||||
## Inicio rápido
|
||||
|
||||
1. Descarga el instalador para tu sistema operativo desde tu correo de compra.
|
||||
2. Ejecútalo (no se requieren conocimientos de Python).
|
||||
3. Lánzalo desde el acceso directo del escritorio → tu navegador predeterminado se abrirá en una página local.
|
||||
1. Descarga desde tu correo de compra. Dos formatos por sistema operativo — elige uno:
|
||||
- **Instalador** (`.dmg` en macOS, `.exe` en Windows) — crea acceso directo en el escritorio + entrada en el menú Inicio / Launchpad.
|
||||
- **.zip portable** — descomprime y haz doble clic. Sin instalación, sin admin, se ejecuta desde cualquier lugar.
|
||||
2. Ábrelo (no necesitas Python; todo viene incluido).
|
||||
3. La app arranca un servidor local y abre tu navegador. Nada sale de tu equipo.
|
||||
|
||||
Instrucciones completas: [USER-GUIDE.es.md](USER-GUIDE.es.md).
|
||||
Paso a paso completo incluyendo SmartScreen / Gatekeeper: [USER-GUIDE.es.md §1](USER-GUIDE.es.md#1-instalaci%C3%B3n).
|
||||
|
||||
## Documentación
|
||||
|
||||
|
||||
@@ -6,11 +6,13 @@
|
||||
|
||||
## Quick Start
|
||||
|
||||
1. Download the installer for your OS from your purchase email.
|
||||
2. Run it (no Python knowledge required).
|
||||
3. Launch via the desktop shortcut → your default browser opens to a local page.
|
||||
1. Download from your purchase email. Two flavors per OS — pick one:
|
||||
- **Installer** (`.dmg` on macOS, `.exe` on Windows) — wires up Desktop + Start Menu / Launchpad shortcuts.
|
||||
- **Portable .zip** — unzip and double-click. No install, no admin rights, runs from anywhere.
|
||||
2. Open it (no Python needed; everything is bundled inside).
|
||||
3. The app starts a local server and opens your browser. Nothing leaves your machine.
|
||||
|
||||
Full instructions: [USER-GUIDE.md](USER-GUIDE.md).
|
||||
Full step-by-step including SmartScreen / Gatekeeper workarounds: [USER-GUIDE.md §1](USER-GUIDE.md#1-install).
|
||||
|
||||
## Docs
|
||||
|
||||
|
||||
@@ -21,8 +21,8 @@ project-root/
|
||||
│ └── CLI-REFERENCE.md
|
||||
├── src/
|
||||
│ ├── core/ # shared logic — both CLI + GUI call into this
|
||||
│ ├── cli.py # Deduplicator CLI
|
||||
│ ├── cli_text_clean.py # Text Cleaner CLI
|
||||
│ ├── cli.py # Find Duplicates CLI
|
||||
│ ├── cli_text_clean.py # Clean Text CLI
|
||||
│ ├── cli_analyze.py # Analyzer CLI
|
||||
│ └── gui/
|
||||
│ ├── app.py # Streamlit entry
|
||||
|
||||
@@ -76,7 +76,7 @@ Sample size: 1,000 rows (configurable).
|
||||
- Full-DataFrame `auto_fix`: ~5 min (~30 µs/cell).
|
||||
- Output write: ~10 s.
|
||||
- Recommended RAM: 3–4× input size for the full-Apply path.
|
||||
- **Format standardizer** (`standardize_dataframe`): ~2.7M rows/sec on
|
||||
- **Standardize Formats** (`standardize_dataframe`): ~2.7M rows/sec on
|
||||
cache-warm repetition-heavy columns (synthetic 1M-row in-memory
|
||||
benchmark, 2 typed columns); the fused single-pass loop replaced a
|
||||
3-pass ``.tolist()`` cycle, so per-call overhead is now dominated by
|
||||
@@ -87,20 +87,20 @@ Sample size: 1,000 rows (configurable).
|
||||
thread-pool scaffolding; on CPython 3.12 with the GIL it's
|
||||
roughly neutral, but the API is ready for the free-threaded
|
||||
(PEP 703) Python 3.13+ build where it will help.
|
||||
- **Text cleaner** (`clean_dataframe`): ~1M rows/sec on
|
||||
- **Clean Text** (`clean_dataframe`): ~1M rows/sec on
|
||||
repetition-heavy columns (per-call string cache: the pipeline runs
|
||||
once per *unique* cell value, not once per row).
|
||||
- **Missing handler** (`handle_missing`): lazy-copy — when sentinel
|
||||
- **Fix Missing Values** (`handle_missing`): lazy-copy — when sentinel
|
||||
standardization runs but finds nothing, AND no drops AND no fills
|
||||
apply, the input frame is returned as-is. On a clean 1 GB file this
|
||||
saves the 1 GB allocation that the unconditional upfront copy used
|
||||
to take.
|
||||
- **Column mapper** (`map_columns`): rename + drop both already
|
||||
- **Map Columns** (`map_columns`): rename + drop both already
|
||||
return fresh frames; the explicit upfront `df.copy()` is now
|
||||
removed and downstream mutating steps (schema-add, coerce) copy on
|
||||
demand via `_ensure_owned()`. Rename-only and identity-mapping
|
||||
paths run with zero explicit copies.
|
||||
- **Deduplicator**:
|
||||
- **Find Duplicates**:
|
||||
- **Exact-only strategies** (every column uses `Algorithm.EXACT` at
|
||||
threshold 100 — covers strong-key dedup like email/phone, the
|
||||
fallback drop-duplicates path, and explicit "match on this exact
|
||||
@@ -117,19 +117,30 @@ Sample size: 1,000 rows (configurable).
|
||||
(the common dedup workload) skip re-parsing.
|
||||
|
||||
## 11. Tools
|
||||
1. Deduplicator — Ready
|
||||
2. Text Cleaner — Ready
|
||||
3. Format Standardizer — Ready
|
||||
4. Missing Value Handler — Ready
|
||||
5. Column Mapper — Ready
|
||||
6. Outlier Detector — Coming Soon
|
||||
7. Multi-File Merger — Coming Soon
|
||||
8. Validator & Reporter — Coming Soon
|
||||
9. Pipeline Runner — Ready
|
||||
1. Find Duplicates — Ready
|
||||
2. Clean Text — Ready
|
||||
3. Standardize Formats — Ready
|
||||
4. Fix Missing Values — Ready
|
||||
5. Map Columns — Ready
|
||||
6. Find Unusual Values — Coming Soon
|
||||
7. Combine Files — Coming Soon
|
||||
8. Quality Check — Coming Soon
|
||||
9. Automated Workflows — Ready
|
||||
|
||||
**Future / not in v1.** Tool ideas captured for after-launch consideration
|
||||
live in `docs/FUTURE-TOOLS.md` — entries there are gated by the new-tool
|
||||
freeze in `PLAN.md` §2.1 and don't ship without a paying-customer +
|
||||
repeated-demand signal. Currently parked there:
|
||||
|
||||
- **#10. PDF → CSV extractor** (bank statements + similar). No PDF
|
||||
dependency exists in the repo today; this tool would need pdfplumber,
|
||||
streamlit-drawable-canvas, and a templates store. Estimated 3–4 weeks
|
||||
for a text-only MVP, 6–10 weeks for the polished version with
|
||||
multi-page template recall.
|
||||
|
||||
### 11.a Recommended pipeline order (soft, not enforced)
|
||||
|
||||
The Pipeline Runner ships with a `SOFT_DEPENDENCIES` table; the
|
||||
Automated Workflows ships with a `SOFT_DEPENDENCIES` table; the
|
||||
following ordering is the default and the basis of the warning
|
||||
surface. Re-ordering is allowed; the runner emits a warning string
|
||||
and proceeds.
|
||||
@@ -174,11 +185,15 @@ and proceeds.
|
||||
- **Dev**: pytest, tox.
|
||||
|
||||
## 16. Test coverage
|
||||
- 1,916 tests passing, 0 skipped, 0 xfailed.
|
||||
- 1,777 core + CLI tests (run with `pytest -m 'not gui'` for a quick loop).
|
||||
- 139 GUI tests under `tests/gui/` driving Streamlit pages via `AppTest`
|
||||
- 2,033 tests passing, 0 skipped, 0 xfailed.
|
||||
- 1,868 core + CLI tests (run with `pytest -m 'not gui'` for a quick loop).
|
||||
Includes 49 license-layer unit tests (Ed25519 sign/verify, dev-key
|
||||
derivation, production-safe tripwire, schema), 25 license-CLI
|
||||
tests, and 17 Lite-tier feature-map + guard tests.
|
||||
- 165 GUI tests under `tests/gui/` driving Streamlit pages via `AppTest`
|
||||
(smoke + EN/ES localization, chrome, gate, workflows, dedup review,
|
||||
advanced panels, error paths, findings panel). Marked `gui`.
|
||||
advanced panels, error paths, findings panel, activation +
|
||||
license gate, Lite-tier per-page lock behaviour). Marked `gui`.
|
||||
- Includes 15 perf-shape regression tests.
|
||||
- Fixture corpora: text-cleaner (21), encodings (31), reference UTF-8 (9), format-cleaner (199 buyer cases + 20-row international stress fixture), missing-handler (3 use cases + 16 edge cases), column-mapper (3 use cases + 5 edge cases).
|
||||
- Run: `python run_tests.py [--tool …] [--fixtures] [--coverage]`.
|
||||
@@ -189,6 +204,58 @@ and proceeds.
|
||||
- Original input never modified.
|
||||
- Audit logs: `logs/` next to each run (timestamped).
|
||||
|
||||
## 17a. Licensing
|
||||
- **Storage**: ``~/.datatools/license.json`` (or
|
||||
``$DATATOOLS_LICENSE_PATH`` override). Signed with Ed25519
|
||||
(asymmetric).
|
||||
- **Crypto**: Ed25519. The seller holds the private key; every
|
||||
shipped binary embeds only the public key. A motivated reverse
|
||||
engineer who pulls everything out of the binary still can't sign
|
||||
new licenses. Keys are 32 bytes raw, exposed as hex via
|
||||
``DATATOOLS_LICENSE_PRIVKEY`` (seller-side) and
|
||||
``DATATOOLS_LICENSE_PUBKEY`` (build-time bake-in).
|
||||
- **Activation**: buyer pastes a base64-encoded license blob
|
||||
(``DTLIC1:...``) on first launch; app verifies the signature
|
||||
offline + matches the buyer-entered name/email to the embedded
|
||||
values.
|
||||
- **No free trial**: every license requires a paid blob from the
|
||||
seller. The user-facing trial flow (button + ``license_cli trial``
|
||||
subcommand) was removed in v1.6 to keep paid-tier economics clean.
|
||||
- **Lifetime**: every license is 1 year by default. Renewal applies a
|
||||
fresh blob without losing the embedded buyer identity. Tier may
|
||||
change during renewal (Lite → Core upgrade path).
|
||||
- **Tiers**:
|
||||
- ``lite`` — Find Duplicates + Clean Text + Standardize Formats.
|
||||
Buyer pays once, gets the three universally-useful tools.
|
||||
- ``core`` — every Ready tool (all 9 in v1.6).
|
||||
- ``pro``, ``enterprise`` — scaffolded for future SKUs; currently
|
||||
mirror Core. Add per-SKU restrictions by editing
|
||||
``FEATURES_BY_TIER`` in ``src/license/features.py``.
|
||||
- ``trial`` — kept in the enum for backwards compat with any
|
||||
field-tested trial licenses but no longer issuable.
|
||||
- **Feature flags**: every tool has a stable feature id matching its
|
||||
``tool_id`` in :mod:`src.gui.tools_registry`. Adding a future per-
|
||||
tool SKU is a one-line change to ``FEATURES_BY_TIER`` — no consumer
|
||||
code edits.
|
||||
- **Per-tool gating**: each tool page (GUI) and tool CLI calls
|
||||
``require_feature(FeatureFlag.<TOOL>)`` at entry. GUI shows an
|
||||
upgrade prompt + button to the Activate page; CLI prints a
|
||||
message naming the locked feature and exits with code 2.
|
||||
- **Lock badge**: the home grid shows a red 🔒 Locked pill on tool
|
||||
cards the current tier doesn't unlock.
|
||||
- **Dev bypass**: ``DATATOOLS_DEV_MODE=1`` skips every check (used by
|
||||
the test suite and during development). **Refused in shipped
|
||||
builds** by the production-safe tripwire.
|
||||
- **Production-safe tripwire**: ``assert_production_safe()`` runs at
|
||||
startup in every frozen build. Refuses to boot when ``DEV_MODE``
|
||||
is set or the verification key is still the embedded dev key
|
||||
(i.e., the build pipeline forgot to override
|
||||
``DATATOOLS_LICENSE_PUBKEY``). No-op in source / pytest runs.
|
||||
- **No internet**: signature verification is fully offline. The
|
||||
shipped binary embeds only the public key; the private key never
|
||||
leaves the seller. See ``docs/DECISIONS.md`` for the threat-model
|
||||
discussion.
|
||||
|
||||
## 18. Error handling
|
||||
- Structured hierarchy: `DataToolsError` → `InputValidationError`, `ConfigError`, `FileFormatError`, `FileAccessError`.
|
||||
- Subclasses extend stdlib `ValueError` / `OSError` so existing handlers still catch them.
|
||||
|
||||
593
docs/SETUP-LICENSE-SERVER.md
Normal file
593
docs/SETUP-LICENSE-SERVER.md
Normal file
@@ -0,0 +1,593 @@
|
||||
# SETUP — Self-hosted license server runbook
|
||||
|
||||
End-to-end build instructions for `licenses.datatools.unalogix.com` on
|
||||
the existing invixiom box (Ubuntu 24.04, public IP `46.225.166.142`).
|
||||
|
||||
Audience: creator/operator. Read top to bottom on first install; use as
|
||||
a reference thereafter.
|
||||
|
||||
Companions:
|
||||
- `LICENSE-SERVER.md` — the architecture / design rationale
|
||||
- `ADMIN.md` — day-2 ops (minting comps, looking at the issuance log)
|
||||
|
||||
---
|
||||
|
||||
## 0. Multi-tenancy: where this lands among existing services
|
||||
|
||||
This box already hosts the `*.invixiom.com` family (kasm, files, lifeos,
|
||||
code, gitea) via one shared nginx + one shared Let's Encrypt cert.
|
||||
DataTools is intentionally separated from that stack at every layer:
|
||||
|
||||
| Layer | Existing | New |
|
||||
|---|---|---|
|
||||
| **DNS zone** | `invixiom.com` | `unalogix.com` (different TLD) |
|
||||
| **nginx file** | `/etc/nginx/sites-available/invixiom` | `/etc/nginx/sites-available/unalogix` |
|
||||
| **nginx symlink** | `sites-enabled/invixiom` | `sites-enabled/unalogix` |
|
||||
| **TLS cert** | `letsencrypt/live/kasm.invixiom.com[-0001]` | `letsencrypt/live/datatools.unalogix.com` |
|
||||
| **Backend port** | 8000, 8002, 8003, 8080, 8081, 8443 | **8090** (mint API), **5433** (Postgres, localhost-only) |
|
||||
| **Docker compose project** | per-service (kasm, lifeos, gitea) | `datatools-license` |
|
||||
| **Docker volume** | per service | `datatools_pg_data` |
|
||||
| **Filesystem root** | various | `/srv/datatools-license/` |
|
||||
| **System user** | various | `datatools-api` (UID auto-assigned, no shell) |
|
||||
|
||||
Nothing in the invixiom stack is read, modified, or referenced by the
|
||||
datatools stack. Restart, upgrade, or remove either without affecting
|
||||
the other.
|
||||
|
||||
---
|
||||
|
||||
## 1. Pre-flight checklist (off-box, before any commands run)
|
||||
|
||||
These have to be done by the operator outside this box. The build
|
||||
won't proceed without them.
|
||||
|
||||
### 1a. DNS records
|
||||
|
||||
In your `unalogix.com` registrar / DNS panel, add:
|
||||
|
||||
```
|
||||
A datatools.unalogix.com 46.225.166.142
|
||||
A licenses.datatools.unalogix.com 46.225.166.142
|
||||
```
|
||||
|
||||
Verify before continuing:
|
||||
|
||||
```bash
|
||||
dig +short datatools.unalogix.com
|
||||
dig +short licenses.datatools.unalogix.com
|
||||
# Both should print: 46.225.166.142
|
||||
```
|
||||
|
||||
DNS propagation can take 1–60 minutes. Let's Encrypt won't issue
|
||||
certs until DNS resolves correctly.
|
||||
|
||||
### 1b. Postmark account (transactional email)
|
||||
|
||||
1. Sign up at https://postmarkapp.com (free 100 emails/mo, $15/mo for
|
||||
the volume range we'll be in).
|
||||
2. Verify the `unalogix.com` domain (DNS TXT/CNAME records — Postmark
|
||||
will tell you exactly what to add).
|
||||
3. Create a Server, copy the **Server API Token**. Stash it; we'll put
|
||||
it in the app's `.env`.
|
||||
4. Configure the sender address: `licenses@datatools.unalogix.com`.
|
||||
|
||||
If you prefer SES, Mailgun, Resend, etc. — fine, just swap the
|
||||
adapter (see §6). Postmark is the recommended default.
|
||||
|
||||
### 1c. Cloudflare in front (recommended)
|
||||
|
||||
Move `unalogix.com` DNS hosting to Cloudflare and enable proxy ("orange
|
||||
cloud") on both subdomains. Gets you free DDoS protection, WAF, and rate
|
||||
limiting. **Origin TLS still goes through Let's Encrypt on this box**;
|
||||
Cloudflare adds a second TLS hop in front. Cert renewal still works
|
||||
because we use HTTP-01 challenge on the origin, which Cloudflare
|
||||
proxies transparently.
|
||||
|
||||
If you skip this, the public webhook endpoint is directly hammerable.
|
||||
Not catastrophic at low scale, but the free protection is worth taking.
|
||||
|
||||
### 1d. Gumroad webhook secret
|
||||
|
||||
In Gumroad's seller dashboard → Settings → Advanced → "Ping URL":
|
||||
|
||||
```
|
||||
URL: https://licenses.datatools.unalogix.com/webhooks/gumroad
|
||||
Secret: <generate a random 32-char hex; save it for the .env>
|
||||
```
|
||||
|
||||
Don't enter this until §10 ("PR 2 cutover") — the endpoint won't exist
|
||||
yet during the Mint API build.
|
||||
|
||||
---
|
||||
|
||||
## 2. One-time host setup
|
||||
|
||||
Run as `root` (or via `sudo`).
|
||||
|
||||
```bash
|
||||
# Update apt cache and pull in the bits the rest of the doc needs.
|
||||
apt-get update
|
||||
apt-get install -y \
|
||||
docker-compose-plugin \
|
||||
certbot \
|
||||
python3-certbot-nginx \
|
||||
postgresql-client-16 # for psql to reach the containerized DB
|
||||
|
||||
# Sanity check: docker + compose v2 are already installed via Docker CE.
|
||||
docker --version
|
||||
docker compose version
|
||||
|
||||
# Create the system user the app process will run as (no shell, no home).
|
||||
adduser --system --group --no-create-home --shell /usr/sbin/nologin datatools-api
|
||||
|
||||
# Filesystem layout under /srv (separate from /opt to make the
|
||||
# multi-tenant boundary obvious on disk).
|
||||
install -d -o datatools-api -g datatools-api -m 750 /srv/datatools-license
|
||||
install -d -o datatools-api -g datatools-api -m 750 /srv/datatools-license/app
|
||||
install -d -o datatools-api -g datatools-api -m 750 /srv/datatools-license/secrets
|
||||
install -d -o datatools-api -g datatools-api -m 750 /srv/datatools-license/backups
|
||||
```
|
||||
|
||||
The `secrets/` dir is mode 750 owned by `datatools-api`. The private
|
||||
signing key and Postmark token live there as mode-400 files — never
|
||||
in environment-variable-via-systemd-EnvironmentFile, never in the
|
||||
docker-compose file, never anywhere `root` doesn't need to look.
|
||||
|
||||
> **Gotcha — secret file ownership UID.** Docker compose's
|
||||
> `uid:`/`gid:`/`mode:` long-form on `secrets:` is silently ignored
|
||||
> for **file-based** secrets (it's a swarm-mode-only feature). The
|
||||
> file inside the container appears with whatever ownership it has
|
||||
> on the host, and the API runs as UID 10001 (the `app` user from
|
||||
> the Dockerfile). So chown the actual files to **10001** (a numeric
|
||||
> UID that doesn't exist on the host — that's fine, chown accepts
|
||||
> it) and rely on the parent dir's mode 750 + ownership for host-side
|
||||
> access control. See §3 below for the corrected `chown` step.
|
||||
|
||||
### Firewall recommendation (separate decision)
|
||||
|
||||
The box currently runs without UFW. Enabling it now would affect all
|
||||
existing services. Two options:
|
||||
|
||||
- **(A) Don't enable UFW.** Leave the cloud provider's network firewall
|
||||
as the perimeter. This is the current state.
|
||||
- **(B) Enable UFW with `allow 22, 80, 443` only.** Forces every Docker
|
||||
service to bind to `127.0.0.1` (some currently bind `0.0.0.0`). Will
|
||||
break any direct-port access until those binds are updated.
|
||||
|
||||
Default for this runbook: **(A)**. Revisit independently of the
|
||||
DataTools rollout. The DataTools containers always bind to `127.0.0.1`
|
||||
regardless.
|
||||
|
||||
---
|
||||
|
||||
## 3. Database (Postgres in Docker)
|
||||
|
||||
Postgres lives inside the datatools compose project — separate from
|
||||
every other service on the box, separate volume, separate port,
|
||||
localhost-only binding.
|
||||
|
||||
`/srv/datatools-license/compose.yml`:
|
||||
|
||||
```yaml
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
container_name: datatools-postgres
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
POSTGRES_DB: datatools_licenses
|
||||
POSTGRES_USER: datatools_api
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/pg_password
|
||||
secrets:
|
||||
- pg_password
|
||||
volumes:
|
||||
- datatools_pg_data:/var/lib/postgresql/data
|
||||
ports:
|
||||
- "127.0.0.1:5433:5432" # localhost-only, non-default port
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U datatools_api -d datatools_licenses"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
|
||||
api:
|
||||
build:
|
||||
context: ./app
|
||||
dockerfile: server/Dockerfile
|
||||
image: datatools-license-api:latest
|
||||
container_name: datatools-api
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
DATABASE_URL: postgresql+psycopg://datatools_api@postgres:5432/datatools_licenses
|
||||
PG_PASSWORD_FILE: /run/secrets/pg_password
|
||||
DATATOOLS_ADMIN_TOKEN_FILE: /run/secrets/admin_token
|
||||
# PR 2 — uncomment when Postmark + Gumroad are provisioned.
|
||||
# POSTMARK_TOKEN_FILE: /run/secrets/postmark_token
|
||||
# GUMROAD_WEBHOOK_SECRET_FILE: /run/secrets/gumroad_secret
|
||||
# Production keypair (replaces in-tree dev key): set
|
||||
# DATATOOLS_LICENSE_PRIVKEY_FILE: /run/secrets/license_privkey
|
||||
# and DATATOOLS_LICENSE_PUBKEY: <hex> before shipping v1.0.
|
||||
secrets:
|
||||
- pg_password
|
||||
- admin_token
|
||||
# PR 2:
|
||||
# - postmark_token
|
||||
# - gumroad_secret
|
||||
ports:
|
||||
- "127.0.0.1:8090:8000" # localhost-only; nginx is the only path in
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
|
||||
interval: 30s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
|
||||
secrets:
|
||||
pg_password: { file: ./secrets/pg_password }
|
||||
admin_token: { file: ./secrets/admin_token }
|
||||
# PR 2:
|
||||
# postmark_token: { file: ./secrets/postmark_token }
|
||||
# gumroad_secret: { file: ./secrets/gumroad_secret }
|
||||
# Production keypair rotation adds:
|
||||
# license_privkey: { file: ./secrets/license_privkey }
|
||||
|
||||
volumes:
|
||||
datatools_pg_data:
|
||||
name: datatools_pg_data
|
||||
```
|
||||
|
||||
Populate the secrets (each file should contain the value with no
|
||||
trailing newline). For PR 1, only `pg_password` and `admin_token`
|
||||
are required; the rest land in PR 2 / production key rotation.
|
||||
|
||||
```bash
|
||||
cd /srv/datatools-license
|
||||
|
||||
# Random 32-char hex DB password
|
||||
openssl rand -hex 32 > secrets/pg_password
|
||||
|
||||
# Random admin Bearer token (CLI auth). Save this — you'll need it
|
||||
# on your laptop to talk to /internal/* via the SSH tunnel.
|
||||
openssl rand -hex 32 > secrets/admin_token
|
||||
|
||||
# --- PR 2 secrets ---
|
||||
# echo -n "<postmark-server-token>" > secrets/postmark_token # from postmarkapp.com
|
||||
# openssl rand -hex 32 > secrets/gumroad_secret # paste into Gumroad's Ping URL: ?secret=<this>
|
||||
#
|
||||
# --- production-key follow-up (defer until v1.0 cutover) ---
|
||||
# echo -n "<ed25519-private-hex>" > secrets/license_privkey
|
||||
|
||||
# Lock everything down. The numeric 10001 matches the in-container
|
||||
# `app` user (Dockerfile-defined), letting the API read the file
|
||||
# while keeping host-side access gated by the parent dir's mode 750.
|
||||
chmod 400 secrets/*
|
||||
chown 10001:10001 secrets/*
|
||||
```
|
||||
|
||||
The corresponding **public** key for `DATATOOLS_LICENSE_PUBKEY` goes
|
||||
in `/srv/datatools-license/.env` (it's not secret — it's already in
|
||||
every shipped binary):
|
||||
|
||||
```bash
|
||||
echo "DATATOOLS_LICENSE_PUBKEY=<hex-pubkey>" > /srv/datatools-license/.env
|
||||
chmod 640 /srv/datatools-license/.env
|
||||
chown datatools-api:datatools-api /srv/datatools-license/.env
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4. App image build
|
||||
|
||||
The Mint API source lives in this repo under `server/` (new directory
|
||||
introduced by PR 1). Build the Docker image:
|
||||
|
||||
```bash
|
||||
cd /srv/datatools-license/app
|
||||
git clone https://git.invixiom.com/giteadmin/datatools-dev.git .
|
||||
docker build -t datatools-license-api:latest -f server/Dockerfile server/
|
||||
```
|
||||
|
||||
Schema bootstrap (one-time, after first `docker compose up`):
|
||||
|
||||
```bash
|
||||
docker compose exec api alembic upgrade head
|
||||
```
|
||||
|
||||
Smoke test:
|
||||
|
||||
```bash
|
||||
curl -s http://127.0.0.1:8090/health
|
||||
# expects: {"status":"ok","db":"ok"}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 5. nginx config
|
||||
|
||||
> **Gotcha — nginx version syntax.** Ubuntu 24.04 ships nginx 1.24,
|
||||
> which uses the legacy `listen 443 ssl http2;` form. The standalone
|
||||
> `http2 on;` directive arrived in nginx 1.25 and will error on 1.24
|
||||
> with `unknown directive "http2"`. The config below uses the 1.24
|
||||
> form.
|
||||
>
|
||||
> **Bring-up sequence.** This config references a TLS cert at
|
||||
> `/etc/letsencrypt/live/datatools.unalogix.com/`, which doesn't
|
||||
> exist on a fresh install — nginx would refuse to start. The
|
||||
> working sequence is: (a) install a temporary HTTP-only config
|
||||
> that serves `.well-known/acme-challenge/` and returns 503 for
|
||||
> everything else, (b) `nginx -s reload`, (c) run `certbot
|
||||
> certonly --webroot`, (d) replace with the HTTPS config below,
|
||||
> (e) `nginx -s reload` again. See §6.
|
||||
|
||||
`/etc/nginx/sites-available/unalogix` — **new file**, do not merge
|
||||
into `invixiom`:
|
||||
|
||||
```nginx
|
||||
# Marketing / product site (datatools.unalogix.com) — static for now.
|
||||
server {
|
||||
listen 80;
|
||||
server_name datatools.unalogix.com licenses.datatools.unalogix.com;
|
||||
return 301 https://$host$request_uri;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 443 ssl http2; # nginx 1.24 syntax (Ubuntu 24.04)
|
||||
server_name datatools.unalogix.com;
|
||||
|
||||
ssl_certificate /etc/letsencrypt/live/datatools.unalogix.com/fullchain.pem;
|
||||
ssl_certificate_key /etc/letsencrypt/live/datatools.unalogix.com/privkey.pem;
|
||||
|
||||
root /srv/datatools-license/site; # static landing page; create later
|
||||
index index.html;
|
||||
|
||||
location / {
|
||||
try_files $uri $uri/ =404;
|
||||
}
|
||||
}
|
||||
|
||||
# License operations subdomain.
|
||||
server {
|
||||
listen 443 ssl http2; # nginx 1.24 syntax (Ubuntu 24.04)
|
||||
server_name licenses.datatools.unalogix.com;
|
||||
|
||||
ssl_certificate /etc/letsencrypt/live/datatools.unalogix.com/fullchain.pem;
|
||||
ssl_certificate_key /etc/letsencrypt/live/datatools.unalogix.com/privkey.pem;
|
||||
|
||||
# Block /internal/* from the public side as defense-in-depth.
|
||||
# (The app also enforces this server-side; this is layered.)
|
||||
location /internal/ {
|
||||
return 404;
|
||||
}
|
||||
|
||||
location / {
|
||||
proxy_pass http://127.0.0.1:8090;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
# Gumroad webhook payloads are tiny but tighten anyway.
|
||||
client_max_body_size 1m;
|
||||
|
||||
# Basic rate limiting: 30 req/min/IP on /webhooks/* and /portal/*.
|
||||
# Tune in nginx.conf with a `limit_req_zone` directive.
|
||||
# limit_req zone=licenses burst=10 nodelay;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Enable + reload:
|
||||
|
||||
```bash
|
||||
ln -s /etc/nginx/sites-available/unalogix /etc/nginx/sites-enabled/unalogix
|
||||
nginx -t # validate
|
||||
systemctl reload nginx
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 6. TLS cert
|
||||
|
||||
Use the standalone http-01 challenge (nginx-plugin works too; this is
|
||||
slightly more explicit):
|
||||
|
||||
```bash
|
||||
certbot certonly \
|
||||
--webroot -w /var/www/html \
|
||||
-d datatools.unalogix.com \
|
||||
-d licenses.datatools.unalogix.com \
|
||||
--agree-tos \
|
||||
--email michael.dombaugh@gmail.com \
|
||||
--non-interactive
|
||||
```
|
||||
|
||||
Cert lands at `/etc/letsencrypt/live/datatools.unalogix.com/`.
|
||||
Auto-renewal is already configured by the certbot package (systemd
|
||||
timer `certbot.timer`). Confirm:
|
||||
|
||||
```bash
|
||||
systemctl list-timers certbot.timer
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 7. Bring it up
|
||||
|
||||
```bash
|
||||
cd /srv/datatools-license
|
||||
docker compose up -d
|
||||
docker compose ps # both services should be 'running (healthy)'
|
||||
docker compose logs -f api
|
||||
```
|
||||
|
||||
Public smoke test:
|
||||
|
||||
```bash
|
||||
curl -s https://licenses.datatools.unalogix.com/health
|
||||
# expects: {"status":"ok","db":"ok"}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 8. Verification — end-to-end internal mint
|
||||
|
||||
From your laptop (NOT the server), open an SSH tunnel for the internal
|
||||
endpoint:
|
||||
|
||||
```bash
|
||||
ssh -L 8090:127.0.0.1:8090 michael@46.225.166.142 -N
|
||||
# Leave running; in another terminal:
|
||||
|
||||
curl -X POST http://127.0.0.1:8090/internal/mint \
|
||||
-H "Authorization: Bearer $DATATOOLS_ADMIN_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"name":"Test Buyer",
|
||||
"email":"test@example.com",
|
||||
"tier":"core",
|
||||
"years":1,
|
||||
"source":"manual",
|
||||
"notes":"smoke test"
|
||||
}'
|
||||
```
|
||||
|
||||
Expected: 200 + a `DTLIC2:...` blob + a row inserted in the `licenses`
|
||||
table. Confirm with:
|
||||
|
||||
```bash
|
||||
docker compose exec postgres \
|
||||
psql -U datatools_api -d datatools_licenses \
|
||||
-c "SELECT license_key, email, tier, source FROM licenses;"
|
||||
```
|
||||
|
||||
Then **revoke the test row** before going further:
|
||||
|
||||
```bash
|
||||
docker compose exec postgres \
|
||||
psql -U datatools_api -d datatools_licenses \
|
||||
-c "DELETE FROM licenses WHERE email = 'test@example.com';"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 9. Operational concerns
|
||||
|
||||
### Backups (Postgres → off-site)
|
||||
|
||||
`/etc/cron.daily/datatools-license-backup`:
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
TS=$(date -u +%Y%m%dT%H%M%SZ)
|
||||
OUT=/srv/datatools-license/backups/db-${TS}.sql.gz
|
||||
docker compose -f /srv/datatools-license/compose.yml exec -T postgres \
|
||||
pg_dump -U datatools_api datatools_licenses | gzip > "$OUT"
|
||||
chmod 600 "$OUT"
|
||||
# Off-site copy — pick one:
|
||||
# rclone copy "$OUT" remote:datatools-license-backups/
|
||||
# aws s3 cp "$OUT" s3://datatools-backups/db/ --sse AES256
|
||||
find /srv/datatools-license/backups -name 'db-*.sql.gz' -mtime +30 -delete
|
||||
```
|
||||
|
||||
Pick an off-site target. Without one, a disk failure loses every
|
||||
customer record. Test the restore at least once on a staging copy.
|
||||
|
||||
### Monitoring
|
||||
|
||||
External uptime probe (free):
|
||||
1. UptimeRobot account → add monitor for `https://licenses.datatools.unalogix.com/health`.
|
||||
2. 5-minute interval, alert to email/SMS.
|
||||
|
||||
Container health is already handled by `restart: unless-stopped` +
|
||||
healthcheck. To see recent failures:
|
||||
|
||||
```bash
|
||||
docker compose ps # last health-check status
|
||||
docker compose logs api --tail 200
|
||||
journalctl -u docker --since '1 hour ago' | grep datatools
|
||||
```
|
||||
|
||||
### Log rotation
|
||||
|
||||
Docker handles container logs; cap their size in
|
||||
`/etc/docker/daemon.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"log-driver": "json-file",
|
||||
"log-opts": {
|
||||
"max-size": "10m",
|
||||
"max-file": "3"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Then `systemctl restart docker` (this restarts all containers — schedule
|
||||
during a quiet window).
|
||||
|
||||
### Key rotation (future)
|
||||
|
||||
If the private signing key is ever compromised:
|
||||
|
||||
1. Generate a new keypair (`scripts/generate_keypair.py`).
|
||||
2. Build and ship a desktop release with the new pubkey embedded.
|
||||
3. Update `/srv/datatools-license/secrets/license_privkey` and
|
||||
`/srv/datatools-license/.env`'s pubkey.
|
||||
4. `docker compose restart api`.
|
||||
5. Re-issue every active license (script that queries the DB, calls
|
||||
`/internal/mint`, emails buyers). Old blobs will fail verification
|
||||
in the new desktop build.
|
||||
|
||||
Plan a 90-day overlap window where the desktop verifies against
|
||||
*both* keys before retiring the old pubkey. (Verification logic
|
||||
change to the desktop app — not in scope for PR 1.)
|
||||
|
||||
---
|
||||
|
||||
## 10. PR cutover sequence
|
||||
|
||||
This runbook covers the box-level scaffolding. Application code lands
|
||||
in three independently shippable PRs:
|
||||
|
||||
| PR | Adds | Ship gate | Webhook live? |
|
||||
|---|---|---|---|
|
||||
| **1** | Source-agnostic Mint API + Postgres + `datatools-admin mint` CLI | Operator can mint a comp license through the server | No |
|
||||
| **2** | Gumroad adapter + webhook receiver + email send | Real Gumroad sale auto-mints + emails buyer | **Yes** (enable in Gumroad dashboard at this PR's deploy) |
|
||||
| **3** | Renewal / re-delivery portal | Buyer self-services renewals and lost-blob re-delivery | (unchanged) |
|
||||
|
||||
§1d (Gumroad webhook URL) is **filled in during PR 2's deploy**, not
|
||||
before. Until then the endpoint returns 404.
|
||||
|
||||
---
|
||||
|
||||
## 11. Rollback
|
||||
|
||||
Each component is independently reversible.
|
||||
|
||||
```bash
|
||||
# Stop and remove containers (DB volume persists)
|
||||
docker compose -f /srv/datatools-license/compose.yml down
|
||||
|
||||
# Full teardown including DB (DESTRUCTIVE — backup first)
|
||||
docker compose -f /srv/datatools-license/compose.yml down -v
|
||||
|
||||
# Remove nginx site
|
||||
rm /etc/nginx/sites-enabled/unalogix
|
||||
nginx -t && systemctl reload nginx
|
||||
|
||||
# Revoke + delete TLS cert
|
||||
certbot delete --cert-name datatools.unalogix.com
|
||||
|
||||
# Remove filesystem
|
||||
rm -rf /srv/datatools-license # NOTE: includes secrets dir; backup first
|
||||
|
||||
# Remove system user
|
||||
deluser datatools-api
|
||||
delgroup datatools-api
|
||||
```
|
||||
|
||||
DNS records can stay or be removed — they're not on this host.
|
||||
@@ -3,6 +3,9 @@
|
||||
> Creator-only. Do not ship to buyers.
|
||||
> **Version**: 1.6 · **Updated**: 2026-05-01
|
||||
|
||||
For the end-to-end picture (desktop app + license server + storefronts
|
||||
+ email), see `ARCHITECTURE.md`. This doc focuses on desktop internals.
|
||||
|
||||
## 1. Architecture
|
||||
|
||||
- **Dual interface**: CLI + GUI, both wrapping the same `src/core/` library.
|
||||
@@ -31,8 +34,8 @@ src/
|
||||
normalizers.py # Per-column normalizers for dedup matching
|
||||
text_clean.py # clean_dataframe + smart_title_case
|
||||
_constants.py # Shared USPS abbrevs + state names
|
||||
cli.py # Deduplicator CLI (Typer)
|
||||
cli_text_clean.py # Text Cleaner CLI
|
||||
cli.py # Find Duplicates CLI (Typer)
|
||||
cli_text_clean.py # Clean Text CLI
|
||||
cli_analyze.py # Analyzer CLI (--json)
|
||||
gui/
|
||||
app.py # Streamlit entry point
|
||||
@@ -119,6 +122,17 @@ Tag a release → 3 platform artifacts upload to GitHub Releases. Manual: copy t
|
||||
|
||||
`demo/streamlit_app.py` → Streamlit Community Cloud. Configure deployment in Streamlit UI. Custom domain via CNAME (verify policy at deploy time). Fall back to $5/mo VPS if rate limits / branding constraints hit.
|
||||
|
||||
### 3.10 Bundled Tesseract (PDF Extractor OCR)
|
||||
|
||||
Frozen builds ship Tesseract 5.5 + `eng.traineddata` inside the PyInstaller bundle so scanned PDFs work without a separate install. Per-platform binary URLs pinned in `build/tesseract.py`; tessdata vendored at `build/vendor/tessdata/eng.traineddata`. License attribution in `LICENSE_TESSERACT.txt` at the repo root.
|
||||
|
||||
**Discovery order at runtime** (see `docs/DEVELOPER.md` for the full Path layout):
|
||||
|
||||
1. `DATATOOLS_TESSERACT_BIN` env var override.
|
||||
2. Bundled path under `sys._MEIPASS / "tesseract" /` (frozen bundles only).
|
||||
3. `tesseract` on `PATH` (source / pip developer environments).
|
||||
4. Windows well-known locations.
|
||||
|
||||
## 4. Libraries
|
||||
|
||||
| Purpose | Library |
|
||||
@@ -189,7 +203,7 @@ GUI / CLI handlers use `format_for_user()` so the user always sees: file path, o
|
||||
|
||||
| Bundle | Status |
|
||||
|--------|--------|
|
||||
| Data Cleaning Mastery | 3/9 tools Ready (Dedup, Text Cleaner, Format Standardizer); 6 stubs |
|
||||
| Data Cleaning Mastery | 3/9 tools Ready (Find Duplicates, Clean Text, Standardize Formats); 6 stubs |
|
||||
| Automated Business Reporting | Not started |
|
||||
| Ecommerce Data Pipeline | Not started |
|
||||
| Small Business Finance | Not started |
|
||||
@@ -211,12 +225,12 @@ Deliberately separate. Confluent original spec was wrong.
|
||||
|
||||
| Script | Owns |
|
||||
|--------|------|
|
||||
| 04 Missing Value Handler | "What's not there." Disguised nulls (`N/A`, `-`, sentinel codes), missingness patterns, imputation, drop-by-threshold. |
|
||||
| 06 Outlier Detector | "What shouldn't be there." z-score / IQR / modified-z, multivariate (Isolation Forest, Mahalanobis), domain rules, winsorization. |
|
||||
| 04 Fix Missing Values | "What's not there." Disguised nulls (`N/A`, `-`, sentinel codes), missingness patterns, imputation, drop-by-threshold. |
|
||||
| 06 Find Unusual Values | "What shouldn't be there." z-score / IQR / modified-z, multivariate (Isolation Forest, Mahalanobis), domain rules, winsorization. |
|
||||
|
||||
**Run order**: 04 before 06. Outlier stats on data with `NaN` / sentinels are mathematically poisoned (means dragged, IQR widens, false negatives).
|
||||
|
||||
**Pipeline order** (Pipeline Runner enforces): 02 → 03 → 04 → 05 → 06 → 07 → 08. 01 is order-flexible.
|
||||
**Pipeline order** (Automated Workflows enforces): 02 → 03 → 04 → 05 → 06 → 07 → 08. 01 is order-flexible.
|
||||
|
||||
**Contested cases**:
|
||||
- Whitespace-only cell — 02 trims to empty; 04 then flags empty as null.
|
||||
@@ -239,6 +253,15 @@ The GUI uses an in-house, JSON-backed translation layer at `src/i18n/`. **No** `
|
||||
|
||||
**Why not gettext**: zero compiled artifacts in the PyInstaller bundle, no build step before tests run, no `.po`/`.mo` round-trip for translators (anyone can edit JSON), and the same lookup works in unit tests without process state. Locked in because the surface won't grow large enough to need the alternative, and the alternative breaks the "drop a file, run pytest, ship" loop.
|
||||
|
||||
## 10c. GUI chrome — sidebar nav indicator swap
|
||||
|
||||
Streamlit's `st.Page`-driven sidebar renders section headers with a Material Symbols ligature (`expand_more` / `expand_less`). The header element is not a button and carries no `aria-expanded`, so a pure-CSS swap can't follow open/closed state. We replace the glyph with plain typographic `+` / `−` (U+2212) via JS:
|
||||
|
||||
- **CSS** (`components/_legacy.py`, `_HIDE_CHROME_CSS`) drops the Material Symbols font on `[data-testid="stIconMaterial"]` inside `[data-testid="stNavSectionHeader"]` so the rewritten character renders as normal text rather than re-resolving as an icon name.
|
||||
- **JS** (`_SWAP_NAV_SECTION_INDICATOR_JS`) walks each section header, reads the icon's text node, and rewrites `expand_more` → `+` / `expand_less` → `−`. A MutationObserver re-runs the swap when Streamlit re-renders the sidebar (RAF-throttled so a burst of mutations is one swap).
|
||||
|
||||
The script ships through the same component-iframe bundle as the brand injector and upload-button rename inside `hide_streamlit_chrome()` — one iframe per page, three DOM mutations.
|
||||
|
||||
## 11. Per-script functional specs
|
||||
|
||||
Specs live in this section as scripts enter active build. Each follows the Tier 1/2/3 structure with explicit strategic framing (what's the market gap given some of this is free elsewhere).
|
||||
|
||||
@@ -4,29 +4,90 @@
|
||||
|
||||
**Versión**: 1.6 · **Actualizado**: 2026-05-13
|
||||
|
||||
## 0. Primer arranque — activación
|
||||
|
||||
DataTools debe activarse antes de desbloquear cualquier herramienta. En el primer arranque verás la pantalla **Activar**.
|
||||
|
||||
Introduce tu nombre completo y correo, pega el código de licencia del correo de compra (empieza con `DTLIC1:`) y pulsa **Activar**. La renovación funciona igual: pega el código de renovación y pulsa **Aplicar renovación**.
|
||||
|
||||
**Niveles**:
|
||||
|
||||
| Nivel | Herramientas |
|
||||
|---|---|
|
||||
| **Lite** | Buscar duplicados · Limpiar texto · Estandarizar formatos |
|
||||
| **Core** | Las 9 herramientas |
|
||||
|
||||
Un usuario Lite que abra una herramienta exclusiva de Core verá un mensaje "Actualiza tu licencia". La página de inicio también muestra una marca 🔒 Bloqueado en las tarjetas de las herramientas que tu nivel no incluye. Para actualizar, pega un código Core en la página Activar.
|
||||
|
||||
Cada licencia dura 1 año. La barra lateral muestra tu nivel y los días restantes en todo momento; aparece un aviso de renovación 30 días antes de la caducidad. El archivo de licencia vive en `~/.datatools/license.json` (Windows: `C:\Users\<tú>\.datatools\license.json`).
|
||||
|
||||
Para usar la misma licencia en otro equipo: desactiva éste (página Activar → **Desactivar este dispositivo**) y vuelve a pegar tu código en el nuevo.
|
||||
|
||||
## 1. Instalación
|
||||
|
||||
No necesitas tener Python instalado — el paquete es autocontenido.
|
||||
No necesitas tener Python ni permisos de administrador — el paquete trae su propio intérprete y todas las dependencias. Cada sistema operativo tiene un único instalador que crea automáticamente el acceso directo en el escritorio + la entrada en el menú Inicio / Launchpad.
|
||||
|
||||
| Sistema operativo | Archivo | Cómo |
|
||||
|----|------|-----|
|
||||
| Windows | `BundleName-Setup-1.0.exe` | Doble clic en el instalador → acceso directo en el escritorio. |
|
||||
| macOS | `BundleName-1.0.dmg` | Monta el DMG y arrástralo a Aplicaciones. Firmado y notarizado. |
|
||||
| Linux | `BundleName-1.0.AppImage` | `chmod +x`, doble clic. (También hay un `.tar.gz` de respaldo.) |
|
||||
### 1.1 Windows
|
||||
|
||||
Al iniciar la app, se abre tu navegador predeterminado en una página local (`http://localhost:8501`).
|
||||
**Instalador (`DataTools-<ver>-win-setup.exe`)**
|
||||
|
||||
### Cómo funciona la interfaz gráfica (GUI)
|
||||
1. Descarga `DataTools-<ver>-win-setup.exe` desde tu correo de licencia o GitHub Releases.
|
||||
2. Doble clic en el instalador. La primera vez, Windows SmartScreen mostrará **"Windows protegió tu PC"** — pulsa **Más información** → **Ejecutar de todas formas**. (Este aviso solo aparece una vez por compilación hasta que tengamos un certificado EV de firma de código.)
|
||||
3. Acepta la ruta de instalación por usuario (`%LOCALAPPDATA%\Programs\DataTools` por defecto — no pide UAC). Marca **Crear acceso directo en el escritorio** si lo quieres (activado por defecto).
|
||||
4. Pulsa **Instalar** y luego **Finalizar**. El instalador te ofrece lanzar DataTools al terminar.
|
||||
5. A partir de ahora ejecútalo desde: **Menú Inicio → DataTools**, el **acceso directo del escritorio**, o escribiendo `DataTools` en Ejecutar (Win+R) / cmd.
|
||||
|
||||
Para anclarlo a la barra de tareas, lanza la app una vez, clic derecho en su icono de la barra de tareas, y **Anclar a la barra de tareas**. Windows requiere este paso manual — ningún instalador puede anclar por programa.
|
||||
|
||||
**Desinstalar**: Configuración → Aplicaciones → DataTools → Desinstalar.
|
||||
|
||||
### 1.2 macOS
|
||||
|
||||
**DMG instalador (`DataTools-<ver>-mac.dmg`)**
|
||||
|
||||
1. Descarga `DataTools-<ver>-mac.dmg`.
|
||||
2. Doble clic en el .dmg. Se abre una ventana de Finder con el icono **DataTools** y un alias **Aplicaciones**.
|
||||
3. Arrastra **DataTools** sobre **Aplicaciones**. Espera a que termine la copia y expulsa el DMG.
|
||||
4. En compilaciones sin firma, el primer arranque muestra **"No se puede abrir 'DataTools' porque no se puede verificar al desarrollador"**. Solución: clic derecho en DataTools en /Aplicaciones → **Abrir** → confirma **Abrir** en el diálogo. macOS recuerda la elección — los siguientes arranques no muestran nada.
|
||||
5. Ejecútalo desde **Launchpad**, **Spotlight** (`⌘ Espacio` → escribe "DataTools"), o **Aplicaciones** en Finder.
|
||||
|
||||
Para mantener DataTools en el Dock: lanza la app, clic derecho en su icono del Dock → **Opciones → Mantener en el Dock**. macOS no permite que los instaladores fijen al Dock automáticamente.
|
||||
|
||||
**Desinstalar**: arrastra `DataTools.app` a la Papelera. Tus archivos de datos siguen donde estén — la app no instala nada más.
|
||||
|
||||
### 1.3 Linux
|
||||
|
||||
`DataTools-<ver>-linux-x86_64.AppImage` ya es portable — no hay .zip aparte.
|
||||
|
||||
1. Descarga el .AppImage.
|
||||
2. `chmod +x DataTools-*.AppImage`.
|
||||
3. Doble clic, o ejecútalo desde la terminal.
|
||||
|
||||
Si tu distro no incluye FUSE 2: `sudo apt install libfuse2` (Debian/Ubuntu) o equivalente.
|
||||
|
||||
### 1.4 Qué pasa al arrancar por primera vez
|
||||
|
||||
El lanzador (llamado `DataTools.exe` / `DataTools.app` / `DataTools.AppImage`) hace tres cosas, en orden:
|
||||
|
||||
1. Elige un puerto TCP libre en `127.0.0.1` — normalmente el 8501; si está ocupado prueba 8502, 8503, …
|
||||
2. Arranca un servidor Streamlit local en ese puerto. El servidor solo está enlazado a localhost, nunca a tu red.
|
||||
3. Abre tu navegador predeterminado en `http://127.0.0.1:<puerto>/`. Si el navegador no se abre en 5 segundos, pega esa URL manualmente.
|
||||
|
||||
La ventana del lanzador queda abierta en segundo plano. Cerrarla detiene el servidor — la pestaña del navegador dirá "no se puede acceder a este sitio" la próxima vez.
|
||||
|
||||
### 1.5 Cómo funciona la GUI
|
||||
|
||||
- Se ejecuta localmente en tu equipo. **Sin internet, sin subidas.**
|
||||
- El navegador es solo la capa de visualización. Cerrarlo detiene el programa subyacente.
|
||||
- ¿Prefieres la terminal? Cada herramienta incluye también una interfaz de línea de comandos (CLI) — ver Sección 3.
|
||||
- El navegador es solo la capa de visualización. Cerrarlo NO detiene la app — cierra la ventana del lanzador (o sal de la .app de macOS desde el Dock) para terminar del todo.
|
||||
- ¿Prefieres la terminal? Cada herramienta incluye también una CLI — ver Sección 3.
|
||||
|
||||
### Requisitos del sistema
|
||||
### 1.6 Requisitos del sistema
|
||||
|
||||
- Windows 10/11 (64 bits), macOS 11+, Linux moderno (2020+).
|
||||
- Navegador moderno (Chrome, Edge, Firefox, Safari, últimos 3 años).
|
||||
- ~400-500 MB de espacio libre en disco.
|
||||
- ~500 MB de espacio libre en disco (el paquete ocupa ~300 MB; el resto es espacio de trabajo para CSV grandes).
|
||||
|
||||
**OCR para PDFs escaneados viene incluido** — Tesseract 5.5 y el modelo en inglés `eng.traineddata` vienen dentro de cada instalador / portable / AppImage. La ruta de extracción de PDFs escaneados del Extractor de PDF funciona sin configuración adicional; no hace falta instalar nada por separado. (Quien ejecute desde un checkout con `pip install -r requirements.txt` sigue necesitando Tesseract del sistema en el `PATH` — ver [DEVELOPER.md §PDF Extractor — bundled Tesseract](DEVELOPER.md#pdf-extractor--bundled-tesseract) (solo en inglés).)
|
||||
|
||||
Matriz de soporte completa: [REQUIREMENTS.md](REQUIREMENTS.md) (solo en inglés).
|
||||
|
||||
@@ -34,15 +95,15 @@ Matriz de soporte completa: [REQUIREMENTS.md](REQUIREMENTS.md) (solo en inglés)
|
||||
|
||||
| # | Herramienta | Propósito | Estado |
|
||||
|---|------|---------|--------|
|
||||
| 01 | Eliminador de duplicados | Coincidencia exacta + difusa, 5 normalizadores, auditoría | Listo |
|
||||
| 02 | Limpiador de texto | Espacios, caracteres tipográficos, BOM, finales de línea, mayúsculas/minúsculas | Listo |
|
||||
| 03 | Estandarizador de formatos | Fechas / teléfonos / correos / direcciones / nombres / monedas / booleanos | Listo |
|
||||
| 04 | Gestor de valores faltantes | Nulos disfrazados, imputación, descarte por umbral | Próximamente |
|
||||
| 05 | Mapeador de columnas | Renombrar + aplicar esquema | Próximamente |
|
||||
| 06 | Detector de valores atípicos | z-score, IQR, multivariante | Próximamente |
|
||||
| 07 | Combinador de varios archivos | Combina varios archivos | Próximamente |
|
||||
| 08 | Validador e informes | Reglas + informe PDF/Excel | Próximamente |
|
||||
| 09 | Ejecutor de canalizaciones | Lanzador multi-herramienta de un clic | Próximamente |
|
||||
| 01 | Buscar duplicados | Coincidencia exacta + difusa, 5 normalizadores, auditoría | Listo |
|
||||
| 02 | Limpiar texto | Espacios, caracteres tipográficos, BOM, finales de línea, mayúsculas/minúsculas | Listo |
|
||||
| 03 | Estandarizar formatos | Fechas / teléfonos / correos / direcciones / nombres / monedas / booleanos | Listo |
|
||||
| 04 | Corregir valores faltantes | Nulos disfrazados, imputación, descarte por umbral | Próximamente |
|
||||
| 05 | Mapear columnas | Renombrar + aplicar esquema | Próximamente |
|
||||
| 06 | Detectar valores atípicos | z-score, IQR, multivariante | Próximamente |
|
||||
| 07 | Combinar archivos | Combina varios archivos | Próximamente |
|
||||
| 08 | Verificación de calidad | Reglas + informe PDF/Excel | Próximamente |
|
||||
| 09 | Flujos automatizados | Lanzador multi-herramienta de un clic | Próximamente |
|
||||
|
||||
**Datos de muestra** (`samples/`): `messy_sales.csv`, `bank_export.xlsx`.
|
||||
|
||||
@@ -58,6 +119,10 @@ Matriz de soporte completa: [REQUIREMENTS.md](REQUIREMENTS.md) (solo en inglés)
|
||||
|
||||
Las opciones avanzadas se encuentran en paneles desplegables. El archivo original nunca se modifica.
|
||||
|
||||
**Ayuda en la herramienta**: cada página tiene un botón **Help** a la derecha del título. Al pulsarlo se abre una ventana emergente con una guía compacta (Cuándo usarla · Pasos · Ejemplos · Consejo). Úsala como recordatorio a media tarea — la ventana se cierra al hacer clic fuera y tus datos no se ven afectados.
|
||||
|
||||
**Navegación lateral**: la barra lateral agrupa las herramientas en secciones (Análisis, Limpiadores de datos, Transformaciones, Automatizaciones). Cada cabecera muestra `+` cuando está plegada y `−` cuando está desplegada — pulsa la cabecera para alternar.
|
||||
|
||||
### 3.2 CLI
|
||||
|
||||
```bash
|
||||
@@ -70,17 +135,17 @@ Ayuda: `deduplicator --help`. Referencia completa: [CLI-REFERENCE.es.md](CLI-REF
|
||||
|
||||
### 3.3 Orden de ejecución (cuando uses las herramientas manualmente)
|
||||
|
||||
Si no usas el Ejecutor de canalizaciones, sigue este orden:
|
||||
Si no usas Flujos automatizados, sigue este orden:
|
||||
|
||||
1. **02 Limpiador de texto** primero — normaliza espacios y caracteres especiales.
|
||||
2. **03 Estandarizador de formatos** — fechas, teléfonos, etc. necesitan texto limpio.
|
||||
3. **04 Gestor de valores faltantes** — códigos centinela se ocultan como números.
|
||||
4. **05 Mapeador de columnas** — esquema antes que estadísticas de atípicos.
|
||||
5. **06 Detector de valores atípicos** — necesita datos numéricos limpios. Calcular estadísticas con `NaN` o `-999` envenena los resultados.
|
||||
6. **07 Combinador de varios archivos**, **08 Validador** según sea necesario.
|
||||
7. **01 Eliminador de duplicados** es flexible en cuanto al orden (normaliza internamente para la coincidencia).
|
||||
1. **02 Limpiar texto** primero — normaliza espacios y caracteres especiales.
|
||||
2. **03 Estandarizar formatos** — fechas, teléfonos, etc. necesitan texto limpio.
|
||||
3. **04 Corregir valores faltantes** — códigos centinela se ocultan como números.
|
||||
4. **05 Mapear columnas** — esquema antes que estadísticas de atípicos.
|
||||
5. **06 Detectar valores atípicos** — necesita datos numéricos limpios. Calcular estadísticas con `NaN` o `-999` envenena los resultados.
|
||||
6. **07 Combinar archivos**, **08 Verificación de calidad** según sea necesario.
|
||||
7. **01 Buscar duplicados** es flexible en cuanto al orden (normaliza internamente para la coincidencia).
|
||||
|
||||
El Ejecutor de canalizaciones aplica este orden automáticamente.
|
||||
Flujos automatizados aplica este orden automáticamente.
|
||||
|
||||
### 3.4 Idioma
|
||||
|
||||
@@ -118,12 +183,15 @@ El archivo original nunca se modifica.
|
||||
|
||||
## 6. Solución de problemas
|
||||
|
||||
- **La GUI no se abre / el navegador no se inicia** — espera 10-15 s; visita manualmente `http://localhost:8501`. Error de puerto ocupado → cierra otras instancias.
|
||||
- **La GUI no se abre / el navegador no se inicia** — espera 10-15 s; visita manualmente `http://127.0.0.1:8501` (o el puerto que muestre la ventana del lanzador). Error de puerto ocupado → cierra otras instancias. El lanzador recorre los puertos 8501–8550 buscando uno libre, así que una instancia colgada puede desplazar la URL.
|
||||
- **¿Por qué se abre el navegador?** — patrón de aplicación web local (igual que Jupyter o RStudio). Nada sale de tu equipo.
|
||||
- **Windows SmartScreen** — pulsa "Más información" → "Ejecutar de todas formas". Estándar para software sin firma EV.
|
||||
- **macOS "La aplicación está dañada"** — descárgala de nuevo (probablemente se corrompió en tránsito).
|
||||
- **El AppImage de Linux no se ejecuta** — `chmod +x archivo.AppImage`. Si falta FUSE → `sudo apt install libfuse2` o usa el `.tar.gz`.
|
||||
- **Windows SmartScreen** — pulsa "Más información" → "Ejecutar de todas formas". Una sola vez por compilación hasta que tengamos un certificado EV.
|
||||
- **macOS "La aplicación está dañada" / "no se puede verificar al desarrollador"** — clic derecho en la app → **Abrir** → confirma. Si el mensaje persiste, el archivo se corrompió en tránsito — vuelve a descargarlo. Último recurso: `xattr -cr /Applications/DataTools.app` limpia el atributo de cuarentena.
|
||||
- **macOS — el .zip portable extraído no abre** — Safari descomprime al descargar; si ves una carpeta `__MACOSX/` o archivos `._DataTools.app` usaste otro descompresor. Vuelve a extraer con la Utilidad de Archivo integrada (clic derecho en el .zip → **Abrir con → Utilidad de Archivo**) para preservar los metadatos de la .app.
|
||||
- **Windows — el antivirus pone en cuarentena `DataTools.exe` del portable** — tu antivirus no reconoce el paquete. Añade la carpeta extraída a la lista blanca. El instalador .exe activa menos antivirus porque es un envoltorio Inno Setup conocido.
|
||||
- **El AppImage de Linux no se ejecuta** — `chmod +x archivo.AppImage`. Si falta FUSE → `sudo apt install libfuse2`.
|
||||
- **Lento con archivos grandes** — por encima de ~100k filas tarda más; la barra de progreso lo indica. Para millones de filas → usa la CLI directamente.
|
||||
- **¿Dónde guarda la app mi licencia / configuración?** — `~/.datatools/` en macOS y Linux, `C:\Users\<tú>\.datatools\` en Windows. Tus archivos de entrada y salida siguen donde los dejes; la app nunca los copia a otro sitio.
|
||||
- **Necesito ayuda** — escribe al correo que aparece en tu recibo de compra.
|
||||
|
||||
## 7. Licencia
|
||||
|
||||
@@ -4,29 +4,90 @@
|
||||
|
||||
**Version**: 1.6 · **Updated**: 2026-05-01
|
||||
|
||||
## 0. First launch — activation
|
||||
|
||||
DataTools must be activated before any tools unlock. On first launch you'll see the **Activate** screen.
|
||||
|
||||
Enter your full name + email, paste the license blob from your purchase email (starts with `DTLIC1:`), and click **Activate**. Renewal works the same way — paste the renewal blob, click **Apply renewal**.
|
||||
|
||||
**Tiers**:
|
||||
|
||||
| Tier | Tools |
|
||||
|---|---|
|
||||
| **Lite** | Find Duplicates · Clean Text · Standardize Formats |
|
||||
| **Core** | All 9 tools |
|
||||
|
||||
A Lite user opening a Core-only tool sees an "Upgrade your license" prompt. The home page also shows a 🔒 Locked badge on tool cards your tier doesn't unlock. To upgrade, paste a Core blob on the Activate page.
|
||||
|
||||
Every license lasts 1 year. The sidebar shows your tier and days remaining at all times; a renewal warning appears 30 days before expiry. The license file lives at `~/.datatools/license.json` (Windows: `C:\Users\<you>\.datatools\license.json`).
|
||||
|
||||
To use the same license on a different machine: deactivate this one (Activate page → **Deactivate this device**) and re-paste your blob on the new machine.
|
||||
|
||||
## 1. Install
|
||||
|
||||
You don't need Python — the bundle is self-contained.
|
||||
You don't need Python and you don't need admin rights — the bundle ships its own interpreter and every dependency. Each OS gets a single installer that wires up the Desktop shortcut + Start Menu / Launchpad entry automatically.
|
||||
|
||||
| OS | File | How |
|
||||
|----|------|-----|
|
||||
| Windows | `BundleName-Setup-1.0.exe` | Double-click installer → desktop shortcut. |
|
||||
| macOS | `BundleName-1.0.dmg` | Mount, drag to Applications. Signed + notarized. |
|
||||
| Linux | `BundleName-1.0.AppImage` | `chmod +x`, double-click. (`.tar.gz` fallback available.) |
|
||||
### 1.1 Windows
|
||||
|
||||
Launching opens your default browser to a local page (`http://localhost:8501`).
|
||||
**Installer (`DataTools-<ver>-win-setup.exe`)**
|
||||
|
||||
### How the GUI works
|
||||
1. Download `DataTools-<ver>-win-setup.exe` from your release email or GitHub Releases.
|
||||
2. Double-click the installer. On the first run Windows SmartScreen will say **"Windows protected your PC"** — click **More info** → **Run anyway**. (This warning only appears once per build until we have an EV code-signing cert.)
|
||||
3. Accept the per-user install location (`%LOCALAPPDATA%\Programs\DataTools` by default — no admin prompt). Check **Create a desktop shortcut** if you want one (on by default).
|
||||
4. Click **Install**, then **Finish**. The installer offers to launch DataTools immediately.
|
||||
5. From now on launch from: **Start Menu → DataTools**, the **Desktop shortcut**, or just type `DataTools` into Windows Run (Win+R) / cmd.
|
||||
|
||||
To pin to the taskbar, launch the app once, right-click its icon in the taskbar, then **Pin to taskbar**. Windows requires this manual step — no installer is allowed to pin programmatically.
|
||||
|
||||
**Uninstall**: Settings → Apps → DataTools → Uninstall.
|
||||
|
||||
### 1.2 macOS
|
||||
|
||||
**Installer DMG (`DataTools-<ver>-mac.dmg`)**
|
||||
|
||||
1. Download `DataTools-<ver>-mac.dmg`.
|
||||
2. Double-click the .dmg. A Finder window opens showing the **DataTools** icon and an **Applications** alias.
|
||||
3. Drag **DataTools** onto **Applications**. Wait for the copy to finish, then eject the DMG.
|
||||
4. On unsigned builds the first launch shows **"DataTools" cannot be opened because the developer cannot be verified**. Fix: right-click DataTools in /Applications → **Open** → confirm **Open** in the dialog. macOS remembers this choice — subsequent launches are clean.
|
||||
5. Launch from **Launchpad**, **Spotlight** (`⌘ Space` → type "DataTools"), or **Applications** in Finder.
|
||||
|
||||
To keep DataTools in the Dock: launch the app, right-click its Dock icon → **Options → Keep in Dock**. macOS doesn't allow installers to pin to the Dock automatically.
|
||||
|
||||
**Uninstall**: drag `DataTools.app` to the Trash. Your data files stay where you put them — nothing else is installed.
|
||||
|
||||
### 1.3 Linux
|
||||
|
||||
`DataTools-<ver>-linux-x86_64.AppImage` is already portable — no separate zip needed.
|
||||
|
||||
1. Download the .AppImage.
|
||||
2. `chmod +x DataTools-*.AppImage`.
|
||||
3. Double-click, or run it from a terminal.
|
||||
|
||||
If your distro doesn't ship FUSE 2: `sudo apt install libfuse2` (Debian/Ubuntu) or equivalent.
|
||||
|
||||
### 1.4 What happens on first launch
|
||||
|
||||
The launcher (called `DataTools.exe` / `DataTools.app` / `DataTools.AppImage`) does three things, in order:
|
||||
|
||||
1. Picks a free TCP port on `127.0.0.1` — usually 8501, falls back through 8502, 8503, … if another app is using 8501.
|
||||
2. Starts a local Streamlit server on that port. The server is **bound to localhost only**, never to your LAN.
|
||||
3. Opens your default browser at `http://127.0.0.1:<port>/`. If the browser doesn't open within 5 seconds, paste that URL into your browser manually.
|
||||
|
||||
The launcher window stays open in the background. Closing it stops the server — the browser tab will say "this site can't be reached" the next time you click it.
|
||||
|
||||
### 1.5 How the GUI works
|
||||
|
||||
- Runs locally on your machine. **No internet, no upload.**
|
||||
- Browser is just the display surface. Closing it stops the underlying program.
|
||||
- The browser is just the display surface. Closing it does NOT stop the app — close the launcher window (or quit the macOS .app from the Dock) to fully exit.
|
||||
- Prefer the terminal? Every tool ships with a CLI too (Section 3).
|
||||
|
||||
### System requirements
|
||||
### 1.6 System requirements
|
||||
|
||||
- Windows 10/11 (64-bit), macOS 11+, modern Linux (2020+).
|
||||
- Modern browser (Chrome, Edge, Firefox, Safari, last 3 years).
|
||||
- ~400-500 MB free disk space.
|
||||
- ~500 MB free disk space (the bundle itself is ~300 MB; the rest is working scratch space for large CSVs).
|
||||
|
||||
**OCR for scanned PDFs is bundled** — Tesseract 5.5 + the English `eng.traineddata` model ship inside every installer / portable / AppImage. The PDF Extractor's scanned-statement path works out of the box; no separate install required. (Developers running from a `pip install -r requirements.txt` checkout still need system Tesseract on `PATH` — see [DEVELOPER.md §PDF Extractor — bundled Tesseract](DEVELOPER.md#pdf-extractor--bundled-tesseract).)
|
||||
|
||||
Full numbered support matrix: [REQUIREMENTS.md](REQUIREMENTS.md).
|
||||
|
||||
@@ -34,15 +95,15 @@ Full numbered support matrix: [REQUIREMENTS.md](REQUIREMENTS.md).
|
||||
|
||||
| # | Tool | Purpose | Status |
|
||||
|---|------|---------|--------|
|
||||
| 01 | Deduplicator | Exact + fuzzy match, 5 normalizers, audit | Ready |
|
||||
| 02 | Text Cleaner | Whitespace, smart chars, BOM, line endings, case ops | Ready |
|
||||
| 03 | Format Standardizer | Dates / phones / emails / addresses / names / currencies / booleans | Ready |
|
||||
| 04 | Missing Value Handler | Disguised nulls, imputation, drop-by-threshold | Coming Soon |
|
||||
| 05 | Column Mapper | Rename + enforce schema | Coming Soon |
|
||||
| 06 | Outlier Detector | z-score, IQR, multivariate | Coming Soon |
|
||||
| 07 | Multi-File Merger | Combine multiple files | Coming Soon |
|
||||
| 08 | Validator & Reporter | Rules + PDF/Excel report | Coming Soon |
|
||||
| 09 | Pipeline Runner | One-click multi-tool launcher | Coming Soon |
|
||||
| 01 | Find Duplicates | Exact + fuzzy match, 5 normalizers, audit | Ready |
|
||||
| 02 | Clean Text | Whitespace, smart chars, BOM, line endings, case ops | Ready |
|
||||
| 03 | Standardize Formats | Dates / phones / emails / addresses / names / currencies / booleans | Ready |
|
||||
| 04 | Fix Missing Values | Disguised nulls, imputation, drop-by-threshold | Coming Soon |
|
||||
| 05 | Map Columns | Rename + enforce schema | Coming Soon |
|
||||
| 06 | Find Unusual Values | z-score, IQR, multivariate | Coming Soon |
|
||||
| 07 | Combine Files | Combine multiple files | Coming Soon |
|
||||
| 08 | Quality Check | Rules + PDF/Excel report | Coming Soon |
|
||||
| 09 | Automated Workflows | One-click multi-tool launcher | Coming Soon |
|
||||
|
||||
**Sample data** (`samples/`): `messy_sales.csv`, `bank_export.xlsx`.
|
||||
|
||||
@@ -58,6 +119,10 @@ Full numbered support matrix: [REQUIREMENTS.md](REQUIREMENTS.md).
|
||||
|
||||
Advanced options are tucked in expander panes. The original file is never modified.
|
||||
|
||||
**In-tool Help**: every tool page has a **Help** button right of the title. Click it to open a popover with a compact how-to (When to use · Steps · Examples · Tip). Use it as a refresher mid-task — the popover closes when you click outside, your inputs are untouched.
|
||||
|
||||
**Sidebar nav**: the sidebar groups tools into sections (Analysis, Data Cleaners, Transformations, Automations). Each section header shows `+` when collapsed and `−` when expanded — click the header to toggle.
|
||||
|
||||
### 3.2 CLI
|
||||
|
||||
```bash
|
||||
@@ -70,17 +135,17 @@ Get help: `deduplicator --help`. Full reference: [CLI-REFERENCE.md](CLI-REFERENC
|
||||
|
||||
### 3.3 Run order (when running tools manually)
|
||||
|
||||
If you skip the Pipeline Runner, follow this order:
|
||||
If you skip Automated Workflows, follow this order:
|
||||
|
||||
1. **02 Text Cleaner** first — normalizes whitespace + special chars.
|
||||
2. **03 Format Standardizer** — dates, phones, etc. need cleaned text.
|
||||
3. **04 Missing Value Handler** — sentinel codes hide as numbers.
|
||||
4. **05 Column Mapper** — schema before outlier stats.
|
||||
5. **06 Outlier Detector** — needs clean numerics. Stats on data with `NaN` or `-999` are mathematically poisoned.
|
||||
6. **07 Multi-File Merger**, **08 Validator** as needed.
|
||||
7. **01 Deduplicator** is order-flexible (normalizes internally for matching).
|
||||
1. **02 Clean Text** first — normalizes whitespace + special chars.
|
||||
2. **03 Standardize Formats** — dates, phones, etc. need cleaned text.
|
||||
3. **04 Fix Missing Values** — sentinel codes hide as numbers.
|
||||
4. **05 Map Columns** — schema before outlier stats.
|
||||
5. **06 Find Unusual Values** — needs clean numerics. Stats on data with `NaN` or `-999` are mathematically poisoned.
|
||||
6. **07 Combine Files**, **08 Quality Check** as needed.
|
||||
7. **01 Find Duplicates** is order-flexible (normalizes internally for matching).
|
||||
|
||||
The Pipeline Runner enforces this automatically.
|
||||
Automated Workflows enforces this automatically.
|
||||
|
||||
### 3.4 Language
|
||||
|
||||
@@ -118,12 +183,15 @@ Original input is never modified.
|
||||
|
||||
## 6. Troubleshooting
|
||||
|
||||
- **GUI won't launch / browser doesn't open** — wait 10-15 s; manually visit `http://localhost:8501`. Port-in-use error → close other instances.
|
||||
- **GUI won't launch / browser doesn't open** — wait 10-15 s; manually visit `http://127.0.0.1:8501` (or whichever port the launcher window prints). Port-in-use error → close other instances. The launcher walks ports 8501–8550 looking for a free one, so a stale instance can shift the URL.
|
||||
- **Why does my browser open?** — local web app pattern (same as Jupyter, RStudio). Nothing leaves your machine.
|
||||
- **Windows SmartScreen** — click "More info" → "Run anyway". Standard for non-EV-signed software.
|
||||
- **macOS "App is damaged"** — re-download (file likely corrupted in transit).
|
||||
- **Linux AppImage won't run** — `chmod +x file.AppImage`. Missing FUSE → `sudo apt install libfuse2` or use `.tar.gz`.
|
||||
- **Windows SmartScreen** — click "More info" → "Run anyway". One-time per build until we have an EV-signed cert.
|
||||
- **macOS "App is damaged" / "developer cannot be verified"** — right-click the app → **Open** → confirm. If the message persists, the file was likely corrupted in transit — re-download. As a last resort: `xattr -cr /Applications/DataTools.app` clears the quarantine attribute.
|
||||
- **macOS portable .zip — extracted but won't open** — Safari unzips on download by default; if you see a `__MACOSX/` folder or `._DataTools.app` file you used a different unarchiver. Re-extract with the built-in Archive Utility (right-click the .zip → **Open With → Archive Utility**) so the .app's metadata is preserved.
|
||||
- **Windows portable .zip — antivirus quarantines DataTools.exe** — your AV doesn't recognize the bundle. Allowlist the extracted folder. The installer .exe trips fewer AV products because it's a known Inno Setup wrapper.
|
||||
- **Linux AppImage won't run** — `chmod +x file.AppImage`. Missing FUSE → `sudo apt install libfuse2`.
|
||||
- **Slow on large file** — over ~100k rows takes longer; progress bar shows. Multi-million rows → use the CLI directly.
|
||||
- **Where does the app store my license / settings?** — `~/.datatools/` on macOS + Linux, `C:\Users\<you>\.datatools\` on Windows. Your input/output files stay where you put them; the app never copies them anywhere else.
|
||||
- **Need help** — email the address on your purchase receipt.
|
||||
|
||||
## 7. License
|
||||
|
||||
@@ -9,9 +9,9 @@ Cloudflare Pages.
|
||||
```
|
||||
landing/
|
||||
├── _shared/styles.css shared CSS (system fonts, no externals)
|
||||
├── shopify-pet/index.html Shopify operator (priority: pet supplies)
|
||||
├── bookkeeper/index.html bookkeeper / freelance accountant
|
||||
├── revops/index.html marketing / RevOps agency
|
||||
├── bookkeeper/index.html bookkeeper — bank reconciliation
|
||||
├── ap-1099/index.html accounts payable — 1099 vendor prep
|
||||
├── ar-aging/index.html accounts receivable — open invoices
|
||||
└── README.md this file
|
||||
```
|
||||
|
||||
@@ -19,8 +19,8 @@ Each page:
|
||||
|
||||
- Inherits `landing/_shared/styles.css`
|
||||
- Overrides the `--accent` colour variable in an inline `<style>` block
|
||||
so each persona has its own visual identity (Shopify = mint green,
|
||||
Bookkeeper = steel blue, RevOps = vivid violet)
|
||||
so each persona has its own visual identity (Bookkeeper = steel blue,
|
||||
AP / 1099 = amber/gold, AR = receivables green)
|
||||
- Has a sticky buy bar with the Gumroad CTA tagged with `?from=<persona>`
|
||||
- Embeds the live demo (Streamlit) via `<iframe>` with a sandbox attribute
|
||||
- Carries persona-specific H1, sub-copy, use cases, FAQ, and a
|
||||
@@ -64,13 +64,13 @@ wrangler pages deploy landing/dist
|
||||
```
|
||||
|
||||
Configure the custom apex domain (`datatools.app`) in the Cloudflare
|
||||
Pages project settings; sub-paths `/shopify-pet/`, `/bookkeeper/`,
|
||||
`/revops/` are served automatically because the directory layout
|
||||
Pages project settings; sub-paths `/bookkeeper/`, `/ap-1099/`,
|
||||
`/ar-aging/` are served automatically because the directory layout
|
||||
mirrors them. Cache rule defaults are fine (HTML 1 day, CSS 7 days).
|
||||
|
||||
If you want **separate Pages projects** per persona for independent
|
||||
A/B testing, point three projects at the same `landing/dist/` and
|
||||
configure each with its own sub-domain (`shopify.datatools.app`, etc.)
|
||||
configure each with its own sub-domain (`bookkeeper.datatools.app`, etc.)
|
||||
and a Pages rule that rewrites the root to that persona's
|
||||
sub-directory.
|
||||
|
||||
@@ -110,7 +110,7 @@ Refresh the page when:
|
||||
| `page_view → run_completed < 30%` for 4 weeks | The demo iframe isn't loading or visitors aren't engaging. Check the iframe URL. Move the demo above the fold if it's currently below. |
|
||||
| New tool ships (06–09) | Add it to the persona's saved pipeline only if it fits — don't bloat the demo with every tool. |
|
||||
| Pricing change | Update `<meta>` schema, the buybar `.price-tag`, the pricing card, and the FAQ. Search-and-replace `$49` across the file. |
|
||||
| New persona added (4th, 5th) | Copy `shopify-pet/index.html`, replace persona-specific copy, add to the `footer` cross-link block on the existing pages. |
|
||||
| New persona added (4th, 5th) | Copy `bookkeeper/index.html`, replace persona-specific copy, add to the `footer` cross-link block on the existing pages. |
|
||||
|
||||
## Why static HTML
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
* with zero build step, no privacy banner needed).
|
||||
* • Mobile-first; layout reflows below 720 px.
|
||||
* • Dark, focused, content-first. Buyer reads this on a laptop
|
||||
* between Shopify exports — keep it readable and skimmable.
|
||||
* between messy accounting exports — keep it readable and skimmable.
|
||||
* • Persona pages all share this sheet — niche differences live in
|
||||
* copy + accent-color variables overridden in each page's <style>.
|
||||
*/
|
||||
@@ -18,7 +18,7 @@
|
||||
--text-mute: #9aa3b2;
|
||||
--text-soft: #c8ced8;
|
||||
--rule: #252a36;
|
||||
--accent: #6ee7b7; /* Shopify pet default — overridden per persona */
|
||||
--accent: #6ee7b7; /* default accent — overridden per persona */
|
||||
--accent-ink: #052e1a;
|
||||
--warn: #fbbf24;
|
||||
--max: 1080px;
|
||||
|
||||
391
landing/ap-1099/index.html
Normal file
391
landing/ap-1099/index.html
Normal file
@@ -0,0 +1,391 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>DataTools for 1099 Prep — Clean Your Vendor Master & Recover Missing EINs Locally · $49</title>
|
||||
<meta name="description" content="Build a clean 1099 vendor list — locally. Consolidates duplicate vendor rows, backfills scattered EINs, and flags the genuinely missing ones. 24 messy records → 8 complete vendors, 7 EINs recovered. Your data never leaves your computer. $49 one-time." />
|
||||
<meta name="keywords" content="1099 vendor list, missing EIN, accounts payable cleanup, vendor master dedupe, 1099-NEC prep, QuickBooks vendor export, deduplicate vendors" />
|
||||
<link rel="canonical" href="https://datatools.app/ap-1099/" />
|
||||
<link rel="stylesheet" href="../_shared/styles.css" />
|
||||
|
||||
<!-- Persona accent: Accounts Payable / 1099 → amber/gold invoice tone -->
|
||||
<style>
|
||||
:root { --accent: #d97706; --accent-ink: #2a1604; }
|
||||
</style>
|
||||
|
||||
<!-- Open Graph -->
|
||||
<meta property="og:title" content="DataTools for 1099 Prep — Clean Your Vendor Master & Recover Missing EINs Locally" />
|
||||
<meta property="og:description" content="Consolidate duplicate vendors, backfill scattered EINs, file 1099-NECs on time. Local. No upload. $49 one-time." />
|
||||
<meta property="og:type" content="product" />
|
||||
<meta property="og:url" content="https://datatools.app/ap-1099/" />
|
||||
|
||||
<!-- Schema.org Product -->
|
||||
<script type="application/ld+json">
|
||||
{
|
||||
"@context": "https://schema.org",
|
||||
"@type": "SoftwareApplication",
|
||||
"name": "DataTools for 1099 Prep",
|
||||
"operatingSystem": "Windows, macOS, Linux",
|
||||
"applicationCategory": "BusinessApplication",
|
||||
"offers": {
|
||||
"@type": "Offer",
|
||||
"price": "49",
|
||||
"priceCurrency": "USD"
|
||||
},
|
||||
"description": "Clean your accounts-payable vendor master locally for 1099-NEC season. Six-tool data-cleaning bundle: dedupe-merge to consolidate duplicate vendor rows and backfill missing EINs, text-clean, format-standardize, missing-value handle, column-map, pipeline.",
|
||||
"softwareVersion": "1.0"
|
||||
}
|
||||
</script>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<!-- ============= Sticky buy bar ============= -->
|
||||
<div class="buybar">
|
||||
<div class="buybar-inner">
|
||||
<div class="brand"><span class="brand-mark">●</span> DataTools <span class="muted">/ for 1099 prep</span></div>
|
||||
<div>
|
||||
<span class="price-tag">$49 — one-time, no subscription</span>
|
||||
<a class="btn" href="https://gumroad.com/l/datatools?from=ap-1099" rel="noopener">Get DataTools →</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- ============= Hero ============= -->
|
||||
<section class="hero">
|
||||
<div class="container">
|
||||
<div class="eyebrow">For accounts payable · 1099-NEC season · vendor master cleanup</div>
|
||||
<h1>Build a clean 1099 vendor list —<br /><strong>with the missing EINs filled in.</strong></h1>
|
||||
<p class="lead">
|
||||
The same vendor got entered three times across the year — one row has
|
||||
the EIN, another the address, another the phone — and now it's January
|
||||
and you can't file because the numbers are scattered. DataTools
|
||||
consolidates each vendor to one row and backfills the gaps from the
|
||||
duplicates: in our sample, <strong>24 messy records become 8 complete
|
||||
vendors with 7 missing EINs recovered</strong> from duplicate rows.
|
||||
<strong>Your data never leaves your computer.</strong>
|
||||
</p>
|
||||
<div class="cta-row">
|
||||
<a class="btn btn-large" href="https://gumroad.com/l/datatools?from=ap-1099" rel="noopener">Get DataTools for Accounting — $49 →</a>
|
||||
<a class="btn btn-ghost btn-large" href="#demo">Try the live demo ↓</a>
|
||||
<span class="price-note">One-time payment · cross-platform · runs offline</span>
|
||||
</div>
|
||||
<div class="stats">
|
||||
<div class="stat"><div class="num">24→8</div><div class="label">messy records to complete vendors</div></div>
|
||||
<div class="stat"><div class="num">7</div><div class="label">missing EINs recovered</div></div>
|
||||
<div class="stat"><div class="num">0</div><div class="label">cloud uploads ever</div></div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- ============= Pain points ============= -->
|
||||
<section>
|
||||
<div class="container">
|
||||
<div class="eyebrow">If any of these sound like your January</div>
|
||||
<h2>Five pains DataTools fixes in one pass</h2>
|
||||
<div class="grid">
|
||||
<div class="card">
|
||||
<span class="icon">🧾</span>
|
||||
<h3>The same vendor is in the list two or three times</h3>
|
||||
<p>Different staff entered "Acme LLC", "Acme, L.L.C.", and "ACME Llc" across the year. Each is a separate row in the vendor master, and each only holds part of the story — so your 1099 totals split across three near-duplicate spellings.</p>
|
||||
<p class="muted"><strong>What it costs:</strong> hours of manual matching, plus the risk of filing the wrong total.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<span class="icon">🔢</span>
|
||||
<h3>The EIN is on a different row than the rest of the details</h3>
|
||||
<p>One record captured the EIN at onboarding; the row you actually paid against doesn't have it. At 1099 time the field is blank even though you collected it months ago — it's just sitting on a duplicate.</p>
|
||||
<p class="muted"><strong>What it costs:</strong> chasing W-9s you already have on file.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<span class="icon">📵</span>
|
||||
<h3>Phones, addresses, and amounts are formatted five different ways</h3>
|
||||
<p>Remittance phone as <code>(212) 555-0147</code> on one row and <code>212.555.0147</code> on another. Amounts with stray <code>$</code> and commas. The export won't reconcile and the 1099-NEC box totals don't tie out.</p>
|
||||
<p class="muted"><strong>What it costs:</strong> a half-day reconciling before you can even start filing.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<span class="icon">⛔</span>
|
||||
<h3>You don't know which EINs are genuinely missing</h3>
|
||||
<p>Some EINs are recoverable from a duplicate row. Some you never collected. Until the list is consolidated you can't tell the two apart — so you either over-chase vendors or under-file.</p>
|
||||
<p class="muted"><strong>What it costs:</strong> late filings and TIN-mismatch penalties.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<span class="icon">📤</span>
|
||||
<h3>Your QuickBooks vendor export doesn't match your AP ledger</h3>
|
||||
<p>The vendor master in QuickBooks, the payments spreadsheet, and the W-9 tracker each use different column names for "vendor name" / "Tax ID" / "amount paid." Merging them is an afternoon of manual rename before any analysis begins.</p>
|
||||
<p class="muted"><strong>What it costs:</strong> 4–8 hours per filing season manually merging exports.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<span class="icon">🔒</span>
|
||||
<h3>Cloud cleaners want you to upload your vendor master</h3>
|
||||
<p>Your vendor master holds EINs, remittance addresses, and payment history — exactly the data you should not be uploading to a SaaS to clean. DataTools is desktop-only — your vendor list never leaves your computer.</p>
|
||||
<p class="muted"><strong>What it costs:</strong> nothing — and that's the point.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- ============= Live demo ============= -->
|
||||
<section id="demo">
|
||||
<div class="container">
|
||||
<div class="eyebrow">Live demo · runs in your browser</div>
|
||||
<h2>Try it on a real-looking vendor master export</h2>
|
||||
<p>
|
||||
The demo below loads a sample 24-row vendor file with the pollution
|
||||
we've seen in real AP systems: the same vendor entered two or three
|
||||
times under slightly different spellings, EINs that live on one
|
||||
duplicate row but not the one you paid against, phones and amounts
|
||||
formatted five ways, and the usual mess of
|
||||
<code>N/A</code> / <code>(blank)</code> / <code>?</code> sentinels.
|
||||
Click <strong>Run pipeline</strong> and watch the 24 records collapse
|
||||
to <strong>8 complete vendors with 7 EINs recovered</strong> in under
|
||||
a second.
|
||||
</p>
|
||||
<div class="demo-frame">
|
||||
<iframe
|
||||
src="https://demo.datatools.app/?p=ap-1099"
|
||||
loading="lazy"
|
||||
title="DataTools live demo — accounts payable / 1099 vendor cleanup"
|
||||
sandbox="allow-scripts allow-same-origin allow-downloads allow-forms"></iframe>
|
||||
<div class="demo-caption">
|
||||
Demo runs on free hosting (Streamlit Community Cloud). Capped at
|
||||
100 input rows · output watermarked with one trailing row. The
|
||||
paid product has no caps and runs entirely offline.
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- ============= Built for AP / 1099 ============= -->
|
||||
<section>
|
||||
<div class="container">
|
||||
<div class="eyebrow">Built for the accounts-payable team</div>
|
||||
<h2>Five workflows you do every filing season</h2>
|
||||
<div class="grid">
|
||||
<div class="card">
|
||||
<span class="icon">🧹</span>
|
||||
<h3>Vendor-master consolidation</h3>
|
||||
<p>Catches the same vendor that shows up as <code>Acme LLC</code>, <code>Acme, L.L.C.</code>, and <code>ACME Llc</code>. Fuzzy match merges the spellings; the dedup merge collapses them to one row and backfills the gaps from each duplicate.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<span class="icon">🔢</span>
|
||||
<h3>EIN backfill & missing-EIN flagging</h3>
|
||||
<p>Pulls the EIN off whichever duplicate row captured it and fills it into the survivor. The EINs that are <em>genuinely</em> missing get flagged so you know exactly which W-9s to chase.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<span class="icon">💵</span>
|
||||
<h3>1099-NEC amount roll-up</h3>
|
||||
<p>Before filing: standardize amounts, drop sentinels-as-missing, and merge so each vendor's total paid lands on one row and ties to your AP ledger.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<span class="icon">📥</span>
|
||||
<h3>QuickBooks vendor export cleanup</h3>
|
||||
<p>Whitespace in Tax IDs, near-identical vendor names, copy-paste smart quotes in remittance addresses — gone. Audit log shows every change for your reviewer.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<span class="icon">🔗</span>
|
||||
<h3>Merging the W-9 tracker into the AP ledger</h3>
|
||||
<p>The vendor master, the payments spreadsheet, and the W-9 tracker each name "Tax ID" differently. Map Columns aligns them; the dedup merge consolidates across all three sources.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<span class="icon">⚙️</span>
|
||||
<h3>Repeatable pipeline</h3>
|
||||
<p>Save the cleanup as a JSON file. Drop next year's vendor export on it. Same consolidation, zero re-configuration. Automatable via the CLI.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- ============= Privacy moat ============= -->
|
||||
<section>
|
||||
<div class="container">
|
||||
<div class="eyebrow">The thing every cloud cleaner can't say</div>
|
||||
<h2>Your vendor master never leaves your computer.</h2>
|
||||
<p>
|
||||
DataTools is a desktop app. There's no upload step, no SaaS account,
|
||||
no subscription, no "trust our security policy." The first thing you
|
||||
can do after install is open your browser's network tab, run the
|
||||
cleaner on your real vendor file, and verify zero outbound
|
||||
requests.
|
||||
</p>
|
||||
<div class="callout">
|
||||
<strong>Why it matters for AP:</strong> your vendor master holds EINs,
|
||||
remittance addresses, and payment history. Cloud cleaners require you
|
||||
to upload it. We don't.
|
||||
</div>
|
||||
<div class="terminal"><span class="prompt">$</span> python -m src.cli_pipeline vendor_1099.csv --pipeline vendor_1099_pipeline.json --apply
|
||||
Reading vendor_1099.csv...
|
||||
24 rows, 9 columns
|
||||
Executing pipeline:
|
||||
<span class="ok">✓</span> text_clean (38 ms) {cells_changed: 41}
|
||||
<span class="ok">✓</span> format_standardize (62 ms) {cells_changed: 36} # phones, EINs, amounts
|
||||
<span class="ok">✓</span> missing (11 ms) {sentinels_standardized: 9}
|
||||
<span class="ok">✓</span> dedup (140 ms) {groups_merged: 8, rows_removed: 16, eins_backfilled: 7}
|
||||
|
||||
Initial rows: 24 → Final rows: 8 (8 complete vendors)
|
||||
EINs recovered from duplicate rows: 7 | Still missing (flagged): 1
|
||||
Unparseable cells: 0
|
||||
Total elapsed: 0.25 s
|
||||
<span class="prompt">$</span> # zero network calls. zero. promise.</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- ============= Audit moat ============= -->
|
||||
<section>
|
||||
<div class="container">
|
||||
<div class="eyebrow">For when your reviewer asks "what changed?"</div>
|
||||
<h2>Every change auditable. Every cell logged.</h2>
|
||||
<p>
|
||||
Every modification is recorded with the original value, the new
|
||||
value, and which rule fired. Hand the audit CSV to your controller,
|
||||
your reviewer, or the IRS-ready workpaper file along with the cleaned
|
||||
vendor list. No <em>"I trust the AI"</em> hand-waving — they see
|
||||
exactly which EIN came from which duplicate row.
|
||||
</p>
|
||||
<div class="callout">
|
||||
<strong>Real example:</strong> the demo above merged 24 records into
|
||||
8 vendors and backfilled 7 EINs. The dedup audit lists every vendor
|
||||
group with the survivor, its merged-in duplicates, and the source row
|
||||
each recovered EIN was pulled from. The standardize audit lists every
|
||||
phone, amount, and Tax ID it reformatted.
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- ============= Format handling ============= -->
|
||||
<section>
|
||||
<div class="container">
|
||||
<div class="eyebrow">If your vendors are messy — most AP files are</div>
|
||||
<h2>EINs, phones, addresses, and amounts in every shape.</h2>
|
||||
<p>
|
||||
One row has the EIN as <code>12-3456789</code>, another as
|
||||
<code>123456789</code>. The remittance phone is <code>(212)
|
||||
555-0147</code> on one and <code>212.555.0147</code> on the next.
|
||||
An amount reads <code>$12,410.75</code> with a stray space. Excel
|
||||
treats half of these as text errors. DataTools normalizes every one —
|
||||
EINs to a single format, phones to E.164, amounts to clean numerics —
|
||||
so the file reconciles and the 1099 box totals tie out.
|
||||
</p>
|
||||
<ul class="bullets">
|
||||
<li><strong>EIN / Tax-ID normalization</strong> to one consistent <code>NN-NNNNNNN</code> shape, with genuinely-missing ones flagged.</li>
|
||||
<li><strong>Phone standardization</strong> to E.164 via Google's libphonenumber.</li>
|
||||
<li><strong>Amount parsing</strong> for <code>$</code> / commas / stray spaces — including amounts Excel mis-types as text.</li>
|
||||
<li><strong>Address shape detection</strong> for US remittance addresses.</li>
|
||||
</ul>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- ============= What you get ============= -->
|
||||
<section>
|
||||
<div class="container">
|
||||
<div class="eyebrow">In the bundle</div>
|
||||
<h2>Six tools. One pipeline. One $49 download.</h2>
|
||||
<div class="grid">
|
||||
<div class="card"><h3>1 · Find Duplicates</h3><p>Fuzzy match (Jaro-Winkler), 5 normalizers, survivor rules, gap-backfill merge, interactive review.</p></div>
|
||||
<div class="card"><h3>2 · Clean Text</h3><p>Whitespace, smart chars, NBSP, BOM, line endings, case ops.</p></div>
|
||||
<div class="card"><h3>3 · Standardize Formats</h3><p>EINs, amounts, dates, phones, emails, addresses, names, booleans.</p></div>
|
||||
<div class="card"><h3>4 · Fix Missing Values</h3><p>Disguised-null detection, profile, flag genuinely-missing fields, drop strategies.</p></div>
|
||||
<div class="card"><h3>5 · Map Columns</h3><p>Fuzzy auto-rename, target schema, type coercion, required-field defaults.</p></div>
|
||||
<div class="card"><h3>6 · Automated Workflows</h3><p>Chain tools in recommended order, save/load JSON, automate next year's vendor cleanup.</p></div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- ============= Pricing ============= -->
|
||||
<section>
|
||||
<div class="container">
|
||||
<div class="eyebrow">Pricing — pay once, own it</div>
|
||||
<h2>$49. No subscription. No ceiling on rows or files.</h2>
|
||||
<div class="pricing">
|
||||
<div class="card featured">
|
||||
<div class="row"><div class="price">$49</div><div class="price-suffix">one-time</div></div>
|
||||
<h3>DataTools for 1099 Prep</h3>
|
||||
<ul>
|
||||
<li>All 6 tools, full pipeline</li>
|
||||
<li>Mac · Windows · Linux installers</li>
|
||||
<li>Code-signed (no Gatekeeper warnings)</li>
|
||||
<li>Free updates for the v1.x line</li>
|
||||
<li>Bonus: ready-made vendor-master & 1099 pipelines</li>
|
||||
</ul>
|
||||
<a class="btn btn-large" href="https://gumroad.com/l/datatools?from=ap-1099" rel="noopener">Buy on Gumroad →</a>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div class="row"><div class="price">$149</div><div class="price-suffix">one-time</div></div>
|
||||
<h3>Full DataTools Suite</h3>
|
||||
<p class="muted">Available when 3+ bundles ship. Includes everything in the 1099-prep pack plus the Bookkeeper and Accounts-Receivable bundles. Save $48.</p>
|
||||
<a class="btn btn-ghost btn-large" href="#" aria-disabled="true">Coming when ready</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- ============= FAQ ============= -->
|
||||
<section>
|
||||
<div class="container">
|
||||
<h2>Questions</h2>
|
||||
|
||||
<details class="faq">
|
||||
<summary>Does this work with my QuickBooks vendor export?</summary>
|
||||
<p>Yes — the input is just CSV / Excel from any source. Your QuickBooks vendor export works the same as a Xero export, a Bill.com download, or a vendor spreadsheet you maintain by hand. The cleaner doesn't care where the file came from.</p>
|
||||
</details>
|
||||
|
||||
<details class="faq">
|
||||
<summary>How does this compare to Excel's "Remove Duplicates"?</summary>
|
||||
<p>Excel does <em>exact</em> deduplication and only deletes — it never backfills. <code>Acme LLC</code> and <code>Acme, L.L.C.</code> are different vendors to Excel, and even when it does catch a duplicate it throws the extra row away, taking the EIN with it. DataTools fuzzy-matches across spelling drift, merges the group to one survivor, and pulls the missing EIN, phone, and address off the rows it merges in.</p>
|
||||
</details>
|
||||
|
||||
<details class="faq">
|
||||
<summary>How does it recover a missing EIN?</summary>
|
||||
<p>When it merges a group of duplicate vendor rows, it keeps the survivor and backfills any empty field — including the EIN — from whichever duplicate row had it. In the sample file, 7 of the 8 vendors had their EIN recovered this way; the 1 that's truly missing gets flagged so you know to chase the W-9.</p>
|
||||
</details>
|
||||
|
||||
<details class="faq">
|
||||
<summary>Do I need to know Python to use it?</summary>
|
||||
<p>No. The GUI is a browser interface that opens automatically when you double-click the app. It loads your vendor file, you click Run, you download the cleaned list. The CLI is there for power users who want to script next year's cleanup.</p>
|
||||
</details>
|
||||
|
||||
<details class="faq">
|
||||
<summary>What about my data privacy?</summary>
|
||||
<p>Your vendor master — EINs, remittance addresses, payment history — never leaves your computer. There is no cloud component, no telemetry, no "anonymous usage stats." When the app is running you can confirm zero outbound network requests in your browser's developer tools.</p>
|
||||
</details>
|
||||
|
||||
<details class="faq">
|
||||
<summary>What's your refund policy?</summary>
|
||||
<p>Try the live demo above on the sample vendor dataset before you buy. If you still find DataTools doesn't fit your workflow within 14 days, email for a refund — no questions asked.</p>
|
||||
</details>
|
||||
|
||||
<details class="faq">
|
||||
<summary>Will there be updates?</summary>
|
||||
<p>Yes. The v1.x line is included free for everyone who buys DataTools today. We ship a patch every 30 days adding format support, edge-case fixes, and small features.</p>
|
||||
</details>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- ============= Final CTA ============= -->
|
||||
<section>
|
||||
<div class="container" style="text-align: center;">
|
||||
<h2>Stop chasing scattered EINs by hand.</h2>
|
||||
<p class="lead" style="margin: 0 auto 28px;">One $49 download. Mac, Windows, or Linux. Runs offline. Consolidates 24 messy records into 8 complete vendors, recovers the 7 EINs hiding on duplicate rows, flags the ones genuinely missing, and saves a pipeline you can re-run on next year's vendor export.</p>
|
||||
<a class="btn btn-large" href="https://gumroad.com/l/datatools?from=ap-1099" rel="noopener">Get DataTools for Accounting — $49 →</a>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- ============= Footer ============= -->
|
||||
<footer>
|
||||
<div class="container">
|
||||
<div>
|
||||
<p><strong>DataTools</strong> — local data-cleaning for accounts payable, bookkeepers, and accounts-receivable teams.</p>
|
||||
<p class="muted">© 2026 · Built solo · Shipped from a small office.</p>
|
||||
</div>
|
||||
<div>
|
||||
<p>
|
||||
<a href="../bookkeeper/">For bookkeepers</a> ·
|
||||
<a href="../ar-aging/">For accounts receivable</a><br />
|
||||
<a href="https://gumroad.com/l/datatools?from=ap-1099">Buy on Gumroad</a> ·
|
||||
<a href="mailto:hello@datatools.app">Email support</a>
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
358
landing/ar-aging/index.html
Normal file
358
landing/ar-aging/index.html
Normal file
@@ -0,0 +1,358 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>DataTools for Accounts Receivable — Kill Duplicate Invoices Inflating Your AR Aging Report · $49</title>
|
||||
<meta name="description" content="One tool to clean your open-invoices export: standardize invoice dates, due dates, and amounts, lowercase client emails, then remove double-entered invoice numbers so your AR aging report is accurate. 26 rows → 21, five duplicate invoices removed. Fully offline. $49 one-time." />
|
||||
<meta name="keywords" content="accounts receivable aging, duplicate invoices, AR cleanup, open invoices export, invoice dedupe, aging report accuracy, receivables csv tool" />
|
||||
<link rel="canonical" href="https://datatools.app/ar-aging/" />
|
||||
<link rel="stylesheet" href="../_shared/styles.css" />
|
||||
|
||||
<!-- Persona accent: Accounts Receivable → receivables green -->
|
||||
<style>
|
||||
:root {
|
||||
--accent: #059669;
|
||||
--accent-ink: #03241a;
|
||||
}
|
||||
</style>
|
||||
|
||||
<meta property="og:title" content="DataTools for Accounts Receivable — Kill Duplicate Invoices Inflating Your AR Aging Report" />
|
||||
<meta property="og:description" content="Standardize invoice dates, due dates, and amounts, lowercase client emails, then dedupe double-entered invoices — one tool, no upload. $49 one-time." />
|
||||
<meta property="og:type" content="product" />
|
||||
<meta property="og:url" content="https://datatools.app/ar-aging/" />
|
||||
|
||||
<script type="application/ld+json">
|
||||
{
|
||||
"@context": "https://schema.org",
|
||||
"@type": "SoftwareApplication",
|
||||
"name": "DataTools for Accounts Receivable",
|
||||
"operatingSystem": "Windows, macOS, Linux",
|
||||
"applicationCategory": "BusinessApplication",
|
||||
"offers": {
|
||||
"@type": "Offer",
|
||||
"price": "49",
|
||||
"priceCurrency": "USD"
|
||||
},
|
||||
"description": "Clean and dedupe your open-invoices export so the AR aging report is accurate. Standardize invoice dates, due dates, and amounts, lowercase client emails, then remove double-entered invoice numbers — backfilling a blank status from its twin row. Six-tool data-cleaning bundle for accounts receivable and accounting teams.",
|
||||
"softwareVersion": "1.0"
|
||||
}
|
||||
</script>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="buybar">
|
||||
<div class="buybar-inner">
|
||||
<div class="brand"><span class="brand-mark">●</span> DataTools <span class="muted">/ for Accounts Receivable</span></div>
|
||||
<div>
|
||||
<span class="price-tag">$49 — one-time, no subscription</span>
|
||||
<a class="btn" href="https://gumroad.com/l/datatools?from=ar-aging" rel="noopener">Get DataTools →</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<section class="hero">
|
||||
<div class="container">
|
||||
<div class="eyebrow">For accounts receivable · controllers · collections · accounting teams</div>
|
||||
<h1>Stop chasing the invoices<br /><strong>your aging report counted twice.</strong></h1>
|
||||
<p class="lead">
|
||||
The same invoice number gets posted twice — once as
|
||||
<code>3/04/2026</code> for <code>$1,250.00</code>, again as
|
||||
<code>2026-03-04</code> for <code>1250</code> — so your AR aging
|
||||
report double-counts the receivable and your team chases a balance
|
||||
that was never really open. DataTools standardizes every invoice
|
||||
date, due date, and amount, lowercases client emails, then removes
|
||||
the double-entered invoice numbers — taking a real open-invoices
|
||||
export from <strong>26 rows to 21, five duplicate invoices
|
||||
removed</strong> — all on your own machine, with nothing uploaded.
|
||||
</p>
|
||||
<div class="cta-row">
|
||||
<a class="btn btn-large" href="https://gumroad.com/l/datatools?from=ar-aging" rel="noopener">Get DataTools for Accounting — $49 →</a>
|
||||
<a class="btn btn-ghost btn-large" href="#demo">Try the live demo ↓</a>
|
||||
<span class="price-note">One-time payment · cross-platform · runs offline</span>
|
||||
</div>
|
||||
<div class="stats">
|
||||
<div class="stat"><div class="num">26→21</div><div class="label">rows after dedupe</div></div>
|
||||
<div class="stat"><div class="num">5</div><div class="label">duplicate invoices removed</div></div>
|
||||
<div class="stat"><div class="num">0</div><div class="label">cloud uploads ever</div></div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- ============= Pain points ============= -->
|
||||
<section>
|
||||
<div class="container">
|
||||
<div class="eyebrow">If your last aging report didn't tie out to cash</div>
|
||||
<h2>Five pains DataTools fixes before you run the aging report</h2>
|
||||
<div class="grid">
|
||||
<div class="card">
|
||||
<span class="icon">💸</span>
|
||||
<h3>Double-entered invoices inflate every aging bucket</h3>
|
||||
<p>The same invoice number posted twice — once in <code>MM/DD/YYYY</code>, once in ISO — lands in two rows and gets counted twice. Your 60-day bucket looks worse than it is, and the receivables total overstates what's actually owed.</p>
|
||||
<p class="muted"><strong>What it costs:</strong> overstated AR, a balance sheet that won't reconcile, and a controller asking why.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<span class="icon">📞</span>
|
||||
<h3>Collections chases invoices that were already paid or never real</h3>
|
||||
<p>When a duplicate invoice number shows as still-open, a collector emails the client about a balance that doesn't exist. The client pushes back, trust erodes, and your team burns a morning untangling it.</p>
|
||||
<p class="muted"><strong>What it costs:</strong> wasted collections hours + an awkward "please disregard" to the client.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<span class="icon">⚖️</span>
|
||||
<h3>Uploading the AR ledger to a cloud cleaner is a compliance headache</h3>
|
||||
<p>Every cloud-based cleaner wants you to upload your full receivables ledger — client names, amounts, contact emails. That's a data-handling review your firm doesn't want to run. DataTools is desktop-only — no upload, no DPA, no review.</p>
|
||||
<p class="muted"><strong>What it costs:</strong> weeks of review per tool, or just not cleaning the data at all.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<span class="icon">🗓️</span>
|
||||
<h3>Mixed date formats make due dates and aging unreliable</h3>
|
||||
<p>Invoice dates arrive as <code>3/4/26</code>, <code>2026-03-04</code>, and <code>Mar 4 2026</code>; due dates are just as mixed. Sort by date and the buckets are wrong, so the wrong invoices show up in the wrong aging column.</p>
|
||||
<p class="muted"><strong>What it costs:</strong> 1–3 hours per close reconciling dates by hand, every period.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<span class="icon">📧</span>
|
||||
<h3>Messy client contacts break your remittance reminders</h3>
|
||||
<p>Client names come in mixed casing and emails arrive as <code>Billing@ClientCo.com</code> in one row and <code>billing@clientco.com</code> in another — so the same client looks like two, and reminders go out twice or not at all.</p>
|
||||
<p class="muted"><strong>What it costs:</strong> duplicate dunning, missed reminders, and a client list that won't group.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<span class="icon">❓</span>
|
||||
<h3>Blank invoice statuses hide whether a receivable is really open</h3>
|
||||
<p>When one of the two twin rows has a blank status, you can't tell if the invoice is open, partial, or paid — so it either gets dropped from the aging report or counted at the wrong stage.</p>
|
||||
<p class="muted"><strong>What it costs:</strong> misclassified receivables and an aging report you can't trust.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section id="demo">
|
||||
<div class="container">
|
||||
<div class="eyebrow">Live demo · runs in your browser</div>
|
||||
<h2>Try it on a real-looking open-invoices export</h2>
|
||||
<p>
|
||||
The demo below loads a 26-row open-invoices export with five
|
||||
double-entered invoice numbers — the same invoice posted twice in
|
||||
different date and amount formats (<code>3/04/2026</code> vs
|
||||
<code>2026-03-04</code>, <code>$1,250.00</code> vs <code>1250</code>),
|
||||
client emails in mixed case, and one blank invoice status. Click
|
||||
<strong>Run pipeline</strong> and watch the 5-step pipeline (text
|
||||
clean → format → missing → column map → dedup) standardize both date
|
||||
columns to ISO, coerce amounts to numbers, lowercase the emails, and
|
||||
collapse 26 rows to 21 — backfilling the blank status from its twin
|
||||
row so the aging report is accurate.
|
||||
</p>
|
||||
<div class="demo-frame">
|
||||
<iframe
|
||||
src="https://demo.datatools.app/?p=ar-aging"
|
||||
loading="lazy"
|
||||
title="DataTools live demo — Accounts Receivable"
|
||||
sandbox="allow-scripts allow-same-origin allow-downloads allow-forms"></iframe>
|
||||
<div class="demo-caption">
|
||||
Demo runs on free hosting. Capped at 100 input rows · output
|
||||
watermarked. The paid product has no caps and runs entirely offline.
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<div class="container">
|
||||
<div class="eyebrow">Built for the receivables close</div>
|
||||
<h2>Three workflows you do every period</h2>
|
||||
<div class="grid">
|
||||
<div class="card">
|
||||
<span class="icon">🪢</span>
|
||||
<h3>Dedupe double-entered invoices</h3>
|
||||
<p>Match on invoice number, drop the second posting, and keep one canonical row per invoice — backfilling a blank status, due date, or amount from its twin so nothing accurate is lost when the duplicate goes.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<span class="icon">🗓️</span>
|
||||
<h3>Standardize invoice and due dates</h3>
|
||||
<p>Coerce every invoice date and due date to ISO and every amount to a clean number, so the aging buckets sort correctly and the receivables total ties out to the ledger.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<span class="icon">📧</span>
|
||||
<h3>Normalize client contacts for remittance</h3>
|
||||
<p>Lowercase client emails and fix name casing so each client groups as one. Send remit-to reminders once, to a clean contact list — not twice because two rows looked like two clients.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<div class="container">
|
||||
<div class="eyebrow">If your export comes from QuickBooks, Xero, or a billing system</div>
|
||||
<h2>Standardized dates and amounts. One row per invoice.</h2>
|
||||
<p>
|
||||
Your billing system exports <code>3/04/2026</code>. The re-post of
|
||||
the same invoice has <code>2026-03-04</code>. The amount is
|
||||
<code>$1,250.00</code> in one row and <code>1250</code> in the other.
|
||||
DataTools reads each row, normalizes both date columns to ISO,
|
||||
coerces the amount to a number, and then matches on invoice number
|
||||
to keep exactly one canonical row per receivable.
|
||||
</p>
|
||||
<ul class="bullets">
|
||||
<li><strong>Invoice date + due date</strong> both standardized to ISO, so every aging bucket sorts and totals correctly.</li>
|
||||
<li><strong>Amounts coerced to numbers</strong>: <code>$1,250.00</code> and <code>1250</code> resolve to the same value — no false mismatch between twin rows.</li>
|
||||
<li><strong>Client emails lowercased</strong> so the same client groups as one for remittance reminders.</li>
|
||||
<li><strong>Status backfill on dedupe</strong>: when a twin row has a blank invoice status, the survivor inherits it — so no open receivable goes missing from the report.</li>
|
||||
</ul>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<div class="container">
|
||||
<div class="eyebrow">For anyone who reports on receivables</div>
|
||||
<h2>Every duplicate invoice you don't catch overstates your AR.</h2>
|
||||
<p>
|
||||
Your aging report is only as good as the export under it. Every
|
||||
double-entered invoice number is a receivable counted twice — it
|
||||
inflates the aging buckets, overstates the total owed, and sends
|
||||
collections after balances that aren't really open. DataTools
|
||||
catches them once, before the report runs, by matching on invoice
|
||||
number with the date and amount noise already standardized away.
|
||||
</p>
|
||||
<div class="callout">
|
||||
<strong>Real numbers from the demo:</strong> a 26-row open-invoices
|
||||
export collapses to 21 — that's five double-entered invoices the
|
||||
mixed date and amount formats were hiding, both date columns now
|
||||
ISO, amounts numeric, emails lowercased, 0 unparseable, and a blank
|
||||
status backfilled from its twin row. The aging report finally ties out.
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<div class="container">
|
||||
<div class="eyebrow">The thing every cloud cleaner can't say</div>
|
||||
<h2>Your clients' receivables never leave your computer.</h2>
|
||||
<p>
|
||||
Cloud cleaning tools require you to upload your AR ledger — client
|
||||
names, invoice amounts, remit-to contacts. That ledger is sensitive
|
||||
client financial data, and once it's on someone else's server, your
|
||||
firm owns a data-handling problem you didn't need. DataTools is a
|
||||
desktop app. There is no upload step.
|
||||
</p>
|
||||
<div class="terminal"><span class="prompt">$</span> python -m src.cli_pipeline ar_open_invoices.csv --pipeline ar_open_invoices_pipeline.json --apply
|
||||
Reading ar_open_invoices.csv...
|
||||
26 rows, 9 columns
|
||||
Executing pipeline:
|
||||
<span class="ok">✓</span> text_clean (40 ms) {cells_changed: 31}
|
||||
<span class="ok">✓</span> format_standardize (120 ms) {dates_to_iso: 41, amounts_to_number: 26, emails_lowercased: 18}
|
||||
<span class="ok">✓</span> missing (30 ms) {sentinels_standardized: 4, status_backfilled: 1}
|
||||
<span class="ok">✓</span> column_map (20 ms) {columns_renamed: 2}
|
||||
<span class="ok">✓</span> dedup (60 ms) {duplicate_invoices_removed: 5, merged: 5}
|
||||
|
||||
Initial rows: 26 → Final rows: 21
|
||||
Unparseable dates/amounts: 0
|
||||
Total elapsed: 0.3 s
|
||||
<span class="prompt">$</span> # 5 double-entered invoices gone. aging report ties out. for $49.</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<div class="container">
|
||||
<div class="eyebrow">In the bundle</div>
|
||||
<h2>Six tools. One pipeline. One $49 download.</h2>
|
||||
<div class="grid">
|
||||
<div class="card"><h3>1 · Find Duplicates</h3><p>Match on invoice number; keep one canonical row per receivable and backfill blanks from the twin.</p></div>
|
||||
<div class="card"><h3>2 · Clean Text</h3><p>Smart quotes from copy-paste, NBSP from spreadsheet exports, BOM from Excel.</p></div>
|
||||
<div class="card"><h3>3 · Standardize Formats</h3><p>Invoice and due dates to ISO, amounts to clean numbers, client emails lowercased.</p></div>
|
||||
<div class="card"><h3>4 · Fix Missing Values</h3><p>Detect <code>TBD</code>, <code>(unknown)</code>, <code>—</code> and backfill blank invoice statuses on dedupe.</p></div>
|
||||
<div class="card"><h3>5 · Map Columns</h3><p>Project to your aging-report schema, coerce amount to a number, reorder fields for import.</p></div>
|
||||
<div class="card"><h3>6 · Automated Workflows</h3><p>Save the cleanup as JSON. Drop next period's open-invoices export on it. Same dedupe, automated.</p></div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<div class="container">
|
||||
<div class="eyebrow">Pricing — pay once, own it</div>
|
||||
<h2>$49. No subscription. No per-close fee.</h2>
|
||||
<div class="pricing">
|
||||
<div class="card featured">
|
||||
<div class="row"><div class="price">$49</div><div class="price-suffix">one-time</div></div>
|
||||
<h3>DataTools for Accounts Receivable</h3>
|
||||
<ul>
|
||||
<li>All 6 tools, full pipeline</li>
|
||||
<li>Mac · Windows · Linux installers</li>
|
||||
<li>Code-signed (no Gatekeeper warnings)</li>
|
||||
<li>Free updates for the v1.x line</li>
|
||||
<li>Bonus: open-invoices dedupe pipeline preset</li>
|
||||
<li><strong>Use on any number of clients</strong> — no seat limits</li>
|
||||
</ul>
|
||||
<a class="btn btn-large" href="https://gumroad.com/l/datatools?from=ar-aging" rel="noopener">Buy on Gumroad →</a>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div class="row"><div class="price">$149</div><div class="price-suffix">one-time</div></div>
|
||||
<h3>Full DataTools Suite</h3>
|
||||
<p class="muted">Available when 3+ bundles ship. Includes everything in the Accounts Receivable pack plus the Bookkeeper and Accounts Payable / 1099 bundles. Save $48.</p>
|
||||
<a class="btn btn-ghost btn-large" href="#" aria-disabled="true">Coming when ready</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<div class="container">
|
||||
<h2>Questions</h2>
|
||||
|
||||
<details class="faq">
|
||||
<summary>Does this replace my accounting system's deduplication?</summary>
|
||||
<p>No — it cleans the export <em>before</em> you run the aging report or import it back. Most billing systems will happily hold two postings of the same invoice number; DataTools catches the double-entered invoice so it never inflates a single aging bucket.</p>
|
||||
</details>
|
||||
|
||||
<details class="faq">
|
||||
<summary>How does it know two rows are the same invoice?</summary>
|
||||
<p>It matches on invoice number after the date and amount formats are standardized away. So a posting dated <code>3/04/2026</code> for <code>$1,250.00</code> and its twin dated <code>2026-03-04</code> for <code>1250</code> are recognized as one invoice — and only one canonical row survives.</p>
|
||||
</details>
|
||||
|
||||
<details class="faq">
|
||||
<summary>What happens to a blank invoice status when the duplicate is removed?</summary>
|
||||
<p>It's backfilled. If one twin row has a blank status and the other says <code>open</code>, the surviving row inherits <code>open</code> — so no real receivable drops off the aging report just because the duplicate carried the better data.</p>
|
||||
</details>
|
||||
|
||||
<details class="faq">
|
||||
<summary>Can I use it on multiple clients without paying again?</summary>
|
||||
<p>Yes. The licence is per-operator, not per-client. Run it on every client's open-invoices export for the same $49.</p>
|
||||
</details>
|
||||
|
||||
<details class="faq">
|
||||
<summary>What's the audit trail look like?</summary>
|
||||
<p>A row-by-row CSV: every modified cell with its original value, new value, and which rule fired — every date coerced to ISO, every amount normalized, every duplicate invoice removed. A separate JSON file describes the pipeline that produced it, so the cleanup reproduces deterministically and your client can verify it on their machine.</p>
|
||||
</details>
|
||||
|
||||
<details class="faq">
|
||||
<summary>What's your refund policy?</summary>
|
||||
<p>Try the live demo above on the sample open-invoices export before you buy. If DataTools doesn't fit your workflow within 14 days, email for a refund — no questions asked.</p>
|
||||
</details>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<div class="container" style="text-align: center;">
|
||||
<h2>Stop counting the same receivable twice.</h2>
|
||||
<p class="lead" style="margin: 0 auto 28px;">One $49 download. Standardizes invoice dates, due dates, and amounts, lowercases client emails, removes the double-entered invoices your aging report was counting twice, and saves a pipeline you can re-run on next period's open-invoices export.</p>
|
||||
<a class="btn btn-large" href="https://gumroad.com/l/datatools?from=ar-aging" rel="noopener">Get DataTools for Accounting — $49 →</a>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<footer>
|
||||
<div class="container">
|
||||
<div>
|
||||
<p><strong>DataTools</strong> — local data-cleaning for bookkeepers, accounts payable, and accounts receivable teams.</p>
|
||||
<p class="muted">© 2026 · Built solo · Shipped from a small office.</p>
|
||||
</div>
|
||||
<div>
|
||||
<p>
|
||||
<a href="../bookkeeper/">For bookkeepers</a> ·
|
||||
<a href="../ap-1099/">For accounts payable / 1099</a><br />
|
||||
<a href="https://gumroad.com/l/datatools?from=ar-aging">Buy on Gumroad</a> ·
|
||||
<a href="mailto:hello@datatools.app">Email support</a>
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
@@ -3,9 +3,9 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>DataTools for Bookkeepers — Reconcile Bank Exports With An Audit Trail · $49</title>
|
||||
<meta name="description" content="Reconcile messy bank exports. Catch duplicate transactions QuickBooks imported twice. Standardize dates, amounts, and vendor casing — locally. Every change auditable. $49 one-time." />
|
||||
<meta name="keywords" content="reconcile bank export csv, quickbooks duplicate transactions, vendor list cleanup, bookkeeper csv tool, bank export deduplicator, bookkeeper audit trail" />
|
||||
<title>DataTools for Bookkeepers — Catch Bank Transactions Posted Twice · $49</title>
|
||||
<meta name="description" content="Catch the transactions your bank export posted twice. Standardize every date to ISO and every amount to numeric, then dedup on the real transaction so the reconciliation ties out — with a row-level audit trail. $49 one-time." />
|
||||
<meta name="keywords" content="bank reconciliation, duplicate transactions, bank export csv cleanup, QuickBooks reconcile, bookkeeper csv tool" />
|
||||
<link rel="canonical" href="https://datatools.app/bookkeeper/" />
|
||||
<link rel="stylesheet" href="../_shared/styles.css" />
|
||||
|
||||
@@ -18,8 +18,8 @@
|
||||
</style>
|
||||
|
||||
<!-- Open Graph -->
|
||||
<meta property="og:title" content="DataTools for Bookkeepers — Reconcile Bank Exports With An Audit Trail" />
|
||||
<meta property="og:description" content="Catch duplicate transactions. Standardize dates and amounts. Hand your client an audit trail. $49 one-time." />
|
||||
<meta property="og:title" content="DataTools for Bookkeepers — Catch Bank Transactions Posted Twice" />
|
||||
<meta property="og:description" content="The same payment posts twice in two date/amount formats and a plain dedupe misses it. DataTools standardizes, dedups on the real transaction, and hands you an audit trail. $49 one-time." />
|
||||
<meta property="og:type" content="product" />
|
||||
<meta property="og:url" content="https://datatools.app/bookkeeper/" />
|
||||
|
||||
@@ -35,7 +35,7 @@
|
||||
"price": "49",
|
||||
"priceCurrency": "USD"
|
||||
},
|
||||
"description": "Reconcile bank exports, dedupe vendor lists, and produce a hand-off-ready audit trail. Six-tool data-cleaning bundle for bookkeepers and freelance accountants.",
|
||||
"description": "Catch the duplicate transactions your bank export posted twice across overlapping months, standardize dates and amounts, and produce a hand-off-ready audit trail. Six-tool data-cleaning bundle for bookkeepers and freelance accountants.",
|
||||
"softwareVersion": "1.0"
|
||||
}
|
||||
</script>
|
||||
@@ -47,7 +47,7 @@
|
||||
<div class="brand"><span class="brand-mark">●</span> DataTools <span class="muted">/ for Bookkeepers</span></div>
|
||||
<div>
|
||||
<span class="price-tag">$49 — one-time, no subscription</span>
|
||||
<a class="btn" href="https://gumroad.com/l/datatools?from=bookkeeper" rel="noopener">Get DataTools →</a>
|
||||
<a class="btn" href="https://gumroad.com/l/datatools?from=bookkeeper" rel="noopener">Get DataTools for Bookkeepers — $49 →</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -55,24 +55,29 @@
|
||||
<section class="hero">
|
||||
<div class="container">
|
||||
<div class="eyebrow">For bookkeepers · freelance accountants · small-firm partners</div>
|
||||
<h1>Reconcile messy bank exports.<br /><strong>Hand your client an audit trail.</strong></h1>
|
||||
<h1>Catch the transactions your bank export<br /><strong>posted twice.</strong></h1>
|
||||
<p class="lead">
|
||||
The Jan and Feb exports overlap and you've got the same transaction
|
||||
booked twice. Vendor names are <em>"Amazon"</em>, <em>"amazon.com"</em>,
|
||||
and <em>"AMAZON.COM*4F2X9"</em> in three different rows. Dates are a
|
||||
smoosh of <code>01/15/2025</code>, <code>2025-01-15</code>, and
|
||||
<code>Jan 18 2025</code>. DataTools fixes all of it in one pass —
|
||||
and produces a row-by-row CSV showing every change so your client
|
||||
can verify your work.
|
||||
The Jan and Feb exports overlap, so the <em>same</em> payment posts
|
||||
twice in two different shapes — <code>01/15/2025 +$3,450.00</code>
|
||||
in one export and <code>2025-01-15 3450.00</code> in the
|
||||
other — and a plain Excel dedupe never catches it because the dates and
|
||||
amounts don't match character-for-character. DataTools standardizes
|
||||
every date to ISO and every amount to numeric (parens-negatives
|
||||
resolved), then dedups on the <em>real</em> transaction so the
|
||||
reconciliation ties out. On the sample export that's
|
||||
<strong>26 rows → 20</strong> — six phantom duplicate transactions
|
||||
removed, 36 date/amount cells standardized, 0 unparseable — and you
|
||||
get a row-by-row CSV showing every change so your client can verify
|
||||
your work.
|
||||
</p>
|
||||
<div class="cta-row">
|
||||
<a class="btn btn-large" href="https://gumroad.com/l/datatools?from=bookkeeper" rel="noopener">Get DataTools — $49 →</a>
|
||||
<a class="btn btn-large" href="https://gumroad.com/l/datatools?from=bookkeeper" rel="noopener">Get DataTools for Bookkeepers — $49 →</a>
|
||||
<a class="btn btn-ghost btn-large" href="#demo">Try the live demo ↓</a>
|
||||
<span class="price-note">One-time payment · cross-platform · runs offline</span>
|
||||
</div>
|
||||
<div class="stats">
|
||||
<div class="stat"><div class="num">6</div><div class="label">tools, one bundle</div></div>
|
||||
<div class="stat"><div class="num">100 %</div><div class="label">auditable changes</div></div>
|
||||
<div class="stat"><div class="num">26→20</div><div class="label">rows, on the sample export</div></div>
|
||||
<div class="stat"><div class="num">6</div><div class="label">phantom duplicates removed</div></div>
|
||||
<div class="stat"><div class="num">0</div><div class="label">cloud uploads ever</div></div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -129,13 +134,15 @@
|
||||
<div class="eyebrow">Live demo · runs in your browser</div>
|
||||
<h2>Try it on a sample bank export with a known overlap</h2>
|
||||
<p>
|
||||
The demo below loads a 25-row export combining January and February
|
||||
The demo below loads a 26-row export combining January and February
|
||||
activity, with the month-boundary rows duplicated across exports —
|
||||
the exact scenario where QuickBooks (or any reconciler) silently
|
||||
double-counts transactions. Click <strong>Run pipeline</strong> and
|
||||
watch the dedup catch every overlap, dates land in ISO format, and
|
||||
the parens-negative amounts (<code>($89.50)</code>) become proper
|
||||
negative numbers.
|
||||
watch it standardize 36 date/amount cells, land every date in ISO
|
||||
format, turn the parens-negative amounts (<code>($89.50)</code>) into
|
||||
proper negatives, flag the disguised-null categories, and dedup the
|
||||
export down to <strong>20 real transactions</strong> — six phantom
|
||||
duplicates removed, 0 unparseable.
|
||||
</p>
|
||||
<div class="demo-frame">
|
||||
<iframe
|
||||
@@ -197,13 +204,17 @@
|
||||
price. DataTools writes the audit by default, downloadable as a
|
||||
separate CSV alongside the cleaned file.
|
||||
</div>
|
||||
<div class="terminal"><span class="prompt">$</span> head -5 client_jan2025_changes.csv
|
||||
<div class="terminal"><span class="prompt">$</span> python -m src.cli_pipeline bank_reconciliation.csv --pipeline bank_reconciliation_pipeline.json --apply
|
||||
standardize · 36 date/amount cells normalized (ISO dates, numeric amounts, parens-negatives resolved)
|
||||
missing · disguised-null categories flagged (—, N/A, (blank))
|
||||
dedup · 6 phantom duplicate transactions removed
|
||||
rows · 26 → 20 · 0 unparseable
|
||||
✓ wrote bank_reconciliation.cleaned.csv + bank_reconciliation.changes.csv (row-level audit)
|
||||
<span class="prompt">$</span> head -4 bank_reconciliation.changes.csv
|
||||
row,column,field_type,old,new
|
||||
0,"Date ",date,"01/15/2025","2025-01-15"
|
||||
0,Description,name," AMAZON.COM*4F2X9 PURCHASE","Amazon.com*4F2X9 Purchase"
|
||||
0,Amount,currency,"-$129.99","-129.99"
|
||||
1,Date ,date,"2025-01-15","2025-01-15"
|
||||
<span class="prompt">$</span> # one row of audit per cell change. handed to the client. signed off.</div>
|
||||
0,Amount,currency,"+$3,450.00","3450.00"
|
||||
0,Category,category,"—","(missing)"
|
||||
</div>
|
||||
</section>
|
||||
|
||||
@@ -251,12 +262,12 @@ row,column,field_type,old,new
|
||||
<div class="eyebrow">In the bundle</div>
|
||||
<h2>Six tools. One pipeline. One $49 download.</h2>
|
||||
<div class="grid">
|
||||
<div class="card"><h3>1 · Deduplicator</h3><p>Fuzzy match (Jaro-Winkler), explicit strategies for Date+Amount+Vendor, survivor rules.</p></div>
|
||||
<div class="card"><h3>2 · Text Cleaner</h3><p>Header whitespace, smart quotes from copy-paste, em-dash sentinels.</p></div>
|
||||
<div class="card"><h3>3 · Format Standardizer</h3><p>ISO dates, numeric amounts (parens-negative), vendor casing, multi-currency.</p></div>
|
||||
<div class="card"><h3>4 · Missing Value Handler</h3><p>Disguised-null detection: <code>—</code>, <code>N/A</code>, <code>(blank)</code>, <code>?</code>.</p></div>
|
||||
<div class="card"><h3>5 · Column Mapper</h3><p>Project to your accounting tool's required schema, coerce types, drop extras.</p></div>
|
||||
<div class="card"><h3>6 · Pipeline Runner</h3><p>Save the cleanup. Run it on next month's export with one command. Same audit, automated.</p></div>
|
||||
<div class="card"><h3>1 · Find Duplicates</h3><p>Fuzzy match (Jaro-Winkler), explicit strategies for Date+Amount+Vendor, survivor rules.</p></div>
|
||||
<div class="card"><h3>2 · Clean Text</h3><p>Header whitespace, smart quotes from copy-paste, em-dash sentinels.</p></div>
|
||||
<div class="card"><h3>3 · Standardize Formats</h3><p>ISO dates, numeric amounts (parens-negative), vendor casing, multi-currency.</p></div>
|
||||
<div class="card"><h3>4 · Fix Missing Values</h3><p>Disguised-null detection: <code>—</code>, <code>N/A</code>, <code>(blank)</code>, <code>?</code>.</p></div>
|
||||
<div class="card"><h3>5 · Map Columns</h3><p>Project to your accounting tool's required schema, coerce types, drop extras.</p></div>
|
||||
<div class="card"><h3>6 · Automated Workflows</h3><p>Save the cleanup. Run it on next month's export with one command. Same audit, automated.</p></div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
@@ -336,13 +347,13 @@ row,column,field_type,old,new
|
||||
<footer>
|
||||
<div class="container">
|
||||
<div>
|
||||
<p><strong>DataTools</strong> — local data-cleaning for Shopify, bookkeepers, and RevOps teams.</p>
|
||||
<p><strong>DataTools</strong> — local data-cleaning for bookkeepers, accounts payable, and accounts receivable teams.</p>
|
||||
<p class="muted">© 2026 · Built solo · Shipped from a small office.</p>
|
||||
</div>
|
||||
<div>
|
||||
<p>
|
||||
<a href="../shopify-pet/">For Shopify operators</a> ·
|
||||
<a href="../revops/">For RevOps agencies</a><br />
|
||||
<a href="../ap-1099/">For accounts payable / 1099</a> ·
|
||||
<a href="../ar-aging/">For accounts receivable</a><br />
|
||||
<a href="https://gumroad.com/l/datatools?from=bookkeeper">Buy on Gumroad</a> ·
|
||||
<a href="mailto:hello@datatools.app">Email support</a>
|
||||
</p>
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
"gumroad_listing": "https://gumroad.com/l/datatools",
|
||||
"support_email": "hello@datatools.app",
|
||||
|
||||
"personas": ["shopify-pet", "bookkeeper", "revops"],
|
||||
"personas": ["bookkeeper", "ap-1099", "ar-aging"],
|
||||
|
||||
"_substitutions_made": [
|
||||
"{{site_origin}}/ → site_origin/",
|
||||
|
||||
@@ -7,9 +7,9 @@ to ``landing/deploy.config.json`` and filling in the real URLs:
|
||||
|
||||
Output:
|
||||
landing/dist/index.html
|
||||
landing/dist/shopify-pet/index.html
|
||||
landing/dist/bookkeeper/index.html
|
||||
landing/dist/revops/index.html
|
||||
landing/dist/ap-1099/index.html
|
||||
landing/dist/ar-aging/index.html
|
||||
landing/dist/_shared/styles.css
|
||||
landing/dist/robots.txt
|
||||
landing/dist/sitemap.xml
|
||||
@@ -50,9 +50,9 @@ EXAMPLE_PATH = LANDING / "deploy.config.example.json"
|
||||
# Files to substitute and copy. Order matters only for readability.
|
||||
HTML_PAGES = [
|
||||
LANDING / "index.html",
|
||||
LANDING / "shopify-pet" / "index.html",
|
||||
LANDING / "bookkeeper" / "index.html",
|
||||
LANDING / "revops" / "index.html",
|
||||
LANDING / "bookkeeper" / "index.html",
|
||||
LANDING / "ap-1099" / "index.html",
|
||||
LANDING / "ar-aging" / "index.html",
|
||||
]
|
||||
SHARED = LANDING / "_shared" / "styles.css"
|
||||
|
||||
@@ -125,7 +125,7 @@ def _stamp_sitemap(cfg: dict) -> str:
|
||||
site = cfg["site_origin"].rstrip("/")
|
||||
today = date.today().isoformat()
|
||||
urls = [site + "/"] + [
|
||||
f"{site}/{p}/" for p in cfg.get("personas", ["shopify-pet", "bookkeeper", "revops"])
|
||||
f"{site}/{p}/" for p in cfg.get("personas", ["bookkeeper", "ap-1099", "ar-aging"])
|
||||
]
|
||||
items = "\n".join(
|
||||
f" <url><loc>{u}</loc><lastmod>{today}</lastmod></url>"
|
||||
@@ -177,11 +177,11 @@ def _build_404_html(cfg: dict) -> str:
|
||||
<h1>That page isn't here.</h1>
|
||||
<p class="lead" style="margin: 0 auto 28px;">Pick a workflow below to land somewhere useful.</p>
|
||||
<p>
|
||||
<a class="btn" href="{site_origin}/shopify-pet/">For Shopify</a>
|
||||
|
||||
<a class="btn" href="{site_origin}/bookkeeper/">For bookkeepers</a>
|
||||
|
||||
<a class="btn" href="{site_origin}/revops/">For RevOps</a>
|
||||
<a class="btn" href="{site_origin}/ap-1099/">For AP / 1099</a>
|
||||
|
||||
<a class="btn" href="{site_origin}/ar-aging/">For AR</a>
|
||||
</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
@@ -3,13 +3,13 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>DataTools — Local CSV / Excel Cleaning for Shopify, Bookkeepers, and RevOps</title>
|
||||
<meta name="description" content="One desktop tool. Three workflows. Clean Shopify customer exports, reconcile messy bank statements, or dedupe lead lists across HubSpot and LinkedIn — all locally. $49 one-time." />
|
||||
<title>DataTools — Local CSV / Excel Cleaning for Bookkeepers and Accountants</title>
|
||||
<meta name="description" content="One desktop tool for messy accounting exports. Reconcile bank statements, build clean 1099 vendor lists, and de-duplicate AR aging — all locally. $49 one-time." />
|
||||
<link rel="canonical" href="https://datatools.app/" />
|
||||
<link rel="stylesheet" href="_shared/styles.css" />
|
||||
|
||||
<meta property="og:title" content="DataTools — Local CSV / Excel Cleaning" />
|
||||
<meta property="og:description" content="One desktop tool, three niche workflows. Runs entirely offline. $49 one-time." />
|
||||
<meta property="og:title" content="DataTools — Local CSV / Excel Cleaning for Accounting" />
|
||||
<meta property="og:description" content="Reconcile bank exports, prep 1099 vendor lists, clean AR aging — offline. $49 one-time." />
|
||||
<meta property="og:type" content="website" />
|
||||
<meta property="og:url" content="https://datatools.app/" />
|
||||
|
||||
@@ -38,9 +38,9 @@
|
||||
box-shadow: var(--shadow);
|
||||
text-decoration: none;
|
||||
}
|
||||
.persona-card.shopify { --card-accent: #6ee7b7; }
|
||||
.persona-card.bookkeeper{ --card-accent: #7dd3fc; }
|
||||
.persona-card.revops { --card-accent: #c4b5fd; }
|
||||
.persona-card.ap1099 { --card-accent: #fbbf24; }
|
||||
.persona-card.ar { --card-accent: #6ee7b7; }
|
||||
.persona-card .pill {
|
||||
display: inline-block;
|
||||
background: rgba(255,255,255,0.04);
|
||||
@@ -93,70 +93,69 @@
|
||||
|
||||
<section class="hero">
|
||||
<div class="container">
|
||||
<div class="eyebrow">For Shopify operators · bookkeepers · marketing & RevOps agencies</div>
|
||||
<h1>Local CSV / Excel cleaning.<br /><strong>One tool. Three workflows.</strong></h1>
|
||||
<div class="eyebrow">For bookkeepers · accounts payable · accounts receivable</div>
|
||||
<h1>Local CSV / Excel cleaning for accounting.<br /><strong>One tool. Three workflows.</strong></h1>
|
||||
<p class="lead">
|
||||
DataTools is a desktop app that fixes the data-cleaning headaches
|
||||
every small business hits — duplicates Excel can't catch,
|
||||
international phones it can't parse, dates and currencies in three
|
||||
different formats per export. One $49 download. Works on Mac,
|
||||
Windows, and Linux. <strong>Your data never leaves your
|
||||
computer.</strong>
|
||||
DataTools is a desktop app that fixes the export headaches that
|
||||
throw off your books — the transaction your bank posted twice,
|
||||
the vendor entered three ways at 1099 time, the invoice your aging
|
||||
report counted twice. One $49 download. Mac, Windows, and Linux.
|
||||
<strong>Your data never leaves your computer.</strong>
|
||||
</p>
|
||||
|
||||
<div class="persona-grid">
|
||||
<a class="persona-card shopify" href="shopify-pet/">
|
||||
<span class="pill">🛍️ Shopify operator</span>
|
||||
<h3>Customer / vendor / subscriber export cleanup</h3>
|
||||
<p>
|
||||
Klaviyo-import-ready customer lists in 30 seconds. Catches
|
||||
cross-device duplicates, standardizes international phones
|
||||
and addresses, fixes the disguised nulls that break product
|
||||
feeds.
|
||||
</p>
|
||||
<ul class="pain">
|
||||
<li>· Fix Klaviyo per-contact billing on phantom dupes</li>
|
||||
<li>· Repair feeds rejected by Google Merchant / Meta</li>
|
||||
<li>· Unify orders from Shopify + Etsy + Amazon + Faire</li>
|
||||
<li>· Resolve VAT-MOSS country-name drift</li>
|
||||
</ul>
|
||||
<span class="open">Open the Shopify demo & pricing</span>
|
||||
</a>
|
||||
|
||||
<a class="persona-card bookkeeper" href="bookkeeper/">
|
||||
<span class="pill">📒 Bookkeeper / accountant</span>
|
||||
<h3>Bank-export reconciliation with audit trail</h3>
|
||||
<span class="pill">📒 Bookkeeper</span>
|
||||
<h3>Bank reconciliation with an audit trail</h3>
|
||||
<p>
|
||||
Catches the duplicate transaction QuickBooks imported twice
|
||||
when Jan and Feb exports overlap. Standardizes dates,
|
||||
amounts, and vendor casing. Hands you a row-level audit log
|
||||
to share with the client.
|
||||
When the Jan and Feb exports overlap, the same payment posts
|
||||
twice in two formats. DataTools standardizes every date and
|
||||
amount, then dedups on the real transaction so it ties out —
|
||||
with a row-level audit log to hand the client.
|
||||
</p>
|
||||
<ul class="pain">
|
||||
<li>· Catch month-overlap re-import dupes</li>
|
||||
<li>· Consolidate vendors for clean 1099 reports</li>
|
||||
<li>· Produce hand-off-ready audit trail</li>
|
||||
<li>· Multi-currency books (EUR / GBP / BRL)</li>
|
||||
<li>· Catch month-overlap re-import duplicates</li>
|
||||
<li>· ISO dates, numeric amounts, parens-negatives resolved</li>
|
||||
<li>· Hand-off-ready audit trail</li>
|
||||
<li>· Sample: 26 rows → 20, six phantom duplicates removed</li>
|
||||
</ul>
|
||||
<span class="open">Open the bookkeeper demo & pricing</span>
|
||||
</a>
|
||||
|
||||
<a class="persona-card revops" href="revops/">
|
||||
<span class="pill">🪢 Marketing / RevOps</span>
|
||||
<h3>Lead-list dedup across HubSpot, LinkedIn, scrapes</h3>
|
||||
<a class="persona-card ap1099" href="ap-1099/">
|
||||
<span class="pill">🧾 Accounts payable / 1099</span>
|
||||
<h3>Clean 1099 vendor list — missing EINs filled in</h3>
|
||||
<p>
|
||||
One canonical lead per real person — across HubSpot,
|
||||
LinkedIn, Apollo, ZoomInfo, and manual scrapes.
|
||||
International phones (50+ country codes), per-row country
|
||||
column, fuzzy match with merge.
|
||||
The same vendor entered three times, each record holding only
|
||||
part of the details. DataTools consolidates each vendor to one
|
||||
row and backfills the gaps from the duplicates, so the EINs you
|
||||
need at filing time are recovered.
|
||||
</p>
|
||||
<ul class="pain">
|
||||
<li>· Stop paying HubSpot tier price for cross-source dupes</li>
|
||||
<li>· Protect sender reputation from invalid emails</li>
|
||||
<li>· Skip the 4–8 wk GDPR review on cloud cleaners</li>
|
||||
<li>· Suppression-list sync across 5+ platforms</li>
|
||||
<li>· Consolidate vendor masters for 1099-NEC</li>
|
||||
<li>· Recover EINs scattered across duplicate records</li>
|
||||
<li>· Standardize phones, emails, and amounts</li>
|
||||
<li>· Sample: 24 records → 8 vendors, 7 EINs recovered</li>
|
||||
</ul>
|
||||
<span class="open">Open the RevOps demo & pricing</span>
|
||||
<span class="open">Open the 1099 / AP demo & pricing</span>
|
||||
</a>
|
||||
|
||||
<a class="persona-card ar" href="ar-aging/">
|
||||
<span class="pill">💵 Accounts receivable</span>
|
||||
<h3>AR aging without the double-counted invoices</h3>
|
||||
<p>
|
||||
Double-entered invoices inflate your aging report and your
|
||||
follow-ups. DataTools standardizes invoice dates, due dates,
|
||||
and amounts, lowercases client emails, then removes the
|
||||
duplicate invoice numbers so the aging is accurate.
|
||||
</p>
|
||||
<ul class="pain">
|
||||
<li>· Remove double-entered invoices from the aging</li>
|
||||
<li>· ISO dates, numeric amounts, lowercased client emails</li>
|
||||
<li>· Backfill a blank status from its twin row</li>
|
||||
<li>· Sample: 26 rows → 21, five duplicate invoices removed</li>
|
||||
</ul>
|
||||
<span class="open">Open the AR demo & pricing</span>
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
@@ -168,9 +167,9 @@
|
||||
<h2>One engine. Same six tools. Same $49.</h2>
|
||||
<p>
|
||||
The persona pages above are positioning, not different products.
|
||||
Whichever you buy, you get the full bundle: Deduplicator, Text
|
||||
Cleaner, Format Standardizer, Missing-Value Handler, Column
|
||||
Mapper, and Pipeline Runner — pre-tuned with a saved pipeline
|
||||
Whichever you buy, you get the full bundle: Find Duplicates, Clean
|
||||
Text, Standardize Formats, Fix Missing Values, Map Columns,
|
||||
and Automated Workflows — pre-tuned with a saved pipeline
|
||||
that matches your workflow.
|
||||
</p>
|
||||
<div class="grid">
|
||||
@@ -218,14 +217,14 @@
|
||||
<footer>
|
||||
<div class="container">
|
||||
<div>
|
||||
<p><strong>DataTools</strong> — local data-cleaning for Shopify, bookkeepers, and RevOps teams.</p>
|
||||
<p><strong>DataTools</strong> — local data-cleaning for bookkeepers, accounts payable, and accounts receivable teams.</p>
|
||||
<p class="muted">© 2026 · Built solo · Shipped from a small office.</p>
|
||||
</div>
|
||||
<div>
|
||||
<p>
|
||||
<a href="shopify-pet/">For Shopify operators</a> ·
|
||||
<a href="bookkeeper/">For bookkeepers</a> ·
|
||||
<a href="revops/">For RevOps agencies</a><br />
|
||||
<a href="ap-1099/">For accounts payable / 1099</a> ·
|
||||
<a href="ar-aging/">For accounts receivable</a><br />
|
||||
<a href="mailto:hello@datatools.app">Email support</a>
|
||||
</p>
|
||||
</div>
|
||||
|
||||
@@ -1,352 +0,0 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>DataTools for RevOps — Dedupe Lead Lists Across HubSpot, LinkedIn, and Manual Scrapes · $49</title>
|
||||
<meta name="description" content="One tool to dedupe lead lists across HubSpot, LinkedIn, and manual scrapes. International phones (50+ country codes), per-row country normalization, fuzzy match across vendors, fully offline. $49 one-time." />
|
||||
<meta name="keywords" content="dedupe lead list, hubspot deduplicate, linkedin lead cleanup, marketing data cleaning, revops csv tool, multi-vendor lead unification, international phone normalization" />
|
||||
<link rel="canonical" href="https://datatools.app/revops/" />
|
||||
<link rel="stylesheet" href="../_shared/styles.css" />
|
||||
|
||||
<!-- Persona accent: RevOps → vivid violet -->
|
||||
<style>
|
||||
:root {
|
||||
--accent: #c4b5fd;
|
||||
--accent-ink: #2e1065;
|
||||
}
|
||||
</style>
|
||||
|
||||
<meta property="og:title" content="DataTools for RevOps — Dedupe Lead Lists Across HubSpot, LinkedIn, and Manual Scrapes" />
|
||||
<meta property="og:description" content="International phones, country normalization, fuzzy dedup with merge — one tool, no upload. $49 one-time." />
|
||||
<meta property="og:type" content="product" />
|
||||
<meta property="og:url" content="https://datatools.app/revops/" />
|
||||
|
||||
<script type="application/ld+json">
|
||||
{
|
||||
"@context": "https://schema.org",
|
||||
"@type": "SoftwareApplication",
|
||||
"name": "DataTools for RevOps",
|
||||
"operatingSystem": "Windows, macOS, Linux",
|
||||
"applicationCategory": "BusinessApplication",
|
||||
"offers": {
|
||||
"@type": "Offer",
|
||||
"price": "49",
|
||||
"priceCurrency": "USD"
|
||||
},
|
||||
"description": "Dedupe and unify lead lists across CRM, scraping, and manual sources. International phone normalization, per-row country, fuzzy match with merge. Six-tool data-cleaning bundle for RevOps and marketing agencies.",
|
||||
"softwareVersion": "1.0"
|
||||
}
|
||||
</script>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="buybar">
|
||||
<div class="buybar-inner">
|
||||
<div class="brand"><span class="brand-mark">●</span> DataTools <span class="muted">/ for RevOps</span></div>
|
||||
<div>
|
||||
<span class="price-tag">$49 — one-time, no subscription</span>
|
||||
<a class="btn" href="https://gumroad.com/l/datatools?from=revops" rel="noopener">Get DataTools →</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<section class="hero">
|
||||
<div class="container">
|
||||
<div class="eyebrow">For RevOps · marketing ops · agency lead-gen · audience-builders</div>
|
||||
<h1>Dedupe lead lists across HubSpot, LinkedIn,<br /><strong>and manual scrapes — locally.</strong></h1>
|
||||
<p class="lead">
|
||||
The same prospect shows up as <code>alice@acme.com</code> in HubSpot,
|
||||
<code>Alice.Johnson@acme.com</code> in LinkedIn Sales Navigator, and
|
||||
<code>alice@acme.com</code> again from your VA's manual scrape. Their
|
||||
phone is <code>(415) 555-1234</code> in one source and
|
||||
<code>4155551234</code> in another. DataTools fuzzy-matches across
|
||||
sources, normalizes phones to E.164 with per-row country awareness,
|
||||
and produces one canonical lead per real person — without uploading
|
||||
a single contact to a third-party tool.
|
||||
</p>
|
||||
<div class="cta-row">
|
||||
<a class="btn btn-large" href="https://gumroad.com/l/datatools?from=revops" rel="noopener">Get DataTools — $49 →</a>
|
||||
<a class="btn btn-ghost btn-large" href="#demo">Try the live demo ↓</a>
|
||||
<span class="price-note">One-time payment · cross-platform · runs offline</span>
|
||||
</div>
|
||||
<div class="stats">
|
||||
<div class="stat"><div class="num">50+</div><div class="label">country codes</div></div>
|
||||
<div class="stat"><div class="num">3</div><div class="label">CRM sources unified</div></div>
|
||||
<div class="stat"><div class="num">0</div><div class="label">cloud uploads ever</div></div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- ============= Pain points ============= -->
|
||||
<section>
|
||||
<div class="container">
|
||||
<div class="eyebrow">If your last campaign launch was held up by data hygiene</div>
|
||||
<h2>Five pains DataTools fixes before you import to HubSpot</h2>
|
||||
<div class="grid">
|
||||
<div class="card">
|
||||
<span class="icon">💸</span>
|
||||
<h3>HubSpot / Marketo / Iterable bills you for every duplicate contact</h3>
|
||||
<p>10 k contacts → enterprise tier at $4–8 k/mo. 18 % cross-source duplicate rate from Apollo + ZoomInfo + LinkedIn means you're at 8.2 k unique people but paying for 10 k. Every month. Forever.</p>
|
||||
<p class="muted"><strong>What it costs:</strong> $200–$800 per 1 k duplicate contacts — recurring, every month.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<span class="icon">🚫</span>
|
||||
<h3>Sender reputation tanks when you mail to invalid or duplicate addresses</h3>
|
||||
<p>One bad sending session — to addresses your team scraped or imported without hygiene — and your domain reputation takes weeks to recover. Your good campaigns sit in spam folders during the recovery.</p>
|
||||
<p class="muted"><strong>What it costs:</strong> catastrophic — entire email programme degraded for 2–6 weeks.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<span class="icon">⚖️</span>
|
||||
<h3>GDPR makes uploading to a cloud cleaner a legal-review marathon</h3>
|
||||
<p>Every cloud-based lead-cleaner needs you to upload your prospect list. Your legal team needs 4–8 weeks to bless that. DataTools is desktop-only — no upload, no DPA, no review, no delay.</p>
|
||||
<p class="muted"><strong>What it costs:</strong> 4–8 weeks of legal-review delay per tool, every time.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<span class="icon">🪢</span>
|
||||
<h3>Apollo + ZoomInfo + LinkedIn + manual scrapes all use different schemas</h3>
|
||||
<p>Each export has its own column names, scoring scale, country format. Unifying them by hand for one campaign costs 1–3 days. Doing it for every campaign is unsustainable.</p>
|
||||
<p class="muted"><strong>What it costs:</strong> 1–3 days per campaign of manual unification + judgement calls that drift across team members.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<span class="icon">🛡️</span>
|
||||
<h3>Suppression lists across 5+ marketing platforms get out of sync</h3>
|
||||
<p>Each platform has its own suppression format. Out-of-sync lists let opted-out contacts slip through, triggering CAN-SPAM / GDPR exposure and the kind of "we got a complaint" email no one wants.</p>
|
||||
<p class="muted"><strong>What it costs:</strong> compliance risk + churn-back cost + stakeholder trust.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<span class="icon">📞</span>
|
||||
<h3>International dialer fails because phone formats vary</h3>
|
||||
<p>Calling list to 15 countries with mixed formats means dialler rejects 8–15 % of numbers, your reps spend the day on "number invalid" tones instead of conversations.</p>
|
||||
<p class="muted"><strong>What it costs:</strong> rep productivity × failure rate × team size.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section id="demo">
|
||||
<div class="container">
|
||||
<div class="eyebrow">Live demo · runs in your browser</div>
|
||||
<h2>Try it on a real-looking 3-vendor lead list</h2>
|
||||
<p>
|
||||
The demo below loads a 25-row lead worksheet combining HubSpot,
|
||||
LinkedIn Sales Navigator, and manual scraping — with the same prospect
|
||||
appearing in two or three sources, country names spelled three
|
||||
different ways (<code>USA</code>, <code>US</code>, <code>United
|
||||
States</code>), and 13 different international phone formats. Click
|
||||
<strong>Run pipeline</strong> and watch the 5-step pipeline (text
|
||||
clean → format → missing → column map → dedup) collapse 25 rows to 19
|
||||
with a single canonical record per prospect.
|
||||
</p>
|
||||
<div class="demo-frame">
|
||||
<iframe
|
||||
src="https://demo.datatools.app/?p=revops"
|
||||
loading="lazy"
|
||||
title="DataTools live demo — RevOps"
|
||||
sandbox="allow-scripts allow-same-origin allow-downloads allow-forms"></iframe>
|
||||
<div class="demo-caption">
|
||||
Demo runs on free hosting. Capped at 100 input rows · output
|
||||
watermarked. The paid product has no caps and runs entirely offline.
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<div class="container">
|
||||
<div class="eyebrow">Built for the agency RevOps day</div>
|
||||
<h2>Three workflows you do every campaign</h2>
|
||||
<div class="grid">
|
||||
<div class="card">
|
||||
<span class="icon">🪢</span>
|
||||
<h3>Email-list dedup across lead sources</h3>
|
||||
<p>HubSpot exports + LinkedIn Sales Navigator + the VA's spreadsheet, all merged. Fuzzy match across email + phone + name catches the cross-source duplicates that broke your last campaign send.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<span class="icon">🌍</span>
|
||||
<h3>Multi-platform audience reconciliation</h3>
|
||||
<p>Build one canonical audience from Meta, Google Ads, LinkedIn, and your CRM. Each platform exports a different shape; column-mapper aligns them all, dedup merges the survivors with their most-complete fields.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<span class="icon">🛡️</span>
|
||||
<h3>Suppression-list management</h3>
|
||||
<p>Suppression lists need to dedupe across email + phone + first-party identifiers. Add a row, dedupe, ship the canonical CSV to every platform — without uploading the suppression list to any of them.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<div class="container">
|
||||
<div class="eyebrow">If your campaigns target outside the US — almost everyone's do</div>
|
||||
<h2>50+ country codes. Per-row country awareness.</h2>
|
||||
<p>
|
||||
Your HubSpot list has <code>(415) 555-1234</code>. Your scraped
|
||||
list from the same prospect has <code>+1 415 555 1234</code>. Your
|
||||
Italian prospect entered <code>+39 06 6982</code>. Your Brazilian
|
||||
lead has <code>11 3071 0000</code>. Each comes from a row tagged
|
||||
with its country — DataTools reads that column per row and parses
|
||||
every phone correctly to E.164.
|
||||
</p>
|
||||
<ul class="bullets">
|
||||
<li><strong>Per-row country column</strong> drives the parser — no global default that bucks UK numbers as malformed US.</li>
|
||||
<li><strong>Country-name normalization</strong>: <code>USA</code> / <code>US</code> / <code>United States</code> all resolve to the same ISO-2 code.</li>
|
||||
<li><strong>50+ country support</strong> via Google's libphonenumber, including KR, CN, IN, MX, BR, IL, TR, PL, DK, SE.</li>
|
||||
<li><strong>Schema enforcement</strong> via the column-mapper: project to your CRM's required shape, coerce score columns to integers, reorder fields to match the import contract.</li>
|
||||
</ul>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<div class="container">
|
||||
<div class="eyebrow">For platforms that charge per contact</div>
|
||||
<h2>Every duplicate you don't catch costs you for the life of the contract.</h2>
|
||||
<p>
|
||||
HubSpot prices on contacts. Klaviyo prices on contacts. Marketo,
|
||||
Iterable, ActiveCampaign — all priced on contacts. Every duplicate
|
||||
you don't catch is a recurring tax on your campaign. DataTools
|
||||
catches them once, before import, with a fuzzy matcher that's
|
||||
tuned to the cross-source noise you actually see.
|
||||
</p>
|
||||
<div class="callout">
|
||||
<strong>Real numbers from the demo:</strong> 25 input rows from
|
||||
three sources collapse to 19 — that's 6 duplicates the cross-source
|
||||
noise was hiding. On a 50,000-row campaign list, that ratio
|
||||
typically saves 12,000+ contacts a month, every month.
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<div class="container">
|
||||
<div class="eyebrow">The thing every cloud cleaner can't say</div>
|
||||
<h2>Your prospects' contact info never leaves your computer.</h2>
|
||||
<p>
|
||||
Cloud lead-cleaning tools require you to upload your audience.
|
||||
That audience is your single most valuable agency asset — and once
|
||||
it's on someone else's server, your client's privacy story is
|
||||
no longer in your hands. DataTools is a desktop app. There is no
|
||||
upload step.
|
||||
</p>
|
||||
<div class="terminal"><span class="prompt">$</span> python -m src.cli_pipeline campaign_q1.csv --pipeline revops_pipeline.json --apply
|
||||
Reading campaign_q1.csv...
|
||||
53,802 rows, 14 columns
|
||||
Executing pipeline:
|
||||
<span class="ok">✓</span> text_clean (160 ms) {cells_changed: 8,205}
|
||||
<span class="ok">✓</span> format_standardize (1.4 s) {cells_changed: 41,889 — 50 country codes}
|
||||
<span class="ok">✓</span> missing (140 ms) {sentinels_standardized: 6,710}
|
||||
<span class="ok">✓</span> column_map (220 ms) {columns_renamed: 4, columns_added: 1}
|
||||
<span class="ok">✓</span> dedup (4.8 s) {duplicates_removed: 12,344, merged: 12,344}
|
||||
|
||||
Initial rows: 53,802 → Final rows: 41,458
|
||||
Total elapsed: 6.7 s
|
||||
<span class="prompt">$</span> # 12,344 fewer contacts to pay for. for $49.</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<div class="container">
|
||||
<div class="eyebrow">In the bundle</div>
|
||||
<h2>Six tools. One pipeline. One $49 download.</h2>
|
||||
<div class="grid">
|
||||
<div class="card"><h3>1 · Deduplicator</h3><p>Fuzzy match across email + phone + name + company; merge survivors with most-complete fields.</p></div>
|
||||
<div class="card"><h3>2 · Text Cleaner</h3><p>Smart quotes from copy-paste, NBSP from spreadsheet exports, BOM from Excel.</p></div>
|
||||
<div class="card"><h3>3 · Format Standardizer</h3><p>E.164 phones with per-row country, canonical emails, name casing, ISO dates.</p></div>
|
||||
<div class="card"><h3>4 · Missing Value Handler</h3><p>Detect <code>TBD</code>, <code>(unknown)</code>, <code>—</code> across vendor exports.</p></div>
|
||||
<div class="card"><h3>5 · Column Mapper</h3><p>Project to your CRM's required schema, coerce score to integer, reorder for import.</p></div>
|
||||
<div class="card"><h3>6 · Pipeline Runner</h3><p>Save the cleanup as JSON. Drop next campaign's combined export on it. Same dedup, automated.</p></div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<div class="container">
|
||||
<div class="eyebrow">Pricing — pay once, own it</div>
|
||||
<h2>$49. No subscription. No per-campaign fee.</h2>
|
||||
<div class="pricing">
|
||||
<div class="card featured">
|
||||
<div class="row"><div class="price">$49</div><div class="price-suffix">one-time</div></div>
|
||||
<h3>DataTools for RevOps</h3>
|
||||
<ul>
|
||||
<li>All 6 tools, full pipeline</li>
|
||||
<li>Mac · Windows · Linux installers</li>
|
||||
<li>Code-signed (no Gatekeeper warnings)</li>
|
||||
<li>Free updates for the v1.x line</li>
|
||||
<li>Bonus: 3-source unification pipeline preset</li>
|
||||
<li><strong>Use on any number of clients</strong> — no seat limits</li>
|
||||
</ul>
|
||||
<a class="btn btn-large" href="https://gumroad.com/l/datatools?from=revops" rel="noopener">Buy on Gumroad →</a>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div class="row"><div class="price">$149</div><div class="price-suffix">one-time</div></div>
|
||||
<h3>Full DataTools Suite</h3>
|
||||
<p class="muted">Available when 3+ bundles ship. Includes everything in the RevOps pack plus the Shopify and Bookkeeper bundles. Save $48.</p>
|
||||
<a class="btn btn-ghost btn-large" href="#" aria-disabled="true">Coming when ready</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<div class="container">
|
||||
<h2>Questions</h2>
|
||||
|
||||
<details class="faq">
|
||||
<summary>Does this replace HubSpot's deduplication?</summary>
|
||||
<p>No — it cleans data <em>before</em> import to HubSpot (or LinkedIn, Marketo, Klaviyo, etc.). HubSpot's dedup runs on already-imported contacts; DataTools catches duplicates that haven't yet cost you a contract slot.</p>
|
||||
</details>
|
||||
|
||||
<details class="faq">
|
||||
<summary>Does it handle international phones correctly?</summary>
|
||||
<p>Yes — via Google's libphonenumber, with 50+ country codes. The killer feature is per-row country: point a column at it (any column with values like <code>US</code>, <code>USA</code>, <code>United States</code>, <code>+1</code>, <code>JP</code>, <code>Japan</code>) and DataTools parses each row in its own region. No more UK numbers bucketed as malformed US.</p>
|
||||
</details>
|
||||
|
||||
<details class="faq">
|
||||
<summary>Can I use it on multiple clients without paying again?</summary>
|
||||
<p>Yes. The licence is per-operator, not per-client. Run it on every agency client's lead list for the same $49.</p>
|
||||
</details>
|
||||
|
||||
<details class="faq">
|
||||
<summary>How does fuzzy match work across columns?</summary>
|
||||
<p>Out of the box, the dedup engine builds default strategies based on column names — typically email + phone with exact match, name with Jaro-Winkler at 85%. You can override via JSON: pick which columns to match on, which algorithm, and what threshold. Strategies survive in the saved pipeline so next campaign uses the same rules.</p>
|
||||
</details>
|
||||
|
||||
<details class="faq">
|
||||
<summary>What's the audit trail look like?</summary>
|
||||
<p>A row-by-row CSV: every modified cell with its original value, new value, and which rule fired. A separate JSON file describes the pipeline that produced it. Together they reproduce the cleanup deterministically — your client can verify it on their machine.</p>
|
||||
</details>
|
||||
|
||||
<details class="faq">
|
||||
<summary>What's your refund policy?</summary>
|
||||
<p>Try the live demo above on the sample dataset before you buy. If DataTools doesn't fit your workflow within 14 days, email for a refund — no questions asked.</p>
|
||||
</details>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<div class="container" style="text-align: center;">
|
||||
<h2>Stop paying twice for the same contact.</h2>
|
||||
<p class="lead" style="margin: 0 auto 28px;">One $49 download. Catches the cross-source duplicates HubSpot and LinkedIn can't see, normalizes phones for 50+ countries, and saves a pipeline you can re-run on next campaign's combined list.</p>
|
||||
<a class="btn btn-large" href="https://gumroad.com/l/datatools?from=revops" rel="noopener">Get DataTools — $49 →</a>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<footer>
|
||||
<div class="container">
|
||||
<div>
|
||||
<p><strong>DataTools</strong> — local data-cleaning for Shopify, bookkeepers, and RevOps teams.</p>
|
||||
<p class="muted">© 2026 · Built solo · Shipped from a small office.</p>
|
||||
</div>
|
||||
<div>
|
||||
<p>
|
||||
<a href="../shopify-pet/">For Shopify operators</a> ·
|
||||
<a href="../bookkeeper/">For bookkeepers</a><br />
|
||||
<a href="https://gumroad.com/l/datatools?from=revops">Buy on Gumroad</a> ·
|
||||
<a href="mailto:hello@datatools.app">Email support</a>
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,381 +0,0 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>DataTools for Shopify — Clean Customer & Product Exports Locally · $49</title>
|
||||
<meta name="description" content="Clean Shopify customer, product, and subscriber exports — locally. Klaviyo-import-ready in 30 seconds. Catches duplicates Excel misses. Your data never leaves your computer. $49 one-time." />
|
||||
<meta name="keywords" content="shopify customer cleanup, shopify csv cleaner, shopify product feed cleaner, klaviyo deduplicate, shopify customer dedup tool, shopify pet supplies" />
|
||||
<link rel="canonical" href="https://datatools.app/shopify/" />
|
||||
<link rel="stylesheet" href="../_shared/styles.css" />
|
||||
|
||||
<!-- Persona accent: Shopify pet → mint green (default in shared sheet) -->
|
||||
|
||||
<!-- Open Graph -->
|
||||
<meta property="og:title" content="DataTools for Shopify — Clean Customer & Product Exports Locally" />
|
||||
<meta property="og:description" content="Klaviyo-import-ready in 30 seconds. Local. No upload. $49 one-time." />
|
||||
<meta property="og:type" content="product" />
|
||||
<meta property="og:url" content="https://datatools.app/shopify/" />
|
||||
|
||||
<!-- Schema.org Product -->
|
||||
<script type="application/ld+json">
|
||||
{
|
||||
"@context": "https://schema.org",
|
||||
"@type": "SoftwareApplication",
|
||||
"name": "DataTools for Shopify",
|
||||
"operatingSystem": "Windows, macOS, Linux",
|
||||
"applicationCategory": "BusinessApplication",
|
||||
"offers": {
|
||||
"@type": "Offer",
|
||||
"price": "49",
|
||||
"priceCurrency": "USD"
|
||||
},
|
||||
"description": "Clean Shopify customer, product, and subscriber CSV exports locally. Six-tool data-cleaning bundle: dedupe, text-clean, format-standardize, missing-value handle, column-map, pipeline.",
|
||||
"softwareVersion": "1.0"
|
||||
}
|
||||
</script>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<!-- ============= Sticky buy bar ============= -->
|
||||
<div class="buybar">
|
||||
<div class="buybar-inner">
|
||||
<div class="brand"><span class="brand-mark">●</span> DataTools <span class="muted">/ for Shopify</span></div>
|
||||
<div>
|
||||
<span class="price-tag">$49 — one-time, no subscription</span>
|
||||
<a class="btn" href="https://gumroad.com/l/datatools?from=shopify-pet" rel="noopener">Get DataTools →</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- ============= Hero ============= -->
|
||||
<section class="hero">
|
||||
<div class="container">
|
||||
<div class="eyebrow">For Shopify operators · pet supplies · subscription stores · DTC</div>
|
||||
<h1>Klaviyo-import-ready customer lists.<br /><strong>In 30 seconds. Locally.</strong></h1>
|
||||
<p class="lead">
|
||||
Your Shopify customer export is a mess of formatting drift, disguised
|
||||
duplicates, and inconsistent phone numbers. DataTools fixes all of it
|
||||
in one pass — fuzzy-dedupes the same customer Klaviyo would charge
|
||||
you for twice, standardises phones across your international
|
||||
subscribers, and hands you a cleaned CSV. <strong>Your data never
|
||||
leaves your computer.</strong>
|
||||
</p>
|
||||
<div class="cta-row">
|
||||
<a class="btn btn-large" href="https://gumroad.com/l/datatools?from=shopify-pet" rel="noopener">Get DataTools — $49 →</a>
|
||||
<a class="btn btn-ghost btn-large" href="#demo">Try the live demo ↓</a>
|
||||
<span class="price-note">One-time payment · cross-platform · runs offline</span>
|
||||
</div>
|
||||
<div class="stats">
|
||||
<div class="stat"><div class="num">6</div><div class="label">tools, one bundle</div></div>
|
||||
<div class="stat"><div class="num">1 GB</div><div class="label">customer file in 2.5 min</div></div>
|
||||
<div class="stat"><div class="num">0</div><div class="label">cloud uploads ever</div></div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- ============= Pain points ============= -->
|
||||
<section>
|
||||
<div class="container">
|
||||
<div class="eyebrow">If any of these sound like your Tuesday</div>
|
||||
<h2>Five pains DataTools fixes in one pass</h2>
|
||||
<div class="grid">
|
||||
<div class="card">
|
||||
<span class="icon">💸</span>
|
||||
<h3>Klaviyo / Mailchimp / Omnisend bills you for every duplicate</h3>
|
||||
<p>Same customer signs up twice — once with a typo, once with a plus-tag, once on mobile. Your subscriber list has 10–18 % duplicate rate and you're paying for every one of them, every month, forever.</p>
|
||||
<p class="muted"><strong>What it costs:</strong> $30–$300/mo per percent of dupes on a 50 k-list — recurring.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<span class="icon">📵</span>
|
||||
<h3>Your product feed got rejected by Google Merchant Center</h3>
|
||||
<p>Smart quotes from a copy-paste in product titles. NBSP in SKU. Inconsistent attribute casing. Feed bounces, the launch sits for 24–72 hours while you try to find the bad row in a 12,000-line CSV.</p>
|
||||
<p class="muted"><strong>What it costs:</strong> 1–3 days of delayed campaign × the campaign value.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<span class="icon">🪢</span>
|
||||
<h3>Orders from Shopify + Etsy + Amazon + Faire don't speak the same language</h3>
|
||||
<p>Each platform's export uses different column names for "customer email" / "ship country" / "order total." Merging takes hours of manual rename and copy-paste before the analysis can even begin.</p>
|
||||
<p class="muted"><strong>What it costs:</strong> 4–8 hours per month manually merging exports.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<span class="icon">🔁</span>
|
||||
<h3>Subscription churn looks higher than it is</h3>
|
||||
<p>Pet-box subscribers cancel, then re-sub three months later under a different email or device. Your cohort report says churn is 20 % when it's actually 12 % — and you're over-paying for acquisition because LTV is mis-calculated.</p>
|
||||
<p class="muted"><strong>What it costs:</strong> wrong CAC ceiling for the next year of paid ads.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<span class="icon">🌍</span>
|
||||
<h3>VAT MOSS / EU tax breaks because country is spelled three ways</h3>
|
||||
<p>Your UK customers are tagged <code>UK</code>, <code>U.K.</code>, and <code>United Kingdom</code> — all in one export. The VAT report aggregates them as three different markets. Compliance friction every quarter.</p>
|
||||
<p class="muted"><strong>What it costs:</strong> compliance risk + repeated manual normalization.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<span class="icon">🔒</span>
|
||||
<h3>Cloud cleaners want you to upload your customer list</h3>
|
||||
<p>Your customer list is your single most valuable business asset. Uploading it to a SaaS to clean it is the privacy story you do not want. DataTools is desktop-only — your list never leaves your computer.</p>
|
||||
<p class="muted"><strong>What it costs:</strong> nothing — and that's the point.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- ============= Live demo ============= -->
|
||||
<section id="demo">
|
||||
<div class="container">
|
||||
<div class="eyebrow">Live demo · runs in your browser</div>
|
||||
<h2>Try it on a real-looking Shopify customer export</h2>
|
||||
<p>
|
||||
The demo below loads a sample 15-row Shopify customer file with
|
||||
pollution we've seen in actual stores: smart quotes from copy-paste,
|
||||
duplicates with email-case drift, international phones from the UK,
|
||||
Spain, Germany, Australia, and Japan, and the usual mess of
|
||||
<code>N/A</code> / <code>(blank)</code> / <code>?</code> sentinels.
|
||||
Click <strong>Run pipeline</strong> and watch every column get
|
||||
cleaned in under a second.
|
||||
</p>
|
||||
<div class="demo-frame">
|
||||
<iframe
|
||||
src="https://demo.datatools.app/?p=shopify-pet"
|
||||
loading="lazy"
|
||||
title="DataTools live demo — Shopify pet supplies"
|
||||
sandbox="allow-scripts allow-same-origin allow-downloads allow-forms"></iframe>
|
||||
<div class="demo-caption">
|
||||
Demo runs on free hosting (Streamlit Community Cloud). Capped at
|
||||
100 input rows · output watermarked with one trailing row. The
|
||||
paid product has no caps and runs entirely offline.
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- ============= Built for Shopify ============= -->
|
||||
<section>
|
||||
<div class="container">
|
||||
<div class="eyebrow">Built for the Shopify operator</div>
|
||||
<h2>Five workflows you do every week</h2>
|
||||
<div class="grid">
|
||||
<div class="card">
|
||||
<span class="icon">🧹</span>
|
||||
<h3>Customer-list cleanup</h3>
|
||||
<p>Catches the same customer who shows up as <code>john@gmail.com</code>, <code>John@Gmail.com</code>, and <code>j.ohn@gmail.com</code>. Fuzzy match merges the spellings, exact match catches the obvious ones.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<span class="icon">📦</span>
|
||||
<h3>Product catalogue dedup</h3>
|
||||
<p>SKU whitespace, near-identical product names, copy-paste smart quotes in titles — gone. Audit log shows every change.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<span class="icon">🛒</span>
|
||||
<h3>Abandoned-cart hygiene</h3>
|
||||
<p>Before re-engagement: dedupe across email + phone, drop sentinels-as-missing, format dates so your sequence triggers fire correctly.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<span class="icon">📥</span>
|
||||
<h3>Subscriber-list import to Klaviyo</h3>
|
||||
<p>Klaviyo charges per contact. Every duplicate you don't catch costs you for the life of the subscription. Catch them once, pay once.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<span class="icon">🔗</span>
|
||||
<h3>Multi-channel order consolidation</h3>
|
||||
<p>Orders from Shopify + Etsy + a wholesale spreadsheet, each with a different column for "customer email." Column-mapper aligns them; dedup merges across channels.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<span class="icon">⚙️</span>
|
||||
<h3>Repeatable pipeline</h3>
|
||||
<p>Save the cleanup as a JSON file. Drop next week's export on it. Same cleanup, zero re-configuration. Automatable via the CLI.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- ============= Privacy moat ============= -->
|
||||
<section>
|
||||
<div class="container">
|
||||
<div class="eyebrow">The thing every cloud cleaner can't say</div>
|
||||
<h2>Your customer list never leaves your computer.</h2>
|
||||
<p>
|
||||
DataTools is a desktop app. There's no upload step, no SaaS account,
|
||||
no subscription, no "trust our security policy." The first thing you
|
||||
can do after install is open your browser's network tab, run the
|
||||
cleaner on your real customer file, and verify zero outbound
|
||||
requests.
|
||||
</p>
|
||||
<div class="callout">
|
||||
<strong>Why it matters for Shopify:</strong> your customer list is
|
||||
your single most valuable business asset. Cloud cleaners require
|
||||
you to upload it. We don't.
|
||||
</div>
|
||||
<div class="terminal"><span class="prompt">$</span> python -m src.cli_pipeline customers.csv --apply
|
||||
Reading customers.csv...
|
||||
47,832 rows, 14 columns
|
||||
Executing pipeline:
|
||||
<span class="ok">✓</span> text_clean (140 ms) {cells_changed: 12,408}
|
||||
<span class="ok">✓</span> format_standardize (810 ms) {cells_changed: 31,202}
|
||||
<span class="ok">✓</span> missing (95 ms) {sentinels_standardized: 8,129}
|
||||
<span class="ok">✓</span> dedup (3.1 s) {duplicates_removed: 2,347}
|
||||
|
||||
Initial rows: 47,832 → Final rows: 45,485
|
||||
Total elapsed: 4.2 s
|
||||
<span class="prompt">$</span> # zero network calls. zero. promise.</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- ============= Audit moat ============= -->
|
||||
<section>
|
||||
<div class="container">
|
||||
<div class="eyebrow">For when your client asks "what changed?"</div>
|
||||
<h2>Every change auditable. Every cell logged.</h2>
|
||||
<p>
|
||||
Every modification is recorded with the original value, the new
|
||||
value, and which rule fired. Hand the audit CSV to your accountant,
|
||||
your marketing manager, or your boss along with the cleaned file.
|
||||
No <em>"I trust the AI"</em> hand-waving — they see exactly what
|
||||
happened.
|
||||
</p>
|
||||
<div class="callout">
|
||||
<strong>Real example:</strong> the demo above standardized 27
|
||||
cells across 15 customers. The audit log lists each one — row,
|
||||
column, before, after, which standardizer fired. The dedup audit
|
||||
lists every duplicate group with the survivor and its losers.
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- ============= International ============= -->
|
||||
<section>
|
||||
<div class="container">
|
||||
<div class="eyebrow">If you sell internationally — most pet brands do</div>
|
||||
<h2>Phones, addresses, and currencies from anywhere on Earth.</h2>
|
||||
<p>
|
||||
Your subscriber from London entered her phone as <code>020 7946
|
||||
0958</code>. Your Tokyo customer entered <code>03-3210-7000</code>.
|
||||
Your German wholesale buyer wrote <code>€2.410,75</code>. Excel
|
||||
thinks all of them are mistakes. DataTools knows what country each
|
||||
row is from (per-row country column) and parses every one correctly
|
||||
to E.164 phones, ISO dates, and numeric amounts.
|
||||
</p>
|
||||
<ul class="bullets">
|
||||
<li><strong>50+ country codes</strong> via Google's libphonenumber.</li>
|
||||
<li><strong>Currency auto-detect</strong> for $ / £ / € / ¥ / R$ / kr / zł — including the EU comma-decimal that breaks Excel.</li>
|
||||
<li><strong>Address shape detection</strong> for US, UK, Canada, Germany, Australia.</li>
|
||||
<li><strong>Locale-aware month names</strong> in English, French, German.</li>
|
||||
</ul>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- ============= What you get ============= -->
|
||||
<section>
|
||||
<div class="container">
|
||||
<div class="eyebrow">In the bundle</div>
|
||||
<h2>Six tools. One pipeline. One $49 download.</h2>
|
||||
<div class="grid">
|
||||
<div class="card"><h3>1 · Deduplicator</h3><p>Fuzzy match (Jaro-Winkler), 5 normalizers, survivor rules, interactive review.</p></div>
|
||||
<div class="card"><h3>2 · Text Cleaner</h3><p>Whitespace, smart chars, NBSP, BOM, line endings, case ops.</p></div>
|
||||
<div class="card"><h3>3 · Format Standardizer</h3><p>Dates, phones, emails, addresses, names, currencies, booleans.</p></div>
|
||||
<div class="card"><h3>4 · Missing Value Handler</h3><p>Disguised-null detection, profile, mean/median/mode/ffill, drop strategies.</p></div>
|
||||
<div class="card"><h3>5 · Column Mapper</h3><p>Fuzzy auto-rename, target schema, type coercion, required-field defaults.</p></div>
|
||||
<div class="card"><h3>6 · Pipeline Runner</h3><p>Chain tools in recommended order, save/load JSON, automate weekly cleanups.</p></div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- ============= Pricing ============= -->
|
||||
<section>
|
||||
<div class="container">
|
||||
<div class="eyebrow">Pricing — pay once, own it</div>
|
||||
<h2>$49. No subscription. No ceiling on rows or files.</h2>
|
||||
<div class="pricing">
|
||||
<div class="card featured">
|
||||
<div class="row"><div class="price">$49</div><div class="price-suffix">one-time</div></div>
|
||||
<h3>DataTools for Shopify</h3>
|
||||
<ul>
|
||||
<li>All 6 tools, full pipeline</li>
|
||||
<li>Mac · Windows · Linux installers</li>
|
||||
<li>Code-signed (no Gatekeeper warnings)</li>
|
||||
<li>Free updates for the v1.x line</li>
|
||||
<li>Bonus: 3 ready-made Shopify pipelines</li>
|
||||
</ul>
|
||||
<a class="btn btn-large" href="https://gumroad.com/l/datatools?from=shopify-pet" rel="noopener">Buy on Gumroad →</a>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div class="row"><div class="price">$149</div><div class="price-suffix">one-time</div></div>
|
||||
<h3>Full DataTools Suite</h3>
|
||||
<p class="muted">Available when 3+ bundles ship. Includes everything in the Shopify pack plus the Bookkeeper and RevOps bundles. Save $48.</p>
|
||||
<a class="btn btn-ghost btn-large" href="#" aria-disabled="true">Coming when ready</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- ============= FAQ ============= -->
|
||||
<section>
|
||||
<div class="container">
|
||||
<h2>Questions</h2>
|
||||
|
||||
<details class="faq">
|
||||
<summary>Does this work with Shopify Plus?</summary>
|
||||
<p>Yes — the input is just CSV / Excel from any source. Your Shopify Plus exports work the same as the standard plan, the same as a Shopify-to-CSV pipeline you've stitched together yourself. The cleaner doesn't care.</p>
|
||||
</details>
|
||||
|
||||
<details class="faq">
|
||||
<summary>How does this compare to Excel's "Remove Duplicates"?</summary>
|
||||
<p>Excel does <em>exact</em> deduplication. <code>John@Gmail.com</code> and <code>john@gmail.com</code> are different customers to Excel. DataTools fuzzy-matches across case, whitespace, formatting, and even close-but-not-identical strings. The demo above merges 4 customer pairs Excel would leave duplicated.</p>
|
||||
</details>
|
||||
|
||||
<details class="faq">
|
||||
<summary>How big a file can it handle?</summary>
|
||||
<p>1 GB CSV with international phones + addresses processes in about 2.5 minutes on a typical workstation. Streaming mode keeps memory bounded regardless of input size — we tested it on 26 million rows.</p>
|
||||
</details>
|
||||
|
||||
<details class="faq">
|
||||
<summary>Do I need to know Python to use it?</summary>
|
||||
<p>No. The GUI is a browser interface that opens automatically when you double-click the app. It loads your file, you click Run, you download the cleaned file. The CLI is there for power users who want to script weekly cleanups.</p>
|
||||
</details>
|
||||
|
||||
<details class="faq">
|
||||
<summary>What about my privacy?</summary>
|
||||
<p>Your customer list never leaves your computer. There is no cloud component, no telemetry, no "anonymous usage stats." When the app is running you can confirm zero outbound network requests in your browser's developer tools.</p>
|
||||
</details>
|
||||
|
||||
<details class="faq">
|
||||
<summary>What's your refund policy?</summary>
|
||||
<p>Try the live demo above on the sample dataset before you buy. If you still find DataTools doesn't fit your workflow within 14 days, email for a refund — no questions asked.</p>
|
||||
</details>
|
||||
|
||||
<details class="faq">
|
||||
<summary>Will there be updates?</summary>
|
||||
<p>Yes. The v1.x line is included free for everyone who buys DataTools today. We ship a patch every 30 days adding country support, edge-case fixes, and small features.</p>
|
||||
</details>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- ============= Final CTA ============= -->
|
||||
<section>
|
||||
<div class="container" style="text-align: center;">
|
||||
<h2>Stop deduplicating customers by hand.</h2>
|
||||
<p class="lead" style="margin: 0 auto 28px;">One $49 download. Mac, Windows, or Linux. Runs offline. Catches the duplicates Excel misses, standardizes the phones from your international customers, and saves a pipeline you can re-run on next week's export.</p>
|
||||
<a class="btn btn-large" href="https://gumroad.com/l/datatools?from=shopify-pet" rel="noopener">Get DataTools — $49 →</a>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- ============= Footer ============= -->
|
||||
<footer>
|
||||
<div class="container">
|
||||
<div>
|
||||
<p><strong>DataTools</strong> — local data-cleaning for Shopify, bookkeepers, and RevOps teams.</p>
|
||||
<p class="muted">© 2026 · Built solo · Shipped from a small office.</p>
|
||||
</div>
|
||||
<div>
|
||||
<p>
|
||||
<a href="../bookkeeper/">For bookkeepers</a> ·
|
||||
<a href="../revops/">For RevOps agencies</a><br />
|
||||
<a href="https://gumroad.com/l/datatools?from=shopify-pet">Buy on Gumroad</a> ·
|
||||
<a href="mailto:hello@datatools.app">Email support</a>
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
192
layout-review/01_deduplicator.html
Normal file
192
layout-review/01_deduplicator.html
Normal file
@@ -0,0 +1,192 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Layout review — Find Duplicates</title>
|
||||
<link rel="stylesheet" href="app.css">
|
||||
</head>
|
||||
<body data-page="01_deduplicator">
|
||||
<div class="dt-app">
|
||||
<aside class="dt-sidebar" id="dt-sidebar"></aside>
|
||||
<main class="dt-main">
|
||||
<div class="dt-review-banner">
|
||||
<span class="dt-mi">visibility</span>
|
||||
<span>Static layout preview of <strong>Find Duplicates</strong>, shown with a file imported and a completed run (results + match-group review). <a href="index.html">All pages →</a></span>
|
||||
</div>
|
||||
<div class="dt-main-inner">
|
||||
|
||||
<!-- Tool header -->
|
||||
<div class="dt-tool-header">
|
||||
<h1>Find Duplicates</h1>
|
||||
<div class="dt-tool-header-actions">
|
||||
<span class="dt-privacy-pill">
|
||||
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor">
|
||||
<rect x="4" y="11" width="16" height="10" rx="2"/>
|
||||
<path d="M8 11V7a4 4 0 018 0v4"/>
|
||||
</svg>
|
||||
Runs 100% locally
|
||||
</span>
|
||||
<button class="dt-help-btn"><span class="dt-mi">help_outline</span> Help</button>
|
||||
</div>
|
||||
</div>
|
||||
<p class="dt-tool-caption">Find rows that repeat, then keep one and remove the extras.</p>
|
||||
|
||||
<div class="dt-spacer"></div>
|
||||
|
||||
<!-- File pickup banner (using file from upload screen) -->
|
||||
<div class="dt-alert info">
|
||||
<span class="dt-mi">description</span>
|
||||
<span>Using <strong>customers_export.csv</strong> from the upload screen.</span>
|
||||
</div>
|
||||
<button class="dt-btn" style="margin-bottom:4px">Use a different file</button>
|
||||
|
||||
<!-- Delimiter selector — delimited-text only (CSV/TSV); omitted for XLSX/XLS.
|
||||
Shown here because the staged file is customers_export.csv. -->
|
||||
<div class="dt-field" style="max-width:320px">
|
||||
<label class="dt-label">Delimiter</label>
|
||||
<div class="dt-select">Comma (,)</div>
|
||||
<div class="dt-help-text">Auto-detected on upload. Change if the preview looks wrong.</div>
|
||||
</div>
|
||||
|
||||
<!-- Preview expander (collapsed after a result exists) -->
|
||||
<details class="dt-expander">
|
||||
<summary>Preview: customers_export.csv</summary>
|
||||
<div class="dt-expander-body">
|
||||
<p class="dt-caption">18,442 rows, 6 columns</p>
|
||||
<div class="dt-table-wrap">
|
||||
<table class="dt-table">
|
||||
<thead><tr><th class="idx"></th><th>name</th><th>email</th><th>city</th><th>phone</th><th>signup_date</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td class="idx">0</td><td>Jane Doe</td><td>jane@acme.io</td><td>Austin</td><td>512-555-0190</td><td>2024-01-04</td></tr>
|
||||
<tr><td class="idx">1</td><td>jane doe</td><td>JANE@ACME.IO</td><td>austin</td><td>(512) 555-0190</td><td>01/04/2024</td></tr>
|
||||
<tr><td class="idx">2</td><td>Bob Smith</td><td>bob@globex.com</td><td>Denver</td><td>720-555-7781</td><td>2024-02-11</td></tr>
|
||||
<tr><td class="idx">3</td><td>R. Smith</td><td>bob@globex.com</td><td>Denver</td><td>720-555-7781</td><td>2024-02-11</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<!-- Basic controls (visible by default) -->
|
||||
<div class="dt-cols-2">
|
||||
<div class="dt-field"><label class="dt-label">Match threshold</label>
|
||||
<div class="dt-slider"><div class="track"><div class="fill" style="width:70%"></div><div class="knob" style="left:70%"></div></div><div class="val">85</div></div>
|
||||
<div class="dt-help-text">Higher means rows must look more alike to count as a duplicate.</div></div>
|
||||
<div class="dt-field"><label class="dt-label">When duplicates are found, keep</label>
|
||||
<div class="dt-select">the most-complete row</div>
|
||||
<div class="dt-help-text">Which row survives in each group of duplicates.</div></div>
|
||||
</div>
|
||||
|
||||
<!-- Advanced options (single expander; basics live above) -->
|
||||
<details class="dt-expander">
|
||||
<summary>Advanced options</summary>
|
||||
<div class="dt-expander-body">
|
||||
<p class="dt-help-text" style="margin-top:0">Leave these empty to auto-detect which columns to compare. Otherwise, list the columns that must match <strong>exactly</strong> and the ones that only need to match <strong>approximately</strong> — together these are the columns used to find duplicates.</p>
|
||||
<div class="dt-cols-2">
|
||||
<div>
|
||||
<div class="dt-field"><label class="dt-label">Columns that must match exactly</label>
|
||||
<div class="dt-multiselect"><span class="dt-ms-chip">email <span class="x">✕</span></span></div></div>
|
||||
<div class="dt-field"><label class="dt-label">Columns to match approximately</label>
|
||||
<div class="dt-multiselect"><span class="dt-ms-chip">name <span class="x">✕</span></span></div></div>
|
||||
</div>
|
||||
<div>
|
||||
<div class="dt-field"><label class="dt-label">Approximate-match algorithm</label><div class="dt-select">jaro_winkler</div></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="dt-check on" style="margin-top:6px"><span class="box"><span class="dt-mi">check</span></span> Merge mode — fill missing fields in the surviving row</div>
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<hr class="dt-divider">
|
||||
<button class="dt-btn dt-btn-primary dt-btn-block">Find Duplicates</button>
|
||||
|
||||
<hr class="dt-divider">
|
||||
|
||||
<!-- Results -->
|
||||
<h2>Results</h2>
|
||||
<div class="dt-metrics">
|
||||
<div class="dt-metric"><div class="label">Original rows</div><div class="value">18,442</div></div>
|
||||
<div class="dt-metric"><div class="label">Duplicate rows</div><div class="value">312</div><div class="delta down">−312 removed</div></div>
|
||||
<div class="dt-metric"><div class="label">Match groups</div><div class="value">147</div></div>
|
||||
<div class="dt-metric"><div class="label">Rows kept</div><div class="value">18,130</div></div>
|
||||
</div>
|
||||
<p class="dt-caption">Preview of an auto-resolved run: each group keeps its auto-picked survivor. Review the groups below to override any pending picks before the final download.</p>
|
||||
<div class="dt-btn-row" style="max-width:560px">
|
||||
<button class="dt-btn">Download auto-resolved CSV</button>
|
||||
<button class="dt-btn">Download removed rows</button>
|
||||
</div>
|
||||
|
||||
<hr class="dt-divider">
|
||||
|
||||
<!-- Match groups -->
|
||||
<h2>Match Groups</h2>
|
||||
<div class="dt-cols-3" style="max-width:520px">
|
||||
<button class="dt-btn">Accept All</button>
|
||||
<button class="dt-btn">Reject All</button>
|
||||
<button class="dt-btn">Clear Decisions</button>
|
||||
</div>
|
||||
<p class="dt-caption" style="margin-top:8px">Differing columns are highlighted. The survivor row is kept; uncheck a row to split it out of the group.</p>
|
||||
|
||||
<!-- Match group card 1 -->
|
||||
<div class="dt-match-card">
|
||||
<div class="dt-match-head">
|
||||
<span class="title">Group 1 · 2 rows</span>
|
||||
<span class="conf"><span class="dt-count-pill success">98% match</span></span>
|
||||
</div>
|
||||
<div class="dt-match-body">
|
||||
<div class="dt-table-wrap">
|
||||
<table class="dt-table">
|
||||
<thead><tr><th>keep</th><th>name</th><th>email</th><th>city</th><th>phone</th><th>signup_date</th></tr></thead>
|
||||
<tbody>
|
||||
<tr class="dt-keep-row"><td><span class="dt-keep-tag">keep</span></td><td>Jane Doe</td><td>jane@acme.io</td><td>Austin</td><td>512-555-0190</td><td>2024-01-04</td></tr>
|
||||
<tr><td><span class="dt-caption">remove</span></td><td class="dt-cell-flag">jane doe</td><td class="dt-cell-flag">JANE@ACME.IO</td><td class="dt-cell-flag">austin</td><td>(512) 555-0190</td><td class="dt-cell-flag">01/04/2024</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Match group card 2 -->
|
||||
<div class="dt-match-card">
|
||||
<div class="dt-match-head">
|
||||
<span class="title">Group 2 · 2 rows</span>
|
||||
<span class="conf"><span class="dt-count-pill warn">87% match</span></span>
|
||||
</div>
|
||||
<div class="dt-match-body">
|
||||
<div class="dt-table-wrap">
|
||||
<table class="dt-table">
|
||||
<thead><tr><th>keep</th><th>name</th><th>email</th><th>city</th><th>phone</th><th>signup_date</th></tr></thead>
|
||||
<tbody>
|
||||
<tr class="dt-keep-row"><td><span class="dt-keep-tag">keep</span></td><td>Bob Smith</td><td>bob@globex.com</td><td>Denver</td><td>720-555-7781</td><td>2024-02-11</td></tr>
|
||||
<tr><td><span class="dt-caption">remove</span></td><td class="dt-cell-flag">R. Smith</td><td>bob@globex.com</td><td>Denver</td><td>720-555-7781</td><td>2024-02-11</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p class="dt-caption" style="margin-top:14px">Decisions: 1 merged, 1 pending · Pending groups keep their auto-picked survivor unless you review them.</p>
|
||||
<button class="dt-btn dt-btn-primary dt-btn-block" style="margin-top:8px">Apply Review Decisions & Download Final CSV</button>
|
||||
|
||||
<!-- Processing log -->
|
||||
<details class="dt-expander" style="margin-top:18px">
|
||||
<summary>Processing Log</summary>
|
||||
<div class="dt-expander-body">
|
||||
<div class="dt-code">[00:00.01] Loaded 18,442 rows from customers_export.csv
|
||||
[00:00.04] Strategy: exact(email) + fuzzy(name, jaro_winkler ≥ 85)
|
||||
[00:00.91] Compared 18,442 rows → 147 match groups
|
||||
[00:01.02] Survivor rule: most-complete · merge=on
|
||||
[00:01.05] 312 rows flagged for removal</div>
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<div class="dt-next-step"><span class="dt-mi">arrow_forward</span><span>Duplicates handled — your file is cleaned. Review the result or <a href="home.html">Back to Start here →</a></span><button class="dt-next-step-dismiss" title="Dismiss">✕</button></div>
|
||||
|
||||
</div>
|
||||
</main>
|
||||
</div>
|
||||
<footer class="dt-footer" id="dt-footer"></footer>
|
||||
<script src="shell.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
223
layout-review/02_text_cleaner.html
Normal file
223
layout-review/02_text_cleaner.html
Normal file
@@ -0,0 +1,223 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Layout review — Clean Text</title>
|
||||
<link rel="stylesheet" href="app.css">
|
||||
<style>
|
||||
/* Hidden-character badges — mirrors src/core/text_clean.py:hidden_char_css(),
|
||||
not part of app.css so reproduced inline against the same palette. */
|
||||
.hidden-char { display: inline-block; padding: 0 2px; margin: 0 1px; border-radius: 3px; font-family: var(--font-mono); font-size: 0.85em; cursor: help; }
|
||||
.hidden-char.hidden-whitespace { background: #fff3cd; color: #856404; border: 1px solid #ffeaa7; }
|
||||
.hidden-char.hidden-special { background: #d1ecf1; color: #0c5460; border: 1px solid #bee5eb; }
|
||||
.hidden-char.hidden-control { background: #f8d7da; color: #721c24; border: 1px solid #f5c6cb; }
|
||||
</style>
|
||||
</head>
|
||||
<body data-page="02_text_cleaner">
|
||||
<div class="dt-app">
|
||||
<aside class="dt-sidebar" id="dt-sidebar"></aside>
|
||||
<main class="dt-main">
|
||||
<div class="dt-review-banner">
|
||||
<span class="dt-mi">visibility</span>
|
||||
<span>Static layout preview of <strong>Clean Text</strong>, shown with a file imported and a completed run (results metrics, changes-by-column, before/after examples, cleaned preview, downloads). <a href="index.html">All pages →</a></span>
|
||||
</div>
|
||||
<div class="dt-main-inner">
|
||||
|
||||
<!-- Tool header -->
|
||||
<div class="dt-tool-header">
|
||||
<h1>Clean Text</h1>
|
||||
<div class="dt-tool-header-actions">
|
||||
<span class="dt-privacy-pill">
|
||||
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor">
|
||||
<rect x="4" y="11" width="16" height="10" rx="2"/>
|
||||
<path d="M8 11V7a4 4 0 018 0v4"/>
|
||||
</svg>
|
||||
Runs 100% locally
|
||||
</span>
|
||||
<button class="dt-help-btn"><span class="dt-mi">help_outline</span> Help</button>
|
||||
</div>
|
||||
</div>
|
||||
<p class="dt-tool-caption">Trim extra spaces and strip out odd characters.</p>
|
||||
|
||||
<div class="dt-spacer"></div>
|
||||
|
||||
<!-- File pickup banner (using file from upload screen) -->
|
||||
<div class="dt-alert info">
|
||||
<span class="dt-mi">description</span>
|
||||
<span>Using <strong>contacts_messy.csv</strong> from the upload screen.</span>
|
||||
</div>
|
||||
<button class="dt-btn" style="margin-bottom:4px">Use a different file</button>
|
||||
|
||||
<!-- Preview expander (collapsed once a result exists) -->
|
||||
<details class="dt-expander">
|
||||
<summary>Preview: contacts_messy.csv</summary>
|
||||
<div class="dt-expander-body">
|
||||
<p class="dt-caption">4,120 rows, 4 columns</p>
|
||||
<div class="dt-check on" style="margin-top:2px"><span class="box"><span class="dt-mi">check</span></span> Show hidden characters</div>
|
||||
<div style="display:flex;flex-wrap:wrap;align-items:center;gap:14px;margin-top:6px;font-size:12px;color:var(--ink-secondary)">
|
||||
<span style="display:inline-flex;align-items:center;gap:6px"><span class="hidden-char hidden-whitespace" style="cursor:default">·</span> Whitespace</span>
|
||||
<span style="display:inline-flex;align-items:center;gap:6px"><span class="hidden-char hidden-special" style="cursor:default">”</span> Smart / special</span>
|
||||
<span style="display:inline-flex;align-items:center;gap:6px"><span class="hidden-char hidden-control" style="cursor:default">␣</span> Control</span>
|
||||
</div>
|
||||
<div class="dt-table-wrap" style="margin-top:8px">
|
||||
<table class="dt-table">
|
||||
<thead><tr><th class="idx"></th><th>name</th><th>email</th><th>company</th><th>notes</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td class="idx">0</td><td><span class="hidden-char hidden-whitespace" title="U+0020 SP LEAD">·</span>Jane Doe<span class="hidden-char hidden-whitespace" title="U+0020 SP TRAIL">·</span></td><td>jane@acme.io</td><td>Acme<span class="hidden-char hidden-whitespace" title="U+00A0 NBSP">·</span>Inc.</td><td>VIP<span class="hidden-char hidden-special" title="U+201D RIGHT DOUBLE QUOTE">”</span></td></tr>
|
||||
<tr><td class="idx">1</td><td>Bob Smith</td><td>bob@globex.com<span class="hidden-char hidden-special" title="U+200B ZWSP">∅</span></td><td>Globex</td><td>—<span class="hidden-char hidden-control" title="U+0007 CTRL">␣</span></td></tr>
|
||||
<tr><td class="idx">2</td><td>Ana López</td><td>ana@initech.com</td><td>Initech<span class="hidden-char hidden-whitespace" title="U+0020 SP TRAIL">·</span></td><td>follow up</td></tr>
|
||||
<tr><td class="idx">3</td><td><span class="hidden-char hidden-whitespace" title="U+0009 TAB">→</span>Wei Chen</td><td>WEI@umbrella.co</td><td>Umbrella</td><td>“key<span class="hidden-char hidden-special" title="U+2014 EM DASH">—</span>account”</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<hr class="dt-divider">
|
||||
|
||||
<!-- Options expander (collapsed once a result exists) -->
|
||||
<details class="dt-expander">
|
||||
<summary>Options</summary>
|
||||
<div class="dt-expander-body">
|
||||
<div class="dt-field">
|
||||
<label class="dt-label">Preset</label>
|
||||
<div class="dt-radio-row">
|
||||
<span class="dt-radio on"><span class="dot"></span> excel-hygiene (recommended)</span>
|
||||
<span class="dt-radio"><span class="dot"></span> minimal</span>
|
||||
<span class="dt-radio"><span class="dot"></span> paranoid</span>
|
||||
</div>
|
||||
<div class="dt-help-text">
|
||||
minimal: trim and collapse whitespace only — no character substitutions.<br>
|
||||
excel-hygiene: trim, collapse whitespace, fold smart quotes, strip invisible chars, normalize line endings, and normalize accented characters.<br>
|
||||
paranoid: everything in excel-hygiene plus strip control characters, strip BOM, and normalize accented and look-alike characters (lossy).
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<details class="dt-expander">
|
||||
<summary>Advanced options</summary>
|
||||
<div class="dt-expander-body">
|
||||
<div class="dt-cols-2">
|
||||
<div>
|
||||
<div class="dt-check on"><span class="box"><span class="dt-mi">check</span></span> Trim leading/trailing whitespace</div>
|
||||
<div class="dt-check on"><span class="box"><span class="dt-mi">check</span></span> Collapse internal whitespace</div>
|
||||
<div class="dt-check on"><span class="box"><span class="dt-mi">check</span></span> Normalize line endings (\r\n → \n)</div>
|
||||
<div class="dt-check on"><span class="box"><span class="dt-mi">check</span></span> Strip control characters</div>
|
||||
<div class="dt-check on"><span class="box"><span class="dt-mi">check</span></span> Strip BOM</div>
|
||||
</div>
|
||||
<div>
|
||||
<div class="dt-check on"><span class="box"><span class="dt-mi">check</span></span> Fold smart characters (curly quotes, em-dash, NBSP)</div>
|
||||
<div class="dt-check on"><span class="box"><span class="dt-mi">check</span></span> Strip zero-width / invisible characters</div>
|
||||
<div class="dt-check on" title="Unicode NFC normalization"><span class="box"><span class="dt-mi">check</span></span> Normalize accented characters (NFC)</div>
|
||||
<div class="dt-check" title="Unicode NFKC compatibility fold"><span class="box"></span> Normalize accented and look-alike characters (lossy: ① → 1, fi → fi)</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<h4>Scope</h4>
|
||||
<div class="dt-field">
|
||||
<label class="dt-label">Columns to clean (default: all string columns)</label>
|
||||
<div class="dt-multiselect">
|
||||
<span class="dt-ms-chip">name <span class="x">✕</span></span>
|
||||
<span class="dt-ms-chip">email <span class="x">✕</span></span>
|
||||
<span class="dt-ms-chip">company <span class="x">✕</span></span>
|
||||
<span class="dt-ms-chip">notes <span class="x">✕</span></span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="dt-field">
|
||||
<label class="dt-label">Columns to skip even if they look like text</label>
|
||||
<div class="dt-multiselect"><span class="dt-ms-placeholder">Choose columns to leave untouched</span></div>
|
||||
</div>
|
||||
|
||||
<h4>Case conversion</h4>
|
||||
<div class="dt-field" style="max-width:360px">
|
||||
<label class="dt-label">Apply case conversion to selected columns</label>
|
||||
<div class="dt-select">None</div>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<hr class="dt-divider">
|
||||
<button class="dt-btn dt-btn-primary dt-btn-block">Clean Text</button>
|
||||
|
||||
<hr class="dt-divider">
|
||||
|
||||
<!-- Results -->
|
||||
<h2>Results</h2>
|
||||
<div class="dt-metrics">
|
||||
<div class="dt-metric"><div class="label">Cells scanned</div><div class="value">16,480</div></div>
|
||||
<div class="dt-metric"><div class="label">Cells changed</div><div class="value">3,947</div></div>
|
||||
<div class="dt-metric"><div class="label">% changed</div><div class="value">24.0%</div></div>
|
||||
<div class="dt-metric"><div class="label">Columns processed</div><div class="value">4</div></div>
|
||||
</div>
|
||||
|
||||
<div class="dt-field">
|
||||
<div class="dt-check on"><span class="box"><span class="dt-mi">check</span></span> Show hidden characters (NBSP, ZWSP, smart quotes, control chars…)</div>
|
||||
<div class="dt-help-text">Same setting as “Show hidden characters” in the preview above — toggling either updates both.</div>
|
||||
</div>
|
||||
|
||||
<h4>Changes by column</h4>
|
||||
<div class="dt-table-wrap" style="max-width:360px">
|
||||
<table class="dt-table">
|
||||
<thead><tr><th>column</th><th>cells_changed</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td>company</td><td>1,604</td></tr>
|
||||
<tr><td>name</td><td>1,210</td></tr>
|
||||
<tr><td>notes</td><td>982</td></tr>
|
||||
<tr><td>email</td><td>151</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<h4>Examples (first 25 changes)</h4>
|
||||
<div class="dt-table-wrap">
|
||||
<table class="dt-table">
|
||||
<thead><tr><th>Row</th><th>Column</th><th>Before</th><th>After</th><th>Ops applied</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td>1</td><td>name</td><td><span class="hidden-char hidden-whitespace" title="U+0020 SP LEAD">·</span>Jane Doe<span class="hidden-char hidden-whitespace" title="U+0020 SP TRAIL">·</span></td><td>Jane Doe</td><td>trim</td></tr>
|
||||
<tr><td>1</td><td>company</td><td>Acme<span class="hidden-char hidden-whitespace" title="U+00A0 NBSP">·</span>Inc.</td><td>Acme Inc.</td><td>fold_smart</td></tr>
|
||||
<tr><td>1</td><td>notes</td><td>VIP<span class="hidden-char hidden-special" title="U+201D RIGHT DOUBLE QUOTE">”</span></td><td>VIP"</td><td>fold_smart</td></tr>
|
||||
<tr><td>2</td><td>name</td><td>Bob<span class="hidden-char hidden-whitespace" title="U+0020 SP">·</span><span class="hidden-char hidden-whitespace" title="U+0020 SP">·</span>Smith</td><td>Bob Smith</td><td>collapse_ws</td></tr>
|
||||
<tr><td>2</td><td>email</td><td>bob@globex.com<span class="hidden-char hidden-special" title="U+200B ZWSP">∅</span></td><td>bob@globex.com</td><td>strip_zero_width</td></tr>
|
||||
<tr><td>2</td><td>notes</td><td>—<span class="hidden-char hidden-control" title="U+0007 CTRL">␣</span></td><td>—</td><td>strip_control</td></tr>
|
||||
<tr><td>3</td><td>company</td><td>Initech<span class="hidden-char hidden-whitespace" title="U+0020 SP TRAIL">·</span></td><td>Initech</td><td>trim</td></tr>
|
||||
<tr><td>4</td><td>name</td><td><span class="hidden-char hidden-whitespace" title="U+0009 TAB">→</span>Wei Chen</td><td>Wei Chen</td><td>trim</td></tr>
|
||||
<tr><td>4</td><td>notes</td><td>“key<span class="hidden-char hidden-special" title="U+2014 EM DASH">—</span>account”</td><td>"key-account"</td><td>fold_smart, nfc</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<h4>Cleaned preview (first 10 rows)</h4>
|
||||
<div class="dt-table-wrap">
|
||||
<table class="dt-table">
|
||||
<thead><tr><th class="idx"></th><th>name</th><th>email</th><th>company</th><th>notes</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td class="idx">0</td><td class="dt-cell-add">Jane Doe</td><td>jane@acme.io</td><td class="dt-cell-add">Acme Inc.</td><td class="dt-cell-add">VIP"</td></tr>
|
||||
<tr><td class="idx">1</td><td class="dt-cell-add">Bob Smith</td><td class="dt-cell-add">bob@globex.com</td><td>Globex</td><td class="dt-cell-add">—</td></tr>
|
||||
<tr><td class="idx">2</td><td>Ana López</td><td>ana@initech.com</td><td class="dt-cell-add">Initech</td><td>follow up</td></tr>
|
||||
<tr><td class="idx">3</td><td class="dt-cell-add">Wei Chen</td><td>WEI@umbrella.co</td><td>Umbrella</td><td class="dt-cell-add">"key-account"</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<p class="dt-caption">Changed cells highlighted. Toggle “Show hidden characters” to inspect the invisibles being removed.</p>
|
||||
|
||||
<hr class="dt-divider">
|
||||
|
||||
<!-- Downloads -->
|
||||
<div class="dt-cols-3">
|
||||
<button class="dt-btn dt-btn-primary">Download cleaned CSV</button>
|
||||
<button class="dt-btn">Download changes audit</button>
|
||||
<button class="dt-btn">Download config JSON</button>
|
||||
</div>
|
||||
|
||||
<!-- Next-step suggestion -->
|
||||
<div class="dt-next-step"><span class="dt-mi">arrow_forward</span><span>Text cleaned. Next, most files need: <a href="03_format_standardizer.html">Standardize Formats →</a></span><button class="dt-next-step-dismiss" title="Dismiss">✕</button></div>
|
||||
|
||||
</div>
|
||||
</main>
|
||||
</div>
|
||||
<footer class="dt-footer" id="dt-footer"></footer>
|
||||
<script src="shell.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
265
layout-review/03_format_standardizer.html
Normal file
265
layout-review/03_format_standardizer.html
Normal file
@@ -0,0 +1,265 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Layout review — Standardize Formats</title>
|
||||
<link rel="stylesheet" href="app.css">
|
||||
</head>
|
||||
<body data-page="03_format_standardizer">
|
||||
<div class="dt-app">
|
||||
<aside class="dt-sidebar" id="dt-sidebar"></aside>
|
||||
<main class="dt-main">
|
||||
<div class="dt-review-banner">
|
||||
<span class="dt-mi">visibility</span>
|
||||
<span>Static layout preview of <strong>Standardize Formats</strong>, shown with a file imported from the upload screen and a completed run (results + changes audit + standardized preview). <a href="index.html">All pages →</a></span>
|
||||
</div>
|
||||
<div class="dt-main-inner">
|
||||
|
||||
<!-- Tool header -->
|
||||
<div class="dt-tool-header">
|
||||
<h1>Standardize Formats</h1>
|
||||
<div class="dt-tool-header-actions">
|
||||
<span class="dt-privacy-pill">
|
||||
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor">
|
||||
<rect x="4" y="11" width="16" height="10" rx="2"/>
|
||||
<path d="M8 11V7a4 4 0 018 0v4"/>
|
||||
</svg>
|
||||
Runs 100% locally
|
||||
</span>
|
||||
<button class="dt-help-btn"><span class="dt-mi">help_outline</span> Help</button>
|
||||
</div>
|
||||
</div>
|
||||
<p class="dt-tool-caption">Make dates, phones, currency, and names look the same throughout.</p>
|
||||
|
||||
<div class="dt-spacer"></div>
|
||||
|
||||
<!-- File pickup banner (using file from upload screen) -->
|
||||
<div class="dt-alert info">
|
||||
<span class="dt-mi">description</span>
|
||||
<span>Using <strong>customers_export.csv</strong> from the upload screen.</span>
|
||||
</div>
|
||||
<button class="dt-btn" style="margin-bottom:4px">Use a different file</button>
|
||||
|
||||
<!-- Preview expander (collapsed once a result exists) -->
|
||||
<details class="dt-expander">
|
||||
<summary>Preview: customers_export.csv</summary>
|
||||
<div class="dt-expander-body">
|
||||
<p class="dt-caption">18,442 rows, 6 columns</p>
|
||||
<div class="dt-table-wrap">
|
||||
<table class="dt-table">
|
||||
<thead><tr><th class="idx"></th><th>full_name</th><th>phone</th><th>amount</th><th>signup_date</th><th>active</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td class="idx">0</td><td>jane DOE</td><td>(512) 555-0190</td><td>$1,234.5</td><td>01/04/2024</td><td>Y</td></tr>
|
||||
<tr><td class="idx">1</td><td>bob smith</td><td>720.555.7781</td><td>$99</td><td>2024-2-11</td><td>yes</td></tr>
|
||||
<tr><td class="idx">2</td><td>ALICIA REYES</td><td>+1 415 555 2233</td><td>$45,000</td><td>Mar 3, 2024</td><td>n</td></tr>
|
||||
<tr><td class="idx">3</td><td>m. okafor</td><td>2125550148</td><td>$7.999</td><td>2024/04/22</td><td>true</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<hr class="dt-divider">
|
||||
|
||||
<!-- Options expander (collapsed after run; opened here to show the most informative content) -->
|
||||
<details class="dt-expander" open>
|
||||
<summary>Options</summary>
|
||||
<div class="dt-expander-body">
|
||||
|
||||
<h3 style="margin-top:0">Column types</h3>
|
||||
<p class="dt-caption">Assign each column to a field type. Auto-detected suggestions are pre-filled; pick <strong>(skip)</strong> to leave a column untouched.</p>
|
||||
|
||||
<!-- Per-column type selectboxes, 3 per row -->
|
||||
<div class="dt-cols-3">
|
||||
<div class="dt-field"><label class="dt-label">full_name</label><div class="dt-select">Name</div></div>
|
||||
<div class="dt-field"><label class="dt-label">phone</label><div class="dt-select">Phone</div></div>
|
||||
<div class="dt-field"><label class="dt-label">amount</label><div class="dt-select">Currency</div></div>
|
||||
</div>
|
||||
<div class="dt-cols-3">
|
||||
<div class="dt-field"><label class="dt-label">signup_date</label><div class="dt-select">Date</div></div>
|
||||
<div class="dt-field"><label class="dt-label">active</label><div class="dt-select">Boolean</div></div>
|
||||
<div class="dt-field"><label class="dt-label">notes</label><div class="dt-select">(skip)</div></div>
|
||||
</div>
|
||||
|
||||
<hr class="dt-divider">
|
||||
<h3>Format options</h3>
|
||||
|
||||
<!-- Standards preset radio (vertical). Demo state: preset has auto-switched
|
||||
to Custom because individual controls below diverge from the European base. -->
|
||||
<div class="dt-field">
|
||||
<label class="dt-label">Standards preset</label>
|
||||
<div style="display:flex;flex-direction:column;gap:8px;margin-top:4px">
|
||||
<span class="dt-radio" title="E.164 phones"><span class="dot"></span> US (default) — ISO 8601 dates · international-format phones (+1…) · USD</span>
|
||||
<span class="dt-radio"><span class="dot"></span> European — DMY input · INTL phones · EUR comma decimal <span class="dt-count-pill info" style="margin-left:4px">base</span></span>
|
||||
<span class="dt-radio"><span class="dot"></span> UK — DD/MM/YYYY · GB phones · Yes/No booleans</span>
|
||||
<span class="dt-radio"><span class="dot"></span> ISO Strict — ISO 8601 · bare-number currency · true/false</span>
|
||||
<span class="dt-radio"><span class="dot"></span> Legacy US — MM/DD/YYYY · National phones · Yes/No</span>
|
||||
<span class="dt-radio on"><span class="dot"></span> Custom — based on <strong>European</strong>, 2 controls changed <span class="dt-count-pill warn" style="margin-left:4px">modified</span></span>
|
||||
</div>
|
||||
<div class="dt-precedence" style="margin-top:10px">
|
||||
<span class="dt-mi">rule</span>
|
||||
<span>Individual controls win over the preset. You started from <strong>European</strong>, then changed <strong>Ambiguous input order</strong> and <strong>Decimal separator</strong> below — so the preset is now <strong>Custom</strong>. The controls' current values are what actually run.</span>
|
||||
</div>
|
||||
<div class="dt-help-text">Pick a published standard or regional convention as the baseline. Every option below is still individually overridable; overriding any one switches the preset to Custom.</div>
|
||||
</div>
|
||||
|
||||
<!-- Two-column format options -->
|
||||
<div class="dt-cols-2" style="margin-top:14px">
|
||||
<!-- Left column: Dates + Phones -->
|
||||
<div>
|
||||
<h4 style="margin-top:0"><strong>Dates</strong></h4>
|
||||
<div class="dt-field"><label class="dt-label">Output format</label><div class="dt-select">YYYY-MM-DD (ISO)</div></div>
|
||||
<div class="dt-field">
|
||||
<label class="dt-label">Ambiguous input order (e.g. 01/02/2024) <span class="dt-count-pill warn" style="margin-left:4px">changed</span></label>
|
||||
<div class="dt-radio-row">
|
||||
<span class="dt-radio on"><span class="dot"></span> MDY (US)</span>
|
||||
<span class="dt-radio"><span class="dot"></span> DMY (EU)</span>
|
||||
</div>
|
||||
<div class="dt-help-text">Winning value: <strong>MDY</strong>. Overrides the European base (DMY) — <code>01/02/2024</code> reads as <strong>2024-01-02</strong>.</div>
|
||||
</div>
|
||||
|
||||
<h4><strong>Phones</strong></h4>
|
||||
<div class="dt-field"><label class="dt-label" title="E.164">Output format</label><div class="dt-select" title="E.164">Standard international format (+15551234567)</div></div>
|
||||
<div class="dt-field">
|
||||
<label class="dt-label">Default region (ISO-2)</label>
|
||||
<div class="dt-input">US</div>
|
||||
<div class="dt-help-text">Region used when the input has no country code. US, GB, DE, etc.</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Right column: Currency + Names + Booleans -->
|
||||
<div>
|
||||
<h4 style="margin-top:0"><strong>Currency</strong></h4>
|
||||
<div class="dt-field">
|
||||
<label class="dt-label">Decimal separator in input <span class="dt-count-pill warn" style="margin-left:4px">changed</span></label>
|
||||
<div class="dt-radio-row">
|
||||
<span class="dt-radio on"><span class="dot"></span> dot (1,234.56)</span>
|
||||
<span class="dt-radio"><span class="dot"></span> comma (1.234,56)</span>
|
||||
</div>
|
||||
<div class="dt-help-text">Winning value: <strong>dot</strong>. Overrides the European base (comma) — <code>$1,234.5</code> reads as <strong>1234.50</strong>.</div>
|
||||
</div>
|
||||
<div class="dt-field" style="max-width:200px"><label class="dt-label">Round to decimals</label><div class="dt-input">2</div></div>
|
||||
<div class="dt-check"><span class="box"></span> Preserve original precision (don't round)</div>
|
||||
<div class="dt-check"><span class="box"></span> Preserve currency code (emit <code>USD 1234.56</code>, <code>EUR 99.00</code>, etc.)</div>
|
||||
|
||||
<h4><strong>Names</strong></h4>
|
||||
<div class="dt-field"><label class="dt-label">Casing</label><div class="dt-select">Title Case</div></div>
|
||||
|
||||
<h4><strong>Booleans</strong></h4>
|
||||
<div class="dt-field"><label class="dt-label">Output style</label><div class="dt-select">True/False</div></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<hr class="dt-divider">
|
||||
<button class="dt-btn dt-btn-primary dt-btn-block">Standardize Formats</button>
|
||||
|
||||
<hr class="dt-divider">
|
||||
|
||||
<!-- Results -->
|
||||
<h2>Results</h2>
|
||||
<div class="dt-metrics">
|
||||
<div class="dt-metric"><div class="label">Cells scanned</div><div class="value">92,210</div></div>
|
||||
<div class="dt-metric"><div class="label">Cells changed</div><div class="value">61,838</div></div>
|
||||
<div class="dt-metric"><div class="label">% changed</div><div class="value">67.1%</div></div>
|
||||
<div class="dt-metric"><div class="label">Unparseable</div><div class="value">47</div></div>
|
||||
</div>
|
||||
|
||||
<div class="dt-alert info">
|
||||
<span class="dt-mi">info</span>
|
||||
<span>47 cell(s) in typed columns didn't match a recognizable shape and were left as-is. See <strong>Unparseable cells</strong> below to review them, or re-classify the column to <strong>(skip)</strong>. (They aren't in the changes audit — nothing was changed.)</span>
|
||||
</div>
|
||||
|
||||
<!-- Unparseable cells surface (the alert points here; these are left-as-is, so they never appear in the CHANGES audit) -->
|
||||
<details class="dt-expander">
|
||||
<summary>Unparseable cells (47)</summary>
|
||||
<div class="dt-expander-body">
|
||||
<p class="dt-caption">Cells in typed columns that didn't match a recognizable shape and were left unchanged.</p>
|
||||
<div class="dt-table-wrap">
|
||||
<table class="dt-table">
|
||||
<thead><tr><th>row</th><th>column</th><th>field_type</th><th>value (left as-is)</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td>318</td><td>signup_date</td><td>date</td><td class="dt-cell-flag">soon</td></tr>
|
||||
<tr><td>902</td><td>phone</td><td>phone</td><td class="dt-cell-flag">ext. 4471</td></tr>
|
||||
<tr><td>1,544</td><td>amount</td><td>currency</td><td class="dt-cell-flag">TBD</td></tr>
|
||||
<tr><td>2,087</td><td>active</td><td>boolean</td><td class="dt-cell-flag">maybe</td></tr>
|
||||
<tr><td>3,610</td><td>signup_date</td><td>date</td><td class="dt-cell-flag">00/00/0000</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<p class="dt-caption" style="margin-top:8px">… and 42 more.</p>
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<!-- Changes by column -->
|
||||
<p style="margin-bottom:6px"><strong>Changes by column</strong></p>
|
||||
<div class="dt-table-wrap" style="max-width:520px">
|
||||
<table class="dt-table">
|
||||
<thead><tr><th>column</th><th>field_type</th><th>cells_changed</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td>amount</td><td>currency</td><td>17,902</td></tr>
|
||||
<tr><td>full_name</td><td>name</td><td>16,041</td></tr>
|
||||
<tr><td>phone</td><td>phone</td><td>14,388</td></tr>
|
||||
<tr><td>signup_date</td><td>date</td><td>11,205</td></tr>
|
||||
<tr><td>active</td><td>boolean</td><td>2,302</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<!-- Examples (first 25 changes) -->
|
||||
<p style="margin:14px 0 6px"><strong>Examples (first 25 changes)</strong></p>
|
||||
<div class="dt-table-wrap">
|
||||
<table class="dt-table">
|
||||
<thead><tr><th>row</th><th>column</th><th>field_type</th><th>before</th><th>after</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td>1</td><td>full_name</td><td>name</td><td class="dt-cell-del">jane DOE</td><td class="dt-cell-add">Jane Doe</td></tr>
|
||||
<tr><td>1</td><td>phone</td><td>phone</td><td class="dt-cell-del">(512) 555-0190</td><td class="dt-cell-add">+15125550190</td></tr>
|
||||
<tr><td>1</td><td>amount</td><td>currency</td><td class="dt-cell-del">$1,234.5</td><td class="dt-cell-add">1234.50</td></tr>
|
||||
<tr><td>1</td><td>signup_date</td><td>date</td><td class="dt-cell-del">01/04/2024</td><td class="dt-cell-add">2024-01-04</td></tr>
|
||||
<tr><td>1</td><td>active</td><td>boolean</td><td class="dt-cell-del">Y</td><td class="dt-cell-add">True</td></tr>
|
||||
<tr><td>2</td><td>full_name</td><td>name</td><td class="dt-cell-del">bob smith</td><td class="dt-cell-add">Bob Smith</td></tr>
|
||||
<tr><td>2</td><td>phone</td><td>phone</td><td class="dt-cell-del">720.555.7781</td><td class="dt-cell-add">+17205557781</td></tr>
|
||||
<tr><td>2</td><td>signup_date</td><td>date</td><td class="dt-cell-del">2024-2-11</td><td class="dt-cell-add">2024-02-11</td></tr>
|
||||
<tr><td>3</td><td>signup_date</td><td>date</td><td class="dt-cell-del">Mar 3, 2024</td><td class="dt-cell-add">2024-03-03</td></tr>
|
||||
<tr><td>4</td><td>amount</td><td>currency</td><td class="dt-cell-del">$7.999</td><td class="dt-cell-add">8.00</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<!-- Standardized preview -->
|
||||
<p style="margin:14px 0 6px"><strong>Standardized preview (first 10 rows)</strong></p>
|
||||
<p class="dt-caption" style="margin:0 0 6px">Showing 5 of 6 columns — <code>notes</code> is set to (skip), so it's omitted here.</p>
|
||||
<div class="dt-table-wrap">
|
||||
<table class="dt-table">
|
||||
<thead><tr><th class="idx"></th><th>full_name</th><th>phone</th><th>amount</th><th>signup_date</th><th>active</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td class="idx">0</td><td>Jane Doe</td><td>+15125550190</td><td>1234.50</td><td>2024-01-04</td><td>True</td></tr>
|
||||
<tr><td class="idx">1</td><td>Bob Smith</td><td>+17205557781</td><td>99.00</td><td>2024-02-11</td><td>True</td></tr>
|
||||
<tr><td class="idx">2</td><td>Alicia Reyes</td><td>+14155552233</td><td>45000.00</td><td>2024-03-03</td><td>False</td></tr>
|
||||
<tr><td class="idx">3</td><td>M. Okafor</td><td>+12125550148</td><td>8.00</td><td>2024-04-22</td><td>True</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<hr class="dt-divider">
|
||||
|
||||
<!-- Downloads (3 columns) -->
|
||||
<div class="dt-cols-3">
|
||||
<button class="dt-btn dt-btn-primary">Download standardized CSV</button>
|
||||
<button class="dt-btn">Download changes audit</button>
|
||||
<button class="dt-btn">Download config JSON</button>
|
||||
</div>
|
||||
|
||||
<!-- Next-step suggestion -->
|
||||
<div class="dt-next-step"><span class="dt-mi">arrow_forward</span><span>Formats standardized. Next, most files need: <a href="04_missing_handler.html">Fix Missing Values →</a></span><button class="dt-next-step-dismiss" title="Dismiss">✕</button></div>
|
||||
|
||||
</div>
|
||||
</main>
|
||||
</div>
|
||||
<footer class="dt-footer" id="dt-footer"></footer>
|
||||
<script src="shell.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
263
layout-review/04_missing_handler.html
Normal file
263
layout-review/04_missing_handler.html
Normal file
@@ -0,0 +1,263 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Layout review — Fix Missing Values</title>
|
||||
<link rel="stylesheet" href="app.css">
|
||||
</head>
|
||||
<body data-page="04_missing_handler">
|
||||
<div class="dt-app">
|
||||
<aside class="dt-sidebar" id="dt-sidebar"></aside>
|
||||
<main class="dt-main">
|
||||
<div class="dt-review-banner">
|
||||
<span class="dt-mi">visibility</span>
|
||||
<span>Static layout preview of <strong>Fix Missing Values</strong>, shown with a file imported and a completed run (per-column missingness profile + before/after results). <a href="index.html">All pages →</a></span>
|
||||
</div>
|
||||
<div class="dt-main-inner">
|
||||
|
||||
<!-- Tool header -->
|
||||
<div class="dt-tool-header">
|
||||
<h1>Fix Missing Values</h1>
|
||||
<div class="dt-tool-header-actions">
|
||||
<span class="dt-privacy-pill">
|
||||
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor">
|
||||
<rect x="4" y="11" width="16" height="10" rx="2"/>
|
||||
<path d="M8 11V7a4 4 0 018 0v4"/>
|
||||
</svg>
|
||||
Runs 100% locally
|
||||
</span>
|
||||
<button class="dt-help-btn"><span class="dt-mi">help_outline</span> Help</button>
|
||||
</div>
|
||||
</div>
|
||||
<p class="dt-tool-caption">Find blank cells (even hidden ones) and fill them in or remove them.</p>
|
||||
|
||||
<div class="dt-spacer"></div>
|
||||
|
||||
<!-- File pickup banner (using file from upload screen) -->
|
||||
<div class="dt-alert info">
|
||||
<span class="dt-mi">description</span>
|
||||
<span>Using <strong>survey_responses.csv</strong> from the upload screen.</span>
|
||||
</div>
|
||||
<button class="dt-btn" style="margin-bottom:4px">Use a different file</button>
|
||||
|
||||
<!-- Preview expander (collapsed after a result exists) -->
|
||||
<details class="dt-expander">
|
||||
<summary>Preview: survey_responses.csv</summary>
|
||||
<div class="dt-expander-body">
|
||||
<p class="dt-caption">2,150 rows, 6 columns</p>
|
||||
<div class="dt-table-wrap">
|
||||
<table class="dt-table">
|
||||
<thead><tr><th class="idx"></th><th>respondent_id</th><th>age</th><th>region</th><th>income</th><th>satisfaction</th><th>comments</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td class="idx">0</td><td>R-1001</td><td>34</td><td>West</td><td>52000</td><td>4</td><td>great service</td></tr>
|
||||
<tr><td class="idx">1</td><td>R-1002</td><td class="dt-cell-flag">N/A</td><td>East</td><td class="dt-cell-flag"></td><td>3</td><td class="dt-cell-flag">?</td></tr>
|
||||
<tr><td class="idx">2</td><td>R-1003</td><td>41</td><td class="dt-cell-flag">-</td><td>61000</td><td class="dt-cell-flag">NULL</td><td>none</td></tr>
|
||||
<tr><td class="idx">3</td><td>R-1004</td><td>29</td><td>South</td><td class="dt-cell-flag">N/A</td><td>5</td><td>quick</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<hr class="dt-divider">
|
||||
|
||||
<!-- Missingness profile — always visible: see the damage before configuring -->
|
||||
<h2>Missingness profile</h2>
|
||||
<div class="dt-metrics">
|
||||
<div class="dt-metric"><div class="label">Rows</div><div class="value">2,150</div></div>
|
||||
<div class="dt-metric"><div class="label">Cells missing</div><div class="value">1,043</div></div>
|
||||
<div class="dt-metric"><div class="label">% cells missing</div><div class="value">8.1%</div></div>
|
||||
<div class="dt-metric"><div class="label">Complete rows</div><div class="value">1,388</div></div>
|
||||
</div>
|
||||
<div class="dt-table-wrap">
|
||||
<table class="dt-table">
|
||||
<thead><tr><th>column</th><th>dtype</th><th>missing</th><th>missing_pct</th><th>disguised</th><th>has_missing</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td>respondent_id</td><td>object</td><td>0</td><td>0.0%</td><td>0</td><td>False</td></tr>
|
||||
<tr><td>age</td><td>float64</td><td>187</td><td>8.7%</td><td>61</td><td>True</td></tr>
|
||||
<tr><td>region</td><td>object</td><td>142</td><td>6.6%</td><td>142</td><td>True</td></tr>
|
||||
<tr><td>income</td><td>float64</td><td>329</td><td>15.3%</td><td>118</td><td>True</td></tr>
|
||||
<tr><td>satisfaction</td><td>float64</td><td>95</td><td>4.4%</td><td>40</td><td>True</td></tr>
|
||||
<tr><td>comments</td><td>object</td><td>290</td><td>13.5%</td><td>290</td><td>True</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<hr class="dt-divider">
|
||||
|
||||
<!-- Options expander (Strategy) — configuration follows the diagnostic -->
|
||||
<details class="dt-expander" open>
|
||||
<summary>Options</summary>
|
||||
<div class="dt-expander-body">
|
||||
|
||||
<h3>Strategy</h3>
|
||||
<div class="dt-precedence">
|
||||
<span class="dt-mi">layers</span>
|
||||
<span>Resolution order: <strong>per-column override</strong> → <strong>global strategy</strong> → <strong>preset</strong>. The most specific setting wins; layers it overrides are dimmed.</span>
|
||||
</div>
|
||||
<div class="dt-field">
|
||||
<label class="dt-label">Preset</label>
|
||||
<div class="dt-help-text" style="color:var(--warn);display:flex;align-items:center;gap:5px;margin-bottom:8px"><span class="dt-mi" style="font-family:'Material Symbols Outlined';font-size:15px;line-height:1">info</span> Overridden by <strong>Global strategy → median</strong> (set under Advanced options). Presets apply only when global is “(use preset)”.</div>
|
||||
<div class="dt-radio-row is-overridden" style="flex-direction:column;gap:10px">
|
||||
<span class="dt-radio on"><span class="dot"></span> detect-only (standardize sentinels to NaN, no fill or drop)</span>
|
||||
<span class="dt-radio"><span class="dot"></span> safe-fill (numeric → median, categorical → mode)</span>
|
||||
<span class="dt-radio"><span class="dot"></span> drop-incomplete (drop any row with missing)</span>
|
||||
</div>
|
||||
<div class="dt-help-text">detect-only: replace 'N/A', '-', 'NULL', etc. with real NaN, then stop. safe-fill: also fill — numeric columns with median, others with mode. drop-incomplete: also drop every row that has any missing cell.</div>
|
||||
</div>
|
||||
|
||||
<!-- Advanced options expander (open — most informative) -->
|
||||
<details class="dt-expander" open>
|
||||
<summary>Advanced options</summary>
|
||||
<div class="dt-expander-body">
|
||||
<div class="dt-cols-2">
|
||||
<div>
|
||||
<h4>Detection</h4>
|
||||
<div class="dt-check on"><span class="box"><span class="dt-mi">check</span></span> Standardize disguised nulls to NaN</div>
|
||||
<div class="dt-field">
|
||||
<label class="dt-label" title="Sentinel values">Blanks in disguise (N/A, dash, NULL) — comma-separated</label>
|
||||
<div class="dt-input">N/A, n/a, NA, NULL, null, None, -, --, ?, #N/A</div>
|
||||
<div class="dt-help-text">Text that really means “empty.” Matched case-insensitively after stripping whitespace.</div>
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
<h4>Strategy override</h4>
|
||||
<div class="dt-field">
|
||||
<label class="dt-label">Global strategy</label>
|
||||
<div class="dt-select">median</div>
|
||||
<div class="dt-help-text">drop_row / drop_col use the thresholds below. mean / median / interpolate are numeric only — non-numeric columns fall back to the categorical strategy.</div>
|
||||
</div>
|
||||
<div class="dt-field">
|
||||
<label class="dt-label">Categorical fallback (for non-numeric columns)</label>
|
||||
<div class="dt-select">mode</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<h4>Drop thresholds</h4>
|
||||
<div class="dt-cols-2">
|
||||
<div class="dt-field">
|
||||
<label class="dt-label">Row drop threshold (drop rows with ≥ this fraction missing across selected cols)</label>
|
||||
<div class="dt-slider"><div class="track"><div class="fill" style="width:100%"></div><div class="knob" style="left:calc(100% - 8px)"></div></div><div class="val">1.00</div></div>
|
||||
</div>
|
||||
<div class="dt-field">
|
||||
<label class="dt-label">Column drop threshold (drop columns with ≥ this fraction missing)</label>
|
||||
<div class="dt-slider"><div class="track"><div class="fill" style="width:100%"></div><div class="knob" style="left:calc(100% - 8px)"></div></div><div class="val">1.00</div></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<h4>Scope</h4>
|
||||
<div class="dt-field">
|
||||
<label class="dt-label">Columns to handle (default: all)</label>
|
||||
<div class="dt-multiselect">
|
||||
<span class="dt-ms-chip">respondent_id <span class="x">✕</span></span>
|
||||
<span class="dt-ms-chip">age <span class="x">✕</span></span>
|
||||
<span class="dt-ms-chip">region <span class="x">✕</span></span>
|
||||
<span class="dt-ms-chip">income <span class="x">✕</span></span>
|
||||
<span class="dt-ms-chip">satisfaction <span class="x">✕</span></span>
|
||||
<span class="dt-ms-chip">comments <span class="x">✕</span></span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="dt-field">
|
||||
<label class="dt-label">Columns to skip</label>
|
||||
<div class="dt-multiselect"><span class="dt-ms-placeholder">Choose columns</span></div>
|
||||
</div>
|
||||
|
||||
<h4>Per-column strategy overrides (optional)</h4>
|
||||
<p class="dt-caption">Set a different strategy for specific columns. Leave any row blank to use the global strategy.</p>
|
||||
<div class="dt-table-wrap">
|
||||
<table class="dt-table">
|
||||
<thead><tr><th>Column</th><th>Override</th><th>Resolves to</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td>age</td><td><span class="dt-select" style="display:inline-block;min-width:160px;padding:4px 24px 4px 10px;color:var(--ink-tertiary)">(global)</span></td><td>median <span style="color:var(--ink-tertiary);font-size:11px">· global</span></td></tr>
|
||||
<tr><td>region</td><td><span class="dt-select" style="display:inline-block;min-width:160px;padding:4px 24px 4px 10px;color:var(--ink-tertiary)">(global)</span></td><td>mode <span style="color:var(--ink-tertiary);font-size:11px">· global → categorical fallback</span></td></tr>
|
||||
<tr><td>income</td><td><span class="dt-select" style="display:inline-block;min-width:160px;padding:4px 24px 4px 10px;color:var(--ink-tertiary)">(global)</span></td><td>median <span style="color:var(--ink-tertiary);font-size:11px">· global</span></td></tr>
|
||||
<tr><td>satisfaction</td><td><span class="dt-select" style="display:inline-block;min-width:160px;padding:4px 24px 4px 10px;color:var(--ink-tertiary)">(global)</span></td><td>median <span style="color:var(--ink-tertiary);font-size:11px">· global</span></td></tr>
|
||||
<tr><td>comments</td><td><span class="dt-select" style="display:inline-block;min-width:160px;padding:4px 24px 4px 10px">constant</span></td><td><strong>constant</strong> <span style="color:var(--ink-tertiary);font-size:11px">· this column</span></td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<hr class="dt-divider">
|
||||
<button class="dt-btn dt-btn-primary dt-btn-block">Handle Missing Values</button>
|
||||
|
||||
<hr class="dt-divider">
|
||||
|
||||
<!-- Results -->
|
||||
<div id="missing-results-anchor"></div>
|
||||
<h2>Results</h2>
|
||||
<div class="dt-metrics">
|
||||
<div class="dt-metric"><div class="label">Sentinels → NaN</div><div class="value">651</div></div>
|
||||
<div class="dt-metric"><div class="label">Cells filled</div><div class="value">1,043</div></div>
|
||||
<div class="dt-metric"><div class="label">Rows dropped</div><div class="value">0</div></div>
|
||||
<div class="dt-metric"><div class="label">Columns dropped</div><div class="value">0</div></div>
|
||||
</div>
|
||||
|
||||
<p><strong>Missingness — before vs. after</strong></p>
|
||||
<div class="dt-table-wrap">
|
||||
<table class="dt-table">
|
||||
<thead><tr><th>column</th><th>before_missing</th><th>before_pct</th><th>after_missing</th><th>after_pct</th><th>strategy</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td>respondent_id</td><td>0</td><td>0.0</td><td>0</td><td>0.0</td><td class="dt-cell-flag">—</td></tr>
|
||||
<tr><td>age</td><td class="dt-cell-flag">187</td><td>8.7</td><td class="dt-cell-add">0</td><td class="dt-cell-add">0.0</td><td>median</td></tr>
|
||||
<tr><td>region</td><td class="dt-cell-flag">142</td><td>6.6</td><td class="dt-cell-add">0</td><td class="dt-cell-add">0.0</td><td>mode</td></tr>
|
||||
<tr><td>income</td><td class="dt-cell-flag">329</td><td>15.3</td><td class="dt-cell-add">0</td><td class="dt-cell-add">0.0</td><td>median</td></tr>
|
||||
<tr><td>satisfaction</td><td class="dt-cell-flag">95</td><td>4.4</td><td class="dt-cell-add">0</td><td class="dt-cell-add">0.0</td><td>median</td></tr>
|
||||
<tr><td>comments</td><td class="dt-cell-flag">290</td><td>13.5</td><td class="dt-cell-add">0</td><td class="dt-cell-add">0.0</td><td>constant</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<p><strong>Audit (first 50 changes)</strong></p>
|
||||
<div class="dt-table-wrap">
|
||||
<table class="dt-table">
|
||||
<thead><tr><th>row</th><th>column</th><th>old_value</th><th>new_value</th><th>reason</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td>2</td><td>age</td><td class="dt-cell-flag">N/A</td><td class="dt-cell-add">37.0</td><td>fill: median</td></tr>
|
||||
<tr><td>2</td><td>income</td><td class="dt-cell-flag">(blank)</td><td class="dt-cell-add">54000.0</td><td>fill: median</td></tr>
|
||||
<tr><td>2</td><td>comments</td><td class="dt-cell-flag">?</td><td class="dt-cell-add">(no comment)</td><td>fill: constant</td></tr>
|
||||
<tr><td>3</td><td>region</td><td class="dt-cell-flag">-</td><td class="dt-cell-add">West</td><td>fill: mode</td></tr>
|
||||
<tr><td>3</td><td>satisfaction</td><td class="dt-cell-flag">NULL</td><td class="dt-cell-add">4.0</td><td>fill: median</td></tr>
|
||||
<tr><td>4</td><td>income</td><td class="dt-cell-flag">N/A</td><td class="dt-cell-add">54000.0</td><td>fill: median</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<p class="dt-caption">… and 1,037 more (download the full audit below).</p>
|
||||
|
||||
<p><strong>Handled preview (first 10 rows)</strong></p>
|
||||
<div class="dt-table-wrap">
|
||||
<table class="dt-table">
|
||||
<thead><tr><th class="idx"></th><th>respondent_id</th><th>age</th><th>region</th><th>income</th><th>satisfaction</th><th>comments</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td class="idx">0</td><td>R-1001</td><td>34.0</td><td>West</td><td>52000.0</td><td>4.0</td><td>great service</td></tr>
|
||||
<tr><td class="idx">1</td><td>R-1002</td><td class="dt-cell-add">37.0</td><td>East</td><td class="dt-cell-add">54000.0</td><td>3.0</td><td class="dt-cell-add">(no comment)</td></tr>
|
||||
<tr><td class="idx">2</td><td>R-1003</td><td>41.0</td><td class="dt-cell-add">West</td><td>61000.0</td><td class="dt-cell-add">4.0</td><td>none</td></tr>
|
||||
<tr><td class="idx">3</td><td>R-1004</td><td>29.0</td><td>South</td><td class="dt-cell-add">54000.0</td><td>5.0</td><td>quick</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<hr class="dt-divider">
|
||||
|
||||
<!-- Downloads (html_download_button anchors) -->
|
||||
<div class="dt-cols-3">
|
||||
<button class="dt-btn dt-btn-primary">Download handled CSV</button>
|
||||
<button class="dt-btn">Download changes audit</button>
|
||||
<button class="dt-btn">Download config JSON</button>
|
||||
</div>
|
||||
|
||||
<div class="dt-next-step"><span class="dt-mi">arrow_forward</span><span>Missing values handled. Next, most files need: <a href="01_deduplicator.html">Find Duplicates →</a></span><button class="dt-next-step-dismiss" title="Dismiss">✕</button></div>
|
||||
|
||||
</div>
|
||||
</main>
|
||||
</div>
|
||||
<footer class="dt-footer" id="dt-footer"></footer>
|
||||
<script src="shell.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
221
layout-review/05_column_mapper.html
Normal file
221
layout-review/05_column_mapper.html
Normal file
@@ -0,0 +1,221 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Layout review — Map Columns</title>
|
||||
<link rel="stylesheet" href="app.css">
|
||||
</head>
|
||||
<body data-page="05_column_mapper">
|
||||
<div class="dt-app">
|
||||
<aside class="dt-sidebar" id="dt-sidebar"></aside>
|
||||
<main class="dt-main">
|
||||
<div class="dt-review-banner">
|
||||
<span class="dt-mi">visibility</span>
|
||||
<span>Static layout preview of <strong>Map Columns</strong>, shown with a file imported, an interactive target schema + mapping configured, and a completed run (results + mapped preview). <a href="index.html">All pages →</a></span>
|
||||
</div>
|
||||
<div class="dt-main-inner">
|
||||
|
||||
<!-- Tool header -->
|
||||
<div class="dt-tool-header">
|
||||
<h1>Map Columns</h1>
|
||||
<div class="dt-tool-header-actions">
|
||||
<span class="dt-privacy-pill">
|
||||
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor">
|
||||
<rect x="4" y="11" width="16" height="10" rx="2"/>
|
||||
<path d="M8 11V7a4 4 0 018 0v4"/>
|
||||
</svg>
|
||||
Runs 100% locally
|
||||
</span>
|
||||
<button class="dt-help-btn"><span class="dt-mi">help_outline</span> Help</button>
|
||||
</div>
|
||||
</div>
|
||||
<p class="dt-tool-caption">Rename columns, change their order, and set each one as text, number, or date.</p>
|
||||
|
||||
<div class="dt-spacer"></div>
|
||||
|
||||
<!-- File pickup banner (using file from upload screen) -->
|
||||
<div class="dt-alert info">
|
||||
<span class="dt-mi">description</span>
|
||||
<span>Using <strong>crm_contacts_raw.csv</strong> from the upload screen.</span>
|
||||
</div>
|
||||
<button class="dt-btn" style="margin-bottom:4px">Use a different file</button>
|
||||
|
||||
<!-- Preview expander (collapsed after a result exists) -->
|
||||
<details class="dt-expander">
|
||||
<summary>Preview: crm_contacts_raw.csv</summary>
|
||||
<div class="dt-expander-body">
|
||||
<p class="dt-caption">4,210 rows, 6 columns</p>
|
||||
<div class="dt-table-wrap">
|
||||
<table class="dt-table">
|
||||
<thead><tr><th class="idx"></th><th>Full Name</th><th>EmailAddr</th><th>Phone #</th><th>Signup</th><th>Amount Spent</th><th>Notes</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td class="idx">0</td><td>Jane Doe</td><td>jane@acme.io</td><td>512-555-0190</td><td>01/04/2024</td><td>$1,204.50</td><td>VIP</td></tr>
|
||||
<tr><td class="idx">1</td><td>Bob Smith</td><td>bob@globex.com</td><td>720-555-7781</td><td>02/11/2024</td><td>$88.00</td><td></td></tr>
|
||||
<tr><td class="idx">2</td><td>Carla Reyes</td><td>carla@initech.net</td><td>415-555-3322</td><td>03/02/2024</td><td>$612.10</td><td>renewal</td></tr>
|
||||
<tr><td class="idx">3</td><td>Dev Patel</td><td>dev@umbrella.co</td><td>206-555-9043</td><td>03/19/2024</td><td>$0.00</td><td></td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<hr class="dt-divider">
|
||||
|
||||
<!-- Options expander (open — heart of the tool) -->
|
||||
<details class="dt-expander" open>
|
||||
<summary>Options</summary>
|
||||
<div class="dt-expander-body">
|
||||
|
||||
<!-- ===== Target schema ===== -->
|
||||
<h3 style="margin-top:0">Target schema</h3>
|
||||
<div class="dt-field">
|
||||
<label class="dt-label">How would you like to define the target schema?</label>
|
||||
<div class="dt-radio-row" style="flex-direction:column; gap:8px">
|
||||
<span class="dt-radio on"><span class="dot"></span> Build interactively (start from current columns)</span>
|
||||
<span class="dt-radio"><span class="dot"></span> Import schema JSON</span>
|
||||
<span class="dt-radio"><span class="dot"></span> Skip (rename / convert types only — no schema)</span>
|
||||
</div>
|
||||
<div class="dt-help-text">An interactive build is fastest for one-off cleanup. Import a JSON when you have a fixed contract (a CRM import format, db schema). Skip when you only want to rename or convert the type of specific columns.</div>
|
||||
</div>
|
||||
|
||||
<p class="dt-caption">Edit the table to define your target schema. Add rows for fields the input doesn't have yet (with a default), or remove rows for columns you want to drop.</p>
|
||||
|
||||
<!-- Schema editor (st.data_editor, num_rows=dynamic) -->
|
||||
<div class="dt-table-wrap">
|
||||
<table class="dt-table">
|
||||
<thead><tr><th>Target name</th><th>Type</th><th>Required</th><th>Default (for added cols)</th><th>Aliases (comma-sep, helps fuzzy-match)</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td>full_name</td><td>string</td><td>✗</td><td></td><td>Full Name, name</td></tr>
|
||||
<tr><td>email</td><td>string</td><td>✓</td><td></td><td>EmailAddr, email_address</td></tr>
|
||||
<tr><td>phone</td><td>string</td><td>✗</td><td></td><td>Phone #, tel</td></tr>
|
||||
<tr><td>signup_date</td><td>date</td><td>✗</td><td></td><td>Signup</td></tr>
|
||||
<tr><td>amount_spent</td><td>float</td><td>✗</td><td>0.0</td><td>Amount Spent</td></tr>
|
||||
<tr><td>source</td><td>string</td><td>✗</td><td>crm-import</td><td></td></tr>
|
||||
<tr><td style="color:var(--ink-tertiary)"><span class="dt-mi" style="font-size:16px;vertical-align:-3px">add</span> add row</td><td></td><td></td><td></td><td></td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<p class="dt-caption">6 target fields · 1 added field (<code>source</code>) not present in the input.</p>
|
||||
|
||||
<hr class="dt-divider">
|
||||
|
||||
<!-- ===== Mapping ===== -->
|
||||
<!-- Mapping follows the schema directly: define the schema, then map sources onto it. -->
|
||||
<h3>Mapping</h3>
|
||||
<!-- schema is set → source→target selectbox editor with auto-suggested flag -->
|
||||
<div class="dt-table-wrap">
|
||||
<table class="dt-table">
|
||||
<thead><tr><th>Source</th><th>Target</th><th>Auto-suggested</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td>Full Name</td><td>full_name</td><td>✓</td></tr>
|
||||
<tr><td>EmailAddr</td><td>email</td><td>✓</td></tr>
|
||||
<tr><td>Phone #</td><td>phone</td><td>✓</td></tr>
|
||||
<tr><td>Signup</td><td>signup_date</td><td>✓</td></tr>
|
||||
<tr><td>Amount Spent</td><td>amount_spent</td><td>✓</td></tr>
|
||||
<tr><td>Notes</td><td>(unmapped)</td><td>✗</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<p class="dt-caption">Pick a target for each source column. <code>Notes</code> stays unmapped — with the keep-extras strategy it is kept as-is. <code>source</code> is added from the schema default.</p>
|
||||
|
||||
<hr class="dt-divider">
|
||||
|
||||
<!-- ===== Strategy ===== -->
|
||||
<!-- Strategy is a modifier on the mapping above (strictness: keep/drop extras, coerce, reorder), so it comes after the user can see what it acts on. -->
|
||||
<h3>Strategy</h3>
|
||||
<div class="dt-field">
|
||||
<label class="dt-label">Preset</label>
|
||||
<div class="dt-radio-row" style="flex-direction:column; gap:8px">
|
||||
<span class="dt-radio"><span class="dot"></span> rename-only (just rename, leave types alone, keep extras)</span>
|
||||
<span class="dt-radio"><span class="dot"></span> lenient-schema (rename + convert types + reorder, keep extras)</span>
|
||||
<span class="dt-radio"><span class="dot"></span> strict-schema (rename + convert types + reorder, drop extras) <span class="dt-count-pill info" style="margin-left:4px">base</span></span>
|
||||
<span class="dt-radio on"><span class="dot"></span> Custom — based on <strong>strict-schema</strong>, 1 control changed <span class="dt-count-pill warn" style="margin-left:4px">modified</span></span>
|
||||
</div>
|
||||
<div class="dt-precedence" style="margin-top:10px">
|
||||
<span class="dt-mi">rule</span>
|
||||
<span>Individual Advanced controls win over the preset. You started from <strong>strict-schema</strong>, then changed <strong>Unmapped source columns</strong> to <strong>keep</strong> below — so the preset is now <strong>Custom</strong>. The controls' current values are what actually run.</span>
|
||||
</div>
|
||||
<div class="dt-help-text">Pick a strategy as the baseline. Every Advanced toggle below is still individually overridable; overriding any one switches the preset to Custom.</div>
|
||||
</div>
|
||||
|
||||
<!-- Advanced options expander -->
|
||||
<details class="dt-expander" open>
|
||||
<summary>Advanced options</summary>
|
||||
<div class="dt-expander-body">
|
||||
<div class="dt-cols-2">
|
||||
<div>
|
||||
<div class="dt-field">
|
||||
<label class="dt-label">Unmapped source columns <span class="dt-count-pill warn" style="margin-left:4px">changed</span></label>
|
||||
<div class="dt-select">keep</div>
|
||||
<div class="dt-help-text">Winning value: <strong>keep</strong>. Overrides the strict-schema base (drop) — so <code>Notes</code> survives into the output.</div>
|
||||
</div>
|
||||
<div class="dt-check on" title="coerce types per schema"><span class="box"><span class="dt-mi">check</span></span> Convert each column to the right type</div>
|
||||
<div class="dt-check on"><span class="box"><span class="dt-mi">check</span></span> Reorder to schema order</div>
|
||||
</div>
|
||||
<div>
|
||||
<div class="dt-check on"><span class="box"><span class="dt-mi">check</span></span> Auto-infer mapping (fuzzy match)</div>
|
||||
<div class="dt-field">
|
||||
<label class="dt-label">Fuzzy match threshold</label>
|
||||
<div class="dt-slider"><div class="track"><div class="fill" style="width:80%"></div><div class="knob" style="left:80%"></div></div><div class="val">0.80</div></div>
|
||||
</div>
|
||||
<div class="dt-check on"><span class="box"><span class="dt-mi">check</span></span> Enforce required fields</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<hr class="dt-divider">
|
||||
<button class="dt-btn dt-btn-primary dt-btn-block">Apply Column Mapping</button>
|
||||
|
||||
<hr class="dt-divider">
|
||||
|
||||
<!-- ===== Results ===== -->
|
||||
<div id="colmap-results-anchor" style="height:1px"></div>
|
||||
<h2>Results</h2>
|
||||
<div class="dt-metrics">
|
||||
<div class="dt-metric"><div class="label">Renamed</div><div class="value">5</div></div>
|
||||
<div class="dt-metric"><div class="label">Dropped</div><div class="value">0</div></div>
|
||||
<div class="dt-metric"><div class="label">Added</div><div class="value">1</div></div>
|
||||
<div class="dt-metric"><div class="label">Coerce fails</div><div class="value">3</div></div>
|
||||
</div>
|
||||
|
||||
<div class="dt-alert info"><span class="dt-mi">info</span><span>Added (with defaults): <code>source</code></span></div>
|
||||
<div class="dt-alert warn"><span class="dt-mi">warning</span><span>Some cells could not be coerced and were left as NaN: amount_spent (3)</span></div>
|
||||
|
||||
<p><strong>Mapped preview (first 10 rows)</strong></p>
|
||||
<div class="dt-table-wrap">
|
||||
<table class="dt-table">
|
||||
<thead><tr><th class="idx"></th><th class="dt-cell-add">full_name</th><th>email</th><th>phone</th><th>signup_date</th><th>amount_spent</th><th class="dt-cell-add">source</th><th>Notes</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td class="idx">0</td><td>Jane Doe</td><td>jane@acme.io</td><td>512-555-0190</td><td>2024-01-04</td><td>1204.5</td><td>crm-import</td><td>VIP</td></tr>
|
||||
<tr><td class="idx">1</td><td>Bob Smith</td><td>bob@globex.com</td><td>720-555-7781</td><td>2024-02-11</td><td>88.0</td><td>crm-import</td><td></td></tr>
|
||||
<tr><td class="idx">2</td><td>Carla Reyes</td><td>carla@initech.net</td><td>415-555-3322</td><td>2024-03-02</td><td>612.1</td><td>crm-import</td><td>renewal</td></tr>
|
||||
<tr><td class="idx">3</td><td>Dev Patel</td><td>dev@umbrella.co</td><td>206-555-9043</td><td>2024-03-19</td><td>0.0</td><td>crm-import</td><td></td></tr>
|
||||
<tr><td class="idx">4</td><td>Mei Lin</td><td>mei@hooli.com</td><td>503-555-1188</td><td>2024-04-07</td><td class="dt-cell-flag">NaN</td><td>crm-import</td><td>trial</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<hr class="dt-divider">
|
||||
|
||||
<!-- Downloads (3 columns) -->
|
||||
<div class="dt-cols-3">
|
||||
<button class="dt-btn dt-btn-primary">Download mapped CSV</button>
|
||||
<button class="dt-btn">Download mapping audit</button>
|
||||
<button class="dt-btn">Download config JSON</button>
|
||||
</div>
|
||||
|
||||
<!-- Next-step suggestion -->
|
||||
<div class="dt-next-step"><span class="dt-mi">arrow_forward</span><span>Columns mapped. <a href="home.html">Run the recommended clean →</a></span><button class="dt-next-step-dismiss" title="Dismiss">✕</button></div>
|
||||
|
||||
</div>
|
||||
</main>
|
||||
</div>
|
||||
<footer class="dt-footer" id="dt-footer"></footer>
|
||||
<script src="shell.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
55
layout-review/06_outlier_detector.html
Normal file
55
layout-review/06_outlier_detector.html
Normal file
@@ -0,0 +1,55 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Layout review — Find Unusual Values</title>
|
||||
<link rel="stylesheet" href="app.css">
|
||||
</head>
|
||||
<body data-page="06_outlier_detector">
|
||||
<div class="dt-app">
|
||||
<aside class="dt-sidebar" id="dt-sidebar"></aside>
|
||||
<main class="dt-main">
|
||||
<div class="dt-review-banner">
|
||||
<span class="dt-mi">visibility</span>
|
||||
<span>Static layout preview of <strong>Find Unusual Values</strong> — a <strong>Coming Soon</strong> tool. The page is a stub: a "coming soon" notice, a plain-English list of what the tool will do, and a single "Notify me" action. <a href="index.html">All pages →</a></span>
|
||||
</div>
|
||||
<div class="dt-main-inner">
|
||||
|
||||
<!-- Tool header -->
|
||||
<div class="dt-tool-header">
|
||||
<h1>Find Unusual Values</h1>
|
||||
<button class="dt-help-btn"><span class="dt-mi">help_outline</span> Help</button>
|
||||
</div>
|
||||
<p class="dt-tool-caption">Spot values that look wrong — way too high, too low, or breaking your rules.</p>
|
||||
|
||||
<div class="dt-spacer"></div>
|
||||
|
||||
<!-- Coming-soon notice (st.info) -->
|
||||
<div class="dt-alert info">
|
||||
<span class="dt-mi">info</span>
|
||||
<span>This tool is coming soon.</span>
|
||||
</div>
|
||||
|
||||
<!-- What it will do (st.markdown) -->
|
||||
<p><strong>What it will do:</strong></p>
|
||||
<ul>
|
||||
<li>Find values that are unusually high or low for a column</li>
|
||||
<li>Spot values that break the rules you set (out of range, wrong type)</li>
|
||||
<li>Choose how sensitive the check is</li>
|
||||
<li>Flag unusual rows by adding a column, without changing your data</li>
|
||||
<li>Cap extreme values at a limit you choose</li>
|
||||
<li>See a summary of how many values were flagged</li>
|
||||
</ul>
|
||||
|
||||
<hr class="dt-divider">
|
||||
|
||||
<button class="dt-btn dt-btn-primary"><span class="dt-mi">notifications</span> Notify me when this ships</button>
|
||||
|
||||
</div>
|
||||
</main>
|
||||
</div>
|
||||
<footer class="dt-footer" id="dt-footer"></footer>
|
||||
<script src="shell.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
55
layout-review/07_multi_file_merger.html
Normal file
55
layout-review/07_multi_file_merger.html
Normal file
@@ -0,0 +1,55 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Layout review — Combine Files</title>
|
||||
<link rel="stylesheet" href="app.css">
|
||||
</head>
|
||||
<body data-page="07_multi_file_merger">
|
||||
<div class="dt-app">
|
||||
<aside class="dt-sidebar" id="dt-sidebar"></aside>
|
||||
<main class="dt-main">
|
||||
<div class="dt-review-banner">
|
||||
<span class="dt-mi">visibility</span>
|
||||
<span>Static layout preview of <strong>Combine Files</strong> — a <strong>Coming Soon</strong> tool. The page is a stub: a "coming soon" notice, a plain-English list of what the tool will do, and a single "Notify me" action. <a href="index.html">All pages →</a></span>
|
||||
</div>
|
||||
<div class="dt-main-inner">
|
||||
|
||||
<!-- Tool header -->
|
||||
<div class="dt-tool-header">
|
||||
<h1>Combine Files</h1>
|
||||
<button class="dt-help-btn"><span class="dt-mi">help_outline</span> Help</button>
|
||||
</div>
|
||||
<p class="dt-tool-caption">Combine several CSV or Excel files into one — even if columns differ.</p>
|
||||
|
||||
<div class="dt-spacer"></div>
|
||||
|
||||
<!-- Coming-soon notice (st.info) -->
|
||||
<div class="dt-alert info">
|
||||
<span class="dt-mi">info</span>
|
||||
<span>This tool is coming soon.</span>
|
||||
</div>
|
||||
|
||||
<!-- What it will do (st.markdown) -->
|
||||
<p><strong>What it will do:</strong></p>
|
||||
<ul>
|
||||
<li>Import several CSV or Excel files at once</li>
|
||||
<li>Line up columns automatically by matching their names</li>
|
||||
<li>Stack files on top of each other into one long file</li>
|
||||
<li>Merge files side by side using shared key columns</li>
|
||||
<li>Handle columns that don't match (fill the gaps with blanks or drop them)</li>
|
||||
<li>Add a column showing which file each row came from</li>
|
||||
</ul>
|
||||
|
||||
<hr class="dt-divider">
|
||||
|
||||
<button class="dt-btn dt-btn-primary"><span class="dt-mi">notifications</span> Notify me when this ships</button>
|
||||
|
||||
</div>
|
||||
</main>
|
||||
</div>
|
||||
<footer class="dt-footer" id="dt-footer"></footer>
|
||||
<script src="shell.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
55
layout-review/08_validator_reporter.html
Normal file
55
layout-review/08_validator_reporter.html
Normal file
@@ -0,0 +1,55 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Layout review — Quality Check</title>
|
||||
<link rel="stylesheet" href="app.css">
|
||||
</head>
|
||||
<body data-page="08_validator_reporter">
|
||||
<div class="dt-app">
|
||||
<aside class="dt-sidebar" id="dt-sidebar"></aside>
|
||||
<main class="dt-main">
|
||||
<div class="dt-review-banner">
|
||||
<span class="dt-mi">visibility</span>
|
||||
<span>Static layout preview of <strong>Quality Check</strong> — a <strong>Coming Soon</strong> tool. The page is a stub: a "coming soon" notice, a plain-English list of what the tool will do, and a single "Notify me" action. <a href="index.html">All pages →</a></span>
|
||||
</div>
|
||||
<div class="dt-main-inner">
|
||||
|
||||
<!-- Tool header -->
|
||||
<div class="dt-tool-header">
|
||||
<h1>Quality Check</h1>
|
||||
<button class="dt-help-btn"><span class="dt-mi">help_outline</span> Help</button>
|
||||
</div>
|
||||
<p class="dt-tool-caption">Check your file against rules you set, and export a PDF or Excel report.</p>
|
||||
|
||||
<div class="dt-spacer"></div>
|
||||
|
||||
<!-- Coming-soon notice (st.info) -->
|
||||
<div class="dt-alert info">
|
||||
<span class="dt-mi">info</span>
|
||||
<span>This tool is coming soon.</span>
|
||||
</div>
|
||||
|
||||
<!-- What it will do (st.markdown) -->
|
||||
<p><strong>What it will do:</strong></p>
|
||||
<ul>
|
||||
<li>Check each column against rules you set (no blanks, no duplicates, matches a pattern, within a range, from a set list)</li>
|
||||
<li>Check rules across columns (for example, start date is before end date)</li>
|
||||
<li>Give each column and the whole file a quality score</li>
|
||||
<li>Export a PDF quality report</li>
|
||||
<li>Export an Excel report with the problem rows highlighted</li>
|
||||
<li>Show a summary of what passed, what failed, and how serious each issue is</li>
|
||||
</ul>
|
||||
|
||||
<hr class="dt-divider">
|
||||
|
||||
<button class="dt-btn dt-btn-primary"><span class="dt-mi">notifications</span> Notify me when this ships</button>
|
||||
|
||||
</div>
|
||||
</main>
|
||||
</div>
|
||||
<footer class="dt-footer" id="dt-footer"></footer>
|
||||
<script src="shell.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
373
layout-review/09_pipeline_runner.html
Normal file
373
layout-review/09_pipeline_runner.html
Normal file
@@ -0,0 +1,373 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Layout review — Automated Workflows</title>
|
||||
<link rel="stylesheet" href="app.css">
|
||||
</head>
|
||||
<body data-page="09_pipeline_runner">
|
||||
<div class="dt-app">
|
||||
<aside class="dt-sidebar" id="dt-sidebar"></aside>
|
||||
<main class="dt-main">
|
||||
<div class="dt-review-banner">
|
||||
<span class="dt-mi">visibility</span>
|
||||
<span>Static layout preview of <strong>Automated Workflows</strong> (Pipeline Runner), shown with a file imported, a four-step pipeline configured, and a completed run (results + per-step summary). <a href="index.html">All pages →</a></span>
|
||||
</div>
|
||||
<div class="dt-main-inner">
|
||||
|
||||
<!-- Tool header -->
|
||||
<div class="dt-tool-header">
|
||||
<h1>Automated Workflows</h1>
|
||||
<div class="dt-tool-header-actions">
|
||||
<span class="dt-privacy-pill">
|
||||
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor">
|
||||
<rect x="4" y="11" width="16" height="10" rx="2"/>
|
||||
<path d="M8 11V7a4 4 0 018 0v4"/>
|
||||
</svg>
|
||||
Runs 100% locally
|
||||
</span>
|
||||
<button class="dt-help-btn"><span class="dt-mi">help_outline</span> Help</button>
|
||||
</div>
|
||||
</div>
|
||||
<p class="dt-tool-caption">Run several tools in a row — save the steps once, reuse them anytime.</p>
|
||||
|
||||
<div class="dt-spacer"></div>
|
||||
|
||||
<!-- Upload (file staged) -->
|
||||
<label class="dt-label">Import CSV or Excel file</label>
|
||||
<div class="dt-uploader">
|
||||
<div class="dt-uploader-text">
|
||||
<span class="hint"><span class="dt-mi" style="vertical-align:-4px">upload_file</span> Drag and drop file here</span>
|
||||
<span class="sub">Up to 1.5 GB · CSV, TSV, XLSX, XLS · encoding & delimiter auto-detected</span>
|
||||
</div>
|
||||
<button class="dt-btn">Browse files</button>
|
||||
</div>
|
||||
<div class="dt-file-chip">
|
||||
<span class="dt-file-icon-chip"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor"><path d="M14 2H6a2 2 0 00-2 2v16a2 2 0 002 2h12a2 2 0 002-2V8z"/><path d="M14 2v6h6"/></svg></span>
|
||||
<span class="name">customers_export.csv</span>
|
||||
<span class="size">2.1 MB</span>
|
||||
<button class="dt-btn dt-btn-tertiary" title="Remove">✕</button>
|
||||
</div>
|
||||
|
||||
<!-- Preview expander (collapsed once a result exists) -->
|
||||
<details class="dt-expander">
|
||||
<summary>Preview: customers_export.csv</summary>
|
||||
<div class="dt-expander-body">
|
||||
<p class="dt-caption">18,442 rows, 6 columns</p>
|
||||
<div class="dt-table-wrap">
|
||||
<table class="dt-table">
|
||||
<thead><tr><th class="idx"></th><th>name</th><th>email</th><th>city</th><th>phone</th><th>signup_date</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td class="idx">0</td><td> Jane Doe </td><td>jane@acme.io</td><td>Austin</td><td>512-555-0190</td><td>2024-01-04</td></tr>
|
||||
<tr><td class="idx">1</td><td>jane doe</td><td>JANE@ACME.IO</td><td>austin</td><td>(512) 555-0190</td><td>01/04/2024</td></tr>
|
||||
<tr><td class="idx">2</td><td>Bob Smith</td><td>bob@globex.com</td><td>Denver</td><td>720.555.7781</td><td>2024-02-11</td></tr>
|
||||
<tr><td class="idx">3</td><td>R. Smith</td><td>bob@globex.com</td><td>—</td><td>720-555-7781</td><td>Feb 11 2024</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<hr class="dt-divider">
|
||||
|
||||
<!-- Options: pipeline builder (collapsed once a result exists; opened here to show structure) -->
|
||||
<details class="dt-expander" open>
|
||||
<summary>Options</summary>
|
||||
<div class="dt-expander-body">
|
||||
|
||||
<!-- Mode radio. Editing the steps below auto-switches the mode from the
|
||||
recommended default to "Build interactively" (same precedence-visibility
|
||||
pattern as Fix Missing Values: the active state is made legible, and the
|
||||
default it superseded is marked "· modified"). -->
|
||||
<div class="dt-field">
|
||||
<label class="dt-label">How would you like to define the pipeline?</label>
|
||||
<div class="dt-radio-row" style="flex-direction:column;gap:9px">
|
||||
<span class="dt-radio"><span class="dot"></span> Use the recommended default (Clean Text → Standardize → Fix Missing → Find Duplicates) <span class="dt-count-pill warn" style="margin-left:4px">· modified</span></span>
|
||||
<span class="dt-radio on"><span class="dot"></span> Build interactively</span>
|
||||
<span class="dt-radio"><span class="dot"></span> Import a saved pipeline JSON</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="dt-precedence">
|
||||
<span class="dt-mi">edit</span>
|
||||
<span>You started from the recommended default and edited a step, so the mode switched to <strong>Build interactively</strong>. The steps below are now yours to change — pick <strong>recommended default</strong> again to discard your edits and restore the suggested order.</span>
|
||||
</div>
|
||||
|
||||
<p class="dt-caption" style="margin:10px 0">
|
||||
Add, remove, reorder (drag the row index), enable, or configure each step.
|
||||
Open a step's <strong>Configure</strong> panel to set its options in plain language.
|
||||
Tool order is recommended, not enforced — violations surface as warnings below the table.
|
||||
</p>
|
||||
|
||||
<!-- Pipeline editor. Each step row carries an enable toggle + a "Configure"
|
||||
expander that reveals that tool's OWN controls as the editing surface
|
||||
(built from .dt-* form classes). Raw per-row JSON has been removed;
|
||||
JSON survives only as import/export under "Advanced" below. -->
|
||||
<div class="dt-table-wrap">
|
||||
<table class="dt-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th class="idx"></th>
|
||||
<th>Step</th>
|
||||
<th style="text-align:center">Enabled</th>
|
||||
<th style="text-align:right">Configure</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td class="idx">≡ 0</td>
|
||||
<td><div style="font-weight:500" title="text_clean">Clean Text</div><div class="dt-caption" style="margin:2px 0 0">Trim spaces, collapse repeats, leave case as-is</div></td>
|
||||
<td><span class="dt-check on" style="margin:0;justify-content:center"><span class="box"><span class="dt-mi">check</span></span></span></td>
|
||||
<td style="text-align:right;color:var(--ink-tertiary)"><span class="dt-mi" style="font-size:16px;vertical-align:-3px">tune</span> Configure <span class="dt-mi" style="font-size:14px;vertical-align:-2px">expand_more</span></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<!-- text_clean config panel (open to show the per-step editing surface) -->
|
||||
<details class="dt-expander" open style="margin:6px 0 10px">
|
||||
<summary>Configure: Clean Text</summary>
|
||||
<div class="dt-expander-body">
|
||||
<div class="dt-check on"><span class="box"><span class="dt-mi">check</span></span> Trim leading & trailing whitespace</div>
|
||||
<div class="dt-check on"><span class="box"><span class="dt-mi">check</span></span> Collapse repeated spaces to one</div>
|
||||
<div class="dt-check"><span class="box"></span> Normalize smart quotes & dashes to plain ASCII</div>
|
||||
<div class="dt-field">
|
||||
<label class="dt-label">Letter case</label>
|
||||
<div class="dt-select">Leave as-is</div>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<div class="dt-table-wrap">
|
||||
<table class="dt-table">
|
||||
<tbody>
|
||||
<tr>
|
||||
<td class="idx">≡ 1</td>
|
||||
<td><div style="font-weight:500" title="format_standardize">Standardize Formats</div><div class="dt-caption" style="margin:2px 0 0">Format phone as phone, signup_date as a date</div></td>
|
||||
<td><span class="dt-check on" style="margin:0;justify-content:center"><span class="box"><span class="dt-mi">check</span></span></span></td>
|
||||
<td style="text-align:right;color:var(--ink-tertiary)"><span class="dt-mi" style="font-size:16px;vertical-align:-3px">tune</span> Configure <span class="dt-mi" style="font-size:14px;vertical-align:-2px">chevron_right</span></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<!-- format_standardize config panel (collapsed) -->
|
||||
<details class="dt-expander" style="margin:6px 0 10px">
|
||||
<summary>Configure: Standardize Formats</summary>
|
||||
<div class="dt-expander-body">
|
||||
<p class="dt-caption" style="margin-bottom:8px">Choose a target format for each column. Columns left as “Leave as-is” are untouched.</p>
|
||||
<div class="dt-table-wrap">
|
||||
<table class="dt-table">
|
||||
<thead><tr><th>Column</th><th>Format as</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td>name</td><td><span class="dt-select" style="display:inline-block;min-width:150px;padding:4px 24px 4px 10px;color:var(--ink-tertiary)">Leave as-is</span></td></tr>
|
||||
<tr><td>email</td><td><span class="dt-select" style="display:inline-block;min-width:150px;padding:4px 24px 4px 10px;color:var(--ink-tertiary)">Leave as-is</span></td></tr>
|
||||
<tr><td>phone</td><td><span class="dt-select" style="display:inline-block;min-width:150px;padding:4px 24px 4px 10px">Phone number</span></td></tr>
|
||||
<tr><td>signup_date</td><td><span class="dt-select" style="display:inline-block;min-width:150px;padding:4px 24px 4px 10px">Date</span></td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<div class="dt-table-wrap">
|
||||
<table class="dt-table">
|
||||
<tbody>
|
||||
<tr>
|
||||
<td class="idx">≡ 2</td>
|
||||
<td><div style="font-weight:500" title="missing">Fix Missing Values</div><div class="dt-caption" style="margin:2px 0 0">Flag blank cells (treat “N/A” and “—” as blank)</div></td>
|
||||
<td><span class="dt-check on" style="margin:0;justify-content:center"><span class="box"><span class="dt-mi">check</span></span></span></td>
|
||||
<td style="text-align:right;color:var(--ink-tertiary)"><span class="dt-mi" style="font-size:16px;vertical-align:-3px">tune</span> Configure <span class="dt-mi" style="font-size:14px;vertical-align:-2px">chevron_right</span></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<!-- missing config panel (collapsed) -->
|
||||
<details class="dt-expander" style="margin:6px 0 10px">
|
||||
<summary>Configure: Fix Missing Values</summary>
|
||||
<div class="dt-expander-body">
|
||||
<div class="dt-field">
|
||||
<label class="dt-label">What should happen to blank cells?</label>
|
||||
<div class="dt-radio-row" style="flex-direction:column;gap:8px">
|
||||
<span class="dt-radio on"><span class="dot"></span> Flag them (mark blanks, change nothing)</span>
|
||||
<span class="dt-radio"><span class="dot"></span> Fill them in (numbers → median, text → most common)</span>
|
||||
<span class="dt-radio"><span class="dot"></span> Drop rows that have any blank</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="dt-field">
|
||||
<label class="dt-label">Treat these as blank (comma-separated)</label>
|
||||
<div class="dt-input">N/A, —</div>
|
||||
<div class="dt-help-text">Matched case-insensitively after stripping whitespace.</div>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<div class="dt-table-wrap">
|
||||
<table class="dt-table">
|
||||
<tbody>
|
||||
<tr>
|
||||
<td class="idx">≡ 3</td>
|
||||
<td><div style="font-weight:500" title="dedup">Find Duplicates</div><div class="dt-caption" style="margin:2px 0 0">Match on email & phone; keep the most complete row, merge in missing fields</div></td>
|
||||
<td><span class="dt-check on" style="margin:0;justify-content:center"><span class="box"><span class="dt-mi">check</span></span></span></td>
|
||||
<td style="text-align:right;color:var(--ink-tertiary)"><span class="dt-mi" style="font-size:16px;vertical-align:-3px">tune</span> Configure <span class="dt-mi" style="font-size:14px;vertical-align:-2px">chevron_right</span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td class="idx" style="color:var(--ink-tertiary)">+</td>
|
||||
<td colspan="3" style="color:var(--ink-tertiary);font-family:var(--font-sans)">Add step</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<!-- dedup config panel (collapsed) -->
|
||||
<details class="dt-expander" style="margin:6px 0 10px">
|
||||
<summary>Configure: Find Duplicates</summary>
|
||||
<div class="dt-expander-body">
|
||||
<div class="dt-field">
|
||||
<label class="dt-label">When rows match, which one survives?</label>
|
||||
<div class="dt-select">Keep the most complete row</div>
|
||||
<div class="dt-help-text">Other options: keep the first seen, keep the last seen.</div>
|
||||
</div>
|
||||
<div class="dt-check on"><span class="box"><span class="dt-mi">check</span></span> Merge matched rows (fill each survivor's blanks from its duplicates)</div>
|
||||
<div class="dt-field">
|
||||
<label class="dt-label">Match on these columns</label>
|
||||
<div class="dt-multiselect">
|
||||
<span class="dt-ms-chip">email <span class="x">✕</span></span>
|
||||
<span class="dt-ms-chip">phone <span class="x">✕</span></span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<!-- Validation: pipeline is in recommended order, so no warning shown (warning block omitted) -->
|
||||
|
||||
<!-- Advanced: JSON is import/export only, never the per-step editing surface -->
|
||||
<details class="dt-expander" style="margin-top:14px">
|
||||
<summary>Advanced — import / export pipeline as JSON</summary>
|
||||
<div class="dt-expander-body">
|
||||
<p class="dt-caption" style="margin-bottom:8px">For sharing or version control. Editing is done in the step panels above — this is just the saved form of the same settings.</p>
|
||||
<div class="dt-code">{
|
||||
"version": 1,
|
||||
"steps": [
|
||||
{"tool": "text_clean", "enabled": true, "options": {"trim": true, "collapse_whitespace": true}},
|
||||
{"tool": "format_standardize", "enabled": true, "options": {"column_types": {"phone": "phone", "signup_date": "date"}}},
|
||||
{"tool": "missing", "enabled": true, "options": {"strategy": "flag", "sentinels": ["N/A", "—"]}},
|
||||
{"tool": "dedup", "enabled": true, "options": {"survivor_rule": "most_complete", "merge": true, "keys": ["email", "phone"]}}
|
||||
]
|
||||
}</div>
|
||||
<div class="dt-btn-row" style="margin-top:10px">
|
||||
<button class="dt-btn"><span class="dt-mi">upload</span> Import JSON</button>
|
||||
<button class="dt-btn"><span class="dt-mi">download</span> Export JSON</button>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<!-- Nested explainer expander -->
|
||||
<details class="dt-expander" style="margin-top:14px">
|
||||
<summary>Recommended tool order — why each step belongs where it does</summary>
|
||||
<div class="dt-expander-body">
|
||||
<p><strong>text_clean</strong> before <strong>format_standardize</strong> — format parsers (phone / currency / date) fail on smart-quote-contaminated or NBSP-padded input — clean text first</p>
|
||||
<p><strong>text_clean</strong> before <strong>missing</strong> — sentinel detection misses cells padded with NBSP / zero-width characters — clean text first</p>
|
||||
<p><strong>text_clean</strong> before <strong>dedup</strong> — fuzzy matching treats NBSP-padded values as different — clean text first</p>
|
||||
<p><strong>format_standardize</strong> before <strong>missing</strong> — numeric imputation needs numeric dtypes; canonical phones / currencies improve sentinel detection</p>
|
||||
<p><strong>format_standardize</strong> before <strong>dedup</strong> — canonical phones / lowercase emails enable cross-format duplicate matching</p>
|
||||
<p style="margin-bottom:0"><strong>missing</strong> before <strong>dedup</strong> — deduping rows with mixed NaN sentinels produces brittle merges — resolve missing values first</p>
|
||||
</div>
|
||||
</details>
|
||||
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<hr class="dt-divider">
|
||||
|
||||
<!-- Run -->
|
||||
<button class="dt-btn dt-btn-primary dt-btn-block">Run Pipeline</button>
|
||||
|
||||
<hr class="dt-divider">
|
||||
|
||||
<!-- Results -->
|
||||
<h2>Results</h2>
|
||||
<div class="dt-metrics">
|
||||
<div class="dt-metric"><div class="label">Initial rows</div><div class="value">18,442</div></div>
|
||||
<div class="dt-metric"><div class="label">Final rows</div><div class="value">18,130</div></div>
|
||||
<div class="dt-metric"><div class="label">Steps run</div><div class="value">4</div></div>
|
||||
<div class="dt-metric"><div class="label">Elapsed</div><div class="value">1.84 s</div></div>
|
||||
</div>
|
||||
|
||||
<h4>Per-step summary</h4>
|
||||
<!-- Standalone error column removed: status is one pill per step. A failed step
|
||||
turns the pill danger and surfaces its message in a detail row directly below
|
||||
that step (shown only on failure); successful steps just show a green pill.
|
||||
Summaries are plain-English phrases, not raw JSON. Demo: this run completed
|
||||
cleanly (all four ok, matching the metrics above) — the format_standardize
|
||||
row carries a warn pill + detail row to illustrate how a non-fatal step issue
|
||||
surfaces inline without a dedicated always-empty column. -->
|
||||
<div class="dt-table-wrap">
|
||||
<table class="dt-table">
|
||||
<thead>
|
||||
<tr><th>step</th><th>status</th><th>elapsed</th><th>summary</th></tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>text_clean</td>
|
||||
<td><span class="dt-count-pill success">ok</span></td>
|
||||
<td>214 ms</td>
|
||||
<td style="font-family:var(--font-sans)">1,204 cells changed in name & city</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>format_standardize</td>
|
||||
<td><span class="dt-count-pill warn"><span class="dt-mi" style="font-size:13px;margin-right:3px">warning</span> ok · 141 skipped</span></td>
|
||||
<td>388 ms</td>
|
||||
<td style="font-family:var(--font-sans)">18,301 phones and 17,996 dates standardized</td>
|
||||
</tr>
|
||||
<tr style="background:var(--warn-fill)">
|
||||
<td></td>
|
||||
<td colspan="3" style="font-family:var(--font-sans);color:var(--warn);white-space:normal">
|
||||
<span class="dt-mi" style="font-size:15px;vertical-align:-3px;margin-right:4px">info</span>
|
||||
141 phone values didn't match any known pattern and were left unchanged. The step still completed — review them in the output preview if needed.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>missing</td>
|
||||
<td><span class="dt-count-pill success">ok</span></td>
|
||||
<td>121 ms</td>
|
||||
<td style="font-family:var(--font-sans)">642 blank cells flagged (sentinel “—”)</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>dedup</td>
|
||||
<td><span class="dt-count-pill success">ok</span></td>
|
||||
<td>911 ms</td>
|
||||
<td style="font-family:var(--font-sans)">312 duplicates removed across 147 groups (18,442 → 18,130 rows)</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<h4>Output preview (first 10 rows)</h4>
|
||||
<div class="dt-table-wrap">
|
||||
<table class="dt-table">
|
||||
<thead><tr><th class="idx"></th><th>name</th><th>email</th><th>city</th><th>phone</th><th>signup_date</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td class="idx">0</td><td>Jane Doe</td><td>jane@acme.io</td><td>Austin</td><td class="dt-cell-add">+1 512-555-0190</td><td class="dt-cell-add">2024-01-04</td></tr>
|
||||
<tr><td class="idx">1</td><td>Bob Smith</td><td>bob@globex.com</td><td>Denver</td><td class="dt-cell-add">+1 720-555-7781</td><td class="dt-cell-add">2024-02-11</td></tr>
|
||||
<tr><td class="idx">2</td><td>Carla Reyes</td><td>carla@initech.co</td><td>Phoenix</td><td class="dt-cell-add">+1 480-555-3320</td><td class="dt-cell-add">2024-03-02</td></tr>
|
||||
<tr><td class="idx">3</td><td>Dan Okafor</td><td>dan@umbrella.net</td><td><span class="dt-cell-flag">⚑ missing</span></td><td class="dt-cell-add">+1 206-555-7745</td><td class="dt-cell-add">2024-03-18</td></tr>
|
||||
<tr><td class="idx">4</td><td>Emily Tran</td><td>emily@hooli.com</td><td>Seattle</td><td class="dt-cell-add">+1 206-555-1182</td><td class="dt-cell-add">2024-04-05</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<hr class="dt-divider">
|
||||
|
||||
<!-- Downloads (3 columns) -->
|
||||
<div class="dt-cols-3">
|
||||
<button class="dt-btn dt-btn-primary"><span class="dt-mi">download</span> Download cleaned CSV</button>
|
||||
<button class="dt-btn"><span class="dt-mi">download</span> Download pipeline JSON</button>
|
||||
<button class="dt-btn"><span class="dt-mi">download</span> Download run audit</button>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</main>
|
||||
</div>
|
||||
<footer class="dt-footer" id="dt-footer"></footer>
|
||||
<script src="shell.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
203
layout-review/10_pdf_extractor.html
Normal file
203
layout-review/10_pdf_extractor.html
Normal file
@@ -0,0 +1,203 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Layout review — PDF to CSV</title>
|
||||
<link rel="stylesheet" href="app.css">
|
||||
</head>
|
||||
<body data-page="10_pdf_extractor">
|
||||
<div class="dt-app">
|
||||
<aside class="dt-sidebar" id="dt-sidebar"></aside>
|
||||
<main class="dt-main">
|
||||
<div class="dt-review-banner">
|
||||
<span class="dt-mi">visibility</span>
|
||||
<span>Static layout preview of <strong>PDF to CSV</strong>, shown with two bank-statement PDFs imported and a completed scan (candidate transactions in the editable preview table). <a href="index.html">All pages →</a></span>
|
||||
</div>
|
||||
<div class="dt-main-inner">
|
||||
|
||||
<!-- Tool header -->
|
||||
<div class="dt-tool-header">
|
||||
<h1>PDF to CSV</h1>
|
||||
<div class="dt-tool-header-actions">
|
||||
<span class="dt-privacy-pill">
|
||||
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor">
|
||||
<rect x="4" y="11" width="16" height="10" rx="2"/>
|
||||
<path d="M8 11V7a4 4 0 018 0v4"/>
|
||||
</svg>
|
||||
Runs 100% locally
|
||||
</span>
|
||||
<button class="dt-help-btn"><span class="dt-mi">help_outline</span> Help</button>
|
||||
</div>
|
||||
</div>
|
||||
<p class="dt-tool-caption">Pull transactions out of bank-statement PDFs into a clean CSV file.</p>
|
||||
|
||||
<div class="dt-spacer"></div>
|
||||
|
||||
<!-- Scan options expander (collapsed by default) -->
|
||||
<details class="dt-expander">
|
||||
<summary>Scan options</summary>
|
||||
<div class="dt-expander-body">
|
||||
<div class="dt-cols-2">
|
||||
<div class="dt-check on">
|
||||
<span class="box"><span class="dt-mi">check</span></span>
|
||||
Treat (4.50) as negative
|
||||
</div>
|
||||
<div class="dt-check on">
|
||||
<span class="box"><span class="dt-mi">check</span></span>
|
||||
Use OCR for scanned pages
|
||||
</div>
|
||||
</div>
|
||||
<p class="dt-help-text" style="margin:0 0 10px">OCR status: ready (bundled Tesseract). Most modern bank PDFs are text-based and don't need OCR — only enable for image-based scans.</p>
|
||||
<div class="dt-cols-2">
|
||||
<div class="dt-field">
|
||||
<label class="dt-label">Output date format</label>
|
||||
<div class="dt-select">YYYY-MM-DD (2026-01-13)</div>
|
||||
</div>
|
||||
<div class="dt-field">
|
||||
<label class="dt-label">Override year for short dates (optional)</label>
|
||||
<input class="dt-input" type="text" placeholder="" value="" disabled>
|
||||
<div class="dt-help-text">Leave blank for automatic (statement period → filename year → this override).</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<!-- Files section head -->
|
||||
<div class="dt-files-section-head">
|
||||
<h2>Files</h2>
|
||||
<span class="dt-section-meta">2 files · 318.4 KB total</span>
|
||||
</div>
|
||||
|
||||
<!-- Files card (Home-style bordered list + Add more files) -->
|
||||
<div class="dt-card" style="padding-bottom:0">
|
||||
<div class="dt-file-row" style="padding:6px 0">
|
||||
<button class="dt-btn dt-btn-tertiary" title="Remove statement-jan-2026.pdf">✕</button>
|
||||
<span class="dt-file-icon-chip"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor"><path d="M14 2H6a2 2 0 00-2 2v16a2 2 0 002 2h12a2 2 0 002-2V8z"/><path d="M14 2v6h6"/></svg></span>
|
||||
<span class="dt-file-name">statement-jan-2026.pdf</span>
|
||||
<span class="dt-file-size" style="margin-left:auto">171.2 KB</span>
|
||||
</div>
|
||||
<div class="dt-file-row" style="padding:6px 0">
|
||||
<button class="dt-btn dt-btn-tertiary" title="Remove statement-feb-2026.pdf">✕</button>
|
||||
<span class="dt-file-icon-chip"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor"><path d="M14 2H6a2 2 0 00-2 2v16a2 2 0 002 2h12a2 2 0 002-2V8z"/><path d="M14 2v6h6"/></svg></span>
|
||||
<span class="dt-file-name">statement-feb-2026.pdf</span>
|
||||
<span class="dt-file-size" style="margin-left:auto">147.2 KB</span>
|
||||
</div>
|
||||
<button class="dt-file-add" style="margin-left:-16px;margin-right:-16px;width:calc(100% + 32px)">
|
||||
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor"><path d="M12 5v14M5 12h14"/></svg> Add more files
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<!-- Action buttons -->
|
||||
<div class="dt-btn-row" style="margin-top:16px;max-width:340px">
|
||||
<button class="dt-btn dt-btn-primary">Scan</button>
|
||||
<button class="dt-btn">Clear all files</button>
|
||||
</div>
|
||||
|
||||
<hr class="dt-divider">
|
||||
|
||||
<!-- Warnings expander (collapsed) -->
|
||||
<details class="dt-expander">
|
||||
<summary>Warnings (1)</summary>
|
||||
<div class="dt-expander-body">
|
||||
<div class="dt-alert warn">
|
||||
<span class="dt-mi">warning</span>
|
||||
<span>[statement-feb-2026.pdf] 2 lines matched a date but no amount — skipped (likely a wrapped description). Check the source if a transaction looks missing.</span>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<!-- Results -->
|
||||
<h4>47 candidate transaction(s) from 2 file(s)</h4>
|
||||
<p class="dt-caption">Uncheck rows to exclude. Edit any cell to fix a value the scanner got wrong. Hover the <span class="dt-mi" style="font-size:15px;vertical-align:-3px;color:var(--ink-tertiary)">info</span> on any row to see the original PDF text it came from.</p>
|
||||
|
||||
<!-- overflow-x:auto belt-and-suspenders: any residual width scrolls instead of clipping (app.css .dt-table-wrap is overflow:hidden) -->
|
||||
<div class="dt-table-wrap" style="overflow-x:auto">
|
||||
<table class="dt-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Include</th>
|
||||
<th></th>
|
||||
<th>date</th>
|
||||
<th>description</th>
|
||||
<th>amount_debit</th>
|
||||
<th>amount_credit</th>
|
||||
<th>account_number</th>
|
||||
<th>source_file</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td><span class="dt-check on" style="margin:0"><span class="box"><span class="dt-mi">check</span></span></span></td>
|
||||
<td class="idx" title="raw: 01/03 OPENING BALANCE 2,140.55" style="cursor:help"><span class="dt-mi" style="font-size:16px">info</span></td>
|
||||
<td>2026-01-03</td><td>OPENING BALANCE</td><td></td><td></td><td>****4821</td><td>statement-jan-2026.pdf</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><span class="dt-check on" style="margin:0"><span class="box"><span class="dt-mi">check</span></span></span></td>
|
||||
<td class="idx" title="raw: 01/05 POS PURCHASE WHOLE FOODS MKT (84.12)" style="cursor:help"><span class="dt-mi" style="font-size:16px">info</span></td>
|
||||
<td>2026-01-05</td><td>POS PURCHASE WHOLE FOODS MKT</td><td>84.12</td><td></td><td>****4821</td><td>statement-jan-2026.pdf</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><span class="dt-check on" style="margin:0"><span class="box"><span class="dt-mi">check</span></span></span></td>
|
||||
<td class="idx" title="raw: 01/08 ACH DEPOSIT PAYROLL ACME CORP 3,250.00" style="cursor:help"><span class="dt-mi" style="font-size:16px">info</span></td>
|
||||
<td>2026-01-08</td><td>ACH DEPOSIT PAYROLL ACME CORP</td><td></td><td>3,250.00</td><td>****4821</td><td>statement-jan-2026.pdf</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><span class="dt-check on" style="margin:0"><span class="box"><span class="dt-mi">check</span></span></span></td>
|
||||
<td class="idx" title="raw: 01/11 ONLINE TRANSFER TO SAVINGS (500.00)" style="cursor:help"><span class="dt-mi" style="font-size:16px">info</span></td>
|
||||
<td>2026-01-11</td><td>ONLINE TRANSFER TO SAVINGS</td><td>500.00</td><td></td><td>****4821</td><td>statement-jan-2026.pdf</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><span class="dt-check" style="margin:0"><span class="box"></span></span></td>
|
||||
<td class="idx" title="raw: 01/12 INTEREST RATE 0.50% APY 0.00" style="cursor:help"><span class="dt-mi" style="font-size:16px">info</span></td>
|
||||
<td class="dt-cell-flag">2026-01-12</td><td class="dt-cell-flag">INTEREST RATE 0.50% APY DETAIL <span style="font-family:var(--font-sans);font-size:11px;font-weight:500;background:var(--warn-fill);color:var(--warn);border-radius:999px;padding:1px 7px;white-space:nowrap">auto-excluded · not a transaction line</span></td><td></td><td></td><td>****4821</td><td>statement-jan-2026.pdf</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><span class="dt-check on" style="margin:0"><span class="box"><span class="dt-mi">check</span></span></span></td>
|
||||
<td class="idx" title="raw: 01/14 DEBIT CARD SHELL OIL #2287 (52.40)" style="cursor:help"><span class="dt-mi" style="font-size:16px">info</span></td>
|
||||
<td>2026-01-14</td><td>DEBIT CARD SHELL OIL #2287</td><td>52.40</td><td></td><td>****4821</td><td>statement-jan-2026.pdf</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><span class="dt-check on" style="margin:0"><span class="box"><span class="dt-mi">check</span></span></span></td>
|
||||
<td class="idx" title="raw: 02/02 POS PURCHASE TRADER JOES #511 (61.88)" style="cursor:help"><span class="dt-mi" style="font-size:16px">info</span></td>
|
||||
<td>2026-02-02</td><td>POS PURCHASE TRADER JOES #511</td><td>61.88</td><td></td><td>****4821</td><td>statement-feb-2026.pdf</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><span class="dt-check on" style="margin:0"><span class="box"><span class="dt-mi">check</span></span></span></td>
|
||||
<td class="idx" title="raw: 02/06 ACH DEPOSIT PAYROLL ACME CORP 3,250.00" style="cursor:help"><span class="dt-mi" style="font-size:16px">info</span></td>
|
||||
<td>2026-02-06</td><td>ACH DEPOSIT PAYROLL ACME CORP</td><td></td><td>3,250.00</td><td>****4821</td><td>statement-feb-2026.pdf</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><span class="dt-check on" style="margin:0"><span class="box"><span class="dt-mi">check</span></span></span></td>
|
||||
<td class="idx" title="raw: 02/09 CHECK #1043 (1,200.00)" style="cursor:help"><span class="dt-mi" style="font-size:16px">info</span></td>
|
||||
<td>2026-02-09</td><td>CHECK #1043</td><td>1,200.00</td><td></td><td>****4821</td><td>statement-feb-2026.pdf</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<!-- Download area: configure-then-act — column selector first, download button below -->
|
||||
<div style="margin-top:14px;max-width:520px">
|
||||
<div class="dt-field" style="margin:0 0 14px">
|
||||
<label class="dt-label">Columns to include in CSV</label>
|
||||
<div class="dt-multiselect">
|
||||
<span class="dt-ms-chip">date <span class="x">✕</span></span>
|
||||
<span class="dt-ms-chip">description <span class="x">✕</span></span>
|
||||
<span class="dt-ms-chip">amount_debit <span class="x">✕</span></span>
|
||||
<span class="dt-ms-chip">amount_credit <span class="x">✕</span></span>
|
||||
<span class="dt-ms-chip">account_number <span class="x">✕</span></span>
|
||||
<span class="dt-ms-chip">source_file <span class="x">✕</span></span>
|
||||
</div>
|
||||
<div class="dt-help-text"><code>page</code> and <code>raw</code> are kept off by default; tick them if you want them in the file.</div>
|
||||
</div>
|
||||
<button class="dt-btn dt-btn-primary dt-btn-block">Download 46 rows as CSV</button>
|
||||
<p class="dt-caption" style="margin-top:8px">1 row excluded (INTEREST RATE detail line).</p>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</main>
|
||||
</div>
|
||||
<footer class="dt-footer" id="dt-footer"></footer>
|
||||
<script src="shell.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
248
layout-review/11_reconciler.html
Normal file
248
layout-review/11_reconciler.html
Normal file
@@ -0,0 +1,248 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Layout review — Reconcile Two Files</title>
|
||||
<link rel="stylesheet" href="app.css">
|
||||
</head>
|
||||
<body data-page="11_reconciler">
|
||||
<div class="dt-app">
|
||||
<aside class="dt-sidebar" id="dt-sidebar"></aside>
|
||||
<main class="dt-main">
|
||||
<div class="dt-review-banner">
|
||||
<span class="dt-mi">visibility</span>
|
||||
<span>Static layout preview of <strong>Reconcile Two Files</strong>, shown with both files imported, key columns mapped, and a completed reconciliation (matched / review / unmatched results). <a href="index.html">All pages →</a></span>
|
||||
</div>
|
||||
<div class="dt-main-inner">
|
||||
|
||||
<!-- Tool header -->
|
||||
<div class="dt-tool-header">
|
||||
<h1>Reconcile Two Files</h1>
|
||||
<div class="dt-tool-header-actions">
|
||||
<span class="dt-privacy-pill">
|
||||
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor">
|
||||
<rect x="4" y="11" width="16" height="10" rx="2"/>
|
||||
<path d="M8 11V7a4 4 0 018 0v4"/>
|
||||
</svg>
|
||||
Runs 100% locally
|
||||
</span>
|
||||
<button class="dt-help-btn"><span class="dt-mi">help_outline</span> Help</button>
|
||||
</div>
|
||||
</div>
|
||||
<p class="dt-tool-caption">Compare two lists of transactions (e.g. bank vs. ledger) and flag what doesn't match.</p>
|
||||
|
||||
<div class="dt-spacer"></div>
|
||||
|
||||
<!-- Side-by-side upload (st.columns(2) → two _side_panel) -->
|
||||
<div class="dt-cols-2">
|
||||
<!-- Left side -->
|
||||
<div>
|
||||
<h4 style="margin-top:0">Left (e.g. bank feed)</h4>
|
||||
<div class="dt-alert info">
|
||||
<span class="dt-mi">description</span>
|
||||
<span>Using <strong>bank_feed_may.csv</strong> from the upload screen.</span>
|
||||
</div>
|
||||
<button class="dt-btn" style="margin-bottom:4px">Use a different file</button>
|
||||
<p class="dt-caption" style="margin-top:6px"><code>bank_feed_may.csv</code> — 1,204 rows, 4 columns</p>
|
||||
<details class="dt-expander">
|
||||
<summary>Preview left (e.g. bank feed)</summary>
|
||||
<div class="dt-expander-body">
|
||||
<div class="dt-table-wrap">
|
||||
<table class="dt-table">
|
||||
<thead><tr><th>posted_date</th><th>description</th><th>amount</th><th>ref</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td>2026-05-01</td><td>ACME SUPPLIES</td><td>-1240.00</td><td>CHK1041</td></tr>
|
||||
<tr><td>2026-05-02</td><td>PAYROLL RUN</td><td>-8800.00</td><td>ACH5520</td></tr>
|
||||
<tr><td>2026-05-03</td><td>CLIENT GLOBEX</td><td>5200.00</td><td>DEP0090</td></tr>
|
||||
<tr><td>2026-05-04</td><td>UTILITY CO</td><td>-318.42</td><td>CHK1042</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
</div>
|
||||
<!-- Right side -->
|
||||
<div>
|
||||
<h4 style="margin-top:0">Right (e.g. ledger)</h4>
|
||||
<div class="dt-alert info">
|
||||
<span class="dt-mi">description</span>
|
||||
<span>Using <strong>ledger_may.xlsx</strong> from the upload screen.</span>
|
||||
</div>
|
||||
<button class="dt-btn" style="margin-bottom:4px">Use a different file</button>
|
||||
<p class="dt-caption" style="margin-top:6px"><code>ledger_may.xlsx</code> — 1,198 rows, 5 columns</p>
|
||||
<details class="dt-expander">
|
||||
<summary>Preview right (e.g. ledger)</summary>
|
||||
<div class="dt-expander-body">
|
||||
<div class="dt-table-wrap">
|
||||
<table class="dt-table">
|
||||
<thead><tr><th>txn_date</th><th>memo</th><th>value</th><th>invoice_no</th><th>account</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td>2026-05-01</td><td>Acme Supplies Inc</td><td>-1240.00</td><td>INV-1041</td><td>5000</td></tr>
|
||||
<tr><td>2026-05-02</td><td>Monthly payroll</td><td>-8800.00</td><td>INV-5520</td><td>6000</td></tr>
|
||||
<tr><td>2026-05-03</td><td>Globex retainer</td><td>5200.00</td><td>INV-0090</td><td>4000</td></tr>
|
||||
<tr><td>2026-05-04</td><td>City Utilities</td><td>-318.40</td><td>INV-1042</td><td>6100</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<hr class="dt-divider">
|
||||
|
||||
<!-- Match settings -->
|
||||
<h2>Match settings</h2>
|
||||
<div class="dt-cols-2">
|
||||
<!-- Left pickers (file order: posted_date, description, amount → date, desc, amount) -->
|
||||
<div>
|
||||
<h4 style="margin-top:0">Left columns</h4>
|
||||
<div class="dt-field"><label class="dt-label">Date column (optional)</label><div class="dt-select">posted_date</div></div>
|
||||
<div class="dt-field"><label class="dt-label">Description column (optional)</label><div class="dt-select">description</div></div>
|
||||
<div class="dt-field"><label class="dt-label">Amount column <span class="req">*</span></label><div class="dt-select">amount</div></div>
|
||||
<div class="dt-field"><label class="dt-label">Reference columns (optional, e.g. check / invoice no.)</label>
|
||||
<div class="dt-multiselect"><span class="dt-ms-chip">ref <span class="x">✕</span></span></div></div>
|
||||
</div>
|
||||
<!-- Right pickers (file order: txn_date, memo, value → date, desc, amount) -->
|
||||
<div>
|
||||
<h4 style="margin-top:0">Right columns</h4>
|
||||
<div class="dt-field"><label class="dt-label">Date column (optional)</label><div class="dt-select">txn_date</div></div>
|
||||
<div class="dt-field"><label class="dt-label">Description column (optional)</label><div class="dt-select">memo</div></div>
|
||||
<div class="dt-field"><label class="dt-label">Amount column <span class="req">*</span></label><div class="dt-select">value</div></div>
|
||||
<div class="dt-field"><label class="dt-label">Reference columns (must match left count)</label>
|
||||
<div class="dt-multiselect"><span class="dt-ms-chip">invoice_no <span class="x">✕</span></span></div>
|
||||
<div class="dt-help-text" style="color:var(--success);display:flex;align-items:center;gap:5px"><span class="dt-mi" style="font-family:'Material Symbols Outlined';font-size:15px;line-height:1">check_circle</span> 1 reference each side — counts match</div></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Tolerances & options (expanded=True) -->
|
||||
<details class="dt-expander" open>
|
||||
<summary>Tolerances & options</summary>
|
||||
<div class="dt-expander-body">
|
||||
<div class="dt-cols-3">
|
||||
<div class="dt-field"><label class="dt-label">Amount tolerance</label>
|
||||
<div class="dt-input">0.0200</div>
|
||||
<div class="dt-help-text">Absolute tolerance on amount (e.g. 0.01 to absorb cent rounding).</div></div>
|
||||
<div class="dt-field"><label class="dt-label">Date tolerance (days)</label>
|
||||
<div class="dt-input">1</div>
|
||||
<div class="dt-help-text">Allow N calendar days of drift between posting dates.</div></div>
|
||||
<div class="dt-field"><label class="dt-label">Invert right amount sign</label>
|
||||
<div class="dt-check" style="margin-top:8px"><span class="box"></span></div>
|
||||
<div class="dt-help-text">Use when one side records debits as positive and the other as negative.</div></div>
|
||||
</div>
|
||||
<div class="dt-field"><label class="dt-label">Description similarity boost (0 disables)</label>
|
||||
<div class="dt-slider"><div class="track"><div class="fill" style="width:80%"></div><div class="knob" style="left:80%"></div></div><div class="val">80</div></div>
|
||||
<div class="dt-help-text">When both sides have a description column set, accept matches with this minimum fuzzy similarity even if amount/date are merely within tolerance. Lower = more permissive.</div></div>
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<hr class="dt-divider">
|
||||
|
||||
<button class="dt-btn dt-btn-primary dt-btn-block">Reconcile</button>
|
||||
|
||||
<hr class="dt-divider">
|
||||
|
||||
<!-- Results -->
|
||||
<h2>Results</h2>
|
||||
<div class="dt-metrics">
|
||||
<div class="dt-metric"><div class="label">Review</div><div class="value">9</div></div>
|
||||
<div class="dt-metric"><div class="label">Unmatched left</div><div class="value">22</div></div>
|
||||
<div class="dt-metric"><div class="label">Unmatched right</div><div class="value">16</div></div>
|
||||
<div class="dt-metric"><div class="label">Matched</div><div class="value">1,173</div></div>
|
||||
</div>
|
||||
<p class="dt-caption">Coverage: 97.4% of the larger side</p>
|
||||
|
||||
<!-- Tabs (st.tabs) — exceptions-first; Review active by default -->
|
||||
<div class="dt-tabs">
|
||||
<span class="dt-tab is-active">Review (9)</span>
|
||||
<span class="dt-tab">Unmatched left (22)</span>
|
||||
<span class="dt-tab">Unmatched right (16)</span>
|
||||
<span class="dt-tab">Matched (1,173)</span>
|
||||
</div>
|
||||
|
||||
<!-- Active tab content: Review (exceptions-first default) -->
|
||||
<p class="dt-caption">Pairs flagged because the algorithm couldn't pick a single best match (e.g. multiple equally-good candidates). Use the left/right indices to disambiguate manually.</p>
|
||||
<div class="dt-table-wrap">
|
||||
<table class="dt-table">
|
||||
<thead><tr><th>left_idx</th><th>left_amount</th><th>right_idx</th><th>right_value</th><th>candidates</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td>118</td><td>-450.00</td><td>121, 209</td><td>-450.00</td><td class="dt-cell-flag">2 equal</td></tr>
|
||||
<tr><td>203</td><td>1000.00</td><td>198, 244</td><td>1000.00</td><td class="dt-cell-flag">2 equal</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<!-- Other tab previews shown as collapsed expanders for review context -->
|
||||
<details class="dt-expander">
|
||||
<summary>Unmatched left (22) — only in bank_feed_may.csv</summary>
|
||||
<div class="dt-expander-body">
|
||||
<p class="dt-caption">Preview of first 25 of 22 rows.</p>
|
||||
<div class="dt-table-wrap">
|
||||
<table class="dt-table">
|
||||
<thead><tr><th>posted_date</th><th>description</th><th>amount</th><th>ref</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td class="dt-cell-del">2026-05-09</td><td class="dt-cell-del">BANK FEE</td><td class="dt-cell-del">-12.00</td><td class="dt-cell-del">FEE0001</td></tr>
|
||||
<tr><td class="dt-cell-del">2026-05-14</td><td class="dt-cell-del">ATM WITHDRAWAL</td><td class="dt-cell-del">-200.00</td><td class="dt-cell-del">ATM7781</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<details class="dt-expander">
|
||||
<summary>Unmatched right (16) — only in ledger_may.xlsx</summary>
|
||||
<div class="dt-expander-body">
|
||||
<p class="dt-caption">Preview of first 25 of 16 rows.</p>
|
||||
<div class="dt-table-wrap">
|
||||
<table class="dt-table">
|
||||
<thead><tr><th>txn_date</th><th>memo</th><th>value</th><th>invoice_no</th><th>account</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td class="dt-cell-del">2026-05-11</td><td class="dt-cell-del">Accrued interest</td><td class="dt-cell-del">37.50</td><td class="dt-cell-del">INV-9001</td><td class="dt-cell-del">7000</td></tr>
|
||||
<tr><td class="dt-cell-del">2026-05-22</td><td class="dt-cell-del">Depreciation</td><td class="dt-cell-del">-410.00</td><td class="dt-cell-del">INV-9044</td><td class="dt-cell-del">8000</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<details class="dt-expander">
|
||||
<summary>Matched (1,173) — cleanly reconciled</summary>
|
||||
<div class="dt-expander-body">
|
||||
<p class="dt-caption">Preview of first 25 of 1,173 rows — download the CSV below for the full set.</p>
|
||||
<div class="dt-table-wrap">
|
||||
<table class="dt-table">
|
||||
<thead><tr>
|
||||
<th>left_posted_date</th><th>left_description</th><th>left_amount</th>
|
||||
<th>right_txn_date</th><th>right_memo</th><th>right_value</th><th>amount_diff</th>
|
||||
</tr></thead>
|
||||
<tbody>
|
||||
<tr><td>2026-05-01</td><td>ACME SUPPLIES</td><td>-1240.00</td><td>2026-05-01</td><td>Acme Supplies Inc</td><td>-1240.00</td><td class="dt-cell-add">0.00</td></tr>
|
||||
<tr><td>2026-05-02</td><td>PAYROLL RUN</td><td>-8800.00</td><td>2026-05-02</td><td>Monthly payroll</td><td>-8800.00</td><td class="dt-cell-add">0.00</td></tr>
|
||||
<tr><td>2026-05-03</td><td>CLIENT GLOBEX</td><td>5200.00</td><td>2026-05-03</td><td>Globex retainer</td><td>5200.00</td><td class="dt-cell-add">0.00</td></tr>
|
||||
<tr><td>2026-05-04</td><td>UTILITY CO</td><td>-318.42</td><td>2026-05-04</td><td>City Utilities</td><td>-318.40</td><td class="dt-cell-flag">0.02</td></tr>
|
||||
<tr><td>2026-05-06</td><td>OFFICE DEPOT</td><td>-89.15</td><td>2026-05-07</td><td>Office supplies</td><td>-89.15</td><td class="dt-cell-add">0.00</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<hr class="dt-divider">
|
||||
|
||||
<!-- Downloads (st.columns(4) of html_download_button) — exceptions-first,
|
||||
matching the tab/metric order; four parallel exports, equal weight -->
|
||||
<div class="dt-btn-row">
|
||||
<button class="dt-btn">Review CSV</button>
|
||||
<button class="dt-btn">Unmatched left</button>
|
||||
<button class="dt-btn">Unmatched right</button>
|
||||
<button class="dt-btn">Matched CSV</button>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</main>
|
||||
</div>
|
||||
<footer class="dt-footer" id="dt-footer"></footer>
|
||||
<script src="shell.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
542
layout-review/app.css
Normal file
542
layout-review/app.css
Normal file
@@ -0,0 +1,542 @@
|
||||
/* ===========================================================================
|
||||
DataTools — static layout-review stylesheet
|
||||
---------------------------------------------------------------------------
|
||||
Faithful reproduction of the live Streamlit app's design system for human
|
||||
review of page layouts. Tokens are copied verbatim from src/gui/theme.py
|
||||
(§3 color + type scale) and the component values from
|
||||
src/gui/components/_legacy.py:_DESIGN_TOKENS_CSS.
|
||||
|
||||
The live app applies these styles to Streamlit's data-testid DOM; here we
|
||||
re-express the same look against clean semantic classes so the static HTML
|
||||
stays readable. Where the app uses real .dt-* classes (page header, files
|
||||
card, findings, stats) the class names are kept identical.
|
||||
=========================================================================== */
|
||||
|
||||
@import url("https://fonts.googleapis.com/css2?family=Geist:wght@400;500;600;700&family=Geist+Mono:wght@400;500&display=swap");
|
||||
@import url("https://fonts.googleapis.com/css2?family=Material+Symbols+Outlined:opsz,wght,FILL,GRAD@20..48,400,0,0&display=block");
|
||||
|
||||
:root {
|
||||
--font-sans: "Geist", -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
|
||||
--font-mono: "Geist Mono", ui-monospace, "SF Mono", Menlo, monospace;
|
||||
|
||||
--ink: #1c1917;
|
||||
--ink-secondary: #57534e;
|
||||
--ink-tertiary: #a8a29e;
|
||||
--bg: #fafaf7;
|
||||
--surface: #ffffff;
|
||||
--surface-hover: #f8f7f3;
|
||||
--border: #e7e5dc;
|
||||
--border-strong: #d6d3c7;
|
||||
--accent: #c2410c;
|
||||
--accent-hover: #9a3412;
|
||||
--accent-fill: #fef4ed;
|
||||
--accent-fill-strong: #fde4d3;
|
||||
|
||||
--warn: #b45309;
|
||||
--warn-fill: #fef3c7;
|
||||
--info: #0369a1;
|
||||
--info-fill: #e0f2fe;
|
||||
--success: #15803d;
|
||||
--success-fill: #dcfce7;
|
||||
--danger: #b91c1c;
|
||||
--danger-fill: #fee2e2;
|
||||
|
||||
--r-sm: 6px;
|
||||
--r-md: 10px;
|
||||
--r-lg: 14px;
|
||||
|
||||
--sidebar-w: 264px;
|
||||
}
|
||||
|
||||
* { box-sizing: border-box; }
|
||||
|
||||
html, body {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
background: var(--bg);
|
||||
color: var(--ink);
|
||||
font-family: var(--font-sans);
|
||||
font-feature-settings: "ss01", "cv01", "cv11";
|
||||
-webkit-font-smoothing: antialiased;
|
||||
}
|
||||
|
||||
/* ---------- Type scale (theme.py §4) ---------- */
|
||||
h1 { font-size: 32px; font-weight: 600; letter-spacing: -0.035em; line-height: 1.1; margin: 0 0 4px; }
|
||||
h2 { font-size: 22px; font-weight: 600; letter-spacing: -0.025em; line-height: 1.2; margin: 1.5rem 0 0.75rem; }
|
||||
h3 { font-size: 18px; font-weight: 500; letter-spacing: -0.018em; line-height: 1.25; margin: 1.25rem 0 0.5rem; }
|
||||
h4 { font-size: 15px; font-weight: 500; letter-spacing: -0.012em; line-height: 1.35; margin: 1rem 0 0.5rem; }
|
||||
p { font-size: 14px; font-weight: 400; line-height: 1.55; color: var(--ink); margin: 0 0 0.6rem; }
|
||||
strong { font-weight: 500; color: var(--ink); }
|
||||
a { color: var(--accent); text-decoration: none; }
|
||||
a:hover { color: var(--accent-hover); text-decoration: underline; }
|
||||
code, .dt-mono { font-family: var(--font-mono); font-size: 0.92em; font-feature-settings: "ss02"; }
|
||||
|
||||
/* ===========================================================================
|
||||
App frame — sidebar + main + sticky footer
|
||||
=========================================================================== */
|
||||
.dt-app { display: flex; min-height: 100vh; }
|
||||
|
||||
/* ---------- Sidebar (cream paper) ---------- */
|
||||
.dt-sidebar {
|
||||
width: var(--sidebar-w);
|
||||
flex-shrink: 0;
|
||||
background: #f5f4ef;
|
||||
border-right: 1px solid var(--border);
|
||||
padding: 18px 14px 90px;
|
||||
position: sticky;
|
||||
top: 0;
|
||||
align-self: flex-start;
|
||||
height: 100vh;
|
||||
overflow-y: auto;
|
||||
}
|
||||
.dt-brand { display: flex; align-items: center; gap: 10px; padding: 0 4px 18px; }
|
||||
.dt-brand-mark {
|
||||
width: 28px; height: 28px; border-radius: 7px;
|
||||
background: var(--ink); color: var(--accent-fill);
|
||||
display: inline-flex; align-items: center; justify-content: center;
|
||||
font-weight: 700; font-size: 16px; letter-spacing: -0.04em; line-height: 1; flex-shrink: 0;
|
||||
}
|
||||
.dt-brand-name { display: flex; flex-direction: column; gap: 1px; line-height: 1.05; }
|
||||
.dt-brand-eyebrow {
|
||||
font-size: 9.5px; font-weight: 600; letter-spacing: 0.14em;
|
||||
text-transform: uppercase; color: var(--ink-tertiary); line-height: 1;
|
||||
}
|
||||
.dt-brand-word { font-weight: 600; font-size: 15px; letter-spacing: -0.02em; color: var(--ink); }
|
||||
|
||||
.dt-nav { display: flex; flex-direction: column; }
|
||||
.dt-nav-section {
|
||||
font-size: 11.5px; text-transform: uppercase; letter-spacing: 0.08em;
|
||||
color: var(--ink-tertiary); font-weight: 500;
|
||||
padding: 14px 10px 4px; margin: 0;
|
||||
display: flex; align-items: center; justify-content: space-between;
|
||||
}
|
||||
.dt-nav-section .dt-nav-indicator { font-size: 16px; color: var(--ink-tertiary); }
|
||||
.dt-nav-link {
|
||||
display: flex; align-items: center; gap: 8px;
|
||||
color: var(--ink-secondary); font-size: 13px; font-weight: 500; line-height: 1.3;
|
||||
padding: 5px 10px; border-radius: var(--r-sm); margin-bottom: 1px;
|
||||
text-decoration: none; transition: background 0.12s ease, color 0.12s ease;
|
||||
}
|
||||
.dt-nav-link:hover { background: rgba(0,0,0,0.04); color: var(--ink); text-decoration: none; }
|
||||
.dt-nav-link.is-active { background: rgba(0,0,0,0.04); color: var(--ink); font-weight: 600; }
|
||||
.dt-nav-link .dt-mi { font-family: "Material Symbols Outlined"; font-size: 18px; color: var(--ink-secondary); line-height: 1; }
|
||||
.dt-nav-link.is-active .dt-mi { color: var(--ink); }
|
||||
.dt-nav-link.is-soon { opacity: 0.55; }
|
||||
|
||||
/* "Start here" front-door item — weightier than ordinary nav links so the
|
||||
obvious entry point reads at a glance. Accent-fill ground + accent-hover ink,
|
||||
slightly larger hit area, with bottom margin to part it from the groups below.
|
||||
Layers on .dt-nav-link, so the .is-active treatment still overrides cleanly. */
|
||||
.dt-nav-start {
|
||||
background: var(--accent-fill); color: var(--accent-hover); font-weight: 600;
|
||||
padding: 8px 10px; margin-bottom: 12px;
|
||||
}
|
||||
.dt-nav-start:hover { background: var(--accent-fill-strong); color: var(--accent-hover); }
|
||||
.dt-nav-start .dt-mi { color: var(--accent); }
|
||||
.dt-nav-start.is-active { background: var(--accent-fill-strong); color: var(--accent-hover); }
|
||||
.dt-nav-start.is-active .dt-mi { color: var(--accent); }
|
||||
.dt-nav-soon-tag {
|
||||
margin-left: auto; font-size: 9px; font-weight: 600; letter-spacing: 0.06em;
|
||||
text-transform: uppercase; color: var(--ink-tertiary);
|
||||
border: 1px solid var(--border-strong); border-radius: 999px; padding: 1px 6px;
|
||||
}
|
||||
|
||||
.dt-sidebar-foot { margin-top: 22px; padding-top: 16px; border-top: 1px solid var(--border); display: flex; flex-direction: column; gap: 10px; }
|
||||
.dt-sidebar-label { font-size: 11.5px; font-weight: 500; text-transform: uppercase; letter-spacing: 0.08em; color: var(--ink-tertiary); margin-bottom: 4px; }
|
||||
.dt-license-badge { font-size: 12.5px; color: var(--ink-secondary); }
|
||||
|
||||
/* ---------- Main column ---------- */
|
||||
.dt-main { flex: 1; min-width: 0; padding: 40px 56px 96px; }
|
||||
.dt-main-inner { max-width: 920px; margin: 0 auto; }
|
||||
|
||||
/* Review banner above every mockup */
|
||||
.dt-review-banner {
|
||||
max-width: 920px; margin: 0 auto 20px; display: flex; gap: 10px; align-items: center;
|
||||
background: var(--info-fill); color: var(--info);
|
||||
border: 1px solid transparent; border-radius: var(--r-md);
|
||||
padding: 8px 14px; font-size: 12.5px; line-height: 1.4;
|
||||
}
|
||||
.dt-review-banner a { color: var(--info); text-decoration: underline; }
|
||||
.dt-review-banner .dt-mi { font-family: "Material Symbols Outlined"; font-size: 18px; }
|
||||
|
||||
/* ---------- Sticky footer ---------- */
|
||||
.dt-footer {
|
||||
position: fixed; bottom: 0; left: var(--sidebar-w); right: 0;
|
||||
background: rgba(255,255,255,0.97); backdrop-filter: blur(8px);
|
||||
border-top: 1px solid var(--border-strong);
|
||||
padding: 8px 20px; z-index: 50;
|
||||
display: flex; align-items: center; gap: 8px;
|
||||
}
|
||||
.dt-footer-btn {
|
||||
display: inline-flex; align-items: center; gap: 8px;
|
||||
color: var(--ink-secondary); font-size: 13px; font-weight: 500; line-height: 1.3;
|
||||
padding: 5px 10px; border-radius: var(--r-sm);
|
||||
background: transparent; border: none; cursor: pointer; text-decoration: none;
|
||||
}
|
||||
.dt-footer-btn:hover { background: rgba(0,0,0,0.04); color: var(--ink); text-decoration: none; }
|
||||
.dt-footer-btn .dt-mi { font-family: "Material Symbols Outlined"; font-size: 16px; }
|
||||
|
||||
/* ===========================================================================
|
||||
Page header (brand + privacy pill) — .dt-page-* mirror the live app
|
||||
=========================================================================== */
|
||||
.dt-page-header {
|
||||
display: flex; align-items: center; justify-content: space-between; gap: 24px;
|
||||
margin: 0 0 24px; padding-bottom: 22px; border-bottom: 1px solid var(--border);
|
||||
}
|
||||
.dt-page-brand { display: flex; flex-direction: column; gap: 8px; }
|
||||
.dt-page-brand-row { display: flex; align-items: center; gap: 18px; }
|
||||
.dt-page-brand-mark {
|
||||
width: 56px; height: 56px; border-radius: 14px; background: var(--ink);
|
||||
color: var(--accent-fill); display: inline-flex; align-items: center; justify-content: center;
|
||||
font-weight: 700; font-size: 32px; letter-spacing: -0.04em; line-height: 1; flex-shrink: 0;
|
||||
}
|
||||
.dt-page-brand-words { display: flex; flex-direction: column; gap: 2px; line-height: 1; }
|
||||
.dt-page-eyebrow { font-size: 11.5px; font-weight: 600; letter-spacing: 0.14em; text-transform: uppercase; color: var(--ink-tertiary); line-height: 1.2; }
|
||||
.dt-page-wordmark { margin: 0; font-weight: 600; font-size: 32px; letter-spacing: -0.035em; line-height: 1.1; color: var(--ink); }
|
||||
.dt-page-subtitle { margin: 4px 0 0; color: var(--ink-secondary); font-size: 14px; line-height: 1.5; }
|
||||
.dt-privacy-pill {
|
||||
display: inline-flex; align-items: center; gap: 6px; padding: 6px 11px;
|
||||
background: var(--success-fill); color: var(--success); border-radius: 999px;
|
||||
font-size: 12px; font-weight: 500; white-space: nowrap; flex-shrink: 0;
|
||||
}
|
||||
.dt-privacy-pill svg { width: 13px; height: 13px; stroke-width: 2; }
|
||||
|
||||
/* ---------- Tool header (title + Help popover) ---------- */
|
||||
.dt-tool-header { display: flex; align-items: flex-start; justify-content: space-between; gap: 16px; }
|
||||
.dt-tool-header h1 { margin: 0; }
|
||||
.dt-help-btn {
|
||||
display: inline-flex; align-items: center; gap: 6px; white-space: nowrap;
|
||||
background: var(--surface); color: var(--ink); border: 1px solid var(--border-strong);
|
||||
border-radius: var(--r-md); padding: 9px 16px; font-size: 13.5px; font-weight: 500;
|
||||
cursor: pointer; flex-shrink: 0; margin-top: 6px;
|
||||
}
|
||||
.dt-help-btn .dt-mi { font-family: "Material Symbols Outlined"; font-size: 18px; }
|
||||
.dt-tool-caption { font-size: 12.5px; color: var(--ink-tertiary); line-height: 1.5; margin: 2px 0 0; }
|
||||
/* Right-side actions cluster in a tool header: the local-first privacy pill +
|
||||
the Help button. One shared class so every tool page aligns identically
|
||||
(replaces per-page inline flex/gap/margin drift). */
|
||||
.dt-tool-header-actions { display: flex; align-items: center; gap: 12px; flex-shrink: 0; margin-top: 6px; }
|
||||
.dt-tool-header-actions .dt-help-btn { margin-top: 0; }
|
||||
|
||||
/* ===========================================================================
|
||||
Buttons
|
||||
=========================================================================== */
|
||||
.dt-btn {
|
||||
border-radius: var(--r-md); font-family: var(--font-sans); font-weight: 500;
|
||||
font-size: 13.5px; letter-spacing: -0.005em; line-height: 1; padding: 9px 16px;
|
||||
border: 1px solid var(--border-strong); background: var(--surface); color: var(--ink);
|
||||
cursor: pointer; transition: background 0.12s ease, border-color 0.12s ease, color 0.12s ease;
|
||||
display: inline-flex; align-items: center; justify-content: center; gap: 8px;
|
||||
}
|
||||
.dt-btn:hover { background: var(--surface-hover); border-color: var(--ink-tertiary); }
|
||||
.dt-btn-primary { background: var(--ink); color: var(--bg); border-color: var(--ink); }
|
||||
.dt-btn-primary:hover { background: #292524; border-color: #292524; color: var(--bg); }
|
||||
.dt-btn-tertiary { background: transparent; border: none; color: var(--ink-tertiary); padding: 4px 8px; }
|
||||
.dt-btn-tertiary:hover { background: var(--danger-fill); color: var(--danger); }
|
||||
.dt-btn:disabled, .dt-btn.is-disabled {
|
||||
background: var(--surface-hover); color: var(--ink-tertiary);
|
||||
border: 1px solid var(--border); cursor: not-allowed;
|
||||
}
|
||||
.dt-btn-block { width: 100%; }
|
||||
.dt-btn .dt-mi { font-family: "Material Symbols Outlined"; font-size: 18px; }
|
||||
|
||||
.dt-btn-row { display: flex; gap: 10px; flex-wrap: wrap; }
|
||||
.dt-btn-row > .dt-btn { flex: 1; }
|
||||
|
||||
/* ===========================================================================
|
||||
File uploader (cream dropzone)
|
||||
=========================================================================== */
|
||||
.dt-uploader {
|
||||
background: var(--surface-hover); border: 1px dashed var(--border-strong);
|
||||
border-radius: var(--r-md); padding: 22px 20px;
|
||||
display: flex; align-items: center; justify-content: space-between; gap: 16px;
|
||||
}
|
||||
.dt-uploader-text { display: flex; flex-direction: column; gap: 2px; }
|
||||
.dt-uploader-text .hint { font-size: 14px; color: var(--ink); }
|
||||
.dt-uploader-text .sub { font-size: 12.5px; color: var(--ink-tertiary); }
|
||||
.dt-uploader .dt-mi { font-family: "Material Symbols Outlined"; font-size: 24px; color: var(--ink-tertiary); }
|
||||
|
||||
/* Staged-file chip */
|
||||
.dt-file-chip {
|
||||
display: flex; align-items: center; gap: 12px;
|
||||
background: var(--surface); border: 1px solid var(--border); border-radius: var(--r-sm);
|
||||
padding: 10px 14px; margin-top: 10px;
|
||||
}
|
||||
.dt-file-chip .name { font-family: var(--font-mono); font-size: 13px; color: var(--ink); font-feature-settings: "ss02"; }
|
||||
.dt-file-chip .size { font-family: var(--font-mono); font-size: 12px; color: var(--ink-tertiary); margin-left: auto; }
|
||||
|
||||
/* ===========================================================================
|
||||
Expanders / bordered cards
|
||||
=========================================================================== */
|
||||
.dt-expander {
|
||||
background: var(--surface); border: 1px solid var(--border); border-radius: var(--r-lg);
|
||||
overflow: hidden; box-shadow: 0 1px 2px rgba(28,25,23,0.03); margin: 10px 0;
|
||||
}
|
||||
.dt-expander > summary, .dt-expander-head {
|
||||
background: var(--surface-hover); border-bottom: 1px solid var(--border);
|
||||
padding: 12px 16px; font-weight: 500; color: var(--ink); font-size: 14px;
|
||||
cursor: pointer; list-style: none; display: flex; align-items: center; gap: 8px;
|
||||
}
|
||||
.dt-expander > summary::-webkit-details-marker { display: none; }
|
||||
.dt-expander > summary::before {
|
||||
content: "expand_more"; font-family: "Material Symbols Outlined"; font-size: 20px;
|
||||
color: var(--ink-tertiary); transition: transform 0.15s ease;
|
||||
}
|
||||
.dt-expander[open] > summary::before { transform: rotate(180deg); }
|
||||
.dt-expander-body, .dt-expander > .dt-expander-body { padding: 14px 16px; }
|
||||
.dt-expander:not([open]) > summary { border-bottom: none; }
|
||||
|
||||
.dt-card {
|
||||
background: var(--surface); border: 1px solid var(--border); border-radius: var(--r-lg);
|
||||
box-shadow: 0 1px 2px rgba(28,25,23,0.03); padding: 16px; margin: 10px 0;
|
||||
}
|
||||
|
||||
/* ===========================================================================
|
||||
Alerts
|
||||
=========================================================================== */
|
||||
.dt-alert {
|
||||
border-radius: var(--r-md); border: 1px solid transparent;
|
||||
padding: 10px 14px; font-size: 13.5px; line-height: 1.45; margin: 10px 0;
|
||||
display: flex; gap: 10px; align-items: flex-start;
|
||||
}
|
||||
.dt-alert .dt-mi { font-family: "Material Symbols Outlined"; font-size: 18px; flex-shrink: 0; margin-top: 1px; }
|
||||
.dt-alert.info { background: var(--info-fill); color: var(--info); }
|
||||
.dt-alert.success { background: var(--success-fill); color: var(--success); }
|
||||
.dt-alert.warn { background: var(--warn-fill); color: var(--warn); }
|
||||
.dt-alert.error { background: var(--danger-fill); color: var(--danger); }
|
||||
.dt-alert code { background: rgba(0,0,0,0.05); padding: 1px 5px; border-radius: 4px; }
|
||||
|
||||
/* Next-step strip — slim single-line "what to do next" suggestion shown at the
|
||||
end of a tool's results. Subtle accent ground + left accent rule so it nudges
|
||||
without competing with alerts; the trailing dismiss control is unobtrusive. */
|
||||
.dt-next-step {
|
||||
display: flex; align-items: center; gap: 10px;
|
||||
background: var(--accent-fill); border-left: 3px solid var(--accent);
|
||||
border-radius: var(--r-md); padding: 10px 14px; margin: 16px 0;
|
||||
font-size: 13.5px; line-height: 1.4; color: var(--ink);
|
||||
}
|
||||
.dt-next-step .dt-mi { font-family: "Material Symbols Outlined"; font-size: 18px; color: var(--accent); flex-shrink: 0; }
|
||||
.dt-next-step a { color: var(--accent); font-weight: 500; }
|
||||
.dt-next-step a:hover { color: var(--accent-hover); }
|
||||
.dt-next-step-dismiss {
|
||||
margin-left: auto; background: transparent; border: none; cursor: pointer;
|
||||
color: var(--ink-tertiary); font-size: 13px; line-height: 1; padding: 2px 4px;
|
||||
}
|
||||
.dt-next-step-dismiss:hover { color: var(--ink-secondary); }
|
||||
|
||||
/* ===========================================================================
|
||||
Inputs (static representations of Streamlit widgets)
|
||||
=========================================================================== */
|
||||
.dt-field { margin: 10px 0; }
|
||||
.dt-label { font-size: 13px; font-weight: 500; color: var(--ink); margin-bottom: 5px; display: block; }
|
||||
.dt-label .req { color: var(--accent); }
|
||||
.dt-input, .dt-select, .dt-textarea {
|
||||
width: 100%; background: var(--surface); border: 1px solid var(--border-strong);
|
||||
border-radius: var(--r-sm); padding: 8px 11px; font-family: var(--font-sans);
|
||||
font-size: 13.5px; color: var(--ink);
|
||||
}
|
||||
.dt-select { appearance: none; background-image: linear-gradient(45deg, transparent 50%, var(--ink-tertiary) 50%), linear-gradient(135deg, var(--ink-tertiary) 50%, transparent 50%); background-position: calc(100% - 16px) 14px, calc(100% - 11px) 14px; background-size: 5px 5px, 5px 5px; background-repeat: no-repeat; }
|
||||
.dt-textarea { min-height: 76px; resize: vertical; font-family: var(--font-mono); font-size: 13px; }
|
||||
.dt-help-text { font-size: 12px; color: var(--ink-tertiary); margin-top: 4px; }
|
||||
|
||||
/* Multiselect — chips inside a box */
|
||||
.dt-multiselect {
|
||||
width: 100%; background: var(--surface); border: 1px solid var(--border-strong);
|
||||
border-radius: var(--r-sm); padding: 6px 8px; min-height: 38px;
|
||||
display: flex; flex-wrap: wrap; gap: 6px; align-items: center;
|
||||
}
|
||||
.dt-ms-chip {
|
||||
display: inline-flex; align-items: center; gap: 5px; background: var(--accent-fill);
|
||||
color: var(--accent-hover); border-radius: var(--r-sm); padding: 3px 8px;
|
||||
font-size: 12.5px; font-weight: 500;
|
||||
}
|
||||
.dt-ms-chip .x { color: var(--accent); font-size: 13px; }
|
||||
.dt-ms-placeholder { color: var(--ink-tertiary); font-size: 13px; padding: 2px 4px; }
|
||||
|
||||
/* Checkbox / radio */
|
||||
.dt-check { display: flex; align-items: center; gap: 9px; margin: 8px 0; font-size: 13.5px; color: var(--ink); }
|
||||
.dt-check .box {
|
||||
width: 18px; height: 18px; border-radius: 5px; border: 1px solid var(--border-strong);
|
||||
background: var(--surface); display: inline-flex; align-items: center; justify-content: center; flex-shrink: 0;
|
||||
}
|
||||
.dt-check.on .box { background: var(--ink); border-color: var(--ink); color: var(--bg); }
|
||||
.dt-check.on .box .dt-mi { font-family: "Material Symbols Outlined"; font-size: 14px; }
|
||||
.dt-radio-row { display: flex; gap: 18px; flex-wrap: wrap; margin: 8px 0; }
|
||||
.dt-radio { display: inline-flex; align-items: center; gap: 7px; font-size: 13.5px; }
|
||||
.dt-radio .dot { width: 16px; height: 16px; border-radius: 50%; border: 1px solid var(--border-strong); display: inline-block; flex-shrink: 0; }
|
||||
.dt-radio.on .dot { border: 5px solid var(--ink); }
|
||||
|
||||
/* Strategy precedence legend + overridden state (Fix Missing Values).
|
||||
Makes the preset -> global -> per-column resolution order legible and
|
||||
visibly dims a layer when a more specific layer wins. */
|
||||
.dt-precedence {
|
||||
display: flex; align-items: center; gap: 8px;
|
||||
background: var(--surface-hover); border: 1px solid var(--border);
|
||||
border-radius: var(--r-md); padding: 9px 13px; margin: 0 0 14px;
|
||||
font-size: 12.5px; color: var(--ink-secondary); line-height: 1.4;
|
||||
}
|
||||
.dt-precedence .dt-mi { font-family: "Material Symbols Outlined"; font-size: 18px; color: var(--ink-tertiary); flex-shrink: 0; }
|
||||
.dt-precedence strong { color: var(--ink); font-weight: 600; }
|
||||
.dt-radio-row.is-overridden { opacity: 0.5; }
|
||||
.dt-radio-row.is-overridden .dt-radio { text-decoration: line-through; text-decoration-color: var(--ink-tertiary); }
|
||||
|
||||
/* Slider */
|
||||
.dt-slider { margin: 14px 0 6px; }
|
||||
.dt-slider .track { position: relative; height: 4px; background: var(--border-strong); border-radius: 2px; }
|
||||
.dt-slider .fill { position: absolute; left: 0; top: 0; height: 4px; background: var(--ink); border-radius: 2px; }
|
||||
.dt-slider .knob { position: absolute; top: 50%; width: 16px; height: 16px; border-radius: 50%; background: var(--ink); transform: translate(-50%, -50%); }
|
||||
.dt-slider .val { font-family: var(--font-mono); font-size: 12px; color: var(--ink-secondary); margin-top: 8px; }
|
||||
|
||||
/* ===========================================================================
|
||||
Layout helpers
|
||||
=========================================================================== */
|
||||
.dt-row { display: flex; gap: 16px; }
|
||||
.dt-row > * { flex: 1; min-width: 0; }
|
||||
.dt-cols-2 { display: grid; grid-template-columns: 1fr 1fr; gap: 16px; }
|
||||
.dt-cols-3 { display: grid; grid-template-columns: repeat(3, 1fr); gap: 16px; }
|
||||
.dt-divider { border: none; border-top: 1px solid var(--border); margin: 22px 0; }
|
||||
.dt-caption { font-size: 12.5px; color: var(--ink-tertiary); line-height: 1.5; }
|
||||
.dt-spacer { height: 12px; }
|
||||
|
||||
/* ===========================================================================
|
||||
DataFrame / preview table
|
||||
=========================================================================== */
|
||||
.dt-table-wrap { border: 1px solid var(--border); border-radius: var(--r-md); overflow: hidden; margin: 8px 0; }
|
||||
table.dt-table { width: 100%; border-collapse: collapse; font-size: 13px; }
|
||||
table.dt-table th {
|
||||
background: var(--surface-hover); color: var(--ink-secondary); font-weight: 500;
|
||||
text-align: left; padding: 8px 12px; border-bottom: 1px solid var(--border);
|
||||
font-size: 12px; text-transform: none; white-space: nowrap;
|
||||
}
|
||||
table.dt-table td {
|
||||
padding: 7px 12px; border-bottom: 1px solid var(--border);
|
||||
font-family: var(--font-mono); font-size: 12.5px; color: var(--ink); font-feature-settings: "ss02"; white-space: nowrap;
|
||||
}
|
||||
table.dt-table tr:last-child td { border-bottom: none; }
|
||||
table.dt-table tr:nth-child(even) td { background: #fcfbf8; }
|
||||
table.dt-table td.idx { color: var(--ink-tertiary); background: var(--surface-hover); }
|
||||
.dt-cell-flag { color: var(--warn); }
|
||||
.dt-cell-del { color: var(--danger); text-decoration: line-through; }
|
||||
.dt-cell-add { color: var(--success); }
|
||||
|
||||
/* ===========================================================================
|
||||
Stats overview (home) — copied from _legacy.py
|
||||
=========================================================================== */
|
||||
.dt-stats { display: grid; grid-template-columns: repeat(4, 1fr); gap: 12px; margin: 8px 0 20px; }
|
||||
.dt-stat { background: var(--surface); border: 1px solid var(--border); border-radius: var(--r-lg); padding: 16px 18px; box-shadow: 0 1px 2px rgba(28,25,23,0.03); }
|
||||
.dt-stat-label { font-size: 11.5px; text-transform: uppercase; letter-spacing: 0.08em; color: var(--ink-tertiary); font-weight: 500; margin-bottom: 6px; line-height: 1.4; }
|
||||
.dt-stat-value { font-size: 28px; font-weight: 600; letter-spacing: -0.03em; line-height: 1; color: var(--ink); display: flex; align-items: baseline; gap: 6px; }
|
||||
.dt-stat-unit { font-size: 12px; font-weight: 400; color: var(--ink-tertiary); letter-spacing: 0; }
|
||||
.dt-stat.is-warn .dt-stat-value { color: var(--warn); }
|
||||
.dt-stat.is-info .dt-stat-value { color: var(--info); }
|
||||
.dt-stat.is-success .dt-stat-value { color: var(--success); }
|
||||
@media (max-width: 900px) { .dt-stats { grid-template-columns: repeat(2, 1fr); } }
|
||||
|
||||
/* Metric (st.metric) */
|
||||
.dt-metrics { display: flex; gap: 28px; flex-wrap: wrap; margin: 6px 0 14px; }
|
||||
.dt-metric .label { font-size: 12.5px; color: var(--ink-tertiary); margin-bottom: 4px; }
|
||||
.dt-metric .value { font-size: 26px; font-weight: 600; letter-spacing: -0.03em; color: var(--ink); line-height: 1; }
|
||||
.dt-metric .delta { font-size: 12.5px; margin-top: 3px; }
|
||||
.dt-metric .delta.up { color: var(--success); }
|
||||
.dt-metric .delta.down { color: var(--danger); }
|
||||
|
||||
/* ===========================================================================
|
||||
Files card (home) — copied from _legacy.py
|
||||
=========================================================================== */
|
||||
.dt-files-section-head { display: flex; align-items: baseline; justify-content: space-between; margin: 4px 0 10px; gap: 12px; }
|
||||
.dt-files-section-head h2 { margin: 0; }
|
||||
.dt-section-meta { font-size: 12.5px; color: var(--ink-tertiary); }
|
||||
.dt-file-row { display: flex; align-items: center; gap: 12px; }
|
||||
.dt-file-icon-chip { width: 28px; height: 28px; border-radius: var(--r-sm); background: var(--accent-fill); color: var(--accent); display: inline-flex; align-items: center; justify-content: center; flex-shrink: 0; }
|
||||
.dt-file-icon-chip svg { width: 14px; height: 14px; stroke-width: 1.8; }
|
||||
.dt-file-name { font-family: var(--font-mono); font-size: 13px; color: var(--ink); font-feature-settings: "ss02"; }
|
||||
.dt-file-size { font-family: var(--font-mono); font-size: 12px; color: var(--ink-tertiary); font-feature-settings: "ss02"; }
|
||||
.dt-file-add {
|
||||
display: flex; align-items: center; justify-content: center; gap: 8px;
|
||||
width: 100%; padding: 12px 16px; background: var(--surface-hover);
|
||||
border: none; border-top: 1px dashed var(--border-strong);
|
||||
border-radius: 0 0 var(--r-lg) var(--r-lg); cursor: pointer;
|
||||
font-size: 13px; font-weight: 500; color: var(--ink-secondary); margin-top: 14px;
|
||||
}
|
||||
.dt-file-add:hover { background: var(--accent-fill); color: var(--accent); }
|
||||
.dt-file-add svg { width: 14px; height: 14px; stroke-width: 2; }
|
||||
|
||||
/* ===========================================================================
|
||||
Findings panel — copied from _legacy.py
|
||||
=========================================================================== */
|
||||
.dt-finding-group-head {
|
||||
display: flex; align-items: center; gap: 12px; padding: 16px 22px;
|
||||
border-bottom: 1px solid var(--border); background: var(--surface-hover);
|
||||
margin: -16px -16px 1.2rem; border-radius: var(--r-lg) var(--r-lg) 0 0;
|
||||
cursor: pointer; user-select: none;
|
||||
}
|
||||
.dt-finding-group-chevron { color: var(--ink-tertiary); font-family: "Material Symbols Outlined"; font-size: 20px; line-height: 1; flex-shrink: 0; }
|
||||
.dt-severity-dot { width: 8px; height: 8px; border-radius: 50%; flex-shrink: 0; display: inline-block; }
|
||||
.dt-severity-dot.warn { background: var(--warn); }
|
||||
.dt-severity-dot.info { background: var(--info); }
|
||||
.dt-severity-dot.error { background: var(--danger); }
|
||||
.dt-severity-dot.success { background: var(--success); }
|
||||
.dt-group-filename { font-family: var(--font-mono); font-size: 13.5px; font-weight: 500; color: var(--ink); font-feature-settings: "ss02"; }
|
||||
.dt-group-counts { margin-left: auto; display: flex; align-items: center; gap: 8px; }
|
||||
.dt-count-pill { display: inline-flex; align-items: center; padding: 3px 9px; border-radius: 999px; font-size: 11.5px; font-weight: 500; line-height: 1.4; white-space: nowrap; }
|
||||
.dt-count-pill.warn { background: var(--warn-fill); color: var(--warn); }
|
||||
.dt-count-pill.info { background: var(--info-fill); color: var(--info); }
|
||||
.dt-count-pill.error { background: var(--danger-fill); color: var(--danger); }
|
||||
.dt-count-pill.success { background: var(--success-fill); color: var(--success); }
|
||||
.dt-finding-row { display: flex; align-items: flex-start; gap: 12px; padding: 12px 0; border-top: 1px solid var(--border); }
|
||||
.dt-finding-row:first-of-type { border-top: none; }
|
||||
.dt-finding-icon { width: 24px; height: 24px; border-radius: var(--r-sm); display: inline-flex; align-items: center; justify-content: center; flex-shrink: 0; }
|
||||
.dt-finding-icon.warn { background: var(--warn-fill); color: var(--warn); }
|
||||
.dt-finding-icon.info { background: var(--info-fill); color: var(--info); }
|
||||
.dt-finding-icon.error { background: var(--danger-fill); color: var(--danger); }
|
||||
.dt-finding-icon .dt-mi { font-family: "Material Symbols Outlined"; font-size: 16px; line-height: 1; }
|
||||
.dt-finding-body { flex: 1; min-width: 0; }
|
||||
.dt-finding-title { font-size: 14px; color: var(--ink); margin: 0 0 2px; line-height: 1.4; letter-spacing: -0.005em; }
|
||||
.dt-finding-title strong { font-weight: 500; }
|
||||
.dt-finding-meta { font-family: var(--font-mono); font-size: 12px; color: var(--ink-tertiary); line-height: 1.4; margin: 0; font-feature-settings: "ss02"; }
|
||||
|
||||
/* Overflow control — sits at the foot of a findings card when rows are hidden.
|
||||
Bleeds to the card edges (cancels the .dt-card 16px padding) like .dt-file-add. */
|
||||
.dt-finding-more {
|
||||
display: flex; align-items: center; justify-content: center; gap: 6px;
|
||||
width: calc(100% + 32px); margin: 4px -16px -16px;
|
||||
padding: 11px 16px; background: var(--surface-hover);
|
||||
border: none; border-top: 1px solid var(--border);
|
||||
border-radius: 0 0 var(--r-lg) var(--r-lg); cursor: pointer;
|
||||
font-family: var(--font-sans); font-size: 12.5px; font-weight: 500; color: var(--ink-secondary);
|
||||
}
|
||||
.dt-finding-more:hover { background: var(--accent-fill); color: var(--accent); }
|
||||
.dt-finding-more .dt-mi { font-family: "Material Symbols Outlined"; font-size: 18px; }
|
||||
|
||||
/* Collapsed findings panel — the group head fills the whole card (head only,
|
||||
no body). Proper state variant so the two states don't drift; replaces the
|
||||
per-instance inline margin-bottom:-16px hack. */
|
||||
.dt-card.is-collapsed { padding: 0; }
|
||||
.dt-finding-group-head.is-collapsed { margin: 0; border-bottom: none; border-radius: var(--r-lg); }
|
||||
|
||||
/* Match-group review card (dedup) */
|
||||
.dt-match-card { background: var(--surface); border: 1px solid var(--border); border-radius: var(--r-lg); box-shadow: 0 1px 2px rgba(28,25,23,0.03); margin: 12px 0; overflow: hidden; }
|
||||
.dt-match-head { background: var(--surface-hover); border-bottom: 1px solid var(--border); padding: 12px 16px; display: flex; align-items: center; gap: 12px; }
|
||||
.dt-match-head .title { font-weight: 500; font-size: 14px; }
|
||||
.dt-match-head .conf { margin-left: auto; }
|
||||
.dt-match-body { padding: 14px 16px; }
|
||||
.dt-keep-row { background: var(--success-fill); }
|
||||
.dt-keep-tag { display: inline-flex; align-items: center; gap: 4px; background: var(--success-fill); color: var(--success); border-radius: 999px; padding: 2px 8px; font-size: 11px; font-weight: 500; }
|
||||
|
||||
/* Progress bar */
|
||||
.dt-progress { height: 6px; background: var(--border); border-radius: 3px; overflow: hidden; margin: 10px 0; }
|
||||
.dt-progress .bar { height: 100%; background: var(--ink); border-radius: 3px; }
|
||||
|
||||
/* Tabs */
|
||||
.dt-tabs { display: flex; gap: 18px; border-bottom: 1px solid var(--border); margin: 10px 0 16px; }
|
||||
.dt-tab { font-size: 13.5px; color: var(--ink-secondary); padding: 8px 2px; border-bottom: 2px solid transparent; cursor: pointer; }
|
||||
.dt-tab.is-active { color: var(--ink); font-weight: 500; border-bottom-color: var(--accent); }
|
||||
|
||||
/* Code block */
|
||||
.dt-code { background: var(--surface-hover); border: 1px solid var(--border); border-radius: var(--r-md); padding: 12px 14px; font-family: var(--font-mono); font-size: 12.5px; color: var(--ink); white-space: pre; overflow-x: auto; font-feature-settings: "ss02"; }
|
||||
|
||||
@media (max-width: 1100px) {
|
||||
.dt-footer { left: 0; }
|
||||
.dt-sidebar { display: none; }
|
||||
.dt-main { padding: 28px 24px 96px; }
|
||||
}
|
||||
206
layout-review/home.html
Normal file
206
layout-review/home.html
Normal file
@@ -0,0 +1,206 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Layout review — File Analysis (Home)</title>
|
||||
<link rel="stylesheet" href="app.css">
|
||||
</head>
|
||||
<body data-page="home">
|
||||
<div class="dt-app">
|
||||
<aside class="dt-sidebar" id="dt-sidebar"></aside>
|
||||
<main class="dt-main">
|
||||
<div class="dt-review-banner">
|
||||
<span class="dt-mi">visibility</span>
|
||||
<span>Static layout preview of the <strong>Home / File Analysis</strong> page, shown with three imported files in the post-analysis state. <a href="index.html">All pages →</a></span>
|
||||
</div>
|
||||
<div class="dt-main-inner">
|
||||
|
||||
<!-- Page header: brand block + privacy pill -->
|
||||
<header class="dt-page-header">
|
||||
<div class="dt-page-brand">
|
||||
<div class="dt-page-brand-row">
|
||||
<div class="dt-page-brand-mark">D</div>
|
||||
<div class="dt-page-brand-words">
|
||||
<span class="dt-page-eyebrow">UNALOGIX</span>
|
||||
<h1 class="dt-page-wordmark">DataTools</h1>
|
||||
</div>
|
||||
</div>
|
||||
<p class="dt-page-subtitle">Clean. Normalize. Transform.</p>
|
||||
</div>
|
||||
<span class="dt-privacy-pill">
|
||||
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor">
|
||||
<rect x="4" y="11" width="16" height="10" rx="2"/>
|
||||
<path d="M8 11V7a4 4 0 018 0v4"/>
|
||||
</svg>
|
||||
Runs 100% locally
|
||||
</span>
|
||||
</header>
|
||||
|
||||
<!-- Files section head -->
|
||||
<div class="dt-files-section-head">
|
||||
<h2>Files</h2>
|
||||
<span class="dt-section-meta">3 files · 4.7 MB total</span>
|
||||
</div>
|
||||
|
||||
<!-- Files card -->
|
||||
<div class="dt-card" style="padding-bottom:0">
|
||||
<div class="dt-file-row" style="padding:6px 0">
|
||||
<button class="dt-btn dt-btn-tertiary" title="Remove">✕</button>
|
||||
<span class="dt-file-icon-chip"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor"><path d="M14 2H6a2 2 0 00-2 2v16a2 2 0 002 2h12a2 2 0 002-2V8z"/><path d="M14 2v6h6"/></svg></span>
|
||||
<span class="dt-file-name">customers_export.csv</span>
|
||||
<span class="dt-file-size" style="margin-left:auto">2.1 MB</span>
|
||||
</div>
|
||||
<div class="dt-file-row" style="padding:6px 0">
|
||||
<button class="dt-btn dt-btn-tertiary" title="Remove">✕</button>
|
||||
<span class="dt-file-icon-chip"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor"><path d="M14 2H6a2 2 0 00-2 2v16a2 2 0 002 2h12a2 2 0 002-2V8z"/><path d="M14 2v6h6"/></svg></span>
|
||||
<span class="dt-file-name">q3_transactions.xlsx</span>
|
||||
<span class="dt-file-size" style="margin-left:auto">1.8 MB</span>
|
||||
</div>
|
||||
<div class="dt-file-row" style="padding:6px 0">
|
||||
<button class="dt-btn dt-btn-tertiary" title="Remove">✕</button>
|
||||
<span class="dt-file-icon-chip"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor"><path d="M14 2H6a2 2 0 00-2 2v16a2 2 0 002 2h12a2 2 0 002-2V8z"/><path d="M14 2v6h6"/></svg></span>
|
||||
<span class="dt-file-name">vendor_list.csv</span>
|
||||
<span class="dt-file-size" style="margin-left:auto">0.8 MB</span>
|
||||
</div>
|
||||
<button class="dt-file-add" style="margin-left:-16px;margin-right:-16px;width:calc(100% + 32px)">
|
||||
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor"><path d="M12 5v14M5 12h14"/></svg> Add more files
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<!-- Action bar -->
|
||||
<div class="dt-btn-row" style="margin-top:16px">
|
||||
<button class="dt-btn dt-btn-primary" style="flex:0 0 auto">Run analysis</button>
|
||||
<button class="dt-btn" style="flex:0 0 auto">Clear results</button>
|
||||
</div>
|
||||
|
||||
<hr class="dt-divider">
|
||||
|
||||
<!-- Stats overview -->
|
||||
<div class="dt-stats">
|
||||
<div class="dt-stat">
|
||||
<div class="dt-stat-label">Rows scanned</div>
|
||||
<div class="dt-stat-value">48,210 <span class="dt-stat-unit">rows</span></div>
|
||||
</div>
|
||||
<div class="dt-stat">
|
||||
<div class="dt-stat-label">Total findings</div>
|
||||
<div class="dt-stat-value">14</div>
|
||||
</div>
|
||||
<div class="dt-stat is-warn">
|
||||
<div class="dt-stat-label">Warnings</div>
|
||||
<div class="dt-stat-value">9 <span class="dt-stat-unit">to review</span></div>
|
||||
</div>
|
||||
<div class="dt-stat is-info">
|
||||
<div class="dt-stat-label">Info</div>
|
||||
<div class="dt-stat-value">5 <span class="dt-stat-unit">suggestions</span></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- ======================================================================
|
||||
FRONT DOOR — primary path. The orchestrator (09_pipeline_runner)
|
||||
wearing a friendly face: maps the analyzer's findings to the
|
||||
recommended pipeline (Clean Text → Standardize → Fix Missing →
|
||||
Find Duplicates) and runs them in order, returning a downloadable
|
||||
result. This is the hero of the page; the per-file findings below
|
||||
remain as the manual "fix one thing at a time" path.
|
||||
====================================================================== -->
|
||||
<div class="dt-card" style="border-color:var(--accent);background:var(--accent-fill);box-shadow:0 1px 2px rgba(28,25,23,0.03),0 0 0 1px var(--accent)">
|
||||
<div style="display:flex;align-items:flex-start;gap:14px;flex-wrap:wrap">
|
||||
<span class="dt-file-icon-chip" style="width:36px;height:36px;border-radius:var(--r-md)">
|
||||
<span class="dt-mi" style="font-family:'Material Symbols Outlined';font-size:20px">auto_awesome</span>
|
||||
</span>
|
||||
<div style="flex:1;min-width:240px">
|
||||
<h3 style="margin:0 0 4px;color:var(--ink)">Recommended</h3>
|
||||
<p style="margin:0;color:var(--ink-secondary)">Runs the recommended clean — fix text, standardize formats, fill blanks, remove duplicates — in the right order, then hands you the cleaned file.</p>
|
||||
</div>
|
||||
<button class="dt-btn dt-btn-primary" style="flex:0 0 auto;align-self:center">
|
||||
<span class="dt-mi">auto_fix_high</span> Clean these files for me
|
||||
</button>
|
||||
</div>
|
||||
<!-- Pipeline-step affordance: the order the findings will be resolved in -->
|
||||
<div style="display:flex;align-items:center;gap:6px;flex-wrap:wrap;margin-top:14px;padding-top:12px;border-top:1px solid var(--accent-fill-strong)">
|
||||
<span class="dt-count-pill" style="background:var(--surface);color:var(--ink-secondary)">1 · Clean Text</span>
|
||||
<span class="dt-mi" style="font-family:'Material Symbols Outlined';font-size:16px;color:var(--accent)">arrow_forward</span>
|
||||
<span class="dt-count-pill" style="background:var(--surface);color:var(--ink-secondary)">2 · Standardize</span>
|
||||
<span class="dt-mi" style="font-family:'Material Symbols Outlined';font-size:16px;color:var(--accent)">arrow_forward</span>
|
||||
<span class="dt-count-pill" style="background:var(--surface);color:var(--ink-secondary)">3 · Fix Missing</span>
|
||||
<span class="dt-mi" style="font-family:'Material Symbols Outlined';font-size:16px;color:var(--accent)">arrow_forward</span>
|
||||
<span class="dt-count-pill" style="background:var(--surface);color:var(--ink-secondary)">4 · Find Duplicates</span>
|
||||
<span class="dt-caption" style="margin-left:auto">Result downloads when finished</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Secondary / manual path — keep full control over each fix -->
|
||||
<h3 style="margin-top:24px">Or fix issues one at a time</h3>
|
||||
<p class="dt-caption" style="margin:-2px 0 4px">Prefer to handle things yourself? Open any finding to jump straight to the right tool.</p>
|
||||
|
||||
<!-- Per-file findings panel #1 -->
|
||||
<div class="dt-card">
|
||||
<div class="dt-finding-group-head">
|
||||
<span class="dt-finding-group-chevron" style="transform:rotate(90deg)">chevron_right</span>
|
||||
<span class="dt-severity-dot warn"></span>
|
||||
<span class="dt-group-filename">customers_export.csv</span>
|
||||
<div class="dt-group-counts">
|
||||
<span class="dt-count-pill warn">6 warnings</span>
|
||||
<span class="dt-count-pill info">2 info</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="dt-finding-row">
|
||||
<span class="dt-finding-icon warn"><span class="dt-mi">priority_high</span></span>
|
||||
<div class="dt-finding-body">
|
||||
<p class="dt-finding-title"><strong>312 duplicate rows</strong> across exact + near matches</p>
|
||||
<p class="dt-finding-meta">column: email · Find Duplicates →</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="dt-finding-row">
|
||||
<span class="dt-finding-icon warn"><span class="dt-mi">format_color_text</span></span>
|
||||
<div class="dt-finding-body">
|
||||
<p class="dt-finding-title"><strong>1,204 cells</strong> with leading / trailing whitespace</p>
|
||||
<p class="dt-finding-meta">columns: name, city · Clean Text →</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="dt-finding-row">
|
||||
<span class="dt-finding-icon info"><span class="dt-mi">event</span></span>
|
||||
<div class="dt-finding-body">
|
||||
<p class="dt-finding-title">Mixed date formats in <strong>signup_date</strong></p>
|
||||
<p class="dt-finding-meta">3 formats detected · Standardize Formats →</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<button class="dt-finding-more">
|
||||
<span class="dt-mi">expand_more</span> Show all 8 findings · 5 more
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<!-- Per-file findings panel #2 (collapsed) -->
|
||||
<div class="dt-card is-collapsed">
|
||||
<div class="dt-finding-group-head is-collapsed">
|
||||
<span class="dt-finding-group-chevron">chevron_right</span>
|
||||
<span class="dt-severity-dot warn"></span>
|
||||
<span class="dt-group-filename">q3_transactions.xlsx</span>
|
||||
<div class="dt-group-counts">
|
||||
<span class="dt-count-pill warn">3 warnings</span>
|
||||
<span class="dt-count-pill info">3 info</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Per-file findings panel #3 (clean) -->
|
||||
<div class="dt-card is-collapsed">
|
||||
<div class="dt-finding-group-head is-collapsed">
|
||||
<span class="dt-severity-dot success"></span>
|
||||
<span class="dt-group-filename">vendor_list.csv</span>
|
||||
<div class="dt-group-counts">
|
||||
<span class="dt-count-pill success">no issues</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</main>
|
||||
</div>
|
||||
<footer class="dt-footer" id="dt-footer"></footer>
|
||||
<script src="shell.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
71
layout-review/index.html
Normal file
71
layout-review/index.html
Normal file
@@ -0,0 +1,71 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>DataTools — Layout Review</title>
|
||||
<link rel="stylesheet" href="app.css">
|
||||
<style>
|
||||
.lr-wrap { max-width: 960px; margin: 0 auto; padding: 48px 32px 80px; }
|
||||
.lr-grid { display: grid; grid-template-columns: repeat(2, 1fr); gap: 14px; margin-top: 18px; }
|
||||
.lr-card { display: flex; align-items: center; gap: 14px; background: var(--surface); border: 1px solid var(--border); border-radius: var(--r-lg); padding: 16px 18px; box-shadow: 0 1px 2px rgba(28,25,23,0.03); text-decoration: none; transition: border-color .12s ease, box-shadow .12s ease; }
|
||||
.lr-card:hover { border-color: var(--border-strong); box-shadow: 0 2px 8px rgba(28,25,23,0.06); text-decoration: none; }
|
||||
.lr-ico { width: 40px; height: 40px; border-radius: var(--r-md); background: var(--accent-fill); color: var(--accent); display: inline-flex; align-items: center; justify-content: center; flex-shrink: 0; }
|
||||
.lr-ico .dt-mi { font-family: "Material Symbols Outlined"; font-size: 22px; }
|
||||
.lr-body { min-width: 0; }
|
||||
.lr-name { font-size: 15px; font-weight: 600; color: var(--ink); letter-spacing: -0.01em; display:flex; align-items:center; gap:8px; }
|
||||
.lr-desc { font-size: 12.5px; color: var(--ink-secondary); margin-top: 2px; line-height: 1.45; }
|
||||
.lr-sec { font-size: 11.5px; text-transform: uppercase; letter-spacing: 0.08em; color: var(--ink-tertiary); font-weight: 600; margin: 26px 0 2px; }
|
||||
.lr-soon { font-size: 9px; font-weight: 600; letter-spacing: .06em; text-transform: uppercase; color: var(--ink-tertiary); border: 1px solid var(--border-strong); border-radius: 999px; padding: 1px 6px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="lr-wrap">
|
||||
<header class="dt-page-header">
|
||||
<div class="dt-page-brand">
|
||||
<div class="dt-page-brand-row">
|
||||
<div class="dt-page-brand-mark">D</div>
|
||||
<div class="dt-page-brand-words">
|
||||
<span class="dt-page-eyebrow">UNALOGIX · LAYOUT REVIEW</span>
|
||||
<h1 class="dt-page-wordmark">DataTools</h1>
|
||||
</div>
|
||||
</div>
|
||||
<p class="dt-page-subtitle">Static HTML reproductions of every tool page, built from the live app's design tokens for human review of layouts.</p>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<div class="dt-alert info">
|
||||
<span class="dt-mi">info</span>
|
||||
<span>These are faithful static mockups — not the running Streamlit app. Colors, type scale, spacing, and components are copied verbatim from <code>theme.py</code> and <code>components/_legacy.py</code>. Each page is shown in a representative <strong>populated</strong> state so the layout can be reviewed end-to-end. Fonts load from Google Fonts (needs network); the chrome (sidebar + footer) is shared across every page.</span>
|
||||
</div>
|
||||
|
||||
<div class="lr-sec">Analysis</div>
|
||||
<div class="lr-grid">
|
||||
<a class="lr-card" href="home.html"><span class="lr-ico"><span class="dt-mi">insert_chart_outlined</span></span><span class="lr-body"><span class="lr-name">File Analysis (Home)</span><span class="lr-desc">Import files, run the analyzer, browse per-file findings.</span></span></a>
|
||||
<a class="lr-card" href="11_reconciler.html"><span class="lr-ico"><span class="dt-mi">compare_arrows</span></span><span class="lr-body"><span class="lr-name">Reconcile Two Files</span><span class="lr-desc">Compare two lists of transactions and flag what doesn't match.</span></span></a>
|
||||
</div>
|
||||
|
||||
<div class="lr-sec">Data Cleaners</div>
|
||||
<div class="lr-grid">
|
||||
<a class="lr-card" href="04_missing_handler.html"><span class="lr-ico"><span class="dt-mi">help_outline</span></span><span class="lr-body"><span class="lr-name">Fix Missing Values</span><span class="lr-desc">Find blank cells (even hidden ones) and fill them in or remove them.</span></span></a>
|
||||
<a class="lr-card" href="06_outlier_detector.html"><span class="lr-ico"><span class="dt-mi">insights</span></span><span class="lr-body"><span class="lr-name">Find Unusual Values <span class="lr-soon">Soon</span></span><span class="lr-desc">Spot values that look wrong — too high, too low, or rule-breaking.</span></span></a>
|
||||
<a class="lr-card" href="02_text_cleaner.html"><span class="lr-ico"><span class="dt-mi">text_format</span></span><span class="lr-body"><span class="lr-name">Clean Text</span><span class="lr-desc">Trim extra spaces and strip out odd characters.</span></span></a>
|
||||
<a class="lr-card" href="03_format_standardizer.html"><span class="lr-ico"><span class="dt-mi">format_list_bulleted</span></span><span class="lr-body"><span class="lr-name">Standardize Formats</span><span class="lr-desc">Make dates, phones, currency, and names look the same throughout.</span></span></a>
|
||||
<a class="lr-card" href="01_deduplicator.html"><span class="lr-ico"><span class="dt-mi">search</span></span><span class="lr-body"><span class="lr-name">Find Duplicates</span><span class="lr-desc">Find rows that repeat, then keep one and remove the extras.</span></span></a>
|
||||
<a class="lr-card" href="08_validator_reporter.html"><span class="lr-ico"><span class="dt-mi">check_circle</span></span><span class="lr-body"><span class="lr-name">Quality Check <span class="lr-soon">Soon</span></span><span class="lr-desc">Check your file against rules and export a PDF or Excel report.</span></span></a>
|
||||
</div>
|
||||
|
||||
<div class="lr-sec">Transformations</div>
|
||||
<div class="lr-grid">
|
||||
<a class="lr-card" href="05_column_mapper.html"><span class="lr-ico"><span class="dt-mi">view_column</span></span><span class="lr-body"><span class="lr-name">Map Columns</span><span class="lr-desc">Rename columns, reorder, and set each one as text, number, or date.</span></span></a>
|
||||
<a class="lr-card" href="07_multi_file_merger.html"><span class="lr-ico"><span class="dt-mi">account_tree</span></span><span class="lr-body"><span class="lr-name">Combine Files <span class="lr-soon">Soon</span></span><span class="lr-desc">Combine several CSV or Excel files into one — even if columns differ.</span></span></a>
|
||||
<a class="lr-card" href="10_pdf_extractor.html"><span class="lr-ico"><span class="dt-mi">picture_as_pdf</span></span><span class="lr-body"><span class="lr-name">PDF to CSV</span><span class="lr-desc">Pull transactions out of bank-statement PDFs into a clean CSV file.</span></span></a>
|
||||
</div>
|
||||
|
||||
<div class="lr-sec">Automations</div>
|
||||
<div class="lr-grid">
|
||||
<a class="lr-card" href="09_pipeline_runner.html"><span class="lr-ico"><span class="dt-mi">auto_awesome</span></span><span class="lr-body"><span class="lr-name">Automated Workflows</span><span class="lr-desc">Run several tools in a row — save the steps and reuse them anytime.</span></span></a>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
83
layout-review/shell.js
Normal file
83
layout-review/shell.js
Normal file
@@ -0,0 +1,83 @@
|
||||
/* Shared app chrome (sidebar nav + sticky footer) for the static layout
|
||||
review pages. Mirrors src/gui/app.py:_build_navigation() ordering and
|
||||
src/gui/components/_legacy.py:render_sticky_footer(). Each page sets
|
||||
<body data-page="<tool_id|home>"> to mark the active nav item. */
|
||||
(function () {
|
||||
// Front-door entry — rendered standalone above the section groups.
|
||||
var START = { id: "home", icon: "insert_chart_outlined", name: "Start here", href: "home.html" };
|
||||
|
||||
// Sections + entries in pipeline / job order.
|
||||
var NAV = [
|
||||
{ label: "Data Cleaners", items: [
|
||||
{ id: "02_text_cleaner", icon: "text_format", name: "Clean Text", href: "02_text_cleaner.html" },
|
||||
{ id: "03_format_standardizer", icon: "format_list_bulleted", name: "Standardize Formats", href: "03_format_standardizer.html" },
|
||||
{ id: "04_missing_handler", icon: "help_outline", name: "Fix Missing Values", href: "04_missing_handler.html" },
|
||||
{ id: "01_deduplicator", icon: "search", name: "Find Duplicates", href: "01_deduplicator.html" },
|
||||
]},
|
||||
{ label: "Transformations", items: [
|
||||
{ id: "05_column_mapper", icon: "view_column", name: "Map Columns", href: "05_column_mapper.html" },
|
||||
]},
|
||||
{ label: "Automations", items: [
|
||||
{ id: "09_pipeline_runner", icon: "auto_awesome", name: "Automated Workflows", href: "09_pipeline_runner.html" },
|
||||
]},
|
||||
{ label: "Finance", items: [
|
||||
{ id: "11_reconciler", icon: "compare_arrows", name: "Reconcile Two Files", href: "11_reconciler.html" },
|
||||
{ id: "10_pdf_extractor", icon: "picture_as_pdf", name: "PDF to CSV", href: "10_pdf_extractor.html" },
|
||||
]},
|
||||
{ label: "Coming soon", items: [
|
||||
{ id: "06_outlier_detector", icon: "insights", name: "Find Unusual Values", href: "06_outlier_detector.html", soon: true },
|
||||
{ id: "08_validator_reporter", icon: "check_circle", name: "Quality Check", href: "08_validator_reporter.html", soon: true },
|
||||
{ id: "07_multi_file_merger", icon: "account_tree", name: "Combine Files", href: "07_multi_file_merger.html", soon: true },
|
||||
]},
|
||||
];
|
||||
|
||||
var active = document.body.getAttribute("data-page") || "";
|
||||
|
||||
// ---- Sidebar -----------------------------------------------------------
|
||||
var sb = document.getElementById("dt-sidebar");
|
||||
if (sb) {
|
||||
var html = '' +
|
||||
'<a class="dt-brand" href="index.html" style="text-decoration:none">' +
|
||||
'<span class="dt-brand-mark">D</span>' +
|
||||
'<span class="dt-brand-name">' +
|
||||
'<span class="dt-brand-eyebrow">UNALOGIX</span>' +
|
||||
'<span class="dt-brand-word">DataTools</span>' +
|
||||
'</span>' +
|
||||
'</a>' +
|
||||
'<nav class="dt-nav">';
|
||||
var startCls = "dt-nav-link dt-nav-start" + (START.id === active ? " is-active" : "");
|
||||
html += '<a class="' + startCls + '" href="' + START.href + '">' +
|
||||
'<span class="dt-mi">' + START.icon + '</span>' +
|
||||
'<span>' + START.name + '</span>' +
|
||||
'</a>';
|
||||
NAV.forEach(function (sec) {
|
||||
var indicator = "−";
|
||||
html += '<div class="dt-nav-section">' + sec.label +
|
||||
'<span class="dt-nav-indicator">' + indicator + '</span></div>';
|
||||
sec.items.forEach(function (it) {
|
||||
var cls = "dt-nav-link" + (it.id === active ? " is-active" : "") + (it.soon ? " is-soon" : "");
|
||||
html += '<a class="' + cls + '" href="' + it.href + '">' +
|
||||
'<span class="dt-mi">' + it.icon + '</span>' +
|
||||
'<span>' + it.name + '</span>' +
|
||||
(it.soon ? '<span class="dt-nav-soon-tag">Soon</span>' : '') +
|
||||
'</a>';
|
||||
});
|
||||
});
|
||||
html += '</nav>' +
|
||||
'<div class="dt-sidebar-foot">' +
|
||||
'<div><div class="dt-sidebar-label">Language</div>' +
|
||||
'<div class="dt-select" style="pointer-events:none">English</div></div>' +
|
||||
'<div class="dt-license-badge">Core · 1,820 days left</div>' +
|
||||
'</div>';
|
||||
sb.innerHTML = html;
|
||||
}
|
||||
|
||||
// ---- Sticky footer -----------------------------------------------------
|
||||
var ft = document.getElementById("dt-footer");
|
||||
if (ft) {
|
||||
ft.innerHTML =
|
||||
'<a class="dt-footer-btn" href="index.html"><span class="dt-mi">close</span>Close</a>' +
|
||||
'<button class="dt-footer-btn" type="button"><span class="dt-mi">help_outline</span>Help</button>' +
|
||||
'<span style="margin-left:auto;font-size:11.5px;color:var(--ink-tertiary)">DataTools · local-first · static layout preview</span>';
|
||||
}
|
||||
})();
|
||||
@@ -1,2 +1,6 @@
|
||||
pytest>=8.0,<9
|
||||
pytest-cov>=5.0,<6
|
||||
# Test-only: generate small fixture PDFs in
|
||||
# tests/test_pdf_extract_smoke.py so we can exercise pdfplumber +
|
||||
# pypdfium2 end-to-end without committing binary fixtures.
|
||||
fpdf2==2.8.7
|
||||
|
||||
@@ -8,3 +8,16 @@ tqdm>=4.66,<5
|
||||
typer>=0.12,<1
|
||||
phonenumbers>=8.13,<9
|
||||
streamlit>=1.35,<2
|
||||
cryptography>=41,<49
|
||||
# PDF Extractor stack — pinned to exact tested versions so a future
|
||||
# upstream release can't quietly change pdfplumber's word-position
|
||||
# behavior or pypdfium2's OCR rendering mid-build. Bump these
|
||||
# explicitly when re-testing against a new release.
|
||||
#
|
||||
# ``pypdfium2`` is here for the OCR fallback path only (rasterizing
|
||||
# pages to images for Tesseract). The drawable-canvas dep was
|
||||
# removed when the visual picker was ripped out — the scanner is
|
||||
# pure heuristic now, no coordinate UI.
|
||||
pdfplumber==0.11.9
|
||||
pypdfium2==5.8.0
|
||||
pytesseract==0.3.13
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
Lead ID,First Name,Last Name,Company,Title,Email,Phone,Country,Source,Score,Last Activity,Tags
|
||||
HUB-001,Alice,Johnson,Acme Corp,VP Marketing,alice@acme.com,(415) 555-1234,USA,HubSpot,87,2025-12-04,Enterprise
|
||||
HUB-002,bob,smith,Beta LLC,Director Growth,bob@beta.com,N/A,United States,HubSpot,N/A,2025-11-22,SMB
|
||||
HUB-003,Carlos,Garcia,Gamma Inc,CEO,carlos@gamma.io,+34 91 411 1111,Spain,HubSpot,82,2025-10-30,Enterprise
|
||||
HUB-004,DIANA,LEE,Delta Co,Marketing Manager,diana@delta.com,020 7946 0958,United Kingdom,HubSpot,74,2025-12-15,Mid-Market
|
||||
HUB-005,Eve,Martinez,Epsilon Group,VP Ops,eve@epsilon.com,(none),Mexico,HubSpot,(blank),2025-09-15,SMB
|
||||
LIN-006,Alice,Johnson,Acme Corporation,VP of Marketing,Alice.Johnson@acme.com,4155551234,US,LinkedIn,—,2025-12-04,Enterprise
|
||||
LIN-007,Frank,Brown,Foxtrot Ltd,Head Sales,frank@foxtrot.de,+49 30 12345678,Germany,LinkedIn,68,2025-12-01,Mid-Market
|
||||
LIN-008,Grace,Davis,Golf Industries,Marketing Lead,grace@golfind.com,+44 20 7946 0958,UK,LinkedIn,79,2025-11-08,Mid-Market
|
||||
LIN-009,henry,wilson,Hotel Logistics,COO,henry@hotellog.com,+86 10 1234 5678,China,LinkedIn,91,2025-12-12,Enterprise
|
||||
LIN-010,IVY CHEN,,India Tech,CTO,ivy@indiatech.in,+91 11 2345 6789,IN,LinkedIn,88,2025-11-30,Enterprise
|
||||
LIN-011,Jack,Taylor,Juliet & Co,Founder,jack@juliet.co,unknown,United States,LinkedIn,?,(unknown),SMB
|
||||
SCR-012,Diana,Lee,Delta Company,Marketing Manager,diana@delta.com,020-7946-0958,UK,Manual Scrape,74,12/15/2025,Mid-Market
|
||||
SCR-013,kate,o'neil,Kilo Ventures,Partner,kate@kilo.vc,+1 415 555 2222,USA,Manual Scrape,N/A,?,Investor
|
||||
SCR-014,Carlos,García,Gamma Incorporated,CEO,Carlos@gamma.io,+34-91-411-1111,Spain,Manual Scrape,82,Oct 30 2025,Enterprise
|
||||
SCR-015,Liam,Park,Lima Solutions,Director Marketing,liam@limasol.kr,+82 2 2287 0114,South Korea,Manual Scrape,77,2025-11-20,Enterprise
|
||||
SCR-016,Mia,nguyen,Mike Corp,VP Marketing,mia@mikecorp.com.au,02 9374 4000,Australia,Manual Scrape,72,2025-10-05,Mid-Market
|
||||
SCR-017,Noah,Brown,November Inc,Head of Growth,noah@november.com,(555) 444-5555,US,Manual Scrape,—,#N/A,SMB
|
||||
HUB-018,Frank,Brown,Foxtrot,Head of Sales,Frank@Foxtrot.de,+49-30-12345678,Germany,HubSpot,68,2025-12-01,Mid-Market
|
||||
HUB-019,Olivia,Rossi,Oscar Italia,CMO,olivia@oscar.it,+39 06 6982,Italy,HubSpot,85,2025-12-08,Enterprise
|
||||
HUB-020,papa,wong,Papa Trading,Founder,papa@papatrading.hk,+852 2123 4567,Hong Kong,HubSpot,69,2025-11-15,SMB
|
||||
LIN-021,Quinn,Reyes,Quebec Group,VP Sales,quinn@quebec.mx,+52 55 5555 0000,Mexico,LinkedIn,80,2025-12-05,Mid-Market
|
||||
LIN-022,Robert,Tan,Romeo Logistics,Director,r.tan@romeo.sg,+65 6123 4567,Singapore,LinkedIn,76,2025-11-28,Mid-Market
|
||||
SCR-023,Sara,Khan,Sierra Foods,Head Marketing,sara@sierra.in,+91-22-1234-5678,India,Manual Scrape,73,2025-12-02,SMB
|
||||
SCR-024,bob,Smith,Beta,Director Growth,Bob@Beta.com,(none),United States,Manual Scrape,(unknown),(unknown),SMB
|
||||
HUB-025,Tara,Levi,Tango Tech,VP Product,tara@tango.il,+972 3 6957 0000,Israel,HubSpot,82,2025-12-10,Enterprise
|
||||
HUB-026,Uma,Patel,Uniform Health,CMO,uma at uniform dot com,+44 20 7946 8888,United Kingdom,HubSpot,71,2025-12-12,Enterprise
|
||||
LIN-027,Victor,Lee,Victor Co,Director,victor@@victorco.com,+1 415 555 8888,USA,LinkedIn,69,2025-11-30,SMB
|
||||
SCR-028,Wendy,Akin,Whiskey Inc,CMO,wendy@whiskey.tr,+90 212 252 1111,Turkey,Manual Scrape,77,2025-12-04,Mid-Market
|
||||
SCR-029,Xander,Ng,Xray Group,Founder,xander@xray.sg,+65 6234 5678,Singapore,Manual Scrape,65,2025-11-15,Suppressed
|
||||
HUB-030,Yara,Costa,Yankee Foods,Marketing Lead,yara@yankee.br,+55 11 3071 2222,Brazil,HubSpot,—,2025-12-15,Opted Out
|
||||
|
@@ -1,74 +0,0 @@
|
||||
{
|
||||
"steps": [
|
||||
{
|
||||
"tool": "text_clean",
|
||||
"options": {},
|
||||
"enabled": true,
|
||||
"name": "1. Clean text (whitespace + smart quotes from copy-paste)"
|
||||
},
|
||||
{
|
||||
"tool": "format_standardize",
|
||||
"options": {
|
||||
"column_types": {
|
||||
"First Name": "name",
|
||||
"Last Name": "name",
|
||||
"Company": "name",
|
||||
"Email": "email",
|
||||
"Phone": "phone"
|
||||
},
|
||||
"phone_country_column": "Country",
|
||||
"phone_format": "E164",
|
||||
"email_gmail_canonical": true
|
||||
},
|
||||
"enabled": true,
|
||||
"name": "2. E.164 phones (per-row country) · canonical emails · name casing"
|
||||
},
|
||||
{
|
||||
"tool": "missing",
|
||||
"options": {
|
||||
"strategy": "none",
|
||||
"standardize_sentinels": true,
|
||||
"sentinels": ["N/A", "n/a", "—", "?", "(unknown)", "unknown", "(blank)", "(none)", "TBD", "#N/A"]
|
||||
},
|
||||
"enabled": true,
|
||||
"name": "3. Standardize sentinels across vendor exports"
|
||||
},
|
||||
{
|
||||
"tool": "column_map",
|
||||
"options": {
|
||||
"schema": {
|
||||
"fields": [
|
||||
{"name": "Lead ID", "dtype": "string", "required": true},
|
||||
{"name": "First Name", "dtype": "string"},
|
||||
{"name": "Last Name", "dtype": "string"},
|
||||
{"name": "Company", "dtype": "string"},
|
||||
{"name": "Title", "dtype": "string"},
|
||||
{"name": "Email", "dtype": "string"},
|
||||
{"name": "Phone", "dtype": "string"},
|
||||
{"name": "Country", "dtype": "string"},
|
||||
{"name": "Source", "dtype": "string"},
|
||||
{"name": "Score", "dtype": "integer"},
|
||||
{"name": "Last Activity", "dtype": "date"},
|
||||
{"name": "Tags", "dtype": "string"}
|
||||
]
|
||||
},
|
||||
"auto_infer": true,
|
||||
"unmapped": "keep",
|
||||
"coerce_types": true,
|
||||
"reorder_to_schema": true,
|
||||
"enforce_required": false
|
||||
},
|
||||
"enabled": true,
|
||||
"name": "4. Coerce types · reorder to canonical schema"
|
||||
},
|
||||
{
|
||||
"tool": "dedup",
|
||||
"options": {
|
||||
"survivor_rule": "most_complete",
|
||||
"merge": true
|
||||
},
|
||||
"enabled": true,
|
||||
"name": "5. Dedup leads across HubSpot / LinkedIn / Manual Scrape (fuzzy + merge)"
|
||||
}
|
||||
]
|
||||
}
|
||||
27
samples/demo/ar_open_invoices.csv
Normal file
27
samples/demo/ar_open_invoices.csv
Normal file
@@ -0,0 +1,27 @@
|
||||
Invoice,Client,Email,Invoice_Date,Due_Date,Amount,Status
|
||||
INV-1007,ACME LLC,AP@Acme.com,03/04/2025,04/03/2025,"$1,250.00",Open
|
||||
INV-1007, Acme LLC ,ap@acme.com,2025-03-04,2025-04-03,"1,250.00",(blank)
|
||||
INV-1001,northwind traders,billing@northwind.com,Mar 6 2025,04/05/2025,$980,Overdue
|
||||
INV-1002,Globex Corp,AR@Globex.com,3/11/25,4/10/25,"2,400.50",Sent
|
||||
INV-1011,initech,accounts@initech.com,04/01/2025,05/01/2025,"$ 1,100.00",?
|
||||
INV-1011,Initech,Accounts@Initech.com,2025-04-01,2025-05-01,1100,Open
|
||||
INV-1003,Stark Industries,ap@stark.com,Mar 6 2025,Apr 6 2025,$75.00,Open
|
||||
INV-1004,Wayne Enterprises,ar@wayne.com,03/15/2025,04/14/2025,($300.00),—
|
||||
INV-1015,Hooli,billing@hooli.com,3/11/25,4/10/25,"$4,300.00",Overdue
|
||||
INV-1015,hooli,Billing@Hooli.com,2025-03-11,2025-04-10,4300,(none)
|
||||
INV-1005,Soylent Corp,ap@soylent.com,2025-03-20,2025-04-19,"$1,875.25",Sent
|
||||
INV-1006,Umbrella Co,ar@umbrella.com,03/22/2025,04/21/2025,$640.00,TBD
|
||||
INV-1019,Cyberdyne Systems,ap@cyberdyne.com,Mar 25 2025,04/24/2025,"$2,050.00",unknown
|
||||
INV-1019,cyberdyne systems,AP@Cyberdyne.com,2025-03-25,2025-04-24,"2,050.00",Open
|
||||
INV-1008,Vandelay Industries,ar@vandelay.com,3/28/25,4/27/25,$915.00,Overdue
|
||||
INV-1009,Gekko & Co,billing@gekko.com,2025-03-30,2025-04-29,"$3,120.75",Open
|
||||
INV-1010,Pied Piper,ap@piedpiper.com,04/02/2025,05/02/2025,$180,Sent
|
||||
INV-1023,Tyrell Corp,ar@tyrell.com,04/05/2025,05/05/2025,($300.00),(blank)
|
||||
INV-1023,Tyrell Corp,AR@Tyrell.com,2025-04-05,2025-05-05,-300.00,Open
|
||||
INV-1012,Oscorp,ap@oscorp.com,Apr 8 2025,05/08/2025,"$5,000.00",Overdue
|
||||
INV-1013,Nakatomi Trading,ar@nakatomi.com,4/9/25,5/9/25,$725.50,Sent
|
||||
INV-1014,Bluth Company,billing@bluth.com,2025-04-10,2025-05-10,"$1,420.00",Open
|
||||
INV-1016,Dunder Mifflin,ap@dundermifflin.com,04/12/2025,05/12/2025,$960.00,Overdue
|
||||
INV-1017,Prestige Worldwide,ar@prestige.com,Apr 14 2025,05/14/2025,"$2,680.00",Sent
|
||||
INV-1018,Sterling Cooper,billing@sterlingcooper.com,4/15/25,5/15/25,"$3,950.00",Open
|
||||
INV-1020,Wonka Industries,ap@wonka.com,2025-04-18,2025-05-18,"$1,050.00",Overdue
|
||||
|
50
samples/demo/ar_open_invoices_pipeline.json
Normal file
50
samples/demo/ar_open_invoices_pipeline.json
Normal file
@@ -0,0 +1,50 @@
|
||||
{
|
||||
"steps": [
|
||||
{
|
||||
"tool": "text_clean",
|
||||
"enabled": true,
|
||||
"options": {
|
||||
"trim": true,
|
||||
"collapse_whitespace": true,
|
||||
"fold_smart_chars": true,
|
||||
"strip_zero_width": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"tool": "format_standardize",
|
||||
"enabled": true,
|
||||
"options": {
|
||||
"column_types": {
|
||||
"Invoice_Date": "date",
|
||||
"Due_Date": "date",
|
||||
"Amount": "currency",
|
||||
"Email": "email"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"tool": "missing",
|
||||
"enabled": true,
|
||||
"options": {
|
||||
"strategy": "none",
|
||||
"standardize_sentinels": true,
|
||||
"sentinels": ["—", "-", "?", "(blank)", "TBD", "unknown", "(none)", "N/A", "#N/A"]
|
||||
}
|
||||
},
|
||||
{
|
||||
"tool": "dedup",
|
||||
"enabled": true,
|
||||
"options": {
|
||||
"survivor_rule": "most_complete",
|
||||
"merge": true,
|
||||
"strategies": [
|
||||
{
|
||||
"columns": [
|
||||
{"column": "Invoice", "algorithm": "exact", "threshold": 100}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
27
samples/demo/bank_reconciliation.csv
Normal file
27
samples/demo/bank_reconciliation.csv
Normal file
@@ -0,0 +1,27 @@
|
||||
Date,Description,Vendor,Category,Amount,Account
|
||||
01/15/2025,“Stripe payout — weekly”,Stripe,Income,"+$3,450.00",Business Checking
|
||||
2025-01-15,Verizon business line,Verizon,—,($89.50),Business Checking
|
||||
Jan 18 2025,Adobe Creative Cloud ,Adobe,(blank),-$129.99,Business Checking
|
||||
1/27/25,Office supplies,Amazon,Supplies,-$74.20,Business Checking
|
||||
02/03/2025, Monthly office rent,Highland Properties,Rent,"$1,200.00",Business Checking
|
||||
Feb 5 2025,Account service fee,First National Bank,?,(50.00),Business Checking
|
||||
2025-01-09,Shipping labels,amazon.com,unknown,-$18.40,Business Checking
|
||||
1/22/25,Contractor — landing page,Bright Lane Design,TBD,- $599.88,Business Checking
|
||||
Jan 30 2025,Late fee adjustment,verizon,Utilities,-$12.00,Business Checking
|
||||
2025-01-11,Packaging tape,AMAZON.COM,Supplies,-$31.75,Business Checking
|
||||
01/06/2025,Client deposit — ACME Co,ACME Co,Income,"$2,500.00",Business Checking
|
||||
2025-01-20,Google Workspace,Google,Software,-$36.00,Business Checking
|
||||
Jan 24 2025,Fuel — delivery van,Shell,Vehicle,-$58.63,Business Checking
|
||||
1/28/25,QuickBooks subscription,Intuit,Software,-$80.00,Business Checking
|
||||
2025-01-15,Stripe payout weekly,Stripe,Income,3450.00,Business Checking
|
||||
01/15/2025,Verizon business line,Verizon,Utilities,-89.50,Business Checking
|
||||
2025-01-18,Adobe Creative Cloud,Adobe,Software,-129.99,Business Checking
|
||||
2025-02-03,Monthly office rent,Highland Properties,Rent,1200.00,Business Checking
|
||||
2025-02-05,Account service fee,First National Bank,Bank Fees,-50.00,Business Checking
|
||||
2025-01-22,Contractor landing page,Bright Lane Design,Contractors,-599.88,Business Checking
|
||||
02/10/2025,Client deposit — Globex,Globex,Income,"$1,800.00",Business Checking
|
||||
2025-02-12,Slack subscription,Slack,Software,-$96.00,Business Checking
|
||||
Feb 14 2025,Coffee — client meeting,Blue Bottle,Meals,-$23.10,Business Checking
|
||||
2/18/25,Insurance premium,Hartford,Insurance,-$240.50,Business Checking
|
||||
02/21/2025,Refund — returned printer,Staples,Supplies,$210.99,Business Checking
|
||||
Feb 25 2025,Domain renewal,Namecheap,Software,-$13.98,Business Checking
|
||||
|
6
samples/demo/bank_reconciliation_pipeline.json
Normal file
6
samples/demo/bank_reconciliation_pipeline.json
Normal file
@@ -0,0 +1,6 @@
|
||||
{"steps":[
|
||||
{"tool":"text_clean","enabled":true,"options":{"trim":true,"collapse_whitespace":true,"fold_smart_chars":true,"strip_zero_width":true}},
|
||||
{"tool":"format_standardize","enabled":true,"options":{"column_types":{"Date":"date","Amount":"currency"}}},
|
||||
{"tool":"missing","enabled":true,"options":{"strategy":"none","standardize_sentinels":true,"sentinels":["—","(blank)","?","unknown","TBD","N/A","#N/A","(none)"]}},
|
||||
{"tool":"dedup","enabled":true,"options":{"survivor_rule":"most_complete","merge":true,"strategies":[{"columns":[{"column":"Date","algorithm":"exact","threshold":100},{"column":"Amount","algorithm":"exact","threshold":100}]}]}}
|
||||
]}
|
||||
@@ -1,56 +0,0 @@
|
||||
{
|
||||
"steps": [
|
||||
{
|
||||
"tool": "text_clean",
|
||||
"options": {},
|
||||
"enabled": true,
|
||||
"name": "1. Clean text (header whitespace, smart quotes, em-dash)"
|
||||
},
|
||||
{
|
||||
"tool": "format_standardize",
|
||||
"options": {
|
||||
"column_types": {
|
||||
"Date": "date",
|
||||
"Amount": "currency",
|
||||
"Balance": "currency",
|
||||
"Vendor": "name"
|
||||
},
|
||||
"currency_decimal": "auto",
|
||||
"currency_preserve_code": false,
|
||||
"currency_decimals": 2,
|
||||
"date_output_format": "%Y-%m-%d"
|
||||
},
|
||||
"enabled": true,
|
||||
"name": "2. ISO dates · numeric amounts (parens-negative) · vendor casing"
|
||||
},
|
||||
{
|
||||
"tool": "missing",
|
||||
"options": {
|
||||
"strategy": "none",
|
||||
"standardize_sentinels": true,
|
||||
"sentinels": ["N/A", "n/a", "—", "-", "?", "(blank)", "(none)", "unknown", "#N/A"]
|
||||
},
|
||||
"enabled": true,
|
||||
"name": "3. Standardize disguised nulls (— / N/A / (blank))"
|
||||
},
|
||||
{
|
||||
"tool": "dedup",
|
||||
"options": {
|
||||
"survivor_rule": "most_complete",
|
||||
"merge": false,
|
||||
"date_column": "Date",
|
||||
"strategies": [
|
||||
{
|
||||
"columns": [
|
||||
{"column": "Date", "algorithm": "exact", "threshold": 100},
|
||||
{"column": "Amount", "algorithm": "exact", "threshold": 100},
|
||||
{"column": "Vendor", "algorithm": "jaro_winkler", "threshold": 80}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"enabled": true,
|
||||
"name": "4. Dedup transactions on Date+Amount+fuzzy Vendor"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
Txn ID,Date ,Description,Amount,Balance,Account,Vendor,Category
|
||||
TXN-2401,01/15/2025," AMAZON.COM*4F2X9 PURCHASE",-$129.99,"$2,450.01",Checking,Amazon,Office Supplies
|
||||
TXN-2402,2025-01-15,"AMAZON.COM*4F2X9 PURCHASE",-$129.99,"2450.01",Checking,amazon.com,Office Supplies
|
||||
TXN-2403,Jan 18 2025,"STAPLES #4422 — paper, toner",($89.50),$2360.51,Checking,STAPLES,Office Supplies
|
||||
TXN-2404,01/22/2025,"Verizon Wireless ""autopay""",-$120.00,"$2,240.51",Checking,Verizon,Utilities
|
||||
TXN-2405,2025-01-22,Verizon Wireless autopay,-120.00,"2,240.51",Checking,verizon,Utilities
|
||||
TXN-2406,01-25-2025,"Stripe Payout — invoice #1077","+$3,450.00","$5,690.51",Checking,Stripe,Income
|
||||
TXN-2407,1/27/25,"Office Lease - Suite 204",-1500.00,"$4,190.51",Checking,Acme Realty,Rent
|
||||
TXN-2408,02/01/2025,"Wire — Acme Realty Mgmt","-$1,500.00","$2,690.51",Checking,acme realty,Rent
|
||||
TXN-2409,2025-02-03,"Adobe Creative Cloud annual","- $599.88","$2,090.63",Credit Card,Adobe Inc.,Software
|
||||
TXN-2410,02/03/2025,"ADOBE CREATIVE CLOUD ANN",-599.88,2090.63,Credit Card,adobe,Software
|
||||
TXN-2411,Feb 5 2025,"FedEx — overnight to client A",-$32.50,"$2,058.13",Checking,FedEx,Shipping
|
||||
TXN-2412,02/07/2025,"Square fee — invoice #1078","-$3.20","$2,054.93",Checking,Square,Fees
|
||||
TXN-2413,02/10/2025,"Stripe Payout invoice #1079","+ $1,200.00","$3,254.93",Checking,Stripe,Income
|
||||
TXN-2414,2025-02-12,"USPS PRIORITY — to vendor B","-12.40","$3,242.53",Checking,USPS,Shipping
|
||||
TXN-2415,02/14/2025,"Zoom Video Comms — annual","-$149.90","$3,092.63",Credit Card,Zoom,Software
|
||||
TXN-2416,2/14/25,"Zoom Video Communications","-149.90","3092.63",Credit Card,zoom,Software
|
||||
TXN-2417,02/18/2025,"Costco Whse #421 — supplies","-$237.84","$2,854.79",Checking,Costco,Office Supplies
|
||||
TXN-2418,2025-02-18,COSTCO WHSE #421,-237.84,"2,854.79",Checking,costco,Office Supplies
|
||||
TXN-2419,02/22/2025,"Bank fee — int'l wire","-$45.00","$2,809.79",Checking,Bank Fee,Fees
|
||||
TXN-2420,02/24/2025,"Stripe Payout — invoice #1080","+$2,100.00","$4,909.79",Checking,Stripe,Income
|
||||
TXN-2421,02/28/2025," Refund — overcharge ","+$45.00","$4,954.79",Checking,—,Refunds
|
||||
TXN-2422,Feb 28 2025,REFUND OVERCHARGE,45.00,4954.79,Checking,N/A,Refunds
|
||||
TXN-2423,03/01/2025,"Office Lease — Suite 204","-$1,500.00","$3,454.79",Checking,Acme Realty,Rent
|
||||
TXN-2424,2025-03-03,"Slack Technologies — annual","-$840.00","$2,614.79",Credit Card,Slack,Software
|
||||
TXN-2425,03/05/2025,"Stripe Payout — invoice #1081","+$1,875.00","$4,489.79",Checking,Stripe,Income
|
||||
TXN-2426,03/08/2025,"Wire — Berlin office rent (EUR vendor)","-€1.450,00","$2,989.79",Checking,Mietverwaltung GmbH,Rent
|
||||
TXN-2427,03/10/2025,"London supplier invoice (GBP)","-£950.00","$1,939.79",Checking,Stationery Co Ltd,Office Supplies
|
||||
TXN-2428,03/12/2025,"São Paulo agency retainer","-R$ 1.299,90","$1,679.79",Credit Card,Estúdio Ágil,Software
|
||||
TXN-2429,03/14/2025,"VAT MOSS prep — multi-EU sales","($89.00)","$1,768.79",Checking,EU VAT Service,Fees
|
||||
TXN-2430,03/14/2025,"VAT MOSS prep multi EU sales",-89.00,"1,768.79",Checking,eu vat service,Fees
|
||||
|
@@ -1,21 +0,0 @@
|
||||
Customer ID,First Name,Last Name,Email,Phone,Address,City,State,ZIP,Country,Total Orders,Lifetime Value,Last Order Date,Tags
|
||||
SHOP-1001, Alice ,Johnson,alice@petshop.com,(415) 555-1234,"123 Main St., Apt 4B",San Francisco,CA,94102,US,12,$1,240.50,2025-12-04,VIP
|
||||
SHOP-1002,Bob,SMITH,Bob@PetShop.com,415.555.1234,"123 Main St, Apt 4B",San Francisco,CA,94102,US,12,"$1,240.50",N/A,VIP
|
||||
SHOP-1003,carlos,garcia,carlos@petshop.com,5559876543,"742 Evergreen Terrace",Springfield,IL,62704,US,5,420.00,12/15/2025,Wholesale
|
||||
SHOP-1004,Diana,Lee,diana@petshop.com,(555) 222-3344,"PO Box 12, Sherwood Forest",Nottingham,,NG1 5BA,GB,8,£890.25,2025-10-30,VIP|Wholesale
|
||||
SHOP-1005,EVE MARTINEZ,,eve.martinez@petshop.com,555-9988,"Calle Mayor 45","Madrid",,"28013",ES,3,€180,2025-09-15,
|
||||
SHOP-1006,Frank,Brown,frank@petshop.com,, ,"Berlin",BE,10115,DE,15,€2.410,75,(blank),Wholesale
|
||||
SHOP-1007,Grace,Davis,grace@petshop.com,+1 555-111-1111,"888 Maple Ave",Toronto,ON,M5V 3A8,CA,1,$49.99,#N/A,New
|
||||
SHOP-1008,henry,wilson,Henry@PetShop.com,5551111111,"888 Maple Avenue","Toronto",ON,M5V 3A8,CA,1,$49.99,2025-12-01,New
|
||||
SHOP-1009,Ivy,Chen,IVY@petshop.com,+1 (555) 777-7777,"550 Elm Street, Suite 200",Brooklyn,NY,11201,US,4,"$320.50 ",10/12/2025,
|
||||
SHOP-1010,Jack,Taylor,jack@petshop.com,(none),"550 elm street, suite 200",brooklyn,NY,11201,US,4,$320.50,2025-10-12,
|
||||
SHOP-1011,kate,o'neil,kate.oneil@petshop.com,415-555-2222,"99 King's Rd","London",,SW3 4LX,GB,7,£675.00,?,VIP
|
||||
SHOP-1012,luis,rodriguez,LUIS@petshop.com,+34 91 411 1111,"Avenida de la Paz 12, 3°D",Madrid,,28013,ES,2,"€89,99",unknown,
|
||||
SHOP-1013,Mia,Park,mia@petshop.com,02-9374-4000,"Sydney Opera House Drive","Sydney",NSW,2000,AU,9,"A$ 1,299.00",2025-11-20,Wholesale
|
||||
SHOP-1014,Noah,nguyen,noah@petshop.com,+81 3 3210 7000,"丸の内 2-7-3","Tokyo",,100-0005,JP,6,"¥75000",2025-12-10,VIP
|
||||
SHOP-1015,Olivia,Brown,OLIVIA@PETSHOP.COM,(555) 333-4444,"742 evergreen terrace",springfield,IL,62704,US,3,$180.00,(none),
|
||||
SHOP-1016,Pavel,Novak,pavel@petshop.com,+44 20 7946 1234,"22 Baker Street",London,,W1U 6AB,United Kingdom,4,£412.00,2025-11-18,VIP
|
||||
SHOP-1017,Quinn,Murphy,quinn@petshop.com,+44 20 7946 5678,"5 Princes Street",Edinburgh,,EH2 2DA,U.K.,2,£189.50,2025-12-09,
|
||||
SHOP-1018,Rachel,O'Brien,rachel@petshop.com,02-9374-9999,"100 George Street","Sydney",NSW,2000,UK,1,£75.00,?,New
|
||||
SHOP-1019,Sam,Klein,sam@petshop.com,+49 30 99887766,"Friedrichstraße 100","Berlin",,10117,Germany,11,"€1.890,40",2025-12-11,VIP|Wholesale
|
||||
SHOP-1020,Tara,Gianni,tara@petshop.com,+39 06 6982 4567,"Via del Corso 250",Roma,,00186,Italia,5,"€649,99",2025-12-03,
|
||||
|
@@ -1,49 +0,0 @@
|
||||
{
|
||||
"steps": [
|
||||
{
|
||||
"tool": "text_clean",
|
||||
"options": {},
|
||||
"enabled": true,
|
||||
"name": "1. Clean text (whitespace, smart quotes, NBSP, BOM)"
|
||||
},
|
||||
{
|
||||
"tool": "format_standardize",
|
||||
"options": {
|
||||
"column_types": {
|
||||
"First Name": "name",
|
||||
"Last Name": "name",
|
||||
"Email": "email",
|
||||
"Phone": "phone",
|
||||
"Address": "address",
|
||||
"Lifetime Value": "currency",
|
||||
"Last Order Date": "date"
|
||||
},
|
||||
"phone_country_column": "Country",
|
||||
"address_country_column": "Country",
|
||||
"currency_preserve_code": true,
|
||||
"currency_decimal": "auto",
|
||||
"email_gmail_canonical": false
|
||||
},
|
||||
"enabled": true,
|
||||
"name": "2. Standardize phones, addresses, dates, currencies, names"
|
||||
},
|
||||
{
|
||||
"tool": "missing",
|
||||
"options": {
|
||||
"strategy": "none",
|
||||
"standardize_sentinels": true
|
||||
},
|
||||
"enabled": true,
|
||||
"name": "3. Standardize disguised nulls (N/A, -, (blank), ?, #N/A)"
|
||||
},
|
||||
{
|
||||
"tool": "dedup",
|
||||
"options": {
|
||||
"survivor_rule": "most_complete",
|
||||
"merge": true
|
||||
},
|
||||
"enabled": true,
|
||||
"name": "4. Dedup customers (fuzzy match, merge missing fields)"
|
||||
}
|
||||
]
|
||||
}
|
||||
25
samples/demo/vendor_1099.csv
Normal file
25
samples/demo/vendor_1099.csv
Normal file
@@ -0,0 +1,25 @@
|
||||
Vendor,Contact,Email,Phone,EIN,Address,Total_Paid
|
||||
Acme Realty,Bob Stein,acme.ap@acmerealty.com,(212) 555-0100,12-3456789,(blank),"$12,400.00"
|
||||
acme realty llc,Bob Stein, ACME.AP@AcmeRealty.com ,,—,"118 Canal St, New York, NY 10013","$8,250"
|
||||
ACME REALTY,R. Stein,Acme.AP@acmerealty.com,212.555.0100,N/A,TBD,"1,999.99"
|
||||
Bright Books Bookkeeping,Dana Cole,hello@brightbooks.com,,98-7654321,(blank),"$6,000.00"
|
||||
bright books,Dana Cole,HELLO@brightbooks.com,(415) 555-0142,unknown,"50 Market St, San Francisco, CA 94105","$6,000"
|
||||
"Bright Books, LLC",D. Cole, hello@BrightBooks.com,4155550142,98-7654321,unknown,"5,500.00"
|
||||
Northwind Logistics,Sam Reyes,ap@northwindlog.com,(312) 555-0198,—,(blank),"$22,750.00"
|
||||
northwind logistics inc,Sam Reyes,AP@NorthwindLog.com,,45-6789012,"900 W Loop, Chicago, IL 60607","$22,750"
|
||||
Pearl Design Studio,“Jo” Marsh,billing@pearldesign.co,,33-2211000,(blank),"$3,200.00"
|
||||
pearl design,Jo Marsh,Billing@PearlDesign.co,(206) 555-0167,TBD,"77 Pike St, Seattle, WA 98101","$3,200"
|
||||
PEARL DESIGN STUDIO,J. Marsh, billing@pearldesign.co ,206.555.0167,33-2211000,unknown,"2,800.00"
|
||||
Cooper Plumbing,Lee Cooper,office@cooperplumb.com,(617) 555-0133,—,(blank),"$1,450.00"
|
||||
cooper plumbing co,Lee Cooper,OFFICE@cooperplumb.com,,TBD,"12 Beacon St, Boston, MA 02108","$1,450"
|
||||
COOPER PLUMBING,L. Cooper, office@CooperPlumb.com,6175550133,N/A,unknown,900.00
|
||||
Vertex Marketing,Pat Nguyen,accounts@vertexmktg.com,(404) 555-0119,77-8899001,(blank),"$15,000.00"
|
||||
vertex marketing group,Pat Nguyen,ACCOUNTS@VertexMktg.com,,unknown,"300 Peachtree St, Atlanta, GA 30308","$15,000"
|
||||
Summit Consulting,Ray Brooks,invoices@summitconsult.net,,21-0099887,(blank),"$9,800.00"
|
||||
summit consulting llc,Ray Brooks,INVOICES@summitconsult.net,(303) 555-0175,—,"1100 17th St, Denver, CO 80202","$9,800"
|
||||
SUMMIT CONSULTING,R. Brooks, invoices@SummitConsult.net ,303.555.0175,21-0099887,TBD,"7,250.00"
|
||||
Garcia Catering,Mia Garcia,ap@garciacatering.com,(305) 555-0188,—,(blank),"$4,600.00"
|
||||
garcia catering services,Mia Garcia,AP@GarciaCatering.com,,66-1234509,"450 Ocean Dr, Miami, FL 33139",$600.00
|
||||
Northwind Logistics,S. Reyes, ap@northwindlog.com ,312.555.0198,45-6789012,TBD,"21,000.00"
|
||||
VERTEX MARKETING,P. Nguyen, accounts@vertexmktg.com ,404.555.0119,77-8899001,TBD,"14,500.00"
|
||||
GARCIA CATERING,M. Garcia,ap@GARCIACATERING.com,305.555.0188,66-1234509,unknown,"4,200.00"
|
||||
|
49
samples/demo/vendor_1099_pipeline.json
Normal file
49
samples/demo/vendor_1099_pipeline.json
Normal file
@@ -0,0 +1,49 @@
|
||||
{
|
||||
"steps": [
|
||||
{
|
||||
"tool": "text_clean",
|
||||
"enabled": true,
|
||||
"options": {
|
||||
"trim": true,
|
||||
"collapse_whitespace": true,
|
||||
"fold_smart_chars": true,
|
||||
"strip_zero_width": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"tool": "format_standardize",
|
||||
"enabled": true,
|
||||
"options": {
|
||||
"column_types": {
|
||||
"Phone": "phone",
|
||||
"Email": "email",
|
||||
"Total_Paid": "currency"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"tool": "missing",
|
||||
"enabled": true,
|
||||
"options": {
|
||||
"strategy": "none",
|
||||
"standardize_sentinels": true,
|
||||
"sentinels": ["—", "-", "--", "(blank)", "TBD", "unknown", "N/A", "#N/A", "(none)"]
|
||||
}
|
||||
},
|
||||
{
|
||||
"tool": "dedup",
|
||||
"enabled": true,
|
||||
"options": {
|
||||
"survivor_rule": "most_complete",
|
||||
"merge": true,
|
||||
"strategies": [
|
||||
{
|
||||
"columns": [
|
||||
{"column": "Email", "algorithm": "exact", "threshold": 100, "normalizer": "email"}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
customer_name,email,vendor,memo
|
||||
Alice Johnson,alice@example.com,ACME Corp ,Welcome aboard
|
||||
Bob Smith,bob@example.com,ACME Corp,Returning customer
|
||||
Charlie Brown,charlie@example.com,Globex,Net 30
|
||||
Diana Prince,diana@example.com,Globex,VIP
|
||||
Edward Norton,ed@example.com,“Best Pet Supplies”,Order#42 - rush
|
||||
Frank Castle,frank@example.com,Stark—Industries,"Line 1
|
||||
Line 2
|
||||
Line 3"
|
||||
grace HOPPER ,grace@example.com,Globex,Loves long memos…
|
||||
Henry Ford,henry@example.com,Ford Motor,Industrial
|
||||
Iris West,iris@example.com,S.T.A.R. Labs,Notewith-bell
|
||||
Jane Doe,jane@example.com,Acme,Standard
|
||||
|
106
scripts/generate_keypair.py
Normal file
106
scripts/generate_keypair.py
Normal file
@@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate a fresh Ed25519 keypair for production license signing.
|
||||
|
||||
**Creator-only.** Run once, write the private key somewhere safe,
|
||||
configure the build pipeline with the public key.
|
||||
|
||||
Usage::
|
||||
|
||||
python scripts/generate_keypair.py
|
||||
python scripts/generate_keypair.py --json
|
||||
python scripts/generate_keypair.py --output keys.txt
|
||||
|
||||
The output looks like::
|
||||
|
||||
DATATOOLS_LICENSE_PRIVKEY=<64 hex chars> # KEEP SECRET
|
||||
DATATOOLS_LICENSE_PUBKEY=<64 hex chars> # BAKE INTO BUILD
|
||||
|
||||
The private key never goes near the buyer-facing binary. Stash it in
|
||||
a password manager / KMS / hardware token; the only places it gets
|
||||
loaded are:
|
||||
|
||||
- ``scripts/generate_license.py`` when minting a buyer's blob
|
||||
- Your CI's signing step, if you've automated blob minting
|
||||
|
||||
The public key gets set as ``DATATOOLS_LICENSE_PUBKEY`` in the
|
||||
PyInstaller build env (so the shipped binary verifies against it),
|
||||
and the production-safe runtime check refuses to start any frozen
|
||||
build that's still using the in-source dev key.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
|
||||
|
||||
|
||||
def generate() -> tuple[str, str]:
|
||||
"""Return ``(private_hex, public_hex)`` for a fresh keypair."""
|
||||
priv = Ed25519PrivateKey.generate()
|
||||
priv_hex = priv.private_bytes(
|
||||
encoding=serialization.Encoding.Raw,
|
||||
format=serialization.PrivateFormat.Raw,
|
||||
encryption_algorithm=serialization.NoEncryption(),
|
||||
).hex()
|
||||
pub_hex = priv.public_key().public_bytes(
|
||||
encoding=serialization.Encoding.Raw,
|
||||
format=serialization.PublicFormat.Raw,
|
||||
).hex()
|
||||
return priv_hex, pub_hex
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
p.add_argument("--json", action="store_true", help="Emit JSON instead of env-file format.")
|
||||
p.add_argument("--output", "-o", type=Path, default=None, help="Write to this file instead of stdout.")
|
||||
args = p.parse_args(argv)
|
||||
|
||||
priv_hex, pub_hex = generate()
|
||||
|
||||
if args.json:
|
||||
payload = json.dumps(
|
||||
{"private_key": priv_hex, "public_key": pub_hex},
|
||||
indent=2,
|
||||
)
|
||||
else:
|
||||
payload = (
|
||||
f"# DataTools license keypair — generated by generate_keypair.py\n"
|
||||
f"# KEEP THE PRIVATE KEY SECRET. Lose it and your existing\n"
|
||||
f"# licenses can't be renewed (you'd have to ship a new build\n"
|
||||
f"# with a new public key and re-issue every active license).\n"
|
||||
f"\n"
|
||||
f"DATATOOLS_LICENSE_PRIVKEY={priv_hex}\n"
|
||||
f"DATATOOLS_LICENSE_PUBKEY={pub_hex}\n"
|
||||
)
|
||||
|
||||
if args.output:
|
||||
args.output.write_text(payload + "\n", encoding="utf-8")
|
||||
# chmod 600 — best-effort; ignored on Windows.
|
||||
try:
|
||||
args.output.chmod(0o600)
|
||||
except OSError:
|
||||
pass
|
||||
print(f"Wrote {args.output} (mode 600)", file=sys.stderr)
|
||||
else:
|
||||
print(payload)
|
||||
|
||||
print(
|
||||
"\nNext steps:\n"
|
||||
" 1. Store the private key in your password manager.\n"
|
||||
" 2. Bake the public key into the PyInstaller build:\n"
|
||||
" DATATOOLS_LICENSE_PUBKEY=<pubkey> pyinstaller ...\n"
|
||||
" 3. Mint buyer licenses by setting the private key:\n"
|
||||
" DATATOOLS_LICENSE_PRIVKEY=<privkey> "
|
||||
"python scripts/generate_license.py --name 'Buyer' --email b@x.com\n",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
215
scripts/generate_license.py
Normal file
215
scripts/generate_license.py
Normal file
@@ -0,0 +1,215 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Mint a signed license blob for a buyer (LOCAL, break-glass).
|
||||
|
||||
.. warning::
|
||||
|
||||
This script mints **locally**, without going through the license
|
||||
server. Prefer :mod:`src.admin_cli` (``datatools-admin mint``)
|
||||
for routine work — it writes to the authoritative ``licenses``
|
||||
Postgres table and emits the same blob.
|
||||
|
||||
Reach for this script only when the server is unreachable and a
|
||||
buyer needs a license *right now*. Mints from here land in the
|
||||
local issuance JSONL log; you'll need to reconcile them into the
|
||||
server's DB afterwards.
|
||||
|
||||
Creator-only tool. Signs with the Ed25519 private key from
|
||||
``$DATATOOLS_LICENSE_PRIVKEY`` (production) or the in-tree dev key
|
||||
(local development).
|
||||
|
||||
Every successful mint also appends a record to the issuance log at
|
||||
``~/.datatools-creator/issued.jsonl`` (override with
|
||||
``$DATATOOLS_ISSUANCE_LOG``). That log is the creator-side system of
|
||||
record for "who has a license" — useful for re-delivery, support, and
|
||||
as the seed for the future server-side ``licenses`` table.
|
||||
|
||||
Examples
|
||||
--------
|
||||
|
||||
Mint a 1-year CORE license for Jane Doe::
|
||||
|
||||
python scripts/generate_license.py \\
|
||||
--name "Jane Doe" --email jane@example.com --tier core
|
||||
|
||||
Mint a 2-year PRO license and write the blob to a file::
|
||||
|
||||
python scripts/generate_license.py \\
|
||||
--name "Acme Corp" --email ops@acme.com --tier pro \\
|
||||
--years 2 --output acme.dtlic
|
||||
|
||||
Mint with the production key (CI / manual fulfillment)::
|
||||
|
||||
DATATOOLS_LICENSE_PRIVKEY=<prod-private-hex> \\
|
||||
python scripts/generate_license.py --name ... --email ...
|
||||
|
||||
The output is a single base64-encoded token starting with ``DTLIC2:``
|
||||
— paste this whole string into the buyer's delivery email or
|
||||
deliver as an attached ``.dtlic`` file.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
# Make ``src.license`` importable when run from the repo root.
|
||||
_PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
if str(_PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_PROJECT_ROOT))
|
||||
|
||||
from src.license import Tier # noqa: E402
|
||||
from src.license.crypto import encode_blob, sign # noqa: E402
|
||||
from src.license.features import all_features_for_tier # noqa: E402
|
||||
from src.license.schema import ( # noqa: E402
|
||||
License,
|
||||
_utcnow_iso,
|
||||
default_expiry_iso,
|
||||
)
|
||||
|
||||
|
||||
def default_issuance_log() -> Path:
|
||||
"""Path to the local issuance log (creator-side ledger).
|
||||
|
||||
Resolution order:
|
||||
|
||||
1. ``$DATATOOLS_ISSUANCE_LOG`` (absolute path; useful for tests
|
||||
and for pointing at a shared / encrypted volume).
|
||||
2. ``~/.datatools-creator/issued.jsonl`` — separate from the
|
||||
buyer-facing ``~/.datatools/`` dir so it never gets bundled
|
||||
into a shipped install.
|
||||
"""
|
||||
override = os.environ.get("DATATOOLS_ISSUANCE_LOG")
|
||||
if override:
|
||||
return Path(override).expanduser().resolve()
|
||||
return Path.home() / ".datatools-creator" / "issued.jsonl"
|
||||
|
||||
|
||||
def append_issuance_log(record: dict, *, path: Path | None = None) -> Path | None:
|
||||
"""Best-effort append of *record* to the issuance log.
|
||||
|
||||
Returns the resolved path on success, ``None`` on IO failure
|
||||
(with a warning printed to stderr). We intentionally do not raise:
|
||||
the blob has already been minted by the time this runs, and losing
|
||||
one ledger row is strictly better than aborting after a successful
|
||||
mint and leaving the creator unsure whether to re-mint.
|
||||
"""
|
||||
p = path or default_issuance_log()
|
||||
try:
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
with p.open("a", encoding="utf-8") as f:
|
||||
f.write(json.dumps(record, sort_keys=True) + "\n")
|
||||
try:
|
||||
p.chmod(0o600)
|
||||
except OSError:
|
||||
pass
|
||||
return p
|
||||
except OSError as e:
|
||||
print(
|
||||
f"WARNING: could not write issuance log at {p}: {e}\n"
|
||||
" The blob above is still valid — record the mint "
|
||||
"manually.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def build_args() -> argparse.ArgumentParser:
|
||||
p = argparse.ArgumentParser(
|
||||
description="Mint a signed DataTools license blob.",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
p.add_argument("--name", required=True, help="Buyer's full name.")
|
||||
p.add_argument("--email", required=True, help="Buyer's email.")
|
||||
p.add_argument(
|
||||
"--tier",
|
||||
default=Tier.CORE.value,
|
||||
choices=[t.value for t in Tier],
|
||||
help="License tier (default: %(default)s).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--years",
|
||||
type=int,
|
||||
default=1,
|
||||
help="License lifetime in years (default: %(default)s).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--key",
|
||||
default=None,
|
||||
help="Override the auto-generated license key (default: random).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--output",
|
||||
"-o",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="Write the blob to this file (default: print to stdout).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--no-log",
|
||||
action="store_true",
|
||||
help=(
|
||||
"Skip writing to the issuance log. Use for one-off test "
|
||||
"mints; do NOT use for real buyer fulfillment."
|
||||
),
|
||||
)
|
||||
return p
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_args().parse_args(argv)
|
||||
tier = Tier(args.tier)
|
||||
rid = uuid.uuid4().hex
|
||||
key = args.key or f"DT1-{tier.value.upper()}-{rid[:8]}-{rid[8:16]}"
|
||||
|
||||
lic = License(
|
||||
name=args.name,
|
||||
email=args.email,
|
||||
license_key=key,
|
||||
tier=tier,
|
||||
features=all_features_for_tier(tier),
|
||||
issued_at=_utcnow_iso(),
|
||||
expires_at=default_expiry_iso(years=args.years),
|
||||
signature="",
|
||||
)
|
||||
signature = sign(lic.to_canonical_dict())
|
||||
payload = lic.to_canonical_dict()
|
||||
payload["signature"] = signature
|
||||
blob = encode_blob(payload)
|
||||
|
||||
if not args.no_log:
|
||||
log_path = append_issuance_log({
|
||||
"license_key": lic.license_key,
|
||||
"name": lic.name,
|
||||
"email": lic.email,
|
||||
"tier": lic.tier.value,
|
||||
"issued_at": lic.issued_at,
|
||||
"expires_at": lic.expires_at,
|
||||
"blob": blob,
|
||||
})
|
||||
else:
|
||||
log_path = None
|
||||
|
||||
if args.output:
|
||||
args.output.write_text(blob + "\n", encoding="utf-8")
|
||||
print(f"Wrote license to {args.output}", file=sys.stderr)
|
||||
else:
|
||||
print(blob)
|
||||
print(
|
||||
f" name: {lic.name}\n"
|
||||
f" email: {lic.email}\n"
|
||||
f" tier: {lic.tier.value}\n"
|
||||
f" key: {lic.license_key}\n"
|
||||
f" expires: {lic.expires_at}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
if log_path:
|
||||
print(f" logged: {log_path}", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
16
server/.dockerignore
Normal file
16
server/.dockerignore
Normal file
@@ -0,0 +1,16 @@
|
||||
**/__pycache__
|
||||
**/*.pyc
|
||||
**/.pytest_cache
|
||||
**/.mypy_cache
|
||||
**/.ruff_cache
|
||||
.git
|
||||
.venv
|
||||
venv
|
||||
docs
|
||||
landing
|
||||
marketing
|
||||
samples
|
||||
test-cases
|
||||
tests
|
||||
logs
|
||||
build
|
||||
38
server/Dockerfile
Normal file
38
server/Dockerfile
Normal file
@@ -0,0 +1,38 @@
|
||||
# syntax=docker/dockerfile:1.6
|
||||
FROM python:3.12-slim AS base
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PIP_NO_CACHE_DIR=1 \
|
||||
PIP_DISABLE_PIP_VERSION_CHECK=1
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
curl \
|
||||
libpq5 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN useradd --system --create-home --shell /usr/sbin/nologin --uid 10001 app
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY server/requirements.txt /app/requirements.txt
|
||||
RUN pip install -r /app/requirements.txt
|
||||
|
||||
# Reused crypto / schema logic from the desktop app — single source of truth.
|
||||
COPY src/license /app/datatools_license
|
||||
|
||||
COPY server/app /app/app
|
||||
COPY server/config /app/config
|
||||
COPY server/alembic /app/alembic
|
||||
COPY server/alembic.ini /app/alembic.ini
|
||||
|
||||
RUN chown -R app:app /app
|
||||
USER app
|
||||
|
||||
EXPOSE 8000
|
||||
|
||||
HEALTHCHECK --interval=30s --timeout=3s --start-period=15s --retries=3 \
|
||||
CMD curl --fail --silent --show-error http://localhost:8000/health || exit 1
|
||||
|
||||
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000", "--proxy-headers", "--forwarded-allow-ips", "*"]
|
||||
38
server/alembic.ini
Normal file
38
server/alembic.ini
Normal file
@@ -0,0 +1,38 @@
|
||||
[alembic]
|
||||
script_location = alembic
|
||||
prepend_sys_path = .
|
||||
sqlalchemy.url =
|
||||
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
[handlers]
|
||||
keys = console
|
||||
|
||||
[formatters]
|
||||
keys = generic
|
||||
|
||||
[logger_root]
|
||||
level = WARN
|
||||
handlers = console
|
||||
qualname =
|
||||
|
||||
[logger_sqlalchemy]
|
||||
level = WARN
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
datefmt = %H:%M:%S
|
||||
46
server/alembic/env.py
Normal file
46
server/alembic/env.py
Normal file
@@ -0,0 +1,46 @@
|
||||
"""Alembic environment.
|
||||
|
||||
Reads the runtime database URL from ``app.db`` (which resolves the
|
||||
password from the secrets file), so ``alembic upgrade head`` Just
|
||||
Works inside the API container with no extra env wiring.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from logging.config import fileConfig
|
||||
|
||||
from alembic import context
|
||||
|
||||
from app.db import Base, engine
|
||||
from app import models # noqa: F401 — imported for side-effect of registering models
|
||||
|
||||
config = context.config
|
||||
|
||||
if config.config_file_name is not None:
|
||||
fileConfig(config.config_file_name)
|
||||
|
||||
target_metadata = Base.metadata
|
||||
|
||||
|
||||
def run_migrations_offline() -> None:
|
||||
context.configure(
|
||||
url=str(engine.url),
|
||||
target_metadata=target_metadata,
|
||||
literal_binds=True,
|
||||
dialect_opts={"paramstyle": "named"},
|
||||
)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
def run_migrations_online() -> None:
|
||||
with engine.connect() as connection:
|
||||
context.configure(connection=connection, target_metadata=target_metadata)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
if context.is_offline_mode():
|
||||
run_migrations_offline()
|
||||
else:
|
||||
run_migrations_online()
|
||||
26
server/alembic/script.py.mako
Normal file
26
server/alembic/script.py.mako
Normal file
@@ -0,0 +1,26 @@
|
||||
"""${message}
|
||||
|
||||
Revision ID: ${up_revision}
|
||||
Revises: ${down_revision | comma,n}
|
||||
Create Date: ${create_date}
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
${imports if imports else ""}
|
||||
|
||||
revision: str = ${repr(up_revision)}
|
||||
down_revision: Union[str, None] = ${repr(down_revision)}
|
||||
branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
|
||||
depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
${upgrades if upgrades else "pass"}
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
${downgrades if downgrades else "pass"}
|
||||
80
server/alembic/versions/0001_initial.py
Normal file
80
server/alembic/versions/0001_initial.py
Normal file
@@ -0,0 +1,80 @@
|
||||
"""Initial schema — licenses + gumroad_events.
|
||||
|
||||
Revision ID: 0001_initial
|
||||
Revises:
|
||||
Create Date: 2026-05-14
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
revision: str = "0001_initial"
|
||||
down_revision: Union[str, None] = None
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"licenses",
|
||||
sa.Column("license_key", sa.String(), primary_key=True),
|
||||
sa.Column("name", sa.String(), nullable=False),
|
||||
sa.Column("email", sa.String(), nullable=False),
|
||||
sa.Column("tier", sa.String(), nullable=False),
|
||||
sa.Column("issued_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("blob", sa.String(), nullable=False),
|
||||
sa.Column("source", sa.String(), nullable=False),
|
||||
sa.Column("source_order_id", sa.String(), nullable=True),
|
||||
sa.Column("promotion", sa.String(), nullable=True),
|
||||
sa.Column("amount_paid", sa.Numeric(10, 2), nullable=True),
|
||||
sa.Column("currency", sa.String(length=3), server_default=sa.text("'USD'"), nullable=True),
|
||||
sa.Column("revoked_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("notes", sa.String(), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.UniqueConstraint("source", "source_order_id", name="uq_licenses_source_order"),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_licenses_email_lower",
|
||||
"licenses",
|
||||
[sa.text("lower(email)")],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_licenses_expires_active",
|
||||
"licenses",
|
||||
["expires_at"],
|
||||
postgresql_where=sa.text("revoked_at IS NULL"),
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"gumroad_events",
|
||||
sa.Column("id", sa.BigInteger(), primary_key=True, autoincrement=True),
|
||||
sa.Column("received_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("event_type", sa.String(), nullable=False),
|
||||
sa.Column("order_id", sa.String(), nullable=True),
|
||||
sa.Column("raw_payload", postgresql.JSONB(), nullable=False),
|
||||
sa.Column("processed", sa.Boolean(), server_default=sa.text("false"), nullable=False),
|
||||
sa.Column("error", sa.String(), nullable=True),
|
||||
)
|
||||
op.create_index("ix_gumroad_events_order_id", "gumroad_events", ["order_id"])
|
||||
op.create_index(
|
||||
"ix_gumroad_events_unprocessed",
|
||||
"gumroad_events",
|
||||
["received_at"],
|
||||
postgresql_where=sa.text("processed = false"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_gumroad_events_unprocessed", table_name="gumroad_events")
|
||||
op.drop_index("ix_gumroad_events_order_id", table_name="gumroad_events")
|
||||
op.drop_table("gumroad_events")
|
||||
op.drop_index("ix_licenses_expires_active", table_name="licenses")
|
||||
op.drop_index("ix_licenses_email_lower", table_name="licenses")
|
||||
op.drop_table("licenses")
|
||||
0
server/app/__init__.py
Normal file
0
server/app/__init__.py
Normal file
0
server/app/adapters/__init__.py
Normal file
0
server/app/adapters/__init__.py
Normal file
71
server/app/adapters/base.py
Normal file
71
server/app/adapters/base.py
Normal file
@@ -0,0 +1,71 @@
|
||||
"""Source-adapter interface.
|
||||
|
||||
The Mint API speaks only the normalized event types defined here.
|
||||
Each storefront has its own adapter that:
|
||||
|
||||
- Verifies the storefront's webhook signature in its native format.
|
||||
- Parses the storefront's payload into a :class:`SaleEvent` or
|
||||
:class:`RefundEvent`.
|
||||
- Maps the storefront's product/variant IDs to a license tier via
|
||||
the per-source config in :mod:`app.adapters.config`.
|
||||
|
||||
Adding a new source (Lemon Squeezy, Stripe, Paddle) is one new
|
||||
module that implements :class:`SourceAdapter`. The Mint API and DB
|
||||
do not change.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from decimal import Decimal
|
||||
from typing import Any, Optional, Protocol
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SaleEvent:
|
||||
"""A storefront sale, normalized.
|
||||
|
||||
The Mint API consumes this directly — it never reaches into the
|
||||
raw storefront payload. Anything storefront-specific that's worth
|
||||
keeping is preserved in :attr:`raw_payload` for audit.
|
||||
"""
|
||||
|
||||
source: str # e.g. "gumroad", "manual"
|
||||
source_order_id: Optional[str] # storefront's order ID; None for manual mints
|
||||
buyer_name: str
|
||||
buyer_email: str
|
||||
tier: str # mapped from product/variant
|
||||
years: int = 1
|
||||
promotion: Optional[str] = None
|
||||
amount_paid: Optional[Decimal] = None
|
||||
currency: Optional[str] = "USD"
|
||||
notes: Optional[str] = None
|
||||
raw_payload: dict = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RefundEvent:
|
||||
"""A storefront refund — marks an existing license revoked."""
|
||||
|
||||
source: str
|
||||
source_order_id: str
|
||||
reason: Optional[str] = None
|
||||
raw_payload: dict = field(default_factory=dict)
|
||||
|
||||
|
||||
class SourceAdapter(Protocol):
|
||||
"""Interface every storefront adapter implements."""
|
||||
|
||||
source_name: str
|
||||
|
||||
def verify_webhook(self, *, body: bytes, headers: dict[str, str]) -> bool:
|
||||
"""Return True iff the request came from the legitimate storefront."""
|
||||
...
|
||||
|
||||
def parse_sale(self, payload: dict[str, Any]) -> Optional[SaleEvent]:
|
||||
"""Return a :class:`SaleEvent` if *payload* is a sale, else None."""
|
||||
...
|
||||
|
||||
def parse_refund(self, payload: dict[str, Any]) -> Optional[RefundEvent]:
|
||||
"""Return a :class:`RefundEvent` if *payload* is a refund, else None."""
|
||||
...
|
||||
173
server/app/adapters/gumroad.py
Normal file
173
server/app/adapters/gumroad.py
Normal file
@@ -0,0 +1,173 @@
|
||||
"""Gumroad adapter.
|
||||
|
||||
Receives "Ping" notifications from Gumroad — form-encoded POSTs sent
|
||||
when a sale occurs. Gumroad's Ping URL is configured in the seller
|
||||
dashboard (Settings → Advanced → Ping URL).
|
||||
|
||||
Authentication
|
||||
--------------
|
||||
|
||||
Gumroad does not HMAC-sign the body. Their recommended pattern is
|
||||
to put a secret in the URL itself::
|
||||
|
||||
https://licenses.datatools.unalogix.com/webhooks/gumroad?secret=...
|
||||
|
||||
The webhook receiver pulls the secret from the query string and
|
||||
:meth:`GumroadAdapter.verify_webhook` constant-time-compares it
|
||||
against the configured value. If they don't match, the request is
|
||||
dropped with 404 (so a probing attacker can't tell whether the
|
||||
endpoint exists, much less that it's the wrong secret).
|
||||
|
||||
The "test" field
|
||||
----------------
|
||||
|
||||
Gumroad sends ``test=true`` on test pings fired from the dashboard.
|
||||
We treat test pings as real sales (they create licenses just like
|
||||
production sales), but tag them with ``notes='gumroad test ping'``
|
||||
so the operator can filter / delete them later. Refusing test pings
|
||||
would block the standard "Send Test Ping" verification flow.
|
||||
|
||||
Refunds, disputes, cancellations
|
||||
--------------------------------
|
||||
|
||||
Stubbed for now (``parse_refund`` returns None). Gumroad doesn't
|
||||
include refund signals in the standard sale Ping — refunds arrive
|
||||
via the separate "Resource subscriptions" mechanism. Wiring that
|
||||
in is PR 2.1; until then, refunds are handled by the operator
|
||||
running ``datatools-admin revoke``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hmac
|
||||
from decimal import Decimal
|
||||
from typing import Any, Optional
|
||||
|
||||
from app.adapters.base import RefundEvent, SaleEvent
|
||||
from app.products import lookup as product_lookup
|
||||
|
||||
|
||||
class GumroadAdapter:
|
||||
source_name = "gumroad"
|
||||
|
||||
def __init__(self, secret: Optional[str]) -> None:
|
||||
self._secret = secret
|
||||
|
||||
# --- Auth ----------------------------------------------------------------
|
||||
|
||||
def verify_webhook(self, *, body: bytes, headers: dict[str, str]) -> bool:
|
||||
"""Not used — Gumroad authentication is via URL query param,
|
||||
which only the route handler has direct access to. Call
|
||||
:meth:`verify_secret` instead."""
|
||||
return False
|
||||
|
||||
def verify_secret(self, presented: Optional[str]) -> bool:
|
||||
"""Constant-time compare against the configured secret.
|
||||
|
||||
Returns False (not an exception) so the route handler can
|
||||
decide the response code — we return 404 to avoid signaling
|
||||
endpoint existence to an unauthenticated prober.
|
||||
"""
|
||||
if not self._secret or not presented:
|
||||
return False
|
||||
return hmac.compare_digest(presented, self._secret)
|
||||
|
||||
# --- Parsing -------------------------------------------------------------
|
||||
|
||||
def parse_sale(self, payload: dict[str, Any]) -> Optional[SaleEvent]:
|
||||
"""Parse a Gumroad Ping form-encoded payload into a SaleEvent.
|
||||
|
||||
Returns None if the payload isn't a sale (e.g. some future
|
||||
event type we don't yet handle). Returns None *with no row
|
||||
side-effect* if the product_id is unmapped — the caller
|
||||
should treat that as an error and record it in the audit
|
||||
row, not silently drop.
|
||||
"""
|
||||
# Sale pings always include sale_id (the order ID) and email.
|
||||
sale_id = payload.get("sale_id")
|
||||
email = payload.get("email")
|
||||
product_id = (
|
||||
payload.get("product_id")
|
||||
or payload.get("product_permalink")
|
||||
or payload.get("permalink")
|
||||
)
|
||||
if not (sale_id and email and product_id):
|
||||
return None
|
||||
|
||||
mapping = product_lookup(self.source_name, str(product_id))
|
||||
if mapping is None:
|
||||
# Unmapped — surface to caller as a SaleEvent with no tier.
|
||||
# We deliberately don't raise here so the caller can
|
||||
# log it to gumroad_events with error info and still
|
||||
# return 200 (no Gumroad retry storm).
|
||||
raise UnmappedProductError(
|
||||
f"Gumroad product_id {product_id!r} has no entry in "
|
||||
"config/products.yaml. Add a mapping and replay this "
|
||||
f"sale (sale_id={sale_id})."
|
||||
)
|
||||
|
||||
name = (payload.get("full_name") or "").strip() or _email_local(email)
|
||||
|
||||
price_cents = _to_int(payload.get("price"))
|
||||
amount_paid = Decimal(price_cents) / Decimal(100) if price_cents is not None else None
|
||||
currency = (payload.get("currency") or "USD").upper()
|
||||
promotion = (payload.get("offer_code") or "").strip() or None
|
||||
|
||||
notes = None
|
||||
if _is_truthy(payload.get("test")):
|
||||
notes = "gumroad test ping"
|
||||
|
||||
return SaleEvent(
|
||||
source=self.source_name,
|
||||
source_order_id=str(sale_id),
|
||||
buyer_name=name,
|
||||
buyer_email=email.strip(),
|
||||
tier=mapping.tier,
|
||||
years=mapping.years,
|
||||
promotion=promotion,
|
||||
amount_paid=amount_paid,
|
||||
currency=currency,
|
||||
notes=notes,
|
||||
raw_payload=dict(payload),
|
||||
)
|
||||
|
||||
def parse_refund(self, payload: dict[str, Any]) -> Optional[RefundEvent]:
|
||||
# PR 2.1.
|
||||
return None
|
||||
|
||||
|
||||
class UnmappedProductError(ValueError):
|
||||
"""Raised when a sale arrives for a product not in products.yaml.
|
||||
|
||||
Caller catches and logs into ``gumroad_events.error`` so the
|
||||
operator can fix the mapping and replay.
|
||||
"""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _email_local(email: str) -> str:
|
||||
"""Fallback display name when ``full_name`` is missing — the part
|
||||
of the email before the ``@``, capitalized. Better than 'Unknown'
|
||||
for support tickets and the buyer's own delivery email."""
|
||||
local = email.split("@", 1)[0]
|
||||
return local.replace(".", " ").title()
|
||||
|
||||
|
||||
def _to_int(v: Any) -> Optional[int]:
|
||||
if v is None or v == "":
|
||||
return None
|
||||
try:
|
||||
return int(v)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _is_truthy(v: Any) -> bool:
|
||||
if isinstance(v, bool):
|
||||
return v
|
||||
if v is None:
|
||||
return False
|
||||
return str(v).strip().lower() in {"1", "true", "yes", "on"}
|
||||
52
server/app/adapters/manual.py
Normal file
52
server/app/adapters/manual.py
Normal file
@@ -0,0 +1,52 @@
|
||||
"""Manual adapter — operator-initiated mints (comps, support replacements).
|
||||
|
||||
There is no webhook to verify and no payload to parse: the operator
|
||||
hands us the buyer details directly via the CLI, and we construct a
|
||||
:class:`SaleEvent` from them. ``source='manual'`` separates these
|
||||
rows from storefront-driven mints in the DB.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from decimal import Decimal
|
||||
from typing import Any, Optional
|
||||
|
||||
from app.adapters.base import RefundEvent, SaleEvent
|
||||
|
||||
|
||||
class ManualAdapter:
|
||||
source_name = "manual"
|
||||
|
||||
def verify_webhook(self, *, body: bytes, headers: dict[str, str]) -> bool:
|
||||
return False # manual flows never come through webhooks
|
||||
|
||||
def parse_sale(self, payload: dict[str, Any]) -> Optional[SaleEvent]:
|
||||
return self.build_sale(**payload)
|
||||
|
||||
def parse_refund(self, payload: dict[str, Any]) -> Optional[RefundEvent]:
|
||||
return None
|
||||
|
||||
def build_sale(
|
||||
self,
|
||||
*,
|
||||
name: str,
|
||||
email: str,
|
||||
tier: str,
|
||||
years: int = 1,
|
||||
promotion: Optional[str] = None,
|
||||
amount_paid: Optional[Decimal] = None,
|
||||
currency: Optional[str] = "USD",
|
||||
notes: Optional[str] = None,
|
||||
) -> SaleEvent:
|
||||
return SaleEvent(
|
||||
source=self.source_name,
|
||||
source_order_id=None,
|
||||
buyer_name=name,
|
||||
buyer_email=email,
|
||||
tier=tier,
|
||||
years=years,
|
||||
promotion=promotion,
|
||||
amount_paid=amount_paid,
|
||||
currency=currency,
|
||||
notes=notes,
|
||||
)
|
||||
65
server/app/auth.py
Normal file
65
server/app/auth.py
Normal file
@@ -0,0 +1,65 @@
|
||||
"""Auth guards for ``/internal/*``.
|
||||
|
||||
Active layer: Bearer token, presented by the operator's CLI and
|
||||
matched against the value in the secrets dir. Token rotation =
|
||||
update the file, restart the container.
|
||||
|
||||
:func:`require_localhost` is preserved but unused by default — it
|
||||
fights the Docker bridge network model (the container sees the
|
||||
gateway IP, not 127.0.0.1, regardless of where traffic originated).
|
||||
Re-enable it only if the API runs in ``network_mode: host``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hmac
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import HTTPException, Request, status
|
||||
|
||||
from app.config import get_settings
|
||||
|
||||
|
||||
def require_localhost(request: Request) -> None:
|
||||
"""Reject the request unless the connecting peer is loopback.
|
||||
|
||||
``request.client.host`` reflects the actual TCP peer (the nginx
|
||||
upstream connecting from 127.0.0.1) when ``proxy_set_header`` is
|
||||
used appropriately. We deliberately do NOT trust
|
||||
``X-Forwarded-For`` here — we want the raw peer.
|
||||
"""
|
||||
peer = request.client.host if request.client else None
|
||||
if peer not in {"127.0.0.1", "::1"}:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="Not found",
|
||||
)
|
||||
|
||||
|
||||
def require_bearer_token(request: Request) -> None:
|
||||
"""Verify ``Authorization: Bearer <admin_token>``.
|
||||
|
||||
Uses constant-time comparison so timing leaks don't reveal token
|
||||
prefixes. The 401 deliberately doesn't echo the supplied token or
|
||||
leak whether a token is configured at all — clients should treat
|
||||
"no token configured" the same as "wrong token".
|
||||
"""
|
||||
settings = get_settings()
|
||||
expected: Optional[str] = settings.resolve_admin_token()
|
||||
if not expected:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Server not configured for internal access.",
|
||||
)
|
||||
auth = request.headers.get("Authorization", "")
|
||||
if not auth.startswith("Bearer "):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Bearer token required.",
|
||||
)
|
||||
presented = auth.removeprefix("Bearer ").strip()
|
||||
if not hmac.compare_digest(presented, expected):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Invalid token.",
|
||||
)
|
||||
64
server/app/config.py
Normal file
64
server/app/config.py
Normal file
@@ -0,0 +1,64 @@
|
||||
"""Runtime configuration loaded from environment + secret files.
|
||||
|
||||
Secrets are read from files (``*_FILE`` env vars pointing at
|
||||
``/run/secrets/<name>``) so they never appear in ``docker inspect``
|
||||
or process environment dumps. Plain ``*`` vars are the fallback for
|
||||
local development where mounting secret files is overkill.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from pydantic import Field
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
model_config = SettingsConfigDict(env_file=".env", extra="ignore")
|
||||
|
||||
database_url: str = Field(
|
||||
default="postgresql+psycopg://datatools_api@localhost:5432/datatools_licenses",
|
||||
validation_alias="DATABASE_URL",
|
||||
)
|
||||
|
||||
admin_token: Optional[str] = Field(default=None, validation_alias="DATATOOLS_ADMIN_TOKEN")
|
||||
admin_token_file: Optional[Path] = Field(default=None, validation_alias="DATATOOLS_ADMIN_TOKEN_FILE")
|
||||
|
||||
license_privkey_hex: Optional[str] = Field(default=None, validation_alias="DATATOOLS_LICENSE_PRIVKEY")
|
||||
license_privkey_file: Optional[Path] = Field(default=None, validation_alias="DATATOOLS_LICENSE_PRIVKEY_FILE")
|
||||
|
||||
license_pubkey_hex: Optional[str] = Field(default=None, validation_alias="DATATOOLS_LICENSE_PUBKEY")
|
||||
|
||||
postmark_token: Optional[str] = Field(default=None, validation_alias="POSTMARK_TOKEN")
|
||||
postmark_token_file: Optional[Path] = Field(default=None, validation_alias="POSTMARK_TOKEN_FILE")
|
||||
|
||||
gumroad_secret: Optional[str] = Field(default=None, validation_alias="GUMROAD_WEBHOOK_SECRET")
|
||||
gumroad_secret_file: Optional[Path] = Field(default=None, validation_alias="GUMROAD_WEBHOOK_SECRET_FILE")
|
||||
|
||||
def resolve_admin_token(self) -> Optional[str]:
|
||||
return _resolve(self.admin_token, self.admin_token_file)
|
||||
|
||||
def resolve_license_privkey(self) -> Optional[str]:
|
||||
return _resolve(self.license_privkey_hex, self.license_privkey_file)
|
||||
|
||||
def resolve_postmark_token(self) -> Optional[str]:
|
||||
return _resolve(self.postmark_token, self.postmark_token_file)
|
||||
|
||||
def resolve_gumroad_secret(self) -> Optional[str]:
|
||||
return _resolve(self.gumroad_secret, self.gumroad_secret_file)
|
||||
|
||||
|
||||
def _resolve(inline: Optional[str], path: Optional[Path]) -> Optional[str]:
|
||||
if inline:
|
||||
return inline.strip()
|
||||
if path and path.exists():
|
||||
return path.read_text(encoding="utf-8").strip()
|
||||
return None
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_settings() -> Settings:
|
||||
return Settings()
|
||||
65
server/app/db.py
Normal file
65
server/app/db.py
Normal file
@@ -0,0 +1,65 @@
|
||||
"""SQLAlchemy engine + session factory.
|
||||
|
||||
The DB password lives in ``/run/secrets/pg_password``; we read it
|
||||
from there (or ``$PG_PASSWORD`` for local dev) and splice it into
|
||||
``DATABASE_URL`` so the password never has to be in plaintext in
|
||||
``compose.yml`` or process environment listings.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Generator
|
||||
from urllib.parse import quote_plus, urlparse, urlunparse
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker
|
||||
|
||||
from app.config import get_settings
|
||||
|
||||
|
||||
def _resolve_password() -> str | None:
|
||||
inline = os.environ.get("PG_PASSWORD")
|
||||
if inline:
|
||||
return inline.strip()
|
||||
path = os.environ.get("PG_PASSWORD_FILE")
|
||||
if path and Path(path).exists():
|
||||
return Path(path).read_text(encoding="utf-8").strip()
|
||||
return None
|
||||
|
||||
|
||||
def _build_url(base_url: str) -> str:
|
||||
"""Inject the resolved password into ``base_url`` if absent."""
|
||||
parsed = urlparse(base_url)
|
||||
if parsed.password:
|
||||
return base_url
|
||||
pw = _resolve_password()
|
||||
if pw is None:
|
||||
return base_url
|
||||
netloc = f"{parsed.username or ''}:{quote_plus(pw)}@{parsed.hostname}"
|
||||
if parsed.port:
|
||||
netloc += f":{parsed.port}"
|
||||
return urlunparse(parsed._replace(netloc=netloc))
|
||||
|
||||
|
||||
_settings = get_settings()
|
||||
engine = create_engine(_build_url(_settings.database_url), pool_pre_ping=True, future=True)
|
||||
SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False, expire_on_commit=False)
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
"""Declarative base for ORM models."""
|
||||
|
||||
|
||||
def get_session() -> Generator[Session, None, None]:
|
||||
"""FastAPI dependency. Commits on success, rolls back on exception."""
|
||||
session = SessionLocal()
|
||||
try:
|
||||
yield session
|
||||
session.commit()
|
||||
except Exception:
|
||||
session.rollback()
|
||||
raise
|
||||
finally:
|
||||
session.close()
|
||||
214
server/app/email.py
Normal file
214
server/app/email.py
Normal file
@@ -0,0 +1,214 @@
|
||||
"""Transactional email delivery.
|
||||
|
||||
Provider: Postmark. Picked for its transactional-deliverability
|
||||
reputation and a tiny, no-SDK-needed HTTP API.
|
||||
|
||||
Configuration
|
||||
-------------
|
||||
|
||||
- ``POSTMARK_TOKEN`` / ``POSTMARK_TOKEN_FILE`` — server API token.
|
||||
- ``EMAIL_FROM`` — verified sender address (default
|
||||
``licenses@datatools.unalogix.com``).
|
||||
- ``EMAIL_REPLY_TO`` — optional Reply-To (default same as From).
|
||||
|
||||
When ``POSTMARK_TOKEN`` is unset the service falls back to
|
||||
:class:`LoggingEmailService`, which prints the email to stdout
|
||||
instead of sending. Lets the webhook handler exercise the full
|
||||
flow before the Postmark account is provisioned.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional, Protocol
|
||||
|
||||
import httpx
|
||||
|
||||
from app.config import get_settings
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LicenseEmail:
|
||||
"""Inputs the renderer needs from the caller."""
|
||||
|
||||
to_name: str
|
||||
to_email: str
|
||||
tier: str
|
||||
license_key: str
|
||||
expires_at_iso: str
|
||||
blob: str
|
||||
|
||||
|
||||
class EmailService(Protocol):
|
||||
"""Provider-agnostic email surface — keeps Postmark out of the
|
||||
callers' import graph."""
|
||||
|
||||
def send_license(self, msg: LicenseEmail) -> str:
|
||||
"""Deliver the license-delivery email. Returns a provider
|
||||
message id (or ``"logged"`` for the dev fallback) so the
|
||||
caller can record it on the licenses row for audit."""
|
||||
...
|
||||
|
||||
|
||||
class LoggingEmailService:
|
||||
"""Stand-in when no real provider is configured. Logs the
|
||||
rendered message body at INFO so it shows up in ``docker compose
|
||||
logs api`` — useful during local dev and during the deploy
|
||||
window before Postmark is wired up."""
|
||||
|
||||
def send_license(self, msg: LicenseEmail) -> str:
|
||||
body = _render_text(msg)
|
||||
log.info(
|
||||
"[email-stub] would send to=%s subject=%r\n%s",
|
||||
msg.to_email,
|
||||
_subject(msg),
|
||||
body,
|
||||
)
|
||||
return "logged"
|
||||
|
||||
|
||||
class PostmarkEmailService:
|
||||
"""Postmark transactional API client.
|
||||
|
||||
Single endpoint, ~3 fields, no SDK needed. We use a per-call
|
||||
httpx Client with a tight timeout — webhook handlers run on
|
||||
the request thread and we never want to block them on a flaky
|
||||
upstream.
|
||||
"""
|
||||
|
||||
API_URL = "https://api.postmarkapp.com/email"
|
||||
TIMEOUT_S = 8.0
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
token: str,
|
||||
*,
|
||||
sender: str,
|
||||
reply_to: Optional[str] = None,
|
||||
message_stream: str = "outbound",
|
||||
) -> None:
|
||||
self._token = token
|
||||
self._sender = sender
|
||||
self._reply_to = reply_to or sender
|
||||
self._stream = message_stream
|
||||
|
||||
def send_license(self, msg: LicenseEmail) -> str:
|
||||
body_text = _render_text(msg)
|
||||
body_html = _render_html(msg)
|
||||
payload = {
|
||||
"From": self._sender,
|
||||
"To": _rfc_addr(msg.to_name, msg.to_email),
|
||||
"ReplyTo": self._reply_to,
|
||||
"Subject": _subject(msg),
|
||||
"TextBody": body_text,
|
||||
"HtmlBody": body_html,
|
||||
"MessageStream": self._stream,
|
||||
}
|
||||
headers = {
|
||||
"Accept": "application/json",
|
||||
"Content-Type": "application/json",
|
||||
"X-Postmark-Server-Token": self._token,
|
||||
}
|
||||
with httpx.Client(timeout=self.TIMEOUT_S) as c:
|
||||
r = c.post(self.API_URL, json=payload, headers=headers)
|
||||
if r.status_code >= 400:
|
||||
raise EmailDeliveryError(
|
||||
f"Postmark rejected the request: HTTP {r.status_code} "
|
||||
f"body={r.text[:300]!r}"
|
||||
)
|
||||
return str(r.json().get("MessageID", ""))
|
||||
|
||||
|
||||
class EmailDeliveryError(RuntimeError):
|
||||
"""Provider returned a non-2xx. Caller should record this on the
|
||||
audit row so the operator can replay after fixing the provider
|
||||
config (verified sender domain, paid plan, etc.)."""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Factory
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def get_email_service() -> EmailService:
|
||||
"""Choose the real provider if a token is configured, else the
|
||||
logger. Reads settings fresh — tests can flip env vars between
|
||||
sends without restarting."""
|
||||
settings = get_settings()
|
||||
token = settings.resolve_postmark_token()
|
||||
if not token:
|
||||
return LoggingEmailService()
|
||||
sender = os.environ.get("EMAIL_FROM", "licenses@datatools.unalogix.com")
|
||||
reply_to = os.environ.get("EMAIL_REPLY_TO")
|
||||
return PostmarkEmailService(token, sender=sender, reply_to=reply_to)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Rendering
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _subject(msg: LicenseEmail) -> str:
|
||||
return f"Your DataTools license ({msg.tier})"
|
||||
|
||||
|
||||
def _render_text(msg: LicenseEmail) -> str:
|
||||
return (
|
||||
f"Hi {msg.to_name},\n\n"
|
||||
f"Thanks for your DataTools purchase. Your license is below.\n\n"
|
||||
f"License key: {msg.license_key}\n"
|
||||
f"Tier: {msg.tier}\n"
|
||||
f"Expires: {msg.expires_at_iso[:10]}\n\n"
|
||||
f"To activate, paste the full blob (starting with DTLIC2:) into\n"
|
||||
f"the Activate screen, or run:\n\n"
|
||||
f" python -m src.license_cli activate \"{msg.blob}\" \\\n"
|
||||
f" --name \"{msg.to_name}\" --email {msg.to_email}\n\n"
|
||||
f"Your blob:\n\n"
|
||||
f"{msg.blob}\n\n"
|
||||
f"Keep this email — you'll need the blob if you move to a new\n"
|
||||
f"computer. Questions: reply to this email.\n\n"
|
||||
f"— DataTools\n"
|
||||
)
|
||||
|
||||
|
||||
def _render_html(msg: LicenseEmail) -> str:
|
||||
return (
|
||||
"<!doctype html><html><body style=\"font-family:system-ui,sans-serif;"
|
||||
"max-width:560px;margin:auto;padding:24px;color:#222;\">"
|
||||
f"<p>Hi {_html_escape(msg.to_name)},</p>"
|
||||
"<p>Thanks for your DataTools purchase. Your license is below.</p>"
|
||||
"<table cellpadding=\"4\" style=\"border-collapse:collapse;\">"
|
||||
f"<tr><td><b>License key</b></td><td><code>{_html_escape(msg.license_key)}</code></td></tr>"
|
||||
f"<tr><td><b>Tier</b></td><td>{_html_escape(msg.tier)}</td></tr>"
|
||||
f"<tr><td><b>Expires</b></td><td>{_html_escape(msg.expires_at_iso[:10])}</td></tr>"
|
||||
"</table>"
|
||||
"<p>To activate, paste the blob below into the <em>Activate</em> "
|
||||
"screen on first launch.</p>"
|
||||
"<pre style=\"background:#f4f4f4;padding:12px;border-radius:6px;"
|
||||
"white-space:pre-wrap;word-break:break-all;font-size:11px;\">"
|
||||
f"{_html_escape(msg.blob)}</pre>"
|
||||
"<p style=\"color:#666;font-size:13px;\">Keep this email — you'll "
|
||||
"need the blob if you move to a new computer. Questions: just reply.</p>"
|
||||
"<p>— DataTools</p></body></html>"
|
||||
)
|
||||
|
||||
|
||||
def _rfc_addr(name: str, email: str) -> str:
|
||||
# Postmark accepts "Name <addr>" or just "addr". Quote names with
|
||||
# special chars; otherwise keep it readable in the inbox.
|
||||
if not name or "@" in name:
|
||||
return email
|
||||
if any(c in name for c in ',<>"'):
|
||||
name = name.replace('"', "").replace(",", "")
|
||||
return f"{name} <{email}>"
|
||||
|
||||
|
||||
def _html_escape(s: str) -> str:
|
||||
return (
|
||||
s.replace("&", "&")
|
||||
.replace("<", "<")
|
||||
.replace(">", ">")
|
||||
.replace('"', """)
|
||||
)
|
||||
19
server/app/main.py
Normal file
19
server/app/main.py
Normal file
@@ -0,0 +1,19 @@
|
||||
"""FastAPI entry point for the DataTools license server."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import FastAPI
|
||||
|
||||
from app.routes import internal, public, webhooks
|
||||
|
||||
app = FastAPI(
|
||||
title="DataTools License Server",
|
||||
version="0.1.0",
|
||||
docs_url=None,
|
||||
redoc_url=None,
|
||||
openapi_url=None,
|
||||
)
|
||||
|
||||
app.include_router(public.router)
|
||||
app.include_router(internal.router)
|
||||
app.include_router(webhooks.router)
|
||||
136
server/app/mint.py
Normal file
136
server/app/mint.py
Normal file
@@ -0,0 +1,136 @@
|
||||
"""Core mint + revoke logic.
|
||||
|
||||
Bridges the source-adapter layer (:mod:`app.adapters`) to the DB
|
||||
layer (:mod:`app.models`), reusing the desktop app's signing /
|
||||
encoding primitives from ``datatools_license.crypto`` so blobs minted
|
||||
here verify against the same embedded pubkey on the buyer's machine.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.adapters.base import SaleEvent
|
||||
from app.config import get_settings
|
||||
from app.models import License
|
||||
|
||||
|
||||
def _init_key_env() -> None:
|
||||
"""Resolve secret-file pointers into env vars before importing crypto.
|
||||
|
||||
``datatools_license.crypto`` looks for ``DATATOOLS_LICENSE_PRIVKEY``
|
||||
/ ``DATATOOLS_LICENSE_PUBKEY`` in ``os.environ``. When those come
|
||||
from secret files (``*_FILE`` env vars), we read them once at
|
||||
module import and stash so crypto can pick them up without
|
||||
changes.
|
||||
"""
|
||||
settings = get_settings()
|
||||
priv = settings.resolve_license_privkey()
|
||||
if priv:
|
||||
os.environ.setdefault("DATATOOLS_LICENSE_PRIVKEY", priv)
|
||||
pub = settings.license_pubkey_hex
|
||||
if pub:
|
||||
os.environ.setdefault("DATATOOLS_LICENSE_PUBKEY", pub)
|
||||
|
||||
|
||||
_init_key_env()
|
||||
|
||||
# Imported after env init so the crypto module reads the correct key.
|
||||
from datatools_license.crypto import encode_blob, sign # noqa: E402
|
||||
from datatools_license.features import all_features_for_tier # noqa: E402
|
||||
from datatools_license.schema import ( # noqa: E402
|
||||
License as LicenseDataclass,
|
||||
Tier,
|
||||
_utcnow_iso,
|
||||
default_expiry_iso,
|
||||
)
|
||||
|
||||
|
||||
def _generate_license_key(tier: str) -> str:
|
||||
rid = uuid.uuid4().hex
|
||||
return f"DT1-{tier.upper()}-{rid[:8]}-{rid[8:16]}"
|
||||
|
||||
|
||||
def _iso_to_dt(iso: str) -> datetime:
|
||||
return datetime.fromisoformat(iso.replace("Z", "+00:00"))
|
||||
|
||||
|
||||
def mint_from_sale(session: Session, sale: SaleEvent) -> License:
|
||||
"""Idempotently mint a license for *sale*.
|
||||
|
||||
If a row with the same ``(source, source_order_id)`` already
|
||||
exists, return it untouched — Gumroad retrying a webhook does not
|
||||
produce a second blob with a different signature. Manual mints
|
||||
(``source_order_id is None``) skip the dedup check and always
|
||||
produce a new row.
|
||||
"""
|
||||
if sale.source_order_id is not None:
|
||||
existing = session.execute(
|
||||
select(License).where(
|
||||
License.source == sale.source,
|
||||
License.source_order_id == sale.source_order_id,
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
return existing
|
||||
|
||||
tier_enum = Tier(sale.tier)
|
||||
license_key = _generate_license_key(sale.tier)
|
||||
issued_iso = _utcnow_iso()
|
||||
expires_iso = default_expiry_iso(years=sale.years)
|
||||
|
||||
unsigned = LicenseDataclass(
|
||||
name=sale.buyer_name,
|
||||
email=sale.buyer_email,
|
||||
license_key=license_key,
|
||||
tier=tier_enum,
|
||||
features=all_features_for_tier(tier_enum),
|
||||
issued_at=issued_iso,
|
||||
expires_at=expires_iso,
|
||||
signature="",
|
||||
)
|
||||
signature = sign(unsigned.to_canonical_dict())
|
||||
payload = unsigned.to_canonical_dict()
|
||||
payload["signature"] = signature
|
||||
blob = encode_blob(payload)
|
||||
|
||||
row = License(
|
||||
license_key=license_key,
|
||||
name=sale.buyer_name,
|
||||
email=sale.buyer_email,
|
||||
tier=sale.tier,
|
||||
issued_at=_iso_to_dt(issued_iso),
|
||||
expires_at=_iso_to_dt(expires_iso),
|
||||
blob=blob,
|
||||
source=sale.source,
|
||||
source_order_id=sale.source_order_id,
|
||||
promotion=sale.promotion,
|
||||
amount_paid=sale.amount_paid,
|
||||
currency=sale.currency,
|
||||
notes=sale.notes,
|
||||
)
|
||||
session.add(row)
|
||||
session.flush()
|
||||
return row
|
||||
|
||||
|
||||
def revoke_license(
|
||||
session: Session,
|
||||
*,
|
||||
license_key: str,
|
||||
reason: Optional[str] = None,
|
||||
) -> Optional[License]:
|
||||
row = session.get(License, license_key)
|
||||
if row is None:
|
||||
return None
|
||||
row.revoked_at = datetime.now(timezone.utc)
|
||||
if reason:
|
||||
suffix = f"\nRevoked: {reason}"
|
||||
row.notes = ((row.notes or "") + suffix).strip()
|
||||
return row
|
||||
97
server/app/models.py
Normal file
97
server/app/models.py
Normal file
@@ -0,0 +1,97 @@
|
||||
"""ORM models for the licenses + gumroad_events tables.
|
||||
|
||||
Schema mirrors ``docs/LICENSE-SERVER.md``, generalized so any
|
||||
``source`` can populate it. The ``(source, source_order_id)``
|
||||
composite uniqueness key gives idempotent webhook retries — a
|
||||
storefront firing the same sale twice maps to the same row.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy import (
|
||||
JSON,
|
||||
BigInteger,
|
||||
DateTime,
|
||||
Index,
|
||||
Integer,
|
||||
Numeric,
|
||||
String,
|
||||
UniqueConstraint,
|
||||
func,
|
||||
text,
|
||||
)
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
# JSONB on Postgres (indexable, queryable), plain JSON elsewhere
|
||||
# (SQLite for tests). Same Python interface either way.
|
||||
_JSON_TYPE = JSON().with_variant(JSONB(), "postgresql")
|
||||
|
||||
# SQLite only auto-increments INTEGER PRIMARY KEY (not BIGINT).
|
||||
# Postgres can autoincrement either, so the variant keeps the
|
||||
# production migration on BigInteger while tests use Integer.
|
||||
_PK_TYPE = BigInteger().with_variant(Integer(), "sqlite")
|
||||
|
||||
from app.db import Base
|
||||
|
||||
|
||||
class License(Base):
|
||||
__tablename__ = "licenses"
|
||||
|
||||
license_key: Mapped[str] = mapped_column(String, primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String, nullable=False)
|
||||
email: Mapped[str] = mapped_column(String, nullable=False)
|
||||
tier: Mapped[str] = mapped_column(String, nullable=False)
|
||||
issued_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
blob: Mapped[str] = mapped_column(String, nullable=False)
|
||||
|
||||
source: Mapped[str] = mapped_column(String, nullable=False)
|
||||
source_order_id: Mapped[Optional[str]] = mapped_column(String, nullable=True)
|
||||
promotion: Mapped[Optional[str]] = mapped_column(String, nullable=True)
|
||||
amount_paid: Mapped[Optional[float]] = mapped_column(Numeric(10, 2), nullable=True)
|
||||
currency: Mapped[Optional[str]] = mapped_column(String(3), nullable=True, server_default=text("'USD'"))
|
||||
|
||||
revoked_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
notes: Mapped[Optional[str]] = mapped_column(String, nullable=True)
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=func.now(),
|
||||
onupdate=func.now(),
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint("source", "source_order_id", name="uq_licenses_source_order"),
|
||||
Index("ix_licenses_email_lower", func.lower(text("email"))),
|
||||
Index("ix_licenses_expires_active", "expires_at", postgresql_where=text("revoked_at IS NULL")),
|
||||
)
|
||||
|
||||
|
||||
class GumroadEvent(Base):
|
||||
"""Append-only audit log of every webhook delivery.
|
||||
|
||||
Stored regardless of processing outcome so we can replay failed
|
||||
events, investigate disputes, and reconstruct the customer
|
||||
record if the ``licenses`` table is ever corrupted.
|
||||
"""
|
||||
|
||||
__tablename__ = "gumroad_events"
|
||||
|
||||
id: Mapped[int] = mapped_column(_PK_TYPE, primary_key=True, autoincrement=True)
|
||||
received_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
|
||||
event_type: Mapped[str] = mapped_column(String, nullable=False)
|
||||
order_id: Mapped[Optional[str]] = mapped_column(String, nullable=True)
|
||||
raw_payload: Mapped[dict] = mapped_column(_JSON_TYPE, nullable=False)
|
||||
processed: Mapped[bool] = mapped_column(server_default=text("false"), nullable=False)
|
||||
error: Mapped[Optional[str]] = mapped_column(String, nullable=True)
|
||||
|
||||
__table_args__ = (
|
||||
Index("ix_gumroad_events_order_id", "order_id"),
|
||||
Index("ix_gumroad_events_unprocessed", "received_at", postgresql_where=text("processed = false")),
|
||||
)
|
||||
71
server/app/products.py
Normal file
71
server/app/products.py
Normal file
@@ -0,0 +1,71 @@
|
||||
"""Storefront product → license tier mapping.
|
||||
|
||||
The mapping lives in ``server/config/products.yaml`` (gitignored
|
||||
for secrets it isn't — it's a routine catalog file) so adding a
|
||||
new SKU is one yaml edit plus a container restart. The lookup is
|
||||
``(source, product_id) -> (tier, years)``.
|
||||
|
||||
Cached at module import. The runtime cost of reloading on every
|
||||
webhook would be trivial, but caching keeps the hot path
|
||||
allocation-free and makes the "edit yaml, restart api" idiom
|
||||
explicit — operators always know exactly when their changes go
|
||||
live.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ProductMapping:
|
||||
tier: str
|
||||
years: int
|
||||
|
||||
|
||||
def _config_path() -> Path:
|
||||
"""Resolve the products config.
|
||||
|
||||
Container layout puts the config at ``/app/config/products.yaml``
|
||||
(the Dockerfile COPYs ``server/config`` to ``/app/config``).
|
||||
For local pytest runs we walk up from this file to ``server/``.
|
||||
"""
|
||||
in_container = Path("/app/config/products.yaml")
|
||||
if in_container.exists():
|
||||
return in_container
|
||||
return Path(__file__).resolve().parent.parent / "config" / "products.yaml"
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _table() -> dict[tuple[str, str], ProductMapping]:
|
||||
raw = yaml.safe_load(_config_path().read_text(encoding="utf-8")) or {}
|
||||
table: dict[tuple[str, str], ProductMapping] = {}
|
||||
for source, entries in raw.items():
|
||||
for entry in entries or []:
|
||||
key = (source, str(entry["product_id"]))
|
||||
table[key] = ProductMapping(
|
||||
tier=entry["tier"],
|
||||
years=int(entry.get("years", 1)),
|
||||
)
|
||||
return table
|
||||
|
||||
|
||||
def lookup(source: str, product_id: str) -> Optional[ProductMapping]:
|
||||
"""Return the mapping for *(source, product_id)*, or None if unmapped.
|
||||
|
||||
Returning None (rather than raising) lets the webhook layer
|
||||
decide whether to surface the failure as an audit row vs a
|
||||
user-visible error — we want unmapped sales to be logged, not
|
||||
to crash the handler and trigger Gumroad retry storms.
|
||||
"""
|
||||
return _table().get((source, product_id))
|
||||
|
||||
|
||||
def reload_for_tests() -> None:
|
||||
"""Drop the cache. Tests that mutate the yaml call this."""
|
||||
_table.cache_clear()
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user