added a check utility so we can scan the export folder and check if the folder that failed during the export run actually failed or managed to complete succesfully

This commit is contained in:
2026-07-11 12:12:41 -05:00
parent bda12518dd
commit 2b04210a23
4 changed files with 170 additions and 0 deletions

View File

@@ -20,6 +20,7 @@ Extracts and organizes documents from FileCabinet CS (Thomson Reuters) using GUI
| `fccs_export.py` | Step 2: Automate FCCS GUI to export all drawers |
| `fccs_reorganize.py` | Step 3: Parse filenames and rebuild folder structure |
| `fccs_verify.py` | Step 4 (optional): Compare manifests against exported files |
| `fccs_check.py` | Utility: Interactively check if specific drawers exported completely |
| `fccs_dump_controls.py` | Utility: Dump control identifiers of an on-screen dialog |
## Setup Per Engagement
@@ -68,6 +69,14 @@ python fccs_verify.py
During export, each drawer's document list is captured from the FCCS dialog and saved as a manifest. This script compares those manifests against the actual exported files to flag any drawers with missing or extra files.
To spot-check specific drawers (e.g. ones the log marked failed, to see whether they actually finished exporting in the background), run:
```
python fccs_check.py
```
It prompts for one or more drawer IDs and reports, per drawer, which manifest documents are present vs missing. It accounts for FCCS page-splitting (a document exported as `Name Page 1`, `Name Page 2`, … counts as present) and for filename sanitization (titles containing characters illegal in filenames, like `:`, still match).
### Step 3: Reorganize Files
```

Binary file not shown.

161
fccs_check.py Normal file
View File

@@ -0,0 +1,161 @@
"""
Utility: Check whether specific drawers actually finished exporting.
Interactive — prompts for one or more drawer IDs, then for each drawer compares
its manifest (the documents FCCS said it would export, captured during Step 2)
against the files actually sitting in the export folder, and reports any
missing documents.
Handles two quirks of the FCCS export:
1. Page-splitting: a single manifest document (e.g. "Donations") is exported
as one file if it's a single page, or as "Donations Page 1",
"Donations Page 2", ... for a multi-page document. All of those count as
that one document being present.
2. Filename sanitization: FCCS strips characters that are illegal in Windows
filenames (: / \\ ? * " < > |) from document titles, so a manifest title
like 'US Tax Return (... 01:35PM)' won't match the exported file
character-for-character. Comparison is done on a normalized key
(lowercase, alphanumerics only) so these still match.
USAGE
-----
python fccs_check.py
Drawer ID(s): 08097 18430
"""
import os
import re
import sys
from fccs_config import parse_args, load_config
from fccs_verify import load_manifest # reuse the both-format manifest loader
# Trailing " Page N" (optionally "Page N of M") appended to multi-page exports.
_PAGE_RE = re.compile(r"\s*Page\s+\d+(?:\s+of\s+\d+)?\s*$", re.IGNORECASE)
# Creation-date field ("_MM-DD-YYYY_") that precedes the document name.
_DATE_ANCHOR = re.compile(r"_\d{2}-\d{2}-\d{4}_")
def match_key(name):
"""Normalize a document name for tolerant comparison.
Strips a trailing 'Page N' page-split suffix, then reduces to lowercase
alphanumerics so punctuation and filename-sanitization differences don't
cause false mismatches.
"""
base = _PAGE_RE.sub("", name)
return re.sub(r"[^a-z0-9]+", "", base.lower())
def manifest_doc_names(path, drawer_id):
"""Return the expected document (Page Title) names from a manifest file."""
rows = load_manifest(path)
names = []
for row in rows:
if len(row) >= 3 and row[0].strip() == drawer_id:
names.append(row[1]) # old format: DrawerID, PageTitle, Application
elif row:
names.append(row[0]) # new format: PageTitle, Application
return names
def exported_doc_name(filename):
"""Extract the document-name portion from an exported filename, or None.
Format: {drawer}_{client}_{folder}_{MM-DD-YYYY}_{docname}.ext
The creation-date field is a reliable anchor; the doc name follows the last
one (the client/folder fields don't carry an "_MM-DD-YYYY_" pattern).
"""
stem = os.path.splitext(filename)[0]
anchors = list(_DATE_ANCHOR.finditer(stem))
if not anchors:
return None
return stem[anchors[-1].end():]
def check_drawer(drawer_id, files, manifest_dir, out):
"""Report completeness of one drawer's export."""
manifest_path = os.path.join(manifest_dir, drawer_id + ".txt")
if not os.path.exists(manifest_path):
out("")
out(f"[{drawer_id}] NO MANIFEST at {manifest_path} — cannot verify "
"(was this drawer exported by the tool?)")
return
expected = manifest_doc_names(manifest_path, drawer_id)
# Group exported files by normalized doc name; page-splits collapse together.
exported = {} # key -> list of full doc names (one entry per file/page)
unparsed = []
for f in files:
doc = exported_doc_name(f)
if doc is None:
unparsed.append(f)
continue
exported.setdefault(match_key(doc), []).append(doc)
missing = [name for name in expected if match_key(name) not in exported]
expected_keys = {match_key(n) for n in expected}
extras = [names[0] for k, names in exported.items() if k not in expected_keys]
out("")
out(f"[{drawer_id}] manifest lists {len(expected)} document(s); "
f"{len(files)} file(s) in export folder.")
if missing:
out(f" INCOMPLETE — {len(missing)} document(s) missing from export:")
for m in missing:
out(f" - {m}")
else:
out(f" COMPLETE — all {len(expected)} manifest document(s) present.")
if extras:
out(f" Note: {len(extras)} exported document(s) not in the manifest:")
for e in extras:
out(f" - {e}")
if unparsed:
out(f" Note: {len(unparsed)} file(s) had no recognizable date anchor "
"and were skipped.")
def main():
args = parse_args()
cfg = load_config(args.config)
export_dir = cfg.get("paths", "export_dir")
manifest_dir = cfg.get("paths", "manifest_dir")
if not os.path.isdir(export_dir):
print(f"ERROR: export directory not found: {export_dir}")
sys.exit(1)
# Index every export file by its leading drawer-ID token (before first "_").
# The underscore boundary keeps clashing IDs separate (04289 vs 04289TS).
files_by_drawer = {}
for f in os.listdir(export_dir):
if not os.path.isfile(os.path.join(export_dir, f)):
continue
token = f.split("_", 1)[0]
files_by_drawer.setdefault(token, []).append(f)
print("FCCS export completeness check")
print(f" export folder : {export_dir}")
print(f" manifests : {manifest_dir}")
print("Enter drawer ID(s) separated by spaces or commas (blank to quit).")
while True:
try:
raw = input("\nDrawer ID(s): ").strip()
except EOFError:
break
if not raw:
break
ids = [i for i in re.split(r"[\s,]+", raw) if i]
for drawer_id in ids:
check_drawer(drawer_id, files_by_drawer.get(drawer_id, []),
manifest_dir, print)
if __name__ == "__main__":
main()