added a check utility so we can scan the export folder and check if the folder that failed during the export run actually failed or managed to complete succesfully
This commit is contained in:
@@ -20,6 +20,7 @@ Extracts and organizes documents from FileCabinet CS (Thomson Reuters) using GUI
|
||||
| `fccs_export.py` | Step 2: Automate FCCS GUI to export all drawers |
|
||||
| `fccs_reorganize.py` | Step 3: Parse filenames and rebuild folder structure |
|
||||
| `fccs_verify.py` | Step 4 (optional): Compare manifests against exported files |
|
||||
| `fccs_check.py` | Utility: Interactively check if specific drawers exported completely |
|
||||
| `fccs_dump_controls.py` | Utility: Dump control identifiers of an on-screen dialog |
|
||||
|
||||
## Setup Per Engagement
|
||||
@@ -68,6 +69,14 @@ python fccs_verify.py
|
||||
|
||||
During export, each drawer's document list is captured from the FCCS dialog and saved as a manifest. This script compares those manifests against the actual exported files to flag any drawers with missing or extra files.
|
||||
|
||||
To spot-check specific drawers (e.g. ones the log marked failed, to see whether they actually finished exporting in the background), run:
|
||||
|
||||
```
|
||||
python fccs_check.py
|
||||
```
|
||||
|
||||
It prompts for one or more drawer IDs and reports, per drawer, which manifest documents are present vs missing. It accounts for FCCS page-splitting (a document exported as `Name Page 1`, `Name Page 2`, … counts as present) and for filename sanitization (titles containing characters illegal in filenames, like `:`, still match).
|
||||
|
||||
### Step 3: Reorganize Files
|
||||
|
||||
```
|
||||
|
||||
BIN
__pycache__/fccs_check.cpython-313.pyc
Normal file
BIN
__pycache__/fccs_check.cpython-313.pyc
Normal file
Binary file not shown.
Binary file not shown.
161
fccs_check.py
Normal file
161
fccs_check.py
Normal file
@@ -0,0 +1,161 @@
|
||||
"""
|
||||
Utility: Check whether specific drawers actually finished exporting.
|
||||
|
||||
Interactive — prompts for one or more drawer IDs, then for each drawer compares
|
||||
its manifest (the documents FCCS said it would export, captured during Step 2)
|
||||
against the files actually sitting in the export folder, and reports any
|
||||
missing documents.
|
||||
|
||||
Handles two quirks of the FCCS export:
|
||||
|
||||
1. Page-splitting: a single manifest document (e.g. "Donations") is exported
|
||||
as one file if it's a single page, or as "Donations Page 1",
|
||||
"Donations Page 2", ... for a multi-page document. All of those count as
|
||||
that one document being present.
|
||||
|
||||
2. Filename sanitization: FCCS strips characters that are illegal in Windows
|
||||
filenames (: / \\ ? * " < > |) from document titles, so a manifest title
|
||||
like 'US Tax Return (... 01:35PM)' won't match the exported file
|
||||
character-for-character. Comparison is done on a normalized key
|
||||
(lowercase, alphanumerics only) so these still match.
|
||||
|
||||
USAGE
|
||||
-----
|
||||
python fccs_check.py
|
||||
Drawer ID(s): 08097 18430
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from fccs_config import parse_args, load_config
|
||||
from fccs_verify import load_manifest # reuse the both-format manifest loader
|
||||
|
||||
|
||||
# Trailing " Page N" (optionally "Page N of M") appended to multi-page exports.
|
||||
_PAGE_RE = re.compile(r"\s*Page\s+\d+(?:\s+of\s+\d+)?\s*$", re.IGNORECASE)
|
||||
# Creation-date field ("_MM-DD-YYYY_") that precedes the document name.
|
||||
_DATE_ANCHOR = re.compile(r"_\d{2}-\d{2}-\d{4}_")
|
||||
|
||||
|
||||
def match_key(name):
|
||||
"""Normalize a document name for tolerant comparison.
|
||||
|
||||
Strips a trailing 'Page N' page-split suffix, then reduces to lowercase
|
||||
alphanumerics so punctuation and filename-sanitization differences don't
|
||||
cause false mismatches.
|
||||
"""
|
||||
base = _PAGE_RE.sub("", name)
|
||||
return re.sub(r"[^a-z0-9]+", "", base.lower())
|
||||
|
||||
|
||||
def manifest_doc_names(path, drawer_id):
|
||||
"""Return the expected document (Page Title) names from a manifest file."""
|
||||
rows = load_manifest(path)
|
||||
names = []
|
||||
for row in rows:
|
||||
if len(row) >= 3 and row[0].strip() == drawer_id:
|
||||
names.append(row[1]) # old format: DrawerID, PageTitle, Application
|
||||
elif row:
|
||||
names.append(row[0]) # new format: PageTitle, Application
|
||||
return names
|
||||
|
||||
|
||||
def exported_doc_name(filename):
|
||||
"""Extract the document-name portion from an exported filename, or None.
|
||||
|
||||
Format: {drawer}_{client}_{folder}_{MM-DD-YYYY}_{docname}.ext
|
||||
The creation-date field is a reliable anchor; the doc name follows the last
|
||||
one (the client/folder fields don't carry an "_MM-DD-YYYY_" pattern).
|
||||
"""
|
||||
stem = os.path.splitext(filename)[0]
|
||||
anchors = list(_DATE_ANCHOR.finditer(stem))
|
||||
if not anchors:
|
||||
return None
|
||||
return stem[anchors[-1].end():]
|
||||
|
||||
|
||||
def check_drawer(drawer_id, files, manifest_dir, out):
|
||||
"""Report completeness of one drawer's export."""
|
||||
manifest_path = os.path.join(manifest_dir, drawer_id + ".txt")
|
||||
if not os.path.exists(manifest_path):
|
||||
out("")
|
||||
out(f"[{drawer_id}] NO MANIFEST at {manifest_path} — cannot verify "
|
||||
"(was this drawer exported by the tool?)")
|
||||
return
|
||||
|
||||
expected = manifest_doc_names(manifest_path, drawer_id)
|
||||
|
||||
# Group exported files by normalized doc name; page-splits collapse together.
|
||||
exported = {} # key -> list of full doc names (one entry per file/page)
|
||||
unparsed = []
|
||||
for f in files:
|
||||
doc = exported_doc_name(f)
|
||||
if doc is None:
|
||||
unparsed.append(f)
|
||||
continue
|
||||
exported.setdefault(match_key(doc), []).append(doc)
|
||||
|
||||
missing = [name for name in expected if match_key(name) not in exported]
|
||||
|
||||
expected_keys = {match_key(n) for n in expected}
|
||||
extras = [names[0] for k, names in exported.items() if k not in expected_keys]
|
||||
|
||||
out("")
|
||||
out(f"[{drawer_id}] manifest lists {len(expected)} document(s); "
|
||||
f"{len(files)} file(s) in export folder.")
|
||||
if missing:
|
||||
out(f" INCOMPLETE — {len(missing)} document(s) missing from export:")
|
||||
for m in missing:
|
||||
out(f" - {m}")
|
||||
else:
|
||||
out(f" COMPLETE — all {len(expected)} manifest document(s) present.")
|
||||
if extras:
|
||||
out(f" Note: {len(extras)} exported document(s) not in the manifest:")
|
||||
for e in extras:
|
||||
out(f" - {e}")
|
||||
if unparsed:
|
||||
out(f" Note: {len(unparsed)} file(s) had no recognizable date anchor "
|
||||
"and were skipped.")
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
cfg = load_config(args.config)
|
||||
export_dir = cfg.get("paths", "export_dir")
|
||||
manifest_dir = cfg.get("paths", "manifest_dir")
|
||||
|
||||
if not os.path.isdir(export_dir):
|
||||
print(f"ERROR: export directory not found: {export_dir}")
|
||||
sys.exit(1)
|
||||
|
||||
# Index every export file by its leading drawer-ID token (before first "_").
|
||||
# The underscore boundary keeps clashing IDs separate (04289 vs 04289TS).
|
||||
files_by_drawer = {}
|
||||
for f in os.listdir(export_dir):
|
||||
if not os.path.isfile(os.path.join(export_dir, f)):
|
||||
continue
|
||||
token = f.split("_", 1)[0]
|
||||
files_by_drawer.setdefault(token, []).append(f)
|
||||
|
||||
print("FCCS export completeness check")
|
||||
print(f" export folder : {export_dir}")
|
||||
print(f" manifests : {manifest_dir}")
|
||||
print("Enter drawer ID(s) separated by spaces or commas (blank to quit).")
|
||||
|
||||
while True:
|
||||
try:
|
||||
raw = input("\nDrawer ID(s): ").strip()
|
||||
except EOFError:
|
||||
break
|
||||
if not raw:
|
||||
break
|
||||
ids = [i for i in re.split(r"[\s,]+", raw) if i]
|
||||
for drawer_id in ids:
|
||||
check_drawer(drawer_id, files_by_drawer.get(drawer_id, []),
|
||||
manifest_dir, print)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user