diff --git a/README.md b/README.md index 439802b..68d9c04 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ Extracts and organizes documents from FileCabinet CS (Thomson Reuters) using GUI | `fccs_verify.py` | Step 4 (optional): Compare manifests against exported files | | `fccs_check.py` | Utility: Interactively check if specific drawers exported completely | | `fccs_report.py` | Utility: Generate a clean client-facing HTML progress report (issues only) | +| `fccs_report_reorganize.py` | Utility: Console report of `_unparsed` leftovers after reorganizing, with suggested missing folder templates | | `fccs_dump_controls.py` | Utility: Dump control identifiers of an on-screen dialog | ## Setup Per Engagement @@ -100,6 +101,14 @@ Files that can't be fully parsed still keep their client: the drawer ID and clie Folder names are sanitized for Windows before use — trailing spaces and periods are stripped from each path component, since Windows can't create a directory ending in a space or dot (e.g. a client named `FARR GROUP, P.L.` becomes `FARR GROUP, P.L`). Reorganization is also resilient per file: if one file can't be placed for any reason, the error is logged and counted (reported as `errored` in the summary) and the run continues with the rest rather than aborting. +**Checking for folder template gaps:** After a reorganize run, get a quick internal summary of what didn't parse: + +``` +python fccs_report_reorganize.py +``` + +This walks `output_dir`, lists every client that has an `_unparsed` subfolder (plus the top-level `_unparsed`), and — because unparsed files keep their original export filename — recovers the folder field from each name and aggregates them into **suggested template lines** (years generalized to `YYYY`) that can be pasted into `fccs_folders.txt`. Add the missing templates and re-run `fccs_reorganize.py`. Console-only output; unparsed files with no recoverable folder field are counted separately (oddball names, not template gaps). + ### Step 4 (optional): Verify Export Completeness During export, each drawer's document list is captured from the FCCS dialog and saved as a manifest (in `manifest_dir`). These tools compare the manifests against the files actually in the export folder to confirm nothing was missed. diff --git a/fccs_report_reorganize.py b/fccs_report_reorganize.py new file mode 100644 index 0000000..af230e1 --- /dev/null +++ b/fccs_report_reorganize.py @@ -0,0 +1,155 @@ +""" +Utility: Analyze the results of fccs_reorganize.py. + +Internal-use console report. Walks the configured output_dir looking for +_unparsed folders (files the reorganizer couldn't match against a folder +template) and summarizes what it finds, so gaps in fccs_folders.txt are easy +to spot. + +Because unparsed files keep their original export filename +({drawer}_{client}_{folder}_{MM-DD-YYYY}_{doc}.ext), the folder field can +still be recovered from the name. The report aggregates those folder fields +(with years generalized back to YYYY) into suggested template lines you can +paste straight into fccs_folders.txt, then re-run the reorganizer. + +Usage: + python fccs_report_reorganize.py [--config path\\to\\config.ini] +""" + +import os +import re +import sys + +from fccs_config import parse_args, load_config + +# Creation-date field that terminates the folder portion of the filename. +_DATE_RE = re.compile(r"_(\d{2}-\d{2}-\d{4})_") +# A standalone year inside a folder name, e.g. "2013 Tax Documents". +_YEAR_RE = re.compile(r"\b(19|20)\d{2}\b") + +UNPARSED_DIRNAME = "_unparsed" + + +def folder_field_from_filename(filename): + """Recover the folder field from an unparsed export filename, or None. + + Format: {drawer}_{client}_{folder}_{MM-DD-YYYY}_{doc}.ext + Drawer and client are the first two underscore tokens (client names never + contain underscores); the folder field runs from there to the first + creation-date anchor. + """ + stem = os.path.splitext(filename)[0] + parts = stem.split("_", 2) + if len(parts) < 3: + return None + rest = parts[2] # folder + date + doc + + m = _DATE_RE.search("_" + rest) + if not m: + return None + folder = rest[: max(m.start() - 1, 0)].strip("_").strip() + return folder or None + + +def suggest_template(folder_field): + """Generalize a concrete folder field into an fccs_folders.txt line. + + e.g. "2013 Tax Documents" -> "YYYY Tax Documents". Fields without a year + are suggested as-is (static folders). + """ + return _YEAR_RE.sub("YYYY", folder_field) + + +def main(): + args = parse_args() + cfg = load_config(args.config) + output_dir = cfg.get("paths", "output_dir") + + if not os.path.isdir(output_dir): + print(f"ERROR: output directory not found: {output_dir}") + sys.exit(1) + + clients_total = 0 + clients_with_unparsed = [] # (client_name, [filenames]) + top_level_unparsed = [] # filenames in output/_unparsed + + for name in sorted(os.listdir(output_dir)): + path = os.path.join(output_dir, name) + if not os.path.isdir(path): + continue + if name == UNPARSED_DIRNAME: + top_level_unparsed = sorted( + f for f in os.listdir(path) + if os.path.isfile(os.path.join(path, f)) + ) + continue + + clients_total += 1 + up = os.path.join(path, UNPARSED_DIRNAME) + if os.path.isdir(up): + files = sorted( + f for f in os.listdir(up) + if os.path.isfile(os.path.join(up, f)) + ) + if files: + clients_with_unparsed.append((name, files)) + + print("=" * 64) + print("REORGANIZE RESULTS REPORT") + print(f"Output directory : {output_dir}") + print(f"Client folders : {clients_total}") + print(f"With _unparsed : {len(clients_with_unparsed)}") + print(f"Top-level _unparsed (no client recovered): {len(top_level_unparsed)}") + print("=" * 64) + + if not clients_with_unparsed and not top_level_unparsed: + print("CLEAN — no _unparsed folders found. All files matched a " + "folder template.") + return + + # Aggregate folder fields across every unparsed file to point at the + # template gaps directly. + suggestions = {} # suggested template line -> file count + unrecovered = 0 # unparsed files whose folder field couldn't be read + + print() + for client, files in clients_with_unparsed: + print(f"{client} ({len(files)} unparsed)") + for f in files: + field = folder_field_from_filename(f) + if field: + key = suggest_template(field) + suggestions[key] = suggestions.get(key, 0) + 1 + else: + unrecovered += 1 + print(f" {f}") + print() + + if top_level_unparsed: + print(f"{UNPARSED_DIRNAME}/ (top level — client unknown, " + f"{len(top_level_unparsed)} files)") + for f in top_level_unparsed: + field = folder_field_from_filename(f) + if field: + key = suggest_template(field) + suggestions[key] = suggestions.get(key, 0) + 1 + else: + unrecovered += 1 + print(f" {f}") + print() + + if suggestions: + print("-" * 64) + print("Possible missing folder templates (add to fccs_folders.txt,") + print("then re-run fccs_reorganize.py):") + for tmpl, count in sorted(suggestions.items(), + key=lambda kv: (-kv[1], kv[0])): + print(f" {count:4d} x {tmpl}") + if unrecovered: + print(f"\n{unrecovered} unparsed file(s) had no recoverable folder " + "field (no date anchor in the name) — likely oddball names, " + "not template gaps.") + + +if __name__ == "__main__": + main()