diff --git a/README.md b/README.md index 3d9e856..7d1b134 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ Extracts and organizes documents from FileCabinet CS (Thomson Reuters) using GUI | `fccs_scan.py` | Step 1: Scan backup directory for drawer IDs | | `fccs_export.py` | Step 2: Automate FCCS GUI to export all drawers | | `fccs_reorganize.py` | Step 3: Parse filenames and rebuild folder structure | +| `fccs_verify.py` | Step 4 (optional): Compare manifests against exported files | ## Setup Per Engagement @@ -47,6 +48,14 @@ Requires FCCS to be open with export destination already configured. Automates t - **Screenshots** -- captures failure states for diagnosis - **Defensive** -- one bad drawer won't crash the entire run +### Step 2b (optional): Verify Export Completeness + +``` +python fccs_verify.py +``` + +During export, each drawer's document list is captured from the FCCS dialog and saved as a manifest. This script compares those manifests against the actual exported files to flag any drawers with missing or extra files. + ### Step 3: Reorganize Files ``` diff --git a/config.ini b/config.ini index bb7b24d..5118c90 100644 --- a/config.ini +++ b/config.ini @@ -7,6 +7,7 @@ drawer_id_file = C:\Migration\drawer_ids.txt completed_file = C:\Migration\completed.txt log_file = C:\Migration\run_log.txt screenshot_dir = C:\Migration\screenshots +manifest_dir = C:\Migration\manifests folder_list = fccs_folders.txt [timeouts] diff --git a/fccs_export.py b/fccs_export.py index fa955dd..15cf6a3 100755 --- a/fccs_export.py +++ b/fccs_export.py @@ -129,19 +129,48 @@ def open_send_to_file(main, app, ctrl, timeouts): return dlg -def perform_export(dlg, ctrl, timeouts): - """Click Select -> then OK to start the export.""" - # Wait for the Select button to be ready — the dialog may still be - # populating its controls after appearing. +def read_manifest(dlg, log): + """Read the list of selected documents from the right-side ListView.""" + try: + lv = dlg.child_window(title="List1", class_name="SysListView32") + count = lv.item_count() + if count == 0: + log(" WARNING: ListView is empty after Select") + return [] + items = lv.texts() + log(f" Manifest captured: {count} documents selected") + return items + except Exception as e: + log(f" WARNING: could not read manifest from ListView: {e}") + return None + + +def save_manifest(drawer_id, items, manifest_dir, log): + """Save the list of expected export items to a manifest file.""" + os.makedirs(manifest_dir, exist_ok=True) + path = os.path.join(manifest_dir, f"{drawer_id}.txt") + with open(path, "w", encoding="utf-8") as f: + for row in items: + if isinstance(row, (list, tuple)): + f.write("\t".join(str(c) for c in row) + "\n") + else: + f.write(str(row) + "\n") + log(f" Manifest saved: {path}") + + +def perform_export(dlg, ctrl, timeouts, log): + """Click Select -> then OK to start the export. Returns manifest list.""" select_btn = dlg.child_window(title=ctrl["select_btn_title"], class_name="Button") select_btn.wait("visible ready enabled", timeout=timeouts["dialog"]) select_btn.click() - # Wait for the OK button to become ready — transferring files to the - # selected side can take a while on large drawers. ok_btn = dlg.child_window(title=ctrl["ok_btn_title"], class_name="Button") ok_btn.wait("visible ready enabled", timeout=timeouts["select"]) + + manifest = read_manifest(dlg, log) + ok_btn.click() + return manifest def dismiss_any_dialog(app, ctrl, log): @@ -228,7 +257,8 @@ def wait_for_export(app, ctrl, timeouts, log): return True -def export_drawer(app, main, drawer_id, ctrl, timeouts, screenshot_dir, log): +def export_drawer(app, main, drawer_id, ctrl, timeouts, screenshot_dir, + manifest_dir, log): """Full per-drawer sequence. Returns True on success.""" log(f"Drawer {drawer_id}: starting") @@ -248,9 +278,11 @@ def export_drawer(app, main, drawer_id, ctrl, timeouts, screenshot_dir, log): take_screenshot(main, drawer_id, "dialog_fail", screenshot_dir, log) return False - # 3 & 4. Select -> and OK + # 3 & 4. Select -> and OK (captures manifest before clicking OK) try: - perform_export(dlg, ctrl, timeouts) + manifest = perform_export(dlg, ctrl, timeouts, log) + if manifest is not None: + save_manifest(drawer_id, manifest, manifest_dir, log) except Exception as e: log(f" ERROR during export selection: {e}") take_screenshot(dlg, drawer_id, "export_fail", screenshot_dir, log) @@ -284,6 +316,7 @@ def main(): "drawer_id_file": cfg.get("paths", "drawer_id_file"), "completed_file": cfg.get("paths", "completed_file"), "screenshot_dir": cfg.get("paths", "screenshot_dir"), + "manifest_dir": cfg.get("paths", "manifest_dir"), } timeouts = { "nav": cfg.getfloat("timeouts", "nav_timeout"), @@ -339,7 +372,8 @@ def main(): log(f"--- [{i}/{len(pending)}] Drawer {drawer_id} ---") try: ok = export_drawer(app, main_win, drawer_id, ctrl, timeouts, - paths["screenshot_dir"], log) + paths["screenshot_dir"], + paths["manifest_dir"], log) except Exception as e: log(f" UNEXPECTED ERROR on drawer {drawer_id}: {e}") try: diff --git a/fccs_verify.py b/fccs_verify.py new file mode 100644 index 0000000..93c9009 --- /dev/null +++ b/fccs_verify.py @@ -0,0 +1,118 @@ +""" +Step 4 (optional): Verify exported files against manifests. + +Compares the per-drawer manifests captured during export (Step 2) +against the actual files in the export directory to identify +missing or extra files. +""" + +import os +import sys + +from fccs_config import parse_args, load_config, make_logger + + +def load_manifest(path): + """Load a manifest file and return the list of document rows.""" + items = [] + with open(path, "r", encoding="utf-8") as f: + for line in f: + line = line.rstrip("\n") + if line: + items.append(line.split("\t")) + return items + + +def main(): + args = parse_args() + cfg = load_config(args.config) + log = make_logger(cfg.get("paths", "log_file")) + + export_dir = cfg.get("paths", "export_dir") + manifest_dir = cfg.get("paths", "manifest_dir") + + if not os.path.isdir(manifest_dir): + log(f"ERROR: manifest directory not found: {manifest_dir}") + log("Run fccs_export.py first to generate manifests.") + sys.exit(1) + + if not os.path.isdir(export_dir): + log(f"ERROR: export directory not found: {export_dir}") + sys.exit(1) + + # Load all manifests + manifests = {} + for fname in sorted(os.listdir(manifest_dir)): + if fname.endswith(".txt"): + drawer_id = os.path.splitext(fname)[0] + items = load_manifest(os.path.join(manifest_dir, fname)) + manifests[drawer_id] = items + + if not manifests: + log("No manifest files found.") + sys.exit(1) + + # Index exported files by drawer ID prefix + exported_by_drawer = {} + all_exported = [ + f for f in os.listdir(export_dir) + if os.path.isfile(os.path.join(export_dir, f)) + ] + + for fname in all_exported: + sep = fname.find("_") + if sep != -1: + did = fname[:sep] + exported_by_drawer.setdefault(did, []).append(fname) + + log("=" * 60) + log("Export Verification Report") + log("=" * 60) + log(f"Manifests loaded : {len(manifests)} drawers") + log(f"Exported files : {len(all_exported)} total") + log("") + + total_expected = 0 + total_actual = 0 + mismatched = [] + + for drawer_id in sorted(manifests): + expected = manifests[drawer_id] + actual = exported_by_drawer.get(drawer_id, []) + n_expected = len(expected) + n_actual = len(actual) + total_expected += n_expected + total_actual += n_actual + + if n_expected == n_actual: + log(f" {drawer_id}: OK ({n_actual} files)") + else: + diff = n_actual - n_expected + sign = "+" if diff > 0 else "" + log(f" {drawer_id}: MISMATCH — expected {n_expected}, " + f"got {n_actual} ({sign}{diff})") + mismatched.append(drawer_id) + + # Check for exported files with no manifest + orphan_drawers = set(exported_by_drawer.keys()) - set(manifests.keys()) + orphan_count = sum(len(exported_by_drawer[d]) for d in orphan_drawers) + + log("") + log("-" * 60) + log(f"Expected total : {total_expected}") + log(f"Actual total : {total_actual}") + + if orphan_drawers: + log(f"No manifest for : {', '.join(sorted(orphan_drawers))} " + f"({orphan_count} files)") + + if mismatched: + log(f"Mismatched drawers: {', '.join(mismatched)}") + elif not orphan_drawers: + log("All drawers match.") + + log("=" * 60) + + +if __name__ == "__main__": + main()