updated our verify logic

fccs_verify.py

@@ -1,35 +1,24 @@
 """
 Step 4 (optional): Verify exported files against manifests.
 
-Compares the per-drawer manifests captured during export (Step 2)
-against the actual files in the export directory to identify
-missing or extra files.
+Batch check across EVERY drawer that has a manifest: compares each drawer's
+manifest (captured during Step 2) against the files in the export directory and
+reports which drawers are complete vs missing documents.
+
+Matching is document-level and identical to fccs_check.py (via
+fccs_config.evaluate_drawer): it accounts for page-splitting (a document
+exported as 'Name Page 1', 'Name Page 2', ... counts as present) and for
+filename sanitization (titles with characters illegal in filenames still
+match). Use fccs_check.py to spot-check individual drawers interactively.
 """
 
 import os
 import sys
 
-from fccs_config import parse_args, load_config, make_logger
-
-
-def load_manifest(path):
-    """Load a manifest file and return one row per document.
-
-    Handles both formats:
-      - New: one document per line (tab-separated columns).
-      - Old: a raw ListView dump led by 'List1', then row-major cells with
-        3 columns per document (Drawer ID, Page Title, Application). We
-        reshape it so the document count is correct.
-    """
-    with open(path, "r", encoding="utf-8") as f:
-        lines = [line.rstrip("\n") for line in f if line.strip()]
-
-    if lines and lines[0].strip() == "List1":
-        cells = lines[1:]
-        rows = [cells[i:i + 3] for i in range(0, len(cells), 3)]
-        return [r for r in rows if len(r) == 3]
-
-    return [line.split("\t") for line in lines]
+from fccs_config import (
+    parse_args, load_config, make_logger,
+    evaluate_drawer, index_files_by_drawer,
+)
 
 
 def main():
@@ -49,77 +38,56 @@ def main():
         log(f"ERROR: export directory not found: {export_dir}")
         sys.exit(1)
 
-    # Load all manifests
-    manifests = {}
-    for fname in sorted(os.listdir(manifest_dir)):
-        if fname.endswith(".txt"):
-            drawer_id = os.path.splitext(fname)[0]
-            items = load_manifest(os.path.join(manifest_dir, fname))
-            manifests[drawer_id] = items
-
-    if not manifests:
+    drawer_ids = sorted(
+        os.path.splitext(f)[0]
+        for f in os.listdir(manifest_dir)
+        if f.endswith(".txt")
+    )
+    if not drawer_ids:
         log("No manifest files found.")
         sys.exit(1)
 
-    # Index exported files by drawer ID prefix
-    exported_by_drawer = {}
-    all_exported = [
-        f for f in os.listdir(export_dir)
-        if os.path.isfile(os.path.join(export_dir, f))
-    ]
-
-    for fname in all_exported:
-        sep = fname.find("_")
-        if sep != -1:
-            did = fname[:sep]
-            exported_by_drawer.setdefault(did, []).append(fname)
+    files_by_drawer = index_files_by_drawer(export_dir)
 
     log("=" * 60)
     log("Export Verification Report")
     log("=" * 60)
-    log(f"Manifests loaded : {len(manifests)} drawers")
-    log(f"Exported files   : {len(all_exported)} total")
+    log(f"Manifests loaded : {len(drawer_ids)} drawers")
     log("")
 
-    total_expected = 0
-    total_actual = 0
-    mismatched = []
+    complete_ids = []
+    incomplete = []   # (drawer_id, missing_list)
 
-    for drawer_id in sorted(manifests):
-        expected = manifests[drawer_id]
-        actual = exported_by_drawer.get(drawer_id, [])
-        n_expected = len(expected)
-        n_actual = len(actual)
-        total_expected += n_expected
-        total_actual += n_actual
-
-        if n_expected == n_actual:
-            log(f"  {drawer_id}: OK ({n_actual} files)")
+    for drawer_id in drawer_ids:
+        r = evaluate_drawer(drawer_id, files_by_drawer.get(drawer_id, []),
+                            manifest_dir)
+        n_expected = len(r["expected"])
+        missing = r["missing"]
+        if not missing:
+            complete_ids.append(drawer_id)
+            log(f"  {drawer_id}: OK ({n_expected} docs, {r['file_count']} files)")
         else:
-            diff = n_actual - n_expected
-            sign = "+" if diff > 0 else ""
-            log(f"  {drawer_id}: MISMATCH — expected {n_expected}, "
-                f"got {n_actual} ({sign}{diff})")
-            mismatched.append(drawer_id)
+            incomplete.append((drawer_id, missing))
+            log(f"  {drawer_id}: INCOMPLETE — {len(missing)}/{n_expected} "
+                f"document(s) missing:")
+            for m in missing:
+                log(f"       - {m}")
 
-    # Check for exported files with no manifest
-    orphan_drawers = set(exported_by_drawer.keys()) - set(manifests.keys())
-    orphan_count = sum(len(exported_by_drawer[d]) for d in orphan_drawers)
+    # Exported files whose drawer has no manifest at all.
+    orphan_drawers = sorted(set(files_by_drawer) - set(drawer_ids))
+    orphan_count = sum(len(files_by_drawer[d]) for d in orphan_drawers)
 
     log("")
     log("-" * 60)
-    log(f"Expected total   : {total_expected}")
-    log(f"Actual total     : {total_actual}")
-
+    log(f"Complete drawers   : {len(complete_ids)}")
+    log(f"Incomplete drawers : {len(incomplete)}")
+    if incomplete:
+        log(f"  Incomplete IDs: {', '.join(d for d, _ in incomplete)}")
     if orphan_drawers:
-        log(f"No manifest for  : {', '.join(sorted(orphan_drawers))} "
+        log(f"No manifest for    : {', '.join(orphan_drawers)} "
             f"({orphan_count} files)")
-
-    if mismatched:
-        log(f"Mismatched drawers: {', '.join(mismatched)}")
-    elif not orphan_drawers:
-        log("All drawers match.")
-
+    if not incomplete and not orphan_drawers:
+        log("All drawers complete.")
     log("=" * 60)