added a py file to audit the folder reorganization and help us fix the files we couldnt parse

fccs_report_reorganize.py

@@ -0,0 +1,155 @@
+"""
+Utility: Analyze the results of fccs_reorganize.py.
+
+Internal-use console report. Walks the configured output_dir looking for
+_unparsed folders (files the reorganizer couldn't match against a folder
+template) and summarizes what it finds, so gaps in fccs_folders.txt are easy
+to spot.
+
+Because unparsed files keep their original export filename
+({drawer}_{client}_{folder}_{MM-DD-YYYY}_{doc}.ext), the folder field can
+still be recovered from the name. The report aggregates those folder fields
+(with years generalized back to YYYY) into suggested template lines you can
+paste straight into fccs_folders.txt, then re-run the reorganizer.
+
+Usage:
+  python fccs_report_reorganize.py [--config path\\to\\config.ini]
+"""
+
+import os
+import re
+import sys
+
+from fccs_config import parse_args, load_config
+
+# Creation-date field that terminates the folder portion of the filename.
+_DATE_RE = re.compile(r"_(\d{2}-\d{2}-\d{4})_")
+# A standalone year inside a folder name, e.g. "2013 Tax Documents".
+_YEAR_RE = re.compile(r"\b(19|20)\d{2}\b")
+
+UNPARSED_DIRNAME = "_unparsed"
+
+
+def folder_field_from_filename(filename):
+    """Recover the folder field from an unparsed export filename, or None.
+
+    Format: {drawer}_{client}_{folder}_{MM-DD-YYYY}_{doc}.ext
+    Drawer and client are the first two underscore tokens (client names never
+    contain underscores); the folder field runs from there to the first
+    creation-date anchor.
+    """
+    stem = os.path.splitext(filename)[0]
+    parts = stem.split("_", 2)
+    if len(parts) < 3:
+        return None
+    rest = parts[2]  # folder + date + doc
+
+    m = _DATE_RE.search("_" + rest)
+    if not m:
+        return None
+    folder = rest[: max(m.start() - 1, 0)].strip("_").strip()
+    return folder or None
+
+
+def suggest_template(folder_field):
+    """Generalize a concrete folder field into an fccs_folders.txt line.
+
+    e.g. "2013 Tax Documents" -> "YYYY Tax Documents". Fields without a year
+    are suggested as-is (static folders).
+    """
+    return _YEAR_RE.sub("YYYY", folder_field)
+
+
+def main():
+    args = parse_args()
+    cfg = load_config(args.config)
+    output_dir = cfg.get("paths", "output_dir")
+
+    if not os.path.isdir(output_dir):
+        print(f"ERROR: output directory not found: {output_dir}")
+        sys.exit(1)
+
+    clients_total = 0
+    clients_with_unparsed = []   # (client_name, [filenames])
+    top_level_unparsed = []      # filenames in output/_unparsed
+
+    for name in sorted(os.listdir(output_dir)):
+        path = os.path.join(output_dir, name)
+        if not os.path.isdir(path):
+            continue
+        if name == UNPARSED_DIRNAME:
+            top_level_unparsed = sorted(
+                f for f in os.listdir(path)
+                if os.path.isfile(os.path.join(path, f))
+            )
+            continue
+
+        clients_total += 1
+        up = os.path.join(path, UNPARSED_DIRNAME)
+        if os.path.isdir(up):
+            files = sorted(
+                f for f in os.listdir(up)
+                if os.path.isfile(os.path.join(up, f))
+            )
+            if files:
+                clients_with_unparsed.append((name, files))
+
+    print("=" * 64)
+    print("REORGANIZE RESULTS REPORT")
+    print(f"Output directory : {output_dir}")
+    print(f"Client folders   : {clients_total}")
+    print(f"With _unparsed   : {len(clients_with_unparsed)}")
+    print(f"Top-level _unparsed (no client recovered): {len(top_level_unparsed)}")
+    print("=" * 64)
+
+    if not clients_with_unparsed and not top_level_unparsed:
+        print("CLEAN — no _unparsed folders found. All files matched a "
+              "folder template.")
+        return
+
+    # Aggregate folder fields across every unparsed file to point at the
+    # template gaps directly.
+    suggestions = {}   # suggested template line -> file count
+    unrecovered = 0    # unparsed files whose folder field couldn't be read
+
+    print()
+    for client, files in clients_with_unparsed:
+        print(f"{client}  ({len(files)} unparsed)")
+        for f in files:
+            field = folder_field_from_filename(f)
+            if field:
+                key = suggest_template(field)
+                suggestions[key] = suggestions.get(key, 0) + 1
+            else:
+                unrecovered += 1
+            print(f"    {f}")
+        print()
+
+    if top_level_unparsed:
+        print(f"{UNPARSED_DIRNAME}/  (top level — client unknown, "
+              f"{len(top_level_unparsed)} files)")
+        for f in top_level_unparsed:
+            field = folder_field_from_filename(f)
+            if field:
+                key = suggest_template(field)
+                suggestions[key] = suggestions.get(key, 0) + 1
+            else:
+                unrecovered += 1
+            print(f"    {f}")
+        print()
+
+    if suggestions:
+        print("-" * 64)
+        print("Possible missing folder templates (add to fccs_folders.txt,")
+        print("then re-run fccs_reorganize.py):")
+        for tmpl, count in sorted(suggestions.items(),
+                                  key=lambda kv: (-kv[1], kv[0])):
+            print(f"  {count:4d} x  {tmpl}")
+    if unrecovered:
+        print(f"\n{unrecovered} unparsed file(s) had no recoverable folder "
+              "field (no date anchor in the name) — likely oddball names, "
+              "not template gaps.")
+
+
+if __name__ == "__main__":
+    main()