will/FCCS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

updated our verify logic

Browse Source
This commit is contained in:
dat972
2026-07-11 12:27:40 -05:00
parent 2b04210a23
commit fec7745092
7 changed files with 230 additions and 183 deletions
Download Patch File Download Diff File Expand all files Collapse all files

115
fccs_check.py
View File

@@ -6,7 +6,7 @@ its manifest (the documents FCCS said it would export, captured during Step 2)
against the files actually sitting in the export folder, and reports any
missing documents.
Handles two quirks of the FCCS export:
Handles two quirks of the FCCS export (see fccs_config.evaluate_drawer):
1. Page-splitting: a single manifest document (e.g. "Donations") is exported
as one file if it's a single page, or as "Donations Page 1",
@@ -14,10 +14,11 @@ Handles two quirks of the FCCS export:
that one document being present.
2. Filename sanitization: FCCS strips characters that are illegal in Windows
filenames (: / \\ ? * " < > |) from document titles, so a manifest title
like 'US Tax Return (... 01:35PM)' won't match the exported file
character-for-character. Comparison is done on a normalized key
(lowercase, alphanumerics only) so these still match.
filenames (: / \\ ? * " < > |) from document titles, so comparison is done
on a normalized key so these still match.
fccs_verify.py applies this same logic in batch across every drawer; this tool
is for spot-checking specific drawers (e.g. ones the log marked failed).
USAGE
-----
@@ -29,94 +30,37 @@ import os
import re
import sys
from fccs_config import parse_args, load_config
from fccs_verify import load_manifest # reuse the both-format manifest loader
from fccs_config import (
parse_args, load_config, evaluate_drawer, index_files_by_drawer,
)
# Trailing " Page N" (optionally "Page N of M") appended to multi-page exports.
_PAGE_RE = re.compile(r"\s*Page\s+\d+(?:\s+of\s+\d+)?\s*$", re.IGNORECASE)
# Creation-date field ("_MM-DD-YYYY_") that precedes the document name.
_DATE_ANCHOR = re.compile(r"_\d{2}-\d{2}-\d{4}_")
def report_drawer(drawer_id, files, manifest_dir, out):
"""Evaluate one drawer and print a human-readable completeness report."""
r = evaluate_drawer(drawer_id, files, manifest_dir)
def match_key(name):
"""Normalize a document name for tolerant comparison.
Strips a trailing 'Page N' page-split suffix, then reduces to lowercase
alphanumerics so punctuation and filename-sanitization differences don't
cause false mismatches.
"""
base = _PAGE_RE.sub("", name)
return re.sub(r"[^a-z0-9]+", "", base.lower())
def manifest_doc_names(path, drawer_id):
"""Return the expected document (Page Title) names from a manifest file."""
rows = load_manifest(path)
names = []
for row in rows:
if len(row) >= 3 and row[0].strip() == drawer_id:
names.append(row[1]) # old format: DrawerID, PageTitle, Application
elif row:
names.append(row[0]) # new format: PageTitle, Application
return names
def exported_doc_name(filename):
"""Extract the document-name portion from an exported filename, or None.
Format: {drawer}_{client}_{folder}_{MM-DD-YYYY}_{docname}.ext
The creation-date field is a reliable anchor; the doc name follows the last
one (the client/folder fields don't carry an "_MM-DD-YYYY_" pattern).
"""
stem = os.path.splitext(filename)[0]
anchors = list(_DATE_ANCHOR.finditer(stem))
if not anchors:
return None
return stem[anchors[-1].end():]
def check_drawer(drawer_id, files, manifest_dir, out):
"""Report completeness of one drawer's export."""
manifest_path = os.path.join(manifest_dir, drawer_id + ".txt")
if not os.path.exists(manifest_path):
out("")
out(f"[{drawer_id}] NO MANIFEST at {manifest_path} — cannot verify "
out("")
if not r["has_manifest"]:
out(f"[{drawer_id}] NO MANIFEST at {r['manifest_path']} — cannot verify "
"(was this drawer exported by the tool?)")
return
expected = manifest_doc_names(manifest_path, drawer_id)
# Group exported files by normalized doc name; page-splits collapse together.
exported = {} # key -> list of full doc names (one entry per file/page)
unparsed = []
for f in files:
doc = exported_doc_name(f)
if doc is None:
unparsed.append(f)
continue
exported.setdefault(match_key(doc), []).append(doc)
missing = [name for name in expected if match_key(name) not in exported]
expected_keys = {match_key(n) for n in expected}
extras = [names[0] for k, names in exported.items() if k not in expected_keys]
out("")
expected = r["expected"]
missing = r["missing"]
out(f"[{drawer_id}] manifest lists {len(expected)} document(s); "
f"{len(files)} file(s) in export folder.")
f"{r['file_count']} file(s) in export folder.")
if missing:
out(f" INCOMPLETE — {len(missing)} document(s) missing from export:")
for m in missing:
out(f" - {m}")
else:
out(f" COMPLETE — all {len(expected)} manifest document(s) present.")
if extras:
out(f" Note: {len(extras)} exported document(s) not in the manifest:")
for e in extras:
if r["extras"]:
out(f" Note: {len(r['extras'])} exported document(s) not in the manifest:")
for e in r["extras"]:
out(f" - {e}")
if unparsed:
out(f" Note: {len(unparsed)} file(s) had no recognizable date anchor "
if r["unparsed"]:
out(f" Note: {r['unparsed']} file(s) had no recognizable date anchor "
"and were skipped.")
@@ -130,14 +74,7 @@ def main():
print(f"ERROR: export directory not found: {export_dir}")
sys.exit(1)
# Index every export file by its leading drawer-ID token (before first "_").
# The underscore boundary keeps clashing IDs separate (04289 vs 04289TS).
files_by_drawer = {}
for f in os.listdir(export_dir):
if not os.path.isfile(os.path.join(export_dir, f)):
continue
token = f.split("_", 1)[0]
files_by_drawer.setdefault(token, []).append(f)
files_by_drawer = index_files_by_drawer(export_dir)
print("FCCS export completeness check")
print(f" export folder : {export_dir}")
@@ -153,8 +90,8 @@ def main():
break
ids = [i for i in re.split(r"[\s,]+", raw) if i]
for drawer_id in ids:
check_drawer(drawer_id, files_by_drawer.get(drawer_id, []),
manifest_dir, print)
report_drawer(drawer_id, files_by_drawer.get(drawer_id, []),
manifest_dir, print)
if __name__ == "__main__":