373 lines
13 KiB
Python
373 lines
13 KiB
Python
"""
|
|
Shared configuration, logging, and utilities for the FCCS extraction tool.
|
|
"""
|
|
|
|
import os
|
|
import re
|
|
import sys
|
|
import shutil
|
|
import argparse
|
|
import configparser
|
|
from datetime import datetime
|
|
|
|
DEFAULT_CONFIG = "config.ini"
|
|
TEMPLATE_CONFIG = "config.template.ini"
|
|
|
|
|
|
def ensure_from_template(path, template):
|
|
"""Create a per-machine working file from its git-tracked template.
|
|
|
|
If `path` doesn't exist but `template` does, copy template -> path and
|
|
announce it. Returns True if the file exists (or was just created).
|
|
Used for config.ini and fccs_folders.txt so local edits never drift the
|
|
repo — the working copies are git-ignored, only the templates are tracked.
|
|
"""
|
|
if os.path.exists(path):
|
|
return True
|
|
if os.path.exists(template):
|
|
shutil.copyfile(template, path)
|
|
print(f"Created {os.path.basename(path)} from "
|
|
f"{os.path.basename(template)}. Edit it for this machine: {path}")
|
|
return True
|
|
return False
|
|
|
|
|
|
def parse_args():
|
|
"""Parse the optional --config argument common to all scripts."""
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument(
|
|
"--config",
|
|
default=None,
|
|
help=f"Path to config file (default: {DEFAULT_CONFIG} in script directory)",
|
|
)
|
|
return parser.parse_args()
|
|
|
|
|
|
def load_config(path=None):
|
|
"""Read the config file and return a ConfigParser object.
|
|
|
|
Config is split into two files:
|
|
- config.template.ini : tracked in git, the pristine template.
|
|
- config.ini : git-ignored, this machine's working copy.
|
|
|
|
When no explicit --config path is given, the per-machine config.ini is used.
|
|
If it doesn't exist yet, it's created from config.template.ini so a freshly
|
|
cloned/pulled repo works out of the box without editing the tracked file.
|
|
Edit config.ini locally; your changes never drift the repo.
|
|
"""
|
|
if path is None:
|
|
script_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
|
|
path = os.path.join(script_dir, DEFAULT_CONFIG)
|
|
# Auto-create the per-machine config from the template on first run.
|
|
ensure_from_template(path, os.path.join(script_dir, TEMPLATE_CONFIG))
|
|
if not os.path.exists(path):
|
|
print(f"ERROR: config file not found: {path}")
|
|
print(f" Expected {DEFAULT_CONFIG} or a --config path "
|
|
f"(template: {TEMPLATE_CONFIG}).")
|
|
sys.exit(1)
|
|
cfg = configparser.ConfigParser()
|
|
cfg.read(path, encoding="utf-8")
|
|
return cfg
|
|
|
|
|
|
def make_logger(log_file):
|
|
"""Return a log() function that writes timestamped lines to console and file."""
|
|
def log(msg):
|
|
line = f"[{datetime.now():%Y-%m-%d %H:%M:%S}] {msg}"
|
|
print(line)
|
|
try:
|
|
with open(log_file, "a", encoding="utf-8") as f:
|
|
f.write(line + "\n")
|
|
except Exception:
|
|
pass
|
|
return log
|
|
|
|
|
|
def load_folder_list(path):
|
|
"""Load known FCCS folder names from file, one per line.
|
|
|
|
Like config.ini, the folder list is split into a tracked template
|
|
(fccs_folders.template.txt) and a git-ignored per-engagement working copy
|
|
(fccs_folders.txt) — auto-created from the template on first use.
|
|
"""
|
|
root, ext = os.path.splitext(path)
|
|
template = root + ".template" + ext
|
|
if not ensure_from_template(path, template):
|
|
print(f"ERROR: folder list file not found: {path}")
|
|
print(f" (no template found at {template} to create it from)")
|
|
sys.exit(1)
|
|
folders = []
|
|
with open(path, "r", encoding="utf-8") as f:
|
|
for line in f:
|
|
s = line.strip()
|
|
if s and not s.startswith("#"):
|
|
folders.append(s)
|
|
return folders
|
|
|
|
|
|
# Thomson Reuters product integrations that auto-generate folders named
|
|
# "{Product} MM-DD-YYYY" in FCCS. Matched built-in (no fccs_folders.txt entry
|
|
# needed); add newly-discovered products here.
|
|
TR_PRODUCT_FOLDERS = ("UltraTax CS", "Planner CS", "Practice CS")
|
|
|
|
|
|
def build_folder_patterns(templates):
|
|
"""
|
|
Convert folder template strings into regex patterns for matching.
|
|
|
|
Templates use YYYY as a year placeholder (e.g. 'YYYY Tax Documents').
|
|
Non-recurring folders (no YYYY) are matched literally.
|
|
Built-in patterns for '{Product} MM-DD-YYYY' Thomson Reuters product
|
|
folders (TR_PRODUCT_FOLDERS, e.g. UltraTax CS) are always included.
|
|
|
|
Returns a list of (compiled_regex, template_name, folder_type) tuples,
|
|
sorted longest-first to prevent partial matches.
|
|
|
|
folder_type is one of:
|
|
"yyyy" — recurring folder with year prefix (captured in group 1)
|
|
"ultratax" — TR product folder with date suffix (captured in group 1)
|
|
"static" — non-recurring folder, no decomposition needed
|
|
"""
|
|
patterns = []
|
|
|
|
# Built-in: TR product folders (auto-generated by each product's integration)
|
|
for product in TR_PRODUCT_FOLDERS:
|
|
patterns.append((
|
|
re.compile(re.escape(product) + r" (\d{2}-\d{2}-\d{4})$"),
|
|
product,
|
|
"ultratax",
|
|
))
|
|
|
|
for tmpl in templates:
|
|
if "YYYY" in tmpl:
|
|
# Replace YYYY with captured 4-digit year pattern, escape the rest
|
|
parts = tmpl.split("YYYY")
|
|
regex_str = re.escape(parts[0]) + r"(\d{4})" + re.escape(parts[1])
|
|
patterns.append((re.compile(regex_str + "$"), tmpl, "yyyy"))
|
|
else:
|
|
# Non-recurring folder — exact match
|
|
regex_str = re.escape(tmpl)
|
|
patterns.append((re.compile(regex_str + "$"), tmpl, "static"))
|
|
|
|
# Sort by regex pattern length (longest first) to avoid partial matches
|
|
patterns.sort(key=lambda p: len(p[0].pattern), reverse=True)
|
|
return patterns
|
|
|
|
|
|
def decompose_folder_path(match, folder_type, template_name):
|
|
"""
|
|
Decompose a matched folder name into nested path components that
|
|
recreate the FCCS UI folder structure.
|
|
|
|
Returns a tuple of path parts to be joined with os.path.join().
|
|
|
|
Examples:
|
|
yyyy: "YYYY Tax Documents" matched "2025 Tax Documents"
|
|
→ ("Tax Documents", "2025")
|
|
ultratax: any TR product folder, e.g. "UltraTax CS 12-31-2008"
|
|
→ ("UltraTax CS", "12-31-2008"),
|
|
"Planner CS 12-31-2016" → ("Planner CS", "12-31-2016")
|
|
static: "Permanent File"
|
|
→ ("Permanent File",)
|
|
"""
|
|
if folder_type == "yyyy":
|
|
year = match.group(1)
|
|
# Strip "YYYY " from template to get the base folder name
|
|
base_name = template_name.replace("YYYY", "").strip()
|
|
return (base_name, year)
|
|
elif folder_type == "ultratax":
|
|
date = match.group(1)
|
|
return (template_name, date)
|
|
else:
|
|
return (template_name,)
|
|
|
|
|
|
def load_lines(path):
|
|
"""Read non-blank, non-comment lines from a file."""
|
|
if not os.path.exists(path):
|
|
return []
|
|
with open(path, "r", encoding="utf-8") as f:
|
|
return [line.strip() for line in f if line.strip() and not line.strip().startswith("#")]
|
|
|
|
|
|
def check_for_clashes(drawer_ids):
|
|
"""Find drawer IDs that are a prefix of another drawer ID.
|
|
|
|
FCCS searches by prefix, so searching the shorter ID (e.g. '02218')
|
|
pops up a selection box when a longer ID exists (e.g. '02218A'),
|
|
which breaks the automated navigation.
|
|
|
|
Returns a list of (short_id, [longer_ids...]) tuples, sorted by short_id.
|
|
Empty list means no clashes.
|
|
"""
|
|
ids = sorted(set(drawer_ids))
|
|
clashes = []
|
|
for short in ids:
|
|
matches = [other for other in ids
|
|
if other != short and other.startswith(short)]
|
|
if matches:
|
|
clashes.append((short, matches))
|
|
return clashes
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# MANIFEST / EXPORT-COMPLETENESS HELPERS
|
|
# ---------------------------------------------------------------------------
|
|
# Shared by fccs_verify.py (batch) and fccs_check.py (interactive) so both
|
|
# judge completeness identically.
|
|
|
|
# Trailing " Page N" (optionally "Page N of M") appended to multi-page exports.
|
|
_PAGE_RE = re.compile(r"\s*Page\s+\d+(?:\s+of\s+\d+)?\s*$", re.IGNORECASE)
|
|
# Creation-date field ("_MM-DD-YYYY_") that precedes the document name.
|
|
_DATE_ANCHOR = re.compile(r"_\d{2}-\d{2}-\d{4}_")
|
|
|
|
|
|
def load_manifest(path):
|
|
"""Load a manifest file and return one row (list of cells) per document.
|
|
|
|
Handles both formats:
|
|
- New: one document per line (tab-separated columns).
|
|
- Old: a raw ListView dump led by 'List1', then row-major cells with
|
|
3 columns per document (Drawer ID, Page Title, Application), reshaped
|
|
so the document count is correct.
|
|
"""
|
|
with open(path, "r", encoding="utf-8") as f:
|
|
lines = [line.rstrip("\n") for line in f if line.strip()]
|
|
|
|
if lines and lines[0].strip() == "List1":
|
|
cells = lines[1:]
|
|
rows = [cells[i:i + 3] for i in range(0, len(cells), 3)]
|
|
return [r for r in rows if len(r) == 3]
|
|
|
|
return [line.split("\t") for line in lines]
|
|
|
|
|
|
def manifest_doc_names(path, drawer_id):
|
|
"""Return the expected document (Page Title) names from a manifest file."""
|
|
rows = load_manifest(path)
|
|
names = []
|
|
for row in rows:
|
|
if len(row) >= 3 and row[0].strip() == drawer_id:
|
|
names.append(row[1]) # old format: DrawerID, PageTitle, Application
|
|
elif row:
|
|
names.append(row[0]) # new format: PageTitle, Application
|
|
return names
|
|
|
|
|
|
def match_key(name):
|
|
"""Normalize a document name for tolerant comparison.
|
|
|
|
Strips a trailing 'Page N' page-split suffix, then reduces to lowercase
|
|
alphanumerics so punctuation and filename-sanitization differences (FCCS
|
|
strips characters illegal in Windows filenames) don't cause false
|
|
mismatches.
|
|
"""
|
|
base = _PAGE_RE.sub("", name)
|
|
return re.sub(r"[^a-z0-9]+", "", base.lower())
|
|
|
|
|
|
def exported_doc_name(filename):
|
|
"""Extract the document-name portion from an exported filename, or None.
|
|
|
|
Format: {drawer}_{client}_{folder}_{MM-DD-YYYY}_{docname}.ext
|
|
The creation-date field is a reliable anchor; the doc name follows the last
|
|
one (client/folder fields don't carry an "_MM-DD-YYYY_" pattern).
|
|
"""
|
|
stem = os.path.splitext(filename)[0]
|
|
anchors = list(_DATE_ANCHOR.finditer(stem))
|
|
if not anchors:
|
|
return None
|
|
return stem[anchors[-1].end():]
|
|
|
|
|
|
def client_name_from_filename(filename):
|
|
"""Best-effort client name from a single export filename, or None.
|
|
|
|
Filenames are '{drawer}_{client}_{folder}_{MM-DD-YYYY}_{doc}.ext', so the
|
|
client name is the second underscore-delimited token. Client names carry
|
|
commas/spaces but not underscores, so this token is reliable even when the
|
|
fuller parse (folder/date matching) fails.
|
|
"""
|
|
parts = filename.split("_")
|
|
if len(parts) >= 2 and parts[1].strip():
|
|
return parts[1].strip()
|
|
return None
|
|
|
|
|
|
def client_name_from_files(files):
|
|
"""Best-effort client name from a drawer's export filenames, or None.
|
|
|
|
The most common per-file client name is returned to shrug off any oddball
|
|
filename.
|
|
"""
|
|
counts = {}
|
|
for f in files:
|
|
name = client_name_from_filename(f)
|
|
if name:
|
|
counts[name] = counts.get(name, 0) + 1
|
|
if not counts:
|
|
return None
|
|
return max(counts, key=counts.get)
|
|
|
|
|
|
def index_files_by_drawer(export_dir):
|
|
"""Map each export file to its leading drawer-ID token (before first '_').
|
|
|
|
The underscore boundary keeps clashing IDs separate (04289 vs 04289TS).
|
|
Returns {drawer_id: [filenames]}.
|
|
"""
|
|
index = {}
|
|
for f in os.listdir(export_dir):
|
|
if not os.path.isfile(os.path.join(export_dir, f)):
|
|
continue
|
|
token = f.split("_", 1)[0]
|
|
index.setdefault(token, []).append(f)
|
|
return index
|
|
|
|
|
|
def evaluate_drawer(drawer_id, files, manifest_dir):
|
|
"""Compare a drawer's manifest against its exported files.
|
|
|
|
`files` is the list of export filenames belonging to this drawer. Accounts
|
|
for page-splitting (a document exported as 'Name Page 1/2/...' counts as
|
|
present) and filename sanitization (via match_key).
|
|
|
|
Returns a dict:
|
|
has_manifest, manifest_path, expected (list), missing (list),
|
|
extras (list), unparsed (int), file_count (int)
|
|
"""
|
|
manifest_path = os.path.join(manifest_dir, drawer_id + ".txt")
|
|
result = {
|
|
"has_manifest": os.path.exists(manifest_path),
|
|
"manifest_path": manifest_path,
|
|
"expected": [],
|
|
"missing": [],
|
|
"extras": [],
|
|
"unparsed": 0,
|
|
"file_count": len(files),
|
|
}
|
|
if not result["has_manifest"]:
|
|
return result
|
|
|
|
expected = manifest_doc_names(manifest_path, drawer_id)
|
|
result["expected"] = expected
|
|
|
|
# Group exported files by normalized doc name; page-splits collapse together.
|
|
exported = {} # key -> list of full doc names (one entry per file/page)
|
|
unparsed = 0
|
|
for f in files:
|
|
doc = exported_doc_name(f)
|
|
if doc is None:
|
|
unparsed += 1
|
|
continue
|
|
exported.setdefault(match_key(doc), []).append(doc)
|
|
|
|
result["missing"] = [name for name in expected
|
|
if match_key(name) not in exported]
|
|
|
|
expected_keys = {match_key(n) for n in expected}
|
|
result["extras"] = [names[0] for k, names in exported.items()
|
|
if k not in expected_keys]
|
|
result["unparsed"] = unparsed
|
|
return result
|