From 9eb7784e9c536511e2f6c1cc547ae82cce68a545 Mon Sep 17 00:00:00 2001 From: dat972 Date: Sun, 5 Jul 2026 23:07:47 -0500 Subject: [PATCH] tweaked some code after looking at real images and added a readme --- .gitignore | 1 + README.md | 78 ++++++++++++++++++++++++++++++++++++++++ fccs_config.py | 35 ++++++++++++++++++ fccs_folders.txt | 19 +++++----- fccs_reorganize.py | 90 +++++++++++++++++++++++++++------------------- 5 files changed, 178 insertions(+), 45 deletions(-) create mode 100644 .gitignore create mode 100644 README.md diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6f49e84 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +references/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..8200639 --- /dev/null +++ b/README.md @@ -0,0 +1,78 @@ +# FCCS Data Extraction Tool + +Extracts and organizes documents from FileCabinet CS (Thomson Reuters) using GUI automation. Designed to work across multiple client engagements without code changes. + +## Prerequisites + +- 32-bit Python (must match FCCS architecture) +- `pywinauto` (`pip install pywinauto`) +- FileCabinet CS installed and restored with client backup data + +## Project Layout + +| File | Purpose | +|------|---------| +| `config.ini` | All paths, timeouts, and FCCS control identifiers | +| `fccs_folders.txt` | FCCS folder templates (manually populated per engagement) | +| `fccs_config.py` | Shared config loading, logging, folder pattern building | +| `fccs_scan.py` | Step 1: Scan backup directory for drawer IDs | +| `fccs_export.py` | Step 2: Automate FCCS GUI to export all drawers | +| `fccs_reorganize.py` | Step 3: Parse filenames and rebuild folder structure | + +## Setup Per Engagement + +1. Edit `config.ini` -- point `backup_dir`, `export_dir`, and `output_dir` at the client's directories. +2. Populate `fccs_folders.txt` -- open FCCS > System Configuration > Document Folders and list each folder template exactly as shown, one per line. Keep the `YYYY` prefix on recurring folders. UltraTax CS folders are detected automatically and do not need to be listed. + +## Workflow + +### Step 1: Scan Drawers + +``` +python fccs_scan.py +``` + +Reads the restored FCCS backup directory and writes all drawer IDs (subfolder names) to `drawer_ids.txt`. + +### Step 2: Export Documents + +``` +python fccs_export.py +``` + +Requires FCCS to be open with export destination already configured. Automates the GUI to export every drawer via File > Send To > File. Features: + +- **Resumable** -- tracks completed drawers in `completed.txt`; safe to restart +- **Screenshots** -- captures failure states for diagnosis +- **Defensive** -- one bad drawer won't crash the entire run + +### Step 3: Reorganize Files + +``` +python fccs_reorganize.py +``` + +Parses the flat exported filenames and copies them into an organized structure: + +``` +output/ + 01069_ABRAHAM, REBEKAH L./ + 2025 Tax Documents/ + 01069_ABRAHAM, REBEKAH L._2025 Tax Documents_03-03-2026_030126 E-mail re Tax Info.pdf + Permanent File/ + 01069_ABRAHAM, REBEKAH L._Permanent File_02-24-2018_Driver's License.pdf + UltraTax CS 12-31-2008/ + 01069_ABRAHAM, REBEKAH L._UltraTax CS 12-31-2008_02-12-2009_2008 Form 1040 Filing Instructions.doc + _unparsed/ + (files that couldn't be parsed go here for manual review) +``` + +Exported filenames follow the format `{drawer_id}_{client_name}_{folder_name}_{creation_date}_{document_name}.ext`. The parser uses folder templates from `fccs_folders.txt` (with `YYYY` expanded via regex) and the creation date (`MM-DD-YYYY`) as anchors to reliably split the underscore-delimited fields. UltraTax CS folders are matched by a built-in pattern. + +## Config Reference + +All scripts read from `config.ini` (or specify `--config path\to\config.ini`). + +- **`[paths]`** -- backup_dir, export_dir, output_dir, drawer_id_file, completed_file, log_file, screenshot_dir, folder_list +- **`[timeouts]`** -- nav_timeout, dialog_timeout, progress_appear, progress_finish, settle, confirm_timeout +- **`[controls]`** -- FCCS window class names and button titles (rarely need changing) diff --git a/fccs_config.py b/fccs_config.py index 20b691d..22715d8 100644 --- a/fccs_config.py +++ b/fccs_config.py @@ -3,6 +3,7 @@ Shared configuration, logging, and utilities for the FCCS extraction tool. """ import os +import re import sys import argparse import configparser @@ -62,6 +63,40 @@ def load_folder_list(path): return folders +def build_folder_patterns(templates): + """ + Convert folder template strings into regex patterns for matching. + + Templates use YYYY as a year placeholder (e.g. 'YYYY Tax Documents'). + Non-recurring folders (no YYYY) are matched literally. + A built-in pattern for 'UltraTax CS MM-DD-YYYY' is always included. + + Returns a list of (compiled_regex, template_name) tuples, sorted + longest-first to prevent partial matches. + """ + patterns = [] + + # Built-in: UltraTax CS folders (auto-generated by UltraTax integration) + patterns.append(( + re.compile(r"UltraTax CS \d{2}-\d{2}-\d{4}$"), + "UltraTax CS", + )) + + for tmpl in templates: + if "YYYY" in tmpl: + # Replace YYYY with 4-digit year pattern, escape the rest + parts = tmpl.split("YYYY") + regex_str = re.escape(parts[0]) + r"\d{4}" + re.escape(parts[1]) + else: + # Non-recurring folder — exact match + regex_str = re.escape(tmpl) + patterns.append((re.compile(regex_str + "$"), tmpl)) + + # Sort by regex pattern length (longest first) to avoid partial matches + patterns.sort(key=lambda p: len(p[0].pattern), reverse=True) + return patterns + + def load_lines(path): """Read non-blank, non-comment lines from a file.""" if not os.path.exists(path): diff --git a/fccs_folders.txt b/fccs_folders.txt index 9143c6b..4c670c0 100644 --- a/fccs_folders.txt +++ b/fccs_folders.txt @@ -1,11 +1,14 @@ -# FCCS Folder Names -# ----------------- -# List one folder name per line, exactly as it appears in FileCabinet CS. -# Open FCCS > Tools > Options > Folder Structure to see all folder names. +# FCCS Folder Templates +# --------------------- +# List folder names exactly as shown in FCCS System Configuration > Document Folders. +# Keep the YYYY prefix for recurring folders. Non-recurring folders go as-is. +# UltraTax CS folders are handled automatically — do not list them here. # Blank lines and lines starting with # are ignored. # # Examples: -# Tax Documents -# Correspondence -# Financial Statements -# Payroll +# YYYY Tax Documents +# YYYY Billing & Invoices +# YYYY PAYROLL +# YYYY Financial Stmt Documents +# CORRESPONDENCE +# Permanent File diff --git a/fccs_reorganize.py b/fccs_reorganize.py index da470e2..0b699d5 100644 --- a/fccs_reorganize.py +++ b/fccs_reorganize.py @@ -1,12 +1,16 @@ """ Step 3: Reorganize flat exported files into a proper folder structure. -Parses FCCS export filenames using known folder names and date anchors +Parses FCCS export filenames using known folder templates and date anchors to reconstruct the original drawer/folder hierarchy. Filename format: - {drawer_id}_{client_name}_{folder_name}_{MM-DD-YYYY}_{document_name}.ext - Example: 01069_SMITH, BOB_2007 Tax Documents_02-10-2009_2007 Form 1040A Page 1.pdf + {drawer_id}_{client_name}_{folder_name}_{creation_date MM-DD-YYYY}_{document_name}.ext + +Examples: + 01069_ABRAHAM, REBEKAH L._2025 Tax Documents_03-03-2026_030126 E-mail re Tax Info.pdf + 01069_ABRAHAM, REBEKAH L._Permanent File_02-24-2018_Driver's License.pdf + 01069_ABRAHAM, REBEKAH L._UltraTax CS 12-31-2008_02-12-2009_2008 Form 1040 Filing Instructions.doc Output structure: output/{drawer_id}_{client_name}/{folder_name}/{original_filename} @@ -19,54 +23,66 @@ import re import shutil import sys -from fccs_config import parse_args, load_config, make_logger, load_folder_list +from fccs_config import ( + parse_args, load_config, make_logger, + load_folder_list, build_folder_patterns, +) + +# Regex for the creation date field (MM-DD-YYYY) bounded by underscores +_DATE_RE = re.compile(r"_(\d{2}-\d{2}-\d{4})_") -def parse_filename(filename, known_folders): +def parse_filename(filename, folder_patterns): """ Parse an FCCS export filename into (drawer_id, client_name, folder_name, date, doc_name_with_ext) or return None if it cannot be parsed. - Strategy: anchor on drawer_id (left), date MM-DD-YYYY (middle), and match - a known folder name between client_name and date. + folder_patterns is a list of (compiled_regex, template_name) from + build_folder_patterns(), sorted longest-first. + + Strategy: + 1. Drawer ID: first token before first underscore + 2. Creation date: find MM-DD-YYYY pattern as anchor + 3. Folder name: regex-match a known folder template at the end of + the text between client_name and creation_date + 4. Client name: whatever is left between drawer_id and folder_name """ stem, ext = os.path.splitext(filename) - # Drawer ID is always the first underscore-delimited token + # 1. Drawer ID — first underscore-delimited token sep = stem.find("_") if sep == -1: return None drawer_id = stem[:sep] rest = stem[sep + 1:] - # Find all date-pattern occurrences (MM-DD-YYYY) in rest - # We search with a leading underscore context to ensure proper boundaries - date_pattern = re.compile(r"_(\d{2}-\d{2}-\d{4})_") - # Prepend underscore so the first potential date at position 0 of rest is caught + # 2. Find all creation date candidates (MM-DD-YYYY bounded by underscores) + # Prepend underscore so a date right at the start of `rest` is also found search_str = "_" + rest - matches = list(date_pattern.finditer(search_str)) - - if not matches: + date_matches = list(_DATE_RE.finditer(search_str)) + if not date_matches: return None - # Try each date match; known_folders is already sorted longest-first - for m in matches: + # 3. Try each date match; for each, try to match a known folder template + for m in date_matches: date_str = m.group(1) - # Position in `rest` where date starts (adjust for prepended _) - date_start_in_rest = m.start() - 1 # -1 for the prepended _ - # But the match includes the leading _, so the actual content before date: - before_date = rest[:date_start_in_rest] - after_date = rest[date_start_in_rest + len(date_str) + 1:] # +1 for trailing _ + # Calculate positions relative to `rest` (adjust for prepended _) + # m.start() is the position of the leading _ in search_str + # In `rest`, the content before the date ends at (m.start() - 1) + before_date = rest[: m.start() - 1] + after_date_pos = m.start() - 1 + len("_") + len(date_str) + len("_") + doc_name = rest[after_date_pos:] - # Try to match a known folder at the end of before_date - for folder in known_folders: - if before_date.endswith(folder): - # Check there's an underscore separator before the folder name - prefix_end = len(before_date) - len(folder) - if prefix_end > 0 and before_date[prefix_end - 1] == "_": - client_name = before_date[: prefix_end - 1] - doc_name = after_date + ext - return (drawer_id, client_name, folder, date_str, doc_name) + # Try each folder pattern against the end of before_date + for pattern, template_name in folder_patterns: + match = pattern.search(before_date) + if match and match.end() == len(before_date): + # Folder matched at the end — check for underscore separator before it + folder_start = match.start() + if folder_start > 0 and before_date[folder_start - 1] == "_": + client_name = before_date[: folder_start - 1] + folder_name = match.group() # the actual expanded name + return (drawer_id, client_name, folder_name, date_str, doc_name + ext) return None @@ -84,11 +100,11 @@ def main(): log(f"ERROR: export directory not found: {export_dir}") sys.exit(1) - known_folders = load_folder_list(folder_list_path) - if not known_folders: - log("WARNING: folder list is empty. All files will go to _unparsed/.") - # Sort longest-first to prevent partial matches - known_folders.sort(key=len, reverse=True) + folder_templates = load_folder_list(folder_list_path) + if not folder_templates: + log("WARNING: folder list is empty. Only built-in patterns (UltraTax CS) will match.") + folder_patterns = build_folder_patterns(folder_templates) + log(f"Loaded {len(folder_patterns)} folder patterns") files = [ f for f in os.listdir(export_dir) @@ -101,7 +117,7 @@ def main(): failed = 0 for filename in files: - result = parse_filename(filename, known_folders) + result = parse_filename(filename, folder_patterns) if result is None: log(f" UNPARSED: {filename}")