added manifests to create a list of expected file outputs as well as a verification script.
This commit is contained in:
@@ -19,6 +19,7 @@ Extracts and organizes documents from FileCabinet CS (Thomson Reuters) using GUI
|
|||||||
| `fccs_scan.py` | Step 1: Scan backup directory for drawer IDs |
|
| `fccs_scan.py` | Step 1: Scan backup directory for drawer IDs |
|
||||||
| `fccs_export.py` | Step 2: Automate FCCS GUI to export all drawers |
|
| `fccs_export.py` | Step 2: Automate FCCS GUI to export all drawers |
|
||||||
| `fccs_reorganize.py` | Step 3: Parse filenames and rebuild folder structure |
|
| `fccs_reorganize.py` | Step 3: Parse filenames and rebuild folder structure |
|
||||||
|
| `fccs_verify.py` | Step 4 (optional): Compare manifests against exported files |
|
||||||
|
|
||||||
## Setup Per Engagement
|
## Setup Per Engagement
|
||||||
|
|
||||||
@@ -47,6 +48,14 @@ Requires FCCS to be open with export destination already configured. Automates t
|
|||||||
- **Screenshots** -- captures failure states for diagnosis
|
- **Screenshots** -- captures failure states for diagnosis
|
||||||
- **Defensive** -- one bad drawer won't crash the entire run
|
- **Defensive** -- one bad drawer won't crash the entire run
|
||||||
|
|
||||||
|
### Step 2b (optional): Verify Export Completeness
|
||||||
|
|
||||||
|
```
|
||||||
|
python fccs_verify.py
|
||||||
|
```
|
||||||
|
|
||||||
|
During export, each drawer's document list is captured from the FCCS dialog and saved as a manifest. This script compares those manifests against the actual exported files to flag any drawers with missing or extra files.
|
||||||
|
|
||||||
### Step 3: Reorganize Files
|
### Step 3: Reorganize Files
|
||||||
|
|
||||||
```
|
```
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ drawer_id_file = C:\Migration\drawer_ids.txt
|
|||||||
completed_file = C:\Migration\completed.txt
|
completed_file = C:\Migration\completed.txt
|
||||||
log_file = C:\Migration\run_log.txt
|
log_file = C:\Migration\run_log.txt
|
||||||
screenshot_dir = C:\Migration\screenshots
|
screenshot_dir = C:\Migration\screenshots
|
||||||
|
manifest_dir = C:\Migration\manifests
|
||||||
folder_list = fccs_folders.txt
|
folder_list = fccs_folders.txt
|
||||||
|
|
||||||
[timeouts]
|
[timeouts]
|
||||||
|
|||||||
@@ -129,19 +129,48 @@ def open_send_to_file(main, app, ctrl, timeouts):
|
|||||||
return dlg
|
return dlg
|
||||||
|
|
||||||
|
|
||||||
def perform_export(dlg, ctrl, timeouts):
|
def read_manifest(dlg, log):
|
||||||
"""Click Select -> then OK to start the export."""
|
"""Read the list of selected documents from the right-side ListView."""
|
||||||
# Wait for the Select button to be ready — the dialog may still be
|
try:
|
||||||
# populating its controls after appearing.
|
lv = dlg.child_window(title="List1", class_name="SysListView32")
|
||||||
|
count = lv.item_count()
|
||||||
|
if count == 0:
|
||||||
|
log(" WARNING: ListView is empty after Select")
|
||||||
|
return []
|
||||||
|
items = lv.texts()
|
||||||
|
log(f" Manifest captured: {count} documents selected")
|
||||||
|
return items
|
||||||
|
except Exception as e:
|
||||||
|
log(f" WARNING: could not read manifest from ListView: {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def save_manifest(drawer_id, items, manifest_dir, log):
|
||||||
|
"""Save the list of expected export items to a manifest file."""
|
||||||
|
os.makedirs(manifest_dir, exist_ok=True)
|
||||||
|
path = os.path.join(manifest_dir, f"{drawer_id}.txt")
|
||||||
|
with open(path, "w", encoding="utf-8") as f:
|
||||||
|
for row in items:
|
||||||
|
if isinstance(row, (list, tuple)):
|
||||||
|
f.write("\t".join(str(c) for c in row) + "\n")
|
||||||
|
else:
|
||||||
|
f.write(str(row) + "\n")
|
||||||
|
log(f" Manifest saved: {path}")
|
||||||
|
|
||||||
|
|
||||||
|
def perform_export(dlg, ctrl, timeouts, log):
|
||||||
|
"""Click Select -> then OK to start the export. Returns manifest list."""
|
||||||
select_btn = dlg.child_window(title=ctrl["select_btn_title"], class_name="Button")
|
select_btn = dlg.child_window(title=ctrl["select_btn_title"], class_name="Button")
|
||||||
select_btn.wait("visible ready enabled", timeout=timeouts["dialog"])
|
select_btn.wait("visible ready enabled", timeout=timeouts["dialog"])
|
||||||
select_btn.click()
|
select_btn.click()
|
||||||
|
|
||||||
# Wait for the OK button to become ready — transferring files to the
|
|
||||||
# selected side can take a while on large drawers.
|
|
||||||
ok_btn = dlg.child_window(title=ctrl["ok_btn_title"], class_name="Button")
|
ok_btn = dlg.child_window(title=ctrl["ok_btn_title"], class_name="Button")
|
||||||
ok_btn.wait("visible ready enabled", timeout=timeouts["select"])
|
ok_btn.wait("visible ready enabled", timeout=timeouts["select"])
|
||||||
|
|
||||||
|
manifest = read_manifest(dlg, log)
|
||||||
|
|
||||||
ok_btn.click()
|
ok_btn.click()
|
||||||
|
return manifest
|
||||||
|
|
||||||
|
|
||||||
def dismiss_any_dialog(app, ctrl, log):
|
def dismiss_any_dialog(app, ctrl, log):
|
||||||
@@ -228,7 +257,8 @@ def wait_for_export(app, ctrl, timeouts, log):
|
|||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
def export_drawer(app, main, drawer_id, ctrl, timeouts, screenshot_dir, log):
|
def export_drawer(app, main, drawer_id, ctrl, timeouts, screenshot_dir,
|
||||||
|
manifest_dir, log):
|
||||||
"""Full per-drawer sequence. Returns True on success."""
|
"""Full per-drawer sequence. Returns True on success."""
|
||||||
log(f"Drawer {drawer_id}: starting")
|
log(f"Drawer {drawer_id}: starting")
|
||||||
|
|
||||||
@@ -248,9 +278,11 @@ def export_drawer(app, main, drawer_id, ctrl, timeouts, screenshot_dir, log):
|
|||||||
take_screenshot(main, drawer_id, "dialog_fail", screenshot_dir, log)
|
take_screenshot(main, drawer_id, "dialog_fail", screenshot_dir, log)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
# 3 & 4. Select -> and OK
|
# 3 & 4. Select -> and OK (captures manifest before clicking OK)
|
||||||
try:
|
try:
|
||||||
perform_export(dlg, ctrl, timeouts)
|
manifest = perform_export(dlg, ctrl, timeouts, log)
|
||||||
|
if manifest is not None:
|
||||||
|
save_manifest(drawer_id, manifest, manifest_dir, log)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log(f" ERROR during export selection: {e}")
|
log(f" ERROR during export selection: {e}")
|
||||||
take_screenshot(dlg, drawer_id, "export_fail", screenshot_dir, log)
|
take_screenshot(dlg, drawer_id, "export_fail", screenshot_dir, log)
|
||||||
@@ -284,6 +316,7 @@ def main():
|
|||||||
"drawer_id_file": cfg.get("paths", "drawer_id_file"),
|
"drawer_id_file": cfg.get("paths", "drawer_id_file"),
|
||||||
"completed_file": cfg.get("paths", "completed_file"),
|
"completed_file": cfg.get("paths", "completed_file"),
|
||||||
"screenshot_dir": cfg.get("paths", "screenshot_dir"),
|
"screenshot_dir": cfg.get("paths", "screenshot_dir"),
|
||||||
|
"manifest_dir": cfg.get("paths", "manifest_dir"),
|
||||||
}
|
}
|
||||||
timeouts = {
|
timeouts = {
|
||||||
"nav": cfg.getfloat("timeouts", "nav_timeout"),
|
"nav": cfg.getfloat("timeouts", "nav_timeout"),
|
||||||
@@ -339,7 +372,8 @@ def main():
|
|||||||
log(f"--- [{i}/{len(pending)}] Drawer {drawer_id} ---")
|
log(f"--- [{i}/{len(pending)}] Drawer {drawer_id} ---")
|
||||||
try:
|
try:
|
||||||
ok = export_drawer(app, main_win, drawer_id, ctrl, timeouts,
|
ok = export_drawer(app, main_win, drawer_id, ctrl, timeouts,
|
||||||
paths["screenshot_dir"], log)
|
paths["screenshot_dir"],
|
||||||
|
paths["manifest_dir"], log)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log(f" UNEXPECTED ERROR on drawer {drawer_id}: {e}")
|
log(f" UNEXPECTED ERROR on drawer {drawer_id}: {e}")
|
||||||
try:
|
try:
|
||||||
|
|||||||
118
fccs_verify.py
Normal file
118
fccs_verify.py
Normal file
@@ -0,0 +1,118 @@
|
|||||||
|
"""
|
||||||
|
Step 4 (optional): Verify exported files against manifests.
|
||||||
|
|
||||||
|
Compares the per-drawer manifests captured during export (Step 2)
|
||||||
|
against the actual files in the export directory to identify
|
||||||
|
missing or extra files.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from fccs_config import parse_args, load_config, make_logger
|
||||||
|
|
||||||
|
|
||||||
|
def load_manifest(path):
|
||||||
|
"""Load a manifest file and return the list of document rows."""
|
||||||
|
items = []
|
||||||
|
with open(path, "r", encoding="utf-8") as f:
|
||||||
|
for line in f:
|
||||||
|
line = line.rstrip("\n")
|
||||||
|
if line:
|
||||||
|
items.append(line.split("\t"))
|
||||||
|
return items
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
args = parse_args()
|
||||||
|
cfg = load_config(args.config)
|
||||||
|
log = make_logger(cfg.get("paths", "log_file"))
|
||||||
|
|
||||||
|
export_dir = cfg.get("paths", "export_dir")
|
||||||
|
manifest_dir = cfg.get("paths", "manifest_dir")
|
||||||
|
|
||||||
|
if not os.path.isdir(manifest_dir):
|
||||||
|
log(f"ERROR: manifest directory not found: {manifest_dir}")
|
||||||
|
log("Run fccs_export.py first to generate manifests.")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
if not os.path.isdir(export_dir):
|
||||||
|
log(f"ERROR: export directory not found: {export_dir}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
# Load all manifests
|
||||||
|
manifests = {}
|
||||||
|
for fname in sorted(os.listdir(manifest_dir)):
|
||||||
|
if fname.endswith(".txt"):
|
||||||
|
drawer_id = os.path.splitext(fname)[0]
|
||||||
|
items = load_manifest(os.path.join(manifest_dir, fname))
|
||||||
|
manifests[drawer_id] = items
|
||||||
|
|
||||||
|
if not manifests:
|
||||||
|
log("No manifest files found.")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
# Index exported files by drawer ID prefix
|
||||||
|
exported_by_drawer = {}
|
||||||
|
all_exported = [
|
||||||
|
f for f in os.listdir(export_dir)
|
||||||
|
if os.path.isfile(os.path.join(export_dir, f))
|
||||||
|
]
|
||||||
|
|
||||||
|
for fname in all_exported:
|
||||||
|
sep = fname.find("_")
|
||||||
|
if sep != -1:
|
||||||
|
did = fname[:sep]
|
||||||
|
exported_by_drawer.setdefault(did, []).append(fname)
|
||||||
|
|
||||||
|
log("=" * 60)
|
||||||
|
log("Export Verification Report")
|
||||||
|
log("=" * 60)
|
||||||
|
log(f"Manifests loaded : {len(manifests)} drawers")
|
||||||
|
log(f"Exported files : {len(all_exported)} total")
|
||||||
|
log("")
|
||||||
|
|
||||||
|
total_expected = 0
|
||||||
|
total_actual = 0
|
||||||
|
mismatched = []
|
||||||
|
|
||||||
|
for drawer_id in sorted(manifests):
|
||||||
|
expected = manifests[drawer_id]
|
||||||
|
actual = exported_by_drawer.get(drawer_id, [])
|
||||||
|
n_expected = len(expected)
|
||||||
|
n_actual = len(actual)
|
||||||
|
total_expected += n_expected
|
||||||
|
total_actual += n_actual
|
||||||
|
|
||||||
|
if n_expected == n_actual:
|
||||||
|
log(f" {drawer_id}: OK ({n_actual} files)")
|
||||||
|
else:
|
||||||
|
diff = n_actual - n_expected
|
||||||
|
sign = "+" if diff > 0 else ""
|
||||||
|
log(f" {drawer_id}: MISMATCH — expected {n_expected}, "
|
||||||
|
f"got {n_actual} ({sign}{diff})")
|
||||||
|
mismatched.append(drawer_id)
|
||||||
|
|
||||||
|
# Check for exported files with no manifest
|
||||||
|
orphan_drawers = set(exported_by_drawer.keys()) - set(manifests.keys())
|
||||||
|
orphan_count = sum(len(exported_by_drawer[d]) for d in orphan_drawers)
|
||||||
|
|
||||||
|
log("")
|
||||||
|
log("-" * 60)
|
||||||
|
log(f"Expected total : {total_expected}")
|
||||||
|
log(f"Actual total : {total_actual}")
|
||||||
|
|
||||||
|
if orphan_drawers:
|
||||||
|
log(f"No manifest for : {', '.join(sorted(orphan_drawers))} "
|
||||||
|
f"({orphan_count} files)")
|
||||||
|
|
||||||
|
if mismatched:
|
||||||
|
log(f"Mismatched drawers: {', '.join(mismatched)}")
|
||||||
|
elif not orphan_drawers:
|
||||||
|
log("All drawers match.")
|
||||||
|
|
||||||
|
log("=" * 60)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Reference in New Issue
Block a user