From fec7745092f9df968020190e9d5c17b398d08c2f Mon Sep 17 00:00:00 2001 From: dat972 Date: Sat, 11 Jul 2026 12:27:40 -0500 Subject: [PATCH] updated our verify logic --- README.md | 42 +++++--- __pycache__/fccs_check.cpython-313.pyc | Bin 7758 -> 4501 bytes __pycache__/fccs_config.cpython-313.pyc | Bin 7844 -> 13922 bytes __pycache__/fccs_verify.cpython-313.pyc | Bin 6307 -> 4975 bytes fccs_check.py | 115 +++++---------------- fccs_config.py | 132 ++++++++++++++++++++++++ fccs_verify.py | 124 +++++++++------------- 7 files changed, 230 insertions(+), 183 deletions(-) diff --git a/README.md b/README.md index 4af0eda..e6ba629 100644 --- a/README.md +++ b/README.md @@ -61,22 +61,6 @@ Requires FCCS to be open with export destination already configured. Automates t - **Defensive** -- one bad drawer won't crash the entire run - **Crash recovery** -- some documents (e.g. UltraTax "Diagnostics" files) crash FCCS's converter (`FileConversionEngine::convert() failed`), which aborts that drawer's export. The script detects the error dialog, screenshots and logs the crashing document, dismisses it, and records the drawer in `crashed.txt` so it's skipped on future runs instead of stalling. Handle crashed drawers manually (export them excluding the poison document); delete a line from `crashed.txt` to retry after fixing. -### Step 2b (optional): Verify Export Completeness - -``` -python fccs_verify.py -``` - -During export, each drawer's document list is captured from the FCCS dialog and saved as a manifest. This script compares those manifests against the actual exported files to flag any drawers with missing or extra files. - -To spot-check specific drawers (e.g. ones the log marked failed, to see whether they actually finished exporting in the background), run: - -``` -python fccs_check.py -``` - -It prompts for one or more drawer IDs and reports, per drawer, which manifest documents are present vs missing. It accounts for FCCS page-splitting (a document exported as `Name Page 1`, `Name Page 2`, … counts as present) and for filename sanitization (titles containing characters illegal in filenames, like `:`, still match). - ### Step 3: Reorganize Files ``` @@ -105,6 +89,32 @@ output/ Exported filenames follow the format `{drawer_id}_{client_name}_{folder_name}_{creation_date}_{document_name}.ext`. The parser uses folder templates from `fccs_folders.txt` (with `YYYY` expanded via regex) and the creation date (`MM-DD-YYYY`) as anchors to reliably split the underscore-delimited fields. Folder names are decomposed into nested paths that match the FCCS UI structure (e.g. `2025 Tax Documents` becomes `Tax Documents/2025/`). UltraTax CS folders are matched by a built-in pattern. +### Step 4 (optional): Verify Export Completeness + +During export, each drawer's document list is captured from the FCCS dialog and saved as a manifest (in `manifest_dir`). These tools compare the manifests against the files actually in the export folder to confirm nothing was missed. + +Both compare at the **document level** and share identical matching logic. They account for: + +- **Page-splitting** -- a multi-page document exported as `Name Page 1`, `Name Page 2`, … counts as that one document being present. +- **Filename sanitization** -- document titles containing characters illegal in Windows filenames (e.g. `:` `/` `?`) still match the exported files. + +Batch-check every drawer that has a manifest: + +``` +python fccs_verify.py +``` + +Reports each drawer as `OK` or `INCOMPLETE` (listing the missing documents), plus a summary and any exported drawers that have no manifest. + +Spot-check specific drawers interactively (e.g. ones the log marked failed, to see whether they actually finished exporting in the background): + +``` +python fccs_check.py +Drawer ID(s): 08097 18430 +``` + +> Note: because page-splitting means the number of files can't be mapped one-to-one to documents, completeness is judged by document *presence* (is each manifest document represented by at least one exported file), not by exact file counts. + ## Config Reference All scripts read from `config.ini` (or specify `--config path\to\config.ini`). diff --git a/__pycache__/fccs_check.cpython-313.pyc b/__pycache__/fccs_check.cpython-313.pyc index 2fe6db5b88c7c0112595468a3fd3247d39b479cf..0799fbc1e2b28bbeedfa1fc235c3acb5e1a70dcd 100644 GIT binary patch delta 2250 zcmZuyO>7fK6rTODcfGa~@*mrPhS2=DfkbTtr2JD9La8VfAr);9+r>L}w%EI7W6Oo6&-l-g*owhe}+yRH{_CRh3by5{3&^IkdMx`XlvF_3g$cE@G^Gp4s=__ukCD z@6C_9Z*A^hiN`er+!(69cYKE&E&<M7ClyM40gIdh7vBj|I*hItrfVF{bhZc;yb&;6(ESbg3!R|C+1T)vl=W)Ss zGt)zw76ds@sFj@^D$Zh~Sj=05fhhnoqd;)p$ypg}+4zj_fsG7x7{dfyWn;Q=y~8-g^0NI?uph@OR1vxb!?X4+el zkCtSv!S)!@4Vq&-ns*Gdb{@R>l>^}&mTi&?dKSEyerC3IP8qrWdEh?+j|9)kZvsNM z&=knkaZ;ERxA__vwmd2Ipb24{fJ(s&j3(s?F$D90V2zO3hEQN6O`<0FzMv2JdYFet zLf{w-_@)YIR9|C0oGO33I)F*L61Y zczvlC4R_jWZ`t3*RdNBU+a)Gf3KnBf?%u~On`2z_OA*R-#&*5SEw@x2CKp_4Ft4R` zyT;WSJ1A|_8*kmyEmD7{T*?zW9pRBxmgtu0Ra;XV1xlgUAR6~ggaQ``^D8+ZI>z2k zrN0zk&AL|fGzscHc-Ri8#X0m$?V67)Y*|!y%pLeeR?6bl@ug_2eELyz{bMy+HY$=(9!jBoz~36UL7)uyM->nh3XSLO~cH8ocv*Yk7Zj{#{@ z`MvV4YG+UX9ckh4*YU4P-`GC#sq6cyTB@4bwk&}+K!irU#rF40ONv_VzIMqEI(?JP zzcqjI%Uz3I{ngHa>cH`8Y~m;N#8OLZW%OpMeBg1cxzab^GA~tPKgTvamr(15r{JEv zb|rAHoSRSHxKeQ-$v*67ivNU6H!Vv@?O1`Wql_==1KKSxAI^B?o_P1$j-=2t)lt_0 z^=pi^<*DZS+ljC@+}$B4Ub;IaL8W?w-HF~v-4dgX@SEy-X|2^Y#}i&fOO}A}p*mAd zfX|_+^>yRp8|$;Rf9N<-XaTi5qz!!(X5uhnB2WcByOVv11r|r6K!osz;!#M}=drEW z;YkH7-vG|5Q=0%)36M`-^@PrLaZ>emC5DBhcQ&y(70#k`>;oS1>z8p2Y8R?qH!Vur zz1xWa@pr*{l2|CUP`@)#zkBlV>J`ML9C5kmFfRMfJY+F&10mq zjy~)iy_fjz-NoMVrDWfo&|i`eANHQEA1HkvO^hk%en=bZlJB>LK|WBlv3BV}n*frl zI&=lg*7bCZE4pqv8C|D65RVS}!Y3Y&Hr(B7M;Wj@IGq~w1}VvlGg5^a&PCEHSHC$3`4vK>ouNHga~YKS$tl(wdp z%hmj?ll62eYt^ni&N>w*CjHo#U1vFk!)Vf$2$>?+uVIvQz?-)9 z7W3|~&A3xM-#XHHm}&I!CSC1F@g+H^{Yf{}fg}%gOVR^%a59wi#xEiya0pEX!{~71 zBN$EDrfd;kX%|eAPiazK(q`lM#I%^XjIT_K+O()(wJ2t!oRq;?MYtj=Di$(YNyz7C zaZZvYbz00~@oKT4Xp%e?Q*o}K;DRh-xXct3u^|ng zIjQzw_>N3Fh(c!Ce3OB3f+DKC1fypJS;~p3hP$CJ&K5GI8BvDzQzuR&u_{PeENS>k zp_Dh3>A@MHsFf7ZE?ZJSHJsGMBJPXu!jvG%Ac+XdNqMnpLaL;ZSzt+~Z;fezV)?A7 z^k6~GVnrk~g(31RKO?Cs>1}BPvp)@;h)C^90l!g_l*=kEQ7uRQNipJeyY$}GsCkwKmNd;LAN3f(e>@H?O925b*1qLQ_!cqR(>NU_w9!X}7XPceLgy11kkWU`uBx;*IKH}Yz>i)^>{_f*(xGCg}r^s=5*;#H)PiC+6U5{Gzq3h{P z1J5WtIia@0|C2IQKSUF>kyHuNx~U|Y6fzN0W+qI`Mu`TBHg^Zi1;kD<30HHgac?5y z;l?*QQN&*U&6v3!ur`)6Q=cWMrWNvlc;NU3Vv*$p!zT^b6|klVfUDs5UBn`ItP7JZ zWbAvamU21iY6AfQtV$UI9tCVIHotMLVU>#|nE)_>0SZZ`!$+I<56()toT!L0kq3^| zrSTb|1m2Mo@=ybGQI8eX7_Xx;Kk>Q{Egy(J|GoVYR_D^^!DrHmVco8lF6lN!)E!hG zZ~>OgT4yhT8>+;#80+EQ>B5ZIdqtS-O#nKoy#(ocH#nH?re8~gGb;fYwGaN)C{z{n zl^=QA=8xSxR<%`6);%4SlWQK|{E@egEJPPWix+N03h|mP7HWkw~ zQVb2HKoe5-ty~CX*vW8HI!2?%iRA8vye!OM^z@$@O*Lp5{W?M(gyC!8jBI(h0YeR} z5djt<#5DpU?xsF=f#6yM8!$*AuL<6Wzr?;IW>APNza)l=unJLGw*z+S4wbmB?i7ke zQO-sjI$IR9Y29ta|Fo3VSq0!$cToAdojj1yBnDx{PvnG5BuIm1Y1&VD8eR|^su`#% z=nK!zYFpj2yK-{X>#qzy2(-@G*F63?_3gohH{O1snp_OtdG-CtJCi?p{qAi2nZZ?W zaQ@uQbJf=BD|fbk=EeW&Yne-KZhYiMEo~2Zan6h0A8IF_it&cPBOy^75 zSVY`F*a1U^uL8DY3?^fGK$I98Q#wXqo`gF=-WZff4@w2bAm9K}og(O_#ql*`fv=~p zW%3dPXt*(`58RB7Mo*rMeisTnz_*Bu>oM`F#$SM#mr+EDN71aH5dtCt4TFFt0A0#U z!X@A%f}EKyC`W08gyGaR39$lPAu(OVc|p~95&$W|GZg4GK5qzYz!=avMVTdY3a8hV zN{0bdH6RFbB*s&>9{citphR4{t)MD9z>GRanHH$5vw2C?bZ`3P(1qdj(AbI7;|ZMu zGMkk^f^H{HZzl}~kz%&TN(MSC$t)9UG)U2wp*Utfm?xk zOGm9GT=VR$v3m`lWu;x=6K+%D#qU_!I1V7$2@vi|F#zFXu;;i5 zYk#FUbG+n2`A<$-F5!om<9vL{!;vT+bGzK0KM#y`4o}jRc!|iDFCXnw@HfE zlp@(i0p{o^pX`Nsfekhn6FWen|J8tMZ)gUyEy4O0|MTL1Sekf2lR(l%unqJ9SyXb1 z?3V+7W;MBqZRR&q){4UVx|`Z9{DXGB-(~8LQOaZbi~zKNe#O%?&SUn3ao&Er`9#Kn zb%|Yqbc`<*^rQYJZ!_&OzKy>3H_2?)F|a{qi=|7Xp-W5CxEAy4pi6M$xP~sFhAyG4 zbxFA_`IJ7uerT3&Sq0>Wn&nS9r<|${gD=Q@4r+SmZQR-Afb7qqNNagN;a`fuL-V30 z%A!isPXen3wnIi}09LHb#xw{YWHal`L^=39JT{I;hsMsF8ctrokhqoN@QP;e89|l{ z8vX%f8@XBBeFZW|ZJOjOM#f+wAigw9kz6Yj^2Z}%6orXglL}yhx+*UFEw%!g#!*~u zrlh=fb^E5JnHhHcw#kK7u$eeTw|#iu_xveY|NZ+odSN-_for1LM{ ze6bo`@pMBFz0Y~(UV6txF2aI)?pm#V|J~8M#~&i*=n<%VN4`Rgd*E}<|3#oD556a{7TUG{>y@_bf7Xe|XNm>ha$^x(fNo_|5U^(6V>;eQ(<;*RsMr zwZuL3<1>qyAC298sm}GTa7UK7BlpzLxTB9e$lvA{`0}5 zo|osf54g%m&42E9`}^x{hyH$WQF!mW)i?fT@~4B9(OTP~)vn&3-1y+ea@WAJFY#&D zz@5SB!A}k@9ew5P={a#>=$+}Mu7S#_C12uqtZUBpS0ne~(lK{t?r3d$*T?p|r+?;K z{L6bXAD75R56S=)Lf5%6A7pFgrF7BxvXbkwR%SY@x{hnGq^6s^T zncCjbns;oO8-Ebov02h1khJp;u7EjU@ZIYR1z3pZFj)=!U$7b-5ABF+`TFs*C@{)= z{rk2N;FfZh7c3Xdq6oAmOO=U|0VFfHRFl2$A zOjzUuh>~q%D48Hx4K3exoNUeuBcM)flTKS4GKW#h(f~r(n-ESNO(V9N1>w@0%KtH9 z*QW09c=1`&YNoc<{DG2;o`T@(NIdn;aY<*=VU9FtitJ3WiD$mqGXQ-Sici@$t zLlyvX3Qas`_DT@o-Bf^9_);!&_CWhMi;hBSS(w{kYar7p4#*y8OUa&F=2C9(1@J>4 zu0Vr~a`B@r&pP;ZE2r!eB=@J~aAUuLZ9!8acl0_!9cTii8pVYK*tr<(f!XBH3-&!I z$25oy2V}-SraMT0Qgz;lOC*9R2ViV@?{Fe9o`{=N0ooye6(A=hAvp&p(AhXvNbJ#> zv_V34H=VuHLpqBb(I8X~VN+Nm;e@acZDyoYY50P+p<(jdhRJ*ANhwxEh?S7_W(oaK zivl@sgEK)m3Y-C+)O{&0$d}219{d=fMPlUue{#b)YuPz5srJCJ+ct&3nL_NQJB?#i z-99C1O{~BnsU(a$ituxW24`~+cv+wbn(iXZio9}4Dhd3`ej>#|^Ie7Hb_Q{9NOG~H zDTJ%*-0=9RVMQq@3fbc{{zg1HOOJN#ncS4}5@{y()&rDFq%X~y!TJbkb!dgla8_i8 z+31X<+oh~ZsfS9Ev!-}&M0<7yesfV?g;&?$U!4ahuAnu~wzrQ#7R+y3$lktu>vEkB zS5B-3+ivf;wWGRYv8NvFt(;wL-F|!E)$NKXnQt$|`#^C1%FQeD zH*Vhe@ul~t?o7R(zmvZkukU_-B@kZ<#7WR^3EpF^J0T&pZkuPvF150;xNnKo}0)_XY`F2>t0XhXT*j0Q@gIca5~6 zUxoOQPWD&W3H7hr_>m6#uQ3BP91N$)I;PW+R^64RCmw0TKdBR)CVt^D4u0wH0!=P$ zG#e>|an$G{r{&KkkT7(KF$S{v=yBPM#5ybEhkoy7R{~h)I20aU|;q}kQ--_3^ z$3J=LpGW_3wC*`u;XViFD8E&%wI8Sl4pzL6ogCBuc)ObkKHlkJIv+nBVLBdPWWr4B a;aLW;-nnw!zUw~A{q-qeHEcW8=zjrkOA&klh0;fz6q^q&0)KsFR{af*z3E2T+{TzYe%o3|ouYx*yleAJ_kExoF|T z0lhb~T*@|rTS24UnKy6V%>3T(y{GTJ@P~B!L#LBL@che_iPxTMN9e!shyKvDmxq5| zYagKkYl|K9jA4Q;d`SDB<(%-^rb}-BJ#5h42{X!pndE`cv3_{Tq_nT_F&Nyt?2FGN zuN=G@skc2>ei3XcWP-lJgwa=6buzs#8OAyg zvLhBrM#;dL@Q=O&7QJi?)i%bzhEUANnN9GmsmXDc{waibjtp2i6OAN#+ESB8oO#-& zzsG2O|D5fxQS0@gb0)+WtL1aGss%%vW*By7SNrm zX^&a9m1BdFuqCZZK5GW$b;biP$T9;QfynZz^RAn`Q6;B=_M@v#exgaZxqNyfi1s zC-A@-Xu?(*?itn(*N%s)=k=r9_p6-79!)q*yTq@wV|JJF&Pk4R;*ZZ&Yc5XS z^|rltVa2rJ8TgA4F$ex^Mpox)VD-XU@Mg?chhpZ#U%SjGLNPeaE_-K%_YVBJ)3l|#o?=neOmK5Kra;*S19&no}DL!}Gf z?Y({W-J>_D^`5djx?zp}XSlOskG>x$`EQ)Qc6v2?*RkupK-nJMprenr+{k;JBKG{s zw*P;(7?kr2P>)o1^wA&eQRN8i@$XE7dyH$r!4~t60?_=M!2sWPES|w0`p$0O;1T*x zj~UuesHVXe=}%g)^e20m!EQ^U#+sl5Axtv(+8%xm4W=;oL!KDYJyQbekc^QSW{(+Q z1bP}z(1fg>mDK!jJscT#O>j1xyk%PShDopHHNX*am$u7L6Ul zy*e2ZiW&KTW*eTG{i;SHG2>nork8WW6Pm+ggbSAz0nXrNb(aloyGWRo_yutFl$==r z|I~3sn4?p^045T=6QGA_0yhoJ80B(5G+`4#v}#Kd7mSNZ`4VPM)kHL_Mhs_c?PDNr z(ASBBO>rV8zX>9D;8#gNlSiN1gGGPE-jW}xI$ZhTU%P!P#(Q?xit^3A)eGNzp~#iI z>*sFGtk1mt>g~&w-F;Pu_r~b8(W0+-cD?c0yAJkuPR|PWL@#tcupy8CK7$yK+|_tHy9l%v zQGm{}tg@6!iI;S)6p~8`LCJweHk|h zXu!rq6bKesaakKlim8+!3lb<~Lnj6YIfl>}km5YhI-p2aSTUUz=J+%#O6<#`l!V*G zXvq<65KEF7sU3F3FDbB5J`J`khzbOd#Jr#Yr=EvXU1p|V)u=4Z%3~k_tcbS~HmEczQ!Te1E5Zj;N8z& z(Y*6&{7ND%LX?M>R3@DiWOx}Hiwq4#&gagbKM!Bvk$9P_ox-IoGYR~WkOcycND|^Y zEFz?nI85>oPSYYkOEQead`3R1+e#<}K%Ry>R7hu@5a`~Hnbg$APD^pz|Sy^ z3ao2a;cLbKrM>-a(9cX6baEyb1SH zgYX$M=%|Ls1yYt< zW(8nH3@=EW*!Flk+%`eqk^~@CASV+b9fc1bAo98(07)hQRr6TasUcvvTHeWqx^g`S z`;M@0C~OygndCr|77Z2M3m``Iv(q{Vl&@UvWK=3nVjMvjBxx91ss)27Dasni%%Xy) z$W1UraygYzgltks@k{Bf77A&V5iaGhJH}y}Y69E|mq@Osn#evC8YTn4Y6uGufjCyH z#Ah$ZwNsJTLFm8XS6PB4kM28=)14o_=XBqA<@zf{=B{(k3U$xzy>ao{#T##4d-I*y zTXXAkx6Ozv4 zgR(0|fpZ)O=de!KksPzu@R>uBEoMsqX~!&_IRXNb1=nmb^B2K-8t3}tiP?03KjsKf z&prKY%Yh>d)f~YDRUo5PfjO`ZvOtM>Om)5Zi$Y5a#SOFWwdK9DNlAK_N727eD!=?{9OB%3 z2i~HVu9G4is#Zv!6k;sK%Vlc#PD;IO8V~RelSRZECqo~yDGK5s5;P8ei`LRcf|8C(8J`E|e zp!4z%LGmwn$|PVLqJJX_?6`Sy?c~kTwb9b9k9*6}fr|fle(dvL(=E%o<(6yRRk~UU z<_=ePK3nxSR(*kLgTLwz+-uxf+*f$*f!pdb=TF>kK#qph&P{u3)g4%Ie(r89MJn#2 z`H||*)|LJ;-FV-OJPq*a_T)!??Q8(cUj-HCY=l#AC|E=@I3x7ZJUSED}!{1ZwKJrn^ zhb_PC9{3ZYf`>ls9(Z@Ebhg^OtK8iE(1>=n7EML|9c#IJa$pM!fWE#m?f-pkS5><= zKL!;)tn_Hhi~@%rJv4xn(gFv3=SAO85B*8NjN5&Ip(FIq-1eat%|C0gK^vyV!%ctAgeN`PL7|M7#C3Y delta 123 zcmaEqv&2^YGcPX}0}y=T4ah1HVPJR+;=lkCl<~P_qPhno@5BNfmUJdfzRgOEMzXA$ zjDDH|lO;?;m5qVQiugbTFOay!VUwGmQks)$S5yGxG6He2@8pxF-dvvr1Q~_F3IXX# B7*YTL diff --git a/__pycache__/fccs_verify.cpython-313.pyc b/__pycache__/fccs_verify.cpython-313.pyc index 684a9efb605ae403828e6d5138ec206d757cbfb2..a41e3e0f174d340d1bbc26369a507a9c4bfee762 100644 GIT binary patch literal 4975 zcmbtYU2NOd6~2@xi=yZsS&{50c6<{%vE{{b9j9|EI8GDW&Ft7tCx!iiiwUem%S>cS zAZf?ehE3=?z*!5#8QMj|ia|v;G<_K0p?yg6)V=LdhJ@PTf}(4S0ews7ZfoCmE=AI^ zmp&A|fOvWT&OPTl=ic*CyN!)3g7V;}v8zAi5c-TXtl}_VcAtfpH&7fYC{D#4mnelY zYKP*0+Ih)2O)K=YOL0v%C=JtY#XZd^3`TJ}gcQ$!O0SMM1-h0xY(dMR78_loJ$?C%Y6G=ckmQuKBBho=O3iSPDuZy?K>`@pBJ39=80W$+orPFX#e|@wRo$D(3aLcW zfKK-=3D?9#CYw%+ir#zxNId{sh^q-BG!rkanfNBn5H~7vyqxgcZD>H~C(Uo5dCD5L zj)IKl$qb|LC`McFd-dby8kw4RTfICL^H}eDH944%n+Mlk<@naQPHV30{MI?NJ%^q* zE(@|-3@(df?3z`0cCF(o1&nE?c*wrf#e? ziqUt{>aevdZ`-}pZ1};uC(&qg2igb68|RPMvRu~OCKPWPwk4asY_>)*8bopf{M@|D zkACzvLNYVoX-i?QG(Reqd1eqwG|7qb<)mO~)TCk8iZB0v~nrO-2OR_<^OiW+}Mycw26Y zffl*jdaU&(Dz?vo_Mq^IIo*{Lv_(~CAhQ_hLpSqm0_cOzSuBLK@+T%^vCFZj&0Xpk zhMd(PwJyu4D9#VYmLXM=bXd#Sgb6H23UDRJifK`Xya!~%lB`C)pa8#ow<)}~=Fy^) zWTsU}b{dA??r-7IjW7^H9zn#OgNUUdht$tC*Tn5;v+z$rhMI+2WZukFkWU_w^vuik!_A^o9%0!ui@+NIOQH3sl&drz6^`rV#0KVOnQIIi(VUp*ymwP7~;L z#$Y?*sup(Wj>uy=wV;ye80+1Gi`gY{@Vc-z7z6gE4n93T9v?K2T`@Q*Agg;u38bqz zWlV2d0!kVg&18X$sbdj)0P-ACpM=L3sDK_Ky1n63UwHlej+eU`y%F75*!Fhrx{#;q zHOEc&4fi(FzQwc`eHG?N@%%^3QQa5*MkPt@<)@u!8?u}%%<12zHeQ^OWN zj^;hMRtE6Cpd^uC1m#okGDg*frJsc59_6|MRESKrg2)UXxI)R5nnomi-lgWklj59-vD&CVq7;cHK9VE z)?=m+o8LtyUJ2set`J}A z&a|j0gi9(U+;o?ul1tX9uEAX(u1cETpc@2XvCh^!l8{oW&RI$tEmpFD&z*ASAG9cG!ay2QVFy2lk{uMm< zbpbu_23{R4Ogvx%8>!pZZe6Re#|z`6mAsw4l`f`B`AYC~g&i)8f8slIyX{t6v8{Bf z;u|bX@A!kaM{bQ2M@n-Q|8QaEb8q9#ksBj7&)zs&?AZ36-145h&v?sC!<#dkV`c6_ zl^G{(?rfDAd%$>aGQVave6M-ymxZ>yJzL%$W7*JV|7J&-8>=$s>Sz1fz9hHj=RW`K z&~HM;nW`_m>w=D5M7K5+rvJktE@*6XvdZ+Qn>sN+P%)aGJBtPmzka_ zeF9#rHrx4Tvef_kY`HUXXR*wTROu(ksNA-6+UF|{_G*5W~g3x`ue^?RAczFK*t7kzs0{Hm8hb+ zarvW`o?Ryjc7Ayn1@TuH1v}mxFEv*>9(#XyGymSXt&R(oz<6Qqfj?LcUi_e`61-UP zUo6ZJw7~v)kh=cpEX`E#P`T@LnHjFqX9$!m>79PN8T2YMLsk0p$8MkDM_fAe&f&KX zZ}m@n5UTV~RE|xQ@no5My2@Pq*c~vYbeAUHnR;vLj$G+ETkakM8_u=tv~;Zhy!hNh zK=dTowzrquB=qe98SK#426hI;h z5CPo~ruZ-uiN$$|;i6njkW$PN zC&VJWl#+ZjJ(*Zaaw06I7I-Gfi+y2{yTyxD7$(FeQ_CU{2``H=AzC5Ic$94iy|hXd z6W~IrWiDRXE+!~M%Sqx$HlSj<_k zUZ#ONi4SsORFW<61Q!m55<(;vl`Tu$4L%r8M5DYIA}U(JHxvBz@ZT|l`rkwI+8V1U zKcFAc>ySXOgdM%6lXQM0>Fns$qX;EB=P=%$p(@-{c^?Lt=cpvLFTtdO;U3;NOXgLon#7UZuw^0xf;Oh>87g`v**l8w3<+DqtXyK+a=Xl@Tz` zo7HDl;~`)^QIQB_HE6N|Wa7#-Hel`p6cPcamI|jv!N3{;)}+C@5E4u^v`wmf0W;=D z(S`$fG65BLnTn(nOx+;UF8$gS{lY$bM}Sr-0tKf=LDvm-Vc;+5BB%rPB2`~u&~F4* z4uNJFH2Zg_^@mV|sK6Q`Dwaum0(`H9n*g+n;64nexwhhK3@3yckp~YKn1sMHV&WDK z116jZEiZwSLWsNyAUJ%kCsGScBq1(wDQT2um_BBfzcs30sn7~)JO*?;Deiio#Dn*8 zH~>S5`0|n@JQO&r~^t&rkI-m$@j#+5S|-G_VL27)JLM&0iWfqOFa)nm(6b`; zY}ugj{S;J%a|$$&o+3Q576 zN+$FrZ_9N`_E2-5cmN1MzDCl2!es@0(S+#wOgKB4@A;Ye=oOSaCe<6hfpTkgsuTjxh_W(SOnxBCQ;)0%K5`Z< z{sQTL@w@@qUM3WFJ~bcx-E$f>P7p8T2LFp+TKv)oNcbJYWS4%`J=t#j2Pa(qMW=)L z$EL=~VMq;zoL3HRyxW|$ zx~KXyhn-{3sAg6z8+q|b(1`*DmaIoZI+aVN{MJ~V&o`>X=9<)}>K!O(nUz8R56<-J zP8w$&70xJDznil^g_C|H80H}>mq!g>a4zf}ov5lGl^Zx&V{n2PS>qg>nR^GEPdHb= z(748@Ya9Aw8UTpBgVl5$pz&D>r`ACgriSnREPY6YtKP4jQ~gBcP@t~8&6;`uWadWs zlwu7U*XPIR+bJ!?*M%5b_G&5DP7mm|`|fAR0FBiGyrM;5~M?=1EoGK)WN^rkA#M5oK<%M4So~*^0ljqPd9=#_Q{j1sj3aZ zR>Mc5x|2}LolSddXY1IeKFI1q9b3=Zm8W{PiM1=WrVZ?Xb>?JEAcu>!!?k%x(*kJr zGCmxzEvaMs|ruuXv9s!(ZTo%M)y!2H0FrYq26!@jnx z$&531%C?+8rdkCwg3H)FLp_aDNvrUz;r{}!NaULLt7KXCaUyWA6)jS%%dRzHDs`b* zk+a6G%Di1&uXxBIwr+POR-hQRe$QK%8d)58wgS@;=$zXpCTl##rS7HPLAPye^InUw z4H|98p$hG;{b{pWWbEedQUj~;7p5vSy477PlGa2~RISL(P=_C$l?}MYlVl1yw;*my zWxCuT2BATet!jCOXX(SQ%*|bz8?6@X)gDqvq@WaE7NCw!`{$MgMk)5nl%Uvy(jMwe z+-XL6fyX7dMnv+7xC=>lRfvD7J*_28`clx9fs~3-8C=bZskD7Y?bsPy#6pt*VpB}HRhQECGAg_-_xTK7HiGmiE-s_A zj)B=IbLj%(Nt?44Z(lsEL_HkF2q2a%_2xu(Bwf;S2@xVx&oyd23DQpCVI~RNu$!_MbWT6LU@T; zFCpTJ2%3GJuU{s@a#PtRd^mXhcCaEOt|=L2MZ_n+Y#zgAeJd%B%gsyhO~W@9PlUL* zH15+D;EYBR4w>IRh3dZGn*QndI`^(rXoFI`*w(+Z|h0U5*4@9fs8PGHJ(5OoHArv;g|ANDV7Z6Y;n@1qXy> zpVuv7b%>aC*;xJ_B4S0$`Y4~0bqPs?mK@r#n1q8$FWr`8l7B0fl8ut`JxDgip~1v! z(8Pr$+)&Dtgqtw@5e9cgGP#(D39>OJDKSb$VnSG^B|a7ABiwQvC~-;X>cWcUUk zO0lRQ*C)i}0w<_$CtE9Xr5P5F0%H;`WWm1sjjxYxYeoNLbamTdl&=`FkK z-tg*hZa6<%vJb6X{LE5!clgfm-P3nY=MHRIjub6N9#fV={m}Zw_3?uB4xQj6BQY3VLnx)rp6^`q+t3fA!r>P!vvGb6IuSI%$Sn%~!bK!1uR7N6FH$?QZ?RblN5>C3 zKkRwX^I`9U-nFlnIwwnxsmv=|U7l>LWMw`#qo%>%SdrDKFneT!>e;=0>wyC4`poXU zH~KfDxy22;cSo;x7-hRR!);eI0BpS9`9aV9o)3EO_ksdjPi$IFLL+RZi+1lbJu=vy zA;8(8QBylGUT?p|JT1Orpeq=q1d2RpPiVReS`o_#-(o~?jH^{^Apw8)o?}zf)U?^AaXp!p9)NfIa995*4 zjP;4dnRA0-9$Oo-o!P(4UdY|dFXUfeyH#i$eq=rM#NL)$EZRMpsjb%byzTp*)uA#17#k?+mJbz@(v1VN3p6LnOz!!+98+>AQZCVc&tp{_l zM^?}0hmfo1i*Dp_`D@F4?D7F8-pdkX_o$VNDh&-v;R~>oFJ4eXT(CZjgPyHYpxc-;wYB(EGsq_vhBf zOT90HdlfpSfa_O?#;wK!nZL?i1zzip0I%jA{FSCjarVyq@efBHjI6z0>N-{E94S~w zKc!y!d0m_0gUkliftjJ+o_XjiEFJK}#`vrp^p&h_N<-|daEhWOsaeiE!hj;iS zl}^D0979+TAD=fL;L JR`GQ}{vS*KS^59~ diff --git a/fccs_check.py b/fccs_check.py index b5bb608..60e6f02 100644 --- a/fccs_check.py +++ b/fccs_check.py @@ -6,7 +6,7 @@ its manifest (the documents FCCS said it would export, captured during Step 2) against the files actually sitting in the export folder, and reports any missing documents. -Handles two quirks of the FCCS export: +Handles two quirks of the FCCS export (see fccs_config.evaluate_drawer): 1. Page-splitting: a single manifest document (e.g. "Donations") is exported as one file if it's a single page, or as "Donations Page 1", @@ -14,10 +14,11 @@ Handles two quirks of the FCCS export: that one document being present. 2. Filename sanitization: FCCS strips characters that are illegal in Windows - filenames (: / \\ ? * " < > |) from document titles, so a manifest title - like 'US Tax Return (... 01:35PM)' won't match the exported file - character-for-character. Comparison is done on a normalized key - (lowercase, alphanumerics only) so these still match. + filenames (: / \\ ? * " < > |) from document titles, so comparison is done + on a normalized key so these still match. + +fccs_verify.py applies this same logic in batch across every drawer; this tool +is for spot-checking specific drawers (e.g. ones the log marked failed). USAGE ----- @@ -29,94 +30,37 @@ import os import re import sys -from fccs_config import parse_args, load_config -from fccs_verify import load_manifest # reuse the both-format manifest loader +from fccs_config import ( + parse_args, load_config, evaluate_drawer, index_files_by_drawer, +) -# Trailing " Page N" (optionally "Page N of M") appended to multi-page exports. -_PAGE_RE = re.compile(r"\s*Page\s+\d+(?:\s+of\s+\d+)?\s*$", re.IGNORECASE) -# Creation-date field ("_MM-DD-YYYY_") that precedes the document name. -_DATE_ANCHOR = re.compile(r"_\d{2}-\d{2}-\d{4}_") +def report_drawer(drawer_id, files, manifest_dir, out): + """Evaluate one drawer and print a human-readable completeness report.""" + r = evaluate_drawer(drawer_id, files, manifest_dir) - -def match_key(name): - """Normalize a document name for tolerant comparison. - - Strips a trailing 'Page N' page-split suffix, then reduces to lowercase - alphanumerics so punctuation and filename-sanitization differences don't - cause false mismatches. - """ - base = _PAGE_RE.sub("", name) - return re.sub(r"[^a-z0-9]+", "", base.lower()) - - -def manifest_doc_names(path, drawer_id): - """Return the expected document (Page Title) names from a manifest file.""" - rows = load_manifest(path) - names = [] - for row in rows: - if len(row) >= 3 and row[0].strip() == drawer_id: - names.append(row[1]) # old format: DrawerID, PageTitle, Application - elif row: - names.append(row[0]) # new format: PageTitle, Application - return names - - -def exported_doc_name(filename): - """Extract the document-name portion from an exported filename, or None. - - Format: {drawer}_{client}_{folder}_{MM-DD-YYYY}_{docname}.ext - The creation-date field is a reliable anchor; the doc name follows the last - one (the client/folder fields don't carry an "_MM-DD-YYYY_" pattern). - """ - stem = os.path.splitext(filename)[0] - anchors = list(_DATE_ANCHOR.finditer(stem)) - if not anchors: - return None - return stem[anchors[-1].end():] - - -def check_drawer(drawer_id, files, manifest_dir, out): - """Report completeness of one drawer's export.""" - manifest_path = os.path.join(manifest_dir, drawer_id + ".txt") - if not os.path.exists(manifest_path): - out("") - out(f"[{drawer_id}] NO MANIFEST at {manifest_path} — cannot verify " + out("") + if not r["has_manifest"]: + out(f"[{drawer_id}] NO MANIFEST at {r['manifest_path']} — cannot verify " "(was this drawer exported by the tool?)") return - expected = manifest_doc_names(manifest_path, drawer_id) - - # Group exported files by normalized doc name; page-splits collapse together. - exported = {} # key -> list of full doc names (one entry per file/page) - unparsed = [] - for f in files: - doc = exported_doc_name(f) - if doc is None: - unparsed.append(f) - continue - exported.setdefault(match_key(doc), []).append(doc) - - missing = [name for name in expected if match_key(name) not in exported] - - expected_keys = {match_key(n) for n in expected} - extras = [names[0] for k, names in exported.items() if k not in expected_keys] - - out("") + expected = r["expected"] + missing = r["missing"] out(f"[{drawer_id}] manifest lists {len(expected)} document(s); " - f"{len(files)} file(s) in export folder.") + f"{r['file_count']} file(s) in export folder.") if missing: out(f" INCOMPLETE — {len(missing)} document(s) missing from export:") for m in missing: out(f" - {m}") else: out(f" COMPLETE — all {len(expected)} manifest document(s) present.") - if extras: - out(f" Note: {len(extras)} exported document(s) not in the manifest:") - for e in extras: + if r["extras"]: + out(f" Note: {len(r['extras'])} exported document(s) not in the manifest:") + for e in r["extras"]: out(f" - {e}") - if unparsed: - out(f" Note: {len(unparsed)} file(s) had no recognizable date anchor " + if r["unparsed"]: + out(f" Note: {r['unparsed']} file(s) had no recognizable date anchor " "and were skipped.") @@ -130,14 +74,7 @@ def main(): print(f"ERROR: export directory not found: {export_dir}") sys.exit(1) - # Index every export file by its leading drawer-ID token (before first "_"). - # The underscore boundary keeps clashing IDs separate (04289 vs 04289TS). - files_by_drawer = {} - for f in os.listdir(export_dir): - if not os.path.isfile(os.path.join(export_dir, f)): - continue - token = f.split("_", 1)[0] - files_by_drawer.setdefault(token, []).append(f) + files_by_drawer = index_files_by_drawer(export_dir) print("FCCS export completeness check") print(f" export folder : {export_dir}") @@ -153,8 +90,8 @@ def main(): break ids = [i for i in re.split(r"[\s,]+", raw) if i] for drawer_id in ids: - check_drawer(drawer_id, files_by_drawer.get(drawer_id, []), - manifest_dir, print) + report_drawer(drawer_id, files_by_drawer.get(drawer_id, []), + manifest_dir, print) if __name__ == "__main__": diff --git a/fccs_config.py b/fccs_config.py index f26d3ca..1a29108 100644 --- a/fccs_config.py +++ b/fccs_config.py @@ -157,3 +157,135 @@ def check_for_clashes(drawer_ids): if matches: clashes.append((short, matches)) return clashes + + +# --------------------------------------------------------------------------- +# MANIFEST / EXPORT-COMPLETENESS HELPERS +# --------------------------------------------------------------------------- +# Shared by fccs_verify.py (batch) and fccs_check.py (interactive) so both +# judge completeness identically. + +# Trailing " Page N" (optionally "Page N of M") appended to multi-page exports. +_PAGE_RE = re.compile(r"\s*Page\s+\d+(?:\s+of\s+\d+)?\s*$", re.IGNORECASE) +# Creation-date field ("_MM-DD-YYYY_") that precedes the document name. +_DATE_ANCHOR = re.compile(r"_\d{2}-\d{2}-\d{4}_") + + +def load_manifest(path): + """Load a manifest file and return one row (list of cells) per document. + + Handles both formats: + - New: one document per line (tab-separated columns). + - Old: a raw ListView dump led by 'List1', then row-major cells with + 3 columns per document (Drawer ID, Page Title, Application), reshaped + so the document count is correct. + """ + with open(path, "r", encoding="utf-8") as f: + lines = [line.rstrip("\n") for line in f if line.strip()] + + if lines and lines[0].strip() == "List1": + cells = lines[1:] + rows = [cells[i:i + 3] for i in range(0, len(cells), 3)] + return [r for r in rows if len(r) == 3] + + return [line.split("\t") for line in lines] + + +def manifest_doc_names(path, drawer_id): + """Return the expected document (Page Title) names from a manifest file.""" + rows = load_manifest(path) + names = [] + for row in rows: + if len(row) >= 3 and row[0].strip() == drawer_id: + names.append(row[1]) # old format: DrawerID, PageTitle, Application + elif row: + names.append(row[0]) # new format: PageTitle, Application + return names + + +def match_key(name): + """Normalize a document name for tolerant comparison. + + Strips a trailing 'Page N' page-split suffix, then reduces to lowercase + alphanumerics so punctuation and filename-sanitization differences (FCCS + strips characters illegal in Windows filenames) don't cause false + mismatches. + """ + base = _PAGE_RE.sub("", name) + return re.sub(r"[^a-z0-9]+", "", base.lower()) + + +def exported_doc_name(filename): + """Extract the document-name portion from an exported filename, or None. + + Format: {drawer}_{client}_{folder}_{MM-DD-YYYY}_{docname}.ext + The creation-date field is a reliable anchor; the doc name follows the last + one (client/folder fields don't carry an "_MM-DD-YYYY_" pattern). + """ + stem = os.path.splitext(filename)[0] + anchors = list(_DATE_ANCHOR.finditer(stem)) + if not anchors: + return None + return stem[anchors[-1].end():] + + +def index_files_by_drawer(export_dir): + """Map each export file to its leading drawer-ID token (before first '_'). + + The underscore boundary keeps clashing IDs separate (04289 vs 04289TS). + Returns {drawer_id: [filenames]}. + """ + index = {} + for f in os.listdir(export_dir): + if not os.path.isfile(os.path.join(export_dir, f)): + continue + token = f.split("_", 1)[0] + index.setdefault(token, []).append(f) + return index + + +def evaluate_drawer(drawer_id, files, manifest_dir): + """Compare a drawer's manifest against its exported files. + + `files` is the list of export filenames belonging to this drawer. Accounts + for page-splitting (a document exported as 'Name Page 1/2/...' counts as + present) and filename sanitization (via match_key). + + Returns a dict: + has_manifest, manifest_path, expected (list), missing (list), + extras (list), unparsed (int), file_count (int) + """ + manifest_path = os.path.join(manifest_dir, drawer_id + ".txt") + result = { + "has_manifest": os.path.exists(manifest_path), + "manifest_path": manifest_path, + "expected": [], + "missing": [], + "extras": [], + "unparsed": 0, + "file_count": len(files), + } + if not result["has_manifest"]: + return result + + expected = manifest_doc_names(manifest_path, drawer_id) + result["expected"] = expected + + # Group exported files by normalized doc name; page-splits collapse together. + exported = {} # key -> list of full doc names (one entry per file/page) + unparsed = 0 + for f in files: + doc = exported_doc_name(f) + if doc is None: + unparsed += 1 + continue + exported.setdefault(match_key(doc), []).append(doc) + + result["missing"] = [name for name in expected + if match_key(name) not in exported] + + expected_keys = {match_key(n) for n in expected} + result["extras"] = [names[0] for k, names in exported.items() + if k not in expected_keys] + result["unparsed"] = unparsed + return result diff --git a/fccs_verify.py b/fccs_verify.py index aa633df..5e28f9b 100644 --- a/fccs_verify.py +++ b/fccs_verify.py @@ -1,35 +1,24 @@ """ Step 4 (optional): Verify exported files against manifests. -Compares the per-drawer manifests captured during export (Step 2) -against the actual files in the export directory to identify -missing or extra files. +Batch check across EVERY drawer that has a manifest: compares each drawer's +manifest (captured during Step 2) against the files in the export directory and +reports which drawers are complete vs missing documents. + +Matching is document-level and identical to fccs_check.py (via +fccs_config.evaluate_drawer): it accounts for page-splitting (a document +exported as 'Name Page 1', 'Name Page 2', ... counts as present) and for +filename sanitization (titles with characters illegal in filenames still +match). Use fccs_check.py to spot-check individual drawers interactively. """ import os import sys -from fccs_config import parse_args, load_config, make_logger - - -def load_manifest(path): - """Load a manifest file and return one row per document. - - Handles both formats: - - New: one document per line (tab-separated columns). - - Old: a raw ListView dump led by 'List1', then row-major cells with - 3 columns per document (Drawer ID, Page Title, Application). We - reshape it so the document count is correct. - """ - with open(path, "r", encoding="utf-8") as f: - lines = [line.rstrip("\n") for line in f if line.strip()] - - if lines and lines[0].strip() == "List1": - cells = lines[1:] - rows = [cells[i:i + 3] for i in range(0, len(cells), 3)] - return [r for r in rows if len(r) == 3] - - return [line.split("\t") for line in lines] +from fccs_config import ( + parse_args, load_config, make_logger, + evaluate_drawer, index_files_by_drawer, +) def main(): @@ -49,77 +38,56 @@ def main(): log(f"ERROR: export directory not found: {export_dir}") sys.exit(1) - # Load all manifests - manifests = {} - for fname in sorted(os.listdir(manifest_dir)): - if fname.endswith(".txt"): - drawer_id = os.path.splitext(fname)[0] - items = load_manifest(os.path.join(manifest_dir, fname)) - manifests[drawer_id] = items - - if not manifests: + drawer_ids = sorted( + os.path.splitext(f)[0] + for f in os.listdir(manifest_dir) + if f.endswith(".txt") + ) + if not drawer_ids: log("No manifest files found.") sys.exit(1) - # Index exported files by drawer ID prefix - exported_by_drawer = {} - all_exported = [ - f for f in os.listdir(export_dir) - if os.path.isfile(os.path.join(export_dir, f)) - ] - - for fname in all_exported: - sep = fname.find("_") - if sep != -1: - did = fname[:sep] - exported_by_drawer.setdefault(did, []).append(fname) + files_by_drawer = index_files_by_drawer(export_dir) log("=" * 60) log("Export Verification Report") log("=" * 60) - log(f"Manifests loaded : {len(manifests)} drawers") - log(f"Exported files : {len(all_exported)} total") + log(f"Manifests loaded : {len(drawer_ids)} drawers") log("") - total_expected = 0 - total_actual = 0 - mismatched = [] + complete_ids = [] + incomplete = [] # (drawer_id, missing_list) - for drawer_id in sorted(manifests): - expected = manifests[drawer_id] - actual = exported_by_drawer.get(drawer_id, []) - n_expected = len(expected) - n_actual = len(actual) - total_expected += n_expected - total_actual += n_actual - - if n_expected == n_actual: - log(f" {drawer_id}: OK ({n_actual} files)") + for drawer_id in drawer_ids: + r = evaluate_drawer(drawer_id, files_by_drawer.get(drawer_id, []), + manifest_dir) + n_expected = len(r["expected"]) + missing = r["missing"] + if not missing: + complete_ids.append(drawer_id) + log(f" {drawer_id}: OK ({n_expected} docs, {r['file_count']} files)") else: - diff = n_actual - n_expected - sign = "+" if diff > 0 else "" - log(f" {drawer_id}: MISMATCH — expected {n_expected}, " - f"got {n_actual} ({sign}{diff})") - mismatched.append(drawer_id) + incomplete.append((drawer_id, missing)) + log(f" {drawer_id}: INCOMPLETE — {len(missing)}/{n_expected} " + f"document(s) missing:") + for m in missing: + log(f" - {m}") - # Check for exported files with no manifest - orphan_drawers = set(exported_by_drawer.keys()) - set(manifests.keys()) - orphan_count = sum(len(exported_by_drawer[d]) for d in orphan_drawers) + # Exported files whose drawer has no manifest at all. + orphan_drawers = sorted(set(files_by_drawer) - set(drawer_ids)) + orphan_count = sum(len(files_by_drawer[d]) for d in orphan_drawers) log("") log("-" * 60) - log(f"Expected total : {total_expected}") - log(f"Actual total : {total_actual}") - + log(f"Complete drawers : {len(complete_ids)}") + log(f"Incomplete drawers : {len(incomplete)}") + if incomplete: + log(f" Incomplete IDs: {', '.join(d for d, _ in incomplete)}") if orphan_drawers: - log(f"No manifest for : {', '.join(sorted(orphan_drawers))} " + log(f"No manifest for : {', '.join(orphan_drawers)} " f"({orphan_count} files)") - - if mismatched: - log(f"Mismatched drawers: {', '.join(mismatched)}") - elif not orphan_drawers: - log("All drawers match.") - + if not incomplete and not orphan_drawers: + log("All drawers complete.") log("=" * 60)