updated how we are doing manifests so they are more clean and easier to check file formats
This commit is contained in:
@@ -13,14 +13,23 @@ from fccs_config import parse_args, load_config, make_logger
|
||||
|
||||
|
||||
def load_manifest(path):
|
||||
"""Load a manifest file and return the list of document rows."""
|
||||
items = []
|
||||
"""Load a manifest file and return one row per document.
|
||||
|
||||
Handles both formats:
|
||||
- New: one document per line (tab-separated columns).
|
||||
- Old: a raw ListView dump led by 'List1', then row-major cells with
|
||||
3 columns per document (Drawer ID, Page Title, Application). We
|
||||
reshape it so the document count is correct.
|
||||
"""
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.rstrip("\n")
|
||||
if line:
|
||||
items.append(line.split("\t"))
|
||||
return items
|
||||
lines = [line.rstrip("\n") for line in f if line.strip()]
|
||||
|
||||
if lines and lines[0].strip() == "List1":
|
||||
cells = lines[1:]
|
||||
rows = [cells[i:i + 3] for i in range(0, len(cells), 3)]
|
||||
return [r for r in rows if len(r) == 3]
|
||||
|
||||
return [line.split("\t") for line in lines]
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
Reference in New Issue
Block a user