forked from admin/miem_workers
fix: prepare indexed council data and skip ineligible publication parsing
This commit is contained in:
1 parent
9f5e508b47
commit
dffe1afb14
10 files changed
+49419
-62
No files matched your search
@@ -0,0 +1,33 @@
|
||||
"""Prepare a readable, indexed council JSON from the owner-supplied CSV.
|
||||
|
||||
python scripts/prepare_council_articles.py app/data/council_articles.csv
|
||||
No database access or app startup; only Python's standard library is required.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
from tempfile import NamedTemporaryFile
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from app.services.council_source import load_reference, prepare_reference
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("source", type=Path)
|
||||
parser.add_argument("--output", type=Path, default=Path("app/data/council_articles.json"))
|
||||
args = parser.parse_args()
|
||||
if args.source.resolve() == args.output.resolve():
|
||||
parser.error("Output must not overwrite the source CSV")
|
||||
result = prepare_reference(args.source)
|
||||
text = json.dumps(result, ensure_ascii=False, indent=2) + "\n"
|
||||
with NamedTemporaryFile(mode="w", encoding="utf-8", dir=args.output.parent,
|
||||
prefix=args.output.name + ".", suffix=".tmp", delete=False) as stream:
|
||||
stream.write(text)
|
||||
temporary = Path(stream.name)
|
||||
try:
|
||||
load_reference(temporary)
|
||||
temporary.replace(args.output)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
print(f"Prepared {result['source_rows']} publications and {len(result['by_author'])} author names")
|
||||
Reference in new issue
Block a user