""" WSP CLI runner. Usage: python -m wsp.runner status python -m wsp.runner incremental [op_name|all] python -m wsp.runner backfill --from YYYY-MM-DD --to YYYY-MM-DD [--workers 3] python -m wsp.runner test """ from __future__ import annotations import argparse import logging import sys from datetime import datetime, timedelta from . import db, sync from .operations import ALL_OPS, PUBLIC_OPS, OWN_OPS logging.basicConfig( level=logging.INFO, format='%(asctime)s %(levelname)s [%(name)s] %(message)s', datefmt='%Y-%m-%d %H:%M:%S', ) log = logging.getLogger('wsp.runner') def cmd_status(args): with db.connection() as conn: cur = conn.cursor() cur.execute(""" SELECT feed, last_run_at, last_cursor_date, items_imported_total, items_imported_24h, consecutive_errors, last_error FROM seap.wsp_sync_state ORDER BY feed """) rows = cur.fetchall() print(f'\n{"Feed":<25} {"Last run":<22} {"Cursor":<22} {"Total":>10} {"Errors"}') print('-' * 100) for r in rows: feed, last_run, cursor, total, _, errs, err_msg = r err_str = err_msg[:40] + '...' if err_msg and len(err_msg) > 40 else (err_msg or '') print(f'{feed:<25} {str(last_run)[:19]:<22} {str(cursor)[:19]:<22} ' f'{total or 0:>10,} {errs}/{err_str}') cur.execute(""" SELECT feed, state, count(*) FROM seap.wsp_backfill_windows GROUP BY feed, state ORDER BY feed, state """) rows = cur.fetchall() if rows: print(f'\n{"Backfill queue":<30}') print('-' * 60) for feed, state, count in rows: print(f' {feed:<25} {state:<15} {count:>8}') cur.execute(""" SELECT source, count(*) FROM seap.announcements WHERE source LIKE 'wsp_%' GROUP BY source ORDER BY source """) rows = cur.fetchall() if rows: print(f'\n{"Source":<25} {"Rows in seap.announcements"}') print('-' * 60) for source, count in rows: print(f' {source:<25} {count:>10,}') def cmd_incremental(args): target_ops = args.ops if args.ops != ['all'] else list(ALL_OPS) for op_name in target_ops: if op_name not in ALL_OPS: log.error('Unknown op: %s', op_name) continue log.info('=== Running incremental: %s ===', op_name) result = sync.run_incremental(op_name, lookback_hours=args.lookback_hours) log.info('Result: %s', result) def cmd_backfill(args): if args.op not in ALL_OPS: log.error('Unknown op: %s. Available: %s', args.op, list(ALL_OPS)) sys.exit(1) start = datetime.fromisoformat(args.start) end = datetime.fromisoformat(args.end) log.info('=== Backfill %s [%s, %s) workers=%d ===', args.op, start, end, args.workers) result = sync.run_backfill(args.op, start, end, workers=args.workers) log.info('Done: %s', result) def cmd_test(args): """Run a 1-day end-to-end test on yesterday (all SU_* feeds).""" yesterday = datetime.now().date() - timedelta(days=1) start = datetime.combine(yesterday, datetime.min.time()) end = datetime.combine(yesterday, datetime.max.time()) feeds_to_test = ['SU_CaNotices', 'SU_CNotices', 'SU_PiNotices', 'SU_RfqNotices', 'SU_RfqInvitations', 'SU_DCNotices', 'SU_PCNotices', 'SU_RdcNotices', 'SU_ENotices'] log.info('=== Smoke test on %s ===', yesterday) for op_name in feeds_to_test: log.info('--- %s ---', op_name) try: r = sync.run_backfill(op_name, start, end, workers=1, enqueue=True) log.info(' → %d items imported', r['items_imported']) except Exception as e: log.exception(' → ERROR: %s', e) def main(): parser = argparse.ArgumentParser(prog='wsp.runner') sub = parser.add_subparsers(dest='cmd', required=True) sub.add_parser('status') p_inc = sub.add_parser('incremental') p_inc.add_argument('ops', nargs='*', default=['all'], help='Operations to sync (default: all)') p_inc.add_argument('--lookback-hours', type=int, default=36) p_bf = sub.add_parser('backfill') p_bf.add_argument('op') p_bf.add_argument('--start', required=True, help='YYYY-MM-DD or full ISO') p_bf.add_argument('--end', required=True) p_bf.add_argument('--workers', type=int, default=3) sub.add_parser('test') args = parser.parse_args() handlers = { 'status': cmd_status, 'incremental': cmd_incremental, 'backfill': cmd_backfill, 'test': cmd_test, } handlers[args.cmd](args) if __name__ == '__main__': main()