#!/usr/bin/env python3
"""Charity Commission register: for each annual-return cycle, share of charities whose accounts and annual
return were received by the reporting due date, days late, and the share still outstanding, count- and
income-weighted; restricted to charities currently registered (charity extract) with income >= 25,000 (the
threshold at which accounts must be filed)."""
import csv, datetime as dt, collections, json, sys
csv.field_size_limit(1<<30)
def d(s):
    try: return dt.date.fromisoformat(s[:10])
    except Exception: return None
reg = {}
for r in csv.DictReader(open('publicextract.charity.txt', encoding='utf-8-sig'), delimiter='\t'):
    if r.get('charity_registration_status') == 'Registered' and r.get('linked_charity_number') in ('0', '', None):
        reg[r['organisation_number']] = r
print('registered main charities', len(reg), file=sys.stderr)
extract = None; by = collections.defaultdict(lambda: {'n': 0, 'ontime': 0, 'late': 0, 'missing': 0, 'inc': 0.0, 'inc_late': 0.0, 'inc_missing': 0.0, 'days': [], 'late_over_1m': 0, 'late_names': []})
for r in csv.DictReader(open('publicextract.charity_annual_return_history.txt', encoding='utf-8-sig'), delimiter='\t'):
    extract = extract or d(r['date_of_extract'])
    if r['organisation_number'] not in reg: continue
    inc = float(r['total_gross_income'] or 0)
    if inc < 25000: continue
    due = d(r['reporting_due_date']); acc = d(r['date_accounts_received']) or d(r['date_annual_return_received'])
    if not due or due > extract: continue
    cyc = r['ar_cycle_reference']; b = by[cyc]; b['n'] += 1; b['inc'] += inc
    if acc is None:
        b['missing'] += 1; b['inc_missing'] += inc
    elif acc > due:
        b['late'] += 1; b['inc_late'] += inc; b['days'].append((acc - due).days)
        if inc >= 1_000_000: b['late_over_1m'] += 1; b['late_names'].append((reg[r['organisation_number']]['charity_name'], inc, (acc - due).days))
    else: b['ontime'] += 1
out = {}
for cyc, b in sorted(by.items()):
    if b['n'] < 1000: continue
    days = sorted(b['days'])
    out[cyc] = {'n': b['n'], 'on_time_pct': round(100 * b['ontime'] / b['n'], 1), 'late_pct': round(100 * b['late'] / b['n'], 1), 'missing_pct': round(100 * b['missing'] / b['n'], 2),
                'income_on_time_pct': round(100 * (b['inc'] - b['inc_late'] - b['inc_missing']) / b['inc'], 1), 'late_p50_days': days[len(days) // 2] if days else None, 'late_p90_days': days[int(.9 * (len(days) - 1))] if days else None,
                'late_over_1m_income': b['late_over_1m'], 'late_names_top': sorted(b['late_names'], key=lambda x: -x[1])[:10]}
    print(cyc, {k: v for k, v in out[cyc].items() if k != 'late_names_top'})
json.dump(out, open('results.json', 'w'), indent=1, default=str)
for cyc in list(out)[-2:]: print(cyc, out[cyc]['late_names_top'][:8])