#!/usr/bin/env python3
"""Acquire official selected-column RTE exports; never calculate storage outcomes."""
from pathlib import Path
from datetime import datetime,timezone
import concurrent.futures,hashlib,json,time,urllib.request,urllib.parse

HERE=Path(__file__).resolve().parent
RAW=HERE/'raw'
BASE='https://odre.opendatasoft.com/api/explore/v2.1/catalog/datasets/eco2mix-national-cons-def/exports/csv'
UA='Incoprea-Research-Codex/1.0 (public finalized RTE data; reproducible storage test)'

def annual_url(year):
    start=f'{year-1}-12-31T23:00:00Z'
    end=f'{year}-12-31T23:00:00Z'
    query={'select':'date_heure,date,heure,nature,consommation,eolien,solaire',
           'where':f"date_heure >= date'{start}' and date_heure < date'{end}'",
           'order_by':'date_heure','delimiter':';','use_labels':'false','timezone':'UTC'}
    return BASE+'?'+urllib.parse.urlencode(query)

def fetch_one(year):
    url=annual_url(year)
    path=RAW/f'eco2mix-{year}-selected.csv'
    receipt=RAW/f'eco2mix-{year}-receipt.json'
    if path.exists() or receipt.exists():
        if not (path.exists() and receipt.exists()):raise RuntimeError('Incomplete existing acquisition; inspect first')
        prior=json.loads(receipt.read_text())
        if hashlib.sha256(path.read_bytes()).hexdigest()!=prior['sha256']:raise RuntimeError('Saved raw digest mismatch')
        return prior
    attempts=[]
    for attempt in range(1,4):
        try:
            request=urllib.request.Request(url,headers={'User-Agent':UA,'Accept-Encoding':'identity'})
            with urllib.request.urlopen(request,timeout=60) as response:
                data=response.read(25_000_001)
                if len(data)>25_000_000:raise RuntimeError('Per-year25MB bounded download limit exceeded')
                if not data:raise RuntimeError('Empty export')
                record={'year':year,'url':url,'resolved_url':response.url,'http_status':response.status,
                        'retrieved_utc':datetime.now(timezone.utc).isoformat(),'bytes':len(data),
                        'sha256':hashlib.sha256(data).hexdigest(),'headers':dict(response.headers),
                        'attempts_before_success':attempts}
            path.write_bytes(data);receipt.write_text(json.dumps(record,indent=2)+'\n')
            print(f'{year}: saved{len(data)} bytes SHA256={record["sha256"]}',flush=True)
            return record
        except Exception as exc:
            attempts.append({'attempt':attempt,'error':repr(exc)})
            if attempt<3:time.sleep(2*attempt)
    failure={'year':year,'url':url,'attempts':attempts,'status':'failed'}
    (RAW/f'eco2mix-{year}-failure.json').write_text(json.dumps(failure,indent=2)+'\n')
    raise RuntimeError(str(failure))

def main():
    RAW.mkdir(exist_ok=True)
    urls=[{'year':year,'url':annual_url(year)} for year in range(2015,2025)]
    (HERE/'acquisition-urls.json').write_text(json.dumps(urls,indent=2)+'\n')
    records=[];errors=[]
    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
        futures={pool.submit(fetch_one,year):year for year in range(2015,2025)}
        for future in concurrent.futures.as_completed(futures):
            try:records.append(future.result())
            except Exception as exc:errors.append({'year':futures[future],'error':repr(exc)})
    result={'annual_exports':sorted(records,key=lambda x:x['year']),'errors':errors,
            'data_outcomes_computed':False,'total_export_bytes':sum(x['bytes'] for x in records)}
    (HERE/'data-acquisition.json').write_text(json.dumps(result,indent=2)+'\n')
    print(json.dumps({'exports':len(records),'errors':errors,'bytes':result['total_export_bytes']},indent=2))
    if errors:raise SystemExit(1)

if __name__=='__main__':main()