#!/usr/bin/env python3 """Check complete finalized RTE half-hour inputs; compute no storage capacities.""" from pathlib import Path from datetime import datetime,date,timedelta,timezone from decimal import Decimal,InvalidOperation from zoneinfo import ZoneInfo from collections import Counter,defaultdict import argparse,calendar,csv,hashlib,json,unicodedata PARIS=ZoneInfo('Europe/Paris') FIELDS=['date_heure','date','heure','nature','consommation','eolien','solaire'] MISSING={'','null','none','nan','nd','na','n/a'} def sha(b):return hashlib.sha256(b).hexdigest() def norm(s):return ''.join(c for c in unicodedata.normalize('NFKD',s.lower()) if not unicodedata.combining(c)) def timestamp(s): result=datetime.fromisoformat(s.replace('Z','+00:00')) if result.tzinfo is None:raise ValueError('timestamp has no timezone') return result.astimezone(timezone.utc) def integer(s): x=Decimal(s) if not x.is_finite() or x!=x.to_integral_value():raise ValueError('not exact integral MW') return int(x) def validate(directory,write_validated=False): directory=Path(directory) plan_bytes=(directory/'plan-v1.json').read_bytes();plan=json.loads(plan_bytes) metadata_bytes=(directory/'raw/dataset-metadata.json').read_bytes();meta=json.loads(metadata_bytes) errors=[];years=[];byyear={};raw_records=[] schema={f['name']:f for f in meta['fields']} for name in FIELDS: if name not in schema:errors.append({'scope':'metadata','error':'required field absent','field':name}) for name in ['consommation','eolien','solaire']: f=schema.get(name,{}) if f.get('type')!='int' or '(MW)' not in f.get('label',''): errors.append({'scope':'metadata','error':'unexpected actual units/type','field':name}) description=meta['metas']['default'].get('description','') if '2024' not in description or 'demi-heure' not in description: errors.append({'scope':'metadata','error':'finalization/frequency description needs review'}) for year in plan['data']['years']: path=directory/'raw'/f'eco2mix-{year}-selected.csv';receipt_path=directory/'raw'/f'eco2mix-{year}-receipt.json' data=path.read_bytes();receipt=json.loads(receipt_path.read_text()) if sha(data)!=receipt['sha256']:errors.append({'year':year,'error':'raw input hash mismatch'}) raw_records.append({'year':year,'file':str(path),'bytes':len(data),'sha256':sha(data),'url':receipt['url']}) rows=[];seen=set();nature=Counter();days=defaultdict(list);quarter_forecasts=0;total=0;year_errors=[] with path.open(encoding='utf-8-sig',newline='') as stream: reader=csv.DictReader(stream,delimiter=';') if reader.fieldnames!=FIELDS: if set(reader.fieldnames or [])!=set(FIELDS):year_errors.append({'error':'export columns differ','columns':reader.fieldnames}) for lineno,row in enumerate(reader,2): total+=1 try:utc=timestamp(row['date_heure']) except Exception as exc:year_errors.append({'line':lineno,'error':'invalid timestamp','detail':str(exc)});continue local=utc.astimezone(PARIS) if local.year!=year:year_errors.append({'line':lineno,'error':'outside selected Paris civil year','timestamp':utc.isoformat()});continue values=[row.get(f,'').strip() for f in ['consommation','eolien','solaire']] blanks=[v.lower() in MISSING for v in values] if utc.second or utc.microsecond or utc.minute not in [0,15,30,45]: year_errors.append({'line':lineno,'error':'timestamp not on quarter-hour grid'});continue if utc.minute in [15,45]: if all(blanks):quarter_forecasts+=1;continue year_errors.append({'line':lineno,'error':'nonempty actual value at quarter-only timestamp'});continue if any(blanks): year_errors.append({'line':lineno,'timestamp':utc.isoformat(),'error':'missing half-hour actual','fields':[f for f,blank in zip(['consommation','eolien','solaire'],blanks) if blank]});continue if utc in seen:year_errors.append({'line':lineno,'error':'duplicate half-hour timestamp','timestamp':utc.isoformat()});continue seen.add(utc) nature[row['nature']]+=1 if norm(row['nature']).strip() not in ['donnees definitives','donnee definitive','definitif','definitive']: year_errors.append({'line':lineno,'error':'not finalized definitive nature','nature':row['nature']}) try:load,wind,solar=map(integer,values) except (InvalidOperation,ValueError) as exc:year_errors.append({'line':lineno,'error':'invalid integer MW','detail':str(exc)});continue if load<=0 or wind<0 or solar<0:year_errors.append({'line':lineno,'error':'invalid signs under frozen plan','load':load,'wind':wind,'solar':solar}) if row['date']!=local.strftime('%Y-%m-%d') or row['heure']!=local.strftime('%H:%M'): year_errors.append({'line':lineno,'error':'supplied local date/hour conflicts with UTC timestamp','date':row['date'],'heure':row['heure'],'derived_date':local.strftime('%Y-%m-%d'),'derived_hour':local.strftime('%H:%M')}) item={'utc':utc,'local':local,'date':local.date(),'load':load,'wind':wind,'solar':solar} rows.append(item);days[local.date()].append(item) rows.sort(key=lambda r:r['utc']) start=datetime(year,1,1,tzinfo=PARIS).astimezone(timezone.utc);end=datetime(year+1,1,1,tzinfo=PARIS).astimezone(timezone.utc) expected_count=int((end-start).total_seconds()/1800) expected={start+timedelta(minutes=30*i) for i in range(expected_count)} actual={r['utc'] for r in rows};missing=sorted(expected-actual);extra=sorted(actual-expected) if missing or extra:year_errors.append({'error':'UTC half-hour coverage mismatch','missing_count':len(missing),'extra_count':len(extra),'first_missing':[x.isoformat() for x in missing[:20]],'first_extra':[x.isoformat() for x in extra[:20]]}) if len(rows)!=expected_count:year_errors.append({'error':'actual row count differs','expected':expected_count,'actual':len(rows)}) bad_gaps=[(a['utc'].isoformat(),b['utc'].isoformat()) for a,b in zip(rows,rows[1:]) if (b['utc']-a['utc']).total_seconds()!=1800] if bad_gaps:year_errors.append({'error':'noncontiguous actual UTC sequence','count':len(bad_gaps),'first':bad_gaps[:20]}) expected_dates=[];day=date(year,1,1) while day.year==year:expected_dates.append(day);day+=timedelta(days=1) bad_days=[] for day in expected_dates: a=datetime.combine(day,datetime.min.time(),PARIS).astimezone(timezone.utc);b=datetime.combine(day+timedelta(days=1),datetime.min.time(),PARIS).astimezone(timezone.utc);n=int((b-a).total_seconds()/1800) if len(days.get(day,[]))!=n:bad_days.append({'day':day.isoformat(),'expected':n,'actual':len(days.get(day,[]))}) if bad_days:year_errors.append({'error':'civil-day block count differs','days':bad_days}) if sum(r['wind'] for r in rows)<=0 or sum(r['solar'] for r in rows)<=0:year_errors.append({'error':'nonpositive annual renewable sum'}) year_report={'year':year,'csv_rows':total,'forecast_only_quarter_rows_excluded':quarter_forecasts,'actual_half_hours':len(rows),'expected_half_hours':expected_count,'civil_days':len(days),'leap_year':calendar.isleap(year),'civil_day_length_counts':dict(Counter(len(v) for v in days.values())),'nature_labels':dict(nature),'first_actual_utc':rows[0]['utc'].isoformat() if rows else None,'last_actual_utc':rows[-1]['utc'].isoformat() if rows else None,'error_count':len(year_errors),'errors':year_errors} years.append(year_report);byyear[year]=rows;errors.extend({'year':year,**e} for e in year_errors) result={'status':'PASS' if not errors else 'ABORT_NO_OUTCOMES','checked_utc':datetime.now(timezone.utc).isoformat(),'plan_sha256':sha(plan_bytes),'metadata_sha256':sha(metadata_bytes),'source_actual_units':'MW','interval_hours':'1/2','calendar':'Europe/Paris civil years/days; UTC cadence check','years':years,'errors':errors,'raw_inputs':raw_records,'storage_outcomes_computed':False,'license':{k:meta['metas']['default'].get(k) for k in ['publisher','license','license_url','data_processed']}} if write_validated and not errors: path=directory/'validated-actuals.csv' with path.open('w',encoding='utf-8',newline='') as stream: writer=csv.DictWriter(stream,fieldnames=['year','local_date','local_time','utc_timestamp','consumption_MW','wind_MW','solar_MW']);writer.writeheader() for year in plan['data']['years']: for r in byyear[year]:writer.writerow({'year':year,'local_date':r['date'].isoformat(),'local_time':r['local'].strftime('%H:%M%z'),'utc_timestamp':r['utc'].isoformat(),'consumption_MW':r['load'],'wind_MW':r['wind'],'solar_MW':r['solar']}) result['validated_actuals']={'file':str(path),'bytes':path.stat().st_size,'sha256':sha(path.read_bytes()),'rows':sum(len(v) for v in byyear.values())} return result,byyear def main(): parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('--directory',type=Path,default=Path(__file__).resolve().parent);args=parser.parse_args() result,_=validate(args.directory,write_validated=True);(args.directory/'preflight-v1.json').write_text(json.dumps(result,indent=2)+'\n') print(json.dumps({'status':result['status'],'errors':result['errors'][:10],'total_errors':len(result['errors']),'years':[{k:v for k,v in r.items() if k!='errors'} for r in result['years']]},indent=2)) if result['errors']:raise SystemExit(1) if __name__=='__main__':main()