import json, urllib.request, urllib.parse, time, csv, re
from pathlib import Path
H=Path(__file__).resolve().parent
UA={"User-Agent":"incoprea-research/1.0"}
BASE="https://clinicaltrials.gov/api/v2/studies"
FIELDS="NCTId,WhyStopped,Phase,LeadSponsorClass,Condition,EnrollmentCount,OverallStatus"
def get(url):
    return json.loads(urllib.request.urlopen(urllib.request.Request(url,headers=UA),timeout=90).read())
def total(status):
    u=f"{BASE}?filter.overallStatus={status}&pageSize=1&countTotal=true&fields=NCTId"
    return get(u).get("totalCount",0)
den={s:total(s) for s in ["TERMINATED","WITHDRAWN","COMPLETED","RECRUITING"]}
print("denominators",den)
CATS=[("efficacy",r"efficac|futility|not effective|no benefit|lack of|interim|superiority|inferiority"),
 ("safety",r"adverse|toxicit|safety|side effect|risk"),
 ("enrollment",r"enrol|accrual|recruit|slow|difficulty (in )?(finding|recruiting)"),
 ("funding",r"fund|grant|financial|budget|sponsor (withdrew|decision)"),
 ("business",r"business|strategic|portfolio|administrat|logistic|supply|shortage|company"),
 ("covid",r"covid|pandemic")]
def cat(t):
    s=(t or "").lower()
    for name,pat in CATS:
        if re.search(pat,s): return name
    return "other/unspecified"
recs=[]; token=None; pages=0
while pages<8:
    u=f"{BASE}?filter.overallStatus=TERMINATED&pageSize=1000&fields={FIELDS}"
    if token: u+="&pageToken="+urllib.parse.quote(token)
    d=get(u); st=d.get("studies",[])
    for s in st:
        p=s.get("protocolSection",{})
        why=(p.get("statusModule",{}) or {}).get("whyStopped","")
        ph=(p.get("designModule",{}) or {}).get("phases",[])
        sp=((p.get("sponsorCollaboratorsModule",{}) or {}).get("leadSponsor",{}) or {}).get("class","")
        cd=((p.get("conditionsModule",{}) or {}).get("conditions",[]) or [""])[0]
        recs.append({"nct":(p.get("identificationModule",{}) or {}).get("nctId",""),
          "why":why,"category":cat(why),"phase":"/".join(ph) if ph else "NA",
          "sponsor":sp,"condition":cd})
    term=d.get("nextPageToken")
    pages+=1
    if not term or len(st)<1000: break
    token=term; time.sleep(0.3)
print("sampled terminated:",len(recs))
from collections import Counter
overall=Counter(r["category"] for r in recs)
by_phase={}
by_sponsor={}
for r in recs:
    by_phase.setdefault(r["phase"],Counter())[r["category"]]+=1
    by_sponsor.setdefault(r["sponsor"],Counter())[r["category"]]+=1
with (H/"trials.csv").open("w",newline="") as f:
    w=csv.DictWriter(f,fieldnames=list(recs[0])); w.writeheader(); w.writerows(recs)
(H/"results.json").write_text(json.dumps({"denominators":den,"sampled":len(recs),"pages":pages,
  "overall":overall.most_common(),"by_phase":{k:v.most_common() for k,v in by_phase.items()},
  "by_sponsor":{k:v.most_common() for k,v in by_sponsor.items()}},indent=2))
N=len(recs)
L=["# Why clinical trials stop","",
f"Sample: **{N:,} terminated interventional studies** from ClinicalTrials.gov, 30 October 2026 (capped at 8 pages).",
f"Registry denominators: {den['TERMINATED']:,} terminated, {den['WITHDRAWN']:,} withdrawn, {den['COMPLETED']:,} completed, {den['RECRUITING']:,} recruiting.","",
"| Reason category | n | share |","|---|---:|---:|"]
for k,v in overall.most_common(): L.append(f"| {k} | {v} | {100*v/N:.1f}% |")
L+=["","## By phase (top reasons)",""]
for ph,c in sorted(by_phase.items()):
    tot=sum(c.values()); top=", ".join(f"{k} {100*v/tot:.0f}%" for k,v in c.most_common(3))
    L.append(f"- **{ph}** (n={tot}): {top}")
L+=["","## By sponsor class",""]
for sp,c in sorted(by_sponsor.items()):
    tot=sum(c.values()); top=", ".join(f"{k} {100*v/tot:.0f}%" for k,v in c.most_common(3))
    L.append(f"- **{sp}** (n={tot}): {top}")
L+=["","## Method note","",
"`whyStopped` is free text, so reasons are assigned by keyword regex into six categories plus 'other'.",
"Categories can overlap in the source text; the first match wins, in the order efficacy, safety,",
"enrollment, funding, business, covid. This is a coarse but reproducible classification.","",
"## Limits","",
"- 8-page cap: the sample is terminated studies in registry order, not a random sample.",
"- Free-text reasons are inconsistently written; 'other/unspecified' is large.",
"- Withdrawn studies are excluded here (separate status), so pre-start cancellations are missed.",
"- Registration practice varies by sponsor and country.",""]
(H/"FINDINGS.md").write_text("\n".join(L))
print("overall:",overall.most_common())