import json,urllib.request,urllib.parse,time,csv,re
from pathlib import Path
from collections import Counter
H=Path(__file__).resolve().parent
UA={"User-Agent":"incoprea-research/1.0"}; BASE="https://clinicaltrials.gov/api/v2/studies"
def get(u): return json.loads(urllib.request.urlopen(urllib.request.Request(u,headers=UA),timeout=90).read())
CATS=[("efficacy",r"efficac|futility|not effective|no benefit|lack of"),
 ("safety",r"adverse|toxicit|safety|side effect"),
 ("enrollment",r"enrol|accrual|recruit|slow"),
 ("funding",r"fund|grant|financial|budget"),
 ("business",r"business|strategic|portfolio|administrat|logistic|supply|shortage|company"),
 ("protocol",r"protocol|design|amend|feasib|site"),
 ("covid",r"covid|pandemic")]
def cat(t):
    s=(t or "").lower()
    for n,p in CATS:
        if re.search(p,s): return n
    return "other/unspecified"
recs=[]; token=None; pages=0
while pages<6:
    u=f"{BASE}?filter.overallStatus=WITHDRAWN&pageSize=1000&fields=NCTId,WhyStopped,Phase,LeadSponsorClass,Condition"
    if token: u+="&pageToken="+urllib.parse.quote(token)
    d=get(u); st=d.get("studies",[])
    for s in st:
        p=s.get("protocolSection",{}); why=(p.get("statusModule",{}) or {}).get("whyStopped","")
        recs.append({"nct":(p.get("identificationModule",{}) or {}).get("nctId",""),"why":why,"category":cat(why),
          "phase":"/".join((p.get("designModule",{}) or {}).get("phases",[]) or ["NA"]),
          "sponsor":((p.get("sponsorCollaboratorsModule",{}) or {}).get("leadSponsor",{}) or {}).get("class","")})
    t=d.get("nextPageToken"); pages+=1
    if not t or len(st)<1000: break
    token=t; time.sleep(0.3)
c=Counter(r["category"] for r in recs)
with (H/"withdrawn.csv").open("w",newline="") as f:
    w=csv.DictWriter(f,fieldnames=list(recs[0])); w.writeheader(); w.writerows(recs)
(H/"results.json").write_text(json.dumps({"sampled":len(recs),"categories":c.most_common()},indent=2))
N=len(recs)
L=["# Why trials are withdrawn before they start","",
f"Sample: {N:,} withdrawn studies (never started), ClinicalTrials.gov, 30 October 2026.","",
"| Reason | n | share |","|---|---:|---:|"]
for k,v in c.most_common(): L.append(f"| {k} | {v} | {100*v/N:.1f}% |")
L+=["","## Reading","",
"Withdrawal is a pre-start event: these trials were registered and then cancelled. Comparing this",
"distribution with terminated trials shows whether the failure modes differ before and after launch.","",
"## Limits","",
"- Free-text reasons; keyword categories are coarse and first-match.",
"- 6-page cap, registry order rather than random.",
"- Registration is voluntary at the planning stage in many jurisdictions.",""]
(H/"FINDINGS.md").write_text("\n".join(L))
print("withdrawn sampled",N,c.most_common(4))