import urllib.request, urllib.parse, json, time, datetime, os, sys

OUT = os.path.abspath(sys.argv[1] if len(sys.argv) > 1 else os.getcwd())
RAW = os.path.join(OUT, "raw.json")
BASE = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi"
UA = "pgx-gap-map-research/1.0 (public-data landscape note; python-urllib)"

GENES = ["ABCG2","CACNA1S","CFTR","CYP2B6","CYP2C19","CYP2C9","CYP2D6","CYP3A5",
         "CYP4F2","DPYD","G6PD","HLA-A","HLA-B","MT-RNR1","NAT2","NUDT15","RYR1",
         "SLCO1B1","TPMT","UGT1A1","VKORC1"]

# Budget: hard cap 40 requests for the whole session. One exploratory request was
# already spent before this script (CFTR test), so MAX_NEW = 39.
MAX_NEW = 39
SLEEP = 0.4

# "all records" (query B) is requested for the first 18 genes only; the last three
# in the brief's list are omitted to stay inside the request cap. Fixed, reproducible rule.
B_GENES = GENES[:18]

def term_pathogenic(g): return f'{g}[gene] AND "pathogenic"[clinsig]'
def term_all(g):        return f'{g}[gene]'

def run(term):
    url = BASE + "?" + urllib.parse.urlencode(
        {"db":"clinvar","retmode":"json","retmax":"0","term":term})
    req = urllib.request.Request(url, headers={"User-Agent": UA})
    try:
        with urllib.request.urlopen(req, timeout=45) as r:
            status = r.status
            text = r.read().decode("utf-8", "replace")
        return {"url": url, "http_status": status, "body_text": text}
    except Exception as e:
        return {"url": url, "http_status": None, "error": f"{type(e).__name__}: {e}"}

def main():
    doc = {"meta": {
        "note": "Raw NCBI E-utilities esearch responses for ClinVar gene record counts",
        "endpoint": BASE,
        "params": {"db":"clinvar","retmode":"json","retmax":"0"},
        "fetched_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds"),
        "user_agent": UA,
        "sleep_seconds_between_requests": SLEEP,
        "request_budget": {"hard_cap":40,"spent_before_script":1,"planned_in_script":MAX_NEW},
        "gene_list": GENES,
        "query_B_genes": B_GENES,
        "query_B_omitted": [g for g in GENES if g not in B_GENES],
    }, "responses": {}, "failures": {}}

    n = 0
    plan = [(g,"pathogenic",term_pathogenic(g)) for g in GENES] + \
           [(g,"all",term_all(g)) for g in B_GENES]

    for gene, kind, term in plan:
        if n >= MAX_NEW:
            doc["failures"][f"not_attempted:{gene}:{kind}"] = "request cap reached"
            continue
        key = f"clinvar:{gene}:{kind}"
        res = run(term)
        res["query"] = term
        res["gene"] = gene
        res["query_kind"] = kind
        res["fetched_utc"] = datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds")
        n += 1
        if "error" in res:
            doc["failures"][key] = res
        else:
            doc["responses"][key] = res
        # save before analysing, every time
        with open(RAW, "w", encoding="utf-8") as f:
            json.dump(doc, f, indent=2, ensure_ascii=False)
        print(f"[{n:02d}/{MAX_NEW}] {key} -> {res.get('http_status')} {str(res.get('body_text',''))[:60]!r}", flush=True)
        time.sleep(SLEEP)

    doc["meta"]["requests_made_in_script"] = n
    doc["meta"]["requests_made_total"] = n + 1
    with open(RAW, "w", encoding="utf-8") as f:
        json.dump(doc, f, indent=2, ensure_ascii=False)
    print("saved", RAW, "requests:", n + 1, "failures:", len(doc["failures"]))

if __name__ == "__main__":
    main()