Evaluating longevity drugs with LLMs

{
"Summary": "The evidence on whether the compound acarbose has a role in longevity is mixed. There are some promising results from cell-based and animal studies, but the overall evidence is not yet conclusive. More research, especially in human subjects, would be needed to establish a clear role for acarbose in longevity.",
"Compound": "Acarbose",
"Evidence": "Acarbose was identified as one of several longevity-enhancing interventions in a proteomic study of mouse tissues (Burns et al., Geroscience, 2023). Cell-based assays also found that acarbose shared clinically relevant activities with other longevity-enhancing compounds like rapamycin (Venn-Watson et al., Nutrients, 2023). However, the review paper by Elliehausen et al. (BMC Biology, 2023) suggests the relationship between acarbose and exercise for longevity is complex and requires further study.",
"Score": 7
}

import json
import re
import requests

from pathlib import Path
from textwrap import dedent
from time import sleep
from urllib.parse import quote

from tqdm.auto import tqdm

import anthropic
client = anthropic.Anthropic()
# os.environ["ANTHROPIC_API_KEY"] = "sk-xxx"
model, gpt_out_dir = "claude-3-opus-20240229", "out_opus"
model, gpt_out_dir = "claude-3-haiku-20240307", "out_haiku"
model, gpt_out_dir = "claude-3-sonnet-20240229", "out_sonnet"

SEARCH_PUBMED_URL = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&term={params}&retmax={max_abstracts}{api_key_param}"
GET_ABSTRACT_URL = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pubmed&id={pmid}&retmode=text&rettype=abstract{api_key_param}"
# https://www.ncbi.nlm.nih.gov/research/bionlp/APIs/BioC-PubMed/
BIOC_PMCID_URL = "https://www.ncbi.nlm.nih.gov/research/bionlp/RESTful/pmcoa.cgi/BioC_json/{pmcid}/unicode"

DEFAULT_MAX_ABSTRACTS = 10

# NCBI recommends that users post no more than three URL requests per second and limit large jobs
# Failure to comply with this policy may result in an IP address being blocked from accessing NCBI.
NCBI_API_KEY = None

def remove_ref_blank_entries_and_offsets(obj):
    """
    Recursively traverse through the JSON object (dict or list) and:
    1. Remove any node if it or any of its nested structures contains a dict with 'section_type': 'REF'.
    2. Remove any key-value pairs where the value is an empty list or empty dictionary.
    3. Remove any key-value pairs where the key is 'offset'.
    """
    if isinstance(obj, dict):
        # Check if any value of this dict is a nested dict with 'section_type': 'REF'
        if any(isinstance(v, dict) and v.get('section_type') == 'REF' for v in obj.values()):
            return None
        else:
            # No nested dict with 'section_type': 'REF', recursively process each key-value pair
            return {k: remove_ref_blank_entries_and_offsets(v) for k, v in obj.items() if k != 'offset' and
                       remove_ref_blank_entries_and_offsets(v) is not None and v != [] and v != {}}
    elif isinstance(obj, list):
        # Recursively process each item in the list
        return [remove_ref_blank_entries_and_offsets(item) for item in obj
                if remove_ref_blank_entries_and_offsets(item) is not None and item != [] and item != {}]
    else:
        # Return the item as is if it's not a dict or list
        return obj


def find_keywords(abstract):
    if "longevity" in abstract.lower():
        return -10
    elif "age" in abstract.lower() or "aging" in abstract.lower():
        return -5
    else:
        return 0


def main(compounds, role, max_abstracts=DEFAULT_MAX_ABSTRACTS, ncbi_api_key=NCBI_API_KEY):
    """Download abstracts and fulltext from pubmed, generate a prompt, query OpenAI.
    """
    Path("downloads").mkdir(exist_ok=True)
    Path(gpt_out_dir).mkdir(exist_ok=True)

    api_key_param = "&api_key={ncbi_api_key}" if ncbi_api_key is not None else ""

    for compound in tqdm(compounds, desc="Compounds", leave=True):
        abstracts = []
        fulltexts = []

        params = quote(f'({compound}) AND ("{role}")')

        pmtxt = requests.get(SEARCH_PUBMED_URL.format(params=params, max_abstracts=max_abstracts, api_key_param=api_key_param)).text
        pmids = re.findall("<Id>(\d+)</Id>", pmtxt)
        sleep(0.3)

        for pmid in tqdm(pmids, desc="pmids", leave=False):
            if Path(f"downloads/abstract.pmid_{pmid}.txt").exists():
                abstracts.append(open(f"downloads/abstract.pmid_{pmid}.txt").read())
                if Path(f"downloads/fulltext.pmid_{pmid}.json").exists():
                    fulltexts.append(json.load(open(f"downloads/fulltext.pmid_{pmid}.json")))
                continue

            abstract = requests.get(GET_ABSTRACT_URL.format(pmid=pmid, api_key_param=api_key_param)).text
            open(f"downloads/abstract.pmid_{pmid}.txt", 'w').write(abstract)
            abstracts.append(abstract)
            sleep(0.3)

            pmcid = re.findall("^PMCID:\s+(PMC\S+)$", abstract, re.MULTILINE)
            assert len(pmcid) <= 1, "there should be max one PMCID per paper"
            pmcid = pmcid[0] if len(pmcid) == 1 else None

            if pmcid is not None:
                fulltext_request = requests.get(BIOC_PMCID_URL.format(pmcid=pmcid))
                if fulltext_request.status_code == 200:
                    fulltext_json = remove_ref_blank_entries_and_offsets(fulltext_request.json())
                    json.dump(fulltext_json, open(f"downloads/fulltext.pmid_{pmid}.json", 'w'), indent=2)
                    fulltexts.append(fulltext_json)
                sleep(0.3)

        # If there are only a couple of papers then we can include the fulltext
        # 250k characters should be well below 100k tokens; 500k characters is too many
        abstracts.sort(key=find_keywords)

        abstract_len = len("\n".join(abstracts))
        fulltext_len = len("\n".join(json.dumps(j) for j in fulltexts))
        if abstract_len + fulltext_len < 250_000:
            abstracts_txt = "\n".join(abstracts) + "\n" + "\n".join(json.dumps(j) for j in fulltexts)
        else:
            abstracts_txt = "\n".join(abstracts)

        # clip data to the first few abstracts
        abstracts_txt = abstracts_txt[:10_000]

        gpt_out = f"{gpt_out_dir}/" + f"{compound}_{role}".replace(' ','_').replace('/','_').replace('\\','_') + ".txt"
        if len(abstracts_txt) > 1_000 and not Path(gpt_out).exists():
            completion = client.messages.create(
                model=model,
                max_tokens=1024,
                system = dedent("""
                You are an autoregressive language model that has been fine-tuned with instruction-tuning and RLHF.
                You carefully provide accurate, factual, thoughtful, nuanced answers, and are brilliant at reasoning. 
                If you think there might not be a correct answer, you say so. 
                Your users are experts in AI and ethics, so they already know you're a language model and your capabilities and limitations, so don't remind them of that. 
                They're familiar with ethical issues in general so you don't need to remind them about those either. 
                Don't be verbose in your answers, but do provide details and examples where it might help the explanation.

                Your users are also experts in science, and especially biology, medicine, statistics. 
                Do NOT add any details about how science or research works, tell me to ask my doctor or consult with a health professional.
                Do NOT add any details that such an expert would already know.
                """),
                messages=[
                {"role": "user", 
                "content": dedent(f"""
                Help me research whether this compound has the role of {role}: {compound}. 
                Here are the top abstracts from pubmed, and any full text that will fit in context:

                {abstracts_txt}

                Read and synthesize all the evidence from these papers.
                Prefer evidence from good journals and highly cited papers, and papers that are recent.
                If there is one standout result in a top journal like Nature, Science or Cell, focus on that.
                There should usually be one primary result and most of the evidence should depend on that.

                Include a score out of 10, where 1 would be a single paper with weak evidence,
                5 would be a mid-tier journal with a single paper with believable evidence,
                and 10 would be multiple Nature, Science or Cell papers with very strong evidence.
                A score above 7 is exceptional, implying at least the result has been replicated and is trustworthy.

                Make sure the results are valid json with fields exactly as in the examples below.
                Do NOT change the fields, change the order, or add any other fields.
                The "Compound" json entry MUST match the query compound. Here, {compound}.                
                Include AT LEAST ONE reference for each entry on evidence (unless there is no evidence), e.g., (Smith et al., Nature, 2021). ALWAYS include the journal name.
                To help you think step-by-step about your response, use the FIRST "Summary" entry to summarize the question
                and the relevant available evidence in your own words, in one hundred words or fewer.
                Do NOT output markdown, just raw json. The first character should be {{ and the last character should be }}.

                Here are some examples of json you might output:

                {{
                "Summary": "Aspirin has no evidence of {role}. The experiments are not convincing. There is some evidence for a related compound, which may indicate something."; // string
                "Compound": "Aspirin", // string
                "Evidence": "No evidence as {role}. No relevant papers found.", // string
                "Score": 0 // integer from 0-10
                }}

                {{
                "Summary": "There is some evidence for Abacavir, which may indicate something. The evidence is medium, e.g. a paper in a mid-tier journal" // string
                "Compound": "Abacavir", // string
                "Evidence": "A C. elegans screen identified Abacavir as {role} with a small effect (Jones et al., Nature Cancer, 2001)", // string
                "Score": 5 // integer from 0-10
                }}

                {{
                "Summary": "Rapamycin has been cited as {role} many times. The experiments are convincing. The evidence is strong: many papers including a recent Nature paper and a Science paper" // string
                "Compound": "Rapamycin" // string
                "Evidence": "A cell-based assay identified Rapamycin as {role} in longevity (Smith et al., Science, 2022, Thomson et al., Nature, 2019)", // string
                "Score": 10 // integer from 0-10
                }}
                """)
                }
            ]
            )
            open(gpt_out, 'w').write(completion.content[0].text)
            sleep(.1)
        elif not Path(gpt_out).exists():
            # just log null results
            open(gpt_out, 'w').write("")


if __name__ == "__main__":
    compounds = [x.strip() for x in open("compound_list_fixed.txt").readlines()]
    main(compounds = compounds, role = "longevity")