Pulling Bibliometrics data from PubMed via API

import json
import pandas as pd
import requests
import pandas as pd
from xml.etree import ElementTree
from urllib.parse import urlencode
import configparser
import warnings
warnings.filterwarnings('ignore')

config = configparser.ConfigParser()
config.read('config.ini')
api_key = config.get('API', 'key')

data = [["OCD", "Obsessive-Compulsive Disorder treatment"], ["GAD", "generalized anxiety disorder treatment"], ["panic disorder", "panic disorder treatment"]]
df = pd.DataFrame(data, columns= ["Name", "Query"])
print(df)

# Your API key
api_key = api_key  # Replace "your_api_key_here" with your actual API key

def query_pubmed(search_term, api_key):
    base_url = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi"
    query_params = {
        "db": "pubmed",
        "term": search_term,
        "retmax": 100,  # Increase the number of maximum results to return to 100
        "sort": "pub+date",  # Sort by publication date (most recent first)
        "api_key": api_key  # Include your API key here
    }
    response = requests.get(base_url, params=query_params)
    if response.status_code == 200:
        root = ElementTree.fromstring(response.content)
        pubmed_ids = [id_elem.text for id_elem in root.findall(".//Id")]
        return pubmed_ids
    else:
        print(f"Error: {response.status_code}")
        return []

def get_paper_details(pubmed_id, api_key):
    base_url = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi"
    query_params = {
        "db": "pubmed",
        "id": pubmed_id,
        "rettype": "abstract",
        "retmode": "xml",
        "api_key": api_key
    }
    response = requests.get(base_url, params=query_params)
    if response.status_code == 200:
        root = ElementTree.fromstring(response.content)
        
        # Check if the article exists
        article = root.find(".//PubmedArticle")
        if article is not None:
            # Extracting information from the XML response
            title = article.find(".//ArticleTitle").text
            abstract_element = article.find(".//AbstractText")
            abstract = abstract_element.text if abstract_element is not None else ""
            date_of_publication = article.find(".//PubDate").findtext(".//Year", default="") + "-" + article.find(".//PubDate").findtext(".//Month", default="") + "-" + article.find(".//PubDate").findtext(".//Day", default="")
            
            # Extracting Clinical Trials ID
            clinical_trials_id = ""
            databank_list = article.find(".//DataBankList")
            if databank_list is not None:
                for databank in databank_list.findall(".//DataBank"):
                    databank_name = databank.find(".//DataBankName")
                    if databank_name is not None and databank_name.text == "ClinicalTrials.gov":
                        accession_number = databank.find(".//AccessionNumber")
                        if accession_number is not None:
                            clinical_trials_id = accession_number.text
            
            # Extracting authors and their affiliations
            authors_info = []
            for author in article.findall(".//Author"):
                last_name = author.findtext(".//LastName", default="")
                first_name = author.findtext(".//ForeName", default="")
                full_name = f"{first_name} {last_name}"
                
                # Extracting author's affiliations
                affiliations = []
                for affiliation in author.findall(".//Affiliation"):
                    affiliations.append(affiliation.text)
                
                authors_info.append({
                    "Author": full_name,
                    "Affiliations": affiliations
                })
            
            # Duplicate data for co-authors
            coauthors_str = "; ".join([author["Author"] for author in authors_info])
            coauthors_affiliations = [affiliation for author in authors_info for affiliation in author["Affiliations"]]
            
            # Extracting Conflict of Interest Statement
            conflict_of_interest_element = article.find(".//CoiStatement")
            conflict_of_interest = conflict_of_interest_element.text if conflict_of_interest_element is not None else ""
            
            # Extracting Journal information
            journal_element = article.find(".//Journal")
            issn = journal_element.findtext(".//ISSN", default="")
            volume = journal_element.findtext(".//Volume", default="")
            issue = journal_element.findtext(".//Issue", default="")
            
            # Extracting Keywords
            keyword_list_element = article.find(".//KeywordList")
            if keyword_list_element is not None:
                keywords = [keyword.text for keyword in keyword_list_element.findall(".//Keyword")]
            else:
                keywords = []
            
            # Generate PubMed URL
            pubmed_url = f"https://pubmed.ncbi.nlm.nih.gov/{pubmed_id}/"
            
            return {
                "Title": title,
                "Abstract": abstract,
                "Date of Publication": date_of_publication,
                "Authors": coauthors_str,
                "Coauthors Affiliations": coauthors_affiliations,
                "Conflict of Interest Statement": conflict_of_interest,
                "ISSN": issn,
                "Volume": volume,
                "Issue": issue,
                "Keywords": keywords,
                "PubMed ID": pubmed_id,
                "PubMed URL": pubmed_url,  # Add PubMed URL to the returned dictionary
                "Clinical Trials ID": clinical_trials_id  # Add Clinical Trials ID to the returned dictionary
            }
        else:
            print("Article not found for PubMed ID:", pubmed_id)
            return {}
    else:
        print(f"Error: {response.status_code}")
        return {}

# Initialize an empty list to store the DataFrames for each query
dfs = []

# Iterate over each row and perform PubMed query
for index, row in df.iterrows():
    query = row['Query']
    pubmed_ids = query_pubmed(query, api_key)
    
    # Initialize an empty list to store the DataFrames for each PubMed ID
    paper_dfs = []
    
    # Iterate over each PubMed ID and retrieve paper details
    for pubmed_id in pubmed_ids:
        paper_details = get_paper_details(pubmed_id, api_key)
        
        # Add the query to the paper details
        paper_details["Query"] = query
        
        # Convert paper details to DataFrame and append to list
        paper_df = pd.DataFrame([paper_details])
        paper_dfs.append(paper_df)
    
    # Concatenate DataFrames for each PubMed ID and append to list
    query_df = pd.concat(paper_dfs, ignore_index=True)
    dfs.append(query_df)

# Concatenate DataFrames for each query
results_df = pd.concat(dfs, ignore_index=True)

# Save the results to a CSV file
results_df.to_csv("pubmed_results.csv", index=False)

results_df.head()

results_df.to_csv("../Data/results.csv", index=False)