Use pmcid to find PMC source

ace/scrape.py

@@ -341,7 +341,7 @@ def get_html(self, url, journal, mode='browser'):
                 else:
                     break
 
-            new_url = self.check_for_substitute_url(url, html, journal)
+            new_url = self.check_for_substitute_url(url, journal)
 
             if url != new_url:
                 driver = create_driver()
@@ -419,7 +419,7 @@ def get_html(self, url, journal, mode='browser'):
             r = requests.get(url, headers=headers)
             # For some journals, we can do better than the returned HTML, so get the final URL and 
             # substitute a better one.
-            url = self.check_for_substitute_url(r.url, r.text, journal)
+            url = self.check_for_substitute_url(r.url, journal)
             if url != r.url:
                 r = requests.get(url, headers=headers)
                 # XML content is usually misidentified as ISO-8859-1, so we need to manually set utf-8.
@@ -429,43 +429,23 @@ def get_html(self, url, journal, mode='browser'):
             return r.text
 
 
-    def get_html_by_pmid(self, pmid, journal, mode='browser', retmode='ref', prefer_pmc_source=True):
+    def get_html_by_pmid(self, pmid, journal, mode='browser', retmode='ref', pmcid=None):
         base_url = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/elink.fcgi"
-        "https://eutils.ncbi.nlm.nih.gov/entrez/eutils"
-
-        if prefer_pmc_source:
+        if pmcid:
             try:
-                response = self._client.elink(pmid, retmode='json', return_content=False)
-                response.raise_for_status()  # Raise an HTTPError for bad responses
-                json_content = response.json()
-
-                providers = {obj['provider']['nameabbr']: obj["url"]["value"] for obj in json_content['linksets'][0]['idurllist'][0]['objurls']}
-                pmc_url = providers.get('PMC')
-
-                if pmc_url:
-                    return self.get_html(pmc_url, journal, mode='requests')
-                elif prefer_pmc_source == "only":
-                    logger.info("\tNo PMC source found! Skipping...")
-                    return
+                pmc_url = f"https://www.ncbi.nlm.nih.gov/pmc/articles/{pmcid}/"
+                return self.get_html(pmc_url, journal, mode='requests')
+
             except requests.RequestException as e:
                 logger.error(f"Request failed: {e}")
-            except KeyError as e:
-                logger.error(f"Key error: {e} - JSON content: {json_content}")
+                logger.info(f"Failed to retrieve article from PMC...")
         else:
             query = f"{base_url}?dbfrom=pubmed&id={pmid}&cmd=prlinks&retmode={retmode}"
             logger.info(query)
             return self.get_html(query, journal, mode=mode)
 
-        if prefer_pmc_source == "only":
-            logger.info("\tNo PMC source found!! Skipping...")
-            return
-
-        # Fallback if no PMC link found
-        query = f"{base_url}?dbfrom=pubmed&id={pmid}&cmd=prlinks&retmode={retmode}"
-        return self.get_html(query, journal, mode=mode)
 
-
-    def check_for_substitute_url(self, url, html, journal):
+    def check_for_substitute_url(self, url, journal):
         ''' For some journals/publishers, we can get a better document version by modifying the 
         URL passed from PubMed. E.g., we can get XML with embedded tables from PLoS ONE instead of 
         the standard HTML, which displays tables as images. For some journals (e.g., Frontiers),  
@@ -497,10 +477,10 @@ def is_pmc_open_acess(self, pmcid):
         oa_url = "https://www.ncbi.nlm.nih.gov/pmc/utils/oa/oa.fcgi?id="
 
         response = get_url(oa_url + pmcid)
-    
+
         return 'idIsNotOpenAccess' not in response
 
-    def process_article(self, id, journal, delay=None, mode='browser', overwrite=False, prefer_pmc_source=True):
+    def process_article(self, id, journal, delay=None, mode='browser', overwrite=False, pmcid=None):
 
         logger.info("Processing %s..." % id)
         journal_path = (self.store / 'html' / journal)
@@ -513,7 +493,7 @@ def process_article(self, id, journal, delay=None, mode='browser', overwrite=Fal
             return None, None
 
         # Save the HTML 
-        doc = self.get_html_by_pmid(id, journal, mode=mode, prefer_pmc_source=prefer_pmc_source)
+        doc = self.get_html_by_pmid(id, journal, mode=mode, pmcid=pmcid)
         valid = None
         if doc:
             valid = _validate_scrape(doc)
@@ -606,10 +586,10 @@ def retrieve_articles(self, journal=None, pmids=None, dois=None, delay=None, mod
             for pmid in pmids 
             if (min_pmid is None or int(pmid) >= min_pmid) and (max_pmid is None or int(pmid) <= max_pmid)
             ]
-    
+
         logger.info(f"Retrieving {len(pmids)} articles...")
-        
-        if skip_pubmed_central:
+
+        if skip_pubmed_central or prefer_pmc_source:
             all_ids = _convert_pmid_to_pmc(pmids)
         else:
             all_ids = [(None, pmid) for pmid in pmids]
@@ -627,7 +607,11 @@ def retrieve_articles(self, journal=None, pmids=None, dois=None, delay=None, mod
                 logger.info(f"\tPubMed Central OpenAccess entry found! Skipping {pmid}...")
                 continue
 
-            filename, valid = self.process_article(pmid, journal, delay, mode, overwrite, prefer_pmc_source)
+            if prefer_pmc_source == "only" and not pmcid:
+                logger.info(f"\tNo PubMed Central entry found! Skipping {pmid}...")
+                continue
+
+            filename, valid = self.process_article(pmid, journal, delay, mode, overwrite, pmcid=pmcid)
 
             if not valid:
                 invalid_articles.append(filename)