More Entrez improvements

Merge PubMedSearch and PubMedXMLFetcher.

Make PubMedXMLParser fully static.

Coordinate calls to the Entrez API instead of blindly waiting between
calls to respect usage policies.

Author: arteymix

gemma-cli/src/main/java/ubic/gemma/apps/ExpressionExperimentPrimaryPubCli.java

@@ -27,7 +27,7 @@
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import ubic.gemma.core.loader.entrez.pubmed.ExpressionExperimentBibRefFinder;
-import ubic.gemma.core.loader.entrez.pubmed.PubMedXMLFetcher;
+import ubic.gemma.core.loader.entrez.pubmed.PubMedSearch;
 import ubic.gemma.model.common.description.BibliographicReference;
 import ubic.gemma.model.expression.experiment.BioAssaySet;
 import ubic.gemma.model.expression.experiment.ExpressionExperiment;
@@ -53,7 +53,7 @@ public class ExpressionExperimentPrimaryPubCli extends ExpressionExperimentManip
     private ExpressionExperimentService ees;
     @Autowired
     private PersisterHelper persisterHelper;
-    private PubMedXMLFetcher fetcher;
+    private PubMedSearch fetcher;
     private ExpressionExperimentBibRefFinder finder;
 
     @Value("${entrez.efetch.apikey}")
@@ -62,7 +62,7 @@ public class ExpressionExperimentPrimaryPubCli extends ExpressionExperimentManip
     @Override
     public void afterPropertiesSet() throws Exception {
         super.afterPropertiesSet();
-        this.fetcher = new PubMedXMLFetcher( ncbiApiKey );
+        this.fetcher = new PubMedSearch( ncbiApiKey );
         this.finder = new ExpressionExperimentBibRefFinder( ncbiApiKey );
     }
 
@@ -146,7 +146,7 @@ protected void processExpressionExperiment( ExpressionExperiment experiment ) {
             experiment = ees.thawLite( experiment );
 
             // get from GEO or get from a file
-            BibliographicReference ref = fetcher.retrieveByHTTP( pubmedIds.get( experiment.getShortName() ) );
+            BibliographicReference ref = fetcher.fetchById( pubmedIds.get( experiment.getShortName() ) );
 
             if ( ref == null ) {
                 if ( this.pubmedIdFilename != null ) {

gemma-cli/src/main/java/ubic/gemma/apps/MeshTermFetcherCli.java

@@ -21,11 +21,12 @@
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.Options;
+import org.apache.commons.collections4.ListUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Value;
-import ubic.gemma.core.loader.entrez.pubmed.PubMedXMLFetcher;
 import ubic.gemma.cli.util.AbstractCLI;
 import ubic.gemma.cli.util.CLI;
+import ubic.gemma.core.loader.entrez.pubmed.PubMedSearch;
 import ubic.gemma.model.common.description.BibliographicReference;
 import ubic.gemma.model.common.description.MedicalSubjectHeading;
 
@@ -36,6 +37,7 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.stream.Collectors;
 
 /**
  * @author pavlidis
@@ -91,45 +93,29 @@ protected void processOptions( CommandLine commandLine ) {
 
     @Override
     protected void doWork() throws Exception {
-        PubMedXMLFetcher fetcher = new PubMedXMLFetcher( ncbiApiKey );
-
-        Collection<Integer> ids = this.readIdsFromFile( file );
-        Collection<Integer> chunk = new ArrayList<>();
-        for ( Integer i : ids ) {
-
-            chunk.add( i );
-
-            if ( chunk.size() == MeshTermFetcherCli.CHUNK_SIZE ) {
-
-                this.processChunk( fetcher, chunk );
-                chunk.clear();
-            }
-        }
-
-        if ( !chunk.isEmpty() ) {
+        PubMedSearch fetcher = new PubMedSearch( ncbiApiKey );
+        List<String> ids = this.readIdsFromFile( file ).stream().distinct().collect( Collectors.toList() );
+        for ( List<String> chunk : ListUtils.partition( ids, MeshTermFetcherCli.CHUNK_SIZE ) ) {
             this.processChunk( fetcher, chunk );
         }
     }
 
-    private Collection<Integer> readIdsFromFile( String inFile ) throws IOException {
+    private Collection<String> readIdsFromFile( String inFile ) throws IOException {
         log.info( "Reading " + inFile );
-
-        Collection<Integer> ids = new ArrayList<>();
+        Collection<String> ids = new ArrayList<>();
         try ( BufferedReader in = new BufferedReader( new FileReader( file ) ) ) {
             String line;
             while ( ( line = in.readLine() ) != null ) {
                 if ( line.startsWith( "#" ) )
                     continue;
-
-                ids.add( Integer.parseInt( line ) );
-
+                ids.add( StringUtils.strip( line ) );
             }
         }
         return ids;
     }
 
-    private void processChunk( PubMedXMLFetcher fetcher, Collection<Integer> ids ) throws IOException {
-        Collection<BibliographicReference> refs = fetcher.retrieveByHTTP( ids );
+    private void processChunk( PubMedSearch fetcher, Collection<String> pubMedIds ) throws IOException {
+        Collection<BibliographicReference> refs = fetcher.fetchById( pubMedIds );
 
         for ( BibliographicReference r : refs ) {
             getCliContext().getOutputStream().print( r.getPubAccession().getAccession() + "\t" );

gemma-cli/src/main/java/ubic/gemma/apps/PubMedSearcher.java

@@ -74,7 +74,7 @@ protected void processOptions( CommandLine commandLine ) {
     protected void doAuthenticatedWork() throws Exception {
         PubMedSearch pubMedSearcher = new PubMedSearch( ncbiApiKey );
         Collection<BibliographicReference> refs = pubMedSearcher
-                .searchAndRetrieveByHTTP( this.args );
+                .searchAndRetrieve( this.args );
 
         getCliContext().getOutputStream().println( refs.size() + " references found" );
 

gemma-cli/src/main/java/ubic/gemma/apps/UpdatePubMedCli.java

@@ -161,7 +161,7 @@ private BibliographicReference getBibliographicReference( String pubmedId ) {
             searchTerms.add( pubmedId );
             Collection<BibliographicReference> publications;
             try {
-                publications = pms.searchAndRetrieveIdByHTTP( searchTerms );
+                publications = pms.fetchById( searchTerms );
             } catch ( IOException e ) {
                 throw new RuntimeException( e );
             }

gemma-core/src/main/java/ubic/gemma/core/loader/entrez/EntrezException.java

@@ -0,0 +1,17 @@
+package ubic.gemma.core.loader.entrez;
+
+import java.util.List;
+
+public class EntrezException extends RuntimeException {
+
+    private final List<String> errors;
+
+    public EntrezException( String message, List<String> errors ) {
+        super( message );
+        this.errors = errors;
+    }
+
+    public List<String> getErrors() {
+        return errors;
+    }
+}

gemma-core/src/main/java/ubic/gemma/core/loader/entrez/EntrezUtils.java

@@ -1,22 +1,26 @@
 package ubic.gemma.core.loader.entrez;
 
+import lombok.extern.apachecommons.CommonsLog;
 import org.apache.commons.lang3.StringUtils;
-import org.springframework.util.Assert;
 import ubic.gemma.core.config.Settings;
+import ubic.gemma.core.util.SimpleRetryCallable;
 
 import javax.annotation.Nullable;
+import java.io.IOException;
 import java.io.UnsupportedEncodingException;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLEncoder;
 import java.nio.charset.StandardCharsets;
+import java.util.concurrent.Callable;
 
 /**
  * Low-level utilities for generating Entrez URLs.
  * <p>
  * Read more about this in <a href="https://www.ncbi.nlm.nih.gov/books/NBK25500/">The E-utilities In-Depth: Parameters, Syntax and More</a>.
  * @author poirigui
  */
+@CommonsLog
 public class EntrezUtils {
 
     public static final String ESEARCH = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi";
@@ -27,46 +31,68 @@ public class EntrezUtils {
     private static final String TOOL = "gemma";
     private static final String EMAIL = Settings.getString( "gemma.support.email" );
 
+    private static final long TIMEOUT_AUTHENTICATED_MS = 100;
+    private static final long TIMEOUT_ANONYMOUS_MS = 333;
+
+    private static long lastCall = 0L;
+
+    @FunctionalInterface
+    public interface EntrezCall<T> extends Callable<T> {
+
+        @Override
+        T call() throws IOException;
+    }
+
     /**
-     * Perform a search on an Entrez database.
+     * Coordinate calls to the Entrez API so that we always respect the recommended usage.
      * <p>
-     * Results must be subsequently retrieved with {@link #fetch(String, String, String, int, int, String, String)} or
-     * {@link #summary(String, String, String, int, int, String, String)}. The query key and WebEnv values must be
-     * extracted from the payload.
+     * Refer to <a href="https://www.ncbi.nlm.nih.gov/books/NBK25497/">A General Introduction to the E-utilities</a> for
+     * more information about usage policies.
      */
-    public static URL search( String db, String term, @Nullable String apiKey ) {
-        return createUrl( ESEARCH
-                + "?db=" + urlEncode( db )
-                + "&term=" + urlEncode( term )
-                + "&usehistory=y", apiKey );
+    public synchronized static <T> T doNicely( EntrezCall<T> task, @Nullable String apiKey ) throws IOException {
+        long timeoutMs = StringUtils.isNotBlank( apiKey ) ? TIMEOUT_AUTHENTICATED_MS : TIMEOUT_ANONYMOUS_MS;
+        long diff = System.currentTimeMillis() - lastCall;
+        if ( diff < timeoutMs ) {
+            try {
+                log.debug( "Last Entrez API call occurred " + diff + " ms ago, waiting " + ( timeoutMs - diff ) + " ms..." );
+                Thread.sleep( timeoutMs - diff );
+            } catch ( InterruptedException e ) {
+                Thread.currentThread().interrupt();
+                throw new RuntimeException( e );
+            }
+        }
+        try {
+            return task.call();
+        } finally {
+            lastCall = System.currentTimeMillis();
+        }
     }
 
     /**
-     * Perform a search on an Entrez database and retrieve results immediately.
+     * Wrap a {@link SimpleRetryCallable} such that it will respect the recommended usage of the Entrez API.
+     * @see #doNicely(EntrezCall, String)
      */
-    public static URL searchAndRetrieve( String db, String term, String retmode, String rettype, @Nullable String apiKey ) {
-        return createUrl( ESEARCH
-                + "?db=" + urlEncode( db )
-                + "&term=" + urlEncode( term )
-                + "&retmode=" + urlEncode( retmode )
-                + "&rettype=" + urlEncode( rettype ), apiKey );
+    public static <T> SimpleRetryCallable<T, IOException> retryNicely( SimpleRetryCallable<T, IOException> callable, @Nullable String apiKey ) {
+        return ( ctx ) -> EntrezUtils.doNicely( () -> callable.call( ctx ), apiKey );
     }
 
     /**
-     * Perform a search on an Entrez database and retrieve results immediately.
+     * Perform a search on an Entrez database.
+     * <p>
+     * Results must be subsequently retrieved with {@link #fetch(String, String, String, int, int, String, String)} or
+     * {@link #summary(String, String, String, int, int, String, String)}. The query key and WebEnv values must be
+     * extracted from the payload.
      */
-    public static URL searchAndRetrieve( String db, String term, String retmode, String rettype, int retstart, int retmax, @Nullable String apiKey ) {
+    public static URL search( String db, String term, String retmode, @Nullable String apiKey ) {
         return createUrl( ESEARCH
                 + "?db=" + urlEncode( db )
                 + "&term=" + urlEncode( term )
-                + "&retstart=" + retstart
-                + "&retmax=" + retmax
                 + "&retmode=" + urlEncode( retmode )
-                + "&rettype=" + urlEncode( rettype ), apiKey );
+                + "&usehistory=y", apiKey );
     }
 
     /**
-     * Summarize a previous {@link #search(String, String, String)} query.
+     * Summarize a previous {@link #search(String, String, String, String)} query.
      */
     public static URL summary( String db, String queryKey, String retmode, int retstart, int retmax, String webEnv, @Nullable String apiKey ) {
         return createUrl( ESUMMARY
@@ -90,7 +116,7 @@ public static URL fetchById( String db, String id, String retmode, String rettyp
     }
 
     /**
-     * Retrieve the results of a previous {@link #search(String, String, String)} query.
+     * Retrieve the results of a previous {@link #search(String, String, String, String)} query.
      */
     public static URL fetch( String db, String queryKey, String retmode, int retstart, int retmax, String webEnv, @Nullable String apiKey ) {
         return createUrl( EFETCH

gemma-core/src/main/java/ubic/gemma/core/loader/entrez/EntrezXmlUtils.java

@@ -0,0 +1,100 @@
+package ubic.gemma.core.loader.entrez;
+
+import org.apache.commons.lang3.StringUtils;
+import org.w3c.dom.Document;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import ubic.gemma.core.util.XMLUtils;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.ParserConfigurationException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+
+/**
+ * Utilities for manipulating Entrez XML responses.
+ * @author poirigui
+ */
+public class EntrezXmlUtils {
+
+    /**
+     * Parse an XML reply from Entrez.
+     * <p>
+     * This will check if there are any {@code ERROR} tags.
+     */
+    public static Document parse( InputStream is ) throws IOException {
+        try {
+            DocumentBuilder builder = createDocumentBuilder();
+            Document doc = builder.parse( is );
+            checkForErrors( doc );
+            return doc;
+        } catch ( ParserConfigurationException | SAXException e ) {
+            throw new RuntimeException( e );
+        }
+    }
+
+    public static Document parse( InputSource is ) throws IOException {
+        try {
+            DocumentBuilder builder = createDocumentBuilder();
+            Document doc = builder.parse( is );
+            checkForErrors( doc );
+            return doc;
+        } catch ( ParserConfigurationException | SAXException e ) {
+            throw new RuntimeException( e );
+        }
+    }
+
+    private static void checkForErrors( Document doc ) {
+        NodeList error = doc.getDocumentElement().getElementsByTagName( "ERROR" );
+        if ( error.item( 0 ) != null ) {
+            List<String> errors = new ArrayList<>();
+            for ( Node elem = error.item( 0 ); elem != null; elem = elem.getNextSibling() ) {
+                errors.add( XMLUtils.getTextValue( elem ) );
+            }
+            throw new EntrezException( errors.get( 0 ), errors );
+        }
+    }
+
+
+    /**
+     * Create a document builder with {@link NcbiEntityResolver} as entity resolver.
+     * <p>
+     * This will work for most of not all XML files from NCBI Entrez and related services.
+     */
+    private static DocumentBuilder createDocumentBuilder() throws ParserConfigurationException {
+        DocumentBuilder builder = ubic.gemma.core.util.XMLUtils.createDocumentBuilder();
+        builder.setEntityResolver( new NcbiEntityResolver() );
+        return builder;
+    }
+
+    public static Collection<String> extractIds( Document doc ) {
+        NodeList idList = doc.getElementsByTagName( "Id" );
+        Collection<String> result = new HashSet<>();
+        for ( Node elem = idList.item( 0 ); elem != null; elem = elem.getNextSibling() ) {
+            String val = XMLUtils.getTextValue( elem );
+            if ( StringUtils.isBlank( val ) ) {
+                continue;
+            }
+            result.add( val );
+        }
+        return result;
+    }
+
+    public static int getCount( Document document ) {
+        return Integer.parseInt( XMLUtils.getTextValue( document.getElementsByTagName( "Count" ).item( 0 ) ) );
+    }
+
+    public static String getQueryId( Document document ) {
+        return XMLUtils.getTextValue( XMLUtils.getUniqueItem( document.getElementsByTagName( "QueryKey" ) ) );
+    }
+
+    public static String getCookie( Document document ) {
+        return XMLUtils.getTextValue( XMLUtils.getUniqueItem( document.getElementsByTagName( "WebEnv" ) ) );
+    }
+}