[VIVO-1311] Implementation of UMLS datasource using NLM APIs

api/src/main/java/edu/cornell/mannlib/semservices/service/impl/UMLSService.java

@@ -2,13 +2,11 @@
 
 package edu.cornell.mannlib.semservices.service.impl;
 
-import java.io.BufferedReader;
+import java.io.IOException;
-import java.io.InputStreamReader;
+import java.io.InputStream;
-import java.io.StringWriter;
-import java.net.URL;
-import java.net.URLEncoder;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Properties;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -20,6 +18,11 @@ import edu.cornell.mannlib.semservices.bo.Concept;
 import edu.cornell.mannlib.semservices.exceptions.ConceptsNotFoundException;
 import edu.cornell.mannlib.semservices.service.ExternalConceptService;
 import edu.cornell.mannlib.vitro.webapp.utils.json.JacksonUtils;
+import org.apache.http.HttpVersion;
+import org.apache.http.client.fluent.Form;
+import org.apache.http.client.fluent.Request;
+import org.apache.http.client.utils.URIBuilder;
+import org.springframework.util.StringUtils;
 
 /**
  * @author jaf30
@@ -27,37 +30,80 @@ import edu.cornell.mannlib.vitro.webapp.utils.json.JacksonUtils;
  */
 public class UMLSService implements ExternalConceptService {
     protected final Log logger = LogFactory.getLog(getClass());
-   private static final String submissionUrl = "http://link.informatics.stonybrook.edu/MeaningLookup/MlServiceServlet?";
-   private static final String baseUri = "http://link.informatics.stonybrook.edu/umls/CUI/";
-   private static final String endpoint = "http://link.informatics.stonybrook.edu/sparql/";
-   private static final String schemeURI = "http://link.informatics.stonybrook.edu/umls";
 
+    private static String UTS_REST_API_URL = "https://uts-ws.nlm.nih.gov/rest";
+    private static String SEARCH_PATH = "/search/current";
+    private static String SEARCH_PARAMETER = "string";
+    private static String SEARCH_TYPE_PARAMETER = "searchType";
+    private static String SEARCH_TYPE = "rightTruncation";
+    private static String PAGE_SIZE_PARAMETER = "pageSize";
+    private static String RETURN_TYPE_PARAMETER = "returnIdType";
+    private static String RETURN_TYPE = "concept";
+    private static String TICKET_PARAMETER = "ticket";
 
+    private static String ticketGrantingTicketURL = null;
+
+    private static long lastUpdate = -1;
+
+    private static String username = null;
+    private static String password = null;
+    private static String apikey = null;
+
+    private static String pageSize = "50";
+
+    private static String UMLS_AUTH_USER_URL = "https://utslogin.nlm.nih.gov/cas/v1/tickets";
+    private static String UMLS_AUTH_KEY_URL = "https://utslogin.nlm.nih.gov/cas/v1/api-key";
+    private static String UTS_SERVICE_URL   = "http://umlsks.nlm.nih.gov";
+
+    {
+        if (username == null || apikey == null) {
+            final Properties properties = new Properties();
+            try (InputStream stream = getClass().getResourceAsStream("/umls.properties")) {
+                properties.load(stream);
+                username = properties.getProperty("username");
+                password = properties.getProperty("password");
+                apikey   = properties.getProperty("apikey");
+
+                String exPageSize = properties.getProperty("pagesize");
+                try {
+                    if (!StringUtils.isEmpty(exPageSize)) {
+                        int iPageSize = Integer.parseInt(exPageSize, 10);
+                        if (iPageSize > 5 && iPageSize < 200) {
+                            pageSize = Integer.toString(iPageSize, 10);
+                        }
+                    }
+                } catch (Exception e) {
+                }
+            } catch (IOException e) {
+            }
+        }
+    }
+
+    public boolean isConfigured() {
+        return !(StringUtils.isEmpty(username) && StringUtils.isEmpty(apikey));
+    }
 
     @Override
     public List<Concept> getConcepts(String term) throws Exception {
+        String ticket = getSingleUseTicket();
+
         List<Concept> conceptList = new ArrayList<Concept>();
 
         String results = null;
-		String dataUrl = submissionUrl + "textToProcess="
-				+ URLEncoder.encode(term, "UTF-8") 
-				+ "&format=json";
 
         try {
+            URIBuilder b = new URIBuilder(UTS_REST_API_URL + SEARCH_PATH);
+            b.addParameter(SEARCH_PARAMETER, term);
+            b.addParameter(RETURN_TYPE_PARAMETER, RETURN_TYPE);
+            b.addParameter(SEARCH_TYPE_PARAMETER, SEARCH_TYPE);
+            b.addParameter(PAGE_SIZE_PARAMETER, pageSize);
+            b.addParameter(TICKET_PARAMETER, ticket);
 
-			StringWriter sw = new StringWriter();
+            results = Request.Get(b.build())
-			URL rss = new URL(dataUrl);
+                    .connectTimeout(3000)
+                    .socketTimeout(3000)
+                    .execute().returnContent().asString();
 
-			BufferedReader in = new BufferedReader(new InputStreamReader(
-					rss.openStream()));
-			String inputLine;
-			while ((inputLine = in.readLine()) != null) {
-				sw.write(inputLine);
-			}
-			in.close();
-
-			results = sw.toString();
-			//System.out.println("results before processing: "+results);
             conceptList = processOutput(results);
             return conceptList;
 
@@ -68,39 +114,13 @@ public class UMLSService implements ExternalConceptService {
     }
 
     public List<Concept> processResults(String term) throws Exception {
-      String results = null;
+        return getConcepts(term);
-      String dataUrl = submissionUrl + "textToProcess="
-            + URLEncoder.encode(term, "UTF-8") + "&format=json";
-
-      try {
-
-         StringWriter sw = new StringWriter();
-         URL rss = new URL(dataUrl);
-
-         BufferedReader in = new BufferedReader(new InputStreamReader(rss.openStream()));
-         String inputLine;
-         while ((inputLine = in.readLine()) != null) {
-            sw.write(inputLine);
-         }
-         in.close();
-
-         results = sw.toString();
-         //System.out.println("results before processing: "+results);
-         List<Concept> conceptList = processOutput(results);
-         return conceptList;
-
-      } catch (Exception ex) {
-         logger.error("error occurred in servlet", ex);
-         return null;
-      }
-
     }
 
     /**
      * @param uri URI
      */
-	public List<Concept> getConceptsByURIWithSparql(String uri)
+    public List<Concept> getConceptsByURIWithSparql(String uri) throws Exception {
-			throws Exception {
         // deprecating this method...just return an empty list
         List<Concept> conceptList = new ArrayList<Concept>();
         return conceptList;
@@ -110,77 +130,37 @@ public class UMLSService implements ExternalConceptService {
      * @param results Results to process
      */
     private List<Concept> processOutput(String results) throws Exception {
-
         List<Concept> conceptList = new ArrayList<Concept>();
         List<String> bestMatchIdList = new ArrayList<String>();
         String bestMatchId = new String();
-      boolean bestMatchFound = false;
-      boolean allFound = false;
 
         try {
             ObjectNode json = (ObjectNode) JacksonUtils.parseJson(results);
-         //System.out.println(json.toString());
+            ArrayNode allArray = (ArrayNode) json.get("result").get("results");
-         if (json.has("Best Match")) {
-            bestMatchFound = true;
-            //System.out.println("Best Match");
-
-            ArrayNode bestMatchArray = (ArrayNode) json.get("Best Match");
-            int len = bestMatchArray.size();
-            if (len > 1) {
-               logger.debug("Found this many best matches: "+ len);
-            }
-            int i;
-            for (i = 0; i < len; i++) {
-               ObjectNode o = (ObjectNode) bestMatchArray.get(i);
-               //System.out.println(o.toString());
-               Concept concept = new Concept();
-               concept.setDefinedBy(schemeURI);
-               concept.setBestMatch("true");
-               String cui = getJsonValue(o, "CUI");
-               bestMatchIdList.add(cui);
-
-               concept.setConceptId(cui);
-               concept.setLabel(getJsonValue(o, "label"));
-               concept.setType(getJsonValue(o, "type"));
-               concept.setDefinition(getJsonValue(o, "definition"));
-               concept.setUri(baseUri + cui);
-               concept.setSchemeURI(schemeURI);
-               conceptList.add(concept);
-            }
-         }
-         if (json.has("All")) {
-            allFound = true;
-            ArrayNode allArray = (ArrayNode) json.get("All");
             int len = allArray.size();
-            //System.out.println("size of best match array: "+ len);
             int i;
             for (i = 0; i < len; i++) {
                 ObjectNode o = (ObjectNode) allArray.get(i);
-               //System.out.println(o.toString());
-               Concept concept = new Concept();
-               concept.setDefinedBy(schemeURI);
-               String cui = getJsonValue(o, "CUI");
-               concept.setConceptId(cui);
 
-               concept.setLabel(getJsonValue(o, "label"));
+                Concept concept = new Concept();
-               concept.setType(getJsonValue(o, "type"));
+                concept.setDefinedBy(UTS_SERVICE_URL);
-               concept.setDefinition(getJsonValue(o, "definition"));
+                concept.setSchemeURI(UTS_SERVICE_URL);
-               concept.setUri(baseUri + cui);
+
-               concept.setSchemeURI(schemeURI);
+                concept.setType(RETURN_TYPE);
-               // prevent duplicate concepts in list
+                concept.setConceptId(getJsonValue(o, "ui"));
-               if (! bestMatchIdList.contains(cui)) {
+                concept.setLabel(getJsonValue(o, "name"));
+                concept.setUri(getJsonValue(o, "uri"));
+
                 concept.setBestMatch("false");
                 conceptList.add(concept);
             }
-            }
+        } catch (Exception ex) {
-         }
-      } catch (Exception ex ) {
             ex.printStackTrace();
             logger.error("Could not get concepts", ex);
             throw ex;
         }
-      if (! bestMatchFound && !allFound) {
+
-         // we did not get a bestMatch or All element
+        if (conceptList.size() == 0) {
             throw new ConceptsNotFoundException();
         }
 
@@ -191,6 +171,7 @@ public class UMLSService implements ExternalConceptService {
 
     /**
      * Get a string from a json object or an empty string if there is no value for the given key
+     *
      * @param obj JSON Object
      * @param key Key to retrieve
      */
@@ -203,7 +184,6 @@ public class UMLSService implements ExternalConceptService {
     }
 
 
-
     protected String stripConceptId(String uri) {
         String conceptId = new String();
         int lastslash = uri.lastIndexOf('/');
@@ -211,4 +191,39 @@ public class UMLSService implements ExternalConceptService {
         return conceptId;
     }
 
+    private synchronized void getTicketGrantingTicket() {
+        if (StringUtils.isEmpty(username) && StringUtils.isEmpty(apikey)) {
+            throw new IllegalStateException("Unable to read umls.properties");
+        }
+
+        if (ticketGrantingTicketURL == null || lastUpdate + 28700000l < System.currentTimeMillis()) {
+            try {
+                if (!StringUtils.isEmpty(apikey)) {
+                    ticketGrantingTicketURL = Request.Post(UMLS_AUTH_KEY_URL).useExpectContinue().version(HttpVersion.HTTP_1_1)
+                            .bodyForm(Form.form().add("apikey", apikey).build())
+                            .execute().returnResponse().getFirstHeader("location").getValue();
+                } else {
+                    ticketGrantingTicketURL = Request.Post(UMLS_AUTH_USER_URL).useExpectContinue().version(HttpVersion.HTTP_1_1)
+                            .bodyForm(Form.form().add("username", username).add("password", password).build())
+                            .execute().returnResponse().getFirstHeader("location").getValue();
+                }
+            } catch (IOException e) {
+                throw new IllegalStateException("Unable to get ticket granting ticket.");
+            }
+            lastUpdate = System.currentTimeMillis();
+        }
+    }
+
+    private String getSingleUseTicket() {
+        getTicketGrantingTicket();
+        String ticket = "";
+        try {
+            ticket = Request.Post(ticketGrantingTicketURL).useExpectContinue().version(HttpVersion.HTTP_1_1)
+                    .bodyForm(Form.form().add("service", UTS_SERVICE_URL).build())
+                    .execute().returnContent().asString();
+        } catch (IOException e) {
+            throw new IllegalStateException("Unable to get ticket.");
+        }
+        return ticket;
+    }
 }

api/src/test/java/edu/cornell/mannlib/semservices/service/impl/UMLSServiceTest.java

@@ -0,0 +1,19 @@
+package edu.cornell.mannlib.semservices.service.impl;
+
+import edu.cornell.mannlib.semservices.bo.Concept;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.List;
+
+public class UMLSServiceTest {
+    @Test
+    public void testUmls() throws Exception {
+        UMLSService service = new UMLSService();
+
+        if (service.isConfigured()) {
+            List<Concept> concepts = service.getConcepts("diabetes");
+            Assert.assertNotNull(concepts);
+        }
+    }
+}

api/src/test/resources/umls.properties

@@ -0,0 +1,9 @@
+# Configure credentials to access UMLS service
+
+# Sign up here - https://uts.nlm.nih.gov//home.html
+
+# You can user either a username / password combination, or an apikey
+
+#username =
+#password =
+#apikey =