litvinovg/vivo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[VIVO-1311] Implementation of UMLS datasource using NLM APIs

Browse source
This commit is contained in:
Graham Triggs 2017-09-12 15:29:11 +01:00
parent 51e9ee9b9f
commit 930507a694
3 changed files with 211 additions and 168 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

225
api/src/main/java/edu/cornell/mannlib/semservices/service/impl/UMLSService.java
View file

@ -2,13 +2,11 @@
package edu.cornell.mannlib.semservices.service.impl;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.io.StringWriter;
import java.net.URL;
import java.net.URLEncoder;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -20,6 +18,11 @@ import edu.cornell.mannlib.semservices.bo.Concept;
import edu.cornell.mannlib.semservices.exceptions.ConceptsNotFoundException;
import edu.cornell.mannlib.semservices.service.ExternalConceptService;
import edu.cornell.mannlib.vitro.webapp.utils.json.JacksonUtils;
import org.apache.http.HttpVersion;
import org.apache.http.client.fluent.Form;
import org.apache.http.client.fluent.Request;
import org.apache.http.client.utils.URIBuilder;
import org.springframework.util.StringUtils;
/**
* @author jaf30
@ -27,37 +30,80 @@ import edu.cornell.mannlib.vitro.webapp.utils.json.JacksonUtils;
*/
public class UMLSService implements ExternalConceptService {
protected final Log logger = LogFactory.getLog(getClass());
private static final String submissionUrl = "http://link.informatics.stonybrook.edu/MeaningLookup/MlServiceServlet?";
private static final String baseUri = "http://link.informatics.stonybrook.edu/umls/CUI/";
private static final String endpoint = "http://link.informatics.stonybrook.edu/sparql/";
private static final String schemeURI = "http://link.informatics.stonybrook.edu/umls";
private static String UTS_REST_API_URL = "https://uts-ws.nlm.nih.gov/rest";
private static String SEARCH_PATH = "/search/current";
private static String SEARCH_PARAMETER = "string";
private static String SEARCH_TYPE_PARAMETER = "searchType";
private static String SEARCH_TYPE = "rightTruncation";
private static String PAGE_SIZE_PARAMETER = "pageSize";
private static String RETURN_TYPE_PARAMETER = "returnIdType";
private static String RETURN_TYPE = "concept";
private static String TICKET_PARAMETER = "ticket";
private static String ticketGrantingTicketURL = null;
private static long lastUpdate = -1;
private static String username = null;
private static String password = null;
private static String apikey = null;
private static String pageSize = "50";
private static String UMLS_AUTH_USER_URL = "https://utslogin.nlm.nih.gov/cas/v1/tickets";
private static String UMLS_AUTH_KEY_URL = "https://utslogin.nlm.nih.gov/cas/v1/api-key";
private static String UTS_SERVICE_URL = "http://umlsks.nlm.nih.gov";
{
if (username == null || apikey == null) {
final Properties properties = new Properties();
try (InputStream stream = getClass().getResourceAsStream("/umls.properties")) {
properties.load(stream);
username = properties.getProperty("username");
password = properties.getProperty("password");
apikey = properties.getProperty("apikey");
String exPageSize = properties.getProperty("pagesize");
try {
if (!StringUtils.isEmpty(exPageSize)) {
int iPageSize = Integer.parseInt(exPageSize, 10);
if (iPageSize > 5 && iPageSize < 200) {
pageSize = Integer.toString(iPageSize, 10);
}
}
} catch (Exception e) {
}
} catch (IOException e) {
}
}
}
public boolean isConfigured() {
return !(StringUtils.isEmpty(username) && StringUtils.isEmpty(apikey));
}
@Override
public List<Concept> getConcepts(String term) throws Exception {
String ticket = getSingleUseTicket();
List<Concept> conceptList = new ArrayList<Concept>();
String results = null;
String dataUrl = submissionUrl + "textToProcess="
+ URLEncoder.encode(term, "UTF-8")
+ "&format=json";
try {
URIBuilder b = new URIBuilder(UTS_REST_API_URL + SEARCH_PATH);
b.addParameter(SEARCH_PARAMETER, term);
b.addParameter(RETURN_TYPE_PARAMETER, RETURN_TYPE);
b.addParameter(SEARCH_TYPE_PARAMETER, SEARCH_TYPE);
b.addParameter(PAGE_SIZE_PARAMETER, pageSize);
b.addParameter(TICKET_PARAMETER, ticket);
StringWriter sw = new StringWriter();
URL rss = new URL(dataUrl);
results = Request.Get(b.build())
.connectTimeout(3000)
.socketTimeout(3000)
.execute().returnContent().asString();
BufferedReader in = new BufferedReader(new InputStreamReader(
rss.openStream()));
String inputLine;
while ((inputLine = in.readLine()) != null) {
sw.write(inputLine);
}
in.close();
results = sw.toString();
//System.out.println("results before processing: "+results);
conceptList = processOutput(results);
return conceptList;
@ -68,39 +114,13 @@ public class UMLSService implements ExternalConceptService {
}
public List<Concept> processResults(String term) throws Exception {
String results = null;
String dataUrl = submissionUrl + "textToProcess="
+ URLEncoder.encode(term, "UTF-8") + "&format=json";
try {
StringWriter sw = new StringWriter();
URL rss = new URL(dataUrl);
BufferedReader in = new BufferedReader(new InputStreamReader(rss.openStream()));
String inputLine;
while ((inputLine = in.readLine()) != null) {
sw.write(inputLine);
}
in.close();
results = sw.toString();
//System.out.println("results before processing: "+results);
List<Concept> conceptList = processOutput(results);
return conceptList;
} catch (Exception ex) {
logger.error("error occurred in servlet", ex);
return null;
}
return getConcepts(term);
}
/**
* @param uri URI
*/
public List<Concept> getConceptsByURIWithSparql(String uri)
throws Exception {
public List<Concept> getConceptsByURIWithSparql(String uri) throws Exception {
// deprecating this method...just return an empty list
List<Concept> conceptList = new ArrayList<Concept>();
return conceptList;
@ -110,77 +130,37 @@ public class UMLSService implements ExternalConceptService {
* @param results Results to process
*/
private List<Concept> processOutput(String results) throws Exception {
List<Concept> conceptList = new ArrayList<Concept>();
List<String> bestMatchIdList = new ArrayList<String>();
String bestMatchId = new String();
boolean bestMatchFound = false;
boolean allFound = false;
try {
ObjectNode json = (ObjectNode) JacksonUtils.parseJson(results);
//System.out.println(json.toString());
if (json.has("Best Match")) {
bestMatchFound = true;
//System.out.println("Best Match");
ArrayNode bestMatchArray = (ArrayNode) json.get("Best Match");
int len = bestMatchArray.size();
if (len > 1) {
logger.debug("Found this many best matches: "+ len);
}
int i;
for (i = 0; i < len; i++) {
ObjectNode o = (ObjectNode) bestMatchArray.get(i);
//System.out.println(o.toString());
Concept concept = new Concept();
concept.setDefinedBy(schemeURI);
concept.setBestMatch("true");
String cui = getJsonValue(o, "CUI");
bestMatchIdList.add(cui);
concept.setConceptId(cui);
concept.setLabel(getJsonValue(o, "label"));
concept.setType(getJsonValue(o, "type"));
concept.setDefinition(getJsonValue(o, "definition"));
concept.setUri(baseUri + cui);
concept.setSchemeURI(schemeURI);
conceptList.add(concept);
}
}
if (json.has("All")) {
allFound = true;
ArrayNode allArray = (ArrayNode) json.get("All");
ArrayNode allArray = (ArrayNode) json.get("result").get("results");
int len = allArray.size();
//System.out.println("size of best match array: "+ len);
int i;
for (i = 0; i < len; i++) {
ObjectNode o = (ObjectNode) allArray.get(i);
//System.out.println(o.toString());
Concept concept = new Concept();
concept.setDefinedBy(schemeURI);
String cui = getJsonValue(o, "CUI");
concept.setConceptId(cui);
concept.setLabel(getJsonValue(o, "label"));
concept.setType(getJsonValue(o, "type"));
concept.setDefinition(getJsonValue(o, "definition"));
concept.setUri(baseUri + cui);
concept.setSchemeURI(schemeURI);
// prevent duplicate concepts in list
if (! bestMatchIdList.contains(cui)) {
Concept concept = new Concept();
concept.setDefinedBy(UTS_SERVICE_URL);
concept.setSchemeURI(UTS_SERVICE_URL);
concept.setType(RETURN_TYPE);
concept.setConceptId(getJsonValue(o, "ui"));
concept.setLabel(getJsonValue(o, "name"));
concept.setUri(getJsonValue(o, "uri"));
concept.setBestMatch("false");
conceptList.add(concept);
}
}
}
} catch (Exception ex) {
ex.printStackTrace();
logger.error("Could not get concepts", ex);
throw ex;
}
if (! bestMatchFound && !allFound) {
// we did not get a bestMatch or All element
if (conceptList.size() == 0) {
throw new ConceptsNotFoundException();
}
@ -191,6 +171,7 @@ public class UMLSService implements ExternalConceptService {
/**
* Get a string from a json object or an empty string if there is no value for the given key
*
* @param obj JSON Object
* @param key Key to retrieve
*/
@ -203,7 +184,6 @@ public class UMLSService implements ExternalConceptService {
}
protected String stripConceptId(String uri) {
String conceptId = new String();
int lastslash = uri.lastIndexOf('/');
@ -211,4 +191,39 @@ public class UMLSService implements ExternalConceptService {
return conceptId;
}
private synchronized void getTicketGrantingTicket() {
if (StringUtils.isEmpty(username) && StringUtils.isEmpty(apikey)) {
throw new IllegalStateException("Unable to read umls.properties");
}
if (ticketGrantingTicketURL == null || lastUpdate + 28700000l < System.currentTimeMillis()) {
try {
if (!StringUtils.isEmpty(apikey)) {
ticketGrantingTicketURL = Request.Post(UMLS_AUTH_KEY_URL).useExpectContinue().version(HttpVersion.HTTP_1_1)
.bodyForm(Form.form().add("apikey", apikey).build())
.execute().returnResponse().getFirstHeader("location").getValue();
} else {
ticketGrantingTicketURL = Request.Post(UMLS_AUTH_USER_URL).useExpectContinue().version(HttpVersion.HTTP_1_1)
.bodyForm(Form.form().add("username", username).add("password", password).build())
.execute().returnResponse().getFirstHeader("location").getValue();
}
} catch (IOException e) {
throw new IllegalStateException("Unable to get ticket granting ticket.");
}
lastUpdate = System.currentTimeMillis();
}
}
private String getSingleUseTicket() {
getTicketGrantingTicket();
String ticket = "";
try {
ticket = Request.Post(ticketGrantingTicketURL).useExpectContinue().version(HttpVersion.HTTP_1_1)
.bodyForm(Form.form().add("service", UTS_SERVICE_URL).build())
.execute().returnContent().asString();
} catch (IOException e) {
throw new IllegalStateException("Unable to get ticket.");
}
return ticket;
}
}

19
api/src/test/java/edu/cornell/mannlib/semservices/service/impl/UMLSServiceTest.java Normal file
View file

@ -0,0 +1,19 @@
package edu.cornell.mannlib.semservices.service.impl;
import edu.cornell.mannlib.semservices.bo.Concept;
import org.junit.Assert;
import org.junit.Test;
import java.util.List;
public class UMLSServiceTest {
@Test
public void testUmls() throws Exception {
UMLSService service = new UMLSService();
if (service.isConfigured()) {
List<Concept> concepts = service.getConcepts("diabetes");
Assert.assertNotNull(concepts);
}
}
}

9
api/src/test/resources/umls.properties Normal file
View file

@ -0,0 +1,9 @@
# Configure credentials to access UMLS service
# Sign up here - https://uts.nlm.nih.gov//home.html
# You can user either a username / password combination, or an apikey
#username =
#password =
#apikey =