[VIVO-1311] Implementation of UMLS datasource using NLM APIs
This commit is contained in:
Graham Triggs
parent
51e9ee9b9f
commit
930507a694
3 changed files with 211 additions and 168 deletions
|
|
@ -2,13 +2,11 @@
|
||||||
|
|
||||||
package edu.cornell.mannlib.semservices.service.impl;
|
package edu.cornell.mannlib.semservices.service.impl;
|
||||||
|
|
||||||
import java.io.BufferedReader;
|
import java.io.IOException;
|
||||||
import java.io.InputStreamReader;
|
import java.io.InputStream;
|
||||||
import java.io.StringWriter;
|
|
||||||
import java.net.URL;
|
|
||||||
import java.net.URLEncoder;
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.Properties;
|
||||||
|
|
||||||
import org.apache.commons.logging.Log;
|
import org.apache.commons.logging.Log;
|
||||||
import org.apache.commons.logging.LogFactory;
|
import org.apache.commons.logging.LogFactory;
|
||||||
|
|
@ -20,195 +18,212 @@ import edu.cornell.mannlib.semservices.bo.Concept;
|
||||||
import edu.cornell.mannlib.semservices.exceptions.ConceptsNotFoundException;
|
import edu.cornell.mannlib.semservices.exceptions.ConceptsNotFoundException;
|
||||||
import edu.cornell.mannlib.semservices.service.ExternalConceptService;
|
import edu.cornell.mannlib.semservices.service.ExternalConceptService;
|
||||||
import edu.cornell.mannlib.vitro.webapp.utils.json.JacksonUtils;
|
import edu.cornell.mannlib.vitro.webapp.utils.json.JacksonUtils;
|
||||||
|
import org.apache.http.HttpVersion;
|
||||||
|
import org.apache.http.client.fluent.Form;
|
||||||
|
import org.apache.http.client.fluent.Request;
|
||||||
|
import org.apache.http.client.utils.URIBuilder;
|
||||||
|
import org.springframework.util.StringUtils;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @author jaf30
|
* @author jaf30
|
||||||
*
|
*
|
||||||
*/
|
*/
|
||||||
public class UMLSService implements ExternalConceptService {
|
public class UMLSService implements ExternalConceptService {
|
||||||
protected final Log logger = LogFactory.getLog(getClass());
|
protected final Log logger = LogFactory.getLog(getClass());
|
||||||
private static final String submissionUrl = "http://link.informatics.stonybrook.edu/MeaningLookup/MlServiceServlet?";
|
|
||||||
private static final String baseUri = "http://link.informatics.stonybrook.edu/umls/CUI/";
|
|
||||||
private static final String endpoint = "http://link.informatics.stonybrook.edu/sparql/";
|
|
||||||
private static final String schemeURI = "http://link.informatics.stonybrook.edu/umls";
|
|
||||||
|
|
||||||
|
private static String UTS_REST_API_URL = "https://uts-ws.nlm.nih.gov/rest";
|
||||||
|
private static String SEARCH_PATH = "/search/current";
|
||||||
|
private static String SEARCH_PARAMETER = "string";
|
||||||
|
private static String SEARCH_TYPE_PARAMETER = "searchType";
|
||||||
|
private static String SEARCH_TYPE = "rightTruncation";
|
||||||
|
private static String PAGE_SIZE_PARAMETER = "pageSize";
|
||||||
|
private static String RETURN_TYPE_PARAMETER = "returnIdType";
|
||||||
|
private static String RETURN_TYPE = "concept";
|
||||||
|
private static String TICKET_PARAMETER = "ticket";
|
||||||
|
|
||||||
|
private static String ticketGrantingTicketURL = null;
|
||||||
|
|
||||||
@Override
|
private static long lastUpdate = -1;
|
||||||
public List<Concept> getConcepts(String term) throws Exception {
|
|
||||||
List<Concept> conceptList = new ArrayList<Concept>();
|
|
||||||
|
|
||||||
String results = null;
|
private static String username = null;
|
||||||
String dataUrl = submissionUrl + "textToProcess="
|
private static String password = null;
|
||||||
+ URLEncoder.encode(term, "UTF-8")
|
private static String apikey = null;
|
||||||
+ "&format=json";
|
|
||||||
|
|
||||||
try {
|
private static String pageSize = "50";
|
||||||
|
|
||||||
StringWriter sw = new StringWriter();
|
private static String UMLS_AUTH_USER_URL = "https://utslogin.nlm.nih.gov/cas/v1/tickets";
|
||||||
URL rss = new URL(dataUrl);
|
private static String UMLS_AUTH_KEY_URL = "https://utslogin.nlm.nih.gov/cas/v1/api-key";
|
||||||
|
private static String UTS_SERVICE_URL = "http://umlsks.nlm.nih.gov";
|
||||||
|
|
||||||
BufferedReader in = new BufferedReader(new InputStreamReader(
|
{
|
||||||
rss.openStream()));
|
if (username == null || apikey == null) {
|
||||||
String inputLine;
|
final Properties properties = new Properties();
|
||||||
while ((inputLine = in.readLine()) != null) {
|
try (InputStream stream = getClass().getResourceAsStream("/umls.properties")) {
|
||||||
sw.write(inputLine);
|
properties.load(stream);
|
||||||
}
|
username = properties.getProperty("username");
|
||||||
in.close();
|
password = properties.getProperty("password");
|
||||||
|
apikey = properties.getProperty("apikey");
|
||||||
|
|
||||||
results = sw.toString();
|
String exPageSize = properties.getProperty("pagesize");
|
||||||
//System.out.println("results before processing: "+results);
|
try {
|
||||||
conceptList = processOutput(results);
|
if (!StringUtils.isEmpty(exPageSize)) {
|
||||||
return conceptList;
|
int iPageSize = Integer.parseInt(exPageSize, 10);
|
||||||
|
if (iPageSize > 5 && iPageSize < 200) {
|
||||||
} catch (Exception ex) {
|
pageSize = Integer.toString(iPageSize, 10);
|
||||||
logger.error("error occurred in servlet", ex);
|
}
|
||||||
return null;
|
}
|
||||||
}
|
} catch (Exception e) {
|
||||||
}
|
}
|
||||||
|
} catch (IOException e) {
|
||||||
public List<Concept> processResults(String term) throws Exception {
|
|
||||||
String results = null;
|
|
||||||
String dataUrl = submissionUrl + "textToProcess="
|
|
||||||
+ URLEncoder.encode(term, "UTF-8") + "&format=json";
|
|
||||||
|
|
||||||
try {
|
|
||||||
|
|
||||||
StringWriter sw = new StringWriter();
|
|
||||||
URL rss = new URL(dataUrl);
|
|
||||||
|
|
||||||
BufferedReader in = new BufferedReader(new InputStreamReader(rss.openStream()));
|
|
||||||
String inputLine;
|
|
||||||
while ((inputLine = in.readLine()) != null) {
|
|
||||||
sw.write(inputLine);
|
|
||||||
}
|
|
||||||
in.close();
|
|
||||||
|
|
||||||
results = sw.toString();
|
|
||||||
//System.out.println("results before processing: "+results);
|
|
||||||
List<Concept> conceptList = processOutput(results);
|
|
||||||
return conceptList;
|
|
||||||
|
|
||||||
} catch (Exception ex) {
|
|
||||||
logger.error("error occurred in servlet", ex);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param uri URI
|
|
||||||
*/
|
|
||||||
public List<Concept> getConceptsByURIWithSparql(String uri)
|
|
||||||
throws Exception {
|
|
||||||
// deprecating this method...just return an empty list
|
|
||||||
List<Concept> conceptList = new ArrayList<Concept>();
|
|
||||||
return conceptList;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param results Results to process
|
|
||||||
*/
|
|
||||||
private List<Concept> processOutput(String results) throws Exception {
|
|
||||||
|
|
||||||
List<Concept> conceptList = new ArrayList<Concept>();
|
|
||||||
List<String> bestMatchIdList = new ArrayList<String>();
|
|
||||||
String bestMatchId = new String();
|
|
||||||
boolean bestMatchFound = false;
|
|
||||||
boolean allFound = false;
|
|
||||||
|
|
||||||
try {
|
|
||||||
ObjectNode json = (ObjectNode) JacksonUtils.parseJson(results);
|
|
||||||
//System.out.println(json.toString());
|
|
||||||
if (json.has("Best Match")) {
|
|
||||||
bestMatchFound = true;
|
|
||||||
//System.out.println("Best Match");
|
|
||||||
|
|
||||||
ArrayNode bestMatchArray = (ArrayNode) json.get("Best Match");
|
|
||||||
int len = bestMatchArray.size();
|
|
||||||
if (len > 1) {
|
|
||||||
logger.debug("Found this many best matches: "+ len);
|
|
||||||
}
|
}
|
||||||
int i;
|
}
|
||||||
for (i = 0; i < len; i++) {
|
}
|
||||||
ObjectNode o = (ObjectNode) bestMatchArray.get(i);
|
|
||||||
//System.out.println(o.toString());
|
|
||||||
Concept concept = new Concept();
|
|
||||||
concept.setDefinedBy(schemeURI);
|
|
||||||
concept.setBestMatch("true");
|
|
||||||
String cui = getJsonValue(o, "CUI");
|
|
||||||
bestMatchIdList.add(cui);
|
|
||||||
|
|
||||||
concept.setConceptId(cui);
|
public boolean isConfigured() {
|
||||||
concept.setLabel(getJsonValue(o, "label"));
|
return !(StringUtils.isEmpty(username) && StringUtils.isEmpty(apikey));
|
||||||
concept.setType(getJsonValue(o, "type"));
|
}
|
||||||
concept.setDefinition(getJsonValue(o, "definition"));
|
|
||||||
concept.setUri(baseUri + cui);
|
@Override
|
||||||
concept.setSchemeURI(schemeURI);
|
public List<Concept> getConcepts(String term) throws Exception {
|
||||||
conceptList.add(concept);
|
String ticket = getSingleUseTicket();
|
||||||
}
|
|
||||||
}
|
List<Concept> conceptList = new ArrayList<Concept>();
|
||||||
if (json.has("All")) {
|
|
||||||
allFound = true;
|
String results = null;
|
||||||
ArrayNode allArray = (ArrayNode) json.get("All");
|
|
||||||
|
try {
|
||||||
|
URIBuilder b = new URIBuilder(UTS_REST_API_URL + SEARCH_PATH);
|
||||||
|
b.addParameter(SEARCH_PARAMETER, term);
|
||||||
|
b.addParameter(RETURN_TYPE_PARAMETER, RETURN_TYPE);
|
||||||
|
b.addParameter(SEARCH_TYPE_PARAMETER, SEARCH_TYPE);
|
||||||
|
b.addParameter(PAGE_SIZE_PARAMETER, pageSize);
|
||||||
|
b.addParameter(TICKET_PARAMETER, ticket);
|
||||||
|
|
||||||
|
results = Request.Get(b.build())
|
||||||
|
.connectTimeout(3000)
|
||||||
|
.socketTimeout(3000)
|
||||||
|
.execute().returnContent().asString();
|
||||||
|
|
||||||
|
conceptList = processOutput(results);
|
||||||
|
return conceptList;
|
||||||
|
|
||||||
|
} catch (Exception ex) {
|
||||||
|
logger.error("error occurred in servlet", ex);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<Concept> processResults(String term) throws Exception {
|
||||||
|
return getConcepts(term);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param uri URI
|
||||||
|
*/
|
||||||
|
public List<Concept> getConceptsByURIWithSparql(String uri) throws Exception {
|
||||||
|
// deprecating this method...just return an empty list
|
||||||
|
List<Concept> conceptList = new ArrayList<Concept>();
|
||||||
|
return conceptList;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param results Results to process
|
||||||
|
*/
|
||||||
|
private List<Concept> processOutput(String results) throws Exception {
|
||||||
|
List<Concept> conceptList = new ArrayList<Concept>();
|
||||||
|
List<String> bestMatchIdList = new ArrayList<String>();
|
||||||
|
String bestMatchId = new String();
|
||||||
|
|
||||||
|
try {
|
||||||
|
ObjectNode json = (ObjectNode) JacksonUtils.parseJson(results);
|
||||||
|
ArrayNode allArray = (ArrayNode) json.get("result").get("results");
|
||||||
int len = allArray.size();
|
int len = allArray.size();
|
||||||
//System.out.println("size of best match array: "+ len);
|
|
||||||
int i;
|
int i;
|
||||||
for (i = 0; i < len; i++) {
|
for (i = 0; i < len; i++) {
|
||||||
ObjectNode o = (ObjectNode) allArray.get(i);
|
ObjectNode o = (ObjectNode) allArray.get(i);
|
||||||
//System.out.println(o.toString());
|
|
||||||
Concept concept = new Concept();
|
|
||||||
concept.setDefinedBy(schemeURI);
|
|
||||||
String cui = getJsonValue(o, "CUI");
|
|
||||||
concept.setConceptId(cui);
|
|
||||||
|
|
||||||
concept.setLabel(getJsonValue(o, "label"));
|
Concept concept = new Concept();
|
||||||
concept.setType(getJsonValue(o, "type"));
|
concept.setDefinedBy(UTS_SERVICE_URL);
|
||||||
concept.setDefinition(getJsonValue(o, "definition"));
|
concept.setSchemeURI(UTS_SERVICE_URL);
|
||||||
concept.setUri(baseUri + cui);
|
|
||||||
concept.setSchemeURI(schemeURI);
|
concept.setType(RETURN_TYPE);
|
||||||
// prevent duplicate concepts in list
|
concept.setConceptId(getJsonValue(o, "ui"));
|
||||||
if (! bestMatchIdList.contains(cui)) {
|
concept.setLabel(getJsonValue(o, "name"));
|
||||||
concept.setBestMatch("false");
|
concept.setUri(getJsonValue(o, "uri"));
|
||||||
conceptList.add(concept);
|
|
||||||
}
|
concept.setBestMatch("false");
|
||||||
|
conceptList.add(concept);
|
||||||
}
|
}
|
||||||
}
|
} catch (Exception ex) {
|
||||||
} catch (Exception ex ) {
|
ex.printStackTrace();
|
||||||
ex.printStackTrace();
|
logger.error("Could not get concepts", ex);
|
||||||
logger.error("Could not get concepts", ex);
|
throw ex;
|
||||||
throw ex;
|
}
|
||||||
}
|
|
||||||
if (! bestMatchFound && !allFound) {
|
|
||||||
// we did not get a bestMatch or All element
|
|
||||||
throw new ConceptsNotFoundException();
|
|
||||||
}
|
|
||||||
|
|
||||||
//
|
if (conceptList.size() == 0) {
|
||||||
return conceptList;
|
throw new ConceptsNotFoundException();
|
||||||
|
}
|
||||||
|
|
||||||
}
|
//
|
||||||
|
return conceptList;
|
||||||
|
|
||||||
/**
|
}
|
||||||
* Get a string from a json object or an empty string if there is no value for the given key
|
|
||||||
* @param obj JSON Object
|
/**
|
||||||
* @param key Key to retrieve
|
* Get a string from a json object or an empty string if there is no value for the given key
|
||||||
*/
|
*
|
||||||
protected String getJsonValue(ObjectNode obj, String key) {
|
* @param obj JSON Object
|
||||||
if (obj.has(key)) {
|
* @param key Key to retrieve
|
||||||
return obj.get(key).asText();
|
*/
|
||||||
} else {
|
protected String getJsonValue(ObjectNode obj, String key) {
|
||||||
return new String("");
|
if (obj.has(key)) {
|
||||||
}
|
return obj.get(key).asText();
|
||||||
}
|
} else {
|
||||||
|
return new String("");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
protected String stripConceptId(String uri) {
|
||||||
|
String conceptId = new String();
|
||||||
|
int lastslash = uri.lastIndexOf('/');
|
||||||
|
conceptId = uri.substring(lastslash + 1, uri.length());
|
||||||
|
return conceptId;
|
||||||
|
}
|
||||||
|
|
||||||
protected String stripConceptId(String uri) {
|
private synchronized void getTicketGrantingTicket() {
|
||||||
String conceptId = new String();
|
if (StringUtils.isEmpty(username) && StringUtils.isEmpty(apikey)) {
|
||||||
int lastslash = uri.lastIndexOf('/');
|
throw new IllegalStateException("Unable to read umls.properties");
|
||||||
conceptId = uri.substring(lastslash + 1, uri.length());
|
}
|
||||||
return conceptId;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
if (ticketGrantingTicketURL == null || lastUpdate + 28700000l < System.currentTimeMillis()) {
|
||||||
|
try {
|
||||||
|
if (!StringUtils.isEmpty(apikey)) {
|
||||||
|
ticketGrantingTicketURL = Request.Post(UMLS_AUTH_KEY_URL).useExpectContinue().version(HttpVersion.HTTP_1_1)
|
||||||
|
.bodyForm(Form.form().add("apikey", apikey).build())
|
||||||
|
.execute().returnResponse().getFirstHeader("location").getValue();
|
||||||
|
} else {
|
||||||
|
ticketGrantingTicketURL = Request.Post(UMLS_AUTH_USER_URL).useExpectContinue().version(HttpVersion.HTTP_1_1)
|
||||||
|
.bodyForm(Form.form().add("username", username).add("password", password).build())
|
||||||
|
.execute().returnResponse().getFirstHeader("location").getValue();
|
||||||
|
}
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new IllegalStateException("Unable to get ticket granting ticket.");
|
||||||
|
}
|
||||||
|
lastUpdate = System.currentTimeMillis();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String getSingleUseTicket() {
|
||||||
|
getTicketGrantingTicket();
|
||||||
|
String ticket = "";
|
||||||
|
try {
|
||||||
|
ticket = Request.Post(ticketGrantingTicketURL).useExpectContinue().version(HttpVersion.HTTP_1_1)
|
||||||
|
.bodyForm(Form.form().add("service", UTS_SERVICE_URL).build())
|
||||||
|
.execute().returnContent().asString();
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new IllegalStateException("Unable to get ticket.");
|
||||||
|
}
|
||||||
|
return ticket;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,19 @@
|
||||||
|
package edu.cornell.mannlib.semservices.service.impl;
|
||||||
|
|
||||||
|
import edu.cornell.mannlib.semservices.bo.Concept;
|
||||||
|
import org.junit.Assert;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
public class UMLSServiceTest {
|
||||||
|
@Test
|
||||||
|
public void testUmls() throws Exception {
|
||||||
|
UMLSService service = new UMLSService();
|
||||||
|
|
||||||
|
if (service.isConfigured()) {
|
||||||
|
List<Concept> concepts = service.getConcepts("diabetes");
|
||||||
|
Assert.assertNotNull(concepts);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
9
api/src/test/resources/umls.properties
Normal file
9
api/src/test/resources/umls.properties
Normal file
|
|
@ -0,0 +1,9 @@
|
||||||
|
# Configure credentials to access UMLS service
|
||||||
|
|
||||||
|
# Sign up here - https://uts.nlm.nih.gov//home.html
|
||||||
|
|
||||||
|
# You can user either a username / password combination, or an apikey
|
||||||
|
|
||||||
|
#username =
|
||||||
|
#password =
|
||||||
|
#apikey =
|
||||||
Loading…
Add table
Reference in a new issue