diff --git a/productMods/templates/freemarker/body/harvester/testfile.ftl b/productMods/templates/freemarker/body/harvester/testfile.ftl index 399b39d2..89934484 100644 --- a/productMods/templates/freemarker/body/harvester/testfile.ftl +++ b/productMods/templates/freemarker/body/harvester/testfile.ftl @@ -28,12 +28,12 @@ var response = harvestProgressResponse; var json = eval("(" + response + ")"); + var logAppend = json.progressSinceLastCheck; + var progressTextArea = document.getElementById("progressTextArea"); + progressTextArea.innerHTML = progressTextArea.innerHTML + logAppend; + progressTextArea.scrollTop = progressTextArea.scrollHeight; + if(!json.finished) { - var logAppend = json.progressSinceLastCheck; - var progressTextArea = document.getElementById("progressTextArea"); - progressTextArea.innerHTML = progressTextArea.innerHTML + logAppend; - progressTextArea.scrollTop = progressTextArea.scrollHeight; - var request = createRequest(); request.onreadystatechange=function() { if(request.readyState == 4 && request.status == 200) { @@ -44,6 +44,16 @@ request.open("POST", "/vivo/harvester/testfile", true); request.setRequestHeader("Content-type","application/x-www-form-urlencoded"); request.send("${paramIsHarvestClick}=false"); + } else { + + var importedGrants = document.getElementById("importedGrants") + + for(var i = 0; i < json.newlyAddedUrls.length; i++) { + + var newLi = document.createElement("li"); + newLi.innerHTML = "" + json.newlyAddedUrls[i] + ""; + importedGrants.appendChild(newLi); + } } } @@ -207,6 +217,11 @@
+
+
Imported grants
+ +
diff --git a/src/edu/cornell/mannlib/vitro/webapp/controller/harvester/TestFileController.java b/src/edu/cornell/mannlib/vitro/webapp/controller/harvester/TestFileController.java index 800ab06a..f7b1c4f5 100644 --- a/src/edu/cornell/mannlib/vitro/webapp/controller/harvester/TestFileController.java +++ b/src/edu/cornell/mannlib/vitro/webapp/controller/harvester/TestFileController.java @@ -17,6 +17,8 @@ import javax.servlet.ServletContext; import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import javax.xml.parsers.DocumentBuilderFactory; + import org.apache.commons.fileupload.FileItem; import org.apache.commons.fileupload.servlet.ServletFileUpload; import org.apache.commons.logging.Log; @@ -24,6 +26,10 @@ import org.apache.commons.logging.LogFactory; import org.json.JSONException; import org.json.JSONObject; import org.skife.csv.SimpleReader; +import org.w3c.dom.Document; +import org.w3c.dom.NamedNodeMap; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; import edu.cornell.mannlib.vitro.webapp.config.ConfigurationProperties; import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest; @@ -284,8 +290,9 @@ public class TestFileController extends FreemarkerHttpServlet { //String path = getUploadPath(vreq); String script = job.getScript(); + String additionsFilePath = job.getAdditionsFilePath(); log.error("start harvest"); - runScript(getSessionId(request), script); + runScript(getSessionId(request), script, additionsFilePath); log.error("end harvest"); JSONObject json = new JSONObject(); @@ -330,9 +337,27 @@ public class TestFileController extends FreemarkerHttpServlet { boolean finished = !sessionIdToHarvestThread.containsKey(sessionId); + VitroRequest vreq = new VitroRequest(request); + ArrayList newlyAddedUrls = new ArrayList(); + if(finished) { + ArrayList newlyAddedUris = sessionIdToNewlyAddedUris.get(sessionId); + if(newlyAddedUris != null) { + for(String uri : newlyAddedUris) { + + String namespaceRoot = vreq.getWebappDaoFactory().getDefaultNamespace(); + + String suffix = uri.substring(namespaceRoot.length()); + String url = "display/" + suffix; + + newlyAddedUrls.add(uri); + } + } + } + JSONObject json = new JSONObject(); json.put("progressSinceLastCheck", progressSinceLastCheck); json.put("finished", finished); + json.put("newlyAddedUrls", newlyAddedUrls); response.getWriter().write(json.toString()); } @@ -359,11 +384,11 @@ public class TestFileController extends FreemarkerHttpServlet { } - private void runScript(String sessionId, String script) { + private void runScript(String sessionId, String script, String additionsFilePath) { if(!sessionIdToHarvestThread.containsKey(sessionId)) { - ScriptRunner runner = new ScriptRunner(sessionId, script); + ScriptRunner runner = new ScriptRunner(sessionId, script, additionsFilePath); sessionIdToHarvestThread.put(sessionId, runner); runner.start(); } @@ -409,7 +434,61 @@ public class TestFileController extends FreemarkerHttpServlet { return request.getSession().getId(); } + private ArrayList extractNewlyAddedUris(File additionsFile) { + ArrayList newlyAddedUris = new ArrayList(); + log.error(additionsFile.getAbsolutePath()); + + try { + Document document = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(additionsFile); + NodeList descriptionNodes = document.getElementsByTagName("http://www.w3.org/1999/02/22-rdf-syntax-ns#Description"); + + int numNodes = descriptionNodes.getLength(); + for(int i = 0; i < numNodes; i++) { + Node node = descriptionNodes.item(i); + + ArrayList types = getRdfTypes(node); + if(types.contains("http://vivoweb.org/ontology/core#Grant")) { //todo: generalize + + NamedNodeMap attributes = node.getAttributes(); + Node aboutAttribute = attributes.getNamedItem("http://www.w3.org/1999/02/22-rdf-syntax-ns#about"); + if(aboutAttribute != null) { + String value = aboutAttribute.getNodeValue(); + newlyAddedUris.add(value); + } + } + } + + + + } catch(Exception e) { + log.error(e, e); + } + + return newlyAddedUris; + } + + private ArrayList getRdfTypes(Node descriptionNode) { + ArrayList rdfTypesList = new ArrayList(); + + NodeList children = descriptionNode.getChildNodes(); + int numChildren = children.getLength(); + for(int i = 0; i < numChildren; i++) { + Node child = children.item(i); + + String name = child.getNodeName(); + if(name.equals("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")) { + NamedNodeMap attributes = child.getAttributes(); + Node resourceAttribute = attributes.getNamedItem("http://www.w3.org/1999/02/22-rdf-syntax-ns#resource"); + if(resourceAttribute != null) { + String value = resourceAttribute.getNodeValue(); + rdfTypesList.add(value); + } + } + } + + return rdfTypesList; + } @@ -435,18 +514,21 @@ public class TestFileController extends FreemarkerHttpServlet { super(cause); } } - - + + private Map sessionIdToHarvestThread = new Hashtable(); //Hashtable is threadsafe, HashMap is not private Map> sessionIdToUnsentLogLines = new Hashtable>(); //Hashtable is threadsafe, HashMap is not + private Map> sessionIdToNewlyAddedUris = new Hashtable>(); private class ScriptRunner extends Thread { private final String sessionId; private final String script; + private final String additionsFilePath; - public ScriptRunner(String sessionId, String script) { + public ScriptRunner(String sessionId, String script, String additionsFilePath) { this.sessionId = sessionId; this.script = script; + this.additionsFilePath = additionsFilePath; } @Override @@ -455,16 +537,16 @@ public class TestFileController extends FreemarkerHttpServlet { ArrayList unsentLogLines = sessionIdToUnsentLogLines.get(sessionId); if(unsentLogLines == null) { unsentLogLines = new ArrayList(); - sessionIdToUnsentLogLines.put(sessionId, unsentLogLines); + sessionIdToUnsentLogLines.put(this.sessionId, unsentLogLines); } - File scriptFile = createScriptFile(script); + File scriptFile = createScriptFile(this.script); String command = "/bin/bash " + getHarvesterPath() + "scripts/temp/" + scriptFile.getName(); log.info("Running command: " + command); Process pr = Runtime.getRuntime().exec(command); - + //try { Thread.sleep(15000); } catch(InterruptedException e) {log.error(e, e);} BufferedReader processOutputReader = new BufferedReader(new InputStreamReader(pr.getInputStream())); @@ -481,13 +563,19 @@ public class TestFileController extends FreemarkerHttpServlet { } int exitVal; - + try { exitVal = pr.waitFor(); } catch(InterruptedException e) { throw new IOException(e.getMessage(), e); } + + File additionsFile = new File(this.additionsFilePath); + ArrayList newlyAddedUris = extractNewlyAddedUris(additionsFile); + log.error("newly added URIs size: " + newlyAddedUris.size()); + sessionIdToNewlyAddedUris.put(this.sessionId, newlyAddedUris); + log.debug("Harvester script exited with error code " + exitVal); log.info("Harvester script execution complete"); } catch (IOException e) { @@ -498,9 +586,7 @@ public class TestFileController extends FreemarkerHttpServlet { } } } - } - } @@ -542,6 +628,7 @@ class CsvHarvestJob implements FileHarvestJob { public CsvHarvestJob(VitroRequest vreq, String templateFileName, String namespace) { this.vreq = vreq; this.templateFile = new File(getTemplateFileDirectory() + templateFileName); + log.error(getTemplateFileDirectory() + templateFileName); this.namespace = namespace; } @@ -607,13 +694,24 @@ class CsvHarvestJob implements FileHarvestJob { * @return an error message if the two lines don't match, or null if they do */ private String validateCsvFirstLine(String[] templateFirstLine, String[] line) { - String errorMessage = "File header does not match specification"; - if(line.length != templateFirstLine.length) - return errorMessage; + String errorMessage = "File header does not match template"; + if(line.length != templateFirstLine.length) { + //return errorMessage + ": " + "file header columns = " + line.length + ", template columns = " + templateFirstLine.length; + String errorMsg = ""; + errorMsg += "file header items: "; + for(int i = 0; i < line.length; i++) { + errorMsg += line[i] + ", "; + } + errorMsg += "template items: "; + for(int i = 0; i < templateFirstLine.length; i++) { + errorMsg += templateFirstLine[i] + ", "; + } + return errorMsg; + } for(int i = 0; i < line.length; i++) { if(!line[i].equals(templateFirstLine[i])) - return errorMessage; + return errorMessage + ": file header column " + (i + 1) + " = " + line[i] + ", template column " + (i + 1) + " = " + templateFirstLine[i]; } return null; } @@ -675,6 +773,12 @@ class CsvHarvestJob implements FileHarvestJob { public void performHarvest(File directory) { } + + @Override + public String getAdditionsFilePath() { + + return TestFileController.getHarvesterPath() + "harvested-data/csv/additions.rdf.xml"; + }