litvinovg/vivo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

TestFileController.java and testfile.ftl: add links to newly ingested items via parsing the additions file

Browse source
This commit is contained in:
mbarbier 2011-06-03 20:44:15 +00:00
parent 661e14c350
commit 589a5cf09a
2 changed files with 140 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
productMods/templates/freemarker/body/harvester/testfile.ftl
View file

@ -28,12 +28,12 @@
var response = harvestProgressResponse; var response = harvestProgressResponse;
var json = eval("(" + response + ")"); var json = eval("(" + response + ")");
if(!json.finished) { var logAppend = json.progressSinceLastCheck;
var logAppend = json.progressSinceLastCheck; var progressTextArea = document.getElementById("progressTextArea");
var progressTextArea = document.getElementById("progressTextArea"); progressTextArea.innerHTML = progressTextArea.innerHTML + logAppend;
progressTextArea.innerHTML = progressTextArea.innerHTML + logAppend; progressTextArea.scrollTop = progressTextArea.scrollHeight;
progressTextArea.scrollTop = progressTextArea.scrollHeight;
if(!json.finished) {
var request = createRequest(); var request = createRequest();
request.onreadystatechange=function() { request.onreadystatechange=function() {
if(request.readyState == 4 && request.status == 200) { if(request.readyState == 4 && request.status == 200) {
@ -44,6 +44,16 @@
request.open("POST", "/vivo/harvester/testfile", true); request.open("POST", "/vivo/harvester/testfile", true);
request.setRequestHeader("Content-type","application/x-www-form-urlencoded"); request.setRequestHeader("Content-type","application/x-www-form-urlencoded");
request.send("${paramIsHarvestClick}=false"); request.send("${paramIsHarvestClick}=false");
} else {
var importedGrants = document.getElementById("importedGrants")
for(var i = 0; i < json.newlyAddedUrls.length; i++) {
var newLi = document.createElement("li");
newLi.innerHTML = "<a href=\"" + json.newlyAddedUrls[i] + "\">" + json.newlyAddedUrls[i] + "</a>";
importedGrants.appendChild(newLi);
}
} }
} }
@ -207,6 +217,11 @@
<div id="progress"> <div id="progress">
<textarea cols="100" rows="50" readonly="readonly" id="progressTextArea"></textarea> <textarea cols="100" rows="50" readonly="readonly" id="progressTextArea"></textarea>
</div> </div>
<div id="summary">
<h5>Imported grants</h5>
<ul id="importedGrants">
</ul>
</div>
</div> </div>
<div class="clearBothDiv" /> <div class="clearBothDiv" />
</div> </div>

128
src/edu/cornell/mannlib/vitro/webapp/controller/harvester/TestFileController.java
View file

@ -17,6 +17,8 @@ import javax.servlet.ServletContext;
import javax.servlet.ServletException; import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpServletResponse;
import javax.xml.parsers.DocumentBuilderFactory;
import org.apache.commons.fileupload.FileItem; import org.apache.commons.fileupload.FileItem;
import org.apache.commons.fileupload.servlet.ServletFileUpload; import org.apache.commons.fileupload.servlet.ServletFileUpload;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
@ -24,6 +26,10 @@ import org.apache.commons.logging.LogFactory;
import org.json.JSONException; import org.json.JSONException;
import org.json.JSONObject; import org.json.JSONObject;
import org.skife.csv.SimpleReader; import org.skife.csv.SimpleReader;
import org.w3c.dom.Document;
import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import edu.cornell.mannlib.vitro.webapp.config.ConfigurationProperties; import edu.cornell.mannlib.vitro.webapp.config.ConfigurationProperties;
import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest; import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
@ -284,8 +290,9 @@ public class TestFileController extends FreemarkerHttpServlet {
//String path = getUploadPath(vreq); //String path = getUploadPath(vreq);
String script = job.getScript(); String script = job.getScript();
String additionsFilePath = job.getAdditionsFilePath();
log.error("start harvest"); log.error("start harvest");
runScript(getSessionId(request), script); runScript(getSessionId(request), script, additionsFilePath);
log.error("end harvest"); log.error("end harvest");
JSONObject json = new JSONObject(); JSONObject json = new JSONObject();
@ -330,9 +337,27 @@ public class TestFileController extends FreemarkerHttpServlet {
boolean finished = !sessionIdToHarvestThread.containsKey(sessionId); boolean finished = !sessionIdToHarvestThread.containsKey(sessionId);
VitroRequest vreq = new VitroRequest(request);
ArrayList<String> newlyAddedUrls = new ArrayList<String>();
if(finished) {
ArrayList<String> newlyAddedUris = sessionIdToNewlyAddedUris.get(sessionId);
if(newlyAddedUris != null) {
for(String uri : newlyAddedUris) {
String namespaceRoot = vreq.getWebappDaoFactory().getDefaultNamespace();
String suffix = uri.substring(namespaceRoot.length());
String url = "display/" + suffix;
newlyAddedUrls.add(uri);
}
}
}
JSONObject json = new JSONObject(); JSONObject json = new JSONObject();
json.put("progressSinceLastCheck", progressSinceLastCheck); json.put("progressSinceLastCheck", progressSinceLastCheck);
json.put("finished", finished); json.put("finished", finished);
json.put("newlyAddedUrls", newlyAddedUrls);
response.getWriter().write(json.toString()); response.getWriter().write(json.toString());
} }
@ -359,11 +384,11 @@ public class TestFileController extends FreemarkerHttpServlet {
} }
private void runScript(String sessionId, String script) { private void runScript(String sessionId, String script, String additionsFilePath) {
if(!sessionIdToHarvestThread.containsKey(sessionId)) { if(!sessionIdToHarvestThread.containsKey(sessionId)) {
ScriptRunner runner = new ScriptRunner(sessionId, script); ScriptRunner runner = new ScriptRunner(sessionId, script, additionsFilePath);
sessionIdToHarvestThread.put(sessionId, runner); sessionIdToHarvestThread.put(sessionId, runner);
runner.start(); runner.start();
} }
@ -409,7 +434,61 @@ public class TestFileController extends FreemarkerHttpServlet {
return request.getSession().getId(); return request.getSession().getId();
} }
private ArrayList<String> extractNewlyAddedUris(File additionsFile) {
ArrayList<String> newlyAddedUris = new ArrayList<String>();
log.error(additionsFile.getAbsolutePath());
try {
Document document = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(additionsFile);
NodeList descriptionNodes = document.getElementsByTagName("http://www.w3.org/1999/02/22-rdf-syntax-ns#Description");
int numNodes = descriptionNodes.getLength();
for(int i = 0; i < numNodes; i++) {
Node node = descriptionNodes.item(i);
ArrayList<String> types = getRdfTypes(node);
if(types.contains("http://vivoweb.org/ontology/core#Grant")) { //todo: generalize
NamedNodeMap attributes = node.getAttributes();
Node aboutAttribute = attributes.getNamedItem("http://www.w3.org/1999/02/22-rdf-syntax-ns#about");
if(aboutAttribute != null) {
String value = aboutAttribute.getNodeValue();
newlyAddedUris.add(value);
}
}
}
} catch(Exception e) {
log.error(e, e);
}
return newlyAddedUris;
}
private ArrayList<String> getRdfTypes(Node descriptionNode) {
ArrayList<String> rdfTypesList = new ArrayList<String>();
NodeList children = descriptionNode.getChildNodes();
int numChildren = children.getLength();
for(int i = 0; i < numChildren; i++) {
Node child = children.item(i);
String name = child.getNodeName();
if(name.equals("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")) {
NamedNodeMap attributes = child.getAttributes();
Node resourceAttribute = attributes.getNamedItem("http://www.w3.org/1999/02/22-rdf-syntax-ns#resource");
if(resourceAttribute != null) {
String value = resourceAttribute.getNodeValue();
rdfTypesList.add(value);
}
}
}
return rdfTypesList;
}
@ -439,14 +518,17 @@ public class TestFileController extends FreemarkerHttpServlet {
private Map<String, ScriptRunner> sessionIdToHarvestThread = new Hashtable<String, ScriptRunner>(); //Hashtable is threadsafe, HashMap is not private Map<String, ScriptRunner> sessionIdToHarvestThread = new Hashtable<String, ScriptRunner>(); //Hashtable is threadsafe, HashMap is not
private Map<String, ArrayList<String>> sessionIdToUnsentLogLines = new Hashtable<String, ArrayList<String>>(); //Hashtable is threadsafe, HashMap is not private Map<String, ArrayList<String>> sessionIdToUnsentLogLines = new Hashtable<String, ArrayList<String>>(); //Hashtable is threadsafe, HashMap is not
private Map<String, ArrayList<String>> sessionIdToNewlyAddedUris = new Hashtable<String, ArrayList<String>>();
private class ScriptRunner extends Thread { private class ScriptRunner extends Thread {
private final String sessionId; private final String sessionId;
private final String script; private final String script;
private final String additionsFilePath;
public ScriptRunner(String sessionId, String script) { public ScriptRunner(String sessionId, String script, String additionsFilePath) {
this.sessionId = sessionId; this.sessionId = sessionId;
this.script = script; this.script = script;
this.additionsFilePath = additionsFilePath;
} }
@Override @Override
@ -455,10 +537,10 @@ public class TestFileController extends FreemarkerHttpServlet {
ArrayList<String> unsentLogLines = sessionIdToUnsentLogLines.get(sessionId); ArrayList<String> unsentLogLines = sessionIdToUnsentLogLines.get(sessionId);
if(unsentLogLines == null) { if(unsentLogLines == null) {
unsentLogLines = new ArrayList<String>(); unsentLogLines = new ArrayList<String>();
sessionIdToUnsentLogLines.put(sessionId, unsentLogLines); sessionIdToUnsentLogLines.put(this.sessionId, unsentLogLines);
} }
File scriptFile = createScriptFile(script); File scriptFile = createScriptFile(this.script);
String command = "/bin/bash " + getHarvesterPath() + "scripts/temp/" + scriptFile.getName(); String command = "/bin/bash " + getHarvesterPath() + "scripts/temp/" + scriptFile.getName();
@ -488,6 +570,12 @@ public class TestFileController extends FreemarkerHttpServlet {
catch(InterruptedException e) { catch(InterruptedException e) {
throw new IOException(e.getMessage(), e); throw new IOException(e.getMessage(), e);
} }
File additionsFile = new File(this.additionsFilePath);
ArrayList<String> newlyAddedUris = extractNewlyAddedUris(additionsFile);
log.error("newly added URIs size: " + newlyAddedUris.size());
sessionIdToNewlyAddedUris.put(this.sessionId, newlyAddedUris);
log.debug("Harvester script exited with error code " + exitVal); log.debug("Harvester script exited with error code " + exitVal);
log.info("Harvester script execution complete"); log.info("Harvester script execution complete");
} catch (IOException e) { } catch (IOException e) {
@ -498,9 +586,7 @@ public class TestFileController extends FreemarkerHttpServlet {
} }
} }
} }
} }
} }
@ -542,6 +628,7 @@ class CsvHarvestJob implements FileHarvestJob {
public CsvHarvestJob(VitroRequest vreq, String templateFileName, String namespace) { public CsvHarvestJob(VitroRequest vreq, String templateFileName, String namespace) {
this.vreq = vreq; this.vreq = vreq;
this.templateFile = new File(getTemplateFileDirectory() + templateFileName); this.templateFile = new File(getTemplateFileDirectory() + templateFileName);
log.error(getTemplateFileDirectory() + templateFileName);
this.namespace = namespace; this.namespace = namespace;
} }
@ -607,13 +694,24 @@ class CsvHarvestJob implements FileHarvestJob {
* @return an error message if the two lines don't match, or null if they do * @return an error message if the two lines don't match, or null if they do
*/ */
private String validateCsvFirstLine(String[] templateFirstLine, String[] line) { private String validateCsvFirstLine(String[] templateFirstLine, String[] line) {
String errorMessage = "File header does not match specification"; String errorMessage = "File header does not match template";
if(line.length != templateFirstLine.length) if(line.length != templateFirstLine.length) {
return errorMessage; //return errorMessage + ": " + "file header columns = " + line.length + ", template columns = " + templateFirstLine.length;
String errorMsg = "";
errorMsg += "file header items: ";
for(int i = 0; i < line.length; i++) {
errorMsg += line[i] + ", ";
}
errorMsg += "template items: ";
for(int i = 0; i < templateFirstLine.length; i++) {
errorMsg += templateFirstLine[i] + ", ";
}
return errorMsg;
}
for(int i = 0; i < line.length; i++) for(int i = 0; i < line.length; i++)
{ {
if(!line[i].equals(templateFirstLine[i])) if(!line[i].equals(templateFirstLine[i]))
return errorMessage; return errorMessage + ": file header column " + (i + 1) + " = " + line[i] + ", template column " + (i + 1) + " = " + templateFirstLine[i];
} }
return null; return null;
} }
@ -676,6 +774,12 @@ class CsvHarvestJob implements FileHarvestJob {
} }
@Override
public String getAdditionsFilePath() {
return TestFileController.getHarvesterPath() + "harvested-data/csv/additions.rdf.xml";
}
} }