litvinovg/vivo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

NIHVIVO-3005 NIHVIVO-3007 Merged r3374 from maint-rel-1.3 branch.

Browse source
This commit is contained in:
cappadona 2011-07-22 15:51:24 +00:00
parent 48b4ded3af
commit eb567572bd
4 changed files with 337 additions and 217 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
example.deploy.properties
View file

@ -162,7 +162,7 @@ visualization.temporal = enabled
# Absolute path on the server of the Harvester root directory. # Absolute path on the server of the Harvester root directory.
# You must include the final slash. # You must include the final slash.
# #
harvester.location = /usr/local/vivo/harvester/ # harvester.location = /usr/local/vivo/harvester/
# #
# Default type(s) for Google Refine Reconciliation Service # Default type(s) for Google Refine Reconciliation Service

10
productMods/css/harvester/fileharvest.css
View file

@ -32,4 +32,14 @@ a.help {
} }
#harvestButton { #harvestButton {
margin-right: 10px; margin-right: 10px;
}
#fileHarvestErrorHelp ol {
margin: 0 0 1.3em 2em;
list-style-type: decimal;
font-size: .9em;
}
#fileHarvestErrorHelp em {
font-family:"Courier New",Courier,mono;
font-style: normal;
color:#666;
} }

431
productMods/templates/freemarker/body/harvester/fileharvest.ftl
View file

@ -1,5 +1,7 @@
<#-- $This file is distributed under the terms of the license in /doc/license.txt$ --> <#-- $This file is distributed under the terms of the license in /doc/license.txt$ -->
<#-- <@dump var="harvesterLocation" /> -->
<#if !(user.loggedIn && user.hasSiteAdminAccess)> <#if !(user.loggedIn && user.hasSiteAdminAccess)>
<p>You must be an administrator to use this tool.</p> <p>You must be an administrator to use this tool.</p>
@ -13,224 +15,265 @@
<p>The end user should not see this error under normal circumstances, so this is probably a bug and should be reported.</p> <p>The end user should not see this error under normal circumstances, so this is probably a bug and should be reported.</p>
<#else> <#else>
<script type="text/javascript"> <h2><a class="ingestMenu" href="${urls.base}/ingest" title="Return to the Data Ingest Tools menu">Ingest Menu</a> > ${jobSpecificHeader}</h2>
var harvestProgressResponse; <#-- check to ensure harvester.location is set in deploy.properties -->
<#if harvesterLocation?has_content>
function doHarvest()
{ <script type="text/javascript">
document.getElementById("harvestButton").disabled = true;
document.getElementById("harvestButtonHelpText").innerHTML = "Please wait while your data is harvested."; var harvestProgressResponse;
var request = createRequest(); function doHarvest() {
request.onreadystatechange=function() { document.getElementById("harvestButton").disabled = true;
if(request.readyState == 4 && request.status == 200) { document.getElementById("harvestButtonHelpText").innerHTML = "Please wait while your data is harvested.";
harvestProgressResponse = request.responseText;
var json = eval("(" + harvestProgressResponse + ")");
var scriptTextArea = document.getElementById("scriptTextArea");
scriptTextArea.innerHTML = json.scriptText;
window.setTimeout(continueHarvest, 1000);
}
}
request.open("POST", "${postTo}", true);
request.setRequestHeader("content-type","application/x-www-form-urlencoded");
//request.send("${paramMode}=${modeHarvest}&${paramJob}=${job}");
request.send("${paramMode}=${modeHarvest}");
}
function continueHarvest()
{
var response = harvestProgressResponse;
var json = eval("(" + response + ")");
var logAppend = json.progressSinceLastCheck;
var progressTextArea = document.getElementById("progressTextArea");
progressTextArea.innerHTML = progressTextArea.innerHTML + logAppend;
progressTextArea.scrollTop = progressTextArea.scrollHeight;
if(!json.finished) {
var request = createRequest(); var request = createRequest();
request.onreadystatechange=function() { request.onreadystatechange=function() {
if(request.readyState == 4 && request.status == 200) { if(request.readyState == 4 && request.status == 200) {
harvestProgressResponse = request.responseText; harvestProgressResponse = request.responseText;
var json = eval("(" + harvestProgressResponse + ")");
if(checkForFatalError(json))
return;
var scriptTextArea = document.getElementById("scriptTextArea");
scriptTextArea.innerHTML = json.scriptText;
window.setTimeout(continueHarvest, 1000); window.setTimeout(continueHarvest, 1000);
} }
} }
request.open("POST", "${postTo}", true); request.open("POST", "${postTo}", true);
request.setRequestHeader("Content-type","application/x-www-form-urlencoded"); request.setRequestHeader("content-type","application/x-www-form-urlencoded");
//request.send("${paramMode}=${modeCheckStatus}&${paramJob}=${job}"); //request.send("${paramMode}=${modeHarvest}&${paramJob}=${job}");
request.send("${paramMode}=${modeCheckStatus}"); request.send("${paramMode}=${modeHarvest}");
} else { }
// var linkHeader = document.getElementById("linkHeader"); function continueHarvest() {
// linkHeader.style.display = "inline"; var response = harvestProgressResponse;
$('#linkHeader').removeClass('hidden'); var json = eval("(" + response + ")");
if(checkForFatalError(json)) {
var importedItems = document.getElementById("importedItems") return;
}
if(json.newlyAddedUrls.length > 0) {
for(var i = 0; i < json.newlyAddedUrls.length; i++) { var logAppend = json.progressSinceLastCheck;
var progressTextArea = document.getElementById("progressTextArea");
progressTextArea.innerHTML = progressTextArea.innerHTML + logAppend;
progressTextArea.scrollTop = progressTextArea.scrollHeight;
if(!json.finished) {
var request = createRequest();
request.onreadystatechange=function() {
if(request.readyState == 4 && request.status == 200) {
harvestProgressResponse = request.responseText;
window.setTimeout(continueHarvest, 1000);
}
}
request.open("POST", "${postTo}", true);
request.setRequestHeader("Content-type","application/x-www-form-urlencoded");
//request.send("${paramMode}=${modeCheckStatus}&${paramJob}=${job}");
request.send("${paramMode}=${modeCheckStatus}");
} else {
// var linkHeader = document.getElementById("linkHeader");
// linkHeader.style.display = "inline";
$('#linkHeader').removeClass('hidden');
var importedItems = document.getElementById("importedItems")
if(json.newlyAddedUrls.length > 0) {
for(var i = 0; i < json.newlyAddedUrls.length; i++) {
var newLi = document.createElement("li");
newLi.innerHTML = "<a href=\"" + json.newlyAddedUrls[i] + "\" target=\"_blank\">" + json.newlyAddedUris[i] + "</a>";
importedItems.appendChild(newLi);
}
} else {
var newLi = document.createElement("li"); var newLi = document.createElement("li");
newLi.innerHTML = "<a href=\"" + json.newlyAddedUrls[i] + "\" target=\"_blank\">" + json.newlyAddedUris[i] + "</a>"; newLi.innerHTML = "${jobSpecificNoNewDataMessage}";
importedItems.appendChild(newLi); importedItems.appendChild(newLi);
} }
} else {
document.getElementById("harvestButtonHelpText").innerHTML = "Harvest complete. For another, please refresh the page.";
var newLi = document.createElement("li");
newLi.innerHTML = "${jobSpecificNoNewDataMessage}";
importedItems.appendChild(newLi);
} }
document.getElementById("harvestButtonHelpText").innerHTML = "Harvest complete. For another, please refresh the page.";
} }
}
function createRequest() {
var request;
if (window.XMLHttpRequest) {
function createRequest() { request = new XMLHttpRequest();
var request; } else {
if (window.XMLHttpRequest) { request = new ActiveXObject("Microsoft.XMLHTTP");
request = new XMLHttpRequest(); }
} else { return request;
request = new ActiveXObject("Microsoft.XMLHTTP");
} }
return request;
} function fileResponse() {
var response = frames["uploadTarget"].document.getElementsByTagName("body")[0].innerHTML;
function fileResponse() var json = eval("(" + response + ")");
{ if(checkForFatalError(json)) {
var response = frames["uploadTarget"].document.getElementsByTagName("body")[0].innerHTML; return;
var json = eval("(" + response + ")"); }
var fileListing = document.getElementById("fileListing") var fileListing = document.getElementById("fileListing")
var newLi = document.createElement("li"); var newLi = document.createElement("li");
if(json.success) if (json.success) {
newLi.innerHTML = json.fileName + " <span style=\"color:green\">" + json.errorMessage + "</span>"; newLi.innerHTML = json.fileName + " <span style=\"color:green\">" + json.errorMessage + "</span>";
else } else {
newLi.innerHTML = json.fileName + " <span style=\"color:red\">upload failed: " + json.errorMessage + "</span>"; newLi.innerHTML = json.fileName + " <span style=\"color:red\">upload failed: " + json.errorMessage + "</span>";
fileListing.appendChild(newLi); }
document.getElementById("${paramFirstUpload}").value = "false"; fileListing.appendChild(newLi);
//document.getElementById("responseArea").innerHTML = response; document.getElementById("${paramFirstUpload}").value = "false";
}
//document.getElementById("responseArea").innerHTML = response;
function init()
{
document.getElementById("harvestButton").disabled = false;
document.getElementById("${paramFirstUpload}").value = "true";
document.getElementById("fileUploadForm").onsubmit = function()
{
document.getElementById("fileUploadForm").target = "uploadTarget";
document.getElementById("uploadTarget").onload = fileResponse;
} }
document.getElementById("downloadTemplateForm").onsubmit = function()
{ function init() {
document.getElementById("downloadTemplateForm").target = "uploadTarget"; document.getElementById("harvestButton").disabled = false;
document.getElementById("${paramFirstUpload}").value = "true";
document.getElementById("fileUploadForm").onsubmit = function()
{
document.getElementById("fileUploadForm").target = "uploadTarget";
document.getElementById("uploadTarget").onload = fileResponse;
}
document.getElementById("downloadTemplateForm").onsubmit = function()
{
document.getElementById("downloadTemplateForm").target = "uploadTarget";
}
} }
}
function checkForFatalError(json) {
window.onload = init; if(json.fatalError) {
handleFatalError();
$(document).ready(function() { return true;
$('a.help').click(function() { } else {
$('#csvHelp-collapsible').toggleClass('hidden'); return false;
return false; }
}
function handleFatalError() {
$('#fileHarvestErrorHelp').removeClass('hidden');
$('#fileHarvestForm').addClass('hidden');
}
window.onload = init;
$(document).ready(function() {
$('a.help').click(function() {
$('#csvHelp-collapsible').toggleClass('hidden');
return false;
});
$('#harvestButton').click(function() {
doHarvest();
return false;
});
}); });
$('#harvestButton').click(function() { </script>
doHarvest();
return false; <div id="fileHarvestErrorHelp" class="hidden">
}); <p>An error has occurred and the file harvest cannot continue.</p>
});
<p>This is most likely due to an improper Harvester configuration. Please ensure the following:</p>
</script>
<ol>
<h2><a class="ingestMenu" href="${urls.base}/ingest">Ingest Menu</a> > ${jobSpecificHeader}</h2> <li>VIVO Harvester is installed.</li>
<li>The <em>harvester.location</em> property in deploy.properties is pointed to the Harvester installation directory.</li>
<div id="step1" class="testfile-step"> <li>In VIVO Harvester, the web server user (typically tomcat6) has read and write access to the <em>vivo/</em> directory and all of its children.</li>
<h3 class="testfile-step-header">Step 1</h3> <li>In VIVO Harvester, the <em>logs/</em> directory exists and the web server user has read and write access to it.</li>
<div id="step1-inner" class="testfile-step-body"> <li>In VIVO Harvester, the file <em>vivo/config/vivo.xml</em> is properly configured with your database information and namespace.</li>
<h4 class="testfile-step-subheader">Download template</h4> </ol>
<form id="downloadTemplateForm" method="post" action=${postTo}>
<input type="hidden" id="${paramMode}" name="${paramMode}" value="${modeDownloadTemplate}" />
<p><input id="submit" type="submit" name="submit" value="Download" /> ${jobSpecificDownloadHelp}</p>
</form>
</div> </div>
<div class="clearBothDiv" />
</div> <div id="fileHarvestForm">
<div id="step1" class="testfile-step">
<div id="step2" class="testfile-step"> <h3 class="testfile-step-header">Step 1</h3>
<h3 class="testfile-step-header">Step 2</h3> <div id="step1-inner" class="testfile-step-body">
<div id="step2-inner" class="testfile-step-body"> <h4 class="testfile-step-subheader">Download template</h4>
<h4 class="testfile-step-subheader">Fill in data <a class="help" href="#">Help</a></h4> <form id="downloadTemplateForm" method="post" action=${postTo}>
<div id="csvHelp-collapsible" class="hidden"> <input type="hidden" id="${paramMode}" name="${paramMode}" value="${modeDownloadTemplate}" />
<div id="csvHelp-indented"> <p><input id="submit" type="submit" name="submit" value="Download" /> ${jobSpecificDownloadHelp}</p>
${jobSpecificFillInHelp} </form>
</div> </div>
</div> </div>
<p>Fill in the template with your data. You may fill in multiple templates if you wish to harvest multiple files at once.</p>
<div id="csvHelp"> <div id="step2" class="testfile-step">
<h3 class="testfile-step-header">Step 2</h3>
<div id="step2-inner" class="testfile-step-body">
<h4 class="testfile-step-subheader">Fill in data <a class="help" href="#">Help</a></h4>
<div id="csvHelp-collapsible" class="hidden">
<div id="csvHelp-indented">
${jobSpecificFillInHelp}
</div>
</div>
<p>Fill in the template with your data. You may fill in multiple templates if you wish to harvest multiple files at once.</p>
<div id="csvHelp">
</div>
</div>
</div>
<div id="step3" class="testfile-step">
<h3 class="testfile-step-header">Step 3</h3>
<div id="step3-inner" class="testfile-step-body">
<h4 class="testfile-step-subheader">Upload file(s)</h4>
<p>Upload your filled-in template(s).</p>
<form id="fileUploadForm" method="post" enctype="multipart/form-data" action=${postTo}>
<input type="hidden" id="${paramFirstUpload}" name="${paramFirstUpload}" value="true" />
<!--<input type="hidden" id="${paramJob}" name="${paramJob}" value="${job}" /> -->
<input type="file" name="${paramUploadedFile}" />
<input type="submit" name="submit" value="Upload" />
<iframe id="uploadTarget" name="uploadTarget" src=""></iframe>
</form>
<h5>Uploaded files</h5>
<ul id="fileListing">
</ul>
</div>
</div>
<div id="step4" class="testfile-step">
<h3 class="testfile-step-header">Step 4</h3>
<div id="step4-inner" class="testfile-step-body">
<h4 class="testfile-step-subheader">Harvest</h4>
<p><input type="button" name="harvestButton" id="harvestButton" class="green button" value="Harvest" /><span id="harvestButtonHelpText">Click the button to harvest your file(s).</span></p>
</div>
</div>
<div id="step5" class="testfile-step">
<h3 class="testfile-step-header">Step 5</h3>
<div id="step5-inner" class="testfile-step-body">
<h4 class="testfile-step-subheader">View results</h4>
<div id="script">
<h5>Script being executed</h5>
<textarea cols="100" rows="20" readonly="readonly" id="scriptTextArea"></textarea>
</div>
<div id="progress">
<h5>Progress</h5>
<textarea cols="100" rows="20" readonly="readonly" id="progressTextArea"></textarea>
</div>
<div id="summary">
<h5 id="linkHeader" class="hidden">${jobSpecificLinkHeader}</h5>
<ul id="importedItems">
</ul>
</div>
</div>
</div> </div>
</div> </div>
<div class="clearBothDiv" />
</div> <#else>
<div id="step3" class="testfile-step"> <div id="fileHarvestErrorHelp">
<h3 class="testfile-step-header">Step 3</h3> <p>The <em>harvester.location</em> property in deploy.properties is undefined.</p>
<div id="step3-inner" class="testfile-step-body">
<h4 class="testfile-step-subheader">Upload file(s)</h4> <p>In order to use this feature, please define a value for this property that points to the Harvester installation directory before redeploying and restarting the application.</p>
<p>Upload your filled-in template(s).</p>
<form id="fileUploadForm" method="post" enctype="multipart/form-data" action=${postTo}>
<input type="hidden" id="${paramFirstUpload}" name="${paramFirstUpload}" value="true" />
<!--<input type="hidden" id="${paramJob}" name="${paramJob}" value="${job}" /> -->
<input type="file" name="${paramUploadedFile}" />
<input type="submit" name="submit" value="Upload" />
<iframe id="uploadTarget" name="uploadTarget" src=""></iframe>
</form>
<h5>Uploaded files</h5>
<ul id="fileListing">
</ul>
</div> </div>
<div class="clearBothDiv" />
</div> <#-- if harvester.location is defined -->
</#if>
<div id="step4" class="testfile-step">
<h3 class="testfile-step-header">Step 4</h3>
<div id="step4-inner" class="testfile-step-body">
<h4 class="testfile-step-subheader">Harvest</h4>
<p><input type="button" name="harvestButton" id="harvestButton" class="green button" value="Harvest" /><span id="harvestButtonHelpText">Click the button to harvest your file(s).</span></p>
</div>
<div class="clearBothDiv" />
</div>
<div id="step5" class="testfile-step">
<h3 class="testfile-step-header">Step 5</h3>
<div id="step5-inner" class="testfile-step-body">
<h4 class="testfile-step-subheader">View results</h4>
<div id="script">
<h5>Script being executed</h5>
<textarea cols="100" rows="20" readonly="readonly" id="scriptTextArea"></textarea>
</div>
<div id="progress">
<h5>Progress</h5>
<textarea cols="100" rows="20" readonly="readonly" id="progressTextArea"></textarea>
</div>
<div id="summary">
<h5 id="linkHeader" class="hidden">${jobSpecificLinkHeader}</h5>
<ul id="importedItems">
</ul>
</div>
</div>
<div class="clearBothDiv" />
</div>
${stylesheets.add('<link rel="stylesheet" href="${urls.base}/css/harvester/fileharvest.css" />')} ${stylesheets.add('<link rel="stylesheet" href="${urls.base}/css/harvester/fileharvest.css" />')}

111
src/edu/cornell/mannlib/vitro/webapp/controller/harvester/FileHarvestController.java
View file

@ -52,6 +52,8 @@ public class FileHarvestController extends FreemarkerHttpServlet {
private static final Log log = LogFactory.getLog(FileHarvestController.class); private static final Log log = LogFactory.getLog(FileHarvestController.class);
private static final String TEMPLATE_DEFAULT = "fileharvest.ftl"; private static final String TEMPLATE_DEFAULT = "fileharvest.ftl";
private static final String NORMAL_TERMINATION_LAST_OUTPUT = "File Harvest completed successfully";
private static final String PARAMETER_FIRST_UPLOAD = "firstUpload"; private static final String PARAMETER_FIRST_UPLOAD = "firstUpload";
private static final String PARAMETER_UPLOADED_FILE = "uploadedFile"; private static final String PARAMETER_UPLOADED_FILE = "uploadedFile";
private static final String PARAMETER_MODE = "mode"; private static final String PARAMETER_MODE = "mode";
@ -140,6 +142,7 @@ public class FileHarvestController extends FreemarkerHttpServlet {
FileHarvestJob jobObject = getJob(vreq, job); FileHarvestJob jobObject = getJob(vreq, job);
Map<String, Object> body = new HashMap<String, Object>(); Map<String, Object> body = new HashMap<String, Object>();
String harvesterPath = getHarvesterPath(vreq);
//body.put("uploadPostback", "false"); //body.put("uploadPostback", "false");
body.put("paramFirstUpload", PARAMETER_FIRST_UPLOAD); body.put("paramFirstUpload", PARAMETER_FIRST_UPLOAD);
body.put("paramUploadedFile", PARAMETER_UPLOADED_FILE); body.put("paramUploadedFile", PARAMETER_UPLOADED_FILE);
@ -150,6 +153,7 @@ public class FileHarvestController extends FreemarkerHttpServlet {
body.put("modeDownloadTemplate", MODE_DOWNLOAD_TEMPLATE); body.put("modeDownloadTemplate", MODE_DOWNLOAD_TEMPLATE);
body.put("job", job); body.put("job", job);
body.put("jobKnown", jobKnown); body.put("jobKnown", jobKnown);
body.put("harvesterLocation", harvesterPath);
body.put("postTo", POST_TO + "?" + PARAMETER_JOB + "=" + job); body.put("postTo", POST_TO + "?" + PARAMETER_JOB + "=" + job);
body.put("jobSpecificHeader", (jobObject != null) ? jobObject.getPageHeader() : ""); body.put("jobSpecificHeader", (jobObject != null) ? jobObject.getPageHeader() : "");
body.put("jobSpecificLinkHeader", (jobObject != null) ? jobObject.getLinkHeader() : ""); body.put("jobSpecificLinkHeader", (jobObject != null) ? jobObject.getLinkHeader() : "");
@ -284,7 +288,7 @@ public class FileHarvestController extends FreemarkerHttpServlet {
private void doFileUploadPost(HttpServletRequest request, HttpServletResponse response) private void doFileUploadPost(HttpServletRequest request, HttpServletResponse response)
throws IOException, ServletException { throws IOException, ServletException {
JSONObject json = new JSONObject(); JSONObject json = generateJson(false);
try { try {
VitroRequest vreq = new VitroRequest(request); VitroRequest vreq = new VitroRequest(request);
@ -392,7 +396,7 @@ public class FileHarvestController extends FreemarkerHttpServlet {
} }
} catch(Exception e) { } catch(Exception e) {
log.error(e, e); log.error(e, e);
return; json = generateJson(true);
} }
//write the prepared response //write the prepared response
@ -407,6 +411,7 @@ public class FileHarvestController extends FreemarkerHttpServlet {
*/ */
private void doHarvestPost(HttpServletRequest request, HttpServletResponse response) { private void doHarvestPost(HttpServletRequest request, HttpServletResponse response) {
JSONObject json;
try { try {
VitroRequest vreq = new VitroRequest(request); VitroRequest vreq = new VitroRequest(request);
FileHarvestJob job = getJob(vreq, vreq.getParameter(PARAMETER_JOB)); FileHarvestJob job = getJob(vreq, vreq.getParameter(PARAMETER_JOB));
@ -416,18 +421,23 @@ public class FileHarvestController extends FreemarkerHttpServlet {
String script = job.getScript(); String script = job.getScript();
String additionsFilePath = job.getAdditionsFilePath(); String additionsFilePath = job.getAdditionsFilePath();
String scriptFileLocation = getScriptFileLocation(vreq); String scriptFileLocation = getScriptFileLocation(vreq);
runScript(getSessionId(request), script, additionsFilePath, scriptFileLocation); runScript(getSessionId(request), script, additionsFilePath, scriptFileLocation, job);
JSONObject json = new JSONObject(); json = generateJson(false);
json.put("progressSinceLastCheck", ""); json.put("progressSinceLastCheck", "");
json.put("scriptText", script); json.put("scriptText", script);
json.put("finished", false); json.put("finished", false);
response.getWriter().write(json.toString());
} catch(Exception e) { } catch(Exception e) {
json = generateJson(true);
log.error(e, e); log.error(e, e);
} }
try {
response.getWriter().write(json.toString());
} catch(IOException e) {
log.error(e, e);
}
} }
/** /**
@ -439,6 +449,7 @@ public class FileHarvestController extends FreemarkerHttpServlet {
*/ */
private void doCheckHarvestStatusPost(HttpServletRequest request, HttpServletResponse response) { private void doCheckHarvestStatusPost(HttpServletRequest request, HttpServletResponse response) {
JSONObject json;
try { try {
String newline = "\n"; String newline = "\n";
@ -463,6 +474,7 @@ public class FileHarvestController extends FreemarkerHttpServlet {
} }
boolean finished = sessionInfo.isFinished(); boolean finished = sessionInfo.isFinished();
boolean abnormalTermination = false;
VitroRequest vreq = new VitroRequest(request); VitroRequest vreq = new VitroRequest(request);
ArrayList<String> newlyAddedUrls = new ArrayList<String>(); ArrayList<String> newlyAddedUrls = new ArrayList<String>();
@ -478,21 +490,35 @@ public class FileHarvestController extends FreemarkerHttpServlet {
//remove all entries in "sessionIdTo..." mappings for this session ID //remove all entries in "sessionIdTo..." mappings for this session ID
clearSessionInfo(sessionId); clearSessionInfo(sessionId);
if(sessionInfo.getAbnormalTermination())
abnormalTermination = true;
} }
JSONObject json = new JSONObject(); if(!abnormalTermination) {
json.put("progressSinceLastCheck", progressSinceLastCheck); json = generateJson(false);
json.put("finished", finished); json.put("progressSinceLastCheck", progressSinceLastCheck);
json.put("newlyAddedUris", newlyAddedUris); json.put("finished", finished);
json.put("newlyAddedUrls", newlyAddedUrls); json.put("newlyAddedUris", newlyAddedUris);
json.put("newlyAddedUrls", newlyAddedUrls);
response.getWriter().write(json.toString()); } else {
json = generateJson(true);
log.error("File harvest terminated abnormally.");
}
} else { //if we have not started a harvest thread, the browser should not have made this request to begin with. Bad browser, very bad browser. } else { //if we have not started a harvest thread, the browser should not have made this request to begin with. Bad browser, very bad browser.
json = generateJson(true);
log.error("Attempt to check status of a harvest that was never started! (Session ID " + sessionId + ")"); log.error("Attempt to check status of a harvest that was never started! (Session ID " + sessionId + ")");
} }
} catch(Exception e) { } catch(Exception e) {
json = generateJson(true);
log.error(e, e); log.error(e, e);
} }
try {
response.getWriter().write(json.toString());
} catch(IOException e) {
log.error(e, e);
}
} }
private void doDownloadTemplatePost(HttpServletRequest request, HttpServletResponse response) { private void doDownloadTemplatePost(HttpServletRequest request, HttpServletResponse response) {
@ -549,10 +575,10 @@ public class FileHarvestController extends FreemarkerHttpServlet {
} }
private void runScript(String sessionId, String script, String additionsFilePath, String scriptFileLocation) { private void runScript(String sessionId, String script, String additionsFilePath, String scriptFileLocation, FileHarvestJob job) {
clearSessionInfo(sessionId); clearSessionInfo(sessionId);
ScriptRunner runner = new ScriptRunner(sessionId, script, additionsFilePath, scriptFileLocation); ScriptRunner runner = new ScriptRunner(sessionId, script, additionsFilePath, scriptFileLocation, job);
SessionInfo info = new SessionInfo(sessionId, runner); SessionInfo info = new SessionInfo(sessionId, runner);
sessionIdToSessionInfo.put(sessionId, info); sessionIdToSessionInfo.put(sessionId, info);
runner.start(); runner.start();
@ -604,7 +630,7 @@ public class FileHarvestController extends FreemarkerHttpServlet {
* @param additionsFile the file containing the newly-added RDF/XML * @param additionsFile the file containing the newly-added RDF/XML
* @param newlyAddedUris a list in which to place the newly added URIs * @param newlyAddedUris a list in which to place the newly added URIs
*/ */
private void extractNewlyAddedUris(File additionsFile, List<String> newlyAddedUris) { private void extractNewlyAddedUris(File additionsFile, List<String> newlyAddedUris, FileHarvestJob job) {
try { try {
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
@ -618,7 +644,16 @@ public class FileHarvestController extends FreemarkerHttpServlet {
Node node = descriptionNodes.item(i); Node node = descriptionNodes.item(i);
ArrayList<String> types = getRdfTypes(node); ArrayList<String> types = getRdfTypes(node);
if(types.contains("http://vivoweb.org/ontology/core#Grant")) { //todo: generalize
boolean match = false;
String[] validRdfTypesForJob = job.getRdfTypesForLinks();
for(String rdfType : validRdfTypesForJob) {
if(types.contains(rdfType))
match = true;
break;
}
if(match) {
NamedNodeMap attributes = node.getAttributes(); NamedNodeMap attributes = node.getAttributes();
Node aboutAttribute = attributes.getNamedItemNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "about"); Node aboutAttribute = attributes.getNamedItemNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "about");
@ -705,6 +740,21 @@ public class FileHarvestController extends FreemarkerHttpServlet {
} }
/**
* Create a new JSON object
* @param fatalError whether the fatal error flag should be set on this object
* @return the new JSON object
*/
private JSONObject generateJson(boolean fatalError) {
JSONObject json = null;
try {
json = new JSONObject();
json.put("fatalError", fatalError);
} catch(JSONException e) {
log.error(e.getMessage(), e);
}
return json;
}
@ -743,6 +793,11 @@ public class FileHarvestController extends FreemarkerHttpServlet {
*/ */
private boolean finished = false; private boolean finished = false;
/**
* Flag indicating that the thread finished abnormally.
*/
private boolean abnormalTermination = false;
/** /**
* Newly added entries to VIVO, for this user session. * Newly added entries to VIVO, for this user session.
@ -757,6 +812,13 @@ public class FileHarvestController extends FreemarkerHttpServlet {
this.harvestThread = harvestThread; this.harvestThread = harvestThread;
} }
public void setAbnormalTermination() {
abnormalTermination = true;
}
public boolean getAbnormalTermination() {
return abnormalTermination;
}
public void finish() { public void finish() {
finished = true; finished = true;
} }
@ -786,14 +848,16 @@ public class FileHarvestController extends FreemarkerHttpServlet {
private final String script; private final String script;
private final String additionsFilePath; private final String additionsFilePath;
private final String scriptFileLocation; private final String scriptFileLocation;
private final FileHarvestJob job;
private volatile boolean abort = false; private volatile boolean abort = false;
public ScriptRunner(String sessionId, String script, String additionsFilePath, String scriptFileLocation) { public ScriptRunner(String sessionId, String script, String additionsFilePath, String scriptFileLocation, FileHarvestJob job) {
this.sessionId = sessionId; this.sessionId = sessionId;
this.script = script; this.script = script;
this.additionsFilePath = additionsFilePath; this.additionsFilePath = additionsFilePath;
this.scriptFileLocation = scriptFileLocation; this.scriptFileLocation = scriptFileLocation;
this.job = job;
} }
public void abortRun() { public void abortRun() {
@ -804,6 +868,7 @@ public class FileHarvestController extends FreemarkerHttpServlet {
@Override @Override
public void run() { public void run() {
SessionInfo sessionInfo = sessionIdToSessionInfo.get(sessionId); SessionInfo sessionInfo = sessionIdToSessionInfo.get(sessionId);
boolean normalTerminationLineFound = false;
if(sessionInfo != null) { if(sessionInfo != null) {
try { try {
ArrayList<String> unsentLogLines = sessionInfo.unsentLogLines; ArrayList<String> unsentLogLines = sessionInfo.unsentLogLines;
@ -820,6 +885,8 @@ public class FileHarvestController extends FreemarkerHttpServlet {
BufferedReader processOutputReader = new BufferedReader(new InputStreamReader(pr.getInputStream())); BufferedReader processOutputReader = new BufferedReader(new InputStreamReader(pr.getInputStream()));
for(String line = processOutputReader.readLine(); line != null; line = processOutputReader.readLine()) { for(String line = processOutputReader.readLine(); line != null; line = processOutputReader.readLine()) {
normalTerminationLineFound = line.endsWith(NORMAL_TERMINATION_LAST_OUTPUT); //set every read to ensure it's the last line
//don't add stuff to this list if the main thread is running a "transaction" of copying out the data to send to client and then clearing the list //don't add stuff to this list if the main thread is running a "transaction" of copying out the data to send to client and then clearing the list
synchronized(unsentLogLines) { synchronized(unsentLogLines) {
unsentLogLines.add(line); unsentLogLines.add(line);
@ -857,7 +924,7 @@ public class FileHarvestController extends FreemarkerHttpServlet {
File additionsFile = new File(this.additionsFilePath); File additionsFile = new File(this.additionsFilePath);
if(additionsFile.exists()) if(additionsFile.exists())
extractNewlyAddedUris(additionsFile, sessionInfo.newlyAddedUris); extractNewlyAddedUris(additionsFile, sessionInfo.newlyAddedUris, this.job);
else else
log.error("Additions file not found: " + this.additionsFilePath); log.error("Additions file not found: " + this.additionsFilePath);
} }
@ -867,10 +934,10 @@ public class FileHarvestController extends FreemarkerHttpServlet {
log.error(e, e); log.error(e, e);
} finally { } finally {
sessionInfo.finish(); sessionInfo.finish();
if(!normalTerminationLineFound)
sessionInfo.setAbnormalTermination();
} }
} }
} }
} }
} }