litvinovg/vivo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

testfile.ftl, CsvFileHarvestJob.java, FileHarvestJob.java, TestFileController.java: continued construction

Browse source
This commit is contained in:
mbarbier 2011-06-16 20:53:49 +00:00
parent 3dcd3e32ba
commit 6ee5771c01
4 changed files with 117 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
productMods/templates/freemarker/body/harvester/testfile.ftl
View file

@ -24,8 +24,8 @@
}
request.open("POST", "${postTo}", true);
request.setRequestHeader("content-type","application/x-www-form-urlencoded");
//request.send("${paramIsHarvestClick}=true&${paramJob}=${job}");
request.send("${paramIsHarvestClick}=true");
//request.send("${paramMode}=${modeHarvest}&${paramJob}=${job}");
request.send("${paramMode}=${modeHarvest}");
}
@ -49,8 +49,8 @@
}
request.open("POST", "${postTo}", true);
request.setRequestHeader("Content-type","application/x-www-form-urlencoded");
//request.send("${paramIsHarvestClick}=false&${paramJob}=${job}");
request.send("${paramIsHarvestClick}=false");
//request.send("${paramMode}=${modeCheckStatus}&${paramJob}=${job}");
request.send("${paramMode}=${modeCheckStatus}");
} else {
var importedGrants = document.getElementById("importedGrants")
@ -124,6 +124,10 @@
document.getElementById("fileUploadForm").target = "uploadTarget";
document.getElementById("uploadTarget").onload = fileResponse;
}
document.getElementById("downloadTemplateForm").onsubmit = function()
{
document.getElementById("downloadTemplateForm").target = "uploadTarget";
}
}
window.onload = init;
</script>
@ -150,12 +154,15 @@
}
</style>
<h2>${jobSpecificHeader}</h2>
<div id="step1" class="testfile-step">
<h3 class="testfile-step-header">Step 1</h3>
<div id="step1-inner" class="testfile-step-body">
<h4 class="testfile-step-subheader">Download template</h4>
<p><input type="button" value="Download" style="margin-right:10px" />We are providing a helpful template file for you to download.</p>
<form id="downloadTemplateForm" method="post" action=${postTo}>
<input type="hidden" id="${paramMode}" name="${paramMode}" value="${modeDownloadTemplate}" />
<p><input type="submit" name="submit" value="Download" style="margin-right:10px" />We are providing a helpful template file for you to download.</p>
</form>
</div>
<div class="clearBothDiv" />
</div>

51
src/edu/cornell/mannlib/vitro/webapp/controller/harvester/CsvFileHarvestJob.java
View file

@ -6,6 +6,7 @@ import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.logging.Log;
@ -46,16 +47,22 @@ class CsvFileHarvestJob implements FileHarvestJob {
*/
private final String namespace;
/**
* A name for the type of data being imported. For example "Grant" or "Person".
*/
private final String friendlyName;
/**
* Constructor.
* @param templateFileName just the name of the template file. The directory is assumed to be standard.
*/
public CsvFileHarvestJob(VitroRequest vreq, String templateFileName, String scriptFileName, String namespace) {
public CsvFileHarvestJob(VitroRequest vreq, String templateFileName, String scriptFileName, String namespace, String friendlyName) {
this.vreq = vreq;
this.templateFile = new File(getTemplateFileDirectory() + templateFileName);
this.scriptFile = new File(getScriptFileDirectory() + scriptFileName);
log.error(getTemplateFileDirectory() + templateFileName);
this.namespace = namespace;
this.friendlyName = friendlyName;
}
/**
@ -79,6 +86,27 @@ class CsvFileHarvestJob implements FileHarvestJob {
}
private boolean[] getLinesEndingInComma(File file) throws IOException {
ArrayList<Boolean> linesEndingInCommaList = new ArrayList<Boolean>();
BufferedReader reader = new BufferedReader(new FileReader(file));
for(String line = reader.readLine(); line != null; line = reader.readLine()) {
boolean lineEndsInComma = line.endsWith(",");
linesEndingInCommaList.add(lineEndsInComma);
}
reader.close();
boolean[] linesEndingInComma = new boolean[linesEndingInCommaList.size()];
for(int i = 0; i < linesEndingInComma.length; i++) {
linesEndingInComma[i] = linesEndingInCommaList.get(i);
}
return linesEndingInComma;
}
@Override
@SuppressWarnings("rawtypes")
public String validateUpload(File file) {
@ -88,7 +116,9 @@ class CsvFileHarvestJob implements FileHarvestJob {
List templateCsv = reader.parse(this.templateFile);
String[] templateFirstLine = (String[])templateCsv.get(0);
//if a line ends in a comma (absolutely a comma, no whitespace), SimpleReader will not consider the part after the comma to be a blank section.
List csv = reader.parse(file);
boolean[] linesEndingInComma = getLinesEndingInComma(file);
int length = csv.size();
@ -97,19 +127,16 @@ class CsvFileHarvestJob implements FileHarvestJob {
for(int i = 0; i < length; i++) {
String[] line = (String[])csv.get(i);
boolean endsInComma = linesEndingInComma[i];
if(i == 0) {
String errorMessage = validateCsvFirstLine(templateFirstLine, line);
if(errorMessage != null)
return errorMessage;
}
else if(line.length != 0) {
if(line.length != templateFirstLine.length) {
String retval = "Mismatch in number of entries in row " + i + ": expected , " + templateFirstLine.length + ", found " + line.length + " ";
for(int j = 0; j < line.length; j++) {
retval += "\"" + line[j] + "\", ";
}
//return retval;
return "Mismatch in number of entries in row " + i + ": expected , " + templateFirstLine.length + ", found " + line.length;
int actualLineLength = line.length + (endsInComma ? 1 : 0);
if(actualLineLength != templateFirstLine.length) {
return "Mismatch in number of entries in row " + i + ": expected " + templateFirstLine.length + ", found " + actualLineLength;
}
}
}
@ -209,7 +236,15 @@ class CsvFileHarvestJob implements FileHarvestJob {
return TestFileController.getHarvesterPath() + TestFileController.PATH_TO_ADDITIONS_FILE;
}
@Override
public String getPageHeader() {
return "Harvest " + this.friendlyName + " data from CSV file(s)";
}
@Override
public String getTemplateFilePath() {
return this.templateFile.getPath();
}
}

12
src/edu/cornell/mannlib/vitro/webapp/controller/harvester/FileHarvestJob.java
View file

@ -18,6 +18,12 @@ interface FileHarvestJob {
* @return null if success, message to be returned to the user if failure
*/
String validateUpload(File file);
/**
* Gets the path on the server of the file which the user can download to serve as a guide for what to upload.
* @return the path on the server of the file which the user can download to serve as a guide for what to upload.
*/
String getTemplateFilePath();
/**
* Gets the console script which can be used to run the harvest job.
@ -30,5 +36,11 @@ interface FileHarvestJob {
* @return the path to the file containing the RDF/XML triples that get added to VIVO
*/
String getAdditionsFilePath();
/**
* A heading to be shown at the top of the page.
* @return a heading to be shown at the top of the page
*/
String getPageHeader();
}

55
src/edu/cornell/mannlib/vitro/webapp/controller/harvester/TestFileController.java
View file

@ -3,7 +3,9 @@
package edu.cornell.mannlib.vitro.webapp.controller.harvester;
import java.io.BufferedReader;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
@ -16,6 +18,7 @@ import java.util.List;
import java.util.Map;
import javax.servlet.ServletContext;
import javax.servlet.ServletException;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.xml.parsers.DocumentBuilderFactory;
@ -49,7 +52,7 @@ public class TestFileController extends FreemarkerHttpServlet {
private static final String PARAMETER_FIRST_UPLOAD = "firstUpload";
private static final String PARAMETER_UPLOADED_FILE = "uploadedFile";
private static final String PARAMETER_IS_HARVEST_CLICK = "isHarvestClick";
private static final String PARAMETER_MODE = "mode";
private static final String PARAMETER_JOB = "job";
private static final String POST_TO = "/vivo/harvester/harvest";
@ -57,6 +60,11 @@ public class TestFileController extends FreemarkerHttpServlet {
private static final String JOB_CSV_GRANT = "csvGrant";
private static final String JOB_CSV_PERSON = "csvPerson";
private static final String MODE_HARVEST = "harvest";
private static final String MODE_CHECK_STATUS = "checkStatus";
private static final String MODE_DOWNLOAD_TEMPLATE = "template";
private static final List<String> knownJobs = Arrays.asList(JOB_CSV_GRANT.toLowerCase(), JOB_CSV_PERSON.toLowerCase());
@ -101,11 +109,15 @@ public class TestFileController extends FreemarkerHttpServlet {
//body.put("uploadPostback", "false");
body.put("paramFirstUpload", PARAMETER_FIRST_UPLOAD);
body.put("paramUploadedFile", PARAMETER_UPLOADED_FILE);
body.put("paramIsHarvestClick", PARAMETER_IS_HARVEST_CLICK);
body.put("paramMode", PARAMETER_MODE);
body.put("paramJob", PARAMETER_JOB);
body.put("modeHarvest", MODE_HARVEST);
body.put("modeCheckStatus", MODE_CHECK_STATUS);
body.put("modeDownloadTemplate", MODE_DOWNLOAD_TEMPLATE);
body.put("job", job);
body.put("jobKnown", jobKnown);
body.put("postTo", POST_TO + "?" + PARAMETER_JOB + "=" + job);
body.put("jobSpecificHeader", getJob(vreq, job).getPageHeader());
return new TemplateResponseValues(TEMPLATE_DEFAULT, body);
} catch (Throwable e) {
log.error(e, e);
@ -163,9 +175,9 @@ public class TestFileController extends FreemarkerHttpServlet {
if(jobParameter == null)
log.error("No job specified.");
else if(jobParameter.equalsIgnoreCase(JOB_CSV_GRANT))
job = new CsvFileHarvestJob(vreq, "granttemplate.csv", "testCSVtoRDFgrant.sh", namespace);
job = new CsvFileHarvestJob(vreq, "granttemplate.csv", "testCSVtoRDFgrant.sh", namespace, "Grant");
else if(jobParameter.equalsIgnoreCase(JOB_CSV_PERSON))
job = new CsvFileHarvestJob(vreq, "persontemplate.csv", "testCSVtoRDFperson.sh", namespace);
job = new CsvFileHarvestJob(vreq, "persontemplate.csv", "testCSVtoRDFperson.sh", namespace, "Person");
else
log.error("Invalid job: " + jobParameter);
@ -197,12 +209,17 @@ public class TestFileController extends FreemarkerHttpServlet {
try {
boolean isMultipart = ServletFileUpload.isMultipartContent(request);
String mode = request.getParameter(PARAMETER_MODE);
if(isMultipart)
doFileUploadPost(request, response);
else if(request.getParameter(PARAMETER_IS_HARVEST_CLICK).toLowerCase().equals("true"))
else if(mode.equals(MODE_HARVEST))
doHarvestPost(request, response);
else
else if(mode.equals(MODE_CHECK_STATUS))
doCheckHarvestStatusPost(request, response);
else if(mode.equals(MODE_DOWNLOAD_TEMPLATE))
doDownloadTemplatePost(request, response);
else
throw new Exception("Unrecognized post mode: " + mode);
} catch(Exception e) {
log.error(e, e);
}
@ -429,6 +446,32 @@ public class TestFileController extends FreemarkerHttpServlet {
}
}
private void doDownloadTemplatePost(HttpServletRequest request, HttpServletResponse response) {
VitroRequest vreq = new VitroRequest(request);
FileHarvestJob job = getJob(vreq, vreq.getParameter(PARAMETER_JOB));
File fileToSend = new File(job.getTemplateFilePath());
response.setContentType("application/octet-stream");
response.setContentLength((int)(fileToSend.length()));
response.setHeader("Content-Disposition", "attachment; filename=\"" + fileToSend.getName() + "\"");
try {
byte[] byteBuffer = new byte[(int)(fileToSend.length())];
DataInputStream inStream = new DataInputStream(new FileInputStream(fileToSend));
ServletOutputStream outputStream = response.getOutputStream();
for(int length = inStream.read(byteBuffer); length != -1; length = inStream.read(byteBuffer)) {
outputStream.write(byteBuffer, 0, length);
}
inStream.close();
outputStream.flush();
outputStream.close();
} catch(IOException e) {
log.error(e, e);
}
}
private File createScriptFile(String script) throws IOException {