diff --git a/productMods/templates/freemarker/body/harvester/testfile.ftl b/productMods/templates/freemarker/body/harvester/testfile.ftl index 736047d1..d9cfa7bc 100644 --- a/productMods/templates/freemarker/body/harvester/testfile.ftl +++ b/productMods/templates/freemarker/body/harvester/testfile.ftl @@ -24,8 +24,8 @@ } request.open("POST", "${postTo}", true); request.setRequestHeader("content-type","application/x-www-form-urlencoded"); - //request.send("${paramIsHarvestClick}=true&${paramJob}=${job}"); - request.send("${paramIsHarvestClick}=true"); + //request.send("${paramMode}=${modeHarvest}&${paramJob}=${job}"); + request.send("${paramMode}=${modeHarvest}"); } @@ -49,8 +49,8 @@ } request.open("POST", "${postTo}", true); request.setRequestHeader("Content-type","application/x-www-form-urlencoded"); - //request.send("${paramIsHarvestClick}=false&${paramJob}=${job}"); - request.send("${paramIsHarvestClick}=false"); + //request.send("${paramMode}=${modeCheckStatus}&${paramJob}=${job}"); + request.send("${paramMode}=${modeCheckStatus}"); } else { var importedGrants = document.getElementById("importedGrants") @@ -124,6 +124,10 @@ document.getElementById("fileUploadForm").target = "uploadTarget"; document.getElementById("uploadTarget").onload = fileResponse; } + document.getElementById("downloadTemplateForm").onsubmit = function() + { + document.getElementById("downloadTemplateForm").target = "uploadTarget"; + } } window.onload = init; @@ -150,12 +154,15 @@ } - +

${jobSpecificHeader}

Step 1

Download template

-

We are providing a helpful template file for you to download.

+
+ +

We are providing a helpful template file for you to download.

+
diff --git a/src/edu/cornell/mannlib/vitro/webapp/controller/harvester/CsvFileHarvestJob.java b/src/edu/cornell/mannlib/vitro/webapp/controller/harvester/CsvFileHarvestJob.java index 05a53d05..bff4bed7 100644 --- a/src/edu/cornell/mannlib/vitro/webapp/controller/harvester/CsvFileHarvestJob.java +++ b/src/edu/cornell/mannlib/vitro/webapp/controller/harvester/CsvFileHarvestJob.java @@ -6,6 +6,7 @@ import java.io.BufferedReader; import java.io.File; import java.io.FileReader; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import org.apache.commons.logging.Log; @@ -46,16 +47,22 @@ class CsvFileHarvestJob implements FileHarvestJob { */ private final String namespace; + /** + * A name for the type of data being imported. For example "Grant" or "Person". + */ + private final String friendlyName; + /** * Constructor. * @param templateFileName just the name of the template file. The directory is assumed to be standard. */ - public CsvFileHarvestJob(VitroRequest vreq, String templateFileName, String scriptFileName, String namespace) { + public CsvFileHarvestJob(VitroRequest vreq, String templateFileName, String scriptFileName, String namespace, String friendlyName) { this.vreq = vreq; this.templateFile = new File(getTemplateFileDirectory() + templateFileName); this.scriptFile = new File(getScriptFileDirectory() + scriptFileName); log.error(getTemplateFileDirectory() + templateFileName); this.namespace = namespace; + this.friendlyName = friendlyName; } /** @@ -79,6 +86,27 @@ class CsvFileHarvestJob implements FileHarvestJob { } + + private boolean[] getLinesEndingInComma(File file) throws IOException { + ArrayList linesEndingInCommaList = new ArrayList(); + + BufferedReader reader = new BufferedReader(new FileReader(file)); + + for(String line = reader.readLine(); line != null; line = reader.readLine()) { + boolean lineEndsInComma = line.endsWith(","); + linesEndingInCommaList.add(lineEndsInComma); + } + reader.close(); + + boolean[] linesEndingInComma = new boolean[linesEndingInCommaList.size()]; + for(int i = 0; i < linesEndingInComma.length; i++) { + linesEndingInComma[i] = linesEndingInCommaList.get(i); + } + return linesEndingInComma; + } + + + @Override @SuppressWarnings("rawtypes") public String validateUpload(File file) { @@ -88,7 +116,9 @@ class CsvFileHarvestJob implements FileHarvestJob { List templateCsv = reader.parse(this.templateFile); String[] templateFirstLine = (String[])templateCsv.get(0); + //if a line ends in a comma (absolutely a comma, no whitespace), SimpleReader will not consider the part after the comma to be a blank section. List csv = reader.parse(file); + boolean[] linesEndingInComma = getLinesEndingInComma(file); int length = csv.size(); @@ -97,19 +127,16 @@ class CsvFileHarvestJob implements FileHarvestJob { for(int i = 0; i < length; i++) { String[] line = (String[])csv.get(i); + boolean endsInComma = linesEndingInComma[i]; if(i == 0) { String errorMessage = validateCsvFirstLine(templateFirstLine, line); if(errorMessage != null) return errorMessage; } else if(line.length != 0) { - if(line.length != templateFirstLine.length) { - String retval = "Mismatch in number of entries in row " + i + ": expected , " + templateFirstLine.length + ", found " + line.length + " "; - for(int j = 0; j < line.length; j++) { - retval += "\"" + line[j] + "\", "; - } - //return retval; - return "Mismatch in number of entries in row " + i + ": expected , " + templateFirstLine.length + ", found " + line.length; + int actualLineLength = line.length + (endsInComma ? 1 : 0); + if(actualLineLength != templateFirstLine.length) { + return "Mismatch in number of entries in row " + i + ": expected " + templateFirstLine.length + ", found " + actualLineLength; } } } @@ -209,7 +236,15 @@ class CsvFileHarvestJob implements FileHarvestJob { return TestFileController.getHarvesterPath() + TestFileController.PATH_TO_ADDITIONS_FILE; } + @Override + public String getPageHeader() { + return "Harvest " + this.friendlyName + " data from CSV file(s)"; + } + @Override + public String getTemplateFilePath() { + return this.templateFile.getPath(); + } } diff --git a/src/edu/cornell/mannlib/vitro/webapp/controller/harvester/FileHarvestJob.java b/src/edu/cornell/mannlib/vitro/webapp/controller/harvester/FileHarvestJob.java index 705d898e..8c7db7c7 100644 --- a/src/edu/cornell/mannlib/vitro/webapp/controller/harvester/FileHarvestJob.java +++ b/src/edu/cornell/mannlib/vitro/webapp/controller/harvester/FileHarvestJob.java @@ -18,6 +18,12 @@ interface FileHarvestJob { * @return null if success, message to be returned to the user if failure */ String validateUpload(File file); + + /** + * Gets the path on the server of the file which the user can download to serve as a guide for what to upload. + * @return the path on the server of the file which the user can download to serve as a guide for what to upload. + */ + String getTemplateFilePath(); /** * Gets the console script which can be used to run the harvest job. @@ -30,5 +36,11 @@ interface FileHarvestJob { * @return the path to the file containing the RDF/XML triples that get added to VIVO */ String getAdditionsFilePath(); + + /** + * A heading to be shown at the top of the page. + * @return a heading to be shown at the top of the page + */ + String getPageHeader(); } diff --git a/src/edu/cornell/mannlib/vitro/webapp/controller/harvester/TestFileController.java b/src/edu/cornell/mannlib/vitro/webapp/controller/harvester/TestFileController.java index 40d33a84..b0d9a8ed 100644 --- a/src/edu/cornell/mannlib/vitro/webapp/controller/harvester/TestFileController.java +++ b/src/edu/cornell/mannlib/vitro/webapp/controller/harvester/TestFileController.java @@ -3,7 +3,9 @@ package edu.cornell.mannlib.vitro.webapp.controller.harvester; import java.io.BufferedReader; +import java.io.DataInputStream; import java.io.File; +import java.io.FileInputStream; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; @@ -16,6 +18,7 @@ import java.util.List; import java.util.Map; import javax.servlet.ServletContext; import javax.servlet.ServletException; +import javax.servlet.ServletOutputStream; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import javax.xml.parsers.DocumentBuilderFactory; @@ -49,7 +52,7 @@ public class TestFileController extends FreemarkerHttpServlet { private static final String PARAMETER_FIRST_UPLOAD = "firstUpload"; private static final String PARAMETER_UPLOADED_FILE = "uploadedFile"; - private static final String PARAMETER_IS_HARVEST_CLICK = "isHarvestClick"; + private static final String PARAMETER_MODE = "mode"; private static final String PARAMETER_JOB = "job"; private static final String POST_TO = "/vivo/harvester/harvest"; @@ -57,6 +60,11 @@ public class TestFileController extends FreemarkerHttpServlet { private static final String JOB_CSV_GRANT = "csvGrant"; private static final String JOB_CSV_PERSON = "csvPerson"; + private static final String MODE_HARVEST = "harvest"; + private static final String MODE_CHECK_STATUS = "checkStatus"; + private static final String MODE_DOWNLOAD_TEMPLATE = "template"; + + private static final List knownJobs = Arrays.asList(JOB_CSV_GRANT.toLowerCase(), JOB_CSV_PERSON.toLowerCase()); @@ -101,11 +109,15 @@ public class TestFileController extends FreemarkerHttpServlet { //body.put("uploadPostback", "false"); body.put("paramFirstUpload", PARAMETER_FIRST_UPLOAD); body.put("paramUploadedFile", PARAMETER_UPLOADED_FILE); - body.put("paramIsHarvestClick", PARAMETER_IS_HARVEST_CLICK); + body.put("paramMode", PARAMETER_MODE); body.put("paramJob", PARAMETER_JOB); + body.put("modeHarvest", MODE_HARVEST); + body.put("modeCheckStatus", MODE_CHECK_STATUS); + body.put("modeDownloadTemplate", MODE_DOWNLOAD_TEMPLATE); body.put("job", job); body.put("jobKnown", jobKnown); body.put("postTo", POST_TO + "?" + PARAMETER_JOB + "=" + job); + body.put("jobSpecificHeader", getJob(vreq, job).getPageHeader()); return new TemplateResponseValues(TEMPLATE_DEFAULT, body); } catch (Throwable e) { log.error(e, e); @@ -163,9 +175,9 @@ public class TestFileController extends FreemarkerHttpServlet { if(jobParameter == null) log.error("No job specified."); else if(jobParameter.equalsIgnoreCase(JOB_CSV_GRANT)) - job = new CsvFileHarvestJob(vreq, "granttemplate.csv", "testCSVtoRDFgrant.sh", namespace); + job = new CsvFileHarvestJob(vreq, "granttemplate.csv", "testCSVtoRDFgrant.sh", namespace, "Grant"); else if(jobParameter.equalsIgnoreCase(JOB_CSV_PERSON)) - job = new CsvFileHarvestJob(vreq, "persontemplate.csv", "testCSVtoRDFperson.sh", namespace); + job = new CsvFileHarvestJob(vreq, "persontemplate.csv", "testCSVtoRDFperson.sh", namespace, "Person"); else log.error("Invalid job: " + jobParameter); @@ -197,12 +209,17 @@ public class TestFileController extends FreemarkerHttpServlet { try { boolean isMultipart = ServletFileUpload.isMultipartContent(request); + String mode = request.getParameter(PARAMETER_MODE); if(isMultipart) doFileUploadPost(request, response); - else if(request.getParameter(PARAMETER_IS_HARVEST_CLICK).toLowerCase().equals("true")) + else if(mode.equals(MODE_HARVEST)) doHarvestPost(request, response); - else + else if(mode.equals(MODE_CHECK_STATUS)) doCheckHarvestStatusPost(request, response); + else if(mode.equals(MODE_DOWNLOAD_TEMPLATE)) + doDownloadTemplatePost(request, response); + else + throw new Exception("Unrecognized post mode: " + mode); } catch(Exception e) { log.error(e, e); } @@ -429,6 +446,32 @@ public class TestFileController extends FreemarkerHttpServlet { } } + private void doDownloadTemplatePost(HttpServletRequest request, HttpServletResponse response) { + + VitroRequest vreq = new VitroRequest(request); + FileHarvestJob job = getJob(vreq, vreq.getParameter(PARAMETER_JOB)); + File fileToSend = new File(job.getTemplateFilePath()); + + response.setContentType("application/octet-stream"); + response.setContentLength((int)(fileToSend.length())); + response.setHeader("Content-Disposition", "attachment; filename=\"" + fileToSend.getName() + "\""); + + try { + byte[] byteBuffer = new byte[(int)(fileToSend.length())]; + DataInputStream inStream = new DataInputStream(new FileInputStream(fileToSend)); + + ServletOutputStream outputStream = response.getOutputStream(); + for(int length = inStream.read(byteBuffer); length != -1; length = inStream.read(byteBuffer)) { + outputStream.write(byteBuffer, 0, length); + } + + inStream.close(); + outputStream.flush(); + outputStream.close(); + } catch(IOException e) { + log.error(e, e); + } + } private File createScriptFile(String script) throws IOException {