litvinovg/vivo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

testfile.ftl, FileHarvestJob.java, TestFileController.java: Major updates, still too general and under-construction for a more specific report

Browse source
This commit is contained in:
mbarbier 2011-06-15 21:02:09 +00:00
parent 47f2cfed3d
commit 83194eb293
3 changed files with 84 additions and 212 deletions
Download patch file Download diff file Expand all files Collapse all files

6
src/edu/cornell/mannlib/vitro/webapp/controller/harvester/FileHarvestJob.java
View file

@ -25,12 +25,6 @@ interface FileHarvestJob {
*/
String getScript();
/**
* Runs a harvest on the files in the specified directory.
* @param directory the directory containing files to harvest
*/
void performHarvest(File directory);
/**
* The path to the file containing the RDF/XML triples that get added to VIVO.
* @return the path to the file containing the RDF/XML triples that get added to VIVO

274
src/edu/cornell/mannlib/vitro/webapp/controller/harvester/TestFileController.java
View file

@ -9,6 +9,7 @@ import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Hashtable;
import java.util.List;
@ -49,15 +50,62 @@ public class TestFileController extends FreemarkerHttpServlet {
private static final String PARAMETER_FIRST_UPLOAD = "firstUpload";
private static final String PARAMETER_UPLOADED_FILE = "uploadedFile";
private static final String PARAMETER_IS_HARVEST_CLICK = "isHarvestClick";
private static final String PARAMETER_JOB = "job";
private static final String POST_TO = "/vivo/harvester/harvest";
private static final String JOB_CSV_GRANT = "csvGrant";
private static final String JOB_CSV_PERSON = "csvPerson";
private static final List<String> knownJobs = Arrays.asList(JOB_CSV_GRANT.toLowerCase(), JOB_CSV_PERSON.toLowerCase());
/**
* Relative path from the VIVO Uploads directory to the root location where user-uploaded files will be stored. Include
* final slash.
*/
private static final String PATH_TO_UPLOADS = "harvester/";
/**
* Absolute path on the server of the Harvester root directory. Include final slash.
*/
private static final String PATH_TO_HARVESTER = "/home/mbarbieri/workspace/HarvesterDev/";
/**
* Relative path from the Harvester root directory to the Additions file containing rdf/xml added to VIVO from Harvest run.
*/
public static final String PATH_TO_ADDITIONS_FILE = "harvested-data/csv/additions.rdf.xml"; //todo: this is job-specific
/**
* Relative path from the Harvester root directory to the directory where user-downloadable template files are stored.
*/
public static final String PATH_TO_TEMPLATE_FILES = "files/";
/**
* Relative path from the Harvester root directory to the directory containing the script templates. Include final slash.
*/
public static final String PATH_TO_HARVESTER_SCRIPTS = "scripts/";
@Override
protected ResponseValues processRequest(VitroRequest vreq) {
try {
String job = vreq.getParameter(PARAMETER_JOB);
String jobKnown = "false";
if((job != null) && TestFileController.knownJobs.contains(job.toLowerCase()))
jobKnown = "true";
Map<String, Object> body = new HashMap<String, Object>();
//body.put("uploadPostback", "false");
body.put("paramFirstUpload", PARAMETER_FIRST_UPLOAD);
body.put("paramUploadedFile", PARAMETER_UPLOADED_FILE);
body.put("paramIsHarvestClick", PARAMETER_IS_HARVEST_CLICK);
body.put("paramJob", PARAMETER_JOB);
body.put("job", job);
body.put("jobKnown", jobKnown);
body.put("postTo", POST_TO + "?" + PARAMETER_JOB + "=" + job);
return new TemplateResponseValues(TEMPLATE_DEFAULT, body);
} catch (Throwable e) {
log.error(e, e);
@ -76,8 +124,7 @@ public class TestFileController extends FreemarkerHttpServlet {
*/
public static String getHarvesterPath()
{
//String harvesterPath = "/usr/share/vivo/harvester/"; //todo: hack
String harvesterPath = "/home/mbarbieri/workspace/HarvesterDevTomcat2/";
String harvesterPath = PATH_TO_HARVESTER;
return harvesterPath;
}
@ -94,7 +141,7 @@ public class TestFileController extends FreemarkerHttpServlet {
throw new Exception("Vitro home directory name could not be found.");
}
String pathBase = vitroHomeDirectoryName + "/" + FileStorageSetup.FILE_STORAGE_SUBDIRECTORY + "/harvester/";
String pathBase = vitroHomeDirectoryName + "/" + FileStorageSetup.FILE_STORAGE_SUBDIRECTORY + "/" + PATH_TO_UPLOADS;
return pathBase;
}
@ -102,14 +149,27 @@ public class TestFileController extends FreemarkerHttpServlet {
* Gets the FileHarvestJob implementation that is needed to handle the specified request. This
* will depend on the type of harvest being performed (CSV, RefWorks, etc.)
* @param vreq the request from the browser
* @param jobParameter the POST or GET parameter "job". Might not be available in vreq at this point,
* thus we are requiring that it be sent in.
* @return the FileHarvestJob that will provide harvest-type-specific services for this request
*/
private FileHarvestJob getJob(VitroRequest vreq)
private FileHarvestJob getJob(VitroRequest vreq, String jobParameter)
{
String namespace = vreq.getWebappDaoFactory().getDefaultNamespace();
FileHarvestJob job = null;
//todo: complete
return new CsvHarvestJob(vreq, "granttemplate.csv", namespace);
if(jobParameter == null)
log.error("No job specified.");
else if(jobParameter.equalsIgnoreCase(JOB_CSV_GRANT))
job = new CsvFileHarvestJob(vreq, "granttemplate.csv", "testCSVtoRDFgrant.sh", namespace);
else if(jobParameter.equalsIgnoreCase(JOB_CSV_PERSON))
job = new CsvFileHarvestJob(vreq, "persontemplate.csv", "testCSVtoRDFperson.sh", namespace);
else
log.error("Invalid job: " + jobParameter);
return job;
}
/**
@ -172,6 +232,9 @@ public class TestFileController extends FreemarkerHttpServlet {
Exception e = req.getFileUploadException();
new ExceptionVisibleToUser(e);
}
//get the job parameter
String jobParameter = req.getParameter(PARAMETER_JOB);
//get the location where we want to save the files (it will end in a slash), then create a File object out of it
String path = getUploadPath(vreq);
@ -197,7 +260,7 @@ public class TestFileController extends FreemarkerHttpServlet {
directory.mkdirs();
//get the file harvest job for this request (this will determine what type of harvest is run)
FileHarvestJob job = getJob(vreq);
FileHarvestJob job = getJob(vreq, jobParameter);
//get the files out of the parsed request (there should only be one)
Map<String, List<FileItem>> fileStreams = req.getFiles();
@ -285,7 +348,7 @@ public class TestFileController extends FreemarkerHttpServlet {
log.error("harvest post.");
try {
VitroRequest vreq = new VitroRequest(request);
FileHarvestJob job = getJob(vreq);
FileHarvestJob job = getJob(vreq, vreq.getParameter(PARAMETER_JOB));
//String path = getUploadPath(vreq);
@ -590,198 +653,3 @@ public class TestFileController extends FreemarkerHttpServlet {
}
/**
* An implementation of FileHarvestJob that can be used for any CSV file harvest.
*/
class CsvHarvestJob implements FileHarvestJob {
/**
* Logger.
*/
private static final Log log = LogFactory.getLog(CsvHarvestJob.class);
/**
* The HTTP request.
*/
private VitroRequest vreq;
/**
* The template file against which uploaded CSV files will be validated.
*/
private File templateFile;
/**
* The namespace to be used for the harvest.
*/
private final String namespace;
/**
* Constructor.
* @param templateFileName just the name of the template file. The directory is assumed to be standard.
*/
public CsvHarvestJob(VitroRequest vreq, String templateFileName, String namespace) {
this.vreq = vreq;
this.templateFile = new File(getTemplateFileDirectory() + templateFileName);
log.error(getTemplateFileDirectory() + templateFileName);
this.namespace = namespace;
}
/**
* Gets the path to the directory containing the template files.
* @return the path to the directory containing the template files
*/
private String getTemplateFileDirectory() {
String harvesterPath = TestFileController.getHarvesterPath();
String pathToTemplateFiles = harvesterPath + "files/";
return pathToTemplateFiles;
}
@Override
@SuppressWarnings("rawtypes")
public String validateUpload(File file) {
try {
SimpleReader reader = new SimpleReader();
List templateCsv = reader.parse(this.templateFile);
String[] templateFirstLine = (String[])templateCsv.get(0);
List csv = reader.parse(file);
int length = csv.size();
if(length == 0)
return "No data in file";
for(int i = 0; i < length; i++) {
String[] line = (String[])csv.get(i);
if(i == 0) {
String errorMessage = validateCsvFirstLine(templateFirstLine, line);
if(errorMessage != null)
return errorMessage;
}
else if(line.length != 0) {
if(line.length != templateFirstLine.length) {
String retval = "Mismatch in number of entries in row " + i + ": expected , " + templateFirstLine.length + ", found " + line.length + " ";
for(int j = 0; j < line.length; j++) {
retval += "\"" + line[j] + "\", ";
}
//return retval;
return "Mismatch in number of entries in row " + i + ": expected , " + templateFirstLine.length + ", found " + line.length;
}
}
}
} catch (IOException e) {
log.error(e, e);
return e.getMessage();
}
return null;
}
/**
* Makes sure that the first line of the CSV file is identical to the first line of the template file. This is
* assuming we are expecting all user CSV files to contain an initial header line. If this is not the case, then
* this method is unnecessary.
* @param templateFirstLine the parsed-out contents of the first line of the template file
* @param line the parsed-out contents of the first line of the input file
* @return an error message if the two lines don't match, or null if they do
*/
private String validateCsvFirstLine(String[] templateFirstLine, String[] line) {
String errorMessage = "File header does not match template";
if(line.length != templateFirstLine.length) {
//return errorMessage + ": " + "file header columns = " + line.length + ", template columns = " + templateFirstLine.length;
String errorMsg = "";
errorMsg += "file header items: ";
for(int i = 0; i < line.length; i++) {
errorMsg += line[i] + ", ";
}
errorMsg += "template items: ";
for(int i = 0; i < templateFirstLine.length; i++) {
errorMsg += templateFirstLine[i] + ", ";
}
return errorMsg;
}
for(int i = 0; i < line.length; i++)
{
if(!line[i].equals(templateFirstLine[i]))
return errorMessage + ": file header column " + (i + 1) + " = " + line[i] + ", template column " + (i + 1) + " = " + templateFirstLine[i];
}
return null;
}
@Override
public String getScript()
{
String path = TestFileController.getHarvesterPath() + "scripts/" + "testCSVtoRDFgrant.sh"; //todo: complete
File scriptTemplate = new File(path);
String scriptTemplateContents = readScriptTemplate(scriptTemplate);
String replacements = performScriptTemplateReplacements(scriptTemplateContents);
return replacements;
}
private String performScriptTemplateReplacements(String scriptTemplateContents) {
String replacements = scriptTemplateContents;
String fileDirectory = TestFileController.getUploadPath(vreq);
replacements = replacements.replace("${UPLOADS_FOLDER}", fileDirectory);
/*
* What needs to be replaced?
*
* task directory name
*/
//todo: complete
return replacements;
}
private String readScriptTemplate(File scriptTemplate) {
String scriptTemplateContents = null;
BufferedReader reader = null;
try {
int fileSize = (int)(scriptTemplate.length());
char[] buffer = new char[fileSize];
reader = new BufferedReader(new FileReader(scriptTemplate), fileSize);
reader.read(buffer);
scriptTemplateContents = new String(buffer);
} catch (IOException e) {
log.error(e, e);
} finally {
try {
if(reader != null)
reader.close();
} catch(IOException e) {
log.error(e, e);
}
}
return scriptTemplateContents;
}
@Override
public void performHarvest(File directory) {
}
@Override
public String getAdditionsFilePath() {
return TestFileController.getHarvesterPath() + "harvested-data/csv/additions.rdf.xml";
}
}