VIVO-974 Modify dump-restore to use only N-Quads format.

This commit is contained in:
Jim Blake 2015-02-16 10:39:23 -05:00
parent 46b075464c
commit b59d755007
4 changed files with 12 additions and 90 deletions

View file

@ -3,7 +3,6 @@
package edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore; package edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore;
import static edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.ACTION_DUMP; import static edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.ACTION_DUMP;
import static edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.PARAMETER_FORMAT;
import static edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.PARAMETER_WHICH; import static edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.PARAMETER_WHICH;
import java.io.IOException; import java.io.IOException;
@ -15,7 +14,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.BadRequestException; import edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.BadRequestException;
import edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.DumpFormat;
import edu.cornell.mannlib.vitro.webapp.modelaccess.ModelAccess.WhichService; import edu.cornell.mannlib.vitro.webapp.modelaccess.ModelAccess.WhichService;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService; import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService.ResultFormat; import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService.ResultFormat;
@ -32,8 +30,10 @@ import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException;
class DumpModelsAction extends AbstractDumpRestoreAction { class DumpModelsAction extends AbstractDumpRestoreAction {
private static final Log log = LogFactory.getLog(DumpModelsAction.class); private static final Log log = LogFactory.getLog(DumpModelsAction.class);
private static final String N_QUADS_EXTENSION = ".nq";
private static final String N_QUADS_MIME_TYPE = "application/n-quads";
private final HttpServletResponse resp; private final HttpServletResponse resp;
private final DumpFormat format;
private final WhichService which; private final WhichService which;
private final String queryString; private final String queryString;
@ -41,14 +41,12 @@ class DumpModelsAction extends AbstractDumpRestoreAction {
throws BadRequestException { throws BadRequestException {
super(req); super(req);
this.resp = resp; this.resp = resp;
this.format = getEnumFromParameter(DumpFormat.class, PARAMETER_FORMAT);
this.which = getEnumFromParameter(WhichService.class, PARAMETER_WHICH); this.which = getEnumFromParameter(WhichService.class, PARAMETER_WHICH);
this.queryString = req.getQueryString(); this.queryString = req.getQueryString();
} }
void redirectToFilename() throws IOException { void redirectToFilename() throws IOException {
String filename = which + "." + format.getExtension(); String filename = which + N_QUADS_EXTENSION;
String urlPath = req.getContextPath() + req.getServletPath() String urlPath = req.getContextPath() + req.getServletPath()
+ ACTION_DUMP; + ACTION_DUMP;
resp.sendRedirect(urlPath + "/" + filename + "?" + queryString); resp.sendRedirect(urlPath + "/" + filename + "?" + queryString);
@ -59,16 +57,11 @@ class DumpModelsAction extends AbstractDumpRestoreAction {
RDFService rdfService = getRdfService(which); RDFService rdfService = getRdfService(which);
String query = "SELECT * WHERE { GRAPH ?g {?s ?p ?o}}"; String query = "SELECT * WHERE { GRAPH ?g {?s ?p ?o}}";
resp.setContentType(format.getMimeType()); resp.setContentType(N_QUADS_MIME_TYPE);
if (format == DumpFormat.NQUADS) {
dumpNQuads(rdfService, query); dumpNQuads(rdfService, query);
} else {
rdfService.sparqlSelectQuery(query,
format.getRdfServiceFormat(), resp.getOutputStream());
}
} catch (Throwable t) { } catch (Throwable t) {
log.error("Failed to dump " + which + " models as " + format + ".", log.error("Failed to dump " + which + " models as N-Quads.", t);
t);
} }
} }

View file

@ -43,7 +43,6 @@ public class DumpRestoreController extends FreemarkerHttpServlet {
static final String ACTION_RESTORE = "/restore"; static final String ACTION_RESTORE = "/restore";
static final String ACTION_SELECT = "/select"; static final String ACTION_SELECT = "/select";
static final String PARAMETER_WHICH = "which"; static final String PARAMETER_WHICH = "which";
static final String PARAMETER_FORMAT = "format";
static final String PARAMETER_SOURCE_FILE = "sourceFile"; static final String PARAMETER_SOURCE_FILE = "sourceFile";
static final String PARAMETER_PURGE = "purge"; static final String PARAMETER_PURGE = "purge";
static final String ATTRIBUTE_TRIPLE_COUNT = "tripleCount"; static final String ATTRIBUTE_TRIPLE_COUNT = "tripleCount";
@ -128,53 +127,4 @@ public class DumpRestoreController extends FreemarkerHttpServlet {
super(message); super(message);
} }
} }
/**
* The formats that we will accept on a dump request.
*/
enum DumpFormat {
NQUADS("application/n-quads", "nq", null),
JSON("application/sparql-results+json", "srj", ResultFormat.JSON),
XML("application/sparql-results+xml", "srx", ResultFormat.XML);
private final String mimeType;
private final String extension;
private final ResultFormat rdfServiceFormat;
private DumpFormat(String mimeType, String extension,
ResultFormat rdfServiceFormat) {
this.mimeType = mimeType;
this.extension = extension;
this.rdfServiceFormat = rdfServiceFormat;
}
public String getMimeType() {
return mimeType;
}
public String getExtension() {
return extension;
}
public ResultFormat getRdfServiceFormat() {
return rdfServiceFormat;
}
}
/**
* The formats that we will accept on a restore request.
*/
enum RestoreFormat {
NQUADS {
@Override
public DumpParser getParser(InputStream is) throws IOException {
return new NquadsParser(is);
}
};
public abstract DumpParser getParser(InputStream is) throws IOException;
}
} }

View file

@ -2,7 +2,6 @@
package edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore; package edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore;
import static edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.PARAMETER_FORMAT;
import static edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.PARAMETER_PURGE; import static edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.PARAMETER_PURGE;
import static edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.PARAMETER_SOURCE_FILE; import static edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.PARAMETER_SOURCE_FILE;
import static edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.PARAMETER_WHICH; import static edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.PARAMETER_WHICH;
@ -32,7 +31,6 @@ import com.hp.hpl.jena.rdf.model.Model;
import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest; import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
import edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.BadRequestException; import edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.BadRequestException;
import edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.RestoreFormat;
import edu.cornell.mannlib.vitro.webapp.dao.jena.RDFServiceDataset; import edu.cornell.mannlib.vitro.webapp.dao.jena.RDFServiceDataset;
import edu.cornell.mannlib.vitro.webapp.modelaccess.ModelAccess.WhichService; import edu.cornell.mannlib.vitro.webapp.modelaccess.ModelAccess.WhichService;
import edu.cornell.mannlib.vitro.webapp.rdfservice.ChangeSet; import edu.cornell.mannlib.vitro.webapp.rdfservice.ChangeSet;
@ -59,7 +57,6 @@ public class RestoreModelsAction extends AbstractDumpRestoreAction {
private static final String DEFAULT_GRAPH_URI = "__default__"; private static final String DEFAULT_GRAPH_URI = "__default__";
private final FileItem sourceFile; private final FileItem sourceFile;
private final RestoreFormat format;
private final WhichService which; private final WhichService which;
private final boolean purge; private final boolean purge;
private final SelfLimitingTripleBuckets bnodeBuckets; private final SelfLimitingTripleBuckets bnodeBuckets;
@ -71,8 +68,6 @@ public class RestoreModelsAction extends AbstractDumpRestoreAction {
throws BadRequestException { throws BadRequestException {
super(req); super(req);
this.sourceFile = getFileItem(PARAMETER_SOURCE_FILE); this.sourceFile = getFileItem(PARAMETER_SOURCE_FILE);
this.format = getEnumFromParameter(RestoreFormat.class,
PARAMETER_FORMAT);
this.which = getEnumFromParameter(WhichService.class, PARAMETER_WHICH); this.which = getEnumFromParameter(WhichService.class, PARAMETER_WHICH);
this.purge = null != req.getParameter(PARAMETER_PURGE); this.purge = null != req.getParameter(PARAMETER_PURGE);
@ -114,7 +109,7 @@ public class RestoreModelsAction extends AbstractDumpRestoreAction {
log.info("Restoring the " + which + " models."); log.info("Restoring the " + which + " models.");
long lineCount = 0; long lineCount = 0;
try (InputStream is = sourceFile.getInputStream(); try (InputStream is = sourceFile.getInputStream();
DumpParser p = format.getParser(is)) { DumpParser p = new NquadsParser(is)) {
for (DumpQuad line : p) { for (DumpQuad line : p) {
bucketize(line); bucketize(line);
lineCount++; lineCount++;
@ -160,7 +155,7 @@ public class RestoreModelsAction extends AbstractDumpRestoreAction {
rdfService.changeSetUpdate(change); rdfService.changeSetUpdate(change);
tripleCount += triples.size(); tripleCount += triples.size();
log.info("processed " + tripleCount +" triples."); log.info("processed " + tripleCount + " triples.");
} }
private InputStream serialize(Collection<DumpTriple> triples) private InputStream serialize(Collection<DumpTriple> triples)

View file

@ -35,7 +35,6 @@ table.choices td {
<table class="choices"> <table class="choices">
<tr> <tr>
<td>Select models</td> <td>Select models</td>
<td>Select format</td>
<td>&nbsp;</td> <td>&nbsp;</td>
</tr> </tr>
<tr> <tr>
@ -45,13 +44,6 @@ table.choices td {
<option value="CONTENT">Content models</option> <option value="CONTENT">Content models</option>
</select> </select>
</td> </td>
<td>
<select name="format">
<option value="NQUADS">N-Quads</option>
<option value="JSON">RS-JSON</option>
<option value="XML">RS-XML</option>
</select>
</td>
<td> <td>
<input type="submit" value="Dump" /> <input type="submit" value="Dump" />
</td> </td>
@ -84,8 +76,7 @@ table.choices td {
<table class="choices"> <table class="choices">
<tr> <tr>
<td>Select models</td> <td>Select models</td>
<td>Select a file to restore from</td> <td>Select a file to restore from (N-Quads format)</td>
<td>Select format</td>
<td>&nbsp;</td> <td>&nbsp;</td>
</tr> </tr>
<tr> <tr>
@ -98,13 +89,6 @@ table.choices td {
<td> <td>
<input type="file" name="sourceFile" size="60"/> <input type="file" name="sourceFile" size="60"/>
</td> </td>
<td>
<select name="format">
<option value="NQUADS">N-Quads</option>
<!-- <option value="JSON">RS-JSON</option> TODO -->
<!-- <option value="XML">RS-XML</option> TODO -->
</select>
</td>
<td> <td>
<input type="submit" value="Restore" /> <input type="submit" value="Restore" />
</td> </td>