VIVO-974 Modify dump-restore to use only N-Quads format.
This commit is contained in:
parent
46b075464c
commit
b59d755007
4 changed files with 12 additions and 90 deletions
|
@ -3,7 +3,6 @@
|
||||||
package edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore;
|
package edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore;
|
||||||
|
|
||||||
import static edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.ACTION_DUMP;
|
import static edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.ACTION_DUMP;
|
||||||
import static edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.PARAMETER_FORMAT;
|
|
||||||
import static edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.PARAMETER_WHICH;
|
import static edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.PARAMETER_WHICH;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
@ -15,7 +14,6 @@ import org.apache.commons.logging.Log;
|
||||||
import org.apache.commons.logging.LogFactory;
|
import org.apache.commons.logging.LogFactory;
|
||||||
|
|
||||||
import edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.BadRequestException;
|
import edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.BadRequestException;
|
||||||
import edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.DumpFormat;
|
|
||||||
import edu.cornell.mannlib.vitro.webapp.modelaccess.ModelAccess.WhichService;
|
import edu.cornell.mannlib.vitro.webapp.modelaccess.ModelAccess.WhichService;
|
||||||
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
|
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
|
||||||
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService.ResultFormat;
|
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService.ResultFormat;
|
||||||
|
@ -32,8 +30,10 @@ import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException;
|
||||||
class DumpModelsAction extends AbstractDumpRestoreAction {
|
class DumpModelsAction extends AbstractDumpRestoreAction {
|
||||||
private static final Log log = LogFactory.getLog(DumpModelsAction.class);
|
private static final Log log = LogFactory.getLog(DumpModelsAction.class);
|
||||||
|
|
||||||
|
private static final String N_QUADS_EXTENSION = ".nq";
|
||||||
|
private static final String N_QUADS_MIME_TYPE = "application/n-quads";
|
||||||
|
|
||||||
private final HttpServletResponse resp;
|
private final HttpServletResponse resp;
|
||||||
private final DumpFormat format;
|
|
||||||
private final WhichService which;
|
private final WhichService which;
|
||||||
private final String queryString;
|
private final String queryString;
|
||||||
|
|
||||||
|
@ -41,14 +41,12 @@ class DumpModelsAction extends AbstractDumpRestoreAction {
|
||||||
throws BadRequestException {
|
throws BadRequestException {
|
||||||
super(req);
|
super(req);
|
||||||
this.resp = resp;
|
this.resp = resp;
|
||||||
this.format = getEnumFromParameter(DumpFormat.class, PARAMETER_FORMAT);
|
|
||||||
this.which = getEnumFromParameter(WhichService.class, PARAMETER_WHICH);
|
this.which = getEnumFromParameter(WhichService.class, PARAMETER_WHICH);
|
||||||
this.queryString = req.getQueryString();
|
this.queryString = req.getQueryString();
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void redirectToFilename() throws IOException {
|
void redirectToFilename() throws IOException {
|
||||||
String filename = which + "." + format.getExtension();
|
String filename = which + N_QUADS_EXTENSION;
|
||||||
String urlPath = req.getContextPath() + req.getServletPath()
|
String urlPath = req.getContextPath() + req.getServletPath()
|
||||||
+ ACTION_DUMP;
|
+ ACTION_DUMP;
|
||||||
resp.sendRedirect(urlPath + "/" + filename + "?" + queryString);
|
resp.sendRedirect(urlPath + "/" + filename + "?" + queryString);
|
||||||
|
@ -59,16 +57,11 @@ class DumpModelsAction extends AbstractDumpRestoreAction {
|
||||||
RDFService rdfService = getRdfService(which);
|
RDFService rdfService = getRdfService(which);
|
||||||
String query = "SELECT * WHERE { GRAPH ?g {?s ?p ?o}}";
|
String query = "SELECT * WHERE { GRAPH ?g {?s ?p ?o}}";
|
||||||
|
|
||||||
resp.setContentType(format.getMimeType());
|
resp.setContentType(N_QUADS_MIME_TYPE);
|
||||||
if (format == DumpFormat.NQUADS) {
|
|
||||||
dumpNQuads(rdfService, query);
|
dumpNQuads(rdfService, query);
|
||||||
} else {
|
|
||||||
rdfService.sparqlSelectQuery(query,
|
|
||||||
format.getRdfServiceFormat(), resp.getOutputStream());
|
|
||||||
}
|
|
||||||
} catch (Throwable t) {
|
} catch (Throwable t) {
|
||||||
log.error("Failed to dump " + which + " models as " + format + ".",
|
log.error("Failed to dump " + which + " models as N-Quads.", t);
|
||||||
t);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -43,7 +43,6 @@ public class DumpRestoreController extends FreemarkerHttpServlet {
|
||||||
static final String ACTION_RESTORE = "/restore";
|
static final String ACTION_RESTORE = "/restore";
|
||||||
static final String ACTION_SELECT = "/select";
|
static final String ACTION_SELECT = "/select";
|
||||||
static final String PARAMETER_WHICH = "which";
|
static final String PARAMETER_WHICH = "which";
|
||||||
static final String PARAMETER_FORMAT = "format";
|
|
||||||
static final String PARAMETER_SOURCE_FILE = "sourceFile";
|
static final String PARAMETER_SOURCE_FILE = "sourceFile";
|
||||||
static final String PARAMETER_PURGE = "purge";
|
static final String PARAMETER_PURGE = "purge";
|
||||||
static final String ATTRIBUTE_TRIPLE_COUNT = "tripleCount";
|
static final String ATTRIBUTE_TRIPLE_COUNT = "tripleCount";
|
||||||
|
@ -128,53 +127,4 @@ public class DumpRestoreController extends FreemarkerHttpServlet {
|
||||||
super(message);
|
super(message);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* The formats that we will accept on a dump request.
|
|
||||||
*/
|
|
||||||
enum DumpFormat {
|
|
||||||
NQUADS("application/n-quads", "nq", null),
|
|
||||||
|
|
||||||
JSON("application/sparql-results+json", "srj", ResultFormat.JSON),
|
|
||||||
|
|
||||||
XML("application/sparql-results+xml", "srx", ResultFormat.XML);
|
|
||||||
|
|
||||||
private final String mimeType;
|
|
||||||
private final String extension;
|
|
||||||
private final ResultFormat rdfServiceFormat;
|
|
||||||
|
|
||||||
private DumpFormat(String mimeType, String extension,
|
|
||||||
ResultFormat rdfServiceFormat) {
|
|
||||||
this.mimeType = mimeType;
|
|
||||||
this.extension = extension;
|
|
||||||
this.rdfServiceFormat = rdfServiceFormat;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getMimeType() {
|
|
||||||
return mimeType;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getExtension() {
|
|
||||||
return extension;
|
|
||||||
}
|
|
||||||
|
|
||||||
public ResultFormat getRdfServiceFormat() {
|
|
||||||
return rdfServiceFormat;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The formats that we will accept on a restore request.
|
|
||||||
*/
|
|
||||||
enum RestoreFormat {
|
|
||||||
NQUADS {
|
|
||||||
@Override
|
|
||||||
public DumpParser getParser(InputStream is) throws IOException {
|
|
||||||
return new NquadsParser(is);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
public abstract DumpParser getParser(InputStream is) throws IOException;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,7 +2,6 @@
|
||||||
|
|
||||||
package edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore;
|
package edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore;
|
||||||
|
|
||||||
import static edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.PARAMETER_FORMAT;
|
|
||||||
import static edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.PARAMETER_PURGE;
|
import static edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.PARAMETER_PURGE;
|
||||||
import static edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.PARAMETER_SOURCE_FILE;
|
import static edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.PARAMETER_SOURCE_FILE;
|
||||||
import static edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.PARAMETER_WHICH;
|
import static edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.PARAMETER_WHICH;
|
||||||
|
@ -32,7 +31,6 @@ import com.hp.hpl.jena.rdf.model.Model;
|
||||||
|
|
||||||
import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
|
import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
|
||||||
import edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.BadRequestException;
|
import edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.BadRequestException;
|
||||||
import edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.RestoreFormat;
|
|
||||||
import edu.cornell.mannlib.vitro.webapp.dao.jena.RDFServiceDataset;
|
import edu.cornell.mannlib.vitro.webapp.dao.jena.RDFServiceDataset;
|
||||||
import edu.cornell.mannlib.vitro.webapp.modelaccess.ModelAccess.WhichService;
|
import edu.cornell.mannlib.vitro.webapp.modelaccess.ModelAccess.WhichService;
|
||||||
import edu.cornell.mannlib.vitro.webapp.rdfservice.ChangeSet;
|
import edu.cornell.mannlib.vitro.webapp.rdfservice.ChangeSet;
|
||||||
|
@ -59,7 +57,6 @@ public class RestoreModelsAction extends AbstractDumpRestoreAction {
|
||||||
private static final String DEFAULT_GRAPH_URI = "__default__";
|
private static final String DEFAULT_GRAPH_URI = "__default__";
|
||||||
|
|
||||||
private final FileItem sourceFile;
|
private final FileItem sourceFile;
|
||||||
private final RestoreFormat format;
|
|
||||||
private final WhichService which;
|
private final WhichService which;
|
||||||
private final boolean purge;
|
private final boolean purge;
|
||||||
private final SelfLimitingTripleBuckets bnodeBuckets;
|
private final SelfLimitingTripleBuckets bnodeBuckets;
|
||||||
|
@ -71,8 +68,6 @@ public class RestoreModelsAction extends AbstractDumpRestoreAction {
|
||||||
throws BadRequestException {
|
throws BadRequestException {
|
||||||
super(req);
|
super(req);
|
||||||
this.sourceFile = getFileItem(PARAMETER_SOURCE_FILE);
|
this.sourceFile = getFileItem(PARAMETER_SOURCE_FILE);
|
||||||
this.format = getEnumFromParameter(RestoreFormat.class,
|
|
||||||
PARAMETER_FORMAT);
|
|
||||||
this.which = getEnumFromParameter(WhichService.class, PARAMETER_WHICH);
|
this.which = getEnumFromParameter(WhichService.class, PARAMETER_WHICH);
|
||||||
this.purge = null != req.getParameter(PARAMETER_PURGE);
|
this.purge = null != req.getParameter(PARAMETER_PURGE);
|
||||||
|
|
||||||
|
@ -114,7 +109,7 @@ public class RestoreModelsAction extends AbstractDumpRestoreAction {
|
||||||
log.info("Restoring the " + which + " models.");
|
log.info("Restoring the " + which + " models.");
|
||||||
long lineCount = 0;
|
long lineCount = 0;
|
||||||
try (InputStream is = sourceFile.getInputStream();
|
try (InputStream is = sourceFile.getInputStream();
|
||||||
DumpParser p = format.getParser(is)) {
|
DumpParser p = new NquadsParser(is)) {
|
||||||
for (DumpQuad line : p) {
|
for (DumpQuad line : p) {
|
||||||
bucketize(line);
|
bucketize(line);
|
||||||
lineCount++;
|
lineCount++;
|
||||||
|
@ -158,9 +153,9 @@ public class RestoreModelsAction extends AbstractDumpRestoreAction {
|
||||||
ModelSerializationFormat.NTRIPLE, graphUri);
|
ModelSerializationFormat.NTRIPLE, graphUri);
|
||||||
|
|
||||||
rdfService.changeSetUpdate(change);
|
rdfService.changeSetUpdate(change);
|
||||||
|
|
||||||
tripleCount += triples.size();
|
tripleCount += triples.size();
|
||||||
log.info("processed " + tripleCount +" triples.");
|
log.info("processed " + tripleCount + " triples.");
|
||||||
}
|
}
|
||||||
|
|
||||||
private InputStream serialize(Collection<DumpTriple> triples)
|
private InputStream serialize(Collection<DumpTriple> triples)
|
||||||
|
|
|
@ -35,7 +35,6 @@ table.choices td {
|
||||||
<table class="choices">
|
<table class="choices">
|
||||||
<tr>
|
<tr>
|
||||||
<td>Select models</td>
|
<td>Select models</td>
|
||||||
<td>Select format</td>
|
|
||||||
<td> </td>
|
<td> </td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
|
@ -45,13 +44,6 @@ table.choices td {
|
||||||
<option value="CONTENT">Content models</option>
|
<option value="CONTENT">Content models</option>
|
||||||
</select>
|
</select>
|
||||||
</td>
|
</td>
|
||||||
<td>
|
|
||||||
<select name="format">
|
|
||||||
<option value="NQUADS">N-Quads</option>
|
|
||||||
<option value="JSON">RS-JSON</option>
|
|
||||||
<option value="XML">RS-XML</option>
|
|
||||||
</select>
|
|
||||||
</td>
|
|
||||||
<td>
|
<td>
|
||||||
<input type="submit" value="Dump" />
|
<input type="submit" value="Dump" />
|
||||||
</td>
|
</td>
|
||||||
|
@ -84,8 +76,7 @@ table.choices td {
|
||||||
<table class="choices">
|
<table class="choices">
|
||||||
<tr>
|
<tr>
|
||||||
<td>Select models</td>
|
<td>Select models</td>
|
||||||
<td>Select a file to restore from</td>
|
<td>Select a file to restore from (N-Quads format)</td>
|
||||||
<td>Select format</td>
|
|
||||||
<td> </td>
|
<td> </td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
|
@ -98,13 +89,6 @@ table.choices td {
|
||||||
<td>
|
<td>
|
||||||
<input type="file" name="sourceFile" size="60"/>
|
<input type="file" name="sourceFile" size="60"/>
|
||||||
</td>
|
</td>
|
||||||
<td>
|
|
||||||
<select name="format">
|
|
||||||
<option value="NQUADS">N-Quads</option>
|
|
||||||
<!-- <option value="JSON">RS-JSON</option> TODO -->
|
|
||||||
<!-- <option value="XML">RS-XML</option> TODO -->
|
|
||||||
</select>
|
|
||||||
</td>
|
|
||||||
<td>
|
<td>
|
||||||
<input type="submit" value="Restore" />
|
<input type="submit" value="Restore" />
|
||||||
</td>
|
</td>
|
||||||
|
|
Loading…
Add table
Reference in a new issue