VIVO-719 Create the first attempt at restore.
This commit is contained in:
Jim Blake
parent
a7915785f4
commit
4809fcb37e
12 changed files with 636 additions and 25 deletions
|
|
@ -2,6 +2,9 @@
|
|||
|
||||
package edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore;
|
||||
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import javax.json.JsonObject;
|
||||
import javax.json.JsonString;
|
||||
|
||||
|
|
@ -14,7 +17,7 @@ import org.apache.jena.riot.out.EscapeStr;
|
|||
public abstract class DumpNode {
|
||||
public static DumpNode fromJson(JsonObject json) throws BadNodeException {
|
||||
if (json == null) {
|
||||
return null;
|
||||
return new DumpNullNode();
|
||||
}
|
||||
|
||||
String type = getString(json, "type");
|
||||
|
|
@ -32,23 +35,106 @@ public abstract class DumpNode {
|
|||
}
|
||||
}
|
||||
|
||||
private static final String PATTERN_NQUAD_URI = "<(.+)>";
|
||||
private static final String PATTERN_NQUAD_LITERAL = "" //
|
||||
+ "\"(.*)\"" // quoted string
|
||||
+ "(@(.+))?" // optional language specifier
|
||||
+ "(\\^\\^<(.+)>)?" // optional datatype
|
||||
;
|
||||
private static final String PATTERN_NQUAD_BLANK = "_:(.+)";
|
||||
|
||||
public static DumpNode fromNquad(String text) throws BadNodeException {
|
||||
if (text == null) {
|
||||
return new DumpNullNode();
|
||||
}
|
||||
|
||||
text = text.trim();
|
||||
if (text.isEmpty()) {
|
||||
return new DumpNullNode();
|
||||
}
|
||||
|
||||
Matcher m1 = Pattern.compile(PATTERN_NQUAD_URI).matcher(text);
|
||||
Matcher m2 = Pattern.compile(PATTERN_NQUAD_LITERAL).matcher(text);
|
||||
Matcher m3 = Pattern.compile(PATTERN_NQUAD_BLANK).matcher(text);
|
||||
|
||||
if (m1.matches()) {
|
||||
return new DumpUriNode(unescape(m1.group(1)));
|
||||
} else if (m2.matches()) {
|
||||
return new DumpLiteralNode(unescape(m2.group(1)),
|
||||
unescape(m2.group(3)), unescape(m2.group(5)));
|
||||
} else if (m3.matches()) {
|
||||
return new DumpBlankNode(unescape(m3.group(1)));
|
||||
} else {
|
||||
throw new BadNodeException("Can't parse node: '" + text + "'");
|
||||
}
|
||||
}
|
||||
|
||||
private static String unescape(String s) {
|
||||
return (s == null) ? null : EscapeStr.unescapeStr(s);
|
||||
}
|
||||
|
||||
private static String getString(JsonObject json, String name) {
|
||||
JsonString jsString = json.getJsonString(name);
|
||||
return (jsString == null) ? null : json.getString(name);
|
||||
}
|
||||
|
||||
public abstract String getValue();
|
||||
|
||||
public abstract String toNquad();
|
||||
|
||||
public boolean isNull() {
|
||||
return false;
|
||||
}
|
||||
|
||||
public boolean isUri() {
|
||||
return false;
|
||||
}
|
||||
|
||||
public boolean isLiteral() {
|
||||
return false;
|
||||
}
|
||||
|
||||
public boolean isBlank() {
|
||||
return false;
|
||||
}
|
||||
|
||||
public static class DumpNullNode extends DumpNode {
|
||||
@Override
|
||||
public boolean isNull() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getValue() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toNquad() {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
public static class DumpUriNode extends DumpNode {
|
||||
private final String uri;
|
||||
|
||||
public DumpUriNode(String uri) throws BadNodeException {
|
||||
private DumpUriNode(String uri) throws BadNodeException {
|
||||
if (uri == null) {
|
||||
throw new BadNodeException("uri may not be null.");
|
||||
}
|
||||
this.uri = uri;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isUri() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getValue() {
|
||||
return uri;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toNquad() {
|
||||
return "<" + EscapeStr.stringEsc(uri) + ">";
|
||||
|
|
@ -60,7 +146,7 @@ public abstract class DumpNode {
|
|||
private final String language;
|
||||
private final String datatype;
|
||||
|
||||
public DumpLiteralNode(String value, String language, String datatype)
|
||||
private DumpLiteralNode(String value, String language, String datatype)
|
||||
throws BadNodeException {
|
||||
if (value == null) {
|
||||
throw new BadNodeException("value may not be null.");
|
||||
|
|
@ -74,6 +160,16 @@ public abstract class DumpNode {
|
|||
this.datatype = datatype;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isLiteral() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getValue() {
|
||||
return value;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toNquad() {
|
||||
String valueString = "\"" + EscapeStr.stringEsc(value) + "\"";
|
||||
|
|
@ -91,13 +187,23 @@ public abstract class DumpNode {
|
|||
public static class DumpBlankNode extends DumpNode {
|
||||
private final String label;
|
||||
|
||||
public DumpBlankNode(String label) throws BadNodeException {
|
||||
private DumpBlankNode(String label) throws BadNodeException {
|
||||
if (label == null) {
|
||||
throw new BadNodeException("label may not be null.");
|
||||
}
|
||||
this.label = label;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isBlank() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getValue() {
|
||||
return label;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toNquad() {
|
||||
return "_:" + label;
|
||||
|
|
@ -113,4 +219,5 @@ public abstract class DumpNode {
|
|||
super(message, cause);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,24 @@
|
|||
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
|
||||
|
||||
package edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* The interface for parsers that process dump/restore files.
|
||||
*/
|
||||
public interface DumpParser extends AutoCloseable, Iterable<DumpQuad> {
|
||||
@Override
|
||||
public void close() throws IOException;
|
||||
|
||||
public static class BadInputException extends RuntimeException {
|
||||
public BadInputException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public BadInputException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
|
||||
|
||||
package edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore;
|
||||
|
||||
/**
|
||||
* TODO
|
||||
*/
|
||||
public class DumpQuad {
|
||||
private final DumpTriple triple;
|
||||
private final DumpNode g;
|
||||
|
||||
public DumpQuad(DumpNode s, DumpNode p, DumpNode o, DumpNode g) {
|
||||
this.triple = new DumpTriple(s, p, o);
|
||||
this.g = g;
|
||||
}
|
||||
|
||||
public DumpTriple getTriple() {
|
||||
return triple;
|
||||
}
|
||||
|
||||
public DumpNode getG() {
|
||||
return g;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -3,6 +3,7 @@
|
|||
package edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
|
|
@ -21,6 +22,7 @@ import edu.cornell.mannlib.vitro.webapp.controller.freemarker.UrlBuilder;
|
|||
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.ResponseValues;
|
||||
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.TemplateResponseValues;
|
||||
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService.ResultFormat;
|
||||
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException;
|
||||
|
||||
/**
|
||||
* Allow the user to dump the knowledge base from either RDFService, or restore
|
||||
|
|
@ -43,6 +45,7 @@ public class DumpRestoreController extends FreemarkerHttpServlet {
|
|||
static final String PARAMETER_WHICH = "which";
|
||||
static final String PARAMETER_FORMAT = "format";
|
||||
static final String PARAMETER_SOURCE_FILE = "sourceFile";
|
||||
static final String PARAMETER_PURGE = "purge";
|
||||
static final String ATTRIBUTE_TRIPLE_COUNT = "tripleCount";
|
||||
|
||||
private static final String TEMPLATE_NAME = "datatools-dumpRestore.ftl";
|
||||
|
|
@ -53,7 +56,8 @@ public class DumpRestoreController extends FreemarkerHttpServlet {
|
|||
*/
|
||||
@Override
|
||||
public long maximumMultipartFileSize() {
|
||||
return 100L * 1024L * 1024L * 1024L; // allow really big uploads.
|
||||
long gigabyte = 1024L * 1024L * 1024L;
|
||||
return 100L * gigabyte; // permit really big uploads.
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -94,9 +98,8 @@ public class DumpRestoreController extends FreemarkerHttpServlet {
|
|||
} else {
|
||||
resp.sendError(HttpServletResponse.SC_METHOD_NOT_ALLOWED);
|
||||
}
|
||||
} catch (BadRequestException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
} catch (BadRequestException | RDFServiceException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -164,7 +167,14 @@ public class DumpRestoreController extends FreemarkerHttpServlet {
|
|||
* The formats that we will accept on a restore request.
|
||||
*/
|
||||
enum RestoreFormat {
|
||||
NQUADS
|
||||
NQUADS {
|
||||
@Override
|
||||
public DumpParser getParser(InputStream is) throws IOException {
|
||||
return new NquadsParser(is);
|
||||
}
|
||||
};
|
||||
|
||||
public abstract DumpParser getParser(InputStream is) throws IOException;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,35 @@
|
|||
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
|
||||
|
||||
package edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore;
|
||||
|
||||
/**
|
||||
* TODO
|
||||
*/
|
||||
public class DumpTriple {
|
||||
private final DumpNode s;
|
||||
private final DumpNode p;
|
||||
private final DumpNode o;
|
||||
|
||||
public DumpTriple(DumpNode s, DumpNode p, DumpNode o) {
|
||||
this.s = s;
|
||||
this.p = p;
|
||||
this.o = o;
|
||||
}
|
||||
|
||||
public DumpNode getS() {
|
||||
return s;
|
||||
}
|
||||
|
||||
public DumpNode getP() {
|
||||
return p;
|
||||
}
|
||||
|
||||
public DumpNode getO() {
|
||||
return o;
|
||||
}
|
||||
|
||||
public String toNtriples() {
|
||||
return String.format("%s %s %s .\n", s.toNquad(), p.toNquad(), o.toNquad());
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,117 @@
|
|||
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
|
||||
|
||||
package edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
|
||||
import edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpNode.BadNodeException;
|
||||
|
||||
/**
|
||||
* A utility class for NquadsParser. Breaks an NQuad line into 3 or 4 strings,
|
||||
* each holding the text representation of a node.
|
||||
*
|
||||
* It's a little tricky where literals are involved, because white space is
|
||||
* significant, except between quotes, and quotes are only quotes if they aren't
|
||||
* escaped.
|
||||
*/
|
||||
class NQuadLineSplitter {
|
||||
private static final Log log = LogFactory.getLog(NQuadLineSplitter.class);
|
||||
|
||||
private final String line;
|
||||
private final List<String> pieces = new ArrayList<>();
|
||||
private int i;
|
||||
|
||||
public NQuadLineSplitter(String line) throws BadNodeException {
|
||||
this.line = line;
|
||||
while (!atEnd()) {
|
||||
advancePastWhiteSpace();
|
||||
switch (line.charAt(i)) {
|
||||
case '#':
|
||||
advancePastComment();
|
||||
break;
|
||||
case '<':
|
||||
case '_':
|
||||
scanUnquotedString();
|
||||
break;
|
||||
case '"':
|
||||
scanLiteralString();
|
||||
break;
|
||||
case '.':
|
||||
assureEndOfLine();
|
||||
break;
|
||||
default:
|
||||
throw new BadNodeException(
|
||||
"found unexpected character at position " + i
|
||||
+ " of line '" + line + "'");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private boolean atEnd() {
|
||||
return i >= line.length();
|
||||
}
|
||||
|
||||
private boolean isWhiteSpace() {
|
||||
return " \t\r\n".indexOf(line.charAt(i)) >= 0;
|
||||
}
|
||||
|
||||
private void advancePastWhiteSpace() {
|
||||
while (!atEnd() && isWhiteSpace()) {
|
||||
i++;
|
||||
}
|
||||
}
|
||||
|
||||
private void advancePastComment() {
|
||||
i = line.length();
|
||||
}
|
||||
|
||||
private void scanUnquotedString() {
|
||||
int start = i;
|
||||
while (!atEnd() && !isWhiteSpace()) {
|
||||
i++;
|
||||
}
|
||||
pieces.add(line.substring(start, i));
|
||||
}
|
||||
|
||||
private void scanLiteralString() {
|
||||
int start = i;
|
||||
boolean inQuotes = false;
|
||||
while (!atEnd() && (inQuotes || !isWhiteSpace())) {
|
||||
if (isQuote()) {
|
||||
inQuotes = !inQuotes;
|
||||
log.debug("column " + i + ", inQuotes=" + inQuotes);
|
||||
}
|
||||
i++;
|
||||
}
|
||||
pieces.add(line.substring(start, i));
|
||||
}
|
||||
|
||||
private void assureEndOfLine() throws BadNodeException {
|
||||
i++;
|
||||
while (!atEnd() && isWhiteSpace()) {
|
||||
i++;
|
||||
}
|
||||
if (atEnd()) {
|
||||
return;
|
||||
} else if (line.charAt(i) == '#') {
|
||||
return;
|
||||
} else {
|
||||
throw new BadNodeException(
|
||||
"Period was not followed by end of line: '" + line + "'");
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isQuote() {
|
||||
boolean isEscaped = i > 0 && line.charAt(i - 1) == '\\';
|
||||
return (line.charAt(i) == '"') && (!isEscaped);
|
||||
}
|
||||
|
||||
public List<String> split() {
|
||||
return pieces;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,106 @@
|
|||
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
|
||||
|
||||
package edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
|
||||
import edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpNode.BadNodeException;
|
||||
|
||||
/**
|
||||
* TODO
|
||||
*/
|
||||
public class NquadsParser implements DumpParser {
|
||||
private final BufferedReader r;
|
||||
|
||||
public NquadsParser(InputStream is) throws IOException {
|
||||
this.r = new BufferedReader(new InputStreamReader(is, "UTF-8"));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
r.close();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Iterator<DumpQuad> iterator() {
|
||||
return new NQIterator();
|
||||
}
|
||||
|
||||
private class NQIterator implements Iterator<DumpQuad> {
|
||||
private DumpQuad next = null;
|
||||
|
||||
NQIterator() {
|
||||
lookAhead();
|
||||
}
|
||||
|
||||
private void lookAhead() {
|
||||
next = null;
|
||||
String line = null;
|
||||
try {
|
||||
while (true) {
|
||||
line = r.readLine();
|
||||
if (line == null) {
|
||||
return;
|
||||
}
|
||||
if (!line.trim().startsWith("#")) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch (IOException e) {
|
||||
return;
|
||||
}
|
||||
next = parseLine(line);
|
||||
}
|
||||
|
||||
private DumpQuad parseLine(String line) {
|
||||
try {
|
||||
List<String> strings = parseNodeStrings(line);
|
||||
int stringCount = strings.size();
|
||||
if (stringCount != 3 && stringCount != 4) {
|
||||
throw new BadInputException("Input line is invalid: has "
|
||||
+ stringCount + " groups: '" + line + "' ==> "
|
||||
+ strings);
|
||||
}
|
||||
|
||||
DumpNode s = DumpNode.fromNquad(strings.get(0));
|
||||
DumpNode p = DumpNode.fromNquad(strings.get(1));
|
||||
DumpNode o = DumpNode.fromNquad(strings.get(2));
|
||||
DumpNode g = DumpNode.fromNquad((stringCount == 4) ? strings
|
||||
.get(3) : null);
|
||||
return new DumpQuad(s, p, o, g);
|
||||
} catch (BadNodeException e) {
|
||||
throw new BadInputException("unable to parse node, line='"
|
||||
+ line + "'", e);
|
||||
}
|
||||
}
|
||||
|
||||
private List<String> parseNodeStrings(String line)
|
||||
throws BadNodeException {
|
||||
return new NQuadLineSplitter(line).split();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
return next != null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public DumpQuad next() {
|
||||
DumpQuad dq = next;
|
||||
lookAhead();
|
||||
return dq;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void remove() {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -3,31 +3,66 @@
|
|||
package edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore;
|
||||
|
||||
import static edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.PARAMETER_FORMAT;
|
||||
import static edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.PARAMETER_PURGE;
|
||||
import static edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.PARAMETER_SOURCE_FILE;
|
||||
import static edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.PARAMETER_WHICH;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.io.Reader;
|
||||
import java.io.OutputStreamWriter;
|
||||
import java.io.Writer;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import org.apache.commons.fileupload.FileItem;
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
|
||||
import com.hp.hpl.jena.rdf.model.Model;
|
||||
|
||||
import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
|
||||
import edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.BadRequestException;
|
||||
import edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpRestoreController.RestoreFormat;
|
||||
import edu.cornell.mannlib.vitro.webapp.dao.jena.RDFServiceDataset;
|
||||
import edu.cornell.mannlib.vitro.webapp.rdfservice.ChangeSet;
|
||||
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
|
||||
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService.ModelSerializationFormat;
|
||||
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException;
|
||||
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils.WhichService;
|
||||
|
||||
/**
|
||||
* TODO
|
||||
* In progress.
|
||||
* Load from a dump file of NQuads, or something equivalent.
|
||||
*
|
||||
* We could process it all a line at a time, except for the blank nodes. If
|
||||
* there are two references to the same blank node, they must be processed in
|
||||
* the same method call to the RDFService.
|
||||
*
|
||||
* So, process each line as it comes in, unless it contains a blank node. Lines
|
||||
* with blank nodes get put into buckets, with one bucket for each model (and
|
||||
* one for the default model). At the end, we'll empty each of the buckets.
|
||||
*
|
||||
* And if they ask to purge the models before restoring, do that.
|
||||
*/
|
||||
public class RestoreModelsAction extends AbstractDumpRestoreAction {
|
||||
private static final Log log = LogFactory.getLog(RestoreModelsAction.class);
|
||||
|
||||
private static final String DEFAULT_GRAPH_URI = "__default__";
|
||||
|
||||
private final FileItem sourceFile;
|
||||
private final RestoreFormat format;
|
||||
private final WhichService which;
|
||||
private final boolean purge;
|
||||
private final TripleBuckets buckets = new TripleBuckets();
|
||||
|
||||
RestoreModelsAction(HttpServletRequest req, HttpServletResponse resp)
|
||||
throws BadRequestException {
|
||||
|
|
@ -35,6 +70,9 @@ public class RestoreModelsAction extends AbstractDumpRestoreAction {
|
|||
this.sourceFile = getFileItem(PARAMETER_SOURCE_FILE);
|
||||
this.format = getEnumFromParameter(RestoreFormat.class,
|
||||
PARAMETER_FORMAT);
|
||||
this.which = getEnumFromParameter(WhichService.class, PARAMETER_WHICH);
|
||||
this.purge = null != req.getParameter(PARAMETER_PURGE);
|
||||
;
|
||||
}
|
||||
|
||||
private FileItem getFileItem(String key) throws BadRequestException {
|
||||
|
|
@ -46,22 +84,99 @@ public class RestoreModelsAction extends AbstractDumpRestoreAction {
|
|||
return fileItem;
|
||||
}
|
||||
|
||||
long restoreModels() throws IOException {
|
||||
long restoreModels() throws IOException, RDFServiceException {
|
||||
purgeIfRequested();
|
||||
return doTheRestore();
|
||||
}
|
||||
|
||||
private void purgeIfRequested() throws RDFServiceException {
|
||||
if (!purge) {
|
||||
return;
|
||||
}
|
||||
|
||||
RDFService rdfService = getRdfService(which);
|
||||
RDFServiceDataset dataset = new RDFServiceDataset(rdfService);
|
||||
for (String graphUri : rdfService.getGraphURIs()) {
|
||||
Model m = dataset.getNamedModel(graphUri);
|
||||
m.removeAll();
|
||||
}
|
||||
}
|
||||
|
||||
private long doTheRestore() throws IOException, RDFServiceException {
|
||||
long lineCount = 0;
|
||||
try (InputStream is = sourceFile.getInputStream();
|
||||
Reader isr = new InputStreamReader(is, "UTF-8");
|
||||
BufferedReader br = new BufferedReader(isr)) {
|
||||
String line;
|
||||
while (null != (line = br.readLine())) {
|
||||
processLine(line);
|
||||
DumpParser p = format.getParser(is)) {
|
||||
for (DumpQuad line : p) {
|
||||
bucketize(line);
|
||||
lineCount++;
|
||||
}
|
||||
emptyBuckets();
|
||||
}
|
||||
return lineCount;
|
||||
}
|
||||
|
||||
private void processLine(String line) {
|
||||
System.out.println("TOTALLY BOGUS RESTORE");
|
||||
private void bucketize(DumpQuad quad) throws IOException,
|
||||
RDFServiceException {
|
||||
DumpTriple triple = quad.getTriple();
|
||||
if (triple.getS().isBlank() || triple.getO().isBlank()) {
|
||||
buckets.add(quad.getG().getValue(), triple);
|
||||
} else {
|
||||
processTriples(quad.getG().getValue(),
|
||||
Collections.singleton(triple));
|
||||
}
|
||||
}
|
||||
|
||||
private void emptyBuckets() throws IOException, RDFServiceException {
|
||||
for (String key : buckets.getKeys()) {
|
||||
processTriples(key, buckets.getTriples(key));
|
||||
}
|
||||
}
|
||||
|
||||
private void processTriples(String graphUri, Collection<DumpTriple> triples)
|
||||
throws IOException, RDFServiceException {
|
||||
if (graphUri.equals(DEFAULT_GRAPH_URI)) {
|
||||
graphUri = null;
|
||||
}
|
||||
RDFService rdfService = getRdfService(which);
|
||||
ChangeSet change = rdfService.manufactureChangeSet();
|
||||
change.addAddition(serialize(triples),
|
||||
ModelSerializationFormat.NTRIPLE, graphUri);
|
||||
|
||||
rdfService.changeSetUpdate(change);
|
||||
}
|
||||
|
||||
private InputStream serialize(Collection<DumpTriple> triples)
|
||||
throws IOException {
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
Writer w = new OutputStreamWriter(out, "UTF-8");
|
||||
for (DumpTriple triple : triples) {
|
||||
w.write(triple.toNtriples());
|
||||
}
|
||||
w.close();
|
||||
return new ByteArrayInputStream(out.toByteArray());
|
||||
}
|
||||
|
||||
private static class TripleBuckets {
|
||||
private final Map<String, List<DumpTriple>> map = new HashMap<>();
|
||||
|
||||
public void add(String value, DumpTriple triple) {
|
||||
String key = (value == null) ? DEFAULT_GRAPH_URI : value;
|
||||
if (!map.containsKey(key)) {
|
||||
map.put(key, new ArrayList<DumpTriple>());
|
||||
}
|
||||
map.get(key).add(triple);
|
||||
}
|
||||
|
||||
public Set<String> getKeys() {
|
||||
return map.keySet();
|
||||
}
|
||||
|
||||
public List<DumpTriple> getTriples(String key) {
|
||||
if (map.containsKey(key)) {
|
||||
return map.get(key);
|
||||
} else {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,8 +29,8 @@ import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException;
|
|||
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils;
|
||||
|
||||
public class RDFServiceGraphBulkUpdater implements BulkUpdateHandler {
|
||||
private static final Log log = LogFactory.getLog(RDFServiceGraphBulkUpdater.class);
|
||||
|
||||
private static final Log log = LogFactory.getLog(RDFServiceGraphBulkUpdater.class);
|
||||
private final RDFServiceGraph graph;
|
||||
private final GraphEventManager manager;
|
||||
|
||||
|
|
@ -190,7 +190,8 @@ public class RDFServiceGraphBulkUpdater implements BulkUpdateHandler {
|
|||
}
|
||||
|
||||
private static void removeAll(Graph g, Node s, Node p, Node o)
|
||||
{
|
||||
{
|
||||
log.debug("removeAll: g=" + g + ", s=" + s + ", p=" + p + ", o=" + o);
|
||||
if (!(g instanceof RDFServiceGraph)) {
|
||||
removeAllTripleByTriple(g, s, p, o);
|
||||
return;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,62 @@
|
|||
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
|
||||
|
||||
package edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore;
|
||||
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.log4j.Level;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
|
||||
import edu.cornell.mannlib.vitro.testing.AbstractTestClass;
|
||||
import edu.cornell.mannlib.vitro.webapp.controller.datatools.dumprestore.DumpNode.BadNodeException;
|
||||
|
||||
/**
|
||||
* TODO
|
||||
*/
|
||||
public class NQuadLineSplitterTest extends AbstractTestClass {
|
||||
private static List<String> testData;
|
||||
|
||||
@BeforeClass
|
||||
public static void readTestData() throws IOException {
|
||||
InputStream stream = NQuadLineSplitterTest.class
|
||||
.getResourceAsStream("NQuadLineSplitterTest.nq");
|
||||
testData = IOUtils.readLines(stream);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void splitLine1() throws BadNodeException {
|
||||
List<String> strings = new NQuadLineSplitter(getLine(1)).split();
|
||||
assertEquals("count", 4, strings.size());
|
||||
assertEquals("subject", "<http://purl.org/ontology/bibo/degree>",
|
||||
strings.get(0));
|
||||
assertEquals("predicate",
|
||||
"<http://purl.obolibrary.org/obo/IAO_0000112>", strings.get(1));
|
||||
assertEquals(
|
||||
"object",
|
||||
"\"The source of the public description and this info is found here: "
|
||||
+ "http://bibotools.googlecode.com/svn/bibo-ontology/trunk/doc/index.html. "
|
||||
+ "Bibo considers this term \\\"unstable\\\". "
|
||||
+ "The bibo editorial note is: \\\"We are not defining, using an enumeration, "
|
||||
+ "the range of the bibo:degree to the defined list of bibo:ThesisDegree. "
|
||||
+ "We won't do it because we want people to be able to define new degress "
|
||||
+ "if needed by some special usecases. "
|
||||
+ "Creating such an enumeration would restrict this to "
|
||||
+ "happen.\\\"\"^^<http://www.w3.org/2001/XMLSchema#string>",
|
||||
strings.get(2));
|
||||
assertEquals(
|
||||
"graph",
|
||||
"<http://vitro.mannlib.cornell.edu/filegraph/tbox/object-properties.owl>",
|
||||
strings.get(3));
|
||||
}
|
||||
|
||||
private String getLine(int i) {
|
||||
return testData.get(i - 1);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1 @@
|
|||
<http://purl.org/ontology/bibo/degree> <http://purl.obolibrary.org/obo/IAO_0000112> "The source of the public description and this info is found here: http://bibotools.googlecode.com/svn/bibo-ontology/trunk/doc/index.html. Bibo considers this term \"unstable\". The bibo editorial note is: \"We are not defining, using an enumeration, the range of the bibo:degree to the defined list of bibo:ThesisDegree. We won't do it because we want people to be able to define new degress if needed by some special usecases. Creating such an enumeration would restrict this to happen.\""^^<http://www.w3.org/2001/XMLSchema#string> <http://vitro.mannlib.cornell.edu/filegraph/tbox/object-properties.owl> .
|
||||
|
|
@ -69,7 +69,7 @@
|
|||
</section>
|
||||
</#if>
|
||||
|
||||
<form action="${restoreUrl}" method="post">
|
||||
<form action="${restoreUrl}" enctype="multipart/form-data" method="post">
|
||||
<table>
|
||||
<tr>
|
||||
<td>Select models</td>
|
||||
|
|
@ -98,5 +98,13 @@
|
|||
<input type="submit" value="Restore" />
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td colspan="4">
|
||||
<label>
|
||||
<input type="checkbox" value="purge" name="purge" />
|
||||
Purge the models before restoring.
|
||||
</label>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</form>
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue