litvinovg/vitro
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

VIVO-719 Add "bucketing" to optimize the restore.

Browse source
This commit is contained in:
Jim Blake 2014-06-06 17:05:51 -04:00
parent 4809fcb37e
commit aa1df06ac2
1 changed files with 34 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

44
webapp/src/edu/cornell/mannlib/vitro/webapp/controller/datatools/dumprestore/RestoreModelsAction.java
View file

@ -62,7 +62,8 @@ public class RestoreModelsAction extends AbstractDumpRestoreAction {
private final RestoreFormat format; private final RestoreFormat format;
private final WhichService which; private final WhichService which;
private final boolean purge; private final boolean purge;
private final TripleBuckets buckets = new TripleBuckets(); private final SelfLimitingTripleBuckets bnodeBuckets;
private final SelfLimitingTripleBuckets easyBuckets;
RestoreModelsAction(HttpServletRequest req, HttpServletResponse resp) RestoreModelsAction(HttpServletRequest req, HttpServletResponse resp)
throws BadRequestException { throws BadRequestException {
@ -72,7 +73,9 @@ public class RestoreModelsAction extends AbstractDumpRestoreAction {
PARAMETER_FORMAT); PARAMETER_FORMAT);
this.which = getEnumFromParameter(WhichService.class, PARAMETER_WHICH); this.which = getEnumFromParameter(WhichService.class, PARAMETER_WHICH);
this.purge = null != req.getParameter(PARAMETER_PURGE); this.purge = null != req.getParameter(PARAMETER_PURGE);
;
this.bnodeBuckets = new SelfLimitingTripleBuckets(this, 1000000);
this.easyBuckets = new SelfLimitingTripleBuckets(this, 10000);
} }
private FileItem getFileItem(String key) throws BadRequestException { private FileItem getFileItem(String key) throws BadRequestException {
@ -119,16 +122,18 @@ public class RestoreModelsAction extends AbstractDumpRestoreAction {
RDFServiceException { RDFServiceException {
DumpTriple triple = quad.getTriple(); DumpTriple triple = quad.getTriple();
if (triple.getS().isBlank() || triple.getO().isBlank()) { if (triple.getS().isBlank() || triple.getO().isBlank()) {
buckets.add(quad.getG().getValue(), triple); bnodeBuckets.add(quad.getG().getValue(), triple);
} else { } else {
processTriples(quad.getG().getValue(), easyBuckets.add(quad.getG().getValue(), triple);
Collections.singleton(triple));
} }
} }
private void emptyBuckets() throws IOException, RDFServiceException { private void emptyBuckets() throws IOException, RDFServiceException {
for (String key : buckets.getKeys()) { for (String key : easyBuckets.getKeys()) {
processTriples(key, buckets.getTriples(key)); processTriples(key, easyBuckets.getTriples(key));
}
for (String key : bnodeBuckets.getKeys()) {
processTriples(key, bnodeBuckets.getTriples(key));
} }
} }
@ -156,15 +161,28 @@ public class RestoreModelsAction extends AbstractDumpRestoreAction {
return new ByteArrayInputStream(out.toByteArray()); return new ByteArrayInputStream(out.toByteArray());
} }
private static class TripleBuckets { private static class SelfLimitingTripleBuckets {
private final RestoreModelsAction parent;
private final int sizeLimit;
private final Map<String, List<DumpTriple>> map = new HashMap<>(); private final Map<String, List<DumpTriple>> map = new HashMap<>();
public void add(String value, DumpTriple triple) { public SelfLimitingTripleBuckets(RestoreModelsAction parent,
String key = (value == null) ? DEFAULT_GRAPH_URI : value; int sizeLimit) {
this.parent = parent;
this.sizeLimit = sizeLimit;
}
public void add(String key, DumpTriple triple) throws IOException,
RDFServiceException {
key = nonNull(key, DEFAULT_GRAPH_URI);
if (!map.containsKey(key)) { if (!map.containsKey(key)) {
map.put(key, new ArrayList<DumpTriple>()); map.put(key, new ArrayList<DumpTriple>());
} }
map.get(key).add(triple); map.get(key).add(triple);
if (map.get(key).size() > sizeLimit) {
parent.processTriples(key, map.remove(key));
}
} }
public Set<String> getKeys() { public Set<String> getKeys() {
@ -172,11 +190,17 @@ public class RestoreModelsAction extends AbstractDumpRestoreAction {
} }
public List<DumpTriple> getTriples(String key) { public List<DumpTriple> getTriples(String key) {
key = nonNull(key, DEFAULT_GRAPH_URI);
if (map.containsKey(key)) { if (map.containsKey(key)) {
return map.get(key); return map.get(key);
} else { } else {
return Collections.emptyList(); return Collections.emptyList();
} }
} }
private String nonNull(String value, String defaultValue) {
return (value == null) ? defaultValue : value;
}
} }
} }