litvinovg/vitro
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

VIVO-719 Add some log statements so we can follow the progress.

Browse source
This commit is contained in:
Jim Blake 2014-06-06 22:02:44 -04:00
parent c05a3a857a
commit 21a3c242df
2 changed files with 21 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
webapp/src/edu/cornell/mannlib/vitro/webapp/controller/datatools/dumprestore/JsonToNquads.java
View file

@ -30,9 +30,11 @@ public class JsonToNquads extends OutputStream {
private final ByteArrayOutputStream header = new ByteArrayOutputStream();
private boolean headerIsComplete;
private long recordCount;
public JsonToNquads(OutputStream out) throws IOException {
this.writer = new OutputStreamWriter(out, "UTF-8");
log.info("Dump beginning.");
}
@Override
@ -43,6 +45,7 @@ public class JsonToNquads extends OutputStream {
@Override
public void close() throws IOException {
writer.close();
log.info("Dump is complete: " + recordCount + " records.");
log.debug("Left over in the buffer: '" + buffer + "'");
}
@ -112,6 +115,11 @@ public class JsonToNquads extends OutputStream {
writer.write(String.format("%s %s %s %s .\n", s.toNquad(),
p.toNquad(), o.toNquad(), g.toNquad()));
}
recordCount++;
if (recordCount % 10000 == 0) {
log.info("dumped " + recordCount + " records.");
}
} catch (Exception e) {
log.error("Failed to parse record: '" + text + "'", e);
throw new RuntimeException(e);

13
webapp/src/edu/cornell/mannlib/vitro/webapp/controller/datatools/dumprestore/RestoreModelsAction.java
View file

@ -65,6 +65,8 @@ public class RestoreModelsAction extends AbstractDumpRestoreAction {
private final SelfLimitingTripleBuckets bnodeBuckets;
private final SelfLimitingTripleBuckets easyBuckets;
private long tripleCount;
RestoreModelsAction(HttpServletRequest req, HttpServletResponse resp)
throws BadRequestException {
super(req);
@ -97,24 +99,32 @@ public class RestoreModelsAction extends AbstractDumpRestoreAction {
return;
}
log.info("Purging the " + which + " models.");
RDFService rdfService = getRdfService(which);
RDFServiceDataset dataset = new RDFServiceDataset(rdfService);
for (String graphUri : rdfService.getGraphURIs()) {
Model m = dataset.getNamedModel(graphUri);
log.info("Remove " + m.size() + " triples from " + graphUri);
m.removeAll();
}
log.info("Purge is complete.");
}
private long doTheRestore() throws IOException, RDFServiceException {
log.info("Restoring the " + which + " models.");
long lineCount = 0;
try (InputStream is = sourceFile.getInputStream();
DumpParser p = format.getParser(is)) {
for (DumpQuad line : p) {
bucketize(line);
lineCount++;
if (lineCount % 10000 == 0) {
log.info("read " + lineCount + " lines.");
}
}
emptyBuckets();
}
log.info("Restore is complete.");
return lineCount;
}
@ -148,6 +158,9 @@ public class RestoreModelsAction extends AbstractDumpRestoreAction {
ModelSerializationFormat.NTRIPLE, graphUri);
rdfService.changeSetUpdate(change);
tripleCount += triples.size();
log.info("processed " + tripleCount +" triples.");
}
private InputStream serialize(Collection<DumpTriple> triples)