litvinovg/vitro
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

VIVO-719 Add some log statements so we can follow the progress.

Browse source
This commit is contained in:
Jim Blake 2014-06-06 22:02:44 -04:00
parent c05a3a857a
commit 21a3c242df
2 changed files with 21 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
webapp/src/edu/cornell/mannlib/vitro/webapp/controller/datatools/dumprestore/JsonToNquads.java
View file

@ -30,9 +30,11 @@ public class JsonToNquads extends OutputStream {
private final ByteArrayOutputStream header = new ByteArrayOutputStream(); private final ByteArrayOutputStream header = new ByteArrayOutputStream();
private boolean headerIsComplete; private boolean headerIsComplete;
private long recordCount;
public JsonToNquads(OutputStream out) throws IOException { public JsonToNquads(OutputStream out) throws IOException {
this.writer = new OutputStreamWriter(out, "UTF-8"); this.writer = new OutputStreamWriter(out, "UTF-8");
log.info("Dump beginning.");
} }
@Override @Override
@ -43,6 +45,7 @@ public class JsonToNquads extends OutputStream {
@Override @Override
public void close() throws IOException { public void close() throws IOException {
writer.close(); writer.close();
log.info("Dump is complete: " + recordCount + " records.");
log.debug("Left over in the buffer: '" + buffer + "'"); log.debug("Left over in the buffer: '" + buffer + "'");
} }
@ -112,6 +115,11 @@ public class JsonToNquads extends OutputStream {
writer.write(String.format("%s %s %s %s .\n", s.toNquad(), writer.write(String.format("%s %s %s %s .\n", s.toNquad(),
p.toNquad(), o.toNquad(), g.toNquad())); p.toNquad(), o.toNquad(), g.toNquad()));
} }
recordCount++;
if (recordCount % 10000 == 0) {
log.info("dumped " + recordCount + " records.");
}
} catch (Exception e) { } catch (Exception e) {
log.error("Failed to parse record: '" + text + "'", e); log.error("Failed to parse record: '" + text + "'", e);
throw new RuntimeException(e); throw new RuntimeException(e);

13
webapp/src/edu/cornell/mannlib/vitro/webapp/controller/datatools/dumprestore/RestoreModelsAction.java
View file

@ -65,6 +65,8 @@ public class RestoreModelsAction extends AbstractDumpRestoreAction {
private final SelfLimitingTripleBuckets bnodeBuckets; private final SelfLimitingTripleBuckets bnodeBuckets;
private final SelfLimitingTripleBuckets easyBuckets; private final SelfLimitingTripleBuckets easyBuckets;
private long tripleCount;
RestoreModelsAction(HttpServletRequest req, HttpServletResponse resp) RestoreModelsAction(HttpServletRequest req, HttpServletResponse resp)
throws BadRequestException { throws BadRequestException {
super(req); super(req);
@ -97,24 +99,32 @@ public class RestoreModelsAction extends AbstractDumpRestoreAction {
return; return;
} }
log.info("Purging the " + which + " models.");
RDFService rdfService = getRdfService(which); RDFService rdfService = getRdfService(which);
RDFServiceDataset dataset = new RDFServiceDataset(rdfService); RDFServiceDataset dataset = new RDFServiceDataset(rdfService);
for (String graphUri : rdfService.getGraphURIs()) { for (String graphUri : rdfService.getGraphURIs()) {
Model m = dataset.getNamedModel(graphUri); Model m = dataset.getNamedModel(graphUri);
log.info("Remove " + m.size() + " triples from " + graphUri);
m.removeAll(); m.removeAll();
} }
log.info("Purge is complete.");
} }
private long doTheRestore() throws IOException, RDFServiceException { private long doTheRestore() throws IOException, RDFServiceException {
log.info("Restoring the " + which + " models.");
long lineCount = 0; long lineCount = 0;
try (InputStream is = sourceFile.getInputStream(); try (InputStream is = sourceFile.getInputStream();
DumpParser p = format.getParser(is)) { DumpParser p = format.getParser(is)) {
for (DumpQuad line : p) { for (DumpQuad line : p) {
bucketize(line); bucketize(line);
lineCount++; lineCount++;
if (lineCount % 10000 == 0) {
log.info("read " + lineCount + " lines.");
}
} }
emptyBuckets(); emptyBuckets();
} }
log.info("Restore is complete.");
return lineCount; return lineCount;
} }
@ -148,6 +158,9 @@ public class RestoreModelsAction extends AbstractDumpRestoreAction {
ModelSerializationFormat.NTRIPLE, graphUri); ModelSerializationFormat.NTRIPLE, graphUri);
rdfService.changeSetUpdate(change); rdfService.changeSetUpdate(change);
tripleCount += triples.size();
log.info("processed " + tripleCount +" triples.");
} }
private InputStream serialize(Collection<DumpTriple> triples) private InputStream serialize(Collection<DumpTriple> triples)