litvinovg/vitro
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Use limited diff when checking for local overrides of firsttime data. Add unit test for firsttime update behavior.

Browse source
This commit is contained in:
Brian Lowe 2022-05-26 20:22:07 +03:00
parent ebc9237cc6
commit f1ef775d95
4 changed files with 219 additions and 150 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

90
api/src/main/java/edu/cornell/mannlib/vitro/webapp/servlet/setup/ConfigurationModelsSetup.java
View file

@ -98,7 +98,7 @@ public class ConfigurationModelsSetup implements ServletContextListener {
OntModel baseModelFirsttime = VitroModelFactory.createOntologyModel(); OntModel baseModelFirsttime = VitroModelFactory.createOntologyModel();
RDFFilesLoader.loadFirstTimeFiles(ctx, modelPath, baseModelFirsttime, true); RDFFilesLoader.loadFirstTimeFiles(ctx, modelPath, baseModelFirsttime, true);
if (baseModelFirsttime.isIsomorphicWith(baseModelFirsttimeBackup)) { if (RDFFilesLoader.areIsomporphic(baseModelFirsttime, baseModelFirsttimeBackup)) {
log.debug("They are the same, so do nothing: '" + modelPath + "'"); log.debug("They are the same, so do nothing: '" + modelPath + "'");
} else { } else {
log.debug("They differ:" + modelPath + ", compare values in configuration models with user's triplestore"); log.debug("They differ:" + modelPath + ", compare values in configuration models with user's triplestore");
@ -132,9 +132,8 @@ public class ConfigurationModelsSetup implements ServletContextListener {
// remove special cases for display, problem with blank nodes // remove special cases for display, problem with blank nodes
if (modelIdString.equals("display")) { if (modelIdString.equals("display")) {
RDFFilesLoader.removeBlankTriples(difOldNew);
removeBlankTriples(difOldNew); RDFFilesLoader.removeBlankTriples(difNewOld);
removeBlankTriples(difNewOld);
} }
if (difOldNew.isEmpty() && difNewOld.isEmpty()) { if (difOldNew.isEmpty() && difNewOld.isEmpty()) {
@ -149,7 +148,7 @@ public class ConfigurationModelsSetup implements ServletContextListener {
log.debug("Difference for " + modelIdString + " (old -> new), these triples should be removed: " + out); log.debug("Difference for " + modelIdString + " (old -> new), these triples should be removed: " + out);
// Check if the UI-changes Overlap with the changes made in the fristtime-files // Check if the UI-changes Overlap with the changes made in the fristtime-files
checkUiChangesOverlapWithFileChanges(baseModel, userModel, difOldNew); RDFFilesLoader.checkUiChangesOverlapWithFileChanges(baseModel, userModel, difOldNew);
// before we remove the triples, we need to compare values in back up firsttime with user's triplestore // before we remove the triples, we need to compare values in back up firsttime with user's triplestore
// if the triples which should be removed are still in user´s triplestore, remove them // if the triples which should be removed are still in user´s triplestore, remove them
@ -167,7 +166,7 @@ public class ConfigurationModelsSetup implements ServletContextListener {
log.debug("Difference for " + modelIdString + " (new -> old), these triples should be added: " + out2); log.debug("Difference for " + modelIdString + " (new -> old), these triples should be added: " + out2);
// Check if the UI-changes Overlap with the changes made in the fristtime-files // Check if the UI-changes Overlap with the changes made in the fristtime-files
checkUiChangesOverlapWithFileChanges(baseModel, userModel, difNewOld); RDFFilesLoader.checkUiChangesOverlapWithFileChanges(baseModel, userModel, difNewOld);
// before we add the triples, we need to compare values in back up firsttime with user's triplestore // before we add the triples, we need to compare values in back up firsttime with user's triplestore
// if the triples which should be added are not already in user´s triplestore, add them // if the triples which should be added are not already in user´s triplestore, add them
@ -186,85 +185,6 @@ public class ConfigurationModelsSetup implements ServletContextListener {
return updatedFiles; return updatedFiles;
} }
/**
* Check if the UI-changes Overlap with the changes made in the fristtime-files, if they overlap these changes are not applied to the user-model (UI)
*
* @param baseModel firsttime backup model
* @param userModel current state in the system (user/UI-model)
* @param changesModel the changes between firsttime-files and firttime-backup
*/
private void checkUiChangesOverlapWithFileChanges(Model baseModel, Model userModel, Model changesModel) {
log.debug("Beginn check if subtractions from Backup-firsttime model to current state of firsttime-files were changed in user-model (via UI)");
Model changesUserModel = userModel.difference(baseModel);
List<Statement> changedInUIandFileStatements = new ArrayList<Statement>();
if(!changesUserModel.isEmpty())
{
removeBlankTriples(changesUserModel);
StringWriter out3 = new StringWriter();
changesUserModel.write(out3, "TTL");
log.debug("There were changes in the user-model via UI which have also changed in the firsttime files, the following triples will not be updated");
// iterate all statements and check if the ones which should be removed were not changed via the UI
StmtIterator iter = changesUserModel.listStatements();
while (iter.hasNext()) {
Statement stmt = iter.nextStatement(); // get next statement
Resource subject = stmt.getSubject(); // get the subject
Property predicate = stmt.getPredicate(); // get the predicate
RDFNode object = stmt.getObject(); // get the object
StmtIterator iter2 = changesModel.listStatements();
while (iter2.hasNext()) {
Statement stmt2 = iter2.nextStatement(); // get next statement
Resource subject2 = stmt2.getSubject(); // get the subject
Property predicate2 = stmt2.getPredicate(); // get the predicate
RDFNode object2 = stmt2.getObject(); // get the object
// if subject and predicate are equal but the object differs and the language tag is the same, do not update these triples
// this case indicates an change in the UI, which should not be overwriten from the firsttime files
if(subject.equals(subject2) && predicate.equals(predicate2) && !object.equals(object2) ) {
// if object is an literal, check the language tag
if (object.isLiteral() && object2.isLiteral()) {
// if the langauge tag is the same, remove this triple from the update list
if(object.asLiteral().getLanguage().equals(object2.asLiteral().getLanguage())) {
log.debug("This two triples changed UI and files: \n UI: " + stmt + " \n file: " +stmt2);
changedInUIandFileStatements.add(stmt2);
}
} else {
log.debug("This two triples changed UI and files: \n UI: " + stmt + " \n file: " +stmt2);
changedInUIandFileStatements.add(stmt2);
}
}
}
}
// remove triples which were changed in the user model (UI) from the list
changesModel.remove(changedInUIandFileStatements);
} else {
log.debug("There were no changes in the user-model via UI compared to the backup-firsttime-model");
}
}
/**
* Remove all triples where subject or object is blank (Anon)
*/
private void removeBlankTriples(Model model) {
StmtIterator iter = model.listStatements();
List<Statement> removeStatement = new ArrayList<Statement>();
while (iter.hasNext()) {
Statement stmt = iter.nextStatement(); // get next statement
Resource subject = stmt.getSubject(); // get the subject
RDFNode object = stmt.getObject(); // get the object
if(subject.isAnon() || object.isAnon())
{
removeStatement.add(stmt);
}
}
model.remove(removeStatement);
}
@Override @Override
public void contextDestroyed(ServletContextEvent sce) { public void contextDestroyed(ServletContextEvent sce) {
// Nothing to tear down. // Nothing to tear down.

68
api/src/main/java/edu/cornell/mannlib/vitro/webapp/servlet/setup/ContentModelSetup.java
View file

@ -21,6 +21,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.jena.ontology.OntModel; import org.apache.jena.ontology.OntModel;
import org.apache.jena.rdf.model.Model; import org.apache.jena.rdf.model.Model;
import org.apache.jena.rdf.model.ModelFactory;
import org.apache.jena.rdf.model.Property; import org.apache.jena.rdf.model.Property;
import org.apache.jena.rdf.model.ResIterator; import org.apache.jena.rdf.model.ResIterator;
import org.apache.jena.rdf.model.Resource; import org.apache.jena.rdf.model.Resource;
@ -249,7 +250,7 @@ public class ContentModelSetup extends JenaDataSourceSetupBase
setPortalUriOnFirstTime(firsttimeFilesModel, ctx); setPortalUriOnFirstTime(firsttimeFilesModel, ctx);
} }
if ( firsttimeBackupModel.isIsomorphicWith(firsttimeFilesModel) ) { if ( RDFFilesLoader.areIsomporphic(firsttimeBackupModel, firsttimeFilesModel) ) {
log.debug("They are the same, so do nothing: '" + modelPath + "'"); log.debug("They are the same, so do nothing: '" + modelPath + "'");
} else { } else {
log.debug("They differ: '" + modelPath + "', compare values in configuration models with user's triplestore"); log.debug("They differ: '" + modelPath + "', compare values in configuration models with user's triplestore");
@ -301,7 +302,7 @@ public class ContentModelSetup extends JenaDataSourceSetupBase
log.debug("Difference for " + modelIdString + " (old -> new), these triples should be removed: " + out); log.debug("Difference for " + modelIdString + " (old -> new), these triples should be removed: " + out);
// Check if the UI-changes Overlap with the changes made in the fristtime-files // Check if the UI-changes Overlap with the changes made in the fristtime-files
checkUiChangesOverlapWithFileChanges(baseModel, userModel, difOldNew); RDFFilesLoader.checkUiChangesOverlapWithFileChanges(baseModel, userModel, difOldNew);
// before we remove the triples, we need to compare values in back up firsttime with user's triplestore // before we remove the triples, we need to compare values in back up firsttime with user's triplestore
// if the triples which should be removed are still in user´s triplestore, remove them // if the triples which should be removed are still in user´s triplestore, remove them
@ -320,7 +321,7 @@ public class ContentModelSetup extends JenaDataSourceSetupBase
log.debug("Difference for " + modelIdString + " (new -> old), these triples should be added: " + out2); log.debug("Difference for " + modelIdString + " (new -> old), these triples should be added: " + out2);
// Check if the UI-changes Overlap with the changes made in the fristtime-files // Check if the UI-changes Overlap with the changes made in the fristtime-files
checkUiChangesOverlapWithFileChanges(baseModel, userModel, difNewOld); RDFFilesLoader.checkUiChangesOverlapWithFileChanges(baseModel, userModel, difNewOld);
// before we add the triples, we need to compare values in back up firsttime with user's triplestore // before we add the triples, we need to compare values in back up firsttime with user's triplestore
// if the triples which should be added are not already in user´s triplestore, add them // if the triples which should be added are not already in user´s triplestore, add them
@ -339,67 +340,6 @@ public class ContentModelSetup extends JenaDataSourceSetupBase
return updatedFiles; return updatedFiles;
} }
/**
* Check if the UI-changes Overlap with the changes made in the fristtime-files, if they overlap these changes are not applied to the user-model (UI)
*
* @param baseModel firsttime backup model
* @param userModel current state in the system (user/UI-model)
* @param changesModel the changes between firsttime-files and firttime-backup
*/
private void checkUiChangesOverlapWithFileChanges(Model baseModel, Model userModel, Model changesModel) {
log.debug("Beginn check if subtractions from Backup-firsttime model to current state of firsttime-files were changed in user-model (via UI)");
Model changesUserModel = userModel.difference(baseModel);
List<Statement> changedInUIandFileStatements = new ArrayList<Statement>();
if(!changesUserModel.isEmpty())
{
StringWriter out3 = new StringWriter();
changesUserModel.write(out3, "TTL");
log.debug("There were changes in the user-model via UI which have also changed in the firsttime files, the following triples will not be updated");
// iterate all statements and check if the ones which should be removed were not changed via the UI
StmtIterator iter = changesUserModel.listStatements();
while (iter.hasNext()) {
Statement stmt = iter.nextStatement(); // get next statement
Resource subject = stmt.getSubject(); // get the subject
Property predicate = stmt.getPredicate(); // get the predicate
RDFNode object = stmt.getObject(); // get the object
StmtIterator iter2 = changesModel.listStatements();
while (iter2.hasNext()) {
Statement stmt2 = iter2.nextStatement(); // get next statement
Resource subject2 = stmt2.getSubject(); // get the subject
Property predicate2 = stmt2.getPredicate(); // get the predicate
RDFNode object2 = stmt2.getObject(); // get the object
// if subject and predicate are equal but the object differs and the language tag is the same, do not update these triples
// this case indicates an change in the UI, which should not be overwriten from the firsttime files
if(subject.equals(subject2) && predicate.equals(predicate2) && !object.equals(object2) ) {
// if object is an literal, check the language tag
if (object.isLiteral() && object2.isLiteral()) {
// if the langauge tag is the same, remove this triple from the update list
if(object.asLiteral().getLanguage().equals(object2.asLiteral().getLanguage())) {
log.debug("This two triples changed UI and files: \n UI: " + stmt + " \n file: " +stmt2);
changedInUIandFileStatements.add(stmt2);
}
} else {
log.debug("This two triples changed UI and files: \n UI: " + stmt + " \n file: " +stmt2);
changedInUIandFileStatements.add(stmt2);
}
}
}
}
// remove triples which were changed in the user model (UI) from the list
changesModel.remove(changedInUIandFileStatements);
} else {
log.debug("There were no changes in the user-model via UI compared to the backup-firsttime-model");
}
}
/* ===================================================================== */
@Override @Override
public void contextDestroyed(ServletContextEvent sce) { public void contextDestroyed(ServletContextEvent sce) {
// Nothing to do. // Nothing to do.

135
api/src/main/java/edu/cornell/mannlib/vitro/webapp/servlet/setup/RDFFilesLoader.java
View file

@ -5,10 +5,12 @@ package edu.cornell.mannlib.vitro.webapp.servlet.setup;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.StringWriter;
import java.nio.file.DirectoryStream; import java.nio.file.DirectoryStream;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Locale; import java.util.Locale;
@ -23,6 +25,11 @@ import org.apache.jena.ontology.OntModel;
import org.apache.jena.ontology.OntModelSpec; import org.apache.jena.ontology.OntModelSpec;
import org.apache.jena.rdf.model.Model; import org.apache.jena.rdf.model.Model;
import org.apache.jena.rdf.model.ModelFactory; import org.apache.jena.rdf.model.ModelFactory;
import org.apache.jena.rdf.model.Property;
import org.apache.jena.rdf.model.RDFNode;
import org.apache.jena.rdf.model.Resource;
import org.apache.jena.rdf.model.Statement;
import org.apache.jena.rdf.model.StmtIterator;
import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils; import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils;
@ -208,4 +215,132 @@ public class RDFFilesLoader {
// Nothing to initialize. // Nothing to initialize.
} }
/**
* Check if the user model (UI) changes conflict with the changes made to
* the firsttime. If there is conflict, the user model UI value will be
* left unchanged.
*
* @param baseModel firsttime backup model
* @param userModel current state in the system (user/UI-model)
* @param changesModel the changes between firsttime-files and firsttime-backup
*/
public static void checkUiChangesOverlapWithFileChanges(Model baseModel,
Model userModel, Model changesModel) {
log.debug("Check if subtractions from backup-firsttime model to"
+ " current state of firsttime-files were changed in user-model"
+ " (via UI)");
// We don't want to diff against the entire user model, which may be
// huge. We only care about subject/predicate pairs that exist in the
// changesModel. So extract these first from userModel into a
// scopedUserModel that we can use for diffing.
Model scopedUserModel = ModelFactory.createDefaultModel();
StmtIterator scopeIt = changesModel.listStatements();
while(scopeIt.hasNext()) {
Statement scopingStmt = scopeIt.next();
scopedUserModel.add(userModel.listStatements(
scopingStmt.getSubject(), scopingStmt.getPredicate(), (RDFNode) null));
}
log.debug("Scoped user model has " + scopedUserModel.size());
Model changesUserModel = scopedUserModel.difference(baseModel);
log.debug("Diff of scoped user model against firsttime backup has "
+ changesUserModel.size() + " triples");
List<Statement> changedInUIandFileStatements = new ArrayList<Statement>();
if(!changesUserModel.isEmpty()) {
removeBlankTriples(changesUserModel);
if(log.isDebugEnabled()) {
StringWriter out3 = new StringWriter();
changesUserModel.write(out3, "TTL");
log.debug("changesUserModel:\n" + out3);
}
log.debug("There were changes in the user-model via UI which have"
+ " also changed in the firsttime files. The following"
+ " triples will not be updated.");
// Iterate over all statements and check if the ones which should be
// removed were not changed via the UI
StmtIterator iter = changesUserModel.listStatements();
while (iter.hasNext()) {
Statement stmt = iter.nextStatement();
Resource subject = stmt.getSubject();
Property predicate = stmt.getPredicate();
RDFNode object = stmt.getObject();
StmtIterator iter2 = changesModel.listStatements(
subject, predicate, (RDFNode) null);
while (iter2.hasNext()) {
Statement stmt2 = iter2.nextStatement();
RDFNode object2 = stmt2.getObject();
// If subject and predicate are equal but the object differs
// and the language tag is the same, do not update these triples.
// This case indicates an change in the UI, which should not
// be overwritten from the firsttime files.
if(!object.equals(object2) ) {
// if object is an literal, check the language tag
if (object.isLiteral() && object2.isLiteral()) {
// if the language tag is the same, remove this
// triple from the update list
if(object.asLiteral().getLanguage().equals(
object2.asLiteral().getLanguage())) {
log.debug("This two triples changed UI and"
+ " files: \n UI: " + stmt
+ " \n file: " +stmt2);
changedInUIandFileStatements.add(stmt2);
}
} else {
log.debug("This two triples changed UI and"
+ " files: \n UI: " + stmt
+ " \n file: " +stmt2);
changedInUIandFileStatements.add(stmt2);
}
}
}
}
// remove triples which were changed in the user model (UI) from the list
changesModel.remove(changedInUIandFileStatements);
} else {
log.debug("There were no changes in the user-model via UI"
+ " compared to the backup-firsttime-model");
}
}
/**
* Remove all triples where subject or object is blank (Anon)
*/
public static void removeBlankTriples(Model model) {
StmtIterator iter = model.listStatements();
List<Statement> removeStatement = new ArrayList<Statement>();
while (iter.hasNext()) {
Statement stmt = iter.nextStatement(); // get next statement
Resource subject = stmt.getSubject(); // get the subject
RDFNode object = stmt.getObject(); // get the object
if(subject.isAnon() || object.isAnon())
{
removeStatement.add(stmt);
}
}
model.remove(removeStatement);
}
/**
* Check 'isomorphism' for purposes of propagating firsttime changes.
* Run Jena's isomorphism check, but if it fails only due to blank nodes,
* ignore and treat as isomorphic anyway. (Auto-updating firsttime
* changes should occur only with named nodes.)
* @param m1
* @param m2
* @return true if models are isomorphic or any lack of isomorphism exists
* only in blank nodes
*/
public static boolean areIsomporphic(Model m1, Model m2) {
boolean isIsomorphic = m1.isIsomorphicWith(m2);
if(isIsomorphic) {
return true;
} else {
Model diff1 = m1.difference(m2);
Model diff2 = m2.difference(m1);
removeBlankTriples(diff1);
removeBlankTriples(diff2);
return (diff1.isEmpty() && diff2.isEmpty());
}
}
} }

74
api/src/test/java/edu/cornell/mannlib/vitro/webapp/servlet/setup/RDFFilesLoaderTest.java Normal file
View file

@ -0,0 +1,74 @@
package edu.cornell.mannlib.vitro.webapp.servlet.setup;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.io.StringReader;
import org.apache.jena.rdf.model.Model;
import org.apache.jena.rdf.model.ModelFactory;
import org.apache.jena.vocabulary.RDFS;
import edu.cornell.mannlib.vitro.testing.AbstractTestClass;
public class RDFFilesLoaderTest extends AbstractTestClass {
@org.junit.Test
public void testFirsttimeUpdate() {
// the current state of the firsttime file on the filesystem
String fileModelRdf = "@prefix rdfs: <" + RDFS.getURI() + "> .\n"
+ "@prefix : <http://example.com/individual/> .\n"
+ ":n1 rdfs:label \"fish 'n' chips\"@en-GB . \n"
+ ":n1 rdfs:label \"fish and fries\"@en-US . \n"
+ ":n2 rdfs:label \"tube\"@en-GB . \n"
+ ":n2 rdfs:label \"subway; eat fresh\"@en-US . \n"
+ ":n2 rdfs:label \"metrou\"@ro-RO . \n";
// the backup of the previous state of the firsttime file
String backupModelRdf = "@prefix rdfs: <" + RDFS.getURI() + "> .\n"
+ "@prefix : <http://example.com/individual/> .\n"
+ ":n1 rdfs:label \"fish 'n' chips\"@en-GB . \n"
+ ":n2 rdfs:label \"tube\"@en-GB . \n";
// the current state of the user-editable model
String userModelRdf = "@prefix rdfs: <" + RDFS.getURI() + "> .\n"
+ "@prefix : <http://example.com/individual/> .\n"
+ ":n1 rdfs:label \"fish and chips\"@en-GB . \n"
+ ":n2 rdfs:label \"tube\"@en-GB . \n"
+ ":n2 rdfs:label \"subway\"@en-US . \n";
// the expected state of the user-editable model after firsttime
// updates have been applied
String userModelExpectedRdf = "@prefix rdfs: <" + RDFS.getURI() + "> .\n"
+ "@prefix : <http://example.com/individual/> .\n"
+ ":n1 rdfs:label \"fish and chips\"@en-GB . \n"
+ ":n1 rdfs:label \"fish and fries\"@en-US . \n"
+ ":n2 rdfs:label \"tube\"@en-GB . \n"
+ ":n2 rdfs:label \"subway\"@en-US . \n"
+ ":n2 rdfs:label \"metrou\"@ro-RO . \n";
Model fileModel = ModelFactory.createDefaultModel();
fileModel.read(new StringReader(fileModelRdf), null, "N3");
Model backupModel = ModelFactory.createDefaultModel();
backupModel.read(new StringReader(backupModelRdf), null, "N3");
Model userModel = ModelFactory.createDefaultModel();
userModel.read(new StringReader(userModelRdf), null, "N3");
Model userModelExpected = ModelFactory.createDefaultModel();
userModelExpected.read(new StringReader(userModelExpectedRdf), null, "N3");
Model additionsModel = fileModel.difference(backupModel);
Model retractionsModel = backupModel.difference(fileModel);
RDFFilesLoader.checkUiChangesOverlapWithFileChanges(backupModel, userModel, additionsModel);
RDFFilesLoader.checkUiChangesOverlapWithFileChanges(backupModel, userModel, retractionsModel);
userModel.remove(retractionsModel);
userModel.add(additionsModel);
assertTrue("expected: " + userModelExpected + " but was: " + userModel,
userModelExpected.isIsomorphicWith(userModel));
}
}