From f1ef775d95fd6b0684093b43d94ea172d339f18a Mon Sep 17 00:00:00 2001 From: Brian Lowe Date: Thu, 26 May 2022 20:22:07 +0300 Subject: [PATCH] Use limited diff when checking for local overrides of firsttime data. Add unit test for firsttime update behavior. --- .../setup/ConfigurationModelsSetup.java | 90 +----------- .../servlet/setup/ContentModelSetup.java | 68 +-------- .../webapp/servlet/setup/RDFFilesLoader.java | 137 +++++++++++++++++- .../servlet/setup/RDFFilesLoaderTest.java | 74 ++++++++++ 4 files changed, 219 insertions(+), 150 deletions(-) create mode 100644 api/src/test/java/edu/cornell/mannlib/vitro/webapp/servlet/setup/RDFFilesLoaderTest.java diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/servlet/setup/ConfigurationModelsSetup.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/servlet/setup/ConfigurationModelsSetup.java index 6c6429327..d2bbcc13a 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/servlet/setup/ConfigurationModelsSetup.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/servlet/setup/ConfigurationModelsSetup.java @@ -98,7 +98,7 @@ public class ConfigurationModelsSetup implements ServletContextListener { OntModel baseModelFirsttime = VitroModelFactory.createOntologyModel(); RDFFilesLoader.loadFirstTimeFiles(ctx, modelPath, baseModelFirsttime, true); - if (baseModelFirsttime.isIsomorphicWith(baseModelFirsttimeBackup)) { + if (RDFFilesLoader.areIsomporphic(baseModelFirsttime, baseModelFirsttimeBackup)) { log.debug("They are the same, so do nothing: '" + modelPath + "'"); } else { log.debug("They differ:" + modelPath + ", compare values in configuration models with user's triplestore"); @@ -132,9 +132,8 @@ public class ConfigurationModelsSetup implements ServletContextListener { // remove special cases for display, problem with blank nodes if (modelIdString.equals("display")) { - - removeBlankTriples(difOldNew); - removeBlankTriples(difNewOld); + RDFFilesLoader.removeBlankTriples(difOldNew); + RDFFilesLoader.removeBlankTriples(difNewOld); } if (difOldNew.isEmpty() && difNewOld.isEmpty()) { @@ -149,7 +148,7 @@ public class ConfigurationModelsSetup implements ServletContextListener { log.debug("Difference for " + modelIdString + " (old -> new), these triples should be removed: " + out); // Check if the UI-changes Overlap with the changes made in the fristtime-files - checkUiChangesOverlapWithFileChanges(baseModel, userModel, difOldNew); + RDFFilesLoader.checkUiChangesOverlapWithFileChanges(baseModel, userModel, difOldNew); // before we remove the triples, we need to compare values in back up firsttime with user's triplestore // if the triples which should be removed are still in user´s triplestore, remove them @@ -167,7 +166,7 @@ public class ConfigurationModelsSetup implements ServletContextListener { log.debug("Difference for " + modelIdString + " (new -> old), these triples should be added: " + out2); // Check if the UI-changes Overlap with the changes made in the fristtime-files - checkUiChangesOverlapWithFileChanges(baseModel, userModel, difNewOld); + RDFFilesLoader.checkUiChangesOverlapWithFileChanges(baseModel, userModel, difNewOld); // before we add the triples, we need to compare values in back up firsttime with user's triplestore // if the triples which should be added are not already in user´s triplestore, add them @@ -186,85 +185,6 @@ public class ConfigurationModelsSetup implements ServletContextListener { return updatedFiles; } - /** - * Check if the UI-changes Overlap with the changes made in the fristtime-files, if they overlap these changes are not applied to the user-model (UI) - * - * @param baseModel firsttime backup model - * @param userModel current state in the system (user/UI-model) - * @param changesModel the changes between firsttime-files and firttime-backup - */ - private void checkUiChangesOverlapWithFileChanges(Model baseModel, Model userModel, Model changesModel) { - log.debug("Beginn check if subtractions from Backup-firsttime model to current state of firsttime-files were changed in user-model (via UI)"); - Model changesUserModel = userModel.difference(baseModel); - List changedInUIandFileStatements = new ArrayList(); - - if(!changesUserModel.isEmpty()) - { - removeBlankTriples(changesUserModel); - - StringWriter out3 = new StringWriter(); - changesUserModel.write(out3, "TTL"); - log.debug("There were changes in the user-model via UI which have also changed in the firsttime files, the following triples will not be updated"); - - // iterate all statements and check if the ones which should be removed were not changed via the UI - StmtIterator iter = changesUserModel.listStatements(); - while (iter.hasNext()) { - Statement stmt = iter.nextStatement(); // get next statement - Resource subject = stmt.getSubject(); // get the subject - Property predicate = stmt.getPredicate(); // get the predicate - RDFNode object = stmt.getObject(); // get the object - - StmtIterator iter2 = changesModel.listStatements(); - - while (iter2.hasNext()) { - Statement stmt2 = iter2.nextStatement(); // get next statement - Resource subject2 = stmt2.getSubject(); // get the subject - Property predicate2 = stmt2.getPredicate(); // get the predicate - RDFNode object2 = stmt2.getObject(); // get the object - - // if subject and predicate are equal but the object differs and the language tag is the same, do not update these triples - // this case indicates an change in the UI, which should not be overwriten from the firsttime files - if(subject.equals(subject2) && predicate.equals(predicate2) && !object.equals(object2) ) { - // if object is an literal, check the language tag - if (object.isLiteral() && object2.isLiteral()) { - // if the langauge tag is the same, remove this triple from the update list - if(object.asLiteral().getLanguage().equals(object2.asLiteral().getLanguage())) { - log.debug("This two triples changed UI and files: \n UI: " + stmt + " \n file: " +stmt2); - changedInUIandFileStatements.add(stmt2); - } - } else { - log.debug("This two triples changed UI and files: \n UI: " + stmt + " \n file: " +stmt2); - changedInUIandFileStatements.add(stmt2); - } - } - } - } - // remove triples which were changed in the user model (UI) from the list - changesModel.remove(changedInUIandFileStatements); - } else { - log.debug("There were no changes in the user-model via UI compared to the backup-firsttime-model"); - } - } - - /** - * Remove all triples where subject or object is blank (Anon) - */ - private void removeBlankTriples(Model model) { - StmtIterator iter = model.listStatements(); - List removeStatement = new ArrayList(); - while (iter.hasNext()) { - Statement stmt = iter.nextStatement(); // get next statement - Resource subject = stmt.getSubject(); // get the subject - RDFNode object = stmt.getObject(); // get the object - - if(subject.isAnon() || object.isAnon()) - { - removeStatement.add(stmt); - } - } - model.remove(removeStatement); - } - @Override public void contextDestroyed(ServletContextEvent sce) { // Nothing to tear down. diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/servlet/setup/ContentModelSetup.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/servlet/setup/ContentModelSetup.java index 2024e59e8..4608b3b90 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/servlet/setup/ContentModelSetup.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/servlet/setup/ContentModelSetup.java @@ -21,6 +21,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.jena.ontology.OntModel; import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.ModelFactory; import org.apache.jena.rdf.model.Property; import org.apache.jena.rdf.model.ResIterator; import org.apache.jena.rdf.model.Resource; @@ -249,7 +250,7 @@ public class ContentModelSetup extends JenaDataSourceSetupBase setPortalUriOnFirstTime(firsttimeFilesModel, ctx); } - if ( firsttimeBackupModel.isIsomorphicWith(firsttimeFilesModel) ) { + if ( RDFFilesLoader.areIsomporphic(firsttimeBackupModel, firsttimeFilesModel) ) { log.debug("They are the same, so do nothing: '" + modelPath + "'"); } else { log.debug("They differ: '" + modelPath + "', compare values in configuration models with user's triplestore"); @@ -301,7 +302,7 @@ public class ContentModelSetup extends JenaDataSourceSetupBase log.debug("Difference for " + modelIdString + " (old -> new), these triples should be removed: " + out); // Check if the UI-changes Overlap with the changes made in the fristtime-files - checkUiChangesOverlapWithFileChanges(baseModel, userModel, difOldNew); + RDFFilesLoader.checkUiChangesOverlapWithFileChanges(baseModel, userModel, difOldNew); // before we remove the triples, we need to compare values in back up firsttime with user's triplestore // if the triples which should be removed are still in user´s triplestore, remove them @@ -320,7 +321,7 @@ public class ContentModelSetup extends JenaDataSourceSetupBase log.debug("Difference for " + modelIdString + " (new -> old), these triples should be added: " + out2); // Check if the UI-changes Overlap with the changes made in the fristtime-files - checkUiChangesOverlapWithFileChanges(baseModel, userModel, difNewOld); + RDFFilesLoader.checkUiChangesOverlapWithFileChanges(baseModel, userModel, difNewOld); // before we add the triples, we need to compare values in back up firsttime with user's triplestore // if the triples which should be added are not already in user´s triplestore, add them @@ -339,67 +340,6 @@ public class ContentModelSetup extends JenaDataSourceSetupBase return updatedFiles; } - /** - * Check if the UI-changes Overlap with the changes made in the fristtime-files, if they overlap these changes are not applied to the user-model (UI) - * - * @param baseModel firsttime backup model - * @param userModel current state in the system (user/UI-model) - * @param changesModel the changes between firsttime-files and firttime-backup - */ - private void checkUiChangesOverlapWithFileChanges(Model baseModel, Model userModel, Model changesModel) { - log.debug("Beginn check if subtractions from Backup-firsttime model to current state of firsttime-files were changed in user-model (via UI)"); - Model changesUserModel = userModel.difference(baseModel); - List changedInUIandFileStatements = new ArrayList(); - - if(!changesUserModel.isEmpty()) - { - - StringWriter out3 = new StringWriter(); - changesUserModel.write(out3, "TTL"); - log.debug("There were changes in the user-model via UI which have also changed in the firsttime files, the following triples will not be updated"); - - // iterate all statements and check if the ones which should be removed were not changed via the UI - StmtIterator iter = changesUserModel.listStatements(); - while (iter.hasNext()) { - Statement stmt = iter.nextStatement(); // get next statement - Resource subject = stmt.getSubject(); // get the subject - Property predicate = stmt.getPredicate(); // get the predicate - RDFNode object = stmt.getObject(); // get the object - - StmtIterator iter2 = changesModel.listStatements(); - - while (iter2.hasNext()) { - Statement stmt2 = iter2.nextStatement(); // get next statement - Resource subject2 = stmt2.getSubject(); // get the subject - Property predicate2 = stmt2.getPredicate(); // get the predicate - RDFNode object2 = stmt2.getObject(); // get the object - - // if subject and predicate are equal but the object differs and the language tag is the same, do not update these triples - // this case indicates an change in the UI, which should not be overwriten from the firsttime files - if(subject.equals(subject2) && predicate.equals(predicate2) && !object.equals(object2) ) { - // if object is an literal, check the language tag - if (object.isLiteral() && object2.isLiteral()) { - // if the langauge tag is the same, remove this triple from the update list - if(object.asLiteral().getLanguage().equals(object2.asLiteral().getLanguage())) { - log.debug("This two triples changed UI and files: \n UI: " + stmt + " \n file: " +stmt2); - changedInUIandFileStatements.add(stmt2); - } - } else { - log.debug("This two triples changed UI and files: \n UI: " + stmt + " \n file: " +stmt2); - changedInUIandFileStatements.add(stmt2); - } - } - } - } - // remove triples which were changed in the user model (UI) from the list - changesModel.remove(changedInUIandFileStatements); - } else { - log.debug("There were no changes in the user-model via UI compared to the backup-firsttime-model"); - } - } - - /* ===================================================================== */ - @Override public void contextDestroyed(ServletContextEvent sce) { // Nothing to do. diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/servlet/setup/RDFFilesLoader.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/servlet/setup/RDFFilesLoader.java index acaa93fb6..36849f23b 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/servlet/setup/RDFFilesLoader.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/servlet/setup/RDFFilesLoader.java @@ -5,10 +5,12 @@ package edu.cornell.mannlib.vitro.webapp.servlet.setup; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.io.StringWriter; import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Locale; @@ -23,6 +25,11 @@ import org.apache.jena.ontology.OntModel; import org.apache.jena.ontology.OntModelSpec; import org.apache.jena.rdf.model.Model; import org.apache.jena.rdf.model.ModelFactory; +import org.apache.jena.rdf.model.Property; +import org.apache.jena.rdf.model.RDFNode; +import org.apache.jena.rdf.model.Resource; +import org.apache.jena.rdf.model.Statement; +import org.apache.jena.rdf.model.StmtIterator; import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils; @@ -207,5 +214,133 @@ public class RDFFilesLoader { private RDFFilesLoader() { // Nothing to initialize. } + + /** + * Check if the user model (UI) changes conflict with the changes made to + * the firsttime. If there is conflict, the user model UI value will be + * left unchanged. + * + * @param baseModel firsttime backup model + * @param userModel current state in the system (user/UI-model) + * @param changesModel the changes between firsttime-files and firsttime-backup + */ + public static void checkUiChangesOverlapWithFileChanges(Model baseModel, + Model userModel, Model changesModel) { + log.debug("Check if subtractions from backup-firsttime model to" + + " current state of firsttime-files were changed in user-model" + + " (via UI)"); + // We don't want to diff against the entire user model, which may be + // huge. We only care about subject/predicate pairs that exist in the + // changesModel. So extract these first from userModel into a + // scopedUserModel that we can use for diffing. + Model scopedUserModel = ModelFactory.createDefaultModel(); + StmtIterator scopeIt = changesModel.listStatements(); + while(scopeIt.hasNext()) { + Statement scopingStmt = scopeIt.next(); + scopedUserModel.add(userModel.listStatements( + scopingStmt.getSubject(), scopingStmt.getPredicate(), (RDFNode) null)); + } + log.debug("Scoped user model has " + scopedUserModel.size()); + Model changesUserModel = scopedUserModel.difference(baseModel); + log.debug("Diff of scoped user model against firsttime backup has " + + changesUserModel.size() + " triples"); + List changedInUIandFileStatements = new ArrayList(); + if(!changesUserModel.isEmpty()) { + removeBlankTriples(changesUserModel); + if(log.isDebugEnabled()) { + StringWriter out3 = new StringWriter(); + changesUserModel.write(out3, "TTL"); + log.debug("changesUserModel:\n" + out3); + } + log.debug("There were changes in the user-model via UI which have" + + " also changed in the firsttime files. The following" + + " triples will not be updated."); + // Iterate over all statements and check if the ones which should be + // removed were not changed via the UI + StmtIterator iter = changesUserModel.listStatements(); + while (iter.hasNext()) { + Statement stmt = iter.nextStatement(); + Resource subject = stmt.getSubject(); + Property predicate = stmt.getPredicate(); + RDFNode object = stmt.getObject(); + StmtIterator iter2 = changesModel.listStatements( + subject, predicate, (RDFNode) null); + while (iter2.hasNext()) { + Statement stmt2 = iter2.nextStatement(); + RDFNode object2 = stmt2.getObject(); + // If subject and predicate are equal but the object differs + // and the language tag is the same, do not update these triples. + // This case indicates an change in the UI, which should not + // be overwritten from the firsttime files. + if(!object.equals(object2) ) { + // if object is an literal, check the language tag + if (object.isLiteral() && object2.isLiteral()) { + // if the language tag is the same, remove this + // triple from the update list + if(object.asLiteral().getLanguage().equals( + object2.asLiteral().getLanguage())) { + log.debug("This two triples changed UI and" + + " files: \n UI: " + stmt + + " \n file: " +stmt2); + changedInUIandFileStatements.add(stmt2); + } + } else { + log.debug("This two triples changed UI and" + + " files: \n UI: " + stmt + + " \n file: " +stmt2); + changedInUIandFileStatements.add(stmt2); + } + } + } + } + // remove triples which were changed in the user model (UI) from the list + changesModel.remove(changedInUIandFileStatements); + } else { + log.debug("There were no changes in the user-model via UI" + + " compared to the backup-firsttime-model"); + } + } -} + /** + * Remove all triples where subject or object is blank (Anon) + */ + public static void removeBlankTriples(Model model) { + StmtIterator iter = model.listStatements(); + List removeStatement = new ArrayList(); + while (iter.hasNext()) { + Statement stmt = iter.nextStatement(); // get next statement + Resource subject = stmt.getSubject(); // get the subject + RDFNode object = stmt.getObject(); // get the object + + if(subject.isAnon() || object.isAnon()) + { + removeStatement.add(stmt); + } + } + model.remove(removeStatement); + } + + /** + * Check 'isomorphism' for purposes of propagating firsttime changes. + * Run Jena's isomorphism check, but if it fails only due to blank nodes, + * ignore and treat as isomorphic anyway. (Auto-updating firsttime + * changes should occur only with named nodes.) + * @param m1 + * @param m2 + * @return true if models are isomorphic or any lack of isomorphism exists + * only in blank nodes + */ + public static boolean areIsomporphic(Model m1, Model m2) { + boolean isIsomorphic = m1.isIsomorphicWith(m2); + if(isIsomorphic) { + return true; + } else { + Model diff1 = m1.difference(m2); + Model diff2 = m2.difference(m1); + removeBlankTriples(diff1); + removeBlankTriples(diff2); + return (diff1.isEmpty() && diff2.isEmpty()); + } + } + +} \ No newline at end of file diff --git a/api/src/test/java/edu/cornell/mannlib/vitro/webapp/servlet/setup/RDFFilesLoaderTest.java b/api/src/test/java/edu/cornell/mannlib/vitro/webapp/servlet/setup/RDFFilesLoaderTest.java new file mode 100644 index 000000000..3ccf9b4b9 --- /dev/null +++ b/api/src/test/java/edu/cornell/mannlib/vitro/webapp/servlet/setup/RDFFilesLoaderTest.java @@ -0,0 +1,74 @@ +package edu.cornell.mannlib.vitro.webapp.servlet.setup; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.StringReader; + +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.ModelFactory; +import org.apache.jena.vocabulary.RDFS; + +import edu.cornell.mannlib.vitro.testing.AbstractTestClass; + +public class RDFFilesLoaderTest extends AbstractTestClass { + + @org.junit.Test + public void testFirsttimeUpdate() { + + // the current state of the firsttime file on the filesystem + String fileModelRdf = "@prefix rdfs: <" + RDFS.getURI() + "> .\n" + + "@prefix : .\n" + + ":n1 rdfs:label \"fish 'n' chips\"@en-GB . \n" + + ":n1 rdfs:label \"fish and fries\"@en-US . \n" + + ":n2 rdfs:label \"tube\"@en-GB . \n" + + ":n2 rdfs:label \"subway; eat fresh\"@en-US . \n" + + ":n2 rdfs:label \"metrou\"@ro-RO . \n"; + + // the backup of the previous state of the firsttime file + String backupModelRdf = "@prefix rdfs: <" + RDFS.getURI() + "> .\n" + + "@prefix : .\n" + + ":n1 rdfs:label \"fish 'n' chips\"@en-GB . \n" + + ":n2 rdfs:label \"tube\"@en-GB . \n"; + + // the current state of the user-editable model + String userModelRdf = "@prefix rdfs: <" + RDFS.getURI() + "> .\n" + + "@prefix : .\n" + + ":n1 rdfs:label \"fish and chips\"@en-GB . \n" + + ":n2 rdfs:label \"tube\"@en-GB . \n" + + ":n2 rdfs:label \"subway\"@en-US . \n"; + + // the expected state of the user-editable model after firsttime + // updates have been applied + String userModelExpectedRdf = "@prefix rdfs: <" + RDFS.getURI() + "> .\n" + + "@prefix : .\n" + + ":n1 rdfs:label \"fish and chips\"@en-GB . \n" + + ":n1 rdfs:label \"fish and fries\"@en-US . \n" + + ":n2 rdfs:label \"tube\"@en-GB . \n" + + ":n2 rdfs:label \"subway\"@en-US . \n" + + ":n2 rdfs:label \"metrou\"@ro-RO . \n"; + + Model fileModel = ModelFactory.createDefaultModel(); + fileModel.read(new StringReader(fileModelRdf), null, "N3"); + Model backupModel = ModelFactory.createDefaultModel(); + backupModel.read(new StringReader(backupModelRdf), null, "N3"); + Model userModel = ModelFactory.createDefaultModel(); + userModel.read(new StringReader(userModelRdf), null, "N3"); + Model userModelExpected = ModelFactory.createDefaultModel(); + userModelExpected.read(new StringReader(userModelExpectedRdf), null, "N3"); + + Model additionsModel = fileModel.difference(backupModel); + Model retractionsModel = backupModel.difference(fileModel); + + RDFFilesLoader.checkUiChangesOverlapWithFileChanges(backupModel, userModel, additionsModel); + RDFFilesLoader.checkUiChangesOverlapWithFileChanges(backupModel, userModel, retractionsModel); + + userModel.remove(retractionsModel); + userModel.add(additionsModel); + + assertTrue("expected: " + userModelExpected + " but was: " + userModel, + userModelExpected.isIsomorphicWith(userModel)); + + } + +}