VIVO-248 Move more RDF files into the home directory

Files for initializing the TBOX, ABOX, ApplicationMetadata and UserAccounts models.

webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/ContentModelSetup.java

@@ -24,11 +24,13 @@ import com.hp.hpl.jena.rdf.model.ResIterator;
 import com.hp.hpl.jena.rdf.model.Resource;
 import com.hp.hpl.jena.shared.Lock;
 import com.hp.hpl.jena.util.ResourceUtils;
+import com.hp.hpl.jena.util.iterator.ClosableIterator;
 import com.hp.hpl.jena.vocabulary.RDF;
 
 import edu.cornell.mannlib.vitro.webapp.dao.ModelAccess;
 import edu.cornell.mannlib.vitro.webapp.dao.ModelAccess.FactoryID;
 import edu.cornell.mannlib.vitro.webapp.dao.ModelAccess.ModelID;
+import edu.cornell.mannlib.vitro.webapp.dao.VitroVocabulary;
 import edu.cornell.mannlib.vitro.webapp.dao.WebappDaoFactory;
 import edu.cornell.mannlib.vitro.webapp.dao.WebappDaoFactoryConfig;
 import edu.cornell.mannlib.vitro.webapp.dao.jena.ModelSynchronizer;
@@ -40,7 +42,6 @@ import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
 import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceFactory;
 import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils;
 import edu.cornell.mannlib.vitro.webapp.startup.StartupStatus;
-import edu.cornell.mannlib.vitro.webapp.utils.jena.InitialJenaModelUtils;
 
 /**
  * Sets up the content models, OntModelSelectors and webapp DAO factories.
@@ -79,9 +80,16 @@ public class ContentModelSetup extends JenaDataSourceSetupBase
         OntModel unionABoxModel = createCombinedBulkUpdatingModel(baseABoxModel, inferenceABoxModel);
         OntModel unionTBoxModel = createCombinedBulkUpdatingModel(baseTBoxModel, inferenceTBoxModel);
 
+
         if (isFirstStartup()) {
-        	loadInitialApplicationMetadataModel(applicationMetadataModel, ctx);
+        	RDFFilesLoader loader = new RDFFilesLoader(ctx);
-        	loadDataFromFilesystem(baseABoxModel, baseTBoxModel, applicationMetadataModel, ctx);
+        	loader.loadFirstTimeFiles("abox", baseABoxModel, true);
+        	loader.loadFirstTimeFiles("tbox", baseTBoxModel, true);
+
+        	loader.loadFirstTimeFiles("applicationMetadata", applicationMetadataModel, true);
+        	setPortalUriOnFirstTime(applicationMetadataModel, ctx);
+        } else {
+        	checkForNamespaceMismatch( applicationMetadataModel, ctx );
         }
         
         log.info("Setting up full models");
@@ -101,7 +109,6 @@ public class ContentModelSetup extends JenaDataSourceSetupBase
         models.setOntModel(ModelID.UNION_TBOX, unionTBoxModel);
         models.setOntModel(ModelID.UNION_FULL, unionFullModel);
         
-        checkForNamespaceMismatch( applicationMetadataModel, ctx );
         
 		log.info("Setting up DAO factories");
 		
@@ -159,35 +166,40 @@ public class ContentModelSetup extends JenaDataSourceSetupBase
 		return ModelFactory.createOntologyModel(MEM_ONT_MODEL_SPEC, unionModel);
 	}
 
-	private void loadInitialApplicationMetadataModel(OntModel applicationMetadataModel,
-			ServletContext ctx) {
-		try {
-			applicationMetadataModel.add(
-					InitialJenaModelUtils.loadInitialModel(ctx, getDefaultNamespace(ctx)));
-		} catch (Throwable e) {
-			throw new RuntimeException("Unable to load application metadata model cache from DB", e);
-		}
-	}
-
-	private void loadDataFromFilesystem(OntModel baseABoxModel, OntModel baseTBoxModel, OntModel applicationMetadataModel, 
-			ServletContext ctx) {
-		Long startTime = System.currentTimeMillis();
-		log.info("Initializing models from RDF files");    
-		
-		readOntologyFilesInPathSet(USER_ABOX_PATH, ctx, baseABoxModel);
-		readOntologyFilesInPathSet(USER_TBOX_PATH, ctx, baseTBoxModel);
-		readOntologyFilesInPathSet(USER_APPMETA_PATH, ctx, applicationMetadataModel);
-		
-		log.debug(((System.currentTimeMillis() - startTime) / 1000)
-				+ " seconds to read RDF files ");
-	}
-	
 	private long secondsSince(long startTime) {
 		return (System.currentTimeMillis() - startTime) / 1000;
 	}
 
     /* ===================================================================== */
 
+	/**
+	 * If we are loading the application metadata for the first time, set the
+	 * URI of the Portal based on the default namespace.
+	 */
+	private void setPortalUriOnFirstTime(OntModel model, ServletContext ctx) {
+		// currently, only a single portal is permitted in the initialization
+		// data
+		Resource portalResource = null;
+		ClosableIterator<Resource> portalResIt = model
+				.listSubjectsWithProperty(RDF.type,
+						model.getResource(VitroVocabulary.PORTAL));
+		try {
+			if (portalResIt.hasNext()) {
+				Resource portalRes = portalResIt.next();
+				if (portalRes.isAnon()) {
+					portalResource = portalRes;
+				}
+			}
+		} finally {
+			portalResIt.close();
+		}
+		
+		if (portalResource != null) {
+			ResourceUtils.renameResource(portalResource, getDefaultNamespace(ctx) + "portal1");
+		}
+	}
+
+	
     /**
      * If we find a "portal1" portal (and we should), its URI should use the
      * default namespace.

webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/JenaDataSourceSetupBase.java

@@ -70,11 +70,6 @@ public class JenaDataSourceSetupBase extends JenaBaseDaoCon {
             DEFAULT_TESTONRETURN = true, DEFAULT_TESTWHILEIDLE = true;
 
     protected static String BASE = "/WEB-INF/ontologies/";
-    protected static String USERPATH = BASE+"user/";
-    protected static String USER_ABOX_PATH = BASE+"user/abox";
-    protected static String USER_TBOX_PATH = BASE+"user/tbox";
-    protected static String USER_APPMETA_PATH = BASE+"user/applicationMetadata";
-    protected static String SYSTEMPATH = BASE+"system/";       
     public static String APPPATH = BASE+"app/";
     //these files are loaded everytime the system starts up
     public static String APPPATH_LOAD = APPPATH + "menuload/";
@@ -395,6 +390,7 @@ public class JenaDataSourceSetupBase extends JenaBaseDaoCon {
        return dbModel;
    }
 
+   // TODO get rid of this?
     public static void readOntologyFilesInPathSet(String path,
             ServletContext ctx, Model model) {
         log.debug("Reading ontology files from '" + path + "'");
@@ -406,6 +402,7 @@ public class JenaDataSourceSetupBase extends JenaBaseDaoCon {
         }
     }
    
+    // TODO get rid of this?
     public static void readOntologyFileFromPath(String p, 
                                                 Model model, 
                                                 ServletContext ctx) {

webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/RDFFilesLoader.java

@@ -0,0 +1,118 @@
+/* $This file is distributed under the terms of the license in /doc/license.txt$ */
+
+package edu.cornell.mannlib.vitro.webapp.servlet.setup;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Set;
+import java.util.TreeSet;
+
+import javax.servlet.ServletContext;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import com.hp.hpl.jena.rdf.model.Model;
+
+import edu.cornell.mannlib.vitro.webapp.config.ConfigurationProperties;
+
+/**
+ * Help to load RDF files on first time and on every startup.
+ */
+public class RDFFilesLoader {
+	private static final Log log = LogFactory.getLog(RDFFilesLoader.class);
+
+	private static final String PROPERTY_VITRO_HOME = "vitro.home";
+	private static final String DEFAULT_RDF_FORMAT = "RDF/XML";
+	private static final String FIRST_TIME = "firsttime";
+
+	/** Directory filter that ignores sub-directories and hidden files. */
+	private static final DirectoryStream.Filter<Path> RDF_FILE_FILTER = new DirectoryStream.Filter<Path>() {
+		@Override
+		public boolean accept(Path p) throws IOException {
+			if (Files.isHidden(p)) {
+				return false;
+			}
+			if (Files.isDirectory(p)) {
+				log.warn("RDF files in subdirectories are not loaded. Directory '"
+						+ p + "' ignored.");
+				return false;
+			}
+			return true;
+		}
+	};
+
+	private final String homeDirProperty;
+
+	public RDFFilesLoader(ServletContext ctx) {
+		ConfigurationProperties props = ConfigurationProperties.getBean(ctx);
+		this.homeDirProperty = props.getProperty(PROPERTY_VITRO_HOME);
+	}
+
+	/**
+	 * Load the "first time" files if the model is empty.
+	 */
+	public void loadFirstTimeFiles(String modelPath, Model model) {
+		loadFirstTimeFiles(modelPath, model, model.isEmpty());
+	}
+	
+	/**
+	 * Load the "first time" files if we say it is the first time.
+	 */
+	public void loadFirstTimeFiles(String modelPath, Model model, boolean firstTime) {
+		if (firstTime) {
+			Set<Path> paths = getPaths(homeDirProperty, modelPath, FIRST_TIME);
+			for (Path p : paths) {
+				readOntologyFileIntoModel(p, model);
+			}
+		}
+	}
+
+	private Set<Path> getPaths(String parentDir, String... strings) {
+		Path dir = Paths.get(parentDir, strings);
+
+		Set<Path> paths = new TreeSet<>();
+		if (Files.isDirectory(dir)) {
+			try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir,
+					RDF_FILE_FILTER)) {
+				for (Path p : stream) {
+					paths.add(p);
+				}
+			} catch (IOException e) {
+				log.warn("Failed to read directory '" + dir + "'", e);
+			}
+		} else {
+			log.debug("Filegraph directory '" + dir + "' doesn't exist.");
+		}
+		log.debug("Paths from '" + dir + "': " + paths);
+		return paths;
+	}
+
+	private static void readOntologyFileIntoModel(Path p, Model model) {
+		String format = getRdfFormat(p);
+		log.info("Loading ontology file at " + p + " as format " + format);
+		try (InputStream stream = new FileInputStream(p.toFile())) {
+			model.read(stream, null, format);
+			log.debug("...successful");
+		} catch (IOException e) {
+			log.error("Failed to load ontology file at '" + p + "' as format "
+					+ format, e);
+		}
+	}
+
+	private static String getRdfFormat(Path p) {
+		String filename = p.getFileName().toString().toLowerCase();
+		if (filename.endsWith("n3"))
+			return "N3";
+		else if (filename.endsWith("ttl"))
+			return "TURTLE";
+		else
+			return DEFAULT_RDF_FORMAT;
+	}
+
+}

webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/UserModelSetup.java

@@ -24,8 +24,6 @@ import edu.cornell.mannlib.vitro.webapp.startup.StartupStatus;
  */
 public class UserModelSetup extends JenaDataSourceSetupBase 
                                            implements ServletContextListener {
-    protected static String AUTHPATH = BASE + "auth/";
-    
     private static final Log log = LogFactory.getLog(
             UserModelSetup.class.getName());
 
@@ -58,9 +56,7 @@ public class UserModelSetup extends JenaDataSourceSetupBase
                 ModelFactory.createOntologyModel( MEM_ONT_MODEL_SPEC);
     
             // This is used in Selenium testing, when we load user accounts from a file.
-        	if (userAccountsDbModel.isEmpty()) {
+            new RDFFilesLoader(ctx).loadFirstTimeFiles("auth", userAccountsDbModel);
-        		readOntologyFilesInPathSet(AUTHPATH, ctx, userAccountsDbModel);
-        	}
 
         	userAccountsModel.add(userAccountsDbModel);
             userAccountsModel.getBaseModel().register(

webapp/src/edu/cornell/mannlib/vitro/webapp/utils/jena/InitialJenaModelUtils.java

@@ -1,64 +0,0 @@
-/* $This file is distributed under the terms of the license in /doc/license.txt$ */
-
-package edu.cornell.mannlib.vitro.webapp.utils.jena;
-
-import javax.servlet.ServletContext;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import com.hp.hpl.jena.ontology.OntModel;
-import com.hp.hpl.jena.ontology.OntModelSpec;
-import com.hp.hpl.jena.rdf.model.Model;
-import com.hp.hpl.jena.rdf.model.ModelFactory;
-import com.hp.hpl.jena.rdf.model.Resource;
-import com.hp.hpl.jena.util.ResourceUtils;
-import com.hp.hpl.jena.util.iterator.ClosableIterator;
-import com.hp.hpl.jena.vocabulary.RDF;
-
-import edu.cornell.mannlib.vitro.webapp.dao.VitroVocabulary;
-import edu.cornell.mannlib.vitro.webapp.servlet.setup.JenaDataSourceSetupBase;
-
-public class InitialJenaModelUtils {
-
-	private static final String INIT_DATA_DIRECTORY = "/WEB-INF/init-data";
-	
-	private static final Log log = LogFactory.getLog(InitialJenaModelUtils.class);
-	
-	public static Model loadInitialModel(ServletContext ctx, String defaultNamespace) {
-	
-		OntModel initialModel = ModelFactory.createOntologyModel(OntModelSpec.OWL_MEM);
-		
-		try {
-			JenaDataSourceSetupBase.readOntologyFilesInPathSet(INIT_DATA_DIRECTORY, ctx, initialModel);
-		} catch (Throwable t) {
-			log.warn("Unable to read initial model data from " + INIT_DATA_DIRECTORY, t);
-		}
-	
-		if (initialModel.size() == 0) {
-			return initialModel;
-		}
-		
-		//find and rename portal object
-		//currently, only a single portal is permitted in the initialization data
-		Resource portalResource = null;
-		ClosableIterator<Resource> portalResIt = initialModel.listSubjectsWithProperty(RDF.type, initialModel.getResource(VitroVocabulary.PORTAL));
-		try {
-			if (portalResIt.hasNext()) {
-				Resource portalRes = portalResIt.next();
-				if (portalRes.isAnon()) {
-					portalResource = portalRes;
-				}
-			}
-		} finally {
-			portalResIt.close();
-		}
-		if (portalResource != null) {
-			ResourceUtils.renameResource(portalResource, defaultNamespace + "portal1");
-		}
-		
-		return initialModel;
-		
-	}
-	
-}