diff --git a/.gitignore b/.gitignore index e58913106..a853e4b57 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ .build utilities/solrtester/.work utilities/rdbmigration/.work +utilities/sdb_to_tdb/.work diff --git a/utilities/sdb_to_tdb/README.txt b/utilities/sdb_to_tdb/README.txt new file mode 100644 index 000000000..3b23497e0 --- /dev/null +++ b/utilities/sdb_to_tdb/README.txt @@ -0,0 +1,19 @@ +A tool to convert triples from SDB to TDB. + +Unlike the RDF-migration tool, this tool is not tied to the VIVO +properties files. Instead, you must provide: +* The URL for the SDB, including username and password parameters +* The directory path for the TDB. + The directory must exist. + The directory must be empty, unless the "force" optiopn is used. + +Examples: + + java -jar sdb2tdb.jar \ + jdbc:mysql://localhost/vitrodb?user=vivoUser&password=vivoPass \ + /usr/local/my/tdb + + java -Xms512m -Xmx4096m -jar .work/sdb2tdb.jar \ + 'jdbc:mysql://localhost/weill17?user=vivoUser&password=vivoPass' \ + /Users/jeb228/Testing/instances/weill-develop/vivo_home/contentTdb \ + force \ No newline at end of file diff --git a/utilities/sdb_to_tdb/build.xml b/utilities/sdb_to_tdb/build.xml new file mode 100644 index 000000000..0993e7cc2 --- /dev/null +++ b/utilities/sdb_to_tdb/build.xml @@ -0,0 +1,89 @@ + + + + + + + + + + + + + + + + + + all - Compiles the tool ad packs it into a JAR + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/utilities/sdb_to_tdb/lib/jena-arq-2.10.1.jar b/utilities/sdb_to_tdb/lib/jena-arq-2.10.1.jar new file mode 100644 index 000000000..b8e9547f3 Binary files /dev/null and b/utilities/sdb_to_tdb/lib/jena-arq-2.10.1.jar differ diff --git a/utilities/sdb_to_tdb/lib/jena-core-2.10.1.jar b/utilities/sdb_to_tdb/lib/jena-core-2.10.1.jar new file mode 100644 index 000000000..d2a389c96 Binary files /dev/null and b/utilities/sdb_to_tdb/lib/jena-core-2.10.1.jar differ diff --git a/utilities/sdb_to_tdb/lib/jena-iri-0.9.6.jar b/utilities/sdb_to_tdb/lib/jena-iri-0.9.6.jar new file mode 100644 index 000000000..e4ee0dc2a Binary files /dev/null and b/utilities/sdb_to_tdb/lib/jena-iri-0.9.6.jar differ diff --git a/utilities/sdb_to_tdb/lib/jena-sdb-1.3.6.jar b/utilities/sdb_to_tdb/lib/jena-sdb-1.3.6.jar new file mode 100644 index 000000000..43ee9ffcc Binary files /dev/null and b/utilities/sdb_to_tdb/lib/jena-sdb-1.3.6.jar differ diff --git a/utilities/sdb_to_tdb/lib/jena-tdb-0.10.0.jar b/utilities/sdb_to_tdb/lib/jena-tdb-0.10.0.jar new file mode 100644 index 000000000..ad7101a20 Binary files /dev/null and b/utilities/sdb_to_tdb/lib/jena-tdb-0.10.0.jar differ diff --git a/utilities/sdb_to_tdb/lib/log4j-1.2.16.jar b/utilities/sdb_to_tdb/lib/log4j-1.2.16.jar new file mode 100644 index 000000000..5429a903e Binary files /dev/null and b/utilities/sdb_to_tdb/lib/log4j-1.2.16.jar differ diff --git a/utilities/sdb_to_tdb/lib/mysql-connector-java-5.1.30-bin.jar b/utilities/sdb_to_tdb/lib/mysql-connector-java-5.1.30-bin.jar new file mode 100644 index 000000000..afef9b2d0 Binary files /dev/null and b/utilities/sdb_to_tdb/lib/mysql-connector-java-5.1.30-bin.jar differ diff --git a/utilities/sdb_to_tdb/lib/slf4j-api-1.6.6.jar b/utilities/sdb_to_tdb/lib/slf4j-api-1.6.6.jar new file mode 100644 index 000000000..4c03fa6bb Binary files /dev/null and b/utilities/sdb_to_tdb/lib/slf4j-api-1.6.6.jar differ diff --git a/utilities/sdb_to_tdb/lib/slf4j-log4j12-1.6.6.jar b/utilities/sdb_to_tdb/lib/slf4j-log4j12-1.6.6.jar new file mode 100644 index 000000000..e72c2d66e Binary files /dev/null and b/utilities/sdb_to_tdb/lib/slf4j-log4j12-1.6.6.jar differ diff --git a/utilities/sdb_to_tdb/lib/xercesImpl-2.11.0.jar b/utilities/sdb_to_tdb/lib/xercesImpl-2.11.0.jar new file mode 100644 index 000000000..0aaa990f3 Binary files /dev/null and b/utilities/sdb_to_tdb/lib/xercesImpl-2.11.0.jar differ diff --git a/utilities/sdb_to_tdb/lib/xml-apis-1.4.01.jar b/utilities/sdb_to_tdb/lib/xml-apis-1.4.01.jar new file mode 100644 index 000000000..46733464f Binary files /dev/null and b/utilities/sdb_to_tdb/lib/xml-apis-1.4.01.jar differ diff --git a/utilities/sdb_to_tdb/src/edu/cornell/mannlib/vitro/utilities/sdb2tdb/Sdb2Tdb.java b/utilities/sdb_to_tdb/src/edu/cornell/mannlib/vitro/utilities/sdb2tdb/Sdb2Tdb.java new file mode 100644 index 000000000..5a9283abf --- /dev/null +++ b/utilities/sdb_to_tdb/src/edu/cornell/mannlib/vitro/utilities/sdb2tdb/Sdb2Tdb.java @@ -0,0 +1,182 @@ +/* $This file is distributed under the terms of the license in /doc/license.txt$ */ + +package edu.cornell.mannlib.vitro.utilities.sdb2tdb; + +import java.io.File; +import java.io.FilenameFilter; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; + +import com.hp.hpl.jena.query.Dataset; +import com.hp.hpl.jena.rdf.model.Model; +import com.hp.hpl.jena.sdb.SDBFactory; +import com.hp.hpl.jena.sdb.Store; +import com.hp.hpl.jena.sdb.StoreDesc; +import com.hp.hpl.jena.sdb.store.DatabaseType; +import com.hp.hpl.jena.sdb.store.LayoutType; +import com.hp.hpl.jena.tdb.TDBFactory; + +/** + *
+ *    java -jar sdb2tdb.jar \ 
+ *    		'jdbc:mysql://localhost/vitrodb?user=vivoUser&password=vivoPass'\ 
+ *    		/usr/local/my/tdb
+ * 
+ */ +public class Sdb2Tdb { + private final String driverClassName; + private final String jdbcUrl; + private final String destination; + private final boolean force; + + private Dataset sdbDataset; + private Dataset tdbDataset; + + public Sdb2Tdb(List hardArgs) throws UsageException { + List args = new ArrayList<>(hardArgs); + + if (!args.isEmpty() && args.indexOf("force") == (args.size() - 1)) { + this.force = true; + args.remove(args.size() - 1); + } else { + this.force = false; + } + + if (args.size() == 3) { + this.driverClassName = args.remove(0); + } else if (args.size() == 2) { + this.driverClassName = "com.mysql.jdbc.Driver"; + } else { + throw new UsageException("Wrong number of arguments: " + + hardArgs.size()); + } + + this.jdbcUrl = args.get(0); + this.destination = args.get(1); + + checkDriverClass(); + checkJdbcUrl(); + checkDestination(); + } + + private void checkDriverClass() throws UsageException { + try { + Class.forName(this.driverClassName).newInstance(); + } catch (Exception e) { + throw new UsageException("Can't instantiate JDBC driver: " + + this.driverClassName); + } + } + + private void checkJdbcUrl() { + if ((!this.jdbcUrl.matches("\\busername\\b")) + || (!this.jdbcUrl.matches("\\bpassword\\b"))) { + System.out.println("\nWARNING: The JDBC url probably should " + + "contain values for username and password.\n"); + } + } + + private void checkDestination() throws UsageException { + File destDir = new File(this.destination); + + if (!destDir.isDirectory()) { + throw new UsageException( + "The destination directory does not exist: '" + + this.destination + "'"); + } + + if (!destDir.canWrite()) { + throw new UsageException("Cannot write to '" + this.destination + + "'"); + } + + if (!(this.force || getDestinationFilenames().isEmpty())) { + throw new UsageException("The destination directory is not empty. " + + "Choose another destination, or use the 'force' option"); + } + } + + private List getDestinationFilenames() { + File destDir = new File(this.destination); + String[] filenames = destDir.list(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return !(name.equals(".") || name.equals("..")); + } + }); + return Arrays.asList(filenames); + } + + private void translate() throws SQLException { + try { + sdbDataset = openSdbDataset(); + tdbDataset = openTdbDataset(); + copyGraphs(); + } finally { + if (tdbDataset != null) { + tdbDataset.close(); + } + if (sdbDataset != null) { + sdbDataset.close(); + } + } + } + + private Dataset openSdbDataset() throws SQLException { + Connection conn = DriverManager.getConnection(this.jdbcUrl); + Store store = SDBFactory.connectStore(conn, makeSdbStoreDesc()); + return SDBFactory.connectDataset(store); + } + + private StoreDesc makeSdbStoreDesc() { + return new StoreDesc(LayoutType.LayoutTripleNodesHash, + DatabaseType.MySQL); + } + + private Dataset openTdbDataset() { + return TDBFactory.createDataset(new File(this.destination) + .getAbsolutePath()); + } + + private void copyGraphs() { + for (Iterator modelNames = sdbDataset.listNames(); modelNames + .hasNext();) { + String modelName = modelNames.next(); + Model model = sdbDataset.getNamedModel(modelName); + System.out.println(String.format("Copying %6d triples: %s", + model.size(), modelName)); + tdbDataset.addNamedModel(modelName, model); + model.close(); + } + } + + public static void main(String[] args) { + try { + Sdb2Tdb sdb2tdb = new Sdb2Tdb(Arrays.asList(args)); + sdb2tdb.translate(); + + } catch (UsageException e) { + System.out.println(); + System.out.println(e.getMessage()); + System.out.println(e.getProperUsage()); + System.out.println(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + private static class UsageException extends Exception { + public UsageException(String message) { + super(message); + } + + public String getProperUsage() { + return "Usage is: java -jar sdb2tdb [driver_class] [force]"; + } + } +}