NIHVIVO-160 Complete the first iteration of coding and testing - especially avoidance of Windows reserved words.

This commit is contained in:
jeb228 2010-05-25 20:33:32 +00:00
parent 60958400f1
commit a16985ccaa
6 changed files with 381 additions and 79 deletions

View file

@ -24,11 +24,21 @@ public interface FileStorage {
*/
String PROPERTY_DEFAULT_NAMESPACE = "Vitro.defaultNamespace";
/**
* The name of the root directory, within the base directory.
*/
public static final String FILE_STORAGE_ROOT = "file_storage_root";
/**
* The name of the file in the base directory that holds the namespace map.
*/
public static final String FILE_STORAGE_NAMESPACES_PROPERTIES = "file_storage_namespaces.properties";
/**
* How often to we insert path separator characters?
*/
int SHORTY_LENGTH = 3;
/**
* Store the bytes from this stream as a file with the specified ID and
* filename. If the file already exists, it is over-written.

View file

@ -42,6 +42,15 @@ public class FileStorageHelper {
public static final char[] NAME_SINGLE_CHARACTER_TARGETS = new char[] {
'=', '+' };
/**
* Windows reserves these names (case-insensitive), so they can't be used
* for directories or files.
*/
public static final String[] WINDOWS_RESERVED_NAMES = new String[] { "CON",
"PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "COM5",
"COM6", "COM7", "COM8", "COM9", "LPT1", "LPT2", "LPT3", "LPT4",
"LPT5", "LPT6", "LPT7", "LPT8", "LPT9" };
/**
* Encode the filename as needed to guard against illegal characters.
*
@ -49,8 +58,9 @@ public class FileStorageHelper {
*/
public static String encodeName(String filename) {
String hexed = addHexEncoding(filename);
return addSingleCharacterConversions(hexed,
String cleaned = addSingleCharacterConversions(hexed,
NAME_SINGLE_CHARACTER_SOURCES, NAME_SINGLE_CHARACTER_TARGETS);
return excludeWindowsReservedNames(cleaned);
}
/**
@ -123,17 +133,47 @@ public class FileStorageHelper {
return c;
}
/**
* If a requested filename, after cleaning, is one of the Windows reserved
* words, add a tilde in front.
*/
private static String excludeWindowsReservedNames(String cleanedName) {
for (String word : WINDOWS_RESERVED_NAMES) {
if (word.equalsIgnoreCase(cleanedName)) {
return '~' + cleanedName;
}
}
return cleanedName;
}
/**
* Restore the filename to its original form, removing the encoding.
*
* @see edu.cornell.mannlib.vitro.webapp.utils.filestorage
*/
public static String decodeName(String coded) {
String hexed = removeSingleCharacterConversions(coded,
public static String decodeName(String stored) {
String unexcluded = unexcludeWindowsReservedNames(stored);
String hexed = removeSingleCharacterConversions(unexcluded,
NAME_SINGLE_CHARACTER_SOURCES, NAME_SINGLE_CHARACTER_TARGETS);
return removeHexEncoding(hexed);
}
/**
* If the stored filename was a tilde followed by a Windows reserved word,
* strip the tilde.
*/
private static String unexcludeWindowsReservedNames(String stored) {
if (stored.startsWith("~")) {
String remainder = stored.substring(1);
for (String word : WINDOWS_RESERVED_NAMES) {
if (word.equalsIgnoreCase(remainder)) {
return remainder;
}
}
}
return stored;
}
/**
* Convert common single-character substitutions back to their original
* values.
@ -186,7 +226,7 @@ public class FileStorageHelper {
* Translate the object ID to a relative directory path. A recognized
* namespace is translated to its prefix, and illegal characters are
* encoded. The resulting string is broken up into 3-character directory
* names (or less).
* names (or less). Windows reserved words are prefixed with tilde.
*
* @see edu.cornell.mannlib.vitro.webapp.utils.filestorage
*/
@ -206,7 +246,11 @@ public class FileStorageHelper {
String cleaned = addSingleCharacterConversions(hexed,
PATH_SINGLE_CHARACTER_SOURCES, PATH_SINGLE_CHARACTER_TARGETS);
String prefixed = applyPrefixChar(prefix, cleaned);
return insertPathDelimiters(prefixed);
String brokenUp = insertPathDelimiters(prefixed);
String result = excludeWindowsWordsFromPath(brokenUp);
LOG.debug("id2Path: id='" + id + "', namespaces='" + namespacesMap
+ "', path='" + result + "'");
return result;
}
/**
@ -232,9 +276,31 @@ public class FileStorageHelper {
}
path.append(prefixed.charAt(i));
}
LOG.debug("Insert path delimiters to '" + prefixed + "' giving '"
+ path + "'");
return path.toString();
}
/**
* Check each part in the path, and if it is a Windows reserved word, add a
* tilde. This only applies to the relative path.
*/
private static String excludeWindowsWordsFromPath(String rawPath) {
String path = rawPath.replace(File.separatorChar, '/');
String[] parts = path.split("/");
StringBuilder newPath = new StringBuilder();
for (int i = 0; i < parts.length; i++) {
String part = excludeWindowsReservedNames(parts[i]);
if (i > 0) {
newPath.append(File.separatorChar);
}
newPath.append(part);
}
return newPath.toString();
}
/**
* Translate the object ID and the file storage root directory into a full
* path to the directory that would represent that ID.

View file

@ -14,6 +14,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.io.Reader;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@ -69,10 +70,10 @@ public class FileStorageImpl implements FileStorage {
checkNamespacesValid(namespaces);
this.baseDir = baseDir;
this.rootDir = new File(this.baseDir, "file_storage_root");
this.rootDir = new File(this.baseDir, FILE_STORAGE_ROOT);
this.namespaceFile = new File(baseDir,
"file_storage_namespaces.properties");
FILE_STORAGE_NAMESPACES_PROPERTIES);
if (rootDir.exists() && namespaceFile.exists()) {
this.namespacesMap = confirmNamespaces(namespaces);
@ -80,15 +81,15 @@ public class FileStorageImpl implements FileStorage {
this.namespacesMap = mapNamespaces(namespaces);
initializeStorage();
} else if (rootDir.exists()) {
throw new IllegalStateException(
"Storage directory '' has been partially initialized. '"
+ rootDir.getPath() + "' exists, but '"
+ namespaceFile.getPath() + "' does not.");
throw new IllegalStateException("Storage directory '"
+ baseDir.getPath() + "' has been partially initialized. '"
+ FILE_STORAGE_ROOT + "' exists, but '"
+ FILE_STORAGE_NAMESPACES_PROPERTIES + "' does not.");
} else {
throw new IllegalStateException(
"Storage directory '' has been partially initialized. '"
+ namespaceFile.getPath() + "' exists, but '"
+ rootDir.getPath() + "' does not.");
throw new IllegalStateException("Storage directory '"
+ baseDir.getPath() + "' has been partially initialized. '"
+ FILE_STORAGE_NAMESPACES_PROPERTIES + "' exists, but '"
+ FILE_STORAGE_ROOT + "' does not.");
}
}
@ -222,9 +223,11 @@ public class FileStorageImpl implements FileStorage {
private Map<Character, String> confirmNamespaces(
Collection<String> namespaces) throws IOException {
Map<Character, String> map;
Reader reader = null;
try {
reader = new FileReader(this.namespaceFile);
Properties props = new Properties();
props.load(new FileReader(this.namespaceFile));
props.load(reader);
map = new HashMap<Character, String>();
for (Object key : props.keySet()) {
char keyChar = key.toString().charAt(0);
@ -232,6 +235,14 @@ public class FileStorageImpl implements FileStorage {
}
} catch (Exception e) {
throw new IOException("Problem loading the namespace file.");
} finally {
if (reader != null) {
try {
reader.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
Set<String> requestedNamespaces = new HashSet<String>(namespaces);
@ -248,12 +259,29 @@ public class FileStorageImpl implements FileStorage {
return map;
}
// ----------------------------------------------------------------------
// package access methods -- used in unit tests.
// ----------------------------------------------------------------------
File getBaseDir() {
return this.baseDir;
}
Map<Character, String> getNamespaces() {
return this.namespacesMap;
}
// ----------------------------------------------------------------------
// Public methods
// ----------------------------------------------------------------------
/**
* {@inheritDoc}
*
* <p>
* Before creating the file, we may need to create one or more parent
* directories to put it in.
* </p>
*/
@Override
public void createFile(String id, String filename, InputStream bytes)
@ -266,6 +294,16 @@ public class FileStorageImpl implements FileStorage {
File file = FileStorageHelper.getFullPath(this.rootDir, id, filename,
this.namespacesMap);
File parent = file.getParentFile();
if (!parent.exists()) {
parent.mkdirs();
if (!parent.exists()) {
throw new IOException(
"Failed to create parent directories for file with ID '"
+ id + "', file location '" + file + "'");
}
}
OutputStream out = null;
try {

View file

@ -134,8 +134,17 @@
* </li>
* <li>
* <strong>Path breakdown</strong> -
* Finally, path separator characters are inserted after every third
* character in the processed ID string.
* Path separator characters are inserted after every third character
* in the processed ID string.
* </li>
* <li>
* <strong>Exclusion of reserved Windows filenames</strong> -
* Windows will not permit certain specific filename or directory names,
* so if any part of the path would be equal to one of those reserved
* names, it is prefixed with a tilde. The reserved names are:
* CON, PRN, AUX, NUL, COM1, COM2, COM3, COM4, COM5, COM6, COM7, COM8,
* COM9, LPT1, LPT2, LPT3, LPT4, LPT5, LPT6, LPT7, LPT8, and LPT9.
* And remember, Windows is case-insensitive.
* </li>
* </ul>
* Examples:
@ -161,7 +170,8 @@
*
* <p>
* The encoding process is the same as the "rare character encoding" and
* "common character encoding" steps used for ID encoding.
* "common character encoding" steps used for ID encoding, except that
* periods are not encoded.
* </p>
*/

View file

@ -48,6 +48,16 @@ public class FileStorageHelperTest {
private static File FULL_RESULT_PATH = new File(
"/usr/local/vivo/uploads/file_storage_root/b~n/323/4,X/XX/myPhoto.jpg");
private static Map<Character, String> WINDOWS_PREFIX_MAP = initWindowsPrefixMap();
/** This reserved word will be modified. */
private static String WINDOWS_NAME = "lpT8";
/** This ID would translate to a path with a reserved word. */
private static String WINDOWS_ID = "prefix:createdConflict";
/** Not allowed to change the root, even if it contains reserved words. */
private static File WINDOWS_ROOT = new File("/usr/aux/root/");
private static File WINDOWS_FULL_PATH = new File(
"/usr/aux/root/a~c/rea/ted/~Con/fli/ct/~lpT8");
private static Map<Character, String> EMPTY_NAMESPACES = Collections
.emptyMap();
private static Map<Character, String> NAMESPACES = initPrefixMap();
@ -59,6 +69,12 @@ public class FileStorageHelperTest {
return map;
}
private static Map<Character, String> initWindowsPrefixMap() {
Map<Character, String> map = new HashMap<Character, String>();
map.put('a', "prefix:");
return map;
}
// ----------------------------------------------------------------------
// encodeName
// ----------------------------------------------------------------------
@ -199,4 +215,11 @@ public class FileStorageHelperTest {
assertEquals("fullPath", FULL_RESULT_PATH, actual);
}
@Test
public void checkWindowsExclusions() {
File actual = FileStorageHelper.getFullPath(WINDOWS_ROOT, WINDOWS_ID,
WINDOWS_NAME, WINDOWS_PREFIX_MAP);
assertEquals("windows exclusion", WINDOWS_FULL_PATH, actual);
}
}

View file

@ -2,9 +2,26 @@
package edu.cornell.mannlib.vitro.webapp.utils.filestorage;
import static org.junit.Assert.*;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import org.junit.Ignore;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.apache.log4j.Level;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import edu.cornell.mannlib.vitro.testing.AbstractTestClass;
@ -14,105 +31,243 @@ import edu.cornell.mannlib.vitro.testing.AbstractTestClass;
* {@link FileStorageFactoryTest}.
*/
public class FileStorageImplTest extends AbstractTestClass {
@Ignore
@Test
public void baseDirDoesntExist() {
fail("baseDirDoesntExist not implemented");
private static final List<String> EMPTY_NAMESPACES = Collections
.emptyList();
private static File tempDir;
private static FileStorageImpl generalFs;
@BeforeClass
public static void createSomeDirectories() throws IOException {
tempDir = createTempDirectory(FileStorageImplTest.class.getSimpleName());
generalFs = createFileStorage("general");
}
@Ignore
@Test
public void partialInitializationRoot() {
fail("partialInitializationRoot not implemented");
@AfterClass
public static void cleanUp() {
if (tempDir != null) {
purgeDirectoryRecursively(tempDir);
}
}
@Ignore
@Test
public void partialInitializationNamespaces() {
fail("partialInitializationNamespaces not implemented");
@Test(expected = IllegalArgumentException.class)
public void baseDirDoesntExist() throws IOException {
File baseDir = new File(tempDir, "doesntExist");
new FileStorageImpl(baseDir, EMPTY_NAMESPACES);
}
@Ignore
@Test
public void notInitialized() {
fail("notInitialized not implemented");
@Test(expected = IllegalStateException.class)
public void partialInitializationRoot() throws IOException {
File baseDir = new File(tempDir, "partialWithRoot");
baseDir.mkdir();
new File(baseDir, FileStorage.FILE_STORAGE_ROOT).mkdir();
new FileStorageImpl(baseDir, EMPTY_NAMESPACES);
}
@Ignore
@Test
public void initializedOK() {
fail("initializedOK not implemented");
@Test(expected = IllegalStateException.class)
public void partialInitializationNamespaces() throws IOException {
File baseDir = new File(tempDir, "partialWithNamespaces");
baseDir.mkdir();
new File(baseDir, FileStorage.FILE_STORAGE_NAMESPACES_PROPERTIES)
.createNewFile();
new FileStorageImpl(baseDir, EMPTY_NAMESPACES);
}
@Ignore
@Test
public void initializedNamespacesDontMatch() {
fail("initializedNamespacesDontMatch not implemented");
public void notInitializedNoNamespaces() throws IOException {
File baseDir = new File(tempDir, "emptyNoNamespaces");
baseDir.mkdir();
FileStorageImpl fs = new FileStorageImpl(baseDir,
new ArrayList<String>());
assertEquals("baseDir", baseDir, fs.getBaseDir());
assertEqualSets("namespaces", new String[0], fs.getNamespaces()
.values());
}
@Ignore
@Test
public void createFileOriginal() {
fail("createFileOriginal not implemented");
public void notInitializedNamespaces() throws IOException {
String[] namespaces = new String[] { "ns1", "ns2" };
String dirName = "emptyWithNamespaces";
FileStorageImpl fs = createFileStorage(dirName, namespaces);
assertEquals("baseDir", new File(tempDir, dirName), fs.getBaseDir());
assertEqualSets("namespaces", namespaces, fs.getNamespaces().values());
}
@Ignore
@Test
public void createFileOverwrite() {
fail("createFileOverwrite not implemented");
public void initializedOK() throws IOException {
createFileStorage("initializeTwiceTheSame", "ns1", "ns2");
createFileStorage("initializeTwiceTheSame", "ns2", "ns1");
}
@Ignore
@Test
public void createFileConflictingName() {
fail("createFileConflictingName not implemented");
@Test(expected = IllegalStateException.class)
public void initializedNamespacesDontMatch() throws IOException {
createFileStorage("initializeTwiceDifferent", "ns1", "ns2");
createFileStorage("initializeTwiceDifferent", "ns2");
}
@Ignore
@Test
public void getFilenameExists() {
fail("getFilenameExists not implemented");
public void createFileOriginal() throws IOException {
String id = "createOriginal";
String filename = "someName.txt";
String contents = "these contents";
InputStream bytes = new ByteArrayInputStream(contents.getBytes());
generalFs.createFile(id, filename, bytes);
assertFileContents(generalFs, id, filename, contents);
}
@Ignore
@Test
public void getFilenameDoesntExist() {
fail("getFilenameDoesntExist not implemented");
public void createFileOverwrite() throws IOException {
String id = "createOverwrite";
String filename = "someName.txt";
String contents1 = "these contents";
InputStream bytes1 = new ByteArrayInputStream(contents1.getBytes());
String contents2 = "a different string";
InputStream bytes2 = new ByteArrayInputStream(contents2.getBytes());
generalFs.createFile(id, filename, bytes1);
generalFs.createFile(id, filename, bytes2);
assertFileContents(generalFs, id, filename, contents2);
}
@Ignore
@Test
public void getFilenameMultipleFiles() {
fail("getFilenameMultipleFiles not implemented");
public void createFileConflictingName() throws IOException {
String id = "createConflict";
String filename1 = "someName.txt";
String filename2 = "secondFileName.txt";
String contents = "these contents";
InputStream bytes = new ByteArrayInputStream(contents.getBytes());
generalFs.createFile(id, filename1, bytes);
try {
generalFs.createFile(id, filename2, bytes);
fail("Expected FileAlreadyExistsException.");
} catch (FileAlreadyExistsException e) {
// expected it.
}
}
@Ignore
@Test
public void getFileFound() {
fail("getFilenameFound not implemented");
public void getFilenameExists() throws IOException {
String id = "filenameExists";
String filename = "theName.txt";
String contents = "the contents";
InputStream bytes = new ByteArrayInputStream(contents.getBytes());
generalFs.createFile(id, filename, bytes);
assertEquals("filename", filename, generalFs.getFilename(id));
}
@Ignore
@Test
public void getFileNotFound() {
fail("getFileNotFound not implemented");
public void getFilenameDoesntExist() throws IOException {
assertNull("null filename", generalFs.getFilename("neverHeardOfIt"));
}
@Ignore
@Test
public void getFileTooLarge() {
fail("getFileTooLarge not implemented");
public void getInputStreamFound() throws IOException {
String id = "inputStreamExists";
String filename = "myFile";
String contents = "Some stuff to put into my file.";
InputStream bytes = new ByteArrayInputStream(contents.getBytes());
generalFs.createFile(id, filename, bytes);
assertFileContents(generalFs, id, filename, contents);
assertEquals("getInputStream", contents, readAll(generalFs
.getInputStream(id, filename)));
}
@Ignore
@Test
public void deleteFileExists() {
fail("deleteFileExists not implemented");
@Test(expected = FileNotFoundException.class)
public void getInputStreamNotFound() throws IOException {
generalFs.getInputStream("notFound", "nothing");
}
@Ignore
@Test
public void deleteFileDoesntExist() {
fail("deleteFileDoesntExist not implemented");
public void deleteFileExists() throws IOException {
String id = "deleteMe";
String filename = "deadFile";
String contents = "Some stuff to put into my file.";
InputStream bytes = new ByteArrayInputStream(contents.getBytes());
generalFs.createFile(id, filename, bytes);
generalFs.deleteFile(id);
assertNull("deleted filename", generalFs.getFilename(id));
}
@Test
public void deleteFileDoesntExist() throws IOException {
generalFs.deleteFile("totallyBogus");
}
@Test
public void exerciseWindowsExclusions() throws FileAlreadyExistsException,
IOException {
// setLoggerLevel(FileStorageHelper.class, Level.DEBUG);
String id = "nul";
String filename = "COM1";
String contents = "Windows doesn't like certain names.";
InputStream bytes = new ByteArrayInputStream(contents.getBytes());
generalFs.createFile(id, filename, bytes);
assertFileContents(generalFs, id, filename, contents);
assertEquals("filename", filename, generalFs.getFilename(id));
assertEquals("getInputStream", contents, readAll(generalFs
.getInputStream(id, filename)));
}
// ----------------------------------------------------------------------
// helper methods
// ----------------------------------------------------------------------
private static FileStorageImpl createFileStorage(String dirName,
String... namespaces) throws IOException {
File baseDir = new File(tempDir, dirName);
baseDir.mkdir();
return new FileStorageImpl(baseDir, Arrays.asList(namespaces));
}
private <T> void assertEqualSets(String message, T[] expected,
Collection<T> actual) {
Set<T> expectedSet = new HashSet<T>(Arrays.asList(expected));
if (expectedSet.size() != expected.length) {
fail("message: expected array contains duplicate elements: "
+ Arrays.deepToString(expected));
}
Set<T> actualSet = new HashSet<T>(actual);
if (actualSet.size() != actual.size()) {
fail("message: actual collection contains duplicate elements: "
+ actual);
}
assertEquals(message, expectedSet, actualSet);
}
/**
* This file storage should contain a file with this ID and this name, and
* it should have these contents.
*/
private void assertFileContents(FileStorageImpl fs, String id,
String filename, String expectedContents) throws IOException {
File rootDir = new File(fs.getBaseDir(), FileStorage.FILE_STORAGE_ROOT);
File path = FileStorageHelper.getFullPath(rootDir, id, filename, fs
.getNamespaces());
assertTrue("file exists: " + path, path.exists());
String actualContents = readFile(path);
assertEquals("file contents", expectedContents, actualContents);
}
}