May 2010 - Diswork-commits - mailman.cloud.codelutin.com

r53 - in trunk/diswork-fs/src/main/java/org/nuiton/disworkfs: . storage
by bleny＠users.nuiton.org May 26, 2010

May 26, 2010

Author: bleny Date: 2010-05-26 10:44:36 +0200 (Wed, 26 May 2010) New Revision: 53 Url: http://nuiton.org/repositories/revision/diswork/53 Log: exceptions, suppression de donnees retardees Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-25 16:24:08 UTC (rev 52) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-26 08:44:36 UTC (rev 53) @@ -1,9 +1,11 @@ package org.nuiton.disworkfs; +import java.io.Closeable; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; +import java.util.ConcurrentModificationException; import java.util.List; import org.apache.commons.logging.Log; @@ -24,7 +26,7 @@ * on directories, files and symlinks</li> * </ul> */ -public class DisworkFileSystem { +public class DisworkFileSystem implements Closeable { /** * storage will permit to save and read directories, files and links @@ -104,8 +106,14 @@ * @param path * @param source * @throws IOException if file already exists + * @throws ConcurrentModificationException if file is already being written */ - public void write(String path, InputStream source) throws IOException { + public void write(String path, InputStream source) + throws IOException, + ConcurrentModificationException { + if (source == null) { + throw new IOException("source is not readable (null stream)"); + } String parent = EntryUtil.getParentFromPath(path); String name = EntryUtil.getNameFromPath(path); write(parent, name, source); @@ -149,13 +157,16 @@ * @param path the absolute path (ending by the name) of the directory * @throws IOException if parent path is not correct */ - public void createDirectory(String path) throws IOException { + public void createDirectory(String path) + throws IOException, + ConcurrentModificationException { String parent = EntryUtil.getParentFromPath(path); String dirName = EntryUtil.getNameFromPath(path); createDirectory(parent, dirName); } - protected void createDirectory(String parent, String dirName) throws IOException { + protected void createDirectory(String parent, String dirName) + throws IOException { log.info("trying to create directory " + dirName + " in " + parent); @@ -190,7 +201,8 @@ * @throws IOException if parent path is not correct */ public void createSymbolicLink(String path, String target) - throws IOException { + throws IOException, + ConcurrentModificationException { String parent = EntryUtil.getParentFromPath(path); String name = EntryUtil.getNameFromPath(path); createSymbolicLink(parent, name, target); @@ -244,7 +256,8 @@ * @param path the complete path to the entity to remove * @throws IOException if path is incorrect or directory not empty */ - public void delete(String path) throws IOException { + public void delete(String path) throws IOException, + ConcurrentModificationException { String parent = EntryUtil.getParentFromPath(path); String name = EntryUtil.getNameFromPath(path); log.info("trying to remove " + path); @@ -352,10 +365,9 @@ * @return null if path is not valid or the entry corresponding to path * @throws IOException */ - - // FIXME 20105021 bleny works fine but is not understandable protected String walk(String path, String current, String content) throws IOException { + // FIXME 20105021 bleny works fine but is not understandable String result = null; if (path.equals("/")) { @@ -437,5 +449,10 @@ } return result; } + + @Override + public void close() throws IOException { + storage.close(); + } } Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-05-25 16:24:08 UTC (rev 52) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-05-26 08:44:36 UTC (rev 53) @@ -1,9 +1,12 @@ package org.nuiton.disworkfs.storage; import java.io.ByteArrayInputStream; +import java.io.Closeable; import java.io.IOException; import java.io.InputStream; +import java.util.ArrayList; import java.util.ConcurrentModificationException; +import java.util.List; import java.util.Map; import org.apache.commons.io.IOUtils; @@ -14,10 +17,14 @@ /** * */ -public class Storage { +public class Storage implements Closeable { private static final Log log = LogFactory.getLog(Storage.class); + + protected static final long LOCK_VALID_TIME = 60 * 60 * 1000; + protected List<String> idsToBeRemoved = new ArrayList<String>(); + /** * This class is a InputStream used to read data from the map. When * someone read from the filesytem, he will bre returned a @@ -147,6 +154,8 @@ } public void putFile(String id, InputStream content) throws IOException { + log.debug("putFile(\"" + id + "\", stream of "+ content.available() + + " bytes)"); put(id, content); } @@ -171,11 +180,9 @@ * @param value * @throws IOException */ - protected void put(String key, InputStream value) throws IOException { - // TODO 20100519 bleny deal with null value properly - if (value == null) { - value = new ByteArrayInputStream(new byte[0]); - } + protected void put(String key, InputStream value) + throws IOException, + ConcurrentModificationException { String lockKey = key + "_lock"; String newDataKey = key + "_newData"; @@ -188,19 +195,20 @@ // file is locked, check date Long currentLock = Long.parseLong(EntryUtil.bytesToString(lock)); - if (currentDate - currentLock > 60 * 60 * 1000) { - // this lock is out dated, let's erase it - String obsoleteMetaBlock = EntryUtil.bytesToString(map.get(newDataKey)); + if (currentDate - currentLock > LOCK_VALID_TIME) { + // this lock is out-dated, let's erase it + String obsoleteMetaBlock = + EntryUtil.bytesToString(map.get(newDataKey)); if (obsoleteMetaBlock != null) { - String[] obsoleteBlocksIds = EntryUtil.getBlockIdsFromMetaBlock(obsoleteMetaBlock); + String[] obsoleteBlocksIds = + EntryUtil.getBlockIdsFromMetaBlock(obsoleteMetaBlock); for (String obsoleteBlockId : obsoleteBlocksIds) { map.remove(obsoleteBlockId); } } } else { - // file is currently written, stopping + // file is currently written, stopping operation map.put(lockKey, lock); - // FIXME 20100525 bleny should be declared in throws throw new ConcurrentModificationException("can't write due to concurrency"); } } @@ -253,9 +261,28 @@ map.remove(lockKey); } - public void remove(Object key) { - // TODO poussin 20100518 mark bloque to remove - map.remove(key); + /** + * due to concurrency, key should not be removed immediately. Someone may + * read it now. It will be removed later, using {@link #clean()}. + * @param key the key to be removed from the map + */ + public void remove(String key) { + idsToBeRemoved.add(key); } + + /** + * this method clean the map by removing all the blocks that are not + * referenced (ie, their key is not in any meta-block or in entries) + */ + public void clean() { + for (String key : idsToBeRemoved) { + map.remove(key); + } + } + @Override + public void close() throws IOException { + clean(); + } + } \ No newline at end of file

1 0

r52 - trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage
by bleny＠users.nuiton.org May 25, 2010

May 25, 2010

Author: bleny Date: 2010-05-25 18:24:08 +0200 (Tue, 25 May 2010) New Revision: 52 Url: http://nuiton.org/repositories/revision/diswork/52 Log: untested implementation of copy-on-write to deal with concurrency Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-05-25 15:24:32 UTC (rev 51) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-05-25 16:24:08 UTC (rev 52) @@ -3,6 +3,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; +import java.util.ConcurrentModificationException; import java.util.Map; import org.apache.commons.io.IOUtils; @@ -165,54 +166,91 @@ } /** - * a put in the map, this involves split considerations + * a put in the map, this involves split considerations and concurrency * @param key * @param value * @throws IOException */ protected void put(String key, InputStream value) throws IOException { - - // TODO 20100519 bleny deal with copy-on-write // TODO 20100519 bleny deal with null value properly if (value == null) { value = new ByteArrayInputStream(new byte[0]); } + String lockKey = key + "_lock"; + String newDataKey = key + "_newData"; + + // trying to acquire lock + Long currentDate = System.currentTimeMillis(); + byte[] lock = map.put(lockKey, EntryUtil.stringToBytes(currentDate.toString())); + + if (lock != null) { + // file is locked, check date + Long currentLock = Long.parseLong(EntryUtil.bytesToString(lock)); + + if (currentDate - currentLock > 60 * 60 * 1000) { + // this lock is out dated, let's erase it + String obsoleteMetaBlock = EntryUtil.bytesToString(map.get(newDataKey)); + if (obsoleteMetaBlock != null) { + String[] obsoleteBlocksIds = EntryUtil.getBlockIdsFromMetaBlock(obsoleteMetaBlock); + for (String obsoleteBlockId : obsoleteBlocksIds) { + map.remove(obsoleteBlockId); + } + } + } else { + // file is currently written, stopping + map.put(lockKey, lock); + // FIXME 20100525 bleny should be declared in throws + throw new ConcurrentModificationException("can't write due to concurrency"); + } + } + + // here, we know we can write or an exception + // would have been thrown earlier String blocksIds = ""; int readResult = 0; int totalSize = 0; + String metaBlock = totalSize + blocksIds; + map.put(newDataKey, EntryUtil.stringToBytes(metaBlock)); + // creating a buffer of the size of a block int bufferSize = disworkConfig.getBlockSize(); byte[] buffer = new byte[bufferSize]; - + while ((readResult = value.read(buffer)) != -1) { totalSize += readResult; - + byte[] newBlock = buffer; - + // if the buffer is not full, truncate the block if (readResult < buffer.length) { newBlock = new byte[readResult]; System.arraycopy(buffer, 0, newBlock, 0, readResult); } - + String id = EntryUtil.generateId(); blocksIds += EntryUtil.BLOCKIDS_SEPARATOR + id; log.debug("saving new block (size = " + newBlock.length + ")" + " at key " + id); - + // copy block in map map.put(id, newBlock); + + metaBlock = totalSize + blocksIds; + + log.debug("putting metablock " + metaBlock + " at key " + key); + map.put(newDataKey, EntryUtil.stringToBytes(metaBlock)); + + // updating lock + currentDate = System.currentTimeMillis(); + map.put(lockKey, EntryUtil.stringToBytes(currentDate.toString())); } - String metaBlock = totalSize + blocksIds; - - log.debug("putting metablock " + metaBlock + " at key " + key); - - map.put(key, EntryUtil.stringToBytes(metaBlock)); + map.put(key, map.get(newDataKey)); + map.remove(lockKey); } public void remove(Object key) {

1 0

r51 - in trunk/diswork-fs/src: main/java/org/nuiton/disworkfs main/java/org/nuiton/disworkfs/storage main/resources test/java/org/nuiton/disworkfs
by bleny＠users.nuiton.org May 25, 2010

May 25, 2010

Author: bleny Date: 2010-05-25 17:24:32 +0200 (Tue, 25 May 2010) New Revision: 51 Url: http://nuiton.org/repositories/revision/diswork/51 Log: menage, documentation, tests Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/package-info.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/package-info.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java Removed: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkConfig.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryMap.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java trunk/diswork-fs/src/main/resources/log4j.properties trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java Deleted: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java 2010-05-21 16:46:41 UTC (rev 50) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java 2010-05-25 15:24:32 UTC (rev 51) @@ -1,28 +0,0 @@ -package org.nuiton.disworkfs; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.util.List; - -public interface DistributedFileSystem { - - public boolean exists(String path) throws IOException; - - public void write(String path, InputStream source) throws IOException; - - public void write(String parent, String fileName, InputStream source) - throws IOException; - - public InputStream read(String path) throws FileNotFoundException, - IOException; - - public void mkdir(String path) throws IOException; - - public void ln(String path, String target) throws IOException; - - public void remove(String path) throws IOException; - - public List<String> ls(String path) throws IOException; - -} Deleted: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkConfig.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkConfig.java 2010-05-21 16:46:41 UTC (rev 50) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkConfig.java 2010-05-25 15:24:32 UTC (rev 51) @@ -1,14 +0,0 @@ -package org.nuiton.disworkfs; - -import org.nuiton.util.ApplicationConfig; - -public class DisworkConfig extends ApplicationConfig { - - public DisworkConfig() { - setDefaultOption("blocks_size", "10485760"); // 10 MiB - } - - public int getBlockSize() { - return Integer.parseInt(getOption("blocks_size")); - } -} Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-21 16:46:41 UTC (rev 50) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-25 15:24:32 UTC (rev 51) @@ -12,122 +12,67 @@ import org.nuiton.disworkfs.storage.InMemoryMap; import org.nuiton.disworkfs.storage.Storage; -public class DisworkFileSystem implements DistributedFileSystem { +/** Main class of Diswork File System, provide methods for all operations + * You can use: + * <ul> + * <li>{@link #createDirectory(String)} and {@link DisworkFileSystem#readDirectory(String)} + * to create and browse directories</li> + * <li>{@link #write(String, InputStream)} and {@link #read(String)} + * to write a file and read it</li> + * <li>{@link #createSymbolicLink(String, String)} to create symbolic links</li> + * <li>{@link #exists(String)} and {@link #delete(String)} can be used + * on directories, files and symlinks</li> + * </ul> + */ +public class DisworkFileSystem { + /** + * storage will permit to save and read directories, files and links + */ protected Storage storage; private static final Log log = LogFactory.getLog(DisworkFileSystem.class); - public DisworkFileSystem(DisworkConfig disworkConfig) throws IOException { + /** default constructor (uses default configuration parameters) + * @throws IOException + */ + public DisworkFileSystem() throws IOException { + this(new DisworkFileSystemConfig()); + } + + /** constructor allowing to provide another configuration to use + * + * @param disworkConfig the configuration + * @throws IOException + */ + public DisworkFileSystem(DisworkFileSystemConfig disworkConfig) + throws IOException { storage = new Storage(disworkConfig, new InMemoryMap()); storage.putDirectory("/", EntryUtil.newEmptyDirectoryContent()); } - @Override + /** tests the existence of a file/dir/link at a given path + * return true if something exists at path p, it will be true if a call + * to <code>mkdir(p)</code>, <code>write(p, ?)</code> or + * <code>ln(p, ?)</code> has been done before. + * @param path a path in the virtual FS + * @return true is something (a link, a file, or a directory) exists at path + * @throws IOException + */ public boolean exists(String path) throws IOException { String entry = walk(path); boolean result = entry != null; return result; } - + /** - * return the entry of the element at the end of <code>path</code> - * - * @param path - * @return null if path is not valid + * use this method to read the content of a file. A call of read may be + * done after a call to {@link #write} + * @param path the path to the file to read + * @return an InputStream on the file + * @throws FileNotFoundException if no file exists at this path + * @throws IOException if path exists but is a directory */ - protected String walk(String path) throws IOException { - String result = walk(path, null, null); - log.info("walking to " + path + " returns " + result); - return result; - } - - // FIXME 20105021 bleny works fine but is not understandable - protected String walk(String path, String current, String content) - throws IOException { - String result = null; - - if (path.equals("/")) { - return EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR - + "/" + EntryUtil.ENTRY_SEPARATOR - + "/"; - } - - - String parentPath = EntryUtil.getParentFromPath(path); - - if (content == null) { - // start of walk - content = storage.getRootDirectory(); - result = walk(path, "/", content); - } else if (parentPath.equals(current)) { - // in the last directory - // recuperation de l'element a traiter dans p - - String tail = path.substring(current.length()); - - /* - log.info("tail " + tail); - String[] elementsNames = tail.split("/"); - String p = elementsNames[0]; - */ - String p = EntryUtil.getNameFromPath(tail); - - log.info("in final dir " + current + ", looking for " + p); - - String entry = EntryUtil.findEntryInDirectory(content, p); - result = entry; - } else { - // in middle of path - - String tail; // the path still remaining when in current - - if (current.equals("/")) { - tail = path.substring(current.length()); - } else { - tail = path.substring(current.length() + 1); - } - - log.debug("current = " + current); - log.debug("tail = " + tail); - String[] elementsNames = tail.split("/"); - String p = elementsNames[0]; - - log.info("in intermediate dir " + current + ", looking for " + p); - - // mise a jour de current - if (current.equals("/")) current = ""; - current += "/" + p; - - String entry = EntryUtil.findEntryInDirectory(content, p); - if (entry == null) { - result = null; - } else { - if (EntryUtil.isDirectory(entry)) { - String id = EntryUtil.getId(entry); - content = storage.getDirectory(id); - result = walk(path, current, content); - } else if (EntryUtil.isLink(entry)) { - String id = EntryUtil.getId(entry); - String linkContent = storage.getLink(id); - String newTarget = EntryUtil.resolveLink(current, linkContent); - newTarget += path.substring(current.length()); - - // restart walk from / - result = walk(newTarget, null, null); - } else if (EntryUtil.isFile(entry)) { - // erreur, find file in path like '/dir1/dir2/filename/dir3' - result = null; - } else { - log.warn("strange case : " + entry); - result = null; - } - } - } - return result; - } - - @Override public InputStream read(String path) throws FileNotFoundException, IOException { @@ -140,7 +85,7 @@ if (EntryUtil.isLink(entry)) { log.info("reading link " + path); - String id = EntryUtil.getId(entry); + String id = EntryUtil.getIdFromEntry(entry); String link = storage.getLink(id); String newTarget = EntryUtil.resolveLink(path, link); result = read(newTarget); @@ -148,21 +93,26 @@ throw new IOException("target is not a file: " + path); } else if (EntryUtil.isFile(entry)) { log.info("reading file " + path); - String id = EntryUtil.getId(entry); + String id = EntryUtil.getIdFromEntry(entry); result = storage.getFile(id); - } + } return result; } - @Override + /** + * write a file on the file system at a given place + * @param path + * @param source + * @throws IOException if file already exists + */ public void write(String path, InputStream source) throws IOException { String parent = EntryUtil.getParentFromPath(path); String name = EntryUtil.getNameFromPath(path); write(parent, name, source); } - - @Override - public void write(String parent, String fileName, InputStream source) throws IOException { + + protected void write(String parent, String fileName, InputStream source) + throws IOException { String entryParent = walk(parent); if (entryParent == null) { @@ -170,11 +120,20 @@ } if (EntryUtil.isDirectory(entryParent)) { - String parentId = EntryUtil.getId(entryParent); + String parentId = EntryUtil.getIdFromEntry(entryParent); String content = storage.getDirectory(parentId); + + // checking that file do not already exists in this directory + String findResult = EntryUtil.findEntryInDirectory + (content, fileName); + if (findResult != null) { + throw new IOException + (parent + " already contains an element named " + fileName); + } + String newFileId = EntryUtil.generateId(); String newContent = EntryUtil.addEntryToDirectoryContent( - content, EntryUtil.TYPE.F, fileName, newFileId); + content, EntryUtil.TYPE.F, fileName, newFileId); // store file before meta info storage.putFile(newFileId, source); @@ -185,14 +144,18 @@ } } - @Override - public void mkdir(String path) throws IOException { + /** + * creates a new empty directory + * @param path the absolute path (ending by the name) of the directory + * @throws IOException if parent path is not correct + */ + public void createDirectory(String path) throws IOException { String parent = EntryUtil.getParentFromPath(path); String dirName = EntryUtil.getNameFromPath(path); - mkdir(parent, dirName); + createDirectory(parent, dirName); } - public void mkdir(String parent, String dirName) throws IOException { + protected void createDirectory(String parent, String dirName) throws IOException { log.info("trying to create directory " + dirName + " in " + parent); @@ -203,7 +166,7 @@ } if (EntryUtil.isDirectory(entryParent)) { - String parentId = EntryUtil.getId(entryParent); + String parentId = EntryUtil.getIdFromEntry(entryParent); String content = storage.getDirectory(parentId); String newDirectoryId = EntryUtil.generateId(); String newContent = EntryUtil.addEntryToDirectoryContent( @@ -219,14 +182,25 @@ } } - @Override - public void ln(String path, String target) throws IOException { + /** + * create a new symbolic link + * @param path the path (ending by the name) where the link will be created + * @param target the path where the link point to + * (may be relative or absolute) + * @throws IOException if parent path is not correct + */ + public void createSymbolicLink(String path, String target) + throws IOException { String parent = EntryUtil.getParentFromPath(path); String name = EntryUtil.getNameFromPath(path); - ln(parent, name, target); + createSymbolicLink(parent, name, target); } - public void ln(String parent, String name, String target) throws IOException { + /** + * @see #createSymbolicLink(String, String) + */ + protected void createSymbolicLink(String parent, String name, String target) + throws IOException { if (target.startsWith("/")) { // target is absolute @@ -244,11 +218,11 @@ } if (EntryUtil.isDirectory(entryParent)) { - String parentId = EntryUtil.getId(entryParent); + String parentId = EntryUtil.getIdFromEntry(entryParent); String content = storage.getDirectory(parentId); String newLinkId = EntryUtil.generateId(); String newContent = EntryUtil.addEntryToDirectoryContent( - content, EntryUtil.TYPE.L, name, newLinkId); + content, EntryUtil.TYPE.L, name, newLinkId); // store link before meta info storage.putLink(newLinkId, target); @@ -264,15 +238,23 @@ } } - @Override - public void remove(String path) throws IOException { + /** + * remove a file, directory, or link. Non-empty directories can't be + * removed + * @param path the complete path to the entity to remove + * @throws IOException if path is incorrect or directory not empty + */ + public void delete(String path) throws IOException { String parent = EntryUtil.getParentFromPath(path); String name = EntryUtil.getNameFromPath(path); log.info("trying to remove " + path); - remove(parent, name); + delete(parent, name); } - - protected void remove(String parent, String name) throws IOException { + + /** + * @see #delete(String) + */ + protected void delete(String parent, String name) throws IOException { String entryParent = walk(parent); if (entryParent == null) { @@ -280,25 +262,28 @@ } if (EntryUtil.isDirectory(entryParent)) { - String parentId = EntryUtil.getId(entryParent); + String parentId = EntryUtil.getIdFromEntry(entryParent); String content = storage.getDirectory(parentId); String entry = EntryUtil.findEntryInDirectory(content, name); - String idToRemove = EntryUtil.getId(entry); + String idToRemove = EntryUtil.getIdFromEntry(entry); String newContent = EntryUtil.removeEntryFromEntries( content, name); // check if not removing a non-empty directory if (EntryUtil.isDirectory(entry)) { - String innerDirectoryId = EntryUtil.getId(entry); - String innerDirectoryContent = storage.getDirectory(innerDirectoryId); - if (! innerDirectoryContent.equals(EntryUtil.newEmptyDirectoryContent())) { + String innerDirectoryId = EntryUtil.getIdFromEntry(entry); + String innerDirectoryContent = + storage.getDirectory(innerDirectoryId); + // checking the emptiness of the directory + if (!innerDirectoryContent.equals( + EntryUtil.newEmptyDirectoryContent())) { // directory is not empty - throw new IOException("trying to remove a non-empty directory"); + throw new IOException + ("trying to remove a non-empty directory"); } } - // update meta info directory storage.putDirectory(parentId, newContent); @@ -309,27 +294,34 @@ throw new IOException(parent + " is not a directory"); } } - - @Override - public List<String> ls(String path) throws IOException { + + /** + * list the content of a directory + * @param path the complete path to the directory + * @return a list of the names of the elements in <code>path</code> + * @throws IOException if path doesn't point to a directory + */ + public List<String> readDirectory(String path) throws IOException { String entry = walk(path); List<String> result = null; - if (entry == null) { throw new IOException(path + " doesn't exists"); } else { + // path may be a link, if it's the case, + // entry become the actual directory if (EntryUtil.isLink(entry)) { - String target = storage.getLink(EntryUtil.getId(entry)); + String target = storage.getLink( + EntryUtil.getIdFromEntry(entry)); entry = walk(target); - } - + } if (EntryUtil.isDirectory(entry)) { - String content = storage.getDirectory(EntryUtil.getId(entry)); + String content = storage.getDirectory( + EntryUtil.getIdFromEntry(entry)); String[] entries = content.split(EntryUtil.ENTRIES_SEPARATOR); result = new ArrayList<String>(); for (String elementEntry : entries) { - result.add(EntryUtil.getName(elementEntry)); + result.add(EntryUtil.getNameFromEntry(elementEntry)); } } else { throw new IOException(path + " is not a directory"); @@ -339,4 +331,111 @@ return result; } + /** + * return the entry of the element at the end of <code>path</code> + * @param path + * @return null if path is not valid + */ + protected String walk(String path) throws IOException { + String result = walk(path, null, null); + log.info("walking to " + path + " returns " + result); + return result; + } + + /** + * This method is a recursive function to walk through the tree structure + * of the directories and starting for root directory, following the + * symbolic links to reach the given path + * @param path + * @param current + * @param content + * @return null if path is not valid or the entry corresponding to path + * @throws IOException + */ + + // FIXME 20105021 bleny works fine but is not understandable + protected String walk(String path, String current, String content) + throws IOException { + String result = null; + + if (path.equals("/")) { + return EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR + + "/" + EntryUtil.ENTRY_SEPARATOR + + "/"; + } + + String parentPath = EntryUtil.getParentFromPath(path); + + if (content == null) { + // start of walk + content = storage.getRootDirectory(); + result = walk(path, "/", content); + } else if (parentPath.equals(current)) { + // in the last directory + // recuperation de l'element a traiter dans p + + String tail = path.substring(current.length()); + + /* + log.info("tail " + tail); + String[] elementsNames = tail.split("/"); + String p = elementsNames[0]; + */ + String p = EntryUtil.getNameFromPath(tail); + + log.info("in final dir " + current + ", looking for " + p); + + String entry = EntryUtil.findEntryInDirectory(content, p); + result = entry; + } else { + // in middle of path + + String tail; // the path still remaining when in current + + if (current.equals("/")) { + tail = path.substring(current.length()); + } else { + tail = path.substring(current.length() + 1); + } + + log.debug("current = " + current); + log.debug("tail = " + tail); + String[] elementsNames = tail.split("/"); + String p = elementsNames[0]; + + log.info("in intermediate dir " + current + ", looking for " + p); + + // mise a jour de current + if (current.equals("/")) current = ""; + current += "/" + p; + + String entry = EntryUtil.findEntryInDirectory(content, p); + if (entry == null) { + result = null; + } else { + if (EntryUtil.isDirectory(entry)) { + String id = EntryUtil.getIdFromEntry(entry); + content = storage.getDirectory(id); + result = walk(path, current, content); + } else if (EntryUtil.isLink(entry)) { + String id = EntryUtil.getIdFromEntry(entry); + String linkContent = storage.getLink(id); + String newTarget = + EntryUtil.resolveLink(current, linkContent); + newTarget += path.substring(current.length()); + + // restart walk from / + result = walk(newTarget, null, null); + } else if (EntryUtil.isFile(entry)) { + // error, found file in path like '/dir1/dir2/filename/dir3' + result = null; + } else { + log.warn("strange case: " + entry); + result = null; + } + } + } + return result; + } + } Copied: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java (from rev 50, trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkConfig.java) =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystemConfig.java 2010-05-25 15:24:32 UTC (rev 51) @@ -0,0 +1,26 @@ +package org.nuiton.disworkfs; + +import org.nuiton.util.ApplicationConfig; + +/** + * This object contains parameters used in {@link DisworkFileSystem}. + * + * Available parameters are: + * <dl> + * <dt>blocks_size</dt> + * <dd> + * The size of blocks to create when splitting big data in bytes + * (by default, 10 MiB) + * </dd> + * </dl> + */ +public class DisworkFileSystemConfig extends ApplicationConfig { + + public DisworkFileSystemConfig() { + setDefaultOption("blocks_size", "10485760"); // 10 MiB + } + + public int getBlockSize() { + return getOptionAsInt("blocks_size"); + } +} Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/package-info.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/package-info.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/package-info.java 2010-05-25 15:24:32 UTC (rev 51) @@ -0,0 +1,10 @@ +/** + * DisworkFS is a distributed file system. You can use it by instantiating + * {@link DisworkFileSystem} and use it to store directories and their content. + * + * You can change the default DisworkFileSystem behavior by provide a + * {@link DisworkFileSystemConfig} instance at construction. If you don't + * provide one, a instance will be created + */ + +package org.nuiton.disworkfs; \ No newline at end of file Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java 2010-05-21 16:46:41 UTC (rev 50) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java 2010-05-25 15:24:32 UTC (rev 51) @@ -7,28 +7,51 @@ import org.apache.commons.io.FilenameUtils; /** + * This class contains String utilities to ease operations on + * <dl> + * <dt>a single entry;</dt> + * <dd>a record in the list of content in a directory</dd> + * <dt>multiple entries;</dt> + * <dd>a string containing multiple entry (as above) separated by + * {@link #ENTRIES_SEPARATOR} + * </dd> + * <dt>a metablock;</dt> + * <dd>a metablock is a String that begins with the total size of the data + * it describes and the ids where some blocks containing the data + * are stored into the map</dd> + * <dt>a path.</dt> + * <dd>a string like /a/b/c/d</dd> + * </dl> + * * @author poussin * @version $Revision$ * * Last update: $Date$ - * by : $Author$ + * by: $Author$ */ public class EntryUtil { + static public enum TYPE { /** Directory */ D, + /** Link */ L, + /** File */ F + }; + static final int TYPE_LENGTH = 1; + static final public String ENTRY_SEPARATOR = ":"; static final public String ENTRIES_SEPARATOR = "\n"; static final public String BLOCKIDS_SEPARATOR = ";"; - - static final int TYPE_LENGTH = 1; - /* - // this line may not be accurate for "/" - static final int UUID_LENGTH = UUID.randomUUID().toString().length(); - */ + static final int ENTRY_SEPARATOR_LENGTH = ENTRY_SEPARATOR.length(); - static public enum TYPE{D /*Directory*/, L /*Link*/, F /*File*/}; + /** + * This charset is forced in string-to-bytes and bytes-to-string conversions + */ + static protected Charset CHARSET = Charset.forName("UTF-8"); - static protected Charset CHARSET = Charset.forName("UTF-8"); + /** + * This is a utility class thus should not be instantiated + */ + protected EntryUtil() {}; /** * generate a new Id usage as a new key for the map @@ -45,7 +68,7 @@ * a link */ static public boolean isDirectory(String entry) { - boolean result = entry.startsWith(TYPE.D.name()); + boolean result = getTypeFromEntry(entry) == TYPE.D; return result; } @@ -55,7 +78,7 @@ * a directory */ static public boolean isLink(String entry) { - boolean result = entry.startsWith(TYPE.L.name()); + boolean result = getTypeFromEntry(entry) == TYPE.L; return result; } @@ -65,47 +88,31 @@ * a directory */ static public boolean isFile(String entry) { - boolean result = entry.startsWith(TYPE.F.name()); + boolean result = getTypeFromEntry(entry) == TYPE.F; return result; } - static public TYPE getType(String entry) { + static public TYPE getTypeFromEntry(String entry) { int index = entry.indexOf(ENTRY_SEPARATOR); String type = entry.substring(0, index); TYPE result = TYPE.valueOf(type); return result; } - static public String getName(String entry) { + static public String getNameFromEntry(String entry) { int start = entry.indexOf(ENTRY_SEPARATOR); int last = entry.lastIndexOf(ENTRY_SEPARATOR); String result = entry.substring(start + ENTRY_SEPARATOR_LENGTH, last); return result; } - static public String getId(String entry) { + static public String getIdFromEntry(String entry) { int index = entry.lastIndexOf(ENTRY_SEPARATOR); String result = entry.substring(index + ENTRY_SEPARATOR_LENGTH); return result; } + - static public int getTotalSizeFromMetaBlock(String metaBlock) { - // dealing with empty meta-block (for empty directory or file) - if (metaBlock.equals("0")) return 0; - String result = metaBlock.substring(0, metaBlock.indexOf(BLOCKIDS_SEPARATOR)); - return Integer.valueOf(result); - } - - static public String[] getBlockIdsFromMetaBlock(String metaBlock) { - // dealing with empty meta-block (for empty directory or file) - if (metaBlock.indexOf(BLOCKIDS_SEPARATOR) == -1) { - return new String[0]; - } - String blockIdsAsString = metaBlock.substring(metaBlock.indexOf(BLOCKIDS_SEPARATOR) + 1, metaBlock.length()); - String[] blockIds = blockIdsAsString.split(BLOCKIDS_SEPARATOR); - return blockIds; - } - /** * find file name in directory content description * @@ -126,6 +133,12 @@ return result; } + public static String removeEntryFromEntries(String content, String name) { + String entry = findEntryInDirectory(content, name); + String newContent = content.replaceFirst(entry + ENTRIES_SEPARATOR, ""); + return newContent; + } + /** * */ @@ -135,51 +148,38 @@ /** * - * @param content the string content of the directory (all entries before add) + * @param content the string content of the directory + * (all entries before add) * @param type the type of the new entry * @param name the name (not a full path) of the new entry * @param id the id where the content of the entry can be found * @return a String with the new content */ static public String addEntryToDirectoryContent - (String content, TYPE type, String name, String id) { - content += type.name() + ENTRY_SEPARATOR + name + - ENTRY_SEPARATOR + id + ENTRIES_SEPARATOR; - return content; + (String content, TYPE type, String name, String id) { + String newEntry = type.name() + ENTRY_SEPARATOR + name + + ENTRY_SEPARATOR + id + ENTRIES_SEPARATOR; + return content + newEntry; } - /** - * Resolve a path from a parent directory, - * - * ie resolve /dir/subdir, ../file returns /dir/file - * @param parent the path to the dir where start from compute the path - * @param link the path to follow (may be absolute or relative to - * <code>parent</code>) - * @return the path to the destination - */ - static public String resolveLink(String parent, String link) { - /* - String result = null; - if (link.startsWith("/")) { - // destination is absolute, ignore parent - result = link; - } else { - // destination is relative, so we have to consider parent - // parent may have a trailing file name, let's remove it - String parentWithNoFile = parent; - if (!parent.endsWith("/")) { - // removing the file at the end of the path, it's not a - // directory and thus should not be part of the path - parentWithNoFile = parent.substring(0, parent.lastIndexOf("/") + 1); - } - - // concatenate and making the complete path consistent - // ie /dir/subdir/../file to /dir/file - result = FilenameUtils.normalize(parentWithNoFile + link); + static public int getTotalSizeFromMetaBlock(String metaBlock) { + // dealing with empty meta-block (for empty directory or file) + if (metaBlock.equals("0")) return 0; + String result = metaBlock.substring(0, + metaBlock.indexOf(BLOCKIDS_SEPARATOR)); + return Integer.valueOf(result); + } + + static public String[] getBlockIdsFromMetaBlock(String metaBlock) { + // dealing with empty meta-block (for empty directory or file) + if (metaBlock.indexOf(BLOCKIDS_SEPARATOR) == -1) { + return new String[0]; } - return result; - */ - return FilenameUtils.concat(parent, link); + String blockIdsAsString = metaBlock.substring( + metaBlock.indexOf(BLOCKIDS_SEPARATOR) + 1, + metaBlock.length()); + String[] blockIds = blockIdsAsString.split(BLOCKIDS_SEPARATOR); + return blockIds; } /** @@ -189,20 +189,31 @@ * @return a byte array containing all the bytes in <code>string</code> */ public static byte[] stringToBytes(String string) { - // FIXME 20100519 bleny this code may cause bad conversion - // due to charsets byte[] bytes = string.getBytes(CHARSET); return bytes; } /** - * @see {@link #stringToBytes(String)} - * @param string the string + * @see #stringToBytes(String) + * @param bytes the bytes to convert * @return a byte array containing all the bytes in <code>string</code> */ public static String bytesToString(byte[] bytes) { return new String(bytes, CHARSET); } + + /** + * Resolve a path from a parent directory and a link + * + * ie resolve /dir/subdir, ../file returns /dir/file + * @param parent the path to the dir where start from compute the path + * @param link the path to follow (may be absolute or relative to + * <code>parent</code>) + * @return the path to the destination + */ + static public String resolveLink(String parent, String link) { + return FilenameUtils.concat(parent, link); + } public static String getParentFromPath(String path) { String parent = FilenameUtils.getFullPathNoEndSeparator(path); @@ -215,11 +226,5 @@ public static String getNameFromPath(String path) { return FilenameUtils.getName(path); } - - public static String removeEntryFromEntries(String content, String name) { - String entry = findEntryInDirectory(content, name); - String newContent = content.replaceFirst(entry + ENTRIES_SEPARATOR, ""); - return newContent; - } } Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryMap.java 2010-05-21 16:46:41 UTC (rev 50) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryMap.java 2010-05-25 15:24:32 UTC (rev 51) @@ -6,6 +6,11 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +/** An in-memory implementation of a Map + * This class is a map where put make a copy of the value. It is used for + * stubbing purpose. It emulates the behavior of a DHT and thus can be used + * by {@link Storage} to store data and blocks. + */ public class InMemoryMap extends HashMap<String, byte[]> { private final Log log = LogFactory.getLog(InMemoryMap.class); @@ -17,4 +22,4 @@ log.info("putting " + valueCopy.length + " bytes"); return super.put(key, valueCopy); } -} +} \ No newline at end of file Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-05-21 16:46:41 UTC (rev 50) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-05-25 15:24:32 UTC (rev 51) @@ -8,12 +8,20 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.nuiton.disworkfs.DisworkConfig; +import org.nuiton.disworkfs.DisworkFileSystemConfig; +/** + * + */ public class Storage { private static final Log log = LogFactory.getLog(Storage.class); + /** + * This class is a InputStream used to read data from the map. When + * someone read from the filesytem, he will bre returned a + * SplitBlocksInputStream. This InputStream loads blocks only when needed + */ protected class SplitBlocksInputStream extends InputStream { protected Map<String, byte[]> map; @@ -45,6 +53,8 @@ if (currentBlock == null || currentBlock.available() == 0) { blockIdsIndex += 1; if (blockIdsIndex < blockIds.length) { + // closing previously opened stream + IOUtils.closeQuietly(currentBlock); currentBlock = new ByteArrayInputStream(map.get(blockIds[blockIdsIndex])); log.debug("new current block (size = " + currentBlock.available() + ")"); } else { @@ -53,16 +63,18 @@ } int result = currentBlock.read(); - /* - log.debug("reading block number " + blockIdsIndex + - " (available = " + available + ")" + - " returns " + result); - */ available -= 1; - + + if(log.isDebugEnabled()) { + log.debug("reading block number " + blockIdsIndex + + " (available = " + available + ")" + + " returns " + result); + } return result; } + // This method is called at the first read or the first available + // it initialize all the necessary field protected void checkInitialization() { if (blockIds == null) { byte[] bytes = map.get(id); @@ -86,17 +98,13 @@ protected Map<String, byte[]> map; - protected DisworkConfig disworkConfig; + protected DisworkFileSystemConfig disworkConfig; - public Storage(DisworkConfig disworkConfig, Map<String, byte[]> map) { + public Storage(DisworkFileSystemConfig disworkConfig, Map<String, byte[]> map) { this.map = map; this.disworkConfig = disworkConfig; } - public boolean contains(Object id) { - return map.containsKey(id); - } - /** * @return the content (entries) of the root directory * @throws IOException @@ -108,8 +116,11 @@ - - + /** + * @param id an id of a directory + * @return a the entries of the directory + * @throws IOException + */ public String getDirectory(String id) throws IOException { InputStream in = get(id); String content = IOUtils.toString(in); @@ -162,36 +173,37 @@ protected void put(String key, InputStream value) throws IOException { // TODO 20100519 bleny deal with copy-on-write - // TODO 20100519 bleny deal with null value properly + // TODO 20100519 bleny deal with null value properly if (value == null) { value = new ByteArrayInputStream(new byte[0]); } String blocksIds = ""; - int read = 0; + int readResult = 0; int totalSize = 0; + // creating a buffer of the size of a block int bufferSize = disworkConfig.getBlockSize(); byte[] buffer = new byte[bufferSize]; - while ((read = value.read(buffer)) != -1) { - totalSize += read; + while ((readResult = value.read(buffer)) != -1) { + totalSize += readResult; byte[] newBlock = buffer; - // if the block is shorter, truncate - if (read < buffer.length) { - newBlock = new byte[read]; - System.arraycopy(buffer, 0, newBlock, 0, read); + // if the buffer is not full, truncate the block + if (readResult < buffer.length) { + newBlock = new byte[readResult]; + System.arraycopy(buffer, 0, newBlock, 0, readResult); } String id = EntryUtil.generateId(); blocksIds += EntryUtil.BLOCKIDS_SEPARATOR + id; - log.debug("saving new block (size = " + newBlock.length - + ") at key " + id); + log.debug("saving new block (size = " + newBlock.length + ")" + + " at key " + id); - // copy buffer in map + // copy block in map map.put(id, newBlock); } @@ -208,5 +220,4 @@ map.remove(key); } - -} +} \ No newline at end of file Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/package-info.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/package-info.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/package-info.java 2010-05-25 15:24:32 UTC (rev 51) @@ -0,0 +1,203 @@ +/** + * + * This package provides to {@link org.nuiton.disworkfs.DisworkFileSystem} a + * way to persistently store data. This is done by the {@link Storage} class + * which permit to store different type of data. + * + * + * We illustrate the use of a file system by setting the content of "/", + * the root directory as follow: + * + * <ul> + * <li> + * a directory named "my_folder" that contains: + * <ul> + * <li> + * a directory named "my_sub_folder" that contains + * <ul> + * <li>a file name "my_deep_file.ext"</li> + * <li>an empty folder named "my_empty_folder"</li> + * </ul> + * </li> + * <li>a file named "my_other_file.ext"</li> + * </ul> + * </li> + * <li> + * a file named "my_file.ext" (containing binary data: 010101) + * </li> + * <li> + * a symbolic link named "my_link" which target the path + * <code>"/my_folder/my_sub_folder/my_deep_file.ext"</code> + * </li> + * </ul> + * + * Storage uses a Map to store data. + * + * <table> + * <caption>Use of map, abstract point of view</caption> + * <tr> + * <th>Key (String)</th> + * <th>Value (byte[])</th> + * </tr> + * <tr> + * <td>"/"</td> + * <td>"D:my_folder:ID1\n + * F:my_file.ext:ID2\n + * L:my_link:ID3"</td> + * </tr> + * <tr> + * <td>"ID1"</td> + * <td>"D:my_sub_folder:ID4\n + * F:my_other_file.ext:ID5"</td> + * </tr> + * <tr> + * <td>"ID2"</td> + * <td>010101</td> + * </tr> + * <tr> + * <td>"ID3"</td> + * <td>"/my_folder/my_sub_folder/my_deep_file.ext"</td> + * </tr> + * <tr> + * <td>"ID4"</td> + * <td>"F:my_deep_file:ID6\n + * D:my_empty_folder:ID7"</td> + * </tr> + * <tr> + * <td>"ID5"</td> + * <td>011010100111010...</td> + * </tr> + * <tr> + * <td>"ID6"</td> + * <td>1100010101010110110010...</td> + * </tr> + * <tr> + * <td>"ID7"</td> + * <td>""</td> + * </tr> + * </table> + * + * + * An entry is, for example, "D:my_folder:ID1". It contains the type of the + * entry (Directory, File or + * Link), the name of the element, and an ID to be used as + * a key on the map to get the actual content. Those three informations are + * separated by ":" which is {@link EntryUtil#ENTRY_SEPARATOR}. + * + * + * + * A directory way have multiple entries. Entries of a directory are + * separated by "\n" which is {@link EntryUtil#ENTRIES_SEPARATOR}. + * + * + * The above description shows the main principle used to store a tree + * structure in a map. + * + * + * In fact, the storage of actual data is a bit more complex, due to + * the need of splitting the data that may be too large (the value of "ID6" + * may be a series of 100 * 220 bytes for a 200 MiB file). + * Actually, files content (like ID2, ID5, ID6) and directory content + * values ("/", ID1, ID4) will be split and the map will look like that ( + * note that only the files have been split for readability): + * + * <table> + * <caption>Use of map, concrete point of view</caption> + * <tr> + * <th>Key (String)</th> + * <th>Value (byte[])</th> + * </tr> + * <tr> + * <td>"/"</td> + * <td>"D:my_folder:ID1\n + * F:my_file.ext:ID2\n + * L:my_link:ID3"</td> + * </tr> + * <tr> + * <td>"ID1"</td> + * <td>"D:my_sub_folder:ID4\n + * F:my_other_file.ext:ID5"</td> + * </tr> + * <tr> + * <td>"ID2"</td> + * <td>"6;ID8"</td> + * </tr> + * <tr> + * <td>"ID3"</td> + * <td>"/my_folder/my_sub_folder/my_deep_file.ext"</td> + * </tr> + * <tr> + * <td>"ID4"</td> + * <td>"F:my_deep_file:ID6\n + * D:my_empty_folder:ID7"</td> + * </tr> + * <tr> + * <td>"ID5"</td> + * <td>"14689;ID9"</td> + * </tr> + * <tr> + * <td>"ID6"</td> + * <td>"79874567;ID10;ID11;ID12"</td> + * </tr> + * <tr> + * <td>"ID7"</td> + * <td>""</td> + * </tr> + * <tr> + * <td>"ID8"</td> + * <td>010101</td> + * </tr> + * <tr> + * <td>"ID9"</td> + * <td>011010100111010...</td> + * </tr> + * <tr> + * <td>"ID10"</td> + * <td>110001010101011...</td> + * </tr> + * <tr> + * <td>"ID11"</td> + * <td>111010110101101...</td> + * </tr> + * <tr> + * <td>"ID12"</td> + * <td>011101100111010...</td> + * </tr> + * </table> + * + * + * Values of ID2, ID5, ID6 are no longer the file content. It's now a set + * of meta-information information data called <code>metablock</code>. + * A metablock is composed of the total size of the file followed by + * an ordered lists of IDs of the different blocks composing the file. + * Those informations are separated by ";" (see + * {@link EntryUtil#BLOCKIDS_SEPARATOR}). + * + * + * In the above example: + * <ul> + * <li> + * "/my_file.ext" is a file which size is 6 bytes, it is + * composed of one block available at ID8; + * </li> + * <li> + * "/my_sub_folder/my_other_file.ext" is a file (size = 14689 bytes) + * which content is available at ID9; + * </li> + * <li> + * "/my_sub_folder/my_sub_folder/my_deep_file.ext" is a big file: its + * size is 79874567. It has been split in three blocks: ID10, ID11 and + * ID12. + * </li> + * </ul> + * + * + * When reading and writing in storage, split is done transparently. When + * reading, a Stream is returned: it loads data blocks after blocks + * when needed (see {@link Storage.SplitBlocksInputStream}). When writing, + * data are split in blocks of a maximum configurable size + * (see {@link org.nuiton.disworkfs.DisworkConfig}). + * + */ + +package org.nuiton.disworkfs.storage; \ No newline at end of file Modified: trunk/diswork-fs/src/main/resources/log4j.properties =================================================================== --- trunk/diswork-fs/src/main/resources/log4j.properties 2010-05-21 16:46:41 UTC (rev 50) +++ trunk/diswork-fs/src/main/resources/log4j.properties 2010-05-25 15:24:32 UTC (rev 51) @@ -5,4 +5,4 @@ log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=%d %5p [%t] (%F:%L) %M - %m%n # package level -log4j.logger.org.nuiton.disworkfs=TRACE \ No newline at end of file +log4j.logger.org.nuiton.disworkfs=WARN \ No newline at end of file Deleted: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java 2010-05-21 16:46:41 UTC (rev 50) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java 2010-05-25 15:24:32 UTC (rev 51) @@ -1,253 +0,0 @@ -package org.nuiton.disworkfs; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.util.List; -import java.util.Random; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.nuiton.util.FileUtil; - - -public class DistributedFileSystemTest { - - /** - * a place to store files for the test it's a subdirectory of the OS temp - * dir e.g. under linux /tmp/disworkfs/tests/ - */ - static protected String tempDirectoryPath = System.getProperty( - "java.io.tmpdir", ".") - + "/disworkfs/tests"; - - /** - * We will create a file at this path for test purpose - */ - static protected String randomFilePath = tempDirectoryPath + "/randomfile"; - - /** - * The file will have this fixed size - */ - static protected int randomFileSize = 10000; - - static protected DistributedFileSystem fileSystem; - - @Before - public void setUp() throws Exception { - File tempDirectory = new File(tempDirectoryPath); - tempDirectory.mkdir(); - - Random random = new Random(); - // creating random data for the file - byte[] randomBytes = new byte[randomFileSize]; - random.nextBytes(randomBytes); - - // dumping random data into the file - File randomFile = new File(randomFilePath); - FileUtils.writeByteArrayToFile(randomFile, randomBytes); - - DisworkConfig disworkConfig = new DisworkConfig(); - - fileSystem = new DisworkFileSystem(disworkConfig); - } - - @After - public void tearDown() throws Exception { - // cleaning - FileUtil.deleteRecursively(tempDirectoryPath); - } - - @Test - public void testWrite() throws Exception { - fileSystem.write("/", "my_file", new FileInputStream(randomFilePath)); - } - - @Test - public void testExists() throws Exception { - fileSystem.write("/", "my_file", new FileInputStream(randomFilePath)); - assertTrue(fileSystem.exists("/my_file")); - assertFalse(fileSystem.exists("/my_other_file")); - } - - /** - * try to read a file that as never been created nor written - */ - @Test(expected = FileNotFoundException.class) - public void testFailAtRead() throws Exception { - fileSystem.read("/not_existing_file"); - } - - /** - * testing {@link org.nuiton.disworkfs.storage.Storage#SplitBlocksInputStream} - * by storing a "-1" byte, can be buggy due to the use of read() - * @throws IOException - */ - @Test - public void testSplit() throws IOException { - - byte[] bytes = new byte[1]; - bytes[0] = -0x1; - - InputStream source; - - source = new ByteArrayInputStream(bytes); - fileSystem.write("/", "my_file", source); - - source.close(); - - - source = new ByteArrayInputStream(bytes); - InputStream readResult = fileSystem.read("/my_file"); - - int read = 0; - byte[] b = new byte[1]; - - read = readResult.read(b); - - assertEquals(1, read); - assertArrayEquals(bytes, b); - - } - - /** - * writing a file at the root directory and reading it. - * finally, compare original source and read result - * byte-to-byte : contents should be equals - * @throws Exception - */ - @Test - public void testWriteRead() throws Exception { - - InputStream source = new FileInputStream(randomFilePath); - - fileSystem.write("/", "my_file", source); - - source.close(); - - InputStream readResult; - - // to be used for debugging purpose - - source = new FileInputStream(randomFilePath); - readResult = fileSystem.read("/my_file"); - - System.out.println("source.available() = " + source.available()); - System.out.println("readResult.available() = " + readResult.available()); - - byte[] sourceAsBytes = IOUtils.toByteArray(source); - byte[] readResultAsBytes = IOUtils.toByteArray(readResult); - /* - System.out.println("source.available() = " + source.available()); - System.out.println("readResult.available() = " + readResult.available()); - - System.out.println("source (" + sourceAsBytes.length + ") = " + Arrays.toString(sourceAsBytes)); - System.out.println("result (" + readResultAsBytes.length + ") = " + Arrays.toString(readResultAsBytes)); - */ - assertArrayEquals(sourceAsBytes, readResultAsBytes); - - - source = new FileInputStream(randomFilePath); - readResult = fileSystem.read("/my_file"); - - boolean actualContentEquality = - IOUtils.contentEquals(source, readResult); - source.close(); - readResult.close(); - - assertTrue(actualContentEquality); - - } - - /** - * this use case should raise an exception because my_folder - * doesn't exists - */ - @Test(expected = IOException.class) - public void testFailAtWrite() throws Exception { - fileSystem.write("/my_folder", "my_file", new FileInputStream(randomFilePath)); - } - - @Test - public void testMkdir() throws Exception { - fileSystem.mkdir("/my_folder"); - assertTrue(fileSystem.exists("/my_folder")); - fileSystem.mkdir("/my_folder/my_sub_folder"); - assertTrue(fileSystem.exists("/my_folder/my_sub_folder")); - } - - @Test - public void testWriteInFolder() throws Exception { - fileSystem.mkdir("/my_folder"); - fileSystem.write("/my_folder", "my_file", new FileInputStream(randomFilePath)); - fileSystem.mkdir("/my_folder/my_sub_folder"); - fileSystem.write("/my_folder/my_sub_folder", "my_file", new FileInputStream(randomFilePath)); - assertTrue(fileSystem.exists("/my_folder/my_sub_folder/my_file")); - } - - @Test - public void testln() throws Exception { - fileSystem.mkdir("/my_folder"); - fileSystem.write("/my_folder", "my_file", new FileInputStream(randomFilePath)); - fileSystem.ln("/my_link", "/my_folder/my_file"); - - InputStream source = new FileInputStream(randomFilePath); - InputStream readResult = fileSystem.read("/my_link"); - - boolean actualContentEquality = - IOUtils.contentEquals(source, readResult); - source.close(); - readResult.close(); - - assertTrue(actualContentEquality); - } - - @Test(expected = IOException.class) - public void testFailAtLn() throws Exception { - fileSystem.ln("/my_link", "/wrong_target_path"); - } - - @Test - public void testRemove() throws Exception { - fileSystem.mkdir("/my_folder"); - fileSystem.write("/my_folder", "my_file", new FileInputStream(randomFilePath)); - fileSystem.remove("/my_folder/my_file"); - assertTrue(fileSystem.exists("/my_folder")); - assertFalse(fileSystem.exists("/my_folder/my_file")); - fileSystem.remove("/my_folder"); - assertFalse(fileSystem.exists("/my_folder")); - } - - @Test(expected = IOException.class) - public void testFailAtRemove() throws Exception { - fileSystem.mkdir("/my_folder"); - fileSystem.write("/my_folder", "my_file", new FileInputStream(randomFilePath)); - - // trying to remove a non-empty directory should raise an exception - fileSystem.remove("/my_folder"); - } - - @Test - public void testLs() throws Exception { - fileSystem.mkdir("/my_folder"); - fileSystem.mkdir("/my_folder/my_sub_dir"); - fileSystem.write("/my_folder", "my_file", new FileInputStream(randomFilePath)); - fileSystem.ln("/my_folder/my_link", "my_file"); - - // trying to remove a non-empty directory should raise an exception - List<String> lsResult = fileSystem.ls("/my_folder"); - assertEquals(3, lsResult.size()); - } - -} Copied: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java (from rev 50, trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java) =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java (rev 0) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DisworkFileSystemTest.java 2010-05-25 15:24:32 UTC (rev 51) @@ -0,0 +1,316 @@ +package org.nuiton.disworkfs; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import java.util.Random; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.nuiton.util.FileUtil; + + +public class DisworkFileSystemTest { + + /** + * a place to store files for the test it's a subdirectory of the OS temp + * dir e.g. under linux /tmp/disworkfs/tests/ + */ + static protected String tempDirectoryPath = + System.getProperty("java.io.tmpdir", ".") // temp directory + + "/disworkfs/tests"; + + /** + * We will create a file at this path for test purpose + */ + static protected String randomFilePath = tempDirectoryPath + "/randomfile"; + + /** + * The file will have this fixed size + */ + static protected int randomFileSize = 9999; + + static protected DisworkFileSystem fileSystem; + + /** + * This setUp creates in a temp directory a file of size + * {@link randomFileSize} (fulfilling it with random bytes) + * This file can be found at {@link #randomFilePath} + * At the end of the method {@link #fileSystem} is ready to be used + * @throws Exception + */ + @Before + public void setUp() throws Exception { + // create a temp directory for our test + File tempDirectory = new File(tempDirectoryPath); + tempDirectory.mkdir(); + + // creating random data for the file + Random random = new Random(); + byte[] randomBytes = new byte[randomFileSize]; + random.nextBytes(randomBytes); + + // dumping random data into the file + File randomFile = new File(randomFilePath); + FileUtils.writeByteArrayToFile(randomFile, randomBytes); + + // finally, initiate the fileSystem + DisworkFileSystemConfig disworkConfig = new DisworkFileSystemConfig(); + fileSystem = new DisworkFileSystem(disworkConfig); + } + + @After + public void tearDown() throws Exception { + // cleaning + FileUtil.deleteRecursively(tempDirectoryPath); + } + + /** + * writing a file at root directory should not raise any exception + * @throws Exception + */ + @Test + public void testWrite() throws Exception { + fileSystem.write("/", "my_file", new FileInputStream(randomFilePath)); + } + + /** + * First, write a file then test if exists return true. + * @throws Exception + */ + @Test + public void testExists() throws Exception { + fileSystem.write("/", "my_file", new FileInputStream(randomFilePath)); + assertTrue(fileSystem.exists("/my_file")); + assertFalse(fileSystem.exists("/my_other_file")); + } + + /** + * try to read a file that as never been created nor written + */ + @Test(expected = FileNotFoundException.class) + public void testFailAtRead() throws Exception { + fileSystem.read("/not_existing_file"); + } + + /** + * tests {@link org.nuiton.disworkfs.storage.Storage#SplitBlocksInputStream} + * by storing a "-1" byte, can be buggy due to the use of read() + * @throws IOException + */ + @Test + public void testSplit() throws IOException { + + byte[] bytes = new byte[1]; + bytes[0] = -0x1; + + InputStream source; + + source = new ByteArrayInputStream(bytes); + fileSystem.write("/", "my_file", source); + + source.close(); + + + source = new ByteArrayInputStream(bytes); + InputStream readResult = fileSystem.read("/my_file"); + + int read = 0; + byte[] b = new byte[1]; + + read = readResult.read(b); + + assertEquals(1, read); + assertArrayEquals(bytes, b); + + } + + /** + * writing a file at the root directory and reading it. + * finally, compare original source and read result + * byte-to-byte: contents should be equals + * @throws Exception + */ + @Test + public void testWriteRead() throws Exception { + + InputStream source = new FileInputStream(randomFilePath); + + fileSystem.write("/", "my_file", source); + + source.close(); + + InputStream readResult; + + // now, the checks. We read the original file and the result of + // a read() and then compare it byte-to-byte + + source = new FileInputStream(randomFilePath); + readResult = fileSystem.read("/my_file"); + + assertEquals(randomFileSize, source.available()); + assertEquals(randomFileSize, readResult.available()); + + byte[] sourceAsBytes = IOUtils.toByteArray(source); + byte[] readResultAsBytes = IOUtils.toByteArray(readResult); + + assertArrayEquals(sourceAsBytes, readResultAsBytes); + + source.close(); + readResult.close(); + + } + + /** + * this use case should raise an exception because my_folder + * doesn't exists + */ + @Test(expected = IOException.class) + public void testFailAtWrite() throws Exception { + fileSystem.write("/my_folder", "my_file", + new FileInputStream(randomFilePath)); + } + + /** + * this use case should raise an exception because writing to a file + * that already exists + */ + @Test(expected = IOException.class) + public void testFailAtDoubleWrite() throws Exception { + InputStream source = new FileInputStream(randomFilePath); + fileSystem.write("/my_file", source); + source.close(); + + source = new FileInputStream(randomFilePath); + try { + fileSystem.write("/my_file", source); + } finally { + source.close(); + } + } + + /** + * This test uses mkdir to create dirs and sub-dirs + * @throws Exception + */ + @Test + public void testCreateDirectory() throws Exception { + fileSystem.createDirectory("/my_folder"); + assertTrue(fileSystem.exists("/my_folder")); + fileSystem.createDirectory("/my_folder/my_sub_folder"); + assertTrue(fileSystem.exists("/my_folder/my_sub_folder")); + } + + /** + * Create some folders with mkdir and write files in those directories + * @throws Exception + */ + @Test + public void testWriteInFolder() throws Exception { + fileSystem.createDirectory("/my_folder"); + fileSystem.write("/my_folder", "my_file", + new FileInputStream(randomFilePath)); + fileSystem.createDirectory("/my_folder/my_sub_folder"); + fileSystem.write("/my_folder/my_sub_folder", "my_file", + new FileInputStream(randomFilePath)); + assertTrue(fileSystem.exists("/my_folder/my_sub_folder/my_file")); + } + + /** + * create a symbolic link to a file. This test show that we can read the + * file using the link + * @throws Exception + */ + @Test + public void testLinking() throws Exception { + fileSystem.createDirectory("/my_folder"); + fileSystem.write("/my_folder", "my_file", + new FileInputStream(randomFilePath)); + fileSystem.createSymbolicLink("/my_link", "/my_folder/my_file"); + + InputStream source = new FileInputStream(randomFilePath); + InputStream readResult = fileSystem.read("/my_link"); + + boolean actualContentEquality = + IOUtils.contentEquals(source, readResult); + source.close(); + readResult.close(); + + assertTrue(actualContentEquality); + } + + /** + * Trying to create a link to a wrong target, this sould raise an exception + * @throws Exception + */ + @Test(expected = IOException.class) + public void testFailAtLinking() throws Exception { + fileSystem.createSymbolicLink("/my_link", "/wrong_target_path"); + } + + /** + * Trying to remove files and directories + * @throws Exception + */ + @Test + public void testRemove() throws Exception { + fileSystem.createDirectory("/my_folder"); + fileSystem.write("/my_folder", "my_file", + new FileInputStream(randomFilePath)); + fileSystem.delete("/my_folder/my_file"); + assertTrue(fileSystem.exists("/my_folder")); + assertFalse(fileSystem.exists("/my_folder/my_file")); + fileSystem.delete("/my_folder"); + assertFalse(fileSystem.exists("/my_folder")); + } + + /** + * By trying to remove a non-empty directory, this test should raise an + * exception + * @throws Exception + */ + @Test(expected = IOException.class) + public void testFailAtRemove() throws Exception { + fileSystem.createDirectory("/my_folder"); + fileSystem.write("/my_folder", "my_file", + new FileInputStream(randomFilePath)); + + // trying to remove a non-empty directory should raise an exception + fileSystem.delete("/my_folder"); + } + + /** + * This tests uses ls + */ + @Test + public void testListDirectory() throws Exception { + fileSystem.createDirectory("/my_folder"); + fileSystem.createDirectory("/my_folder/my_sub_dir"); + fileSystem.write("/my_folder", "my_file", + new FileInputStream(randomFilePath)); + fileSystem.createSymbolicLink("/my_folder/my_link", "my_file"); + + List<String> lsResult = fileSystem.readDirectory("/my_folder"); + + // checking that result contains all the required data + assertTrue(lsResult.contains("my_sub_dir")); + assertTrue(lsResult.contains("my_file")); + assertTrue(lsResult.contains("my_link")); + + // ... and only those + assertEquals(3, lsResult.size()); + } + +} Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java 2010-05-21 16:46:41 UTC (rev 50) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java 2010-05-25 15:24:32 UTC (rev 51) @@ -18,6 +18,7 @@ package org.nuiton.disworkfs; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -87,24 +88,24 @@ @Test public void testGetId() { - assertEquals("myid1", EntryUtil.getId(directoryEntry)); - assertEquals("myid2", EntryUtil.getId(fileEntry)); - assertEquals("myid3", EntryUtil.getId(linkEntry)); + assertEquals("myid1", EntryUtil.getIdFromEntry(directoryEntry)); + assertEquals("myid2", EntryUtil.getIdFromEntry(fileEntry)); + assertEquals("myid3", EntryUtil.getIdFromEntry(linkEntry)); } @Test public void testGetName() { - assertEquals("mydir", EntryUtil.getName(directoryEntry)); - assertEquals("myfile", EntryUtil.getName(fileEntry)); - assertEquals("mylink", EntryUtil.getName(linkEntry)); + assertEquals("mydir", EntryUtil.getNameFromEntry(directoryEntry)); + assertEquals("myfile", EntryUtil.getNameFromEntry(fileEntry)); + assertEquals("mylink", EntryUtil.getNameFromEntry(linkEntry)); } @Test public void testGetType() { - assertEquals(EntryUtil.TYPE.D, EntryUtil.getType(directoryEntry)); - assertEquals(EntryUtil.TYPE.F, EntryUtil.getType(fileEntry)); - assertEquals(EntryUtil.TYPE.L, EntryUtil.getType(linkEntry)); + assertEquals(EntryUtil.TYPE.D, EntryUtil.getTypeFromEntry(directoryEntry)); + assertEquals(EntryUtil.TYPE.F, EntryUtil.getTypeFromEntry(fileEntry)); + assertEquals(EntryUtil.TYPE.L, EntryUtil.getTypeFromEntry(linkEntry)); } @Test @@ -123,7 +124,7 @@ @Test public void testBytesToArray() { - String s = "abcdefg@^:éèvwxyz,!;*$"; + String s = "abcdefg@^:éèàÉÈÀvwxyz,!;*$"; String copy = EntryUtil.bytesToString(EntryUtil.stringToBytes(s)); assertEquals(s, copy); } @@ -137,10 +138,7 @@ public void testGetBlockIdsFromMetaBlock() { String[] expectedBlocksIds = {"myid1", "myid2", "myid3"}; String[] actualBlocksIds = EntryUtil.getBlockIdsFromMetaBlock(metaBlock); - assertEquals(expectedBlocksIds.length, actualBlocksIds.length); - assertEquals(expectedBlocksIds[0], actualBlocksIds[0]); - assertEquals(expectedBlocksIds[1], actualBlocksIds[1]); - assertEquals(expectedBlocksIds[2], actualBlocksIds[2]); + assertArrayEquals(expectedBlocksIds, actualBlocksIds); } @Test

1 0

r50 - in trunk/diswork-fs/src: main/java/org/nuiton/disworkfs main/java/org/nuiton/disworkfs/storage test/java/org/nuiton/disworkfs
by bleny＠users.nuiton.org May 21, 2010

May 21, 2010

Author: bleny Date: 2010-05-21 18:46:41 +0200 (Fri, 21 May 2010) New Revision: 50 Url: http://nuiton.org/repositories/revision/diswork/50 Log: ls + test Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryMap.java Removed: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/InMemoryMap.java Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkConfig.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java 2010-05-21 15:14:22 UTC (rev 49) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java 2010-05-21 16:46:41 UTC (rev 50) @@ -3,13 +3,17 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.util.List; public interface DistributedFileSystem { public boolean exists(String path) throws IOException; - public void write(String parent, String file, InputStream source) throws IOException; + public void write(String path, InputStream source) throws IOException; + public void write(String parent, String fileName, InputStream source) + throws IOException; + public InputStream read(String path) throws FileNotFoundException, IOException; @@ -19,4 +23,6 @@ public void remove(String path) throws IOException; + public List<String> ls(String path) throws IOException; + } Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkConfig.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkConfig.java 2010-05-21 15:14:22 UTC (rev 49) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkConfig.java 2010-05-21 16:46:41 UTC (rev 50) @@ -1,21 +1,14 @@ package org.nuiton.disworkfs; -import java.io.File; -import java.util.Random; - import org.nuiton.util.ApplicationConfig; public class DisworkConfig extends ApplicationConfig { public DisworkConfig() { - Random random = new Random(); - setDefaultOption("storage", "/tmp/disworkfs/storage" + random.nextInt()); - - // replication strategy... - // TODO 20100519 bleny chunk size parameter + setDefaultOption("blocks_size", "10485760"); // 10 MiB } - public File getStoragePath() { - return getOptionAsFile("storage"); + public int getBlockSize() { + return Integer.parseInt(getOption("blocks_size")); } } Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-21 15:14:22 UTC (rev 49) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-21 16:46:41 UTC (rev 50) @@ -3,10 +3,13 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.disworkfs.storage.EntryUtil; +import org.nuiton.disworkfs.storage.InMemoryMap; import org.nuiton.disworkfs.storage.Storage; public class DisworkFileSystem implements DistributedFileSystem { @@ -16,7 +19,7 @@ private static final Log log = LogFactory.getLog(DisworkFileSystem.class); public DisworkFileSystem(DisworkConfig disworkConfig) throws IOException { - storage = new Storage(new InMemoryMap()); + storage = new Storage(disworkConfig, new InMemoryMap()); storage.putDirectory("/", EntryUtil.newEmptyDirectoryContent()); } @@ -40,7 +43,8 @@ } // FIXME 20105021 bleny works fine but is not understandable - protected String walk(String path, String current, String content) throws IOException { + protected String walk(String path, String current, String content) + throws IOException { String result = null; if (path.equals("/")) { @@ -49,6 +53,7 @@ + "/"; } + String parentPath = EntryUtil.getParentFromPath(path); if (content == null) { @@ -150,6 +155,13 @@ } @Override + public void write(String path, InputStream source) throws IOException { + String parent = EntryUtil.getParentFromPath(path); + String name = EntryUtil.getNameFromPath(path); + write(parent, name, source); + } + + @Override public void write(String parent, String fileName, InputStream source) throws IOException { String entryParent = walk(parent); @@ -215,6 +227,14 @@ } public void ln(String parent, String name, String target) throws IOException { + + if (target.startsWith("/")) { + // target is absolute + } else { + // target is relative, taking this in consideration + target = EntryUtil.resolveLink(parent, target); + } + if (exists(target)) { String entryParent = walk(parent); @@ -290,4 +310,33 @@ } } + @Override + public List<String> ls(String path) throws IOException { + String entry = walk(path); + List<String> result = null; + + + if (entry == null) { + throw new IOException(path + " doesn't exists"); + } else { + if (EntryUtil.isLink(entry)) { + String target = storage.getLink(EntryUtil.getId(entry)); + entry = walk(target); + } + + if (EntryUtil.isDirectory(entry)) { + String content = storage.getDirectory(EntryUtil.getId(entry)); + String[] entries = content.split(EntryUtil.ENTRIES_SEPARATOR); + result = new ArrayList<String>(); + for (String elementEntry : entries) { + result.add(EntryUtil.getName(elementEntry)); + } + } else { + throw new IOException(path + " is not a directory"); + } + } + + return result; + } + } Deleted: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/InMemoryMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/InMemoryMap.java 2010-05-21 15:14:22 UTC (rev 49) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/InMemoryMap.java 2010-05-21 16:46:41 UTC (rev 50) @@ -1,20 +0,0 @@ -package org.nuiton.disworkfs; - -import java.util.Arrays; -import java.util.HashMap; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -public class InMemoryMap extends HashMap<String, byte[]> { - - private final Log log = LogFactory.getLog(InMemoryMap.class); - - @Override - public byte[] put(String key, byte[] value) { - log.info("put(\"" + key + "\"," + Arrays.toString(value) + ")"); - byte[] valueCopy = value.clone(); - log.info("putting " + valueCopy.length + " bytes"); - return super.put(key, valueCopy); - } -} Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java 2010-05-21 15:14:22 UTC (rev 49) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java 2010-05-21 16:46:41 UTC (rev 50) @@ -1,6 +1,7 @@ package org.nuiton.disworkfs.storage; +import java.nio.charset.Charset; import java.util.UUID; import org.apache.commons.io.FilenameUtils; @@ -27,6 +28,8 @@ static public enum TYPE{D /*Directory*/, L /*Link*/, F /*File*/}; + static protected Charset CHARSET = Charset.forName("UTF-8"); + /** * generate a new Id usage as a new key for the map * @return a String containing the id @@ -87,14 +90,14 @@ } static public int getTotalSizeFromMetaBlock(String metaBlock) { - // FIXME 20100520 bleny this workaround should not be + // dealing with empty meta-block (for empty directory or file) if (metaBlock.equals("0")) return 0; String result = metaBlock.substring(0, metaBlock.indexOf(BLOCKIDS_SEPARATOR)); return Integer.valueOf(result); } static public String[] getBlockIdsFromMetaBlock(String metaBlock) { - // FIXME 20100520 bleny this workaround should not be + // dealing with empty meta-block (for empty directory or file) if (metaBlock.indexOf(BLOCKIDS_SEPARATOR) == -1) { return new String[0]; } @@ -188,7 +191,7 @@ public static byte[] stringToBytes(String string) { // FIXME 20100519 bleny this code may cause bad conversion // due to charsets - byte[] bytes = string.getBytes(); + byte[] bytes = string.getBytes(CHARSET); return bytes; } @@ -198,7 +201,7 @@ * @return a byte array containing all the bytes in <code>string</code> */ public static String bytesToString(byte[] bytes) { - return new String(bytes); + return new String(bytes, CHARSET); } public static String getParentFromPath(String path) { Copied: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryMap.java (from rev 47, trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/InMemoryMap.java) =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryMap.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/InMemoryMap.java 2010-05-21 16:46:41 UTC (rev 50) @@ -0,0 +1,20 @@ +package org.nuiton.disworkfs.storage; + +import java.util.Arrays; +import java.util.HashMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +public class InMemoryMap extends HashMap<String, byte[]> { + + private final Log log = LogFactory.getLog(InMemoryMap.class); + + @Override + public byte[] put(String key, byte[] value) { + log.info("put(\"" + key + "\"," + Arrays.toString(value) + ")"); + byte[] valueCopy = value.clone(); + log.info("putting " + valueCopy.length + " bytes"); + return super.put(key, valueCopy); + } +} Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-05-21 15:14:22 UTC (rev 49) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-05-21 16:46:41 UTC (rev 50) @@ -8,6 +8,7 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.nuiton.disworkfs.DisworkConfig; public class Storage { @@ -84,9 +85,12 @@ } protected Map<String, byte[]> map; + + protected DisworkConfig disworkConfig; - public Storage(Map<String, byte[]> map) { + public Storage(DisworkConfig disworkConfig, Map<String, byte[]> map) { this.map = map; + this.disworkConfig = disworkConfig; } public boolean contains(Object id) { @@ -167,8 +171,8 @@ int read = 0; int totalSize = 0; - // TODO 20100519 bleny the size of blocks should be a config directive - byte[] buffer = new byte[10 * 1024 * 1024]; + int bufferSize = disworkConfig.getBlockSize(); + byte[] buffer = new byte[bufferSize]; while ((read = value.read(buffer)) != -1) { totalSize += read; Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java 2010-05-21 15:14:22 UTC (rev 49) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java 2010-05-21 16:46:41 UTC (rev 50) @@ -11,21 +11,18 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.util.List; import java.util.Random; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.nuiton.util.FileUtil; -public class DistributedFileSystemTest { - - private static final Log log = LogFactory.getLog(DistributedFileSystemTest.class); +public class DistributedFileSystemTest { /** * a place to store files for the test it's a subdirectory of the OS temp @@ -240,5 +237,17 @@ // trying to remove a non-empty directory should raise an exception fileSystem.remove("/my_folder"); } + + @Test + public void testLs() throws Exception { + fileSystem.mkdir("/my_folder"); + fileSystem.mkdir("/my_folder/my_sub_dir"); + fileSystem.write("/my_folder", "my_file", new FileInputStream(randomFilePath)); + fileSystem.ln("/my_folder/my_link", "my_file"); + + // trying to remove a non-empty directory should raise an exception + List<String> lsResult = fileSystem.ls("/my_folder"); + assertEquals(3, lsResult.size()); + } } Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java 2010-05-21 15:14:22 UTC (rev 49) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java 2010-05-21 16:46:41 UTC (rev 50) @@ -4,33 +4,23 @@ import java.util.Arrays; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.junit.After; -import org.junit.Before; import org.junit.Test; +import org.nuiton.disworkfs.storage.InMemoryMap; public class InMemoryMapTest { - private final Log log = LogFactory.getLog(InMemoryMapTest.class); - - - @Before - public void setUp() throws Exception {} - @After - public void tearDown() throws Exception {} - + /** + * this test show that the InMemoryMap put implies a copy of the data. + * After the put, the original value is modified. A final check shows + * there is no side effect + */ @Test public void testPut() { InMemoryMap map = new InMemoryMap(); - byte[] expected = {0x1, 0x2}; - + byte[] expected = {0x1, 0x2}; map.put("key", expected); - expected[0] = 0xf; - byte[] actual = map.get("key"); - assertFalse(Arrays.equals(expected, actual)); }

1 0

r49 - in trunk/diswork-fs/src: main/java/org/nuiton/disworkfs test/java/org/nuiton/disworkfs
by bleny＠users.nuiton.org May 21, 2010

May 21, 2010

Author: bleny Date: 2010-05-21 17:14:22 +0200 (Fri, 21 May 2010) New Revision: 49 Url: http://nuiton.org/repositories/revision/diswork/49 Log: clean imports Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-21 15:04:11 UTC (rev 48) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-21 15:14:22 UTC (rev 49) @@ -3,7 +3,6 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; -import java.util.Map.Entry; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java 2010-05-21 15:04:11 UTC (rev 48) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java 2010-05-21 15:14:22 UTC (rev 49) @@ -1,9 +1,9 @@ package org.nuiton.disworkfs; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertArrayEquals; import java.io.ByteArrayInputStream; import java.io.File; @@ -11,7 +11,6 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; -import java.util.Arrays; import java.util.Random; import org.apache.commons.io.FileUtils; @@ -23,10 +22,7 @@ import org.junit.Test; import org.nuiton.util.FileUtil; -import com.sun.org.apache.xml.internal.security.keys.storage.StorageResolverException; -import com.sun.xml.internal.ws.util.ByteArrayBuffer; - public class DistributedFileSystemTest { private static final Log log = LogFactory.getLog(DistributedFileSystemTest.class); Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java 2010-05-21 15:04:11 UTC (rev 48) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java 2010-05-21 15:14:22 UTC (rev 49) @@ -1,5 +1,7 @@ package org.nuiton.disworkfs; +import static org.junit.Assert.assertFalse; + import java.util.Arrays; import org.apache.commons.logging.Log; @@ -8,8 +10,6 @@ import org.junit.Before; import org.junit.Test; -import static org.junit.Assert.assertFalse; - public class InMemoryMapTest { private final Log log = LogFactory.getLog(InMemoryMapTest.class);

1 0

r48 - in trunk/diswork-fs/src: main/java/org/nuiton/disworkfs main/java/org/nuiton/disworkfs/storage test/java/org/nuiton/disworkfs
by bleny＠users.nuiton.org May 21, 2010

May 21, 2010

Author: bleny Date: 2010-05-21 17:04:11 +0200 (Fri, 21 May 2010) New Revision: 48 Url: http://nuiton.org/repositories/revision/diswork/48 Log: ln, remove with tests Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java 2010-05-21 13:01:45 UTC (rev 47) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java 2010-05-21 15:04:11 UTC (rev 48) @@ -15,4 +15,8 @@ public void mkdir(String path) throws IOException; + public void ln(String path, String target) throws IOException; + + public void remove(String path) throws IOException; + } Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-21 13:01:45 UTC (rev 47) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-21 15:04:11 UTC (rev 48) @@ -3,6 +3,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.util.Map.Entry; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -206,5 +207,88 @@ throw new IOException(parent + " is not a directory"); } } + + @Override + public void ln(String path, String target) throws IOException { + String parent = EntryUtil.getParentFromPath(path); + String name = EntryUtil.getNameFromPath(path); + ln(parent, name, target); + } + public void ln(String parent, String name, String target) throws IOException { + if (exists(target)) { + + String entryParent = walk(parent); + + if (entryParent == null) { + throw new IOException(parent + " directory doesn't exists"); + } + + if (EntryUtil.isDirectory(entryParent)) { + String parentId = EntryUtil.getId(entryParent); + String content = storage.getDirectory(parentId); + String newLinkId = EntryUtil.generateId(); + String newContent = EntryUtil.addEntryToDirectoryContent( + content, EntryUtil.TYPE.L, name, newLinkId); + + // store link before meta info + storage.putLink(newLinkId, target); + // update meta info directory + storage.putDirectory(parentId, newContent); + } else { + throw new IOException(parent + " is not a directory"); + } + + + } else { + throw new IOException(target + " is not a valid target"); + } + } + + @Override + public void remove(String path) throws IOException { + String parent = EntryUtil.getParentFromPath(path); + String name = EntryUtil.getNameFromPath(path); + log.info("trying to remove " + path); + remove(parent, name); + } + + protected void remove(String parent, String name) throws IOException { + String entryParent = walk(parent); + + if (entryParent == null) { + throw new IOException(parent + " directory doesn't exists"); + } + + if (EntryUtil.isDirectory(entryParent)) { + String parentId = EntryUtil.getId(entryParent); + String content = storage.getDirectory(parentId); + + String entry = EntryUtil.findEntryInDirectory(content, name); + String idToRemove = EntryUtil.getId(entry); + + String newContent = EntryUtil.removeEntryFromEntries( + content, name); + + // check if not removing a non-empty directory + if (EntryUtil.isDirectory(entry)) { + String innerDirectoryId = EntryUtil.getId(entry); + String innerDirectoryContent = storage.getDirectory(innerDirectoryId); + if (! innerDirectoryContent.equals(EntryUtil.newEmptyDirectoryContent())) { + // directory is not empty + throw new IOException("trying to remove a non-empty directory"); + } + } + + + // update meta info directory + storage.putDirectory(parentId, newContent); + // store file before meta info + storage.remove(idToRemove); + + } else { + throw new IOException(parent + " is not a directory"); + } + } + } Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java 2010-05-21 13:01:45 UTC (rev 47) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java 2010-05-21 15:04:11 UTC (rev 48) @@ -212,5 +212,11 @@ public static String getNameFromPath(String path) { return FilenameUtils.getName(path); } + + public static String removeEntryFromEntries(String content, String name) { + String entry = findEntryInDirectory(content, name); + String newContent = content.replaceFirst(entry + ENTRIES_SEPARATOR, ""); + return newContent; + } } Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java 2010-05-21 13:01:45 UTC (rev 47) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java 2010-05-21 15:04:11 UTC (rev 48) @@ -182,7 +182,7 @@ * doesn't exists */ @Test(expected = IOException.class) - public void testWriteFail() throws Exception { + public void testFailAtWrite() throws Exception { fileSystem.write("/my_folder", "my_file", new FileInputStream(randomFilePath)); } @@ -203,5 +203,46 @@ assertTrue(fileSystem.exists("/my_folder/my_sub_folder/my_file")); } + @Test + public void testln() throws Exception { + fileSystem.mkdir("/my_folder"); + fileSystem.write("/my_folder", "my_file", new FileInputStream(randomFilePath)); + fileSystem.ln("/my_link", "/my_folder/my_file"); + + InputStream source = new FileInputStream(randomFilePath); + InputStream readResult = fileSystem.read("/my_link"); + + boolean actualContentEquality = + IOUtils.contentEquals(source, readResult); + source.close(); + readResult.close(); + + assertTrue(actualContentEquality); + } + + @Test(expected = IOException.class) + public void testFailAtLn() throws Exception { + fileSystem.ln("/my_link", "/wrong_target_path"); + } + + @Test + public void testRemove() throws Exception { + fileSystem.mkdir("/my_folder"); + fileSystem.write("/my_folder", "my_file", new FileInputStream(randomFilePath)); + fileSystem.remove("/my_folder/my_file"); + assertTrue(fileSystem.exists("/my_folder")); + assertFalse(fileSystem.exists("/my_folder/my_file")); + fileSystem.remove("/my_folder"); + assertFalse(fileSystem.exists("/my_folder")); + } + + @Test(expected = IOException.class) + public void testFailAtRemove() throws Exception { + fileSystem.mkdir("/my_folder"); + fileSystem.write("/my_folder", "my_file", new FileInputStream(randomFilePath)); + + // trying to remove a non-empty directory should raise an exception + fileSystem.remove("/my_folder"); + } } Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java 2010-05-21 13:01:45 UTC (rev 47) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java 2010-05-21 15:04:11 UTC (rev 48) @@ -142,4 +142,20 @@ assertEquals(expectedBlocksIds[1], actualBlocksIds[1]); assertEquals(expectedBlocksIds[2], actualBlocksIds[2]); } + + @Test + public void testRemoveEntryFromEntries() { + + String content = directoryEntry + EntryUtil.ENTRIES_SEPARATOR + + fileEntry + EntryUtil.ENTRIES_SEPARATOR + + linkEntry + EntryUtil.ENTRIES_SEPARATOR; + + String expected = directoryEntry + EntryUtil.ENTRIES_SEPARATOR + + linkEntry + EntryUtil.ENTRIES_SEPARATOR; + + String actual = EntryUtil.removeEntryFromEntries(content, "myfile"); + + assertEquals(expected, actual); + + } }

1 0

r47 - in trunk/diswork-fs/src: main/java/org/nuiton/disworkfs main/java/org/nuiton/disworkfs/storage main/resources test/java/org/nuiton/disworkfs
by bleny＠users.nuiton.org May 21, 2010

May 21, 2010

Author: bleny Date: 2010-05-21 15:01:45 +0200 (Fri, 21 May 2010) New Revision: 47 Url: http://nuiton.org/repositories/revision/diswork/47 Log: write, read in deep hierarchy ; mkdir ; less bugs ; tests Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/InMemoryMap.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java trunk/diswork-fs/src/main/resources/log4j.properties trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java 2010-05-19 17:21:30 UTC (rev 46) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java 2010-05-21 13:01:45 UTC (rev 47) @@ -1,51 +1,18 @@ package org.nuiton.disworkfs; -import java.io.Closeable; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; public interface DistributedFileSystem { - - /** - * checks the existence of a file on the virtual FS - * - * @param path - * a path to a file on the virtual FS - * @return true if the file exists, false if not - */ + public boolean exists(String path) throws IOException; -; - /** - * write a file - * - * @param path - * path of the virtual FS where the file should be written - * @param source - * the file to copy on the VFS - * @throws IOException - * if problems occurs while reading the source or writing on VFS - */ public void write(String parent, String file, InputStream source) throws IOException; - /** - * read a file on the VFS, the file is downloaded if needed (may take some - * times) - * - * @param path - * the path in the virtual FS to the file you want to read - * @return ?? - * @throws FileNotFoundException - * if no file have been written to this path (you may use - * {@link #exists(String) to check before read} - * @throws IOException - * if a problem occur while reading the local file - */ public InputStream read(String path) throws FileNotFoundException, IOException; public void mkdir(String path) throws IOException; - } Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-19 17:21:30 UTC (rev 46) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-21 13:01:45 UTC (rev 47) @@ -1,12 +1,9 @@ package org.nuiton.disworkfs; -import java.io.ByteArrayInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; -import java.util.HashMap; -import org.apache.commons.io.FilenameUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.disworkfs.storage.EntryUtil; @@ -18,57 +15,89 @@ private static final Log log = LogFactory.getLog(DisworkFileSystem.class); - public DisworkFileSystem(DisworkConfig disworkConfig) { - storage = new Storage(new HashMap<String, byte[]>()); - // TODO 20100519 bleny create root dir + public DisworkFileSystem(DisworkConfig disworkConfig) throws IOException { + storage = new Storage(new InMemoryMap()); + storage.putDirectory("/", EntryUtil.newEmptyDirectoryContent()); } @Override public boolean exists(String path) throws IOException { - String entry = walk(path, null, null); + String entry = walk(path); boolean result = entry != null; return result; } - + /** - * Return id of the last element in path. + * return the entry of the element at the end of <code>path</code> * * @param path * @return null if path is not valid */ + protected String walk(String path) throws IOException { + String result = walk(path, null, null); + log.info("walking to " + path + " returns " + result); + return result; + } + + // FIXME 20105021 bleny works fine but is not understandable protected String walk(String path, String current, String content) throws IOException { String result = null; + + if (path.equals("/")) { + return EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR + + "/" + EntryUtil.ENTRY_SEPARATOR + + "/"; + } + + String parentPath = EntryUtil.getParentFromPath(path); + if (content == null) { // start of walk content = storage.getRootDirectory(); result = walk(path, "/", content); - } else if (FilenameUtils.getFullPathNoEndSeparator(path).equals(current)) { + } else if (parentPath.equals(current)) { // in the last directory - - // recuperation du directory a traiter dans p + // recuperation de l'element a traiter dans p + String tail = path.substring(current.length()); - String[] dirNames = tail.split(EntryUtil.ENTRY_SEPARATOR); - String p = dirNames[0]; - + + /* + log.info("tail " + tail); + String[] elementsNames = tail.split("/"); + String p = elementsNames[0]; + */ + String p = EntryUtil.getNameFromPath(tail); + + log.info("in final dir " + current + ", looking for " + p); + String entry = EntryUtil.findEntryInDirectory(content, p); - if (entry != null) { - result = entry; + result = entry; + } else { + // in middle of path + + String tail; // the path still remaining when in current + + if (current.equals("/")) { + tail = path.substring(current.length()); } else { - result = null; + tail = path.substring(current.length() + 1); } - } else { - // in middle of path + + log.debug("current = " + current); + log.debug("tail = " + tail); + String[] elementsNames = tail.split("/"); + String p = elementsNames[0]; - // recuperation du directory a traiter dans p - String tail = path.substring(current.length()); - String[] dirNames = tail.split(EntryUtil.ENTRY_SEPARATOR); - String p = dirNames[0]; - + log.info("in intermediate dir " + current + ", looking for " + p); + // mise a jour de current + if (current.equals("/")) current = ""; current += "/" + p; String entry = EntryUtil.findEntryInDirectory(content, p); - if (entry != null) { + if (entry == null) { + result = null; + } else { if (EntryUtil.isDirectory(entry)) { String id = EntryUtil.getId(entry); content = storage.getDirectory(id); @@ -81,12 +110,13 @@ // restart walk from / result = walk(newTarget, null, null); - } else { + } else if (EntryUtil.isFile(entry)) { // erreur, find file in path like '/dir1/dir2/filename/dir3' result = null; + } else { + log.warn("strange case : " + entry); + result = null; } - } else { - result = null; } } return result; @@ -94,44 +124,50 @@ @Override public InputStream read(String path) throws FileNotFoundException, - IOException { + IOException { - String entry = walk(path, null, null); + String entry = walk(path); if (entry == null) { throw new FileNotFoundException(path); } - InputStream result; + InputStream result = null; + if (EntryUtil.isLink(entry)) { + log.info("reading link " + path); String id = EntryUtil.getId(entry); String link = storage.getLink(id); String newTarget = EntryUtil.resolveLink(path, link); result = read(newTarget); - } else if(EntryUtil.isDirectory(entry)) { + } else if (EntryUtil.isDirectory(entry)) { throw new IOException("target is not a file: " + path); - } else { + } else if (EntryUtil.isFile(entry)) { + log.info("reading file " + path); String id = EntryUtil.getId(entry); result = storage.getFile(id); - } + } return result; } @Override - public void write(String parent, String file, InputStream source) throws IOException { - String entryParent = walk(parent, null, null); + public void write(String parent, String fileName, InputStream source) throws IOException { + String entryParent = walk(parent); + + if (entryParent == null) { + throw new IOException(parent + " directory doesn't exists"); + } + if (EntryUtil.isDirectory(entryParent)) { - String idParent = EntryUtil.getId(entryParent); - String content = storage.getDirectory(idParent); - String id = EntryUtil.generateId(); - String newContent = EntryUtil.addEntryToDirectoryContent(content, EntryUtil.TYPE.D, file, id); + String parentId = EntryUtil.getId(entryParent); + String content = storage.getDirectory(parentId); + String newFileId = EntryUtil.generateId(); + String newContent = EntryUtil.addEntryToDirectoryContent( + content, EntryUtil.TYPE.F, fileName, newFileId); // store file before meta info - // FIXME 20100519 bleny this should use putFile - storage.put(id, source); + storage.putFile(newFileId, source); // update meta info directory - InputStream parentContent = new ByteArrayInputStream(newContent.getBytes()); - // FIXME 20100519 bleny this should use putDirectory - storage.put(idParent, parentContent); + storage.putDirectory(parentId, newContent); } else { throw new IOException(parent + " is not a directory"); } @@ -139,7 +175,36 @@ @Override public void mkdir(String path) throws IOException { - throw new UnsupportedOperationException("not yet implemented"); + String parent = EntryUtil.getParentFromPath(path); + String dirName = EntryUtil.getNameFromPath(path); + mkdir(parent, dirName); } + public void mkdir(String parent, String dirName) throws IOException { + + log.info("trying to create directory " + dirName + " in " + parent); + + String entryParent = walk(parent); + + if (entryParent == null) { + throw new IOException(parent + " directory doesn't exists"); + } + + if (EntryUtil.isDirectory(entryParent)) { + String parentId = EntryUtil.getId(entryParent); + String content = storage.getDirectory(parentId); + String newDirectoryId = EntryUtil.generateId(); + String newContent = EntryUtil.addEntryToDirectoryContent( + content, EntryUtil.TYPE.D, dirName, newDirectoryId); + + // store file before meta info + storage.putDirectory(newDirectoryId, + EntryUtil.newEmptyDirectoryContent()); + // update meta info directory + storage.putDirectory(parentId, newContent); + } else { + throw new IOException(parent + " is not a directory"); + } + } + } Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/InMemoryMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/InMemoryMap.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/InMemoryMap.java 2010-05-21 13:01:45 UTC (rev 47) @@ -0,0 +1,20 @@ +package org.nuiton.disworkfs; + +import java.util.Arrays; +import java.util.HashMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +public class InMemoryMap extends HashMap<String, byte[]> { + + private final Log log = LogFactory.getLog(InMemoryMap.class); + + @Override + public byte[] put(String key, byte[] value) { + log.info("put(\"" + key + "\"," + Arrays.toString(value) + ")"); + byte[] valueCopy = value.clone(); + log.info("putting " + valueCopy.length + " bytes"); + return super.put(key, valueCopy); + } +} Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java 2010-05-19 17:21:30 UTC (rev 46) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java 2010-05-21 13:01:45 UTC (rev 47) @@ -87,11 +87,17 @@ } static public int getTotalSizeFromMetaBlock(String metaBlock) { + // FIXME 20100520 bleny this workaround should not be + if (metaBlock.equals("0")) return 0; String result = metaBlock.substring(0, metaBlock.indexOf(BLOCKIDS_SEPARATOR)); return Integer.valueOf(result); } static public String[] getBlockIdsFromMetaBlock(String metaBlock) { + // FIXME 20100520 bleny this workaround should not be + if (metaBlock.indexOf(BLOCKIDS_SEPARATOR) == -1) { + return new String[0]; + } String blockIdsAsString = metaBlock.substring(metaBlock.indexOf(BLOCKIDS_SEPARATOR) + 1, metaBlock.length()); String[] blockIds = blockIdsAsString.split(BLOCKIDS_SEPARATOR); return blockIds; @@ -119,16 +125,23 @@ /** * + */ + static public String newEmptyDirectoryContent() { + return ""; + } + + /** + * * @param content the string content of the directory (all entries before add) * @param type the type of the new entry * @param name the name (not a full path) of the new entry * @param id the id where the content of the entry can be found * @return a String with the new content - */ + */ static public String addEntryToDirectoryContent (String content, TYPE type, String name, String id) { - content += ENTRIES_SEPARATOR + type.name() + ENTRY_SEPARATOR + name + - ENTRY_SEPARATOR + id; + content += type.name() + ENTRY_SEPARATOR + name + + ENTRY_SEPARATOR + id + ENTRIES_SEPARATOR; return content; } @@ -142,6 +155,7 @@ * @return the path to the destination */ static public String resolveLink(String parent, String link) { + /* String result = null; if (link.startsWith("/")) { // destination is absolute, ignore parent @@ -161,6 +175,8 @@ result = FilenameUtils.normalize(parentWithNoFile + link); } return result; + */ + return FilenameUtils.concat(parent, link); } /** @@ -170,7 +186,8 @@ * @return a byte array containing all the bytes in <code>string</code> */ public static byte[] stringToBytes(String string) { - // FIXME 20100519 bleny this code may cause inconsistences with charsets + // FIXME 20100519 bleny this code may cause bad conversion + // due to charsets byte[] bytes = string.getBytes(); return bytes; } @@ -184,4 +201,16 @@ return new String(bytes); } + public static String getParentFromPath(String path) { + String parent = FilenameUtils.getFullPathNoEndSeparator(path); + if (parent.isEmpty()) { + parent = "/"; + } + return parent; + } + + public static String getNameFromPath(String path) { + return FilenameUtils.getName(path); + } + } Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-05-19 17:21:30 UTC (rev 46) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-05-21 13:01:45 UTC (rev 47) @@ -6,9 +6,13 @@ import java.util.Map; import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; public class Storage { + private static final Log log = LogFactory.getLog(Storage.class); + protected class SplitBlocksInputStream extends InputStream { protected Map<String, byte[]> map; @@ -17,8 +21,7 @@ protected String[] blockIds; protected int blockIdsIndex; - protected byte[] currentBlock; - protected int currentBlockIndex; + protected ByteArrayInputStream currentBlock; protected int available; @@ -32,31 +35,43 @@ @Override public int read() throws IOException { checkInitialization(); + + if (blockIdsIndex >= blockIds.length) { + return -1; + } // update current block if needed - if (currentBlock == null || currentBlockIndex >= currentBlock.length) { + if (currentBlock == null || currentBlock.available() == 0) { blockIdsIndex += 1; if (blockIdsIndex < blockIds.length) { - currentBlock = map.get(blockIds[blockIdsIndex]); - currentBlockIndex = 0; + currentBlock = new ByteArrayInputStream(map.get(blockIds[blockIdsIndex])); + log.debug("new current block (size = " + currentBlock.available() + ")"); } else { return -1; } } - byte result = currentBlock[currentBlockIndex]; - currentBlockIndex += 1; + int result = currentBlock.read(); + /* + log.debug("reading block number " + blockIdsIndex + + " (available = " + available + ")" + + " returns " + result); + */ + available -= 1; return result; } protected void checkInitialization() { - if (blockIds == null) { + if (blockIds == null) { byte[] bytes = map.get(id); String metaBlock = EntryUtil.bytesToString(bytes); blockIds = EntryUtil.getBlockIdsFromMetaBlock(metaBlock); blockIdsIndex = -1; available = EntryUtil.getTotalSizeFromMetaBlock(metaBlock); + log.debug("initializing stream with meta block \"" + metaBlock + + "\" (" + blockIds.length + " blocks, " + available + + " bytes)"); } } @@ -74,10 +89,14 @@ this.map = map; } - public boolean exists(Object key) { - return map.containsKey(key); + public boolean contains(Object id) { + return map.containsKey(id); } + /** + * @return the content (entries) of the root directory + * @throws IOException + */ public String getRootDirectory() throws IOException { String result = getDirectory("/"); return result; @@ -89,8 +108,9 @@ public String getDirectory(String id) throws IOException { InputStream in = get(id); - String result = IOUtils.toString(in); - return result; + String content = IOUtils.toString(in); + log.debug("getDirectory(\"" + id + "\") returns \n" + content); + return content; } public InputStream getFile(String id) { @@ -99,12 +119,28 @@ } public String getLink(String id) throws IOException { - return EntryUtil.bytesToString(map.get(id)); + String content = EntryUtil.bytesToString(map.get(id)); + log.debug("getDirectory(\"" + id + "\") returns \"" + content + "\""); + return content; } - - // TODO 20100519 bleny write getDirectory, getFile, getLink + public void putDirectory(String id, String content) throws IOException { + log.debug("putDirectory(\"" + id + "\", \"" + content + "\")"); + InputStream value = IOUtils.toInputStream(content); + put(id, value); + } + public void putFile(String id, InputStream content) throws IOException { + put(id, content); + } + + public void putLink(String id, String content) { + log.debug("putLink(\"" + id + "\", \"" + content + "\")"); + byte[] contentAsBytes = EntryUtil.stringToBytes(content); + map.put(id, contentAsBytes); + } + + /** * see {@link #get(String)} */ @@ -119,43 +155,47 @@ * @param value * @throws IOException */ - public void put(String key, InputStream value) throws IOException { + protected void put(String key, InputStream value) throws IOException { // TODO 20100519 bleny deal with copy-on-write - // TODO 20100519 bleny deal with null value properly - + // TODO 20100519 bleny deal with null value properly if (value == null) { value = new ByteArrayInputStream(new byte[0]); } String blocksIds = ""; - + int read = 0; + int totalSize = 0; + // TODO 20100519 bleny the size of blocks should be a config directive byte[] buffer = new byte[10 * 1024 * 1024]; - int read = value.read(buffer); - int totalSize = read; - - while (read == buffer.length) { + while ((read = value.read(buffer)) != -1) { + totalSize += read; + + byte[] newBlock = buffer; + + // if the block is shorter, truncate + if (read < buffer.length) { + newBlock = new byte[read]; + System.arraycopy(buffer, 0, newBlock, 0, read); + } + String id = EntryUtil.generateId(); - map.put(id, buffer); blocksIds += EntryUtil.BLOCKIDS_SEPARATOR + id; + + log.debug("saving new block (size = " + newBlock.length + + ") at key " + id); - // for next iteration - read = value.read(buffer); - totalSize += read; + // copy buffer in map + map.put(id, newBlock); + } - byte[] lastBuffer = new byte[read]; - System.arraycopy(buffer, 0, lastBuffer, 0, read); + String metaBlock = totalSize + blocksIds; - String id = EntryUtil.generateId(); - map.put(id, buffer); + log.debug("putting metablock " + metaBlock + " at key " + key); - blocksIds += EntryUtil.BLOCKIDS_SEPARATOR + id; - - String metaBlock = totalSize + blocksIds; - map.put(key, EntryUtil.stringToBytes(metaBlock)); } Modified: trunk/diswork-fs/src/main/resources/log4j.properties =================================================================== --- trunk/diswork-fs/src/main/resources/log4j.properties 2010-05-19 17:21:30 UTC (rev 46) +++ trunk/diswork-fs/src/main/resources/log4j.properties 2010-05-21 13:01:45 UTC (rev 47) @@ -5,5 +5,4 @@ log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=%d %5p [%t] (%F:%L) %M - %m%n # package level -log4j.logger.org.nuiton.disworkfs=TRACE -log4j.logger.org.nuiton.disworkfs.storage=WARN \ No newline at end of file +log4j.logger.org.nuiton.disworkfs=TRACE \ No newline at end of file Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java 2010-05-19 17:21:30 UTC (rev 46) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java 2010-05-21 13:01:45 UTC (rev 47) @@ -1,137 +1,207 @@ -//package org.nuiton.disworkfs; -// -//import static org.junit.Assert.assertFalse; -//import static org.junit.Assert.assertTrue; -// -//import java.io.File; -//import java.io.FileInputStream; -//import java.io.FileNotFoundException; -//import java.io.IOException; -//import java.io.InputStream; -//import java.util.Random; -// -//import org.apache.commons.io.FileUtils; -//import org.apache.commons.io.IOUtils; -//import org.junit.After; -//import org.junit.Before; -//import org.junit.Test; -//import org.nuiton.util.FileUtil; -// -//public class DistributedFileSystemTest { -// -// /** -// * a place to store files for the test it's a subdirectory of the OS temp -// * dir e.g. under linux /tmp/disworkfs/tests/ -// */ -// static protected String tempDirectoryPath = System.getProperty( -// "java.io.tmpdir", ".") -// + "/disworkfs/tests"; -// -// /** -// * We will create a file at this path for test purpose -// */ -// static protected String randomFilePath = tempDirectoryPath + "/randomfile"; -// -// /** -// * The file will have this fixed size -// */ -// static protected int randomFileSize = 3000; -// -// static protected DistributedFileSystem fileSystem; -// -// @Before -// public void setUp() throws Exception { -// File tempDirectory = new File(tempDirectoryPath); -// tempDirectory.mkdir(); -// -// Random random = new Random(); -// // creating random data for the file -// byte[] randomBytes = new byte[randomFileSize]; -// random.nextBytes(randomBytes); -// -// // dumping random data into the file -// File randomFile = new File(randomFilePath); -// FileUtils.writeByteArrayToFile(randomFile, randomBytes); -// -// DisworkConfig disworkConfig = new DisworkConfig(); -// -// fileSystem = new DisworkFileSystem(disworkConfig); -// } -// -// @After -// public void tearDown() throws Exception { -// // cleaning -// fileSystem.close(); -// -// FileUtil.deleteRecursively(tempDirectoryPath); -// } -// -// @Test -// public void testWrite() throws Exception { -// fileSystem.write("/my_file", new FileInputStream(randomFilePath)); -// } -// -// @Test -// public void testExists() throws Exception { -// fileSystem.write("/my_file", new FileInputStream(randomFilePath)); -// assertTrue(fileSystem.exists("/my_file")); -// assertFalse(fileSystem.exists("/my_other_file")); -// } -// -// /** -// * try to read a file that as never been created nor written -// */ -// @Test(expected = FileNotFoundException.class) -// public void testFailAtRead() throws Exception { -// fileSystem.read("/not_existing_file"); -// } -// -// /** -// * writing a file at the root directory and reading it. -// * finally, compare original source and read result -// * byte-to-byte : contents should be equals -// * @throws Exception -// */ -// @Test -// public void testWriteRead() throws Exception { -// -// InputStream source = new FileInputStream(randomFilePath); -// -// fileSystem.write("/my_file", source); -// -// InputStream readResult = fileSystem.read("/my_file"); -// -// source = new FileInputStream(randomFilePath); -// boolean actualContentEquality = -// IOUtils.contentEquals(source, readResult); -// -// assertTrue(actualContentEquality); -// } -// -// @Test -// public void testMkdir() throws Exception { -// fileSystem.mkdir("/my_folder"); -// assertTrue(fileSystem.exists("/my_folder")); -// } -// -// /** -// * this use case should raise an exception because my_folder -// * doesn't exists -// */ -// @Test(expected = IOException.class) -// public void testWriteFail() throws Exception { -// fileSystem.write("/my_folder/my_file", new FileInputStream(randomFilePath)); -// } -// -// @Test -// public void testWriteInFolder() throws Exception { -// fileSystem.mkdir("/my_folder"); -// fileSystem.write("/my_folder/my_file", new FileInputStream(randomFilePath)); -// fileSystem.mkdir("/my_folder/my_sub_folder"); -// fileSystem.write("/my_folder/my_sub_folder/my_file", new FileInputStream(randomFilePath)); -// -// ((DisworkFileSystem) fileSystem).dump(); -// } -// -// -//} +package org.nuiton.disworkfs; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertArrayEquals; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.Random; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.nuiton.util.FileUtil; + +import com.sun.org.apache.xml.internal.security.keys.storage.StorageResolverException; +import com.sun.xml.internal.ws.util.ByteArrayBuffer; + + +public class DistributedFileSystemTest { + + private static final Log log = LogFactory.getLog(DistributedFileSystemTest.class); + + /** + * a place to store files for the test it's a subdirectory of the OS temp + * dir e.g. under linux /tmp/disworkfs/tests/ + */ + static protected String tempDirectoryPath = System.getProperty( + "java.io.tmpdir", ".") + + "/disworkfs/tests"; + + /** + * We will create a file at this path for test purpose + */ + static protected String randomFilePath = tempDirectoryPath + "/randomfile"; + + /** + * The file will have this fixed size + */ + static protected int randomFileSize = 10000; + + static protected DistributedFileSystem fileSystem; + + @Before + public void setUp() throws Exception { + File tempDirectory = new File(tempDirectoryPath); + tempDirectory.mkdir(); + + Random random = new Random(); + // creating random data for the file + byte[] randomBytes = new byte[randomFileSize]; + random.nextBytes(randomBytes); + + // dumping random data into the file + File randomFile = new File(randomFilePath); + FileUtils.writeByteArrayToFile(randomFile, randomBytes); + + DisworkConfig disworkConfig = new DisworkConfig(); + + fileSystem = new DisworkFileSystem(disworkConfig); + } + + @After + public void tearDown() throws Exception { + // cleaning + FileUtil.deleteRecursively(tempDirectoryPath); + } + + @Test + public void testWrite() throws Exception { + fileSystem.write("/", "my_file", new FileInputStream(randomFilePath)); + } + + @Test + public void testExists() throws Exception { + fileSystem.write("/", "my_file", new FileInputStream(randomFilePath)); + assertTrue(fileSystem.exists("/my_file")); + assertFalse(fileSystem.exists("/my_other_file")); + } + + /** + * try to read a file that as never been created nor written + */ + @Test(expected = FileNotFoundException.class) + public void testFailAtRead() throws Exception { + fileSystem.read("/not_existing_file"); + } + + /** + * testing {@link org.nuiton.disworkfs.storage.Storage#SplitBlocksInputStream} + * by storing a "-1" byte, can be buggy due to the use of read() + * @throws IOException + */ + @Test + public void testSplit() throws IOException { + + byte[] bytes = new byte[1]; + bytes[0] = -0x1; + + InputStream source; + + source = new ByteArrayInputStream(bytes); + fileSystem.write("/", "my_file", source); + + source.close(); + + + source = new ByteArrayInputStream(bytes); + InputStream readResult = fileSystem.read("/my_file"); + + int read = 0; + byte[] b = new byte[1]; + + read = readResult.read(b); + + assertEquals(1, read); + assertArrayEquals(bytes, b); + + } + + /** + * writing a file at the root directory and reading it. + * finally, compare original source and read result + * byte-to-byte : contents should be equals + * @throws Exception + */ + @Test + public void testWriteRead() throws Exception { + + InputStream source = new FileInputStream(randomFilePath); + + fileSystem.write("/", "my_file", source); + + source.close(); + + InputStream readResult; + + // to be used for debugging purpose + + source = new FileInputStream(randomFilePath); + readResult = fileSystem.read("/my_file"); + + System.out.println("source.available() = " + source.available()); + System.out.println("readResult.available() = " + readResult.available()); + + byte[] sourceAsBytes = IOUtils.toByteArray(source); + byte[] readResultAsBytes = IOUtils.toByteArray(readResult); + /* + System.out.println("source.available() = " + source.available()); + System.out.println("readResult.available() = " + readResult.available()); + + System.out.println("source (" + sourceAsBytes.length + ") = " + Arrays.toString(sourceAsBytes)); + System.out.println("result (" + readResultAsBytes.length + ") = " + Arrays.toString(readResultAsBytes)); + */ + assertArrayEquals(sourceAsBytes, readResultAsBytes); + + + source = new FileInputStream(randomFilePath); + readResult = fileSystem.read("/my_file"); + + boolean actualContentEquality = + IOUtils.contentEquals(source, readResult); + source.close(); + readResult.close(); + + assertTrue(actualContentEquality); + + } + + /** + * this use case should raise an exception because my_folder + * doesn't exists + */ + @Test(expected = IOException.class) + public void testWriteFail() throws Exception { + fileSystem.write("/my_folder", "my_file", new FileInputStream(randomFilePath)); + } + + @Test + public void testMkdir() throws Exception { + fileSystem.mkdir("/my_folder"); + assertTrue(fileSystem.exists("/my_folder")); + fileSystem.mkdir("/my_folder/my_sub_folder"); + assertTrue(fileSystem.exists("/my_folder/my_sub_folder")); + } + + @Test + public void testWriteInFolder() throws Exception { + fileSystem.mkdir("/my_folder"); + fileSystem.write("/my_folder", "my_file", new FileInputStream(randomFilePath)); + fileSystem.mkdir("/my_folder/my_sub_folder"); + fileSystem.write("/my_folder/my_sub_folder", "my_file", new FileInputStream(randomFilePath)); + assertTrue(fileSystem.exists("/my_folder/my_sub_folder/my_file")); + } + + +} Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java 2010-05-19 17:21:30 UTC (rev 46) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java 2010-05-21 13:01:45 UTC (rev 47) @@ -51,7 +51,7 @@ + "myid3"; static protected String metaBlock = - "12345" + EntryUtil.BLOCKIDS_SEPARATOR + "12345" + EntryUtil.BLOCKIDS_SEPARATOR + "myid1" + EntryUtil.BLOCKIDS_SEPARATOR + "myid2" + EntryUtil.BLOCKIDS_SEPARATOR + "myid3"; @@ -65,20 +65,24 @@ @Test public void testFind() { - String content = directoryEntry + "\n" - + fileEntry + "\n" - + linkEntry; + String content = directoryEntry + EntryUtil.ENTRIES_SEPARATOR + + fileEntry + EntryUtil.ENTRIES_SEPARATOR + + linkEntry + EntryUtil.ENTRIES_SEPARATOR; - { - // recherche et trouve le nom - String entry = EntryUtil.findEntryInDirectory(content, "myfile"); - assertEquals(fileEntry, entry); - } - { - // recherche et ne trouve pas le nom - String entry = EntryUtil.findEntryInDirectory(content, "this_does_not_exists"); - assertEquals(null, entry); - } + // seek and return entry + String findResult; + + findResult = EntryUtil.findEntryInDirectory(content, "mydir"); + assertEquals(directoryEntry, findResult); + findResult = EntryUtil.findEntryInDirectory(content, "myfile"); + assertEquals(fileEntry, findResult); + findResult = EntryUtil.findEntryInDirectory(content, "mylink"); + assertEquals(linkEntry, findResult); + + + // return null if file not found + findResult = EntryUtil.findEntryInDirectory(content, "this_does_not_exists"); + assertEquals(null, findResult); } @Test @@ -109,8 +113,6 @@ path = EntryUtil.resolveLink("/dir/subdir", "/anotherdir"); assertEquals("/anotherdir", path); - path = EntryUtil.resolveLink("/dir/afile", "anotherdir"); - assertEquals("/dir/anotherdir", path); path = EntryUtil.resolveLink("/dir/subdir/", "anotherdir"); assertEquals("/dir/subdir/anotherdir", path); path = EntryUtil.resolveLink("/dir/subdir/", "../anotherdir"); Added: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java (rev 0) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/InMemoryMapTest.java 2010-05-21 13:01:45 UTC (rev 47) @@ -0,0 +1,37 @@ +package org.nuiton.disworkfs; + +import java.util.Arrays; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertFalse; + +public class InMemoryMapTest { + private final Log log = LogFactory.getLog(InMemoryMapTest.class); + + + @Before + public void setUp() throws Exception {} + + @After + public void tearDown() throws Exception {} + + @Test + public void testPut() { + InMemoryMap map = new InMemoryMap(); + byte[] expected = {0x1, 0x2}; + + map.put("key", expected); + + expected[0] = 0xf; + + byte[] actual = map.get("key"); + + assertFalse(Arrays.equals(expected, actual)); + } + +}

1 0

r46 - in trunk/diswork-fs/src: main/java/org/nuiton/disworkfs main/java/org/nuiton/disworkfs/storage test/java/org/nuiton/disworkfs
by bleny＠users.nuiton.org May 19, 2010

May 19, 2010

Author: bleny Date: 2010-05-19 19:21:30 +0200 (Wed, 19 May 2010) New Revision: 46 Url: http://nuiton.org/repositories/revision/diswork/46 Log: cleanup, refactoring in code and tests Added: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java Removed: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/EntryUtil.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/nodes/ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/operations/ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/split/ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/StorageStub.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/split/ Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkConfig.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java 2010-05-18 19:44:15 UTC (rev 45) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DistributedFileSystem.java 2010-05-19 17:21:30 UTC (rev 46) @@ -5,7 +5,7 @@ import java.io.IOException; import java.io.InputStream; -public interface DistributedFileSystem extends Closeable { +public interface DistributedFileSystem { /** * checks the existence of a file on the virtual FS Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkConfig.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkConfig.java 2010-05-18 19:44:15 UTC (rev 45) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkConfig.java 2010-05-19 17:21:30 UTC (rev 46) @@ -12,6 +12,7 @@ setDefaultOption("storage", "/tmp/disworkfs/storage" + random.nextInt()); // replication strategy... + // TODO 20100519 bleny chunk size parameter } public File getStoragePath() { Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-18 19:44:15 UTC (rev 45) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/DisworkFileSystem.java 2010-05-19 17:21:30 UTC (rev 46) @@ -1,25 +1,16 @@ package org.nuiton.disworkfs; -import java.io.BufferedInputStream; import java.io.ByteArrayInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.util.HashMap; -import java.util.UUID; import org.apache.commons.io.FilenameUtils; -import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.nuiton.disworkfs.nodes.DirectoryNode; -import org.nuiton.disworkfs.nodes.FileNode; -import org.nuiton.disworkfs.nodes.Node; -import org.nuiton.disworkfs.operations.Read; -import org.nuiton.disworkfs.operations.Write; +import org.nuiton.disworkfs.storage.EntryUtil; import org.nuiton.disworkfs.storage.Storage; -import org.nuiton.disworkfs.storage.StorageStub; -import org.nuiton.util.FileUtil; -import sun.management.snmp.jvminstr.JvmThreadInstanceEntryImpl.ThreadStateMap.Byte0; public class DisworkFileSystem implements DistributedFileSystem { @@ -28,11 +19,8 @@ private static final Log log = LogFactory.getLog(DisworkFileSystem.class); public DisworkFileSystem(DisworkConfig disworkConfig) { - storage = new StorageStub(); - - DirectoryNode root = new DirectoryNode(); - storage.put("/", root); - + storage = new Storage(new HashMap<String, byte[]>()); + // TODO 20100519 bleny create root dir } @Override @@ -52,7 +40,7 @@ String result = null; if (content == null) { // start of walk - content = storage.getRoot(); + content = storage.getRootDirectory(); result = walk(path, "/", content); } else if (FilenameUtils.getFullPathNoEndSeparator(path).equals(current)) { // in the last directory @@ -62,7 +50,7 @@ String[] dirNames = tail.split(EntryUtil.ENTRY_SEPARATOR); String p = dirNames[0]; - String entry = EntryUtil.find(content, p); + String entry = EntryUtil.findEntryInDirectory(content, p); if (entry != null) { result = entry; } else { @@ -79,7 +67,7 @@ // mise a jour de current current += "/" + p; - String entry = EntryUtil.find(content, p); + String entry = EntryUtil.findEntryInDirectory(content, p); if (entry != null) { if (EntryUtil.isDirectory(entry)) { String id = EntryUtil.getId(entry); @@ -88,7 +76,7 @@ } else if (EntryUtil.isLink(entry)) { String id = EntryUtil.getId(entry); String linkContent = storage.getLink(id); - String newTarget = EntryUtil.getAbsolutePath(current, linkContent); + String newTarget = EntryUtil.resolveLink(current, linkContent); newTarget += path.substring(current.length()); // restart walk from / @@ -117,7 +105,7 @@ if (EntryUtil.isLink(entry)) { String id = EntryUtil.getId(entry); String link = storage.getLink(id); - String newTarget = EntryUtil.getAbsolutePath(path, link); + String newTarget = EntryUtil.resolveLink(path, link); result = read(newTarget); } else if(EntryUtil.isDirectory(entry)) { throw new IOException("target is not a file: " + path); @@ -135,12 +123,14 @@ String idParent = EntryUtil.getId(entryParent); String content = storage.getDirectory(idParent); String id = EntryUtil.generateId(); - String newContent = EntryUtil.add(content, EntryUtil.TYPE.D, file, id); + String newContent = EntryUtil.addEntryToDirectoryContent(content, EntryUtil.TYPE.D, file, id); // store file before meta info + // FIXME 20100519 bleny this should use putFile storage.put(id, source); // update meta info directory InputStream parentContent = new ByteArrayInputStream(newContent.getBytes()); + // FIXME 20100519 bleny this should use putDirectory storage.put(idParent, parentContent); } else { throw new IOException(parent + " is not a directory"); @@ -148,63 +138,8 @@ } @Override - public void close() throws IOException { - storage.close(); - } - - @Override public void mkdir(String path) throws IOException { - - // First check everything is OK with parent dir - log.info("trying create(" + path + ")"); - - if (exists(path)) { - - throw new IOException("can't create " + path + ", already exists"); - - } else { - - String parentPath = FilenameUtils.getFullPathNoEndSeparator(path); - - // needed if parent dir is root - if (parentPath.equals("")) { - parentPath = "/"; - } - - Node parentNode = storage.get(parentPath); - - if (parentNode == null) { - throw new IOException("can't create " + path - + " directory " + parentPath + " doesn't exists"); - } - - if (parentNode.isDirectory()) { - - DirectoryNode parent = (DirectoryNode) parentNode; - - String fileName = FilenameUtils.getName(path); - - log.info("adding \"" + fileName + "\" to " + parentPath + " (" + parent + ")"); - - parent.add(fileName); - - storage.put(path, new DirectoryNode()); - - } else { - - throw new IOException("can't write : " + parentPath + " is not a directory"); - - } - } - + throw new UnsupportedOperationException("not yet implemented"); } - - public void dump() { - - for (String path : storage.keySet()) { - System.out.println(path + " → " + storage.get(path)); - } - - } } Deleted: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/EntryUtil.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/EntryUtil.java 2010-05-18 19:44:15 UTC (rev 45) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/EntryUtil.java 2010-05-19 17:21:30 UTC (rev 46) @@ -1,108 +0,0 @@ -/* *##% - * Copyright (c) 2010 poussin. All rights reserved. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - *##%*/ - -package org.nuiton.disworkfs; - - -import java.util.UUID; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -/** - * - * @author poussin - * @version $Revision$ - * - * Last update: $Date$ - * by : $Author$ - */ -public class EntryUtil { - - /** to use log facility, just put in your code: log.info(\"...\"); */ - static private Log log = LogFactory.getLog(EntryUtil.class); - - static final public String ENTRY_SEPARATOR = ":"; - - static final int TYPE_LENGTH = 1; - static final int UUID_LENGTH = UUID.randomUUID().toString().length(); - static final int ENTRY_SEPARATOR_LENGTH = ENTRY_SEPARATOR.length(); - - static public enum TYPE{D /*Directory*/, L /*Link*/, F /*File*/}; - - static public String add(String content, TYPE type, String name, String id) { - content += type.name() + ENTRY_SEPARATOR + name + ENTRY_SEPARATOR + id; - return content; - } - - static public String generateId() { - String result = UUID.randomUUID().toString(); - return result; - } - - static public boolean isDirectory(String entry) { - boolean result = entry.startsWith(TYPE.D.name()); - return result; - } - - static public boolean isLink(String entry) { - boolean result = entry.startsWith(TYPE.L.name()); - return result; - } - - /** - * find file name in directory content description - * - * @param directoryContent - * @param name - * @return null if name is not found - */ - static public String find(String directoryContent, String name) { - int index = directoryContent.indexOf(ENTRY_SEPARATOR + name + ENTRY_SEPARATOR); - - String result = null; - if (index != -1) { - result = directoryContent.substring( - index - TYPE_LENGTH, index + ENTRY_SEPARATOR_LENGTH + name.length()); - } - return result; - } - - static public String getId(String entry) { - int index = entry.lastIndexOf(ENTRY_SEPARATOR); - String result = entry.substring(index); - return result; - } - - static public String getName(String entry) { - int start = entry.indexOf(ENTRY_SEPARATOR); - int last = entry.lastIndexOf(ENTRY_SEPARATOR); - String result = entry.substring(start, last); - return result; - } - - static public TYPE getType(String entry) { - int index = entry.lastIndexOf(ENTRY_SEPARATOR); - String type = entry.substring(0, index); - TYPE result = TYPE.valueOf(type); - return result; - } - - static public String getAbsolutePath(String parent, String link) { - - } - -} Copied: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java (from rev 45, trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/EntryUtil.java) =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/EntryUtil.java 2010-05-19 17:21:30 UTC (rev 46) @@ -0,0 +1,187 @@ +package org.nuiton.disworkfs.storage; + + +import java.util.UUID; + +import org.apache.commons.io.FilenameUtils; + +/** + * @author poussin + * @version $Revision$ + * + * Last update: $Date$ + * by : $Author$ + */ +public class EntryUtil { + + static final public String ENTRY_SEPARATOR = ":"; + static final public String ENTRIES_SEPARATOR = "\n"; + static final public String BLOCKIDS_SEPARATOR = ";"; + + static final int TYPE_LENGTH = 1; + /* + // this line may not be accurate for "/" + static final int UUID_LENGTH = UUID.randomUUID().toString().length(); + */ + static final int ENTRY_SEPARATOR_LENGTH = ENTRY_SEPARATOR.length(); + + static public enum TYPE{D /*Directory*/, L /*Link*/, F /*File*/}; + + /** + * generate a new Id usage as a new key for the map + * @return a String containing the id + */ + static public String generateId() { + String result = UUID.randomUUID().toString(); + return result; + } + + /** + * @param entry a string taken from the entries of a directory + * @return true if the entry is of type directory, false if it's a file or + * a link + */ + static public boolean isDirectory(String entry) { + boolean result = entry.startsWith(TYPE.D.name()); + return result; + } + + /** + * @param entry a string taken from the entries of a directory + * @return true if the entry is of type link, false if it's a file or + * a directory + */ + static public boolean isLink(String entry) { + boolean result = entry.startsWith(TYPE.L.name()); + return result; + } + + /** + * @param entry a string taken from the entries of a directory + * @return true if the entry is of type file, false if it's a link or + * a directory + */ + static public boolean isFile(String entry) { + boolean result = entry.startsWith(TYPE.F.name()); + return result; + } + + static public TYPE getType(String entry) { + int index = entry.indexOf(ENTRY_SEPARATOR); + String type = entry.substring(0, index); + TYPE result = TYPE.valueOf(type); + return result; + } + + static public String getName(String entry) { + int start = entry.indexOf(ENTRY_SEPARATOR); + int last = entry.lastIndexOf(ENTRY_SEPARATOR); + String result = entry.substring(start + ENTRY_SEPARATOR_LENGTH, last); + return result; + } + + static public String getId(String entry) { + int index = entry.lastIndexOf(ENTRY_SEPARATOR); + String result = entry.substring(index + ENTRY_SEPARATOR_LENGTH); + return result; + } + + static public int getTotalSizeFromMetaBlock(String metaBlock) { + String result = metaBlock.substring(0, metaBlock.indexOf(BLOCKIDS_SEPARATOR)); + return Integer.valueOf(result); + } + + static public String[] getBlockIdsFromMetaBlock(String metaBlock) { + String blockIdsAsString = metaBlock.substring(metaBlock.indexOf(BLOCKIDS_SEPARATOR) + 1, metaBlock.length()); + String[] blockIds = blockIdsAsString.split(BLOCKIDS_SEPARATOR); + return blockIds; + } + + /** + * find file name in directory content description + * + * @param directoryContent the entries of the directory to search in + * @param name + * @return the entry or null if name is not found + */ + static public String findEntryInDirectory(String directoryContent, + String name) { + int index = directoryContent.indexOf( + ENTRY_SEPARATOR + name + ENTRY_SEPARATOR); + String result = null; + if (index != -1) { + result = directoryContent.substring( + index - TYPE_LENGTH, directoryContent.length()); + result = result.substring(0, result.indexOf(ENTRIES_SEPARATOR)); + } + return result; + } + + /** + * + * @param content the string content of the directory (all entries before add) + * @param type the type of the new entry + * @param name the name (not a full path) of the new entry + * @param id the id where the content of the entry can be found + * @return a String with the new content + */ + static public String addEntryToDirectoryContent + (String content, TYPE type, String name, String id) { + content += ENTRIES_SEPARATOR + type.name() + ENTRY_SEPARATOR + name + + ENTRY_SEPARATOR + id; + return content; + } + + /** + * Resolve a path from a parent directory, + * + * ie resolve /dir/subdir, ../file returns /dir/file + * @param parent the path to the dir where start from compute the path + * @param link the path to follow (may be absolute or relative to + * <code>parent</code>) + * @return the path to the destination + */ + static public String resolveLink(String parent, String link) { + String result = null; + if (link.startsWith("/")) { + // destination is absolute, ignore parent + result = link; + } else { + // destination is relative, so we have to consider parent + // parent may have a trailing file name, let's remove it + String parentWithNoFile = parent; + if (!parent.endsWith("/")) { + // removing the file at the end of the path, it's not a + // directory and thus should not be part of the path + parentWithNoFile = parent.substring(0, parent.lastIndexOf("/") + 1); + } + + // concatenate and making the complete path consistent + // ie /dir/subdir/../file to /dir/file + result = FilenameUtils.normalize(parentWithNoFile + link); + } + return result; + } + + /** + * may be used to store strings in a byte[]. String can be found back + * using {@link #bytesToString(byte[])} + * @param string the string + * @return a byte array containing all the bytes in <code>string</code> + */ + public static byte[] stringToBytes(String string) { + // FIXME 20100519 bleny this code may cause inconsistences with charsets + byte[] bytes = string.getBytes(); + return bytes; + } + + /** + * @see {@link #stringToBytes(String)} + * @param string the string + * @return a byte array containing all the bytes in <code>string</code> + */ + public static String bytesToString(byte[] bytes) { + return new String(bytes); + } + +} Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-05-18 19:44:15 UTC (rev 45) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-05-19 17:21:30 UTC (rev 46) @@ -1,67 +1,73 @@ package org.nuiton.disworkfs.storage; import java.io.ByteArrayInputStream; -import java.io.Closeable; import java.io.IOException; import java.io.InputStream; -import java.util.Collection; import java.util.Map; -import java.util.Set; -import java.util.UUID; import org.apache.commons.io.IOUtils; -import org.nuiton.disworkfs.nodes.Node; -/** - * This is a common memory where all the file system will be - * stored. Keys are paths of the tree and values are nodes - * (directory, file or symlink) you find at this precise path - * - * Exemple : - * <table> - * <tr> - * <th> - * Path (key) - * </th> - * <th> - * Node (value) - * </th> - * </tr> - * <tr> - * <td>"/"</td> - * <td> - * an instance of {@link org.nuiton.disworkfs.nodes.DirectoryNode} - * (content ["my_folder", "my_file"]) - * </td> - * </tr> - * <tr> - * <td>"/my_folder"</td> - * <td> - * an instance of {@link org.nuiton.disworkfs.nodes.DirectoryNode} - * (content ["my_second_file"]) - * </td> - * </tr> - * <tr> - * <td>"/my_file"</td> - * <td>an instance of {@link org.nuiton.disworkfs.nodes.FileNode}</td> - * </tr> - * <tr> - * <td>"/my_folder/my_second_file"</td> - * <td>an instance of {@link org.nuiton.disworkfs.nodes.FileNode}</td> - * </tr> - * </table> - * - * The Map should contains at least one entry : a DirectoryNode with the - * key "/". It will be the root folder. - * - * This interface should be implemented using a DHT or any other way to - * store a map on multiple nodes. - * - * Since this storage may use ressources (sockets or stream on files), it - * has to be closeable. - */ public class Storage { + protected class SplitBlocksInputStream extends InputStream { + + protected Map<String, byte[]> map; + protected String id; + + protected String[] blockIds; + protected int blockIdsIndex; + + protected byte[] currentBlock; + protected int currentBlockIndex; + + protected int available; + + + public SplitBlocksInputStream(Map<String, byte[]> map, String id) { + super(); + this.map = map; + this.id = id; + } + + @Override + public int read() throws IOException { + checkInitialization(); + + // update current block if needed + if (currentBlock == null || currentBlockIndex >= currentBlock.length) { + blockIdsIndex += 1; + if (blockIdsIndex < blockIds.length) { + currentBlock = map.get(blockIds[blockIdsIndex]); + currentBlockIndex = 0; + } else { + return -1; + } + } + + byte result = currentBlock[currentBlockIndex]; + currentBlockIndex += 1; + + return result; + } + + protected void checkInitialization() { + if (blockIds == null) { + byte[] bytes = map.get(id); + String metaBlock = EntryUtil.bytesToString(bytes); + blockIds = EntryUtil.getBlockIdsFromMetaBlock(metaBlock); + blockIdsIndex = -1; + available = EntryUtil.getTotalSizeFromMetaBlock(metaBlock); + } + } + + @Override + public int available() { + checkInitialization(); + return available; + } + + } + protected Map<String, byte[]> map; public Storage(Map<String, byte[]> map) { @@ -72,81 +78,91 @@ return map.containsKey(key); } - public String getRoot() throws IOException { + public String getRootDirectory() throws IOException { String result = getDirectory("/"); return result; } - public String getDirectory(String uuid) throws IOException { - InputStream in = get(uuid); - // lecture de l'entry pour / - byte[] entryAsByte = new byte[in.available()]; - in.read(entry); - String entry = EntryUtil.byteToSting(entryAsByte); - byte[] contentAsByte = storage.get(entry); - if (EntryUtil.isDirectory(entry)) { - String content = EntryUtil.byteToSting(contentAsByte); - return content; - } else { - throw IOException("Root is not a directory"); - } + + + + + public String getDirectory(String id) throws IOException { + InputStream in = get(id); + String result = IOUtils.toString(in); + return result; } - /** - * return path to target file - * - * @param uuid - * @return - * @throws IOException - */ - public String getLink(String uuid) throws IOException { - InputStream in = get(uuid); - // lecture de l'entry pour / - byte[] entryAsByte = new byte[in.available()]; - in.read(entry); - String entry = EntryUtil.byteToSting(entryAsByte); - return entry; + public InputStream getFile(String id) { + InputStream result = get(id); + return result; } - public InputStream getFile(String uuid) { - InputStream result = get(uuid); - return result; + public String getLink(String id) throws IOException { + return EntryUtil.bytesToString(map.get(id)); } - protected InputStream get(String uuid) { - byte[] uuidAsByte = map.get(key); - String uuid = EntryUtil.byteToSting(uuidAsByte); - byte[] content = map.get(uuid); - // TODO poussin 20100518 create specifique InputStream - // Lorsque le content pourra etre en plusieurs bon, il faudra - // ecrire un InputStream qui permet le chargement au fur et a mesure - // de la demande new InputStream(map, uuid) - ByteArrayInputStream result = new ByteArrayInputStream(content); - return result; + // TODO 20100519 bleny write getDirectory, getFile, getLink + + + /** + * see {@link #get(String)} + */ + protected InputStream get(String key) { + InputStream in = new SplitBlocksInputStream(map, key); + return in; } - public Object put(String key, InputStream value) { - // TODO poussin 20100518 split value in multiple block (see also get) - // il faudra decouper en plusieurs block plutot que de tout lire - // dans ce cas, il faut aussi stocker la liste des UUID genere pour - // chaque bloque - byte[] content = null; - if (value != null) { - content = new byte[value.available()]; - value.read(content); + /** + * a put in the map, this involves split considerations + * @param key + * @param value + * @throws IOException + */ + public void put(String key, InputStream value) throws IOException { + + // TODO 20100519 bleny deal with copy-on-write + // TODO 20100519 bleny deal with null value properly + + if (value == null) { + value = new ByteArrayInputStream(new byte[0]); } - map.put(key, content); + + String blocksIds = ""; + + // TODO 20100519 bleny the size of blocks should be a config directive + byte[] buffer = new byte[10 * 1024 * 1024]; + + int read = value.read(buffer); + int totalSize = read; + + while (read == buffer.length) { + String id = EntryUtil.generateId(); + map.put(id, buffer); + blocksIds += EntryUtil.BLOCKIDS_SEPARATOR + id; + + // for next iteration + read = value.read(buffer); + totalSize += read; + } + + byte[] lastBuffer = new byte[read]; + System.arraycopy(buffer, 0, lastBuffer, 0, read); + + String id = EntryUtil.generateId(); + map.put(id, buffer); + + blocksIds += EntryUtil.BLOCKIDS_SEPARATOR + id; + + String metaBlock = totalSize + blocksIds; + + map.put(key, EntryUtil.stringToBytes(metaBlock)); } - public Object remove(Object key) { + public void remove(Object key) { // TODO poussin 20100518 mark bloque to remove map.remove(key); } - - public void close() throws IOException { - throw new UnsupportedOperationException("Not supported yet."); - } - } Deleted: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/StorageStub.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/StorageStub.java 2010-05-18 19:44:15 UTC (rev 45) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/StorageStub.java 2010-05-19 17:21:30 UTC (rev 46) @@ -1,58 +0,0 @@ -package org.nuiton.disworkfs.storage; - -import java.io.IOException; -import java.util.HashMap; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.nuiton.disworkfs.nodes.Node; - -/** - * This class is a simple stub of {@link Storage}, it simply uses a HashMap - * in the local memory to store all the data. - * - * All the overridden methods of Map are rewritten only for logging purpose - * - */ -public class StorageStub extends HashMap<String, Node> implements Storage { - - private Log log = LogFactory.getLog(StorageStub.class); - - @Override - public boolean containsKey(Object key) { - log.info("containsKey" + key); - - if (!(key instanceof String)) { - log.warn("trying to look for a wrong-typed key"); - } - - return super.containsKey(key); - } - - @Override - public Node put(String key, Node value) { - log.info("put(" + key + ", " + value + ")"); - - return super.put(key, value); - } - - @Override - public Node get(Object key) { - - if (!(key instanceof String)) { - log.warn("trying to get for a wrong-typed key"); - } - - Node value = super.get(key); - - log.info("get(" + key + ") returns " + value + ")"); - - return value; - } - - @Override - public void close() throws IOException { - // nothing to do - } - -} Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java 2010-05-18 19:44:15 UTC (rev 45) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/DistributedFileSystemTest.java 2010-05-19 17:21:30 UTC (rev 46) @@ -1,136 +1,137 @@ -package org.nuiton.disworkfs; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.util.Random; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.nuiton.util.FileUtil; - -public class DistributedFileSystemTest { - - /** - * a place to store files for the test it's a subdirectory of the OS temp - * dir e.g. under linux /tmp/disworkfs/tests/ - */ - static protected String tempDirectoryPath = System.getProperty( - "java.io.tmpdir", ".") - + "/disworkfs/tests"; - - /** - * We will create a file at this path for test purpose - */ - static protected String randomFilePath = tempDirectoryPath + "/randomfile"; - - /** - * The file will have this fixed size - */ - static protected int randomFileSize = 3000; - - static protected DistributedFileSystem fileSystem; - - @Before - public void setUp() throws Exception { - File tempDirectory = new File(tempDirectoryPath); - tempDirectory.mkdir(); - - Random random = new Random(); - // creating random data for the file - byte[] randomBytes = new byte[randomFileSize]; - random.nextBytes(randomBytes); - - // dumping random data into the file - File randomFile = new File(randomFilePath); - FileUtils.writeByteArrayToFile(randomFile, randomBytes); - - DisworkConfig disworkConfig = new DisworkConfig(); - - fileSystem = new DisworkFileSystem(disworkConfig); - } - - @After - public void tearDown() throws Exception { - // cleaning - fileSystem.close(); - - FileUtil.deleteRecursively(tempDirectoryPath); - } - - @Test - public void testWrite() throws Exception { - fileSystem.write("/my_file", new FileInputStream(randomFilePath)); - } - - @Test - public void testExists() throws Exception { - fileSystem.write("/my_file", new FileInputStream(randomFilePath)); - assertTrue(fileSystem.exists("/my_file")); - assertFalse(fileSystem.exists("/my_other_file")); - } - - /** - * try to read a file that as never been created nor written - */ - @Test(expected = FileNotFoundException.class) - public void testFailAtRead() throws Exception { - fileSystem.read("/not_existing_file"); - } - - /** - * writing a file at the root directory and reading it. - * finally, compare original source and read result - * byte-to-byte : contents should be equals - * @throws Exception - */ - @Test - public void testWriteRead() throws Exception { - - InputStream source = new FileInputStream(randomFilePath); - - fileSystem.write("/my_file", source); - - InputStream readResult = fileSystem.read("/my_file"); - - source = new FileInputStream(randomFilePath); - boolean actualContentEquality = - IOUtils.contentEquals(source, readResult); - - assertTrue(actualContentEquality); - } - - @Test - public void testMkdir() throws Exception { - fileSystem.mkdir("/my_folder"); - assertTrue(fileSystem.exists("/my_folder")); - } - - /** - * this use case should raise an exception because my_folder - * doesn't exists - */ - @Test(expected = IOException.class) - public void testWriteFail() throws Exception { - fileSystem.write("/my_folder/my_file", new FileInputStream(randomFilePath)); - } - - @Test - public void testWriteInFolder() throws Exception { - fileSystem.mkdir("/my_folder"); - fileSystem.write("/my_folder/my_file", new FileInputStream(randomFilePath)); - fileSystem.mkdir("/my_folder/my_sub_folder"); - fileSystem.write("/my_folder/my_sub_folder/my_file", new FileInputStream(randomFilePath)); - - ((DisworkFileSystem) fileSystem).dump(); - } - -} +//package org.nuiton.disworkfs; +// +//import static org.junit.Assert.assertFalse; +//import static org.junit.Assert.assertTrue; +// +//import java.io.File; +//import java.io.FileInputStream; +//import java.io.FileNotFoundException; +//import java.io.IOException; +//import java.io.InputStream; +//import java.util.Random; +// +//import org.apache.commons.io.FileUtils; +//import org.apache.commons.io.IOUtils; +//import org.junit.After; +//import org.junit.Before; +//import org.junit.Test; +//import org.nuiton.util.FileUtil; +// +//public class DistributedFileSystemTest { +// +// /** +// * a place to store files for the test it's a subdirectory of the OS temp +// * dir e.g. under linux /tmp/disworkfs/tests/ +// */ +// static protected String tempDirectoryPath = System.getProperty( +// "java.io.tmpdir", ".") +// + "/disworkfs/tests"; +// +// /** +// * We will create a file at this path for test purpose +// */ +// static protected String randomFilePath = tempDirectoryPath + "/randomfile"; +// +// /** +// * The file will have this fixed size +// */ +// static protected int randomFileSize = 3000; +// +// static protected DistributedFileSystem fileSystem; +// +// @Before +// public void setUp() throws Exception { +// File tempDirectory = new File(tempDirectoryPath); +// tempDirectory.mkdir(); +// +// Random random = new Random(); +// // creating random data for the file +// byte[] randomBytes = new byte[randomFileSize]; +// random.nextBytes(randomBytes); +// +// // dumping random data into the file +// File randomFile = new File(randomFilePath); +// FileUtils.writeByteArrayToFile(randomFile, randomBytes); +// +// DisworkConfig disworkConfig = new DisworkConfig(); +// +// fileSystem = new DisworkFileSystem(disworkConfig); +// } +// +// @After +// public void tearDown() throws Exception { +// // cleaning +// fileSystem.close(); +// +// FileUtil.deleteRecursively(tempDirectoryPath); +// } +// +// @Test +// public void testWrite() throws Exception { +// fileSystem.write("/my_file", new FileInputStream(randomFilePath)); +// } +// +// @Test +// public void testExists() throws Exception { +// fileSystem.write("/my_file", new FileInputStream(randomFilePath)); +// assertTrue(fileSystem.exists("/my_file")); +// assertFalse(fileSystem.exists("/my_other_file")); +// } +// +// /** +// * try to read a file that as never been created nor written +// */ +// @Test(expected = FileNotFoundException.class) +// public void testFailAtRead() throws Exception { +// fileSystem.read("/not_existing_file"); +// } +// +// /** +// * writing a file at the root directory and reading it. +// * finally, compare original source and read result +// * byte-to-byte : contents should be equals +// * @throws Exception +// */ +// @Test +// public void testWriteRead() throws Exception { +// +// InputStream source = new FileInputStream(randomFilePath); +// +// fileSystem.write("/my_file", source); +// +// InputStream readResult = fileSystem.read("/my_file"); +// +// source = new FileInputStream(randomFilePath); +// boolean actualContentEquality = +// IOUtils.contentEquals(source, readResult); +// +// assertTrue(actualContentEquality); +// } +// +// @Test +// public void testMkdir() throws Exception { +// fileSystem.mkdir("/my_folder"); +// assertTrue(fileSystem.exists("/my_folder")); +// } +// +// /** +// * this use case should raise an exception because my_folder +// * doesn't exists +// */ +// @Test(expected = IOException.class) +// public void testWriteFail() throws Exception { +// fileSystem.write("/my_folder/my_file", new FileInputStream(randomFilePath)); +// } +// +// @Test +// public void testWriteInFolder() throws Exception { +// fileSystem.mkdir("/my_folder"); +// fileSystem.write("/my_folder/my_file", new FileInputStream(randomFilePath)); +// fileSystem.mkdir("/my_folder/my_sub_folder"); +// fileSystem.write("/my_folder/my_sub_folder/my_file", new FileInputStream(randomFilePath)); +// +// ((DisworkFileSystem) fileSystem).dump(); +// } +// +// +//} Modified: trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java 2010-05-18 19:44:15 UTC (rev 45) +++ trunk/diswork-fs/src/test/java/org/nuiton/disworkfs/EntryUtilTest.java 2010-05-19 17:21:30 UTC (rev 46) @@ -18,14 +18,13 @@ package org.nuiton.disworkfs; -import java.util.UUID; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.junit.Test; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertEquals; +import org.junit.Test; +import org.nuiton.disworkfs.storage.EntryUtil; + /** * * @author poussin @@ -36,89 +35,109 @@ */ public class EntryUtilTest { - /** to use log facility, just put in your code: log.info(\"...\"); */ - static private Log log = LogFactory.getLog(EntryUtilTest.class); - + static protected String directoryEntry = + EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR + + "mydir" + EntryUtil.ENTRY_SEPARATOR + + "myid1"; + + static protected String fileEntry = + EntryUtil.TYPE.F + EntryUtil.ENTRY_SEPARATOR + + "myfile" + EntryUtil.ENTRY_SEPARATOR + + "myid2"; + + static protected String linkEntry = + EntryUtil.TYPE.L + EntryUtil.ENTRY_SEPARATOR + + "mylink" + EntryUtil.ENTRY_SEPARATOR + + "myid3"; + + static protected String metaBlock = + "12345" + EntryUtil.BLOCKIDS_SEPARATOR + + "myid1" + EntryUtil.BLOCKIDS_SEPARATOR + + "myid2" + EntryUtil.BLOCKIDS_SEPARATOR + + "myid3"; + @Test public void testIsDirectory() { - String d = EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR - + "toto" + EntryUtil.ENTRY_SEPARATOR - + UUID.randomUUID().toString(); - - assertTrue(EntryUtil.isDirectory(d)); - - String f = EntryUtil.TYPE.F + EntryUtil.ENTRY_SEPARATOR - + "toto" + EntryUtil.ENTRY_SEPARATOR - + UUID.randomUUID().toString(); - - assertFalse(EntryUtil.isDirectory(f)); - + assertTrue(EntryUtil.isDirectory(directoryEntry)); + assertFalse(EntryUtil.isDirectory(fileEntry)); } @Test public void testFind() { - String d1 = EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR - + "toto" + EntryUtil.ENTRY_SEPARATOR - + UUID.randomUUID().toString(); - - String d2 = EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR - + "titi" + EntryUtil.ENTRY_SEPARATOR - + UUID.randomUUID().toString(); + + String content = directoryEntry + "\n" + + fileEntry + "\n" + + linkEntry; - String f = EntryUtil.TYPE.F + EntryUtil.ENTRY_SEPARATOR - + "tata" + EntryUtil.ENTRY_SEPARATOR - + UUID.randomUUID().toString(); - - String content = d1 + "\n" + d2 + "\n" + f; - { // recherche et trouve le nom - String entry = EntryUtil.find(content, "titi"); - assertEquals(entry, d2); + String entry = EntryUtil.findEntryInDirectory(content, "myfile"); + assertEquals(fileEntry, entry); } { // recherche et ne trouve pas le nom - String entry = EntryUtil.find(content, "pastrouve"); - assertEquals(entry, null); + String entry = EntryUtil.findEntryInDirectory(content, "this_does_not_exists"); + assertEquals(null, entry); } } @Test public void testGetId() { - String id = UUID.randomUUID().toString(); - String d = EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR - + "toto" + EntryUtil.ENTRY_SEPARATOR - + id; - - String id2 = EntryUtil.getId(d); - assertEquals(id, id2); - + assertEquals("myid1", EntryUtil.getId(directoryEntry)); + assertEquals("myid2", EntryUtil.getId(fileEntry)); + assertEquals("myid3", EntryUtil.getId(linkEntry)); } @Test public void testGetName() { - String id = UUID.randomUUID().toString(); - String name = "truc"; - String d = EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR - + name + EntryUtil.ENTRY_SEPARATOR - + id; + assertEquals("mydir", EntryUtil.getName(directoryEntry)); + assertEquals("myfile", EntryUtil.getName(fileEntry)); + assertEquals("mylink", EntryUtil.getName(linkEntry)); - String name2 = EntryUtil.getName(d); - assertEquals(name, name2); - } @Test public void testGetType() { - EntryUtil.TYPE type = EntryUtil.TYPE.D; - String id = UUID.randomUUID().toString(); - - String d = type + EntryUtil.ENTRY_SEPARATOR - + "toto" + EntryUtil.ENTRY_SEPARATOR - + id; + assertEquals(EntryUtil.TYPE.D, EntryUtil.getType(directoryEntry)); + assertEquals(EntryUtil.TYPE.F, EntryUtil.getType(fileEntry)); + assertEquals(EntryUtil.TYPE.L, EntryUtil.getType(linkEntry)); + } + + @Test + public void testResolveLink() { + String path; - EntryUtil.TYPE type2 = EntryUtil.getType(d); - assertEquals(type, type2); + path = EntryUtil.resolveLink("/dir/subdir", "/anotherdir"); + assertEquals("/anotherdir", path); + path = EntryUtil.resolveLink("/dir/afile", "anotherdir"); + assertEquals("/dir/anotherdir", path); + path = EntryUtil.resolveLink("/dir/subdir/", "anotherdir"); + assertEquals("/dir/subdir/anotherdir", path); + path = EntryUtil.resolveLink("/dir/subdir/", "../anotherdir"); + assertEquals("/dir/anotherdir", path); + path = EntryUtil.resolveLink("/dir/", "subdir/file"); + assertEquals("/dir/subdir/file", path); + } + + @Test + public void testBytesToArray() { + String s = "abcdefg@^:éèvwxyz,!;*$"; + String copy = EntryUtil.bytesToString(EntryUtil.stringToBytes(s)); + assertEquals(s, copy); + } + @Test + public void testGetTotalSizeFromMetaBlock() { + assertEquals(12345, EntryUtil.getTotalSizeFromMetaBlock(metaBlock)); } + + @Test + public void testGetBlockIdsFromMetaBlock() { + String[] expectedBlocksIds = {"myid1", "myid2", "myid3"}; + String[] actualBlocksIds = EntryUtil.getBlockIdsFromMetaBlock(metaBlock); + assertEquals(expectedBlocksIds.length, actualBlocksIds.length); + assertEquals(expectedBlocksIds[0], actualBlocksIds[0]); + assertEquals(expectedBlocksIds[1], actualBlocksIds[1]); + assertEquals(expectedBlocksIds[2], actualBlocksIds[2]); + } }

1 0

r45 - trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage
by bpoussin＠users.nuiton.org May 18, 2010

May 18, 2010

Author: bpoussin Date: 2010-05-18 21:44:15 +0200 (Tue, 18 May 2010) New Revision: 45 Url: http://nuiton.org/repositories/revision/diswork/45 Log: support put of null value Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java Modified: trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-05-18 19:35:33 UTC (rev 44) +++ trunk/diswork-fs/src/main/java/org/nuiton/disworkfs/storage/Storage.java 2010-05-18 19:44:15 UTC (rev 45) @@ -130,8 +130,11 @@ // il faudra decouper en plusieurs block plutot que de tout lire // dans ce cas, il faut aussi stocker la liste des UUID genere pour // chaque bloque - byte[] content = new byte[value.available()]; - value.read(content); + byte[] content = null; + if (value != null) { + content = new byte[value.available()]; + value.read(content); + } map.put(key, content); }

1 0

r44 - trunk/diswork-fs
by bpoussin＠users.nuiton.org May 18, 2010

May 18, 2010

Author: bpoussin Date: 2010-05-18 21:35:33 +0200 (Tue, 18 May 2010) New Revision: 44 Url: http://nuiton.org/repositories/revision/diswork/44 Log: suppression de la dependance peerunit Modified: trunk/diswork-fs/pom.xml Modified: trunk/diswork-fs/pom.xml =================================================================== --- trunk/diswork-fs/pom.xml 2010-05-18 19:34:53 UTC (rev 43) +++ trunk/diswork-fs/pom.xml 2010-05-18 19:35:33 UTC (rev 44) @@ -11,7 +11,20 @@ <packaging>jar</packaging> <name>disworkfs</name> - <dependencies> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <version>2.0.2</version> + <configuration> + <source>1.6</source> + <target>1.6</target> + </configuration> + </plugin> + </plugins> + </build> + <dependencies> <dependency> <groupId>commons-logging</groupId> <artifactId>commons-logging</artifactId> @@ -28,16 +41,18 @@ <groupId>org.nuiton</groupId> <artifactId>nuiton-utils</artifactId> </dependency> +  <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <scope>test</scope> </dependency> - </dependencies> + </dependencies> </project> +

1 0