June 2010 - Diswork-commits - mailman.cloud.codelutin.com

r92 - trunk
by bleny＠users.nuiton.org June 29, 2010

June 29, 2010

Author: bleny Date: 2010-06-29 13:13:09 +0200 (Tue, 29 Jun 2010) New Revision: 92 Url: http://nuiton.org/repositories/revision/diswork/92 Log: update pom, dependance commons daemon Modified: trunk/pom.xml Modified: trunk/pom.xml =================================================================== --- trunk/pom.xml 2010-06-28 10:23:31 UTC (rev 91) +++ trunk/pom.xml 2010-06-29 11:13:09 UTC (rev 92) @@ -35,45 +35,37 @@ <version>1.2.16</version> <scope>test</scope> </dependency> +  <dependency> - <groupId>jgroups</groupId> - <artifactId>jgroups</artifactId> - <version>2.9.0.GA</version> - <scope>compile</scope> - </dependency> - <dependency> <groupId>commons-io</groupId> <artifactId>commons-io</artifactId> <version>1.4</version> - <type>jar</type> - <scope>compile</scope> </dependency> <dependency> - <groupId>commons-lang</groupId> - <artifactId>commons-lang</artifactId> - <version>2.5</version> - <type>jar</type> - <scope>compile</scope> - </dependency> - <dependency> <groupId>org.nuiton</groupId> <artifactId>nuiton-utils</artifactId> <version>1.2.2</version> - <type>jar</type> - <scope>compile</scope> </dependency> <dependency> <groupId>fr.inria.peerunit</groupId> <artifactId>PeerUnit</artifactId> <version>1.0</version> - <scope>compile</scope> </dependency> +  <dependency> - <groupId>org.kth</groupId> - <artifactId>dks</artifactId> - <version>0.0.1</version> - </dependency> - <dependency> <groupId>org.freepastry</groupId> <artifactId>pastry</artifactId> <version>2.1</version> @@ -84,17 +76,13 @@ <version>0.4.12_cl20100625</version> </dependency> <dependency> - <groupId>commons-digester</groupId> - <artifactId>commons-digester</artifactId> - <version>2.0</version> + <groupId>commons-daemon</groupId> + <artifactId>commons-daemon</artifactId> + <version>1.0.1</version> </dependency> - <dependency> - <groupId>org.freemarker</groupId> - <artifactId>freemarker</artifactId> - <version>2.3.16</version> - </dependency> +  <dependency> <groupId>junit</groupId>

1 0

r91 - in trunk/diswork-daemon: . src/main/java/org/nuiton/diswork/daemon
by bleny＠users.nuiton.org June 28, 2010

June 28, 2010

Author: bleny Date: 2010-06-28 12:23:31 +0200 (Mon, 28 Jun 2010) New Revision: 91 Url: http://nuiton.org/repositories/revision/diswork/91 Log: menage eclipse Removed: trunk/diswork-daemon/.project Modified: trunk/diswork-daemon/pom.xml trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkDaemonRunner.java trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkSimpleClient.java trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/JobDescription.java Deleted: trunk/diswork-daemon/.project =================================================================== --- trunk/diswork-daemon/.project 2010-06-27 06:35:18 UTC (rev 90) +++ trunk/diswork-daemon/.project 2010-06-28 10:23:31 UTC (rev 91) @@ -1,23 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<projectDescription> - <name>diswork-daemon</name> - <comment></comment> - <projects> - </projects> - <buildSpec> - <buildCommand> - <name>org.eclipse.jdt.core.javabuilder</name> - <arguments> - </arguments> - </buildCommand> - <buildCommand> - <name>org.maven.ide.eclipse.maven2Builder</name> - <arguments> - </arguments> - </buildCommand> - </buildSpec> - <natures> - <nature>org.eclipse.jdt.core.javanature</nature> - <nature>org.maven.ide.eclipse.maven2Nature</nature> - </natures> -</projectDescription> Modified: trunk/diswork-daemon/pom.xml =================================================================== --- trunk/diswork-daemon/pom.xml 2010-06-27 06:35:18 UTC (rev 90) +++ trunk/diswork-daemon/pom.xml 2010-06-28 10:23:31 UTC (rev 91) @@ -31,21 +31,13 @@ <artifactId>commons-io</artifactId> </dependency> <dependency> - <groupId>commons-lang</groupId> - <artifactId>commons-lang</artifactId> - </dependency> - <dependency> <groupId>org.nuiton</groupId> <artifactId>nuiton-utils</artifactId> </dependency> <dependency> - <groupId>commons-digester</groupId> - <artifactId>commons-digester</artifactId> + <groupId>commons-daemon</groupId> + <artifactId>commons-daemon</artifactId> </dependency> - <dependency> - <groupId>org.freemarker</groupId> - <artifactId>freemarker</artifactId> - </dependency>  <dependency> Modified: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkDaemonRunner.java =================================================================== --- trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkDaemonRunner.java 2010-06-27 06:35:18 UTC (rev 90) +++ trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkDaemonRunner.java 2010-06-28 10:23:31 UTC (rev 91) @@ -24,24 +24,46 @@ */ package org.nuiton.diswork.daemon; +import org.apache.commons.daemon.Daemon; +import org.apache.commons.daemon.DaemonContext; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.diswork.fs.DisworkFileSystemConfig; /** + * This class is used to run diswork daemon. It can be used to run a deamon + * using a simple command-line. * + * This class can be used to run a diswork daemon as a service on the OS. + * Thus, it will be started when the machine boot and stopped just before the + * computer is shut down. + * * @author bleny */ -public class DisworkDaemonRunner { +public class DisworkDaemonRunner implements Daemon { private static final Log log = LogFactory.getLog(DisworkDaemonRunner.class); + + protected static DisworkConfig config; + protected static DisworkDaemon daemon; + /** * @param args */ public static void main(String[] args) throws Exception { - - DisworkConfig config = new DisworkConfig(); + DisworkDaemonRunner instance = new DisworkDaemonRunner(); + instance.init(args); + instance.start(); + } + + /** + * set <code>config</code> according args + * @param args + * @throws DisworkException + */ + protected void init(String[] args) throws DisworkException { + config = new DisworkConfig(); config.setFileSystemConfig( DisworkFileSystemConfig.newKademliaDisworkConfig()); if (args.length == 2) { @@ -50,7 +72,28 @@ Integer.parseInt(args[1]))); } config.setActivityStrategy("unlimited"); - config.fileSystemConfig.setBlockSize(65000); - DisworkDaemon daemon = new DisworkDaemon(config); } + + // constructor need to have empty params + public DisworkDaemonRunner() {} + + @Override + public void destroy() { + // nothing to do + } + + @Override + public void init(DaemonContext context) throws Exception { + init(context.getArguments()); + } + + @Override + public void start() throws Exception { + daemon = new DisworkDaemon(config); + } + + @Override + public void stop() throws Exception { + daemon.close(); + } } Modified: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkSimpleClient.java =================================================================== --- trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkSimpleClient.java 2010-06-27 06:35:18 UTC (rev 90) +++ trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkSimpleClient.java 2010-06-28 10:23:31 UTC (rev 91) @@ -71,6 +71,7 @@ while(! daemon.isFinished(job)) { Thread.sleep(5 * 1000); + System.out.print("."); } System.out.println(IOUtils.toString(daemon.getResults(job).get("output.txt"))); @@ -136,7 +137,6 @@ DisworkFileSystemConfig.newKademliaDisworkConfig(args[0], port)); config.setActivityStrategy("none"); config.setUsedPort(30000); - config.fileSystemConfig.setBlockSize(65000); daemon = new DisworkDaemon(config); //userPrompt(); Modified: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/JobDescription.java =================================================================== --- trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/JobDescription.java 2010-06-27 06:35:18 UTC (rev 90) +++ trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/JobDescription.java 2010-06-28 10:23:31 UTC (rev 91) @@ -81,7 +81,7 @@ protected String applicationName; /** the version of the application - * can't be null applicationName is set + * can't be null if applicationName is set */ protected String applicationVersion; @@ -97,7 +97,7 @@ /** the name of some input files and the URI where to get it */ protected Map<String, URL> inputUrls = new HashMap<String, URL>(); - /** */ + /** data needed for the job, provided through the file-system */ protected Map<String, InputStream> inputData = new HashMap<String, InputStream>(); /** file where to read the standard input, may be null */

1 0

r90 - trunk
by tchemit＠users.nuiton.org June 27, 2010

June 27, 2010

Author: tchemit Date: 2010-06-27 08:35:18 +0200 (Sun, 27 Jun 2010) New Revision: 90 Url: http://nuiton.org/repositories/revision/diswork/90 Log: Utilisation de mavenpom4redmine 2.2.1 Modified: trunk/pom.xml Modified: trunk/pom.xml =================================================================== --- trunk/pom.xml 2010-06-25 10:15:23 UTC (rev 89) +++ trunk/pom.xml 2010-06-27 06:35:18 UTC (rev 90) @@ -1,6 +1,5 @@ <?xml version="1.0" encoding="UTF-8"?> -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> @@ -11,7 +10,7 @@ <parent> <groupId>org.nuiton</groupId> <artifactId>mavenpom4redmine</artifactId> - <version>2.2</version> + <version>2.2.1</version> </parent> <artifactId>diswork</artifactId>

1 0

r89 - trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon
by bleny＠users.nuiton.org June 25, 2010

June 25, 2010

Author: bleny Date: 2010-06-25 12:15:23 +0200 (Fri, 25 Jun 2010) New Revision: 89 Url: http://nuiton.org/repositories/revision/diswork/89 Log: bugfix tests Modified: trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonMultipleNodesTest.java Modified: trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonMultipleNodesTest.java =================================================================== --- trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonMultipleNodesTest.java 2010-06-25 09:59:07 UTC (rev 88) +++ trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonMultipleNodesTest.java 2010-06-25 10:15:23 UTC (rev 89) @@ -33,8 +33,8 @@ config2.setBootstrapIp(DisworkFileSystemConfig.getIp()); config2.setBootstrapPort(port); port += 1; - config.setUsedPort(port); - config.setActivityStrategy("unlimited"); + config2.setUsedPort(port); + config2.setActivityStrategy("unlimited"); daemon2 = new DisworkDaemon(config2); }

1 0

r88 - in trunk: diswork-daemon/src/test/java/org/nuiton/diswork/daemon diswork-fs/src/test/java/org/nuiton/diswork/fs/storage diswork-fs/src/test/resources
by bleny＠users.nuiton.org June 25, 2010

June 25, 2010

Author: bleny Date: 2010-06-25 11:59:07 +0200 (Fri, 25 Jun 2010) New Revision: 88 Url: http://nuiton.org/repositories/revision/diswork/88 Log: m?\195?\160j tests Modified: trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonMultipleNodesTest.java trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonTest.java trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/AbstractDisworkMapTest.java trunk/diswork-fs/src/test/resources/log4j.properties Modified: trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonMultipleNodesTest.java =================================================================== --- trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonMultipleNodesTest.java 2010-06-25 09:38:13 UTC (rev 87) +++ trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonMultipleNodesTest.java 2010-06-25 09:59:07 UTC (rev 88) @@ -2,6 +2,7 @@ import java.io.InputStream; +import org.junit.After; import org.junit.Before; import org.junit.Test; import org.nuiton.diswork.fs.DisworkFileSystemConfig; @@ -36,5 +37,10 @@ config.setActivityStrategy("unlimited"); daemon2 = new DisworkDaemon(config2); } + + @After + public void tearDown() throws Exception { + daemon2.close(); + } } Modified: trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonTest.java =================================================================== --- trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonTest.java 2010-06-25 09:38:13 UTC (rev 87) +++ trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonTest.java 2010-06-25 09:59:07 UTC (rev 88) @@ -31,7 +31,6 @@ @After public void tearDown() throws Exception { - Thread.sleep(5000); daemon.close(); } Modified: trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/AbstractDisworkMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/AbstractDisworkMapTest.java 2010-06-25 09:38:13 UTC (rev 87) +++ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/AbstractDisworkMapTest.java 2010-06-25 09:59:07 UTC (rev 88) @@ -101,53 +101,21 @@ @Test public void testStoreLargeValue() { - - // 237499 - // 312499 - // 351561 - // 249999 - - int min = 0; - int max = 10 * 1000 * 1000; - - - int milieu = (min + max) / 2; - - while (max - min > 1) { + byte[] newBytes = new byte[11 * 1000 * 1000]; + Random random = new Random(); + random.nextBytes(newBytes); - byte[] newBytes = new byte[milieu]; - Random random = new Random(); - random.nextBytes(newBytes); - - map1.put("test", newBytes); + log.debug("putting large value"); + map1.put("test", newBytes); + log.debug("getting large value"); + + byte[] getResult = map2.get("test"); - byte[] getResult = map1.get("test"); - assertNotNull(getResult); - assertEquals(milieu, getResult.length); - - getResult = map2.get("test"); - if (getResult == null) { - System.out.println(max + " trop grand"); - max = milieu; - } else { - System.out.println(min + " trop petit"); - min = milieu; - } - - milieu = (min + max) / 2; - System.out.println(min + " < " + milieu + " < " + max); - } + assertNotNull(getResult); + assertEquals(newBytes.length, getResult.length); + assertArrayEquals(newBytes, getResult); - System.out.println(milieu); - /* - assertNotNull(getResult); - assertEquals(bytesSize, getResult.length); - - assertArrayEquals(newBytes, getResult); - */ - - } } Modified: trunk/diswork-fs/src/test/resources/log4j.properties =================================================================== --- trunk/diswork-fs/src/test/resources/log4j.properties 2010-06-25 09:38:13 UTC (rev 87) +++ trunk/diswork-fs/src/test/resources/log4j.properties 2010-06-25 09:59:07 UTC (rev 88) @@ -8,4 +8,5 @@ log4j.logger.org.nuiton.diswork.fs=WARN log4j.logger.org.nuiton.diswork.fs.storage.KademliaDisworkMap=INFO log4j.logger.org.nuiton.diswork.fs.Demo=INFO -log4j.logger.org.planx.xmlstore.routing=DEBUG \ No newline at end of file +log4j.logger.org.planx.xmlstore.routing=DEBUG +log4j.logger.org.nuiton.diswork.fs.storage.AbstractDisworkMapTest=DEBUG \ No newline at end of file

1 0

r87 - trunk
by bleny＠users.nuiton.org June 25, 2010

June 25, 2010

Author: bleny Date: 2010-06-25 11:38:13 +0200 (Fri, 25 Jun 2010) New Revision: 87 Url: http://nuiton.org/repositories/revision/diswork/87 Log: changement de version xmlstore Modified: trunk/pom.xml Modified: trunk/pom.xml =================================================================== --- trunk/pom.xml 2010-06-24 14:13:06 UTC (rev 86) +++ trunk/pom.xml 2010-06-25 09:38:13 UTC (rev 87) @@ -82,7 +82,7 @@ <dependency> <groupId>org.planx</groupId> <artifactId>koala-xmlstore</artifactId> - <version>0.4.12_cl20100612</version> + <version>0.4.12_cl20100625</version> </dependency> <dependency> <groupId>commons-digester</groupId>

1 0

r86 - in trunk: . diswork-daemon diswork-daemon/src/main/java/org/nuiton/diswork/daemon diswork-daemon/src/test/java/org/nuiton/diswork/daemon diswork-daemon/src/test/resources diswork-fs/src/main/java/org/nuiton/diswork/fs diswork-fs/src/test/java/org/nuiton/diswork/fs diswork-fs/src/test/java/org/nuiton/diswork/fs/storage diswork-fs/src/test/resources
by bleny＠users.nuiton.org June 24, 2010

June 24, 2010

Author: bleny Date: 2010-06-24 16:13:06 +0200 (Thu, 24 Jun 2010) New Revision: 86 Url: http://nuiton.org/repositories/revision/diswork/86 Log: lecture continue de la sortie standard, tests avec plusieurs d?\195?\169mons, m?\195?\160j pom (mina) Added: trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonMultipleNodesTest.java Modified: trunk/diswork-daemon/pom.xml trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkDaemonRunner.java trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkSimpleClient.java trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/WorkersManager.java trunk/diswork-daemon/src/test/resources/log4j.properties trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystemConfig.java trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/AbstractDisworkFileSystemTest.java trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemKademliaTest.java trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/AbstractDisworkMapTest.java trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/KademliaDisworkMapTest.java trunk/diswork-fs/src/test/resources/log4j.properties trunk/pom.xml Modified: trunk/diswork-daemon/pom.xml =================================================================== --- trunk/diswork-daemon/pom.xml 2010-06-21 10:14:34 UTC (rev 85) +++ trunk/diswork-daemon/pom.xml 2010-06-24 14:13:06 UTC (rev 86) @@ -42,6 +42,10 @@ <groupId>commons-digester</groupId> <artifactId>commons-digester</artifactId> </dependency> + <dependency> + <groupId>org.freemarker</groupId> + <artifactId>freemarker</artifactId> + </dependency>  <dependency> Modified: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkDaemonRunner.java =================================================================== --- trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkDaemonRunner.java 2010-06-21 10:14:34 UTC (rev 85) +++ trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkDaemonRunner.java 2010-06-24 14:13:06 UTC (rev 86) @@ -50,6 +50,7 @@ Integer.parseInt(args[1]))); } config.setActivityStrategy("unlimited"); + config.fileSystemConfig.setBlockSize(65000); DisworkDaemon daemon = new DisworkDaemon(config); } } Modified: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkSimpleClient.java =================================================================== --- trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkSimpleClient.java 2010-06-21 10:14:34 UTC (rev 85) +++ trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkSimpleClient.java 2010-06-24 14:13:06 UTC (rev 86) @@ -28,7 +28,7 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; -import java.io.IOException; +import java.io.InputStream; import java.io.InputStreamReader; import java.util.Map; @@ -36,7 +36,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.diswork.fs.DisworkFileSystemConfig; -import org.nuiton.util.FileUtil; /** * @@ -80,36 +79,49 @@ } public static void isisSubmit() throws Exception { - File isis = new File("/tmp/isis/isis-fish-3.3.0.3-bin.zip"); - daemon.submitApplication("isis-fish", "3.3.0.3", new FileInputStream(isis)); + File isis = new File("/tmp/isis/isis-fish-3.3.0.4-dev3069-bin.zip"); + // File isis = new File("/tmp/isis/fake-app-1.0.zip"); + + InputStream in = new FileInputStream(isis); + log.info("isis is " + in.available() + " bytes"); + daemon.submitApplication("isis-fish", "3.3.0.4-dev3069", in); JobDescription jobDescription = new JobDescription(); - jobDescription.setJobName("Mon Job Isis"); + jobDescription.setApplication("isis-fish", "3.3.0.4-dev3069"); - jobDescription.setApplication("isis-fish", "3.3.0.3"); + File input = new File("/tmp/isis/isisfish-simulation-ica10ans.zip"); + jobDescription.addInput("isisfish-simulation-ica10ans.zip", new FileInputStream(input)); + jobDescription.addOutput("isisfish-simulation-ica10ans-result.zip"); - File input = new File("/tmp/isis/sim_test-gdg-3.2-3.3.zip"); - jobDescription.addInput("sim_test-gdg-3.2-3.3.zip", new FileInputStream(input)); - jobDescription.addOutput("sim_test-gdg-3.2-3.3-result.zip"); + jobDescription.setCommandLine( + "%java -Xmx2500M -jar isis-fish-3.3.0.4-SNAPSHOT/isis-fish-3.3.0.4-SNAPSHOT.jar" + + " --option launch.ui false" + + " --option perform.vcsupdate false" + + " --option perform.migration false" + + " --option perform.cron false" + + " --simulateRemotelly my_isis_job isisfish-simulation-ica10ans.zip" + + " isisfish-simulation-ica10ans-result.zip"); - jobDescription.setCommandLine("%java -Xmx2500M -jar isis-fish*.jar" + - " --option launch.ui false --option perform.vcsupdate false" + - " --option perform.migration false --option perform.cron false" - + " --simulateRemotelly my_isis_job sim_test-gdg-3.2-3.3.zip" - + " sim_test-gdg-3.2-3.3-result.zip"); + jobDescription.setStandardOutput("output.txt"); + jobDescription.addOutput("output.txt"); daemon.submitJob(jobDescription); while (! daemon.isFinished(jobDescription)) { - Thread.sleep(5 * 1000); + Thread.sleep(15 * 1000); System.out.print("."); } - + + System.out.println("getting result"); File result = new File("/tmp/isis/sim_test-gdg-3.2-3.3-result.zip"); + IOUtils.copy(daemon.getResults(jobDescription).get("isisfish-simulation-ica10ans-result.zip"), new FileOutputStream(result)); + + System.out.println("getting output"); + result = new File("/tmp/isis/output.txt"); + IOUtils.copy(daemon.getResults(jobDescription).get("output.txt"), new FileOutputStream(result)); - IOUtils.copy(daemon.getResults(jobDescription).get("sim_test-gdg-3.2-3.3-result.zip"), new FileOutputStream(result)); - + System.out.println("end"); } /** @@ -124,6 +136,7 @@ DisworkFileSystemConfig.newKademliaDisworkConfig(args[0], port)); config.setActivityStrategy("none"); config.setUsedPort(30000); + config.fileSystemConfig.setBlockSize(65000); daemon = new DisworkDaemon(config); //userPrompt(); Modified: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/WorkersManager.java =================================================================== --- trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/WorkersManager.java 2010-06-21 10:14:34 UTC (rev 85) +++ trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/WorkersManager.java 2010-06-24 14:13:06 UTC (rev 86) @@ -24,13 +24,17 @@ */ package org.nuiton.diswork.daemon; +import java.io.BufferedReader; +import java.io.BufferedWriter; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; import java.io.OutputStream; +import java.io.OutputStreamWriter; import java.net.URL; import java.util.ArrayList; import java.util.Collections; @@ -127,6 +131,81 @@ /** set this field to true will make run() return and thread stop */ protected boolean shouldStop = false; + /** read the standard output of the subprocess + * + * By reading the standard output, this thread has multiple goals : + * <ul> + * <li>if the process produces many data on standard output, the + * process will block (and waitfor will never return). This + * thread, by reading the output, unblock it. + * http://www.javaworld.com/javaworld/jw-12-2000/jw-1229-traps.html?page=4 + * </li> + * <li>if asked, write the output to a file</li> + * <li>add the output to the log</li> + * </ul> + * + * @author bleny + * + */ + protected class OutputReader extends Thread { + + protected InputStream output; + protected OutputStream outputFile; + + /** + * @param output the stream of the standard output of the process + * @param outputFile the file where to copy, null if you don't + * care about the what is on the standard output + */ + public OutputReader(InputStream output, OutputStream outputFile) { + this.output = output; + this.outputFile = outputFile; + } + + @Override + public void run() { + InputStreamReader osr = new InputStreamReader(output); + BufferedReader br = new BufferedReader(osr); + + BufferedWriter wr = null; + OutputStreamWriter outputStreamWriter = null; + if (outputFile != null) { + outputStreamWriter = new OutputStreamWriter(outputFile); + wr = new BufferedWriter(outputStreamWriter); + } + + String line = null; + try { + // read a line from standard output + while ((line = br.readLine()) != null) { + // copy this line to output file + if (wr != null) { + wr.write(line + "\n"); + } + + // add reading line to logging output + if (log.isInfoEnabled()) { + log.debug(this.toString() + ">" + line); + } + } + } catch (IOException e) { + log.warn("error while reading the output of the subprocess", e); + } finally { + try { + if (wr != null) { + wr.close(); + } + if (outputStreamWriter != null) { + outputStreamWriter.close(); + } + } catch (IOException e) { + log.warn("error while closing the output of the subprocess", e); + } + } + } + + } + /** * this method add a line to a job-specific log * @param jobPath the path to the job concerned @@ -142,6 +221,24 @@ fileSystem.write(logPath, IOUtils.toInputStream(newLog)); } + protected void streamCopy(InputStream in, OutputStream out) throws IOException { + System.out.println("hop"); + int read = 0; + int totalRead = 0; + byte[] buffer = new byte[50000]; + // BufferedInputStream hop = new BufferedInputStream(in); + + while ((read = in.read(buffer)) != -1) { + + totalRead += read; + System.out.println("read " + read + " bytes (total=" + totalRead + ")"); + + out.write(buffer, 0, read); + } + in.close(); + out.close(); + } + /** * Download all the files needed for a job in a temp directory, run * the job, wait for it to end, write all the results. Mark the job @@ -189,14 +286,20 @@ application.createNewFile(); log.info("will create " + application.getAbsolutePath()); OutputStream out = new FileOutputStream(application); - IOUtils.copy(applicationData, out); + log.debug("starting copy of " + applicationData.available() + " bytes"); + // IOUtils.copy(applicationData, out); + streamCopy(applicationData, out); + log.info("unzip application start"); // unzip application ZipUtil.uncompress(application, jobDir); + log.info("unzip application finished"); } else { log.info("no dependency specified for " + jobDescription); } + // staging input files for (String fileName : jobDescription.getStagingInput()) { + log.info("staging " + fileName); File localCopy = new File(jobDir, fileName); localCopy.createNewFile(); InputStream source = null; @@ -212,6 +315,7 @@ IOUtils.copy(source, new FileOutputStream(localCopy)); } + log.info("preparing the job"); // prepare and run it the job String commandLine = jobDescription.getCommandLine(); String[] commandLineElements = commandLine.split(" "); @@ -220,10 +324,25 @@ builder.redirectErrorStream(true); log.info("calling " + commandLine); Process job = builder.start(); + + // dump the standard output in a file + String standardOutputFileName = jobDescription.getStandardOutput(); + log.info("standardOutputFileName is " + standardOutputFileName); + OutputStream outputFileStream = null; + if (standardOutputFileName != null) { + File outputFile = new File(jobDir, standardOutputFileName); + log.info("writing standard output in " + outputFile); + outputFileStream = new FileOutputStream(outputFile); + } + + OutputReader outputReader = new OutputReader(job.getInputStream(), outputFileStream); + outputReader.start(); // plugging a file on the standard input String standardInputFileName = jobDescription.getStandardInput(); if (standardInputFileName != null) { + log.info("writing " + standardInputFileName + " on standard " + + "input"); InputStream input = new FileInputStream( new File(jobDir, standardInputFileName)); IOUtils.copy(input, job.getOutputStream()); @@ -232,6 +351,7 @@ // run the process and wait for it to return int exitValue = -1; try { + log.info("waiting for the end of the process"); exitValue = job.waitFor(); } catch (InterruptedException e) { log.error("job " + jobDescription + " was interupted", e); @@ -240,27 +360,19 @@ exitValue = 1; } - // dump the standard output in a file - String standardOutputFileName = jobDescription.getStandardOutput(); - log.info("standardOutputFileName is " + standardOutputFileName); - if (standardOutputFileName != null) { - File outputFile = new File(jobDir, standardOutputFileName); - OutputStream output = new FileOutputStream(outputFile); - log.info("writing standard output in " + outputFile); - IOUtils.copy(job.getInputStream(), output); - } + log.info("job returned " + exitValue); // output file staging for (String fileName : jobDescription.getStagingOutput()) { + log.info("staging file " + fileName); File localCopy = new File(jobDir, fileName); // FIXME 20100616 bleny may not exists if job has fail InputStream localCopyStream = new FileInputStream(localCopy); String filePath = jobPath + "/" + fileName; - log.info("out-staging " + fileName); // erase before write if (fileSystem.exists(filePath)) { fileSystem.delete(filePath); Added: trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonMultipleNodesTest.java =================================================================== --- trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonMultipleNodesTest.java (rev 0) +++ trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonMultipleNodesTest.java 2010-06-24 14:13:06 UTC (rev 86) @@ -0,0 +1,40 @@ +package org.nuiton.diswork.daemon; + +import java.io.InputStream; + +import org.junit.Before; +import org.junit.Test; +import org.nuiton.diswork.fs.DisworkFileSystemConfig; + +public class DisworkDaemonMultipleNodesTest extends DisworkDaemonTest { + + /** another deamon + * this second deamon will be connected to {@link DisworkDaemonTest#daemon} + * and will to his jobs. + */ + protected static DisworkDaemon daemon2; + + @Before + @Override + public void setUp() throws Exception { + DisworkConfig config = DisworkConfig.newConfig(); + port += 1; + config.setUsedPort(port); + config.setActivityStrategy("none"); + + config.fileSystemConfig.setBlockSize(500); + daemon = new DisworkDaemon(config); + InputStream application = ClassLoader.getSystemResourceAsStream("fake-app-1.0.zip"); + daemon.submitApplication("fake-app", "1.0", application); + + + DisworkConfig config2 = DisworkConfig.newConfig(); + config2.setBootstrapIp(DisworkFileSystemConfig.getIp()); + config2.setBootstrapPort(port); + port += 1; + config.setUsedPort(port); + config.setActivityStrategy("unlimited"); + daemon2 = new DisworkDaemon(config2); + } + +} Modified: trunk/diswork-daemon/src/test/resources/log4j.properties =================================================================== --- trunk/diswork-daemon/src/test/resources/log4j.properties 2010-06-21 10:14:34 UTC (rev 85) +++ trunk/diswork-daemon/src/test/resources/log4j.properties 2010-06-24 14:13:06 UTC (rev 86) @@ -6,4 +6,5 @@ log4j.appender.stdout.layout.ConversionPattern=%d %5p [%t] (%F:%L) %M - %m%n # package level log4j.logger.org.nuiton.diswork.fs.storage.KademliaDisworkMap=INFO -log4j.logger.org.nuiton.diswork.daemon=INFO \ No newline at end of file +log4j.logger.org.nuiton.diswork.daemon=DEBUG +log4j.logger.org.planx.xmlstore.routing.messaging.MessageServer=INFO \ No newline at end of file Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystemConfig.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystemConfig.java 2010-06-21 10:14:34 UTC (rev 85) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystemConfig.java 2010-06-24 14:13:06 UTC (rev 86) @@ -102,13 +102,16 @@ } public DisworkFileSystemConfig() { - // setDefaultOption("diswork.fs.blocks_size", "10485760"); // 10 MiB - setDefaultOption("diswork.fs.blocks_size", "50000"); + setDefaultOption("diswork.fs.blocks_size", "10485760"); // 10 MiB setDefaultOption("diswork.fs.map_type", "inmemory"); setDefaultOption("diswork.fs.use_port", port.toString()); } + public void setBlockSize(Integer size) { + setOption("diswork.fs.blocks_size", size.toString()); + } + public int getBlockSize() { return getOptionAsInt("diswork.fs.blocks_size"); } Modified: trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/AbstractDisworkFileSystemTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/AbstractDisworkFileSystemTest.java 2010-06-21 10:14:34 UTC (rev 85) +++ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/AbstractDisworkFileSystemTest.java 2010-06-24 14:13:06 UTC (rev 86) @@ -16,14 +16,11 @@ import java.util.List; import java.util.Random; -import javax.xml.transform.Source; - import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.junit.After; import org.junit.Before; import org.junit.Test; -import org.nuiton.diswork.fs.DisworkFileSystem; import org.nuiton.util.FileUtil; @@ -45,7 +42,7 @@ /** * The file will have this fixed size */ - static protected int randomFileSize = 11 * 1000 * 1000; + static protected int randomFileSize = 45 * 1000 * 1000; static protected DisworkFileSystem fileSystem; @@ -75,6 +72,7 @@ @After public void tearDown() throws Exception { + fileSystem.close(); // cleaning FileUtil.deleteRecursively(tempDirectoryPath); } Modified: trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemKademliaTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemKademliaTest.java 2010-06-21 10:14:34 UTC (rev 85) +++ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemKademliaTest.java 2010-06-24 14:13:06 UTC (rev 86) @@ -7,9 +7,11 @@ import java.io.ByteArrayInputStream; import java.io.FileInputStream; +import java.io.InputStream; import java.util.ConcurrentModificationException; import org.apache.commons.io.IOUtils; +import org.apache.log4j.lf5.util.StreamUtils; import org.junit.Before; import org.junit.Test; import org.nuiton.diswork.fs.DisworkFileSystem; @@ -48,19 +50,41 @@ DisworkFileSystemConfig disworkConfig = DisworkFileSystemConfig.newKademliaDisworkConfig(bootstrapPort); DisworkFileSystem fileSystem2 = new DisworkFileSystem(disworkConfig); - - byte[] bytes = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; - fileSystem.write("/my_file", new ByteArrayInputStream(bytes)); + InputStream source = null; + try { + source = new FileInputStream(randomFilePath); + fileSystem.write("/my_file", source); + } finally { + IOUtils.closeQuietly(source); + } assertTrue(fileSystem.exists("/my_file")); assertTrue(fileSystem2.exists("/my_file")); - assertEquals(1, fileSystem2.readDirectory("/").size()); - byte[] getResult = IOUtils.toByteArray(fileSystem2.read("/my_file")); + // now, the checks. We read the original file and the result of + // a read() and then compare it byte-to-byte + + InputStream readResult = null; + try { + source = new FileInputStream(randomFilePath); + readResult = fileSystem2.read("/my_file"); + + assertEquals(randomFileSize, source.available()); + assertEquals(randomFileSize, readResult.available()); + + assertTrue(IOUtils.contentEquals(source, readResult)); + /* + byte[] sourceAsBytes = IOUtils.toByteArray(source); + byte[] readResultAsBytes = IOUtils.toByteArray(readResult); - assertArrayEquals(bytes, getResult); + assertArrayEquals(sourceAsBytes, readResultAsBytes); + */ + } finally { + IOUtils.closeQuietly(source); + IOUtils.closeQuietly(readResult); + } } Modified: trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/AbstractDisworkMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/AbstractDisworkMapTest.java 2010-06-21 10:14:34 UTC (rev 85) +++ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/AbstractDisworkMapTest.java 2010-06-24 14:13:06 UTC (rev 86) @@ -1,10 +1,13 @@ package org.nuiton.diswork.fs.storage; import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import java.util.Arrays; +import java.util.Random; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -96,4 +99,55 @@ } + @Test + public void testStoreLargeValue() { + + // 237499 + // 312499 + // 351561 + // 249999 + + int min = 0; + int max = 10 * 1000 * 1000; + + + int milieu = (min + max) / 2; + + while (max - min > 1) { + + + byte[] newBytes = new byte[milieu]; + Random random = new Random(); + random.nextBytes(newBytes); + + map1.put("test", newBytes); + + byte[] getResult = map1.get("test"); + assertNotNull(getResult); + assertEquals(milieu, getResult.length); + + getResult = map2.get("test"); + if (getResult == null) { + System.out.println(max + " trop grand"); + max = milieu; + } else { + System.out.println(min + " trop petit"); + min = milieu; + } + + milieu = (min + max) / 2; + System.out.println(min + " < " + milieu + " < " + max); + } + + System.out.println(milieu); + /* + assertNotNull(getResult); + assertEquals(bytesSize, getResult.length); + + assertArrayEquals(newBytes, getResult); + */ + + + } + } Modified: trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/KademliaDisworkMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/KademliaDisworkMapTest.java 2010-06-21 10:14:34 UTC (rev 85) +++ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/KademliaDisworkMapTest.java 2010-06-24 14:13:06 UTC (rev 86) @@ -70,6 +70,7 @@ * a bad bootstrap * @throws Exception */ + /* @Test(expected = org.planx.xmlstore.routing.RoutingException.class) public void testBadBootrap() throws Exception { DisworkFileSystemConfig config1 = @@ -78,4 +79,5 @@ config1.setBootstrapPort(80); new KademliaDisworkMap(config1); } + */ } Modified: trunk/diswork-fs/src/test/resources/log4j.properties =================================================================== --- trunk/diswork-fs/src/test/resources/log4j.properties 2010-06-21 10:14:34 UTC (rev 85) +++ trunk/diswork-fs/src/test/resources/log4j.properties 2010-06-24 14:13:06 UTC (rev 86) @@ -7,4 +7,5 @@ # package level log4j.logger.org.nuiton.diswork.fs=WARN log4j.logger.org.nuiton.diswork.fs.storage.KademliaDisworkMap=INFO -log4j.logger.org.nuiton.diswork.fs.Demo=INFO \ No newline at end of file +log4j.logger.org.nuiton.diswork.fs.Demo=INFO +log4j.logger.org.planx.xmlstore.routing=DEBUG \ No newline at end of file Modified: trunk/pom.xml =================================================================== --- trunk/pom.xml 2010-06-21 10:14:34 UTC (rev 85) +++ trunk/pom.xml 2010-06-24 14:13:06 UTC (rev 86) @@ -1,5 +1,6 @@ <?xml version="1.0" encoding="UTF-8"?> -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> @@ -32,7 +33,7 @@ <dependency> <groupId>log4j</groupId> <artifactId>log4j</artifactId> - <version>1.2.14</version> + <version>1.2.16</version> <scope>test</scope> </dependency> <dependency> @@ -88,7 +89,13 @@ <artifactId>commons-digester</artifactId> <version>2.0</version> </dependency> + <dependency> + <groupId>org.freemarker</groupId> + <artifactId>freemarker</artifactId> + <version>2.3.16</version> + </dependency> +  <dependency> <groupId>junit</groupId>

1 0

r85 - in trunk: . diswork-daemon/src/main/java/org/nuiton/diswork/daemon diswork-daemon/src/test/java/org/nuiton/diswork/daemon diswork-fs/src/main/java/org/nuiton/diswork/fs diswork-fs/src/main/java/org/nuiton/diswork/fs/storage diswork-fs/src/test/java/org/nuiton/diswork/fs
by bleny＠users.nuiton.org June 21, 2010

June 21, 2010

Author: bleny Date: 2010-06-21 12:14:34 +0200 (Mon, 21 Jun 2010) New Revision: 85 Url: http://nuiton.org/repositories/revision/diswork/85 Log: documentation, client simple, lanceur de d?\195?\169mon Modified: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/ActivityStrategy.java trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkDaemon.java trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkDaemonRunner.java trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkSimpleClient.java trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/JobDescription.java trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/WorkersManager.java trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonTest.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystemConfig.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/Storage.java trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/AbstractDisworkFileSystemTest.java trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemKademliaTest.java trunk/pom.xml Modified: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/ActivityStrategy.java =================================================================== --- trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/ActivityStrategy.java 2010-06-19 18:55:14 UTC (rev 84) +++ trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/ActivityStrategy.java 2010-06-21 10:14:34 UTC (rev 85) @@ -30,6 +30,34 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +/** + * This interface introduce the concept of "activity strategy" in + * diswork, an activity strategy answer to "Can i run a job now ?". + * + * This provide a simple method that returns true if Diswork can run a job + * and take hardware resources. If false is returned, it means that resources + * are not available or we want diswork not to consume more resources. + * + * This interface come with 4 implementations. Two first are very simple, + * one is used when diswork is "shut down", it always returns false. The + * second is used when diswork is in "free-wheel" and take as much resources + * as jobs need (always return true). + * + * LimitedActivity implementation is user-activity aware and returns true + * only if it seems that the user incompletely use the resources of + * his computer. This implementation is based on the a load-average + * computation of the computer. If load-average is low, computer is + * idling : it returns true and diswork will use remaining resources. + * If load-average is high, false is returned and diswork don't make + * the computer over-loaded. + * + * ScheduledActivity implementation make diswork run jobs only at + * fixed period of time. An user can configure it to make + * diswork run job only at night are during the week-end, for example. + * + * @author bleny + * + */ public interface ActivityStrategy { /** use this strategy to never run a job */ @@ -48,7 +76,7 @@ } } - /** use this strategy to run a job only if load average is low */ + /** use this strategy to run a job only if computer is idling */ public static class LimitedActivity implements ActivityStrategy { private static final Log log = LogFactory.getLog(LimitedActivity.class); Modified: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkDaemon.java =================================================================== --- trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkDaemon.java 2010-06-19 18:55:14 UTC (rev 84) +++ trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkDaemon.java 2010-06-21 10:14:34 UTC (rev 85) @@ -31,6 +31,7 @@ import java.lang.management.ManagementFactory; import java.lang.management.OperatingSystemMXBean; import java.net.UnknownHostException; +import java.util.ArrayList; import java.util.ConcurrentModificationException; import java.util.HashMap; import java.util.List; @@ -46,9 +47,88 @@ import org.nuiton.util.FileUtil; /** + * The diswork daemon is the gateway to the global diswork system. Instantiate + * this class creates a new node on the system. The new node can be used + * to submit jobs and monitor them. * + * For how-to use considerations see ... + * + * Here is a list of what the daemon do when booting : + * <ul> + * <li>join the network, given some information (IP+port) provided + * with diswork-config (a diswork-config is an instance of + * {@link DisworkConfig})</li> + * <li>define what is the ownedId for this node. It has to be + * recovered from the last run of the daemon. It's + * the link between two different sessions : if you submit a job, stop + * and restart the daemon with the same ownerId, you will find your + * job back. If ownerId can't be recovered or if it's the first + * run, an ownerId is generated and saved for future run.</li> + * <li>define the home-directory, it's deduced from the ownerId. Thus, if you + * change you ownerId, you'll change your home-dir and loose your jobs + * because they are all stored in your home-dir. If it's the first + * time the daemon is run by this ownerId, the homeDir is created</li> + * <li>If diswork is configured for, the daemon will start a workers-manager. + * This manager don't care about the jobs you submit but look + * for jobs that are proposed on the global system and try to + * help. For more details, see {@link WorkersManager}</li> + * <li>Write some stats about itself. Since all nodes do that, each nodes + * can performs statistics computation to estimate the powerfulness of + * the global diswork system. Each node write those info in a file + * in his home-dir at {@link #HARDINFO_PATH}</li> + * </ul> + * + * The main use of the daemon is to submit job to the global diswork + * system. + * + * A job is a task that need an application. An application is a simple file + * that should be provided to the daemon once for all + * (using {@link #submitApplication(String, String, InputStream)}. Then, all + * nodes that would like to perform this job can download the application. + * + * You can create your application ready for diswork, submit it to the daemon + * and then submit as much jobs as you want that depends of this application. + * Think an application as the common stuff (programs, scripts, files etc.) + * all the different job will need. An application is just a set of file. + * + * TODO 20100617 bleny explain it better + * + * Once the application has been made available, jobs can be created and + * submitted. To do so, a programmer must write a class that + * + * <ol> + * <li>may republish the application (it's always good to be sure an + * application is still present if the last time that it has + * been used is too long)</li> + * <li>Create a new {@link JobDescription} and set it right for his + * new job.</li> + * <li>Submit the job by calling {@link #submitJob(JobDescription)}</li> + * <li>Monitor the job with {@link #isFinished(JobDescription)}</li> + * <li>Check the job state with {@link #isFailed(JobDescription)} + * and {@link #isSuccessful(JobDescription)} and get the results with + * {@link #getResults(JobDescription)}</li> + * <li>Finally, remove the job</li> + * </ol> + * + * Jobs will be saved in home-dir, in a path generated with + * {@link #getPathForJob(JobDescription)}. In this directory will be placed + * <ul> + * <li>All the data provided with the job {@link JobDescription#inputData}</li> + * <li>The JSDL file that describe the job (placed according to the + * value of {@link #JSDL_PATH})</li> + * <li>A log file placed at {@link #LOG_PATH}, it will be empty at the + * submission but lines will be added each time a node try to do + * this job. If the job is finished, the log will contain "FINISHED", + * if it has been completed, it will contain "DONE", if it has been + * failed, it will contain "FAILED" one or few times.</li> + * </ul> + * + * The daemon should be closed properly by using the {@link #close()} method. + * It will release resources, update stats and stop the worker-manager. + * * As far as possible, the use of the file system follow the UNIX Filesystem - * Hierarchy Standard + * Hierarchy Standard. All usual paths used by Diswork are defined in + * constants. * * @see http://en.wikipedia.org/wiki/Filesystem_Hierarchy_Standard * @@ -87,6 +167,9 @@ /** a place where are all user-directories */ protected static final String HOME = "/home"; + + /** in home, where jobs should be placed */ + protected static final String JOBS_DIR = "jobs"; /** in a job directory, the place where the JSDL must be placed */ protected static final String JSDL_PATH = ".diswork/job.jsdl"; @@ -259,7 +342,7 @@ */ protected String getPathForJob(String jobId) { // all jobs are stored in home dir - return homeDir + "/" + jobId; + return homeDir + "/" + JOBS_DIR + "/" + jobId; } /** @@ -273,7 +356,7 @@ /** * Provide an application to all nodes. Once provided, all nodes will be - * able to perform a job with this application + * able to perform a job with this application. * * An application can be uploaded in different version. If a given * application in the given version is already on the File System, nothing @@ -285,7 +368,7 @@ * @throws DisworkException if an error occurs while uploading the file */ public void submitApplication(String applicationName, - String applicationVersion, InputStream applicationData) + String applicationVersion, InputStream applicationData) throws DisworkException { // the place where dependency will be stored @@ -310,6 +393,39 @@ } } + /** + * Returns a list with all jobs submitted before. + * @return + * @throws DisworkException + */ + public List<JobDescription> getAllJobs() throws DisworkException { + List<JobDescription> result = new ArrayList<JobDescription>(); + try { + List<String> jobs = fileSystem.readDirectory(homeDir + "/" + JOBS_DIR); + for (String jobId : jobs) { + String jsdl = IOUtils.toString(fileSystem.read( + getPathForJob(jobId) + "/" + JSDL_PATH)); + JobDescription jobDescription = JobDescription.parseJSDL(jsdl); + jobDescription.setJobId(jobId); + result.add(jobDescription); + } + } catch (IOException e) { + log.info("error file reading home-dir", e); + throw new DisworkException("error file reading home-dir", e); + } + return result; + } + + /** + * Cancel the submission of a job + * @param jobDescription + * @throws DisworkException + */ + public void deleteJob(JobDescription jobDescription) { + // TODO 20100618 bleny stub + throw new UnsupportedOperationException(); + } + public void submitJob(JobDescription jobDescription) throws DisworkException { if (jobDescription.getInputData().size() + jobDescription.getStagingInputUrls().size() @@ -413,6 +529,14 @@ return isFinished(job) && !isSuccessful(job); } + /** + * Must not be called until {@link #isFinished(JobDescription)} returns + * true for this job. + * @param job + * @return a map associating the file-name of a result to a stream to + * read the file + * @throws DisworkException if job is not finished + */ public Map<String, InputStream> getResults(JobDescription job) throws DisworkException { if (isFinished(job)) { Modified: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkDaemonRunner.java =================================================================== --- trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkDaemonRunner.java 2010-06-19 18:55:14 UTC (rev 84) +++ trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkDaemonRunner.java 2010-06-21 10:14:34 UTC (rev 85) @@ -42,10 +42,9 @@ public static void main(String[] args) throws Exception { DisworkConfig config = new DisworkConfig(); - if (args.length == 0) { config.setFileSystemConfig( DisworkFileSystemConfig.newKademliaDisworkConfig()); - } else if (args.length == 2){ + if (args.length == 2) { config.setFileSystemConfig( DisworkFileSystemConfig.newKademliaDisworkConfig(args[0], Integer.parseInt(args[1]))); Modified: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkSimpleClient.java =================================================================== --- trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkSimpleClient.java 2010-06-19 18:55:14 UTC (rev 84) +++ trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkSimpleClient.java 2010-06-21 10:14:34 UTC (rev 85) @@ -25,6 +25,10 @@ package org.nuiton.diswork.daemon; import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; import java.io.InputStreamReader; import java.util.Map; @@ -32,36 +36,25 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.diswork.fs.DisworkFileSystemConfig; +import org.nuiton.util.FileUtil; /** * * @author bleny */ public class DisworkSimpleClient { - + private static final Log log = LogFactory.getLog(DisworkSimpleClient.class); + + static DisworkDaemon daemon; - /** - * @param args - * @throws Exception - */ - public static void main(String[] args) throws Exception { + public static void userPrompt() throws Exception { - DisworkConfig config = new DisworkConfig(); - config.setFileSystemConfig( - DisworkFileSystemConfig.newKademliaDisworkConfig(args[0], - Integer.parseInt(args[1]))); - config.setActivityStrategy("none"); - config.setUsedPort(30000); - DisworkDaemon daemon = new DisworkDaemon(config); - - // prompt the user to enter their name - // open up standard input BufferedReader br = new BufferedReader(new InputStreamReader(System.in)); String userEntry = ""; - do { + while (!userEntry.equals("quit")) { System.out.print("diswork: "); userEntry = br.readLine(); @@ -83,8 +76,57 @@ System.out.println(IOUtils.toString(daemon.getResults(job).get("output.txt"))); } - - } while (!userEntry.equals("quit")); + } + } + + public static void isisSubmit() throws Exception { + File isis = new File("/tmp/isis/isis-fish-3.3.0.3-bin.zip"); + daemon.submitApplication("isis-fish", "3.3.0.3", new FileInputStream(isis)); + + JobDescription jobDescription = new JobDescription(); + + jobDescription.setJobName("Mon Job Isis"); + + jobDescription.setApplication("isis-fish", "3.3.0.3"); + + File input = new File("/tmp/isis/sim_test-gdg-3.2-3.3.zip"); + jobDescription.addInput("sim_test-gdg-3.2-3.3.zip", new FileInputStream(input)); + jobDescription.addOutput("sim_test-gdg-3.2-3.3-result.zip"); + + jobDescription.setCommandLine("%java -Xmx2500M -jar isis-fish*.jar" + + " --option launch.ui false --option perform.vcsupdate false" + + " --option perform.migration false --option perform.cron false" + + " --simulateRemotelly my_isis_job sim_test-gdg-3.2-3.3.zip" + + " sim_test-gdg-3.2-3.3-result.zip"); + + daemon.submitJob(jobDescription); + + while (! daemon.isFinished(jobDescription)) { + Thread.sleep(5 * 1000); + System.out.print("."); + } + + File result = new File("/tmp/isis/sim_test-gdg-3.2-3.3-result.zip"); + + IOUtils.copy(daemon.getResults(jobDescription).get("sim_test-gdg-3.2-3.3-result.zip"), new FileOutputStream(result)); + + } + + /** + * @param args + * @throws Exception + */ + public static void main(String[] args) throws Exception { + DisworkConfig config = new DisworkConfig(); + Integer port = Integer.parseInt(args[1]); + System.out.println("port = " + port); + config.setFileSystemConfig( + DisworkFileSystemConfig.newKademliaDisworkConfig(args[0], port)); + config.setActivityStrategy("none"); + config.setUsedPort(30000); + daemon = new DisworkDaemon(config); + //userPrompt(); + isisSubmit(); } } Modified: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/JobDescription.java =================================================================== --- trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/JobDescription.java 2010-06-19 18:55:14 UTC (rev 84) +++ trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/JobDescription.java 2010-06-21 10:14:34 UTC (rev 85) @@ -52,7 +52,7 @@ * * <dl> * <dt>%java</dt> - * </dd>will be replaced by the actual path of java executable + * <dd>will be replaced by the actual path of java executable * into the JRE</dd> * </dl> * @@ -117,8 +117,8 @@ /** * constructor is protected to prevent bad jobIds. To get a JobDescription - * instance, a client should use the {@link DisworkDaemon#newJob()} - * factory method. The given instance will have a valid jobId + * instance, a client should use the {@link #JobDescription()} constructor + * The given instance will have a valid jobId when needed. * @param jobId */ protected JobDescription(String jobId) { @@ -194,16 +194,7 @@ return "job : " + jobName + " (" + jobId + ")"; } - protected static Integer count = 0; - protected static Map<String, JobDescription> map = new HashMap<String, JobDescription>(); - - public String toJSDL() { - /* - count += 1; - map.put(count.toString(), this); - return count.toString(); - */ - + public String toJSDL() { String jsdl = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + "<jsdl:JobDefinition xmlns=\"http://www.example.org/\"\n" @@ -288,7 +279,7 @@ Element jobName = jobIdentification.getChild("JobName", jsdlNamespace); result.setJobName(jobName.getText()); - + // application Element application = jobDescription.getChild("Application", jsdlNamespace); @@ -300,10 +291,10 @@ result.setApplication(applicationName.getText(), applicationVersion.getText()); } - + Element POSIXApplication = application.getChild("POSIXApplication", jsdlPosixNamespace); - + Element argument = POSIXApplication.getChild("Argument", jsdlPosixNamespace); result.setCommandLine(argument.getText()); @@ -318,22 +309,16 @@ if (output != null) { result.setStandardOutput(output.getText()); } - + // staging List<Element> dataStagings = jobDescription.getChildren ("DataStaging", jsdlNamespace); for (Element dataStaging : dataStagings) { Attribute type = dataStaging.getAttribute("type", disworkNamespace); - Element fileName = dataStaging.getChild("FileName", jsdlNamespace); - - if (type != null && "out".equals(type.getValue())) { - // type="out" - result.addOutput(fileName.getText()); - } else { - // type="in" + if ("in".equals(type.getValue())) { Element source = dataStaging.getChild("Source", jsdlNamespace); if (source != null) { @@ -342,14 +327,11 @@ } else { result.addInput(fileName.getText(), new NullInputStream(0)); } - - // type not set - if (type == null) { - result.addOutput(fileName.getText()); - } } + if ("out".equals(type.getValue())) { + result.addOutput(fileName.getText()); + } } - } catch (JDOMException e) { log.error("can't read malformed JSDL file", e); throw new IOException("can't read malformed JSDL file", e); @@ -359,7 +341,7 @@ } return result; } - + public void addInput(String fileName, URL source) { input.add(fileName); inputUrls.put(fileName, source); @@ -369,7 +351,7 @@ input.add(fileName); inputData.put(fileName, source); } - + public void addOutput(String fileName) { output.add(fileName); } Modified: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/WorkersManager.java =================================================================== --- trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/WorkersManager.java 2010-06-19 18:55:14 UTC (rev 84) +++ trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/WorkersManager.java 2010-06-21 10:14:34 UTC (rev 85) @@ -37,7 +37,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Random; import org.apache.commons.io.FilenameUtils; import org.apache.commons.io.IOUtils; @@ -48,23 +47,37 @@ import org.nuiton.util.ZipUtil; /** + * The workers-manager aims to run and manage the different workers. A worker + * is a thread that try to find a jobs and execute them. * + * The manager can manage multiple workers : one worker + * doing one job at a time. By running n workers, the daemon can run up + * to n job at the same time. The number of workers should be set carefully + * (according to the actual hardware resources available). The number of workers + * can be set through {@link DisworkConfig#setNumberOfWorkers(Integer)}. + * + * Workers are not always active, before trying to find and job, they check + * if they will have enough resources to execute the job without bugging + * the other processes of the host machine. To do so, the worker-manager + * allow to define an activity-strategy to be followed by each worker. The + * activity strategy tell the worker if they are allowed to take resources or + * not, different strategies are available (see {@link ActivityStrategy}), + * current strategy can be changed on the fly, while the daemon run. + * * @author bleny */ public class WorkersManager { + private static final Log log = LogFactory.getLog(WorkersManager.class); + + /** a job found in dir "key", should be move to "value" before running */ protected static Map<String, String> RUNNING_MOVE = new HashMap<String, String>(); + + /** a job found in dir "key" that fail should be moved to "value" */ protected static Map<String, String> FAILED_MOVE = new HashMap<String, String>(); - - // TODO 20100611 bleny make it configurable - /** time to wait beetween two look for a job */ - protected static final int JOB_WAIT = 10 * 1000; - // TODO 20100614 bleny make it configurable - /** after this time (ms), a job is considered as no longer running */ - protected static final long MAX_JOB_RUNNING_TIME = 24 * 60 * 60 * 1000; - static { + // initialize RUNNING_MOVE and FAILED_MOVE constants RUNNING_MOVE.put(DisworkDaemon.TODO, DisworkDaemon.TODO_RUNNING); RUNNING_MOVE.put(DisworkDaemon.FAILED_1, DisworkDaemon.FAILED_1_RUNNING); RUNNING_MOVE.put(DisworkDaemon.FAILED_2, DisworkDaemon.FAILED_2_RUNNING); @@ -76,23 +89,44 @@ FAILED_MOVE.put(DisworkDaemon.FAILED_1_RUNNING, DisworkDaemon.FAILED_2); FAILED_MOVE.put(DisworkDaemon.FAILED_2_RUNNING, DisworkDaemon.FAILED_3); } - - private static final Log log = LogFactory.getLog(WorkersManager.class); protected DisworkFileSystem fileSystem; protected DisworkConfig config; - // Pool of workers + /** a pool of workers */ protected List<Worker> workers = new ArrayList<Worker>(); + /** the current activity strategy followed by all the workers */ protected ActivityStrategy activityStrategy; + /** A worker search a job and execute it. + * + * Jobs are found on the file-system by browsing some special directories, + * like {@link DisworkDaemon#TODO}. They do not contains jobs but symbolic + * links to jobs. If a worker find a job, it must move it to another + * directory ({@link DisworkDaemon#TODO_RUNNING}) for no other worker to + * take it. Once finished, the job link, if successful, will be moved to + * {@link DisworkDaemon#DONE}. If failed, the link will be moved to + * {@link DisworkDaemon#FAILED_1} (read "failed once") for another worker + * to try it and move it the same may. If the job is failed three times, + * it's moved to {@link DisworkDaemon#FAILED_3} meaning it will not been + * tried again. + * + * @author bleny + */ protected class Worker extends Thread { - + + // TODO 20100611 bleny make it configurable + /** time to wait between two look for a job */ + protected static final int JOB_WAIT = 10 * 1000; + + // TODO 20100614 bleny make it configurable + /** after this time (ms), a job is considered as no longer running */ + protected static final long MAX_JOB_RUNNING_TIME = 24 * 60 * 60 * 1000; + + /** set this field to true will make run() return and thread stop */ protected boolean shouldStop = false; - - protected WorkersManager manager; - + /** * this method add a line to a job-specific log * @param jobPath the path to the job concerned @@ -135,10 +169,9 @@ log.info("will run job " + jobDescription); - // create temp dir - Random random = new Random(); - File jobDir = new File(config.getTempDirectory(), - String.valueOf(random.nextInt())); + // create temp dir + File jobDir = FileUtil.createTempDirectory("job", "", + new File(config.getTempDirectory())); jobDir.mkdirs(); // download application @@ -249,6 +282,10 @@ return success; } + /** + * In a directory, list the content, sort the content, and returns + * the first element. + */ protected String getFistJobName(String path) throws IOException { List<String> jobsNames = fileSystem.readDirectory(path); if (jobsNames.size() == 0) { @@ -369,19 +406,15 @@ } } + /** + * find + */ @Override public void run() { while (! shouldStop) { - if (manager.getActivityStrategy().canWork()) { + if (activityStrategy.canWork()) { findAJobAndRunIt(); } - try { - Thread.sleep(10*1000); - } catch (InterruptedException e) { - // TODO 20100615 bleny Auto-generated catch block - log.info("exception catch", e); - e.printStackTrace(); - } } } } @@ -389,7 +422,8 @@ public WorkersManager(DisworkFileSystem fileSystem, DisworkConfig config) { this.fileSystem = fileSystem; this.config = config; - + + // initialize activityStrategy according to config String initialStrategy = config.getActivityStrategy(); if ( "none".equals(initialStrategy)) { activeNoActivityStrategy(); @@ -405,14 +439,13 @@ activeNoActivityStrategy(); } + // start as many workers as needed log.info("will start " + config.getNumberOfWorkers() + " workers"); for (int i = 1 ; i <= config.getNumberOfWorkers() ; i++) { Worker worker = new Worker(); - worker.manager = this; worker.start(); workers.add(worker); } - } public void stop() { Modified: trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonTest.java =================================================================== --- trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonTest.java 2010-06-19 18:55:14 UTC (rev 84) +++ trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonTest.java 2010-06-21 10:14:34 UTC (rev 85) @@ -5,6 +5,7 @@ import java.io.InputStream; import java.net.URL; +import java.util.List; import java.util.Map; import org.apache.commons.io.IOUtils; @@ -39,6 +40,7 @@ JobDescription job = new JobDescription(); job.setCommandLine("java -version"); daemon.submitJob(job); + } @Test(expected = DisworkException.class) @@ -60,6 +62,14 @@ } assertTrue(daemon.isSuccessful(job)); + + // check getAllJobs return + List<JobDescription> currentJobs = daemon.getAllJobs(); + assertEquals(1, currentJobs.size()); + + JobDescription currentJob = currentJobs.get(0); + assertEquals(job.getJobId(), currentJob.getJobId()); + assertEquals(job.getCommandLine(), currentJob.getCommandLine()); } Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystemConfig.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystemConfig.java 2010-06-19 18:55:14 UTC (rev 84) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystemConfig.java 2010-06-21 10:14:34 UTC (rev 85) @@ -45,11 +45,11 @@ * </dd> * </dl> * - * * This class provides utilities to write tests easily. * * You can get multiples diswork configs ready to use. All the instances uses * a new port and the local IP. + * * <pre> * c = newDisworkConfig(); // create a config for a bootstrap node * c2 = newDisworkConfig(c.getUsedPort()) // creates a config for a node that @@ -57,7 +57,6 @@ * // first node * c3 = newDisworkConfig(c.getUsedPort()) * </pre> - * */ public class DisworkFileSystemConfig extends ApplicationConfig { @@ -103,7 +102,8 @@ } public DisworkFileSystemConfig() { - setDefaultOption("diswork.fs.blocks_size", "10485760"); // 10 MiB + // setDefaultOption("diswork.fs.blocks_size", "10485760"); // 10 MiB + setDefaultOption("diswork.fs.blocks_size", "50000"); setDefaultOption("diswork.fs.map_type", "inmemory"); setDefaultOption("diswork.fs.use_port", port.toString()); Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/Storage.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/Storage.java 2010-06-19 18:55:14 UTC (rev 84) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/Storage.java 2010-06-21 10:14:34 UTC (rev 85) @@ -152,16 +152,16 @@ protected DisworkMap map; - protected DisworkFileSystemConfig disworkConfig; + protected DisworkFileSystemConfig config; - public Storage(DisworkFileSystemConfig disworkConfig, DisworkMap map) + public Storage(DisworkFileSystemConfig config, DisworkMap map) throws IOException { - this.disworkConfig = disworkConfig; + this.config = config; if (map == null) { // instantiating a map according to config directives - String mapType = disworkConfig.getOption("diswork.fs.map_type"); + String mapType = config.getOption("diswork.fs.map_type"); if (mapType == null) { log.info("no map type specified"); @@ -172,10 +172,10 @@ this.map = new InMemoryDisworkMap(); } else if ("pastry".equals(mapType)) { log.info("using Pastry map"); - this.map = new PastryDisworkMap(disworkConfig); + this.map = new PastryDisworkMap(config); } else if ("kademlia".equals(mapType)) { log.info("using Kademlia map"); - this.map = new KademliaDisworkMap(disworkConfig); + this.map = new KademliaDisworkMap(config); } } } else { @@ -191,8 +191,8 @@ } } - public Storage(DisworkFileSystemConfig disworkConfig) throws IOException { - this(disworkConfig, null); + public Storage(DisworkFileSystemConfig config) throws IOException { + this(config, null); } /** @@ -298,7 +298,7 @@ map.put(newDataKey, EntryUtil.stringToBytes(metaBlock)); // creating a buffer of the size of a block - int bufferSize = disworkConfig.getBlockSize(); + int bufferSize = config.getBlockSize(); byte[] buffer = new byte[bufferSize]; while ((readResult = value.read(buffer)) != -1) { Modified: trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/AbstractDisworkFileSystemTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/AbstractDisworkFileSystemTest.java 2010-06-19 18:55:14 UTC (rev 84) +++ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/AbstractDisworkFileSystemTest.java 2010-06-21 10:14:34 UTC (rev 85) @@ -16,12 +16,13 @@ import java.util.List; import java.util.Random; +import javax.xml.transform.Source; + import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.junit.After; import org.junit.Before; import org.junit.Test; -import org.junit.experimental.categories.Categories.ExcludeCategory; import org.nuiton.diswork.fs.DisworkFileSystem; import org.nuiton.util.FileUtil; @@ -44,7 +45,7 @@ /** * The file will have this fixed size */ - static protected int randomFileSize = 25 * 1000; + static protected int randomFileSize = 11 * 1000 * 1000; static protected DisworkFileSystem fileSystem; @@ -84,7 +85,13 @@ */ @Test public void testWrite() throws Exception { - fileSystem.write("/my_file", new FileInputStream(randomFilePath)); + InputStream source = null; + try { + source = new FileInputStream(randomFilePath); + fileSystem.write("/my_file", source); + } finally { + IOUtils.closeQuietly(source); + } } /** @@ -93,9 +100,15 @@ */ @Test public void testExists() throws Exception { - fileSystem.write("/my_file", new FileInputStream(randomFilePath)); - assertTrue(fileSystem.exists("/my_file")); - assertFalse(fileSystem.exists("/my_other_file")); + InputStream source = null; + try { + source = new FileInputStream(randomFilePath); + fileSystem.write("/my_file", new FileInputStream(randomFilePath)); + assertTrue(fileSystem.exists("/my_file")); + assertFalse(fileSystem.exists("/my_other_file")); + } finally { + IOUtils.closeQuietly(source); + } } /** @@ -117,25 +130,28 @@ byte[] bytes = new byte[1]; bytes[0] = -0x1; - InputStream source; + InputStream source = null; + try { + source = new ByteArrayInputStream(bytes); + fileSystem.write("/my_file", source); + } finally { + IOUtils.closeQuietly(source); + } - source = new ByteArrayInputStream(bytes); - fileSystem.write("/my_file", source); + try { + source = new ByteArrayInputStream(bytes); + InputStream readResult = fileSystem.read("/my_file"); - source.close(); + int read = 0; + byte[] b = new byte[1]; - - source = new ByteArrayInputStream(bytes); - InputStream readResult = fileSystem.read("/my_file"); - - int read = 0; - byte[] b = new byte[1]; - - read = readResult.read(b); + read = readResult.read(b); - assertEquals(1, read); - assertArrayEquals(bytes, b); - + assertEquals(1, read); + assertArrayEquals(bytes, b); + } finally { + IOUtils.closeQuietly(source); + } } /** @@ -147,31 +163,35 @@ @Test public void testWriteRead() throws Exception { - InputStream source = new FileInputStream(randomFilePath); - - fileSystem.write("/my_file", source); + InputStream source = null; + try { + source = new FileInputStream(randomFilePath); + fileSystem.write("/my_file", source); + } finally { + IOUtils.closeQuietly(source); + } - source.close(); - - InputStream readResult; // now, the checks. We read the original file and the result of // a read() and then compare it byte-to-byte - - source = new FileInputStream(randomFilePath); - readResult = fileSystem.read("/my_file"); - assertEquals(randomFileSize, source.available()); - assertEquals(randomFileSize, readResult.available()); + InputStream readResult = null; + try { + source = new FileInputStream(randomFilePath); + readResult = fileSystem.read("/my_file"); + + assertEquals(randomFileSize, source.available()); + assertEquals(randomFileSize, readResult.available()); - byte[] sourceAsBytes = IOUtils.toByteArray(source); - byte[] readResultAsBytes = IOUtils.toByteArray(readResult); + byte[] sourceAsBytes = IOUtils.toByteArray(source); + byte[] readResultAsBytes = IOUtils.toByteArray(readResult); - assertArrayEquals(sourceAsBytes, readResultAsBytes); + assertArrayEquals(sourceAsBytes, readResultAsBytes); + } finally { + IOUtils.closeQuietly(source); + IOUtils.closeQuietly(readResult); + } - source.close(); - readResult.close(); - } /** @@ -180,22 +200,34 @@ */ @Test(expected = IOException.class) public void testFailAtWrite() throws Exception { - fileSystem.write("/my_folder/my_file", - new FileInputStream(randomFilePath)); + InputStream source = null; + try { + source = new FileInputStream(randomFilePath); + fileSystem.write("/my_folder/my_file", source); + } finally { + IOUtils.closeQuietly(source); + } } /** * this writing a file that already exists should not be a problem */ @Test - public void testFailAtDoubleWrite() throws Exception { - InputStream source = new FileInputStream(randomFilePath); - fileSystem.write("/my_file", source); - source.close(); + public void testDoubleWrite() throws Exception { + InputStream source = null; + try { + source = new FileInputStream(randomFilePath); + fileSystem.write("/my_file", source); + } finally { + IOUtils.closeQuietly(source); + } - source = new FileInputStream(randomFilePath); - fileSystem.write("/my_file", source); - source.close(); + try { + source = new FileInputStream(randomFilePath); + fileSystem.write("/my_file", source); + } finally { + IOUtils.closeQuietly(source); + } } /** @@ -217,12 +249,23 @@ @Test public void testWriteInFolder() throws Exception { fileSystem.createDirectory("/my_folder"); - fileSystem.write("/my_folder/my_file", - new FileInputStream(randomFilePath)); fileSystem.createDirectory("/my_folder/my_sub_folder"); - fileSystem.write("/my_folder/my_sub_folder/my_file", - new FileInputStream(randomFilePath)); - assertTrue(fileSystem.exists("/my_folder/my_sub_folder/my_file")); + + InputStream source = null; + try { + source = new FileInputStream(randomFilePath); + fileSystem.write("/my_folder/my_file", source); + } finally { + IOUtils.closeQuietly(source); + } + + try { + source = new FileInputStream(randomFilePath); + fileSystem.write("/my_folder/my_sub_folder/my_file", source); + assertTrue(fileSystem.exists("/my_folder/my_sub_folder/my_file")); + } finally { + IOUtils.closeQuietly(source); + } } /** @@ -233,19 +276,29 @@ @Test public void testLinking() throws Exception { fileSystem.createDirectory("/my_folder"); - fileSystem.write("/my_folder/my_file", - new FileInputStream(randomFilePath)); + + InputStream source = null; + try { + source = new FileInputStream(randomFilePath); + fileSystem.write("/my_folder/my_file", source); + } finally { + IOUtils.closeQuietly(source); + } + fileSystem.createSymbolicLink("/my_link", "/my_folder/my_file"); - InputStream source = new FileInputStream(randomFilePath); - InputStream readResult = fileSystem.read("/my_link"); + InputStream readResult = null; + try { + source = new FileInputStream(randomFilePath); + readResult = fileSystem.read("/my_link"); - boolean actualContentEquality = - IOUtils.contentEquals(source, readResult); - source.close(); - readResult.close(); - - assertTrue(actualContentEquality); + boolean actualContentEquality = + IOUtils.contentEquals(source, readResult); + assertTrue(actualContentEquality); + } finally { + IOUtils.closeQuietly(source); + IOUtils.closeQuietly(readResult); + } } /** @@ -264,8 +317,14 @@ @Test public void testRemove() throws Exception { fileSystem.createDirectory("/my_folder"); - fileSystem.write("/my_folder/my_file", - new FileInputStream(randomFilePath)); + + InputStream source = null; + try { + source = new FileInputStream(randomFilePath); + fileSystem.write("/my_folder/my_file", source); + } finally { + IOUtils.closeQuietly(source); + } fileSystem.delete("/my_folder/my_file"); assertTrue(fileSystem.exists("/my_folder")); assertFalse(fileSystem.exists("/my_folder/my_file")); @@ -281,9 +340,15 @@ @Test(expected = IOException.class) public void testFailAtRemove() throws Exception { fileSystem.createDirectory("/my_folder"); - fileSystem.write("/my_folder/my_file", - new FileInputStream(randomFilePath)); + InputStream source = null; + try { + source = new FileInputStream(randomFilePath); + fileSystem.write("/my_folder/my_file", source); + } finally { + IOUtils.closeQuietly(source); + } + // trying to remove a non-empty directory should raise an exception fileSystem.delete("/my_folder"); } @@ -295,8 +360,15 @@ public void testListDirectory() throws Exception { fileSystem.createDirectory("/my_folder"); fileSystem.createDirectory("/my_folder/my_sub_dir"); - fileSystem.write("/my_folder/my_file", - new FileInputStream(randomFilePath)); + + InputStream source = null; + try { + source = new FileInputStream(randomFilePath); + fileSystem.write("/my_folder/my_file", source); + } finally { + IOUtils.closeQuietly(source); + } + fileSystem.createSymbolicLink("/my_folder/my_link", "my_file"); List<String> lsResult = fileSystem.readDirectory("/my_folder"); @@ -362,7 +434,14 @@ fileSystem.createDirectory("/dir/subdir"); fileSystem.createDirectory("/dir/subdir/subsubdir"); fileSystem.createSymbolicLink("/dir/link", "subdir"); - fileSystem.write("/dir/file", new FileInputStream(randomFilePath)); + + InputStream source = null; + try { + source = new FileInputStream(randomFilePath); + fileSystem.write("/dir/file", source); + } finally { + IOUtils.closeQuietly(source); + } fileSystem.createDirectory("/otherdir"); fileSystem.move("/dir/subdir", "/otherdir/subdir"); Modified: trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemKademliaTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemKademliaTest.java 2010-06-19 18:55:14 UTC (rev 84) +++ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemKademliaTest.java 2010-06-21 10:14:34 UTC (rev 85) @@ -6,6 +6,7 @@ import static org.junit.Assert.fail; import java.io.ByteArrayInputStream; +import java.io.FileInputStream; import java.util.ConcurrentModificationException; import org.apache.commons.io.IOUtils; @@ -38,8 +39,8 @@ DisworkFileSystemConfig.newKademliaDisworkConfig(bootstrapPort); DisworkFileSystem fileSystem2 = new DisworkFileSystem(disworkConfig); - assertTrue(fileSystem.exists("/")); - assertTrue(fileSystem2.exists("/")); + fileSystem.write("/file", new FileInputStream(randomFilePath)); + assertTrue(fileSystem2.exists("/file")); } @Test Modified: trunk/pom.xml =================================================================== --- trunk/pom.xml 2010-06-19 18:55:14 UTC (rev 84) +++ trunk/pom.xml 2010-06-21 10:14:34 UTC (rev 85) @@ -81,7 +81,7 @@ <dependency> <groupId>org.planx</groupId> <artifactId>koala-xmlstore</artifactId> - <version>0.4.12</version> + <version>0.4.12_cl20100612</version> </dependency> <dependency> <groupId>commons-digester</groupId>

1 0

r84 - trunk
by tchemit＠users.nuiton.org June 19, 2010

June 19, 2010

Author: tchemit Date: 2010-06-19 20:55:14 +0200 (Sat, 19 Jun 2010) New Revision: 84 Url: http://nuiton.org/repositories/revision/diswork/84 Log: Utilisation de mavenpom4redmine 2.2 Modified: trunk/pom.xml Modified: trunk/pom.xml =================================================================== --- trunk/pom.xml 2010-06-17 13:37:59 UTC (rev 83) +++ trunk/pom.xml 2010-06-19 18:55:14 UTC (rev 84) @@ -1,6 +1,5 @@ <?xml version="1.0" encoding="UTF-8"?> -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> @@ -11,7 +10,7 @@ <parent> <groupId>org.nuiton</groupId> <artifactId>mavenpom4redmine</artifactId> - <version>2.1.5</version> + <version>2.2</version> </parent> <artifactId>diswork</artifactId>

1 0

r83 - in trunk/diswork-fs/src: main/java/org/nuiton/diswork/fs test/java/org/nuiton/diswork/fs
by bleny＠users.nuiton.org June 17, 2010

June 17, 2010

Author: bleny Date: 2010-06-17 15:37:59 +0200 (Thu, 17 Jun 2010) New Revision: 83 Url: http://nuiton.org/repositories/revision/diswork/83 Log: bugfix move, overwrite Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystem.java trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/AbstractDisworkFileSystemTest.java trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemInMemoryTest.java Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystem.java 2010-06-17 13:33:48 UTC (rev 82) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystem.java 2010-06-17 13:37:59 UTC (rev 83) @@ -49,6 +49,11 @@ * <li>{@link #exists(String)} and {@link #delete(String)} can be used * on directories, files and symlinks</li> * </ul> + * + * Those methods are not thread-safe, so manipulating the content of + * same directory in two different thread using the same instance may + * lead to errors. + * */ public class DisworkFileSystem implements Closeable { @@ -164,14 +169,6 @@ if (EntryUtil.isDirectory(entryParent)) { String parentId = EntryUtil.getIdFromEntry(entryParent); - // checking that file do not already exists in this directory - String content = storage.getDirectory(parentId); - String findResult = EntryUtil.findEntryInDirectory - (content, fileName); - if (findResult != null) { - throw new IOException - (parent + " already contains an element named " + fileName); - } // file do not exists, write file on the FS String newFileId = EntryUtil.generateId(); @@ -184,11 +181,26 @@ lockAcquired = storage.tryToLock(parentId); if (lockAcquired) { // parent dir is locked, do the update - content = storage.getDirectory(parentId); - String newContent = EntryUtil.addEntryToDirectoryContent( + + // checking that file do not already exists in this directory + String content = storage.getDirectory(parentId); + String oldEntry = EntryUtil.findEntryInDirectory + (content, fileName); + + if (oldEntry != null) { + content = EntryUtil.removeEntryFromEntries(content, fileName); + } + + content = EntryUtil.addEntryToDirectoryContent( content, EntryUtil.TYPE.F, fileName, newFileId); - storage.putDirectory(parentId, newContent); + storage.putDirectory(parentId, content); storage.unLock(parentId); + + // removing old data + if (oldEntry != null) { + storage.removeFile(EntryUtil.getIdFromEntry(oldEntry)); + } + } else { log.info(parent + " is locked and can't be written"); try { @@ -410,13 +422,16 @@ /** * remove a file, directory, or link. Non-empty directories can't be - * removed + * removed. * @param path the complete path to the entity to remove * @throws IOException if path is incorrect or directory not empty */ public void delete(String path) throws IOException, ConcurrentModificationException { checkPathSyntax(path); + if (!exists(path)) { + throw new IOException(path + " doesn't exists"); + } String parent = EntryUtil.getParentFromPath(path); String name = EntryUtil.getNameFromPath(path); log.info("trying to remove " + path); @@ -703,6 +718,10 @@ checkPathSyntax(path); checkPathSyntax(destination); + if (exists(destination)) { + throw new IOException(destination + " already exists"); + } + String pathParent = EntryUtil.getParentFromPath(path); String pathName = EntryUtil.getNameFromPath(path); String destinationParent = EntryUtil.getParentFromPath(destination); @@ -750,6 +769,7 @@ while (numberOfTry <= LOCK_MAX_NUMBER_OF_TRY && !lockAcquired) { lockAcquired = storage.tryToLock(parentId) && storage.tryToLock(destinationParentId); + if (lockAcquired) { // we have locked, do the update parentContent = storage.getDirectory(parentId); Modified: trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/AbstractDisworkFileSystemTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/AbstractDisworkFileSystemTest.java 2010-06-17 13:33:48 UTC (rev 82) +++ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/AbstractDisworkFileSystemTest.java 2010-06-17 13:37:59 UTC (rev 83) @@ -21,6 +21,7 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.experimental.categories.Categories.ExcludeCategory; import org.nuiton.diswork.fs.DisworkFileSystem; import org.nuiton.util.FileUtil; @@ -184,21 +185,17 @@ } /** - * this use case should raise an exception because writing to a file - * that already exists + * this writing a file that already exists should not be a problem */ - @Test(expected = IOException.class) + @Test public void testFailAtDoubleWrite() throws Exception { InputStream source = new FileInputStream(randomFilePath); fileSystem.write("/my_file", source); source.close(); source = new FileInputStream(randomFilePath); - try { - fileSystem.write("/my_file", source); - } finally { - source.close(); - } + fileSystem.write("/my_file", source); + source.close(); } /** @@ -395,6 +392,13 @@ */ } + @Test(expected = IOException.class) + public void failAtMove() throws Exception { + fileSystem.createDirectory("/dir"); + fileSystem.createDirectory("/dir2"); + fileSystem.move("/dir", "/dir2"); + } + @Test public void testCreateDirectories() throws Exception { try { Modified: trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemInMemoryTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemInMemoryTest.java 2010-06-17 13:33:48 UTC (rev 82) +++ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemInMemoryTest.java 2010-06-17 13:37:59 UTC (rev 83) @@ -29,6 +29,7 @@ fileSystem.createDirectory("/dir"); fileSystem.write("/dir/file", new FileInputStream(randomFilePath)); + fileSystem.write("/dir/file", new FileInputStream(randomFilePath)); fileSystem.createSymbolicLink("/dir/link", "/dir/file"); fileSystem.delete("/dir/link");

1 0