July 2010 - Diswork-commits - mailman.cloud.codelutin.com

r97 - in trunk: diswork-daemon/src/main/java/org/nuiton/diswork/daemon diswork-daemon/src/test/java/org/nuiton/diswork/daemon diswork-fs/src/main/java/org/nuiton/diswork/fs src/site/rst/user
by bleny＠users.nuiton.org July 1, 2010

July 1, 2010

Author: bleny Date: 2010-07-01 16:35:38 +0200 (Thu, 01 Jul 2010) New Revision: 97 Url: http://nuiton.org/repositories/revision/diswork/97 Log: documentation, global stats Modified: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkConfig.java trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkDaemon.java trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkSimpleClient.java trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonTest.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystem.java trunk/src/site/rst/user/how_to_use.rst Modified: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkConfig.java =================================================================== --- trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkConfig.java 2010-07-01 13:15:34 UTC (rev 96) +++ trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkConfig.java 2010-07-01 14:35:38 UTC (rev 97) @@ -107,10 +107,6 @@ setDefaultOption("diswork.total_uptime", "0"); } - public String getTempDirectory() { - return getOption("diswork.temp_directory"); - } - public static DisworkConfig newConfig() { DisworkConfig newConfig = new DisworkConfig(); newConfig.setFileSystemConfig(DisworkFileSystemConfig.newKademliaDisworkConfig()); @@ -186,6 +182,11 @@ + + public String getTempDirectory() { + return getOption("diswork.temp_directory"); + } + public String getOwnerId() { return getOption("diswork.owner"); } Modified: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkDaemon.java =================================================================== --- trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkDaemon.java 2010-07-01 13:15:34 UTC (rev 96) +++ trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkDaemon.java 2010-07-01 14:35:38 UTC (rev 97) @@ -42,6 +42,7 @@ import org.nuiton.diswork.fs.DisworkFileSystem; import org.nuiton.diswork.fs.DisworkFileSystemConfig; import org.nuiton.diswork.fs.DisworkFileSystemException; +import org.nuiton.diswork.fs.DisworkFileSystemException.Type; import org.nuiton.util.FileUtil; /** @@ -230,7 +231,8 @@ // init fileSystem with all needed directories String[] directories = { BIN, HOME, TODO, TODO_RUNNING, FAILED_1, - FAILED_1_RUNNING, FAILED_2, FAILED_2_RUNNING, FAILED_3, DONE}; + FAILED_1_RUNNING, FAILED_2, FAILED_2_RUNNING, FAILED_3, DONE, + "/proc"}; // if HOME exists, we suppose all others exists if (! fileSystem.exists(HOME)) { for (String directory : directories) { @@ -294,6 +296,9 @@ * @throws DisworkException */ protected void initWorkersManager() throws DisworkException { + // cleaning the temp dir + FileUtil.deleteRecursively(config.getTempDirectory()); + // check if config implies to run a worker if (config.getNumberOfWorkers() >= 0) { workers = new WorkersManager(fileSystem, config); @@ -501,12 +506,23 @@ jobDescription.getInputData().get(fileName)); } - // FIXME 20100609 bleny may throws exception if jobs are proposed - // at a same time - // propose job - String linkName = newJobLinkName(); - fileSystem.createSymbolicLink(TODO + "/" + linkName, jobDir); + boolean success = false; + while (!success) { + try { + String linkName = newJobLinkName(); + // there is a risk that jobs be proposed at the same time + fileSystem.createSymbolicLink(TODO + "/" + linkName, jobDir); + success = true; + } catch (DisworkFileSystemException e) { + if (e.getType() == Type.ALREADY_EXISTS) { + // retry + success = false; + } else { + throw e; + } + } + } log.info("job submited"); @@ -620,34 +636,74 @@ * @return * @throws DisworkException */ - public Map<String, Integer> getGlobalStats() throws DisworkException { + public Map<String, String> getGlobalStats() throws DisworkException { + final String globalStatsPath = "/proc/globalstats"; + final int timeBeforeGlobalStatsAreObsolete = 1 * 60 * 60 * 1000; + + // in this file, key and values are split with \t and entries are split + // with \n, if a global-stats file is found, read it. If not, create + // it and date it. If file is obsolete read it and delete it + try { - Map<String, Integer> stats = new HashMap<String, Integer>(); - stats.put("available_processors", 0); - - List<String> homeDirs = fileSystem.readDirectory(HOME); - for (String homeDir : homeDirs) { - String hardinfo = IOUtils.toString(fileSystem.read( - HOME + "/" + homeDir + "/" + HARDINFO_PATH)); - String[] infos = hardinfo.split("\n"); + Long currentTime = System.currentTimeMillis(); + Map<String, String> result = new HashMap<String, String>(); + + if (fileSystem.exists(globalStatsPath)) { + String globalStats = IOUtils.toString(fileSystem.read(globalStatsPath)); + log.debug("global stats file found, reading " + globalStats); + String[] lines = globalStats.split("\n"); + for (String line : lines) { + String[] keyValue = line.split("\t"); + result.put(keyValue[0], keyValue[1]); + } - // reading the OS name - if (!stats.containsKey(infos[0])) { - stats.put(infos[0], 0); + // delete the file if it's too old + Long statsTime = Long.parseLong(result.get("date")); + if (currentTime - statsTime > timeBeforeGlobalStatsAreObsolete) { + log.info("deleting global stats file"); + fileSystem.delete(globalStatsPath); } - // reading the architecture - stats.put(infos[0], stats.get(infos[0]) + 1); - if (!stats.containsKey(infos[1])) { - stats.put(infos[1], 0); + } else { + log.info("global stats file doesn't exists, creating one"); + Map<String, Long> stats = new HashMap<String, Long>(); + Long availableProcessors = 0L; + + List<String> homeDirs = fileSystem.readDirectory(HOME); + for (String homeDir : homeDirs) { + String hardinfo = IOUtils.toString(fileSystem.read( + HOME + "/" + homeDir + "/" + HARDINFO_PATH)); + String[] infos = hardinfo.split("\n"); + + // reading the OS name + if (!stats.containsKey(infos[0])) { + stats.put(infos[0], 0L); + } + stats.put(infos[0], stats.get(infos[0]) + 1); + + // reading the architecture + if (!stats.containsKey(infos[1])) { + stats.put(infos[1], 0L); + } + stats.put(infos[1], stats.get(infos[1]) + 1); + + // reading the number of processors + availableProcessors += Integer.parseInt(infos[2]); } - stats.put(infos[1], stats.get(infos[1]) + 1); - - // reading the number of processors - stats.put("available_processors", - stats.get("available_processors") + - Integer.parseInt(infos[2])); + stats.put("available_processors", availableProcessors); + stats.put("date", currentTime); + + // write the result + String statsFileContent = ""; + for (String key : stats.keySet()) { + result.put(key, stats.get(key).toString()); + statsFileContent += key + "\t" + stats.get(key) + "\n"; + } + + log.debug("writing stats file " + statsFileContent); + fileSystem.write(globalStatsPath, IOUtils.toInputStream(statsFileContent)); } - return stats; + + return result; } catch (DisworkFileSystemException e) { log.error("file system error ", e); throw new DisworkException("file system error ", e); Modified: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkSimpleClient.java =================================================================== --- trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkSimpleClient.java 2010-07-01 13:15:34 UTC (rev 96) +++ trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkSimpleClient.java 2010-07-01 14:35:38 UTC (rev 97) @@ -58,7 +58,7 @@ userEntry = br.readLine(); if ("stats".equals(userEntry)) { - Map<String, Integer> stats = daemon.getGlobalStats(); + Map<String, String> stats = daemon.getGlobalStats(); for (String key : stats.keySet()) { System.out.println(key + " => " + stats.get(key)); } Modified: trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonTest.java =================================================================== --- trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonTest.java 2010-07-01 13:15:34 UTC (rev 96) +++ trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonTest.java 2010-07-01 14:35:38 UTC (rev 97) @@ -149,10 +149,17 @@ public void testStats() throws Exception { daemon.getUptimeRatio(); - Map<String, Integer> stats = daemon.getGlobalStats(); - // deamon should read 3 stats : 1 OS, 1 architecture and the number + Map<String, String> stats = daemon.getGlobalStats(); + // deamon should read 3 stats : 1 OS, 1 architecture, the number + // of processors and the date when the stats was computed + assertEquals(4, stats.size()); + + // a second read should return the same data, without re generate them + // check logs + stats = daemon.getGlobalStats(); + // deamon should read 4 stats : 1 OS, 1 architecture and the number // of processors - assertEquals(3, stats.size()); + assertEquals(4, stats.size()); } } Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystem.java 2010-07-01 13:15:34 UTC (rev 96) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystem.java 2010-07-01 14:35:38 UTC (rev 97) @@ -572,149 +572,8 @@ return result; } - - /** - * return the entry of the element at the end of <code>path</code> - * @param path - * @return null if path is not valid - */ - protected String walk(String path) throws DisworkFileSystemException { - String result = walk(path, null, null); - log.info("walking to " + path + " returns " + result); - return result; - } /** - * This method is a recursive function to walk through the tree structure - * of the directories and starting for root directory, following the - * symbolic links to reach the given path - * @param path - * @param current - * @param content - * @return null if path is not valid or the entry corresponding to path - * @throws IOException - */ - protected String walk(String path, String current, String content) - throws DisworkFileSystemException { - // FIXME 20105021 bleny works fine but is not understandable - String result = null; - - // if path is "/", recursion can be initiated with this value : - // (it returns "/" as the id where to get the content of "/" - if (path.equals(EntryUtil.ROOT_DIRECTORY)) { - return EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR - + EntryUtil.ROOT_DIRECTORY + EntryUtil.ENTRY_SEPARATOR - + EntryUtil.ROOT_DIRECTORY; - } - - String parentPath = EntryUtil.getParentFromPath(path); - - if (content == null) { - // start the recursion from root directory - content = storage.getRootDirectory(); - result = walk(path, EntryUtil.ROOT_DIRECTORY, content); - } else if (parentPath.equals(current)) { - // we are now in the last directory - // ie if path is a/b/p we are at a/b - - // p is the name of the element in a/b we search for - String tail = path.substring(current.length()); - String p = EntryUtil.getNameFromPath(tail); - - log.info("in final dir " + current + ", looking for " + p); - - String entry = EntryUtil.findEntryInDirectory(content, p); - result = entry; - } else { - // in middle of path - // if path is a/b/c/d/e/f, we may be at b, c, d... - - String tail; // the path still remaining when in current - // ie if we are in c, tail is "d/e/f" - - // if we are at root directory, deal with the "/" - if (current.equals(EntryUtil.ROOT_DIRECTORY)) { - tail = path.substring(current.length()); - } else { - tail = path.substring(current.length() + 1); - } - - log.debug("current = " + current); - log.debug("tail = " + tail); - String[] elementsNames = tail.split(EntryUtil.PATH_SEPARATOR); - String p = elementsNames[0]; - - log.info("in intermediate dir " + current + ", looking for " + p); - - // updating current for recursion - if (current.equals(EntryUtil.ROOT_DIRECTORY)) { - current = ""; // avoid "//path" next line - } - current += EntryUtil.PATH_SEPARATOR + p; - - String entry = EntryUtil.findEntryInDirectory(content, p); - if (entry == null) { - result = null; - } else { - // we have found the entry to call recursion - - if (EntryUtil.isDirectory(entry)) { - String id = EntryUtil.getIdFromEntry(entry); - content = storage.getDirectory(id); - result = walk(path, current, content); - } else if (EntryUtil.isLink(entry)) { - String id = EntryUtil.getIdFromEntry(entry); - String linkContent = storage.getLink(id); - String newTarget = - EntryUtil.resolveLink(current, linkContent); - newTarget += path.substring(current.length()); - - // restart walk from / - result = walk(newTarget, null, null); - } else if (EntryUtil.isFile(entry)) { - // error, found file in path like '/dir1/dir2/filename/dir3' - result = null; - } else { - log.warn("strange case: " + entry); - result = null; - } - } - } - return result; - } - - @Override - public void close() throws IOException { - storage.close(); - } - - protected void setMap(DisworkMap map) { - storage.setMap(map); - } - - /** - * check a path is absolute and syntactically correct, throw exception if - * that's not the case. - */ - protected void checkPathSyntax(String path) - throws DisworkFileSystemException { - if (!path.startsWith(EntryUtil.ROOT_DIRECTORY)) { - throw new DisworkFileSystemException(Type.INVALID_PATH, - "\"" + path + "\" is not correct, all pathes " + - "have to be absolute (thus, starts with)" + - EntryUtil.ROOT_DIRECTORY); - } - - String doubleSeparator = EntryUtil.PATH_SEPARATOR - + EntryUtil.PATH_SEPARATOR; - if (path.contains(doubleSeparator)) { - throw new DisworkFileSystemException(Type.INVALID_PATH, - "\"" + path + "\" is not correct, it contains " - + doubleSeparator); - } - } - - /** * move a file, a directory a link. May be used for renaming purpose * @param path the path to the object to move * @param destination the full path of the target @@ -857,4 +716,150 @@ } } } + + /** + * return the entry of the element at the end of <code>path</code> + * @param path + * @return null if path is not valid + */ + protected String walk(String path) throws DisworkFileSystemException { + String result = walk(path, null, null); + log.info("walking to " + path + " returns " + result); + return result; + } + + /** + * This method is a recursive function to walk through the tree structure + * of the directories and starting for root directory, following the + * symbolic links to reach the given path + * @param path + * @param current + * @param content + * @return null if path is not valid or the entry corresponding to path + * @throws IOException + */ + protected String walk(String path, String current, String content) + throws DisworkFileSystemException { + // FIXME 20105021 bleny works fine but is not understandable + String result = null; + + // if path is "/", recursion can be initiated with this value : + // (it returns "/" as the id where to get the content of "/" + if (path.equals(EntryUtil.ROOT_DIRECTORY)) { + return EntryUtil.TYPE.D + EntryUtil.ENTRY_SEPARATOR + + EntryUtil.ROOT_DIRECTORY + EntryUtil.ENTRY_SEPARATOR + + EntryUtil.ROOT_DIRECTORY; + } + + String parentPath = EntryUtil.getParentFromPath(path); + + if (content == null) { + // start the recursion from root directory + content = storage.getRootDirectory(); + result = walk(path, EntryUtil.ROOT_DIRECTORY, content); + } else if (parentPath.equals(current)) { + // we are now in the last directory + // ie if path is a/b/p we are at a/b + + // p is the name of the element in a/b we search for + String tail = path.substring(current.length()); + String p = EntryUtil.getNameFromPath(tail); + + log.info("in final dir " + current + ", looking for " + p); + + String entry = EntryUtil.findEntryInDirectory(content, p); + result = entry; + } else { + // in middle of path + // if path is a/b/c/d/e/f, we may be at b, c, d... + + String tail; // the path still remaining when in current + // ie if we are in c, tail is "d/e/f" + + // if we are at root directory, deal with the "/" + if (current.equals(EntryUtil.ROOT_DIRECTORY)) { + tail = path.substring(current.length()); + } else { + tail = path.substring(current.length() + 1); + } + + log.debug("current = " + current); + log.debug("tail = " + tail); + String[] elementsNames = tail.split(EntryUtil.PATH_SEPARATOR); + String p = elementsNames[0]; + + log.info("in intermediate dir " + current + ", looking for " + p); + + // updating current for recursion + if (current.equals(EntryUtil.ROOT_DIRECTORY)) { + current = ""; // avoid "//path" next line + } + current += EntryUtil.PATH_SEPARATOR + p; + + String entry = EntryUtil.findEntryInDirectory(content, p); + if (entry == null) { + result = null; + } else { + // we have found the entry to call recursion + + if (EntryUtil.isDirectory(entry)) { + String id = EntryUtil.getIdFromEntry(entry); + content = storage.getDirectory(id); + result = walk(path, current, content); + } else if (EntryUtil.isLink(entry)) { + String id = EntryUtil.getIdFromEntry(entry); + String linkContent = storage.getLink(id); + String newTarget = + EntryUtil.resolveLink(current, linkContent); + newTarget += path.substring(current.length()); + + // restart walk from / + result = walk(newTarget, null, null); + } else if (EntryUtil.isFile(entry)) { + // error, found file in path like '/dir1/dir2/filename/dir3' + result = null; + } else { + log.warn("strange case: " + entry); + result = null; + } + } + } + return result; + } + + @Override + public void close() throws IOException { + storage.close(); + } + + protected void setMap(DisworkMap map) { + storage.setMap(map); + } + + /** + * check a path is absolute and syntactically correct, throw exception if + * that's not the case. + */ + protected void checkPathSyntax(String path) + throws DisworkFileSystemException { + if (path == null) { + throw new DisworkFileSystemException(Type.INVALID_PATH, + new NullPointerException()); + } + + if (!path.startsWith(EntryUtil.ROOT_DIRECTORY)) { + throw new DisworkFileSystemException(Type.INVALID_PATH, + "\"" + path + "\" is not correct, all pathes " + + "have to be absolute (thus, starts with)" + + EntryUtil.ROOT_DIRECTORY); + } + + String doubleSeparator = EntryUtil.PATH_SEPARATOR + + EntryUtil.PATH_SEPARATOR; + if (path.contains(doubleSeparator)) { + throw new DisworkFileSystemException(Type.INVALID_PATH, + "\"" + path + "\" is not correct, it contains " + + doubleSeparator); + } + } } \ No newline at end of file Modified: trunk/src/site/rst/user/how_to_use.rst =================================================================== --- trunk/src/site/rst/user/how_to_use.rst 2010-07-01 13:15:34 UTC (rev 96) +++ trunk/src/site/rst/user/how_to_use.rst 2010-07-01 14:35:38 UTC (rev 97) @@ -8,8 +8,79 @@ How to install a diswork node on my server ------------------------------------------ +:: + + svn checkout http://svn.nuiton.org/svn/diswork/trunk/ diswork + cd diswork/diswork-daemon + mvn assembly:assembly -Dmaven.test.skip -DdescriptorId=jar-with-dependencies + + cd ../.. + # under debian commons-daemon can be found at /usr/share/java/commons-daemon.jar + jsvc -cp /usr/share/java/commons-daemon.jar:diswork/diswork-daemon/target/diswork-daemon-0.0.1-SNAPSHOT-jar-with-dependencies.jar -pidfile ./pid -outfile ./out -errfile ./err DisworkDaemonRunner + + How to make my own application ready for being run by all diswork nodes ----------------------------------------------------------------------- +Applications for diswork are very easy to create. An application is a zip file. +The only specification of the content is that it has to be set considering that +the whole application will be unzipped in the job-specific directory. + +You can put executables in it, a jar file with its dependencies, scripts, etc. + +You have to write the command-line for the job being aware that the application +and all input files will be in the current working directory. + How to make my application able to submit jobs to Diswork and retrieve results ------------------------------------------------------------------------------- \ No newline at end of file +------------------------------------------------------------------------------ + +Your application have to use the Daemon API, here is a sample code : + +:: + + // First, start a daemon + DisworkConfig config = DisworkConfig.newConfig(); + daemon = new DisworkDaemon(config); + + // submit the application + daemon.submitApplication("fake-app", "1.0", new FileInputStream("fake-app-1.0.zip")); + + // now all the nodes will be able to run jobs that need fake-app version 1.0 + + // create a new job + JobDescription job = new JobDescription(); + job.setJobName("My Job"); + job.setApplication("fake-app", "1.0"); + job.setCommandLine("%java -jar fake-app.jar"); + + // now, defining data in input + + // this will be downloaded by the worker before job start + job.addInput("example.com_index", new URL("http://www.example.com/")); + // this will, needed for fake-app will be provided to the worker + job.addInput("input.txt", new FileInputStream("input.txt")); + + // defining expected data in output + job.addOutput("output.txt"); + job.addOutput("example.com_index"); + + // setting standard input and output file + job.setStandardInput("input.txt"); + job.setStandardOutput("output.txt"); + + // submit the job + daemon.submitJob(job); + + // waiting for the job to finish + while(! daemon.isFinished(job)) { + Thread.sleep(5 * 60 * 1000); + } + + // check that job is successful and read the result files + if(daemon.isSuccessful(job)) { + // results will contains keys "output.txt" and "example.com_index" + // values are InputStream on the content of those files + Map<String, InputStream> results = daemon.getResults(job); + } + + \ No newline at end of file

1 0

r96 - trunk
by tchemit＠users.nuiton.org July 1, 2010

July 1, 2010

Author: tchemit Date: 2010-07-01 15:15:34 +0200 (Thu, 01 Jul 2010) New Revision: 96 Url: http://nuiton.org/repositories/revision/diswork/96 Log: Utilisation de mavenpom4labs 2.2.2.1 Modified: trunk/pom.xml Modified: trunk/pom.xml =================================================================== --- trunk/pom.xml 2010-07-01 12:30:16 UTC (rev 95) +++ trunk/pom.xml 2010-07-01 13:15:34 UTC (rev 96) @@ -10,7 +10,7 @@ <parent> <groupId>org.nuiton</groupId> <artifactId>mavenpom4redmine</artifactId> - <version>2.2.2</version> + <version>2.2.2.1</version> </parent> <artifactId>diswork</artifactId>

1 0

r95 - trunk
by tchemit＠users.nuiton.org July 1, 2010

July 1, 2010

Author: tchemit Date: 2010-07-01 14:30:16 +0200 (Thu, 01 Jul 2010) New Revision: 95 Url: http://nuiton.org/repositories/revision/diswork/95 Log: Utilisation de mavenpom4redmine 2.2.2 Modified: trunk/pom.xml Modified: trunk/pom.xml =================================================================== --- trunk/pom.xml 2010-07-01 10:45:20 UTC (rev 94) +++ trunk/pom.xml 2010-07-01 12:30:16 UTC (rev 95) @@ -10,7 +10,7 @@ <parent> <groupId>org.nuiton</groupId> <artifactId>mavenpom4redmine</artifactId> - <version>2.2.1</version> + <version>2.2.2</version> </parent> <artifactId>diswork</artifactId>

1 0

r94 - in trunk: diswork-daemon diswork-daemon/src/main/java/org/nuiton/diswork/daemon diswork-daemon/src/test/java/org/nuiton/diswork/daemon diswork-fs diswork-fs/src/main/java/org/nuiton/diswork/fs diswork-fs/src/main/java/org/nuiton/diswork/fs/storage diswork-fs/src/test/java/org/nuiton/diswork/fs diswork-fs/src/test/java/org/nuiton/diswork/fs/storage
by bleny＠users.nuiton.org July 1, 2010

July 1, 2010

Author: bleny Date: 2010-07-01 12:45:20 +0200 (Thu, 01 Jul 2010) New Revision: 94 Url: http://nuiton.org/repositories/revision/diswork/94 Log: refactoring exception management, tokens property files, diswork runnable as an OS service Added: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystemException.java Removed: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkFileSystemException.java Modified: trunk/diswork-daemon/ trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/ActivityStrategy.java trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkConfig.java trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkDaemon.java trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/JobDescription.java trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/WorkersManager.java trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonTest.java trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/JobDescriptionTest.java trunk/diswork-fs/pom.xml trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/Demo.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystem.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/KademliaDisworkMap.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/PastryDisworkMap.java trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/Storage.java trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/AbstractDisworkFileSystemTest.java trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemKademliaTest.java trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/AbstractDisworkMapTest.java trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/KademliaDisworkMapTest.java Property changes on: trunk/diswork-daemon ___________________________________________________________________ Modified: svn:ignore - target .classpath .settings + target .classpath .settings .project diswork-daemon.iml Modified: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/ActivityStrategy.java =================================================================== --- trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/ActivityStrategy.java 2010-07-01 10:38:20 UTC (rev 93) +++ trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/ActivityStrategy.java 2010-07-01 10:45:20 UTC (rev 94) @@ -95,9 +95,7 @@ try { Thread.sleep(5 * 60 * 1000); // 5 min } catch (InterruptedException e) { - // TODO 20100615 bleny Auto-generated catch block - log.info("exception catch", e); - e.printStackTrace(); + log.warn("load average monitoring interrupted", e); } loadAverage10MinutesBefore = loadAverage5MinutesBefore; loadAverage5MinutesBefore = loadAverageNow; Modified: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkConfig.java =================================================================== --- trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkConfig.java 2010-07-01 10:38:20 UTC (rev 93) +++ trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkConfig.java 2010-07-01 10:45:20 UTC (rev 94) @@ -25,8 +25,17 @@ package org.nuiton.diswork.daemon; import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; import java.lang.management.ManagementFactory; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.nuiton.diswork.fs.DisworkFileSystemConfig; import org.nuiton.util.ApplicationConfig; @@ -58,14 +67,28 @@ * nights, week-end, etc.). It needs to define a pattern.</dd> * </dl> * </dd> + * <dt>diswork.tokens_file</dt> + * <dd>a path to a file containing token-names and the replacement strings + * this is a property file</dd> * </dl> * * @author bleny */ public class DisworkConfig extends ApplicationConfig { - + + private static final Log log = LogFactory.getLog(DisworkConfig.class); + protected DisworkFileSystemConfig fileSystemConfig; + + /** TOKENS are piece of string you can use for writing command lines + * some tokens are defined by default, some others can be defined in a + * properties file + * + * @see #initTokens() + */ + protected Map<String, String> tokens; + public DisworkConfig() { setConfigFileName("diswork.config"); @@ -94,6 +117,75 @@ return newConfig; } + /** + * Read the tokens file if one is given in the config and merge the content + * of this file into {@link #tokens} + * + * @throws DisworkException + */ + protected void initTokens() throws DisworkException { + tokens = new HashMap<String, String>(); + + String java = // full java path + System.getProperty("java.home") + + File.separator + "bin" + File.separator + "java" + // setting some system properties + + " -Duser.dir=%tmp" + + " -Duser.home=%tmp" + + " -Djava.io.tmpdir=%tmp" + ; + tokens.put("%java", java); + + File tokensFile = getTokensFile(); + if (tokensFile != null) { + try { + InputStream tokensStream = new FileInputStream(tokensFile); + Properties userTokens = new Properties(); + userTokens.load(tokensStream); + for (String token : userTokens.stringPropertyNames()) { + log.debug("adding token " + token + " → " + + userTokens.getProperty(token)); + tokens.put(token, userTokens.getProperty(token)); + } + } catch (FileNotFoundException e) { + log.warn("tokens file not found, 0 tokens loaded", e); + throw new DisworkException("tokens file not found," + + "no token loaded", e); + } catch (IOException e) { + log.error("can't read tokens file", e); + throw new DisworkException("can't read tokens file", e); + } + } + } + + protected String applyTokensRecursively(String commandLine) { + String result = commandLine; + for (String token : tokens.keySet()) { + result = result.replaceAll(token, tokens.get(token)); + } + if (result.equals(commandLine)) { + return result; + } else { + // a token has been applied, re-do a pass + return applyTokensRecursively(result); + } + } + + protected String parseCommandLine(String commandLine, String tempDir) + throws DisworkException { + if (tokens == null) { + initTokens(); + } + + tokens.put("%tmp", tempDir); + + return applyTokensRecursively(commandLine); + } + + + + + public String getOwnerId() { return getOption("diswork.owner"); } @@ -102,10 +194,6 @@ setOption("diswork.owner", ownerId); } - - - - public String getBootstrapIp() { return fileSystemConfig.getBootstrapIp(); } @@ -171,5 +259,22 @@ String firstRunTime = getOption("diswork.first_run_time"); return Long.parseLong(firstRunTime); } - + + /** + * + * @return null if no file specified + */ + public File getTokensFile() { + String path = getOption("diswork.tokens_file"); + if (path == null) { + return null; + } else { + File file = new File(path); + return file; + } + } + + public void setTokensFile(String tokensFilePath) { + setOption("diswork.tokens_file", tokensFilePath); + } } Modified: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkDaemon.java =================================================================== --- trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkDaemon.java 2010-07-01 10:38:20 UTC (rev 93) +++ trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkDaemon.java 2010-07-01 10:45:20 UTC (rev 94) @@ -25,14 +25,11 @@ package org.nuiton.diswork.daemon; import java.io.Closeable; -import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.lang.management.ManagementFactory; import java.lang.management.OperatingSystemMXBean; -import java.net.UnknownHostException; import java.util.ArrayList; -import java.util.ConcurrentModificationException; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -44,6 +41,7 @@ import org.apache.commons.logging.LogFactory; import org.nuiton.diswork.fs.DisworkFileSystem; import org.nuiton.diswork.fs.DisworkFileSystemConfig; +import org.nuiton.diswork.fs.DisworkFileSystemException; import org.nuiton.util.FileUtil; /** @@ -130,7 +128,7 @@ * Hierarchy Standard. All usual paths used by Diswork are defined in * constants. * - * @see http://en.wikipedia.org/wiki/Filesystem_Hierarchy_Standard + * @link http://en.wikipedia.org/wiki/Filesystem_Hierarchy_Standard * * @author bleny */ @@ -202,49 +200,81 @@ public DisworkDaemon(DisworkConfig config) throws DisworkException { this.config = config; - // init fileSystem with all needed directories + // step by step, set all the dependencies of the daemon + + // init the file-system + initFileSystem(); + + initOwnerIdAndHomeDir(); + + initWorkersManager(); + + writeHardwareInfos(); + + sessionStartTime = System.currentTimeMillis(); + } + + /* *** init methods, used once by the constructor */ + + /** + * set {@link #fileSystem} and create all base directories + * on the File System + */ + protected void initFileSystem() throws DisworkException { + try { DisworkFileSystemConfig fileSystemConfig = config.getFileSystemConfig(); fileSystem = new DisworkFileSystem(fileSystemConfig); - initFileSystem(); - } catch (UnknownHostException e) { - log.error("bootstrap failed", e); - throw new DisworkException("bootstrap failed", e); - } catch (IOException e) { + + // init fileSystem with all needed directories + String[] directories = { BIN, HOME, TODO, TODO_RUNNING, FAILED_1, + FAILED_1_RUNNING, FAILED_2, FAILED_2_RUNNING, FAILED_3, DONE}; + // if HOME exists, we suppose all others exists + if (! fileSystem.exists(HOME)) { + for (String directory : directories) { + if (! fileSystem.exists(directory)) { + fileSystem.createDirectories(directory); + log.info("created " + directory); + } + } + } + } catch (DisworkFileSystemException e) { log.error("booting diswork file system failed", e); throw new DisworkException("booting diswork file system failed", e); } + } + /** + * set {@link #ownerId} and {@link #homeDir} + * @throws DisworkException + */ + protected void initOwnerIdAndHomeDir() throws DisworkException { ownerId = config.getOwnerId(); // get job owner id from config if (ownerId == null) { // first time running the daemon log.info("can't find owner id, generating a new one"); - + // check home dir do not exists try { - Random random = new Random(); // generate a new one by cheking if home dir exists String simpleName = System.getProperty("user.name", "anonymous"); ownerId = simpleName; boolean alreadyExists = fileSystem.exists(HOME + "/" + ownerId); - + // if simpleName is already taken, try simpleName + random + Random random = new Random(); while (alreadyExists) { + ownerId = simpleName + random.nextInt(); alreadyExists = fileSystem.exists(HOME + "/" + ownerId); - ownerId = simpleName + random.nextInt(); } - homeDir = HOME + "/" + ownerId; - if (!fileSystem.exists(homeDir)) { - fileSystem.createDirectory(homeDir); - } - } catch (ConcurrentModificationException e) { - log.info("can't create home dir", e); + + fileSystem.createDirectory(HOME + "/" + ownerId); + + } catch (DisworkFileSystemException e) { + log.error("can't create home dir", e); throw new DisworkException("can't create home dir", e); - } catch (IOException e) { - log.info("can't create home dir", e); - throw new DisworkException("can't create home dir", e); } config.setOwnerId(ownerId); @@ -254,57 +284,42 @@ // config.saveForUser(); } - log.info("owner id is " + ownerId); + homeDir = HOME + "/" + ownerId; + log.info("owner id is " + ownerId); + } + + /** + * + * @throws DisworkException + */ + protected void initWorkersManager() throws DisworkException { // check if config implies to run a worker if (config.getNumberOfWorkers() >= 0) { workers = new WorkersManager(fileSystem, config); } else { log.info("worker manager disabled"); } + } - sessionStartTime = System.currentTimeMillis(); - + protected void writeHardwareInfos() throws DisworkException { // writing hardware info to homeDir try { OperatingSystemMXBean os = ManagementFactory.getOperatingSystemMXBean(); String hardinfos = os.getName() + "\n" + os.getArch() + "\n" + - os.getAvailableProcessors(); - + config.getNumberOfWorkers(); + // TODO 20100615 bleny add RAM size and HDD capacities to hardinfos - + fileSystem.write(homeDir + "/" + HARDINFO_PATH, IOUtils.toInputStream(hardinfos)); - log.info("writing hardware infos " + hardinfos); - } catch (ConcurrentModificationException e) { - log.info("can't write hardware info", e); - throw new DisworkException("can't write hardware info", e); - } catch (IOException e) { - log.info("can't write hardware info", e); - throw new DisworkException("can't write hardware info", e); + log.info("writing hardware infos " + hardinfos.replaceAll("\n", " ")); + } catch (DisworkFileSystemException e) { + log.error("can't write hardware infos", e); + throw new DisworkException("can't write hardware infos", e); } } - /** - * Create all base directories on the File System - * @throws ConcurrentModificationException - * @throws IOException - */ - protected void initFileSystem() throws ConcurrentModificationException, - IOException { - String[] directories = { TODO, TODO_RUNNING, FAILED_1, FAILED_1_RUNNING, - FAILED_2, FAILED_2_RUNNING, FAILED_3, DONE, HOME, BIN }; - // if HOME exists, we suppose all others exists - if (! fileSystem.exists(HOME)) { - for (String directory : directories) { - if (! fileSystem.exists(directory)) { - fileSystem.createDirectories(directory); - log.info("created " + directory); - } - } - } - } - /* *** methods for defining some usual paths *** */ /** @@ -317,7 +332,7 @@ protected static String getPathForDependency(String applicationName, String applicationVersion) { - String result = "/bin/" + applicationName // application directory + String result = BIN + "/" + applicationName // application directory + "/" // application file name + applicationName + "-" + applicationVersion + ".zip"; @@ -337,7 +352,7 @@ /** * Given a job id, returns the place on disworkFS where all data * for this jobs should be stored - * @param jobDescription + * @param jobId * @return a path */ protected String getPathForJob(String jobId) { @@ -353,7 +368,10 @@ protected static String newJobLinkName() { return ((Long) System.currentTimeMillis()).toString(); } - + + + /* ** public methods for use of the daemon */ + /** * Provide an application to all nodes. Once provided, all nodes will be * able to perform a job with this application. @@ -384,12 +402,9 @@ if (!fileSystem.exists(path)) { fileSystem.write(path, applicationData); } - } catch (ConcurrentModificationException e) { - log.info("unable to write", e); - throw new DisworkException("unable to write", e); - } catch (IOException e) { - log.info("unable to write", e); - throw new DisworkException("unable to write", e); + } catch (DisworkFileSystemException e) { + log.error("unable to publish application", e); + throw new DisworkException("unable to publish application", e); } } @@ -410,8 +425,11 @@ result.add(jobDescription); } } catch (IOException e) { - log.info("error file reading home-dir", e); - throw new DisworkException("error file reading home-dir", e); + log.error("error in file while reading home-dir", e); + throw new DisworkException("error in file while reading home-dir", e); + } catch (DisworkFileSystemException e) { + log.error("file-system error", e); + throw new DisworkException("file-system error", e); } return result; } @@ -428,12 +446,12 @@ public void submitJob(JobDescription jobDescription) throws DisworkException { - if (jobDescription.getInputData().size() + jobDescription.getStagingInputUrls().size() - < jobDescription.getStagingInput().size()) { + if (jobDescription.getInputData().size() + jobDescription.getInputUrls().size() + < jobDescription.getInput().size()) { // dependencies are missing } - try { + try { // trying to put the job in a new directory of home Random random = new Random(); @@ -468,10 +486,6 @@ } String jobDir = getPathForJob(jobDescription); - - if(!fileSystem.exists(jobDir)) { - // strange ! - } // creating an empty log file log.info("creating log file " + jobDir + "/" + LOG_PATH); @@ -495,10 +509,10 @@ fileSystem.createSymbolicLink(TODO + "/" + linkName, jobDir); log.info("job submited"); - - } catch (IOException e) { + + } catch (DisworkFileSystemException e) { log.error("file system error", e); - throw new DisworkFileSystemException("file system error", e); + throw new DisworkException(e); } } @@ -508,12 +522,12 @@ String jobPath = getPathForJob(job); List<?> entries = IOUtils.readLines(fileSystem.read(jobPath + "/" + LOG_PATH)); return entries.contains(pattern); - } catch (FileNotFoundException e) { - log.info("log file was not found in job " + job, e); - throw new DisworkException("log file was not found in job " + job, e); } catch (IOException e) { - log.info("file system error ", e); - throw new DisworkException("file system error ", e); + log.info("unable to read log ", e); + throw new DisworkException("unable to read log ", e); + } catch (DisworkFileSystemException e) { + log.error("unable to read log file" + job, e); + throw new DisworkException("unable to read log file" + job, e); } } @@ -541,15 +555,13 @@ throws DisworkException { if (isFinished(job)) { Map<String, InputStream> results = new HashMap<String, InputStream>(); - for (String fileName : job.getStagingOutput()) { + for (String fileName : job.getOutput()) { String jobPath = getPathForJob(job); try { InputStream result = fileSystem.read(jobPath + "/" + fileName); results.put(fileName, result); - } catch (FileNotFoundException e) { - throw new DisworkException("an expected file is missing", e); - } catch (IOException e) { - log.info("file system error ", e); + } catch (DisworkFileSystemException e) { + log.error("file system error ", e); throw new DisworkException("file system error ", e); } } @@ -562,11 +574,13 @@ } /** close the daemon (stop all workers) - * update statistices and delete all temporary data + * update statistics and delete all temporary data */ @Override public void close() throws IOException { - workers.stop(); + if (workers != null) { + workers.stop(); + } // updating total uptime statistic Long totalUptime = getTotalUptime(); @@ -601,7 +615,7 @@ return uptimeRatio; } - /** get informations on hardware available on the global Diswork system + /** get infos on hardware available on the global Diswork system * * @return * @throws DisworkException @@ -634,9 +648,12 @@ Integer.parseInt(infos[2])); } return stats; + } catch (DisworkFileSystemException e) { + log.error("file system error ", e); + throw new DisworkException("file system error ", e); } catch (IOException e) { - log.info("file system error ", e); - throw new DisworkException("file system error ", e); + log.error("can't read hardware infos ", e); + throw new DisworkException("can't read hardware infos ", e); } } } \ No newline at end of file Deleted: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkFileSystemException.java =================================================================== --- trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkFileSystemException.java 2010-07-01 10:38:20 UTC (rev 93) +++ trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkFileSystemException.java 2010-07-01 10:45:20 UTC (rev 94) @@ -1,47 +0,0 @@ -/* - * #%L - * Diswork daemon - * - * $Id$ - * $HeadURL$ - * %% - * Copyright (C) 2010 CodeLutin - * %% - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Lesser Public License for more details. - * - * You should have received a copy of the GNU General Lesser Public - * License along with this program. If not, see - * <http://www.gnu.org/licenses/lgpl-3.0.html>. - * #L% - */ -package org.nuiton.diswork.daemon; - -/** - * - * @author bleny - */ -public class DisworkFileSystemException extends DisworkException { - - private static final long serialVersionUID = -4027003687525235092L; - - public DisworkFileSystemException(String message, Throwable cause) { - super(message, cause); - } - - public DisworkFileSystemException(String message) { - super(message); - } - - public DisworkFileSystemException(Throwable cause) { - super(cause); - } - -} Modified: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/JobDescription.java =================================================================== --- trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/JobDescription.java 2010-07-01 10:38:20 UTC (rev 93) +++ trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/JobDescription.java 2010-07-01 10:45:20 UTC (rev 94) @@ -54,14 +54,17 @@ * <dt>%java</dt> * <dd>will be replaced by the actual path of java executable * into the JRE</dd> + * <dt>%tmp</dt> + * <dd>will be replaced by the the path to dir where the job + * is executed, it's nice to use this to set a temp directory</dd> * </dl> * * This class provides methods to read and parse those data to an XML file * following (as far as possible), the Job Submission Description Language * Specification. * - * @see http://en.wikipedia.org/wiki/Job_Submission_Description_Language - * @see http://www.gridforum.org/documents/GFD.56.pdf + * @link http://en.wikipedia.org/wiki/Job_Submission_Description_Language + * @link http://www.gridforum.org/documents/GFD.56.pdf * * @author bleny */ @@ -106,15 +109,6 @@ /** file where to write the standard output, may be null */ protected String standardOutput; - /** TOKENS are piece of string you can use for writing command lines */ - protected static Map<String, String> TOKENS; - - static { - TOKENS = new HashMap<String, String>(); - - TOKENS.put("%java", System.getProperty("java.home") + "/bin/java"); - } - /** * constructor is protected to prevent bad jobIds. To get a JobDescription * instance, a client should use the {@link #JobDescription()} constructor @@ -136,11 +130,7 @@ } public String getCommandLine() { - String result = commandLine; - for (String token : TOKENS.keySet()) { - result = result.replace(token, TOKENS.get(token)); - } - return result; + return commandLine; } public void setCommandLine(String commandLine) { @@ -157,9 +147,7 @@ /** * this method is protected to force the use of - * {@link #setApplication(String, String)} which need a version. - * This method is still here because it is used by - * {@link #parseJSDL(String)} + * {@link #setApplication(String, String)} which need to give the version. */ protected void setApplicationName(String applicationName) { this.applicationName = applicationName; @@ -172,8 +160,6 @@ /** * this method is protected to force the use of * {@link #setApplication(String, String)} which need a version. - * This method is still here because it is used by - * {@link #parseJSDL(String)} */ protected void setApplicationVersion(String applicationVersion) { this.applicationVersion = applicationVersion; @@ -194,6 +180,10 @@ return "job : " + jobName + " (" + jobId + ")"; } + /** + * Get a JSDL that describes this job + * @return a string containing the complete XML + */ public String toJSDL() { String jsdl = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" @@ -356,15 +346,15 @@ output.add(fileName); } - public List<String> getStagingInput() { + public List<String> getInput() { return input; } - public List<String> getStagingOutput() { + public List<String> getOutput() { return output; } - public Map<String, URL> getStagingInputUrls() { + public Map<String, URL> getInputUrls() { return inputUrls; } Modified: trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/WorkersManager.java =================================================================== --- trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/WorkersManager.java 2010-07-01 10:38:20 UTC (rev 93) +++ trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/WorkersManager.java 2010-07-01 10:45:20 UTC (rev 94) @@ -28,7 +28,6 @@ import java.io.BufferedWriter; import java.io.File; import java.io.FileInputStream; -import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; @@ -47,6 +46,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.diswork.fs.DisworkFileSystem; +import org.nuiton.diswork.fs.DisworkFileSystemException; import org.nuiton.util.FileUtil; import org.nuiton.util.ZipUtil; @@ -161,7 +161,7 @@ this.output = output; this.outputFile = outputFile; } - + @Override public void run() { InputStreamReader osr = new InputStreamReader(output); @@ -189,6 +189,7 @@ } } } catch (IOException e) { + // FIXME 20100701 bleny throw exception log.warn("error while reading the output of the subprocess", e); } finally { try { @@ -210,9 +211,12 @@ * this method add a line to a job-specific log * @param jobPath the path to the job concerned * @param message the line to add the the log - * @throws IOException if an error occurred while writing the log + * @throws DisworkFileSystemException if an error occurred while writing + * the log + * @throws IOException */ - protected void log(String jobPath, String message) throws IOException { + protected void log(String jobPath, String message) + throws DisworkFileSystemException, IOException { String logPath = jobPath + "/" + DisworkDaemon.LOG_PATH; InputStream oldLogAsStream = fileSystem.read(logPath); String oldLog = IOUtils.toString(oldLogAsStream); @@ -221,24 +225,6 @@ fileSystem.write(logPath, IOUtils.toInputStream(newLog)); } - protected void streamCopy(InputStream in, OutputStream out) throws IOException { - System.out.println("hop"); - int read = 0; - int totalRead = 0; - byte[] buffer = new byte[50000]; - // BufferedInputStream hop = new BufferedInputStream(in); - - while ((read = in.read(buffer)) != -1) { - - totalRead += read; - System.out.println("read " + read + " bytes (total=" + totalRead + ")"); - - out.write(buffer, 0, read); - } - in.close(); - out.close(); - } - /** * Download all the files needed for a job in a temp directory, run * the job, wait for it to end, write all the results. Mark the job @@ -246,20 +232,20 @@ * the end, depending of the results * @param jobPath * @return + * @throws DisworkFileSystemException * @throws IOException + * @throws DisworkException */ - protected boolean runJob(String jobPath) throws IOException { + protected boolean runJob(String jobPath) + throws IOException, + DisworkFileSystemException, + DisworkException { log.info("running job at " + jobPath); - String jsdl; - try { - String jsdlPath = jobPath + "/" + DisworkDaemon.JSDL_PATH; - jsdl = IOUtils.toString(fileSystem.read(jsdlPath)); - } catch (FileNotFoundException e) { - log.warn("job " + jobPath + " misses a job description"); - return false; - } + String jsdlPath = jobPath + "/" + DisworkDaemon.JSDL_PATH; + String jsdl = IOUtils.toString(fileSystem.read(jsdlPath)); + log.info("read jsdl " + jsdl); JobDescription jobDescription = JobDescription.parseJSDL(jsdl); @@ -287,8 +273,7 @@ log.info("will create " + application.getAbsolutePath()); OutputStream out = new FileOutputStream(application); log.debug("starting copy of " + applicationData.available() + " bytes"); - // IOUtils.copy(applicationData, out); - streamCopy(applicationData, out); + IOUtils.copy(applicationData, out); log.info("unzip application start"); // unzip application ZipUtil.uncompress(application, jobDir); @@ -298,14 +283,14 @@ } // staging input files - for (String fileName : jobDescription.getStagingInput()) { + for (String fileName : jobDescription.getInput()) { log.info("staging " + fileName); File localCopy = new File(jobDir, fileName); localCopy.createNewFile(); InputStream source = null; - if (jobDescription.getStagingInputUrls().containsKey(fileName)) { + if (jobDescription.getInputUrls().containsKey(fileName)) { // download this file from URL - URL url = jobDescription.getStagingInputUrls().get(fileName); + URL url = jobDescription.getInputUrls().get(fileName); log.info("downloading from " + url); source = url.openStream(); } else { @@ -316,8 +301,10 @@ } log.info("preparing the job"); - // prepare and run it the job - String commandLine = jobDescription.getCommandLine(); + // prepare the job and run it + String commandLine = config.parseCommandLine( + jobDescription.getCommandLine(), + jobDir.getAbsolutePath()); String[] commandLineElements = commandLine.split(" "); ProcessBuilder builder = new ProcessBuilder(commandLineElements); builder.directory(jobDir); @@ -325,7 +312,7 @@ log.info("calling " + commandLine); Process job = builder.start(); - // dump the standard output in a file + // start a thread to constantly read on the standard output String standardOutputFileName = jobDescription.getStandardOutput(); log.info("standardOutputFileName is " + standardOutputFileName); OutputStream outputFileStream = null; @@ -335,7 +322,8 @@ outputFileStream = new FileOutputStream(outputFile); } - OutputReader outputReader = new OutputReader(job.getInputStream(), outputFileStream); + OutputReader outputReader = new OutputReader(job.getInputStream(), + outputFileStream); outputReader.start(); // plugging a file on the standard input @@ -354,18 +342,15 @@ log.info("waiting for the end of the process"); exitValue = job.waitFor(); } catch (InterruptedException e) { - log.error("job " + jobDescription + " was interupted", e); + log.error("job " + jobDescription + " was interrupted", e); // FIXME 20100611 bleny job is considered has failed - exitValue = 1; + exitValue = 1; } - - - log.info("job returned " + exitValue); - + // output file staging - for (String fileName : jobDescription.getStagingOutput()) { + for (String fileName : jobDescription.getOutput()) { log.info("staging file " + fileName); File localCopy = new File(jobDir, fileName); // FIXME 20100616 bleny may not exists if job has fail @@ -397,8 +382,10 @@ /** * In a directory, list the content, sort the content, and returns * the first element. + * @throws DisworkFileSystemException */ - protected String getFistJobName(String path) throws IOException { + protected String getFistJobName(String path) + throws DisworkFileSystemException { List<String> jobsNames = fileSystem.readDirectory(path); if (jobsNames.size() == 0) { return null; @@ -408,114 +395,110 @@ } } - protected void findAJobAndRunIt() { - // try to find a new job - try { + protected void findAJobAndRunIt() throws IOException, + DisworkFileSystemException, + DisworkException { - // Once a job is found, those two var will be set - String jobLinkDir = null; - String jobLinkName = null; + // Once a job is found, those two var will be set + String jobLinkDir = null; + String jobLinkName = null; + + // use a synchronized block because multiple workers + // may try to take a same job + synchronized (fileSystem) { - // use a synchronized block because multiple workers - // may try to take a same job - synchronized (fileSystem) { - - // fist, try to find a job declared has running since - // too long to re-run it - String[] runningJobsDirs = { DisworkDaemon.FAILED_2_RUNNING, - DisworkDaemon.FAILED_1_RUNNING, - DisworkDaemon.TODO_RUNNING - }; - // browsing all "running" dirs - for (String path : runningJobsDirs) { - String oldName = getFistJobName(path); - if (oldName != null) { - Long linkAge = System.currentTimeMillis() - - Long.parseLong(oldName); - // check is oldest job is too old and should be - // considered has to-be-rerun - if (linkAge > MAX_JOB_RUNNING_TIME) { - log.info("taking old job (age = " + linkAge + ")"); - jobLinkDir = path; - jobLinkName = oldName; - // FIXME 20100617 bleny break s*cks - break; - } + // fist, try to find a job declared has running since + // too long to re-run it + String[] runningJobsDirs = { DisworkDaemon.FAILED_2_RUNNING, + DisworkDaemon.FAILED_1_RUNNING, + DisworkDaemon.TODO_RUNNING + }; + // browsing all "running" dirs + for (String path : runningJobsDirs) { + String oldName = getFistJobName(path); + if (oldName != null) { + Long linkAge = System.currentTimeMillis() + - Long.parseLong(oldName); + // check is oldest job is too old and should be + // considered has to-be-rerun + if (linkAge > MAX_JOB_RUNNING_TIME) { + log.info("taking old job (age = " + linkAge + ")"); + jobLinkDir = path; + jobLinkName = oldName; + // FIXME 20100617 bleny break s*cks + break; } } - - // if no job was found, search now in not running jobs - if (jobLinkDir == null) { - String[] jobsDirs = { DisworkDaemon.FAILED_2, - DisworkDaemon.FAILED_1, - DisworkDaemon.TODO - }; - for (String path : jobsDirs) { - String oldName = getFistJobName(path); - if (oldName != null) { // take it - jobLinkDir = path; - jobLinkName = oldName; - // FIXME 20100617 bleny break s*cks - break; - } + } + + // if no job was found, search now in not running jobs + if (jobLinkDir == null) { + String[] jobsDirs = { DisworkDaemon.FAILED_2, + DisworkDaemon.FAILED_1, + DisworkDaemon.TODO + }; + for (String path : jobsDirs) { + String oldName = getFistJobName(path); + if (oldName != null) { // take it + jobLinkDir = path; + jobLinkName = oldName; + // FIXME 20100617 bleny break s*cks + break; } } - - if (jobLinkDir != null) { - // move the link before running the job - String oldPath = jobLinkDir + "/" + jobLinkName; - log.info("job found at " + oldPath); + } + + if (jobLinkDir != null) { + // move the link before running the job + String oldPath = jobLinkDir + "/" + jobLinkName; + log.info("job found at " + oldPath); - jobLinkDir = RUNNING_MOVE.get(jobLinkDir); - jobLinkName = DisworkDaemon.newJobLinkName(); - String newPath = jobLinkDir + "/" + jobLinkName; - - log.info("moving " + oldPath + " to " + newPath); - fileSystem.move(oldPath, newPath); - - } + jobLinkDir = RUNNING_MOVE.get(jobLinkDir); + jobLinkName = DisworkDaemon.newJobLinkName(); + String newPath = jobLinkDir + "/" + jobLinkName; + + log.info("moving " + oldPath + " to " + newPath); + fileSystem.move(oldPath, newPath); + } - - // now, if no job was found, do nothing - if (jobLinkDir == null) { - log.info("nothing to do"); + } + + // now, if no job was found, do nothing + if (jobLinkDir == null) { + log.info("nothing to do"); + try { Thread.sleep(JOB_WAIT); + } catch (InterruptedException e) { + log.warn("worker interrupted while waiting", e); + // TODO 20100629 bleny ? + } // if a job was found, take it - } else { + } else { - String jobPath = jobLinkDir + "/" + jobLinkName; - - boolean jobSuccess = runJob(jobPath); - - // move the link after the job - String newDir = null; - if (jobSuccess) { - newDir = DisworkDaemon.DONE; - } else { - newDir = FAILED_MOVE.get(jobLinkDir); - } - String newPath = newDir + "/" + DisworkDaemon.newJobLinkName(); - - log.info("moving " + jobPath + " to " + newPath); - fileSystem.move(jobPath, newPath); - - // mark the job has finished if done or failed too many - // times - if ( newDir == DisworkDaemon.DONE || - newDir == DisworkDaemon.FAILED_3) { - log.info("marking " + newPath + " as finished"); - log(newPath, "FINISHED"); - } + String jobPath = jobLinkDir + "/" + jobLinkName; + + boolean jobSuccess = runJob(jobPath); + + // move the link after the job + String newDir = null; + if (jobSuccess) { + newDir = DisworkDaemon.DONE; + } else { + newDir = FAILED_MOVE.get(jobLinkDir); } + String newPath = newDir + "/" + DisworkDaemon.newJobLinkName(); + log.info("moving " + jobPath + " to " + newPath); + fileSystem.move(jobPath, newPath); - } catch (IOException e) { - log.error("error while reading jobs", e); - // TODO 20100611 bleny manage exception - } catch (InterruptedException e) { - log.info("exception catch", e); - // TODO 20100611 bleny manage exception - } + // mark the job has finished if done or failed too many + // times + if ( newDir == DisworkDaemon.DONE || + newDir == DisworkDaemon.FAILED_3) { + log.info("marking " + newPath + " as finished"); + log(newPath, "FINISHED"); + } + } } /** @@ -525,7 +508,18 @@ public void run() { while (! shouldStop) { if (activityStrategy.canWork()) { - findAJobAndRunIt(); + try { + findAJobAndRunIt(); + } catch (DisworkException e) { + log.error("worker error " + e); + throw new RuntimeException("worker error " + e); + } catch (IOException e) { + log.error("worker error " + e); + throw new RuntimeException("worker error " + e); + } catch (DisworkFileSystemException e) { + log.error("worker error " + e); + throw new RuntimeException("worker error " + e); + } } } } @@ -537,7 +531,7 @@ // initialize activityStrategy according to config String initialStrategy = config.getActivityStrategy(); - if ( "none".equals(initialStrategy)) { + if ("none".equals(initialStrategy)) { activeNoActivityStrategy(); } else if ("unlimited".equals(initialStrategy)) { activeUnlimitedActivityStrategy(); @@ -577,9 +571,8 @@ try { Thread.sleep(10 * 1000); } catch (InterruptedException e) { - // TODO 20100615 bleny Auto-generated catch block - log.info("exception catch", e); - e.printStackTrace(); + log.warn("interrupted while waiting for a worker to " + + "stop", e); } } } @@ -595,19 +588,19 @@ } public void activeNoActivityStrategy() { - activityStrategy = new ActivityStrategy.NoActivity(); + setActivityStrategy(new ActivityStrategy.NoActivity()); } public void activeUnlimitedActivityStrategy() { - activityStrategy = new ActivityStrategy.UnlimitedActivity(); + setActivityStrategy(new ActivityStrategy.UnlimitedActivity()); } public void activeLimitedActivityStrategy() { - activityStrategy = new ActivityStrategy.LimitedActivity(); + setActivityStrategy(new ActivityStrategy.LimitedActivity()); } public void activeScheduledActivityStrategy() { - activityStrategy = new ActivityStrategy.ScheduledActivity(); + setActivityStrategy(new ActivityStrategy.ScheduledActivity()); } } \ No newline at end of file Modified: trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonTest.java =================================================================== --- trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonTest.java 2010-07-01 10:38:20 UTC (rev 93) +++ trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/DisworkDaemonTest.java 2010-07-01 10:45:20 UTC (rev 94) @@ -93,9 +93,9 @@ * <li>the job come with input data (input.txt);</li> * <li>the job need another input file to be downloaded from http;</li> * <li>the job ask for a file to be provided as a result at the end - * of the job.</li> - * <li>standard output is asked has result</li> - * </ul> + * of the job;</li> + * <li>standard output is asked has results.</li> + * </ul> * */ @Test Modified: trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/JobDescriptionTest.java =================================================================== --- trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/JobDescriptionTest.java 2010-07-01 10:38:20 UTC (rev 93) +++ trunk/diswork-daemon/src/test/java/org/nuiton/diswork/daemon/JobDescriptionTest.java 2010-07-01 10:45:20 UTC (rev 94) @@ -79,16 +79,16 @@ assertEquals(job2.getStandardOutput(), job2Copy.getStandardOutput()); - assertTrue(ListUtils.isEqualList(job2.getStagingInput(), - job2Copy.getStagingInput())); - assertTrue(ListUtils.isEqualList(job2.getStagingOutput(), - job2Copy.getStagingOutput())); + assertTrue(ListUtils.isEqualList(job2.getInput(), + job2Copy.getInput())); + assertTrue(ListUtils.isEqualList(job2.getOutput(), + job2Copy.getOutput())); assertTrue(ListUtils.isEqualList( - job2.getStagingInputUrls().keySet(), - job2Copy.getStagingInputUrls().keySet())); + job2.getInputUrls().keySet(), + job2Copy.getInputUrls().keySet())); assertTrue(ListUtils.isEqualList( - job2.getStagingInputUrls().values(), - job2Copy.getStagingInputUrls().values())); + job2.getInputUrls().values(), + job2Copy.getInputUrls().values())); } catch (IOException e) { fail(); throw e; Modified: trunk/diswork-fs/pom.xml =================================================================== --- trunk/diswork-fs/pom.xml 2010-07-01 10:38:20 UTC (rev 93) +++ trunk/diswork-fs/pom.xml 2010-07-01 10:45:20 UTC (rev 94) @@ -11,7 +11,7 @@ <artifactId>diswork-fs</artifactId> <packaging>jar</packaging> - <name>disworkfs</name> + <name>Diswork File-System</name> <dependencies> <dependency> <groupId>commons-logging</groupId> @@ -42,7 +42,6 @@ <artifactId>koala-xmlstore</artifactId> </dependency> -  <dependency> <groupId>junit</groupId> Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/Demo.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/Demo.java 2010-07-01 10:38:20 UTC (rev 93) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/Demo.java 2010-07-01 10:45:20 UTC (rev 94) @@ -88,6 +88,9 @@ } else { fileSystem.createDirectory("/todo"); } + } catch (DisworkFileSystemException e) { + log.error(e); + System.out.println("error occured " + e); } catch (IOException e) { log.error(e); System.out.println("error occured " + e); @@ -145,6 +148,9 @@ } catch (IOException e) { log.error(e); System.out.println("error occured " + e); + } catch (DisworkFileSystemException e) { + log.error(e); + System.out.println("error occured " + e); } try { Thread.sleep(10 * 1000); @@ -187,7 +193,7 @@ * @param args * @throws IOException */ - public static void main(String[] args) throws IOException { + public static void main(String[] args) throws Exception { if (args.length == 2) { DisworkFileSystemConfig config = DisworkFileSystemConfig.newKademliaDisworkConfig(); Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystem.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystem.java 2010-07-01 10:38:20 UTC (rev 93) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystem.java 2010-07-01 10:45:20 UTC (rev 94) @@ -34,6 +34,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.nuiton.diswork.fs.DisworkFileSystemException.Type; import org.nuiton.diswork.fs.storage.DisworkMap; import org.nuiton.diswork.fs.storage.EntryUtil; import org.nuiton.diswork.fs.storage.Storage; @@ -59,6 +60,9 @@ private static final Log log = LogFactory.getLog(DisworkFileSystem.class); + /** the symbol to use to separate directories in a path */ + public static final String separator = EntryUtil.PATH_SEPARATOR; + /** storage will permit to save and read directories, files and links */ protected Storage storage; @@ -71,7 +75,7 @@ /** default constructor (uses default configuration parameters) * @throws IOException caused by network issue */ - public DisworkFileSystem() throws IOException { + public DisworkFileSystem() throws DisworkFileSystemException { this(new DisworkFileSystemConfig()); } @@ -81,7 +85,7 @@ * @throws IOException caused by network issue */ public DisworkFileSystem(DisworkFileSystemConfig config) - throws IOException { + throws DisworkFileSystemException { storage = new Storage(config); } @@ -93,7 +97,7 @@ * @return true is something (a link, a file, or a directory) exists at path * @throws IOException */ - public boolean exists(String path) throws IOException { + public boolean exists(String path) throws DisworkFileSystemException { checkPathSyntax(path); String entry = walk(path); boolean result = entry != null; @@ -108,13 +112,12 @@ * @throws FileNotFoundException if no file exists at this path * @throws IOException if path exists but is a directory */ - public InputStream read(String path) throws FileNotFoundException, - IOException { + public InputStream read(String path) throws DisworkFileSystemException { checkPathSyntax(path); String entry = walk(path); if (entry == null) { - throw new FileNotFoundException(path); + throw new DisworkFileSystemException(Type.NO_SUCH_FILE, path); } InputStream result = null; @@ -126,15 +129,13 @@ String newTarget = EntryUtil.resolveLink(path, link); result = read(newTarget); } else if (EntryUtil.isDirectory(entry)) { - throw new IOException("target is not a file: " + path); + throw new DisworkFileSystemException(Type.NOT_FILE, "target is not a file: " + path); } else if (EntryUtil.isFile(entry)) { log.info("reading file " + path); String id = EntryUtil.getIdFromEntry(entry); result = storage.getFile(id); } - log.info("read " + path + " returns " + result.available() + " bytes"); - return result; } @@ -146,24 +147,24 @@ * @throws ConcurrentModificationException if file is already being written */ public void write(String path, InputStream source) - throws IOException, - ConcurrentModificationException { + throws DisworkFileSystemException { checkPathSyntax(path); if (source == null) { - throw new NullPointerException("source stream is null"); + throw new DisworkFileSystemException(Type.READ_LOCAL_DATA_FAILURE, + "source stream is null"); } String parent = EntryUtil.getParentFromPath(path); String name = EntryUtil.getNameFromPath(path); - log.info("writing " + source.available() + " bytes at " + path); write(parent, name, source); } protected void write(String parent, String fileName, InputStream source) - throws IOException { + throws DisworkFileSystemException { String entryParent = walk(parent); if (entryParent == null) { - throw new IOException(parent + " directory doesn't exists"); + throw new DisworkFileSystemException(Type.NO_SUCH_DIRECTORY, parent + + " directory doesn't exists"); } if (EntryUtil.isDirectory(entryParent)) { @@ -207,8 +208,8 @@ Thread.sleep(LOCK_WAIT); } catch (InterruptedException e) { log.info("wait for lock interrupted", e); - throw new IOException - ("interrupted while trying to acquire lock", e); + throw new DisworkFileSystemException(Type.INTERRUPTION, + "interrupted while trying to acquire lock", e); } } } @@ -224,10 +225,11 @@ String newTarget = EntryUtil.resolveLink(parent, linkTarget); write(newTarget, fileName, source); } else if (EntryUtil.isFile(entryParent)) { - throw new IOException(parent + " is not a directory"); + throw new DisworkFileSystemException(Type.NOT_DIRECTORY, parent); } else { log.warn("strange entry" + entryParent); - throw new IOException("strange entry" + entryParent); + throw new DisworkFileSystemException(Type.CORRUPTED_DATA, + "strange entry" + entryParent); } } @@ -236,9 +238,7 @@ * @param path the absolute path (ending by the name) of the directory * @throws IOException if parent path is not correct */ - public void createDirectory(String path) - throws IOException, - ConcurrentModificationException { + public void createDirectory(String path) throws DisworkFileSystemException { checkPathSyntax(path); String parent = EntryUtil.getParentFromPath(path); String dirName = EntryUtil.getNameFromPath(path); @@ -246,14 +246,14 @@ } protected void createDirectory(String parent, String dirName) - throws IOException { + throws DisworkFileSystemException { log.info("trying to create directory " + dirName + " in " + parent); String entryParent = walk(parent); if (entryParent == null) { - throw new IOException(parent + " directory doesn't exists"); + throw new DisworkFileSystemException(Type.NO_SUCH_DIRECTORY, parent); } if (EntryUtil.isDirectory(entryParent)) { @@ -264,8 +264,8 @@ String findResult = EntryUtil.findEntryInDirectory (content, dirName); if (findResult != null) { - throw new IOException - (parent + " already contains an element named " + dirName); + throw new DisworkFileSystemException(Type.ALREADY_EXISTS, parent + + " already contains an element named " + dirName); } // store file before meta info @@ -291,8 +291,8 @@ Thread.sleep(LOCK_WAIT); } catch (InterruptedException e) { log.info("wait for lock interrupted", e); - throw new IOException - ("interrupted while trying to acquire lock", e); + throw new DisworkFileSystemException(Type.INTERRUPTION, + "interrupted while trying to acquire lock", e); } } } @@ -308,10 +308,11 @@ String newTarget = EntryUtil.resolveLink(parent, linkTarget); createDirectory(newTarget, dirName); } else if (EntryUtil.isFile(entryParent)) { - throw new IOException(parent + " is not a directory"); + throw new DisworkFileSystemException(Type.NOT_DIRECTORY, parent); } else { log.warn("strange entry" + entryParent); - throw new IOException("strange entry" + entryParent); + throw new DisworkFileSystemException(Type.CORRUPTED_DATA, + "strange entry" + entryParent); } } @@ -320,11 +321,10 @@ * @param path the path (ending by the name) where the link will be created * @param target the path where the link point to * (may be relative or absolute) - * @throws IOException if parent path is not correct + * @throws DisworkFileSystemException */ public void createSymbolicLink(String path, String target) - throws IOException, - ConcurrentModificationException { + throws DisworkFileSystemException { checkPathSyntax(path); String parent = EntryUtil.getParentFromPath(path); String name = EntryUtil.getNameFromPath(path); @@ -335,7 +335,7 @@ * @see #createSymbolicLink(String, String) */ protected void createSymbolicLink(String parent, String name, String target) - throws IOException { + throws DisworkFileSystemException { if (target.startsWith(EntryUtil.PATH_SEPARATOR)) { // target is absolute @@ -352,7 +352,8 @@ String entryParent = walk(parent); if (entryParent == null) { - throw new IOException(parent + " directory doesn't exists"); + throw new DisworkFileSystemException(Type.NO_SUCH_DIRECTORY, + parent); } if (EntryUtil.isDirectory(entryParent)) { @@ -364,8 +365,8 @@ String findResult = EntryUtil.findEntryInDirectory (content, name); if (findResult != null) { - throw new IOException - (parent + " already contains an element named " + name); + throw new DisworkFileSystemException(Type.ALREADY_EXISTS, + parent + " already contains an element named " + name); } // store file before meta info @@ -390,8 +391,8 @@ Thread.sleep(LOCK_WAIT); } catch (InterruptedException e) { log.info("wait for lock interrupted", e); - throw new IOException - ("interrupted while trying to acquire lock", e); + throw new DisworkFileSystemException(Type.INTERRUPTION, + "interrupted while trying to acquire lock", e); } } } @@ -408,15 +409,16 @@ String newTarget = EntryUtil.resolveLink(parent, linkTarget); createSymbolicLink(newTarget, name, target); } else if (EntryUtil.isFile(entryParent)) { - throw new IOException(parent + " is not a directory"); + throw new DisworkFileSystemException(Type.NOT_DIRECTORY, parent); } else { log.warn("strange entry" + entryParent); - throw new IOException("strange entry" + entryParent); + throw new DisworkFileSystemException(Type.CORRUPTED_DATA, + "strange entry" + entryParent); } } else { - throw new IOException(target + " is not a valid target"); + throw new DisworkFileSystemException(Type.INVALID_TARGET, target); } } @@ -424,13 +426,12 @@ * remove a file, directory, or link. Non-empty directories can't be * removed. * @param path the complete path to the entity to remove - * @throws IOException if path is incorrect or directory not empty + * @throws DisworkFileSystemException */ - public void delete(String path) throws IOException, - ConcurrentModificationException { + public void delete(String path) throws DisworkFileSystemException { checkPathSyntax(path); if (!exists(path)) { - throw new IOException(path + " doesn't exists"); + throw new DisworkFileSystemException(Type.NO_SUCH_ENTITY, path); } String parent = EntryUtil.getParentFromPath(path); String name = EntryUtil.getNameFromPath(path); @@ -441,13 +442,10 @@ /** * @see #delete(String) */ - protected void delete(String parent, String name) throws IOException { + protected void delete(String parent, String name) + throws DisworkFileSystemException { String entryParent = walk(parent); - if (entryParent == null) { - throw new IOException(parent + " directory doesn't exists"); - } - if (EntryUtil.isDirectory(entryParent)) { String parentId = EntryUtil.getIdFromEntry(entryParent); String content = storage.getDirectory(parentId); @@ -465,8 +463,8 @@ if (!innerDirectoryContent.equals( EntryUtil.EMPTY_DIRECTORY_CONTENT)) { // directory is not empty - throw new IOException - ("trying to remove a non-empty directory"); + throw new DisworkFileSystemException(Type.DIRECTORY_NOT_EMPTY, + "trying to remove a non-empty directory"); } // remove it @@ -478,7 +476,8 @@ storage.removeLink(idToRemove); } else { log.warn("strange entry" + entryParent); - throw new IOException("strange entry" + entryParent); + throw new DisworkFileSystemException(Type.CORRUPTED_DATA, + "strange entry" + entryParent); } @@ -499,8 +498,8 @@ Thread.sleep(LOCK_WAIT); } catch (InterruptedException e) { log.info("wait for lock interrupted", e); - throw new IOException - ("interrupted while trying to acquire lock", e); + throw new DisworkFileSystemException(Type.INTERRUPTION, + "interrupted while trying to acquire lock", e); } } } @@ -517,10 +516,11 @@ String newTarget = EntryUtil.resolveLink(parent, linkTarget); delete(newTarget, name); } else if (EntryUtil.isFile(entryParent)) { - throw new IOException(parent + " is not a directory"); + throw new DisworkFileSystemException(Type.NOT_DIRECTORY, parent); } else { log.warn("strange entry" + entryParent); - throw new IOException("strange entry" + entryParent); + throw new DisworkFileSystemException(Type.CORRUPTED_DATA, + "strange entry" + entryParent); } } @@ -528,15 +528,16 @@ * list the content of a directory * @param path the complete path to the directory * @return a list of the names of the elements in <code>path</code> - * @throws IOException if path doesn't point to a directory + * @throws DisworkFileSystemException */ - public List<String> readDirectory(String path) throws IOException { + public List<String> readDirectory(String path) + throws DisworkFileSystemException { checkPathSyntax(path); String entry = walk(path); List<String> result = null; if (entry == null) { - throw new IOException(path + " doesn't exists"); + throw new DisworkFileSystemException(Type.NO_SUCH_DIRECTORY, path); } else { // path may be a link, if it's the case, // entry become the actual directory @@ -558,10 +559,12 @@ } } } else if (EntryUtil.isFile(entry)) { - throw new IOException(path + " is not a directory but a file"); + throw new DisworkFileSystemException(Type.NOT_DIRECTORY, path + + " is not a directory but a file"); } else { log.warn("strange entry" + entry); - throw new IOException("strange entry" + entry); + throw new DisworkFileSystemException(Type.CORRUPTED_DATA, + "strange entry" + entry); } } @@ -575,7 +578,7 @@ * @param path * @return null if path is not valid */ - protected String walk(String path) throws IOException { + protected String walk(String path) throws DisworkFileSystemException { String result = walk(path, null, null); log.info("walking to " + path + " returns " + result); return result; @@ -592,7 +595,7 @@ * @throws IOException */ protected String walk(String path, String current, String content) - throws IOException { + throws DisworkFileSystemException { // FIXME 20105021 bleny works fine but is not understandable String result = null; @@ -693,9 +696,11 @@ * check a path is absolute and syntactically correct, throw exception if * that's not the case. */ - protected void checkPathSyntax(String path) throws IOException { + protected void checkPathSyntax(String path) + throws DisworkFileSystemException { if (!path.startsWith(EntryUtil.ROOT_DIRECTORY)) { - throw new IOException("\"" + path + "\" is not correct, all pathes " + + throw new DisworkFileSystemException(Type.INVALID_PATH, + "\"" + path + "\" is not correct, all pathes " + "have to be absolute (thus, starts with)" + EntryUtil.ROOT_DIRECTORY); } @@ -703,7 +708,8 @@ String doubleSeparator = EntryUtil.PATH_SEPARATOR + EntryUtil.PATH_SEPARATOR; if (path.contains(doubleSeparator)) { - throw new IOException("\"" + path + "\" is not correct, it contains " + throw new DisworkFileSystemException(Type.INVALID_PATH, + "\"" + path + "\" is not correct, it contains " + doubleSeparator); } } @@ -712,14 +718,16 @@ * move a file, a directory a link. May be used for renaming purpose * @param path the path to the object to move * @param destination the full path of the target - * @throws IOException + * @throws DisworkFileSystemException */ - public void move(String path, String destination) throws IOException { + public void move(String path, String destination) + throws DisworkFileSystemException { checkPathSyntax(path); checkPathSyntax(destination); if (exists(destination)) { - throw new IOException(destination + " already exists"); + throw new DisworkFileSystemException(Type.ALREADY_EXISTS, + destination + " already exists"); } String pathParent = EntryUtil.getParentFromPath(path); @@ -732,20 +740,22 @@ } protected void move(String pathParent, String pathName, - String destinationParent, String destinationName) throws IOException { + String destinationParent, String destinationName) + throws DisworkFileSystemException { String entryParent = walk(pathParent); if (entryParent == null) { - throw new IOException(pathParent + " directory doesn't exists"); + throw new DisworkFileSystemException(Type.NO_SUCH_DIRECTORY, + pathParent); } if (EntryUtil.isDirectory(entryParent)) { String destinationParentEntry = walk(destinationParent); if (destinationParentEntry == null) { - throw new IOException(destinationParentEntry + " directory" - + "doesn't exists"); + throw new DisworkFileSystemException(Type.NO_SUCH_DIRECTORY, + destinationParentEntry); } if (EntryUtil.isDirectory(destinationParentEntry)) { @@ -757,8 +767,8 @@ String oldEntry = EntryUtil.findEntryInDirectory(parentContent, pathName); if (oldEntry == null) { - throw new IOException("no element " + pathName + " in " + - pathParent); + throw new DisworkFileSystemException(Type.NO_SUCH_ENTITY, + "no element " + pathName + " in " + pathParent); } String parentId = EntryUtil.getIdFromEntry(entryParent); @@ -792,8 +802,8 @@ Thread.sleep(LOCK_WAIT); } catch (InterruptedException e) { log.info("wait for lock interrupted", e); - throw new IOException - ("interrupted while trying to acquire lock", e); + throw new DisworkFileSystemException(Type.INTERRUPTION, + "interrupted while trying to acquire lock", e); } } } @@ -810,10 +820,11 @@ String newTarget = EntryUtil.resolveLink(destinationParent, linkTarget); move(newTarget, pathName, destinationParent, destinationName); } else if (EntryUtil.isFile(destinationParentEntry)) { - throw new IOException(destinationParent + " is not a directory"); + throw new DisworkFileSystemException(Type.NOT_DIRECTORY, destinationParent); } else { log.warn("strange entry" + destinationParentEntry); - throw new IOException("strange entry" + destinationParentEntry); + throw new DisworkFileSystemException(Type.CORRUPTED_DATA, + "strange entry" + destinationParentEntry); } } else if (EntryUtil.isLink(entryParent)) { @@ -822,17 +833,17 @@ String newTarget = EntryUtil.resolveLink(pathParent, linkTarget); move(pathParent, pathName, newTarget, destinationName); } else if (EntryUtil.isFile(entryParent)) { - throw new IOException(pathParent + " is not a directory"); + throw new DisworkFileSystemException(Type.NOT_DIRECTORY, pathParent); } else { log.warn("strange entry" + entryParent); - throw new IOException("strange entry" + entryParent); + throw new DisworkFileSystemException(Type.CORRUPTED_DATA, + "strange entry" + entryParent); } } public void createDirectories(String path) - throws ConcurrentModificationException, - IOException { + throws DisworkFileSystemException { log.info("trying create directories for " + path); String pathWithoutRoot = path.substring(1, path.length()); Copied: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystemException.java (from rev 92, trunk/diswork-daemon/src/main/java/org/nuiton/diswork/daemon/DisworkFileSystemException.java) =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystemException.java (rev 0) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/DisworkFileSystemException.java 2010-07-01 10:45:20 UTC (rev 94) @@ -0,0 +1,97 @@ +/* + * #%L + * Diswork daemon + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 CodeLutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/lgpl-3.0.html>. + * #L% + */ +package org.nuiton.diswork.fs; + +/** + * + * A port to Java 1.7 should use java.nio.file.FileSystemException and + * sub-classes. + * + * @author bleny + */ +public class DisworkFileSystemException extends Exception { + + private static final long serialVersionUID = 1L; + + public static enum Type { + /** Something with this path already exists */ + ALREADY_EXISTS, + /** Error while writing because another node is writing at this place */ + CONCURRENT_MODIFICATION, + /** String do not describe a valid path */ + INVALID_PATH, + /** The directory has a content */ + DIRECTORY_NOT_EMPTY, + /** directory doesn't exists */ + NO_SUCH_DIRECTORY, + /** file doesn't exists */ + NO_SUCH_FILE, + /** nothing at a given path */ + NO_SUCH_ENTITY, + /** The path leads to something that exists but is not a directory */ + NOT_DIRECTORY, + /** The path leads to something that exists but is not a file */ + NOT_FILE, + /** The connection to the DHT failed + * this is due to a network failure or a bad configuration + */ + BOOTSTRAP_FAILURE, + /** an IO errors at low-level network occured */ + NETWORK_FAILURE, + /** a source data on the local OS is not readable */ + READ_LOCAL_DATA_FAILURE, + /** error while reading data in the map */ + READ_MAP_DATA_FAILURE, + /** a process was interrupted */ + INTERRUPTION, + /** if map contains corrupted data */ + CORRUPTED_DATA, + /** the symlink point to a target that is not valid */ + INVALID_TARGET + } + + protected Type type; + + public DisworkFileSystemException(Type type, String message, + Throwable cause) { + super(message, cause); + this.type = type; + } + + public DisworkFileSystemException(Type type, String message) { + super(message); + this.type = type; + } + + public DisworkFileSystemException(Type type, Throwable cause) { + super(cause); + this.type = type; + } + + public Type getType() { + return type; + } + +} Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/KademliaDisworkMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/KademliaDisworkMap.java 2010-07-01 10:38:20 UTC (rev 93) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/KademliaDisworkMap.java 2010-07-01 10:45:20 UTC (rev 94) @@ -38,6 +38,8 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.diswork.fs.DisworkFileSystemConfig; +import org.nuiton.diswork.fs.DisworkFileSystemException; +import org.nuiton.diswork.fs.DisworkFileSystemException.Type; import org.planx.xmlstore.routing.Identifier; import org.planx.xmlstore.routing.Kademlia; import org.planx.xmlstore.routing.RoutingException; @@ -48,28 +50,35 @@ protected Kademlia kad; - public KademliaDisworkMap(DisworkFileSystemConfig config) throws IOException { + public KademliaDisworkMap(DisworkFileSystemConfig config) + throws DisworkFileSystemException { - log.info("booting on port " + config.getUsedPort()); - kad = new Kademlia(Identifier.randomIdentifier(), config.getUsedPort()); - - if (config.getBootstrapIp() != null) { - InetSocketAddress bootstrap = new InetSocketAddress( - config.getBootstrapIp(), - config.getBootstrapPort()); - try { - log.info("trying to connect to " + bootstrap); - kad.connect(bootstrap); - } catch (RoutingException e) { - log.error("bootstrap node is unreachable", e); - throw e; + try { + log.info("booting on port " + config.getUsedPort()); + kad = new Kademlia(Identifier.randomIdentifier(), config.getUsedPort()); + + if (config.getBootstrapIp() != null) { + InetSocketAddress bootstrap = new InetSocketAddress( + config.getBootstrapIp(), + config.getBootstrapPort()); + try { + log.info("trying to connect to " + bootstrap); + kad.connect(bootstrap); + } catch (RoutingException e) { + log.error("bootstrap node is unreachable", e); + throw new DisworkFileSystemException(Type.BOOTSTRAP_FAILURE, + "bootstrap node is unreachable", e); + } } + + log.info("kademlia status : " + kad); + + //Identifier.IDSIZE = 1024; + log.info("using " + Identifier.IDSIZE + " bytes for identifiers"); + } catch (IOException e) { + throw new DisworkFileSystemException(Type.NETWORK_FAILURE, + "failure while bootstraping", e); } - - log.info("kademlia status : " + kad); - - //Identifier.IDSIZE = 1024; - log.info("using " + Identifier.IDSIZE + " bytes for identifiers"); } protected static Identifier stringToIdentifier(String s) { Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/PastryDisworkMap.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/PastryDisworkMap.java 2010-07-01 10:38:20 UTC (rev 93) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/PastryDisworkMap.java 2010-07-01 10:45:20 UTC (rev 94) @@ -37,6 +37,8 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.diswork.fs.DisworkFileSystemConfig; +import org.nuiton.diswork.fs.DisworkFileSystemException; +import org.nuiton.diswork.fs.DisworkFileSystemException.Type; import rice.Continuation; import rice.environment.Environment; @@ -118,104 +120,107 @@ protected PastryIdFactory pastryIdFactory; - public PastryDisworkMap(DisworkFileSystemConfig config) throws IOException { + public PastryDisworkMap(DisworkFileSystemConfig config) + throws DisworkFileSystemException { + + try { + Environment env = new Environment(); - Environment env = new Environment(); + // disable the UPnP setting (in case you are testing this on a NATted LAN) + // env.getParameters().setString("nat_search_policy", "never"); - // disable the UPnP setting (in case you are testing this on a NATted LAN) - // env.getParameters().setString("nat_search_policy", "never"); + // the port to use locally + int bindport = config.getUsedPort(); - // the port to use locally - int bindport = config.getUsedPort(); + InetAddress bootaddr = InetAddress.getByName(config.getBootstrapIp()); + int bootport = config.getBootstrapPort(); - InetAddress bootaddr = InetAddress.getByName(config.getBootstrapIp()); - int bootport = config.getBootstrapPort(); + InetSocketAddress bootaddress = new InetSocketAddress(bootaddr, bootport); - InetSocketAddress bootaddress = new InetSocketAddress(bootaddr, bootport); + log.info("boot address : " + bootaddress); - log.info("boot address : " + bootaddress); + NodeIdFactory nidFactory = new RandomNodeIdFactory(env); - NodeIdFactory nidFactory = new RandomNodeIdFactory(env); - - // construct the PastryNodeFactory, this is how we use rice.pastry.socket - PastryNodeFactory factory = null; + // construct the PastryNodeFactory, this is how we use rice.pastry.socket + PastryNodeFactory factory = null; - // FIXME 20100602 bleny this is a work-around to deal with time out on - // connect exceptions that occurs "sometimes" - int numberOfTry = 0; - while(factory == null && numberOfTry <= 10) { - try { - factory = new SocketPastryNodeFactory(nidFactory, bindport, env); - } catch (BindException e) { - // the bootstrap node can't be joined - throw new IOException(""/*"bootstrap node can't be joined"*/, e); - } catch (IllegalStateException e) { - // the bootstrap node can't be joined - throw new IOException - ("unable to bind to already used port " + bindport, e); - } catch (ConnectException e) { - // this occurs some times, it may work after some time... - numberOfTry += 1; + // FIXME 20100602 bleny this is a work-around to deal with time out on + // connect exceptions that occurs "sometimes" + int numberOfTry = 0; + while(factory == null && numberOfTry <= 10) { try { - Thread.sleep(10 * 1000); - } catch (InterruptedException ee) { - throw new IOException("Pastry boot process interrupted", ee); + factory = new SocketPastryNodeFactory(nidFactory, bindport, env); + } catch (BindException e) { + // the bootstrap node can't be joined + throw new IOException(""/*"bootstrap node can't be joined"*/, e); + } catch (IllegalStateException e) { + // the bootstrap node can't be joined + throw new IOException + ("unable to bind to already used port " + bindport, e); + } catch (ConnectException e) { + // this occurs some times, it may work after some time... + numberOfTry += 1; + try { + Thread.sleep(10 * 1000); + } catch (InterruptedException ee) { + throw new IOException("Pastry boot process interrupted", ee); + } } } - } - if (factory == null) { - throw new IOException("unable to connect"); - } - - // construct a node, but this does not cause it to boot - PastryNode node = factory.newNode(); - - // in later tutorials, we will register applications before calling boot - node.boot(bootaddress); - - // the node may require sending several messages to fully boot into the ring - synchronized(node) { - while(!node.isReady() && !node.joinFailed()) { - // delay so we don't busy-wait - try { - node.wait(500); - } catch (InterruptedException e) { - throw new IOException("Pastry boot process interrupted", e); + if (factory == null) { + throw new IOException("unable to connect"); + } + + // construct a node, but this does not cause it to boot + PastryNode node = factory.newNode(); + + // in later tutorials, we will register applications before calling boot + node.boot(bootaddress); + + // the node may require sending several messages to fully boot into the ring + synchronized(node) { + while(!node.isReady() && !node.joinFailed()) { + // delay so we don't busy-wait + try { + node.wait(500); + } catch (InterruptedException e) { + throw new IOException("Pastry boot process interrupted", e); + } + + // abort if can't join + if (node.joinFailed()) { + throw new IOException("Could not join the FreePastry ring. " + + "Reason : " + node.joinFailedReason()); + } } - - // abort if can't join - if (node.joinFailed()) { - throw new IOException("Could not join the FreePastry ring. " - + "Reason : " + node.joinFailedReason()); - } } + + log.info("finished creating new node " + node); + + pastryIdFactory = new PastryIdFactory(env); + + rice.persistence.Storage storage = new MemoryStorage(pastryIdFactory); + past = new PastImpl( + node, + new StorageManagerImpl( + pastryIdFactory, + storage, + // Using a cache do not permit to remove while putting + /* + new LRUCache( + new MemoryStorage(pastryIdFactory), + 512 * 1024, + node.getEnvironment() + ) + */ + new EmptyCache(pastryIdFactory) + ), + 0, // replication factor + "" + ); + } catch (IOException e) { + throw new DisworkFileSystemException(Type.NETWORK_FAILURE, e); } - - log.info("finished creating new node " + node); - - pastryIdFactory = new PastryIdFactory(env); - - rice.persistence.Storage storage = new MemoryStorage(pastryIdFactory); - past = new PastImpl( - node, - new StorageManagerImpl( - pastryIdFactory, - storage, - // Using a cache do not permit to remove while putting - /* - new LRUCache( - new MemoryStorage(pastryIdFactory), - 512 * 1024, - node.getEnvironment() - ) - */ - new EmptyCache(pastryIdFactory) - ), - 0, // replication factor - "" - ); - - } protected class ContainsKeyContinuation extends Modified: trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/Storage.java =================================================================== --- trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/Storage.java 2010-07-01 10:38:20 UTC (rev 93) +++ trunk/diswork-fs/src/main/java/org/nuiton/diswork/fs/storage/Storage.java 2010-07-01 10:45:20 UTC (rev 94) @@ -29,7 +29,6 @@ import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; -import java.util.ConcurrentModificationException; import java.util.List; import java.util.Map; @@ -37,6 +36,8 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.diswork.fs.DisworkFileSystemConfig; +import org.nuiton.diswork.fs.DisworkFileSystemException; +import org.nuiton.diswork.fs.DisworkFileSystemException.Type; /** * This class is the middle layer between the File System operations and @@ -106,8 +107,14 @@ if (blockIdsIndex < blockIds.length) { // closing previously opened stream IOUtils.closeQuietly(currentBlock); - currentBlock = new ByteArrayInputStream( - map.get(blockIds[blockIdsIndex])); + byte[] data = map.get(blockIds[blockIdsIndex]); + if (data == null) { + // maybe data has not been posted or DHT replication + // failed + log.error("a block is missing into the map"); + throw new IOException("missing block"); + } + currentBlock = new ByteArrayInputStream(data); log.debug("new current block (size = " + currentBlock.available() + ")"); } else { @@ -154,8 +161,15 @@ protected DisworkFileSystemConfig config; + /** + * + * @param config an instance of + * @param null at normal use. Will set map according to the content of + * config + * @throws IOException if network fail + */ public Storage(DisworkFileSystemConfig config, DisworkMap map) - throws IOException { + throws DisworkFileSystemException { this.config = config; @@ -191,7 +205,8 @@ } } - public Storage(DisworkFileSystemConfig config) throws IOException { + public Storage(DisworkFileSystemConfig config) + throws DisworkFileSystemException { this(config, null); } @@ -199,7 +214,7 @@ * @return the content (entries) of the root directory * @throws IOException */ - public String getRootDirectory() throws IOException { + public String getRootDirectory() throws DisworkFileSystemException { String result = getDirectory(EntryUtil.ROOT_DIRECTORY); return result; } @@ -211,11 +226,15 @@ * @return a the entries of the directory * @throws IOException */ - public String getDirectory(String id) throws IOException { + public String getDirectory(String id) throws DisworkFileSystemException { InputStream in = get(id); - String content = IOUtils.toString(in); - log.debug("getDirectory(\"" + id + "\") returns \n" + content); - return content; + try { + String content = IOUtils.toString(in); + log.debug("getDirectory(\"" + id + "\") returns \n" + content); + return content; + } catch (IOException e) { + throw new DisworkFileSystemException(Type.READ_MAP_DATA_FAILURE, e); + } } public InputStream getFile(String id) { @@ -223,22 +242,28 @@ return result; } - public String getLink(String id) throws IOException { + public String getLink(String id) { String content = EntryUtil.bytesToString(map.get(id)); - log.debug("getDirectory(\"" + id + "\") returns \"" + content + "\""); + log.debug("getLink(\"" + id + "\") returns \"" + content + "\""); return content; } - public void putDirectory(String id, String content) throws IOException { + public void putDirectory(String id, String content) + throws DisworkFileSystemException { log.debug("putDirectory(\"" + id + "\", \"" + content + "\")"); InputStream value = IOUtils.toInputStream(content); put(id, value); } - public void putFile(String id, InputStream content) throws IOException { - log.debug("putFile(\"" + id + "\", stream of "+ content.available() + - " bytes)"); - put(id, content); + public void putFile(String id, InputStream content) + throws DisworkFileSystemException { + try { + log.debug("putFile(\"" + id + "\", stream of "+ content.available() + + " bytes)"); + put(id, content); + } catch (IOException e) { + throw new DisworkFileSystemException(Type.READ_LOCAL_DATA_FAILURE, e); + } } public void putLink(String id, String content) { @@ -247,12 +272,12 @@ map.put(id, contentAsBytes); } - public void removeDirectory(String id) throws IOException { + public void removeDirectory(String id) throws DisworkFileSystemException { log.debug("removeDirectory(\"" + id + "\")"); remove(id); } - public void removeFile(String id) throws IOException { + public void removeFile(String id) throws DisworkFileSystemException { log.debug("removeFile(\"" + id + "\")"); remove(id); } @@ -277,79 +302,82 @@ * Lock is released at the end * @param key * @param value - * @throws IOException + * @throws DisworkFileSystemException */ protected void put(String key, InputStream value) - throws IOException, - ConcurrentModificationException { + throws DisworkFileSystemException { boolean lockAcquired = tryToLock(key); if (lockAcquired) { - log.info("lock on " + key + " acquired"); + try { + log.info("lock on " + key + " acquired"); - // here, we know we can write or an exception - // would have been thrown earlier - String blocksIds = ""; - int readResult = 0; - int totalSize = 0; + // here, we know we can write or an exception + // would have been thrown earlier + String blocksIds = ""; + int readResult = 0; + int totalSize = 0; - String metaBlock = totalSize + blocksIds; - String newDataKey = keyToNewDataKey(key); - map.put(newDataKey, EntryUtil.stringToBytes(metaBlock)); - - // creating a buffer of the size of a block - int bufferSize = config.getBlockSize(); - byte[] buffer = new byte[bufferSize]; - - while ((readResult = value.read(buffer)) != -1) { - totalSize += readResult; - - byte[] newBlock = buffer; - - // if the buffer is not full, truncate the block - if (readResult < buffer.length) { - newBlock = new byte[readResult]; - System.arraycopy(buffer, 0, newBlock, 0, readResult); + String metaBlock = totalSize + blocksIds; + String newDataKey = keyToNewDataKey(key); + map.put(newDataKey, EntryUtil.stringToBytes(metaBlock)); + + // creating a buffer of the size of a block + int bufferSize = config.getBlockSize(); + byte[] buffer = new byte[bufferSize]; + + while ((readResult = value.read(buffer)) != -1) { + totalSize += readResult; + + byte[] newBlock = buffer; + + // if the buffer is not full, truncate the block + if (readResult < buffer.length) { + newBlock = new byte[readResult]; + System.arraycopy(buffer, 0, newBlock, 0, readResult); + } + + String id = EntryUtil.generateId(); + blocksIds += EntryUtil.BLOCKIDS_SEPARATOR + id; + + log.debug("saving new block (size = " + newBlock.length + + ") at key " + id); + + // copy block in map + map.put(id, newBlock); + + metaBlock = totalSize + blocksIds; + + log.debug("putting metablock " + metaBlock + " at key " + + key); + + map.put(newDataKey, EntryUtil.stringToBytes(metaBlock)); + + // updating lock + updateLock(key); } - - String id = EntryUtil.generateId(); - blocksIds += EntryUtil.BLOCKIDS_SEPARATOR + id; - - log.debug("saving new block (size = " + newBlock.length + ")" - + " at key " + id); - - // copy block in map - map.put(id, newBlock); - - metaBlock = totalSize + blocksIds; - - log.debug("putting metablock " + metaBlock + " at key " + key); - map.put(newDataKey, EntryUtil.stringToBytes(metaBlock)); + eraseDependenciesOfMetablock(key); - // updating lock - updateLock(key); + if (isLockStillOwned(key)) { + map.put(key, map.get(newDataKey)); + } + + map.remove(newDataKey); + + unLock(key); + } catch (IOException e) { + throw new DisworkFileSystemException + (Type.READ_LOCAL_DATA_FAILURE, "can't read value", e); } - eraseDependenciesOfMetablock(key); - - if (isLockStillOwned(key)) { - map.put(key, map.get(newDataKey)); - } - - map.remove(newDataKey); - - unLock(key); - } else { - throw new ConcurrentModificationException("key " + key + - " is locked"); + throw new DisworkFileSystemException(Type.CONCURRENT_MODIFICATION, + "key " + key + " is locked"); } - } - protected void remove(String key) throws IOException, - ConcurrentModificationException { + protected void remove(String key) throws DisworkFileSystemException { eraseDependenciesOfMetablock(key); removeKey(key); } @@ -396,7 +424,7 @@ if (lock == null || (EntryUtil.getOwnerFromLock(lock).equals(ownerId))) { - // file is not locked, we have took the lock + // file was not locked, we have took the lock result = true; } else { // file is locked by other node, check date @@ -464,6 +492,9 @@ return lockAge > LOCK_VALID_TIME; } + /** + * a setter to inject the map dependency + */ public void setMap(DisworkMap map) { this.map = map; } Modified: trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/AbstractDisworkFileSystemTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/AbstractDisworkFileSystemTest.java 2010-07-01 10:38:20 UTC (rev 93) +++ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/AbstractDisworkFileSystemTest.java 2010-07-01 10:45:20 UTC (rev 94) @@ -9,8 +9,6 @@ import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; import java.io.InputStream; import java.util.ConcurrentModificationException; import java.util.List; @@ -21,6 +19,7 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.nuiton.diswork.fs.DisworkFileSystemException.Type; import org.nuiton.util.FileUtil; @@ -42,7 +41,7 @@ /** * The file will have this fixed size */ - static protected int randomFileSize = 45 * 1000 * 1000; + static protected int randomFileSize = 15 * 1000 * 1000; static protected DisworkFileSystem fileSystem; @@ -112,18 +111,22 @@ /** * try to read a file that as never been created nor written */ - @Test(expected = FileNotFoundException.class) + @Test public void testFailAtRead() throws Exception { - fileSystem.read("/not_existing_file"); + try { + fileSystem.read("/not_existing_file"); + fail(); + } catch (DisworkFileSystemException e) { + assertEquals(Type.NO_SUCH_FILE, e.getType()); + } } /** * tests {@link org.nuiton.diswork.fs.storage.Storage#SplitBlocksInputStream} * by storing a "-1" byte, can be buggy due to the use of read() - * @throws IOException */ @Test - public void testSplit() throws IOException { + public void testSplit() throws Exception { byte[] bytes = new byte[1]; bytes[0] = -0x1; @@ -196,12 +199,15 @@ * this use case should raise an exception because my_folder * doesn't exists */ - @Test(expected = IOException.class) + @Test public void testFailAtWrite() throws Exception { InputStream source = null; try { source = new FileInputStream(randomFilePath); fileSystem.write("/my_folder/my_file", source); + fail(); + } catch (DisworkFileSystemException e ){ + assertEquals(Type.NO_SUCH_DIRECTORY, e.getType()); } finally { IOUtils.closeQuietly(source); } @@ -303,9 +309,14 @@ * Trying to create a link to a wrong target, this sould raise an exception * @throws Exception */ - @Test(expected = IOException.class) + @Test public void testFailAtLinking() throws Exception { - fileSystem.createSymbolicLink("/my_link", "/wrong_target_path"); + try { + fileSystem.createSymbolicLink("/my_link", "/wrong_target_path"); + fail(); + } catch (DisworkFileSystemException e) { + assertEquals(Type.INVALID_TARGET, e.getType()); + } } /** @@ -335,7 +346,7 @@ * exception * @throws Exception */ - @Test(expected = IOException.class) + @Test public void testFailAtRemove() throws Exception { fileSystem.createDirectory("/my_folder"); @@ -348,7 +359,12 @@ } // trying to remove a non-empty directory should raise an exception - fileSystem.delete("/my_folder"); + try { + fileSystem.delete("/my_folder"); + fail(); + } catch (DisworkFileSystemException e) { + assertEquals(Type.DIRECTORY_NOT_EMPTY, e.getType()); + } } /** @@ -469,11 +485,15 @@ */ } - @Test(expected = IOException.class) + @Test public void failAtMove() throws Exception { fileSystem.createDirectory("/dir"); fileSystem.createDirectory("/dir2"); - fileSystem.move("/dir", "/dir2"); + try { + fileSystem.move("/dir", "/dir2"); + } catch (DisworkFileSystemException e) { + assertEquals(Type.ALREADY_EXISTS, e.getType()); + } } @Test @@ -481,7 +501,7 @@ try { fileSystem.createDirectories("/dir/subdir/subsubdir"); assertTrue(fileSystem.exists("/dir/subdir/subsubdir")); - } catch (IOException e) { + } catch (DisworkFileSystemException e) { fail(); } } Modified: trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemKademliaTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemKademliaTest.java 2010-07-01 10:38:20 UTC (rev 93) +++ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/DisworkFileSystemKademliaTest.java 2010-07-01 10:45:20 UTC (rev 94) @@ -1,21 +1,16 @@ package org.nuiton.diswork.fs; -import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import java.io.ByteArrayInputStream; import java.io.FileInputStream; import java.io.InputStream; import java.util.ConcurrentModificationException; import org.apache.commons.io.IOUtils; -import org.apache.log4j.lf5.util.StreamUtils; import org.junit.Before; import org.junit.Test; -import org.nuiton.diswork.fs.DisworkFileSystem; -import org.nuiton.diswork.fs.DisworkFileSystemConfig; public class DisworkFileSystemKademliaTest extends AbstractDisworkFileSystemTest { @@ -90,15 +85,15 @@ @Test public void testMultipleNodes3() throws Exception { - + DisworkFileSystemConfig disworkConfig = + DisworkFileSystemConfig.newKademliaDisworkConfig(bootstrapPort); + DisworkFileSystem fileSystem2 = new DisworkFileSystem(disworkConfig); + fileSystem.createDirectory("/mydir"); try { - fileSystem.createDirectory("/myseconddir"); + fileSystem2.createDirectory("/myseconddir"); } catch (ConcurrentModificationException e) { fail(); } - - } - - + } } Modified: trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/AbstractDisworkMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/AbstractDisworkMapTest.java 2010-07-01 10:38:20 UTC (rev 93) +++ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/AbstractDisworkMapTest.java 2010-07-01 10:45:20 UTC (rev 94) @@ -14,7 +14,6 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; -import org.nuiton.diswork.fs.storage.DisworkMap; public abstract class AbstractDisworkMapTest { Modified: trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/KademliaDisworkMapTest.java =================================================================== --- trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/KademliaDisworkMapTest.java 2010-07-01 10:38:20 UTC (rev 93) +++ trunk/diswork-fs/src/test/java/org/nuiton/diswork/fs/storage/KademliaDisworkMapTest.java 2010-07-01 10:45:20 UTC (rev 94) @@ -2,6 +2,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; import java.util.ArrayList; import java.util.HashSet; @@ -11,6 +12,8 @@ import org.junit.Before; import org.junit.Test; import org.nuiton.diswork.fs.DisworkFileSystemConfig; +import org.nuiton.diswork.fs.DisworkFileSystemException; +import org.nuiton.diswork.fs.DisworkFileSystemException.Type; import org.planx.xmlstore.routing.Identifier; public class KademliaDisworkMapTest extends AbstractDisworkMapTest { @@ -30,11 +33,11 @@ * this tests the key generation algorithm, it has different properties to * check : * * two call with same param should return same key - * * two call with different parameter should return differents keys + * * two call with different parameter should return different keys * * a call should never return null */ @Test - public void testStringToIndentifier() { + public void testStringToIdentifier() { List<String> paths = new ArrayList<String>(); @@ -66,18 +69,21 @@ } /** - * Tests that in exception is raised when attempting to connect using + * Tests that an exception is raised when attempting to connect using * a bad bootstrap * @throws Exception */ - /* - @Test(expected = org.planx.xmlstore.routing.RoutingException.class) + @Test public void testBadBootrap() throws Exception { DisworkFileSystemConfig config1 = DisworkFileSystemConfig.newKademliaDisworkConfig(); config1.setBootstrapIp("microsoft.com"); config1.setBootstrapPort(80); - new KademliaDisworkMap(config1); + try { + new KademliaDisworkMap(config1); + fail(); + } catch (DisworkFileSystemException e) { + assertEquals(Type.BOOTSTRAP_FAILURE, e.getType()); + } } - */ }

1 0

r93 - in trunk/src: . site site/rst site/rst/diswork-daemon site/rst/diswork-fs site/rst/user uml
by bleny＠users.nuiton.org July 1, 2010

July 1, 2010

Author: bleny Date: 2010-07-01 12:38:20 +0200 (Thu, 01 Jul 2010) New Revision: 93 Url: http://nuiton.org/repositories/revision/diswork/93 Log: maj documentation, zargo Added: trunk/src/site/rst/diswork-daemon/devel.rst trunk/src/site/rst/diswork-daemon/draft.rst trunk/src/site/rst/diswork-fs/devel.rst trunk/src/site/rst/user/how_to_use.rst trunk/src/uml/ trunk/src/uml/diswork.zargo Removed: trunk/src/site/rst/devel/ trunk/src/site/rst/diswork-daemon/how_it_works.rst trunk/src/site/rst/diswork-fs/how_it_works.rst trunk/src/site/rst/user/install.rst trunk/src/site/rst/user/starting.rst Modified: trunk/src/site/site_en.xml Copied: trunk/src/site/rst/diswork-daemon/devel.rst (from rev 92, trunk/src/site/rst/diswork-daemon/how_it_works.rst) =================================================================== --- trunk/src/site/rst/diswork-daemon/devel.rst (rev 0) +++ trunk/src/site/rst/diswork-daemon/devel.rst 2010-07-01 10:38:20 UTC (rev 93) @@ -0,0 +1,96 @@ +======================== +How Diswork Daemon works +======================== + +Abstract +-------- + +This document is for people who want to develop Diswork Daemon. + +Overview +-------- + +Diswork Daemon uses Diswork File System. + +Submit a job +------------ + +First, a node, before joining a network generate, for himself, a unique ID. And +then, create a directory "/home/ID". If this is not the first time the node +is run, ID should be reused so jobs can be found back after disconnect/connect. + +In his home directory, a node can put and remove his jobs. + +To create a job, a node should : + +#. Check that the application in the good version is available at + /bin/application/application-version.jar + +#. Create a directory in home for each of his jobs + /home/ID/jobs/job-1, /home/ID/jobs/job-2... + +#. In a job directory, put job.jsdl, a command line to run. This file contains + necessary information like : + + * The name of the job + * The application name, version + * File staging (needed file and where to get them) + * ... (to be completed) + +#. Declare the job as available by adding a symlink in /var/jobs/TODO + with the link named after the current date, the id of the job owner and + the job name. Places possible in /var/jobs/ are : + +#. Check the content of /home/ID/job-i/status + +#. Finally, clean home by deleting the directories of finished jobs + +Perform a job +------------- + +To perform a job, a node should find a job in /var/jobs/, this directory has +5 sub-directories : + +- /var/jobs/todo + +- /var/jobs/failed_1 + +- /var/jobs/failed_2 + +- /var/jobs/failed_3 + +Three more directories are available : todo_running, failed_1_running, +failed_2_running. + +#. Choose a job in one of those directories except "done" and "failed_3". + Use the data in the link same to know which job should be done first; + +#. Once the job is chosen, move the symlink to the "running" dir where you + found the link. Then, follow the link to get the actual data needed for + the job; + +#. Run the job; + +#. Now there is two way to proceed : + + - If the job successfully ran (no exception raised and return code is 0) : + + #. write the results in the job directory; + + #. mark job as finished + + - If the job has failed (a job taking too much time may be considered as + a failure). move it to according to where the job was found. If + the job was in "todo", move it to "failed_1" (read "failed once") ; + if the job was in "failed_1", move it to "failed_2", if it was in + "failed_2", move it to "failed_3". + + In both cases, write what occurred in the job directory in the status.txt + file (append your message, don't remove anything); + + +Additional step : before choosing a job, check in the all the "running" +directories if any symlink is not too old (use the name of the link). +It may means that a node crashed while processing this job. Move it to +the RUNNING directory of the next failure level ("todo" → "failed_1" → +"failed_2" → "failed_3") and try process it. \ No newline at end of file Copied: trunk/src/site/rst/diswork-daemon/draft.rst (from rev 92, trunk/src/site/rst/devel/draft.rst) =================================================================== --- trunk/src/site/rst/diswork-daemon/draft.rst (rev 0) +++ trunk/src/site/rst/diswork-daemon/draft.rst 2010-07-01 10:38:20 UTC (rev 93) @@ -0,0 +1,84 @@ +Diswork first draft (fin mars 2010) +=================================== + + +Principe +-------- +3 types de machines : annuaires (uptime très grand), stockage (beaucoup de HDD), +calculateur (beaucoup de CPU). Un noeud (une machine physique) peut être d'un ou +plusieurs type. +Les annuaires sont reliés entre eux en DHT (Distributed Hashtable). +Potentiellement 1 ou 2 serveurs existent pour indiquer les point d'entrées +dans le réseau pour pouvoir fonctionner hors d'un réseau local. +Un client envoie aux annuaires les simulations qu'il doit effectuer : version du +simulateur et data. +Quand un calculateur a du temps, il demande à l'annuaire ce qu'il doit faire. +Récupère une simu, le simulateur approprié si il ne l'a pas (P2P ?) fait tourner +la simu et dit aux stockages que les résultats sont dispo. Réplication des +résultats entre les stockages. Client demande à l'annuaire où sont les résultats +et les télécharges (P2P ?) + +Storage +~~~~~~~ +Pour les serveurs de stockage, il recherche a duplique l'information +qui ne l'est pas assez. Par exemple dans l'annuaire, un resultat est +marque comme ayant une disponibilite de 1 (une seul machine active a +l'avoir) alors un serveur de stockage au repos va essaye de recuperer +ce resultat pour augmenter ce taux de partage. + +Il faut bien penser que l'intelligence est dans les serveurs qui +veulent faire du travail, pas dans un serveur qui repartira les choses +(surtout pour le stockage on se n'est pas une approche vraiment +naturelle) + +Tracker +~~~~~~~ +Les trackers possèdent les logiciels de simulation. + +Détails +------- +- Utilisation d'algorithme efficace pour découvrir l'ensemble des noeuds +- Un noued peut se contenter de ne connaitre que ces voisins (annuaire, stockage, calculateur) +- Un peu de config pour que le calculateur puisse choisir les applis qui ont le droit de tourner +- Quand résultats sont produits, aucun calculateur ne peut plus démarrer la simu +- Il faut optimiser les simulations a faire pour que le maximum de calculateur en fasse le plus de differente +- Plusieurs calculateur peuvent effectuer la même simulation +- Les calculateurs gere eux même les simulations en erreur (telle simulation a echoué 2 fois, je ne la fait plus) +- Le client peut télécharger les résultats depuis plusieurs noeud de stockage (P2P) +- Communication : utiliser HTTP pour éviter les firewall +- Vérification d'un ckecksum sur les logiciels de simulation (et les résultats?) +- Voir comment faire entrer Caparmor dans le réseau (noeud qui délègue à Caparmor ?) +- Mettre en pause la simulation quand l'utilisateur se ressert de son poste +- Detecter que l'utilisateur ne se sert plus de son poste pour lancer les simulations (inactivité, pas de charge cpu) +- Postes windows, utilisation d'un screensaver ? (diswork en cours) +- Les noeuds savent se mettre à jour (ou détecter qu'il ne sont pas à jour) +- Bonne gestion de l'espace disque sur les noeuds de stockage +- Systèmes de statistique pour insiter à offrir du stockage, du cpu, des annuaires (bonus) + + +Contacts +-------- +- Gerson Sunye pour le DHT +- Lamarre pour le plus haut niveau (les algos de recherche/decouverte...) +- TMG pour le P2P (on va avoir les mêmes problématiques : connexion entrante sur un pair qui n'est pas un serveur) + + +Technique +--------- +- JXTA : protocole P2P +- rechercher s'il n'y a pas d'autre librairie utilisable. Par exemple +regarder dans les implantations java bittorrent s'il n'y a pas des +choses recuperables (vuze, ...) + + +Premier objectif +---------------- +Bonne structure de l'annuaire. +Etude des protocoles. +Solutionnement des problématiques de communication (firewall). + +Fonctionne sans réplication des données. +Fonctionne sans téléchargement depuis plusieurs sources. + +La version 1.0.0 doit être entierement fonctionnelle (sans contenir toutes les +fonctionnalités) \ No newline at end of file Deleted: trunk/src/site/rst/diswork-daemon/how_it_works.rst =================================================================== --- trunk/src/site/rst/diswork-daemon/how_it_works.rst 2010-06-29 11:13:09 UTC (rev 92) +++ trunk/src/site/rst/diswork-daemon/how_it_works.rst 2010-07-01 10:38:20 UTC (rev 93) @@ -1,91 +0,0 @@ -======================== -How Diswork Daemon works -======================== - -Overview --------- - -Diswork Daemon uses Diswork File System. - -Submit a job ------------- - -First, a node, before joining a network generate, for himself, a unique ID. And -then, create a directory "/home/ID". If this is not the first time the node -is run, ID should be reused so jobs can be found back after disconnect/connect. - -In his home directory, a node can put and remove his jobs. - -To create a job, a node should : - -#. Check that the application in the good version is available at - /bin/application/application-version.jar - -#. Create a directory in home for each of his jobs - /home/ID/job-1, /home/ID/job-2... - -#. In a job directory, put job.jsdl, a command line to run. This file contains - necessary information like : - - * The name of the job - * The application name, version - * File staging (needed file and where to get them) - * ... (to be completed) - -#. Declare the job as available by adding a symlink in /var/jobs/TODO - with the link named after the current date, the id of the job owner and - the job name. Places possible in /var/jobs/ are : - -#. Check the content of /home/ID/job-i/status - -#. Finally, clean home by deleting the directories of finished jobs - -Perform a job -------------- - -To perform a job, a node should find a job in /var/jobs/, this directory has -6 sub-directories : - -- /var/jobs/TODO - -- /var/jobs/FAILED_1 - -- /var/jobs/FAILED_2 - -- /var/jobs/FAILED_3 - -- /var/jobs/DONE - -TODO, FAILED_1, FAILED_2 all have a sub-directory RUNNING. - -#. Choose a job in one of those directories except DONE and FAILED_3. Use the - data in the link same to know which job should be done first; - -#. Once the job is chosen, move the symlink to the RUNNING subdir where you - found the link. Then, follow the link to get the actual datas needed for - the job; - -#. Run the job; - -#. Now there is two way to proceed : - - - If the job successfully ran (no exception raised and return code is 0) : - - #. write the results in the job directory; - - #. move the link to DONE. - - - If the job has failed (a job taking too much time may be considered as - a failure). move it to according to where the job was found. If - the job was in TODO, move it to FAILED_1 (read "failed once") ; - if the job was in FAILED_1, move it to FAILED_2, if it was it FAILED_2, - move it to FAILED_3. - - In both cases, write what occurred in the job directory in the status.txt - file (append your message, don't remove anything); - -#. Sometimes, while choosing a job, check in the RUNNING directory if any - symlink is not too old (use the name of the link). It may means that a - node crashed while processing this job. Move it to the RUNNING directory - of the next failure level (TODO → FAILED_1 → FAILED_2 → FAILED_3) and try - process it. \ No newline at end of file Copied: trunk/src/site/rst/diswork-fs/devel.rst (from rev 92, trunk/src/site/rst/diswork-fs/how_it_works.rst) =================================================================== --- trunk/src/site/rst/diswork-fs/devel.rst (rev 0) +++ trunk/src/site/rst/diswork-fs/devel.rst 2010-07-01 10:38:20 UTC (rev 93) @@ -0,0 +1,148 @@ +============================= +How Diswork File System works +============================= + +Abstract +-------- + +This document is for people who want to develop Diswork File System. It is not +an interesting reading for people who write to write code that use Diswork +FS. + +Diswork File System uses a map to store every data +-------------------------------------------------- + +Diswork File System uses a map to store data persistently. + +We illustrate the use of a file system by setting the content of "/", +the root directory as follow: + +* a directory named "my_folder" that contains: + + - a directory named "my_sub_folder" that contains: + + + a file name "my_deep_file.ext" + + + an empty folder named "my_empty_folder" + + - a file named "my_other_file.ext" + +* a file named "my_file.ext" (containing binary data: 010101) + +* a symbolic link named "my_link" which target the path + "/my_folder/my_sub_folder/my_deep_file.ext" + +Storage uses a Map to store data. + ++--------------+---------------------------------------------+ +| Key (String) | Value (byte[]) | ++==============+=============================================+ +| | "D:my_folder:ID1\n | +| | F:my_file.ext:ID2\n | +| "/" | L:my_link:ID3" | ++--------------+---------------------------------------------+ +| | "D:my_sub_folder:ID4\n | +| "ID1" | F:my_other_file.ext:ID5" | ++--------------+---------------------------------------------+ +| "ID2" | 010101 | ++--------------+---------------------------------------------+ +| "ID3" | "/my_folder/my_sub_folder/my_deep_file.ext" | ++--------------+---------------------------------------------+ +| | "F:my_deep_file:ID6\n | +| "ID4" | D:my_empty_folder:ID7" | ++--------------+---------------------------------------------+ +| "ID5" | 011010100111010... | ++--------------+---------------------------------------------+ +| "ID6" | 1100010101010110110010... | ++--------------+---------------------------------------------+ +| "ID7" | "" | ++--------------+---------------------------------------------+ + +An entry is, for example, "D:my_folder:ID1". It contains the type of the +entry (Directory, File or +Link), the name of the element, and an ID to be used as +a key on the map to get the actual content. Those three informations are +separated by ":" which is EntryUtil.ENTRY_SEPARATOR. + +A directory way have multiple entries. Entries of a directory are +separated by "\n" which is EntryUtil.ENTRIES_SEPARATOR. + +The above description shows the main principle used to store a tree +structure in a map. + +In fact, the storage of actual data is a bit more complex, due to +the need of splitting the data that may be too large (the value of "ID6" +may be a series of 100220 bytes for a 200 MiB file). +Actually, files content (like ID2, ID5, ID6) and directory content +values ("/", ID1, ID4) will be split and the map will look like that ( +note that only the files have been split for readability): + ++--------------+---------------------------------------------+ +| Key (String) | Value (byte[]) | ++==============+=============================================+ +| | "D:my_folder:ID1\n | +| | F:my_file.ext:ID2\n | +| "/" | L:my_link:ID3" | ++--------------+---------------------------------------------+ +| | "D:my_sub_folder:ID4\n | +| "ID1" | F:my_other_file.ext:ID5" | ++--------------+---------------------------------------------+ +| "ID2" | "6;ID8" | ++--------------+---------------------------------------------+ +| "ID3" | "/my_folder/my_sub_folder/my_deep_file.ext" | ++--------------+---------------------------------------------+ +| | "F:my_deep_file:ID6\n | +| "ID4" | D:my_empty_folder:ID7" | ++--------------+---------------------------------------------+ +| "ID5" | "14689;ID9" | ++--------------+---------------------------------------------+ +| "ID6" | "79874567;ID10;ID11;ID12" | ++--------------+---------------------------------------------+ +| "ID7" | "" | ++--------------+---------------------------------------------+ +| "ID8" | 010101 | ++--------------+---------------------------------------------+ +| "ID9" | 011010100111010... | ++--------------+---------------------------------------------+ +| "ID10" | 110001010101011... | ++--------------+---------------------------------------------+ +| "ID11" | 111010110101101... | ++--------------+---------------------------------------------+ +| "ID12" | 011101100111010... | ++--------------+---------------------------------------------+ + +Values of ID2, ID5, ID6 are no longer the file content. It's now a set +of meta-information information data called metablock. +A metablock is composed of the total size of the file followed by +an ordered lists of IDs of the different blocks composing the file. +Those informations are separated by ";" (see +{@link org.nuiton.disworkfs.storage.EntryUtil#BLOCKIDS_SEPARATOR}). + +In the above example: + +* "/my_file.ext" is a file which size is 6 bytes, it is + composed of one block available at ID8; + +* "/my_sub_folder/my_other_file.ext" is a file (size = 14689 bytes) + which content is available at ID9; + +* "/my_sub_folder/my_sub_folder/my_deep_file.ext" is a big file: its + size is 79874567. It has been split in three blocks: ID10, ID11 and + ID12. + +When reading and writing in storage, split is done transparently. When +reading, a Stream is returned: it loads data blocks after blocks +when needed inner class Storage.SplitBlocksInputStream}). +When writing, data are split in blocks of a maximum configurable size. + +The map is distributed +---------------------- + +The map described above is distributed over all nodes through a DHT. + +Diswork File System manage concurrency +-------------------------------------- + +DFS uses copy-on-write to deal with concurrency. Nothing is overridden, when +a file is replaced, the new file is written on the FS without removing the +old file. The old file data is kept and actually removed later. \ No newline at end of file Deleted: trunk/src/site/rst/diswork-fs/how_it_works.rst =================================================================== --- trunk/src/site/rst/diswork-fs/how_it_works.rst 2010-06-29 11:13:09 UTC (rev 92) +++ trunk/src/site/rst/diswork-fs/how_it_works.rst 2010-07-01 10:38:20 UTC (rev 93) @@ -1,136 +0,0 @@ -============================= -How Diswork File System works -============================= - -Diswork File System uses a map to store every data --------------------------------------------------- - -Diswork File System uses a map to store data persistently. - -We illustrate the use of a file system by setting the content of "/", -the root directory as follow: - -* a directory named "my_folder" that contains: - - - a directory named "my_sub_folder" that contains: - - + a file name "my_deep_file.ext" - - + an empty folder named "my_empty_folder" - - - a file named "my_other_file.ext" - -* a file named "my_file.ext" (containing binary data: 010101) - -* a symbolic link named "my_link" which target the path - "/my_folder/my_sub_folder/my_deep_file.ext" - -Storage uses a Map to store data. - -+--------------+---------------------------------------------+ -| Key (String) | Value (byte[]) | -+==============+=============================================+ -| | "D:my_folder:ID1\n | -| | F:my_file.ext:ID2\n | -| "/" | L:my_link:ID3" | -+--------------+---------------------------------------------+ -| | "D:my_sub_folder:ID4\n | -| "ID1" | F:my_other_file.ext:ID5" | -+--------------+---------------------------------------------+ -| "ID2" | 010101 | -+--------------+---------------------------------------------+ -| "ID3" | "/my_folder/my_sub_folder/my_deep_file.ext" | -+--------------+---------------------------------------------+ -| | "F:my_deep_file:ID6\n | -| "ID4" | D:my_empty_folder:ID7" | -+--------------+---------------------------------------------+ -| "ID5" | 011010100111010... | -+--------------+---------------------------------------------+ -| "ID6" | 1100010101010110110010... | -+--------------+---------------------------------------------+ -| "ID7" | "" | -+--------------+---------------------------------------------+ - -An entry is, for example, "D:my_folder:ID1". It contains the type of the -entry (Directory, File or -Link), the name of the element, and an ID to be used as -a key on the map to get the actual content. Those three informations are -separated by ":" which is EntryUtil.ENTRY_SEPARATOR. - -A directory way have multiple entries. Entries of a directory are -separated by "\n" which is EntryUtil.ENTRIES_SEPARATOR. - -The above description shows the main principle used to store a tree -structure in a map. - -In fact, the storage of actual data is a bit more complex, due to -the need of splitting the data that may be too large (the value of "ID6" -may be a series of 100220 bytes for a 200 MiB file). -Actually, files content (like ID2, ID5, ID6) and directory content -values ("/", ID1, ID4) will be split and the map will look like that ( -note that only the files have been split for readability): - -+--------------+---------------------------------------------+ -| Key (String) | Value (byte[]) | -+==============+=============================================+ -| | "D:my_folder:ID1\n | -| | F:my_file.ext:ID2\n | -| "/" | L:my_link:ID3" | -+--------------+---------------------------------------------+ -| | "D:my_sub_folder:ID4\n | -| "ID1" | F:my_other_file.ext:ID5" | -+--------------+---------------------------------------------+ -| "ID2" | "6;ID8" | -+--------------+---------------------------------------------+ -| "ID3" | "/my_folder/my_sub_folder/my_deep_file.ext" | -+--------------+---------------------------------------------+ -| | "F:my_deep_file:ID6\n | -| "ID4" | D:my_empty_folder:ID7" | -+--------------+---------------------------------------------+ -| "ID5" | "14689;ID9" | -+--------------+---------------------------------------------+ -| "ID6" | "79874567;ID10;ID11;ID12" | -+--------------+---------------------------------------------+ -| "ID7" | "" | -+--------------+---------------------------------------------+ -| "ID8" | 010101 | -+--------------+---------------------------------------------+ -| "ID9" | 011010100111010... | -+--------------+---------------------------------------------+ -| "ID10" | 110001010101011... | -+--------------+---------------------------------------------+ -| "ID11" | 111010110101101... | -+--------------+---------------------------------------------+ -| "ID12" | 011101100111010... | -+--------------+---------------------------------------------+ - -Values of ID2, ID5, ID6 are no longer the file content. It's now a set -of meta-information information data called metablock. -A metablock is composed of the total size of the file followed by -an ordered lists of IDs of the different blocks composing the file. -Those informations are separated by ";" (see -{@link org.nuiton.disworkfs.storage.EntryUtil#BLOCKIDS_SEPARATOR}). - -In the above example: - -* "/my_file.ext" is a file which size is 6 bytes, it is - composed of one block available at ID8; - -* "/my_sub_folder/my_other_file.ext" is a file (size = 14689 bytes) - which content is available at ID9; - -* "/my_sub_folder/my_sub_folder/my_deep_file.ext" is a big file: its - size is 79874567. It has been split in three blocks: ID10, ID11 and - ID12. - -When reading and writing in storage, split is done transparently. When -reading, a Stream is returned: it loads data blocks after blocks -when needed inner class Storage.SplitBlocksInputStream}). -When writing, data are split in blocks of a maximum configurable size. - -Diswork File System manage concurrency --------------------------------------- - -DFS uses copy-on-write to deal with concurrency. Nothing is overridden, when -a file is replaced, the new file is written on the FS without removing the -old file. The old file data is kept and actually removed later. \ No newline at end of file Added: trunk/src/site/rst/user/how_to_use.rst =================================================================== --- trunk/src/site/rst/user/how_to_use.rst (rev 0) +++ trunk/src/site/rst/user/how_to_use.rst 2010-07-01 10:38:20 UTC (rev 93) @@ -0,0 +1,15 @@ +================== +How to use Diswork +================== + +How to install a diswork node on my computer +-------------------------------------------- + +How to install a diswork node on my server +------------------------------------------ + +How to make my own application ready for being run by all diswork nodes +----------------------------------------------------------------------- + +How to make my application able to submit jobs to Diswork and retrieve results +------------------------------------------------------------------------------ \ No newline at end of file Deleted: trunk/src/site/rst/user/install.rst =================================================================== --- trunk/src/site/rst/user/install.rst 2010-06-29 11:13:09 UTC (rev 92) +++ trunk/src/site/rst/user/install.rst 2010-07-01 10:38:20 UTC (rev 93) @@ -1,5 +0,0 @@ -Install -======= - -TODO: client use -TODO: server use \ No newline at end of file Deleted: trunk/src/site/rst/user/starting.rst =================================================================== --- trunk/src/site/rst/user/starting.rst 2010-06-29 11:13:09 UTC (rev 92) +++ trunk/src/site/rst/user/starting.rst 2010-07-01 10:38:20 UTC (rev 93) @@ -1,4 +0,0 @@ -Starting with Diswork -===================== - -TODO : maybe put here a "hello world" application running on diswork \ No newline at end of file Modified: trunk/src/site/site_en.xml =================================================================== --- trunk/src/site/site_en.xml 2010-06-29 11:13:09 UTC (rev 92) +++ trunk/src/site/site_en.xml 2010-07-01 10:38:20 UTC (rev 93) @@ -19,21 +19,16 @@ </breadcrumbs> <menu name="Users"> - <item href="user/install.html" name="Install"/> - <item href="user/starting.html" name="Starting"/> + <item href="user/how_to_use.html" name="How to use"/> </menu> - <menu name="Developer"> - <item href="devel/draft.html" name="First draft"/> - </menu> - <menu name="diswork daemon"> - <item href="diswork-daemon/how_it_works.html" name="How it works"/> + <item href="diswork-daemon/devel.html" name="For developers"/> </menu> <menu name="diswork FS"> <item href="diswork-fs/history.html" name="History of the project"/> - <item href="diswork-fs/how_it_works.html" name="How it works"/> + <item href="diswork-fs/devel.html" name="For developers"/> </menu> <menu ref="reports"/> Added: trunk/src/uml/diswork.zargo =================================================================== (Binary files differ) Property changes on: trunk/src/uml/diswork.zargo ___________________________________________________________________ Added: svn:mime-type + application/octet-stream

1 0