October 2009 - Isis-fish-commits - mailman.cloud.codelutin.com

r2721 - isis-fish/trunk
by chatellier＠users.labs.libre-entreprise.org 30 Oct '09

30 Oct '09

Author: chatellier Date: 2009-10-30 17:12:08 +0000 (Fri, 30 Oct 2009) New Revision: 2721 Modified: isis-fish/trunk/pom.xml Log: Remove default deprecation display (unreadable) Modified: isis-fish/trunk/pom.xml =================================================================== --- isis-fish/trunk/pom.xml 2009-10-30 10:51:04 UTC (rev 2720) +++ isis-fish/trunk/pom.xml 2009-10-30 17:12:08 UTC (rev 2721) @@ -480,7 +480,8 @@ <configuration> <source>1.6</source> <target>1.6</target> - <showDeprecation>true</showDeprecation> +  </configuration> </plugin> <plugin>

1 0

r2720 - isis-fish/trunk/src/main/java/fr/ifremer/isisfish/actions
by chatellier＠users.labs.libre-entreprise.org 30 Oct '09

30 Oct '09

Author: chatellier Date: 2009-10-30 10:51:04 +0000 (Fri, 30 Oct 2009) New Revision: 2720 Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/actions/ImportAction.java isis-fish/trunk/src/main/java/fr/ifremer/isisfish/actions/OtherAction.java isis-fish/trunk/src/main/java/fr/ifremer/isisfish/actions/VCSAction.java isis-fish/trunk/src/main/java/fr/ifremer/isisfish/actions/package-info.java Log: Update actions Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/actions/ImportAction.java =================================================================== --- isis-fish/trunk/src/main/java/fr/ifremer/isisfish/actions/ImportAction.java 2009-10-30 10:26:13 UTC (rev 2719) +++ isis-fish/trunk/src/main/java/fr/ifremer/isisfish/actions/ImportAction.java 2009-10-30 10:51:04 UTC (rev 2720) @@ -36,7 +36,8 @@ import fr.ifremer.isisfish.datastore.SimulatorStorage; /** - * + * Import actions. + * * @author poussin * @version $Revision: 1862 $ * Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/actions/OtherAction.java =================================================================== --- isis-fish/trunk/src/main/java/fr/ifremer/isisfish/actions/OtherAction.java 2009-10-30 10:26:13 UTC (rev 2719) +++ isis-fish/trunk/src/main/java/fr/ifremer/isisfish/actions/OtherAction.java 2009-10-30 10:51:04 UTC (rev 2720) @@ -1,5 +1,5 @@ /* *##% - * Copyright (C) 2002-2008 Ifremer, Code Lutin, Benjamin Poussin + * Copyright (C) 2002-2009 Ifremer, Code Lutin, Benjamin Poussin * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -19,58 +19,38 @@ package fr.ifremer.isisfish.actions; -import static org.nuiton.i18n.I18n._; +import java.util.Arrays; -import java.io.File; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - import fr.ifremer.isisfish.IsisConfig; -import fr.ifremer.isisfish.IsisFish; -import fr.ifremer.isisfish.vcs.VCS; -import java.util.Arrays; /** - * + * Other actions. + * * @author poussin * @version $Revision: 1862 $ - * + * * Last update: $Date: 2009-02-23 16:42:34 +0100 (lun. 23 févr. 2009) $ * by : $Author: chatellier $ */ public class OtherAction { - /** to use log facility, just put in your code: log.info(\"...\"); */ - private static Log log = LogFactory.getLog(OtherAction.class); - protected IsisConfig config = null; - + public OtherAction(IsisConfig config) { this.config = config; } - + static public void help() { System.out.println("Usage: isis-fish [-h]"); - System.out.println("Options (set with --option <key> <value>:"); + System.out.println("Options (set with --option <key> <value>) :"); for (IsisConfig.Option o : IsisConfig.Option.values()) { - System.out.println("\t" + o.key + "(" + o.defaultValue + "):" + o.description); + System.out.println("\t" + o.key + "(" + o.defaultValue + ") : " + o.description); } System.out.println("Actions:"); for (IsisConfig.Action a : IsisConfig.Action.values()) { - System.out.println("\t" + Arrays.toString(a.aliases) + "(" + a.action + "):" + a.description); + System.out.println("\t" + Arrays.toString(a.aliases) + "(" + a.action + ") : " + a.description); } System.exit(0); } - - public void chekoutMavenFile() throws Exception { - VCS vcs = IsisFish.vcs; - File dest = config.getDatabaseDirectory(); - File file = new File(dest, "pom.xml"); - log.info(_("Checkout pom.xml to %s", file)); - vcs.update(file, false); - } - - } Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/actions/VCSAction.java =================================================================== --- isis-fish/trunk/src/main/java/fr/ifremer/isisfish/actions/VCSAction.java 2009-10-30 10:26:13 UTC (rev 2719) +++ isis-fish/trunk/src/main/java/fr/ifremer/isisfish/actions/VCSAction.java 2009-10-30 10:51:04 UTC (rev 2720) @@ -31,7 +31,8 @@ import fr.ifremer.isisfish.IsisConfig; /** - * + * VCS actions. + * * @author poussin * @version $Revision: 1862 $ * @@ -78,10 +79,11 @@ } public void vcsUpdate() throws Exception { - // ATTENTION: il est inutil de d'accepter cette option au premier + // ATTENTION: il est inutile de d'accepter cette option au premier // lancement! car on a effectue soit un checkout, soit - // une migration // !IsisContext.get().isFirstLaunch() -// Welcome.updateVCS(null); + // une migration + // !IsisContext.get().isFirstLaunch() + // Welcome.updateVCS(null); } public void vcsAdd(File ... file) throws Exception { Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/actions/package-info.java =================================================================== --- isis-fish/trunk/src/main/java/fr/ifremer/isisfish/actions/package-info.java 2009-10-30 10:26:13 UTC (rev 2719) +++ isis-fish/trunk/src/main/java/fr/ifremer/isisfish/actions/package-info.java 2009-10-30 10:51:04 UTC (rev 2720) @@ -1,5 +1,5 @@ /** - * Command line action classes implementation. + * Command line actions classes implementation. */ package fr.ifremer.isisfish.actions;

1 0

r2719 - isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher
by chatellier＠users.labs.libre-entreprise.org 30 Oct '09

30 Oct '09

Author: chatellier Date: 2009-10-30 10:26:13 +0000 (Fri, 30 Oct 2009) New Revision: 2719 Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java Log: Remove unused imports Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java =================================================================== --- isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java 2009-10-29 17:48:44 UTC (rev 2718) +++ isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java 2009-10-30 10:26:13 UTC (rev 2719) @@ -20,24 +20,18 @@ import static org.nuiton.i18n.I18n._; -import java.io.BufferedOutputStream; import java.io.File; import java.io.FileInputStream; -import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; -import java.io.OutputStream; import java.io.StringWriter; import java.io.Writer; import java.rmi.RemoteException; -import java.util.Enumeration; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.concurrent.Semaphore; -import java.util.zip.ZipEntry; -import java.util.zip.ZipFile; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils;

1 0

r2718 - in isis-fish/trunk/src: main/java/fr/ifremer/isisfish/simulator/launcher test/java/fr/ifremer/isisfish/simulator/launcher
by chatellier＠users.labs.libre-entreprise.org 29 Oct '09

29 Oct '09

Author: chatellier Date: 2009-10-29 17:48:44 +0000 (Thu, 29 Oct 2009) New Revision: 2718 Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java isis-fish/trunk/src/test/java/fr/ifremer/isisfish/simulator/launcher/SshSimulatorLauncherTest.java Log: Use uncompress method moved to nuiton-utils. Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java =================================================================== --- isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java 2009-10-29 17:10:06 UTC (rev 2717) +++ isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java 2009-10-29 17:48:44 UTC (rev 2718) @@ -45,6 +45,7 @@ import org.apache.commons.logging.LogFactory; import org.nuiton.util.FileUtil; import org.nuiton.util.MD5; +import org.nuiton.util.ZipUtil; import com.jcraft.jsch.JSch; import com.jcraft.jsch.JSchException; @@ -414,10 +415,10 @@ // FIXME big hacks here, only extract non first simulation // only for AS if (control.getId().startsWith("as_") && !control.getId().endsWith("_0")) { - uncompressFiltred(resultArchiveFile, SimulationStorage.getSimulationDirectory(), ".*/data/.*"); + ZipUtil.uncompressFiltred(resultArchiveFile, SimulationStorage.getSimulationDirectory(), ".*/data/.*"); } else { - uncompressFiltred(resultArchiveFile, SimulationStorage.getSimulationDirectory()); + ZipUtil.uncompressFiltred(resultArchiveFile, SimulationStorage.getSimulationDirectory()); } if (log.isDebugEnabled()) { @@ -1065,56 +1066,4 @@ return remotePath; } - - /** - * Unzip compressed archive and keep non exluded patterns. - * - * @param file archive file - * @param targetDir destination file - * @param excludes excludes pattern (pattern must match complete entry name including root folder) - * @throws IOException - * - * @deprecated this code may be in lutin util later... - */ - protected void uncompressFiltred(File file, File targetDir, String... excludes) throws IOException { - - ZipFile zipFile = new ZipFile(file); - - Enumeration<? extends ZipEntry> entries = zipFile.entries(); - - while (entries.hasMoreElements()) { - ZipEntry entry = entries.nextElement(); - - String name = entry.getName(); - // add continue to break loop - boolean excludeEntry = false; - if (excludes != null) { - for (String exclude : excludes) { - if (name.matches(exclude)) { - excludeEntry = true; - } - } - } - - if (!excludeEntry) { - File target = new File(targetDir, name); - if (entry.isDirectory()) { - target.mkdirs(); - } else { - // get inputstream only here - InputStream in = zipFile.getInputStream(entry); - target.getParentFile().mkdirs(); - OutputStream out = new BufferedOutputStream(new FileOutputStream(target)); - byte[] buffer = new byte[8 * 1024]; - int len; - - while ((len = in.read(buffer, 0, 8 * 1024)) != -1) { - out.write(buffer, 0, len); - } - out.close(); - in.close(); - } - } - } - } } Modified: isis-fish/trunk/src/test/java/fr/ifremer/isisfish/simulator/launcher/SshSimulatorLauncherTest.java =================================================================== --- isis-fish/trunk/src/test/java/fr/ifremer/isisfish/simulator/launcher/SshSimulatorLauncherTest.java 2009-10-29 17:10:06 UTC (rev 2717) +++ isis-fish/trunk/src/test/java/fr/ifremer/isisfish/simulator/launcher/SshSimulatorLauncherTest.java 2009-10-29 17:48:44 UTC (rev 2718) @@ -183,49 +183,4 @@ Assert.assertTrue("Invalid qsub script is used", content .indexOf("!/bin/csh") > 0); } - - /** - * Test la fonction de decompression sans filtres. - * - * @throws IOException - */ - @Test - public void testPartialUncompress() throws IOException { - File archive = new File("src" + File.separator + "test" - + File.separator + "resources" + File.separator + "simulations" - + File.separator + "test-nonregression-20090203.zip"); - File dest = new File(getCurrentDatabaseDirectory(), "testunzip"); - - if (log.isInfoEnabled()) { - log.info("Extracting " + archive + " to " + dest); - } - - SSHSimulatorLauncher launcher = new SSHSimulatorLauncher(); - launcher.uncompressFiltred(archive, dest); - File ruleFile = new File(dest, "test-nonregression-20090203" + File.separator + "scripts" + File.separator + "RuleUtil.java"); - Assert.assertTrue(ruleFile.exists()); - } - - /** - * Test la fonction de decompression avec filtres. - * - * @throws IOException - */ - @Test - public void testPartialUncompressFiltred() throws IOException { - File archive = new File("src" + File.separator + "test" - + File.separator + "resources" + File.separator + "simulations" - + File.separator + "test-nonregression-20090203.zip"); - File dest = new File(getCurrentDatabaseDirectory(), "testunzip2"); - String pattern = ".*/scripts/.*"; - - if (log.isInfoEnabled()) { - log.info("Extracting " + archive + " to " + dest + " without " + pattern); - } - - SSHSimulatorLauncher launcher = new SSHSimulatorLauncher(); - launcher.uncompressFiltred(archive, dest, pattern); - File ruleFile = new File(dest, "test-nonregression-20090203" + File.separator + "scripts" + File.separator + "RuleUtil.java"); - Assert.assertFalse(ruleFile.exists()); - } } \ No newline at end of file

1 0

r2717 - isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher
by chatellier＠users.labs.libre-entreprise.org 29 Oct '09

29 Oct '09

Author: chatellier Date: 2009-10-29 17:10:06 +0000 (Thu, 29 Oct 2009) New Revision: 2717 Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java Log: Fix one path name Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java =================================================================== --- isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java 2009-10-29 16:47:37 UTC (rev 2716) +++ isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java 2009-10-29 17:10:06 UTC (rev 2717) @@ -706,7 +706,7 @@ } // always rename simulation to "simulation-$id-preparation.zip" - remotePath += "simulation-" + simulationid + "preparation.zip"; + remotePath += "simulation-" + simulationid + "-preparation.zip"; SSHUtils.scpTo(session, simulationFile, remotePath);

1 0

r2716 - in isis-fish/trunk: . src/main/java/fr/ifremer/isisfish/actions src/main/java/fr/ifremer/isisfish/simulator/launcher src/main/resources/templates/ssh
by chatellier＠users.labs.libre-entreprise.org 29 Oct '09

29 Oct '09

Author: chatellier Date: 2009-10-29 16:47:37 +0000 (Thu, 29 Oct 2009) New Revision: 2716 Modified: isis-fish/trunk/changelog.txt isis-fish/trunk/src/main/java/fr/ifremer/isisfish/actions/SimulationAction.java isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java isis-fish/trunk/src/main/resources/templates/ssh/isis-launch-with-qsub.seq Log: Rename all uploaded file on caparmor to common non random format : - $ISIS-TMP/simulation-$id-preparation.zip - $ISIS-TMP/simulation-$id-prescript.bsh - $ISIS-TMP/simulation-$id-script.seq - $ISIS-TMP/simulation-$id-result.zip - $ISIS-TMP/simulation-$id-result.zip.md5 - $ISIS-TMP/simulation-$id-output.txt - $ISIS-TMP/simulation-$id-pbs.id Modified: isis-fish/trunk/changelog.txt =================================================================== --- isis-fish/trunk/changelog.txt 2009-10-29 14:03:01 UTC (rev 2715) +++ isis-fish/trunk/changelog.txt 2009-10-29 16:47:37 UTC (rev 2716) @@ -1,5 +1,7 @@ isis-fish (3.3.0.0) stable; urgency=low + * Add missing log category in simulation filter appender + * Add -m n option to qsub to not send email for each simulation * Remove duplicated SSH key configuration, common configuration for VCS and caparmor * Global VCS improvement (code, script UI, input UI) * Update to svnkit 1.3.1 Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/actions/SimulationAction.java =================================================================== --- isis-fish/trunk/src/main/java/fr/ifremer/isisfish/actions/SimulationAction.java 2009-10-29 14:03:01 UTC (rev 2715) +++ isis-fish/trunk/src/main/java/fr/ifremer/isisfish/actions/SimulationAction.java 2009-10-29 16:47:37 UTC (rev 2716) @@ -229,6 +229,7 @@ log.debug("Timing : before zipping results : " + new java.util.Date()); } } + // md5 file name in complete simulationStorage.getFile() + ".md5" ZipUtil.compress(simulationResultArchive, simulationStorage.getFile(), null, true); if (log.isDebugEnabled()) { log.debug("Timing : after zipping results : " + new java.util.Date()); Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java =================================================================== --- isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java 2009-10-29 14:03:01 UTC (rev 2715) +++ isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java 2009-10-29 16:47:37 UTC (rev 2716) @@ -71,6 +71,17 @@ * * Isis-Fish must be installed on remote server. * + * Caparmor file layout ($i = plan/as increment) : + * - $ISIS-TMP/simulation-$id-preparation.zip + * - $ISIS-TMP/simulation-$id-prescript.bsh + * - $ISIS-TMP/simulation-$id-script.seq + * - $ISIS-TMP/simulation-$id-result.zip + * - $ISIS-TMP/simulation-$id-result.zip.md5 + * - $ISIS-TMP/simulation-$id-output.txt + * - $ISIS-TMP/simulation-$id-pbs.id + * + * TODO revome all $ISIS-TMP/simulation-$id* after simulation end ? + * * @see JSch * * @author chatellier @@ -95,7 +106,7 @@ * connection. */ protected static Session sshSession; - + /** * Opened session to sftp service. Stored in static context to not reask passphrase at each * connection. @@ -218,7 +229,7 @@ */ @Override public void simulationStopRequest(SimulationJob job) throws RemoteException { - + // make sure user is connected try { getSSHSession(); @@ -226,14 +237,13 @@ catch(JSchException e) { throw new RemoteException("Can't connect", e); } - + try { sendStopSimulationRequest(sshSession, job.getId()); } catch (SSHException e) { throw new RemoteException("Can't connect", e); } - - + simulationEnded(); } @@ -254,7 +264,7 @@ if (StringUtils.isBlank(IsisFish.config.getSimulatorSshUsername())) { throw new RemoteException("Username is empty"); } - + // ask for available resources on caparmor // blocking until available message(control, _("isisfish.simulation.remote.message.waitingavailable")); @@ -262,6 +272,9 @@ // start ssh session try { + + String simulationid = control.getId(); + // connection message(control, _("isisfish.simulation.remote.message.connection")); Session sshSession = getSSHSession(); @@ -270,22 +283,22 @@ // upload simulation on server message(control, _("isisfish.simulation.remote.message.upload")); String simulationRemotePath = uploadSimulation(sshSession, - simulationZip); + simulationid, simulationZip); - String remoteResultZip = getRemoteResultArchivePath(control.getId()); - + String remoteResultZip = getRemoteResultArchivePath(simulationid); + // build du contenu du script message(control,_("isisfish.simulation.remote.message.waitingstart")); String simulationPreScriptPath = uploadPreScriptIfNecessary( sshSession, control.getId(), simulationPrescript); - File simulationScript = getLaunchSimulationScriptFile(control.getId(), + File simulationScript = getLaunchSimulationScriptFile(simulationid, simulationRemotePath, remoteResultZip, simulationPreScriptPath); String scriptRemotePath = uploadSimulationScript(sshSession, - simulationScript); - // prescrit uploaded, delete + simulationid, simulationScript); + // prescript uploaded, delete simulationScript.delete(); - sendStartSimulationRequest(sshSession, scriptRemotePath, control.getId()); + sendStartSimulationRequest(sshSession, scriptRemotePath, simulationid); } catch (Exception e) { if (log.isErrorEnabled()) { @@ -324,7 +337,7 @@ @Override public void updateControl(SimulationService simulationService, SimulationControl control) throws RemoteException { - + // make sure user is connected try { getSSHSession(); @@ -334,7 +347,7 @@ } try { - + // CONTROL file try { // download control file @@ -357,7 +370,7 @@ // deleteTempFile controlFile.delete(); } catch (SSHException e) { - + // file doesn't exist if (log.isDebugEnabled()) { // not add ,e plz :) @@ -369,7 +382,7 @@ // can be because, simulation has not begun // or simulation is ended // try do download md5 control file - + // MD5 + SIMULATION zip file try { File md5ControlFile = downloadResultsMD5File(sshSession, control.getId()); @@ -384,7 +397,7 @@ } File resultArchiveFile = downloadResultsArchive(sshSession, control.getId(), md5sum); - + if (resultArchiveFile != null) { // FIXME done here to not break upload during import @@ -406,7 +419,7 @@ else { uncompressFiltred(resultArchiveFile, SimulationStorage.getSimulationDirectory()); } - + if (log.isDebugEnabled()) { log.debug("Simulation imported : " + resultArchiveFile.getAbsolutePath()); } @@ -631,7 +644,6 @@ try { // use usefull readLines from commons-io - @SuppressWarnings("unchecked") List<String> contents = FileUtils.readLines(publicKey); // only one line @@ -664,16 +676,15 @@ * Upload simulation file to server. * * @param session already open valid ssh session + * @param simulationid simulation id * @param simulationFile simulation file to upload * * @return remote file path or <tt>null</tt> if errors * @throws SSHException if upload fail */ - protected String uploadSimulation(Session session, File simulationFile) + protected String uploadSimulation(Session session, String simulationid, File simulationFile) throws SSHException { - String localPath = simulationFile.getAbsolutePath(); - // first check that remote directory exists String remotePath = getRemoteTempDirectory(); @@ -694,19 +705,9 @@ throw new SSHException(_("Command '%s' fail to execute", command)); } - // upload directory in that dir - if (!remotePath.endsWith("/")) { - remotePath += "/"; - } + // always rename simulation to "simulation-$id-preparation.zip" + remotePath += "simulation-" + simulationid + "preparation.zip"; - if (localPath.lastIndexOf("/") > 0) { - remotePath += localPath.substring(localPath.lastIndexOf("/") + 1); - } else if (localPath.lastIndexOf("\\") > 0) { // windows - remotePath += localPath.substring(localPath.lastIndexOf("\\") + 1); - } else { - remotePath += localPath; - } - SSHUtils.scpTo(session, simulationFile, remotePath); return remotePath; @@ -754,7 +755,7 @@ localFile = null; } } - + return localFile; } @@ -776,7 +777,6 @@ // build remote file path // FIXME this path should be given by remote IsisFish app - // TODO to change String remoteFile = IsisFish.config.getSimulatorSshDataPath(); remoteFile += "/" + SimulationStorage.SIMULATION_PATH; remoteFile += "/" + simulationId; @@ -794,7 +794,7 @@ return localFile; } - + /** * Download remote simulation md5 control file and store its content into temp * file. @@ -809,6 +809,9 @@ throws IOException, SSHException { // build remote file path + // this path is not configurable + // same as fr.ifremer.isisfish.actions.SimulationAction.simulateRemotellyWithPreScript(String, File, File, File) + // defined by org.nuiton.util.ZipUtil.compressFiles(File, File, Collection<File>, boolean) String remoteFile = getRemoteResultArchivePath(simulationId) + ".md5"; // local tmp file @@ -833,22 +836,14 @@ * @throws SSHException if upload fail */ protected String uploadSimulationScript(Session session, - File simulationScript) throws SSHException { + String simulationid, File simulationScript) throws SSHException { - String localPath = simulationScript.getAbsolutePath(); - - String remotePath = getRemoteTempDirectory(); - // remote temp directory should have been created // by #uploadSimulation(Session, String) + String remotePath = getRemoteTempDirectory(); - if (localPath.lastIndexOf("/") > 0) { - remotePath += localPath.substring(localPath.lastIndexOf("/") + 1); - } else if (localPath.lastIndexOf("\\") > 0) { // windows - remotePath += localPath.substring(localPath.lastIndexOf("\\") + 1); - } else { - remotePath += localPath; - } + // always rename uploaded script to "simulation-$id-script.seq" + remotePath += "simulation-" + simulationid + "-script.seq"; SSHUtils.scpTo(session, simulationScript, remotePath); @@ -861,14 +856,11 @@ * @param simulationId simulation id */ protected String getRemoteResultArchivePath(String simulationId) { - String remotePath = getRemoteTempDirectory(); - remotePath += "simulation-" + simulationId + "-result.zip"; - return remotePath; } - + /** * Upload pre script on remote server. * @@ -890,24 +882,17 @@ return null; } - File tempPreScriptFile = File.createTempFile("isis-" + simulationId - + "-", ".prescript"); + File tempPreScriptFile = File.createTempFile("simulation-" + simulationId + + "-prescript", ".bsh"); tempPreScriptFile.deleteOnExit(); FileUtil.writeString(tempPreScriptFile, simulationPreScript); - String localPath = tempPreScriptFile.getAbsolutePath(); - String remotePath = getRemoteTempDirectory(); - // remote temp directory should have been created // by #uploadSimulation(Session, String) + String remotePath = getRemoteTempDirectory(); - if (localPath.lastIndexOf("/") > 0) { - remotePath += localPath.substring(localPath.lastIndexOf("/") + 1); - } else if (localPath.lastIndexOf("\\") > 0) { // windows - remotePath += localPath.substring(localPath.lastIndexOf("\\") + 1); - } else { - remotePath += localPath; - } + // always rename simulation prescript to "simulation-$id-prescript.bsh" + remotePath += "simulation-" + simulationId + "-prescript.bsh"; SSHUtils.scpTo(session, tempPreScriptFile, remotePath); @@ -931,7 +916,7 @@ protected File getLaunchSimulationScriptFile(String simuationId, String simulationZip, String simulationResultZip, String preScriptPath) throws IOException { - File tempScript = File.createTempFile("isis-launch-" + simuationId + "-", ".seq"); + File tempScript = File.createTempFile("simulation-" + simuationId + "-script", ".seq"); tempScript.deleteOnExit(); // auto delete String fileContent = getSimulationScriptLaunchContent( @@ -961,9 +946,9 @@ String scriptContent = null; // test null values for prescript - String localPreScript = preScriptPath; - if (localPreScript == null) { - localPreScript = ""; + String remotePreScript = preScriptPath; + if (remotePreScript == null) { + remotePreScript = ""; } try { @@ -978,7 +963,7 @@ root.put("simulationid", simuationId); root.put("simulationzip", simulationZip); root.put("simulationresultzip", simulationZipResult); - root.put("simulationprescript", localPreScript); + root.put("simulationprescript", remotePreScript); // process template Writer out = new StringWriter(); @@ -1012,8 +997,9 @@ // command to : // - add script in qsub queue // - use option -m n (to not send email) + String remoteFilenameId = getRemoteTempDirectory() + "simulation-" + simulationId + "-pbs.id"; String command = IsisFish.config.getSimulatorSshPbsBinPath() - + "/qsub -m n \"" + scriptRemotePath + "\"|tee \"" + getRemoteTempDirectory() + "/" + simulationId + ".id\""; + + "/qsub -m n \"" + scriptRemotePath + "\"|tee \"" + remoteFilenameId + "\""; Writer output = new StringWriter(); int exit = SSHUtils.exec(session, command, output); @@ -1021,13 +1007,13 @@ if (exit != 0) { throw new SSHException(_("Command '%s' fail to execute", command)); } - + String out = output.toString(); if (out.trim().matches("\\d+\\.\\w+")) { log.info("Job submitted with job id : " + out); } } - + /** * Send qdel request on job. * @@ -1040,16 +1026,16 @@ throws SSHException { // command to : - String command = IsisFish.config.getSimulatorSshPbsBinPath() - + "/qdel `cat \"" + getRemoteTempDirectory() + "/" + simulationId + ".id\"`"; - + String remoteFilenameId = getRemoteTempDirectory() + "simulation-" + simulationId + "-pbs.id"; + String command = IsisFish.config.getSimulatorSshPbsBinPath() + "/qdel `cat \"" + remoteFilenameId + "\"`"; + // and delete simulation //command += " && rm -rf \"" + IsisFish.config.getSimulatorSshDataPath() + "/simulations/" + simulationId + "\""; - + if (log.isDebugEnabled()) { log.debug("Send stop request : " + command); } - + SSHUtils.exec(session, command); // can fail, already stopped @@ -1057,7 +1043,7 @@ throw new SSHException(_("Command '%s' fail to execute", command)); }*/ } - + /** * Get remote directory absolute path. * @@ -1067,7 +1053,7 @@ */ protected String getRemoteTempDirectory() { String remotePath = IsisFish.config.getSimulatorSshTmpPath(); - + if (!remotePath.startsWith("/")) { remotePath = IsisFish.config.getSimulatorSshUserHome() + "/" + remotePath; } @@ -1076,7 +1062,7 @@ if (!remotePath.endsWith("/")) { remotePath += "/"; } - + return remotePath; } @@ -1091,14 +1077,14 @@ * @deprecated this code may be in lutin util later... */ protected void uncompressFiltred(File file, File targetDir, String... excludes) throws IOException { - + ZipFile zipFile = new ZipFile(file); - + Enumeration<? extends ZipEntry> entries = zipFile.entries(); - + while (entries.hasMoreElements()) { ZipEntry entry = entries.nextElement(); - + String name = entry.getName(); // add continue to break loop boolean excludeEntry = false; @@ -1109,7 +1095,7 @@ } } } - + if (!excludeEntry) { File target = new File(targetDir, name); if (entry.isDirectory()) { Modified: isis-fish/trunk/src/main/resources/templates/ssh/isis-launch-with-qsub.seq =================================================================== --- isis-fish/trunk/src/main/resources/templates/ssh/isis-launch-with-qsub.seq 2009-10-29 14:03:01 UTC (rev 2715) +++ isis-fish/trunk/src/main/resources/templates/ssh/isis-launch-with-qsub.seq 2009-10-29 16:47:37 UTC (rev 2716) @@ -6,7 +6,7 @@ #module load java/1.6.0 <#if simulationprescript?length > 0> -/home3/caparmor/poussin/jdk/bin/java -Xmx2500M -jar isis-fish*.jar --option launch.ui false --simulateRemotellyWithPreScript "${simulationid}" "${simulationzip}" "${simulationresultzip}" "${simulationprescript}" >& "${isistemp}output-${simulationid}" +/home3/caparmor/poussin/jdk/bin/java -Xmx2500M -jar isis-fish*.jar --option launch.ui false --simulateRemotellyWithPreScript "${simulationid}" "${simulationzip}" "${simulationresultzip}" "${simulationprescript}" >& "${isistemp}simulation-${simulationid}-output.txt" <#else> -/home3/caparmor/poussin/jdk/bin/java -jar isis-fish*.jar --option launch.ui false --simulateRemotelly "${simulationid}" "${simulationzip}" "${simulationresultzip}" >& "${isistemp}output-${simulationid}" +/home3/caparmor/poussin/jdk/bin/java -jar isis-fish*.jar --option launch.ui false --simulateRemotelly "${simulationid}" "${simulationzip}" "${simulationresultzip}" >& "${isistemp}simulation-${simulationid}-output.txt" </#if>

1 0

r2715 - isis-fish/trunk/src/main/resources/templates/ssh
by chatellier＠users.labs.libre-entreprise.org 29 Oct '09

29 Oct '09

Author: chatellier Date: 2009-10-29 14:03:01 +0000 (Thu, 29 Oct 2009) New Revision: 2715 Modified: isis-fish/trunk/src/main/resources/templates/ssh/isis-launch-with-qsub.seq Log: Remove loading of java/1.6.0 Modified: isis-fish/trunk/src/main/resources/templates/ssh/isis-launch-with-qsub.seq =================================================================== --- isis-fish/trunk/src/main/resources/templates/ssh/isis-launch-with-qsub.seq 2009-10-29 14:00:30 UTC (rev 2714) +++ isis-fish/trunk/src/main/resources/templates/ssh/isis-launch-with-qsub.seq 2009-10-29 14:03:01 UTC (rev 2715) @@ -1,10 +1,10 @@ #!/bin/csh - cd "${isishome}" -source /usr/share/modules/init/csh -module load java/1.6.0 +#source /usr/share/modules/init/csh +#module load java/1.6.0 + <#if simulationprescript?length > 0> /home3/caparmor/poussin/jdk/bin/java -Xmx2500M -jar isis-fish*.jar --option launch.ui false --simulateRemotellyWithPreScript "${simulationid}" "${simulationzip}" "${simulationresultzip}" "${simulationprescript}" >& "${isistemp}output-${simulationid}" <#else>

1 0

r2714 - isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher
by chatellier＠users.labs.libre-entreprise.org 29 Oct '09

29 Oct '09

Author: chatellier Date: 2009-10-29 14:00:30 +0000 (Thu, 29 Oct 2009) New Revision: 2714 Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java Log: Add -m n option to qsub to not send email for each simulation Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java =================================================================== --- isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java 2009-10-29 13:57:48 UTC (rev 2713) +++ isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java 2009-10-29 14:00:30 UTC (rev 2714) @@ -1011,8 +1011,9 @@ // command to : // - add script in qsub queue + // - use option -m n (to not send email) String command = IsisFish.config.getSimulatorSshPbsBinPath() - + "/qsub \"" + scriptRemotePath + "\"|tee \"" + getRemoteTempDirectory() + "/" + simulationId + ".id\""; + + "/qsub -m n \"" + scriptRemotePath + "\"|tee \"" + getRemoteTempDirectory() + "/" + simulationId + ".id\""; Writer output = new StringWriter(); int exit = SSHUtils.exec(session, command, output);

1 0

r2713 - isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher
by chatellier＠users.labs.libre-entreprise.org 29 Oct '09

29 Oct '09

Author: chatellier Date: 2009-10-29 13:57:48 +0000 (Thu, 29 Oct 2009) New Revision: 2713 Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java Log: Make semaphore deprecated. Prepare sftp use. Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java =================================================================== --- isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java 2009-10-29 13:40:45 UTC (rev 2712) +++ isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java 2009-10-29 13:57:48 UTC (rev 2713) @@ -91,13 +91,21 @@ //protected static final String QSUB_SCRIPT_TEMPLATE = "templates/ssh/isis-launch-with-cron.seq"; /** - * Opened session. Stored in static context to not reask passphrase at each + * Opened session to ssh service. Stored in static context to not reask passphrase at each * connection. */ protected static Session sshSession; + + /** + * Opened session to sftp service. Stored in static context to not reask passphrase at each + * connection. + */ + protected static Session sshSftpSession; /** * Semaphore used to upload only 20 (default) simulations at a time. + * + * @deprecated this just a hack to not crash PBS */ protected static Semaphore semaphore = new Semaphore(IsisFish.config .getSimulatorSshMaxSimultaneousSimulation()); @@ -126,8 +134,6 @@ TemplateLoader templateLoader = new ClasspathTemplateLoader(); freemarkerConfiguration.setTemplateLoader(templateLoader); - //freemarkerConfiguration.setObjectWrapper(new BeansWrapper()); - } /** @@ -258,7 +264,8 @@ try { // connection message(control, _("isisfish.simulation.remote.message.connection")); - Session sshSession = getSSHSession(); //openSSHSession(); + Session sshSession = getSSHSession(); + //Session sshSftpSession = getSSHSftpSession(); // upload simulation on server message(control, _("isisfish.simulation.remote.message.upload")); @@ -496,6 +503,23 @@ } /** + * Get opened ssh session or try to open a new one. + * + * This method must synchronized. + * + * @return opened ssh session. + * @throws JSchException + */ + protected synchronized Session getSSHSftpSession() throws JSchException { + + if (sshSftpSession == null || !sshSftpSession.isConnected()) { + sshSftpSession = openSSHSession(); + } + + return sshSftpSession; + } + + /** * Connect to remote server throw SSH, and return session. * * @return valid opened session

1 0

r2712 - isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator
by chatellier＠users.labs.libre-entreprise.org 29 Oct '09

29 Oct '09

Author: chatellier Date: 2009-10-29 13:40:45 +0000 (Thu, 29 Oct 2009) New Revision: 2712 Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/SimulationControl.java Log: Format Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/SimulationControl.java =================================================================== --- isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/SimulationControl.java 2009-10-29 13:03:55 UTC (rev 2711) +++ isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/SimulationControl.java 2009-10-29 13:40:45 UTC (rev 2712) @@ -157,9 +157,8 @@ */ public void stopSimulation() { setRunning(false); - } + } - /** * @return Returns the text. */ @@ -177,7 +176,7 @@ String oldValue = this.text; this.text = text; listeners.firePropertyChange("text", oldValue, this.text); - } + } /** * @return Returns the progressMax.

1 0