Author: chatellier Date: 2009-04-29 13:06:25 +0000 (Wed, 29 Apr 2009) New Revision: 2151 Added: isis-fish/trunk/src/main/resources/templates/ssh/isis-launch-with-cron.seq isis-fish/trunk/src/main/resources/templates/ssh/isis-launch-with-qsub.seq Removed: isis-fish/trunk/src/main/resources/templates/ssh/launch-isis-cron.seq isis-fish/trunk/src/main/resources/templates/ssh/launch-isis-qsub.seq Modified: isis-fish/trunk/pom.xml isis-fish/trunk/src/main/java/fr/ifremer/isisfish/IsisConfig.java isis-fish/trunk/src/main/java/fr/ifremer/isisfish/IsisFish.java isis-fish/trunk/src/main/java/fr/ifremer/isisfish/actions/SimulationAction.java isis-fish/trunk/src/main/java/fr/ifremer/isisfish/datastore/DataStorage.java isis-fish/trunk/src/main/java/fr/ifremer/isisfish/datastore/VersionStorage.java isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SimulationMonitor.java isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/ssh/SSHUtils.java isis-fish/trunk/src/test/java/fr/ifremer/isisfish/simulator/launcher/SshSimulatorLauncherTest.java Log: New simulation result donwload implementation. Make zip on server, and import zip on client. Modified: isis-fish/trunk/pom.xml =================================================================== --- isis-fish/trunk/pom.xml 2009-04-28 15:17:38 UTC (rev 2150) +++ isis-fish/trunk/pom.xml 2009-04-29 13:06:25 UTC (rev 2151) @@ -342,18 +342,17 @@ <topia.version>2.1.4</topia.version> <topia.service.version>1.0.2</topia.service.version> <lutinmatrix.version>1.3</lutinmatrix.version> - <lutinutil.version>1.0.4</lutinutil.version> + <lutinutil.version>1.0.5-SNAPSHOT</lutinutil.version> <i18n.version>0.9</i18n.version> <lutinwidget.version>0.14</lutinwidget.version> <generator.version>0.64</generator.version> - <lutinj2r.version>0.3</lutinj2r.version> + <lutinj2r.version>0.4-SNAPSHOT</lutinj2r.version> <jrst.version>0.8.4</jrst.version> <license-switcher.version>0.6</license-switcher.version> <openmap.version>4.6.4</openmap.version> <aspectwerkz.version>2.0</aspectwerkz.version> <sshtool.version>0.2.2</sshtool.version> <xmlrpc.version>3.1</xmlrpc.version> - <javadoc.version>2.5</javadoc.version> <!-- for compilation test to run --> <maven.test.forkMode>once</maven.test.forkMode> Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/IsisConfig.java =================================================================== --- isis-fish/trunk/src/main/java/fr/ifremer/isisfish/IsisConfig.java 2009-04-28 15:17:38 UTC (rev 2150) +++ isis-fish/trunk/src/main/java/fr/ifremer/isisfish/IsisConfig.java 2009-04-29 13:06:25 UTC (rev 2151) @@ -874,9 +874,8 @@ VCS_COMMIT(_(""), VCSAction.class.getName() + "#vcsCommit", "--vcsCommit"), SIMULATE_WITH_REGION(_(""), SimulationAction.class.getName() + "#simulateWithRegion", "--simulateWithRegion"), - SIMULATE_WITH_SIMULATION(_(""), SimulationAction.class.getName() + "#simulateWithSimulation", "--simulateWithSimulation"), - SIMULATE_WITH_SIMULATION_AND_SCRIPT(_(""), SimulationAction.class.getName() + "#simulateWithSimulationAndScript", "--simulateWithSimulationAndScript"), - ; + SIMULATE_REMOTELLY(_(""), SimulationAction.class.getName() + "#simulateRemotelly", "--simulateRemotelly"), + SIMULATE_REMOTELLY_WITH_PRESCRIPT(_(""), SimulationAction.class.getName() + "#simulateRemotellyWithPreScript", "--simulateRemotellyWithPreScript"); public String description; public String action; Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/IsisFish.java =================================================================== --- isis-fish/trunk/src/main/java/fr/ifremer/isisfish/IsisFish.java 2009-04-28 15:17:38 UTC (rev 2150) +++ isis-fish/trunk/src/main/java/fr/ifremer/isisfish/IsisFish.java 2009-04-29 13:06:25 UTC (rev 2151) @@ -1,6 +1,5 @@ /* *##% - * Copyright (C) 2005 - * Ifremer, Code Lutin, Cedric Pineau, Benjamin Poussin + * Copyright (C) 2005 - 2009 Ifremer, Code Lutin * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,18 +16,6 @@ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. *##%*/ -/* * - * IsisFish.java - * - * Created: 1 aout 2005 18:37:25 CEST - * - * @author Benjamin POUSSIN <poussin at codelutin.com> - * @version $Revision$ - * - * Last update: $Date$ - * by : $Author$ - */ - package fr.ifremer.isisfish; import static org.codelutin.i18n.I18n._; @@ -88,9 +75,15 @@ /** * This is the main class of <code>IsisFish</code> application. + * + * Created: 1 aout 2005 18:37:25 CEST * - * @author poussin + * @author Benjamin POUSSIN <poussin at codelutin.com> * @author chemit + * @version $Revision$ + * + * Last update: $Date$ + * by : $Author$ */ public class IsisFish { // IsisFish @@ -135,6 +128,9 @@ // action after init config.doAction(IsisConfig.STEP_AFTER_INIT); + // initVCS ask for passphrase, ui must be set before + initLoookAndFeel(); + // static vcs init (needed for some actions) try { initVCS(); @@ -142,9 +138,6 @@ log.warn(_("Error during vcs initialisation"), eee); } - // initVCS ask for passphrase, ui must be set before - initLoookAndFeel(); - if (log.isInfoEnabled()) { log.info(_("isisfish.launching", config.getElapsedTimeAsString())); } Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/actions/SimulationAction.java =================================================================== --- isis-fish/trunk/src/main/java/fr/ifremer/isisfish/actions/SimulationAction.java 2009-04-28 15:17:38 UTC (rev 2150) +++ isis-fish/trunk/src/main/java/fr/ifremer/isisfish/actions/SimulationAction.java 2009-04-29 13:06:25 UTC (rev 2151) @@ -1,5 +1,5 @@ /* *##% - * Copyright (C) 2002-2008 Code Lutin, Benjamin Poussin + * Copyright (C) 2002-2009 Code Lutin, Benjamin Poussin * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -28,6 +28,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.codelutin.util.FileUtil; +import org.codelutin.util.ApplicationConfig.Action.Step; import fr.ifremer.isisfish.IsisConfig; import fr.ifremer.isisfish.IsisFish; @@ -95,45 +96,62 @@ } /** - * Launch a simulation with specified simulationId and simulation zip. + * Launch a simulation specialized for remote launch (caparmor). * + * Done operations : + * - simulation zip import + * - pre script set + * - simulation + * - zip creation + * - checksum creation (zip creation name + .md5 extension) + * - delete simulation + * * @param simulationId id de simulation - * @param simulationZip Zip de la simulation + * @param simulationZip zip de la simulation + * @param simulationResultZip simulation result zip * @throws Exception */ - public static void simulateWithSimulation(String simulationId, File simulationZip) - throws Exception { - // just call with null pre script - simulateWithSimulationAndScript(simulationId, simulationZip, null); + @Step(IsisConfig.STEP_AFTER_INIT_VCS) + public static void simulateRemotelly(String simulationId, File simulationZip, File simulationResultZip) throws Exception { + simulateRemotellyWithPreScript(simulationId, simulationZip, simulationResultZip, null); } - + /** - * Launch a simulation with specified simulationId, simulationZip - * and simulationPrescript. + * Launch a simulation specialized for remote launch (caparmor). * + * Done operations : + * - simulation zip import + * - pre script set + * - simulation + * - zip creation + * - checksum creation (zip creation name + .md5 extension) + * - delete simulation + * * @param simulationId id de simulation - * @param simulationZip Zip de la simulation - * @param simulationPrescript simulation prescript + * @param simulationZip zip de la simulation + * @param simulationPrescript simulation prescript (can be null, empty) + * @param simulationResultZip simulation result zip * @throws Exception */ - public static void simulateWithSimulationAndScript(String simulationId, File simulationZip, File simulationPrescript) + @Step(IsisConfig.STEP_AFTER_INIT_VCS) + public static void simulateRemotellyWithPreScript(String simulationId, File simulationZip, File simulationResultZip, File simulationPrescript) throws Exception { - + if (log.isInfoEnabled()) { - log.info("Command line action : simulateWithSimulation (" + simulationId + ", " + simulationZip.getAbsolutePath() + ", " + simulationPrescript + ")"); + log.info("Command line action : simulateRemotelly (" + simulationId + ", " + simulationZip.getAbsolutePath() + ", " + simulationPrescript + ")"); } String name = simulationId; // ne pas mettre la date, car le sub process la met deja + " " + new SimpleDateFormat("yyyy-MM-dd-HH-mm").format(new Date()); - + SimulationControl control = new SimulationControl(name); // pour forcer le fichier de control a être sauver // pour ensuite être lu par le processus parent control.setAutoSaveState(true); - + // lancement de la simulation SimulatorLauncher launcher = new InProcessSimulatorLauncher(); - + //SimulationStorage simulation = launcher.simulate(null, control, simulationZip); //simulation.getStorage().closeContext(); String simulationPrescriptContent = null; @@ -141,9 +159,17 @@ simulationPrescriptContent = FileUtil.readAsString(simulationPrescript); } launcher.simulate(null, control, simulationZip, simulationPrescriptContent); - SimulationStorage simulation = launcher.getSimulationStorage(SimulationService.getService(), control); - simulation.getStorage().closeContext(); - + SimulationStorage simulationStorage = launcher.getSimulationStorage(SimulationService.getService(), control); + + // zip + md5 creation + simulationStorage.createZip(simulationResultZip, true); + + // remove simulation + simulationStorage.delete(false); + + // close all resources + simulationStorage.getStorage().closeContext(); + // FIXME integrer ca dans le process normal d'init IsisFish.quit(); } Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/datastore/DataStorage.java =================================================================== --- isis-fish/trunk/src/main/java/fr/ifremer/isisfish/datastore/DataStorage.java 2009-04-28 15:17:38 UTC (rev 2150) +++ isis-fish/trunk/src/main/java/fr/ifremer/isisfish/datastore/DataStorage.java 2009-04-29 13:06:25 UTC (rev 2151) @@ -234,8 +234,6 @@ } /* - * (non-Javadoc) - * * @see fr.ifremer.isisfish.datastore.VersionStorage#delete(boolean) */ @Override @@ -252,12 +250,12 @@ } /** - * copy le DataStorage courante Attention, all script with package name will - * not be modified you must change package name in each script to reflect - * new name + * Copy le DataStorage courant. * - * @param toName - * le nouveau nom de la region + * Attention, all script with package name will not be modified you must + * change package name in each script to reflect new name + * + * @param toName le nouveau nom de la region * @throws IOException si problème lors de la copie * @throws TopiaException ??? */ @@ -342,10 +340,22 @@ * @throws IOException si problème lors de la création du zip */ public File createZip(File file) throws IOException { + createZip(file, false); + return file; + } + + /** + * Cree un zip. Le zip contient les memes fichiers lorsqu'on utilise le CVS. + * + * @param file le fichier dans lequel l'export doit se faire + * @return l'argument file + * @throws IOException si problème lors de la création du zip + */ + public File createZip(File file, boolean createMD5) throws IOException { prepare(); List<File> files = getFiles(false); - ZipUtil.compressFiles(file, getDirectory().getParentFile(), files); + ZipUtil.compressFiles(file, getDirectory().getParentFile(), files, createMD5); log.info("Zip DataStorage in " + file); Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/datastore/VersionStorage.java =================================================================== --- isis-fish/trunk/src/main/java/fr/ifremer/isisfish/datastore/VersionStorage.java 2009-04-28 15:17:38 UTC (rev 2150) +++ isis-fish/trunk/src/main/java/fr/ifremer/isisfish/datastore/VersionStorage.java 2009-04-29 13:06:25 UTC (rev 2151) @@ -202,14 +202,14 @@ } /** - * Permet de supprimer une classe + * Permet de supprimer un fichier ou répertoire versionné ou non. * - * @param cvsDelete si vrai alors la classe sera aussi supprimé sur le - * cvs si elle existait. Cela prend effet immediatement (un commit est fait) - * @throws StorageException si pb pendant l'op + * @param vcsDelete si vrai alors le fichier sera aussi supprimé sur le + * vcs si elle existait. Cela prend effet immediatement (un commit est fait) + * @throws StorageException if delete operation fail */ - public void delete(boolean cvsDelete) throws StorageException { - if (cvsDelete) { + public void delete(boolean vcsDelete) throws StorageException { + if (vcsDelete) { List<File> files = getFiles(true); try { getVCS().delete(files, _("isisfish.versionStorage.removed")); Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java =================================================================== --- isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java 2009-04-28 15:17:38 UTC (rev 2150) +++ isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java 2009-04-29 13:06:25 UTC (rev 2151) @@ -36,7 +36,10 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.codelutin.topia.TopiaException; import org.codelutin.util.FileUtil; +import org.codelutin.util.MD5; +import org.codelutin.util.MD5InputStream; import com.jcraft.jsch.JSch; import com.jcraft.jsch.JSchException; @@ -79,8 +82,8 @@ protected Configuration freemarkerConfiguration; /** Freemarker qsub template. */ - protected static final String QSUB_SCRIPT_TEMPLATE = "templates/ssh/launch-isis-qsub.seq"; - //protected static final String QSUB_SCRIPT_TEMPLATE = "templates/ssh/launch-isis-cron.seq"; + protected static final String QSUB_SCRIPT_TEMPLATE = "templates/ssh/isis-launch-with-qsub.seq"; + //protected static final String QSUB_SCRIPT_TEMPLATE = "templates/ssh/isis-launch-with-cron.seq"; /** * Opened session. Stored in static context to not reask passphrase at each @@ -230,13 +233,14 @@ String simulationRemotePath = uploadSimulation(sshSession, simulationZip); + String remoteResultZip = getRemoteResultZipPath(control.getId()); + // build du contenu du script - message(control, - _("isisfish.simulation.remote.message.waitingstart")); + message(control,_("isisfish.simulation.remote.message.waitingstart")); String simulationPreScriptPath = uploadPreScriptIfNecessary( sshSession, control.getId(), simulationPrescript); - File simulationScript = getSimulationScriptFile(control.getId(), - simulationRemotePath, simulationPreScriptPath); + File simulationScript = getLaunchSimulationScriptFile(control.getId(), + simulationRemotePath, remoteResultZip, simulationPreScriptPath); String scriptRemotePath = uploadSimulationScript(sshSession, simulationScript); @@ -262,21 +266,11 @@ SimulationService simulationService, SimulationControl control) throws RemoteException { + // TODO change, now getSimulationStorage() + // is common to all launchers SimulationStorage simulationStorage = null; - - try { - // recuperation des resultats - message(control, - _("isisfish.simulation.remote.message.downloadresults")); - - downloadResults(sshSession, control.getId()); - simulationStorage = SimulationStorage .getSimulation(control.getId()); - } catch (SSHException e) { - throw new RemoteException( - _("isisfish.error.simulation.remote.global"), e); - } return simulationStorage; } @@ -292,6 +286,8 @@ SimulationControl control) throws RemoteException { try { + + // CONTROL file try { // download control file File controlFile = downloadSimulationFile(control.getId(), @@ -313,6 +309,7 @@ // deleteTempFile controlFile.delete(); } catch (SSHException e) { + // file doesn't exist if (log.isDebugEnabled()) { // not add ,e plz :) @@ -321,8 +318,61 @@ } } + // can be because, simulation has not begun + // or simulation is ended + // try do download md5 control file + + // MD5 + SIMULATION zip file try { - // download information file + File md5ControlFile = downloadMD5ControlFile(control.getId()); + + if (md5ControlFile != null) { + control.setText(_("isisfish.simulation.remote.message.downloadresults")); + + String md5sum = FileUtil.readAsString(md5ControlFile); + + if (log.isDebugEnabled()) { + log.debug("MD5 Control file have been downloaded : " + md5ControlFile.getAbsolutePath()); + } + + File resultZipFile = downloadZIPResults(sshSession, control.getId(), md5sum); + + if (resultZipFile != null) { + SimulationStorage.importZip(resultZipFile); + + if (log.isDebugEnabled()) { + log.debug("Simulation imported : " + resultZipFile.getAbsolutePath()); + } + + // read control from downloaded simulation + synchronized (control) { + SimulationStorage.readControl(control.getId(), control, "stop"); + } + + resultZipFile.delete(); + } + else { + if (log.isWarnEnabled()) { + log.warn("Simulation zip download failed"); + } + } + + // remove temp file + md5ControlFile.delete(); + } + } + catch(SSHException e) { + if (log.isDebugEnabled()) { + log.debug(_("Error while downloading file : %s", e.getMessage())); + } + } catch (TopiaException e) { + if (log.isErrorEnabled()) { + log.error("Can't import result zip file", e); + } + } + + // INFORMATION file + try { File infoFile = downloadSimulationFile(control.getId(), SimulationStorage.INFORMATION_FILENAME); if (log.isDebugEnabled()) { @@ -572,46 +622,41 @@ } /** - * Download simulation results. + * Download simulation zip results. * - * Simulation result is save in {@link SimulationStorage#SIMULATION_PATH}. - * - * @throws SSHException - * if download fail (can happen if remote file doesn't exist + * @throws SSHException if download fail (can happen if remote file doesn't exist + * @throws IOException if download fail (can happen if remote file doesn't exist */ - protected void downloadResults(Session session, String simulationId) - throws SSHException { + protected File downloadZIPResults(Session session, String simulationId, String md5sum) + throws SSHException, IOException { - // simulation directory - File localFile = new File(IsisFish.config.getDatabaseDirectory(), - SimulationStorage.SIMULATION_PATH); + File localFile = File.createTempFile("simulation-results", ".zip"); if (log.isDebugEnabled()) { log.debug("Downloading results in " + localFile.getAbsolutePath()); } - // Cant happen if result have already been downloded - // Or try to, but has been interrupted - // delete and download again - File simulationBaseDirectory = new File(localFile, simulationId); - if (simulationBaseDirectory.exists()) { - if (log.isWarnEnabled()) { - log.warn("Warning, simulation results " - + simulationBaseDirectory.getAbsolutePath() - + " already exists"); - log.warn("Delete it, and download again"); - } - - FileUtil.deleteRecursively(simulationBaseDirectory); - } - // build remote file path // FIXME this path should be given by remote IsisFish app - String remoteFile = IsisFish.config.getSimulatorSshDataPath(); - remoteFile += "/" + SimulationStorage.SIMULATION_PATH; - remoteFile += "/" + simulationId; + String remoteFile = getRemoteResultZipPath(simulationId); SSHUtils.scpFrom(session, remoteFile, localFile); + + if (!StringUtils.isEmpty(md5sum)) { + MD5InputStream in = new MD5InputStream(new FileInputStream(localFile)); + while(in.read() != -1) {} + in.close(); + String localMd5 = MD5.asHex(in.hash()); + if (!localMd5.equals(md5sum)) { + if (log.isWarnEnabled()) { + log.warn("Warning md5 checksum failed (got " + localMd5 + ", expected : " + md5sum + ")"); + } + localFile.delete(); + localFile = null; + } + } + + return localFile; } /** @@ -648,7 +693,36 @@ return localFile; } + + /** + * Download remote simulation md5 control file and store its content into temp + * file. + * + * @param simulationId id de la simulation + * @return downloaded temp file (file have to be manually deleted) + * @throws IOException + * @throws SSHException if remote file doesn't exists + */ + protected File downloadMD5ControlFile(String simulationId) + throws IOException, SSHException { + File localFile = null; + + // build remote file path + String remoteFile = getRemoteResultZipPath(simulationId) + ".md5"; + + // local tmp file + localFile = File.createTempFile(simulationId, ".md5"); + + try { + SSHUtils.scpFrom(getSSHSession(), remoteFile, localFile); + } catch (JSchException e) { + throw new SSHException("Can't download file", e); + } + + return localFile; + } + /** * Upload script on remote server. * @@ -685,11 +759,37 @@ } /** + * Get remote simulation zip path. + * + * Return path if uploaded or null if no upload needed. + * + * @param simulationId simulation id + * + * @throws SSHException if upload fail + * @throws IOException if upload fail + */ + protected String getRemoteResultZipPath(String simulationId) + throws SSHException, IOException { + + String remotePath = IsisFish.config.getSimulatorSshTmpPath(); + + // upload directory in that dir + if (!remotePath.endsWith("/")) { + remotePath += "/"; + } + + remotePath += "simulation-" + simulationId + "-result.zip"; + + return remotePath; + } + + /** * Upload pre script on remote server. * * Return path if uploaded or null if no upload needed. * * @param session valid opened ssh session + * @param simulationId simulation id * @param simulationPreScript script content * * @throws SSHException if upload fail @@ -745,14 +845,14 @@ * * @throws IOException if can't build script */ - protected File getSimulationScriptFile(String simuationId, - String simulationZip, String preScriptPath) throws IOException { + protected File getLaunchSimulationScriptFile(String simuationId, + String simulationZip, String simulationResultZip, String preScriptPath) throws IOException { - File tempScript = File.createTempFile("launch-isis", ".seq"); + File tempScript = File.createTempFile("isis-launch-", ".seq"); tempScript.deleteOnExit(); // auto delete String fileContent = getSimulationScriptLaunchContent( - QSUB_SCRIPT_TEMPLATE, simuationId, simulationZip, preScriptPath); + QSUB_SCRIPT_TEMPLATE, simuationId, simulationZip, simulationResultZip, preScriptPath); FileUtil.writeString(tempScript, fileContent); return tempScript; @@ -766,12 +866,13 @@ * @param templateName url du template * @param simuationId id de la simulation * @param simulationZip zip de la simulation + * @param simulationZipResult zip resultat de la simulation * @param preScriptPath simulation pre script path (can be null) * * @throws IOException if can't get script content */ protected String getSimulationScriptLaunchContent(String templateName, - String simuationId, String simulationZip, String preScriptPath) + String simuationId, String simulationZip, String simulationZipResult, String preScriptPath) throws IOException { String scriptContent = null; @@ -792,6 +893,7 @@ root.put("isishome", IsisFish.config.getSimulatorSshIsisHome()); root.put("simulationid", simuationId); root.put("simulationzip", simulationZip); + root.put("simulationresultzip", simulationZipResult); root.put("simulationprescript", localPreScript); // process template Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SimulationMonitor.java =================================================================== --- isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SimulationMonitor.java 2009-04-28 15:17:38 UTC (rev 2150) +++ isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SimulationMonitor.java 2009-04-29 13:06:25 UTC (rev 2151) @@ -375,9 +375,17 @@ // if not finished, re-add it at end if (jobIsFinished) { + + if (log.isInfoEnabled()) { + log.info("Job " + job.getId() + " finished"); + } + doPostSimulationOperation(job, launcher); - } else { - + } + + // Si le job est reelement fini + // et que les post operation ont réussie + else { // next check date long nextJobTimeMs = date.getTime() + launcher.getCheckProgressionInterval() * 1000; @@ -425,8 +433,10 @@ // WARNING this condition is VERY important // and set by end of // fr.ifremer.isisfish.simulator.launcher.InProcessSimulatorLauncher#localSimulateSameThread(SimulationControl, SimulationStorage) + + // FIXME check SimulationStorage.exists(control.getId()) condition if ((control.getProgress() > 0 - && control.getProgress() >= control.getProgressMax()) || control.isStopSimulationRequest()) { + && control.getProgress() >= control.getProgressMax() && SimulationStorage.exists(control.getId())) || control.isStopSimulationRequest()) { simulationEnded = true; } } catch (RemoteException e) { @@ -446,10 +456,14 @@ * * @param job job * @param launcher {@link SimulatorLauncher} that manage simulation + * + * @return true if post operation complete and job has be removed from queue */ - protected void doPostSimulationOperation(SimulationJob job, + protected boolean doPostSimulationOperation(SimulationJob job, SimulatorLauncher launcher) { + boolean result = false; + SimulationService service = SimulationService.getService(); SimulationControl control = job.getItem().getControl(); @@ -475,23 +489,29 @@ boolean simulationAvailble = exportResult(job, simulation); // sensitivity calculator analyze result call - // can't do analyze seond pass if simulation + // can't do analyze second pass if simulation // has been deleted if (simulationAvailble) { analyzeSensitivityResult(job, simulation); } } + + // notify simulation ended + // to not check later... + // don't do in finally, getSimulationStorage may fail (md5 on ssh) + simulationStop(job); + service.fireStopEvent(job); + + result = true; + } catch (RemoteException e) { if (log.isErrorEnabled()) { log.error("Can't get simulation results after simulation end", e); } - } finally { - // notify simulation ended - // to not check later... - simulationStop(job); - service.fireStopEvent(job); } + + return result; } /** Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/ssh/SSHUtils.java =================================================================== --- isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/ssh/SSHUtils.java 2009-04-28 15:17:38 UTC (rev 2150) +++ isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/ssh/SSHUtils.java 2009-04-29 13:06:25 UTC (rev 2151) @@ -29,8 +29,10 @@ import java.io.InputStreamReader; import java.io.OutputStream; +import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.codelutin.util.MD5InputStream; import com.jcraft.jsch.Channel; import com.jcraft.jsch.ChannelExec; @@ -121,13 +123,13 @@ } /** - * Download a local file from remote server. + * Download a local file from remote server and check md5sum. * * @param session opened valid jsch session * @param remoteFileName remote file name to download * @param localFile local file name to download into * - * @throws SSHException + * @throws SSHException if transfer fail (include md5 control failed) */ public static void scpFrom(Session session, String remoteFileName, File localFile) throws SSHException { Copied: isis-fish/trunk/src/main/resources/templates/ssh/isis-launch-with-cron.seq (from rev 2143, isis-fish/trunk/src/main/resources/templates/ssh/launch-isis-cron.seq) =================================================================== --- isis-fish/trunk/src/main/resources/templates/ssh/isis-launch-with-cron.seq (rev 0) +++ isis-fish/trunk/src/main/resources/templates/ssh/isis-launch-with-cron.seq 2009-04-29 13:06:25 UTC (rev 2151) @@ -0,0 +1,9 @@ +#!/bin/bash + +cd "${isishome}" + +<#if simulationprescript?length > 0> +java -jar isis-fish*.jar --option launch.ui false --simulateRemotellyWithPreScript "${simulationid}" "${simulationzip}" "${simulationresultzip}" "${simulationprescript}" +<#else> +java -jar isis-fish*.jar --option launch.ui false --simulateRemotelly "${simulationid}" "${simulationzip}" "${simulationresultzip}" +</#if> \ No newline at end of file Copied: isis-fish/trunk/src/main/resources/templates/ssh/isis-launch-with-qsub.seq (from rev 2143, isis-fish/trunk/src/main/resources/templates/ssh/launch-isis-qsub.seq) =================================================================== --- isis-fish/trunk/src/main/resources/templates/ssh/isis-launch-with-qsub.seq (rev 0) +++ isis-fish/trunk/src/main/resources/templates/ssh/isis-launch-with-qsub.seq 2009-04-29 13:06:25 UTC (rev 2151) @@ -0,0 +1,13 @@ +#!/bin/csh + +#PBS -l mem=1000mb + +cd "${isishome}" +source /usr/share/modules/init/csh +module load java/1.6.0 + +<#if simulationprescript?length > 0> +java -jar isis-fish*.jar --option launch.ui false --simulateRemotellyWithPreScript "${simulationid}" "${simulationzip}" "${simulationresultzip}" "${simulationprescript}" +<#else> +java -jar isis-fish*.jar --option launch.ui false --simulateRemotelly "${simulationid}" "${simulationzip}" "${simulationresultzip}" +</#if> \ No newline at end of file Deleted: isis-fish/trunk/src/main/resources/templates/ssh/launch-isis-cron.seq =================================================================== --- isis-fish/trunk/src/main/resources/templates/ssh/launch-isis-cron.seq 2009-04-28 15:17:38 UTC (rev 2150) +++ isis-fish/trunk/src/main/resources/templates/ssh/launch-isis-cron.seq 2009-04-29 13:06:25 UTC (rev 2151) @@ -1,9 +0,0 @@ -#!/bin/bash - -cd "${isishome}" - -<#if simulationprescript?length > 0> -java -jar isis-fish*.jar --option launch.ui false --simulateWithSimulationAndScript "${simulationid}" "${simulationzip}" "${simulationprescript}" -<#else> -java -jar isis-fish*.jar --option launch.ui false --simulateWithSimulation "${simulationid}" "${simulationzip}" -</#if> \ No newline at end of file Deleted: isis-fish/trunk/src/main/resources/templates/ssh/launch-isis-qsub.seq =================================================================== --- isis-fish/trunk/src/main/resources/templates/ssh/launch-isis-qsub.seq 2009-04-28 15:17:38 UTC (rev 2150) +++ isis-fish/trunk/src/main/resources/templates/ssh/launch-isis-qsub.seq 2009-04-29 13:06:25 UTC (rev 2151) @@ -1,13 +0,0 @@ -#!/bin/csh - -#PBS -l mem=1000mb - -cd "${isishome}" -source /usr/share/modules/init/csh -module load java/1.6.0 - -<#if simulationprescript?length > 0> -java -jar isis-fish*.jar --option launch.ui false --simulateWithSimulationAndScript "${simulationid}" "${simulationzip}" "${simulationprescript}" -<#else> -java -jar isis-fish*.jar --option launch.ui false --simulateWithSimulation "${simulationid}" "${simulationzip}" -</#if> \ No newline at end of file Modified: isis-fish/trunk/src/test/java/fr/ifremer/isisfish/simulator/launcher/SshSimulatorLauncherTest.java =================================================================== --- isis-fish/trunk/src/test/java/fr/ifremer/isisfish/simulator/launcher/SshSimulatorLauncherTest.java 2009-04-28 15:17:38 UTC (rev 2150) +++ isis-fish/trunk/src/test/java/fr/ifremer/isisfish/simulator/launcher/SshSimulatorLauncherTest.java 2009-04-29 13:06:25 UTC (rev 2151) @@ -61,13 +61,14 @@ final String SIMULATIONID = "testid"; final String SIMULATIONZIP = "test.zip"; + final String SIMULATIONRESULTZIP = "/tmp/test-result.zip"; final String SIMULATIONPRESCRIPT = "prescript-1234567890.txt"; final String ISISHOME = IsisFish.config.getSimulatorSshIsisHome(); SSHSimulatorLauncher launcher = new SSHSimulatorLauncher(); String content = launcher.getSimulationScriptLaunchContent( SSHSimulatorLauncher.QSUB_SCRIPT_TEMPLATE, SIMULATIONID, - SIMULATIONZIP, SIMULATIONPRESCRIPT); + SIMULATIONZIP, SIMULATIONRESULTZIP, SIMULATIONPRESCRIPT); // simulation parameters Assert.assertTrue("String \"" + SIMULATIONID @@ -76,12 +77,15 @@ Assert.assertTrue("String \"" + SIMULATIONZIP + "\" not found in template", content.indexOf(SIMULATIONZIP) > 0); + Assert.assertTrue("String \"" + SIMULATIONRESULTZIP + + "\" not found in template", + content.indexOf(SIMULATIONRESULTZIP) > 0); Assert.assertTrue("String \"" + SIMULATIONPRESCRIPT + "\" not found in template", content .indexOf(SIMULATIONPRESCRIPT) > 0); Assert.assertTrue( - "Action --simulateWithSimulationAndScript not on script", - content.indexOf(" --simulateWithSimulationAndScript ") > 0); + "Action --simulateRemotellyWithPreScript not on script", + content.indexOf(" --simulateRemotellyWithPreScript ") > 0); // isis location Assert.assertTrue("String \"" + ISISHOME + "\" not found in template", @@ -105,13 +109,14 @@ final String SIMULATIONID = "testid"; final String SIMULATIONZIP = "test.zip"; + final String SIMULATIONRESULTZIP = "/tmp/test-result.zip"; final String SIMULATIONPRESCRIPT = null; final String ISISHOME = IsisFish.config.getSimulatorSshIsisHome(); SSHSimulatorLauncher launcher = new SSHSimulatorLauncher(); String content = launcher.getSimulationScriptLaunchContent( SSHSimulatorLauncher.QSUB_SCRIPT_TEMPLATE, SIMULATIONID, - SIMULATIONZIP, SIMULATIONPRESCRIPT); + SIMULATIONZIP, SIMULATIONRESULTZIP, SIMULATIONPRESCRIPT); // simulation parameters Assert.assertTrue("String \"" + SIMULATIONID @@ -120,14 +125,17 @@ Assert.assertTrue("String \"" + SIMULATIONZIP + "\" not found in template", content.indexOf(SIMULATIONZIP) > 0); - Assert.assertTrue("Action --simulateWithSimulation not on script", - content.indexOf(" --simulateWithSimulation ") > 0); + Assert.assertTrue("String \"" + SIMULATIONRESULTZIP + + "\" not found in template", + content.indexOf(SIMULATIONRESULTZIP) > 0); + Assert.assertTrue("Action --simulateRemotelly not on script", + content.indexOf(" --simulateRemotelly ") > 0); // isis location Assert.assertTrue("String \"" + ISISHOME + "\" not found in template", content.indexOf(ISISHOME) > 0); } - + /** * Pour tester, un autre script est present, mais * celui ci ne doit jamais être utilisé hors des tests. @@ -140,7 +148,7 @@ SSHSimulatorLauncher launcher = new SSHSimulatorLauncher(); String content = launcher.getSimulationScriptLaunchContent( SSHSimulatorLauncher.QSUB_SCRIPT_TEMPLATE, "", - "", ""); + "", "", ""); // simulation parameters Assert.assertTrue("Invalid qsub script is used", content