Author: chatellier Date: 2009-10-29 16:47:37 +0000 (Thu, 29 Oct 2009) New Revision: 2716 Modified: isis-fish/trunk/changelog.txt isis-fish/trunk/src/main/java/fr/ifremer/isisfish/actions/SimulationAction.java isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java isis-fish/trunk/src/main/resources/templates/ssh/isis-launch-with-qsub.seq Log: Rename all uploaded file on caparmor to common non random format : - $ISIS-TMP/simulation-$id-preparation.zip - $ISIS-TMP/simulation-$id-prescript.bsh - $ISIS-TMP/simulation-$id-script.seq - $ISIS-TMP/simulation-$id-result.zip - $ISIS-TMP/simulation-$id-result.zip.md5 - $ISIS-TMP/simulation-$id-output.txt - $ISIS-TMP/simulation-$id-pbs.id Modified: isis-fish/trunk/changelog.txt =================================================================== --- isis-fish/trunk/changelog.txt 2009-10-29 14:03:01 UTC (rev 2715) +++ isis-fish/trunk/changelog.txt 2009-10-29 16:47:37 UTC (rev 2716) @@ -1,5 +1,7 @@ isis-fish (3.3.0.0) stable; urgency=low + * Add missing log category in simulation filter appender + * Add -m n option to qsub to not send email for each simulation * Remove duplicated SSH key configuration, common configuration for VCS and caparmor * Global VCS improvement (code, script UI, input UI) * Update to svnkit 1.3.1 Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/actions/SimulationAction.java =================================================================== --- isis-fish/trunk/src/main/java/fr/ifremer/isisfish/actions/SimulationAction.java 2009-10-29 14:03:01 UTC (rev 2715) +++ isis-fish/trunk/src/main/java/fr/ifremer/isisfish/actions/SimulationAction.java 2009-10-29 16:47:37 UTC (rev 2716) @@ -229,6 +229,7 @@ log.debug("Timing : before zipping results : " + new java.util.Date()); } } + // md5 file name in complete simulationStorage.getFile() + ".md5" ZipUtil.compress(simulationResultArchive, simulationStorage.getFile(), null, true); if (log.isDebugEnabled()) { log.debug("Timing : after zipping results : " + new java.util.Date()); Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java =================================================================== --- isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java 2009-10-29 14:03:01 UTC (rev 2715) +++ isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHSimulatorLauncher.java 2009-10-29 16:47:37 UTC (rev 2716) @@ -71,6 +71,17 @@ * * Isis-Fish must be installed on remote server. * + * Caparmor file layout ($i = plan/as increment) : + * - $ISIS-TMP/simulation-$id-preparation.zip + * - $ISIS-TMP/simulation-$id-prescript.bsh + * - $ISIS-TMP/simulation-$id-script.seq + * - $ISIS-TMP/simulation-$id-result.zip + * - $ISIS-TMP/simulation-$id-result.zip.md5 + * - $ISIS-TMP/simulation-$id-output.txt + * - $ISIS-TMP/simulation-$id-pbs.id + * + * TODO revome all $ISIS-TMP/simulation-$id* after simulation end ? + * * @see JSch * * @author chatellier @@ -95,7 +106,7 @@ * connection. */ protected static Session sshSession; - + /** * Opened session to sftp service. Stored in static context to not reask passphrase at each * connection. @@ -218,7 +229,7 @@ */ @Override public void simulationStopRequest(SimulationJob job) throws RemoteException { - + // make sure user is connected try { getSSHSession(); @@ -226,14 +237,13 @@ catch(JSchException e) { throw new RemoteException("Can't connect", e); } - + try { sendStopSimulationRequest(sshSession, job.getId()); } catch (SSHException e) { throw new RemoteException("Can't connect", e); } - - + simulationEnded(); } @@ -254,7 +264,7 @@ if (StringUtils.isBlank(IsisFish.config.getSimulatorSshUsername())) { throw new RemoteException("Username is empty"); } - + // ask for available resources on caparmor // blocking until available message(control, _("isisfish.simulation.remote.message.waitingavailable")); @@ -262,6 +272,9 @@ // start ssh session try { + + String simulationid = control.getId(); + // connection message(control, _("isisfish.simulation.remote.message.connection")); Session sshSession = getSSHSession(); @@ -270,22 +283,22 @@ // upload simulation on server message(control, _("isisfish.simulation.remote.message.upload")); String simulationRemotePath = uploadSimulation(sshSession, - simulationZip); + simulationid, simulationZip); - String remoteResultZip = getRemoteResultArchivePath(control.getId()); - + String remoteResultZip = getRemoteResultArchivePath(simulationid); + // build du contenu du script message(control,_("isisfish.simulation.remote.message.waitingstart")); String simulationPreScriptPath = uploadPreScriptIfNecessary( sshSession, control.getId(), simulationPrescript); - File simulationScript = getLaunchSimulationScriptFile(control.getId(), + File simulationScript = getLaunchSimulationScriptFile(simulationid, simulationRemotePath, remoteResultZip, simulationPreScriptPath); String scriptRemotePath = uploadSimulationScript(sshSession, - simulationScript); - // prescrit uploaded, delete + simulationid, simulationScript); + // prescript uploaded, delete simulationScript.delete(); - sendStartSimulationRequest(sshSession, scriptRemotePath, control.getId()); + sendStartSimulationRequest(sshSession, scriptRemotePath, simulationid); } catch (Exception e) { if (log.isErrorEnabled()) { @@ -324,7 +337,7 @@ @Override public void updateControl(SimulationService simulationService, SimulationControl control) throws RemoteException { - + // make sure user is connected try { getSSHSession(); @@ -334,7 +347,7 @@ } try { - + // CONTROL file try { // download control file @@ -357,7 +370,7 @@ // deleteTempFile controlFile.delete(); } catch (SSHException e) { - + // file doesn't exist if (log.isDebugEnabled()) { // not add ,e plz :) @@ -369,7 +382,7 @@ // can be because, simulation has not begun // or simulation is ended // try do download md5 control file - + // MD5 + SIMULATION zip file try { File md5ControlFile = downloadResultsMD5File(sshSession, control.getId()); @@ -384,7 +397,7 @@ } File resultArchiveFile = downloadResultsArchive(sshSession, control.getId(), md5sum); - + if (resultArchiveFile != null) { // FIXME done here to not break upload during import @@ -406,7 +419,7 @@ else { uncompressFiltred(resultArchiveFile, SimulationStorage.getSimulationDirectory()); } - + if (log.isDebugEnabled()) { log.debug("Simulation imported : " + resultArchiveFile.getAbsolutePath()); } @@ -631,7 +644,6 @@ try { // use usefull readLines from commons-io - @SuppressWarnings("unchecked") List<String> contents = FileUtils.readLines(publicKey); // only one line @@ -664,16 +676,15 @@ * Upload simulation file to server. * * @param session already open valid ssh session + * @param simulationid simulation id * @param simulationFile simulation file to upload * * @return remote file path or <tt>null</tt> if errors * @throws SSHException if upload fail */ - protected String uploadSimulation(Session session, File simulationFile) + protected String uploadSimulation(Session session, String simulationid, File simulationFile) throws SSHException { - String localPath = simulationFile.getAbsolutePath(); - // first check that remote directory exists String remotePath = getRemoteTempDirectory(); @@ -694,19 +705,9 @@ throw new SSHException(_("Command '%s' fail to execute", command)); } - // upload directory in that dir - if (!remotePath.endsWith("/")) { - remotePath += "/"; - } + // always rename simulation to "simulation-$id-preparation.zip" + remotePath += "simulation-" + simulationid + "preparation.zip"; - if (localPath.lastIndexOf("/") > 0) { - remotePath += localPath.substring(localPath.lastIndexOf("/") + 1); - } else if (localPath.lastIndexOf("\\") > 0) { // windows - remotePath += localPath.substring(localPath.lastIndexOf("\\") + 1); - } else { - remotePath += localPath; - } - SSHUtils.scpTo(session, simulationFile, remotePath); return remotePath; @@ -754,7 +755,7 @@ localFile = null; } } - + return localFile; } @@ -776,7 +777,6 @@ // build remote file path // FIXME this path should be given by remote IsisFish app - // TODO to change String remoteFile = IsisFish.config.getSimulatorSshDataPath(); remoteFile += "/" + SimulationStorage.SIMULATION_PATH; remoteFile += "/" + simulationId; @@ -794,7 +794,7 @@ return localFile; } - + /** * Download remote simulation md5 control file and store its content into temp * file. @@ -809,6 +809,9 @@ throws IOException, SSHException { // build remote file path + // this path is not configurable + // same as fr.ifremer.isisfish.actions.SimulationAction.simulateRemotellyWithPreScript(String, File, File, File) + // defined by org.nuiton.util.ZipUtil.compressFiles(File, File, Collection<File>, boolean) String remoteFile = getRemoteResultArchivePath(simulationId) + ".md5"; // local tmp file @@ -833,22 +836,14 @@ * @throws SSHException if upload fail */ protected String uploadSimulationScript(Session session, - File simulationScript) throws SSHException { + String simulationid, File simulationScript) throws SSHException { - String localPath = simulationScript.getAbsolutePath(); - - String remotePath = getRemoteTempDirectory(); - // remote temp directory should have been created // by #uploadSimulation(Session, String) + String remotePath = getRemoteTempDirectory(); - if (localPath.lastIndexOf("/") > 0) { - remotePath += localPath.substring(localPath.lastIndexOf("/") + 1); - } else if (localPath.lastIndexOf("\\") > 0) { // windows - remotePath += localPath.substring(localPath.lastIndexOf("\\") + 1); - } else { - remotePath += localPath; - } + // always rename uploaded script to "simulation-$id-script.seq" + remotePath += "simulation-" + simulationid + "-script.seq"; SSHUtils.scpTo(session, simulationScript, remotePath); @@ -861,14 +856,11 @@ * @param simulationId simulation id */ protected String getRemoteResultArchivePath(String simulationId) { - String remotePath = getRemoteTempDirectory(); - remotePath += "simulation-" + simulationId + "-result.zip"; - return remotePath; } - + /** * Upload pre script on remote server. * @@ -890,24 +882,17 @@ return null; } - File tempPreScriptFile = File.createTempFile("isis-" + simulationId - + "-", ".prescript"); + File tempPreScriptFile = File.createTempFile("simulation-" + simulationId + + "-prescript", ".bsh"); tempPreScriptFile.deleteOnExit(); FileUtil.writeString(tempPreScriptFile, simulationPreScript); - String localPath = tempPreScriptFile.getAbsolutePath(); - String remotePath = getRemoteTempDirectory(); - // remote temp directory should have been created // by #uploadSimulation(Session, String) + String remotePath = getRemoteTempDirectory(); - if (localPath.lastIndexOf("/") > 0) { - remotePath += localPath.substring(localPath.lastIndexOf("/") + 1); - } else if (localPath.lastIndexOf("\\") > 0) { // windows - remotePath += localPath.substring(localPath.lastIndexOf("\\") + 1); - } else { - remotePath += localPath; - } + // always rename simulation prescript to "simulation-$id-prescript.bsh" + remotePath += "simulation-" + simulationId + "-prescript.bsh"; SSHUtils.scpTo(session, tempPreScriptFile, remotePath); @@ -931,7 +916,7 @@ protected File getLaunchSimulationScriptFile(String simuationId, String simulationZip, String simulationResultZip, String preScriptPath) throws IOException { - File tempScript = File.createTempFile("isis-launch-" + simuationId + "-", ".seq"); + File tempScript = File.createTempFile("simulation-" + simuationId + "-script", ".seq"); tempScript.deleteOnExit(); // auto delete String fileContent = getSimulationScriptLaunchContent( @@ -961,9 +946,9 @@ String scriptContent = null; // test null values for prescript - String localPreScript = preScriptPath; - if (localPreScript == null) { - localPreScript = ""; + String remotePreScript = preScriptPath; + if (remotePreScript == null) { + remotePreScript = ""; } try { @@ -978,7 +963,7 @@ root.put("simulationid", simuationId); root.put("simulationzip", simulationZip); root.put("simulationresultzip", simulationZipResult); - root.put("simulationprescript", localPreScript); + root.put("simulationprescript", remotePreScript); // process template Writer out = new StringWriter(); @@ -1012,8 +997,9 @@ // command to : // - add script in qsub queue // - use option -m n (to not send email) + String remoteFilenameId = getRemoteTempDirectory() + "simulation-" + simulationId + "-pbs.id"; String command = IsisFish.config.getSimulatorSshPbsBinPath() - + "/qsub -m n \"" + scriptRemotePath + "\"|tee \"" + getRemoteTempDirectory() + "/" + simulationId + ".id\""; + + "/qsub -m n \"" + scriptRemotePath + "\"|tee \"" + remoteFilenameId + "\""; Writer output = new StringWriter(); int exit = SSHUtils.exec(session, command, output); @@ -1021,13 +1007,13 @@ if (exit != 0) { throw new SSHException(_("Command '%s' fail to execute", command)); } - + String out = output.toString(); if (out.trim().matches("\\d+\\.\\w+")) { log.info("Job submitted with job id : " + out); } } - + /** * Send qdel request on job. * @@ -1040,16 +1026,16 @@ throws SSHException { // command to : - String command = IsisFish.config.getSimulatorSshPbsBinPath() - + "/qdel `cat \"" + getRemoteTempDirectory() + "/" + simulationId + ".id\"`"; - + String remoteFilenameId = getRemoteTempDirectory() + "simulation-" + simulationId + "-pbs.id"; + String command = IsisFish.config.getSimulatorSshPbsBinPath() + "/qdel `cat \"" + remoteFilenameId + "\"`"; + // and delete simulation //command += " && rm -rf \"" + IsisFish.config.getSimulatorSshDataPath() + "/simulations/" + simulationId + "\""; - + if (log.isDebugEnabled()) { log.debug("Send stop request : " + command); } - + SSHUtils.exec(session, command); // can fail, already stopped @@ -1057,7 +1043,7 @@ throw new SSHException(_("Command '%s' fail to execute", command)); }*/ } - + /** * Get remote directory absolute path. * @@ -1067,7 +1053,7 @@ */ protected String getRemoteTempDirectory() { String remotePath = IsisFish.config.getSimulatorSshTmpPath(); - + if (!remotePath.startsWith("/")) { remotePath = IsisFish.config.getSimulatorSshUserHome() + "/" + remotePath; } @@ -1076,7 +1062,7 @@ if (!remotePath.endsWith("/")) { remotePath += "/"; } - + return remotePath; } @@ -1091,14 +1077,14 @@ * @deprecated this code may be in lutin util later... */ protected void uncompressFiltred(File file, File targetDir, String... excludes) throws IOException { - + ZipFile zipFile = new ZipFile(file); - + Enumeration<? extends ZipEntry> entries = zipFile.entries(); - + while (entries.hasMoreElements()) { ZipEntry entry = entries.nextElement(); - + String name = entry.getName(); // add continue to break loop boolean excludeEntry = false; @@ -1109,7 +1095,7 @@ } } } - + if (!excludeEntry) { File target = new File(targetDir, name); if (entry.isDirectory()) { Modified: isis-fish/trunk/src/main/resources/templates/ssh/isis-launch-with-qsub.seq =================================================================== --- isis-fish/trunk/src/main/resources/templates/ssh/isis-launch-with-qsub.seq 2009-10-29 14:03:01 UTC (rev 2715) +++ isis-fish/trunk/src/main/resources/templates/ssh/isis-launch-with-qsub.seq 2009-10-29 16:47:37 UTC (rev 2716) @@ -6,7 +6,7 @@ #module load java/1.6.0 <#if simulationprescript?length > 0> -/home3/caparmor/poussin/jdk/bin/java -Xmx2500M -jar isis-fish*.jar --option launch.ui false --simulateRemotellyWithPreScript "${simulationid}" "${simulationzip}" "${simulationresultzip}" "${simulationprescript}" >& "${isistemp}output-${simulationid}" +/home3/caparmor/poussin/jdk/bin/java -Xmx2500M -jar isis-fish*.jar --option launch.ui false --simulateRemotellyWithPreScript "${simulationid}" "${simulationzip}" "${simulationresultzip}" "${simulationprescript}" >& "${isistemp}simulation-${simulationid}-output.txt" <#else> -/home3/caparmor/poussin/jdk/bin/java -jar isis-fish*.jar --option launch.ui false --simulateRemotelly "${simulationid}" "${simulationzip}" "${simulationresultzip}" >& "${isistemp}output-${simulationid}" +/home3/caparmor/poussin/jdk/bin/java -jar isis-fish*.jar --option launch.ui false --simulateRemotelly "${simulationid}" "${simulationzip}" "${simulationresultzip}" >& "${isistemp}simulation-${simulationid}-output.txt" </#if>