Author: chatellier Date: 2009-01-14 14:22:05 +0000 (Wed, 14 Jan 2009) New Revision: 1739 Added: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHUtils.java Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SshSimulatorLauncher.java Log: Move ssh specific code to utility class. Use ANT optionnal ssh task code (better than jsch samples) Download simulation result when simulation complete. Added: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHUtils.java =================================================================== --- isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHUtils.java (rev 0) +++ isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SSHUtils.java 2009-01-14 14:22:05 UTC (rev 1739) @@ -0,0 +1,495 @@ +/* *##% + * Copyright (C) 2008 Code Lutin + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + *##%*/ + +package fr.ifremer.isisfish.simulator.launcher; + +import java.io.BufferedReader; +import java.io.ByteArrayOutputStream; +import java.io.EOFException; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import com.jcraft.jsch.Channel; +import com.jcraft.jsch.ChannelExec; +import com.jcraft.jsch.JSchException; +import com.jcraft.jsch.Session; + +/** + * SSH utils class. + * + * All this code has be taken from ant optionnal ssh task. + * + * Use full for: + * - scpTo command + * - scpFrom command + * - exec command + * + * @author chatellier + * @version $Revision: 1.0 $ + * + * Last update : $Date: 13 janv. 2009 $ + * By : $Author: chatellier $ + */ +public class SSHUtils { + + /** log. */ + private static Log log = LogFactory.getLog(SSHUtils.class); + + protected static final byte LINE_FEED = 0x0a; + protected static final int BUFFER_SIZE = 1024; + private static final int HUNDRED_KILOBYTES = 102400; + + /** Utility class */ + protected SSHUtils() { + + } + + /** + * Exec command on remote server. + * + * @param session opened valid session + * @param command command to exec + * @throws SSHException + */ + public static void exec(Session session, String command) + throws SSHException { + + try { + // exec previous command + Channel channel = session.openChannel("exec"); + ((ChannelExec) channel).setCommand(command); + + BufferedReader br = new BufferedReader(new InputStreamReader( + channel.getInputStream())); + channel.connect(); + String line = null; + while (true) { + while ((line = br.readLine()) != null) { + if (log.isInfoEnabled()) { + log.info("Remote output : " + line); + } + } + if (channel.isClosed()) { + if (log.isInfoEnabled()) { + log.info("JSch channel exit-status: " + + channel.getExitStatus()); + } + break; + } + try { + Thread.sleep(500); + } catch (Exception ee) { + } + } + channel.disconnect(); + // end read buffer + } catch (JSchException e) { + throw new SSHException("I/O error while executing command", e); + } catch (IOException e) { + throw new SSHException("I/O error while executing command", e); + } + + } + + /** + * Download a local file from remote server. + * + * @param session opened valid jsch session + * @param remoteFileName remote file name to download + * @param localFile local file name to download into + * + * @throws SSHException + */ + public static void scpFrom(Session session, String remoteFileName, + File localFile) throws SSHException { + + String command = "scp -f -r \"" + remoteFileName + "\""; + + ChannelExec channel = null; + try { + channel = (ChannelExec) session.openChannel("exec"); + channel.setCommand(command); + + // get I/O streams for remote scp + OutputStream out = channel.getOutputStream(); + InputStream in = channel.getInputStream(); + + channel.connect(); + + sendAck(out); + startRemoteCpProtocol(in, out, localFile); + } catch (IOException e) { + throw new SSHException(e); + } catch (JSchException e) { + throw new SSHException(e); + } finally { + if (channel != null) { + channel.disconnect(); + } + } + } + + /** + * Upload file on remote server. + * + * @param session opened valid session + * @param localFile file to upload + * @param remoteFilePath remote file path + * + * @throws SSHException + */ + public static void scpTo(Session session, File localFile, + String remoteFilePath) throws SSHException { + + try { + doSingleTransfer(session, localFile, remoteFilePath); + } catch (IOException e) { + throw new SSHException(e); + } catch (JSchException e) { + throw new SSHException(e); + } + } + + /** + * Send an ack. + * @param out the output stream to use + * @throws IOException on error + */ + protected static void sendAck(OutputStream out) throws IOException { + byte[] buf = new byte[1]; + buf[0] = 0; + out.write(buf); + out.flush(); + } + + /** + * Reads the response, throws a BuildException if the response + * indicates an error. + * @param in the input stream to use + * @throws IOException on I/O error + */ + protected static void waitForAck(InputStream in) throws IOException, + SSHException { + int b = in.read(); + + // b may be 0 for success, + // 1 for error, + // 2 for fatal error, + + if (b == -1) { + // didn't receive any response + throw new SSHException("No response from server"); + } else if (b != 0) { + StringBuffer sb = new StringBuffer(); + + int c = in.read(); + while (c > 0 && c != '\n') { + sb.append((char) c); + c = in.read(); + } + + if (b == 1) { + throw new SSHException("server indicated an error: " + + sb.toString()); + } else if (b == 2) { + throw new SSHException("server indicated a fatal error: " + + sb.toString()); + } else { + throw new SSHException("unknown response, code " + b + + " message: " + sb.toString()); + } + } + } + + /** + * Track progress every 10% if 100kb < filesize < 1mb. For larger + * files track progress for every percent transmitted. + * @param filesize the size of the file been transmitted + * @param totalLength the total transmission size + * @param percentTransmitted the current percent transmitted + * @return the percent that the file is of the total + */ + protected static int trackProgress(long filesize, long totalLength, + int percentTransmitted) { + + // CheckStyle:MagicNumber OFF + int percent = (int) Math.round(Math + .floor((totalLength / (double) filesize) * 100)); + + if (percent > percentTransmitted) { + if (filesize < 1048576) { + if (percent % 5 == 0) { + if (percent == 100) { + System.out.println(" 100%"); + } else { + System.out.print("*"); + } + } + } else { + if (percent == 50) { + System.out.println(" 50%"); + } else if (percent == 100) { + System.out.println(" 100%"); + } else { + System.out.print("."); + } + } + } + // CheckStyle:MagicNumber ON + + return percent; + } + + protected static void startRemoteCpProtocol(InputStream in, + OutputStream out, File localFile) throws IOException, SSHException { + File startFile = localFile; + while (true) { + // C0644 filesize filename - header for a regular file + // T time 0 time 0\n - present if perserve time. + // D directory - this is the header for a directory. + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + while (true) { + int read = in.read(); + if (read < 0) { + return; + } + if ((byte) read == LINE_FEED) { + break; + } + stream.write(read); + } + String serverResponse = stream.toString("UTF-8"); + if (serverResponse.charAt(0) == 'C') { + parseAndFetchFile(serverResponse, startFile, out, in); + } else if (serverResponse.charAt(0) == 'D') { + startFile = parseAndCreateDirectory(serverResponse, startFile); + sendAck(out); + } else if (serverResponse.charAt(0) == 'E') { + startFile = startFile.getParentFile(); + sendAck(out); + } else if (serverResponse.charAt(0) == '\01' + || serverResponse.charAt(0) == '\02') { + // this indicates an error. + throw new IOException(serverResponse.substring(1)); + } + } + } + + protected static File parseAndCreateDirectory(String serverResponse, + File localFile) { + int start = serverResponse.indexOf(" "); + // appears that the next token is not used and it's zero. + start = serverResponse.indexOf(" ", start + 1); + String directoryName = serverResponse.substring(start + 1); + if (localFile.isDirectory()) { + File dir = new File(localFile, directoryName); + dir.mkdir(); + log.debug("Creating: " + dir); + return dir; + } + return null; + } + + protected static void parseAndFetchFile(String serverResponse, + File localFile, OutputStream out, InputStream in) + throws IOException, SSHException { + int start = 0; + int end = serverResponse.indexOf(" ", start + 1); + start = end + 1; + end = serverResponse.indexOf(" ", start + 1); + long filesize = Long.parseLong(serverResponse.substring(start, end)); + String filename = serverResponse.substring(end + 1); + log.debug("Receiving: " + filename + " : " + filesize); + File transferFile = (localFile.isDirectory()) ? new File(localFile, + filename) : localFile; + fetchFile(transferFile, filesize, out, in); + waitForAck(in); + sendAck(out); + } + + protected static void fetchFile(File localFile, long filesize, + OutputStream out, InputStream in) throws IOException { + byte[] buf = new byte[BUFFER_SIZE]; + sendAck(out); + + // read a content of lfile + FileOutputStream fos = new FileOutputStream(localFile); + int length; + long totalLength = 0; + + // only track progress for files larger than 100kb in verbose mode + boolean trackProgress = filesize > HUNDRED_KILOBYTES; + // since filesize keeps on decreasing we have to store the + // initial filesize + long initFilesize = filesize; + int percentTransmitted = 0; + + try { + while (true) { + length = in.read(buf, 0, (BUFFER_SIZE < filesize) ? BUFFER_SIZE + : (int) filesize); + if (length < 0) { + throw new EOFException("Unexpected end of stream."); + } + fos.write(buf, 0, length); + filesize -= length; + totalLength += length; + if (filesize == 0) { + break; + } + + if (trackProgress) { + percentTransmitted = trackProgress(initFilesize, + totalLength, percentTransmitted); + } + } + } finally { + fos.flush(); + fos.close(); + } + } + + protected static void doSingleTransfer(Session session, File localFile, + String remoteFilePath) throws IOException, JSchException, + SSHException { + + String command = "scp -t \"" + remoteFilePath + "\""; + ChannelExec channel = (ChannelExec) session.openChannel("exec"); + channel.setCommand(command); + try { + + OutputStream out = channel.getOutputStream(); + InputStream in = channel.getInputStream(); + + channel.connect(); + + waitForAck(in); + sendFileToRemote(localFile, in, out); + } finally { + channel.disconnect(); + } + } + + protected static void sendFileToRemote(File localFile, InputStream in, + OutputStream out) throws IOException, SSHException { + // send "C0644 filesize filename", where filename should not include '/' + long filesize = localFile.length(); + String command = "C0644 " + filesize + " "; + command += localFile.getName(); + command += "\n"; + + out.write(command.getBytes()); + out.flush(); + + waitForAck(in); + + // send a content of lfile + FileInputStream fis = new FileInputStream(localFile); + byte[] buf = new byte[BUFFER_SIZE]; + long totalLength = 0; + + // only track progress for files larger than 100kb in verbose mode + boolean trackProgress = filesize > HUNDRED_KILOBYTES; + // since filesize keeps on decreasing we have to store the + // initial filesize + long initFilesize = filesize; + int percentTransmitted = 0; + + try { + while (true) { + int len = fis.read(buf, 0, buf.length); + if (len <= 0) { + break; + } + out.write(buf, 0, len); + totalLength += len; + + if (trackProgress) { + percentTransmitted = trackProgress(initFilesize, + totalLength, percentTransmitted); + } + } + out.flush(); + sendAck(out); + waitForAck(in); + } finally { + fis.close(); + } + } +} + +/** + * SSHException. + * + * @author chatellier + * @version $Revision: 1.0 $ + * + * Last update : $Date: 14 janv. 2009 $ + * By : $Author: chatellier $ + */ +class SSHException extends Exception { + + /** serialVersionUID. */ + private static final long serialVersionUID = -198651402309210758L; + + /** + * Constructs a new exception with null as its detail message. + */ + public SSHException() { + super(); + } + + /** + * Constructs a new exception with the specified detail message. + * + * @param message message + * @param cause cause + */ + public SSHException(String message, Throwable cause) { + super(message, cause); + } + + /** + * Constructs a new exception with the specified detail message and cause. + * + * @param message message + */ + public SSHException(String message) { + super(message); + } + + /** + * Constructs a new exception with the specified cause and a detail message + * of (cause==null ? null : cause.toString()) (which typically contains the + * class and detail message of cause). + * + * @param cause cause + */ + public SSHException(Throwable cause) { + super(cause); + } + +} \ No newline at end of file Modified: isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SshSimulatorLauncher.java =================================================================== --- isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SshSimulatorLauncher.java 2009-01-13 19:48:00 UTC (rev 1738) +++ isis-fish/trunk/src/main/java/fr/ifremer/isisfish/simulator/launcher/SshSimulatorLauncher.java 2009-01-14 14:22:05 UTC (rev 1739) @@ -25,11 +25,7 @@ import java.awt.GridBagLayout; import java.awt.Insets; import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; import java.rmi.RemoteException; import java.util.List; @@ -43,8 +39,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import com.jcraft.jsch.Channel; -import com.jcraft.jsch.ChannelExec; import com.jcraft.jsch.JSch; import com.jcraft.jsch.JSchException; import com.jcraft.jsch.Session; @@ -132,11 +126,16 @@ // si l'upload a fonctionné if (remotePath != null) { message(control, _("isisfish.simulation.remote.message.launch")); + // just start a thread checkSimulationProgression(sshSession, simulationService, control); launchSimulation(sshSession, simulationService, control, remotePath); + + // recuperation des resultats + message(control, _("isisfish.simulation.remote.message.downloadresults")); + downloadResults(sshSession, control.getId()); // force thread to stop synchronized(control) { @@ -262,38 +261,7 @@ log.debug("command is : " + command); } - // exec previous commande - Channel channel = session.openChannel("exec"); - ((ChannelExec) channel).setCommand(command); - - // read buffer - // seems to not work without it - InputStream in = channel.getInputStream(); - channel.connect(); - byte[] tmp = new byte[1024]; - while (true) { - while (in.available() > 0) { - int i = in.read(tmp, 0, 1024); - if (i < 0) - break; - if (log.isInfoEnabled()) { - log.info(new String(tmp, 0, i)); - } - } - if (channel.isClosed()) { - if (log.isInfoEnabled()) { - log.info("JSch channel exit-status: " - + channel.getExitStatus()); - } - break; - } - try { - Thread.sleep(1000); - } catch (Exception ee) { - } - } - channel.disconnect(); - // end read buffer + SSHUtils.exec(session, command); } } catch (IOException e) { if (log.isErrorEnabled()) { @@ -301,14 +269,17 @@ e); } } + catch (SSHException e) { + if (log.isErrorEnabled()) { + log.error(_("Error while uploading public key to remote serveur authorized_keys"), + e); + } + } } /** * Upload simulation file to server. * - * ScpTo code taken from : - * http://www.jcraft.com/jsch/examples/ScpTo.java - * * @param session already open valid ssh session * @param simulationFile simulation file to upload * @@ -316,141 +287,78 @@ */ protected String uploadSimulation(Session session, File simulationFile) { - // return flag - String remotePath = null; + String localPath = simulationFile.getAbsolutePath(); + + // Copy simulation file in same arch as local arch + // on windows, it's a bad idee :))) + // copy it always on caparmor remote temp dir ? + // /tmp ? + // TODO check it - // file info - String filePath = simulationFile.getAbsolutePath(); + String remotePath = "/tmp/"; - FileInputStream fis = null; - try { + if(localPath.lastIndexOf("/") > 0) { + remotePath += localPath.substring(localPath.lastIndexOf("/") + 1); + } + else if(localPath.lastIndexOf("\\") > 0) { // windows + remotePath += localPath.substring(localPath.lastIndexOf("\\") + 1); + } + else { + remotePath += localPath; + } - // exec 'scp -t rfile' remotely - String command = "scp -p -t " + filePath; - Channel channel = session.openChannel("exec"); - ((ChannelExec) channel).setCommand(command); - - // get I/O streams for remote scp - OutputStream out = channel.getOutputStream(); - InputStream in = channel.getInputStream(); - - channel.connect(); - - // Check input stream validity - if (checkAck(in) == 0) { - - if (log.isDebugEnabled()) { - log.debug("Uploading " + filePath); - } - - // send "C0644 filesize filename", where filename should not include '/' - long filesize = (new File(filePath)).length(); - command = "C0644 " + filesize + " "; - if (filePath.lastIndexOf('/') > 0) { - command += filePath - .substring(filePath.lastIndexOf('/') + 1); - } else { - command += filePath; - } - command += "\n"; - out.write(command.getBytes()); - out.flush(); - - if (checkAck(in) == 0) { - - // send a content of lfile - fis = new FileInputStream(filePath); - byte[] buf = new byte[1024]; - while (true) { - int len = fis.read(buf, 0, buf.length); - if (len <= 0) { - break; - } - out.write(buf, 0, len); //out.flush(); - } - fis.close(); - fis = null; - // send '\0' - buf[0] = 0; - out.write(buf, 0, 1); - out.flush(); - if (checkAck(in) == 0) { - remotePath = filePath; - } - out.close(); - - channel.disconnect(); - } - } - - } catch (JSchException e) { + try { + SSHUtils.scpTo(session, simulationFile, remotePath); + } + catch(SSHException e) { if (log.isErrorEnabled()) { - log.error(_("isisfish.error.simulation.remote.upload", - filePath)); + log.error(_("Error while uploading simulation"), + e); } - } catch (IOException e) { - if (log.isErrorEnabled()) { - log.error(_("isisfish.error.simulation.remote.upload", - filePath)); - } - } finally { - try { - if (fis != null) { - fis.close(); - } - } catch (IOException e) { - // exception ignoree - } + + remotePath = null; } return remotePath; } /** - * Check input stream validity. + * Download resulation results. * - * Code taken from Jsch samples. - * - * @param in input stream + * @return downloaded temp file (file have to be manually deleted) */ - protected int checkAck(InputStream in) throws IOException { - int b = in.read(); - // b may be 0 for success, - // 1 for error, - // 2 for fatal error, - // -1 - if (b == 0) { - return b; + protected File downloadResults(Session session, String simulationId) { + + // simulation directory + File localFile = new File(IsisFish.config.getDatabaseDirectory(), + SimulationStorage.SIMULATION_PATH); + + if(log.isDebugEnabled()) { + log.debug("Downloading results in " + localFile.getAbsolutePath()); } - if (b == -1) { - return b; - } + + // build remote file path + // FIXME this path should be given by remote IsisFish app + // TODO to change + String remoteFile = IsisFish.config.getSimulatorSshDataPath(); + remoteFile += "/" + SimulationStorage.SIMULATION_PATH; + remoteFile += "/" + simulationId; - if (b == 1 || b == 2) { - StringBuffer sb = new StringBuffer(); - int c; - do { - c = in.read(); - sb.append((char) c); - } while (c != '\n'); - if (b == 1) { // error - if (log.isErrorEnabled()) { - log.error(_( - "isisfish.error.simulation.remote.upload.stream", - sb.toString())); - } + try { + SSHUtils.scpFrom(session, remoteFile, localFile); + } + catch(SSHException e) { + // error can append because control file does'nt exist yet + if (log.isDebugEnabled()) { + log.debug(_("Error while downloading simulation control")); } - if (b == 2) { // fatal error - if (log.isFatalEnabled()) { - log.fatal(_( - "isisfish.error.simulation.remote.upload.stream", - sb.toString())); - } - } + + localFile = null; } - return b; + + return localFile; } - + /** * Launch simulation on remote server. * @@ -477,44 +385,15 @@ if (log.isDebugEnabled()) { log.debug("Launch : " + command); } - - Channel channel = session.openChannel("exec"); - ((ChannelExec) channel).setCommand(command); - - channel.setInputStream(null); - + try { - InputStream in = channel.getInputStream(); - - channel.connect(); - - byte[] tmp = new byte[1024]; - while (true) { - while (in.available() > 0) { - int i = in.read(tmp, 0, 1024); - if (i < 0) { - break; - } - if (log.isInfoEnabled()) { - log.info(_("isisfish.error.simulation.remote.output") + new String(tmp, 0, i)); - } - } - if (channel.isClosed()) { - if (log.isInfoEnabled()) { - log.info("JSch channel exit-status: " - + channel.getExitStatus()); - } - break; - } - try { - Thread.sleep(1000); - } catch (Exception ee) { - } + SSHUtils.exec(session, command); + } + catch(SSHException e) { + if (log.isErrorEnabled()) { + log.error(_("Error while launching simulation"), + e); } - channel.disconnect(); - } catch (IOException e) { - throw new SimulationException( - _("isisfish.error.simulation.remote.launch"), e); } } @@ -623,9 +502,6 @@ * Download remote simulation control file and store * its content into temp file. * - * ScpFrom code taken from : - * http://www.jcraft.com/jsch/examples/ScpFrom.java - * * @return downloaded temp file (file have to be manually deleted) * @throws IOException */ @@ -643,123 +519,19 @@ remoteFile += "/control"; // local tmp file - File tempFile = File.createTempFile(simulationId, "control"); - FileOutputStream fos=null; - + localFile = File.createTempFile(simulationId, "control"); + try { - // exec 'scp -f rfile' remotely - String command = "scp -f \"" + remoteFile + "\""; - Channel channel = sshSession.openChannel("exec"); - ((ChannelExec) channel).setCommand(command); - - // get I/O streams for remote scp - OutputStream out = channel.getOutputStream(); - InputStream in = channel.getInputStream(); - - channel.connect(); - - byte[] buf = new byte[1024]; - - // send '\0' - buf[0] = 0; - out.write(buf, 0, 1); - out.flush(); - - while (true) { - int c = checkAck(in); - if (c != 'C') { - break; - } - - // read '0644 ' - in.read(buf, 0, 5); - - long filesize = 0L; - while (true) { - if (in.read(buf, 0, 1) < 0) { - // error - break; - } - if (buf[0] == ' ') { - break; - } - filesize = filesize * 10L + (buf[0] - '0'); - } - - //String file = null; - for (int i = 0;; i++) { - in.read(buf, i, 1); - if (buf[i] == (byte) 0x0a) { - //file = new String(buf, 0, i); - break; - } - } - - // send '\0' - buf[0] = 0; - out.write(buf, 0, 1); - out.flush(); - - // read a content of lfile - fos = new FileOutputStream(tempFile); - int foo; - while (true) { - if (buf.length < filesize) { - foo = buf.length; - } - else { - foo = (int) filesize; - } - foo = in.read(buf, 0, foo); - if (foo < 0) { - // error - break; - } - fos.write(buf, 0, foo); - filesize -= foo; - if (filesize == 0L) { - break; - } - } - fos.close(); - fos = null; - - if (checkAck(in) == 0) { - localFile = tempFile; - } - - // send '\0' - buf[0] = 0; - out.write(buf, 0, 1); - out.flush(); - } + SSHUtils.scpFrom(sshSession, remoteFile, localFile); } - catch(IOException e){ - if(log.isErrorEnabled()) { - log.error("I/O error while downloading control file", e); + catch(SSHException e) { + if (log.isErrorEnabled()) { + log.error(_("Error while launching simulation"), + e); } - } catch (JSchException e) { - if(log.isErrorEnabled()) { - log.error("Jsch error while downloading control file", e); - } + localFile = null; } - finally { - try{ - if(fos!=null) { - fos.close(); - } - } - catch(Exception ignored) { - } - } - - if(localFile == null) { - // an arror occurs - // so remove temp file - tempFile.delete(); - } - return localFile; } }