[Isis-fish-commits] r4164 - in trunk/src/main/java/fr/ifremer/isisfish: datastore export util

18 Dec 2014

Author: bpoussin
Date: 2014-12-18 11:19:15 +0000 (Thu, 18 Dec 2014)
New Revision: 4164

Url: http://forge.codelutin.com/projects/isis-fish/repository/revisions/4164

Log:
fixes #6341: Am?\195?\169liorer l'affichage des informations d'une simulation


Modified:
   trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultStorageCSV.java
   trunk/src/main/java/fr/ifremer/isisfish/datastore/SimulationInformation.java
   trunk/src/main/java/fr/ifremer/isisfish/export/ExportHelper.java
   trunk/src/main/java/fr/ifremer/isisfish/util/IsisCacheBackendOnGuava.java

Modified: trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultStorageCSV.java
===================================================================

--- trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultStorageCSV.java	2014-12-17 15:35:57 UTC (rev 4163)
+++ trunk/src/main/java/fr/ifremer/isisfish/datastore/ResultStorageCSV.java	2014-12-18 11:19:15 UTC (rev 4164)
@@ -28,6 +28,7 @@
 import fr.ifremer.isisfish.simulator.SimulationContext;
 import fr.ifremer.isisfish.simulator.SimulationControl;
 import fr.ifremer.isisfish.types.TimeStep;
+import fr.ifremer.isisfish.util.ConverterUtil;
 import fr.ifremer.isisfish.util.SimpleParser;
 import java.io.BufferedWriter;
 import java.io.File;
@@ -36,6 +37,7 @@
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.LineNumberReader;
+import java.io.OutputStream;
 import java.io.OutputStreamWriter;
 import java.io.PrintWriter;
 import java.nio.charset.Charset;
@@ -51,11 +53,13 @@
 import java.util.zip.GZIPOutputStream;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
+import org.apache.commons.io.output.CountingOutputStream;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.nuiton.math.matrix.MatrixIterator;
 import org.nuiton.math.matrix.MatrixND;
+import org.nuiton.util.StringUtil;
 
 /**
  * Ecrit les resultats dans des fichiers au format pseudo csv compresse (gz)
@@ -124,6 +128,8 @@
     protected int numberOfResultReadInCache = 0;
     /** le nombre de resultat lu sur le disque */
     protected int numberOfResultReadOnDisk = 0;
+    /** le nombre d'octet ecris sur le disque */
+    protected long bytesWritten = 0;
     /**
      * le plus vieux pas de temps demande par rapport au pas de temps courant
      * lu sur le disque
@@ -156,10 +162,10 @@
 
         return String.format(
                 "ResultStorageCSV:\n"
-                        + "\t%s results written to disk on a total of %s\n"
+                        + "\t%s results written to disk on a total of %s (for %s)\n"
                         + "\t%s results read (%s in cache and %s on disk)\n"
                         + "%s",
-                numberOfResultOnDisk, numberOfResult,
+                numberOfResultOnDisk, numberOfResult, StringUtil.convertMemory(bytesWritten),
                 numberOfResultAsked, numberOfResultReadInCache, numberOfResultReadOnDisk,
                 oldest);
     }
@@ -340,28 +346,6 @@
         return result;
     }
 
-//    @Override
-//    protected Map<TimeStep, MatrixND> readResult(String name) {
-//        File dir = getMatrixDirectory(name);
-//        String[] files = dir.list();
-//
-//        Map<TimeStep, MatrixND> result = new HashMap<TimeStep, MatrixND>();
-//        for(String file : files) {
-//            try {
-//                TimeStep step = getTimeStep(file);
-//                if (step != null) {
-//                    MatrixND mat = readResult(step, name);
-//                    if (mat != null) {
-//                        result.put(step, mat);
-//                    }
-//                }
-//            } catch (Exception eee) {
-//                log.error("Can't read result file: " + file, eee);
-//            }
-//        }
-//        return result;
-//    }
-
     /**
      * Indique s'il faut sauver sur disque les resultats
      * @param step
@@ -403,9 +387,10 @@
             File file = getMatrixFile(step, name);
             file.getParentFile().mkdirs();
             PrintWriter out = null;
+            CountingOutputStream counter = null;
             try {
                 out = new PrintWriter(new BufferedWriter(new OutputStreamWriter(
-                        new GZIPOutputStream(new FileOutputStream(file)),
+                        new GZIPOutputStream(counter = new CountingOutputStream(new FileOutputStream(file))),
                         charset
                 )));
 
@@ -448,6 +433,9 @@
                 throw new IsisFishRuntimeException("Can't write result: " + file, eee);
             } finally {
                 IOUtils.closeQuietly(out);
+                if (counter != null) {
+                    this.bytesWritten += counter.getByteCount();
+                }
             }
         }
     }
@@ -471,4 +459,8 @@
         return result;
     }
 
+    public long getBytesWritten() {
+        return bytesWritten;
+    }
+
 }

Modified: trunk/src/main/java/fr/ifremer/isisfish/datastore/SimulationInformation.java
===================================================================
--- trunk/src/main/java/fr/ifremer/isisfish/datastore/SimulationInformation.java	2014-12-17 15:35:57 UTC (rev 4163)
+++ trunk/src/main/java/fr/ifremer/isisfish/datastore/SimulationInformation.java	2014-12-18 11:19:15 UTC (rev 4164)
@@ -38,15 +38,18 @@
 import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
+import java.util.TreeMap;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.time.DurationFormatUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.nuiton.util.StringUtil;
 
 /**
  * Cette classe permet de conserver des informations sur le deroulement d'une
@@ -68,6 +71,7 @@
     private static final String START_SIMULATION = "simulationStart";
     private static final String END_SIMULATION = "simulationEnd";
     private static final String EXPORT_TIME = "exportTime";
+    private static final String EXPORT_SIZE = "exportSize";
     private static final String RULE_TIME = "ruleTime";
     private static final String RULE_TIME_INIT = RULE_TIME + ".init";
     private static final String RULE_TIME_PRE = RULE_TIME + ".pre";
@@ -117,13 +121,21 @@
         String result = "Simulation Information:\n";
 
         // date start/end
-        result += "Start: " + dateFormat.format(getSimulationStart())
-                + " End: " + dateFormat.format(getSimulationEnd()) + "\n";
+        Date start = getSimulationStart();
+        Date end = getSimulationEnd();
 
+        result += "Duration: " + StringUtil.convertTime(start.getTime()*1000000, end.getTime()*1000000)
+                + " Start: " + dateFormat.format(start)
+                + " End: " + dateFormat.format(end) + "\n";
+
+        result += "\n";
+
         // exports
         Map<String, Long> exportTime = getExportTimes();
         if (exportTime.size() > 0) {
-            result += "Export time:\n";
+            result += "Export time: " +
+                    DurationFormatUtils.formatDuration(getAllExportTime(),
+                                "s'.'S") + "s\n";
             for (Map.Entry<String, Long> entry : exportTime.entrySet()) {
                 result += "\t"
                         + entry.getKey()
@@ -133,6 +145,18 @@
             }
         }
 
+        // exports
+        Map<String, Long> exportSize = getExportSizes();
+        if (exportSize.size() > 0) {
+            result += "Export size: " + StringUtil.convertMemory(getAllExportSize()) + "\n";
+            for (Map.Entry<String, Long> entry : exportSize.entrySet()) {
+                result += "\t"
+                        + entry.getKey()
+                        + " : "
+                        + StringUtil.convertMemory(entry.getValue()) + "\n";
+            }
+        }
+
         // rules
         Set<String> ruleNames = getRuleNames();
         if (ruleNames.size() > 0) {
@@ -279,37 +303,51 @@
         setInfo(EXPORT_TIME + "." + exportName, String.valueOf(time));
     }
 
+    public void addExportSize(String exportName, long size) {
+        setInfo(EXPORT_SIZE + "." + exportName, String.valueOf(size));
+    }
+
+    public void addAllExportTime(long time) {
+        setInfo(EXPORT_TIME, String.valueOf(time));
+    }
+
+    public void addAllExportSize(long size) {
+        setInfo(EXPORT_SIZE, String.valueOf(size));
+    }
+
     /**
      * Get all export time in map.
      * 
      * @return a map with all export time
      */
     protected Map<String, Long> getExportTimes() {
-        Map<String, Long> result = new HashMap<String, Long>();
+        Map<String, Long> result = new TreeMap<String, Long>();
         for (String key : info.stringPropertyNames()) {
             if (key.startsWith(EXPORT_TIME + ".")) {
                 String exportName = key.substring(EXPORT_TIME.length() + 1);
-                String t = info.getProperty(key);
-                Long value = null;
-                if (t != null) {
-                    try {
-                        value = Long.valueOf(t);
-                    } catch (NumberFormatException eee) {
-                        if (log.isWarnEnabled()) {
-                            log.warn(t("isisfish.error.parse.long", t), eee);
-                        }
-                    }
-                }
-                if (value == null) {
-                    value = Long.valueOf(0);
-                }
-                result.put(exportName, value);
+                result.put(exportName, getExportTime(exportName));
             }
         }
         return result;
     }
 
     /**
+     * Get all export size in map.
+     *
+     * @return a map with all export size
+     */
+    protected Map<String, Long> getExportSizes() {
+        Map<String, Long> result = new TreeMap<String, Long>();
+        for (String key : info.stringPropertyNames()) {
+            if (key.startsWith(EXPORT_SIZE + ".")) {
+                String exportName = key.substring(EXPORT_SIZE.length() + 1);
+                result.put(exportName, getExportSize(exportName));
+            }
+        }
+        return result;
+    }
+
+    /**
      * Get all export time in map.
      * 
      * @return a map with all export time
@@ -319,8 +357,8 @@
         return getExportTimes();
     }
 
-    public long getExportTime(String exportName) {
-        String t = info.getProperty(EXPORT_TIME + "." + exportName);
+    protected long getLong(String prop) {
+        String t = info.getProperty(prop);
         long result = 0;
         if (t != null) {
             try {
@@ -332,8 +370,33 @@
             }
         }
         return result;
+
     }
 
+    public long getExportTime(String exportName) {
+        String prop = EXPORT_TIME;
+        if (StringUtils.isNotEmpty(exportName)) {
+            prop += "." + exportName;
+        }
+        return getLong(prop);
+    }
+
+    public long getExportSize(String exportName) {
+        String prop = EXPORT_SIZE;
+        if (StringUtils.isNotEmpty(exportName)) {
+            prop += "." + exportName;
+        }
+        return getLong(prop);
+    }
+    
+    public long getAllExportTime() {
+        return getExportTime(null);
+    }
+
+    public long getAllExportSize() {
+        return getExportSize(null);
+    }
+
     /**
      * Add rule time.
      * 
@@ -548,6 +611,6 @@
      * @param info new info
      */
     public void addInformation(String info) {
-        setInfo(OTHER_INFO, getInformation() + info + "\n");
+        setInfo(OTHER_INFO, getInformation() + info + "\n\n");
     }
 }

Modified: trunk/src/main/java/fr/ifremer/isisfish/export/ExportHelper.java
===================================================================
--- trunk/src/main/java/fr/ifremer/isisfish/export/ExportHelper.java	2014-12-17 15:35:57 UTC (rev 4163)
+++ trunk/src/main/java/fr/ifremer/isisfish/export/ExportHelper.java	2014-12-18 11:19:15 UTC (rev 4164)
@@ -46,6 +46,7 @@
 import java.util.Properties;
 import java.util.zip.GZIPOutputStream;
 import org.apache.commons.io.IOUtils;
+import org.apache.commons.io.output.CountingOutputStream;
 import org.apache.commons.lang3.StringUtils;
 
 
@@ -83,22 +84,27 @@
 
         // on ne compte plus ici les temps d'instanciations
         // deplacer dans SimulationExportResultWrapper#afterSimulation(SimulationContext)
-        
+
+        long writtenAll = 0;
         long timeStart = System.currentTimeMillis();
         for (Export export : exports) {
             String exportName = ExportStorage.getName(export);
+            long written = 0;
             long time = System.currentTimeMillis();
             try {
-                exportToFile(simulation, destdir, export);
+                written = exportToFile(simulation, destdir, export);
+                writtenAll += written;
             } catch (Exception eee) {
                 if (log.isWarnEnabled()) {
                     log.warn("Can't export object: " + exportName, eee);
                 }
             }
+            simulation.getInformation().addExportSize(exportName, written);
             simulation.getInformation().addExportTime(exportName,
                     System.currentTimeMillis() - time);
         }
-        simulation.getInformation().addExportTime("All export",
+        simulation.getInformation().addAllExportSize(writtenAll);
+        simulation.getInformation().addAllExportTime(
                 System.currentTimeMillis() - timeStart);
     }
 
@@ -108,10 +114,14 @@
      * @param simulation la simulation pour lequel il faut faire les exports
      * @param destdir le repertoire dans lequel il faut ecrire les exports
      * @param export le nom des exports a faire
+     *
+     * @return number of byte written on disk
+     *
      * @throws Exception si une erreur survient
      */
-    protected static void exportToFile(SimulationStorage simulation,
+    protected static long exportToFile(SimulationStorage simulation,
             File destdir, Export export) throws Exception {
+        long result = 0;
 
         String filename = export.getExportFilename();
         String extension = export.getExtensionFilename();
@@ -131,9 +141,11 @@
         }
 
         Writer out = null;
+        CountingOutputStream counter = null;
         try {
 
             OutputStream os = new FileOutputStream(file);
+            os = counter = new CountingOutputStream(os);
 
             // if compression is needed by extension, add compression writer
             if (StringUtils.endsWithIgnoreCase(extension, COMPRESSION_EXTENSION)) {
@@ -146,7 +158,11 @@
             export.export(simulation, out);
         } finally {
             IOUtils.closeQuietly(out);
+            if (counter != null) {
+                result = counter.getByteCount();
+            }
         }
+        return result;
     }
 
     /**

Modified: trunk/src/main/java/fr/ifremer/isisfish/util/IsisCacheBackendOnGuava.java
===================================================================
--- trunk/src/main/java/fr/ifremer/isisfish/util/IsisCacheBackendOnGuava.java	2014-12-17 15:35:57 UTC (rev 4163)
+++ trunk/src/main/java/fr/ifremer/isisfish/util/IsisCacheBackendOnGuava.java	2014-12-18 11:19:15 UTC (rev 4164)
@@ -299,7 +299,7 @@
                     StringUtil.convertMemory(biggestStepCached + 500 * 1024 *1024));
         }
         String result = String.format("%s - \n%s"
-                + "\tmaxMemory: %s, cacheMemory: %s, totalCached: %s, numberStep: %s, numberEntries: %s\n"
+                + "\tinitial cache size: %s, cache size reevaluated: %s, totalCached: %s, numberStep: %s, numberEntries: %s\n"
                 + "\tStep min: %s, max: %s, mean: %s, standard deviation: %s\n"
                 + "\tEntry min: %s, max: %s, mean: %s, standard deviation: %s\n",
                 trick,

    

[Isis-fish-commits] r4164 - in trunk/src/main/java/fr/ifremer/isisfish: datastore export util

bpoussin＠users.forge.codelutin.com