This is an automated email from the git hooks/post-receive script. New commit to branch develop in repository echobase. See https://gitlab.nuiton.org/codelutin/echobase.git commit 3495e8a1d06126c5c68c3966252a35cd97d96e50 Author: Tony CHEMIT <chemit@codelutin.com> Date: Mon May 9 12:27:59 2016 +0200 Bien vérifier que les fichiers importés sont correctes (See #8170) --- .../fr/ifremer/echobase/io/EchoBaseIOUtil.java | 82 ++++++++++++++++++++++ .../service/importdata/CheckFileException.java | 32 +++++++++ .../service/importdata/ImportException.java | 26 +++++++ .../actions/ImportDataActionSupport.java | 36 +++++++++- .../ui/actions/importData/DownloadInputFile.java | 73 +++++++++++++++++++ .../ui/actions/workingDb/DownloadFileSupport.java | 17 ++--- .../main/resources/config/struts-importData.xml | 11 +++ .../resources/i18n/echobase-ui_en_GB.properties | 4 ++ .../resources/i18n/echobase-ui_fr_FR.properties | 4 ++ .../jsp/importData/importDataActionResult.jsp | 62 ++++++++++++++++ .../WEB-INF/jsp/importData/resultCommonImport.jsp | 2 +- 11 files changed, 334 insertions(+), 15 deletions(-) diff --git a/echobase-domain/src/main/java/fr/ifremer/echobase/io/EchoBaseIOUtil.java b/echobase-domain/src/main/java/fr/ifremer/echobase/io/EchoBaseIOUtil.java index 9f1c4ea..3c7c006 100644 --- a/echobase-domain/src/main/java/fr/ifremer/echobase/io/EchoBaseIOUtil.java +++ b/echobase-domain/src/main/java/fr/ifremer/echobase/io/EchoBaseIOUtil.java @@ -29,8 +29,10 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.nuiton.util.GZUtil; import org.nuiton.util.ZipUtil; +import java.io.BufferedInputStream; import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.File; @@ -41,9 +43,14 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.LineNumberReader; +import java.io.OutputStream; import java.io.OutputStreamWriter; +import java.nio.charset.StandardCharsets; +import java.sql.Blob; +import java.sql.SQLException; import java.util.Collection; import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; @@ -281,6 +288,81 @@ public class EchoBaseIOUtil { } } + public static File copyAndGzipFileToDirectory(File sourceFile, File targetDirectory) throws IOException { + + try (InputStream inputStream = java.nio.file.Files.newInputStream(sourceFile.toPath())) { + + FileUtils.forceMkdir(targetDirectory); + File targetFile = new File(targetDirectory, sourceFile.getName()); + + try (OutputStream outputStream = new GZIPOutputStream(java.nio.file.Files.newOutputStream(targetFile.toPath()))) { + IOUtils.copy(inputStream, outputStream); + } + + return targetFile; + + } + + } + + public static long fileLength(File sourceFile) throws IOException { + + boolean gzip; + + try (InputStream inputStream = new BufferedInputStream(new FileInputStream(sourceFile))) { + gzip = GZUtil.isGzipStream(inputStream); + } + + long fileLength; + + if (gzip) { + + try (GZIPInputStream inputStream = new GZIPInputStream(new FileInputStream(sourceFile))) { + + byte[] bytes = IOUtils.toString(inputStream, StandardCharsets.UTF_8).getBytes(); + fileLength = new String(bytes).length(); + + } + + } else { + + fileLength = sourceFile.length(); + + } + + return fileLength; + + } + + public static long blobLength(Blob sourceFile) throws IOException, SQLException { + + boolean gzip; + + try (InputStream inputStream = new BufferedInputStream(sourceFile.getBinaryStream())) { + gzip = GZUtil.isGzipStream(inputStream); + } + + long fileLength; + + if (gzip) { + + try (GZIPInputStream inputStream = new GZIPInputStream(sourceFile.getBinaryStream())) { + + byte[] bytes = IOUtils.toString(inputStream, StandardCharsets.UTF_8).getBytes(); + fileLength = new String(bytes).length(); + + } + + } else { + + fileLength = sourceFile.length(); + + } + + return fileLength; + + } + protected EchoBaseIOUtil() { // no instance of helper class } diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/CheckFileException.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/CheckFileException.java new file mode 100644 index 0000000..310281d --- /dev/null +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/CheckFileException.java @@ -0,0 +1,32 @@ +package fr.ifremer.echobase.services.service.importdata; + +/** + * Created on 07/05/16. + * + * @author Tony Chemit - chemit@codelutin.com + */ +public class CheckFileException extends RuntimeException { + + private final String importFilename; + private final String expectedFileSuffix; + private final String actualFileSuffix; + + public CheckFileException(String message, String importFilename, String expectedFileSuffix, String actualFileSuffix) { + super(message); + this.importFilename = importFilename; + this.expectedFileSuffix = expectedFileSuffix; + this.actualFileSuffix = actualFileSuffix; + } + + public String getImportFilename() { + return importFilename; + } + + public String getExpectedFileSuffix() { + return expectedFileSuffix; + } + + public String getActualFileSuffix() { + return actualFileSuffix; + } +} diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/ImportException.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/ImportException.java index 2b53f47..e9e1aa3 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/ImportException.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/ImportException.java @@ -47,4 +47,30 @@ public class ImportException extends Exception { public ImportException(Locale locale, InputFile inputfile, Exception importError) { this(l(locale, "echobase.importError.fromFile", inputfile.getFile().getName(), importError.getMessage()), importError); } + + public boolean isCheckImportError() { + Throwable cause = getCause(); + return cause != null && cause instanceof CheckFileException; + } + + public String getImportFilename() { + return isCheckImportError() ? ((CheckFileException) getCause()).getImportFilename() : null; + } + + public String getExpectedFileName() { + return isCheckImportError() ? "expected-" + ((CheckFileException) getCause()).getImportFilename() : null; + } + + public String getActualFileName() { + return isCheckImportError() ? "actual-" + ((CheckFileException) getCause()).getImportFilename() : null; + } + + public String getExpectedFileSuffix() { + return isCheckImportError() ? ((CheckFileException) getCause()).getExpectedFileSuffix() : null; + } + + public String getActualFileSuffix() { + return isCheckImportError() ? ((CheckFileException) getCause()).getActualFileSuffix() : null; + } + } diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ImportDataActionSupport.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ImportDataActionSupport.java index bab2269..8811fdd 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ImportDataActionSupport.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ImportDataActionSupport.java @@ -9,13 +9,16 @@ import fr.ifremer.echobase.entities.data.Cell; import fr.ifremer.echobase.entities.data.Data; import fr.ifremer.echobase.entities.references.DataMetadata; import fr.ifremer.echobase.entities.references.DataQuality; +import fr.ifremer.echobase.io.EchoBaseIOUtil; import fr.ifremer.echobase.io.InputFile; import fr.ifremer.echobase.services.csv.BatchCsvExport; import fr.ifremer.echobase.services.service.UserDbPersistenceService; +import fr.ifremer.echobase.services.service.importdata.CheckFileException; import fr.ifremer.echobase.services.service.importdata.ImportDataFileResult; import fr.ifremer.echobase.services.service.importdata.ImportException; import fr.ifremer.echobase.services.service.importdata.configurations.ImportDataConfigurationSupport; import fr.ifremer.echobase.services.service.importdata.contexts.ImportDataContextSupport; +import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -27,6 +30,7 @@ import org.nuiton.topia.persistence.TopiaEntity; import java.io.BufferedWriter; import java.io.Closeable; +import java.io.File; import java.io.IOException; import java.io.Reader; import java.nio.charset.StandardCharsets; @@ -186,7 +190,11 @@ public abstract class ImportDataActionSupport<M extends ImportDataConfigurationS flushImportedExport(result); } - checkImport(result.getProcessedImportFile(), result.getImportedExportFile()); + try { + checkImport(result, result.getProcessedImportFile(), result.getImportedExportFile()); + } catch (Exception e) { + throw new ImportException(importDataContext.getLocale(), inputFile, e); + } flushResult(result); @@ -305,11 +313,35 @@ public abstract class ImportDataActionSupport<M extends ImportDataConfigurationS return csvImport = Import.newImport(importModel, reader); } - protected void checkImport(InputFile processedExportFile, InputFile importedExportFile) { + protected void checkImport(ImportDataFileResult result, InputFile processedExportFile, InputFile importedExportFile) { //TODO checkt that processed and imported file are the same if (log.isInfoEnabled()) { log.info("Check processed export file: " + processedExportFile + " vs imported export file: " + importedExportFile); } + try { + File dataDirectory = persistenceService.getConfiguration().getDataDirectory(); + String expectedContent = new String(Files.readAllBytes(processedExportFile.getFile().toPath()), StandardCharsets.UTF_8); + String actualContent = new String(Files.readAllBytes(importedExportFile.getFile().toPath()), StandardCharsets.UTF_8); + if (!expectedContent.equals(actualContent)) { + + // On recopie les fichiers pour que l'utilisateur puisse les télécharger... + File workingDirectory = new File(dataDirectory, "import-check-" + System.nanoTime()); + FileUtils.forceMkdir(workingDirectory); + + File targetProcessedExportFile = EchoBaseIOUtil.copyAndGzipFileToDirectory(processedExportFile.getFile(), workingDirectory); + File targetImportedExportFile = EchoBaseIOUtil.copyAndGzipFileToDirectory(importedExportFile.getFile(), workingDirectory); + + int directoryPrefix = dataDirectory.getAbsolutePath().length() + 1; + String targetProcessedExportFileSuffix = targetProcessedExportFile.getAbsolutePath().substring(directoryPrefix); + String targetImportedExportFileSuffix = targetImportedExportFile.getAbsolutePath().substring(directoryPrefix); + + throw new CheckFileException("Check file for file: " + importedExportFile.getFileName() + " failed, contents are not the same.", result.getImportFile().getName(), targetProcessedExportFileSuffix, targetImportedExportFileSuffix); + + } + } catch (IOException e) { + throw new EchoBaseTechnicalException("Could not read files", e); + } + } protected final Locale getLocale() { diff --git a/echobase-ui/src/main/java/fr/ifremer/echobase/ui/actions/importData/DownloadInputFile.java b/echobase-ui/src/main/java/fr/ifremer/echobase/ui/actions/importData/DownloadInputFile.java new file mode 100644 index 0000000..dc64eb3 --- /dev/null +++ b/echobase-ui/src/main/java/fr/ifremer/echobase/ui/actions/importData/DownloadInputFile.java @@ -0,0 +1,73 @@ +package fr.ifremer.echobase.ui.actions.importData; + +import fr.ifremer.echobase.io.EchoBaseIOUtil; +import fr.ifremer.echobase.ui.actions.EchoBaseActionSupport; + +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import java.util.zip.GZIPInputStream; + +/** + * Created on 08/05/16. + * + * @author Tony Chemit - chemit@codelutin.com + */ +public class DownloadInputFile extends EchoBaseActionSupport { + + private static final long serialVersionUID = 1L; + + /** Prefix of input file to download. */ + protected String fileSuffix; + + /** Input stream of the file to download. */ + protected transient InputStream inputStream; + + /** Length of the file to download. */ + protected long contentLength; + + /** Content type of the file to download. */ + protected String contentType; + + private String filename; + + public void setFileSuffix(String fileSuffix) { + this.fileSuffix = fileSuffix; + } + + public void setFilename(String filename) { + this.filename = filename; + } + + public String getFilename() { + return filename; + } + + public InputStream getInputStream() { + return inputStream; + } + + public long getContentLength() { + return contentLength; + } + + public String getContentType() { + return contentType; + } + + @Override + public String execute() throws Exception { + + File dataDirectory = getEchoBaseApplicationContext().getConfiguration().getDataDirectory(); + File file = new File(dataDirectory, fileSuffix); + + contentType = "text/csv"; + contentLength = EchoBaseIOUtil.fileLength(file); + + inputStream = new GZIPInputStream(new FileInputStream(file), 65535); + + return SUCCESS; + + } + +} diff --git a/echobase-ui/src/main/java/fr/ifremer/echobase/ui/actions/workingDb/DownloadFileSupport.java b/echobase-ui/src/main/java/fr/ifremer/echobase/ui/actions/workingDb/DownloadFileSupport.java index dad8dfb..0151552 100644 --- a/echobase-ui/src/main/java/fr/ifremer/echobase/ui/actions/workingDb/DownloadFileSupport.java +++ b/echobase-ui/src/main/java/fr/ifremer/echobase/ui/actions/workingDb/DownloadFileSupport.java @@ -22,9 +22,9 @@ package fr.ifremer.echobase.ui.actions.workingDb; */ import fr.ifremer.echobase.entities.ImportFile; +import fr.ifremer.echobase.io.EchoBaseIOUtil; import fr.ifremer.echobase.services.service.UserDbPersistenceService; import fr.ifremer.echobase.ui.actions.EchoBaseActionSupport; -import org.apache.commons.io.IOUtils; import javax.inject.Inject; import java.io.InputStream; @@ -92,18 +92,11 @@ public abstract class DownloadFileSupport extends EchoBaseActionSupport { filename = getFilename(importFile); - contentType = "application/text"; + contentType = "text/csv"; + Blob blob = getFile(importFile); + contentLength = EchoBaseIOUtil.blobLength(blob); - Blob file = getFile(importFile); - { - byte[] bytes; - try (InputStream binaryStream = file.getBinaryStream()) { - bytes = IOUtils.toString(new GZIPInputStream(binaryStream)).getBytes(); - } - contentLength = bytes.length; - } - - inputStream = new GZIPInputStream(file.getBinaryStream(), 65535); + inputStream = new GZIPInputStream(blob.getBinaryStream(), 65535); return SUCCESS; diff --git a/echobase-ui/src/main/resources/config/struts-importData.xml b/echobase-ui/src/main/resources/config/struts-importData.xml index a383702..67c1dc5 100644 --- a/echobase-ui/src/main/resources/config/struts-importData.xml +++ b/echobase-ui/src/main/resources/config/struts-importData.xml @@ -68,6 +68,17 @@ <result>/WEB-INF/jsp/importData/result{1}Import.jsp</result> </action> + <!-- Download checked file file --> + <action name="downloadCheckedFile" method="execute" + class="fr.ifremer.echobase.ui.actions.importData.DownloadInputFile"> + <interceptor-ref name="prepareParamsStackLogguedWithDb"/> + <result type="stream"> + <param name="contentType">${contentType}</param> + <param name="contentLength">${contentLength}</param> + <param name="contentDisposition">attachment; filename="${filename}"</param> + </result> + </action> + <!-- Get all vessels used by any transects of a voyage --> <action name="getVesselsForVoyage" class="fr.ifremer.echobase.ui.actions.importData.GetVesselsForVoyage"> diff --git a/echobase-ui/src/main/resources/i18n/echobase-ui_en_GB.properties b/echobase-ui/src/main/resources/i18n/echobase-ui_en_GB.properties index b05accf..728b201 100644 --- a/echobase-ui/src/main/resources/i18n/echobase-ui_en_GB.properties +++ b/echobase-ui/src/main/resources/i18n/echobase-ui_en_GB.properties @@ -254,6 +254,7 @@ echobase.error.workingDbConfiguration.createOnlyOnPostgresql=Database creation i echobase.error.workingDbConfiguration.description.required=Mandatory description echobase.error.workingDbConfiguration.url.already.exists=A configuration with same url already exists echobase.error.workingDbConfiguration.url.required=Mandatory jdbc url +echobase.exported.file=Imported file echobase.header.request.result=SQL query result echobase.importDb.freeResult=Free db import was succesful in %s. echobase.importDb.referentialResult=Referential db import was succesul in %s. @@ -391,6 +392,9 @@ echobase.message.warnEmbeddedApplicationInProgress=Please do not close the windo echobase.message.warnExportInProgress=Please do not close the window to access the export file echobase.message.warnImportInProgress=Please do not close the window to access the imported file echobase.message.warnRemoveDataInProgress=Please do not close the window to access remove data result +echobase.mismatch.import.file=A file (%s) was just imported, but the check file produced is not equals to input file. +echobase.mismatch.import.file.error= +echobase.processed.file=File to import echobase.spatialView.loading=Loading data... echobase.title.confirm.deleteImportLogs=Delete data echobase.title.confirm.deleteQuery=Delete a query diff --git a/echobase-ui/src/main/resources/i18n/echobase-ui_fr_FR.properties b/echobase-ui/src/main/resources/i18n/echobase-ui_fr_FR.properties index 7a38c9b..01edb49 100644 --- a/echobase-ui/src/main/resources/i18n/echobase-ui_fr_FR.properties +++ b/echobase-ui/src/main/resources/i18n/echobase-ui_fr_FR.properties @@ -255,6 +255,7 @@ echobase.error.workingDbConfiguration.createOnlyOnPostgresql=La création d'une echobase.error.workingDbConfiguration.description.required=Description obligatoire echobase.error.workingDbConfiguration.url.already.exists=Une configuration existe déjà avec cette url echobase.error.workingDbConfiguration.url.required=Url jdbc obligatoire +echobase.exported.file=Fichier importé echobase.header.request.result=Résultat de la requête SQL echobase.importDb.freeResult=L'import de type libre s'est déroulé avec succès en %s. echobase.importDb.referentialResult=L'import de type référentiel s'est déroulé avec succès en %s. @@ -394,6 +395,9 @@ echobase.message.warnEmbeddedApplicationInProgress=Merci de ne pas fermer la fen echobase.message.warnExportInProgress=Merci de ne pas fermer la fenêtre pour pouvoir accéder aux résultats de l'export echobase.message.warnImportInProgress=Merci de ne pas fermer la fenêtre pour pouvoir accéder aux résultats de l'import echobase.message.warnRemoveDataInProgress=Merci de ne pas fermer la fenètre pour pouvoir accéder aux résultats de la suppression de l'import +echobase.mismatch.import.file=Le fichier (%s) vient d'être importé mais le fichier de validation correspondant produit n'est pas identique à ce fichier. +echobase.mismatch.import.file.error=Problème de validation de données importées. +echobase.processed.file=Fichier à importer echobase.spatialView.loading=Chargement des données... echobase.title.confirm.deleteImportLogs=Supprimer des données echobase.title.confirm.deleteQuery=Supprimer une requête diff --git a/echobase-ui/src/main/webapp/WEB-INF/jsp/importData/importDataActionResult.jsp b/echobase-ui/src/main/webapp/WEB-INF/jsp/importData/importDataActionResult.jsp new file mode 100644 index 0000000..ae085ec --- /dev/null +++ b/echobase-ui/src/main/webapp/WEB-INF/jsp/importData/importDataActionResult.jsp @@ -0,0 +1,62 @@ +<%-- + #%L + EchoBase :: UI + %% + Copyright (C) 2011 Ifremer, Codelutin + %% + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + #L% + --%> +<%@page contentType="text/html" pageEncoding="UTF-8" %> +<%@ taglib prefix="s" uri="/struts-tags" %> + +<s:if test="error.checkImportError"> + + <fieldset> + <legend> + <s:text name="echobase.mismatch.import.file.error"/> + </legend> + + <p> + <s:text name="echobase.mismatch.import.file"> + <s:param value="%{error.importFilename}"/> + </s:text> + </p> + + <br/> + + <div> + <ul> + <li> + <s:a action="downloadCheckedFile" namespace="/importData"> + <s:param name="fileSuffix" value="%{error.expectedFileSuffix}"/> + <s:param name="filename" value="%{error.expectedFileName}"/> + <s:text name="echobase.processed.file"/> + </s:a> + </li> + <li> + <s:a action="downloadCheckedFile" namespace="/importData"> + <s:param name="fileSuffix" value="%{error.actualFileSuffix}"/> + <s:param name="filename" value="%{error.actualFileName}"/> + <s:text name="echobase.exported.file"/> + </s:a> + </li> + </ul> + </div> + + </fieldset> + +</s:if> + +<%@ include file="/WEB-INF/includes/actionResult.jsp" %> diff --git a/echobase-ui/src/main/webapp/WEB-INF/jsp/importData/resultCommonImport.jsp b/echobase-ui/src/main/webapp/WEB-INF/jsp/importData/resultCommonImport.jsp index b1b20cf..1dfe589 100644 --- a/echobase-ui/src/main/webapp/WEB-INF/jsp/importData/resultCommonImport.jsp +++ b/echobase-ui/src/main/webapp/WEB-INF/jsp/importData/resultCommonImport.jsp @@ -25,4 +25,4 @@ <s:text name="echobase.common.importType.common"/> </title> -<%@ include file="/WEB-INF/includes/actionResult.jsp" %> +<%@ include file="importDataActionResult.jsp" %> -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.