branch develop updated (88a3288 -> 51ff2fd)
This is an automated email from the git hooks/post-receive script. New change to branch develop in repository coselmar. See https://gitlab.nuiton.org/codelutin/coselmar.git from 88a3288 refs #9206 Fix NPE when CSV file is not found in zip + review tool description new 51ff2fd refs #9206 Review file mangement The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "adds" were already present in the repository and have only been added to this reference. Detailed log of new commits: commit 51ff2fd3487938b37be8e53bd7b8ac6a4925fd17 Author: Yannick Martel <martel@©odelutin.com> Date: Fri Jun 9 12:25:16 2017 +0200 refs #9206 Review file mangement Summary of changes: .../coselmar/services/v1/DocumentsWebService.java | 221 +++++++++++---------- 1 file changed, 112 insertions(+), 109 deletions(-) -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.
This is an automated email from the git hooks/post-receive script. New commit to branch develop in repository coselmar. See https://gitlab.nuiton.org/codelutin/coselmar.git commit 51ff2fd3487938b37be8e53bd7b8ac6a4925fd17 Author: Yannick Martel <martel@©odelutin.com> Date: Fri Jun 9 12:25:16 2017 +0200 refs #9206 Review file mangement --- .../coselmar/services/v1/DocumentsWebService.java | 221 +++++++++++---------- 1 file changed, 112 insertions(+), 109 deletions(-) diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/DocumentsWebService.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/DocumentsWebService.java index f522483..93e4b24 100644 --- a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/DocumentsWebService.java +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/DocumentsWebService.java @@ -54,6 +54,7 @@ import fr.ifremer.coselmar.services.errors.UnauthorizedException; import fr.ifremer.coselmar.services.indexation.DocumentsIndexationService; import fr.ifremer.coselmar.services.indexation.TikaUtils; import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.lucene.queryparser.classic.ParseException; @@ -979,137 +980,139 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { return importResult; } - // Get descriptions.csv : it should contains all DocumentBean information - ZipEntry descriptionEntry = zipFile.getEntry(DESCRIPTION_CSV_FILE_NAME); - if (descriptionEntry == null) { - importResult.setSystemErrorType(MassiveDocumentsImportResult.ImportSystemErrorType.UNABLE_TO_FIND_CSV); - return importResult; - } - - InputStream descriptionInputStream; + InputStream descriptionInputStream = null; try { - descriptionInputStream = zipFile.getInputStream(descriptionEntry); - } catch (IOException e) { - String message = String.format("Unable to read '%s' from zip file", DESCRIPTION_CSV_FILE_NAME); - if (log.isErrorEnabled()) { - log.error(message, e); + // Get descriptions.csv : it should contains all DocumentBean information + ZipEntry descriptionEntry = zipFile.getEntry(DESCRIPTION_CSV_FILE_NAME); + if (descriptionEntry == null) { + importResult.setSystemErrorType(MassiveDocumentsImportResult.ImportSystemErrorType.UNABLE_TO_FIND_CSV); + return importResult; } - importResult.setSystemErrorType(MassiveDocumentsImportResult.ImportSystemErrorType.UNABLE_TO_FIND_CSV); - return importResult; // Direct break : cannot continue without this file - } - // Now, read CSV ... - DocumentImportModel csvModel = new DocumentImportModel(); - Import<DocumentBean> importer = Import.newImport(csvModel, descriptionInputStream); + try { + descriptionInputStream = zipFile.getInputStream(descriptionEntry); + } catch (IOException e) { + String message = String.format("Unable to read '%s' from zip file", DESCRIPTION_CSV_FILE_NAME); + if (log.isErrorEnabled()) { + log.error(message, e); + } + importResult.setSystemErrorType(MassiveDocumentsImportResult.ImportSystemErrorType.UNABLE_TO_FIND_CSV); + return importResult; // Direct break : cannot continue without this file + } - File dataDirectory = getCoselmarServicesConfig().getDataDirectory(); - String dataPath = dataDirectory.getAbsolutePath(); - String zipTempPath = dataPath + File.separator + DateUtil.formatDate(getNow(), "yyyyMMddHHmm"); - Path dir = Paths.get(zipTempPath); - try { - Files.createDirectories(dir); - } catch (IOException e) { - if (log.isErrorEnabled()) { - String message = "Unable to create temp path for zip import"; - log.error(message, e); + // Now, read CSV ... + DocumentImportModel csvModel = new DocumentImportModel(); + Import<DocumentBean> importer = Import.newImport(csvModel, descriptionInputStream); + + File dataDirectory = getCoselmarServicesConfig().getDataDirectory(); + String dataPath = dataDirectory.getAbsolutePath(); + String zipTempPath = dataPath + File.separator + DateUtil.formatDate(getNow(), "yyyyMMddHHmm"); + Path dir = Paths.get(zipTempPath); + try { + Files.createDirectories(dir); + } catch (IOException e) { + if (log.isErrorEnabled()) { + String message = "Unable to create temp path for zip import"; + log.error(message, e); + } + throw new CoselmarTechnicalException("Unable to unzip file : error with unzip directory", e); } - throw new CoselmarTechnicalException("Unable to unzip file : error with unzip directory", e); - } - // ... and read each entries and retrieve associated File - try { - for (DocumentBean documentBean : importer) { - FileInfos fileInfos = new FileInfos(); - String fileName = StringUtils.strip(documentBean.getFileName()); - // cf https://forge.codelutin.com/issues/9205?issue_count=18&issue_position=4&next_issue_id=9198&prev_issue_id=9206#note-2 - documentBean.setPrivacy(Privacy.PUBLIC.name()); - - ZipEntry zipFileEntry = zipFile.getEntry(fileName); - if (zipFileEntry == null) { - importResult.addMissingFile(fileName); - - } else { - try { - InputStream zipFileInputStream = zipFile.getInputStream(zipFileEntry); - fileInfos.setFileName(fileName); - String fileMimeType = TikaUtils.getFileMimeType(fileName); - String futureFilePath = getDocumentFileDestPath(currentUser, fileName); - - String storageFileName = getFileStorageName(fileName); - - // Push file stream in temporary folder - File zipEntryFile = new File(zipTempPath + File.separator + storageFileName); - zipEntryFile.createNewFile(); - Files.copy(zipFileInputStream, zipEntryFile.toPath(), StandardCopyOption.REPLACE_EXISTING); - - // Create fileInfos - fileInfos.setFileName(fileName); - fileInfos.setFinalFilePath(futureFilePath); - fileInfos.setActualFilePath(zipEntryFile.getAbsolutePath()); - fileInfos.setMimeType(fileMimeType); - - // Create document - createDocument(documentBean, fileInfos, currentUser); + // ... and read each entries and retrieve associated File + try { + for (DocumentBean documentBean : importer) { + FileInfos fileInfos = new FileInfos(); + String fileName = StringUtils.strip(documentBean.getFileName()); + // cf https://forge.codelutin.com/issues/9205?issue_count=18&issue_position=4&next_issue_id=9198&prev_issue_id=9206#note-2 + documentBean.setPrivacy(Privacy.PUBLIC.name()); + + ZipEntry zipFileEntry = zipFile.getEntry(fileName); + if (zipFileEntry == null) { + importResult.addMissingFile(fileName); + } else { try { - zipFileInputStream.close(); + InputStream zipFileInputStream = zipFile.getInputStream(zipFileEntry); + fileInfos.setFileName(fileName); + String fileMimeType = TikaUtils.getFileMimeType(fileName); + String futureFilePath = getDocumentFileDestPath(currentUser, fileName); + + String storageFileName = getFileStorageName(fileName); + + // Push file stream in temporary folder + File zipEntryFile = new File(zipTempPath + File.separator + storageFileName); + zipEntryFile.createNewFile(); + Files.copy(zipFileInputStream, zipEntryFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + + // Create fileInfos + fileInfos.setFileName(fileName); + fileInfos.setFinalFilePath(futureFilePath); + fileInfos.setActualFilePath(zipEntryFile.getAbsolutePath()); + fileInfos.setMimeType(fileMimeType); + + // Create document + createDocument(documentBean, fileInfos, currentUser); + + try { + zipFileInputStream.close(); + } catch (IOException e) { + if (log.isErrorEnabled()) { + log.error("Unable to close stream from ZipEntry : " + fileName, e); + } + } } catch (IOException e) { + String message = String.format("Unable to retrieve '%s' from zip file", fileName); if (log.isErrorEnabled()) { - log.error("Unable to close stream from ZipEntry : " + fileName, e); + log.error(message, e); } + importResult.setSystemErrorType(MassiveDocumentsImportResult.ImportSystemErrorType.UNABLE_TO_READ_ZIP_ENTRY); + importResult.addMissingFile(fileName); } - } catch (IOException e) { - String message = String.format("Unable to retrieve '%s' from zip file", fileName); - if (log.isErrorEnabled()) { - log.error(message, e); - } - importResult.setSystemErrorType(MassiveDocumentsImportResult.ImportSystemErrorType.UNABLE_TO_READ_ZIP_ENTRY); - importResult.addMissingFile(fileName); } - } - } - } catch (ImportRuntimeException ire) { - if (log.isErrorEnabled()) { - log.error("Error with CSV file", ire); - } - importResult.setSystemErrorType(MassiveDocumentsImportResult.ImportSystemErrorType.UNABLE_TO_READ_CSV); - return importResult; - } finally { - // Close importer - importer.close(); - // and csv stream - try { - descriptionInputStream.close(); - } catch (IOException e) { + } + } catch (ImportRuntimeException ire) { if (log.isErrorEnabled()) { - log.error("Unable to close 'descriptions.csv' input stream from zip", e); + log.error("Error with CSV file", ire); } + importResult.setSystemErrorType(MassiveDocumentsImportResult.ImportSystemErrorType.UNABLE_TO_READ_CSV); + return importResult; + } finally { + // Close importer + importer.close(); } - } - if (importResult.isSuccess()) { - // All is ok ! - try { - // Ok, let move all files from temp storage to real one ! - File tmpDir = new File(zipTempPath); - FileUtils.copyDirectory(tmpDir, new File(getUserDocumentPath(currentUser))); - FileUtils.deleteDirectory(tmpDir); - commit(); - } catch (IOException e) { - // Big problem if we can't move files into real folder ! - String message = String.format("Unable to move files from temp folder '%s' to final folder", zipTempPath); - if (log.isErrorEnabled()) { - log.error(message, e); + + if (importResult.isSuccess()) { + // All is ok ! + try { + // Ok, let move all files from temp storage to real one ! + File tmpDir = new File(zipTempPath); + FileUtils.copyDirectory(tmpDir, new File(getUserDocumentPath(currentUser))); + FileUtils.deleteDirectory(tmpDir); + commit(); + } catch (IOException e) { + // Big problem if we can't move files into real folder ! + String message = String.format("Unable to move files from temp folder '%s' to final folder", zipTempPath); + if (log.isErrorEnabled()) { + log.error(message, e); + } + refreshDocumentsIndex(); + throw new CoselmarTechnicalException(message, e); } + + } else { + // Something wrong happened... rollback, and refresh lucene to avoid new data + rollback(); refreshDocumentsIndex(); - throw new CoselmarTechnicalException(message, e); } - } else { - // Something wrong happened... rollback, and refresh lucene to avoid new data - rollback(); - refreshDocumentsIndex(); + } finally { + // Be sure we close description stream + IOUtils.closeQuietly(descriptionInputStream); + // and zip file + IOUtils.closeQuietly(zipFile); } + return importResult; } -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.
participants (1)
-
codelutin.com scm