branch feature/9206-upload-validation-zip-documents created (now fdce731)
This is an automated email from the git hooks/post-receive script. New change to branch feature/9206-upload-validation-zip-documents in repository coselmar. See https://gitlab.nuiton.org/codelutin/coselmar.git at fdce731 refs-50 #9206 First draft for error management in Documents Zip import This branch includes the following new commits: new dc2afeb Prepare import massif de document new d5a986f refs-30 #9206 First draft for Documents Zip management new fdce731 refs-50 #9206 First draft for error management in Documents Zip import The 3 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "adds" were already present in the repository and have only been added to this reference. Detailed log of new commits: commit fdce731043b1840716f3cf526ff78523dbfbee9a Author: Yannick Martel <martel@©odelutin.com> Date: Tue Jun 6 12:05:37 2017 +0200 refs-50 #9206 First draft for error management in Documents Zip import commit d5a986fb539a3ea59c61b4d954b6f92e1ed09368 Author: Yannick Martel <martel@©odelutin.com> Date: Fri Jun 2 16:54:23 2017 +0200 refs-30 #9206 First draft for Documents Zip management commit dc2afeb93523010de541f4cba659d38420b1a51a Author: Yannick Martel <martel@©odelutin.com> Date: Thu Jun 1 14:29:22 2017 +0200 Prepare import massif de document -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.
This is an automated email from the git hooks/post-receive script. New commit to branch feature/9206-upload-validation-zip-documents in repository coselmar. See https://gitlab.nuiton.org/codelutin/coselmar.git commit dc2afeb93523010de541f4cba659d38420b1a51a Author: Yannick Martel <martel@©odelutin.com> Date: Thu Jun 1 14:29:22 2017 +0200 Prepare import massif de document --- .../fr/ifremer/coselmar/beans/DocumentBean.java | 6 + .../coselmar/beans/DocumentImportModel.java | 44 +++++ .../java/fr/ifremer/coselmar/beans/FileInfos.java | 35 ++++ .../coselmar/converter/BeanEntityConverter.java | 4 +- .../coselmar/services/indexation/TikaUtils.java | 1 + .../coselmar/services/v1/AdminWebService.java | 10 +- .../coselmar/services/v1/DocumentsWebService.java | 216 ++++++++++++--------- 7 files changed, 215 insertions(+), 101 deletions(-) diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/DocumentBean.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/DocumentBean.java index f8c855f..cc724c8 100644 --- a/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/DocumentBean.java +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/DocumentBean.java @@ -64,6 +64,12 @@ public class DocumentBean implements Serializable { // If restricted, could have a list of restricted user protected Set<UserBean> authorizedUsers; + public static DocumentBean newEmptyInstance() { + return new DocumentBean(); + } + + private DocumentBean(){} + public DocumentBean(String id, String name, String ownerName, String ownerId, String privacy, Date depositDate, Collection<String> keywords, String type, String summary, String language, Date publicationDate, diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/DocumentImportModel.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/DocumentImportModel.java new file mode 100644 index 0000000..f8b5749 --- /dev/null +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/DocumentImportModel.java @@ -0,0 +1,44 @@ +package fr.ifremer.coselmar.beans; + +import fr.ifremer.coselmar.persistence.entity.Privacy; +import org.nuiton.csv.ValueParser; +import org.nuiton.csv.ext.AbstractImportModel; + +import java.text.ParseException; + +/** + * @author ymartel (martel@codelutin.com) + */ +public class DocumentImportModel extends AbstractImportModel<DocumentBean> { + + protected static final ValueParser<Privacy> DOCUMENT_PRIVACY_PARSER = new ValueParser<Privacy>() { + @Override + public Privacy parse(String value) throws ParseException { + return Privacy.valueOf(value.toUpperCase()); + } + }; + + public DocumentImportModel() { + super(';'); + + newMandatoryColumn("name", "name"); + newMandatoryColumn("type", "type"); + newMandatoryColumn("privacy", "privacy", DOCUMENT_PRIVACY_PARSER); + newMandatoryColumn("keywords", "keywords"); + newOptionalColumn("authors", "authors"); + newMandatoryColumn("summary", "summary"); + newOptionalColumn("license", "license"); + newOptionalColumn("copyright", "copyright"); + newOptionalColumn("language", "language"); + newOptionalColumn("publicationDate", "publicationDate"); + newOptionalColumn("comment", "comment"); + newMandatoryColumn("fileName", "fileName"); + newMandatoryColumn("citation", "citation"); + } + + @Override + public DocumentBean newEmptyInstance() { + return DocumentBean.newEmptyInstance(); + } + +} diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/FileInfos.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/FileInfos.java new file mode 100644 index 0000000..5bebc04 --- /dev/null +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/FileInfos.java @@ -0,0 +1,35 @@ +package fr.ifremer.coselmar.beans; + +/** + * @author ymartel (martel@codelutin.com) + */ +public class FileInfos { + + protected String FileName; + protected String FilePath; + protected String MimeType; + + public String getFileName() { + return FileName; + } + + public void setFileName(String fileName) { + FileName = fileName; + } + + public String getFilePath() { + return FilePath; + } + + public void setFilePath(String filePath) { + FilePath = filePath; + } + + public String getMimeType() { + return MimeType; + } + + public void setMimeType(String mimeType) { + MimeType = mimeType; + } +} diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/converter/BeanEntityConverter.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/converter/BeanEntityConverter.java index 9af8254..b8d8e2a 100644 --- a/coselmar-rest/src/main/java/fr/ifremer/coselmar/converter/BeanEntityConverter.java +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/converter/BeanEntityConverter.java @@ -309,7 +309,7 @@ public class BeanEntityConverter { * </ul> * If documentBean is null, simply return null Document. */ - public static Document fromBean(DocumentBean documentBean) { + public static Document fromSearchBean(DocumentBean documentBean) { if (documentBean == null) { return null; } @@ -337,7 +337,7 @@ public class BeanEntityConverter { String privacy = documentBean.getPrivacy(); if (StringUtils.isNotBlank(privacy)) { - document.setPrivacy(Privacy.valueOf(privacy)); + document.setPrivacy(Privacy.valueOf(privacy.toUpperCase())); } Date publicationDate = documentBean.getPublicationDate(); diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/indexation/TikaUtils.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/indexation/TikaUtils.java index 4a517a4..68e9ca4 100644 --- a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/indexation/TikaUtils.java +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/indexation/TikaUtils.java @@ -51,4 +51,5 @@ public class TikaUtils { } return fileContent; } + } diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/AdminWebService.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/AdminWebService.java index c4ad5f1..40c4534 100644 --- a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/AdminWebService.java +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/AdminWebService.java @@ -24,18 +24,14 @@ package fr.ifremer.coselmar.services.v1; * #L% */ -import java.io.File; -import java.io.IOException; -import java.util.List; - import fr.ifremer.coselmar.beans.DocumentBean; import fr.ifremer.coselmar.beans.QuestionBean; import fr.ifremer.coselmar.beans.UserWebToken; import fr.ifremer.coselmar.converter.BeanEntityConverter; +import fr.ifremer.coselmar.exceptions.CoselmarTechnicalException; import fr.ifremer.coselmar.persistence.entity.CoselmarUserRole; import fr.ifremer.coselmar.persistence.entity.Document; import fr.ifremer.coselmar.persistence.entity.Question; -import fr.ifremer.coselmar.exceptions.CoselmarTechnicalException; import fr.ifremer.coselmar.services.CoselmarWebServiceSupport; import fr.ifremer.coselmar.services.errors.InvalidCredentialException; import fr.ifremer.coselmar.services.errors.UnauthorizedException; @@ -44,7 +40,9 @@ import fr.ifremer.coselmar.services.indexation.QuestionsIndexationService; import fr.ifremer.coselmar.services.indexation.TikaUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; -import org.apache.tika.exception.TikaException; + +import java.io.IOException; +import java.util.List; import static org.apache.commons.logging.LogFactory.getLog; diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/DocumentsWebService.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/DocumentsWebService.java index 63e133d..1e52d9c 100644 --- a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/DocumentsWebService.java +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/DocumentsWebService.java @@ -33,6 +33,7 @@ import com.google.common.collect.Lists; import fr.ifremer.coselmar.beans.DocumentBean; import fr.ifremer.coselmar.beans.DocumentSearchBean; import fr.ifremer.coselmar.beans.DocumentSearchExample; +import fr.ifremer.coselmar.beans.FileInfos; import fr.ifremer.coselmar.beans.QuestionBean; import fr.ifremer.coselmar.beans.UserBean; import fr.ifremer.coselmar.beans.UserWebToken; @@ -52,10 +53,8 @@ import fr.ifremer.coselmar.services.indexation.DocumentsIndexationService; import fr.ifremer.coselmar.services.indexation.TikaUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.logging.Log; import org.apache.lucene.queryparser.classic.ParseException; -import org.apache.tika.exception.TikaException; import org.debux.webmotion.server.call.UploadFile; import org.debux.webmotion.server.render.Render; import org.nuiton.topia.persistence.TopiaNoResultException; @@ -67,7 +66,6 @@ import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; -import java.net.URL; import java.util.ArrayList; import java.util.Collection; import java.util.Date; @@ -97,6 +95,7 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { CoselmarUserRole.SUPERVISOR.name(), CoselmarUserRole.EXPERT.name() ); + protected static final String DESCRIPTION_CSV_FILE_NAME = "description.csv"; public DocumentBean getDocument(String documentId) throws InvalidCredentialException, UnauthorizedException { @@ -181,7 +180,7 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { } searchExample.setFullTextSearch(searchBean.getFullTextSearch()); - Document example = BeanEntityConverter.fromBean(searchBean); + Document example = BeanEntityConverter.fromSearchBean(searchBean); searchExample.setExample(example); searchExample.setOwnerName(searchBean.getOwnerName()); @@ -276,7 +275,7 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { searchExample.setFullTextSearch(searchBean.getFullTextSearch()); - Document example = BeanEntityConverter.fromBean(searchBean); + Document example = BeanEntityConverter.fromSearchBean(searchBean); searchExample.setExample(example); searchExample.setOwnerName(searchBean.getOwnerName()); @@ -376,91 +375,13 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { throw new InvalidCredentialException(message); } - String documentName = document.getName(); - String contentType = null; - String filePath = null; - String fileContent = null; - - // If document has a file, manage it ! - if (uploadFile != null) { - Pair<String, String> pathAndContentType = managerDocumentFile(uploadFile, owner); - filePath = pathAndContentType.getLeft(); - contentType = pathAndContentType.getRight(); - fileContent = TikaUtils.getFileContent(filePath); - } - - // Document Metadata - Document documentEntity = getDocumentDao().create(); - - documentEntity.setOwner(owner); - - documentEntity.setName(documentName); - - String privacy = document.getPrivacy().toUpperCase(); - documentEntity.setPrivacy(Privacy.valueOf(privacy)); - - // Manage privacy : if restricted, create an UserGroup with authorized users - if (StringUtils.equals(Privacy.RESTRICTED.name(), privacy)) { - Set<UserBean> authorizedUsers = document.getAuthorizedUsers(); - Set<CoselmarUser> coselmarUsers = retrieveUsers(authorizedUsers); - CoselmarUserGroup restrictedUsers = getCoselmarUserGroupDao().create(); - restrictedUsers.setName(documentEntity.getTopiaId()); - - restrictedUsers.addAllMembers(coselmarUsers); - documentEntity.addRestrictedList(restrictedUsers); - - } - - documentEntity.addAllKeywords(document.getKeywords()); - - Date depositDate = document.getDepositDate(); - if (depositDate != null) { - documentEntity.setDepositDate(new Date(depositDate.getTime())); - } else { - documentEntity.setDepositDate(new Date()); - } - - documentEntity.setType(document.getType()); - documentEntity.setSummary(document.getSummary()); - documentEntity.setLanguage(document.getLanguage()); - documentEntity.setPublicationDate(document.getPublicationDate()); - - - // Legal / copyright part - documentEntity.setAuthors(document.getAuthors()); - documentEntity.setCopyright(document.getCopyright()); - documentEntity.setLicense(document.getLicense()); - - // Document resource part + FileInfos fileInfos = null; if (uploadFile != null) { - documentEntity.setWithFile(true); - documentEntity.setMimeType(contentType); - documentEntity.setFilePath(filePath); - documentEntity.setFileContent(fileContent); - } else { - documentEntity.setWithFile(false); + fileInfos = manageDocumentFile(uploadFile, owner); } - documentEntity.setExternalUrl(document.getExternalUrl()); - - documentEntity.setComment(document.getComment()); - - documentEntity.setCitation(document.getCitation()); + DocumentBean result = createDocument(document, fileInfos, owner); commit(); - DocumentBean result = BeanEntityConverter.toBean(getPersistenceContext().getTopiaIdFactory(), documentEntity); - - DocumentsIndexationService documentsIndexationService = getServicesContext().newService(DocumentsIndexationService.class); - try { - documentsIndexationService.indexDocument(result, fileContent); - if (log.isDebugEnabled()) { - String message = String.format("Document '%s' added to index", documentName); - log.debug(message); - } - } catch (IOException e) { - if (log.isErrorEnabled()) { - log.error("Unable to index new document", e); - } - } return result; @@ -492,9 +413,9 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { // Get owner to place correctly the file CoselmarUser owner = documentEntity.getOwner(); - Pair<String, String> pathAndContentType = managerDocumentFile(uploadFile, owner); - String filePath = pathAndContentType.getLeft(); - String contentType = pathAndContentType.getRight(); + FileInfos fileInfos = manageDocumentFile(uploadFile, owner); + String filePath = fileInfos.getFilePath(); + String contentType = fileInfos.getMimeType(); // Read file content String fileContent = TikaUtils.getFileContent(filePath); @@ -757,13 +678,117 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { //////////////////////////////////////////////////////////////////////////// /** + * From a {@link DocumentBean}, create a {@link Document}. + * If there is a {@link FileInfos}, get the file data. + * + * @return a DocumentBean with all created {@link Document} data. + */ + protected DocumentBean createDocument(DocumentBean documentBean, FileInfos fileInfos, CoselmarUser owner) { + + // Document Metadata + Document documentEntity = createDocumentMetadataFromBean(documentBean, owner); + + // If document has a file, manage it ! + String fileContent = ""; + + if (fileInfos != null) { + String filePath = fileInfos.getFilePath(); + String contentType = fileInfos.getMimeType(); + fileContent = TikaUtils.getFileContent(filePath); + + documentEntity.setWithFile(true); + documentEntity.setMimeType(contentType); + documentEntity.setFilePath(filePath); + documentEntity.setFileContent(fileContent); + } else { + documentEntity.setWithFile(false); + } + + DocumentBean result = BeanEntityConverter.toBean(getPersistenceContext().getTopiaIdFactory(), documentEntity); + + // Indexation job + DocumentsIndexationService documentsIndexationService = getServicesContext().newService(DocumentsIndexationService.class); + try { + documentsIndexationService.indexDocument(result, fileContent); + if (log.isDebugEnabled()) { + String message = String.format("Document '%s' added to index", documentBean.getName()); + log.debug(message); + } + } catch (IOException e) { + if (log.isErrorEnabled()) { + log.error("Unable to index new document", e); + } + } + return result; + } + + /** + * Convert metadata fields from {@link DocumentBean} into a {@link Document} and put default values where no given (such as {@link Document#PROPERTY_DEPOSIT_DATE} for example). + * This also manage owner and restriction but <strong>does not manage</strong> attached file part. + * No commit are done here. + */ + public Document createDocumentMetadataFromBean(DocumentBean documentBean, CoselmarUser owner) { + Preconditions.checkNotNull(documentBean); + + // Document Metadata + Document document = getDocumentDao().create(); + + document.setOwner(owner); + + document.setName(documentBean.getName()); + + String privacy = documentBean.getPrivacy().toUpperCase(); + document.setPrivacy(Privacy.valueOf(privacy)); + + // Manage privacy : if restricted, create an UserGroup with authorized users + if (StringUtils.equals(Privacy.RESTRICTED.name(), privacy)) { + Set<UserBean> authorizedUsers = documentBean.getAuthorizedUsers(); + Set<CoselmarUser> coselmarUsers = retrieveUsers(authorizedUsers); + CoselmarUserGroup restrictedUsers = getCoselmarUserGroupDao().create(); + restrictedUsers.setName(document.getTopiaId()); + + restrictedUsers.addAllMembers(coselmarUsers); + document.addRestrictedList(restrictedUsers); + } + + document.addAllKeywords(documentBean.getKeywords()); + + Date depositDate = documentBean.getDepositDate(); + if (depositDate != null) { + document.setDepositDate(new Date(depositDate.getTime())); + } else { + document.setDepositDate(new Date()); + } + + document.setType(documentBean.getType()); + document.setSummary(documentBean.getSummary()); + document.setLanguage(documentBean.getLanguage()); + Date publicationDate = documentBean.getPublicationDate(); + if (publicationDate != null) { + document.setPublicationDate(new Date(publicationDate.getTime())); + } + + document.setCitation(documentBean.getCitation()); + document.setComment(documentBean.getComment()); + + // Legal / copyright part + document.setAuthors(documentBean.getAuthors()); + document.setCopyright(documentBean.getCopyright()); + document.setLicense(documentBean.getLicense()); + + document.setExternalUrl(documentBean.getExternalUrl()); + + return document; + } + + /** * When a Document is sent, it could have a File part : this manage the * upload file. The file is stored in the user specific directory, and the * contentType of document is returned, cause need in Document Metadata. * * @return the upload file Metadata */ - protected Pair<String, String> managerDocumentFile(UploadFile uploadFile, CoselmarUser owner) { + protected FileInfos manageDocumentFile(UploadFile uploadFile, CoselmarUser owner) { Preconditions.checkNotNull(uploadFile); // Document File @@ -771,9 +796,9 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { File uploadedFile = uploadFile.getFile(); String contentType = uploadFile.getContentType(); - if (log.isInfoEnabled()) { + if (log.isDebugEnabled()) { String message = String.format("File name : %s, content-type : %s", fileName, contentType); - log.info(message); + log.debug(message); } // put the document in the good directory @@ -792,7 +817,12 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { } throw new CoselmarTechnicalException("Internal error during file transfer"); } - return Pair.of(destFile.getAbsolutePath(), contentType); + + FileInfos fileInfos = new FileInfos(); + fileInfos.setFileName(fileName); + fileInfos.setMimeType(contentType); + fileInfos.setFilePath(destFile.getAbsolutePath()); + return fileInfos; } protected String getUserDocumentPath(CoselmarUser user) { -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.
This is an automated email from the git hooks/post-receive script. New commit to branch feature/9206-upload-validation-zip-documents in repository coselmar. See https://gitlab.nuiton.org/codelutin/coselmar.git commit d5a986fb539a3ea59c61b4d954b6f92e1ed09368 Author: Yannick Martel <martel@©odelutin.com> Date: Fri Jun 2 16:54:23 2017 +0200 refs-30 #9206 First draft for Documents Zip management --- coselmar-rest/pom.xml | 4 - .../fr/ifremer/coselmar/beans/DocumentBean.java | 5 +- .../coselmar/beans/DocumentImportModel.java | 23 ++- .../services/CoselmarWebServiceSupport.java | 8 +- .../coselmar/services/indexation/TikaUtils.java | 4 + .../coselmar/services/v1/AdminWebService.java | 34 ++-- .../coselmar/services/v1/DocumentsWebService.java | 194 ++++++++++++++++++++- .../services/v1/DocumentsWebServiceTest.java | 93 ++++++++++ coselmar-rest/src/test/resources/documents.zip | Bin 0 -> 151823 bytes 9 files changed, 329 insertions(+), 36 deletions(-) diff --git a/coselmar-rest/pom.xml b/coselmar-rest/pom.xml index 9424e3e..6cba769 100644 --- a/coselmar-rest/pom.xml +++ b/coselmar-rest/pom.xml @@ -247,10 +247,6 @@ <testResources> <testResource> <directory>src/test/resources</directory> - <includes> - <include>**/*.properties</include> - </includes> - <filtering>true</filtering> </testResource> </testResources> <pluginManagement> diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/DocumentBean.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/DocumentBean.java index cc724c8..80d9425 100644 --- a/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/DocumentBean.java +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/DocumentBean.java @@ -28,6 +28,7 @@ import java.io.Serializable; import java.util.Collection; import java.util.Date; import java.util.HashSet; +import java.util.List; import java.util.Set; /** @@ -165,8 +166,8 @@ public class DocumentBean implements Serializable { return keywords; } - public void setKeywords(Collection<String> keywords) { - this.keywords = new HashSet<>(keywords); + public void setKeywords(Set<String> keywords) { + this.keywords = keywords; } public void addKeywords(Collection<String> keywords) { diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/DocumentImportModel.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/DocumentImportModel.java index f8b5749..a5ff66a 100644 --- a/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/DocumentImportModel.java +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/DocumentImportModel.java @@ -1,39 +1,46 @@ package fr.ifremer.coselmar.beans; -import fr.ifremer.coselmar.persistence.entity.Privacy; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import org.nuiton.csv.Common; import org.nuiton.csv.ValueParser; import org.nuiton.csv.ext.AbstractImportModel; import java.text.ParseException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Set; /** * @author ymartel (martel@codelutin.com) */ public class DocumentImportModel extends AbstractImportModel<DocumentBean> { - protected static final ValueParser<Privacy> DOCUMENT_PRIVACY_PARSER = new ValueParser<Privacy>() { + protected static final ValueParser<Set<String>> LIST_STRING_PARSER = new ValueParser<Set<String>>() { @Override - public Privacy parse(String value) throws ParseException { - return Privacy.valueOf(value.toUpperCase()); + public Set<String> parse(String value) throws ParseException { + return Sets.newHashSet(value.split(",")); } }; + protected static final Common.DateValue DATE_PARSER = new Common.DateValue("yyyy/mm/dd"); + public DocumentImportModel() { super(';'); newMandatoryColumn("name", "name"); newMandatoryColumn("type", "type"); - newMandatoryColumn("privacy", "privacy", DOCUMENT_PRIVACY_PARSER); - newMandatoryColumn("keywords", "keywords"); + newMandatoryColumn("privacy", "privacy"); + newMandatoryColumn("keywords", "keywords", LIST_STRING_PARSER); newOptionalColumn("authors", "authors"); newMandatoryColumn("summary", "summary"); newOptionalColumn("license", "license"); newOptionalColumn("copyright", "copyright"); newOptionalColumn("language", "language"); - newOptionalColumn("publicationDate", "publicationDate"); + newOptionalColumn("publicationDate", "publicationDate", DATE_PARSER); newOptionalColumn("comment", "comment"); - newMandatoryColumn("fileName", "fileName"); newMandatoryColumn("citation", "citation"); + newMandatoryColumn("fileName", "fileName"); } @Override diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/CoselmarWebServiceSupport.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/CoselmarWebServiceSupport.java index 1d52f0a..4c85fc7 100644 --- a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/CoselmarWebServiceSupport.java +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/CoselmarWebServiceSupport.java @@ -68,7 +68,13 @@ public abstract class CoselmarWebServiceSupport extends WebMotionController impl protected CoselmarServicesContext getServicesContext() { //try to get it from Request context - HttpContext context = getContext(); + HttpContext context; + try { + context = getContext(); + } catch (NullPointerException e) { + // not web context ? //XXX ymartel use because unit test on Documents mass import are not in web context ... + context = null; + } if (context != null) { CoselmarRestRequestContext requestContext = CoselmarRestRequestContext.getRequestContext(context); this.servicesContext = requestContext.getServicesContext(); diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/indexation/TikaUtils.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/indexation/TikaUtils.java index 68e9ca4..eb46437 100644 --- a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/indexation/TikaUtils.java +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/indexation/TikaUtils.java @@ -52,4 +52,8 @@ public class TikaUtils { return fileContent; } + public static String getFileMimeType(String filePath) { + String mimeType = tika.detect(filePath); + return mimeType; + } } diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/AdminWebService.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/AdminWebService.java index 40c4534..6fd6325 100644 --- a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/AdminWebService.java +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/AdminWebService.java @@ -69,25 +69,11 @@ public class AdminWebService extends CoselmarWebServiceSupport { } - DocumentsIndexationService documentsIndexationService = getServicesContext().newService(DocumentsIndexationService.class); QuestionsIndexationService questionsIndexationService = getServicesContext().newService(QuestionsIndexationService.class); try { getServicesContext().getLuceneUtils().clearIndex(); + refreshDocumentsIndex(); - // get All documents - List<Document> documents = getDocumentDao().findAll(); - for (Document document : documents) { - DocumentBean documentBean = BeanEntityConverter.toBean(getPersistenceContext().getTopiaIdFactory(), document); - if (document.isWithFile()) { - // Refresh file information - String fileContent = TikaUtils.getFileContent(document.getFilePath()); - documentsIndexationService.indexDocument(documentBean, fileContent); - // Refresh database content - document.setFileContent(fileContent); - getDocumentDao().update(document); - } - } - commit(); // Get all questions List<Question> questions = getQuestionDao().findAll(); @@ -108,4 +94,22 @@ public class AdminWebService extends CoselmarWebServiceSupport { } } + protected void refreshDocumentsIndex() throws IOException { + DocumentsIndexationService documentsIndexationService = getServicesContext().newService(DocumentsIndexationService.class); + // get All documents + List<Document> documents = getDocumentDao().findAll(); + for (Document document : documents) { + DocumentBean documentBean = BeanEntityConverter.toBean(getPersistenceContext().getTopiaIdFactory(), document); + if (document.isWithFile()) { + // Refresh file information + String fileContent = TikaUtils.getFileContent(document.getFilePath()); + documentsIndexationService.indexDocument(documentBean, fileContent); + // Refresh database content + document.setFileContent(fileContent); + getDocumentDao().update(document); + } + } + commit(); + } + } diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/DocumentsWebService.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/DocumentsWebService.java index 1e52d9c..80be99e 100644 --- a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/DocumentsWebService.java +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/DocumentsWebService.java @@ -31,6 +31,7 @@ import com.google.common.collect.Collections2; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import fr.ifremer.coselmar.beans.DocumentBean; +import fr.ifremer.coselmar.beans.DocumentImportModel; import fr.ifremer.coselmar.beans.DocumentSearchBean; import fr.ifremer.coselmar.beans.DocumentSearchExample; import fr.ifremer.coselmar.beans.FileInfos; @@ -51,12 +52,17 @@ import fr.ifremer.coselmar.services.errors.NoResultException; import fr.ifremer.coselmar.services.errors.UnauthorizedException; import fr.ifremer.coselmar.services.indexation.DocumentsIndexationService; import fr.ifremer.coselmar.services.indexation.TikaUtils; +import fr.ifremer.coselmar.services.indexation.TransverseIndexationService; import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.lucene.queryparser.classic.ParseException; import org.debux.webmotion.server.call.UploadFile; import org.debux.webmotion.server.render.Render; +import org.nuiton.csv.Import; +import org.nuiton.csv.ImportModel; +import org.nuiton.csv.ImportRuntimeException; import org.nuiton.topia.persistence.TopiaNoResultException; import org.nuiton.util.DateUtil; import org.nuiton.util.pagination.PaginationResult; @@ -64,14 +70,21 @@ import org.nuiton.util.pagination.PaginationResult; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; +import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.Collection; import java.util.Date; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; import static org.apache.commons.logging.LogFactory.getLog; @@ -672,6 +685,42 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { return types; } + public void uploadZipDocuments(UploadFile uploadFile) throws InvalidCredentialException, UnauthorizedException { + + Preconditions.checkNotNull(uploadFile); + + // Check authentication + String authorization = getContext().getHeader("Authorization"); + CoselmarUser currentUser = checkUserAuthentication(authorization); + + // Only Supervisor/Admin can add Zip documents + if (!DOCUMENT_SUPER_USER_ROLES.contains(currentUser.getRole().name())) { + String message = String.format("User %s %s ('%s') is not allowed to upload mass document files", + currentUser.getFirstname(), currentUser.getName(), getShortIdFromFull(currentUser.getTopiaId())); + if (log.isWarnEnabled()) { + log.warn(message); + } + throw new UnauthorizedException(message); + } + List<String> missingFiles = importFromZip(uploadFile.getFile(), currentUser); + + if (missingFiles.isEmpty()) { + // All is ok ! + commit(); + + } else { + // Something wrong happened... rollback, and refresh lucene to avoid new data + rollback(); + AdminWebService adminWebService = getServicesContext().newService(AdminWebService.class); + try { + adminWebService.refreshDocumentsIndex(); + } catch (IOException e) { + if (log.isErrorEnabled()) { + log.error("Unable to refresh Lucene Documents index. Data should be corrupted", e); + } + } + } + } //////////////////////////////////////////////////////////////////////////// /////////////////////// Internal Parts ///////////////////////////// @@ -802,13 +851,9 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { } // put the document in the good directory - String userPath = getUserDocumentPath(owner); + String filePath = getDocumentFileDestPath(owner, fileName); - Date now = getNow(); - String formattedDay = DateUtil.formatDate(now, "yyyyMMddHHmm"); - String prefix = formattedDay + "-"; - - File destFile = new File(userPath + File.separator + prefix + fileName); + File destFile = new File(filePath); try { FileUtils.moveFile(uploadedFile, destFile); } catch (IOException e) { @@ -825,6 +870,20 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { return fileInfos; } + protected String getDocumentFileDestPath(CoselmarUser owner, String fileName) { + String userPath = getUserDocumentPath(owner); + + String storageFileName = getFileStorageName(fileName); + return userPath + File.separator + storageFileName; + } + + protected String getFileStorageName(String fileName) { + Date now = getNow(); + String formattedDay = DateUtil.formatDate(now, "yyyyMMddHHmm"); + String prefix = formattedDay + "-"; + return prefix + fileName; + } + protected String getUserDocumentPath(CoselmarUser user) { File dataDirectory = getCoselmarServicesConfig().getDataDirectory(); String absolutePath = dataDirectory.getAbsolutePath(); @@ -920,4 +979,127 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { return new HashSet<>(coselmarUsers); } + + protected List<String> importFromZip(File file, CoselmarUser currentUser) { + // File should be a Zip + ZipFile zipFile; + try { + zipFile = new ZipFile(file);; + } catch (IOException e) { + if (log.isErrorEnabled()) { + log.error("error during ZipFile transfer", e); + } + throw new CoselmarTechnicalException("Internal error during ZipFile transfer", e); + } + + // Get descriptions.csv : it should contains all DocumentBean information + ZipEntry descriptionEntry = zipFile.getEntry(DESCRIPTION_CSV_FILE_NAME); + InputStream descriptionInputStream; + try { + descriptionInputStream = zipFile.getInputStream(descriptionEntry); + } catch (IOException e) { + String message = String.format("Unable to read '%s' from zip file", DESCRIPTION_CSV_FILE_NAME); + if (log.isErrorEnabled()) { + log.error(message, e); + } + throw new CoselmarTechnicalException(message, e); + } + + // Now, read CSV ... + DocumentImportModel csvModel = new DocumentImportModel(); + Import<DocumentBean> importer = Import.newImport(csvModel, descriptionInputStream); + + File dataDirectory = getCoselmarServicesConfig().getDataDirectory(); + String dataPath = dataDirectory.getAbsolutePath(); + String zipTempPath = dataPath + File.separator + DateUtil.formatDate(getNow(), "yyyyMMddHHmm"); + Path dir = Paths.get(zipTempPath); + try { + Files.createDirectories(dir); + } catch (IOException e) { + if (log.isErrorEnabled()) { + String message = "Unable to create temp path for zip import"; + log.error(message, e); + } + throw new CoselmarTechnicalException("Unable to unzip file : error with unzip directory", e); + } + + List<String> missingFiles = new ArrayList<>(); + // ... and read each entries and retrieve associated File + try { + for (DocumentBean documentBean : importer) { + FileInfos fileInfos = new FileInfos(); + String fileName = documentBean.getFileName(); + ZipEntry zipFileEntry = zipFile.getEntry(fileName); + if (zipFileEntry == null) { + // TODO ymartel 20170601 : manage errors + regenerate Lucene Index + missingFiles.add(fileName); + + } else { + try { + InputStream zipFileInputStream = zipFile.getInputStream(zipFileEntry); + fileInfos.setFileName(fileName); + String fileMimeType = TikaUtils.getFileMimeType(fileName); + String futureFilePath = getDocumentFileDestPath(currentUser, fileName); + + String storageFileName = getFileStorageName(fileName); + + // Push file stream in temporary folder + File zipEntryFile = new File(zipTempPath + File.separator + storageFileName); + zipEntryFile.createNewFile(); + Files.copy(zipFileInputStream, zipEntryFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + + // Create fileInfos + fileInfos.setFileName(fileName); + fileInfos.setFilePath(futureFilePath); + fileInfos.setMimeType(fileMimeType); + + // Create document + createDocument(documentBean, fileInfos, currentUser); + + try { + zipFileInputStream.close(); + } catch (IOException e) { + if (log.isErrorEnabled()) { + log.error("Unable to close stream from ZipEntry : " + fileName, e); + } + } + } catch (IOException e) { + String message = String.format("Unable to retrieve '%s' from zip file", fileName); + if (log.isErrorEnabled()) { + log.error(message, e); + } + // TODO ymartel 20170601 : manage errors + regenerate Lucene Index + missingFiles.add(fileName); + } + } + + } + } catch (ImportRuntimeException ire) { + if (log.isErrorEnabled()) { + log.error("Error with CSV file", ire); + } + throw new CoselmarTechnicalException("Error with CSV file", ire); + } finally { + // Close importer + importer.close(); + // and csv stream + try { + descriptionInputStream.close(); + } catch (IOException e) { + if (log.isErrorEnabled()) { + log.error("Unable to close 'descriptions.csv' input stream from zip"); + } + } + } + if (missingFiles.isEmpty()) { + // Ok, let move all files from temp storage to real one ! + try { + FileUtils.moveDirectory(new File(zipTempPath), new File(getUserDocumentPath(currentUser))); + } catch (IOException e) { + // Big problem if we can't move files into real folder ! + e.printStackTrace(); + } + } + return missingFiles; + } } diff --git a/coselmar-rest/src/test/java/fr/ifremer/coselmar/services/v1/DocumentsWebServiceTest.java b/coselmar-rest/src/test/java/fr/ifremer/coselmar/services/v1/DocumentsWebServiceTest.java new file mode 100644 index 0000000..5357c1c --- /dev/null +++ b/coselmar-rest/src/test/java/fr/ifremer/coselmar/services/v1/DocumentsWebServiceTest.java @@ -0,0 +1,93 @@ +package fr.ifremer.coselmar.services.v1; + +/* + * #%L + * Coselmar :: Rest Services + * $Id:$ + * $HeadURL:$ + * %% + * Copyright (C) 2014 Ifremer, Code Lutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/gpl-3.0.html>. + * #L% + */ + +import fr.ifremer.coselmar.persistence.entity.CoselmarUser; +import fr.ifremer.coselmar.persistence.entity.CoselmarUserImpl; +import fr.ifremer.coselmar.persistence.entity.CoselmarUserTopiaDao; +import fr.ifremer.coselmar.persistence.entity.Document; +import fr.ifremer.coselmar.persistence.entity.DocumentTopiaDao; +import fr.ifremer.coselmar.services.AbstractCoselmarServiceTest; +import fr.ifremer.coselmar.services.FakeCoselmarServicesContext; +import org.junit.Assert; +import org.junit.Test; + +import java.io.File; +import java.io.InputStream; +import java.net.URL; +import java.nio.file.CopyOption; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.List; +import java.util.Locale; + +/** + * @author ymartel <martel@codelutin.com> + */ +public class DocumentsWebServiceTest extends AbstractCoselmarServiceTest { + + protected FakeCoselmarServicesContext serviceContext; + + protected FakeCoselmarServicesContext getServiceContext() { + + if (serviceContext == null) { + serviceContext = application.newServiceContext(application.newPersistenceContext(), Locale.FRANCE); + } + + return serviceContext; + } + + @Test + public void testUploadZipDocuments() throws Exception { + + DocumentsWebService documentsWebService = getServiceContext().newService(DocumentsWebService.class); +// URL zipFileURL = this.getClass().getResource("/documents.zip"); + InputStream resourceAsStream = this.getClass().getResourceAsStream("/documents.zip"); + Path tempZipPath = Files.createTempFile(null, null); + Files.copy(resourceAsStream, tempZipPath, StandardCopyOption.REPLACE_EXISTING); + resourceAsStream.close(); + File zipFile = tempZipPath.toFile(); + Assert.assertNotNull(zipFile); + Assert.assertTrue(zipFile.exists()); + + // create a test user + CoselmarUserTopiaDao coselmarUserDao = getServiceContext().getPersistenceContext().getCoselmarUserDao(); + CoselmarUser coselmarUser = coselmarUserDao.create(); + coselmarUser.setName("Peuplu"); + coselmarUser.setFirstname("Jean"); + getServiceContext().getPersistenceContext().commit(); + + List<String> errors = documentsWebService.importFromZip(zipFile, coselmarUser); + Assert.assertNotNull(errors); + Assert.assertTrue(errors.isEmpty()); + DocumentTopiaDao documentDao = getServiceContext().getPersistenceContext().getDocumentDao(); + List<Document> documents = documentDao.findAll(); + Assert.assertEquals(3, documents.size()); + String oneFilePath = documents.get(0).getFilePath(); + File oneFile = new File(oneFilePath); + Assert.assertTrue(oneFile.exists()); + } +} diff --git a/coselmar-rest/src/test/resources/documents.zip b/coselmar-rest/src/test/resources/documents.zip new file mode 100644 index 0000000..6467307 Binary files /dev/null and b/coselmar-rest/src/test/resources/documents.zip differ -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.
This is an automated email from the git hooks/post-receive script. New commit to branch feature/9206-upload-validation-zip-documents in repository coselmar. See https://gitlab.nuiton.org/codelutin/coselmar.git commit fdce731043b1840716f3cf526ff78523dbfbee9a Author: Yannick Martel <martel@©odelutin.com> Date: Tue Jun 6 12:05:37 2017 +0200 refs-50 #9206 First draft for error management in Documents Zip import --- .../beans/MassiveDocumentsImportResult.java | 47 +++++++++++++++ .../coselmar/services/v1/DocumentsWebService.java | 65 +++++++++++---------- .../services/v1/DocumentsWebServiceTest.java | 37 +++++++++++- .../src/test/resources/documents_errors.zip | Bin 0 -> 87789 bytes 4 files changed, 116 insertions(+), 33 deletions(-) diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/MassiveDocumentsImportResult.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/MassiveDocumentsImportResult.java new file mode 100644 index 0000000..fff4400 --- /dev/null +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/MassiveDocumentsImportResult.java @@ -0,0 +1,47 @@ +package fr.ifremer.coselmar.beans; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +/** + * @author ymartel (martel@codelutin.com) + */ +public class MassiveDocumentsImportResult { + + protected List<String> missingFiles; + protected boolean errorSystem; + protected boolean invalidFile; + + public MassiveDocumentsImportResult() { + this.missingFiles = new ArrayList<>(); + } + + public List<String> getMissingFiles() { + return missingFiles; + } + + public void setMissingFiles(List<String> missingFiles) { + this.missingFiles = missingFiles; + } + + public void addMissingFile(String missingFileName) { + this.missingFiles.add(missingFileName); + } + + public boolean isErrorSystem() { + return errorSystem; + } + + public void setErrorSystem(boolean errorSystem) { + this.errorSystem = errorSystem; + } + + public boolean isInvalidFile() { + return invalidFile; + } + + public void setInvalidFile(boolean invalidFile) { + this.invalidFile = invalidFile; + } +} diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/DocumentsWebService.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/DocumentsWebService.java index 80be99e..6455deb 100644 --- a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/DocumentsWebService.java +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/DocumentsWebService.java @@ -35,6 +35,7 @@ import fr.ifremer.coselmar.beans.DocumentImportModel; import fr.ifremer.coselmar.beans.DocumentSearchBean; import fr.ifremer.coselmar.beans.DocumentSearchExample; import fr.ifremer.coselmar.beans.FileInfos; +import fr.ifremer.coselmar.beans.MassiveDocumentsImportResult; import fr.ifremer.coselmar.beans.QuestionBean; import fr.ifremer.coselmar.beans.UserBean; import fr.ifremer.coselmar.beans.UserWebToken; @@ -685,7 +686,7 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { return types; } - public void uploadZipDocuments(UploadFile uploadFile) throws InvalidCredentialException, UnauthorizedException { + public MassiveDocumentsImportResult uploadZipDocuments(UploadFile uploadFile) throws InvalidCredentialException, UnauthorizedException { Preconditions.checkNotNull(uploadFile); @@ -702,24 +703,8 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { } throw new UnauthorizedException(message); } - List<String> missingFiles = importFromZip(uploadFile.getFile(), currentUser); + return importFromZip(uploadFile.getFile(), currentUser); - if (missingFiles.isEmpty()) { - // All is ok ! - commit(); - - } else { - // Something wrong happened... rollback, and refresh lucene to avoid new data - rollback(); - AdminWebService adminWebService = getServicesContext().newService(AdminWebService.class); - try { - adminWebService.refreshDocumentsIndex(); - } catch (IOException e) { - if (log.isErrorEnabled()) { - log.error("Unable to refresh Lucene Documents index. Data should be corrupted", e); - } - } - } } //////////////////////////////////////////////////////////////////////////// @@ -980,7 +965,8 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { } - protected List<String> importFromZip(File file, CoselmarUser currentUser) { + protected MassiveDocumentsImportResult importFromZip(File file, CoselmarUser currentUser) { + MassiveDocumentsImportResult importResult = new MassiveDocumentsImportResult(); // File should be a Zip ZipFile zipFile; try { @@ -1002,7 +988,9 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { if (log.isErrorEnabled()) { log.error(message, e); } - throw new CoselmarTechnicalException(message, e); + importResult.setInvalidFile(true); + importResult.addMissingFile(DESCRIPTION_CSV_FILE_NAME); + return importResult; // Direct break : cannot continue without this file } // Now, read CSV ... @@ -1023,7 +1011,6 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { throw new CoselmarTechnicalException("Unable to unzip file : error with unzip directory", e); } - List<String> missingFiles = new ArrayList<>(); // ... and read each entries and retrieve associated File try { for (DocumentBean documentBean : importer) { @@ -1031,8 +1018,7 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { String fileName = documentBean.getFileName(); ZipEntry zipFileEntry = zipFile.getEntry(fileName); if (zipFileEntry == null) { - // TODO ymartel 20170601 : manage errors + regenerate Lucene Index - missingFiles.add(fileName); + importResult.addMissingFile(fileName); } else { try { @@ -1068,8 +1054,8 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { if (log.isErrorEnabled()) { log.error(message, e); } - // TODO ymartel 20170601 : manage errors + regenerate Lucene Index - missingFiles.add(fileName); + importResult.setErrorSystem(true); + importResult.addMissingFile(fileName); } } @@ -1078,6 +1064,7 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { if (log.isErrorEnabled()) { log.error("Error with CSV file", ire); } + importResult.setErrorSystem(true); throw new CoselmarTechnicalException("Error with CSV file", ire); } finally { // Close importer @@ -1087,19 +1074,37 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { descriptionInputStream.close(); } catch (IOException e) { if (log.isErrorEnabled()) { - log.error("Unable to close 'descriptions.csv' input stream from zip"); + log.error("Unable to close 'descriptions.csv' input stream from zip", e); } } } - if (missingFiles.isEmpty()) { - // Ok, let move all files from temp storage to real one ! + if (!importResult.isErrorSystem() && (importResult.getMissingFiles() == null || importResult.getMissingFiles().isEmpty())) { + // All is ok ! try { + // Ok, let move all files from temp storage to real one ! FileUtils.moveDirectory(new File(zipTempPath), new File(getUserDocumentPath(currentUser))); + commit(); } catch (IOException e) { // Big problem if we can't move files into real folder ! - e.printStackTrace(); + if (log.isErrorEnabled()) { + log.error("Unable to move files from temp folder '%s' to final folder", e); + } + importResult.setErrorSystem(true); + rollback(); + } + + } else { + // Something wrong happened... rollback, and refresh lucene to avoid new data + rollback(); + AdminWebService adminWebService = getServicesContext().newService(AdminWebService.class); + try { + adminWebService.refreshDocumentsIndex(); + } catch (IOException e) { + if (log.isErrorEnabled()) { + log.error("Unable to refresh Lucene Documents index. Data should be corrupted", e); + } } } - return missingFiles; + return importResult; } } diff --git a/coselmar-rest/src/test/java/fr/ifremer/coselmar/services/v1/DocumentsWebServiceTest.java b/coselmar-rest/src/test/java/fr/ifremer/coselmar/services/v1/DocumentsWebServiceTest.java index 5357c1c..ca3562c 100644 --- a/coselmar-rest/src/test/java/fr/ifremer/coselmar/services/v1/DocumentsWebServiceTest.java +++ b/coselmar-rest/src/test/java/fr/ifremer/coselmar/services/v1/DocumentsWebServiceTest.java @@ -24,6 +24,7 @@ package fr.ifremer.coselmar.services.v1; * #L% */ +import fr.ifremer.coselmar.beans.MassiveDocumentsImportResult; import fr.ifremer.coselmar.persistence.entity.CoselmarUser; import fr.ifremer.coselmar.persistence.entity.CoselmarUserImpl; import fr.ifremer.coselmar.persistence.entity.CoselmarUserTopiaDao; @@ -80,9 +81,9 @@ public class DocumentsWebServiceTest extends AbstractCoselmarServiceTest { coselmarUser.setFirstname("Jean"); getServiceContext().getPersistenceContext().commit(); - List<String> errors = documentsWebService.importFromZip(zipFile, coselmarUser); - Assert.assertNotNull(errors); - Assert.assertTrue(errors.isEmpty()); + MassiveDocumentsImportResult importResult = documentsWebService.importFromZip(zipFile, coselmarUser); + Assert.assertNotNull(importResult); + Assert.assertTrue(importResult.getMissingFiles().isEmpty()); DocumentTopiaDao documentDao = getServiceContext().getPersistenceContext().getDocumentDao(); List<Document> documents = documentDao.findAll(); Assert.assertEquals(3, documents.size()); @@ -90,4 +91,34 @@ public class DocumentsWebServiceTest extends AbstractCoselmarServiceTest { File oneFile = new File(oneFilePath); Assert.assertTrue(oneFile.exists()); } + + @Test + public void testUploadZipDocumentsWithErrors() throws Exception { + + DocumentsWebService documentsWebService = getServiceContext().newService(DocumentsWebService.class); + InputStream resourceAsStream = this.getClass().getResourceAsStream("/documents_errors.zip"); + Path tempZipPath = Files.createTempFile(null, null); + Files.copy(resourceAsStream, tempZipPath, StandardCopyOption.REPLACE_EXISTING); + resourceAsStream.close(); + File zipFile = tempZipPath.toFile(); + Assert.assertNotNull(zipFile); + Assert.assertTrue(zipFile.exists()); + + // create a test user + CoselmarUserTopiaDao coselmarUserDao = getServiceContext().getPersistenceContext().getCoselmarUserDao(); + CoselmarUser coselmarUser = coselmarUserDao.create(); + coselmarUser.setName("Peuplu"); + coselmarUser.setFirstname("Jean"); + getServiceContext().getPersistenceContext().commit(); + + MassiveDocumentsImportResult importResult = documentsWebService.importFromZip(zipFile, coselmarUser); + Assert.assertNotNull(importResult); + Assert.assertFalse(importResult.getMissingFiles().isEmpty()); + Assert.assertEquals(1, importResult.getMissingFiles().size()); + Assert.assertEquals("Reunion-v0.3.pdf", importResult.getMissingFiles().get(0)); + // Should import have been rollback + DocumentTopiaDao documentDao = getServiceContext().getPersistenceContext().getDocumentDao(); + List<Document> documents = documentDao.findAll(); + Assert.assertEquals(0, documents.size()); + } } diff --git a/coselmar-rest/src/test/resources/documents_errors.zip b/coselmar-rest/src/test/resources/documents_errors.zip new file mode 100644 index 0000000..836cde6 Binary files /dev/null and b/coselmar-rest/src/test/resources/documents_errors.zip differ -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.
participants (1)
-
codelutin.com scm