[Coselmar-commits] 01/03: Prepare import massif de document

This is an automated email from the git hooks/post-receive script. New commit to branch feature/9206-upload-validation-zip-documents in repository coselmar. See https://gitlab.nuiton.org/codelutin/coselmar.git commit dc2afeb93523010de541f4cba659d38420b1a51a Author: Yannick Martel <martel@©odelutin.com> Date: Thu Jun 1 14:29:22 2017 +0200 Prepare import massif de document --- .../fr/ifremer/coselmar/beans/DocumentBean.java | 6 + .../coselmar/beans/DocumentImportModel.java | 44 +++++ .../java/fr/ifremer/coselmar/beans/FileInfos.java | 35 ++++ .../coselmar/converter/BeanEntityConverter.java | 4 +- .../coselmar/services/indexation/TikaUtils.java | 1 + .../coselmar/services/v1/AdminWebService.java | 10 +- .../coselmar/services/v1/DocumentsWebService.java | 216 ++++++++++++--------- 7 files changed, 215 insertions(+), 101 deletions(-) diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/DocumentBean.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/DocumentBean.java index f8c855f..cc724c8 100644 --- a/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/DocumentBean.java +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/DocumentBean.java @@ -64,6 +64,12 @@ public class DocumentBean implements Serializable { // If restricted, could have a list of restricted user protected Set<UserBean> authorizedUsers; + public static DocumentBean newEmptyInstance() { + return new DocumentBean(); + } + + private DocumentBean(){} + public DocumentBean(String id, String name, String ownerName, String ownerId, String privacy, Date depositDate, Collection<String> keywords, String type, String summary, String language, Date publicationDate, diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/DocumentImportModel.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/DocumentImportModel.java new file mode 100644 index 0000000..f8b5749 --- /dev/null +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/DocumentImportModel.java @@ -0,0 +1,44 @@ +package fr.ifremer.coselmar.beans; + +import fr.ifremer.coselmar.persistence.entity.Privacy; +import org.nuiton.csv.ValueParser; +import org.nuiton.csv.ext.AbstractImportModel; + +import java.text.ParseException; + +/** + * @author ymartel (martel@codelutin.com) + */ +public class DocumentImportModel extends AbstractImportModel<DocumentBean> { + + protected static final ValueParser<Privacy> DOCUMENT_PRIVACY_PARSER = new ValueParser<Privacy>() { + @Override + public Privacy parse(String value) throws ParseException { + return Privacy.valueOf(value.toUpperCase()); + } + }; + + public DocumentImportModel() { + super(';'); + + newMandatoryColumn("name", "name"); + newMandatoryColumn("type", "type"); + newMandatoryColumn("privacy", "privacy", DOCUMENT_PRIVACY_PARSER); + newMandatoryColumn("keywords", "keywords"); + newOptionalColumn("authors", "authors"); + newMandatoryColumn("summary", "summary"); + newOptionalColumn("license", "license"); + newOptionalColumn("copyright", "copyright"); + newOptionalColumn("language", "language"); + newOptionalColumn("publicationDate", "publicationDate"); + newOptionalColumn("comment", "comment"); + newMandatoryColumn("fileName", "fileName"); + newMandatoryColumn("citation", "citation"); + } + + @Override + public DocumentBean newEmptyInstance() { + return DocumentBean.newEmptyInstance(); + } + +} diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/FileInfos.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/FileInfos.java new file mode 100644 index 0000000..5bebc04 --- /dev/null +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/FileInfos.java @@ -0,0 +1,35 @@ +package fr.ifremer.coselmar.beans; + +/** + * @author ymartel (martel@codelutin.com) + */ +public class FileInfos { + + protected String FileName; + protected String FilePath; + protected String MimeType; + + public String getFileName() { + return FileName; + } + + public void setFileName(String fileName) { + FileName = fileName; + } + + public String getFilePath() { + return FilePath; + } + + public void setFilePath(String filePath) { + FilePath = filePath; + } + + public String getMimeType() { + return MimeType; + } + + public void setMimeType(String mimeType) { + MimeType = mimeType; + } +} diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/converter/BeanEntityConverter.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/converter/BeanEntityConverter.java index 9af8254..b8d8e2a 100644 --- a/coselmar-rest/src/main/java/fr/ifremer/coselmar/converter/BeanEntityConverter.java +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/converter/BeanEntityConverter.java @@ -309,7 +309,7 @@ public class BeanEntityConverter { * </ul> * If documentBean is null, simply return null Document. */ - public static Document fromBean(DocumentBean documentBean) { + public static Document fromSearchBean(DocumentBean documentBean) { if (documentBean == null) { return null; } @@ -337,7 +337,7 @@ public class BeanEntityConverter { String privacy = documentBean.getPrivacy(); if (StringUtils.isNotBlank(privacy)) { - document.setPrivacy(Privacy.valueOf(privacy)); + document.setPrivacy(Privacy.valueOf(privacy.toUpperCase())); } Date publicationDate = documentBean.getPublicationDate(); diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/indexation/TikaUtils.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/indexation/TikaUtils.java index 4a517a4..68e9ca4 100644 --- a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/indexation/TikaUtils.java +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/indexation/TikaUtils.java @@ -51,4 +51,5 @@ public class TikaUtils { } return fileContent; } + } diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/AdminWebService.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/AdminWebService.java index c4ad5f1..40c4534 100644 --- a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/AdminWebService.java +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/AdminWebService.java @@ -24,18 +24,14 @@ package fr.ifremer.coselmar.services.v1; * #L% */ -import java.io.File; -import java.io.IOException; -import java.util.List; - import fr.ifremer.coselmar.beans.DocumentBean; import fr.ifremer.coselmar.beans.QuestionBean; import fr.ifremer.coselmar.beans.UserWebToken; import fr.ifremer.coselmar.converter.BeanEntityConverter; +import fr.ifremer.coselmar.exceptions.CoselmarTechnicalException; import fr.ifremer.coselmar.persistence.entity.CoselmarUserRole; import fr.ifremer.coselmar.persistence.entity.Document; import fr.ifremer.coselmar.persistence.entity.Question; -import fr.ifremer.coselmar.exceptions.CoselmarTechnicalException; import fr.ifremer.coselmar.services.CoselmarWebServiceSupport; import fr.ifremer.coselmar.services.errors.InvalidCredentialException; import fr.ifremer.coselmar.services.errors.UnauthorizedException; @@ -44,7 +40,9 @@ import fr.ifremer.coselmar.services.indexation.QuestionsIndexationService; import fr.ifremer.coselmar.services.indexation.TikaUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; -import org.apache.tika.exception.TikaException; + +import java.io.IOException; +import java.util.List; import static org.apache.commons.logging.LogFactory.getLog; diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/DocumentsWebService.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/DocumentsWebService.java index 63e133d..1e52d9c 100644 --- a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/DocumentsWebService.java +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/DocumentsWebService.java @@ -33,6 +33,7 @@ import com.google.common.collect.Lists; import fr.ifremer.coselmar.beans.DocumentBean; import fr.ifremer.coselmar.beans.DocumentSearchBean; import fr.ifremer.coselmar.beans.DocumentSearchExample; +import fr.ifremer.coselmar.beans.FileInfos; import fr.ifremer.coselmar.beans.QuestionBean; import fr.ifremer.coselmar.beans.UserBean; import fr.ifremer.coselmar.beans.UserWebToken; @@ -52,10 +53,8 @@ import fr.ifremer.coselmar.services.indexation.DocumentsIndexationService; import fr.ifremer.coselmar.services.indexation.TikaUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.logging.Log; import org.apache.lucene.queryparser.classic.ParseException; -import org.apache.tika.exception.TikaException; import org.debux.webmotion.server.call.UploadFile; import org.debux.webmotion.server.render.Render; import org.nuiton.topia.persistence.TopiaNoResultException; @@ -67,7 +66,6 @@ import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; -import java.net.URL; import java.util.ArrayList; import java.util.Collection; import java.util.Date; @@ -97,6 +95,7 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { CoselmarUserRole.SUPERVISOR.name(), CoselmarUserRole.EXPERT.name() ); + protected static final String DESCRIPTION_CSV_FILE_NAME = "description.csv"; public DocumentBean getDocument(String documentId) throws InvalidCredentialException, UnauthorizedException { @@ -181,7 +180,7 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { } searchExample.setFullTextSearch(searchBean.getFullTextSearch()); - Document example = BeanEntityConverter.fromBean(searchBean); + Document example = BeanEntityConverter.fromSearchBean(searchBean); searchExample.setExample(example); searchExample.setOwnerName(searchBean.getOwnerName()); @@ -276,7 +275,7 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { searchExample.setFullTextSearch(searchBean.getFullTextSearch()); - Document example = BeanEntityConverter.fromBean(searchBean); + Document example = BeanEntityConverter.fromSearchBean(searchBean); searchExample.setExample(example); searchExample.setOwnerName(searchBean.getOwnerName()); @@ -376,91 +375,13 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { throw new InvalidCredentialException(message); } - String documentName = document.getName(); - String contentType = null; - String filePath = null; - String fileContent = null; - - // If document has a file, manage it ! - if (uploadFile != null) { - Pair<String, String> pathAndContentType = managerDocumentFile(uploadFile, owner); - filePath = pathAndContentType.getLeft(); - contentType = pathAndContentType.getRight(); - fileContent = TikaUtils.getFileContent(filePath); - } - - // Document Metadata - Document documentEntity = getDocumentDao().create(); - - documentEntity.setOwner(owner); - - documentEntity.setName(documentName); - - String privacy = document.getPrivacy().toUpperCase(); - documentEntity.setPrivacy(Privacy.valueOf(privacy)); - - // Manage privacy : if restricted, create an UserGroup with authorized users - if (StringUtils.equals(Privacy.RESTRICTED.name(), privacy)) { - Set<UserBean> authorizedUsers = document.getAuthorizedUsers(); - Set<CoselmarUser> coselmarUsers = retrieveUsers(authorizedUsers); - CoselmarUserGroup restrictedUsers = getCoselmarUserGroupDao().create(); - restrictedUsers.setName(documentEntity.getTopiaId()); - - restrictedUsers.addAllMembers(coselmarUsers); - documentEntity.addRestrictedList(restrictedUsers); - - } - - documentEntity.addAllKeywords(document.getKeywords()); - - Date depositDate = document.getDepositDate(); - if (depositDate != null) { - documentEntity.setDepositDate(new Date(depositDate.getTime())); - } else { - documentEntity.setDepositDate(new Date()); - } - - documentEntity.setType(document.getType()); - documentEntity.setSummary(document.getSummary()); - documentEntity.setLanguage(document.getLanguage()); - documentEntity.setPublicationDate(document.getPublicationDate()); - - - // Legal / copyright part - documentEntity.setAuthors(document.getAuthors()); - documentEntity.setCopyright(document.getCopyright()); - documentEntity.setLicense(document.getLicense()); - - // Document resource part + FileInfos fileInfos = null; if (uploadFile != null) { - documentEntity.setWithFile(true); - documentEntity.setMimeType(contentType); - documentEntity.setFilePath(filePath); - documentEntity.setFileContent(fileContent); - } else { - documentEntity.setWithFile(false); + fileInfos = manageDocumentFile(uploadFile, owner); } - documentEntity.setExternalUrl(document.getExternalUrl()); - - documentEntity.setComment(document.getComment()); - - documentEntity.setCitation(document.getCitation()); + DocumentBean result = createDocument(document, fileInfos, owner); commit(); - DocumentBean result = BeanEntityConverter.toBean(getPersistenceContext().getTopiaIdFactory(), documentEntity); - - DocumentsIndexationService documentsIndexationService = getServicesContext().newService(DocumentsIndexationService.class); - try { - documentsIndexationService.indexDocument(result, fileContent); - if (log.isDebugEnabled()) { - String message = String.format("Document '%s' added to index", documentName); - log.debug(message); - } - } catch (IOException e) { - if (log.isErrorEnabled()) { - log.error("Unable to index new document", e); - } - } return result; @@ -492,9 +413,9 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { // Get owner to place correctly the file CoselmarUser owner = documentEntity.getOwner(); - Pair<String, String> pathAndContentType = managerDocumentFile(uploadFile, owner); - String filePath = pathAndContentType.getLeft(); - String contentType = pathAndContentType.getRight(); + FileInfos fileInfos = manageDocumentFile(uploadFile, owner); + String filePath = fileInfos.getFilePath(); + String contentType = fileInfos.getMimeType(); // Read file content String fileContent = TikaUtils.getFileContent(filePath); @@ -757,13 +678,117 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { //////////////////////////////////////////////////////////////////////////// /** + * From a {@link DocumentBean}, create a {@link Document}. + * If there is a {@link FileInfos}, get the file data. + * + * @return a DocumentBean with all created {@link Document} data. + */ + protected DocumentBean createDocument(DocumentBean documentBean, FileInfos fileInfos, CoselmarUser owner) { + + // Document Metadata + Document documentEntity = createDocumentMetadataFromBean(documentBean, owner); + + // If document has a file, manage it ! + String fileContent = ""; + + if (fileInfos != null) { + String filePath = fileInfos.getFilePath(); + String contentType = fileInfos.getMimeType(); + fileContent = TikaUtils.getFileContent(filePath); + + documentEntity.setWithFile(true); + documentEntity.setMimeType(contentType); + documentEntity.setFilePath(filePath); + documentEntity.setFileContent(fileContent); + } else { + documentEntity.setWithFile(false); + } + + DocumentBean result = BeanEntityConverter.toBean(getPersistenceContext().getTopiaIdFactory(), documentEntity); + + // Indexation job + DocumentsIndexationService documentsIndexationService = getServicesContext().newService(DocumentsIndexationService.class); + try { + documentsIndexationService.indexDocument(result, fileContent); + if (log.isDebugEnabled()) { + String message = String.format("Document '%s' added to index", documentBean.getName()); + log.debug(message); + } + } catch (IOException e) { + if (log.isErrorEnabled()) { + log.error("Unable to index new document", e); + } + } + return result; + } + + /** + * Convert metadata fields from {@link DocumentBean} into a {@link Document} and put default values where no given (such as {@link Document#PROPERTY_DEPOSIT_DATE} for example). + * This also manage owner and restriction but <strong>does not manage</strong> attached file part. + * No commit are done here. + */ + public Document createDocumentMetadataFromBean(DocumentBean documentBean, CoselmarUser owner) { + Preconditions.checkNotNull(documentBean); + + // Document Metadata + Document document = getDocumentDao().create(); + + document.setOwner(owner); + + document.setName(documentBean.getName()); + + String privacy = documentBean.getPrivacy().toUpperCase(); + document.setPrivacy(Privacy.valueOf(privacy)); + + // Manage privacy : if restricted, create an UserGroup with authorized users + if (StringUtils.equals(Privacy.RESTRICTED.name(), privacy)) { + Set<UserBean> authorizedUsers = documentBean.getAuthorizedUsers(); + Set<CoselmarUser> coselmarUsers = retrieveUsers(authorizedUsers); + CoselmarUserGroup restrictedUsers = getCoselmarUserGroupDao().create(); + restrictedUsers.setName(document.getTopiaId()); + + restrictedUsers.addAllMembers(coselmarUsers); + document.addRestrictedList(restrictedUsers); + } + + document.addAllKeywords(documentBean.getKeywords()); + + Date depositDate = documentBean.getDepositDate(); + if (depositDate != null) { + document.setDepositDate(new Date(depositDate.getTime())); + } else { + document.setDepositDate(new Date()); + } + + document.setType(documentBean.getType()); + document.setSummary(documentBean.getSummary()); + document.setLanguage(documentBean.getLanguage()); + Date publicationDate = documentBean.getPublicationDate(); + if (publicationDate != null) { + document.setPublicationDate(new Date(publicationDate.getTime())); + } + + document.setCitation(documentBean.getCitation()); + document.setComment(documentBean.getComment()); + + // Legal / copyright part + document.setAuthors(documentBean.getAuthors()); + document.setCopyright(documentBean.getCopyright()); + document.setLicense(documentBean.getLicense()); + + document.setExternalUrl(documentBean.getExternalUrl()); + + return document; + } + + /** * When a Document is sent, it could have a File part : this manage the * upload file. The file is stored in the user specific directory, and the * contentType of document is returned, cause need in Document Metadata. * * @return the upload file Metadata */ - protected Pair<String, String> managerDocumentFile(UploadFile uploadFile, CoselmarUser owner) { + protected FileInfos manageDocumentFile(UploadFile uploadFile, CoselmarUser owner) { Preconditions.checkNotNull(uploadFile); // Document File @@ -771,9 +796,9 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { File uploadedFile = uploadFile.getFile(); String contentType = uploadFile.getContentType(); - if (log.isInfoEnabled()) { + if (log.isDebugEnabled()) { String message = String.format("File name : %s, content-type : %s", fileName, contentType); - log.info(message); + log.debug(message); } // put the document in the good directory @@ -792,7 +817,12 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { } throw new CoselmarTechnicalException("Internal error during file transfer"); } - return Pair.of(destFile.getAbsolutePath(), contentType); + + FileInfos fileInfos = new FileInfos(); + fileInfos.setFileName(fileName); + fileInfos.setMimeType(contentType); + fileInfos.setFilePath(destFile.getAbsolutePath()); + return fileInfos; } protected String getUserDocumentPath(CoselmarUser user) { -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.