[Coselmar-commits] 02/07: refs-30 #9206 First draft for Documents Zip management

This is an automated email from the git hooks/post-receive script. New commit to branch develop in repository coselmar. See https://gitlab.nuiton.org/codelutin/coselmar.git commit d5a986fb539a3ea59c61b4d954b6f92e1ed09368 Author: Yannick Martel <martel@©odelutin.com> Date: Fri Jun 2 16:54:23 2017 +0200 refs-30 #9206 First draft for Documents Zip management --- coselmar-rest/pom.xml | 4 - .../fr/ifremer/coselmar/beans/DocumentBean.java | 5 +- .../coselmar/beans/DocumentImportModel.java | 23 ++- .../services/CoselmarWebServiceSupport.java | 8 +- .../coselmar/services/indexation/TikaUtils.java | 4 + .../coselmar/services/v1/AdminWebService.java | 34 ++-- .../coselmar/services/v1/DocumentsWebService.java | 194 ++++++++++++++++++++- .../services/v1/DocumentsWebServiceTest.java | 93 ++++++++++ coselmar-rest/src/test/resources/documents.zip | Bin 0 -> 151823 bytes 9 files changed, 329 insertions(+), 36 deletions(-) diff --git a/coselmar-rest/pom.xml b/coselmar-rest/pom.xml index 9424e3e..6cba769 100644 --- a/coselmar-rest/pom.xml +++ b/coselmar-rest/pom.xml @@ -247,10 +247,6 @@ <testResources> <testResource> <directory>src/test/resources</directory> - <includes> - <include>**/*.properties</include> - </includes> - <filtering>true</filtering> </testResource> </testResources> <pluginManagement> diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/DocumentBean.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/DocumentBean.java index cc724c8..80d9425 100644 --- a/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/DocumentBean.java +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/DocumentBean.java @@ -28,6 +28,7 @@ import java.io.Serializable; import java.util.Collection; import java.util.Date; import java.util.HashSet; +import java.util.List; import java.util.Set; /** @@ -165,8 +166,8 @@ public class DocumentBean implements Serializable { return keywords; } - public void setKeywords(Collection<String> keywords) { - this.keywords = new HashSet<>(keywords); + public void setKeywords(Set<String> keywords) { + this.keywords = keywords; } public void addKeywords(Collection<String> keywords) { diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/DocumentImportModel.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/DocumentImportModel.java index f8b5749..a5ff66a 100644 --- a/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/DocumentImportModel.java +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/DocumentImportModel.java @@ -1,39 +1,46 @@ package fr.ifremer.coselmar.beans; -import fr.ifremer.coselmar.persistence.entity.Privacy; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import org.nuiton.csv.Common; import org.nuiton.csv.ValueParser; import org.nuiton.csv.ext.AbstractImportModel; import java.text.ParseException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Set; /** * @author ymartel (martel@codelutin.com) */ public class DocumentImportModel extends AbstractImportModel<DocumentBean> { - protected static final ValueParser<Privacy> DOCUMENT_PRIVACY_PARSER = new ValueParser<Privacy>() { + protected static final ValueParser<Set<String>> LIST_STRING_PARSER = new ValueParser<Set<String>>() { @Override - public Privacy parse(String value) throws ParseException { - return Privacy.valueOf(value.toUpperCase()); + public Set<String> parse(String value) throws ParseException { + return Sets.newHashSet(value.split(",")); } }; + protected static final Common.DateValue DATE_PARSER = new Common.DateValue("yyyy/mm/dd"); + public DocumentImportModel() { super(';'); newMandatoryColumn("name", "name"); newMandatoryColumn("type", "type"); - newMandatoryColumn("privacy", "privacy", DOCUMENT_PRIVACY_PARSER); - newMandatoryColumn("keywords", "keywords"); + newMandatoryColumn("privacy", "privacy"); + newMandatoryColumn("keywords", "keywords", LIST_STRING_PARSER); newOptionalColumn("authors", "authors"); newMandatoryColumn("summary", "summary"); newOptionalColumn("license", "license"); newOptionalColumn("copyright", "copyright"); newOptionalColumn("language", "language"); - newOptionalColumn("publicationDate", "publicationDate"); + newOptionalColumn("publicationDate", "publicationDate", DATE_PARSER); newOptionalColumn("comment", "comment"); - newMandatoryColumn("fileName", "fileName"); newMandatoryColumn("citation", "citation"); + newMandatoryColumn("fileName", "fileName"); } @Override diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/CoselmarWebServiceSupport.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/CoselmarWebServiceSupport.java index 1d52f0a..4c85fc7 100644 --- a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/CoselmarWebServiceSupport.java +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/CoselmarWebServiceSupport.java @@ -68,7 +68,13 @@ public abstract class CoselmarWebServiceSupport extends WebMotionController impl protected CoselmarServicesContext getServicesContext() { //try to get it from Request context - HttpContext context = getContext(); + HttpContext context; + try { + context = getContext(); + } catch (NullPointerException e) { + // not web context ? //XXX ymartel use because unit test on Documents mass import are not in web context ... + context = null; + } if (context != null) { CoselmarRestRequestContext requestContext = CoselmarRestRequestContext.getRequestContext(context); this.servicesContext = requestContext.getServicesContext(); diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/indexation/TikaUtils.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/indexation/TikaUtils.java index 68e9ca4..eb46437 100644 --- a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/indexation/TikaUtils.java +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/indexation/TikaUtils.java @@ -52,4 +52,8 @@ public class TikaUtils { return fileContent; } + public static String getFileMimeType(String filePath) { + String mimeType = tika.detect(filePath); + return mimeType; + } } diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/AdminWebService.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/AdminWebService.java index 40c4534..6fd6325 100644 --- a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/AdminWebService.java +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/AdminWebService.java @@ -69,25 +69,11 @@ public class AdminWebService extends CoselmarWebServiceSupport { } - DocumentsIndexationService documentsIndexationService = getServicesContext().newService(DocumentsIndexationService.class); QuestionsIndexationService questionsIndexationService = getServicesContext().newService(QuestionsIndexationService.class); try { getServicesContext().getLuceneUtils().clearIndex(); + refreshDocumentsIndex(); - // get All documents - List<Document> documents = getDocumentDao().findAll(); - for (Document document : documents) { - DocumentBean documentBean = BeanEntityConverter.toBean(getPersistenceContext().getTopiaIdFactory(), document); - if (document.isWithFile()) { - // Refresh file information - String fileContent = TikaUtils.getFileContent(document.getFilePath()); - documentsIndexationService.indexDocument(documentBean, fileContent); - // Refresh database content - document.setFileContent(fileContent); - getDocumentDao().update(document); - } - } - commit(); // Get all questions List<Question> questions = getQuestionDao().findAll(); @@ -108,4 +94,22 @@ public class AdminWebService extends CoselmarWebServiceSupport { } } + protected void refreshDocumentsIndex() throws IOException { + DocumentsIndexationService documentsIndexationService = getServicesContext().newService(DocumentsIndexationService.class); + // get All documents + List<Document> documents = getDocumentDao().findAll(); + for (Document document : documents) { + DocumentBean documentBean = BeanEntityConverter.toBean(getPersistenceContext().getTopiaIdFactory(), document); + if (document.isWithFile()) { + // Refresh file information + String fileContent = TikaUtils.getFileContent(document.getFilePath()); + documentsIndexationService.indexDocument(documentBean, fileContent); + // Refresh database content + document.setFileContent(fileContent); + getDocumentDao().update(document); + } + } + commit(); + } + } diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/DocumentsWebService.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/DocumentsWebService.java index 1e52d9c..80be99e 100644 --- a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/DocumentsWebService.java +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/DocumentsWebService.java @@ -31,6 +31,7 @@ import com.google.common.collect.Collections2; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import fr.ifremer.coselmar.beans.DocumentBean; +import fr.ifremer.coselmar.beans.DocumentImportModel; import fr.ifremer.coselmar.beans.DocumentSearchBean; import fr.ifremer.coselmar.beans.DocumentSearchExample; import fr.ifremer.coselmar.beans.FileInfos; @@ -51,12 +52,17 @@ import fr.ifremer.coselmar.services.errors.NoResultException; import fr.ifremer.coselmar.services.errors.UnauthorizedException; import fr.ifremer.coselmar.services.indexation.DocumentsIndexationService; import fr.ifremer.coselmar.services.indexation.TikaUtils; +import fr.ifremer.coselmar.services.indexation.TransverseIndexationService; import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.lucene.queryparser.classic.ParseException; import org.debux.webmotion.server.call.UploadFile; import org.debux.webmotion.server.render.Render; +import org.nuiton.csv.Import; +import org.nuiton.csv.ImportModel; +import org.nuiton.csv.ImportRuntimeException; import org.nuiton.topia.persistence.TopiaNoResultException; import org.nuiton.util.DateUtil; import org.nuiton.util.pagination.PaginationResult; @@ -64,14 +70,21 @@ import org.nuiton.util.pagination.PaginationResult; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; +import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.Collection; import java.util.Date; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; import static org.apache.commons.logging.LogFactory.getLog; @@ -672,6 +685,42 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { return types; } + public void uploadZipDocuments(UploadFile uploadFile) throws InvalidCredentialException, UnauthorizedException { + + Preconditions.checkNotNull(uploadFile); + + // Check authentication + String authorization = getContext().getHeader("Authorization"); + CoselmarUser currentUser = checkUserAuthentication(authorization); + + // Only Supervisor/Admin can add Zip documents + if (!DOCUMENT_SUPER_USER_ROLES.contains(currentUser.getRole().name())) { + String message = String.format("User %s %s ('%s') is not allowed to upload mass document files", + currentUser.getFirstname(), currentUser.getName(), getShortIdFromFull(currentUser.getTopiaId())); + if (log.isWarnEnabled()) { + log.warn(message); + } + throw new UnauthorizedException(message); + } + List<String> missingFiles = importFromZip(uploadFile.getFile(), currentUser); + + if (missingFiles.isEmpty()) { + // All is ok ! + commit(); + + } else { + // Something wrong happened... rollback, and refresh lucene to avoid new data + rollback(); + AdminWebService adminWebService = getServicesContext().newService(AdminWebService.class); + try { + adminWebService.refreshDocumentsIndex(); + } catch (IOException e) { + if (log.isErrorEnabled()) { + log.error("Unable to refresh Lucene Documents index. Data should be corrupted", e); + } + } + } + } //////////////////////////////////////////////////////////////////////////// /////////////////////// Internal Parts ///////////////////////////// @@ -802,13 +851,9 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { } // put the document in the good directory - String userPath = getUserDocumentPath(owner); + String filePath = getDocumentFileDestPath(owner, fileName); - Date now = getNow(); - String formattedDay = DateUtil.formatDate(now, "yyyyMMddHHmm"); - String prefix = formattedDay + "-"; - - File destFile = new File(userPath + File.separator + prefix + fileName); + File destFile = new File(filePath); try { FileUtils.moveFile(uploadedFile, destFile); } catch (IOException e) { @@ -825,6 +870,20 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { return fileInfos; } + protected String getDocumentFileDestPath(CoselmarUser owner, String fileName) { + String userPath = getUserDocumentPath(owner); + + String storageFileName = getFileStorageName(fileName); + return userPath + File.separator + storageFileName; + } + + protected String getFileStorageName(String fileName) { + Date now = getNow(); + String formattedDay = DateUtil.formatDate(now, "yyyyMMddHHmm"); + String prefix = formattedDay + "-"; + return prefix + fileName; + } + protected String getUserDocumentPath(CoselmarUser user) { File dataDirectory = getCoselmarServicesConfig().getDataDirectory(); String absolutePath = dataDirectory.getAbsolutePath(); @@ -920,4 +979,127 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { return new HashSet<>(coselmarUsers); } + + protected List<String> importFromZip(File file, CoselmarUser currentUser) { + // File should be a Zip + ZipFile zipFile; + try { + zipFile = new ZipFile(file);; + } catch (IOException e) { + if (log.isErrorEnabled()) { + log.error("error during ZipFile transfer", e); + } + throw new CoselmarTechnicalException("Internal error during ZipFile transfer", e); + } + + // Get descriptions.csv : it should contains all DocumentBean information + ZipEntry descriptionEntry = zipFile.getEntry(DESCRIPTION_CSV_FILE_NAME); + InputStream descriptionInputStream; + try { + descriptionInputStream = zipFile.getInputStream(descriptionEntry); + } catch (IOException e) { + String message = String.format("Unable to read '%s' from zip file", DESCRIPTION_CSV_FILE_NAME); + if (log.isErrorEnabled()) { + log.error(message, e); + } + throw new CoselmarTechnicalException(message, e); + } + + // Now, read CSV ... + DocumentImportModel csvModel = new DocumentImportModel(); + Import<DocumentBean> importer = Import.newImport(csvModel, descriptionInputStream); + + File dataDirectory = getCoselmarServicesConfig().getDataDirectory(); + String dataPath = dataDirectory.getAbsolutePath(); + String zipTempPath = dataPath + File.separator + DateUtil.formatDate(getNow(), "yyyyMMddHHmm"); + Path dir = Paths.get(zipTempPath); + try { + Files.createDirectories(dir); + } catch (IOException e) { + if (log.isErrorEnabled()) { + String message = "Unable to create temp path for zip import"; + log.error(message, e); + } + throw new CoselmarTechnicalException("Unable to unzip file : error with unzip directory", e); + } + + List<String> missingFiles = new ArrayList<>(); + // ... and read each entries and retrieve associated File + try { + for (DocumentBean documentBean : importer) { + FileInfos fileInfos = new FileInfos(); + String fileName = documentBean.getFileName(); + ZipEntry zipFileEntry = zipFile.getEntry(fileName); + if (zipFileEntry == null) { + // TODO ymartel 20170601 : manage errors + regenerate Lucene Index + missingFiles.add(fileName); + + } else { + try { + InputStream zipFileInputStream = zipFile.getInputStream(zipFileEntry); + fileInfos.setFileName(fileName); + String fileMimeType = TikaUtils.getFileMimeType(fileName); + String futureFilePath = getDocumentFileDestPath(currentUser, fileName); + + String storageFileName = getFileStorageName(fileName); + + // Push file stream in temporary folder + File zipEntryFile = new File(zipTempPath + File.separator + storageFileName); + zipEntryFile.createNewFile(); + Files.copy(zipFileInputStream, zipEntryFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + + // Create fileInfos + fileInfos.setFileName(fileName); + fileInfos.setFilePath(futureFilePath); + fileInfos.setMimeType(fileMimeType); + + // Create document + createDocument(documentBean, fileInfos, currentUser); + + try { + zipFileInputStream.close(); + } catch (IOException e) { + if (log.isErrorEnabled()) { + log.error("Unable to close stream from ZipEntry : " + fileName, e); + } + } + } catch (IOException e) { + String message = String.format("Unable to retrieve '%s' from zip file", fileName); + if (log.isErrorEnabled()) { + log.error(message, e); + } + // TODO ymartel 20170601 : manage errors + regenerate Lucene Index + missingFiles.add(fileName); + } + } + + } + } catch (ImportRuntimeException ire) { + if (log.isErrorEnabled()) { + log.error("Error with CSV file", ire); + } + throw new CoselmarTechnicalException("Error with CSV file", ire); + } finally { + // Close importer + importer.close(); + // and csv stream + try { + descriptionInputStream.close(); + } catch (IOException e) { + if (log.isErrorEnabled()) { + log.error("Unable to close 'descriptions.csv' input stream from zip"); + } + } + } + if (missingFiles.isEmpty()) { + // Ok, let move all files from temp storage to real one ! + try { + FileUtils.moveDirectory(new File(zipTempPath), new File(getUserDocumentPath(currentUser))); + } catch (IOException e) { + // Big problem if we can't move files into real folder ! + e.printStackTrace(); + } + } + return missingFiles; + } } diff --git a/coselmar-rest/src/test/java/fr/ifremer/coselmar/services/v1/DocumentsWebServiceTest.java b/coselmar-rest/src/test/java/fr/ifremer/coselmar/services/v1/DocumentsWebServiceTest.java new file mode 100644 index 0000000..5357c1c --- /dev/null +++ b/coselmar-rest/src/test/java/fr/ifremer/coselmar/services/v1/DocumentsWebServiceTest.java @@ -0,0 +1,93 @@ +package fr.ifremer.coselmar.services.v1; + +/* + * #%L + * Coselmar :: Rest Services + * $Id:$ + * $HeadURL:$ + * %% + * Copyright (C) 2014 Ifremer, Code Lutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/gpl-3.0.html>. + * #L% + */ + +import fr.ifremer.coselmar.persistence.entity.CoselmarUser; +import fr.ifremer.coselmar.persistence.entity.CoselmarUserImpl; +import fr.ifremer.coselmar.persistence.entity.CoselmarUserTopiaDao; +import fr.ifremer.coselmar.persistence.entity.Document; +import fr.ifremer.coselmar.persistence.entity.DocumentTopiaDao; +import fr.ifremer.coselmar.services.AbstractCoselmarServiceTest; +import fr.ifremer.coselmar.services.FakeCoselmarServicesContext; +import org.junit.Assert; +import org.junit.Test; + +import java.io.File; +import java.io.InputStream; +import java.net.URL; +import java.nio.file.CopyOption; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.List; +import java.util.Locale; + +/** + * @author ymartel <martel@codelutin.com> + */ +public class DocumentsWebServiceTest extends AbstractCoselmarServiceTest { + + protected FakeCoselmarServicesContext serviceContext; + + protected FakeCoselmarServicesContext getServiceContext() { + + if (serviceContext == null) { + serviceContext = application.newServiceContext(application.newPersistenceContext(), Locale.FRANCE); + } + + return serviceContext; + } + + @Test + public void testUploadZipDocuments() throws Exception { + + DocumentsWebService documentsWebService = getServiceContext().newService(DocumentsWebService.class); +// URL zipFileURL = this.getClass().getResource("/documents.zip"); + InputStream resourceAsStream = this.getClass().getResourceAsStream("/documents.zip"); + Path tempZipPath = Files.createTempFile(null, null); + Files.copy(resourceAsStream, tempZipPath, StandardCopyOption.REPLACE_EXISTING); + resourceAsStream.close(); + File zipFile = tempZipPath.toFile(); + Assert.assertNotNull(zipFile); + Assert.assertTrue(zipFile.exists()); + + // create a test user + CoselmarUserTopiaDao coselmarUserDao = getServiceContext().getPersistenceContext().getCoselmarUserDao(); + CoselmarUser coselmarUser = coselmarUserDao.create(); + coselmarUser.setName("Peuplu"); + coselmarUser.setFirstname("Jean"); + getServiceContext().getPersistenceContext().commit(); + + List<String> errors = documentsWebService.importFromZip(zipFile, coselmarUser); + Assert.assertNotNull(errors); + Assert.assertTrue(errors.isEmpty()); + DocumentTopiaDao documentDao = getServiceContext().getPersistenceContext().getDocumentDao(); + List<Document> documents = documentDao.findAll(); + Assert.assertEquals(3, documents.size()); + String oneFilePath = documents.get(0).getFilePath(); + File oneFile = new File(oneFilePath); + Assert.assertTrue(oneFile.exists()); + } +} diff --git a/coselmar-rest/src/test/resources/documents.zip b/coselmar-rest/src/test/resources/documents.zip new file mode 100644 index 0000000..6467307 Binary files /dev/null and b/coselmar-rest/src/test/resources/documents.zip differ -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.