This is an automated email from the git hooks/post-receive script. New commit to branch develop in repository echobase. See https://gitlab.nuiton.org/codelutin/echobase.git commit 42961ba52814b9ab11781ba84c88e9231bff3df1 Author: Tony CHEMIT <chemit@codelutin.com> Date: Mon May 2 11:26:34 2016 +0200 Début d'optimisation des check d'import --- .../echobase/entities/ImportFileIdTopiaDao.java | 19 ++ .../services/service/UserDbPersistenceService.java | 7 + .../actions/ImportDataActionSupport.java | 6 + .../VoyageCatchesBiometrySampleImportAction.java | 31 +-- .../VoyageCatchesSubSampleImportAction.java | 257 +++++++++++++++------ .../VoyageCatchesTotalSampleImportAction.java | 19 +- .../actions/VoyageCommonsTransectImportAction.java | 9 - .../actions/VoyageCommonsTransitImportAction.java | 11 - .../VoyageOperationsGearMetadataImportAction.java | 36 +-- .../VoyageOperationsOperationImportAction.java | 25 +- ...ageOperationsOperationMetadataImportAction.java | 37 +-- .../VoyageResultsCellImportDataActionSupport.java | 70 +++--- ...VoyageResultsRegionAssociationImportAction.java | 31 ++- .../actions/VoyageResultsRegionImportAction.java | 41 +--- ...oyageResultsVoyageLengthAgeKeyImportAction.java | 8 +- ...geResultsVoyageLengthWeightKeyImportAction.java | 8 +- .../csv/VoyageCatchesBiometrySampleImportRow.java | 5 +- .../csv/VoyageCatchesTotalSampleImportRow.java | 4 +- ...VoyageOperationsGearMetadataValueImportRow.java | 5 +- .../csv/VoyageOperationsOperationImportRow.java | 3 +- ...eOperationsOperationMetadataValueImportRow.java | 5 +- .../csv/VoyageResultsRegionCellImportRow.java | 3 +- .../echobase/services/EchoBaseServiceFixtures.java | 8 + .../VoyageCatchesAllImportServiceIT.java | 16 +- .../VoyageResultsRegionCellImportServiceIT.java | 7 +- 25 files changed, 332 insertions(+), 339 deletions(-) diff --git a/echobase-domain/src/main/java/fr/ifremer/echobase/entities/ImportFileIdTopiaDao.java b/echobase-domain/src/main/java/fr/ifremer/echobase/entities/ImportFileIdTopiaDao.java index 3df60bf..5555d79 100644 --- a/echobase-domain/src/main/java/fr/ifremer/echobase/entities/ImportFileIdTopiaDao.java +++ b/echobase-domain/src/main/java/fr/ifremer/echobase/entities/ImportFileIdTopiaDao.java @@ -3,6 +3,9 @@ package fr.ifremer.echobase.entities; import com.google.common.collect.ImmutableMap; import org.nuiton.topia.persistence.HqlAndParametersBuilder; import org.nuiton.topia.persistence.TopiaEntity; +import org.nuiton.util.pagination.PaginationResult; + +import java.util.Map; public class ImportFileIdTopiaDao extends AbstractImportFileIdTopiaDao<ImportFileId> { @@ -24,6 +27,22 @@ public class ImportFileIdTopiaDao extends AbstractImportFileIdTopiaDao<ImportFil } + public <E extends TopiaEntity> Iterable<E> getImportedEntities(ImportFile importFile, Class<E> entityType) { + + HqlAndParametersBuilder<ImportFileId> builder = newHqlAndParametersBuilder(); + builder.setSelectClause(" SELECT entityId"); + builder.addEquals(ImportFileId.PROPERTY_IMPORT_FILE, importFile); + builder.addWhereClause(ImportFileId.PROPERTY_ENTITY_ID + " LIKE :" + ImportFileId.PROPERTY_ENTITY_ID, ImmutableMap.<String, Object>builder().put(ImportFileId.PROPERTY_ENTITY_ID, entityType.getName() + "#%").build()); + builder.setOrderByArguments(ImportFileId.PROPERTY_IMPORT_ORDER); + + String idHql = builder.getHql(); + Map<String, Object> hqlParameters = builder.getHqlParameters(); + PaginationResult<ImportFileId> pager = initPagination(idHql, hqlParameters, batchSize); + String hql = "From " + entityType.getName() + " Where topiaId IN (" + idHql + ")"; + return findAllLazy(hql, hqlParameters, pager); + + } + public long getImportFileIdsCountForImportFile(ImportFile importFile) { return forImportFileEquals(importFile).count(); } diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/UserDbPersistenceService.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/UserDbPersistenceService.java index bd06702..7f49194 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/UserDbPersistenceService.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/UserDbPersistenceService.java @@ -1036,6 +1036,13 @@ public class UserDbPersistenceService extends EchoBaseServiceSupport { } + public <E extends TopiaEntity> Iterable<E> getImportedEntities(ImportFile importFile, Class<E> entityType) { + + ImportFileIdTopiaDao importFileIdDao = persistenceContext.getImportFileIdDao(); + return importFileIdDao.getImportedEntities(importFile, entityType); + + } + public long getImportFileIdsCount(ImportFile importFile) { ImportFileIdTopiaDao importFileIdDao = persistenceContext.getImportFileIdDao(); diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ImportDataActionSupport.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ImportDataActionSupport.java index 6f4a0b1..9c225db 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ImportDataActionSupport.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/ImportDataActionSupport.java @@ -391,6 +391,12 @@ public abstract class ImportDataActionSupport<M extends ImportDataConfigurationS } + protected <EE extends TopiaEntity> Iterable<EE> getImportedEntities(Class<EE> type, ImportDataFileResult result) { + + return persistenceService.getImportedEntities(result.getImportFile(), type); + + } + protected Iterable<ImportFileId> getImportFileIds(ImportDataFileResult result) { return persistenceService.getImportFileIdsForImportFile(result.getImportFile()); } diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesBiometrySampleImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesBiometrySampleImportAction.java index be86dbd..bcd88eb 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesBiometrySampleImportAction.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesBiometrySampleImportAction.java @@ -2,7 +2,6 @@ package fr.ifremer.echobase.services.service.importdata.actions; import com.google.common.base.Preconditions; import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; -import fr.ifremer.echobase.entities.ImportFileId; import fr.ifremer.echobase.entities.data.Operation; import fr.ifremer.echobase.entities.data.Sample; import fr.ifremer.echobase.entities.data.SampleData; @@ -34,8 +33,6 @@ public class VoyageCatchesBiometrySampleImportAction extends VoyageCatchesImport /** Logger. */ private static final Log log = LogFactory.getLog(VoyageCatchesBiometrySampleImportAction.class); - private final Map<String, String> sampleIdToOperationId = new TreeMap<>(); - private final Map<String, String> sampleDataIdToSampleId = new TreeMap<>(); public VoyageCatchesBiometrySampleImportAction(VoyageCatchesImportDataContext importDataContext) { super(importDataContext, importDataContext.getConfiguration().getBiometrySampleFile()); @@ -102,20 +99,14 @@ public class VoyageCatchesBiometrySampleImportAction extends VoyageCatchesImport addProcessedRow(result, row); - // add to sample ids cache (need to compute imported export later) - sampleIdToOperationId.put(sample.getTopiaId(), operation.getTopiaId()); - SampleData sampleDataFishIndex = persistenceService.createSampleData(sampleDataTypeFishIndex, null, (float) numFish); sample.addSampleData(sampleDataFishIndex); addId(result, EchoBaseUserEntityEnum.SampleData, sampleDataFishIndex, rowNumber); - SampleData sampleData = row.getSampleData(); SampleData sampleDataCreated = persistenceService.createSampleData(row.getSampleData()); sample.addSampleData(sampleDataCreated); addId(result, EchoBaseUserEntityEnum.SampleData, sampleDataCreated, rowNumber); - sampleDataIdToSampleId.put(sampleData.getTopiaId(), sample.getTopiaId()); - } } @@ -129,31 +120,15 @@ public class VoyageCatchesBiometrySampleImportAction extends VoyageCatchesImport SampleData sampleDataFishIndex = null; - for (ImportFileId importFileId : filterIds(SampleData.class, result)) { + for (SampleData sampleData : getImportedEntities(SampleData.class, result)) { - String sampleDataId = importFileId.getEntityId(); - SampleData sampleData = persistenceService.getSampleData(sampleDataId); if (sampleDataTypeFishIndex.equals(sampleData.getSampleDataType())) { sampleDataFishIndex = sampleData; continue; } - String sampleId = sampleDataIdToSampleId.get(sampleDataId); - Preconditions.checkNotNull(sampleId); - - String operationId = sampleIdToOperationId.get(sampleId); - Preconditions.checkNotNull(operationId); - - Operation operation = persistenceService.getOperation(operationId); - Preconditions.checkNotNull(operation); - - Sample sample = persistenceService.getSample(sampleId); - Preconditions.checkNotNull(sample); - - Preconditions.checkNotNull(sample.getSampleDataByTopiaId(sampleDataId)); - if (log.isInfoEnabled()) { - log.info("Adding sampleData: " + sampleId + " to imported export."); + log.info("Adding sampleData: " + sampleData.getTopiaId() + " to imported export."); } Preconditions.checkNotNull(sampleDataFishIndex); @@ -161,7 +136,7 @@ public class VoyageCatchesBiometrySampleImportAction extends VoyageCatchesImport Float numFishSampleDataValue = sampleDataFishIndex.getDataValue(); Preconditions.checkNotNull(numFishSampleDataValue); - VoyageCatchesBiometrySampleImportRow importedRow = VoyageCatchesBiometrySampleImportRow.of(operation, sample, sampleData, numFishSampleDataValue.intValue()); + VoyageCatchesBiometrySampleImportRow importedRow = VoyageCatchesBiometrySampleImportRow.of(sampleData, numFishSampleDataValue.intValue()); addImportedRow(result, importedRow); diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesSubSampleImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesSubSampleImportAction.java index 2eaacbd..6852f5f 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesSubSampleImportAction.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesSubSampleImportAction.java @@ -1,7 +1,6 @@ package fr.ifremer.echobase.services.service.importdata.actions; import com.google.common.base.Preconditions; -import fr.ifremer.echobase.entities.ImportFileId; import fr.ifremer.echobase.entities.data.Operation; import fr.ifremer.echobase.entities.data.Sample; import fr.ifremer.echobase.entities.data.SampleData; @@ -23,10 +22,9 @@ import org.nuiton.csv.Import; import java.util.Collection; import java.util.LinkedHashSet; -import java.util.Map; +import java.util.LinkedList; +import java.util.List; import java.util.Set; -import java.util.TreeMap; -import java.util.TreeSet; /** * Created on 25/03/16. @@ -38,8 +36,6 @@ public class VoyageCatchesSubSampleImportAction extends VoyageCatchesImportDataA /** Logger. */ private static final Log log = LogFactory.getLog(VoyageCatchesSubSampleImportAction.class); - private final Map<String, String> sampleIdToOperationId = new TreeMap<>(); - private final Map<String, String> sampleDataIdToSampleId = new TreeMap<>(); public VoyageCatchesSubSampleImportAction(VoyageCatchesImportDataContext importDataContext) { super(importDataContext, importDataContext.getConfiguration().getSubSampleFile()); @@ -71,7 +67,7 @@ public class VoyageCatchesSubSampleImportAction extends VoyageCatchesImportDataA SampleDataType sampleDataTypeNumberAtLength1cm = importDataContext.getSampleDataTypeNumberAtLength1cm(); SampleDataType sampleDataTypeWeightAtLength = importDataContext.getSampleDataTypeWeightAtLength(); - Set<String> createdSampleIds = new TreeSet<>(); +// Set<String> createdSampleIds = new TreeSet<>(); try (Import<VoyageCatchesSubSampleImportRow> importer = open()) { @@ -102,16 +98,16 @@ public class VoyageCatchesSubSampleImportAction extends VoyageCatchesImportDataA sample.setSampleType(sampleTypeSubsample); sample = addSample(operation, sample, result, rowNumber); - createdSampleIds.add(sample.getTopiaId()); +// createdSampleIds.add(sample.getTopiaId()); } - // add to sample ids cache (need to compute imported export later) - sampleIdToOperationId.put(sample.getTopiaId(), operation.getTopiaId()); +// // add to sample ids cache (need to compute imported export later) +// sampleIdToOperationId.put(sample.getTopiaId(), operation.getTopiaId()); addProcessedRow(result, row); - boolean sampleWasCreated = createdSampleIds.contains(sample.getTopiaId()); +// boolean sampleWasCreated = createdSampleIds.contains(sample.getTopiaId()); Integer round = row.getRound(); SampleDataType dataType; @@ -129,15 +125,17 @@ public class VoyageCatchesSubSampleImportAction extends VoyageCatchesImportDataA { //create numberAtLength data - SampleData sampleData = addSampleData(dataType, "" + row.getLengthClass(), row.getNumberAtLength(), sample, result, true, rowNumber); - sampleDataIdToSampleId.put(sampleData.getTopiaId(), sample.getTopiaId()); +// SampleData sampleData = + addSampleData(dataType, "" + row.getLengthClass(), row.getNumberAtLength(), sample, result, true, rowNumber); +// sampleDataIdToSampleId.put(sampleData.getTopiaId(), sample.getTopiaId()); } if (row.getWeightAtLength() != null) { //create weightAtLength data - SampleData sampleData = addSampleData(sampleDataTypeWeightAtLength, "" + row.getLengthClass(), row.getWeightAtLength(), sample, result, true, rowNumber); - sampleDataIdToSampleId.put(sampleData.getTopiaId(), sample.getTopiaId()); +// SampleData sampleData = + addSampleData(sampleDataTypeWeightAtLength, "" + row.getLengthClass(), row.getWeightAtLength(), sample, result, true, rowNumber); +// sampleDataIdToSampleId.put(sampleData.getTopiaId(), sample.getTopiaId()); } @@ -150,91 +148,53 @@ public class VoyageCatchesSubSampleImportAction extends VoyageCatchesImportDataA @Override protected void computeImportedExport(VoyageCatchesImportDataContext importDataContext, ImportDataFileResult result) { - String currentSampleId = null; +// String currentSampleId = null; - Set<String> sampleDataIds = new LinkedHashSet<>(); + List<SampleData> sampleDataIds = new LinkedList<>(); - for (ImportFileId importedFileId : getImportFileIds(result)) { + Sample sample = null; + + for (SampleData sampleData : getImportedEntities(SampleData.class, result)) { - String importedId = importedFileId.getEntityId(); if (log.isInfoEnabled()) { - log.info("Test imported id: " + importedId); + log.info("Test imported id: " + sampleData.getTopiaId()); } - if (importedId.startsWith(Sample.class.getName() + "#")) { - - // new sample to import + if (sample == null) { - if (currentSampleId == null) { - - // first sample to collect - currentSampleId = importedId; + // first row + sample = sampleData.getSample(); - } else { + } else { - // flush previous sample - addImportedSample(importDataContext, currentSampleId, result, sampleDataIds); + if (!sample.equals(sampleData.getSample())) { - // starts to collect new sample - currentSampleId = importedId; + // flush sample + flushSample(importDataContext, sample, result, sampleDataIds); sampleDataIds.clear(); - } - - } else if (importedId.startsWith(SampleData.class.getName() + "#")) { - - // coming from an existing sample - String sampleId = sampleDataIdToSampleId.get(importedId); - Preconditions.checkNotNull(sampleId); - - if (currentSampleId == null) { - - // first sample to treat - currentSampleId = sampleId; - - } else { - - if (!currentSampleId.equals(sampleId)) { - - // flush previous sample - addImportedSample(importDataContext, currentSampleId, result, sampleDataIds); - - // starts to collect new sample - currentSampleId = sampleId; - sampleDataIds.clear(); - - } + sample = sampleData.getSample(); } - // add this sample data to current sample to flush - sampleDataIds.add(importedId); - - } else { - throw new IllegalStateException("Cant deal with imported id: " + importedId); } + sampleDataIds.add(sampleData); + } if (!sampleDataIds.isEmpty()) { // flush last sample - addImportedSample(importDataContext, currentSampleId, result, sampleDataIds); + flushSample(importDataContext, sample, result, sampleDataIds); } } - protected void addImportedSample(VoyageCatchesImportDataContext importDataContext, String sampleId, ImportDataFileResult result, Set<String> sampleDataIds) { - - String operationId = sampleIdToOperationId.get(sampleId); - Preconditions.checkNotNull(operationId); - - Operation operation = persistenceService.getOperation(operationId); - Preconditions.checkNotNull(operation); + protected void flushSample(VoyageCatchesImportDataContext importDataContext, Sample sample, ImportDataFileResult result, List<SampleData> sampleDataIds) { - Sample sample = persistenceService.getSample(sampleId); Preconditions.checkNotNull(sample); if (log.isInfoEnabled()) { @@ -250,11 +210,11 @@ public class VoyageCatchesSubSampleImportAction extends VoyageCatchesImportDataA VoyageCatchesSubSampleImportRow importedRow = null; + Operation operation = sample.getOperation(); + String lastLengthClass = null; boolean lengthDataTypeFound = false; - for (String sampleDataId : sampleDataIds) { - - SampleData aSampleData = sample.getSampleDataByTopiaId(sampleDataId); + for (SampleData aSampleData : sampleDataIds) { String lengthClass = aSampleData.getDataLabel(); @@ -296,6 +256,155 @@ public class VoyageCatchesSubSampleImportAction extends VoyageCatchesImportDataA } +// @Override +// protected void computeImportedExport(VoyageCatchesImportDataContext importDataContext, ImportDataFileResult result) { +// +// String currentSampleId = null; +// +// Set<String> sampleDataIds = new LinkedHashSet<>(); +// +// for (ImportFileId importedFileId : getImportFileIds(result)) { +// +// String importedId = importedFileId.getEntityId(); +// if (log.isInfoEnabled()) { +// log.info("Test imported id: " + importedId); +// } +// +// if (importedId.startsWith(Sample.class.getName() + "#")) { +// +// // new sample to import +// +// if (currentSampleId == null) { +// +// // first sample to collect +// currentSampleId = importedId; +// +// } else { +// +// // flush previous sample +// addImportedSample(importDataContext, currentSampleId, result, sampleDataIds); +// +// // starts to collect new sample +// currentSampleId = importedId; +// sampleDataIds.clear(); +// +// } +// +// } else if (importedId.startsWith(SampleData.class.getName() + "#")) { +// +// // coming from an existing sample +// String sampleId = sampleDataIdToSampleId.get(importedId); +// Preconditions.checkNotNull(sampleId); +// +// if (currentSampleId == null) { +// +// // first sample to treat +// currentSampleId = sampleId; +// +// } else { +// +// if (!currentSampleId.equals(sampleId)) { +// +// // flush previous sample +// addImportedSample(importDataContext, currentSampleId, result, sampleDataIds); +// +// // starts to collect new sample +// currentSampleId = sampleId; +// sampleDataIds.clear(); +// +// } +// +// } +// +// // add this sample data to current sample to flush +// sampleDataIds.add(importedId); +// +// } else { +// throw new IllegalStateException("Cant deal with imported id: " + importedId); +// } +// +// +// } +// +// if (!sampleDataIds.isEmpty()) { +// +// // flush last sample +// addImportedSample(importDataContext, currentSampleId, result, sampleDataIds); +// +// } +// +// } +// +// protected void addImportedSample(VoyageCatchesImportDataContext importDataContext, String sampleId, ImportDataFileResult result, Set<String> sampleDataIds) { +// +// String operationId = sampleIdToOperationId.get(sampleId); +// Preconditions.checkNotNull(operationId); +// +// Operation operation = persistenceService.getOperation(operationId); +// Preconditions.checkNotNull(operation); +// +// Sample sample = persistenceService.getSample(sampleId); +// Preconditions.checkNotNull(sample); +// +// if (log.isInfoEnabled()) { +// log.info("Adding sample: " + sample + " to imported export."); +// } +// +// Set<SampleDataType> lengthSampleDataTypes = new LinkedHashSet<>(); +// lengthSampleDataTypes.add(importDataContext.getSampleDataTypeNumberAtLength()); +// lengthSampleDataTypes.add(importDataContext.getSampleDataTypeNumberAtLength05cm()); +// lengthSampleDataTypes.add(importDataContext.getSampleDataTypeNumberAtLength1cm()); +// +// Preconditions.checkState(sample.isSampleDataNotEmpty()); +// +// VoyageCatchesSubSampleImportRow importedRow = null; +// +// String lastLengthClass = null; +// boolean lengthDataTypeFound = false; +// for (String sampleDataId : sampleDataIds) { +// +// SampleData aSampleData = sample.getSampleDataByTopiaId(sampleDataId); +// +// String lengthClass = aSampleData.getDataLabel(); +// +// boolean isLengthDataType = lengthSampleDataTypes.contains(aSampleData.getSampleDataType()); +// +// if (lastLengthClass == null) { +// +// // first row +// lastLengthClass = lengthClass; +// importedRow = VoyageCatchesSubSampleImportRow.of(operation, sample); +// lengthDataTypeFound = false; +// +// } else { +// +// if ((isLengthDataType && lengthDataTypeFound) || !lengthClass.equals(lastLengthClass)) { +// +// // changing length class or length data type already found +// addImportedRow(result, importedRow); +// +// lastLengthClass = lengthClass; +// importedRow = VoyageCatchesSubSampleImportRow.of(operation, sample); +// lengthDataTypeFound = false; +// } +// } +// +// if (log.isInfoEnabled()) { +// log.info("Treated sample data: " + aSampleData); +// } +// +// addImportedSampleData(importDataContext, importedRow, aSampleData); +// +// if (isLengthDataType) { +// lengthDataTypeFound = true; +// } +// +// } +// +// addImportedRow(result, importedRow); +// +// } + protected void addImportedSampleData(VoyageCatchesImportDataContext importDataContext, VoyageCatchesSubSampleImportRow importedRow, SampleData sampleData) { SampleDataType sampleDataTypeNumberAtLength = importDataContext.getSampleDataTypeNumberAtLength(); diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesTotalSampleImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesTotalSampleImportAction.java index 7d0cfba..49d8098 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesTotalSampleImportAction.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCatchesTotalSampleImportAction.java @@ -3,7 +3,6 @@ package fr.ifremer.echobase.services.service.importdata.actions; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; -import fr.ifremer.echobase.entities.ImportFileId; import fr.ifremer.echobase.entities.data.Operation; import fr.ifremer.echobase.entities.data.Sample; import fr.ifremer.echobase.entities.data.SampleData; @@ -39,7 +38,6 @@ public class VoyageCatchesTotalSampleImportAction extends VoyageCatchesImportDat /** Logger. */ private static final Log log = LogFactory.getLog(VoyageCatchesTotalSampleImportAction.class); - private final Map<String, String> sampleIdToOperationId = new TreeMap<>(); private final Map<String, String> sampleIdToSortedSampleId = new TreeMap<>(); public VoyageCatchesTotalSampleImportAction(VoyageCatchesImportDataContext importDataContext) { @@ -63,7 +61,6 @@ public class VoyageCatchesTotalSampleImportAction extends VoyageCatchesImportDat log.info("Starts import of totalSample from file " + inputFile.getFileName()); } - sampleIdToOperationId.clear(); sampleIdToSortedSampleId.clear(); SpeciesCategoryCache speciesCategoryCache = importDataContext.getSpeciesCategoryCache(); @@ -147,7 +144,6 @@ public class VoyageCatchesTotalSampleImportAction extends VoyageCatchesImportDat Sample sortedSample = addSample(operation, createdSortedSample, result, rowNumber); // add to sample ids cache (need to compte imported export later) - sampleIdToOperationId.put(sample.getTopiaId(), operation.getTopiaId()); sampleIdToSortedSampleId.put(sample.getTopiaId(), sortedSample.getTopiaId()); } @@ -164,9 +160,9 @@ public class VoyageCatchesTotalSampleImportAction extends VoyageCatchesImportDat SampleDataType sampleDataTypeNoPerKg = importDataContext.getSampleDataTypeNoPerKg(); Collection<String> sortedSampleIds = sampleIdToSortedSampleId.values(); - for (ImportFileId importFileId : filterIds(Sample.class, result)) { + for (Sample sample : getImportedEntities(Sample.class, result)) { - String sampleId = importFileId.getEntityId(); + String sampleId = sample.getTopiaId(); if (sortedSampleIds.contains(sampleId)) { // skip sorted sample @@ -177,22 +173,13 @@ public class VoyageCatchesTotalSampleImportAction extends VoyageCatchesImportDat log.info("Adding sample: " + sampleId + " to imported export."); } - String operationId = sampleIdToOperationId.get(sampleId); - Preconditions.checkNotNull(operationId); - - Operation operation = persistenceService.getOperation(operationId); - Preconditions.checkNotNull(operation); - - Sample sample = persistenceService.getSample(sampleId); - Preconditions.checkNotNull(sample); - String sortedSampleId = sampleIdToSortedSampleId.get(sampleId); Preconditions.checkNotNull(sortedSampleId); Sample sortedSample = persistenceService.getSample(sortedSampleId); Preconditions.checkNotNull(sortedSample); - VoyageCatchesTotalSampleImportRow importedRow = VoyageCatchesTotalSampleImportRow.of(operation, sample, sortedSample); + VoyageCatchesTotalSampleImportRow importedRow = VoyageCatchesTotalSampleImportRow.of(sample, sortedSample); if (sample.isSampleDataNotEmpty()) { diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCommonsTransectImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCommonsTransectImportAction.java index 6a96174..62b029f 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCommonsTransectImportAction.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCommonsTransectImportAction.java @@ -88,15 +88,6 @@ public class VoyageCommonsTransectImportAction extends VoyageCommonsImportDataAc addProcessedRow(result, row); addId(result, EchoBaseUserEntityEnum.Transect, createdTransect, rowNumber); -// if (collectIds) { -// -// // collect id of the import -// result.addId(EchoBaseUserEntityEnum.Transect, createdTransect); -// } else { -// -// result.incrementsNumberCreated(EchoBaseUserEntityEnum.Transect); -// } - } } diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCommonsTransitImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCommonsTransitImportAction.java index f8ccd1a..29367df 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCommonsTransitImportAction.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageCommonsTransitImportAction.java @@ -70,17 +70,6 @@ public class VoyageCommonsTransitImportAction extends VoyageCommonsImportDataAct addId(result, EchoBaseUserEntityEnum.Transit, createdTransit, rowNumber); addProcessedRow(result, row); -// if (collectIds) { -// -// // collect id of the import -// result.addId(EchoBaseUserEntityEnum.Transit, createdTransit); -// -// -// } else { -// -// result.incrementsNumberCreated(EchoBaseUserEntityEnum.Transit); -// } - } } diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageOperationsGearMetadataImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageOperationsGearMetadataImportAction.java index 27e89fd..c8f7447 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageOperationsGearMetadataImportAction.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageOperationsGearMetadataImportAction.java @@ -1,11 +1,8 @@ package fr.ifremer.echobase.services.service.importdata.actions; -import com.google.common.base.Preconditions; import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; -import fr.ifremer.echobase.entities.ImportFileId; import fr.ifremer.echobase.entities.data.GearMetadataValue; import fr.ifremer.echobase.entities.data.Operation; -import fr.ifremer.echobase.entities.references.Vessel; import fr.ifremer.echobase.io.InputFile; import fr.ifremer.echobase.services.service.importdata.ImportDataFileResult; import fr.ifremer.echobase.services.service.importdata.contexts.VoyageOperationsImportDataContext; @@ -15,9 +12,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.csv.Import; -import java.util.Map; -import java.util.TreeMap; - /** * Created on 25/03/16. * @@ -28,8 +22,6 @@ public class VoyageOperationsGearMetadataImportAction extends VoyageOperationsIm /** Logger. */ private static final Log log = LogFactory.getLog(VoyageOperationsGearMetadataImportAction.class); - private final Map<String, String> gearMetadataValueIdToOperationId = new TreeMap<>(); - private final Map<String, String> operationIdToVesselId = new TreeMap<>(); public VoyageOperationsGearMetadataImportAction(VoyageOperationsImportDataContext importDataContext) { super(importDataContext, importDataContext.getConfiguration().getGearMetadataFile()); @@ -69,11 +61,6 @@ public class VoyageOperationsGearMetadataImportAction extends VoyageOperationsIm addProcessedRow(result, row); - gearMetadataValueIdToOperationId.put(gearMetadataValue.getTopiaId(), operation.getTopiaId()); - - Vessel vessel = row.getVessel(); - operationIdToVesselId.put(operation.getTopiaId(), vessel.getTopiaId()); - } } @@ -83,30 +70,13 @@ public class VoyageOperationsGearMetadataImportAction extends VoyageOperationsIm @Override protected void computeImportedExport(VoyageOperationsImportDataContext importDataContext, ImportDataFileResult result) { - for (ImportFileId importFileId : getImportFileIds(result)) { - - String gearMetadataValueId = importFileId.getEntityId(); + for (GearMetadataValue gearMetadataValue : getImportedEntities(GearMetadataValue.class, result)) { if (log.isInfoEnabled()) { - log.info("Adding gear metadata value : " + gearMetadataValueId + " to imported export."); + log.info("Adding gear metadata value : " + gearMetadataValue.getTopiaId() + " to imported export."); } - String operationId = gearMetadataValueIdToOperationId.get(gearMetadataValueId); - Preconditions.checkNotNull(operationId); - - Operation operation = persistenceService.getOperation(operationId); - Preconditions.checkNotNull(operation); - - String vesselId = operationIdToVesselId.get(operationId); - Preconditions.checkNotNull(vesselId); - - Vessel vessel = persistenceService.getVessel(vesselId); - Preconditions.checkNotNull(vessel); - - GearMetadataValue gearMetadataValue = operation.getGearMetadataValueByTopiaId(gearMetadataValueId); - Preconditions.checkNotNull(gearMetadataValue); - - VoyageOperationsGearMetadataValueImportRow importedRow = VoyageOperationsGearMetadataValueImportRow.of(operation, vessel, gearMetadataValue); + VoyageOperationsGearMetadataValueImportRow importedRow = VoyageOperationsGearMetadataValueImportRow.of(gearMetadataValue); addImportedRow(result, importedRow); } diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageOperationsOperationImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageOperationsOperationImportAction.java index eb78547..0bb72a5 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageOperationsOperationImportAction.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageOperationsOperationImportAction.java @@ -1,8 +1,6 @@ package fr.ifremer.echobase.services.service.importdata.actions; -import com.google.common.base.Preconditions; import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; -import fr.ifremer.echobase.entities.ImportFileId; import fr.ifremer.echobase.entities.data.Operation; import fr.ifremer.echobase.entities.data.Transect; import fr.ifremer.echobase.entities.data.Transit; @@ -18,8 +16,6 @@ import org.apache.commons.logging.LogFactory; import org.nuiton.csv.Import; import java.util.Date; -import java.util.Map; -import java.util.TreeMap; /** * Created on 25/03/16. @@ -31,7 +27,6 @@ public class VoyageOperationsOperationImportAction extends VoyageOperationsImpor /** Logger. */ private static final Log log = LogFactory.getLog(VoyageOperationsOperationImportAction.class); - private final Map<String, String> opperationIdToTransectId = new TreeMap<>(); public VoyageOperationsOperationImportAction(VoyageOperationsImportDataContext importDataContext) { super(importDataContext, importDataContext.getConfiguration().getOperationFile()); @@ -84,10 +79,6 @@ public class VoyageOperationsOperationImportAction extends VoyageOperationsImpor addProcessedRow(result, row); - // add to operation ids cache (need to compute imported export later) - opperationIdToTransectId.put(operation.getTopiaId(), transect.getTopiaId()); - - } } @@ -97,21 +88,13 @@ public class VoyageOperationsOperationImportAction extends VoyageOperationsImpor @Override protected void computeImportedExport(VoyageOperationsImportDataContext importDataContext, ImportDataFileResult result) { - for (ImportFileId importFileId : getImportFileIds(result)) { - String operationId = importFileId.getEntityId(); + for (Operation operation : getImportedEntities(Operation.class, result)) { + if (log.isInfoEnabled()) { - log.info("Adding operation: " + operationId + " to imported export."); + log.info("Adding operation: " + operation.getTopiaId() + " to imported export."); } - Operation operation = persistenceService.getOperation(operationId); - Preconditions.checkNotNull(operation); - - String transectId = opperationIdToTransectId.get(operationId); - Preconditions.checkNotNull(transectId); - - Transect transect = persistenceService.getTransect(transectId); - Preconditions.checkNotNull(transect); - VoyageOperationsOperationImportRow importedRow = VoyageOperationsOperationImportRow.of(transect, operation); + VoyageOperationsOperationImportRow importedRow = VoyageOperationsOperationImportRow.of(operation); addImportedRow(result, importedRow); } diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageOperationsOperationMetadataImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageOperationsOperationMetadataImportAction.java index 46423fc..d65576c 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageOperationsOperationMetadataImportAction.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageOperationsOperationMetadataImportAction.java @@ -1,11 +1,8 @@ package fr.ifremer.echobase.services.service.importdata.actions; -import com.google.common.base.Preconditions; import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; -import fr.ifremer.echobase.entities.ImportFileId; import fr.ifremer.echobase.entities.data.Operation; import fr.ifremer.echobase.entities.data.OperationMetadataValue; -import fr.ifremer.echobase.entities.references.Vessel; import fr.ifremer.echobase.io.InputFile; import fr.ifremer.echobase.services.service.importdata.ImportDataFileResult; import fr.ifremer.echobase.services.service.importdata.contexts.VoyageOperationsImportDataContext; @@ -15,9 +12,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.csv.Import; -import java.util.Map; -import java.util.TreeMap; - /** * Created on 25/03/16. * @@ -28,8 +22,6 @@ public class VoyageOperationsOperationMetadataImportAction extends VoyageOperati /** Logger. */ private static final Log log = LogFactory.getLog(VoyageOperationsOperationMetadataImportAction.class); - private final Map<String, String> operationMetadataValueIdToOperationId = new TreeMap<>(); - private final Map<String, String> operationIdToVesselId = new TreeMap<>(); public VoyageOperationsOperationMetadataImportAction(VoyageOperationsImportDataContext importDataContext) { super(importDataContext, importDataContext.getConfiguration().getOperationMetadataFile()); @@ -69,11 +61,6 @@ public class VoyageOperationsOperationMetadataImportAction extends VoyageOperati addProcessedRow(result, row); addId(result, EchoBaseUserEntityEnum.OperationMetadataValue, operationMetadataValue, rowNumber); - operationMetadataValueIdToOperationId.put(operationMetadataValue.getTopiaId(), operation.getTopiaId()); - - Vessel vessel = row.getVessel(); - operationIdToVesselId.put(operation.getTopiaId(), vessel.getTopiaId()); - } } @@ -83,28 +70,12 @@ public class VoyageOperationsOperationMetadataImportAction extends VoyageOperati @Override protected void computeImportedExport(VoyageOperationsImportDataContext importDataContext, ImportDataFileResult result) { - for (ImportFileId importFileId : getImportFileIds(result)) { - String operationMetadataValueId = importFileId.getEntityId(); + for (OperationMetadataValue operationMetadataValue : getImportedEntities(OperationMetadataValue.class, result)) { + if (log.isInfoEnabled()) { - log.info("Adding operation metadata value : " + operationMetadataValueId + " to imported export."); + log.info("Adding operation metadata value : " + operationMetadataValue.getTopiaId() + " to imported export."); } - - String operationId = operationMetadataValueIdToOperationId.get(operationMetadataValueId); - Preconditions.checkNotNull(operationId); - - Operation operation = persistenceService.getOperation(operationId); - Preconditions.checkNotNull(operation); - - String vesselId = operationIdToVesselId.get(operationId); - Preconditions.checkNotNull(vesselId); - - Vessel vessel = persistenceService.getVessel(vesselId); - Preconditions.checkNotNull(vessel); - - OperationMetadataValue operationMetadataValue = operation.getOperationMetadataValueByTopiaId(operationMetadataValueId); - Preconditions.checkNotNull(operationMetadataValue); - - VoyageOperationsOperationMetadataValueImportRow importedRow = VoyageOperationsOperationMetadataValueImportRow.of(operation, vessel, operationMetadataValue); + VoyageOperationsOperationMetadataValueImportRow importedRow = VoyageOperationsOperationMetadataValueImportRow.of(operationMetadataValue); addImportedRow(result, importedRow); } diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsCellImportDataActionSupport.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsCellImportDataActionSupport.java index 7112c9d..df50de1 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsCellImportDataActionSupport.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsCellImportDataActionSupport.java @@ -1,6 +1,5 @@ package fr.ifremer.echobase.services.service.importdata.actions; -import com.google.common.base.Preconditions; import fr.ifremer.echobase.entities.ImportFileId; import fr.ifremer.echobase.entities.data.Category; import fr.ifremer.echobase.entities.data.Cell; @@ -19,8 +18,6 @@ import org.nuiton.csv.Import; import java.util.LinkedList; import java.util.List; -import java.util.Map; -import java.util.TreeMap; /** * Created on 25/03/16. @@ -32,8 +29,8 @@ public abstract class VoyageResultsCellImportDataActionSupport<E extends ResultA /** Logger. */ private static final Log log = LogFactory.getLog(VoyageResultsCellImportDataActionSupport.class); + protected final List<DataMetadata> metas; - private final Map<String, String> resultIdToCellId = new TreeMap<>(); protected VoyageResultsCellImportDataActionSupport(VoyageResultsImportDataContext importDataContext, InputFile inputFile, String... columnNamesToExclude) { super(importDataContext, inputFile); @@ -66,11 +63,7 @@ public abstract class VoyageResultsCellImportDataActionSupport<E extends ResultA Category category = getResultCategory(result, resultCategoryCache, row); Cell cell = row.getCell(); - String cellTopiaId = cell.getTopiaId(); - List<Result> cellResults = addResults(row, cell, category, resultLabel, result, true, true, rowNumber); - Result lastResult = cellResults.get(cellResults.size() - 1); - - resultIdToCellId.put(lastResult.getTopiaId(), cellTopiaId); + addResults(row, cell, category, resultLabel, result, true, true, rowNumber); addProcessedRow(result, row); @@ -90,53 +83,64 @@ public abstract class VoyageResultsCellImportDataActionSupport<E extends ResultA int count = 0; int max = result.sizeImportFileIds(); - for (ImportFileId importFileId : getImportFileIds(result)) { + int currentLineNumber = -1; + + for (ImportFileId importFileId : filterIds(Result.class, result)) { + String resultId = importFileId.getEntityId(); if (log.isDebugEnabled()) { log.debug("Treat imported result: " + resultId + " (" + (++count) + "/" + max + ")"); } - cellResultIds.add(resultId); + if (currentLineNumber == -1) { - String cellId = resultIdToCellId.get(resultId); + currentLineNumber = importFileId.getLineNumber(); - if (cellId != null) { + } else if (currentLineNumber != importFileId.getLineNumber()) { + // flush cell + flushCell(voyage, result, cellResultIds); - // on the last result of a row, flush current cell + cellResultIds.clear(); - Preconditions.checkState(!cellResultIds.isEmpty(), "Can't flush a cell (" + cellId + " ) with no result..."); + currentLineNumber = importFileId.getLineNumber(); - Cell cell = persistenceService.getCell(cellId); - Preconditions.checkNotNull(cell); + } - if (log.isInfoEnabled()) { - log.info("Flush Cell " + cell + " with " + cellResultIds.size() + " results"); - } + cellResultIds.add(resultId); - List<Result> cellResults = new LinkedList<>(); + } - Category category = null; - for (Result cellResult : cell.getResult()) { + flushCell(voyage, result, cellResultIds); - if (cellResultIds.contains(cellResult.getTopiaId())) { - if (category == null) { - category = cellResult.getCategory(); - } - cellResults.add(cellResult); - } + } - } + protected void flushCell(Voyage voyage, ImportDataFileResult result, List<String> cellResultIds) { - E row = newImportedRow(voyage, cell, category, cellResults); - addImportedRow(result, row); + String firstResultId = cellResultIds.get(0); + Result firstCellResult = persistenceService.getResult(firstResultId); + Cell cell = firstCellResult.getCell(); - cellResultIds.clear(); + if (log.isInfoEnabled()) { + log.info("Flush " + cell + " with " + cellResultIds.size() + " result(s)."); + } + List<Result> cellResults = new LinkedList<>(); + + Category category = null; + for (Result cellResult : cell.getResult()) { + if (cellResultIds.contains(cellResult.getTopiaId())) { + if (category == null) { + category = cellResult.getCategory(); + } + cellResults.add(cellResult); } } + E row = newImportedRow(voyage, cell, category, cellResults); + addImportedRow(result, row); + } } diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsRegionAssociationImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsRegionAssociationImportAction.java index b9f75de..2faaf91 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsRegionAssociationImportAction.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsRegionAssociationImportAction.java @@ -13,8 +13,8 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.csv.Import; -import java.util.LinkedHashMap; -import java.util.Map; +import java.util.LinkedHashSet; +import java.util.Set; /** * Created on 25/03/16. @@ -26,7 +26,7 @@ public class VoyageResultsRegionAssociationImportAction extends VoyageResultsImp /** Logger. */ private static final Log log = LogFactory.getLog(VoyageResultsRegionAssociationImportAction.class); - private final Map<String, String> esduCellDataIdToRegionCellId = new LinkedHashMap<>(); + private final Set<String> regionCellIds = new LinkedHashSet<>(); public VoyageResultsRegionAssociationImportAction(VoyageResultsImportDataContext importDataContext) { super(importDataContext, importDataContext.getConfiguration().getRegionAssociationFile()); @@ -61,12 +61,10 @@ public class VoyageResultsRegionAssociationImportAction extends VoyageResultsImp Cell esduCell = row.getEsduCell(); regionCell.addChilds(esduCell); - if (!esduCellDataIdToRegionCellId.containsValue(regionCell.getTopiaId())) { - result.incrementsNumberUpdated(EchoBaseUserEntityEnum.Cell); - } + result.incrementsNumberUpdated(EchoBaseUserEntityEnum.Cell); addProcessedRow(result, row); - esduCellDataIdToRegionCellId.put(esduCell.getTopiaId(), regionCell.getTopiaId()); + regionCellIds.add(regionCell.getTopiaId()); } @@ -80,23 +78,20 @@ public class VoyageResultsRegionAssociationImportAction extends VoyageResultsImp String voyageId = importDataContext.getConfiguration().getVoyageId(); Voyage voyage = persistenceService.getVoyage(voyageId); - for (Map.Entry<String, String> entry : esduCellDataIdToRegionCellId.entrySet()) { - - String esduCellId = entry.getKey(); - String regionCellId = entry.getValue(); + for (String regionCellId : regionCellIds) { if (log.isInfoEnabled()) { - log.info("Treat imported esduCell association: " + esduCellId + " to regionCell: " + regionCellId); + log.info("Treat imported region regionCell: " + regionCellId); } - - Cell esduCell = persistenceService.getCell(esduCellId); - Preconditions.checkNotNull(esduCell); - Cell regionCell = persistenceService.getCell(regionCellId); Preconditions.checkNotNull(regionCell); - VoyageResultsRegionCellAssociationImportRow row = VoyageResultsRegionCellAssociationImportRow.of(voyage, regionCell, esduCell); - addImportedRow(result, row); + for (Cell esduCell : regionCell.getChilds()) { + + VoyageResultsRegionCellAssociationImportRow row = VoyageResultsRegionCellAssociationImportRow.of(voyage, regionCell, esduCell); + addImportedRow(result, row); + + } } diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsRegionImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsRegionImportAction.java index cedfc12..57ea597 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsRegionImportAction.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsRegionImportAction.java @@ -5,7 +5,6 @@ import com.google.common.base.Preconditions; import com.google.common.base.Predicate; import com.google.common.collect.Iterables; import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; -import fr.ifremer.echobase.entities.ImportFileId; import fr.ifremer.echobase.entities.data.Cell; import fr.ifremer.echobase.entities.data.Data; import fr.ifremer.echobase.entities.data.Datas; @@ -21,9 +20,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.csv.Import; -import java.util.Map; -import java.util.TreeMap; - /** * Created on 25/03/16. * @@ -34,7 +30,6 @@ public class VoyageResultsRegionImportAction extends VoyageResultsImportDataActi /** Logger. */ private static final Log log = LogFactory.getLog(VoyageResultsRegionImportAction.class); - private final Map<String, String> cellDataCoordinateIdToCellId = new TreeMap<>(); public VoyageResultsRegionImportAction(VoyageResultsImportDataContext importDataContext) { super(importDataContext, importDataContext.getConfiguration().getRegionsFile()); @@ -96,15 +91,12 @@ public class VoyageResultsRegionImportAction extends VoyageResultsImportDataActi } // add coordinate data - Data cellData = createCellData(cell, - dataCoordinateMeta, - row.getDataCoordinate(), - dataQuality, - result, - true, rowNumber); - - cellDataCoordinateIdToCellId.put(cellData.getTopiaId(), cell.getTopiaId()); - + createCellData(cell, + dataCoordinateMeta, + row.getDataCoordinate(), + dataQuality, + result, + true, rowNumber); addProcessedRow(result, row); @@ -125,38 +117,27 @@ public class VoyageResultsRegionImportAction extends VoyageResultsImportDataActi String voyageId = importDataContext.getConfiguration().getVoyageId(); Voyage voyage = persistenceService.getVoyage(voyageId); - for (ImportFileId importFileId : filterIds(Data.class, result)) { + for (Data coordinateCellData : getImportedEntities(Data.class, result)) { - String cellDataId = importFileId.getEntityId(); - String cellId = cellDataCoordinateIdToCellId.get(cellDataId); - if (cellId == null) { + if (!dataCoordinateMeta.equals(coordinateCellData.getDataMetadata())) { // Treat only coordindate data, since there is exactly one per row // The surface data is the same for all rows of a same region cell continue; } + Cell cell = coordinateCellData.getCell(); if (log.isInfoEnabled()) { - log.info("Treat imported coordinate data: " + cellDataId + " for cell: " + cellId); + log.info("Treat imported coordinate data: " + coordinateCellData.getTopiaId() + " for cell: " + cell.getTopiaId()); } - Preconditions.checkNotNull(cellId); - - Cell cell = persistenceService.getCell(cellId); - Preconditions.checkNotNull(cell); - Preconditions.checkState(cell.isDataNotEmpty()); - - Data coordinateData = cell.getDataByTopiaId(cellDataId); - Preconditions.checkNotNull(coordinateData); - Preconditions.checkState(dataCoordinateMeta.equals(coordinateData.getDataMetadata())); Optional<Data> optionalSurfaceData = Iterables.tryFind(cell.getData(), dataPredicate); Preconditions.checkState(optionalSurfaceData.isPresent()); String surfaceDataValue = optionalSurfaceData.get().getDataValue(); - Preconditions.checkNotNull(coordinateData); - VoyageResultsRegionCellImportRow row = VoyageResultsRegionCellImportRow.of(voyage, cell, coordinateData, Float.valueOf(surfaceDataValue)); + VoyageResultsRegionCellImportRow row = VoyageResultsRegionCellImportRow.of(voyage, coordinateCellData, Float.valueOf(surfaceDataValue)); addImportedRow(result, row); diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsVoyageLengthAgeKeyImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsVoyageLengthAgeKeyImportAction.java index f900e76..5afb54d 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsVoyageLengthAgeKeyImportAction.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsVoyageLengthAgeKeyImportAction.java @@ -1,8 +1,6 @@ package fr.ifremer.echobase.services.service.importdata.actions; -import com.google.common.base.Preconditions; import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; -import fr.ifremer.echobase.entities.ImportFileId; import fr.ifremer.echobase.entities.data.LengthAgeKey; import fr.ifremer.echobase.entities.data.Voyage; import fr.ifremer.echobase.io.InputFile; @@ -80,11 +78,7 @@ public class VoyageResultsVoyageLengthAgeKeyImportAction extends VoyageResultsIm String voyageId = importDataContext.getConfiguration().getVoyageId(); Voyage voyage = persistenceService.getVoyage(voyageId); - for (ImportFileId importFileId : getImportFileIds(result)) { - - String lengthAgeKeyId = importFileId.getEntityId(); - LengthAgeKey lengthAgeKey = persistenceService.getLengthAgeKey(lengthAgeKeyId); - Preconditions.checkNotNull(lengthAgeKey); + for (LengthAgeKey lengthAgeKey : getImportedEntities(LengthAgeKey.class, result)) { VoyageResultsVoyageLengthAgeKeyImportRow row = VoyageResultsVoyageLengthAgeKeyImportRow.of(voyage, lengthAgeKey); addImportedRow(result, row); diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsVoyageLengthWeightKeyImportAction.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsVoyageLengthWeightKeyImportAction.java index 0442050..560fe2d 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsVoyageLengthWeightKeyImportAction.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/actions/VoyageResultsVoyageLengthWeightKeyImportAction.java @@ -1,8 +1,6 @@ package fr.ifremer.echobase.services.service.importdata.actions; -import com.google.common.base.Preconditions; import fr.ifremer.echobase.entities.EchoBaseUserEntityEnum; -import fr.ifremer.echobase.entities.ImportFileId; import fr.ifremer.echobase.entities.data.LengthWeightKey; import fr.ifremer.echobase.entities.data.Voyage; import fr.ifremer.echobase.entities.references.SpeciesCategory; @@ -89,11 +87,7 @@ public class VoyageResultsVoyageLengthWeightKeyImportAction extends VoyageResult String voyageId = importDataContext.getConfiguration().getVoyageId(); Voyage voyage = persistenceService.getVoyage(voyageId); - for (ImportFileId importFileId : getImportFileIds(result)) { - - String lengthWeightKeyId = importFileId.getEntityId(); - LengthWeightKey lengthWeightKey = persistenceService.getLengthWeightKey(lengthWeightKeyId); - Preconditions.checkNotNull(lengthWeightKey); + for (LengthWeightKey lengthWeightKey : getImportedEntities(LengthWeightKey.class, result)) { VoyageResultsVoyageLengthWeightKeyImportRow row = VoyageResultsVoyageLengthWeightKeyImportRow.of(voyage, lengthWeightKey); addImportedRow(result, row); diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageCatchesBiometrySampleImportRow.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageCatchesBiometrySampleImportRow.java index b68496a..bb0969c 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageCatchesBiometrySampleImportRow.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageCatchesBiometrySampleImportRow.java @@ -49,10 +49,11 @@ public class VoyageCatchesBiometrySampleImportRow { protected int numFish; - public static VoyageCatchesBiometrySampleImportRow of(Operation operation, Sample sample, SampleData sampleData, int numFish) { + public static VoyageCatchesBiometrySampleImportRow of(SampleData sampleData, int numFish) { + Sample sample = sampleData.getSample(); VoyageCatchesBiometrySampleImportRow row = new VoyageCatchesBiometrySampleImportRow(sampleData); - row.setOperation(operation); + row.setOperation(sample.getOperation()); row.setSpecies(sample.getSpeciesCategory().getSpecies()); row.setNumFish(numFish); diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageCatchesTotalSampleImportRow.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageCatchesTotalSampleImportRow.java index 2f11800..d6363e7 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageCatchesTotalSampleImportRow.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageCatchesTotalSampleImportRow.java @@ -140,9 +140,9 @@ public class VoyageCatchesTotalSampleImportRow { this.sortedWeight = sortedWeight; } - public static VoyageCatchesTotalSampleImportRow of(Operation operation, Sample sample, Sample sortedSample) { + public static VoyageCatchesTotalSampleImportRow of(Sample sample, Sample sortedSample) { VoyageCatchesTotalSampleImportRow row = new VoyageCatchesTotalSampleImportRow(sample); - row.setOperation(operation); + row.setOperation(sample.getOperation()); row.setSpecies(sample.getSpeciesCategory().getSpecies()); row.setSizeCategory(sample.getSpeciesCategory().getSizeCategory()); row.setSortedWeight(sortedSample.getSampleWeight()); diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageOperationsGearMetadataValueImportRow.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageOperationsGearMetadataValueImportRow.java index d8066d0..7dc0fc4 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageOperationsGearMetadataValueImportRow.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageOperationsGearMetadataValueImportRow.java @@ -45,10 +45,11 @@ public class VoyageOperationsGearMetadataValueImportRow { protected Vessel vessel; protected Operation operation; - public static VoyageOperationsGearMetadataValueImportRow of(Operation operation, Vessel vessel, GearMetadataValue gearMetadataValue) { + public static VoyageOperationsGearMetadataValueImportRow of(GearMetadataValue gearMetadataValue) { VoyageOperationsGearMetadataValueImportRow row = new VoyageOperationsGearMetadataValueImportRow(gearMetadataValue); + Operation operation = gearMetadataValue.getOperation(); row.setOperation(operation); - row.setVessel(vessel); + row.setVessel(operation.getTransect().getVessel()); return row; } diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageOperationsOperationImportRow.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageOperationsOperationImportRow.java index 0b7f5b8..78b9b29 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageOperationsOperationImportRow.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageOperationsOperationImportRow.java @@ -42,8 +42,9 @@ public class VoyageOperationsOperationImportRow { protected final Operation operation; protected Vessel vessel; - public static VoyageOperationsOperationImportRow of(Transect transect, Operation operation) { + public static VoyageOperationsOperationImportRow of(Operation operation) { VoyageOperationsOperationImportRow row = new VoyageOperationsOperationImportRow(operation); + Transect transect = operation.getTransect(); row.setVessel(transect.getVessel()); return row; } diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageOperationsOperationMetadataValueImportRow.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageOperationsOperationMetadataValueImportRow.java index d49a5d9..3ce0b7f 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageOperationsOperationMetadataValueImportRow.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageOperationsOperationMetadataValueImportRow.java @@ -44,10 +44,11 @@ public class VoyageOperationsOperationMetadataValueImportRow { protected Operation operation; protected Vessel vessel; - public static VoyageOperationsOperationMetadataValueImportRow of(Operation operation, Vessel vessel, OperationMetadataValue operationMetadataValue) { + public static VoyageOperationsOperationMetadataValueImportRow of(OperationMetadataValue operationMetadataValue) { VoyageOperationsOperationMetadataValueImportRow row = new VoyageOperationsOperationMetadataValueImportRow(operationMetadataValue); + Operation operation = operationMetadataValue.getOperation(); row.setOperation(operation); - row.setVessel(vessel); + row.setVessel(operation.getTransect().getVessel()); return row; } diff --git a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageResultsRegionCellImportRow.java b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageResultsRegionCellImportRow.java index 8b16040..05fc438 100644 --- a/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageResultsRegionCellImportRow.java +++ b/echobase-services/src/main/java/fr/ifremer/echobase/services/service/importdata/csv/VoyageResultsRegionCellImportRow.java @@ -109,9 +109,10 @@ public class VoyageResultsRegionCellImportRow { this.dataQuality = dataQuality; } - public static VoyageResultsRegionCellImportRow of(Voyage voyage, Cell cell, Data coordinateData, float surfaceDataValue) { + public static VoyageResultsRegionCellImportRow of(Voyage voyage, Data coordinateData, float surfaceDataValue) { VoyageResultsRegionCellImportRow row = new VoyageResultsRegionCellImportRow(); row.setVoyage(voyage); + Cell cell = coordinateData.getCell(); row.setName(cell.getName()); row.setCellType(cell.getCellType()); row.setDataSurface(surfaceDataValue); diff --git a/echobase-services/src/test/java/fr/ifremer/echobase/services/EchoBaseServiceFixtures.java b/echobase-services/src/test/java/fr/ifremer/echobase/services/EchoBaseServiceFixtures.java index edb522b..13a8a83 100644 --- a/echobase-services/src/test/java/fr/ifremer/echobase/services/EchoBaseServiceFixtures.java +++ b/echobase-services/src/test/java/fr/ifremer/echobase/services/EchoBaseServiceFixtures.java @@ -111,10 +111,18 @@ public class EchoBaseServiceFixtures { return NB_SAMPLE_TOTAL() + NB_SAMPLE_UNSORTED() + NB_SAMPLE_BIOMETRY(); } + public int NB_SAMPLE_ALL() { + return NB_SAMPLE_TOTAL() + NB_SAMPLE_UNSORTED_ALL() + NB_SAMPLE_BIOMETRY(); + } + public int NB_SAMPLE_DATA() { return NB_SAMPLE_DATA_TOTAL() + NB_SAMPLE_DATA_UNSORTED() + NB_SAMPLE_DATA_BIOMETRY(); } + public int NB_SAMPLE_DATA_ALL() { + return NB_SAMPLE_DATA_TOTAL() + NB_SAMPLE_DATA_UNSORTED_ALL() + NB_SAMPLE_DATA_BIOMETRY(); + } + public int NB_DATA_ACQUISITION() { return 1; } diff --git a/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/VoyageCatchesAllImportServiceIT.java b/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/VoyageCatchesAllImportServiceIT.java index 9be5649..06fe741 100644 --- a/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/VoyageCatchesAllImportServiceIT.java +++ b/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/VoyageCatchesAllImportServiceIT.java @@ -48,7 +48,7 @@ public class VoyageCatchesAllImportServiceIT extends VoyageCatchesImportServiceI protected VoyageCatchesImportConfiguration createConfiguration() throws IOException { VoyageCatchesImportConfiguration configuration = super.createConfiguration(); prepareInputFile(configuration.getTotalSampleFile(), getImportPath("totalsample.csv.gz")); - prepareInputFile(configuration.getSubSampleFile(), getImportPath("subsample.csv.gz")); + prepareInputFile(configuration.getSubSampleFile(), getImportPath("subsample_all.csv.gz")); prepareInputFile(configuration.getBiometrySampleFile(), getImportPath("biometrysample.csv.gz")); return configuration; } @@ -63,37 +63,37 @@ public class VoyageCatchesAllImportServiceIT extends VoyageCatchesImportServiceI @Override protected void assertAfertImport(ImportDataResult<VoyageCatchesImportConfiguration> result) throws IOException { - int nbSample = fixtures.NB_SAMPLE(); - int nbSampleData = fixtures.NB_SAMPLE_DATA(); + int nbSample = fixtures.NB_SAMPLE_ALL(); + int nbSampleData = fixtures.NB_SAMPLE_DATA_ALL(); int nbSampleTotal = fixtures.NB_SAMPLE_TOTAL(); int nbSampleDataTotal = fixtures.NB_SAMPLE_DATA_TOTAL(); - int nbSampleUnsorted = fixtures.NB_SAMPLE_UNSORTED(); - int nbSampleDataUnsorted = fixtures.NB_SAMPLE_DATA_UNSORTED(); + int nbSampleUnsorted = fixtures.NB_SAMPLE_UNSORTED_ALL(); + int nbSampleDataUnsorted = fixtures.NB_SAMPLE_DATA_UNSORTED_ALL(); int nbSampleBiometry = fixtures.NB_SAMPLE_BIOMETRY(); int nbSampleDataBiometry = fixtures.NB_SAMPLE_DATA_BIOMETRY(); { ImportDataFileResult importDataFileResult = Iterables.get(result.getImportResults(), 0); + assertCsvImportResult0(importDataFileResult, nbSampleTotal); assertCsvImportResultPerEntity(importDataFileResult, Sample.class, nbSampleTotal, 0, nbSample); assertCsvImportResultPerEntity(importDataFileResult, SampleData.class, nbSampleDataTotal, 0, nbSampleData); - assertCsvImportResult0(importDataFileResult, nbSampleTotal); } { ImportDataFileResult importDataFileResult = Iterables.get(result.getImportResults(), 1); + assertCsvImportResult0(importDataFileResult, nbSampleUnsorted + nbSampleDataUnsorted); assertCsvImportResultPerEntity(importDataFileResult, Sample.class, nbSampleUnsorted, 0, nbSample); assertCsvImportResultPerEntity(importDataFileResult, SampleData.class, nbSampleDataUnsorted, 0, nbSampleData); - assertCsvImportResult0(importDataFileResult, nbSampleUnsorted + nbSampleDataUnsorted); } { ImportDataFileResult importDataFileResult = Iterables.get(result.getImportResults(), 2); + assertCsvImportResult0(importDataFileResult, nbSampleBiometry + nbSampleDataBiometry); assertCsvImportResultPerEntity(importDataFileResult, Sample.class, nbSampleBiometry, 0, nbSample); assertCsvImportResultPerEntity(importDataFileResult, SampleData.class, nbSampleDataBiometry, 0, nbSampleData); - assertCsvImportResult0(importDataFileResult, nbSampleBiometry + nbSampleDataBiometry); } diff --git a/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/VoyageResultsRegionCellImportServiceIT.java b/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/VoyageResultsRegionCellImportServiceIT.java index bc5bf4a..109f463 100644 --- a/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/VoyageResultsRegionCellImportServiceIT.java +++ b/echobase-services/src/test/java/fr/ifremer/echobase/services/service/importdata/VoyageResultsRegionCellImportServiceIT.java @@ -66,6 +66,7 @@ public class VoyageResultsRegionCellImportServiceIT extends VoyageResultsImportS return configuration; } + // public static final int NB_ESDU_CELLS = 2073; public static final int NB_ESDU_CELLS = 22; @Override @@ -73,6 +74,10 @@ public class VoyageResultsRegionCellImportServiceIT extends VoyageResultsImportS int nbRegionCell = 10; int nbRegionCellData = 238; + +// //FIXME Comprendre pourquoi toutes les cellules esdu sont dans 2 régions différentes ? +// int nbUpdatedEsduCell = 2 * NB_ESDU_CELLS; + int nbUpdatedEsduCell = NB_ESDU_CELLS; int nbRegionCellResult = 2128/*1873*/; int nbCell = NB_ESDU_CELLS + nbRegionCell; @@ -86,8 +91,8 @@ public class VoyageResultsRegionCellImportServiceIT extends VoyageResultsImportS { ImportDataFileResult importDataFileResult = Iterables.get(result.getImportResults(), 1); - assertCsvImportResultPerEntity(importDataFileResult, Cell.class, 0, nbRegionCell, nbCell); assertCsvImportResult0(importDataFileResult, 0); + assertCsvImportResultPerEntity(importDataFileResult, Cell.class, 0, nbUpdatedEsduCell, nbCell); } { -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.