This is an automated email from the git hooks/post-receive script. New change to branch develop in repository observe. See http://git.codelutin.com/observe.git from 69ff107 Utiliser le temps de calcul en nanosecondes et pas milisecondes new 39fb53d La création d'une base est effective avec les imports au démarrage (See #7482) new 969e71b Création d'une base via les services (termine #7482). Merge branch 'feature/7482' into develop The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "adds" were already present in the repository and have only been added to this reference. Detailed log of new commits: commit 969e71b966e31721c30aacbd03971c8eb06102d4 Merge: 69ff107 39fb53d Author: Tony CHEMIT <chemit@codelutin.com> Date: Thu Aug 27 10:25:36 2015 +0200 Création d'une base via les services (termine #7482). Merge branch 'feature/7482' into develop commit 39fb53da85bd0305d59054c742d33e7d76bfa5e8 Author: Tony CHEMIT <chemit@codelutin.com> Date: Thu Aug 27 10:25:33 2015 +0200 La création d'une base est effective avec les imports au démarrage (See #7482) Summary of changes: .../service/DataSourceDumpProducerService.java | 7 +- .../dto/DataSourceCreateConfigurationDto.java | 7 +- .../ird/observe/services/ObserveServiceTopia.java | 167 ++++++++++++++++++++- .../DataSourceDumpProducerServiceTopia.java | 160 +++----------------- .../services/service/DataSourceServiceTopia.java | 67 +++++++-- .../services/ApplicationContextResource.java | 50 ++++++ .../ird/observe/services/DataSourceResource.java | 24 +-- .../DataSourceDumpProducerServiceTopiaTest.java | 13 +- .../service/DataSourceServiceTopiaTest.java | 58 +++---- 9 files changed, 342 insertions(+), 211 deletions(-) -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@list.forge.codelutin.com>.
This is an automated email from the git hooks/post-receive script. New commit to branch develop in repository observe. See http://git.codelutin.com/observe.git commit 39fb53da85bd0305d59054c742d33e7d76bfa5e8 Author: Tony CHEMIT <chemit@codelutin.com> Date: Thu Aug 27 10:25:33 2015 +0200 La création d'une base est effective avec les imports au démarrage (See #7482) --- .../service/DataSourceDumpProducerService.java | 7 +- .../dto/DataSourceCreateConfigurationDto.java | 7 +- .../ird/observe/services/ObserveServiceTopia.java | 167 ++++++++++++++++++++- .../DataSourceDumpProducerServiceTopia.java | 160 +++----------------- .../services/service/DataSourceServiceTopia.java | 67 +++++++-- .../services/ApplicationContextResource.java | 50 ++++++ .../ird/observe/services/DataSourceResource.java | 24 +-- .../DataSourceDumpProducerServiceTopiaTest.java | 13 +- .../service/DataSourceServiceTopiaTest.java | 58 +++---- 9 files changed, 342 insertions(+), 211 deletions(-) diff --git a/observe-services-api/src/main/java/fr/ird/observe/services/service/DataSourceDumpProducerService.java b/observe-services-api/src/main/java/fr/ird/observe/services/service/DataSourceDumpProducerService.java index 816dd3d..911f6ef 100644 --- a/observe-services-api/src/main/java/fr/ird/observe/services/service/DataSourceDumpProducerService.java +++ b/observe-services-api/src/main/java/fr/ird/observe/services/service/DataSourceDumpProducerService.java @@ -1,5 +1,6 @@ package fr.ird.observe.services.service; +import com.google.common.collect.ImmutableSet; import fr.ird.observe.services.ObserveService; /** @@ -7,10 +8,12 @@ import fr.ird.observe.services.ObserveService; * * @author Tony Chemit - chemit@codelutin.com */ -public interface DataSourceDumpProducerService extends ObserveService{ +public interface DataSourceDumpProducerService extends ObserveService { byte[] getReferentialDump(); - byte[] getDataDump(String... importDataIds); + byte[] getAllDataDump(); + + byte[] getDataDump(ImmutableSet<String> importDataIds); } diff --git a/observe-services-model/src/main/java/fr/ird/observe/services/dto/DataSourceCreateConfigurationDto.java b/observe-services-model/src/main/java/fr/ird/observe/services/dto/DataSourceCreateConfigurationDto.java index ec129f9..381a974 100644 --- a/observe-services-model/src/main/java/fr/ird/observe/services/dto/DataSourceCreateConfigurationDto.java +++ b/observe-services-model/src/main/java/fr/ird/observe/services/dto/DataSourceCreateConfigurationDto.java @@ -1,6 +1,7 @@ package fr.ird.observe.services.dto; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; import fr.ird.observe.services.configuration.ObserveDataSourceConfiguration; /** @@ -30,7 +31,7 @@ public class DataSourceCreateConfigurationDto extends AbstractObserveDto { /** * Les données à importer (ids des marées) (Optionel). */ - protected String[] optionalImportDataIds; + protected ImmutableSet<String> optionalImportDataIds; /** * Pour autoriser la création d'une base vide (utilisé pour les bases temporaires) @@ -69,7 +70,7 @@ public class DataSourceCreateConfigurationDto extends AbstractObserveDto { return optionalImportDataDataSourceConfiguration; } - public String[] getImportDataIds() { + public ImmutableSet<String> getImportDataIds() { return optionalImportDataIds; } @@ -83,7 +84,7 @@ public class DataSourceCreateConfigurationDto extends AbstractObserveDto { this.optionalImportReferentialDataSourceConfiguration = importReferentialDataSourceConfiguration; } - public void setImportDataConfiguration(ObserveDataSourceConfiguration importDataDataSourceConfiguration, String... importDataIds) { + public void setImportDataConfiguration(ObserveDataSourceConfiguration importDataDataSourceConfiguration, ImmutableSet<String> importDataIds) { Preconditions.checkNotNull(importDataDataSourceConfiguration, "'importDataDataSourceConfiguration' can't be null."); Preconditions.checkNotNull(importDataIds, "'importDataIds' can't be null."); this.optionalImportDataDataSourceConfiguration = importDataDataSourceConfiguration; diff --git a/observe-services-topia/src/main/java/fr/ird/observe/services/ObserveServiceTopia.java b/observe-services-topia/src/main/java/fr/ird/observe/services/ObserveServiceTopia.java index 263af71..8c2f811 100644 --- a/observe-services-topia/src/main/java/fr/ird/observe/services/ObserveServiceTopia.java +++ b/observe-services-topia/src/main/java/fr/ird/observe/services/ObserveServiceTopia.java @@ -4,17 +4,25 @@ import com.google.common.base.Optional; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Multimap; import com.google.common.collect.Sets; +import fr.ird.observe.ObserveTopiaApplicationContext; import fr.ird.observe.ObserveTopiaPersistenceContext; +import fr.ird.observe.entities.Entities; import fr.ird.observe.entities.constants.ReferenceLocale; +import fr.ird.observe.entities.longline.TripLongline; import fr.ird.observe.entities.referentiel.ReferenceEntity; import fr.ird.observe.services.builder.DtoToEntityBuilder; import fr.ird.observe.services.builder.EntityToDtoBuilder; import fr.ird.observe.services.builder.EntityToReferenceDtoBuilder; import fr.ird.observe.services.builder.EntityToReferentialReferenceDtoBuilder; +import fr.ird.observe.services.configuration.ObserveDataSourceConfigurationTopiaH2; +import fr.ird.observe.services.dto.DataSourceCreateConfigurationDto; +import fr.ird.observe.services.dto.DataSourceCreateWithNoReferentialImportException; import fr.ird.observe.services.dto.FormDto; import fr.ird.observe.services.dto.FormDtos; import fr.ird.observe.services.dto.IdDto; +import fr.ird.observe.services.dto.IncompatibleDataSourceCreateConfigurationException; import fr.ird.observe.services.dto.ObserveDtosInitializer; import fr.ird.observe.services.dto.ObserveModelInitializerRunner; import fr.ird.observe.services.dto.ReferenceDto; @@ -23,10 +31,17 @@ import fr.ird.observe.services.dto.ReferenceSetDtos; import fr.ird.observe.services.dto.constants.ReferentialLocale; import fr.ird.observe.services.dto.referential.ReferentialDto; import fr.ird.observe.services.service.DataNotFoundException; +import fr.ird.observe.services.service.DataSourceService; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.nuiton.topia.persistence.TopiaDao; import org.nuiton.topia.persistence.TopiaEntity; +import org.nuiton.topia.replication.TopiaReplicationService; +import org.nuiton.topia.replication.model.ReplicationModel; +import org.nuiton.util.StringUtil; +import java.io.File; import java.util.Date; import java.util.LinkedHashSet; import java.util.List; @@ -41,6 +56,9 @@ import java.util.Set; */ public class ObserveServiceTopia implements ObserveService { + /** Logger. */ + private static final Log log = LogFactory.getLog(ObserveServiceTopia.class); + protected ObserveServiceContextTopia serviceContext; protected static ImmutableMap<Class<?>, Class<?>> DTO_TO_ENTITY_TYPES; @@ -220,13 +238,13 @@ public class ObserveServiceTopia implements ObserveService { } - protected <E extends TopiaEntity, D extends IdDto> ReferenceDto<D> entityToReferenceDto(Class<D> dtoType, E entity) { + protected <E extends TopiaEntity, D extends IdDto> ReferenceDto<D> entityToReferenceDto(Class<D> dtoType, E entity) { ReferenceDto<D> referenceDto; if (entity instanceof ReferenceEntity) { - referenceDto = EntityToReferentialReferenceDtoBuilder.build((Class)dtoType, serviceContext.getReferentialLocale(), entity); + referenceDto = EntityToReferentialReferenceDtoBuilder.build((Class) dtoType, serviceContext.getReferentialLocale(), entity); } else { @@ -260,4 +278,149 @@ public class ObserveServiceTopia implements ObserveService { public ObserveTopiaPersistenceContext getTopiaPersistenceContext() { return serviceContext.getTopiaPersistenceContext(); } + + protected ObserveDataSourceConfigurationTopiaH2 createTemporaryDatabaseFromDump(String prefix, byte... importContent) { + ObserveDataSourceConfigurationTopiaH2 temporaryDataSourceConfiguration = new ObserveDataSourceConfigurationTopiaH2(); + temporaryDataSourceConfiguration.setDbName("obstuna"); + temporaryDataSourceConfiguration.setLabel(prefix); + temporaryDataSourceConfiguration.setUsername("sa"); + temporaryDataSourceConfiguration.setPassword('s', 'a'); + File temporaryDataSourceDirectory = serviceContext.createTemporaryDirectory(prefix); + temporaryDataSourceConfiguration.setDirectory(temporaryDataSourceDirectory); + + DataSourceService dataSourceService = serviceContext.newService(temporaryDataSourceConfiguration, DataSourceService.class); + + try { + DataSourceCreateConfigurationDto dataSourceCreateConfiguration = new DataSourceCreateConfigurationDto(); + dataSourceCreateConfiguration.setImportDatabase(importContent); + dataSourceService.create(dataSourceCreateConfiguration); + } catch (IncompatibleDataSourceCreateConfigurationException | DataSourceCreateWithNoReferentialImportException e) { + //FIXME Avoir une exception plus concrete + throw new RuntimeException("Could not create temporary database: " + prefix, e); + } + + return temporaryDataSourceConfiguration; + } + protected ObserveDataSourceConfigurationTopiaH2 createTemporaryDatabase(String prefix) { + + ObserveDataSourceConfigurationTopiaH2 temporaryDataSourceConfiguration = new ObserveDataSourceConfigurationTopiaH2(); + temporaryDataSourceConfiguration.setDbName("obstuna"); + temporaryDataSourceConfiguration.setLabel(prefix); + temporaryDataSourceConfiguration.setUsername("sa"); + temporaryDataSourceConfiguration.setPassword('s', 'a'); + File temporaryDataSourceDirectory = serviceContext.createTemporaryDirectory(prefix); + temporaryDataSourceConfiguration.setDirectory(temporaryDataSourceDirectory); + + DataSourceService dataSourceService = serviceContext.newService(temporaryDataSourceConfiguration, DataSourceService.class); + + try { + DataSourceCreateConfigurationDto dataSourceCreateConfiguration = new DataSourceCreateConfigurationDto(); + dataSourceCreateConfiguration.setCanCreateEmptyDatabase(true); + dataSourceService.create(dataSourceCreateConfiguration); + } catch (IncompatibleDataSourceCreateConfigurationException | DataSourceCreateWithNoReferentialImportException e) { + //FIXME Avoir une exception plus concrete + throw new RuntimeException("Could not create temporary database: " + prefix, e); + } + + return temporaryDataSourceConfiguration; + + } + + protected void replicateReferential(ObserveTopiaApplicationContext sourceTopiaApplicationContext, ObserveTopiaApplicationContext temporaryTopiaApplicationContext) { + + long t0 = System.nanoTime(); + + TopiaReplicationService service = sourceTopiaApplicationContext.getReplicationService(); + ReplicationModel model = service.prepareForAll(Entities.REFERENCE_ENTITIES); + + try { + service.doReplicate(model, temporaryTopiaApplicationContext); + } catch (Exception e) { + //TODO Avoir une exception concrete + throw new RuntimeException("Could not replicate referantial", e); + } + + + if (log.isInfoEnabled()) { + log.info(String.format("Referential replication done in %s", StringUtil.convertTime(t0, System.nanoTime()))); + } + + } + + protected void replicateData(ObserveTopiaApplicationContext sourceTopiaApplicationContext, + ObserveTopiaApplicationContext temporaryTopiaApplicationContext, + Set<String> ids) { + try { + + + for (String id : ids) { + long t1 = System.nanoTime(); + + replicateOneData(sourceTopiaApplicationContext, temporaryTopiaApplicationContext, id); + + if (log.isInfoEnabled()) { + log.info(String.format("Data replication [%s] done in %s", id, StringUtil.convertTime(t1, System.nanoTime()))); + } + } + + } catch (Exception e) { + throw new RuntimeException("Could not replicate data", e); + } + } + + /** + * Duplication de l'unique donnée observateur depuis ce service vers le service donné. + * + * La duplication utilise une transaction dédiée afin de ne pas saturer le serveur + * et aussi une meilleure maitrise du rollback en cas d'une erreur. + * + * Voir http://forge.codelutin.com/issues/4837 + * + * @param id l'id de la donnée à répliquer + */ + protected void replicateOneData(ObserveTopiaApplicationContext sourceTopiaApplicationContext, ObserveTopiaApplicationContext temporaryTopiaApplicationContext, String id) { + + TopiaReplicationService service = sourceTopiaApplicationContext.getReplicationService(); + + ReplicationModel model = service.prepare(Entities.DATA_ENTITIES, false, id); + + // To fix missing tdr associations (see https://forge.codelutin.com/issues/6611) + Multimap<String, String> tdrAssociation = null; + + try (ObserveTopiaPersistenceContext sourcePersistenceContext = sourceTopiaApplicationContext.newPersistenceContext()) { + + TopiaEntity e = sourcePersistenceContext.findByTopiaId(id); + + if (e instanceof TripLongline) { + + // Grab tdr missing associations (see https://forge.codelutin.com/issues/6611) + if (log.isInfoEnabled()) { + log.info("Should keep SetLongline - Tdr association ids for: " + id); + } + + tdrAssociation = sourcePersistenceContext.getTdrDao().getTdrIdsBySeineIds((TripLongline) e); + + } + + } + + // do the replicate + try { + service.doReplicate(model, temporaryTopiaApplicationContext); + } catch (Exception e) { + throw new RuntimeException("Could not replicate data: " + id, e); + } + + if (tdrAssociation != null) { + + // Apply back tdr missing associations (see https://forge.codelutin.com/issues/6611) + try (ObserveTopiaPersistenceContext temporaryPersistenceContext = temporaryTopiaApplicationContext.newPersistenceContext()) { + + temporaryPersistenceContext.getTdrDao().applyTdrAssociationFix(tdrAssociation); + + } + + } + } + } diff --git a/observe-services-topia/src/main/java/fr/ird/observe/services/service/DataSourceDumpProducerServiceTopia.java b/observe-services-topia/src/main/java/fr/ird/observe/services/service/DataSourceDumpProducerServiceTopia.java index 536e8e0..203099d 100644 --- a/observe-services-topia/src/main/java/fr/ird/observe/services/service/DataSourceDumpProducerServiceTopia.java +++ b/observe-services-topia/src/main/java/fr/ird/observe/services/service/DataSourceDumpProducerServiceTopia.java @@ -1,31 +1,19 @@ package fr.ird.observe.services.service; -import com.google.common.collect.Multimap; +import com.google.common.collect.ImmutableSet; import fr.ird.observe.ObserveTopiaApplicationContext; import fr.ird.observe.ObserveTopiaPersistenceContext; -import fr.ird.observe.entities.Entities; -import fr.ird.observe.entities.longline.TripLongline; import fr.ird.observe.services.ObserveServiceTopia; import fr.ird.observe.services.ObserveTopiaApplicationContextFactory; import fr.ird.observe.services.configuration.ObserveDataSourceConfigurationTopiaH2; -import fr.ird.observe.services.dto.DataSourceCreateConfigurationDto; -import fr.ird.observe.services.dto.DataSourceCreateWithNoReferentialImportException; -import fr.ird.observe.services.dto.IncompatibleDataSourceCreateConfigurationException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.nuiton.topia.persistence.TopiaEntity; -import org.nuiton.topia.replication.TopiaReplicationService; -import org.nuiton.topia.replication.model.ReplicationModel; import org.nuiton.util.StringUtil; -import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Arrays; -import java.util.LinkedHashSet; import java.util.List; -import java.util.Set; /** * Created on 23/08/15. @@ -37,7 +25,7 @@ public class DataSourceDumpProducerServiceTopia extends ObserveServiceTopia impl /** Logger. */ private static final Log log = LogFactory.getLog(DataSourceDumpProducerServiceTopia.class); - public static final String EXPORT_SQL_STATEMENT = "SCRIPT NOPASSWORDS NOSETTINGS TO '%s' CHARSET 'UTF-8' SCHEMA OBSERVE_COMMON, OBSERVE_SEINE, OBSERVE_LONGLINE;"; + protected static final String EXPORT_SQL_STATEMENT = "SCRIPT NOPASSWORDS NOSETTINGS TO '%s' COMPRESSION GZIP CHARSET 'UTF-8' SCHEMA OBSERVE_COMMON, OBSERVE_SEINE, OBSERVE_LONGLINE, PUBLIC;"; @Override public byte[] getReferentialDump() { @@ -62,43 +50,23 @@ public class DataSourceDumpProducerServiceTopia extends ObserveServiceTopia impl } - protected void replicateReferential(ObserveTopiaApplicationContext sourceTopiaApplicationContext, ObserveTopiaApplicationContext temporaryTopiaApplicationContext) { - - long t0 = System.nanoTime(); - - TopiaReplicationService service = sourceTopiaApplicationContext.getReplicationService(); - ReplicationModel model = service.prepareForAll(Entities.REFERENCE_ENTITIES); - - try { - service.doReplicate(model, temporaryTopiaApplicationContext); - } catch (Exception e) { - //TODO Avoir une exception concrete - throw new RuntimeException("Could not replicate referantial", e); - } + @Override + public byte[] getAllDataDump() { + ImmutableSet.Builder<String> builder = ImmutableSet.builder(); + List<String> tripSeineIds = getTopiaPersistenceContext().getTripSeineDao().findAllIds(); + builder.addAll(tripSeineIds); + List<String> tripLonglineIds = getTopiaPersistenceContext().getTripLonglineDao().findAllIds(); + builder.addAll(tripLonglineIds); - if (log.isInfoEnabled()) { - log.info(String.format("Referential replication done in %s", StringUtil.convertTime(t0, System.nanoTime()))); - } + ImmutableSet<String> ids = builder.build(); + byte[] content = getDataDump(ids); + return content; } @Override - public byte[] getDataDump(String... importDataIds) { - - Set<String> ids = new LinkedHashSet<>(); - - if (importDataIds.length == 0) { - - // cas limite où on exporte toutes les données - List<String> tripSeineIds = getTopiaPersistenceContext().getTripSeineDao().findAllIds(); - ids.addAll(tripSeineIds); - List<String> tripLonglineIds = getTopiaPersistenceContext().getTripLonglineDao().findAllIds(); - ids.addAll(tripLonglineIds); - } else { - - ids.addAll(Arrays.asList(importDataIds)); - } + public byte[] getDataDump(ImmutableSet<String> importDataIds) { long t0 = System.nanoTime(); @@ -113,26 +81,12 @@ public class DataSourceDumpProducerServiceTopia extends ObserveServiceTopia impl replicateReferential(sourceTopiaApplicationContext, temporaryTopiaApplicationContext); - try { - - t0 = System.nanoTime(); - - for (String id : ids) { - long t1 = System.nanoTime(); - - replicateOneData(sourceTopiaApplicationContext, temporaryTopiaApplicationContext, id); - - if (log.isInfoEnabled()) { - log.info(String.format("Data replication [%s] done in %s", id, StringUtil.convertTime(t1, System.nanoTime()))); - } - } + t0 = System.nanoTime(); - if (log.isInfoEnabled()) { - log.info(String.format("Data replication for %s data done in %s", ids.size(), StringUtil.convertTime(t0, System.nanoTime()))); - } + replicateData(sourceTopiaApplicationContext, temporaryTopiaApplicationContext, importDataIds); - } catch (Exception e) { - throw new RuntimeException("Could not replicate data", e); + if (log.isInfoEnabled()) { + log.info(String.format("Data replication for %s data done in %s", importDataIds.size(), StringUtil.convertTime(t0, System.nanoTime()))); } Path dumpFile = exportToFile(temporaryDataSourceConfiguration, temporaryTopiaApplicationContext); @@ -142,31 +96,6 @@ public class DataSourceDumpProducerServiceTopia extends ObserveServiceTopia impl } - protected ObserveDataSourceConfigurationTopiaH2 createTemporaryDatabase(String prefix) { - - ObserveDataSourceConfigurationTopiaH2 temporaryDataSourceConfiguration = new ObserveDataSourceConfigurationTopiaH2(); - temporaryDataSourceConfiguration.setDbName("obstuna"); - temporaryDataSourceConfiguration.setLabel(prefix); - temporaryDataSourceConfiguration.setUsername("sa"); - temporaryDataSourceConfiguration.setPassword('s', 'a'); - File temporaryDataSourceDirectory = serviceContext.createTemporaryDirectory(prefix); - temporaryDataSourceConfiguration.setDirectory(temporaryDataSourceDirectory); - - DataSourceService dataSourceService = serviceContext.newService(temporaryDataSourceConfiguration, DataSourceService.class); - - try { - DataSourceCreateConfigurationDto dataSourceCreateConfiguration = new DataSourceCreateConfigurationDto(); - dataSourceCreateConfiguration.setCanCreateEmptyDatabase(true); - dataSourceService.create(dataSourceCreateConfiguration); - } catch (IncompatibleDataSourceCreateConfigurationException | DataSourceCreateWithNoReferentialImportException e) { - //FIXME Avoir une exception plus concrete - throw new RuntimeException("Could not create temporary database: " + prefix, e); - } - - return temporaryDataSourceConfiguration; - - } - protected Path exportToFile(ObserveDataSourceConfigurationTopiaH2 temporaryDataSourceConfiguration, ObserveTopiaApplicationContext temporaryTopiaApplicationContext) { @@ -196,59 +125,4 @@ public class DataSourceDumpProducerServiceTopia extends ObserveServiceTopia impl } } - /** - * Duplication de l'unique donnée observateur depuis ce service vers le service donné. - * - * La duplication utilise une transaction dédiée afin de ne pas saturer le serveur - * et aussi une meilleure maitrise du rollback en cas d'une erreur. - * - * Voir http://forge.codelutin.com/issues/4837 - * - * @param id l'id de la donnée à répliquer - */ - protected void replicateOneData(ObserveTopiaApplicationContext sourceTopiaApplicationContext, ObserveTopiaApplicationContext temporaryTopiaApplicationContext, String id) { - - TopiaReplicationService service = sourceTopiaApplicationContext.getReplicationService(); - - ReplicationModel model = service.prepare(Entities.DATA_ENTITIES, false, id); - - // To fix missing tdr associations (see https://forge.codelutin.com/issues/6611) - Multimap<String, String> tdrAssociation = null; - - try (ObserveTopiaPersistenceContext sourcePersistenceContext = sourceTopiaApplicationContext.newPersistenceContext()) { - - TopiaEntity e = getTopiaPersistenceContext().findByTopiaId(id); - - if (e instanceof TripLongline) { - - // Grab tdr missing associations (see https://forge.codelutin.com/issues/6611) - if (log.isInfoEnabled()) { - log.info("Should keep SetLongline - Tdr association ids for: " + id); - } - - tdrAssociation = sourcePersistenceContext.getTdrDao().getTdrIdsBySeineIds((TripLongline) e); - - } - - } - - // do the replicate - try { - service.doReplicate(model, temporaryTopiaApplicationContext); - } catch (Exception e) { - throw new RuntimeException("Could not replicate data: " + id, e); - } - - if (tdrAssociation != null) { - - // Apply back tdr missing associations (see https://forge.codelutin.com/issues/6611) - try (ObserveTopiaPersistenceContext observeTopiaPersistenceContext = temporaryTopiaApplicationContext.newPersistenceContext()) { - - observeTopiaPersistenceContext.getTdrDao().applyTdrAssociationFix(tdrAssociation); - - } - - } - } - } diff --git a/observe-services-topia/src/main/java/fr/ird/observe/services/service/DataSourceServiceTopia.java b/observe-services-topia/src/main/java/fr/ird/observe/services/service/DataSourceServiceTopia.java index 52ea0f3..849d92b 100644 --- a/observe-services-topia/src/main/java/fr/ird/observe/services/service/DataSourceServiceTopia.java +++ b/observe-services-topia/src/main/java/fr/ird/observe/services/service/DataSourceServiceTopia.java @@ -1,10 +1,12 @@ package fr.ird.observe.services.service; import com.google.common.base.Optional; +import com.google.common.collect.ImmutableSet; import fr.ird.observe.ObserveTopiaApplicationContext; import fr.ird.observe.services.ObserveServiceTopia; import fr.ird.observe.services.ObserveTopiaApplicationContextFactory; import fr.ird.observe.services.configuration.ObserveDataSourceConfiguration; +import fr.ird.observe.services.configuration.ObserveDataSourceConfigurationTopiaH2; import fr.ird.observe.services.configuration.ObserveDataSourceConfigurationTopiaSupport; import fr.ird.observe.services.dto.DataSourceCreateConfigurationDto; import fr.ird.observe.services.dto.DataSourceCreateWithNoReferentialImportException; @@ -61,36 +63,79 @@ public class DataSourceServiceTopia extends ObserveServiceTopia implements DataS } else { - if (dataSourceCreateConfiguration.isImportReferential()) { + boolean importReferential = dataSourceCreateConfiguration.isImportReferential(); + boolean importData = dataSourceCreateConfiguration.isImportData(); + + boolean importStandaloneReferantial = importReferential; + if (importReferential && importData) { + + // Si on est sur la même source de données pour les deux imports, on fait tout sur la même base temporaire + + ObserveDataSourceConfiguration importReferentialDataSourceConfiguration = dataSourceCreateConfiguration.getImportReferentialDataSourceConfiguration(); + ObserveDataSourceConfiguration importDataSourceConfiguration = dataSourceCreateConfiguration.getImportDataDataSourceConfiguration(); + + importStandaloneReferantial = !importReferentialDataSourceConfiguration.equals(importDataSourceConfiguration); + + } + + boolean referantialImported = false; + + if (importStandaloneReferantial) { if (log.isInfoEnabled()) { - log.info("Import referentials."); + log.info("Import referential."); } ObserveDataSourceConfiguration importDataSourceConfiguration = dataSourceCreateConfiguration.getImportReferentialDataSourceConfiguration(); + // Récupération du dump qui contient le référentiel DataSourceDumpProducerService dumpProducerService = serviceContext.newService(importDataSourceConfiguration, DataSourceDumpProducerService.class); - byte[] referentialDump = dumpProducerService.getReferentialDump(); - executeGzipSqlStatements(topiaApplicationContext, "ImportReferential_" + importDataSourceConfiguration.getLabel(), referentialDump); + // création d'une base temporaire pour y importer le référentiel + ObserveDataSourceConfigurationTopiaH2 temporaryDataSourceConfiguration = createTemporaryDatabaseFromDump("ImportReferential",referentialDump ); + ObserveTopiaApplicationContext temporaryTopiaApplicationContext = ObserveTopiaApplicationContextFactory.getOrCreateTopiaApplicationContext(temporaryDataSourceConfiguration); +// executeGzipSqlStatements(temporaryTopiaApplicationContext, "ImportReferential_" + importDataSourceConfiguration.getLabel(), referentialDump); + + // Réplication du référentiel + replicateReferential(temporaryTopiaApplicationContext, topiaApplicationContext); + + referantialImported = true; } - if (dataSourceCreateConfiguration.isImportData()) { + + if (importData) { ObserveDataSourceConfiguration importDataSourceConfiguration = dataSourceCreateConfiguration.getImportDataDataSourceConfiguration(); - String[] importDataIds = dataSourceCreateConfiguration.getImportDataIds(); + ImmutableSet<String> importDataIds = dataSourceCreateConfiguration.getImportDataIds(); + + // Récupération du dump qui contient le référentiel et les données + DataSourceDumpProducerService dumpProducerService = serviceContext.newService(importDataSourceConfiguration, DataSourceDumpProducerService.class); + byte[] dataDump = dumpProducerService.getDataDump(importDataIds); + + // création d'une base temporaire pour y importer le référentiel et les données + ObserveDataSourceConfigurationTopiaH2 temporaryDataSourceConfiguration = createTemporaryDatabaseFromDump("ImportData",dataDump); + ObserveTopiaApplicationContext temporaryTopiaApplicationContext = ObserveTopiaApplicationContextFactory.getOrCreateTopiaApplicationContext(temporaryDataSourceConfiguration); +// executeGzipSqlStatements(temporaryTopiaApplicationContext, "ImportData_" + importDataSourceConfiguration.getLabel(), dataDump); + + if (!referantialImported) { + + if (log.isInfoEnabled()) { + log.info("Import referential."); + } + + // Réplication du référentiel + replicateReferential(temporaryTopiaApplicationContext, topiaApplicationContext); + + } if (log.isInfoEnabled()) { log.info("Import data."); } - DataSourceDumpProducerService dumpProducerService = serviceContext.newService(importDataSourceConfiguration, DataSourceDumpProducerService.class); - - byte[] dataDump = dumpProducerService.getDataDump(importDataIds); - - executeGzipSqlStatements(topiaApplicationContext, "ImportData_" + importDataSourceConfiguration.getLabel(), dataDump); + // Réplication des données + replicateData(temporaryTopiaApplicationContext, topiaApplicationContext, importDataIds); } diff --git a/observe-services-topia/src/test/java/fr/ird/observe/services/ApplicationContextResource.java b/observe-services-topia/src/test/java/fr/ird/observe/services/ApplicationContextResource.java index 4b13c45..2dc55a7 100644 --- a/observe-services-topia/src/test/java/fr/ird/observe/services/ApplicationContextResource.java +++ b/observe-services-topia/src/test/java/fr/ird/observe/services/ApplicationContextResource.java @@ -1,14 +1,19 @@ package fr.ird.observe.services; +import fr.ird.observe.ObserveTopiaApplicationContext; +import fr.ird.observe.ObserveTopiaPersistenceContext; import fr.ird.observe.services.configuration.ObserveDataSourceConfiguration; +import fr.ird.observe.services.configuration.ObserveDataSourceConfigurationTopiaH2; import fr.ird.observe.services.configuration.ObserveDataSourceConfigurationTopiaSupport; import fr.ird.observe.services.dto.constants.ReferentialLocale; import fr.ird.observe.test.TestHelper; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.junit.Assert; import org.junit.rules.TestRule; import org.junit.runner.Description; import org.junit.runners.model.Statement; +import org.nuiton.topia.persistence.TopiaException; import org.nuiton.util.version.Version; import java.io.File; @@ -125,4 +130,49 @@ public class ApplicationContextResource implements TestRule { public void setTestDirectory(File testDirectory) { this.testDirectory = testDirectory; } + + public ObserveDataSourceConfigurationTopiaH2 createDataSourceConfigurationH2(Class<?> testClass, String methodName) { + + ObserveDataSourceConfigurationTopiaH2 configurationTopiaH2 = new ObserveDataSourceConfigurationTopiaH2(); + + File localDbFile = new File(getTestDirectory(), methodName); + + if (log.isInfoEnabled()) { + log.info("db directory: " + localDbFile); + } + configurationTopiaH2.setLabel(testClass.getSimpleName() + "#" + methodName); + configurationTopiaH2.setUsername(TestHelper.TEST_H2_LOGIN); + configurationTopiaH2.setPassword(TestHelper.TEST_H2_PASSWORD); + configurationTopiaH2.setDirectory(localDbFile); + configurationTopiaH2.setDbName("obstuna"); + configurationTopiaH2.setAutoMigrate(true); + return configurationTopiaH2; + + } + + public void assertSchemaCreated(ObserveTopiaApplicationContext topiaApplicationContext) { + + try (ObserveTopiaPersistenceContext topiaPersistenceContext = topiaApplicationContext.newPersistenceContext()) { + TestHelper.assertSchemaCreated(topiaApplicationContext, topiaPersistenceContext, false); + } + + } + + public void assertEntitiesExist(ObserveTopiaApplicationContext topiaApplicationContext, String... ids) { + + try (ObserveTopiaPersistenceContext persistenceContext = topiaApplicationContext.newPersistenceContext()) { + + for (String id : ids) { + try { + persistenceContext.findByTopiaId(id); + if (log.isInfoEnabled()) { + log.info("Entity " + id + " found."); + } + } catch (TopiaException e) { + Assert.fail("Could not find entity with id: " + id); + } + } + } + } + } diff --git a/observe-services-topia/src/test/java/fr/ird/observe/services/DataSourceResource.java b/observe-services-topia/src/test/java/fr/ird/observe/services/DataSourceResource.java index 95511b0..f3ec450 100644 --- a/observe-services-topia/src/test/java/fr/ird/observe/services/DataSourceResource.java +++ b/observe-services-topia/src/test/java/fr/ird/observe/services/DataSourceResource.java @@ -96,13 +96,13 @@ public class DataSourceResource implements TestRule { } - public <E extends TopiaEntity> boolean exists(Class<E> entityType, String id) { + public <E extends TopiaEntity> boolean exists(Class<E> entityType, String id) { TopiaPersistenceContext persistenceContext = newPersistenceContext(); TopiaDao<E> dao = persistenceContext.getDao(entityType); - boolean exist = dao.forTopiaIdEquals(id).exists(); + boolean exist = dao.forTopiaIdEquals(id).exists(); return exist; @@ -145,28 +145,10 @@ public class DataSourceResource implements TestRule { } - dataSourceConfiguration = createDataSourceConfigurationH2(testClass, methodName); + dataSourceConfiguration = applicationContextResource.createDataSourceConfigurationH2(testClass, methodName); } - protected ObserveDataSourceConfigurationTopiaH2 createDataSourceConfigurationH2(Class<?> testClass, String methodName) { - - ObserveDataSourceConfigurationTopiaH2 configurationTopiaH2 = new ObserveDataSourceConfigurationTopiaH2(); - - File localDbFile = new File(applicationContextResource.getTestDirectory(), "localdb"); - - if (log.isInfoEnabled()) { - log.info("db directory: " + localDbFile); - } - configurationTopiaH2.setLabel(testClass.getSimpleName() + "#" + methodName); - configurationTopiaH2.setUsername(TestHelper.TEST_H2_LOGIN); - configurationTopiaH2.setPassword(TestHelper.TEST_H2_PASSWORD); - configurationTopiaH2.setDirectory(localDbFile); - configurationTopiaH2.setDbName("obstuna"); - configurationTopiaH2.setAutoMigrate(true); - return configurationTopiaH2; - - } protected void after(Description description) { diff --git a/observe-services-topia/src/test/java/fr/ird/observe/services/service/DataSourceDumpProducerServiceTopiaTest.java b/observe-services-topia/src/test/java/fr/ird/observe/services/service/DataSourceDumpProducerServiceTopiaTest.java index 4eb5c36..9de9d68 100644 --- a/observe-services-topia/src/test/java/fr/ird/observe/services/service/DataSourceDumpProducerServiceTopiaTest.java +++ b/observe-services-topia/src/test/java/fr/ird/observe/services/service/DataSourceDumpProducerServiceTopiaTest.java @@ -41,11 +41,20 @@ public class DataSourceDumpProducerServiceTopiaTest extends AbstractServiceTopia } + @Test public void testGetDataDump() throws Exception { - byte[] referentialDump = service.getDataDump(); - Assert.assertNotNull(referentialDump); + byte[] dataDump = service.getDataDump(ImmutableSet.of(TRIP_SEINE_ID_1)); + Assert.assertNotNull(dataDump); + + } + + @Test + public void testGetAllDataDump() throws Exception { + + byte[] allDataDump = service.getAllDataDump(); + Assert.assertNotNull(allDataDump); } } \ No newline at end of file diff --git a/observe-services-topia/src/test/java/fr/ird/observe/services/service/DataSourceServiceTopiaTest.java b/observe-services-topia/src/test/java/fr/ird/observe/services/service/DataSourceServiceTopiaTest.java index 4441b80..a5cb968 100644 --- a/observe-services-topia/src/test/java/fr/ird/observe/services/service/DataSourceServiceTopiaTest.java +++ b/observe-services-topia/src/test/java/fr/ird/observe/services/service/DataSourceServiceTopiaTest.java @@ -2,18 +2,16 @@ package fr.ird.observe.services.service; import com.google.common.collect.ImmutableSet; import fr.ird.observe.ObserveTopiaApplicationContext; -import fr.ird.observe.ObserveTopiaPersistenceContext; import fr.ird.observe.entities.migration.H2DataSourceMigration; import fr.ird.observe.services.ApplicationContextResource; import fr.ird.observe.services.DataSourceResource; +import fr.ird.observe.services.ObserveTopiaApplicationContextFactory; import fr.ird.observe.services.configuration.ObserveDataSourceConfigurationTopiaH2; import fr.ird.observe.services.dto.DataSourceCreateConfigurationDto; import fr.ird.observe.services.dto.DataSourceCreateWithNoReferentialImportException; import fr.ird.observe.services.dto.IncompatibleDataSourceCreateConfigurationException; -import fr.ird.observe.test.TestHelper; import org.junit.Before; import org.junit.ClassRule; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; @@ -64,7 +62,7 @@ public class DataSourceServiceTopiaTest { service.create(dataSourceCreateConfiguration); service.open(); - assertSchemaCreated(); + applicationContextResource.assertSchemaCreated(dataSourceResource.getTopiaApplicationContext()); } @@ -75,7 +73,7 @@ public class DataSourceServiceTopiaTest { dataSourceCreateConfiguration.setCanCreateEmptyDatabase(true); service.create(dataSourceCreateConfiguration); - assertSchemaCreated(); + applicationContextResource.assertSchemaCreated(dataSourceResource.getTopiaApplicationContext()); } @@ -90,50 +88,56 @@ public class DataSourceServiceTopiaTest { dataSourceCreateConfiguration.setImportDatabase(dumpContent); service.create(dataSourceCreateConfiguration); - assertSchemaCreated(); + applicationContextResource.assertSchemaCreated(dataSourceResource.getTopiaApplicationContext()); } - //TODO - @Ignore @Test public void testCreateDataSourceFromImportReferential() throws IOException, IncompatibleDataSourceCreateConfigurationException, DataSourceCreateWithNoReferentialImportException { - String scriptPath = applicationContextResource.getScriptPath("referentiel", H2DataSourceMigration.V_4_0); - byte[] dumpContent = dataSourceResource.getDatabaseToImportContent(scriptPath); + // On crée un base qui fait office de source de référentiel + String scriptPath = applicationContextResource.getScriptPath("referentiel", H2DataSourceMigration.V_4_0_1); + dataSourceResource.createDataSourceFromScript(scriptPath); + + ObserveDataSourceConfigurationTopiaH2 dataSourceConfiguration = dataSourceResource.getDataSourceConfiguration(); DataSourceCreateConfigurationDto dataSourceCreateConfiguration = new DataSourceCreateConfigurationDto(); - dataSourceCreateConfiguration.setImportDatabase(dumpContent); + dataSourceCreateConfiguration.setImportReferentialDataSourceConfiguration(dataSourceConfiguration); - service.create(dataSourceCreateConfiguration); - assertSchemaCreated(); + ObserveDataSourceConfigurationTopiaH2 createdDatabaseDataSourceConfiguration = applicationContextResource.createDataSourceConfigurationH2(getClass(), "resultDatabase"); + + DataSourceService dataSourceService = applicationContextResource.newService(createdDatabaseDataSourceConfiguration, DataSourceService.class); + + dataSourceService.create(dataSourceCreateConfiguration); + + ObserveTopiaApplicationContext createdDatabaseTopiaApplicationContext = ObserveTopiaApplicationContextFactory.getOrCreateTopiaApplicationContext(createdDatabaseDataSourceConfiguration); + applicationContextResource.assertSchemaCreated(createdDatabaseTopiaApplicationContext); } - //TODO - @Ignore @Test public void testCreateDataSourceFromImportReferentialAndData() throws IOException, IncompatibleDataSourceCreateConfigurationException, DataSourceCreateWithNoReferentialImportException { - String scriptPath = applicationContextResource.getScriptPath("referentiel", H2DataSourceMigration.V_4_0); - byte[] dumpContent = dataSourceResource.getDatabaseToImportContent(scriptPath); + // On crée un base qui fait office de source de référentiel et de source de données (uniquement des données seine) + String scriptPath = applicationContextResource.getScriptPath("dataForTestSeine", H2DataSourceMigration.V_4_0_1); + dataSourceResource.createDataSourceFromScript(scriptPath); + + ObserveDataSourceConfigurationTopiaH2 dataSourceConfiguration = dataSourceResource.getDataSourceConfiguration(); DataSourceCreateConfigurationDto dataSourceCreateConfiguration = new DataSourceCreateConfigurationDto(); - dataSourceCreateConfiguration.setImportDatabase(dumpContent); + dataSourceCreateConfiguration.setImportReferentialDataSourceConfiguration(dataSourceConfiguration); + dataSourceCreateConfiguration.setImportDataConfiguration(dataSourceConfiguration, ImmutableSet.of(AbstractServiceTopiaTest.TRIP_SEINE_ID_1)); - service.create(dataSourceCreateConfiguration); - assertSchemaCreated(); + ObserveDataSourceConfigurationTopiaH2 createdDatabaseDataSourceConfiguration = applicationContextResource.createDataSourceConfigurationH2(getClass(), "resultDatabase"); - } + DataSourceService dataSourceService = applicationContextResource.newService(createdDatabaseDataSourceConfiguration, DataSourceService.class); - protected void assertSchemaCreated() { + dataSourceService.create(dataSourceCreateConfiguration); - ObserveTopiaApplicationContext topiaApplicationContext = dataSourceResource.getTopiaApplicationContext(); - try (ObserveTopiaPersistenceContext topiaPersistenceContext = topiaApplicationContext.newPersistenceContext()) { - TestHelper.assertSchemaCreated(topiaApplicationContext, topiaPersistenceContext, false); - } + ObserveTopiaApplicationContext createdDatabaseTopiaApplicationContext = ObserveTopiaApplicationContextFactory.getOrCreateTopiaApplicationContext(createdDatabaseDataSourceConfiguration); + applicationContextResource.assertSchemaCreated(createdDatabaseTopiaApplicationContext); + applicationContextResource.assertEntitiesExist(createdDatabaseTopiaApplicationContext, AbstractServiceTopiaTest.TRIP_SEINE_ID_1); } - } \ No newline at end of file -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@list.forge.codelutin.com>.
This is an automated email from the git hooks/post-receive script. New commit to branch develop in repository observe. See http://git.codelutin.com/observe.git commit 969e71b966e31721c30aacbd03971c8eb06102d4 Merge: 69ff107 39fb53d Author: Tony CHEMIT <chemit@codelutin.com> Date: Thu Aug 27 10:25:36 2015 +0200 Création d'une base via les services (termine #7482). Merge branch 'feature/7482' into develop .../service/DataSourceDumpProducerService.java | 7 +- .../dto/DataSourceCreateConfigurationDto.java | 7 +- .../ird/observe/services/ObserveServiceTopia.java | 167 ++++++++++++++++++++- .../DataSourceDumpProducerServiceTopia.java | 160 +++----------------- .../services/service/DataSourceServiceTopia.java | 67 +++++++-- .../services/ApplicationContextResource.java | 50 ++++++ .../ird/observe/services/DataSourceResource.java | 24 +-- .../DataSourceDumpProducerServiceTopiaTest.java | 13 +- .../service/DataSourceServiceTopiaTest.java | 58 +++---- 9 files changed, 342 insertions(+), 211 deletions(-) -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@list.forge.codelutin.com>.
participants (1)
-
codelutin.com scm