branch feature/deal_with_blobs created (now 1e3d400)
This is an automated email from the git hooks/post-receive script. New change to branch feature/deal_with_blobs in repository observe. See https://gitlab.nuiton.org/codelutin/observe.git at 1e3d400 Add entity enum + deal with blobs in import - export of trips (introduce a new api to be used soon also for bi-directionnal data synchro) This branch includes the following new commits: new 1e3d400 Add entity enum + deal with blobs in import - export of trips (introduce a new api to be used soon also for bi-directionnal data synchro) The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "adds" were already present in the repository and have only been added to this reference. Detailed log of new commits: commit 1e3d4000076a382c71b1ee91118feaf6ceafacec Author: Tony CHEMIT <chemit@codelutin.com> Date: Thu Aug 25 12:15:21 2016 +0200 Add entity enum + deal with blobs in import - export of trips (introduce a new api to be used soon also for bi-directionnal data synchro) -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.
This is an automated email from the git hooks/post-receive script. New commit to branch feature/deal_with_blobs in repository observe. See https://gitlab.nuiton.org/codelutin/observe.git commit 1e3d4000076a382c71b1ee91118feaf6ceafacec Author: Tony CHEMIT <chemit@codelutin.com> Date: Thu Aug 25 12:15:21 2016 +0200 Add entity enum + deal with blobs in import - export of trips (introduce a new api to be used soon also for bi-directionnal data synchro) --- .../fr/ird/observe/db/ObserveSwingDataSource.java | 12 +- .../fr/ird/observe/ui/admin/AdminTabUIHandler.java | 4 +- .../java/fr/ird/observe/ui/admin/AdminUIModel.java | 4 +- .../observe/ui/admin/export/ExportUIHandler.java | 86 ++++++++------- .../fr/ird/observe/ui/admin/export/TripEntry.java | 4 + .../observe/ui/admin/save/SaveLocalUIHandler.java | 3 +- .../ird/observe/ui/storage/StorageUIHandler.java | 4 +- .../fr/ird/observe/ui/storage/StorageUIModel.java | 2 +- .../observe-application-swing_en_GB.properties | 1 + .../observe-application-swing_es_ES.properties | 1 + .../observe-application-swing_fr_FR.properties | 3 +- .../v1/SqlScriptProducerServiceController.java | 3 +- .../ExportTripServiceController.java} | 23 ++-- .../v1/trip/ImportTripServiceController.java | 30 +++-- .../observe/ObserveTopiaPersistenceContext.java | 3 +- .../observe/services/ObserveServicesProvider.java | 7 +- .../service/AddSqlScriptProducerResult.java | 27 +++++ .../services/service/ObserveBlobsContainer.java | 37 +++++++ .../services/service/SqlScriptProducerService.java | 2 +- .../actions/export/ReplicateTripsExportResult.java | 100 ----------------- .../ExportTripService.java} | 12 +- .../ExportTripsRequest.java} | 61 +++++------ .../services/service/trip/ExportTripsResult.java | 121 +++++++++++++++++++++ .../ImportTripService.java} | 16 ++- .../ImportTripsRequest.java} | 52 +++++---- .../ImportTripsResult.java} | 33 +++--- .../services/service/DataSourceServiceTopia.java | 12 +- .../service/SqlScriptProducerServiceTopia.java | 15 ++- .../actions/export/ReplicateTripServiceTopia.java | 115 -------------------- .../synchro/data/DataSynchronizeServiceTopia.java | 2 +- .../service/trip/ExportTripServiceTopia.java | 77 +++++++++++++ .../service/trip/ImportTripScriptTopiaSqlWork.java | 68 ++++++++++++ .../service/trip/ImportTripServiceTopia.java | 121 +++++++++++++++++++++ .../AddSqlScriptProducerServiceTopiaTest.java | 2 +- .../services/service/MigrateTestsDatabases.java | 6 +- .../persistence/metadata/TopiaMetadataEntity.java | 16 +++ .../nuiton/topia/service/sql/batch/SqlResult.java | 23 ++++ .../service/sql/batch/TopiaSqlBatchService.java | 6 +- .../sql/batch/TopiaSqlBatchServiceImpl.java | 75 +++++++++++-- .../sql/batch/actions/AbstractSqlAction.java | 52 ++++++--- .../sql/batch/actions/AbstractTablesAction.java | 48 +------- .../service/sql/batch/actions/BlobsContainer.java | 80 ++++++++++++++ .../sql/batch/actions/ReplicateTablesAction.java | 71 +++++++++++- .../service/sql/batch/tables/TopiaSqlTable.java | 19 +++- .../service/sql/batch/tables/TopiaSqlTables.java | 53 +++++---- .../sql/batch/tables/TopiaSqlTablesFactory.java | 5 +- 46 files changed, 1008 insertions(+), 509 deletions(-) diff --git a/observe-application-swing/src/main/java/fr/ird/observe/db/ObserveSwingDataSource.java b/observe-application-swing/src/main/java/fr/ird/observe/db/ObserveSwingDataSource.java index c6ab886..4d6932e 100644 --- a/observe-application-swing/src/main/java/fr/ird/observe/db/ObserveSwingDataSource.java +++ b/observe-application-swing/src/main/java/fr/ird/observe/db/ObserveSwingDataSource.java @@ -60,7 +60,8 @@ import fr.ird.observe.services.service.PingService; import fr.ird.observe.services.service.ReferentialService; import fr.ird.observe.services.service.SqlScriptProducerService; import fr.ird.observe.services.service.actions.consolidate.ConsolidateDataService; -import fr.ird.observe.services.service.actions.export.ReplicateTripService; +import fr.ird.observe.services.service.trip.ExportTripService; +import fr.ird.observe.services.service.trip.ImportTripService; import fr.ird.observe.services.service.actions.report.ReportService; import fr.ird.observe.services.service.actions.synchro.data.DataSynchronizeService; import fr.ird.observe.services.service.actions.synchro.referential.diff.ReferentialSynchronizeDiffService; @@ -557,8 +558,13 @@ public class ObserveSwingDataSource extends AbstractSerializableBean implements } @Override - public ReplicateTripService newReplicateTripService() { - return newService(ReplicateTripService.class); + public ExportTripService newExportTripService() { + return newService(ExportTripService.class); + } + + @Override + public ImportTripService newImportTripService() { + return newService(ImportTripService.class); } @Override diff --git a/observe-application-swing/src/main/java/fr/ird/observe/ui/admin/AdminTabUIHandler.java b/observe-application-swing/src/main/java/fr/ird/observe/ui/admin/AdminTabUIHandler.java index 1be451a..c551c61 100644 --- a/observe-application-swing/src/main/java/fr/ird/observe/ui/admin/AdminTabUIHandler.java +++ b/observe-application-swing/src/main/java/fr/ird/observe/ui/admin/AdminTabUIHandler.java @@ -44,6 +44,7 @@ import java.io.StringWriter; import java.util.concurrent.Callable; import static org.nuiton.i18n.I18n.n; +import static org.nuiton.i18n.I18n.reload; import static org.nuiton.i18n.I18n.t; /** @@ -196,10 +197,11 @@ public class AdminTabUIHandler { } } - protected void openSource(ObserveSwingDataSource service) throws DatabaseConnexionNotAuthorizedException, DatabaseNotFoundException, BabModelVersionException { + protected ObserveSwingDataSource openSource(ObserveSwingDataSource service) throws DatabaseConnexionNotAuthorizedException, DatabaseNotFoundException, BabModelVersionException { if (!service.isOpen()) { service.open(); } + return service; } public void addAdminWorker(String label, Callable<WizardState> callable) { diff --git a/observe-application-swing/src/main/java/fr/ird/observe/ui/admin/AdminUIModel.java b/observe-application-swing/src/main/java/fr/ird/observe/ui/admin/AdminUIModel.java index 9e936dd..b9e618d 100644 --- a/observe-application-swing/src/main/java/fr/ird/observe/ui/admin/AdminUIModel.java +++ b/observe-application-swing/src/main/java/fr/ird/observe/ui/admin/AdminUIModel.java @@ -962,10 +962,10 @@ public class AdminUIModel extends WizardExtModel<AdminStep> { // pour exporter les données utilisateurs // l'opération de synchronisation de référentiel est nécessaire - getOperations().add(AdminStep.SYNCHRONIZE); +// getOperations().add(AdminStep.SYNCHRONIZE); // l'opération de calcul des données est aussi obligatoire - getOperations().add(AdminStep.CONSOLIDATE); +// getOperations().add(AdminStep.CONSOLIDATE); } if (AdminStep.REPORT == step) { diff --git a/observe-application-swing/src/main/java/fr/ird/observe/ui/admin/export/ExportUIHandler.java b/observe-application-swing/src/main/java/fr/ird/observe/ui/admin/export/ExportUIHandler.java index e6df440..3b04eab 100644 --- a/observe-application-swing/src/main/java/fr/ird/observe/ui/admin/export/ExportUIHandler.java +++ b/observe-application-swing/src/main/java/fr/ird/observe/ui/admin/export/ExportUIHandler.java @@ -24,6 +24,9 @@ package fr.ird.observe.ui.admin.export; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; +import fr.ird.observe.application.swing.decoration.DecoratorService; +import fr.ird.observe.application.swing.decoration.decorators.DataReferenceDecorator; +import fr.ird.observe.application.swing.decoration.decorators.ReferentialReferenceDecorator; import fr.ird.observe.db.ObserveSwingDataSource; import fr.ird.observe.services.dto.DataReference; import fr.ird.observe.services.dto.longline.TripLonglineDto; @@ -32,21 +35,21 @@ import fr.ird.observe.services.dto.seine.TripSeineDto; import fr.ird.observe.services.service.BabModelVersionException; import fr.ird.observe.services.service.DatabaseConnexionNotAuthorizedException; import fr.ird.observe.services.service.DatabaseNotFoundException; -import fr.ird.observe.services.service.actions.export.ReplicateTripService; -import fr.ird.observe.services.service.actions.export.ReplicateTripsExportRequest; -import fr.ird.observe.services.service.actions.export.ReplicateTripsExportResult; -import fr.ird.observe.services.service.actions.export.ReplicateTripsImportResult; -import fr.ird.observe.application.swing.decoration.DecoratorService; +import fr.ird.observe.services.service.trip.ExportTripService; +import fr.ird.observe.services.service.trip.ExportTripsRequest; +import fr.ird.observe.services.service.trip.ExportTripsResult; +import fr.ird.observe.services.service.trip.ImportTripService; +import fr.ird.observe.services.service.trip.ImportTripsRequest; +import fr.ird.observe.services.service.trip.ImportTripsResult; import fr.ird.observe.ui.UIHelper; import fr.ird.observe.ui.admin.AdminStep; import fr.ird.observe.ui.admin.AdminTabUI; import fr.ird.observe.ui.admin.AdminTabUIHandler; import fr.ird.observe.ui.admin.AdminUI; import fr.ird.observe.ui.admin.AdminUIModel; +import fr.ird.observe.ui.admin.config.ConfigUI; import fr.ird.observe.ui.admin.config.SelectDataUI; import fr.ird.observe.ui.storage.tabs.DataSelectionModel; -import fr.ird.observe.application.swing.decoration.decorators.DataReferenceDecorator; -import fr.ird.observe.application.swing.decoration.decorators.ReferentialReferenceDecorator; import jaxx.runtime.swing.editor.MyDefaultCellEditor; import jaxx.runtime.swing.wizard.ext.WizardState; import org.apache.commons.collections.CollectionUtils; @@ -89,7 +92,12 @@ public class ExportUIHandler extends AdminTabUIHandler { tabUI.getPrepareAction().setText(t("observe.actions.synchro.prepare.operation", t(tabUI.getStep().getOperationLabel()))); tabUI.getStartAction().setText(t("observe.actions.synchro.launch.operation", t(tabUI.getStep().getOperationLabel()))); - final SelectDataUI selectTabUI = (SelectDataUI) ui.getStepUI(AdminStep.SELECT_DATA); + ConfigUI configabUI = (ConfigUI) ui.getStepUI(AdminStep.CONFIG); + + configabUI.getCentralSourceInfoLabel().setText(t("observe.action.config.export.required.write.data")); + configabUI.getCentralSourceLabel().setText(t("observe.action.config.export.required.write.data")); + + SelectDataUI selectTabUI = (SelectDataUI) ui.getStepUI(AdminStep.SELECT_DATA); tabUI.getModel().addPropertyChangeListener(AdminUIModel.SELECTION_MODEL_CHANGED_PROPERTY_NAME, evt -> { AdminUIModel model1 = (AdminUIModel) evt.getSource(); @@ -158,10 +166,6 @@ public class ExportUIHandler extends AdminTabUIHandler { if (newState == WizardState.NEED_FIX) { // met a jour la liste des marées tabUI.tripsModel.init(tabUI.getStepModel().getData()); - return; - } - if (newState == WizardState.RUNNING) { - } } @@ -249,11 +253,11 @@ public class ExportUIHandler extends AdminTabUIHandler { // on filtre les marées sélectionnées pour export List<TripEntry> selectedTripEntries = stepModel.getSelectedTrips(); - ReplicateTripsImportResult results = replicateData(stepModel.getSource(), stepModel.getCentralSource(), selectedTripEntries); + ImportTripsResult results = replicateData(stepModel.getSource(), stepModel.getCentralSource(), selectedTripEntries); - ImmutableMap<String, TripEntry> tripEntriesById = Maps.uniqueIndex(selectedTripEntries, tripEntry -> tripEntry.getTrip().getId()); + ImmutableMap<String, TripEntry> tripEntriesById = Maps.uniqueIndex(selectedTripEntries, TripEntry::getTripId); - for (ReplicateTripsImportResult.ReplicateTripImportResult result : results) { + for (ImportTripsResult.ImportTripResult result : results) { String tripId = result.getTripId(); @@ -282,51 +286,55 @@ public class ExportUIHandler extends AdminTabUIHandler { return WizardState.SUCCESSED; } - private ReplicateTripsImportResult replicateData(ObserveSwingDataSource source, - ObserveSwingDataSource target, - List<TripEntry> tripEntries) throws DatabaseNotFoundException, BabModelVersionException, DatabaseConnexionNotAuthorizedException { + private ImportTripsResult replicateData(ObserveSwingDataSource source, + ObserveSwingDataSource target, + List<TripEntry> tripEntries) throws DatabaseNotFoundException, BabModelVersionException, DatabaseConnexionNotAuthorizedException { Preconditions.checkState(CollectionUtils.isNotEmpty(tripEntries)); - ReplicateTripsExportRequest.ExportTripsRequestBuilder requestBuilder = ReplicateTripsExportRequest.builder(); - - for (TripEntry tripEntry : tripEntries) { + // Create export request - requestBuilder.addTripToReplicate(tripEntry.getTrip().getId(), tripEntry.isExist()); + ExportTripsRequest.Builder requestBuilder = ExportTripsRequest.builder(true); + for (TripEntry tripEntry : tripEntries) { + requestBuilder.addTripToExport(tripEntry.getTrip().getId()); } - ReplicateTripsExportRequest request = requestBuilder.build(); + ExportTripsRequest exportTripsRequest = requestBuilder.build(); - ReplicateTripsExportResult tripExportResults; + // Do export - try { + ExportTripsResult exportTripsResult; - openSource(source); - ReplicateTripService sourceReplicateService = source.newReplicateTripService(); - tripExportResults = sourceReplicateService.exportTrips(request); + try (ObserveSwingDataSource observeSwingDataSource = openSource(source)) { - } finally { - - source.close(); + ExportTripService exportTripService = observeSwingDataSource.newExportTripService(); + exportTripsResult = exportTripService.exportTrips(exportTripsRequest); } - ReplicateTripsImportResult result; + //TODO Print export results + - try { + // Create import request - openSource(target); - ReplicateTripService targetReplicateService = target.newReplicateTripService(); - result = targetReplicateService.importTrips(tripExportResults); + ImportTripsRequest.Builder importRequestBuilder = ImportTripsRequest.builder(); + for (ExportTripsResult.ExportTripResult tripExportResult : exportTripsResult) { + importRequestBuilder.addTripToImport(tripExportResult.toImportRequest()); + } - } finally { + ImportTripsRequest importTripRequests = importRequestBuilder.build(); - target.close(); + // Do import + ImportTripsResult importTripsResult; + try (ObserveSwingDataSource observeSwingDataSource = openSource(target)) { + ImportTripService importTripService = observeSwingDataSource.newImportTripService(); + importTripsResult = importTripService.importTrips(importTripRequests); } - return result; + + return importTripsResult; } diff --git a/observe-application-swing/src/main/java/fr/ird/observe/ui/admin/export/TripEntry.java b/observe-application-swing/src/main/java/fr/ird/observe/ui/admin/export/TripEntry.java index 27f2e32..3054c38 100644 --- a/observe-application-swing/src/main/java/fr/ird/observe/ui/admin/export/TripEntry.java +++ b/observe-application-swing/src/main/java/fr/ird/observe/ui/admin/export/TripEntry.java @@ -52,6 +52,10 @@ public class TripEntry implements Serializable { return trip; } + public String getTripId() { + return trip.getId(); + } + public boolean isExist() { return exist; } diff --git a/observe-application-swing/src/main/java/fr/ird/observe/ui/admin/save/SaveLocalUIHandler.java b/observe-application-swing/src/main/java/fr/ird/observe/ui/admin/save/SaveLocalUIHandler.java index b1aa980..b251192 100644 --- a/observe-application-swing/src/main/java/fr/ird/observe/ui/admin/save/SaveLocalUIHandler.java +++ b/observe-application-swing/src/main/java/fr/ird/observe/ui/admin/save/SaveLocalUIHandler.java @@ -23,6 +23,7 @@ package fr.ird.observe.ui.admin.save; import fr.ird.observe.db.ObserveSwingDataSource; import fr.ird.observe.services.service.AddSqlScriptProducerRequest; +import fr.ird.observe.services.service.AddSqlScriptProducerResult; import fr.ird.observe.services.service.SqlScriptProducerService; import fr.ird.observe.services.service.actions.synchro.referential.legacy.UnidirectionalReferentialSynchronizeContext; import fr.ird.observe.services.service.actions.synchro.referential.legacy.UnidirectionalReferentialSynchronizeEngine; @@ -156,7 +157,7 @@ public class SaveLocalUIHandler extends AdminTabUIHandler { SqlScriptProducerService dumpService = source.newSqlScriptProducerService(); AddSqlScriptProducerRequest request = AddSqlScriptProducerRequest.forH2().addSchema().addReferential().addAllData(); - byte[] dump = dumpService.produceAddSqlScript(request); + byte[] dump = dumpService.produceAddSqlScript(request).getSqlCode(); try (FileOutputStream outputStream = new FileOutputStream(backupFile)) { IOUtils.write(dump, outputStream); diff --git a/observe-application-swing/src/main/java/fr/ird/observe/ui/storage/StorageUIHandler.java b/observe-application-swing/src/main/java/fr/ird/observe/ui/storage/StorageUIHandler.java index a64a8a7..e853267 100644 --- a/observe-application-swing/src/main/java/fr/ird/observe/ui/storage/StorageUIHandler.java +++ b/observe-application-swing/src/main/java/fr/ird/observe/ui/storage/StorageUIHandler.java @@ -605,7 +605,7 @@ public class StorageUIHandler { } AddSqlScriptProducerRequest request = AddSqlScriptProducerRequest.forH2().addSchema().addReferential().addAllData(); - byte[] dataDump = dumpProducerService.produceAddSqlScript(request); + byte[] dataDump = dumpProducerService.produceAddSqlScript(request).getSqlCode(); try (FileOutputStream outputStream = new FileOutputStream(dst)) { @@ -659,7 +659,7 @@ public class StorageUIHandler { ImmutableSet<String> tripIds = ImmutableSet.copyOf(trips.stream().map(DataReference.ID_FUNCTION).collect(Collectors.toSet())); AddSqlScriptProducerRequest request = AddSqlScriptProducerRequest.forH2().addSchema().addReferential().dataIdsToAdd(tripIds); - byte[] dump = dumpService.produceAddSqlScript(request); + byte[] dump = dumpService.produceAddSqlScript(request).getSqlCode(); try (FileOutputStream fileOutputStream = new FileOutputStream(dst)) { diff --git a/observe-application-swing/src/main/java/fr/ird/observe/ui/storage/StorageUIModel.java b/observe-application-swing/src/main/java/fr/ird/observe/ui/storage/StorageUIModel.java index 1ac2533..2436d26 100644 --- a/observe-application-swing/src/main/java/fr/ird/observe/ui/storage/StorageUIModel.java +++ b/observe-application-swing/src/main/java/fr/ird/observe/ui/storage/StorageUIModel.java @@ -1565,7 +1565,7 @@ public class StorageUIModel extends WizardModel<StorageStep> { SqlScriptProducerService dumpService = source.newSqlScriptProducerService(); AddSqlScriptProducerRequest request = AddSqlScriptProducerRequest.forH2().addSchema().addReferential(); - byte[] dump = dumpService.produceAddSqlScript(request); + byte[] dump = dumpService.produceAddSqlScript(request).getSqlCode(); result.setImportDatabase(dump); } diff --git a/observe-application-swing/src/main/resources/i18n/observe-application-swing_en_GB.properties b/observe-application-swing/src/main/resources/i18n/observe-application-swing_en_GB.properties index 7ccf536..e49fb1c 100644 --- a/observe-application-swing/src/main/resources/i18n/observe-application-swing_en_GB.properties +++ b/observe-application-swing/src/main/resources/i18n/observe-application-swing_en_GB.properties @@ -59,6 +59,7 @@ observe.action.commandline.launch.h2.server.mode= observe.action.commandline.launch.obstuna.admin.ui= observe.action.commandline.use.jmx=Add JMX support observe.action.config.export.required.read.referentiel= +observe.action.config.export.required.write.data= observe.action.configuration=Preferences observe.action.configuration.tip=Change preferences observe.action.configure=Configure diff --git a/observe-application-swing/src/main/resources/i18n/observe-application-swing_es_ES.properties b/observe-application-swing/src/main/resources/i18n/observe-application-swing_es_ES.properties index 35fc99f..75b8703 100644 --- a/observe-application-swing/src/main/resources/i18n/observe-application-swing_es_ES.properties +++ b/observe-application-swing/src/main/resources/i18n/observe-application-swing_es_ES.properties @@ -59,6 +59,7 @@ observe.action.commandline.launch.h2.server.mode=Ejecutar ObServe en modo servid observe.action.commandline.launch.obstuna.admin.ui=Ejecutar una operación de administración sobre Obstuna observe.action.commandline.use.jmx=Mostrar el soporte JMX observe.action.config.export.required.read.referentiel=Debe tener derechos de escritura sobre los datos de la base remota +observe.action.config.export.required.write.data= observe.action.configuration=Configuración observe.action.configuration.tip=Modificar la configuración observe.action.configure=Configurar diff --git a/observe-application-swing/src/main/resources/i18n/observe-application-swing_fr_FR.properties b/observe-application-swing/src/main/resources/i18n/observe-application-swing_fr_FR.properties index 79aedb9..550dd80 100644 --- a/observe-application-swing/src/main/resources/i18n/observe-application-swing_fr_FR.properties +++ b/observe-application-swing/src/main/resources/i18n/observe-application-swing_fr_FR.properties @@ -58,7 +58,8 @@ observe.action.commandline.launch.admin.ui=Lancer une opération d'administratio observe.action.commandline.launch.h2.server.mode=Lancer ObServe en mode serveur observe.action.commandline.launch.obstuna.admin.ui=Lancer une opération d'administration sur Obstuna observe.action.commandline.use.jmx=Ajouter le support JMX -observe.action.config.export.required.read.referentiel= +observe.action.config.export.required.read.referentiel=Lecture sur le référentiel requis pour cette opération +observe.action.config.export.required.write.data=Écriture sur les données requis pour cette opération observe.action.configuration=Configuration observe.action.configuration.tip=Modifier la configuration observe.action.configure=Configurer diff --git a/observe-application-web/src/main/java/fr/ird/observe/application/web/controller/v1/SqlScriptProducerServiceController.java b/observe-application-web/src/main/java/fr/ird/observe/application/web/controller/v1/SqlScriptProducerServiceController.java index 82b13ec..e021e25 100644 --- a/observe-application-web/src/main/java/fr/ird/observe/application/web/controller/v1/SqlScriptProducerServiceController.java +++ b/observe-application-web/src/main/java/fr/ird/observe/application/web/controller/v1/SqlScriptProducerServiceController.java @@ -22,6 +22,7 @@ package fr.ird.observe.application.web.controller.v1; * #L% */ +import fr.ird.observe.services.service.AddSqlScriptProducerResult; import fr.ird.observe.services.service.DeleteSqlScriptProducerRequest; import fr.ird.observe.services.service.AddSqlScriptProducerRequest; import fr.ird.observe.services.service.SqlScriptProducerService; @@ -36,7 +37,7 @@ public class SqlScriptProducerServiceController extends ObserveAuthenticatedServ } @Override - public byte[] produceAddSqlScript(AddSqlScriptProducerRequest request) { + public AddSqlScriptProducerResult produceAddSqlScript(AddSqlScriptProducerRequest request) { return service.produceAddSqlScript(request); } diff --git a/observe-application-web/src/main/java/fr/ird/observe/application/web/controller/v1/actions/export/ReplicateTripServiceController.java b/observe-application-web/src/main/java/fr/ird/observe/application/web/controller/v1/trip/ExportTripServiceController.java similarity index 53% rename from observe-application-web/src/main/java/fr/ird/observe/application/web/controller/v1/actions/export/ReplicateTripServiceController.java rename to observe-application-web/src/main/java/fr/ird/observe/application/web/controller/v1/trip/ExportTripServiceController.java index faf2ec3..9f38e15 100644 --- a/observe-application-web/src/main/java/fr/ird/observe/application/web/controller/v1/actions/export/ReplicateTripServiceController.java +++ b/observe-application-web/src/main/java/fr/ird/observe/application/web/controller/v1/trip/ExportTripServiceController.java @@ -1,4 +1,4 @@ -package fr.ird.observe.application.web.controller.v1.actions.export; +package fr.ird.observe.application.web.controller.v1.trip; /*- * #%L @@ -23,30 +23,25 @@ package fr.ird.observe.application.web.controller.v1.actions.export; */ import fr.ird.observe.application.web.controller.v1.ObserveAuthenticatedServiceControllerSupport; -import fr.ird.observe.services.service.actions.export.ReplicateTripService; -import fr.ird.observe.services.service.actions.export.ReplicateTripsExportRequest; -import fr.ird.observe.services.service.actions.export.ReplicateTripsExportResult; -import fr.ird.observe.services.service.actions.export.ReplicateTripsImportResult; +import fr.ird.observe.services.service.trip.ExportTripService; +import fr.ird.observe.services.service.trip.ExportTripsRequest; +import fr.ird.observe.services.service.trip.ExportTripsResult; /** * Created on 20/07/16. * * @author Tony Chemit - chemit@codelutin.com + * @since 5.0 */ -public class ReplicateTripServiceController extends ObserveAuthenticatedServiceControllerSupport<ReplicateTripService> implements ReplicateTripService { +public class ExportTripServiceController extends ObserveAuthenticatedServiceControllerSupport<ExportTripService> implements ExportTripService { - public ReplicateTripServiceController() { - super(ReplicateTripService.class); + public ExportTripServiceController() { + super(ExportTripService.class); } @Override - public ReplicateTripsExportResult exportTrips(ReplicateTripsExportRequest exportRequest) { + public ExportTripsResult exportTrips(ExportTripsRequest exportRequest) { return getAuthenticatedService().exportTrips(exportRequest); } - @Override - public ReplicateTripsImportResult importTrips(ReplicateTripsExportResult importRequest) { - return getAuthenticatedService().importTrips(importRequest); - } - } diff --git a/observe-services-api/src/main/java/fr/ird/observe/services/service/actions/export/ReplicateTripService.java b/observe-application-web/src/main/java/fr/ird/observe/application/web/controller/v1/trip/ImportTripServiceController.java similarity index 53% copy from observe-services-api/src/main/java/fr/ird/observe/services/service/actions/export/ReplicateTripService.java copy to observe-application-web/src/main/java/fr/ird/observe/application/web/controller/v1/trip/ImportTripServiceController.java index ae681de..d76c456 100644 --- a/observe-services-api/src/main/java/fr/ird/observe/services/service/actions/export/ReplicateTripService.java +++ b/observe-application-web/src/main/java/fr/ird/observe/application/web/controller/v1/trip/ImportTripServiceController.java @@ -1,8 +1,8 @@ -package fr.ird.observe.services.service.actions.export; +package fr.ird.observe.application.web.controller.v1.trip; /*- * #%L - * ObServe :: Services API + * ObServe :: Application Web * %% * Copyright (C) 2008 - 2016 IRD, Codelutin, Tony Chemit * %% @@ -22,11 +22,10 @@ package fr.ird.observe.services.service.actions.export; * #L% */ -import fr.ird.observe.services.ObserveService; -import fr.ird.observe.services.spi.PostRequest; -import fr.ird.observe.services.spi.ReadDataPermission; -import fr.ird.observe.services.spi.ReadReferentialPermission; -import fr.ird.observe.services.spi.WriteDataPermission; +import fr.ird.observe.application.web.controller.v1.ObserveAuthenticatedServiceControllerSupport; +import fr.ird.observe.services.service.trip.ImportTripService; +import fr.ird.observe.services.service.trip.ImportTripsRequest; +import fr.ird.observe.services.service.trip.ImportTripsResult; /** * Created on 20/07/16. @@ -34,16 +33,15 @@ import fr.ird.observe.services.spi.WriteDataPermission; * @author Tony Chemit - chemit@codelutin.com * @since 5.0 */ -public interface ReplicateTripService extends ObserveService { +public class ImportTripServiceController extends ObserveAuthenticatedServiceControllerSupport<ImportTripService> implements ImportTripService { - @ReadDataPermission - @ReadReferentialPermission - @PostRequest - ReplicateTripsExportResult exportTrips(ReplicateTripsExportRequest exportRequest); + public ImportTripServiceController() { + super(ImportTripService.class); + } - @WriteDataPermission - @ReadReferentialPermission - @PostRequest - ReplicateTripsImportResult importTrips(ReplicateTripsExportResult importRequest); + @Override + public ImportTripsResult importTrips(ImportTripsRequest importRequest) { + return service.importTrips(importRequest); + } } diff --git a/observe-entities/src/main/java/fr/ird/observe/ObserveTopiaPersistenceContext.java b/observe-entities/src/main/java/fr/ird/observe/ObserveTopiaPersistenceContext.java index e1cc150..2977b3b 100644 --- a/observe-entities/src/main/java/fr/ird/observe/ObserveTopiaPersistenceContext.java +++ b/observe-entities/src/main/java/fr/ird/observe/ObserveTopiaPersistenceContext.java @@ -32,7 +32,6 @@ import org.nuiton.topia.persistence.TopiaDao; import org.nuiton.topia.persistence.internal.AbstractTopiaPersistenceContextConstructorParameter; import org.nuiton.topia.persistence.support.TopiaSqlQuery; -import java.io.IOException; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -146,7 +145,7 @@ public class ObserveTopiaPersistenceContext extends AbstractObserveTopiaPersiste } } - public void executeSqlScript(byte... content) throws IOException { + public void executeSqlScript(byte... content) { getSqlSupport().doSqlWork(new RunScriptTopiaSqlWork(1000, content)); } diff --git a/observe-services-api/src/main/java/fr/ird/observe/services/ObserveServicesProvider.java b/observe-services-api/src/main/java/fr/ird/observe/services/ObserveServicesProvider.java index 28d28e2..888252c 100644 --- a/observe-services-api/src/main/java/fr/ird/observe/services/ObserveServicesProvider.java +++ b/observe-services-api/src/main/java/fr/ird/observe/services/ObserveServicesProvider.java @@ -5,7 +5,8 @@ import fr.ird.observe.services.service.PingService; import fr.ird.observe.services.service.ReferentialService; import fr.ird.observe.services.service.SqlScriptProducerService; import fr.ird.observe.services.service.actions.consolidate.ConsolidateDataService; -import fr.ird.observe.services.service.actions.export.ReplicateTripService; +import fr.ird.observe.services.service.trip.ExportTripService; +import fr.ird.observe.services.service.trip.ImportTripService; import fr.ird.observe.services.service.actions.report.ReportService; import fr.ird.observe.services.service.actions.synchro.data.DataSynchronizeService; import fr.ird.observe.services.service.actions.synchro.referential.diff.ReferentialSynchronizeDiffService; @@ -57,7 +58,9 @@ public interface ObserveServicesProvider { PingService newPingService(); - ReplicateTripService newReplicateTripService(); + ImportTripService newImportTripService(); + + ExportTripService newExportTripService(); ConsolidateDataService newConsolidateDataService(); diff --git a/observe-services-api/src/main/java/fr/ird/observe/services/service/AddSqlScriptProducerResult.java b/observe-services-api/src/main/java/fr/ird/observe/services/service/AddSqlScriptProducerResult.java new file mode 100644 index 0000000..29f4782 --- /dev/null +++ b/observe-services-api/src/main/java/fr/ird/observe/services/service/AddSqlScriptProducerResult.java @@ -0,0 +1,27 @@ +package fr.ird.observe.services.service; + +import com.google.common.collect.ImmutableSet; + +/** + * Created on 24/08/16. + * + * @author Tony Chemit - chemit@codelutin.com + */ +public class AddSqlScriptProducerResult { + + private final byte[] sqlCode; + + private final ImmutableSet<ObserveBlobsContainer> blobsContainers; + public AddSqlScriptProducerResult(byte[] sqlCode, ImmutableSet<ObserveBlobsContainer> blobsContainers) { + this.sqlCode = sqlCode; + this.blobsContainers = blobsContainers; + } + + public byte[] getSqlCode() { + return sqlCode; + } + + public ImmutableSet<ObserveBlobsContainer> getBlobsContainers() { + return blobsContainers; + } +} diff --git a/observe-services-api/src/main/java/fr/ird/observe/services/service/ObserveBlobsContainer.java b/observe-services-api/src/main/java/fr/ird/observe/services/service/ObserveBlobsContainer.java new file mode 100644 index 0000000..6834e03 --- /dev/null +++ b/observe-services-api/src/main/java/fr/ird/observe/services/service/ObserveBlobsContainer.java @@ -0,0 +1,37 @@ +package fr.ird.observe.services.service; + +import com.google.common.collect.ImmutableMap; + +/** + * Pour contenir des blobs pour une colonne de type blob sur une table donnée. + * + * Created on 24/08/16. + * + * @author Tony Chemit - chemit@codelutin.com + * @since 5.0 + */ +public class ObserveBlobsContainer { + + private final String tableName; + private final String columnName; + private final ImmutableMap<String, byte[]> blobsById; + + public ObserveBlobsContainer(String tableName, String columnName, ImmutableMap<String, byte[]> blobsById) { + this.tableName = tableName; + this.columnName = columnName; + this.blobsById = blobsById; + } + + public String getTableName() { + return tableName; + } + + public String getColumnName() { + return columnName; + } + + public ImmutableMap<String, byte[]> getBlobsById() { + return blobsById; + } + +} diff --git a/observe-services-api/src/main/java/fr/ird/observe/services/service/SqlScriptProducerService.java b/observe-services-api/src/main/java/fr/ird/observe/services/service/SqlScriptProducerService.java index 50ce8b9..73762f3 100644 --- a/observe-services-api/src/main/java/fr/ird/observe/services/service/SqlScriptProducerService.java +++ b/observe-services-api/src/main/java/fr/ird/observe/services/service/SqlScriptProducerService.java @@ -31,7 +31,7 @@ import fr.ird.observe.services.ObserveService; */ public interface SqlScriptProducerService extends ObserveService { - byte[] produceAddSqlScript(AddSqlScriptProducerRequest request); + AddSqlScriptProducerResult produceAddSqlScript(AddSqlScriptProducerRequest request); byte[] produceDeleteSqlScript(DeleteSqlScriptProducerRequest request); } diff --git a/observe-services-api/src/main/java/fr/ird/observe/services/service/actions/export/ReplicateTripsExportResult.java b/observe-services-api/src/main/java/fr/ird/observe/services/service/actions/export/ReplicateTripsExportResult.java deleted file mode 100644 index 1502c16..0000000 --- a/observe-services-api/src/main/java/fr/ird/observe/services/service/actions/export/ReplicateTripsExportResult.java +++ /dev/null @@ -1,100 +0,0 @@ -package fr.ird.observe.services.service.actions.export; - -/*- - * #%L - * ObServe :: Services API - * %% - * Copyright (C) 2008 - 2016 IRD, Codelutin, Tony Chemit - * %% - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public - * License along with this program. If not, see - * <http://www.gnu.org/licenses/gpl-3.0.html>. - * #L% - */ - -import com.google.common.collect.ImmutableSet; - -import java.util.Iterator; -import java.util.Objects; - -/** - * Created on 20/07/16. - * - * @author Tony Chemit - chemit@codelutin.com - */ -public class ReplicateTripsExportResult implements Iterable<ReplicateTripsExportResult.ReplicateTripExportResult> { - - public static ExportTripsRequestBuilder builder() { - return new ExportTripsRequestBuilder(); - } - - public static class ExportTripsRequestBuilder { - - private final ImmutableSet.Builder<ReplicateTripExportResult> trips = ImmutableSet.builder(); - - public ExportTripsRequestBuilder addTripToReplicate(ReplicateTripsExportRequest.ReplicateTripExportRequest request, byte[] sqlContent) { - trips.add(new ReplicateTripExportResult(request, sqlContent)); - return this; - } - - public ReplicateTripsExportResult build() { - return new ReplicateTripsExportResult(trips.build()); - } - - } - - public static class ReplicateTripExportResult { - - private final ReplicateTripsExportRequest.ReplicateTripExportRequest request; - - private final byte[] sqlContent; - - public ReplicateTripExportResult(ReplicateTripsExportRequest.ReplicateTripExportRequest request, byte[] sqlContent) { - this.request = request; - this.sqlContent = sqlContent; - } - - public ReplicateTripsExportRequest.ReplicateTripExportRequest getRequest() { - return request; - } - - public byte[] getSqlContent() { - return sqlContent; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ReplicateTripExportResult that = (ReplicateTripExportResult) o; - return Objects.equals(getRequest(), that.getRequest()); - } - - @Override - public int hashCode() { - return Objects.hash(getRequest()); - } - - } - - private final ImmutableSet<ReplicateTripExportResult> exportedTrips; - - @Override - public Iterator<ReplicateTripExportResult> iterator() { - return exportedTrips.iterator(); - } - - private ReplicateTripsExportResult(ImmutableSet<ReplicateTripExportResult> exportedTrips) { - this.exportedTrips = exportedTrips; - } -} diff --git a/observe-services-api/src/main/java/fr/ird/observe/services/service/actions/export/ReplicateTripService.java b/observe-services-api/src/main/java/fr/ird/observe/services/service/trip/ExportTripService.java similarity index 73% rename from observe-services-api/src/main/java/fr/ird/observe/services/service/actions/export/ReplicateTripService.java rename to observe-services-api/src/main/java/fr/ird/observe/services/service/trip/ExportTripService.java index ae681de..3236cca 100644 --- a/observe-services-api/src/main/java/fr/ird/observe/services/service/actions/export/ReplicateTripService.java +++ b/observe-services-api/src/main/java/fr/ird/observe/services/service/trip/ExportTripService.java @@ -1,4 +1,4 @@ -package fr.ird.observe.services.service.actions.export; +package fr.ird.observe.services.service.trip; /*- * #%L @@ -26,7 +26,6 @@ import fr.ird.observe.services.ObserveService; import fr.ird.observe.services.spi.PostRequest; import fr.ird.observe.services.spi.ReadDataPermission; import fr.ird.observe.services.spi.ReadReferentialPermission; -import fr.ird.observe.services.spi.WriteDataPermission; /** * Created on 20/07/16. @@ -34,16 +33,11 @@ import fr.ird.observe.services.spi.WriteDataPermission; * @author Tony Chemit - chemit@codelutin.com * @since 5.0 */ -public interface ReplicateTripService extends ObserveService { +public interface ExportTripService extends ObserveService { @ReadDataPermission @ReadReferentialPermission @PostRequest - ReplicateTripsExportResult exportTrips(ReplicateTripsExportRequest exportRequest); - - @WriteDataPermission - @ReadReferentialPermission - @PostRequest - ReplicateTripsImportResult importTrips(ReplicateTripsExportResult importRequest); + ExportTripsResult exportTrips(ExportTripsRequest exportRequest); } diff --git a/observe-services-api/src/main/java/fr/ird/observe/services/service/actions/export/ReplicateTripsExportRequest.java b/observe-services-api/src/main/java/fr/ird/observe/services/service/trip/ExportTripsRequest.java similarity index 54% copy from observe-services-api/src/main/java/fr/ird/observe/services/service/actions/export/ReplicateTripsExportRequest.java copy to observe-services-api/src/main/java/fr/ird/observe/services/service/trip/ExportTripsRequest.java index 2ec4413..6bfa26e 100644 --- a/observe-services-api/src/main/java/fr/ird/observe/services/service/actions/export/ReplicateTripsExportRequest.java +++ b/observe-services-api/src/main/java/fr/ird/observe/services/service/trip/ExportTripsRequest.java @@ -1,4 +1,4 @@ -package fr.ird.observe.services.service.actions.export; +package fr.ird.observe.services.service.trip; /*- * #%L @@ -32,51 +32,61 @@ import java.util.Objects; * * @author Tony Chemit - chemit@codelutin.com */ -public class ReplicateTripsExportRequest implements Iterable<ReplicateTripsExportRequest.ReplicateTripExportRequest> { +public class ExportTripsRequest implements Iterable<ExportTripsRequest.ExportTripRequest> { - public static ExportTripsRequestBuilder builder() { - return new ExportTripsRequestBuilder(); + public static Builder builder(boolean forPg) { + return new Builder(forPg); } - public static class ExportTripsRequestBuilder { + public static class Builder { - private final ImmutableSet.Builder<ReplicateTripExportRequest> trips = ImmutableSet.builder(); + private final ImmutableSet.Builder<ExportTripRequest> trips = ImmutableSet.builder(); - public ExportTripsRequestBuilder addTripToReplicate(String tripId, boolean deleteBeofreImport) { - trips.add(new ReplicateTripExportRequest(tripId, deleteBeofreImport)); + private final boolean forPg; + + public Builder(boolean forPg) { + this.forPg = forPg; + } + + public Builder addTripToExport(String tripId) { + trips.add(new ExportTripRequest(tripId)); return this; } - public ReplicateTripsExportRequest build() { - return new ReplicateTripsExportRequest(trips.build()); + public ExportTripsRequest build() { + return new ExportTripsRequest(trips.build()); } } - public static class ReplicateTripExportRequest { + private final ImmutableSet<ExportTripRequest> tripsToExport; - private final String tripId; + @Override + public Iterator<ExportTripRequest> iterator() { + return tripsToExport.iterator(); + } + + private ExportTripsRequest(ImmutableSet<ExportTripRequest> tripsToExport) { + this.tripsToExport = tripsToExport; + } + + public static class ExportTripRequest { - private final boolean deleteBeforeImport; + private final String tripId; - public ReplicateTripExportRequest(String tripId, boolean deleteBeforeImport) { + public ExportTripRequest(String tripId) { this.tripId = tripId; - this.deleteBeforeImport = deleteBeforeImport; } public String getTripId() { return tripId; } - public boolean isDeleteBeforeImport() { - return deleteBeforeImport; - } - @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - ReplicateTripExportRequest that = (ReplicateTripExportRequest) o; + ExportTripRequest that = (ExportTripRequest) o; return Objects.equals(getTripId(), that.getTripId()); } @@ -85,15 +95,4 @@ public class ReplicateTripsExportRequest implements Iterable<ReplicateTripsExpor return Objects.hash(getTripId()); } } - - private final ImmutableSet<ReplicateTripExportRequest> tripsToExport; - - @Override - public Iterator<ReplicateTripExportRequest> iterator() { - return tripsToExport.iterator(); - } - - private ReplicateTripsExportRequest(ImmutableSet<ReplicateTripExportRequest> tripsToExport) { - this.tripsToExport = tripsToExport; - } } diff --git a/observe-services-api/src/main/java/fr/ird/observe/services/service/trip/ExportTripsResult.java b/observe-services-api/src/main/java/fr/ird/observe/services/service/trip/ExportTripsResult.java new file mode 100644 index 0000000..b0995c3 --- /dev/null +++ b/observe-services-api/src/main/java/fr/ird/observe/services/service/trip/ExportTripsResult.java @@ -0,0 +1,121 @@ +package fr.ird.observe.services.service.trip; + +/*- + * #%L + * ObServe :: Services API + * %% + * Copyright (C) 2008 - 2016 IRD, Codelutin, Tony Chemit + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/gpl-3.0.html>. + * #L% + */ + +import com.google.common.collect.ImmutableSet; +import fr.ird.observe.services.service.ObserveBlobsContainer; + +import java.util.Iterator; +import java.util.Objects; + +/** + * Created on 20/07/16. + * + * @author Tony Chemit - chemit@codelutin.com + */ +public class ExportTripsResult implements Iterable<ExportTripsResult.ExportTripResult> { + + public static ExportTripsRequestBuilder builder() { + return new ExportTripsRequestBuilder(); + } + + public static class ExportTripsRequestBuilder { + + private final ImmutableSet.Builder<ExportTripResult> trips = ImmutableSet.builder(); + + public ExportTripsRequestBuilder addTripToReplicate(ExportTripsRequest.ExportTripRequest request, + byte[] sqlContent, + ImmutableSet<ObserveBlobsContainer> blobsContainers, + long time) { + trips.add(new ExportTripResult(request, sqlContent, blobsContainers, time)); + return this; + } + + public ExportTripsResult build() { + return new ExportTripsResult(trips.build()); + } + + } + + public static class ExportTripResult { + + private final ExportTripsRequest.ExportTripRequest request; + private final byte[] sqlContent; + private final ImmutableSet<ObserveBlobsContainer> blobsContainers; + private final long time; + + public ExportTripResult(ExportTripsRequest.ExportTripRequest request, + byte[] sqlContent, + ImmutableSet<ObserveBlobsContainer> blobsContainers, + long time) { + this.request = request; + this.sqlContent = sqlContent; + this.blobsContainers = blobsContainers; + this.time = time; + } + + public ExportTripsRequest.ExportTripRequest getRequest() { + return request; + } + + public byte[] getSqlContent() { + return sqlContent; + } + + public ImmutableSet<ObserveBlobsContainer> getBlobsContainers() { + return blobsContainers; + } + + public long getTime() { + return time; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ExportTripResult that = (ExportTripResult) o; + return Objects.equals(getRequest(), that.getRequest()); + } + + @Override + public int hashCode() { + return Objects.hash(getRequest()); + } + + public ImportTripsRequest.ImportTripRequest toImportRequest() { + return new ImportTripsRequest.ImportTripRequest(request.getTripId(), sqlContent, blobsContainers); + } + } + + private final ImmutableSet<ExportTripResult> exportedTrips; + + @Override + public Iterator<ExportTripResult> iterator() { + return exportedTrips.iterator(); + } + + private ExportTripsResult(ImmutableSet<ExportTripResult> exportedTrips) { + this.exportedTrips = exportedTrips; + } +} diff --git a/observe-services-api/src/main/java/fr/ird/observe/services/service/SqlScriptProducerService.java b/observe-services-api/src/main/java/fr/ird/observe/services/service/trip/ImportTripService.java similarity index 68% copy from observe-services-api/src/main/java/fr/ird/observe/services/service/SqlScriptProducerService.java copy to observe-services-api/src/main/java/fr/ird/observe/services/service/trip/ImportTripService.java index 50ce8b9..5e4594a 100644 --- a/observe-services-api/src/main/java/fr/ird/observe/services/service/SqlScriptProducerService.java +++ b/observe-services-api/src/main/java/fr/ird/observe/services/service/trip/ImportTripService.java @@ -1,4 +1,4 @@ -package fr.ird.observe.services.service; +package fr.ird.observe.services.service.trip; /*- * #%L @@ -23,15 +23,21 @@ package fr.ird.observe.services.service; */ import fr.ird.observe.services.ObserveService; +import fr.ird.observe.services.spi.PostRequest; +import fr.ird.observe.services.spi.Write; +import fr.ird.observe.services.spi.WriteDataPermission; /** - * Created on 31/12/15. + * Created on 20/07/16. * * @author Tony Chemit - chemit@codelutin.com + * @since 5.0 */ -public interface SqlScriptProducerService extends ObserveService { +public interface ImportTripService extends ObserveService { - byte[] produceAddSqlScript(AddSqlScriptProducerRequest request); + @WriteDataPermission + @PostRequest + @Write + ImportTripsResult importTrips(ImportTripsRequest importRequest); - byte[] produceDeleteSqlScript(DeleteSqlScriptProducerRequest request); } diff --git a/observe-services-api/src/main/java/fr/ird/observe/services/service/actions/export/ReplicateTripsExportRequest.java b/observe-services-api/src/main/java/fr/ird/observe/services/service/trip/ImportTripsRequest.java similarity index 51% rename from observe-services-api/src/main/java/fr/ird/observe/services/service/actions/export/ReplicateTripsExportRequest.java rename to observe-services-api/src/main/java/fr/ird/observe/services/service/trip/ImportTripsRequest.java index 2ec4413..8671bd2 100644 --- a/observe-services-api/src/main/java/fr/ird/observe/services/service/actions/export/ReplicateTripsExportRequest.java +++ b/observe-services-api/src/main/java/fr/ird/observe/services/service/trip/ImportTripsRequest.java @@ -1,4 +1,4 @@ -package fr.ird.observe.services.service.actions.export; +package fr.ird.observe.services.service.trip; /*- * #%L @@ -23,6 +23,7 @@ package fr.ird.observe.services.service.actions.export; */ import com.google.common.collect.ImmutableSet; +import fr.ird.observe.services.service.ObserveBlobsContainer; import java.util.Iterator; import java.util.Objects; @@ -32,51 +33,56 @@ import java.util.Objects; * * @author Tony Chemit - chemit@codelutin.com */ -public class ReplicateTripsExportRequest implements Iterable<ReplicateTripsExportRequest.ReplicateTripExportRequest> { +public class ImportTripsRequest implements Iterable<ImportTripsRequest.ImportTripRequest> { - public static ExportTripsRequestBuilder builder() { - return new ExportTripsRequestBuilder(); + public static Builder builder() { + return new Builder(); } - public static class ExportTripsRequestBuilder { + public static class Builder { - private final ImmutableSet.Builder<ReplicateTripExportRequest> trips = ImmutableSet.builder(); + private final ImmutableSet.Builder<ImportTripRequest> tripIds = ImmutableSet.builder(); - public ExportTripsRequestBuilder addTripToReplicate(String tripId, boolean deleteBeofreImport) { - trips.add(new ReplicateTripExportRequest(tripId, deleteBeofreImport)); + public Builder addTripToImport(ImportTripRequest importTripRequest) { + tripIds.add(importTripRequest); return this; } - public ReplicateTripsExportRequest build() { - return new ReplicateTripsExportRequest(trips.build()); + public ImportTripsRequest build() { + return new ImportTripsRequest(tripIds.build()); } } - public static class ReplicateTripExportRequest { + public static class ImportTripRequest { private final String tripId; + private final byte[] sqlContent; + private final ImmutableSet<ObserveBlobsContainer> blobsContainers; - private final boolean deleteBeforeImport; - - public ReplicateTripExportRequest(String tripId, boolean deleteBeforeImport) { + protected ImportTripRequest(String tripId, byte[] sqlContent, ImmutableSet<ObserveBlobsContainer> blobsContainers) { this.tripId = tripId; - this.deleteBeforeImport = deleteBeforeImport; + this.sqlContent = sqlContent; + this.blobsContainers = blobsContainers; } public String getTripId() { return tripId; } - public boolean isDeleteBeforeImport() { - return deleteBeforeImport; + public byte[] getSqlContent() { + return sqlContent; + } + + public ImmutableSet<ObserveBlobsContainer> getBlobsContainers() { + return blobsContainers; } @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - ReplicateTripExportRequest that = (ReplicateTripExportRequest) o; + ImportTripRequest that = (ImportTripRequest) o; return Objects.equals(getTripId(), that.getTripId()); } @@ -86,14 +92,14 @@ public class ReplicateTripsExportRequest implements Iterable<ReplicateTripsExpor } } - private final ImmutableSet<ReplicateTripExportRequest> tripsToExport; + private final ImmutableSet<ImportTripRequest> tripIds; @Override - public Iterator<ReplicateTripExportRequest> iterator() { - return tripsToExport.iterator(); + public Iterator<ImportTripRequest> iterator() { + return tripIds.iterator(); } - private ReplicateTripsExportRequest(ImmutableSet<ReplicateTripExportRequest> tripsToExport) { - this.tripsToExport = tripsToExport; + private ImportTripsRequest(ImmutableSet<ImportTripRequest> tripIds) { + this.tripIds = tripIds; } } diff --git a/observe-services-api/src/main/java/fr/ird/observe/services/service/actions/export/ReplicateTripsImportResult.java b/observe-services-api/src/main/java/fr/ird/observe/services/service/trip/ImportTripsResult.java similarity index 58% rename from observe-services-api/src/main/java/fr/ird/observe/services/service/actions/export/ReplicateTripsImportResult.java rename to observe-services-api/src/main/java/fr/ird/observe/services/service/trip/ImportTripsResult.java index 7ff8af6..4bbe01d 100644 --- a/observe-services-api/src/main/java/fr/ird/observe/services/service/actions/export/ReplicateTripsImportResult.java +++ b/observe-services-api/src/main/java/fr/ird/observe/services/service/trip/ImportTripsResult.java @@ -1,4 +1,4 @@ -package fr.ird.observe.services.service.actions.export; +package fr.ird.observe.services.service.trip; /*- * #%L @@ -34,37 +34,40 @@ import java.util.Iterator; * @author Tony Chemit - chemit@codelutin.com * @since 5.0 */ -public class ReplicateTripsImportResult implements Iterable<ReplicateTripsImportResult.ReplicateTripImportResult> { +public class ImportTripsResult implements Iterable<ImportTripsResult.ImportTripResult> { - public static ExportTripsResultBuilder builder() { - return new ExportTripsResultBuilder(); + public static Builder builder() { + return new Builder(); } - public static class ExportTripsResultBuilder { + public static class Builder { - private final ImmutableSet.Builder<ReplicateTripImportResult> trips = ImmutableSet.builder(); + private final ImmutableSet.Builder<ImportTripResult> trips = ImmutableSet.builder(); - public ExportTripsResultBuilder addTripResult(String tripId, boolean deleted) { - trips.add(new ReplicateTripImportResult(tripId, deleted)); + public Builder addTripResult(String tripId, boolean deleted, long time) { + trips.add(new ImportTripResult(tripId, deleted, time)); return this; } - public ReplicateTripsImportResult build() { + public ImportTripsResult build() { - return new ReplicateTripsImportResult(trips.build()); + return new ImportTripsResult(trips.build()); } } - public static class ReplicateTripImportResult { + public static class ImportTripResult { private final String tripId; private final boolean deleted; - public ReplicateTripImportResult(String tripId, boolean deleted) { + private final long time; + + public ImportTripResult(String tripId, boolean deleted, long time) { this.tripId = tripId; this.deleted = deleted; + this.time = time; } public String getTripId() { @@ -77,14 +80,14 @@ public class ReplicateTripsImportResult implements Iterable<ReplicateTripsImport } - private final ImmutableSet<ReplicateTripImportResult> trips; + private final ImmutableSet<ImportTripResult> trips; @Override - public Iterator<ReplicateTripImportResult> iterator() { + public Iterator<ImportTripResult> iterator() { return trips.iterator(); } - private ReplicateTripsImportResult(ImmutableSet<ReplicateTripImportResult> trips) { + private ImportTripsResult(ImmutableSet<ImportTripResult> trips) { this.trips = trips; } } diff --git a/observe-services-topia/src/main/java/fr/ird/observe/services/service/DataSourceServiceTopia.java b/observe-services-topia/src/main/java/fr/ird/observe/services/service/DataSourceServiceTopia.java index 39a989c..659df69 100644 --- a/observe-services-topia/src/main/java/fr/ird/observe/services/service/DataSourceServiceTopia.java +++ b/observe-services-topia/src/main/java/fr/ird/observe/services/service/DataSourceServiceTopia.java @@ -216,7 +216,7 @@ public class DataSourceServiceTopia extends ObserveServiceTopia implements DataS SqlScriptProducerService dumpProducerService = serviceContext.newService(temporaryConfiguration, SqlScriptProducerService.class); AddSqlScriptProducerRequest request = AddSqlScriptProducerRequest.forPostgres().addSchema().addReferential().addAllData(); - byte[] dump = dumpProducerService.produceAddSqlScript(request); + byte[] dump = dumpProducerService.produceAddSqlScript(request).getSqlCode(); topiaApplicationContext.executeSqlStatements(dump); topiaApplicationContext.getMigrationService().createSchemaIfNotExist(); topiaApplicationContext.getMigrationService().runSchemaMigration(); @@ -281,7 +281,7 @@ public class DataSourceServiceTopia extends ObserveServiceTopia implements DataS ObserveDataSourceConnection importDataSourceConnection = dataSourceService.open(importDataSourceConfiguration); SqlScriptProducerService dumpProducerService = serviceContext.newService(importDataSourceConnection, SqlScriptProducerService.class); - referentialDump = dumpProducerService.produceAddSqlScript(request); + referentialDump = dumpProducerService.produceAddSqlScript(request).getSqlCode(); } finally { dataSourceService.close(); @@ -323,18 +323,14 @@ public class DataSourceServiceTopia extends ObserveServiceTopia implements DataS request.dataIdsToAdd(importDataIds); - DataSourceService dataSourceService = serviceContext.newService(importDataSourceConfiguration, DataSourceService.class); byte[] dataDump; - - try { + try (DataSourceService dataSourceService = serviceContext.newService(importDataSourceConfiguration, DataSourceService.class)) { ObserveDataSourceConnection importDataSourceConnection = dataSourceService.open(importDataSourceConfiguration); SqlScriptProducerService dumpProducerService = serviceContext.newService(importDataSourceConnection, SqlScriptProducerService.class); - dataDump = dumpProducerService.produceAddSqlScript(request); + dataDump = dumpProducerService.produceAddSqlScript(request).getSqlCode(); - } finally { - dataSourceService.close(); } if (log.isInfoEnabled()) { diff --git a/observe-services-topia/src/main/java/fr/ird/observe/services/service/SqlScriptProducerServiceTopia.java b/observe-services-topia/src/main/java/fr/ird/observe/services/service/SqlScriptProducerServiceTopia.java index 2d13358..8b9701e 100644 --- a/observe-services-topia/src/main/java/fr/ird/observe/services/service/SqlScriptProducerServiceTopia.java +++ b/observe-services-topia/src/main/java/fr/ird/observe/services/service/SqlScriptProducerServiceTopia.java @@ -29,7 +29,9 @@ import fr.ird.observe.services.ObserveServiceTopia; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.topia.service.sql.batch.SqlRequests; +import org.nuiton.topia.service.sql.batch.SqlResult; import org.nuiton.topia.service.sql.batch.TopiaSqlBatchService; +import org.nuiton.topia.service.sql.batch.actions.BlobsContainer; import org.nuiton.topia.service.sql.batch.actions.TopiaSqlTableSelectArgument; import java.io.ByteArrayOutputStream; @@ -53,23 +55,30 @@ public class SqlScriptProducerServiceTopia extends ObserveServiceTopia implement private static final Log log = LogFactory.getLog(SqlScriptProducerServiceTopia.class); @Override - public byte[] produceAddSqlScript(AddSqlScriptProducerRequest request) { + public AddSqlScriptProducerResult produceAddSqlScript(AddSqlScriptProducerRequest request) { if (log.isTraceEnabled()) { log.trace("produceAddSqlScript(" + request + ")"); } try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { + ImmutableSet.Builder<ObserveBlobsContainer> observeBlobContainers = ImmutableSet.builder(); try (Writer writer = new OutputStreamWriter(new GZIPOutputStream(out))) { TopiaSqlBatchService sqlBatchService = serviceContext.getTopiaApplicationContext().getSqlBatchService(); SqlRequests.Builder builder = sqlBatchService.requestBuilder().to(writer); SqlRequests sqlRequests = sqlRequests(request, builder); - sqlBatchService.execute(sqlRequests); + SqlResult sqlResult = sqlBatchService.execute(sqlRequests); + ImmutableSet<BlobsContainer> blobsContainers = sqlResult.getBlobsContainers(); + for (BlobsContainer blobsContainer : blobsContainers) { + observeBlobContainers.add(new ObserveBlobsContainer(blobsContainer.getTableName(), + blobsContainer.getColumnName(), + blobsContainer.getBlobsById())); + } } - return out.toByteArray(); + return new AddSqlScriptProducerResult(out.toByteArray(), observeBlobContainers.build()); } catch (IOException e) { throw new RuntimeException("Could not produce sql script", e); diff --git a/observe-services-topia/src/main/java/fr/ird/observe/services/service/actions/export/ReplicateTripServiceTopia.java b/observe-services-topia/src/main/java/fr/ird/observe/services/service/actions/export/ReplicateTripServiceTopia.java deleted file mode 100644 index fbaf1bf..0000000 --- a/observe-services-topia/src/main/java/fr/ird/observe/services/service/actions/export/ReplicateTripServiceTopia.java +++ /dev/null @@ -1,115 +0,0 @@ -package fr.ird.observe.services.service.actions.export; - -/*- - * #%L - * ObServe :: Services ToPIA Implementation - * %% - * Copyright (C) 2008 - 2016 IRD, Codelutin, Tony Chemit - * %% - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public - * License along with this program. If not, see - * <http://www.gnu.org/licenses/gpl-3.0.html>. - * #L% - */ - -import com.google.common.collect.ImmutableSet; -import fr.ird.observe.ObserveTopiaApplicationContext; -import fr.ird.observe.services.ObserveServiceTopia; -import fr.ird.observe.services.dto.IdDtos; -import fr.ird.observe.services.service.AddSqlScriptProducerRequest; -import fr.ird.observe.services.service.SqlScriptProducerService; -import fr.ird.observe.services.service.longline.TripLonglineService; -import fr.ird.observe.services.service.seine.TripSeineService; - -/** - * Created on 20/07/16. - * - * @author Tony Chemit - chemit@codelutin.com - */ -public class ReplicateTripServiceTopia extends ObserveServiceTopia implements ReplicateTripService { - - @Override - public ReplicateTripsExportResult exportTrips(ReplicateTripsExportRequest exportRequest) { - - SqlScriptProducerService sqlScriptProducerService = serviceContext.newService(SqlScriptProducerService.class); - - ReplicateTripsExportResult.ExportTripsRequestBuilder resultBuilder = ReplicateTripsExportResult.builder(); - - for (ReplicateTripsExportRequest.ReplicateTripExportRequest replicateTripExportRequest : exportRequest) { - - String tripId = replicateTripExportRequest.getTripId(); - - AddSqlScriptProducerRequest request = AddSqlScriptProducerRequest.forPostgres().dataIdsToAdd(ImmutableSet.of(tripId)); - - byte[] tripContent = sqlScriptProducerService.produceAddSqlScript(request); - - resultBuilder.addTripToReplicate(replicateTripExportRequest, tripContent); - - } - - return resultBuilder.build(); - - } - - @Override - public ReplicateTripsImportResult importTrips(ReplicateTripsExportResult importTripsRequest) { - - TripSeineService tripSeineService = serviceContext.newService(TripSeineService.class); - TripLonglineService tripLonglineService = serviceContext.newService(TripLonglineService.class); - ObserveTopiaApplicationContext topiaApplicationContext = serviceContext.getTopiaApplicationContext(); - - ReplicateTripsImportResult.ExportTripsResultBuilder resultBuilder = ReplicateTripsImportResult.builder(); - - for (ReplicateTripsExportResult.ReplicateTripExportResult importTripRequest : importTripsRequest) { - - String tripId = importTripRequest.getRequest().getTripId(); - boolean deleteBeforeImport = importTripRequest.getRequest().isDeleteBeforeImport(); - - if (IdDtos.isTripSeineId(tripId)) { - -// deleted = tripSeineService.exists(tripId); - - if (deleteBeforeImport) { - - //FIXME Do not use this, prefer usage of topai SqlBatchService - tripSeineService.delete(tripId); - - } - - - } else if (IdDtos.isTripLonglineId(tripId)) { - -// deleted = tripLonglineService.exists(tripId); - - if (deleteBeforeImport) { - - //FIXME Do not use this, prefer usage of topai SqlBatchService - tripLonglineService.delete(tripId); - - } - - } else { - throw new IllegalStateException("Can't come here!"); - } - - topiaApplicationContext.executeSqlStatements(importTripRequest.getSqlContent()); - - resultBuilder.addTripResult(tripId, deleteBeforeImport); - - } - - return resultBuilder.build(); - - } - -} diff --git a/observe-services-topia/src/main/java/fr/ird/observe/services/service/actions/synchro/data/DataSynchronizeServiceTopia.java b/observe-services-topia/src/main/java/fr/ird/observe/services/service/actions/synchro/data/DataSynchronizeServiceTopia.java index ed2c4b4..a18483a 100644 --- a/observe-services-topia/src/main/java/fr/ird/observe/services/service/actions/synchro/data/DataSynchronizeServiceTopia.java +++ b/observe-services-topia/src/main/java/fr/ird/observe/services/service/actions/synchro/data/DataSynchronizeServiceTopia.java @@ -54,7 +54,7 @@ public class DataSynchronizeServiceTopia extends ObserveServiceTopia implements sqlRequest = AddSqlScriptProducerRequest.forPostgres(); } sqlRequest.dataIdsToAdd(ImmutableSet.of(task.getTripId())); - byte[] sqlScript = sqlScriptProducerService.produceAddSqlScript(sqlRequest); + byte[] sqlScript = sqlScriptProducerService.produceAddSqlScript(sqlRequest).getSqlCode(); sqlCodeBuilder.add(sqlScript); diff --git a/observe-services-topia/src/main/java/fr/ird/observe/services/service/trip/ExportTripServiceTopia.java b/observe-services-topia/src/main/java/fr/ird/observe/services/service/trip/ExportTripServiceTopia.java new file mode 100644 index 0000000..f631993 --- /dev/null +++ b/observe-services-topia/src/main/java/fr/ird/observe/services/service/trip/ExportTripServiceTopia.java @@ -0,0 +1,77 @@ +package fr.ird.observe.services.service.trip; + +/*- + * #%L + * ObServe :: Services ToPIA Implementation + * %% + * Copyright (C) 2008 - 2016 IRD, Codelutin, Tony Chemit + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/gpl-3.0.html>. + * #L% + */ + +import com.google.common.collect.ImmutableSet; +import fr.ird.observe.services.ObserveServiceTopia; +import fr.ird.observe.services.service.AddSqlScriptProducerRequest; +import fr.ird.observe.services.service.AddSqlScriptProducerResult; +import fr.ird.observe.services.service.ObserveBlobsContainer; +import fr.ird.observe.services.service.SqlScriptProducerService; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * Created on 20/07/16. + * + * @author Tony Chemit - chemit@codelutin.com + * @since 5.0 + */ +public class ExportTripServiceTopia extends ObserveServiceTopia implements ExportTripService { + + /** Logger. */ + private static final Log log = LogFactory.getLog(ExportTripServiceTopia.class); + + @Override + public ExportTripsResult exportTrips(ExportTripsRequest exportRequest) { + + SqlScriptProducerService sqlScriptProducerService = serviceContext.newService(SqlScriptProducerService.class); + + ExportTripsResult.ExportTripsRequestBuilder resultBuilder = ExportTripsResult.builder(); + + for (ExportTripsRequest.ExportTripRequest exportTripRequest : exportRequest) { + + long t0 = System.nanoTime(); + + String tripId = exportTripRequest.getTripId(); + + if (log.isInfoEnabled()) { + log.info("Start export of trip: " + tripId); + } + + AddSqlScriptProducerRequest request = AddSqlScriptProducerRequest.forPostgres().dataIdsToAdd(ImmutableSet.of(tripId)); + + AddSqlScriptProducerResult producerResult = sqlScriptProducerService.produceAddSqlScript(request); + byte[] tripContent = producerResult.getSqlCode(); + ImmutableSet<ObserveBlobsContainer> blobsContainers = producerResult.getBlobsContainers(); + + long time = System.nanoTime() - t0; + resultBuilder.addTripToReplicate(exportTripRequest, tripContent, blobsContainers, time); + + } + + return resultBuilder.build(); + + } + +} diff --git a/observe-services-topia/src/main/java/fr/ird/observe/services/service/trip/ImportTripScriptTopiaSqlWork.java b/observe-services-topia/src/main/java/fr/ird/observe/services/service/trip/ImportTripScriptTopiaSqlWork.java new file mode 100644 index 0000000..befce08 --- /dev/null +++ b/observe-services-topia/src/main/java/fr/ird/observe/services/service/trip/ImportTripScriptTopiaSqlWork.java @@ -0,0 +1,68 @@ +package fr.ird.observe.services.service.trip; + +import com.google.common.collect.ImmutableSet; +import fr.ird.observe.RunScriptTopiaSqlWork; +import fr.ird.observe.services.service.ObserveBlobsContainer; + +import javax.sql.rowset.serial.SerialBlob; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.Map; + +/** + * Created on 24/08/16. + * + * @author Tony Chemit - chemit@codelutin.com + */ +public class ImportTripScriptTopiaSqlWork extends RunScriptTopiaSqlWork { + + private final ImmutableSet<ObserveBlobsContainer> blobsContainers; + + public ImportTripScriptTopiaSqlWork(int batchSize, byte[] content, ImmutableSet<ObserveBlobsContainer> blobsContainers) { + super(batchSize, content); + this.blobsContainers = blobsContainers; + } + + @Override + public void execute(Connection connection) throws SQLException { + super.execute(connection); + + for (ObserveBlobsContainer blobsContainer : blobsContainers) { + + String tableName = blobsContainer.getTableName(); + String columnName = blobsContainer.getColumnName(); + int batchSize = 0; + + String sql = String.format("update %s SET %s = ? WHERE topiaId= ?", tableName, columnName); + try (PreparedStatement preparedStatement = connection.prepareStatement(sql)) { + + for (Map.Entry<String, byte[]> containerEntry : blobsContainer.getBlobsById().entrySet()) { + + String topiaId = containerEntry.getKey(); + byte[] content = containerEntry.getValue(); + + preparedStatement.clearParameters(); + preparedStatement.setBlob(1, new SerialBlob(content)); + preparedStatement.setString(2, topiaId); + preparedStatement.addBatch(); + + batchSize++; + + if (batchSize % this.batchSize == 0) { + flushStatement(preparedStatement); + } + + } + + + flushStatement(preparedStatement); + + } + + } + + + } + +} diff --git a/observe-services-topia/src/main/java/fr/ird/observe/services/service/trip/ImportTripServiceTopia.java b/observe-services-topia/src/main/java/fr/ird/observe/services/service/trip/ImportTripServiceTopia.java new file mode 100644 index 0000000..fcb1172 --- /dev/null +++ b/observe-services-topia/src/main/java/fr/ird/observe/services/service/trip/ImportTripServiceTopia.java @@ -0,0 +1,121 @@ +package fr.ird.observe.services.service.trip; + +/*- + * #%L + * ObServe :: Services ToPIA Implementation + * %% + * Copyright (C) 2008 - 2016 IRD, Codelutin, Tony Chemit + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/gpl-3.0.html>. + * #L% + */ + +import com.google.common.collect.ImmutableSet; +import fr.ird.observe.ObserveTopiaPersistenceContext; +import fr.ird.observe.services.ObserveServiceTopia; +import fr.ird.observe.services.dto.IdDtos; +import fr.ird.observe.services.service.ObserveBlobsContainer; +import fr.ird.observe.services.service.longline.TripLonglineService; +import fr.ird.observe.services.service.seine.TripSeineService; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * Created on 20/07/16. + * + * @author Tony Chemit - chemit@codelutin.com + * @since 5.0 + */ +public class ImportTripServiceTopia extends ObserveServiceTopia implements ImportTripService { + + /** Logger. */ + private static final Log log = LogFactory.getLog(ImportTripServiceTopia.class); + + @Override + public ImportTripsResult importTrips(ImportTripsRequest importTripsRequest) { + + TripSeineService tripSeineService = serviceContext.newService(TripSeineService.class); + TripLonglineService tripLonglineService = serviceContext.newService(TripLonglineService.class); + + ImportTripsResult.Builder resultBuilder = ImportTripsResult.builder(); + + try (ObserveTopiaPersistenceContext persistenceContext = serviceContext.getTopiaApplicationContext().newPersistenceContext()) { + + for (ImportTripsRequest.ImportTripRequest importTripRequest : importTripsRequest) { + + String tripId = importTripRequest.getTripId(); + + if (log.isInfoEnabled()) { + log.info("Start import of trip: " + tripId); + } + + long t0 = System.nanoTime(); + + // check if trip exist + + boolean deleteBeforeImport; + + if (IdDtos.isTripSeineId(tripId)) { + + deleteBeforeImport = tripSeineService.exists(tripId); + + if (deleteBeforeImport) { + + //FIXME Do not use this, prefer usage of topai SqlBatchService + tripSeineService.delete(tripId); + + } + + } else if (IdDtos.isTripLonglineId(tripId)) { + + deleteBeforeImport = tripLonglineService.exists(tripId); + + if (deleteBeforeImport) { + + //FIXME Do not use this, prefer usage of topai SqlBatchService + tripLonglineService.delete(tripId); + + } + + } else { + throw new IllegalStateException("Can't come here!"); + } + + ImmutableSet<ObserveBlobsContainer> blobsContainers = importTripRequest.getBlobsContainers(); + + if (blobsContainers.isEmpty()) { + + persistenceContext.executeSqlScript(importTripRequest.getSqlContent()); + + } else { + + persistenceContext.getSqlSupport().doSqlWork(new ImportTripScriptTopiaSqlWork(1000, importTripRequest.getSqlContent(), blobsContainers)); + + } + + long time = System.nanoTime() - t0; + resultBuilder.addTripResult(tripId, deleteBeforeImport, time); + + } + + persistenceContext.commit(); + + } + + return resultBuilder.build(); + + } + +} diff --git a/observe-services-topia/src/test/java/fr/ird/observe/services/service/AddSqlScriptProducerServiceTopiaTest.java b/observe-services-topia/src/test/java/fr/ird/observe/services/service/AddSqlScriptProducerServiceTopiaTest.java index 9d97f2c..cd159ef 100644 --- a/observe-services-topia/src/test/java/fr/ird/observe/services/service/AddSqlScriptProducerServiceTopiaTest.java +++ b/observe-services-topia/src/test/java/fr/ird/observe/services/service/AddSqlScriptProducerServiceTopiaTest.java @@ -92,7 +92,7 @@ public class AddSqlScriptProducerServiceTopiaTest extends AbstractServiceTopiaTe File outputFile = applicationContextResource.createTemporaryFile("-out.sql.gz"); - byte[] gzContent = service.produceAddSqlScript(request); + byte[] gzContent = service.produceAddSqlScript(request).getSqlCode(); try (FileOutputStream fileOutputStream = new FileOutputStream(outputFile)) { IOUtils.write(gzContent, fileOutputStream); diff --git a/observe-services-topia/src/test/java/fr/ird/observe/services/service/MigrateTestsDatabases.java b/observe-services-topia/src/test/java/fr/ird/observe/services/service/MigrateTestsDatabases.java index a9e0b6c..f4e2ec4 100644 --- a/observe-services-topia/src/test/java/fr/ird/observe/services/service/MigrateTestsDatabases.java +++ b/observe-services-topia/src/test/java/fr/ird/observe/services/service/MigrateTestsDatabases.java @@ -103,11 +103,11 @@ public class MigrateTestsDatabases extends AbstractServiceTopiaTest { migrate(DatabaseName.empty_pg.name(), service.produceAddSqlScript(request)); } - protected void migrate(byte[] dump) throws IOException, IncompatibleDataSourceCreateConfigurationException, DataSourceCreateWithNoReferentialImportException, DatabaseNotFoundException, BabModelVersionException, DatabaseConnexionNotAuthorizedException { + protected void migrate(AddSqlScriptProducerResult dump) throws IOException, IncompatibleDataSourceCreateConfigurationException, DataSourceCreateWithNoReferentialImportException, DatabaseNotFoundException, BabModelVersionException, DatabaseConnexionNotAuthorizedException { migrate(dataSourceResource.getDbName(), dump); } - protected void migrate(String dbName, byte[] dump) throws IOException { + protected void migrate(String dbName, AddSqlScriptProducerResult dump) throws IOException { File databasesRootDirectory = dataSourceResource.getDataSourceConfiguration().getDatabaseFile(); @@ -134,7 +134,7 @@ public class MigrateTestsDatabases extends AbstractServiceTopiaTest { Files.createDirectories(scriptfile.toPath().getParent()); - FileUtils.writeByteArrayToFile(scriptfile, dump); + FileUtils.writeByteArrayToFile(scriptfile, dump.getSqlCode()); } diff --git a/observe-topia-extension/src/main/java/org/nuiton/topia/persistence/metadata/TopiaMetadataEntity.java b/observe-topia-extension/src/main/java/org/nuiton/topia/persistence/metadata/TopiaMetadataEntity.java index 06785de..14ddcce 100644 --- a/observe-topia-extension/src/main/java/org/nuiton/topia/persistence/metadata/TopiaMetadataEntity.java +++ b/observe-topia-extension/src/main/java/org/nuiton/topia/persistence/metadata/TopiaMetadataEntity.java @@ -27,6 +27,8 @@ import com.google.common.collect.ImmutableSet; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import java.sql.Blob; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.Map; @@ -80,6 +82,10 @@ public class TopiaMetadataEntity { */ protected final Map<String, String> properties = new LinkedHashMap<>(); /** + * Le dictionnaire des propriétés de type {@link Blob} + */ + protected final Set<String> blobProperties = new HashSet<>(); + /** * Le nom des colunnes correspondants aux propriétés de l'entité. * <b>Note: </b> On ne conserve que les correspondances qui diffèrent du nom de la propriété. * @@ -184,6 +190,13 @@ public class TopiaMetadataEntity { return names; } + public boolean withBlob() { + return !blobProperties.isEmpty(); + } + + public Set<String> getBlobProperties() { + return blobProperties; + } public Map<String, String> getDbColumnsName() { return dbColumnsName; @@ -233,6 +246,9 @@ public class TopiaMetadataEntity { public void addProperty(String name, String type, String dbColumnName) { log.info(getType() + "/" + name + "(" + dbColumnName + ") →" + type); properties.put(name, type); + if (Blob.class.getName().equals(type)) { + blobProperties.add(name); + } addDbColumnName(name, dbColumnName); } diff --git a/observe-topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/SqlResult.java b/observe-topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/SqlResult.java new file mode 100644 index 0000000..6514187 --- /dev/null +++ b/observe-topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/SqlResult.java @@ -0,0 +1,23 @@ +package org.nuiton.topia.service.sql.batch; + +import com.google.common.collect.ImmutableSet; +import org.nuiton.topia.service.sql.batch.actions.BlobsContainer; + +/** + * Created on 24/08/16. + * + * @author Tony Chemit - chemit@codelutin.com + * @since 5.0 + */ +public class SqlResult { + + private final ImmutableSet<BlobsContainer> blobsContainers; + + public SqlResult(ImmutableSet<BlobsContainer> blobsContainers) { + this.blobsContainers = blobsContainers; + } + + public ImmutableSet<BlobsContainer> getBlobsContainers() { + return blobsContainers; + } +} diff --git a/observe-topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/TopiaSqlBatchService.java b/observe-topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/TopiaSqlBatchService.java index 4960a3e..e242429 100644 --- a/observe-topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/TopiaSqlBatchService.java +++ b/observe-topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/TopiaSqlBatchService.java @@ -78,7 +78,7 @@ public interface TopiaSqlBatchService extends TopiaService { * * @param requests the request to execute */ - void execute(SqlRequests requests); + SqlResult execute(SqlRequests requests); /** * Execute a {@link CreateSchemaRequest}. @@ -99,14 +99,14 @@ public interface TopiaSqlBatchService extends TopiaService { * * @param request the request to execute */ - void execute(ReplicateTablesRequest request); + SqlResult execute(ReplicateTablesRequest request); /** * Execute a {@link UpdateTablesRequest}. * * @param request the request to execute */ - void execute(UpdateTablesRequest request); + SqlResult execute(UpdateTablesRequest request); /** * Execute a {@link DeleteTablesRequest}. diff --git a/observe-topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/TopiaSqlBatchServiceImpl.java b/observe-topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/TopiaSqlBatchServiceImpl.java index 00018e9..4762588 100644 --- a/observe-topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/TopiaSqlBatchServiceImpl.java +++ b/observe-topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/TopiaSqlBatchServiceImpl.java @@ -22,12 +22,15 @@ package org.nuiton.topia.service.sql.batch; * #L% */ -import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.nuiton.topia.persistence.TopiaApplicationContext; import org.nuiton.topia.persistence.TopiaException; import org.nuiton.topia.service.sql.batch.actions.AbstractSqlAction; import org.nuiton.topia.service.sql.batch.actions.AbstractSqlRequest; +import org.nuiton.topia.service.sql.batch.actions.BlobsContainer; import org.nuiton.topia.service.sql.batch.actions.CreateSchemaAction; import org.nuiton.topia.service.sql.batch.actions.CreateSchemaRequest; import org.nuiton.topia.service.sql.batch.actions.DeleteTablesAction; @@ -43,6 +46,10 @@ import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.util.Iterator; import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.TreeMap; /** * Created on 04/01/16. @@ -52,6 +59,9 @@ import java.util.Map; */ public class TopiaSqlBatchServiceImpl implements TopiaSqlBatchService { + /** Logger. */ + private static final Log log = LogFactory.getLog(TopiaSqlBatchServiceImpl.class); + private static final ImmutableMap<Class, Class> ACTION_MAPPING = ImmutableMap .<Class, Class>builder() .put(CreateSchemaRequest.class, CreateSchemaAction.class) @@ -138,31 +148,70 @@ public class TopiaSqlBatchServiceImpl implements TopiaSqlBatchService { } @Override - public void execute(SqlRequests requests) { + public SqlResult execute(SqlRequests requests) { + + Map<String, BlobsContainer.Builder> blobContainersBuilder = new TreeMap<>(); Iterator<AbstractSqlRequest> sqlRequestIterator = requests.iterator(); - while (sqlRequestIterator.hasNext()) { + boolean hasNext = sqlRequestIterator.hasNext(); + while (hasNext) { AbstractSqlRequest sqlRequest = sqlRequestIterator.next(); + AbstractSqlAction<?> action = createAction(sqlRequest); action.run(); + Optional<Set<BlobsContainer.Builder>> optionalBlobsContainers = action.getBlobsContainersBuilder(); + if (optionalBlobsContainers.isPresent()) { + Set<BlobsContainer.Builder> blobsContainers = optionalBlobsContainers.get(); + + for (BlobsContainer.Builder blobsContainerBuilder : blobsContainers) { + String tableName = blobsContainerBuilder.getTableName(); + String columnName = blobsContainerBuilder.getColumnName(); + + String key = tableName + "##" + columnName; + if (blobContainersBuilder.containsKey(key)) { + BlobsContainer.Builder blobsContainerBuilder1 = blobContainersBuilder.get(key); + blobsContainerBuilder1.addAllBlob(blobsContainerBuilder.build().getBlobsById()); + } else { + blobContainersBuilder.put(tableName, blobsContainerBuilder); + } + } + } //FIXME Review transaction management - boolean needCommit = !sqlRequestIterator.hasNext(); - if (needCommit) { + hasNext = sqlRequestIterator.hasNext(); + if (!hasNext) { action.commit(); } + + } + + ImmutableMap.Builder<String, BlobsContainer> blobsContainersBuilder = ImmutableMap.builder(); + + for (Map.Entry<String, BlobsContainer.Builder> entry : blobContainersBuilder.entrySet()) { + + String tableName = entry.getKey(); + BlobsContainer blobsContainer = entry.getValue().build(); + if (!blobsContainer.isEmpty()) { + if (log.isInfoEnabled()) { + log.info("Register blobsContainer for table: " + tableName + ", with " + blobsContainer.getBlobsById().size() + " blob(s)"); + } + blobsContainersBuilder.put(tableName, blobsContainer); + } } + ImmutableSet<BlobsContainer> blobsContainers = ImmutableSet.copyOf(blobsContainersBuilder.build().values()); + return new SqlResult(blobsContainers); + } protected <R extends AbstractSqlRequest, A extends AbstractSqlAction<R>> A createAction(R request) { - Preconditions.checkNotNull(request, "Request can't be null"); + Objects.requireNonNull(request, "Request can't be null"); Class<A> actionType = ACTION_MAPPING.get(request.getClass()); - Preconditions.checkNotNull(actionType, "Could not find action for request type: " + request.getClass().getName()); + Objects.requireNonNull(actionType, "Could not find action for request type: " + request.getClass().getName()); Constructor<A> constructor; try { @@ -189,13 +238,14 @@ public class TopiaSqlBatchServiceImpl implements TopiaSqlBatchService { } @Override - public void execute(ReplicateTablesRequest request) { - executeOneRequest(request); + public SqlResult execute(ReplicateTablesRequest request) { + return executeOneRequest(request); + } @Override - public void execute(UpdateTablesRequest request) { - executeOneRequest(request); + public SqlResult execute(UpdateTablesRequest request) { + return executeOneRequest(request); } @Override @@ -203,8 +253,9 @@ public class TopiaSqlBatchServiceImpl implements TopiaSqlBatchService { executeOneRequest(request); } - protected void executeOneRequest(AbstractSqlRequest request) { + protected SqlResult executeOneRequest(AbstractSqlRequest request) { SqlRequests sqlRequests = SqlRequests.of(request); execute(sqlRequests); + return new SqlResult(ImmutableSet.of()); } } diff --git a/observe-topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/actions/AbstractSqlAction.java b/observe-topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/actions/AbstractSqlAction.java index f708367..122f267 100644 --- a/observe-topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/actions/AbstractSqlAction.java +++ b/observe-topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/actions/AbstractSqlAction.java @@ -39,6 +39,9 @@ import java.io.IOException; import java.io.Writer; import java.sql.Connection; import java.sql.SQLException; +import java.util.LinkedHashSet; +import java.util.Optional; +import java.util.Set; /** * Support to create action. @@ -69,28 +72,29 @@ public abstract class AbstractSqlAction<R extends AbstractSqlRequest> implements protected long endTime; + private final Set<BlobsContainer.Builder> blobsContainerBuilders = new LinkedHashSet<>(); + protected AbstractSqlAction(R request) { this.request = request; - ImmutableSet.Builder<Closeable> closeableBuilder = ImmutableSet.builder(); - closeableBuilder.add(() -> { - if (sourcePersistenceContext != null) { - sourcePersistenceContext.close(); - } - }); - closeableBuilder.add(() -> { - if (targetConnection != null) { - try { - targetConnection.close(); - } catch (SQLException e) { - throw new TopiaException("Could not close targetConnection", e); - } - } - }); - this.closeables = closeableBuilder.build(); + this.closeables = ImmutableSet.<Closeable>builder() + .add(() -> { + if (sourcePersistenceContext != null) { + sourcePersistenceContext.close(); + } + }) + .add(() -> { + if (targetConnection != null) { + try { + targetConnection.close(); + } catch (SQLException e) { + throw new TopiaException("Could not close targetConnection", e); + } + } + }).build(); } - protected static void flush(Writer writer) { + public static void flush(Writer writer) { try { writer.flush(); } catch (IOException e) { @@ -102,6 +106,20 @@ public abstract class AbstractSqlAction<R extends AbstractSqlRequest> implements return request; } + public Optional<Set<BlobsContainer.Builder>> getBlobsContainersBuilder() { + + if (blobsContainerBuilders.isEmpty()) { + return Optional.empty(); + } + + return Optional.of(blobsContainerBuilders); + + } + + public void registerBlobsContainer(BlobsContainer.Builder builder) { + blobsContainerBuilders.add(builder); + } + protected boolean useOutputWriter() { return request.getWriter() != null; } diff --git a/observe-topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/actions/AbstractTablesAction.java b/observe-topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/actions/AbstractTablesAction.java index ae640f3..e9cc0e2 100644 --- a/observe-topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/actions/AbstractTablesAction.java +++ b/observe-topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/actions/AbstractTablesAction.java @@ -22,7 +22,6 @@ package org.nuiton.topia.service.sql.batch.actions; * #L% */ -import org.apache.commons.io.output.ByteArrayOutputStream; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.topia.persistence.support.TopiaSqlWork; @@ -30,17 +29,12 @@ import org.nuiton.topia.service.sql.batch.tables.TopiaSqlTable; import org.nuiton.topia.service.sql.batch.tables.TopiaSqlTables; import org.nuiton.util.TimeLog; -import javax.sql.rowset.serial.SerialBlob; -import java.io.IOException; import java.io.Writer; -import java.sql.Blob; import java.sql.Connection; import java.sql.PreparedStatement; -import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.util.ArrayList; -import java.util.Date; import java.util.List; /** @@ -62,7 +56,7 @@ public abstract class AbstractTablesAction<R extends AbstractTablesRequest> exte super(request); } - protected abstract TopiaSqlTables getTables() ; + protected abstract TopiaSqlTables getTables(); protected abstract void executeOnTable(R request, TopiaSqlTable table, PreparedStatement readStatement) throws SQLException; @@ -82,46 +76,6 @@ public abstract class AbstractTablesAction<R extends AbstractTablesRequest> exte } - protected String generateSqlArguments(ResultSet readResultSet, Iterable<String> columnNames) throws SQLException, IOException { - - String statement = ""; - - for (String columnName : columnNames) { - - Object columnValue = readResultSet.getObject(columnName); - if (columnValue == null) { - statement += ", NULL"; - continue; - } - - if (columnValue instanceof String) { - String stringValue = (String) columnValue; - statement += ", '" + stringValue.replaceAll("'", "''") + "'"; - continue; - } - - if (columnValue instanceof Date) { - statement += ", '" + columnValue + "'"; - continue; - } - - if (columnValue instanceof Blob) { - Blob blob = (Blob) columnValue; - SerialBlob serialBlob = new SerialBlob(blob); - try (ByteArrayOutputStream stringWriter = new ByteArrayOutputStream((int) serialBlob.length())) { - stringWriter.write(serialBlob.getBinaryStream()); - statement += ", '" + new String(stringWriter.toByteArray()) + "'"; - } - continue; - } - - statement += ", " + columnValue; - } - - return statement.substring(2); - - } - protected String generateWildcardArguments(Iterable<String> columnNames) { StringBuilder argsBuilder = new StringBuilder(); diff --git a/observe-topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/actions/BlobsContainer.java b/observe-topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/actions/BlobsContainer.java new file mode 100644 index 0000000..044d279 --- /dev/null +++ b/observe-topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/actions/BlobsContainer.java @@ -0,0 +1,80 @@ +package org.nuiton.topia.service.sql.batch.actions; + +import com.google.common.collect.ImmutableMap; + +import java.util.Map; + +/** + * Pour contenir des blobs pour une colonne de type blob sur une table donnée. + * + * Created on 24/08/16. + * + * @author Tony Chemit - chemit@codelutin.com + * @since 5.0 + */ +public class BlobsContainer { + + private final String tableName; + private final String columnName; + private final ImmutableMap<String, byte[]> blobsById; + + public static Builder builder(String tableName, String columnName) { + return new Builder(tableName, columnName); + } + + private BlobsContainer(String tableName, String columnName, ImmutableMap<String, byte[]> blobsById) { + this.tableName = tableName; + this.columnName = columnName; + this.blobsById = blobsById; + } + + public ImmutableMap<String, byte[]> getBlobsById() { + return blobsById; + } + + public String getTableName() { + return tableName; + } + + public String getColumnName() { + return columnName; + } + + public boolean isEmpty() { + return blobsById.isEmpty(); + } + + public static class Builder { + + private final String tableName; + private final String columnName; + private final ImmutableMap.Builder<String, byte[]> blobsContainerBuilder = ImmutableMap.builder(); + + private Builder(String tableName, String columnName) { + this.tableName = tableName; + this.columnName = columnName; + } + + public Builder addBlob(String id, byte[] content) { + blobsContainerBuilder.put(id, content); + return this; + } + + public Builder addAllBlob(Map<String, byte[]> content) { + blobsContainerBuilder.putAll(content); + return this; + } + + public String getTableName() { + return tableName; + } + + public String getColumnName() { + return columnName; + } + + public BlobsContainer build() { + return new BlobsContainer(tableName, columnName, blobsContainerBuilder.build()); + } + } +} diff --git a/observe-topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/actions/ReplicateTablesAction.java b/observe-topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/actions/ReplicateTablesAction.java index cad3c45..46d717e 100644 --- a/observe-topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/actions/ReplicateTablesAction.java +++ b/observe-topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/actions/ReplicateTablesAction.java @@ -22,17 +22,22 @@ package org.nuiton.topia.service.sql.batch.actions; * #L% */ +import org.apache.commons.io.output.ByteArrayOutputStream; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.topia.service.sql.batch.tables.TopiaSqlTable; import org.nuiton.topia.service.sql.batch.tables.TopiaSqlTables; +import javax.sql.rowset.serial.SerialBlob; import java.io.IOException; +import java.sql.Blob; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; +import java.util.Date; import java.util.List; +import java.util.Objects; /** * Created on 01/01/16. @@ -62,9 +67,18 @@ public class ReplicateTablesAction extends AbstractTablesAction<ReplicateTablesR List<String> columnNames = getColumnNames(readResultSetMetaData, columnCount); + boolean useBlob = table.getMetadataEntity().withBlob(); boolean useOutputWriter = useOutputWriter(); boolean useOutputDb = useOutputDb(); + BlobsContainer.Builder blobsBuilder = null; + if (useBlob) { + + //FIXME On devrait gérer pour plusieurs colonnes + String columnName = table.getMetadataEntity().getBlobProperties().iterator().next(); + blobsBuilder = BlobsContainer.builder(table.getFullyTableName(), columnName); + registerBlobsContainer(blobsBuilder); + } PreparedStatement writeStatement = null; String insertStatementSql = newInsertStatementSql(table, columnNames); @@ -102,7 +116,7 @@ public class ReplicateTablesAction extends AbstractTablesAction<ReplicateTablesR try { - String arguments = generateSqlArguments(readResultSet, columnNames); + String arguments = generateSqlArguments(readResultSet, columnNames, blobsBuilder); String sql = String.format(insertStatementSql, arguments); writer.append(sql); @@ -136,9 +150,9 @@ public class ReplicateTablesAction extends AbstractTablesAction<ReplicateTablesR } String sql = String.format(INSERT_STATEMENT, - table.getSchemaName(), - table.getTableName(), - columnNamesBuilder.substring(2)); + table.getSchemaName(), + table.getTableName(), + columnNamesBuilder.substring(2)); if (log.isDebugEnabled()) { log.debug("Insert sql: " + sql); } @@ -147,4 +161,53 @@ public class ReplicateTablesAction extends AbstractTablesAction<ReplicateTablesR } + protected String generateSqlArguments(ResultSet readResultSet, + Iterable<String> columnNames, + BlobsContainer.Builder blobsBuilder) throws SQLException, IOException { + + String statement = ""; + + for (String columnName : columnNames) { + + Object columnValue = readResultSet.getObject(columnName); + if (columnValue == null) { + statement += ", NULL"; + continue; + } + + if (columnValue instanceof String) { + String stringValue = (String) columnValue; + statement += ", '" + stringValue.replaceAll("'", "''") + "'"; + continue; + } + + if (columnValue instanceof Date) { + statement += ", '" + columnValue + "'"; + continue; + } + + if (columnValue instanceof Blob) { + Blob blob = (Blob) columnValue; + SerialBlob serialBlob = new SerialBlob(blob); + try (ByteArrayOutputStream stringWriter = new ByteArrayOutputStream((int) serialBlob.length())) { + stringWriter.write(serialBlob.getBinaryStream()); +// statement += ", '" + new String(stringWriter.toByteArray()) + "'"; + statement += ", NULL"; + String topiaId = readResultSet.getString("topiaId"); + Objects.nonNull(topiaId); + blobsBuilder.addBlob(topiaId, stringWriter.toByteArray()); + if (log.isInfoEnabled()) { + log.info("Add blob: " + topiaId); + } + } + continue; + } + + statement += ", " + columnValue; + } + + return statement.substring(2); + + } + } diff --git a/observe-topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/tables/TopiaSqlTable.java b/observe-topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/tables/TopiaSqlTable.java index 6b14739..a4d837f 100644 --- a/observe-topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/tables/TopiaSqlTable.java +++ b/observe-topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/tables/TopiaSqlTable.java @@ -24,6 +24,7 @@ package org.nuiton.topia.service.sql.batch.tables; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableSet; +import org.nuiton.topia.persistence.metadata.TopiaMetadataEntity; import java.util.Objects; @@ -37,6 +38,8 @@ import java.util.Objects; */ public class TopiaSqlTable { + protected final TopiaMetadataEntity metadataEntity; + /** * Table schema name. */ @@ -46,7 +49,15 @@ public class TopiaSqlTable { * Table name. */ protected final String tableName; + + /** + * Association table (optional only present for association table). + */ private final boolean associationTable; + + /** + * Join column name (optional only present for association table). + */ private final String joinColumnName; /** @@ -69,13 +80,15 @@ public class TopiaSqlTable { */ protected final ImmutableSet<String> joinClauses; - public TopiaSqlTable(String schemaName, + public TopiaSqlTable(TopiaMetadataEntity metadataEntity, + String schemaName, String tableName, String fromClause, String whereClauseAlias, ImmutableSet<String> joinClauses, boolean associationTable, String joinColumnName) { + this.metadataEntity = metadataEntity; this.schemaName = schemaName.toLowerCase(); this.tableName = tableName.toLowerCase(); this.associationTable = associationTable; @@ -86,6 +99,10 @@ public class TopiaSqlTable { this.joinColumnName = joinColumnName; } + public TopiaMetadataEntity getMetadataEntity() { + return metadataEntity; + } + public String getSchemaName() { return schemaName; } diff --git a/observe-topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/tables/TopiaSqlTables.java b/observe-topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/tables/TopiaSqlTables.java index 9cef9c7..0dd78fe 100644 --- a/observe-topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/tables/TopiaSqlTables.java +++ b/observe-topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/tables/TopiaSqlTables.java @@ -30,6 +30,8 @@ import com.google.common.collect.Lists; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.topia.persistence.TopiaEntityEnum; +import org.nuiton.topia.persistence.metadata.TopiaMetadataEntity; +import org.nuiton.topia.persistence.metadata.TopiaMetadataModel; import java.util.Collections; import java.util.Iterator; @@ -61,12 +63,8 @@ public class TopiaSqlTables implements Iterable<TopiaSqlTable> { this.orderedTables = orderedTables; } - public static Builder builder() { - return new BuilderImpl(); - } - - public static BuilderStepOnTable builder(TopiaEntityEnum mainEntityEnum) { - return builder().addMainTable(mainEntityEnum); + public static Builder builder(TopiaMetadataModel metadataModel, TopiaMetadataEntity metadataEntity) { + return new BuilderImpl(metadataModel, metadataEntity); } public TopiaSqlTable getTable(String key) { @@ -118,7 +116,7 @@ public class TopiaSqlTables implements Iterable<TopiaSqlTable> { BuilderStepOnTable addAndEnterReverseJoinTable(TopiaEntityEnum entityEnum); - BuilderStepOnTable addAssociationTable(String associationName); + BuilderStepOnTable addAssociationTable(TopiaEntityEnum entityEnum, String associationName); BuilderStepOnTable backToParent(); @@ -135,13 +133,15 @@ public class TopiaSqlTables implements Iterable<TopiaSqlTable> { */ protected static class BuilderImpl implements Builder { - protected final TreeMap<String, TopiaSqlTable> tablesByFullyTableName; - - protected final TreeMap<Integer, TopiaSqlTable> tablesByOrder; + private final TreeMap<String, TopiaSqlTable> tablesByFullyTableName; + private final TreeMap<Integer, TopiaSqlTable> tablesByOrder; + private final TopiaMetadataModel metadataModel; + private final TopiaMetadataEntity metadataEntity; + private int internalOrder; - protected int internalOrder; - - public BuilderImpl() { + public BuilderImpl(TopiaMetadataModel metadataModel, TopiaMetadataEntity metadataEntity) { + this.metadataModel = metadataModel; + this.metadataEntity = metadataEntity; this.tablesByFullyTableName = new TreeMap<>(); this.tablesByOrder = new TreeMap<>(); } @@ -156,7 +156,7 @@ public class TopiaSqlTables implements Iterable<TopiaSqlTable> { String whereClauseAlias = tableName + ".topiaid"; String fromClause = schemaName + "." + tableName + " " + tableName; - registerTable(schemaName, tableName, whereClauseAlias, fromClause, ImmutableSet.of()); + registerTable(entityEnum, schemaName, tableName, whereClauseAlias, fromClause, ImmutableSet.of()); return new BuilderStepOnTableImpl(null, entityEnum); } @@ -175,27 +175,30 @@ public class TopiaSqlTables implements Iterable<TopiaSqlTable> { orderedTablesBuilder.build()); } - protected Builder registerTable(String schemaName, + protected Builder registerTable(TopiaEntityEnum entityEnum, + String schemaName, String tableName, String whereClauseAlias, String fromClause, ImmutableSet<String> joinClauses) { - return registerTable(schemaName, tableName, whereClauseAlias, fromClause, joinClauses, false, null); + return registerTable(entityEnum, schemaName, tableName, whereClauseAlias, fromClause, joinClauses, false, null); } - protected Builder registerAssociationTable(String schemaName, + protected Builder registerAssociationTable(TopiaEntityEnum entityEnum, + String schemaName, String tableName, String whereClauseAlias, String fromClause, ImmutableSet<String> joinClauses, String joinColumnName) { - return registerTable(schemaName, tableName, whereClauseAlias, fromClause, joinClauses, true, joinColumnName); + return registerTable(entityEnum, schemaName, tableName, whereClauseAlias, fromClause, joinClauses, true, joinColumnName); } - private Builder registerTable(String schemaName, + private Builder registerTable(TopiaEntityEnum entityEnum, + String schemaName, String tableName, String whereClauseAlias, String fromClause, @@ -205,7 +208,10 @@ public class TopiaSqlTables implements Iterable<TopiaSqlTable> { //TODO check that this table is not already registred + TopiaMetadataEntity metadataEntity = metadataModel.getEntity(entityEnum.name()); + TopiaSqlTable table = new TopiaSqlTable( + metadataEntity, schemaName, tableName, fromClause, @@ -302,7 +308,7 @@ public class TopiaSqlTables implements Iterable<TopiaSqlTable> { } - registerTable(schemaName, tableName, whereClauseAlias, fromClause, joinClauses); + registerTable(entityEnum, schemaName, tableName, whereClauseAlias, fromClause, joinClauses); return this; } @@ -328,7 +334,7 @@ public class TopiaSqlTables implements Iterable<TopiaSqlTable> { ImmutableSet<String> joinClauses = addJoinCause(parentTable.getJoinClauses(), joinClause); - registerTable(schemaName, tableName, whereClauseAlias, fromClause, joinClauses); + registerTable(entityEnum, schemaName, tableName, whereClauseAlias, fromClause, joinClauses); invertOrderWithParent(parentTable, entityEnum); return this; @@ -342,7 +348,7 @@ public class TopiaSqlTables implements Iterable<TopiaSqlTable> { } @Override - public BuilderStepOnTable addAssociationTable(String associationName) { + public BuilderStepOnTable addAssociationTable(TopiaEntityEnum entityEnum,String associationName) { TopiaSqlTable parentTable = getTable(); @@ -368,7 +374,8 @@ public class TopiaSqlTables implements Iterable<TopiaSqlTable> { String joinColumnName = parentTable.getTableName(); - registerAssociationTable(schemaName, + registerAssociationTable(entityEnum, + schemaName, tableName, whereClauseAlias, fromClause, diff --git a/observe-topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/tables/TopiaSqlTablesFactory.java b/observe-topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/tables/TopiaSqlTablesFactory.java index ea878b7..6b3a246 100644 --- a/observe-topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/tables/TopiaSqlTablesFactory.java +++ b/observe-topia-extension/src/main/java/org/nuiton/topia/service/sql/batch/tables/TopiaSqlTablesFactory.java @@ -117,7 +117,7 @@ public class TopiaSqlTablesFactory { log.info("E → " + metadataEntity.getType()); TopiaEntityEnum entityEnum = entityEnumProvider.getEntityEnum(metadataEntity.getType()); - builder = (builder == null ? TopiaSqlTables.builder() : builder).addMainTable(entityEnum); + builder = (builder == null ? TopiaSqlTables.builder(metadataModel, metadataEntity) : builder).addMainTable(entityEnum); } } @@ -176,7 +176,8 @@ public class TopiaSqlTablesFactory { log.info(metadataEntity.getType() + "/" + propertyName + "→" + propertyType.getType()); - builder = builder.addAssociationTable(propertyName); + TopiaEntityEnum entityEnum = entityEnumProvider.getEntityEnum(propertyType.getType()); + builder = builder.addAssociationTable(entityEnum, propertyName); } } -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.
participants (1)
-
codelutin.com scm