Tony CHEMIT pushed to branch feature/issue_2575 at ultreiaio / ird-observe Commits: 46eae65c by Tony Chemit at 2023-01-24T07:20:44+01:00 Pom - downgrade hsqldb to 2.7.0 (2.7.1 add some errors on uncanaccess...) - - - - - 16639467 by Tony Chemit at 2023-01-24T07:21:25+01:00 AVDTH - Review and clarify dcp migration - - - - - 2d6f2f35 by Tony Chemit at 2023-01-24T07:21:25+01:00 AVDTH - Review TRANSMITTING_BUOY_OPERATION_CODE_MAPPING mapping - - - - - 7d5117f0 by Tony Chemit at 2023-01-24T07:21:25+01:00 AVDTH - Review OBSERVED_SYSTEM_CODE_MAPPING (for ACTIVITE.C_TYP_OBJET == 12) : * add Observed system 20 * add ObjectMaterial ANLOG - - - - - 7aa72520 by Tony Chemit at 2023-01-24T07:21:25+01:00 AVDTH - Improve some error messages - - - - - 0cdeb14b by Tony Chemit at 2023-01-24T07:21:25+01:00 AVDTH - Add *addMessage* method to reader - - - - - 5e10972e by Tony Chemit at 2023-01-24T07:21:25+01:00 AVDTH - Improve when to create (or not) Floating object - - - - - ddabfa63 by Tony Chemit at 2023-01-24T07:21:25+01:00 AVDTH - Do not migrate ACTIVITE.V_POIDS_ESTIM_DCP. - - - - - 99b81e9c by Tony Chemit at 2023-01-24T07:21:25+01:00 AVDTH - Improve a comment - - - - - 916120d3 by Tony Chemit at 2023-01-24T07:21:25+01:00 AVDTH - Add more tests (using avdth multi-year databases) - - - - - 7f504866 by Tony Chemit at 2023-01-24T07:21:25+01:00 AVDTH - Improve when to create (or not) Floating object - - - - - 4d03b605 by Tony Chemit at 2023-01-24T07:21:25+01:00 Avdth tests - fix when less than 3 db found - - - - - 3c92fc44 by Tony Chemit at 2023-01-24T07:21:25+01:00 Avdth tests - add missing test_forcePrepare to pom properties for avdth all years tests - - - - - e3582a2b by Tony Chemit at 2023-01-24T07:21:25+01:00 Avdth - add comment (for debug purpose) - - - - - ac135d4a by Tony Chemit at 2023-01-24T07:21:25+01:00 Avdth - found more ACTIVITE.C_OPERA with DCP (for 9.0.x we still import them on bad ObServe activities, will be review in 9.1.x with new vesselActivities...) - - - - - b5d57532 by Tony Chemit at 2023-01-24T07:21:26+01:00 Avdth - add ACTIVITE.C_OPERA when creating ObjectMaterial to add it on log if something is wrong - - - - - 14 changed files: - core/api/services/src/main/java/fr/ird/observe/services/service/data/ps/AvdthDataImportResult.java - core/persistence/avdth/src/main/java/fr/ird/observe/persistence/avdth/data/DataReader.java - core/persistence/avdth/src/main/java/fr/ird/observe/persistence/avdth/data/ImportReferentialContext.java - core/persistence/avdth/src/main/java/fr/ird/observe/persistence/avdth/data/logbook/FloatingObjectReader.java - core/persistence/avdth/src/main/java/fr/ird/observe/persistence/avdth/data/logbook/FloatingObjectWriter.java - core/persistence/avdth/src/main/java/fr/ird/observe/persistence/avdth/data/logbook/SampleActivityReader.java - core/persistence/avdth/src/main/java/fr/ird/observe/persistence/avdth/data/logbook/WellPlanReader.java - core/persistence/test/pom.xml - + core/persistence/test/src/test/java/fr/ird/observe/persistence/avdth/data/AvdthAtlanticDataBuilderByYearLatestTest.java - core/persistence/test/src/test/java/fr/ird/observe/persistence/avdth/data/AvdthDataBuilderByYearTestSupport.java - core/persistence/test/src/test/java/fr/ird/observe/persistence/avdth/data/AvdthDataBuilderTestSupport.java - + core/persistence/test/src/test/java/fr/ird/observe/persistence/avdth/data/AvdthIndianDataBuilderByYearLatestTest.java - pom.xml - src/site/markdown/avdth/logbook.md Changes: ===================================== core/api/services/src/main/java/fr/ird/observe/services/service/data/ps/AvdthDataImportResult.java ===================================== @@ -87,6 +87,10 @@ public class AvdthDataImportResult implements ObserveDto { this.notReadResult = notReadResult; this.exportResult = exportResult; this.sqlResultPath = sqlResultPath; + // Used for debugging... +// if (!exportResult.isEmpty()) { +// System.out.println(exportResult.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining("\n"))); +// } } public Path getMessageFile() { ===================================== core/persistence/avdth/src/main/java/fr/ird/observe/persistence/avdth/data/DataReader.java ===================================== @@ -93,5 +93,9 @@ public abstract class DataReader<E extends DataEntity> { protected final AvdthDataImportConfiguration getContext() { return context.getConfiguration(); } + + protected final void addMessage(String message) { + context.addMessage(message); + } } ===================================== core/persistence/avdth/src/main/java/fr/ird/observe/persistence/avdth/data/ImportReferentialContext.java ===================================== @@ -58,7 +58,6 @@ import fr.ird.observe.entities.referential.ps.logbook.SetSuccessStatus; import fr.ird.observe.entities.referential.ps.logbook.WellContentStatus; import fr.ird.observe.entities.referential.ps.logbook.WellSamplingConformity; import fr.ird.observe.entities.referential.ps.logbook.WellSamplingStatus; -import fr.ird.observe.persistence.avdth.data.logbook.FloatingObjectReader; import fr.ird.observe.persistence.avdth.data.logbook.WellPlanReader; import fr.ird.observe.persistence.avdth.referential.AvdthReferentialImportResult; import fr.ird.observe.persistence.avdth.referential.SpeciesCache; @@ -80,10 +79,8 @@ import java.util.function.Consumer; * @author Tony Chemit - dev@tchemit.fr * @since 9.0.0 */ -@SuppressWarnings("SpellCheckingInspection") public class ImportReferentialContext { - private Map<String, WeightCategory> landingCategories; private Map<String, Destination> destinations; private Map<String, WeightCategory> wellCategories; @@ -127,9 +124,11 @@ public class ImportReferentialContext { private ObjectMaterial objectMaterialDFAD; private ObjectMaterial objectMaterialALOG; private ObjectMaterial objectMaterialAFAD; + private ObjectMaterial objectMaterialBiodegradable; private ObjectMaterial objectMaterialAlone; private ObjectMaterial objectMaterialLOG; private ObjectMaterial objectMaterialFALOG; + private ObjectMaterial objectMaterialANLOG; private ObjectMaterial objectMaterialHALOG; private ObjectMaterial objectMaterialVNLOG; private ObservedSystem observedSystem0; @@ -226,8 +225,10 @@ public class ImportReferentialContext { objectMaterialLOG = objectMaterial.get("fr.ird.referential.ps.common.ObjectMaterial#0#0.50"); objectMaterialFALOG = objectMaterial.get("fr.ird.referential.ps.common.ObjectMaterial#0#0.63"); objectMaterialHALOG = objectMaterial.get("fr.ird.referential.ps.common.ObjectMaterial#0#0.67"); + objectMaterialANLOG = objectMaterial.get("fr.ird.referential.ps.common.ObjectMaterial#0#0.56"); objectMaterialVNLOG = objectMaterial.get("fr.ird.referential.ps.common.ObjectMaterial#0#0.52"); objectMaterialAlone = objectMaterial.get("fr.ird.referential.ps.common.ObjectMaterial#1561561977652#0.5876332198776647"); + objectMaterialBiodegradable = objectMaterial.get("fr.ird.referential.ps.common.ObjectMaterial#0#0.69"); // catch catchCategories = new TreeMap<>(); @@ -333,10 +334,34 @@ public class ImportReferentialContext { return objectMaterialAFAD; } + public ObjectMaterial getObjectMaterialLOG() { + return objectMaterialLOG; + } + + public ObjectMaterial getObjectMaterialFALOG() { + return objectMaterialFALOG; + } + + public ObjectMaterial getObjectMaterialHALOG() { + return objectMaterialHALOG; + } + + public ObjectMaterial getObjectMaterialVNLOG() { + return objectMaterialVNLOG; + } + + public ObjectMaterial getObjectMaterialBiodegradable() { + return objectMaterialBiodegradable; + } + public ObjectMaterial getObjectMaterialAlone() { return objectMaterialAlone; } + public ObjectMaterial getObjectMaterialANLOG() { + return objectMaterialANLOG; + } + public SampleType getLocalmarketSampleType() { return localmarketSampleType; } @@ -529,86 +554,37 @@ public class ImportReferentialContext { entity.addObservedSystem(getObservedSystem("102")); } - public ObjectOperation getObjectOperationByVesselActivityCode(String vesselActivityCode) { - String objectOperationCode = FloatingObjectReader.DCP_OBJECT_OPERATION_CODE_MAPPING.get(vesselActivityCode); - if (objectOperationCode == null) { - return null; - } + public ObjectOperation getObjectOperation(String objectOperationCode) { return objectOperation.get(objectOperationCode); } - public TransmittingBuoyOperation getTransmittingBuoyOperationByVesselActivityCode(String vesselActivityCode) { - String transmittingBuoyTypeCode = FloatingObjectReader.TRANSMITTING_BUOY_OPERATION_CODE_MAPPING.get(vesselActivityCode); + public TransmittingBuoyOperation getTransmittingBuoyOperation(String transmittingBuoyTypeCode) { if (transmittingBuoyTypeCode == null) { return null; } return transmittingBuoyOperation.get(transmittingBuoyTypeCode); } - public TransmittingBuoyType getTransmittingBuoyTypeByBuoyTypeCode(String buoyTypeCode) { - String transmittingBuoyTypeCode = FloatingObjectReader.TRANSMITTING_BUOY_TYPE_CODE_MAPPING.get(buoyTypeCode); - if (transmittingBuoyTypeCode == null) { - transmittingBuoyTypeCode = "999"; - } - TransmittingBuoyType transmittingBuoyType = this.transmittingBuoyType.get(transmittingBuoyTypeCode); - return Objects.requireNonNull(transmittingBuoyType, String.format("Can't find transmittingBuoyType with code: %s", buoyTypeCode)); + public TransmittingBuoyType getTransmittingBuoyType(String transmittingBuoyTypeCode) { + TransmittingBuoyType result = transmittingBuoyType.get(transmittingBuoyTypeCode); + return Objects.requireNonNull(result, String.format("Can't find TransmittingBuoyType with code: %s", transmittingBuoyTypeCode)); } - public TransmittingBuoyOwnership getTransmittingBuoyOwnership(String buoyOwnershipCode) { - if (buoyOwnershipCode == null) { - return null; - } - String transmittingBuoyOwnershipCode = FloatingObjectReader.TRANSMITTING_BUOY_OWNERSHIP_CODE_MAPPING.get(buoyOwnershipCode); + public TransmittingBuoyOwnership getTransmittingBuoyOwnership(String transmittingBuoyOwnershipCode) { if (transmittingBuoyOwnershipCode == null) { return null; } return transmittingBuoyOwnership.get(transmittingBuoyOwnershipCode); } - public ObservedSystem getObservedSystem(String objectTypeCode, String buoyTypeCode) { - if (objectTypeCode.equals("999")) { - if (buoyTypeCode.equals("999")) { - return getObservedSystem0(); - } - return getObservedSystem20(); - } - String observedSystemCode = FloatingObjectReader.OBSERVED_SYSTEM_CODE_MAPPING.get(objectTypeCode); - return getObservedSystem(observedSystemCode, false); - } - - /** - * To get objectMaterial id associated by a DCP from avdth ACTIVITE.C_TYP_OBJET code. - * - * @param objectTypeCode ACTIVITE.C_TYP_OBJET - * @return material or null - */ - public ObjectMaterial getObjectMaterialByObjectTypeCode(String objectTypeCode) { - switch (objectTypeCode) { - case "1": - return objectMaterialAFAD; - case "2": - return objectMaterialDFAD; - case "3": - return objectMaterialLOG; - case "9": - return objectMaterialFOB; - case "10": - return objectMaterialFALOG; - case "11": - return objectMaterialHALOG; - case "13": - return objectMaterialVNLOG; - } - return null; - } - public WeightCategory getWellWeightCategory(String weightCategoryCode) { String categoryCode = "W-" + weightCategoryCode; WeightCategory weightCategory = wellCategories.get(categoryCode); - return Objects.requireNonNull(weightCategory, "Can't find weight category for code: " + categoryCode); + return Objects.requireNonNull(weightCategory, String.format("Can't find weight category for code: %s", categoryCode)); } public VesselActivity getVesselActivity6() { return vesselActivity6; } + } ===================================== core/persistence/avdth/src/main/java/fr/ird/observe/persistence/avdth/data/logbook/FloatingObjectReader.java ===================================== @@ -23,6 +23,7 @@ package fr.ird.observe.persistence.avdth.data.logbook; */ import com.google.common.collect.ImmutableMap; +import fr.ird.observe.entities.data.ps.logbook.Activity; import fr.ird.observe.entities.data.ps.logbook.FloatingObject; import fr.ird.observe.entities.data.ps.logbook.FloatingObjectPart; import fr.ird.observe.entities.data.ps.logbook.TransmittingBuoy; @@ -33,16 +34,21 @@ import fr.ird.observe.entities.referential.ps.common.TransmittingBuoyOperation; import fr.ird.observe.entities.referential.ps.common.TransmittingBuoyOwnership; import fr.ird.observe.entities.referential.ps.common.TransmittingBuoyType; import fr.ird.observe.entities.referential.ps.common.VesselActivity; +import fr.ird.observe.persistence.avdth.Query; import fr.ird.observe.persistence.avdth.data.DataReader; import fr.ird.observe.persistence.avdth.data.ImportDataContext; import fr.ird.observe.persistence.avdth.data.ImportEngine; import org.apache.commons.lang3.mutable.MutableInt; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.TreeMap; import java.util.TreeSet; /** @@ -53,6 +59,7 @@ import java.util.TreeSet; */ @SuppressWarnings("SpellCheckingInspection") public class FloatingObjectReader extends DataReader<FloatingObject> { + private static final Logger log = LogManager.getLogger(FloatingObjectReader.class); /** * To get FloatingObject.objectOperation code from avdth ACTIVITE.C_OPERA. */ @@ -66,7 +73,10 @@ public class FloatingObjectReader extends DataReader<FloatingObject> { .put("14", "99") .put("7", "99") .put("8", "99") + .put("9", "99") + .put("12", "99") .put("13", "99") + .put("15", "99") .put("5", "1") .put("6", "4") @@ -87,32 +97,23 @@ public class FloatingObjectReader extends DataReader<FloatingObject> { .build(); /** * To get TransmittingBuoy.TransmittingBuoyOperation code from avdth ACTIVITE.C_OPERA code. + * <p> + * If for a given {@code ACTIVITE.C_OPERA}, no value is found here, this means that there is no buoy to create. */ public static final Map<String, String> TRANSMITTING_BUOY_OPERATION_CODE_MAPPING = ImmutableMap.<String, String>builder() - .put("0", "1") // Visite | 99 inconnu | W_ARRIVING | - .put("1", "1") // Visite | 99 inconnu | W_ARRIVING | - .put("2", "1") // Visite | 99 inconnu | W_ARRIVING | - .put("3", "1") // Visite | 99 inconnu | W_ARRIVING | - .put("4", "1") // Visite | 99 inconnu | W_ARRIVING | - .put("10", "1")// Visite | 99 inconnu | W_ARRIVING | - .put("14", "1")// Visite | 99 inconnu | W_ARRIVING | - .put("7", "1") // Visite | 99 inconnu | W_ARRIVING | - .put("8", "1") // Visite | 99 inconnu | W_ARRIVING | - .put("13", "1")// Visite | 99 inconnu | W_ARRIVING | - - .put("5", "3") // Mise à l'eau | null ou 3 Mise à l'eau | W_LEAVING | Selon présence infos balise dans AVDTH - .put("6", "2") // Retrait | null ou 2 Récupération | W_ARRIVING | Selon présence infos balise dans AVDTH - .put("23", "3") // Mise à l'eau | 3 Mise à l'eau | W_LEAVING | - .put("24", "2") // Retrait | 2 Récupération | W_ARRIVING | - .put("25", "3") // Visite | 3 Mise à l'eau | W_ARRIVING | - .put("26", "2") // Visite | 2 Récupération | W_ARRIVING | - .put("29", "1") // Visite | 1 Visite | W_ARRIVING | - .put("32", "3") // Modif ou renfor | 3 Mise à l'eau | W_ARR + W_LEA | - .put("33", "4") // Perte | 4 Perte signal | W_ARRIVING | codes 11 et 4 à créer ; matériaux sur W_ARRIVING par convention, comme pour Visite - .put("34", "2") // Retrait | 2 Récupération | W_ARRIVING | rattacher cette balise à un FOB de type '2-2-4-4 Bouée émettrice seule' + .put("5", "3") // Mise à l'eau | null ou 3 Mise à l'eau + .put("6", "2") // Retrait | null ou 2 Récupération + .put("23", "3") // Mise à l'eau | 3 Mise à l'eau + .put("24", "2") // Retrait | 2 Récupération + .put("25", "3") // Visite | 3 Mise à l'eau + .put("26", "2") // Visite | 2 Récupération + .put("29", "1") // Visite | 1 Visite + .put("32", "3") // Modif ou renfor | 3 Mise à l'eau + .put("33", "4") // Perte | 4 Perte signal + .put("34", "2") // Retrait | 2 Récupération // See https://gitlab.com/ultreiaio/ird-observe/-/issues/2568 - .put("40", "4") // Perte | 4 Perte signal | W_ARRIVING | rattacher cette balise à un FOB de type '1-1 DFAD' - .put("41", "5") // Arret commandé de la transmission | 5 Fin d'utilisation | W_ARRIVING | rattacher cette balise à un FOB de type '1-1 DFAD' + .put("40", "4") // Perte | 4 Perte signal + .put("41", "5") // Arret commandé de la transmission | 5 Fin d'utilisation .build(); /** * Is floatingObjectPart on leaving from avdth ACTIVITE.C_OPERA. @@ -127,8 +128,11 @@ public class FloatingObjectReader extends DataReader<FloatingObject> { "2", "3", "4", + "9", "10", + "12", "14", + "15", "7", "8", "13", @@ -214,6 +218,7 @@ public class FloatingObjectReader extends DataReader<FloatingObject> { .put("9", "20") .put("10", "20") .put("11", "20") + .put("12", "20") .put("13", "20") .build(); public static final Set<String> OBSERVED_SYTEM_CODES_WITH_DCP = Set.of("20", "21", "22", "23", "24", "25"); @@ -225,6 +230,8 @@ public class FloatingObjectReader extends DataReader<FloatingObject> { private final MutableInt floatingObjectPartCount = new MutableInt(); private final MutableInt transmittingBuoyCount = new MutableInt(); + private final Map<String, MutableInt> badVesselActivityCodes = new TreeMap<>(); + public FloatingObjectReader(ImportEngine context) { super(context); } @@ -232,124 +239,202 @@ public class FloatingObjectReader extends DataReader<FloatingObject> { @Override public FloatingObject read(ImportDataContext dataContext, ResultSet resultSet) throws SQLException { - String vesselActivityCode = resultSet.getString(13); - VesselActivity vesselActivity = dataContext.getVesselActivity(vesselActivityCode); + // Get the current activity where to add the optional floating object + Activity activity = dataContext.getActivity(); + + // Get his vessel activity + VesselActivity vesselActivity = activity.getVesselActivity(); + + // Avdth observed system codes associated to the current activity (need them to compute ex nihilo floating object) Set<String> observedSystemCodes = dataContext.getObservedSystemCodes(); - if (!Objects.equals(vesselActivity.getCode(), "13")) { - boolean noDcpInObservedSystems = observedSystemCodes.stream().noneMatch(OBSERVED_SYTEM_CODES_WITH_DCP::contains); - if (noDcpInObservedSystems) { - reset(dataContext); - return null; - } - } - FloatingObject entity = newEntity(FloatingObject.SPI); + // We need to get the original vessel activity code from AVDTH (to compute some mapping) + String vesselActivityCode = resultSet.getString(13); - ObjectOperation objectOperation = dataContext.getObjectOperationByVesselActivityCode(vesselActivityCode); - entity.setObjectOperation(Objects.requireNonNull(objectOperation, String.format("Can't find objectOperation with code: %s", vesselActivityCode))); - boolean addCoordinate = "11".equals(objectOperation.getCode()); - boolean whenArriving = WHEN_ARRIVING_CODE.contains(vesselActivityCode); - boolean whenLeaving = WHEN_LEAVING_CODE.contains(vesselActivityCode); + // Is the floating object can be created by the observed systems? + boolean floatingObjectCreatedByObservedSystem = observedSystemCodes.stream().anyMatch(OBSERVED_SYTEM_CODES_WITH_DCP::contains); + + // Is the current vessel activity accept to create Floating object? + boolean vesselActivityAllowFad = vesselActivity.isAllowFad(); + + // Get object type code String objectTypeCode = resultSet.getString(28); if (objectTypeCode == null) { - //FIXME should fail ? + // If null, then consider it as not found in avdth objectTypeCode = "999"; } - //FIXME 29 - F_DCP_ECO -// int dcpEcoCode = resultSet.getInt(29); - String buoyOwnershipCode = resultSet.getString(30); + + // Is floating object exists in avdth? + boolean floatingObjectExistsInAvdth = !objectTypeCode.equals("999"); + + if (!floatingObjectExistsInAvdth && !floatingObjectCreatedByObservedSystem) { + // If Floating object does not exist in avdth, neither need to be created by observed system, + // Do not create the floating object (no log is required here) + reset(dataContext); + return null; + } + + if (!vesselActivityAllowFad) { + // If ObServe veseel activity does not support Floating object, reject import + List<Object> activityPrimaryKey = ActivityReader.getActivityPk(resultSet); + String activityPk = Query.primaryKeyString(activityPrimaryKey); + String message = String.format("On activity %s (ACTIVITE.C_OPERA %s), can not add Floating object since ObServe Vessel activity with code : %s does not allow FAD (floatingObjectExistsInAvdth? %b, floatingObjectCreatedByObservedSystem? %b).", + activityPk, + vesselActivityCode, + vesselActivity.getCode(), + floatingObjectExistsInAvdth, + floatingObjectCreatedByObservedSystem); + log.warn(message); + badVesselActivityCodes.computeIfAbsent(vesselActivityCode, MutableInt::new).increment(); + +// // do not create the floating object +// reset(dataContext); +// return null; + } + + FloatingObject entity = newEntity(FloatingObject.SPI); + + // Get floating object operation + ObjectOperation objectOperation = getObjectOperationByVesselActivityCode(dataContext, vesselActivityCode); + entity.setObjectOperation(Objects.requireNonNull(objectOperation, String.format("No mapping for FloatingObject.objectOperation from ACTIVITE.C_OPERA: %s", vesselActivityCode))); + String buoyTypeCode = resultSet.getString(31); if (buoyTypeCode == null) { - //FIXME should fail? + // If null, then consider it as not found in avdth buoyTypeCode = "999"; } - String buoyId = resultSet.getString(32); - String dcpWeight = resultSet.getString(33); - if (dcpWeight != null) { - //FIXME Check if ok - entity.setComment(String.format("ACTIVITE.POIDS_ESTIM_DCP value from avdth: %s", dcpWeight)); - } - ObservedSystem observedSystem = dataContext.getObservedSystem(objectTypeCode, buoyTypeCode); + // IS buoy exists in avdth? + boolean buoyExistsInAvdth = !"999".equals(buoyTypeCode); + + ObservedSystem observedSystem = getObservedSystem(dataContext, objectTypeCode, floatingObjectExistsInAvdth, buoyExistsInAvdth); if (observedSystem != null) { - dataContext.getActivity().addObservedSystem(observedSystem); + activity.addObservedSystem(observedSystem); } - // avdth data has a buoy - boolean willCreateBuoy = !"999".equals(buoyTypeCode); - // to add default material if there is a declared buoy and no object material created by ObservedSystem + + boolean addBuoyFromObservedSystem = addFloatingObjectMaterials(dataContext, + resultSet, + vesselActivityCode, + objectTypeCode, + observedSystemCodes, + entity, + floatingObjectExistsInAvdth, + buoyExistsInAvdth); + + TransmittingBuoy transmittingBuoy = null; + if (buoyExistsInAvdth) { + transmittingBuoy = getBuoyFromAvdthActivity(dataContext, resultSet, buoyTypeCode, vesselActivityCode, objectOperation, activity); + } else if (addBuoyFromObservedSystem) { + transmittingBuoy = getBuoyFromObservedSystem(dataContext); + } + if (transmittingBuoy != null) { + // only add buoy if buoy type known + entity.addTransmittingBuoy(transmittingBuoy); + } + reset(dataContext); + return entity; + } + + private boolean addFloatingObjectMaterials(ImportDataContext dataContext, + ResultSet resultSet, + String vesselActivityCode, + String objectTypeCode, + Set<String> observedSystemCodes, + FloatingObject entity, + boolean floatingObjectExistsInAvdth, + boolean buoyExistsInAvdth) throws SQLException { + boolean whenArriving = WHEN_ARRIVING_CODE.contains(vesselActivityCode); + boolean whenLeaving = WHEN_LEAVING_CODE.contains(vesselActivityCode); + + // to add default material if there is a declared buoy (but no floating object declared) + // and no object material created by ObservedSystem, we will then add a standalone buoy boolean addDefaultObjectMaterialOnUnknownObjectType = true; // should create a buoy from ObservedSystem boolean addBuoyFromObservedSystem = false; Set<String> objectMaterialUsed = new TreeSet<>(); + + if (floatingObjectExistsInAvdth) { + Object dcpEcoCode = resultSet.getObject(29); + if (dcpEcoCode != null && Objects.equals(2, dcpEcoCode)) { + // add material 4-1 (Biodegradable materials) + addObjectMaterial(vesselActivityCode, objectMaterialUsed, dataContext.getObjectMaterialBiodegradable(), whenArriving, whenLeaving, entity); + } + } + if (observedSystemCodes.contains("20")) { // add ObjectMaterial FOB - addObjectMaterial(objectMaterialUsed, dataContext.getObjectMaterialFOB(), whenArriving, whenLeaving, entity); + addObjectMaterial(vesselActivityCode, objectMaterialUsed, dataContext.getObjectMaterialFOB(), whenArriving, whenLeaving, entity); addDefaultObjectMaterialOnUnknownObjectType = false; } else if (observedSystemCodes.contains("21")) { // add ObjectMaterial 2-1 NLOG - addObjectMaterial(objectMaterialUsed, dataContext.getObjectMaterialNLOG(), whenArriving, whenLeaving, entity); + addObjectMaterial(vesselActivityCode, objectMaterialUsed, dataContext.getObjectMaterialNLOG(), whenArriving, whenLeaving, entity); addDefaultObjectMaterialOnUnknownObjectType = false; } else if (observedSystemCodes.contains("22")) { // add ObjectMaterial 2-1 NLOG + 1-1 DFAD - addObjectMaterial(objectMaterialUsed, dataContext.getObjectMaterialNLOG(), whenArriving, whenLeaving, entity); - addObjectMaterial(objectMaterialUsed, dataContext.getObjectMaterialDFAD(), whenArriving, whenLeaving, entity); + addObjectMaterial(vesselActivityCode, objectMaterialUsed, dataContext.getObjectMaterialNLOG(), whenArriving, whenLeaving, entity); + addObjectMaterial(vesselActivityCode, objectMaterialUsed, dataContext.getObjectMaterialDFAD(), whenArriving, whenLeaving, entity); addDefaultObjectMaterialOnUnknownObjectType = false; addBuoyFromObservedSystem = true; } else if (observedSystemCodes.contains("23")) { // add ObjectMaterial 2-2 ALOG - addObjectMaterial(objectMaterialUsed, dataContext.getObjectMaterialALOG(), whenArriving, whenLeaving, entity); + addObjectMaterial(vesselActivityCode, objectMaterialUsed, dataContext.getObjectMaterialALOG(), whenArriving, whenLeaving, entity); addDefaultObjectMaterialOnUnknownObjectType = false; } else if (observedSystemCodes.contains("24")) { // add ObjectMaterial 2-2 ALOG + 1-1 DFAD - addObjectMaterial(objectMaterialUsed, dataContext.getObjectMaterialALOG(), whenArriving, whenLeaving, entity); - addObjectMaterial(objectMaterialUsed, dataContext.getObjectMaterialDFAD(), whenArriving, whenLeaving, entity); + addObjectMaterial(vesselActivityCode, objectMaterialUsed, dataContext.getObjectMaterialALOG(), whenArriving, whenLeaving, entity); + addObjectMaterial(vesselActivityCode, objectMaterialUsed, dataContext.getObjectMaterialDFAD(), whenArriving, whenLeaving, entity); addDefaultObjectMaterialOnUnknownObjectType = false; addBuoyFromObservedSystem = true; } else if (observedSystemCodes.contains("25")) { // add ObjectMaterial 1-2 AFAD - addObjectMaterial(objectMaterialUsed, dataContext.getObjectMaterialAFAD(), whenArriving, whenLeaving, entity); + addObjectMaterial(vesselActivityCode, objectMaterialUsed, dataContext.getObjectMaterialAFAD(), whenArriving, whenLeaving, entity); addDefaultObjectMaterialOnUnknownObjectType = false; } - if ("999".equals(objectTypeCode)) { - if (willCreateBuoy && addDefaultObjectMaterialOnUnknownObjectType) { - // add ObjectMaterial 2-2-4-4 - addObjectMaterial(objectMaterialUsed, dataContext.getObjectMaterialAlone(), whenArriving, whenLeaving, entity); + if (floatingObjectExistsInAvdth) { + // simple mapping + ObjectMaterial objectMaterial = getObjectMaterialByObjectTypeCode(dataContext, objectTypeCode); + if (objectMaterial != null) { + addObjectMaterial(vesselActivityCode, objectMaterialUsed, objectMaterial, whenArriving, whenLeaving, entity); } } else { - // simple mapping - ObjectMaterial objectMaterial1 = dataContext.getObjectMaterialByObjectTypeCode(objectTypeCode); - if (objectMaterial1 != null) { - addObjectMaterial(objectMaterialUsed, objectMaterial1, whenArriving, whenLeaving, entity); + if (buoyExistsInAvdth && addDefaultObjectMaterialOnUnknownObjectType) { + // add ObjectMaterial 2-2-4-4 + addObjectMaterial(vesselActivityCode, objectMaterialUsed, dataContext.getObjectMaterialAlone(), whenArriving, whenLeaving, entity); } } - if (willCreateBuoy) { - // only add buoy if buoy type known - TransmittingBuoyType transmittingBuoyType = dataContext.getTransmittingBuoyTypeByBuoyTypeCode(buoyTypeCode); - TransmittingBuoyOperation transmittingBuoyOperation = dataContext.getTransmittingBuoyOperationByVesselActivityCode(vesselActivityCode); - if (transmittingBuoyOperation != null) { - TransmittingBuoy transmittingBuoy = newEntity(TransmittingBuoy.SPI, transmittingBuoyCount); - transmittingBuoy.setTransmittingBuoyOperation(transmittingBuoyOperation); - transmittingBuoy.setTransmittingBuoyType(transmittingBuoyType); - transmittingBuoy.setCode(buoyId); - if (addCoordinate) { - transmittingBuoy.setLatitude(dataContext.getActivity().getLatitude()); - transmittingBuoy.setLongitude(dataContext.getActivity().getLongitude()); - } - TransmittingBuoyOwnership transmittingBuoyOwnership = dataContext.getTransmittingBuoyOwnership(buoyOwnershipCode); - transmittingBuoy.setTransmittingBuoyOwnership(transmittingBuoyOwnership); - entity.addTransmittingBuoy(transmittingBuoy); - } - } else if (addBuoyFromObservedSystem) { - // only add buoy if observed system ask for it (22 and 24) - TransmittingBuoyType transmittingBuoyType = dataContext.getTransmittingBuoyType98(); - TransmittingBuoyOperation transmittingBuoyOperation = dataContext.getTransmittingBuoyOperation1(); + return addBuoyFromObservedSystem; + } + + private TransmittingBuoy getBuoyFromAvdthActivity(ImportDataContext dataContext, ResultSet resultSet, String buoyTypeCode, String vesselActivityCode, ObjectOperation objectOperation, Activity activity) throws SQLException { + boolean addCoordinate = "11".equals(objectOperation.getCode()); + TransmittingBuoyType transmittingBuoyType = getTransmittingBuoyTypeByBuoyTypeCode(dataContext, buoyTypeCode); + TransmittingBuoyOperation transmittingBuoyOperation = getTransmittingBuoyOperationByVesselActivityCode(dataContext, vesselActivityCode); + if (transmittingBuoyOperation != null) { + String buoyOwnershipCode = resultSet.getString(30); + String buoyId = resultSet.getString(32); TransmittingBuoy transmittingBuoy = newEntity(TransmittingBuoy.SPI, transmittingBuoyCount); transmittingBuoy.setTransmittingBuoyOperation(transmittingBuoyOperation); transmittingBuoy.setTransmittingBuoyType(transmittingBuoyType); - transmittingBuoy.setCode(null); - entity.addTransmittingBuoy(transmittingBuoy); + transmittingBuoy.setCode(buoyId); + if (addCoordinate) { + transmittingBuoy.setLatitude(activity.getLatitude()); + transmittingBuoy.setLongitude(activity.getLongitude()); + } + TransmittingBuoyOwnership transmittingBuoyOwnership = getTransmittingBuoyOwnership(dataContext, buoyOwnershipCode); + transmittingBuoy.setTransmittingBuoyOwnership(transmittingBuoyOwnership); + return transmittingBuoy; } - reset(dataContext); - return entity; + return null; + } + + private TransmittingBuoy getBuoyFromObservedSystem(ImportDataContext dataContext) { + // only add buoy if observed system ask for it (22 and 24) + TransmittingBuoyType transmittingBuoyType = dataContext.getTransmittingBuoyType98(); + TransmittingBuoyOperation transmittingBuoyOperation = dataContext.getTransmittingBuoyOperation1(); + TransmittingBuoy transmittingBuoy = newEntity(TransmittingBuoy.SPI, transmittingBuoyCount); + transmittingBuoy.setTransmittingBuoyOperation(transmittingBuoyOperation); + transmittingBuoy.setTransmittingBuoyType(transmittingBuoyType); + transmittingBuoy.setCode(null); + return transmittingBuoy; } private void reset(ImportDataContext dataContext) { @@ -357,9 +442,9 @@ public class FloatingObjectReader extends DataReader<FloatingObject> { dataContext.setObservedSystemCodes(null); } - private void addObjectMaterial(Set<String> objectMaterialUsed, ObjectMaterial objectMaterial, boolean whenArriving, boolean whenLeaving, FloatingObject floatingObject) { + private void addObjectMaterial(String vesselActivityCode, Set<String> objectMaterialUsed, ObjectMaterial objectMaterial, boolean whenArriving, boolean whenLeaving, FloatingObject floatingObject) { if (!whenArriving && !whenLeaving) { - throw new IllegalStateException("Can't have not whenArriving and not whenLeaving "); + throw new IllegalStateException(String.format("Can't have not whenArriving and not whenLeaving from ACTIVITE.C_OPERA: %s", vesselActivityCode)); } if (!objectMaterialUsed.add(objectMaterial.getTopiaId())) { // already used @@ -380,4 +465,80 @@ public class FloatingObjectReader extends DataReader<FloatingObject> { return floatingObjectPartCount.intValue(); } + public Map<String, MutableInt> getBadVesselActivityCodes() { + return badVesselActivityCodes; + } + + private ObjectOperation getObjectOperationByVesselActivityCode(ImportDataContext dataContext, String vesselActivityCode) { + String objectOperationCode = DCP_OBJECT_OPERATION_CODE_MAPPING.get(vesselActivityCode); + return dataContext.getObjectOperation(Objects.requireNonNull(objectOperationCode, String.format("Can't find objectOperation with ACTIVITE.C_OPERA: %s", vesselActivityCode))); + } + + private ObservedSystem getObservedSystem(ImportDataContext dataContext, String objectTypeCode, boolean floatingObjectExistsInAvdth,boolean buoyExistsInAvdth) { + if (!floatingObjectExistsInAvdth) { + if (!buoyExistsInAvdth) { + return dataContext.getObservedSystem0(); + } + return dataContext.getObservedSystem20(); + } + String observedSystemCode = OBSERVED_SYSTEM_CODE_MAPPING.get(objectTypeCode); + return dataContext.getObservedSystem(observedSystemCode, false); + } + + private TransmittingBuoyOwnership getTransmittingBuoyOwnership(ImportDataContext dataContext, String buoyOwnershipCode) { + if (buoyOwnershipCode == null) { + return null; + } + String transmittingBuoyOwnershipCode = TRANSMITTING_BUOY_OWNERSHIP_CODE_MAPPING.get(buoyOwnershipCode); + if (transmittingBuoyOwnershipCode == null) { + return null; + } + return dataContext.getTransmittingBuoyOwnership(transmittingBuoyOwnershipCode); + } + + private TransmittingBuoyOperation getTransmittingBuoyOperationByVesselActivityCode(ImportDataContext dataContext, String vesselActivityCode) { + String transmittingBuoyTypeCode = TRANSMITTING_BUOY_OPERATION_CODE_MAPPING.get(vesselActivityCode); + if (transmittingBuoyTypeCode == null) { + return null; + } + return dataContext.getTransmittingBuoyOperation(transmittingBuoyTypeCode); + } + + + private TransmittingBuoyType getTransmittingBuoyTypeByBuoyTypeCode(ImportDataContext dataContext, String buoyTypeCode) { + String transmittingBuoyTypeCode = TRANSMITTING_BUOY_TYPE_CODE_MAPPING.get(buoyTypeCode); + if (transmittingBuoyTypeCode == null) { + transmittingBuoyTypeCode = "999"; + } + TransmittingBuoyType transmittingBuoyType = dataContext.getTransmittingBuoyType(transmittingBuoyTypeCode); + return Objects.requireNonNull(transmittingBuoyType, String.format("Can't find transmittingBuoyType with code: %s", buoyTypeCode)); + } + + /** + * To get objectMaterial associated by a DCP (or null) from avdth ACTIVITE.C_TYP_OBJET code. + * + * @param objectTypeCode ACTIVITE.C_TYP_OBJET + * @return material or null + */ + private ObjectMaterial getObjectMaterialByObjectTypeCode(ImportDataContext dataContext, String objectTypeCode) { + switch (objectTypeCode) { + case "1": + return dataContext.getObjectMaterialAFAD(); + case "2": + return dataContext.getObjectMaterialDFAD(); + case "3": + return dataContext.getObjectMaterialLOG(); + case "9": + return dataContext.getObjectMaterialFOB(); + case "10": + return dataContext.getObjectMaterialFALOG(); + case "11": + return dataContext.getObjectMaterialHALOG(); + case "12": + return dataContext.getObjectMaterialANLOG(); + case "13": + return dataContext.getObjectMaterialVNLOG(); + } + return null; + } } ===================================== core/persistence/avdth/src/main/java/fr/ird/observe/persistence/avdth/data/logbook/FloatingObjectWriter.java ===================================== @@ -192,6 +192,9 @@ public class FloatingObjectWriter extends DataWriter<FloatingObject, FloatingObj super.toResult(resultBuilder); toResult(resultBuilder, FloatingObjectPart.class, getReader().getFloatingObjectPartCount()); toResult(resultBuilder, TransmittingBuoy.class, getReader().getTransmittingBuoyCount()); +// getReader().getBadVesselActivityCodes().forEach((k,v)-> { +// resultBuilder.put(String.format("Bad Vessel activity used to build FAD from ACTIVITY.C_OPERA %s used but ObServe not allow fad.", k), v.getValue()); +// }); } } ===================================== core/persistence/avdth/src/main/java/fr/ird/observe/persistence/avdth/data/logbook/SampleActivityReader.java ===================================== @@ -133,7 +133,7 @@ public class SampleActivityReader extends DataReader<SampleActivity> { if (!dataContext.isCanCreateActivity()) { String message = String.format("Sample [%20s], not allowed to create missing activity and could not find Activity [%20s]", samplePk, activityPk); log.debug(message); - context().addMessage(message); + addMessage(message); return null; } // only on this case we can create missing route and activity (See #2070) ===================================== core/persistence/avdth/src/main/java/fr/ird/observe/persistence/avdth/data/logbook/WellPlanReader.java ===================================== @@ -254,7 +254,7 @@ public class WellPlanReader extends DataReader<WellPlan> { String wellSetPk = Query.primaryKeyString(Query.getPrimaryKey(resultSet, 1, 3)); String message = String.format("WellPlan [%20s], not allowed to create missing activity and could not find Activity [%20s]", wellSetPk, activityPk); log.debug(message); - context().addMessage(message); + addMessage(message); return null; } else { activity = new ActivityImpl(); ===================================== core/persistence/test/pom.xml ===================================== @@ -30,6 +30,7 @@ <description>ObServe Core Persistence Test module</description> <properties> <test_forceImport>false</test_forceImport> + <test_forcePrepare>false</test_forcePrepare> <test_allYears>false</test_allYears> </properties> <dependencies> @@ -109,6 +110,7 @@ <systemPropertyVariables> <test_forceImport>${test_forceImport}</test_forceImport> <test_allYears>${test_allYears}</test_allYears> + <test_forcePrepare>${test_forcePrepare}</test_forcePrepare> </systemPropertyVariables> </configuration> </plugin> ===================================== core/persistence/test/src/test/java/fr/ird/observe/persistence/avdth/data/AvdthAtlanticDataBuilderByYearLatestTest.java ===================================== @@ -0,0 +1,54 @@ +package fr.ird.observe.persistence.avdth.data; + +/*- + * #%L + * ObServe Core :: Persistence :: Test + * %% + * Copyright (C) 2008 - 2023 IRD, Ultreia.io + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/gpl-3.0.html>. + * #L% + */ + +import fr.ird.observe.dto.ProtectedIdsCommon; +import org.junit.runners.Parameterized; + +import java.nio.file.Path; + +/** + * Created on 19/01/2023. + * + * @author Tony Chemit - dev@tchemit.fr + * @since 9.0.24 + */ +public class AvdthAtlanticDataBuilderByYearLatestTest extends AvdthDataBuilderByYearTestSupport { + + public static final Path ROOT_PATH = getPath("OA").resolve("latest"); + + @Parameterized.Parameters(name = "{index}: {0}") + public static Iterable<?> data() { + return getDatabases(ROOT_PATH); + } + + @Override + protected Path getRootPath() { + return ROOT_PATH; + } + + @Override + protected String getOceanId() { + return ProtectedIdsCommon.COMMON_OCEAN_ATLANTIC; + } +} ===================================== core/persistence/test/src/test/java/fr/ird/observe/persistence/avdth/data/AvdthDataBuilderByYearTestSupport.java ===================================== @@ -37,7 +37,7 @@ public abstract class AvdthDataBuilderByYearTestSupport extends AvdthDataBuilder public static List<String> getDatabases(Path path) { List<String> databases = allDatabases(path); - return AvdthFixtures.IMPORT_ALL_YEARS ? databases : databases.subList(0, 3); + return AvdthFixtures.IMPORT_ALL_YEARS || databases.size() < 3 ? databases : databases.subList(0, 3); } @Override ===================================== core/persistence/test/src/test/java/fr/ird/observe/persistence/avdth/data/AvdthDataBuilderTestSupport.java ===================================== @@ -169,7 +169,15 @@ public abstract class AvdthDataBuilderTestSupport extends PersistenceTestSupport } protected void checkNotValidResult(AvdthDataImportResult result) { - Assume.assumeTrue(result.getNotReadResult() + "", forceImport()); + Map<String, Integer> notReadResult = result.getNotReadResult(); + String message; + if (notReadResult.isEmpty()) { + message = result.getMessages() + ""; + } else { + message = notReadResult + ""; + return; + } + Assume.assumeTrue(message, forceImport()); } protected long count() { ===================================== core/persistence/test/src/test/java/fr/ird/observe/persistence/avdth/data/AvdthIndianDataBuilderByYearLatestTest.java ===================================== @@ -0,0 +1,54 @@ +package fr.ird.observe.persistence.avdth.data; + +/*- + * #%L + * ObServe Core :: Persistence :: Test + * %% + * Copyright (C) 2008 - 2023 IRD, Ultreia.io + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/gpl-3.0.html>. + * #L% + */ + +import fr.ird.observe.dto.ProtectedIdsCommon; +import org.junit.runners.Parameterized; + +import java.nio.file.Path; + +/** + * Created on 19/01/2023. + * + * @author Tony Chemit - dev@tchemit.fr + * @since 9.0.24 + */ +public class AvdthIndianDataBuilderByYearLatestTest extends AvdthDataBuilderByYearTestSupport { + + public static final Path ROOT_PATH = getPath("OI").resolve("latest"); + + @Parameterized.Parameters(name = "{index}: {0}") + public static Iterable<?> data() { + return getDatabases(ROOT_PATH); + } + + @Override + protected Path getRootPath() { + return ROOT_PATH; + } + + @Override + protected String getOceanId() { + return ProtectedIdsCommon.COMMON_OCEAN_INDIAN; + } +} ===================================== pom.xml ===================================== @@ -127,6 +127,8 @@ <!-- FIXME <lib.version.jaxb-runtime>4.0.0</lib.version.jaxb-runtime>--> <lib.version.jts>1.16.1</lib.version.jts> <!-- FIXME <lib.version.jts>1.18.2</lib.version.jts>--> + <lib.version.hsqldb>2.7.0</lib.version.hsqldb> + <!-- FIXME <lib.version.hsqldb>2.7.1</lib.version.hsqldb>--> <!-- license header configuration --> <license.licenseName>gpl_v3</license.licenseName> <license.organizationName>IRD, Ultreia.io</license.organizationName> ===================================== src/site/markdown/avdth/logbook.md ===================================== @@ -97,7 +97,7 @@ Voir ajout d'un DCP * Note 11 ### <a name="n_0_11"></a> -Donnée non persistée dans ObServe, on la conserve dans le commentaire du DCP +Donnée non persistée dans ObServe, (voir https://gitlab.com/ultreiaio/ird-observe/-/issues/2575#note_1247253501). * Note 12 ### <a name="n_0_12"></a> View it on GitLab: https://gitlab.com/ultreiaio/ird-observe/-/compare/d0a946c1012db57c80393438d... -- View it on GitLab: https://gitlab.com/ultreiaio/ird-observe/-/compare/d0a946c1012db57c80393438d... You're receiving this email because of your account on gitlab.com.