Author: tchemit Date: 2011-12-16 18:54:16 +0100 (Fri, 16 Dec 2011) New Revision: 199 Url: http://forge.codelutin.com/repositories/revision/echobase/199 Log: - fix test resources - continue Catches legacy datas import Modified: trunk/echobase-entities/src/main/java/fr/ifremer/echobase/EchoBaseFunctions.java trunk/echobase-entities/src/main/resources/i18n/echobase-entities_fr_FR.properties trunk/echobase-entities/src/main/xmi/echobase.zargo trunk/echobase-tools/pom.xml trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/BiometryScaleLoader.java trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/OperationLoader.java trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/SampleLoader.java trunk/echobase-tools/src/test/java/fr/ifremer/echobase/tools/loaders/SampleLoaderIT.java trunk/src/doc/reunions/reunion-2011-12-14.txt Modified: trunk/echobase-entities/src/main/java/fr/ifremer/echobase/EchoBaseFunctions.java =================================================================== --- trunk/echobase-entities/src/main/java/fr/ifremer/echobase/EchoBaseFunctions.java 2011-12-16 01:46:20 UTC (rev 198) +++ trunk/echobase-entities/src/main/java/fr/ifremer/echobase/EchoBaseFunctions.java 2011-12-16 17:54:16 UTC (rev 199) @@ -45,7 +45,6 @@ import org.apache.commons.lang3.StringUtils; import org.nuiton.topia.persistence.TopiaEntity; -import javax.annotation.Nullable; import java.io.File; import java.util.Map; @@ -165,7 +164,7 @@ return input.getName(); } }; - public static final Function<SampleDataType, String> SAMPLE_DATA_TYPE_RAPRI_CODE = new Function<SampleDataType, String>() { + public static final Function<SampleDataType, String> SAMPLE_DATA_TYPE_RAPTRI_CODE = new Function<SampleDataType, String>() { @Override public String apply(SampleDataType input) { return String.valueOf(input.getRaptriBiometryDataType()); @@ -186,6 +185,7 @@ return StringUtils.isEmpty(casinoGearName) ? input.getName() : casinoGearName; } }; + public static final Function<Operation, String> OPERATION_ID = new Function<Operation, String>() { @Override public String apply(Operation input) { Modified: trunk/echobase-entities/src/main/resources/i18n/echobase-entities_fr_FR.properties =================================================================== --- trunk/echobase-entities/src/main/resources/i18n/echobase-entities_fr_FR.properties 2011-12-16 01:46:20 UTC (rev 198) +++ trunk/echobase-entities/src/main/resources/i18n/echobase-entities_fr_FR.properties 2011-12-16 17:54:16 UTC (rev 199) @@ -138,6 +138,8 @@ echobase.common.metadata=metadata echobase.common.metadataGroup= echobase.common.metadataType= +echobase.common.midHauleLatitude= +echobase.common.midHauleLongitude= echobase.common.minLatitude= echobase.common.minLongitude= echobase.common.mission=mission Modified: trunk/echobase-entities/src/main/xmi/echobase.zargo =================================================================== (Binary files differ) Modified: trunk/echobase-tools/pom.xml =================================================================== --- trunk/echobase-tools/pom.xml 2011-12-16 01:46:20 UTC (rev 198) +++ trunk/echobase-tools/pom.xml 2011-12-16 17:54:16 UTC (rev 199) @@ -205,6 +205,12 @@ <include>**/*.csv</include> </includes> </testResource> + <testResource> + <directory>src/test/resources</directory> + <includes> + <include>**/*.properties</include> + </includes> + </testResource> </testResources> </build> </profile> Modified: trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/BiometryScaleLoader.java =================================================================== --- trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/BiometryScaleLoader.java 2011-12-16 01:46:20 UTC (rev 198) +++ trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/BiometryScaleLoader.java 2011-12-16 17:54:16 UTC (rev 199) @@ -58,7 +58,7 @@ /* raptriBiometryDataType;ScaleCode;ScaleMeaningEng;ScaleMeaningFr */ - model.addForeignKeyForImport("raptriBiometryDataType", BiometryScale.PROPERTY_SAMPLE_DATA_TYPE, SampleDataType.class, sampleDataTypes, EchoBaseFunctions.SAMPLE_DATA_TYPE_RAPRI_CODE); + model.addForeignKeyForImport("raptriBiometryDataType", BiometryScale.PROPERTY_SAMPLE_DATA_TYPE, SampleDataType.class, sampleDataTypes, EchoBaseFunctions.SAMPLE_DATA_TYPE_RAPTRI_CODE); model.addDefaultColumn("ScaleCode",BiometryScale.PROPERTY_CODE, int.class); model.newMandatoryColumn("ScaleMeaningEng",BiometryScale.PROPERTY_MEANING_EN); model.newMandatoryColumn("ScaleMeaningFr",BiometryScale.PROPERTY_MEANING_FR); Modified: trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/OperationLoader.java =================================================================== --- trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/OperationLoader.java 2011-12-16 01:46:20 UTC (rev 198) +++ trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/OperationLoader.java 2011-12-16 17:54:16 UTC (rev 199) @@ -119,40 +119,63 @@ copyAndLoadEntities(outputDir, meta.getAssociations(Operation.PROPERTY_OPERATION_METADATA_VALUE), false, FileType.ASSOCIATION); copyAndLoadEntities(outputDir, meta.getAssociations(Operation.PROPERTY_GEAR_METADATA_VALUE), false, FileType.ASSOCIATION); - } + //SONDE + OperationMetadata MeanWaterDepthOperationMeta; + //JOUR_NUIT + OperationMetadata DielPeriodOperationMeta; + //SONDEDEB + OperationMetadata StartWaterDepthOperationMeta; + //SONDEFIN + OperationMetadata EndWaterDepthOperationMeta; + + //LFUNES + GearMetadata CableLengthGearMeta; + //VMIN + GearMetadata MinSpeedGearMeta; + //VMAX + GearMetadata MaxSpeedGearMeta; + + TopiaDAO<Operation> dao; + TopiaDAO<OperationMetadataValue> operationMetadataValueDAO; + TopiaDAO<GearMetadataValue> gearMetadataValueDAO; + + Map<String, DepthStratum> depthStratumByName; + Map<String, Gear> gearByCasinoGearName; + @Override - protected void importBaracoudaEntities(Database db) throws IOException, TopiaException { + protected void importBaracoudaEntities(Database db) throws + IOException, TopiaException { Map<String, Voyage> voyagesByName = getUniverse( Voyage.class, EchoBaseFunctions.VOYAGE_NAME); - Map<String, DepthStratum> depthStratumByName = getUniverse( + depthStratumByName = getUniverse( DepthStratum.class, EchoBaseFunctions.DEPTH_STRATUM_ID); - Map<String, Gear> gearByCasinoGearName = getUniverse( + gearByCasinoGearName = getUniverse( Gear.class, EchoBaseFunctions.GEAR_CASINO_GEAR_NAME); OperationMetadataDAO operationMetadataDAO = (OperationMetadataDAO) getDAO(OperationMetadata.class); //SONDE - OperationMetadata MeanWaterDepthOperationMeta = + MeanWaterDepthOperationMeta = operationMetadataDAO.findByName("MeanWaterDepth"); Preconditions.checkNotNull(MeanWaterDepthOperationMeta); //JOUR_NUIT - OperationMetadata DielPeriodOperationMeta = + DielPeriodOperationMeta = operationMetadataDAO.findByName("DielPeriod"); Preconditions.checkNotNull(DielPeriodOperationMeta); //SONDEDEB - OperationMetadata StartWaterDepthOperationMeta = operationMetadataDAO.findByProperties( + StartWaterDepthOperationMeta = operationMetadataDAO.findByProperties( OperationMetadata.PROPERTY_NAME, "WaterDepth", OperationMetadata.PROPERTY_OPERATION_EVENT + "." + OperationEvent.PROPERTY_NAME, "ShootingStart" ); Preconditions.checkNotNull(StartWaterDepthOperationMeta); //SONDEFIN - OperationMetadata EndWaterDepthOperationMeta = operationMetadataDAO.findByProperties( + EndWaterDepthOperationMeta = operationMetadataDAO.findByProperties( OperationMetadata.PROPERTY_NAME, "WaterDepth", OperationMetadata.PROPERTY_OPERATION_EVENT + "." + OperationEvent.PROPERTY_NAME, "HaulingEnd" ); @@ -162,24 +185,19 @@ (GearMetadataDAO) getDAO(GearMetadata.class); //LFUNES - GearMetadata CableLengthGearMeta = gearMetadataDAO.findByType("CableLength"); + CableLengthGearMeta = gearMetadataDAO.findByType("CableLength"); Preconditions.checkNotNull(CableLengthGearMeta); - //VMIN - GearMetadata MinSpeedGearMeta = gearMetadataDAO.findByType("MinSpeed"); + MinSpeedGearMeta = gearMetadataDAO.findByType("MinSpeed"); Preconditions.checkNotNull(MinSpeedGearMeta); - //VMAX - GearMetadata MaxSpeedGearMeta = gearMetadataDAO.findByType("MaxSpeed"); + MaxSpeedGearMeta = gearMetadataDAO.findByType("MaxSpeed"); Preconditions.checkNotNull(MaxSpeedGearMeta); - TopiaDAO<Operation> dao = getDAO(); - TopiaDAO<OperationMetadataValue> operationMetadataValueDAO = - getDAO(OperationMetadataValue.class); + dao = getDAO(); + operationMetadataValueDAO = getDAO(OperationMetadataValue.class); + gearMetadataValueDAO = getDAO(GearMetadataValue.class); - TopiaDAO<GearMetadataValue> gearMetadataValueDAO = - getDAO(GearMetadataValue.class); - Multimap<String, Map<String, Object>> stationByCampagneName = Multimaps.index( db.getTable("STATIONS"), EchoBaseFunctions.newRowFunction("CAMPAGNE")); @@ -196,142 +214,169 @@ Preconditions.checkNotNull(transect); Collection<Map<String, Object>> rows = stationByCampagneName.get(voyageName); + for (Map<String, Object> row : rows) { - String status = (String) row.get("ETAT"); - if ("FICTIF".equalsIgnoreCase(status)) { + Operation operation = addOperation(row, messages); + if (operation != null) { + operations.add(operation); + transect.addOperation(operation); - // skip fictif operation - continue; - } - String operationId = (String) row.get("NOSTA"); + if (!messages.isEmpty()) { - DepthStratum depthStratum = null; - String depthStratumId = (String) row.get("STA_IMAGES"); - if (StringUtils.isBlank(depthStratumId) || - "nul".equals(depthStratumId)) { - messages.add("No depthStraum defined"); - } else { - depthStratum = getFK(depthStratumByName, depthStratumId.trim()); + if (log.isWarnEnabled()) { + log.warn("For operation " + operation.getId() + " : " + messages); + } + messages.clear(); + } } + } + } - String gearId = (String) row.get("ENGIN"); - if (gearId == null) { - messages.add("No gear id"); - } else { - Gear gear = getFK(gearByCasinoGearName, gearId); + if (log.isInfoEnabled()) { + log.info("Found " + operations.size() + " operation(s)."); + } + } - Float shootingStartLatitude = (Float) row.get("LATDEB"); - if (shootingStartLatitude == null) { - messages.add("No shootingStartLatitude"); - shootingStartLatitude = 0f; - } - Float shootingEndLatitude = (Float) row.get("LATDD"); - if (shootingEndLatitude == null) { - messages.add("No shootingEndLatitude"); - shootingEndLatitude = 0f; - } - Float shootingStartLongitude = (Float) row.get("LGDEB"); - if (shootingStartLongitude == null) { - messages.add("No shootingStartLongitude"); - shootingStartLongitude = 0f; - } - Float shootingEndLongitude = (Float) row.get("LGDD"); - if (shootingEndLongitude == null) { - messages.add("No shootingEndLongitude"); - shootingEndLongitude = 0f; - } + protected Operation addOperation(Map<String, Object> row, + List<String> messages) throws TopiaException { + String status = (String) row.get("ETAT"); - Date shootingStartTime = (Date) row.get("DHTUDEB"); - Date shootingEndTime = (Date) row.get("DHTUFIN"); + if ("FICTIF".equalsIgnoreCase(status)) { - String comment = (String) row.get("COMMENTAIRE"); + // skip fictif operation + return null; + } - // create a operation - Operation operation = dao.create( - Operation.PROPERTY_ID, operationId, - Operation.PROPERTY_DEPTH_STRATUM, depthStratum, - Operation.PROPERTY_GEAR, gear, - Operation.PROPERTY_GEAR_SHOOTING_START_LATITUDE, shootingStartLatitude, - Operation.PROPERTY_GEAR_SHOOTING_END_LATITUDE, shootingEndLatitude, - Operation.PROPERTY_GEAR_SHOOTING_START_LONGITUDE, shootingStartLongitude, - Operation.PROPERTY_GEAR_SHOOTING_END_LONGITUDE, shootingEndLongitude, - Operation.PROPERTY_GEAR_SHOOTING_START_TIME, shootingStartTime, - Operation.PROPERTY_GEAR_SHOOTING_END_TIME, shootingEndTime, - Operation.PROPERTY_COMMENT, comment - ); + String operationId = (String) row.get("NOSTA"); - addOperationMetadataValue(operationMetadataValueDAO, - operation, - MeanWaterDepthOperationMeta, - row.get("SONDE")); + String gearId = (String) row.get("ENGIN"); + if (gearId == null) { + if (log.isWarnEnabled()) { + log.warn("No gear id for Operation " + operationId); + } + return null; + } - addOperationMetadataValue(operationMetadataValueDAO, - operation, - DielPeriodOperationMeta, - row.get("JOUR_NUIT")); + DepthStratum depthStratum = null; + String depthStratumId = (String) row.get("STA_IMAGES"); + if (StringUtils.isBlank(depthStratumId) || + "nul".equals(depthStratumId)) { + messages.add("No depthStraum defined"); + } else { + depthStratum = getFK(depthStratumByName, depthStratumId.trim()); + } - addOperationMetadataValue(operationMetadataValueDAO, - operation, - StartWaterDepthOperationMeta, - row.get("SONDEDEB")); + Gear gear = getFK(gearByCasinoGearName, gearId); - addOperationMetadataValue(operationMetadataValueDAO, - operation, - EndWaterDepthOperationMeta, - row.get("SONDEFIN")); + Float shootingStartLatitude = (Float) row.get("LATDEB"); + if (shootingStartLatitude == null) { + messages.add("No shootingStartLatitude"); + } - addGearMetadataValue(gearMetadataValueDAO, - operation, - CableLengthGearMeta, - row.get("LFUNES")); + Double doubleValue = (Double) row.get("LATFIN"); + Float shootingEndLatitude; + if (doubleValue == null) { + messages.add("No shootingEndLatitude"); + shootingEndLatitude = null; + } else { + shootingEndLatitude = doubleValue.floatValue(); + } - addGearMetadataValue(gearMetadataValueDAO, - operation, - MinSpeedGearMeta, - row.get("VMIN")); + Float shootingStartLongitude = (Float) row.get("LGDEB"); + if (shootingStartLongitude == null) { + messages.add("No shootingStartLongitude"); + } - addGearMetadataValue(gearMetadataValueDAO, - operation, - MaxSpeedGearMeta, - row.get("VMAX")); + doubleValue = (Double) row.get("LGFIN"); + Float shootingEndLongitude; + if (doubleValue == null) { + messages.add("No shootingEndLongitude"); + shootingEndLongitude = null; + } else { + shootingEndLongitude = doubleValue.floatValue(); + } - operations.add(operation); - transect.addOperation(operation); + Float midHauleLatitude = (Float) row.get("LATDD"); + Preconditions.checkNotNull(midHauleLatitude); +// if (midHauleLatitude == null) { +// messages.add("No midHauleLatitude"); +// midHauleLatitude = 0f; +// } - } - if (!messages.isEmpty()) { + Float midHauleLongitude = (Float) row.get("LGDD"); + Preconditions.checkNotNull(midHauleLongitude); +// if (midHauleLongitude== null) { +// messages.add("No midHauleLongitude"); +// midHauleLongitude = 0f; +// } - if (log.isWarnEnabled()) { - log.warn("For operation " + operationId + ", status " + status + " : " + messages); - } - messages.clear(); - } - } - } + Date shootingStartTime = (Date) row.get("DHTUDEB"); + Date shootingEndTime = (Date) row.get("DHTUFIN"); + String comment = (String) row.get("COMMENTAIRE"); - if (log.isInfoEnabled()) { - log.info("Found " + operations.size() + " operation(s)."); - } + // create a operation + Operation operation = dao.create( + Operation.PROPERTY_ID, operationId, + Operation.PROPERTY_DEPTH_STRATUM, depthStratum, + Operation.PROPERTY_GEAR, gear, + Operation.PROPERTY_GEAR_SHOOTING_START_LATITUDE, shootingStartLatitude, + Operation.PROPERTY_GEAR_SHOOTING_END_LATITUDE, shootingEndLatitude, + Operation.PROPERTY_GEAR_SHOOTING_START_LONGITUDE, shootingStartLongitude, + Operation.PROPERTY_GEAR_SHOOTING_END_LONGITUDE, shootingEndLongitude, + Operation.PROPERTY_GEAR_SHOOTING_START_TIME, shootingStartTime, + Operation.PROPERTY_GEAR_SHOOTING_END_TIME, shootingEndTime, + Operation.PROPERTY_MID_HAULE_LATITUDE, midHauleLatitude, + Operation.PROPERTY_MID_HAULE_LONGITUDE, midHauleLongitude, + Operation.PROPERTY_COMMENT, comment + ); + + addOperationMetadataValue(operation, + MeanWaterDepthOperationMeta, + row.get("SONDE")); + + addOperationMetadataValue(operation, + DielPeriodOperationMeta, + row.get("JOUR_NUIT")); + + addOperationMetadataValue(operation, + StartWaterDepthOperationMeta, + row.get("SONDEDEB")); + + addOperationMetadataValue(operation, + EndWaterDepthOperationMeta, + row.get("SONDEFIN")); + + addGearMetadataValue(operation, + CableLengthGearMeta, + row.get("LFUNES")); + + addGearMetadataValue(operation, + MinSpeedGearMeta, + row.get("VMIN")); + + addGearMetadataValue(operation, + MaxSpeedGearMeta, + row.get("VMAX")); + + return operation; } - protected void addOperationMetadataValue(TopiaDAO<OperationMetadataValue> dao, - Operation operation, + protected void addOperationMetadataValue(Operation operation, OperationMetadata meta, Object value ) throws TopiaException { if (value == null) { return; } - OperationMetadataValue operationMetadataValue = dao.create( + OperationMetadataValue operationMetadataValue = operationMetadataValueDAO.create( OperationMetadataValue.PROPERTY_OPERATION_METADATA, meta, OperationMetadataValue.PROPERTY_DATA_VALUE, value.toString() ); operation.addOperationMetadataValue(operationMetadataValue); } - protected void addGearMetadataValue(TopiaDAO<GearMetadataValue> dao, - Operation operation, + protected void addGearMetadataValue(Operation operation, GearMetadata meta, Object value ) throws TopiaException { @@ -339,7 +384,7 @@ if (value == null) { return; } - GearMetadataValue gearMetadataValue = dao.create( + GearMetadataValue gearMetadataValue = gearMetadataValueDAO.create( GearMetadataValue.PROPERTY_GEAR_METADATA, meta, GearMetadataValue.PROPERTY_DATA_VALUE, value.toString(), GearMetadataValue.PROPERTY_GEAR, operation.getGear() Modified: trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/SampleLoader.java =================================================================== --- trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/SampleLoader.java 2011-12-16 01:46:20 UTC (rev 198) +++ trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/SampleLoader.java 2011-12-16 17:54:16 UTC (rev 199) @@ -26,6 +26,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Multimap; import com.google.common.collect.Multimaps; +import com.google.common.collect.Sets; import com.healthmarketscience.jackcess.Database; import fr.ifremer.echobase.EchoBaseFunctions; import fr.ifremer.echobase.entities.EchoBaseEntityEnum; @@ -35,7 +36,6 @@ import fr.ifremer.echobase.entities.data.SampleDataDAO; import fr.ifremer.echobase.entities.meta.TableMeta; import fr.ifremer.echobase.entities.references.SampleDataType; -import fr.ifremer.echobase.entities.references.SampleDataTypeDAO; import fr.ifremer.echobase.entities.references.SampleType; import fr.ifremer.echobase.entities.references.SampleTypeDAO; import fr.ifremer.echobase.entities.references.Species; @@ -49,6 +49,7 @@ import java.io.IOException; import java.util.Collection; import java.util.Map; +import java.util.Set; /** * To load {@link Sample} entities. @@ -100,9 +101,9 @@ TableMeta meta = getMeta(); copyAndLoadEntities(outputDir, meta.getAssociations(Sample.PROPERTY_SAMPLE_DATA), false, FileType.ASSOCIATION); - } + @Override protected void importBaracoudaEntities(Database db) throws IOException, TopiaException { @@ -112,6 +113,9 @@ Map<String, Species> speciesByName = getUniverse( Species.class, EchoBaseFunctions.SPECIES_BARACOUDA_CODE); + Map<String, SampleDataType> sampleDataTypeByRaptriCode = getUniverse( + SampleDataType.class, EchoBaseFunctions.SAMPLE_DATA_TYPE_RAPTRI_CODE); + Multimap<String, Map<String, Object>> mensurationByNosta = Multimaps.index( db.getTable("MENSURATIONS", true), EchoBaseFunctions.newRowFunction("NOSTA") @@ -121,6 +125,16 @@ db.getTable("CAPTURES", true), EchoBaseFunctions.newRowFunction("NOSTA") ); + + Multimap<Integer, Map<String, Object>> biometryScalesByRaptri = Multimaps.index( + db.getTable("BIOMETRIE_ECHELLE", true), + EchoBaseFunctions.newRowIntegerFunction("Code_Param_Biometrie_Echelle") + ); + + Multimap<String, Map<String, Object>> biometryValuesByNosta = Multimaps.index( + db.getTable("BIOMETRIE_VALEUR", true), + EchoBaseFunctions.newRowFunction("NOSTA_VALEUR_BIOMETRIE") + ); SampleTypeDAO sampleTypeDAO = (SampleTypeDAO) getDAO(SampleType.class); @@ -130,28 +144,32 @@ Preconditions.checkNotNull(sampleTypeUnsorted); SampleType sampleTypeSubsample = sampleTypeDAO.findByName("Subsample"); Preconditions.checkNotNull(sampleTypeSubsample); + SampleType sampleTypeIndividual = sampleTypeDAO.findByName("Individual"); + Preconditions.checkNotNull(sampleTypeIndividual); - SampleDataTypeDAO sampleDataTypeDAO = - (SampleDataTypeDAO) getDAO(SampleDataType.class); +// SampleDataTypeDAO sampleDataTypeDAO = +// (SampleDataTypeDAO) getDAO(SampleDataType.class); SampleDataDAO sampleDataDAO = (SampleDataDAO) getDAO(SampleData.class); - SampleDataType sampleDataTypeMeanLength = sampleDataTypeDAO.findByName("MeanLengthcm"); - Preconditions.checkNotNull(sampleDataTypeMeanLength); - SampleDataType sampleDataTypeMeanWeight = sampleDataTypeDAO.findByName("MeanWeightg"); - Preconditions.checkNotNull(sampleDataTypeMeanWeight); - SampleDataType sampleDataTypeNoPerKg = sampleDataTypeDAO.findByName("NoPerKg"); - Preconditions.checkNotNull(sampleDataTypeNoPerKg); - SampleDataType sampleDataTypeNumberAtLength = sampleDataTypeDAO.findByName("NumberAtLength"); - Preconditions.checkNotNull(sampleDataTypeNumberAtLength); - SampleDataType sampleDataTypeWeightAtLength = sampleDataTypeDAO.findByName("WeightAtLengthkg"); - Preconditions.checkNotNull(sampleDataTypeWeightAtLength); +// SampleDataType sampleDataTypeMeanLength = sampleDataTypeDAO.findByName("MeanLengthcm"); + SampleDataType sampleDataTypeMeanLength = getFK(sampleDataTypeByRaptriCode, "114"); +// SampleDataType sampleDataTypeMeanWeight = sampleDataTypeDAO.findByName("MeanWeightg"); + SampleDataType sampleDataTypeMeanWeight = getFK(sampleDataTypeByRaptriCode, "115"); +// SampleDataType sampleDataTypeNoPerKg = sampleDataTypeDAO.findByName("NoPerKg"); + SampleDataType sampleDataTypeNoPerKg = getFK(sampleDataTypeByRaptriCode, "116"); +// SampleDataType sampleDataTypeNumberAtLength = sampleDataTypeDAO.findByName("NumberAtLength"); + SampleDataType sampleDataTypeNumberAtLength = getFK(sampleDataTypeByRaptriCode, "112"); +// SampleDataType sampleDataTypeWeightAtLength = sampleDataTypeDAO.findByName("WeightAtLengthkg"); + SampleDataType sampleDataTypeWeightAtLength = getFK(sampleDataTypeByRaptriCode, "113"); + TopiaDAO<Sample> dao = getDAO(); int nbSampleUnsorted = 0; int nbSampleTotal = 0; int nbSubSample = 0; + int nbIndividualSample = 0; for (String operationId : capturesByNosta.keySet()) { Preconditions.checkNotNull(operationId); @@ -162,6 +180,13 @@ } continue; } + Collection<Map<String, Object>> biometryValues = + biometryValuesByNosta.get(operationId); + + Multimap<String, Map<String, Object>> biometryValuesBySpecies = Multimaps.index( + biometryValues, + EchoBaseFunctions.newRowFunction("GENRE_ESP_VALEUR_BIOMETRIE") + ); for (Map<String, Object> row : capturesByNosta.get(operationId)) { String type = (String) row.get("CATEG"); @@ -214,7 +239,7 @@ "NA", row.get("MOULE_MENS")); - Collection<Map<String, Object>> mesnurationRows = + Collection<Map<String, Object>> mensurationRows = mensurationByNosta.get(operationId); Float sampleWeight = (Float) row.get("PESP_MENS"); @@ -224,6 +249,7 @@ } sampleWeight = 0f; } + addSubSamples(dao, sampleDataDAO, sampleTypeSubsample, @@ -233,22 +259,105 @@ type, species, sampleWeight, - mesnurationRows); + mensurationRows); int nbSamples = operation.sizeSample(); nbSubSample += nbSamples - 1; + + Collection<Map<String, Object>> biometryValuesRows = + biometryValuesBySpecies.get(speciesId); + + int nbSampleIndiv = addSampleIndividuals(dao, + sampleDataDAO, + operation, + species, + sampleTypeIndividual, + sampleDataTypeByRaptriCode, + biometryScalesByRaptri, + biometryValuesRows); + nbIndividualSample += nbSampleIndiv; + } } - if (log.isInfoEnabled()) { log.info("Found " + nbSampleTotal + " 'total' sample(s)."); log.info("Found " + nbSampleUnsorted + " 'unsorted' sample(s)."); log.info("Found " + nbSubSample + " 'sub' sample(s)."); - log.info("Total " + (nbSampleTotal + nbSampleUnsorted + nbSubSample) + " sample(s)."); + log.info("Found " + nbIndividualSample + " 'Individual' sample(s)."); + log.info("Total " + (nbSampleTotal + nbSampleUnsorted + nbIndividualSample + nbSubSample) + " sample(s)."); } } + private int addSampleIndividuals(TopiaDAO<Sample> dao, + SampleDataDAO sampleDataDAO, + Operation operation, + Species species, + SampleType sampleTypeIndividual, + Map<String, SampleDataType> sampleDataTypeByRaptriCode, + Multimap<Integer, Map<String, Object>> biometryScalesByRaptri, + Collection<Map<String, Object>> biometryValuesRows) throws TopiaException { + int result = 0; + Sample sample = null; + Set<Integer> individus = Sets.newHashSet(); + for (Map<String, Object> biometryValuesRow : biometryValuesRows) { + Integer ind = Integer.valueOf(biometryValuesRow.get("NUMIND_VALEUR_BIOMETRIE").toString()); + Integer codeParameter = (Integer) biometryValuesRow.get("CODE_PARAM_VALEUR_BIOMETRIE"); + String label = null; + Float value = (Float) biometryValuesRow.get("VALEUR_BIOMETRIE"); + if (value==null) { + if (log.isWarnEnabled()) { + log.warn("Null value of VALEUR_BIOMETRIE for operationId/Species/Individu/Code param " + operation.getId() + "/" + species.getBaracoudaCode() + "/" + ind+"/"+codeParameter); + } + continue; + } + Preconditions.checkNotNull(value, "Null value of VALEUR_BIOMETRIE for operationId/Species/Individu/Code param " + operation.getId() + "/" + species.getBaracoudaCode() + "/" + ind+"/"+codeParameter); + SampleDataType sampleDataType = getFK(sampleDataTypeByRaptriCode, String.valueOf(codeParameter)); + + boolean add = individus.add(ind); + if (add) { + + // new sample + sample = dao.create( + Sample.PROPERTY_SPECIES, species, + Sample.PROPERTY_SAMPLE_TYPE, sampleTypeIndividual + ); + + operation.addSample(sample); + result++; + } + + Collection<Map<String, Object>> scalesRows = + biometryScalesByRaptri.get(codeParameter); + + if (scalesRows == null) { + + label = "NA"; + } else { + + // find the correct line + int valueAsInteger = value.intValue(); + + for (Map<String, Object> scalesRow : scalesRows) { + Float scaleCode = (Float) scalesRow.get("Code_echelle"); + if (valueAsInteger == scaleCode.intValue()) { + + // ok find the correct scale + label = (String) scalesRow.get("Descriptif"); + break; + } + } + } + addSampleData(sampleDataDAO, + sample, + sampleDataType, + label, + value + ); + } + return result; + } + private void addSubSamples(TopiaDAO<Sample> dao, SampleDataDAO sampleDataDAO, SampleType sampleTypeSubsample, Modified: trunk/echobase-tools/src/test/java/fr/ifremer/echobase/tools/loaders/SampleLoaderIT.java =================================================================== --- trunk/echobase-tools/src/test/java/fr/ifremer/echobase/tools/loaders/SampleLoaderIT.java 2011-12-16 01:46:20 UTC (rev 198) +++ trunk/echobase-tools/src/test/java/fr/ifremer/echobase/tools/loaders/SampleLoaderIT.java 2011-12-16 17:54:16 UTC (rev 199) @@ -35,7 +35,7 @@ public class SampleLoaderIT extends AbstractLoaderIT<Sample, SampleLoader> { public SampleLoaderIT() { - super(SampleLoader.class, 0, 208199); + super(SampleLoader.class, 0, 324809); } @Test Modified: trunk/src/doc/reunions/reunion-2011-12-14.txt =================================================================== --- trunk/src/doc/reunions/reunion-2011-12-14.txt 2011-12-16 01:46:20 UTC (rev 198) +++ trunk/src/doc/reunions/reunion-2011-12-14.txt 2011-12-16 17:54:16 UTC (rev 199) @@ -50,6 +50,7 @@ - Remplissage du référentiel Gear - Import des opérations (depuis table STATIONS) +- Import des samples (depuis table CAPTURES/MENSURATIONS/BIOMETRIE_VALEURS) Action ------