This is an automated email from the git hooks/post-receive script. New commit to branch feature/6688 in repository tutti. See http://git.codelutin.com/tutti.git commit 1e67a06126dcb434f3fcaf3aae7b25bccfacc50c Author: Tony CHEMIT <chemit@codelutin.com> Date: Wed Feb 25 19:25:14 2015 +0100 introduce file result --- .../genericformat/GenericFormatFileResult.java | 52 ++++++ .../genericformat/GenericFormatImportService.java | 178 +++++++++++++++------ .../GenericFormatReferentialImportResult.java | 32 +++- 3 files changed, 203 insertions(+), 59 deletions(-) diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatFileResult.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatFileResult.java new file mode 100644 index 0000000..9f78be6 --- /dev/null +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatFileResult.java @@ -0,0 +1,52 @@ +package fr.ifremer.tutti.service.genericformat; + +import java.io.Serializable; +import java.util.Collections; +import java.util.Map; +import java.util.Set; + +/** + * Created on 2/25/15. + * + * @author Tony Chemit - chemit@codelutin.com + * @since 3.14 + */ +public class GenericFormatFileResult implements Serializable { + + private static final long serialVersionUID = 1L; + + private final String filename; + + private final int linesCount; + + private final Map<Long, String> errors; + + public GenericFormatFileResult(String filename, int linesCount, Map<Long, String> errors) { + this.filename = filename; + this.linesCount = linesCount; + if (errors == null) { + errors = Collections.emptyMap(); + } + this.errors = errors; + } + + public String getFilename() { + return filename; + } + + public int getLinesCount() { + return linesCount; + } + + public Set<Map.Entry<Long, String>> getErrorsEntries() { + return errors.entrySet(); + } + + public boolean isValid() { + return errors.isEmpty(); + } + + public void addErrors(Map<Long, String> errors) { + errors.putAll(errors); + } +} diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatImportService.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatImportService.java index 8d60247..be0677f 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatImportService.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatImportService.java @@ -89,14 +89,14 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { GenericFormatArchive archive = createArchive(importConfiguration); int result = 6 // check sampleCategoryModel + import ( gear + person + species + vessel + protocol ) - + archive.countImportLines(archive.getSurveyPath()) // load cruises - + archive.countImportLines(archive.getGearCaracteristicsPath()) // load gear caracteristics - + archive.countImportLines(archive.getOperationPath()) // load operations - + archive.countImportLines(archive.getParameterPath()) // load parameters - + archive.countImportLines(archive.getCatchPath()) // load catches - + archive.countImportLines(archive.getMarineLitterPath()) // load marine litters - + archive.countImportLines(archive.getIndividualObservationPath()) // load individualObservations - + archive.countImportLines(archive.getAccidentalCatchPath()); // load accidental catches + + archive.getSurveyLineCount() // load cruises + + archive.getGearCaracteristicsPathLineCount()// load gear caracteristics + + archive.getOperationPathLineCount() // load operations + + archive.getParameterPathLineCount()// load parameters + + archive.getCatchPathLineCount()// load catches + + archive.getMarineLitterPathLineCount()// load marine litters + + archive.getIndividualObservationPathLineCount()// load individualObservations + + archive.getAccidentalCatchPathLineCount(); // load accidental catches return result; @@ -152,11 +152,11 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { GenericFormatArchive archive = createArchive(importConfiguration); - int nbCruises = archive.countImportLines(archive.getSurveyPath()); + int nbCruises = archive.getSurveyLineCount(); if (log.isInfoEnabled()) { log.info("Count " + nbCruises + " cruises to import."); } - int nbOperations = archive.countImportLines(archive.getOperationPath()); + int nbOperations = archive.getOperationPathLineCount(); if (log.isInfoEnabled()) { log.info("Count " + nbOperations + " operations to import."); } @@ -224,10 +224,10 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { try (GenericFormatValidateFileContext importContext = new GenericFormatValidateFileContext(request, progressionModel, persistenceService, cruiseDecorator, fishingOperationDecorator)) { checkSampleCategoryModel(importContext); - importTemporaryGears(importContext); - importTemporaryPersons(importContext); - importTemporarySpecies(importContext); - importTemporaryVessels(importContext); + importTemporaryGears(importContext, false); + importTemporaryPersons(importContext, false); + importTemporarySpecies(importContext, false); + importTemporaryVessels(importContext, false); importProtocol(importContext); validateCruises(importContext); @@ -253,10 +253,10 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { GenericformatImportPersitenceHelper persitenceHelper = new GenericformatImportPersitenceHelper(importContext, persistenceService, cruiseDecorator, fishingOperationDecorator); checkSampleCategoryModel(importContext); - importTemporaryGears(importContext); - importTemporaryPersons(importContext); - importTemporarySpecies(importContext); - importTemporaryVessels(importContext); + importTemporaryGears(importContext, true); + importTemporaryPersons(importContext, true); + importTemporarySpecies(importContext, true); + importTemporaryVessels(importContext, true); importProtocol(importContext); importCruises(importContext, persitenceHelper); @@ -329,7 +329,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { } importContext.increments(t("tutti.service.genericFormat.import.sampleCategoryModel")); - try (CsvConsumerForSampleCategory consumer = importContext.loadSampleCategories()) { + try (CsvConsumerForSampleCategory consumer = importContext.loadSampleCategories(true)) { int nbCategories = 0; for (ImportRow<SampleCategoryRow> row : consumer) { @@ -349,7 +349,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { } - protected void importTemporaryGears(GenericFormatImportContext importContext) { + protected void importTemporaryGears(GenericFormatImportContext importContext, boolean reportError) { importContext.increments(t("tutti.service.genericFormat.import.temporaryGears")); GenericFormatArchive archive = importContext.getImportRequest().getArchive(); @@ -362,19 +362,27 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { log.info("Import temporary gears."); } + Map<Long, String> rowsInError = null; + ReferentialImportRequest<Gear, Integer> referentialImportRequest = referentialTemporaryGearService.createReferentialImportRequest(); - try (CsvConsumerForTemporaryGear consumer = importContext.loadTemporaryGears()) { + try (CsvConsumerForTemporaryGear consumer = importContext.loadTemporaryGears(reportError)) { for (ImportRow<GearRow> row : consumer) { consumer.checkRowForGenericFormatImport(row, referentialImportRequest); } + + if (consumer.foundSomeErrors()) { + rowsInError = consumer.getRowsInError(); + } + } catch (IOException e) { throw new ApplicationTechnicalException("Could not close gears.csv file", e); } ReferentialImportResult<Gear> referentialImportResult = referentialTemporaryGearService.executeImportRequest(referentialImportRequest); - importResult = new GenericFormatReferentialImportResult<>(referentialImportRequest, referentialImportResult); + int linesCount = importContext.getImportRequest().getArchive().getTemporaryReferentialGearLineCount(); + importResult = new GenericFormatReferentialImportResult<>("gear.csv", linesCount, referentialImportRequest, referentialImportResult, rowsInError); if (log.isInfoEnabled()) { log.info("Temporary gears import result: " + importResult.getReport()); } @@ -384,7 +392,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { if (log.isInfoEnabled()) { log.info("Skip import temporary gears (no file found)."); } - importResult = new GenericFormatReferentialImportResult<>(); + importResult = new GenericFormatReferentialImportResult<>("gear.csv"); } @@ -392,7 +400,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { } - protected void importTemporaryPersons(GenericFormatImportContext importContext) { + protected void importTemporaryPersons(GenericFormatImportContext importContext, boolean reportError) { importContext.increments(t("tutti.service.genericFormat.import.temporaryPersons")); GenericFormatArchive archive = importContext.getImportRequest().getArchive(); @@ -405,19 +413,26 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { log.info("Import temporary persons."); } + Map<Long, String> rowsInError = null; + ReferentialImportRequest<Person, Integer> referentialImportRequest = referentialTemporaryPersonService.createReferentialImportRequest(); - try (CsvConsumerForTemporaryPerson consumer = importContext.loadTemporaryPersons()) { + try (CsvConsumerForTemporaryPerson consumer = importContext.loadTemporaryPersons(reportError)) { for (ImportRow<PersonRow> row : consumer) { consumer.checkRowForGenericFormatImport(row, referentialImportRequest); } + + if (consumer.foundSomeErrors()) { + rowsInError = consumer.getRowsInError(); + } } catch (IOException e) { throw new ApplicationTechnicalException("Could not close person.csv file", e); } ReferentialImportResult<Person> referentialImportResult = referentialTemporaryPersonService.executeImportRequest(referentialImportRequest); - importResult = new GenericFormatReferentialImportResult<>(referentialImportRequest, referentialImportResult); + int linesCount = importContext.getImportRequest().getArchive().getTemporaryReferentialPersonLineCount(); + importResult = new GenericFormatReferentialImportResult<>("person.csv", linesCount, referentialImportRequest, referentialImportResult, rowsInError); if (log.isInfoEnabled()) { log.info("Temporary persons import result: " + importResult.getReport()); } @@ -427,7 +442,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { if (log.isInfoEnabled()) { log.info("Skip import temporary persons (no file found)."); } - importResult = new GenericFormatReferentialImportResult<>(); + importResult = new GenericFormatReferentialImportResult<>("person.csv"); } @@ -435,7 +450,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { } - protected void importTemporarySpecies(GenericFormatImportContext importContext) { + protected void importTemporarySpecies(GenericFormatImportContext importContext, boolean reportError) { importContext.increments(t("tutti.service.genericFormat.import.temporarySpecies")); @@ -450,17 +465,23 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { ReferentialImportRequest<Species, Integer> referentialImportRequest = referentialTemporarySpeciesService.createReferentialImportRequest(); - try (CsvConsumerForTemporarySpecies consumer = importContext.loadTemporarySpecies()) { + Map<Long, String> rowsInError = null; + try (CsvConsumerForTemporarySpecies consumer = importContext.loadTemporarySpecies(reportError)) { for (ImportRow<SpeciesRow> row : consumer) { consumer.checkRowForGenericFormatImport(row, referentialImportRequest); } + + if (consumer.foundSomeErrors()) { + rowsInError = consumer.getRowsInError(); + } } catch (IOException e) { throw new ApplicationTechnicalException("Could not close species.csv file", e); } ReferentialImportResult<Species> referentialImportResult = referentialTemporarySpeciesService.executeImportRequest(referentialImportRequest); - importResult = new GenericFormatReferentialImportResult<>(referentialImportRequest, referentialImportResult); + int linesCount = importContext.getImportRequest().getArchive().getTemporaryReferentialSpeciesLineCount(); + importResult = new GenericFormatReferentialImportResult<>("species.csv", linesCount, referentialImportRequest, referentialImportResult, rowsInError); if (log.isInfoEnabled()) { log.info("Temporary species import result: " + importResult.getReport()); @@ -470,7 +491,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { if (log.isInfoEnabled()) { log.info("Skip import temporary species (no file found)."); } - importResult = new GenericFormatReferentialImportResult<>(); + importResult = new GenericFormatReferentialImportResult<>("species.csv"); } @@ -478,7 +499,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { } - protected void importTemporaryVessels(GenericFormatImportContext importContext) { + protected void importTemporaryVessels(GenericFormatImportContext importContext, boolean reportError) { importContext.increments(t("tutti.service.genericFormat.import.temporaryVessels")); GenericFormatArchive archive = importContext.getImportRequest().getArchive(); @@ -493,17 +514,23 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { ReferentialImportRequest<Vessel, String> referentialImportRequest = referentialTemporaryVesselService.createReferentialImportRequest(); - try (CsvConsumerForTemporaryVessel consumer = importContext.loadTemporaryVessels()) { + Map<Long, String> rowsInError = null; + try (CsvConsumerForTemporaryVessel consumer = importContext.loadTemporaryVessels(reportError)) { for (ImportRow<VesselRow> row : consumer) { consumer.checkRowForGenericFormatImport(row, referentialImportRequest); } + + if (consumer.foundSomeErrors()) { + rowsInError = consumer.getRowsInError(); + } } catch (IOException e) { throw new ApplicationTechnicalException("Could not close vessel.csv file", e); } ReferentialImportResult<Vessel> referentialImportResult = referentialTemporaryVesselService.executeImportRequest(referentialImportRequest); - importResult = new GenericFormatReferentialImportResult<>(referentialImportRequest, referentialImportResult); + int linesCount = importContext.getImportRequest().getArchive().getTemporaryReferentialVesselLineCount(); + importResult = new GenericFormatReferentialImportResult<>("vessel.csv", linesCount, referentialImportRequest, referentialImportResult, rowsInError); if (log.isInfoEnabled()) { log.info("Temporary vessels import result: " + importResult.getReport()); @@ -514,7 +541,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { if (log.isInfoEnabled()) { log.info("Skip import temporary vessels (no file found)."); } - importResult = new GenericFormatReferentialImportResult<>(); + importResult = new GenericFormatReferentialImportResult<>("vessel.csv"); } @@ -563,7 +590,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { log.info("Validate survey.csv file."); } - try (CsvConsumerForSurvey consumer = importContext.loadSurveys()) { + try (CsvConsumerForSurvey consumer = importContext.loadSurveys(false)) { for (ImportRow<SurveyRow> row : consumer) { importContext.increments(t("tutti.service.genericFormat.validate.cruises", row.getLineNumber())); @@ -577,10 +604,17 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { importContext.addImportedCruise(cruise, !createCruise); } + + if (consumer.foundSomeErrors()) { + Map<Long, String> rowsInError = consumer.getRowsInError(); + importContext.getSurveyFileResult().addErrors(rowsInError); + } + } catch (IOException e) { throw new ApplicationTechnicalException("Could not close survey.csv file", e); } + } protected void validateGearCaracteristics(GenericFormatValidateFileContext importContext) { @@ -589,7 +623,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { log.info("Validate gearCaracteristics.csv file."); } - try (CsvConsumerForGearCaracteristic consumer = importContext.loadGearCaracteristics()) { + try (CsvConsumerForGearCaracteristic consumer = importContext.loadGearCaracteristics(false)) { for (ImportRow<GearCaracteristicRow> row : consumer) { importContext.increments(t("tutti.service.genericFormat.validate.gearCaracteristics", row.getLineNumber())); @@ -598,6 +632,12 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { consumer.prepareRowForPersist(cruiseContext, row); } + + if (consumer.foundSomeErrors()) { + Map<Long, String> rowsInError = consumer.getRowsInError(); + importContext.getGearCaracteristicFileResult().addErrors(rowsInError); + } + } catch (IOException e) { throw new ApplicationTechnicalException("Could not close gearCaracteristic.csv file", e); } @@ -610,7 +650,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { log.info("Validate operation.csv file."); } - try (CsvConsumerForOperation consumer = importContext.loadOperations()) { + try (CsvConsumerForOperation consumer = importContext.loadOperations(false)) { for (ImportRow<OperationRow> row : consumer) { importContext.increments(t("tutti.service.genericFormat.validate.operations", row.getLineNumber())); @@ -626,6 +666,12 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { importContext.addImportedFishingOperation(fishingOperation, catchBatch); } + + if (consumer.foundSomeErrors()) { + Map<Long, String> rowsInError = consumer.getRowsInError(); + importContext.getOperationFileResult().addErrors(rowsInError); + } + } catch (IOException e) { throw new ApplicationTechnicalException("Could not close operation.csv file", e); } @@ -638,7 +684,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { log.info("Validate parameter.csv file."); } - try (CsvConsumerForParameter consumer = importContext.loadParameters()) { + try (CsvConsumerForParameter consumer = importContext.loadParameters(false)) { for (ImportRow<ParameterRow> row : consumer) { importContext.increments(t("tutti.service.genericFormat.validate.parameters", row.getLineNumber())); @@ -647,6 +693,12 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { consumer.prepareRowForPersist(operationContext, row); } + + if (consumer.foundSomeErrors()) { + Map<Long, String> rowsInError = consumer.getRowsInError(); + importContext.getParameterFileResult().addErrors(rowsInError); + } + } catch (IOException e) { throw new ApplicationTechnicalException("Could not close parameter.csv file", e); } @@ -659,7 +711,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { log.info("Validate catch.csv file."); } - try (CsvConsumerForCatch consumer = importContext.loadCatches()) { + try (CsvConsumerForCatch consumer = importContext.loadCatches(false)) { for (ImportRow<CatchRow> row : consumer) { importContext.increments(t("tutti.service.genericFormat.validate.catches", row.getLineNumber())); @@ -668,6 +720,12 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { consumer.prepareRowForPersist(operationContext, row); } + + if (consumer.foundSomeErrors()) { + Map<Long, String> rowsInError = consumer.getRowsInError(); + importContext.getCatchFileResult().addErrors(rowsInError); + } + } catch (IOException e) { throw new ApplicationTechnicalException("Could not close catch.csv file", e); } @@ -680,7 +738,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { log.info("Validate marineLitter.csv file."); } - try (CsvConsumerForMarineLitter consumer = importContext.loadMarineLitters()) { + try (CsvConsumerForMarineLitter consumer = importContext.loadMarineLitters(false)) { for (ImportRow<MarineLitterRow> row : consumer) { importContext.increments(t("tutti.service.genericFormat.validate.marineLitters", row.getLineNumber())); @@ -689,6 +747,12 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { consumer.prepareRowForPersist(operationContext, row); } + + if (consumer.foundSomeErrors()) { + Map<Long, String> rowsInError = consumer.getRowsInError(); + importContext.getMarineLitterFileResult().addErrors(rowsInError); + } + } catch (IOException e) { throw new ApplicationTechnicalException("Could not close marineLitter.csv file", e); } @@ -701,7 +765,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { log.info("Validate individualObservation.csv file."); } - try (CsvConsumerForIndividualObservation consumer = importContext.loadIndividualObservations()) { + try (CsvConsumerForIndividualObservation consumer = importContext.loadIndividualObservations(false)) { for (ImportRow<IndividualObservationRow> row : consumer) { importContext.increments(t("tutti.service.genericFormat.validate.individualObservations", row.getLineNumber())); @@ -710,6 +774,12 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { consumer.prepareRowForPersist(operationContext, row); } + + if (consumer.foundSomeErrors()) { + Map<Long, String> rowsInError = consumer.getRowsInError(); + importContext.getIndividualObservationFileResult().addErrors(rowsInError); + } + } catch (IOException e) { throw new ApplicationTechnicalException("Could not close individualObservation.csv file", e); } @@ -722,7 +792,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { log.info("Validate accidentalCatch.csv file."); } - try (CsvConsumerForAccidentalCatch consumer = importContext.loadAccidentalCatches()) { + try (CsvConsumerForAccidentalCatch consumer = importContext.loadAccidentalCatches(false)) { for (ImportRow<AccidentalCatchRow> row : consumer) { importContext.increments(t("tutti.service.genericFormat.validate.accidentalCatches", row.getLineNumber())); @@ -731,6 +801,12 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { consumer.prepareRowForPersist(operationContext, row); } + + if (consumer.foundSomeErrors()) { + Map<Long, String> rowsInError = consumer.getRowsInError(); + importContext.getAccidentalCatchFileResult().addErrors(rowsInError); + } + } catch (IOException e) { throw new ApplicationTechnicalException("Could not close accidentalCatch.csv file", e); } @@ -744,7 +820,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { } importContext.increments(t("tutti.service.genericFormat.import.cruises")); - try (CsvConsumerForSurvey consumer = importContext.loadSurveys()) { + try (CsvConsumerForSurvey consumer = importContext.loadSurveys(true)) { for (ImportRow<SurveyRow> row : consumer) { consumer.validateRow(row, importContext); @@ -766,7 +842,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { } importContext.increments(t("tutti.service.genericFormat.import.gearCaracteristics")); - try (CsvConsumerForGearCaracteristic consumer = importContext.loadGearCaracteristics()) { + try (CsvConsumerForGearCaracteristic consumer = importContext.loadGearCaracteristics(true)) { for (ImportRow<GearCaracteristicRow> row : consumer) { GenericFormatImportCruiseContext cruiseContext = consumer.validateRow(row, importContext); @@ -788,7 +864,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { } importContext.increments(t("tutti.service.genericFormat.import.operations")); - try (CsvConsumerForOperation consumer = importContext.loadOperations()) { + try (CsvConsumerForOperation consumer = importContext.loadOperations(true)) { for (ImportRow<OperationRow> row : consumer) { GenericFormatImportCruiseContext cruiseContext = consumer.validateRow(row, importContext); @@ -815,7 +891,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { } importContext.increments(t("tutti.service.genericFormat.import.parameters")); - try (CsvConsumerForParameter consumer = importContext.loadParameters()) { + try (CsvConsumerForParameter consumer = importContext.loadParameters(true)) { for (ImportRow<ParameterRow> row : consumer) { GenericFormatImportOperationContext operationContext = consumer.validateRow(row, importContext); @@ -837,7 +913,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { } importContext.increments(t("tutti.service.genericFormat.import.catches")); - try (CsvConsumerForCatch consumer = importContext.loadCatches()) { + try (CsvConsumerForCatch consumer = importContext.loadCatches(true)) { for (ImportRow<CatchRow> row : consumer) { GenericFormatImportOperationContext operationContext = consumer.validateRow(row, importContext); @@ -860,7 +936,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { } importContext.increments(t("tutti.service.genericFormat.import.marineLitters")); - try (CsvConsumerForMarineLitter consumer = importContext.loadMarineLitters()) { + try (CsvConsumerForMarineLitter consumer = importContext.loadMarineLitters(true)) { for (ImportRow<MarineLitterRow> row : consumer) { GenericFormatImportOperationContext operationContext = consumer.validateRow(row, importContext); @@ -882,7 +958,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { } importContext.increments(t("tutti.service.genericFormat.import.individualObservations")); - try (CsvConsumerForIndividualObservation consumer = importContext.loadIndividualObservations()) { + try (CsvConsumerForIndividualObservation consumer = importContext.loadIndividualObservations(true)) { for (ImportRow<IndividualObservationRow> row : consumer) { GenericFormatImportOperationContext operationContext = consumer.validateRow(row, importContext); @@ -904,7 +980,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { } importContext.increments(t("tutti.service.genericFormat.import.accidentalCatches")); - try (CsvConsumerForAccidentalCatch consumer = importContext.loadAccidentalCatches()) { + try (CsvConsumerForAccidentalCatch consumer = importContext.loadAccidentalCatches(true)) { for (ImportRow<AccidentalCatchRow> row : consumer) { GenericFormatImportOperationContext operationContext = consumer.validateRow(row, importContext); diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatReferentialImportResult.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatReferentialImportResult.java index bf4bb86..7b13241 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatReferentialImportResult.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatReferentialImportResult.java @@ -6,11 +6,11 @@ import fr.ifremer.tutti.persistence.entities.referential.TuttiReferentialEntity; import fr.ifremer.tutti.service.referential.ReferentialImportRequest; import fr.ifremer.tutti.service.referential.ReferentialImportResult; -import java.io.Serializable; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TreeMap; /** @@ -19,7 +19,7 @@ import java.util.TreeMap; * @author Tony Chemit - chemit@codelutin.com * @since 3.14 */ -public class GenericFormatReferentialImportResult<E extends TuttiReferentialEntity, K extends Comparable<K>> implements Serializable { +public class GenericFormatReferentialImportResult<E extends TuttiReferentialEntity, K extends Comparable<K>> extends GenericFormatFileResult { private static final long serialVersionUID = 1L; @@ -29,13 +29,16 @@ public class GenericFormatReferentialImportResult<E extends TuttiReferentialEnti private final Map<String, String> idTranslationMap; - public GenericFormatReferentialImportResult() { + public GenericFormatReferentialImportResult(String filename) { + super(filename, 0, null); this.entitiesAdded = Collections.emptyMap(); this.entitiesLinked = Collections.emptyMap(); this.idTranslationMap = Collections.emptyMap(); } - public GenericFormatReferentialImportResult(ReferentialImportRequest<E, K> importRequest, ReferentialImportResult<E> importResult) { + public GenericFormatReferentialImportResult(String filename, int linesCount, ReferentialImportRequest<E, K> importRequest, ReferentialImportResult<E> importResult, Map<Long, String> rowsInError) { + + super(filename, linesCount, rowsInError); Map<String, String> translationMap = new TreeMap<>(); @@ -55,12 +58,12 @@ public class GenericFormatReferentialImportResult<E extends TuttiReferentialEnti } - public Map<String, E> getEntitiesAdded() { - return entitiesAdded; + public Set<Map.Entry<String, E>> getEntitiesAddedEntries() { + return entitiesAdded.entrySet(); } - public Map<String, E> getEntitiesLinked() { - return entitiesLinked; + public Set<Map.Entry<String, E>> getEntitiesLinkedEntries() { + return entitiesLinked.entrySet(); } public Map<String, String> getIdTranslationMap() { @@ -98,6 +101,19 @@ public class GenericFormatReferentialImportResult<E extends TuttiReferentialEnti StringBuilder builder = new StringBuilder(); + Set<Map.Entry<Long, String>> errors = getErrorsEntries(); + + if (!errors.isEmpty()) { + + builder.append(String.format("%s lines in error:", errors.size())); + for (Map.Entry<Long, String> entry : errors) { + Long lineNumber = entry.getKey(); + String error = entry.getValue(); + builder.append(String.format("\nline %s : \n%s", lineNumber, error)); + } + + } + List<String> added = getMapReport(entitiesAdded); if (!added.isEmpty()) { builder.append(String.format("\n%s entities added: \n%s", added.size(), Joiner.on("\n").join(added))); -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.

This is an automated email from the git hooks/post-receive script. New commit to branch feature/6688 in repository tutti. See http://git.codelutin.com/tutti.git commit 92390daaf1915fad9955dfd67caf0ecab60b0891 Author: Tony CHEMIT <chemit@codelutin.com> Date: Wed Feb 25 19:26:11 2015 +0100 use failfats mode in normal temporary referential import --- .../tutti/service/referential/ReferentialTemporaryGearService.java | 2 +- .../tutti/service/referential/ReferentialTemporaryPersonService.java | 2 +- .../tutti/service/referential/ReferentialTemporarySpeciesService.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/referential/ReferentialTemporaryGearService.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/referential/ReferentialTemporaryGearService.java index e5849ba..68b54af 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/referential/ReferentialTemporaryGearService.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/referential/ReferentialTemporaryGearService.java @@ -85,7 +85,7 @@ public class ReferentialTemporaryGearService extends AbstractTuttiService { ReferentialImportRequest<Gear, Integer> requestResult = createReferentialImportRequest(); - try (CsvConsumerForTemporaryGear consumer = new CsvConsumerForTemporaryGear(file.toPath(), getCsvSeparator())) { + try (CsvConsumerForTemporaryGear consumer = new CsvConsumerForTemporaryGear(file.toPath(), getCsvSeparator(), true)) { for (ImportRow<GearRow> bean : consumer) { diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/referential/ReferentialTemporaryPersonService.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/referential/ReferentialTemporaryPersonService.java index 834d9c1..b8ac05f 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/referential/ReferentialTemporaryPersonService.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/referential/ReferentialTemporaryPersonService.java @@ -85,7 +85,7 @@ public class ReferentialTemporaryPersonService extends AbstractTuttiService { ReferentialImportRequest<Person, Integer> requestResult = createReferentialImportRequest(); - try (CsvConsumerForTemporaryPerson consumer = new CsvConsumerForTemporaryPerson(file.toPath(), getCsvSeparator())) { + try (CsvConsumerForTemporaryPerson consumer = new CsvConsumerForTemporaryPerson(file.toPath(), getCsvSeparator(), true)) { for (ImportRow<PersonRow> bean : consumer) { diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/referential/ReferentialTemporarySpeciesService.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/referential/ReferentialTemporarySpeciesService.java index 8e15e5b..a5f57a0 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/referential/ReferentialTemporarySpeciesService.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/referential/ReferentialTemporarySpeciesService.java @@ -84,7 +84,7 @@ public class ReferentialTemporarySpeciesService extends AbstractTuttiService { ReferentialImportRequest<Species, Integer> requestResult = createReferentialImportRequest(); - try (CsvConsumerForTemporarySpecies consumer = new CsvConsumerForTemporarySpecies(file.toPath(), getCsvSeparator())) { + try (CsvConsumerForTemporarySpecies consumer = new CsvConsumerForTemporarySpecies(file.toPath(), getCsvSeparator(), true)) { for (ImportRow<SpeciesRow> bean : consumer) { -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.

This is an automated email from the git hooks/post-receive script. New commit to branch feature/6688 in repository tutti. See http://git.codelutin.com/tutti.git commit acb35e6b2e07387910eaf99c8ff7d5ebc9031d70 Author: Tony CHEMIT <chemit@codelutin.com> Date: Wed Feb 25 19:29:32 2015 +0100 add failfast option when loading files + show inmport configuration for report generation --- .../genericformat/GenericFormatImportContext.java | 109 ++++++++++++++++----- .../genericformat/GenericFormatImportRequest.java | 4 + 2 files changed, 87 insertions(+), 26 deletions(-) diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatImportContext.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatImportContext.java index 1e8f306..ee74be5 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatImportContext.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatImportContext.java @@ -70,6 +70,22 @@ public class GenericFormatImportContext implements Closeable { private final Map<String, GenericFormatImportCruiseContext> cruiseContexts; + private final GenericFormatFileResult surveyFileResult; + + private final GenericFormatFileResult accidentalCatchFileResult; + + private final GenericFormatFileResult individualObservationFileResult; + + private final GenericFormatFileResult marineLitterFileResult; + + private final GenericFormatFileResult catchFileResult; + + private final GenericFormatFileResult parameterFileResult; + + private final GenericFormatFileResult operationFileResult; + + private final GenericFormatFileResult gearCaracteristicFileResult; + public GenericFormatImportContext(GenericFormatImportRequest importRequest, ProgressionModel progressionModel, PersistenceService persistenceService, @@ -90,6 +106,15 @@ public class GenericFormatImportContext implements Closeable { this.validationHelper = new GenericFormatImportValidationHelper(this, new ValidationService()); this.cruiseContexts = new LinkedHashMap<>(); + this.surveyFileResult = new GenericFormatFileResult("survey.csv", importRequest.getArchive().getSurveyLineCount(), null); + this.gearCaracteristicFileResult = new GenericFormatFileResult("sgearCaracteristics.csv", importRequest.getArchive().getGearCaracteristicsPathLineCount(), null); + this.operationFileResult = new GenericFormatFileResult("operation.csv", importRequest.getArchive().getOperationPathLineCount(), null); + this.parameterFileResult = new GenericFormatFileResult("parameter.csv", importRequest.getArchive().getParameterPathLineCount(), null); + this.catchFileResult = new GenericFormatFileResult("catch.csv", importRequest.getArchive().getCatchPathLineCount(), null); + this.marineLitterFileResult = new GenericFormatFileResult("marineLitter.csv", importRequest.getArchive().getMarineLitterPathLineCount(), null); + this.individualObservationFileResult = new GenericFormatFileResult("individualObservation.csv", importRequest.getArchive().getIndividualObservationPathLineCount(), null); + this.accidentalCatchFileResult = new GenericFormatFileResult("accidentalCatch.csv", importRequest.getArchive().getAccidentalCatchPathLineCount(), null); + } @Override @@ -104,6 +129,38 @@ public class GenericFormatImportContext implements Closeable { } + public GenericFormatFileResult getSurveyFileResult() { + return surveyFileResult; + } + + public GenericFormatFileResult getGearCaracteristicFileResult() { + return gearCaracteristicFileResult; + } + + public GenericFormatFileResult getOperationFileResult() { + return operationFileResult; + } + + public GenericFormatFileResult getParameterFileResult() { + return parameterFileResult; + } + + public GenericFormatFileResult getCatchFileResult() { + return catchFileResult; + } + + public GenericFormatFileResult getMarineLitterFileResult() { + return marineLitterFileResult; + } + + public GenericFormatFileResult getIndividualObservationFileResult() { + return individualObservationFileResult; + } + + public GenericFormatFileResult getAccidentalCatchFileResult() { + return accidentalCatchFileResult; + } + public GenericFormatImportRequest getImportRequest() { return importRequest; } @@ -208,68 +265,68 @@ public class GenericFormatImportContext implements Closeable { } - public CsvConsumerForSampleCategory loadSampleCategories() { - CsvConsumerForSampleCategory consumer = new CsvConsumerForSampleCategory(importRequest.getArchive().getSampleCategoryModelPath(), importRequest.getCsvSeparator(), importEntityParserFactory); + public CsvConsumerForSampleCategory loadSampleCategories(boolean failFast) { + CsvConsumerForSampleCategory consumer = new CsvConsumerForSampleCategory(importRequest.getArchive().getSampleCategoryModelPath(), importRequest.getCsvSeparator(), importEntityParserFactory, failFast); return consumer; } - public CsvConsumerForSurvey loadSurveys() { - CsvConsumerForSurvey consumer = new CsvConsumerForSurvey(importRequest.getArchive().getSurveyPath(), importRequest.getCsvSeparator(), importEntityParserFactory); + public CsvConsumerForSurvey loadSurveys(boolean failFast) { + CsvConsumerForSurvey consumer = new CsvConsumerForSurvey(importRequest.getArchive().getSurveyPath(), importRequest.getCsvSeparator(), importEntityParserFactory, failFast); return consumer; } - public CsvConsumerForGearCaracteristic loadGearCaracteristics() { - CsvConsumerForGearCaracteristic consumer = new CsvConsumerForGearCaracteristic(importRequest.getArchive().getGearCaracteristicsPath(), importRequest.getCsvSeparator(), importEntityParserFactory); + public CsvConsumerForGearCaracteristic loadGearCaracteristics(boolean failFast) { + CsvConsumerForGearCaracteristic consumer = new CsvConsumerForGearCaracteristic(importRequest.getArchive().getGearCaracteristicsPath(), importRequest.getCsvSeparator(), importEntityParserFactory, failFast); return consumer; } - public CsvConsumerForOperation loadOperations() { - CsvConsumerForOperation consumer = new CsvConsumerForOperation(importRequest.getArchive().getOperationPath(), importRequest.getCsvSeparator(), importEntityParserFactory); + public CsvConsumerForOperation loadOperations(boolean failFast) { + CsvConsumerForOperation consumer = new CsvConsumerForOperation(importRequest.getArchive().getOperationPath(), importRequest.getCsvSeparator(), importEntityParserFactory, failFast); return consumer; } - public CsvConsumerForParameter loadParameters() { - CsvConsumerForParameter consumer = new CsvConsumerForParameter(importRequest.getArchive().getParameterPath(), importRequest.getCsvSeparator(), importEntityParserFactory); + public CsvConsumerForParameter loadParameters(boolean failFast) { + CsvConsumerForParameter consumer = new CsvConsumerForParameter(importRequest.getArchive().getParameterPath(), importRequest.getCsvSeparator(), importEntityParserFactory, failFast); return consumer; } - public CsvConsumerForCatch loadCatches() { - CsvConsumerForCatch consumer = new CsvConsumerForCatch(importRequest.getArchive().getCatchPath(), importRequest.getCsvSeparator(), importRequest.getSampleCategoryModel(), importEntityParserFactory); + public CsvConsumerForCatch loadCatches(boolean failFast) { + CsvConsumerForCatch consumer = new CsvConsumerForCatch(importRequest.getArchive().getCatchPath(), importRequest.getCsvSeparator(), importRequest.getSampleCategoryModel(), importEntityParserFactory, failFast); return consumer; } - public CsvConsumerForMarineLitter loadMarineLitters() { - CsvConsumerForMarineLitter consumer = new CsvConsumerForMarineLitter(importRequest.getArchive().getMarineLitterPath(), importRequest.getCsvSeparator(), importEntityParserFactory); + public CsvConsumerForMarineLitter loadMarineLitters(boolean failFast) { + CsvConsumerForMarineLitter consumer = new CsvConsumerForMarineLitter(importRequest.getArchive().getMarineLitterPath(), importRequest.getCsvSeparator(), importEntityParserFactory, failFast); return consumer; } - public CsvConsumerForIndividualObservation loadIndividualObservations() { - CsvConsumerForIndividualObservation consumer = new CsvConsumerForIndividualObservation(importRequest.getArchive().getIndividualObservationPath(), importRequest.getCsvSeparator(), importEntityParserFactory); + public CsvConsumerForIndividualObservation loadIndividualObservations(boolean failFast) { + CsvConsumerForIndividualObservation consumer = new CsvConsumerForIndividualObservation(importRequest.getArchive().getIndividualObservationPath(), importRequest.getCsvSeparator(), importEntityParserFactory, failFast); return consumer; } - public CsvConsumerForAccidentalCatch loadAccidentalCatches() { - CsvConsumerForAccidentalCatch consumer = new CsvConsumerForAccidentalCatch(importRequest.getArchive().getAccidentalCatchPath(), importRequest.getCsvSeparator(), importEntityParserFactory); + public CsvConsumerForAccidentalCatch loadAccidentalCatches(boolean failFast) { + CsvConsumerForAccidentalCatch consumer = new CsvConsumerForAccidentalCatch(importRequest.getArchive().getAccidentalCatchPath(), importRequest.getCsvSeparator(), importEntityParserFactory, failFast); return consumer; } - public CsvConsumerForTemporaryGear loadTemporaryGears() { - CsvConsumerForTemporaryGear consumer = new CsvConsumerForTemporaryGear(importRequest.getArchive().getTemporaryReferentialGearsPath(), importRequest.getCsvSeparator()); + public CsvConsumerForTemporaryGear loadTemporaryGears(boolean failFast) { + CsvConsumerForTemporaryGear consumer = new CsvConsumerForTemporaryGear(importRequest.getArchive().getTemporaryReferentialGearsPath(), importRequest.getCsvSeparator(), failFast); return consumer; } - public CsvConsumerForTemporaryPerson loadTemporaryPersons() { - CsvConsumerForTemporaryPerson consumer = new CsvConsumerForTemporaryPerson(importRequest.getArchive().getTemporaryReferentialPersonsPath(), importRequest.getCsvSeparator()); + public CsvConsumerForTemporaryPerson loadTemporaryPersons(boolean failFast) { + CsvConsumerForTemporaryPerson consumer = new CsvConsumerForTemporaryPerson(importRequest.getArchive().getTemporaryReferentialPersonsPath(), importRequest.getCsvSeparator(), failFast); return consumer; } - public CsvConsumerForTemporarySpecies loadTemporarySpecies() { - CsvConsumerForTemporarySpecies consumer = new CsvConsumerForTemporarySpecies(importRequest.getArchive().getTemporaryReferentialSpeciesPath(), importRequest.getCsvSeparator()); + public CsvConsumerForTemporarySpecies loadTemporarySpecies(boolean reportError) { + CsvConsumerForTemporarySpecies consumer = new CsvConsumerForTemporarySpecies(importRequest.getArchive().getTemporaryReferentialSpeciesPath(), importRequest.getCsvSeparator(), reportError); return consumer; } - public CsvConsumerForTemporaryVessel loadTemporaryVessels() { - CsvConsumerForTemporaryVessel consumer = new CsvConsumerForTemporaryVessel(importRequest.getArchive().getTemporaryReferentialVesselsPath(), importRequest.getCsvSeparator()); + public CsvConsumerForTemporaryVessel loadTemporaryVessels(boolean reportError) { + CsvConsumerForTemporaryVessel consumer = new CsvConsumerForTemporaryVessel(importRequest.getArchive().getTemporaryReferentialVesselsPath(), importRequest.getCsvSeparator(), reportError); return consumer; } diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatImportRequest.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatImportRequest.java index 39c53df..52bdb9e 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatImportRequest.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatImportRequest.java @@ -83,6 +83,10 @@ public class GenericFormatImportRequest implements Serializable { return startingDate; } + public GenericFormatImportConfiguration getImportConfiguration() { + return importConfiguration; + } + public Cruise getExistingCruise(Cruise importRowCruise) { Cruise result = null; -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.

This is an automated email from the git hooks/post-receive script. New commit to branch feature/6688 in repository tutti. See http://git.codelutin.com/tutti.git commit 8605d6019af9030ed3f9796478df56b9de6a82d2 Author: Tony CHEMIT <chemit@codelutin.com> Date: Fri Feb 27 08:53:25 2015 +0100 continue validation --- .../fr/ifremer/tutti/service/csv/CsvComsumer.java | 36 ++- .../genericformat/GenericFormatArchive.java | 98 ++++--- ...GenericFormatArchiveInvalidLayoutException.java | 23 +- .../genericformat/GenericFormatFileResult.java | 46 +++- .../genericformat/GenericFormatImportContext.java | 136 ++++++---- .../GenericFormatImportEntityParserFactory.java | 8 +- .../genericformat/GenericFormatImportService.java | 281 ++++++++++++--------- .../GenericFormatReferentialImportResult.java | 20 +- .../genericformat/GenericFormatResultSupport.java | 87 +++++-- .../GenericFormatValidateFileResult.java | 14 - .../referential/ReferentialImportRequest.java | 24 +- .../consumer/CsvConsumerForTemporaryGear.java | 12 +- .../consumer/CsvConsumerForTemporaryPerson.java | 8 + .../consumer/CsvConsumerForTemporarySpecies.java | 8 + .../consumer/CsvConsumerForTemporaryVessel.java | 8 + ...ort_fr.ftl => genericFormatImportReport_fr.ftl} | 174 ++++++------- .../ftl/genericFormatValidateReport_fr.ftl | 222 ++++++++-------- .../GenericFormatImportServiceTest.java | 55 +++- .../GenericFormatImportServiceValidTest.java | 133 ++++++++++ .../sampleCategory.csv | 4 +- .../temporaryGears.csv | 4 +- .../temporaryGears.csv | 5 + .../genericFormat/default/accidentalCatch.csv | 2 +- .../default/individualObservation.csv | 2 +- .../genericFormat/default/marineLitter.csv | 2 +- .../resources/genericFormat/default/operation.csv | 2 +- .../resources/genericFormat/default/survey.csv | 2 +- .../genericFormat/empty/accidentalCatch.csv | 2 +- .../test/resources/genericFormat/empty/catch.csv | 2 +- .../genericFormat/empty/gearCaracteristics.csv | 2 +- .../genericFormat/empty/individualObservation.csv | 2 +- .../resources/genericFormat/empty/marineLitter.csv | 2 +- .../resources/genericFormat/empty/operation.csv | 2 +- .../resources/genericFormat/empty/parameter.csv | 2 +- .../test/resources/genericFormat/empty/survey.csv | 2 +- .../resources/genericFormat/onlyCruise/survey.csv | 2 +- .../{technical => referentials}/temporaryGears.csv | 2 +- .../temporaryPersons.csv | 2 +- .../temporarySpecies.csv | 2 +- .../temporaryVessels.csv | 2 +- .../sampleCategory.csv | 2 +- .../sampleCategory.csv | 4 +- 42 files changed, 927 insertions(+), 521 deletions(-) diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/csv/CsvComsumer.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/csv/CsvComsumer.java index 0b865dd..d48d807 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/csv/CsvComsumer.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/csv/CsvComsumer.java @@ -7,6 +7,7 @@ import com.google.common.io.Files; import org.apache.commons.io.IOUtils; import org.nuiton.csv.AbstractImportErrorInfo; import org.nuiton.csv.Import2; +import org.nuiton.csv.ImportConf; import org.nuiton.csv.ImportRow; import org.nuiton.csv.ImportableColumn; import org.nuiton.jaxx.application.ApplicationBusinessException; @@ -39,7 +40,7 @@ public abstract class CsvComsumer<O, M extends AbstractTuttiImportExportModel<O> private final boolean failFast; - private final Map<Long, String> rowsInError; + private final Map<Long, Set<String>> rowsInError; public CsvComsumer(Path file, M model, boolean failFast) { this.failFast = failFast; @@ -51,7 +52,9 @@ public abstract class CsvComsumer<O, M extends AbstractTuttiImportExportModel<O> throw new ApplicationTechnicalException("file not found " + file, e); } - this.importer = Import2.newImport(model, reader); + ImportConf importConf = new ImportConf(); + importConf.setStrictMode(failFast); + this.importer = Import2.newImport(importConf, model, reader); } @@ -96,11 +99,34 @@ public abstract class CsvComsumer<O, M extends AbstractTuttiImportExportModel<O> } + + public Set<String> rowErrorsToMessage(ImportRow<O> bean) { + + Set<String> errors = new HashSet<>(); + for (AbstractImportErrorInfo<O> errorInfo : bean.getErrors()) { + + Throwable cause = errorInfo.getCause(); + + if (errorInfo.getField() == null) { + + errors.add(cause.getMessage()); + + } else { + + errors.add(t("tutti.csv.import.error.on.field", errorInfo.getField().getHeaderName(), cause.getMessage())); + } + + } + + return errors; + + } + public boolean foundSomeErrors() { return !rowsInError.isEmpty(); } - public Map<Long, String> getRowsInError() { + public Map<Long, Set<String>> getRowsInError() { return ImmutableMap.copyOf(rowsInError); } @@ -108,15 +134,15 @@ public abstract class CsvComsumer<O, M extends AbstractTuttiImportExportModel<O> if (!row.isValid()) { - String message = rowErrorsToExceptionMessage(row); if (failFast) { + String message = rowErrorsToExceptionMessage(row); throw new ApplicationBusinessException(message); } else { - rowsInError.put(row.getLineNumber(), message); + rowsInError.put(row.getLineNumber(), rowErrorsToMessage(row)); } diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatArchive.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatArchive.java index 4bf7e08..20a89c3 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatArchive.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatArchive.java @@ -1,6 +1,5 @@ package fr.ifremer.tutti.service.genericformat; -import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import fr.ifremer.tutti.persistence.ProgressionModel; import org.apache.commons.logging.Log; @@ -18,10 +17,10 @@ import java.io.Serializable; import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.Path; -import java.util.ArrayList; import java.util.EnumMap; import java.util.EnumSet; -import java.util.List; +import java.util.LinkedHashSet; +import java.util.Set; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; @@ -213,6 +212,10 @@ public class GenericFormatArchive implements Serializable { return getPath(ArchiveFilePath.DATA_MARINE_LITTER); } + public int getSampleCategoryLineCount() { + return countImportLines(ArchiveFilePath.SAMPLE_CATEGORY); + } + public int getSurveyLineCount() { return countImportLines(ArchiveFilePath.DATA_SURVEY); } @@ -268,53 +271,44 @@ public class GenericFormatArchive implements Serializable { public void validateArchiveLayout() throws GenericFormatArchiveInvalidLayoutException { - List<String> errors = new ArrayList<>(); - - try (ZipFile zipFile = new ZipFile(archiveFile)) { - - for (ArchiveFilePath archiveFilePath : ArchiveFilePath.values()) { - - if (archiveFilePath.isMandatory()) { - String zipEntryPath = getZipEntryPath(archiveFilePath); - - if (log.isDebugEnabled()) { - log.debug("Check mandatory entry " + zipEntryPath + " exists."); - } - - ZipEntry zipEntry = zipFile.getEntry(zipEntryPath); + Set<String> errors = new LinkedHashSet<>(); - if (zipEntry == null) { + for (ArchiveFilePath archiveFilePath : ArchiveFilePath.values()) { - if (log.isErrorEnabled()) { - log.error("Mandatory entry " + zipEntryPath + " not found."); - } - errors.add(t("tutti.service.genericFormat.importError.missArchiveFile", archiveFilePath.getFilename())); + if (archiveFilePath.isMandatory() && missingPaths.contains(archiveFilePath)) { - } else { - - if (log.isInfoEnabled()) { - log.info("Mandatory entry " + zipEntryPath + " found."); - } - } + // Missing a mandatory entry + if (log.isErrorEnabled()) { + log.error("Mandatory entry " + archiveFilePath.getFilename() + " not found."); } + errors.add(t("tutti.service.genericFormat.importError.missArchiveFile", archiveFilePath.getFilename())); } + } - } catch (IOException e) { + if (!errors.isEmpty()) { + + throw new GenericFormatArchiveInvalidLayoutException(this, errors); - throw new ApplicationTechnicalException("Could not open zip file: " + archiveFile, e); } - if (!errors.isEmpty()) { + } - String message = t("tutti.service.genericFormat.importError.archiveNotSane", Joiner.on("\n").join(errors)); + public Set<String> getImportFiles() { - throw new GenericFormatArchiveInvalidLayoutException(message); + Set<String> result = new LinkedHashSet<>(); + for (ArchiveFilePath archiveFilePath : ArchiveFilePath.values()) { + + if (!missingPaths.contains(archiveFilePath)) { + result.add(archiveFilePath.getFilename()); + } } + return result; } + public void createZip(ProgressionModel progressionModel) { if (progressionModel != null) { @@ -345,7 +339,7 @@ public class GenericFormatArchive implements Serializable { log.info("Zip entries prefix path: " + prefixPath); } - missingPaths = getMissingPaths(); + missingPaths = computeMissingPaths(); } else { this.prefixPath = null; @@ -425,7 +419,7 @@ public class GenericFormatArchive implements Serializable { } - protected EnumSet<ArchiveFilePath> getMissingPaths() { + protected EnumSet<ArchiveFilePath> computeMissingPaths() { EnumSet<ArchiveFilePath> result = EnumSet.noneOf(ArchiveFilePath.class); @@ -433,30 +427,26 @@ public class GenericFormatArchive implements Serializable { for (ArchiveFilePath archiveFilePath : ArchiveFilePath.values()) { - if (!archiveFilePath.isMandatory()) { - - String zipEntryPath = getZipEntryPath(archiveFilePath); - - if (log.isDebugEnabled()) { - log.debug("Check optional entry " + zipEntryPath + " exists."); - } + String zipEntryPath = getZipEntryPath(archiveFilePath); - ZipEntry zipEntry = zipFile.getEntry(zipEntryPath); + if (log.isDebugEnabled()) { + log.debug("Check if entry " + zipEntryPath + " exists."); + } - if (zipEntry == null) { + ZipEntry zipEntry = zipFile.getEntry(zipEntryPath); - if (log.isInfoEnabled()) { - log.info("Optional entry " + zipEntryPath + " not found."); - } + if (zipEntry == null) { - result.add(archiveFilePath); + if (log.isInfoEnabled()) { + log.info("Entry " + zipEntryPath + " not found."); + } - } else { + result.add(archiveFilePath); - if (log.isInfoEnabled()) { - log.info("Optional entry " + zipEntryPath + " found."); - } + } else { + if (log.isInfoEnabled()) { + log.info("Entry " + zipEntryPath + " found."); } } @@ -465,10 +455,12 @@ public class GenericFormatArchive implements Serializable { } catch (IOException e) { - throw new ApplicationTechnicalException("Could not open zip file: " + archiveFile, e); + throw new ApplicationTechnicalException("Could not open or close zip file: " + archiveFile, e); + } return result; + } protected int countImportLines(ArchiveFilePath archiveFilePath) { diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatArchiveInvalidLayoutException.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatArchiveInvalidLayoutException.java index f2c4a22..efebddc 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatArchiveInvalidLayoutException.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatArchiveInvalidLayoutException.java @@ -1,7 +1,12 @@ package fr.ifremer.tutti.service.genericformat; +import com.google.common.base.Joiner; import org.nuiton.jaxx.application.ApplicationBusinessException; +import java.util.Set; + +import static org.nuiton.i18n.I18n.t; + /** * Created on 2/24/15. * @@ -12,8 +17,22 @@ public class GenericFormatArchiveInvalidLayoutException extends ApplicationBusin private static final long serialVersionUID = 1L; - public GenericFormatArchiveInvalidLayoutException(String message) { - super(message); + private final GenericFormatArchive archive; + + private final Set<String> errors; + + public GenericFormatArchiveInvalidLayoutException(GenericFormatArchive archive, Set<String> errors) { + super(t("tutti.service.genericFormat.importError.archiveNotSane", Joiner.on("\n").join(errors))); + this.archive = archive; + this.errors = errors; + } + + public GenericFormatArchive getArchive() { + return archive; + } + + public Set<String> getErrors() { + return errors; } } diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatFileResult.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatFileResult.java index 9f78be6..3c1ae39 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatFileResult.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatFileResult.java @@ -1,7 +1,10 @@ package fr.ifremer.tutti.service.genericformat; +import com.google.common.collect.Sets; +import fr.ifremer.tutti.service.csv.CsvComsumer; + import java.io.Serializable; -import java.util.Collections; +import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; @@ -17,17 +20,15 @@ public class GenericFormatFileResult implements Serializable { private final String filename; - private final int linesCount; + private final Map<Long, Set<String>> errors; + + private int linesCount; - private final Map<Long, String> errors; + private boolean imported; - public GenericFormatFileResult(String filename, int linesCount, Map<Long, String> errors) { + public GenericFormatFileResult(String filename) { this.filename = filename; - this.linesCount = linesCount; - if (errors == null) { - errors = Collections.emptyMap(); - } - this.errors = errors; + this.errors = new LinkedHashMap<>(); } public String getFilename() { @@ -38,7 +39,11 @@ public class GenericFormatFileResult implements Serializable { return linesCount; } - public Set<Map.Entry<Long, String>> getErrorsEntries() { + public void setLinesCount(int linesCount) { + this.linesCount = linesCount; + } + + public Set<Map.Entry<Long, Set<String>>> getErrorsEntries() { return errors.entrySet(); } @@ -46,7 +51,24 @@ public class GenericFormatFileResult implements Serializable { return errors.isEmpty(); } - public void addErrors(Map<Long, String> errors) { - errors.putAll(errors); + public void addGlobalError(String error) { + this.errors.put(0l, Sets.newHashSet(error)); + } + + public boolean isImported() { + return imported; + } + + public void setImported(boolean imported) { + this.imported = imported; + } + + public void flushErrors(CsvComsumer<?, ?> consumer) { + + Map<Long, Set<String>> rowsInError = consumer.getRowsInError(); + if (rowsInError.isEmpty()) { + errors.putAll(rowsInError); + } + } } diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatImportContext.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatImportContext.java index ee74be5..e70cb70 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatImportContext.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatImportContext.java @@ -1,6 +1,7 @@ package fr.ifremer.tutti.service.genericformat; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; import fr.ifremer.tutti.persistence.ProgressionModel; import fr.ifremer.tutti.persistence.entities.data.CatchBatch; import fr.ifremer.tutti.persistence.entities.data.Cruise; @@ -30,6 +31,7 @@ import fr.ifremer.tutti.service.referential.consumer.CsvConsumerForTemporaryGear import fr.ifremer.tutti.service.referential.consumer.CsvConsumerForTemporaryPerson; import fr.ifremer.tutti.service.referential.consumer.CsvConsumerForTemporarySpecies; import fr.ifremer.tutti.service.referential.consumer.CsvConsumerForTemporaryVessel; +import org.apache.commons.collections4.CollectionUtils; import org.nuiton.decorator.Decorator; import java.io.Closeable; @@ -37,6 +39,7 @@ import java.io.Serializable; import java.text.ParseException; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Set; /** * Created on 2/5/15. @@ -58,17 +61,19 @@ public class GenericFormatImportContext implements Closeable { private final GenericFormatImportValidationHelper validationHelper; + private Set<String> archiveLayoutErrors; + private TuttiProtocol importedProtocol; - private GenericFormatReferentialImportResult<Gear, Integer> importedGears; + private final GenericFormatFileResult sampleCategoryFileResult; - private GenericFormatReferentialImportResult<Person, Integer> importedPersons; + private final GenericFormatReferentialImportResult<Gear, Integer> referentialTemporaryGearFileResult; - private GenericFormatReferentialImportResult<Species, Integer> importedSpecies; + private final GenericFormatReferentialImportResult<Person, Integer> referentialTemporaryPersonFileResult; - private GenericFormatReferentialImportResult<Vessel, String> importedVessels; + private final GenericFormatReferentialImportResult<Species, Integer> referentialTemporarySpeciesFileResult; - private final Map<String, GenericFormatImportCruiseContext> cruiseContexts; + private final GenericFormatReferentialImportResult<Vessel, String> referentialTemporaryVesselFileResult; private final GenericFormatFileResult surveyFileResult; @@ -86,6 +91,8 @@ public class GenericFormatImportContext implements Closeable { private final GenericFormatFileResult gearCaracteristicFileResult; + private final Map<String, GenericFormatImportCruiseContext> cruiseContexts; + public GenericFormatImportContext(GenericFormatImportRequest importRequest, ProgressionModel progressionModel, PersistenceService persistenceService, @@ -106,14 +113,19 @@ public class GenericFormatImportContext implements Closeable { this.validationHelper = new GenericFormatImportValidationHelper(this, new ValidationService()); this.cruiseContexts = new LinkedHashMap<>(); - this.surveyFileResult = new GenericFormatFileResult("survey.csv", importRequest.getArchive().getSurveyLineCount(), null); - this.gearCaracteristicFileResult = new GenericFormatFileResult("sgearCaracteristics.csv", importRequest.getArchive().getGearCaracteristicsPathLineCount(), null); - this.operationFileResult = new GenericFormatFileResult("operation.csv", importRequest.getArchive().getOperationPathLineCount(), null); - this.parameterFileResult = new GenericFormatFileResult("parameter.csv", importRequest.getArchive().getParameterPathLineCount(), null); - this.catchFileResult = new GenericFormatFileResult("catch.csv", importRequest.getArchive().getCatchPathLineCount(), null); - this.marineLitterFileResult = new GenericFormatFileResult("marineLitter.csv", importRequest.getArchive().getMarineLitterPathLineCount(), null); - this.individualObservationFileResult = new GenericFormatFileResult("individualObservation.csv", importRequest.getArchive().getIndividualObservationPathLineCount(), null); - this.accidentalCatchFileResult = new GenericFormatFileResult("accidentalCatch.csv", importRequest.getArchive().getAccidentalCatchPathLineCount(), null); + this.sampleCategoryFileResult = new GenericFormatFileResult("sampleCategory.csv"); + this.surveyFileResult = new GenericFormatFileResult("survey.csv"); + this.gearCaracteristicFileResult = new GenericFormatFileResult("gearCaracteristics.csv"); + this.operationFileResult = new GenericFormatFileResult("operation.csv"); + this.parameterFileResult = new GenericFormatFileResult("parameter.csv"); + this.catchFileResult = new GenericFormatFileResult("catch.csv"); + this.marineLitterFileResult = new GenericFormatFileResult("marineLitter.csv"); + this.individualObservationFileResult = new GenericFormatFileResult("individualObservation.csv"); + this.accidentalCatchFileResult = new GenericFormatFileResult("accidentalCatch.csv"); + this.referentialTemporaryGearFileResult = new GenericFormatReferentialImportResult<>("temporaryGears.csv"); + this.referentialTemporaryPersonFileResult = new GenericFormatReferentialImportResult<>("temporaryPersons.csv"); + this.referentialTemporarySpeciesFileResult = new GenericFormatReferentialImportResult<>("temporarySpecies.csv"); + this.referentialTemporaryVesselFileResult = new GenericFormatReferentialImportResult<>("temporaryVessels.csv"); } @@ -129,6 +141,38 @@ public class GenericFormatImportContext implements Closeable { } + public boolean isArchiveLayoutValid() { + return CollectionUtils.isEmpty(archiveLayoutErrors); + } + + public void setArchiveLayoutErrors(Set<String> archiveLayoutErrors) { + this.archiveLayoutErrors = archiveLayoutErrors; + } + + public Set<String> getArchiveLayoutErrors() { + return ImmutableSet.copyOf(archiveLayoutErrors); + } + + public GenericFormatFileResult getSampleCategoryFileResult() { + return sampleCategoryFileResult; + } + + public GenericFormatReferentialImportResult<Gear, Integer> getReferentialTemporaryGearFileResult() { + return referentialTemporaryGearFileResult; + } + + public GenericFormatReferentialImportResult<Person, Integer> getReferentialTemporaryPersonFileResult() { + return referentialTemporaryPersonFileResult; + } + + public GenericFormatReferentialImportResult<Species, Integer> getReferentialTemporarySpeciesFileResult() { + return referentialTemporarySpeciesFileResult; + } + + public GenericFormatReferentialImportResult<Vessel, String> getReferentialTemporaryVesselFileResult() { + return referentialTemporaryVesselFileResult; + } + public GenericFormatFileResult getSurveyFileResult() { return surveyFileResult; } @@ -173,38 +217,6 @@ public class GenericFormatImportContext implements Closeable { this.importedProtocol = importedProtocol; } - public GenericFormatReferentialImportResult<Gear, Integer> getImportedGears() { - return importedGears; - } - - public void setImportedGears(GenericFormatReferentialImportResult<Gear, Integer> importedGears) { - this.importedGears = importedGears; - } - - public GenericFormatReferentialImportResult<Person, Integer> getImportedPersons() { - return importedPersons; - } - - public void setImportedPersons(GenericFormatReferentialImportResult<Person, Integer> importedPersons) { - this.importedPersons = importedPersons; - } - - public GenericFormatReferentialImportResult<Species, Integer> getImportedSpecies() { - return importedSpecies; - } - - public void setImportedSpecies(GenericFormatReferentialImportResult<Species, Integer> importedSpecies) { - this.importedSpecies = importedSpecies; - } - - public GenericFormatReferentialImportResult<Vessel, String> getImportedVessels() { - return importedVessels; - } - - public void setImportedVessels(GenericFormatReferentialImportResult<Vessel, String> importedVessels) { - this.importedVessels = importedVessels; - } - public Cruise getExistingCruise(Cruise cruise) { return importRequest.getExistingCruise(cruise); } @@ -266,67 +278,93 @@ public class GenericFormatImportContext implements Closeable { } public CsvConsumerForSampleCategory loadSampleCategories(boolean failFast) { + sampleCategoryFileResult.setImported(true); + surveyFileResult.setLinesCount(importRequest.getArchive().getSampleCategoryLineCount()); CsvConsumerForSampleCategory consumer = new CsvConsumerForSampleCategory(importRequest.getArchive().getSampleCategoryModelPath(), importRequest.getCsvSeparator(), importEntityParserFactory, failFast); return consumer; } public CsvConsumerForSurvey loadSurveys(boolean failFast) { + surveyFileResult.setImported(true); + surveyFileResult.setLinesCount(importRequest.getArchive().getSurveyLineCount()); CsvConsumerForSurvey consumer = new CsvConsumerForSurvey(importRequest.getArchive().getSurveyPath(), importRequest.getCsvSeparator(), importEntityParserFactory, failFast); return consumer; } public CsvConsumerForGearCaracteristic loadGearCaracteristics(boolean failFast) { + gearCaracteristicFileResult.setImported(true); + gearCaracteristicFileResult.setLinesCount(importRequest.getArchive().getGearCaracteristicsPathLineCount()); CsvConsumerForGearCaracteristic consumer = new CsvConsumerForGearCaracteristic(importRequest.getArchive().getGearCaracteristicsPath(), importRequest.getCsvSeparator(), importEntityParserFactory, failFast); return consumer; } public CsvConsumerForOperation loadOperations(boolean failFast) { + operationFileResult.setImported(true); + operationFileResult.setLinesCount(importRequest.getArchive().getOperationPathLineCount()); CsvConsumerForOperation consumer = new CsvConsumerForOperation(importRequest.getArchive().getOperationPath(), importRequest.getCsvSeparator(), importEntityParserFactory, failFast); return consumer; } public CsvConsumerForParameter loadParameters(boolean failFast) { + parameterFileResult.setImported(true); + parameterFileResult.setLinesCount(importRequest.getArchive().getParameterPathLineCount()); CsvConsumerForParameter consumer = new CsvConsumerForParameter(importRequest.getArchive().getParameterPath(), importRequest.getCsvSeparator(), importEntityParserFactory, failFast); return consumer; } public CsvConsumerForCatch loadCatches(boolean failFast) { + catchFileResult.setImported(true); + catchFileResult.setLinesCount(importRequest.getArchive().getCatchPathLineCount()); CsvConsumerForCatch consumer = new CsvConsumerForCatch(importRequest.getArchive().getCatchPath(), importRequest.getCsvSeparator(), importRequest.getSampleCategoryModel(), importEntityParserFactory, failFast); return consumer; } public CsvConsumerForMarineLitter loadMarineLitters(boolean failFast) { + marineLitterFileResult.setImported(true); + marineLitterFileResult.setLinesCount(importRequest.getArchive().getMarineLitterPathLineCount()); CsvConsumerForMarineLitter consumer = new CsvConsumerForMarineLitter(importRequest.getArchive().getMarineLitterPath(), importRequest.getCsvSeparator(), importEntityParserFactory, failFast); return consumer; } public CsvConsumerForIndividualObservation loadIndividualObservations(boolean failFast) { + individualObservationFileResult.setImported(true); + individualObservationFileResult.setLinesCount(importRequest.getArchive().getIndividualObservationPathLineCount()); CsvConsumerForIndividualObservation consumer = new CsvConsumerForIndividualObservation(importRequest.getArchive().getIndividualObservationPath(), importRequest.getCsvSeparator(), importEntityParserFactory, failFast); return consumer; } public CsvConsumerForAccidentalCatch loadAccidentalCatches(boolean failFast) { + accidentalCatchFileResult.setImported(true); + accidentalCatchFileResult.setLinesCount(importRequest.getArchive().getAccidentalCatchPathLineCount()); CsvConsumerForAccidentalCatch consumer = new CsvConsumerForAccidentalCatch(importRequest.getArchive().getAccidentalCatchPath(), importRequest.getCsvSeparator(), importEntityParserFactory, failFast); return consumer; } public CsvConsumerForTemporaryGear loadTemporaryGears(boolean failFast) { + referentialTemporaryGearFileResult.setImported(true); + referentialTemporaryGearFileResult.setLinesCount(importRequest.getArchive().getTemporaryReferentialGearLineCount()); CsvConsumerForTemporaryGear consumer = new CsvConsumerForTemporaryGear(importRequest.getArchive().getTemporaryReferentialGearsPath(), importRequest.getCsvSeparator(), failFast); return consumer; } public CsvConsumerForTemporaryPerson loadTemporaryPersons(boolean failFast) { + referentialTemporaryPersonFileResult.setImported(true); + referentialTemporaryPersonFileResult.setLinesCount(importRequest.getArchive().getTemporaryReferentialPersonLineCount()); CsvConsumerForTemporaryPerson consumer = new CsvConsumerForTemporaryPerson(importRequest.getArchive().getTemporaryReferentialPersonsPath(), importRequest.getCsvSeparator(), failFast); return consumer; } - public CsvConsumerForTemporarySpecies loadTemporarySpecies(boolean reportError) { - CsvConsumerForTemporarySpecies consumer = new CsvConsumerForTemporarySpecies(importRequest.getArchive().getTemporaryReferentialSpeciesPath(), importRequest.getCsvSeparator(), reportError); + public CsvConsumerForTemporarySpecies loadTemporarySpecies(boolean failFast) { + referentialTemporarySpeciesFileResult.setImported(true); + referentialTemporarySpeciesFileResult.setLinesCount(importRequest.getArchive().getTemporaryReferentialSpeciesLineCount()); + CsvConsumerForTemporarySpecies consumer = new CsvConsumerForTemporarySpecies(importRequest.getArchive().getTemporaryReferentialSpeciesPath(), importRequest.getCsvSeparator(), failFast); return consumer; } - public CsvConsumerForTemporaryVessel loadTemporaryVessels(boolean reportError) { - CsvConsumerForTemporaryVessel consumer = new CsvConsumerForTemporaryVessel(importRequest.getArchive().getTemporaryReferentialVesselsPath(), importRequest.getCsvSeparator(), reportError); + public CsvConsumerForTemporaryVessel loadTemporaryVessels(boolean failFast) { + referentialTemporaryVesselFileResult.setImported(true); + referentialTemporaryVesselFileResult.setLinesCount(importRequest.getArchive().getTemporaryReferentialVesselLineCount()); + CsvConsumerForTemporaryVessel consumer = new CsvConsumerForTemporaryVessel(importRequest.getArchive().getTemporaryReferentialVesselsPath(), importRequest.getCsvSeparator(), failFast); return consumer; } diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatImportEntityParserFactory.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatImportEntityParserFactory.java index 17da0d6..0e3e682 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatImportEntityParserFactory.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatImportEntityParserFactory.java @@ -72,7 +72,7 @@ public class GenericFormatImportEntityParserFactory { public VesselParserFormatter getVesselParser() { if (vesselParser == null) { - vesselParser = VesselParserFormatter.newParser(persistenceService, importContext.getImportedVessels().getIdTranslationMap()); + vesselParser = VesselParserFormatter.newParser(persistenceService, importContext.getReferentialTemporaryVesselFileResult().getIdTranslationMap()); } return vesselParser; } @@ -86,7 +86,7 @@ public class GenericFormatImportEntityParserFactory { public GearParserFormatter getGearParser() { if (gearParser == null) { - gearParser = GearParserFormatter.newParser(persistenceService, importContext.getImportedGears().getIdTranslationMap()); + gearParser = GearParserFormatter.newParser(persistenceService, importContext.getReferentialTemporaryGearFileResult().getIdTranslationMap()); } return gearParser; } @@ -101,7 +101,7 @@ public class GenericFormatImportEntityParserFactory { public PersonListParserFormatter getPersonListParser() { if (personListParser == null) { - PersonParserFormatter delegateParserFormatter = PersonParserFormatter.newParser(persistenceService, importContext.getImportedPersons().getIdTranslationMap()); + PersonParserFormatter delegateParserFormatter = PersonParserFormatter.newParser(persistenceService, importContext.getReferentialTemporaryPersonFileResult().getIdTranslationMap()); personListParser = PersonListParserFormatter.newParser(delegateParserFormatter); } return personListParser; @@ -172,7 +172,7 @@ public class GenericFormatImportEntityParserFactory { public SpeciesParserFormatter getSpeciesParser() { if (speciesParser == null) { - speciesParser = SpeciesParserFormatter.newParser(persistenceService, importContext.getImportedSpecies().getIdTranslationMap()); + speciesParser = SpeciesParserFormatter.newParser(persistenceService, importContext.getReferentialTemporarySpeciesFileResult().getIdTranslationMap()); } return speciesParser; } diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatImportService.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatImportService.java index be0677f..88a22d2 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatImportService.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatImportService.java @@ -189,6 +189,8 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { GenericFormatArchive archive = createArchive(importConfiguration); + archive.validateArchiveLayout(); + GenericFormatImportRequest importRequest = createImportRequest(importConfiguration, archive); GenericFormatImportResult result = doImport(importRequest, progressionModel); @@ -219,25 +221,43 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { } + protected void validateArchiveLayout(GenericFormatValidateFileContext importContext) { + + try { + importContext.getImportRequest().getArchive().validateArchiveLayout(); + } catch (GenericFormatArchiveInvalidLayoutException e) { + + importContext.setArchiveLayoutErrors(e.getErrors()); + + } + + } + protected GenericFormatValidateFileResult doValidate(GenericFormatImportRequest request, ProgressionModel progressionModel) { try (GenericFormatValidateFileContext importContext = new GenericFormatValidateFileContext(request, progressionModel, persistenceService, cruiseDecorator, fishingOperationDecorator)) { - checkSampleCategoryModel(importContext); - importTemporaryGears(importContext, false); - importTemporaryPersons(importContext, false); - importTemporarySpecies(importContext, false); - importTemporaryVessels(importContext, false); - importProtocol(importContext); + validateArchiveLayout(importContext); + + if (importContext.isArchiveLayoutValid()) { - validateCruises(importContext); - validateGearCaracteristics(importContext); - validateOperations(importContext); - validateParameters(importContext); - validateCatches(importContext); - validateMarineLitters(importContext); - validateIndividualObservations(importContext); - validateAccidentalCatches(importContext); + checkSampleCategoryModel(importContext); + importTemporaryGears(importContext, false); + importTemporaryPersons(importContext, false); + importTemporarySpecies(importContext, false); + importTemporaryVessels(importContext, false); + importProtocol(importContext); + + validateCruises(importContext); + validateGearCaracteristics(importContext); + validateOperations(importContext); + validateParameters(importContext); + validateCatches(importContext); + validateMarineLitters(importContext); + validateIndividualObservations(importContext); + validateAccidentalCatches(importContext); + + } GenericFormatValidateFileResult result = new GenericFormatValidateFileResult(importContext); return result; @@ -316,7 +336,6 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { Preconditions.checkState(importFile.exists()); GenericFormatArchive archive = GenericFormatArchive.forImport(importFile, context.getConfig().getTmpDirectory()); - archive.validateArchiveLayout(); return archive; @@ -329,7 +348,8 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { } importContext.increments(t("tutti.service.genericFormat.import.sampleCategoryModel")); - try (CsvConsumerForSampleCategory consumer = importContext.loadSampleCategories(true)) { + GenericFormatFileResult importFileResult = importContext.getSampleCategoryFileResult(); + try (CsvConsumerForSampleCategory consumer = importContext.loadSampleCategories(false)) { int nbCategories = 0; for (ImportRow<SampleCategoryRow> row : consumer) { @@ -343,8 +363,14 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { } + importFileResult.flushErrors(consumer); + } catch (IOException e) { - throw new ApplicationTechnicalException("Could not close sampleCategoryModel.csv file", e); + throw new ApplicationTechnicalException("Could not close sampleCategory.csv file", e); + } catch (ImportRuntimeException e) { + + importFileResult.addGlobalError(e.getMessage()); + } } @@ -354,37 +380,35 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { importContext.increments(t("tutti.service.genericFormat.import.temporaryGears")); GenericFormatArchive archive = importContext.getImportRequest().getArchive(); - GenericFormatReferentialImportResult<Gear, Integer> importResult; - if (archive.isTemporaryReferentialGearsPathExists()) { if (log.isInfoEnabled()) { log.info("Import temporary gears."); } - Map<Long, String> rowsInError = null; ReferentialImportRequest<Gear, Integer> referentialImportRequest = referentialTemporaryGearService.createReferentialImportRequest(); + GenericFormatReferentialImportResult<Gear, Integer> importFileResult = importContext.getReferentialTemporaryGearFileResult(); try (CsvConsumerForTemporaryGear consumer = importContext.loadTemporaryGears(reportError)) { for (ImportRow<GearRow> row : consumer) { consumer.checkRowForGenericFormatImport(row, referentialImportRequest); } - if (consumer.foundSomeErrors()) { - rowsInError = consumer.getRowsInError(); - } + importFileResult.flushErrors(consumer); } catch (IOException e) { throw new ApplicationTechnicalException("Could not close gears.csv file", e); + } catch (ImportRuntimeException e) { + + importFileResult.addGlobalError(e.getMessage()); + } ReferentialImportResult<Gear> referentialImportResult = referentialTemporaryGearService.executeImportRequest(referentialImportRequest); - - int linesCount = importContext.getImportRequest().getArchive().getTemporaryReferentialGearLineCount(); - importResult = new GenericFormatReferentialImportResult<>("gear.csv", linesCount, referentialImportRequest, referentialImportResult, rowsInError); + importFileResult.flushResult(referentialImportRequest, referentialImportResult); if (log.isInfoEnabled()) { - log.info("Temporary gears import result: " + importResult.getReport()); + log.info("Temporary gears import result: " + importFileResult.getReport()); } } else { @@ -392,12 +416,8 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { if (log.isInfoEnabled()) { log.info("Skip import temporary gears (no file found)."); } - importResult = new GenericFormatReferentialImportResult<>("gear.csv"); - } - importContext.setImportedGears(importResult); - } protected void importTemporaryPersons(GenericFormatImportContext importContext, boolean reportError) { @@ -405,36 +425,34 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { importContext.increments(t("tutti.service.genericFormat.import.temporaryPersons")); GenericFormatArchive archive = importContext.getImportRequest().getArchive(); - GenericFormatReferentialImportResult<Person, Integer> importResult; - if (archive.isTemporaryReferentialPersonsPathExists()) { if (log.isInfoEnabled()) { log.info("Import temporary persons."); } - Map<Long, String> rowsInError = null; + GenericFormatReferentialImportResult<Person, Integer> importFileResult = importContext.getReferentialTemporaryPersonFileResult(); ReferentialImportRequest<Person, Integer> referentialImportRequest = referentialTemporaryPersonService.createReferentialImportRequest(); - try (CsvConsumerForTemporaryPerson consumer = importContext.loadTemporaryPersons(reportError)) { for (ImportRow<PersonRow> row : consumer) { consumer.checkRowForGenericFormatImport(row, referentialImportRequest); } - if (consumer.foundSomeErrors()) { - rowsInError = consumer.getRowsInError(); - } + importFileResult.flushErrors(consumer); + } catch (IOException e) { throw new ApplicationTechnicalException("Could not close person.csv file", e); + } catch (ImportRuntimeException e) { + + importFileResult.addGlobalError(e.getMessage()); + } ReferentialImportResult<Person> referentialImportResult = referentialTemporaryPersonService.executeImportRequest(referentialImportRequest); - - int linesCount = importContext.getImportRequest().getArchive().getTemporaryReferentialPersonLineCount(); - importResult = new GenericFormatReferentialImportResult<>("person.csv", linesCount, referentialImportRequest, referentialImportResult, rowsInError); + importFileResult.flushResult(referentialImportRequest, referentialImportResult); if (log.isInfoEnabled()) { - log.info("Temporary persons import result: " + importResult.getReport()); + log.info("Temporary persons import result: " + importFileResult.getReport()); } } else { @@ -442,20 +460,15 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { if (log.isInfoEnabled()) { log.info("Skip import temporary persons (no file found)."); } - importResult = new GenericFormatReferentialImportResult<>("person.csv"); } - importContext.setImportedPersons(importResult); - } protected void importTemporarySpecies(GenericFormatImportContext importContext, boolean reportError) { importContext.increments(t("tutti.service.genericFormat.import.temporarySpecies")); - GenericFormatReferentialImportResult<Species, Integer> importResult; - GenericFormatArchive archive = importContext.getImportRequest().getArchive(); if (archive.isTemporaryReferentialSpeciesPathExists()) { @@ -465,38 +478,36 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { ReferentialImportRequest<Species, Integer> referentialImportRequest = referentialTemporarySpeciesService.createReferentialImportRequest(); - Map<Long, String> rowsInError = null; + GenericFormatReferentialImportResult<Species, Integer> importFileResult = importContext.getReferentialTemporarySpeciesFileResult(); try (CsvConsumerForTemporarySpecies consumer = importContext.loadTemporarySpecies(reportError)) { for (ImportRow<SpeciesRow> row : consumer) { consumer.checkRowForGenericFormatImport(row, referentialImportRequest); } - if (consumer.foundSomeErrors()) { - rowsInError = consumer.getRowsInError(); - } + importFileResult.flushErrors(consumer); + } catch (IOException e) { throw new ApplicationTechnicalException("Could not close species.csv file", e); + } catch (ImportRuntimeException e) { + + importFileResult.addGlobalError(e.getMessage()); + } ReferentialImportResult<Species> referentialImportResult = referentialTemporarySpeciesService.executeImportRequest(referentialImportRequest); - - int linesCount = importContext.getImportRequest().getArchive().getTemporaryReferentialSpeciesLineCount(); - importResult = new GenericFormatReferentialImportResult<>("species.csv", linesCount, referentialImportRequest, referentialImportResult, rowsInError); + importFileResult.flushResult(referentialImportRequest, referentialImportResult); if (log.isInfoEnabled()) { - log.info("Temporary species import result: " + importResult.getReport()); + log.info("Temporary species import result: " + importFileResult.getReport()); } } else { if (log.isInfoEnabled()) { log.info("Skip import temporary species (no file found)."); } - importResult = new GenericFormatReferentialImportResult<>("species.csv"); } - importContext.setImportedSpecies(importResult); - } protected void importTemporaryVessels(GenericFormatImportContext importContext, boolean reportError) { @@ -504,8 +515,6 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { importContext.increments(t("tutti.service.genericFormat.import.temporaryVessels")); GenericFormatArchive archive = importContext.getImportRequest().getArchive(); - GenericFormatReferentialImportResult<Vessel, String> importResult; - if (archive.isTemporaryReferentialVesselsPathExists()) { if (log.isInfoEnabled()) { @@ -514,26 +523,27 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { ReferentialImportRequest<Vessel, String> referentialImportRequest = referentialTemporaryVesselService.createReferentialImportRequest(); - Map<Long, String> rowsInError = null; + GenericFormatReferentialImportResult<Vessel, String> importFileResult = importContext.getReferentialTemporaryVesselFileResult(); try (CsvConsumerForTemporaryVessel consumer = importContext.loadTemporaryVessels(reportError)) { for (ImportRow<VesselRow> row : consumer) { consumer.checkRowForGenericFormatImport(row, referentialImportRequest); } - if (consumer.foundSomeErrors()) { - rowsInError = consumer.getRowsInError(); - } + importFileResult.flushErrors(consumer); + } catch (IOException e) { throw new ApplicationTechnicalException("Could not close vessel.csv file", e); + } catch (ImportRuntimeException e) { + + importFileResult.addGlobalError(e.getMessage()); + } ReferentialImportResult<Vessel> referentialImportResult = referentialTemporaryVesselService.executeImportRequest(referentialImportRequest); - - int linesCount = importContext.getImportRequest().getArchive().getTemporaryReferentialVesselLineCount(); - importResult = new GenericFormatReferentialImportResult<>("vessel.csv", linesCount, referentialImportRequest, referentialImportResult, rowsInError); + importFileResult.flushResult(referentialImportRequest, referentialImportResult); if (log.isInfoEnabled()) { - log.info("Temporary vessels import result: " + importResult.getReport()); + log.info("Temporary vessels import result: " + importFileResult.getReport()); } } else { @@ -541,12 +551,9 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { if (log.isInfoEnabled()) { log.info("Skip import temporary vessels (no file found)."); } - importResult = new GenericFormatReferentialImportResult<>("vessel.csv"); } - importContext.setImportedVessels(importResult); - } protected void importProtocol(GenericFormatImportContext importContext) { @@ -560,7 +567,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { TuttiProtocol tuttiProtocol = protocolImportExportService.importProtocol(archive.getProtocolPath().toFile()); - Map<String, String> idTranslationMap = importContext.getImportedSpecies().getIdTranslationMap(); + Map<String, String> idTranslationMap = importContext.getReferentialTemporarySpeciesFileResult().getIdTranslationMap(); TuttiProtocols.translateReferenceTaxonIds(tuttiProtocol, idTranslationMap); String newName = persistenceService.getFirstAvailableName(tuttiProtocol.getName()); @@ -582,6 +589,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { importContext.increments(t("tutti.service.genericFormat.skip.import.protocol")); } + } protected void validateCruises(GenericFormatValidateFileContext importContext) { @@ -590,6 +598,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { log.info("Validate survey.csv file."); } + GenericFormatFileResult importFileResult = importContext.getSurveyFileResult(); try (CsvConsumerForSurvey consumer = importContext.loadSurveys(false)) { for (ImportRow<SurveyRow> row : consumer) { @@ -605,15 +614,15 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { } - if (consumer.foundSomeErrors()) { - Map<Long, String> rowsInError = consumer.getRowsInError(); - importContext.getSurveyFileResult().addErrors(rowsInError); - } + importFileResult.flushErrors(consumer); } catch (IOException e) { throw new ApplicationTechnicalException("Could not close survey.csv file", e); - } + } catch (ImportRuntimeException e) { + importFileResult.addGlobalError(e.getMessage()); + + } } @@ -623,6 +632,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { log.info("Validate gearCaracteristics.csv file."); } + GenericFormatFileResult importFileResult = importContext.getGearCaracteristicFileResult(); try (CsvConsumerForGearCaracteristic consumer = importContext.loadGearCaracteristics(false)) { for (ImportRow<GearCaracteristicRow> row : consumer) { @@ -633,13 +643,14 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { } - if (consumer.foundSomeErrors()) { - Map<Long, String> rowsInError = consumer.getRowsInError(); - importContext.getGearCaracteristicFileResult().addErrors(rowsInError); - } + importFileResult.flushErrors(consumer); } catch (IOException e) { throw new ApplicationTechnicalException("Could not close gearCaracteristic.csv file", e); + } catch (ImportRuntimeException e) { + + importFileResult.addGlobalError(e.getMessage()); + } } @@ -650,6 +661,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { log.info("Validate operation.csv file."); } + GenericFormatFileResult importFileResult = importContext.getOperationFileResult(); try (CsvConsumerForOperation consumer = importContext.loadOperations(false)) { for (ImportRow<OperationRow> row : consumer) { @@ -667,13 +679,14 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { } - if (consumer.foundSomeErrors()) { - Map<Long, String> rowsInError = consumer.getRowsInError(); - importContext.getOperationFileResult().addErrors(rowsInError); - } + importFileResult.flushErrors(consumer); } catch (IOException e) { throw new ApplicationTechnicalException("Could not close operation.csv file", e); + } catch (ImportRuntimeException e) { + + importFileResult.addGlobalError(e.getMessage()); + } } @@ -684,6 +697,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { log.info("Validate parameter.csv file."); } + GenericFormatFileResult importFileResult = importContext.getParameterFileResult(); try (CsvConsumerForParameter consumer = importContext.loadParameters(false)) { for (ImportRow<ParameterRow> row : consumer) { @@ -694,13 +708,14 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { } - if (consumer.foundSomeErrors()) { - Map<Long, String> rowsInError = consumer.getRowsInError(); - importContext.getParameterFileResult().addErrors(rowsInError); - } + importFileResult.flushErrors(consumer); } catch (IOException e) { throw new ApplicationTechnicalException("Could not close parameter.csv file", e); + } catch (ImportRuntimeException e) { + + importFileResult.addGlobalError(e.getMessage()); + } } @@ -711,6 +726,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { log.info("Validate catch.csv file."); } + GenericFormatFileResult importFileResult = importContext.getCatchFileResult(); try (CsvConsumerForCatch consumer = importContext.loadCatches(false)) { for (ImportRow<CatchRow> row : consumer) { @@ -721,13 +737,14 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { } - if (consumer.foundSomeErrors()) { - Map<Long, String> rowsInError = consumer.getRowsInError(); - importContext.getCatchFileResult().addErrors(rowsInError); - } + importFileResult.flushErrors(consumer); } catch (IOException e) { throw new ApplicationTechnicalException("Could not close catch.csv file", e); + } catch (ImportRuntimeException e) { + + importFileResult.addGlobalError(e.getMessage()); + } } @@ -738,6 +755,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { log.info("Validate marineLitter.csv file."); } + GenericFormatFileResult importFileResult = importContext.getMarineLitterFileResult(); try (CsvConsumerForMarineLitter consumer = importContext.loadMarineLitters(false)) { for (ImportRow<MarineLitterRow> row : consumer) { @@ -748,13 +766,14 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { } - if (consumer.foundSomeErrors()) { - Map<Long, String> rowsInError = consumer.getRowsInError(); - importContext.getMarineLitterFileResult().addErrors(rowsInError); - } + importFileResult.flushErrors(consumer); } catch (IOException e) { throw new ApplicationTechnicalException("Could not close marineLitter.csv file", e); + } catch (ImportRuntimeException e) { + + importFileResult.addGlobalError(e.getMessage()); + } } @@ -765,6 +784,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { log.info("Validate individualObservation.csv file."); } + GenericFormatFileResult importFileResult = importContext.getIndividualObservationFileResult(); try (CsvConsumerForIndividualObservation consumer = importContext.loadIndividualObservations(false)) { for (ImportRow<IndividualObservationRow> row : consumer) { @@ -775,13 +795,14 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { } - if (consumer.foundSomeErrors()) { - Map<Long, String> rowsInError = consumer.getRowsInError(); - importContext.getIndividualObservationFileResult().addErrors(rowsInError); - } + importFileResult.flushErrors(consumer); } catch (IOException e) { throw new ApplicationTechnicalException("Could not close individualObservation.csv file", e); + } catch (ImportRuntimeException e) { + + importFileResult.addGlobalError(e.getMessage()); + } } @@ -792,6 +813,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { log.info("Validate accidentalCatch.csv file."); } + GenericFormatFileResult importFileResult = importContext.getAccidentalCatchFileResult(); try (CsvConsumerForAccidentalCatch consumer = importContext.loadAccidentalCatches(false)) { for (ImportRow<AccidentalCatchRow> row : consumer) { @@ -802,13 +824,14 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { } - if (consumer.foundSomeErrors()) { - Map<Long, String> rowsInError = consumer.getRowsInError(); - importContext.getAccidentalCatchFileResult().addErrors(rowsInError); - } + importFileResult.flushErrors(consumer); } catch (IOException e) { throw new ApplicationTechnicalException("Could not close accidentalCatch.csv file", e); + } catch (ImportRuntimeException e) { + + importFileResult.addGlobalError(e.getMessage()); + } } @@ -819,7 +842,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { log.info("Import survey.csv file."); } importContext.increments(t("tutti.service.genericFormat.import.cruises")); - + GenericFormatFileResult importFileResult = importContext.getSurveyFileResult(); try (CsvConsumerForSurvey consumer = importContext.loadSurveys(true)) { for (ImportRow<SurveyRow> row : consumer) { @@ -831,6 +854,10 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { } } catch (IOException e) { throw new ApplicationTechnicalException("Could not close survey.csv file", e); + } catch (ImportRuntimeException e) { + + importFileResult.addGlobalError(e.getMessage()); + } } @@ -841,7 +868,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { log.info("Import gearCaracteristics.csv file."); } importContext.increments(t("tutti.service.genericFormat.import.gearCaracteristics")); - + GenericFormatFileResult importFileResult = importContext.getGearCaracteristicFileResult(); try (CsvConsumerForGearCaracteristic consumer = importContext.loadGearCaracteristics(true)) { for (ImportRow<GearCaracteristicRow> row : consumer) { @@ -851,6 +878,10 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { } } catch (IOException e) { throw new ApplicationTechnicalException("Could not close gearCaracteristic.csv file", e); + } catch (ImportRuntimeException e) { + + importFileResult.addGlobalError(e.getMessage()); + } persitenceHelper.persistGearCaracteristics(); @@ -863,7 +894,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { log.info("Import operation.csv file."); } importContext.increments(t("tutti.service.genericFormat.import.operations")); - + GenericFormatFileResult importFileResult = importContext.getOperationFileResult(); try (CsvConsumerForOperation consumer = importContext.loadOperations(true)) { for (ImportRow<OperationRow> row : consumer) { @@ -880,6 +911,10 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { } } catch (IOException e) { throw new ApplicationTechnicalException("Could not close operation.csv file", e); + } catch (ImportRuntimeException e) { + + importFileResult.addGlobalError(e.getMessage()); + } } @@ -890,7 +925,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { log.info("Import parameter.csv file."); } importContext.increments(t("tutti.service.genericFormat.import.parameters")); - + GenericFormatFileResult importFileResult = importContext.getParameterFileResult(); try (CsvConsumerForParameter consumer = importContext.loadParameters(true)) { for (ImportRow<ParameterRow> row : consumer) { @@ -900,6 +935,10 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { } } catch (IOException e) { throw new ApplicationTechnicalException("Could not close parameter.csv file", e); + } catch (ImportRuntimeException e) { + + importFileResult.addGlobalError(e.getMessage()); + } persitenceHelper.persistFishingOperationParameters(); @@ -912,7 +951,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { log.info("Import catch.csv file."); } importContext.increments(t("tutti.service.genericFormat.import.catches")); - + GenericFormatFileResult importFileResult = importContext.getCatchFileResult(); try (CsvConsumerForCatch consumer = importContext.loadCatches(true)) { for (ImportRow<CatchRow> row : consumer) { @@ -922,6 +961,10 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { } } catch (IOException e) { throw new ApplicationTechnicalException("Could not close catch.csv file", e); + } catch (ImportRuntimeException e) { + + importFileResult.addGlobalError(e.getMessage()); + } persitenceHelper.persistSpeciesBatches(); @@ -935,7 +978,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { log.info("Import marineLitter.csv file."); } importContext.increments(t("tutti.service.genericFormat.import.marineLitters")); - + GenericFormatFileResult importFileResult = importContext.getMarineLitterFileResult(); try (CsvConsumerForMarineLitter consumer = importContext.loadMarineLitters(true)) { for (ImportRow<MarineLitterRow> row : consumer) { @@ -945,6 +988,10 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { } } catch (IOException e) { throw new ApplicationTechnicalException("Could not close marineLitter.csv file", e); + } catch (ImportRuntimeException e) { + + importFileResult.addGlobalError(e.getMessage()); + } persitenceHelper.persistMarineLitterBatches(); @@ -957,7 +1004,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { log.info("Import individualObservation.csv file."); } importContext.increments(t("tutti.service.genericFormat.import.individualObservations")); - + GenericFormatFileResult importFileResult = importContext.getIndividualObservationFileResult(); try (CsvConsumerForIndividualObservation consumer = importContext.loadIndividualObservations(true)) { for (ImportRow<IndividualObservationRow> row : consumer) { @@ -967,6 +1014,10 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { } } catch (IOException e) { throw new ApplicationTechnicalException("Could not close individualObservation.csv file", e); + } catch (ImportRuntimeException e) { + + importFileResult.addGlobalError(e.getMessage()); + } persitenceHelper.persistIndividualObservationBatches(); @@ -979,7 +1030,7 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { log.info("Import accidentalCatch.csv file."); } importContext.increments(t("tutti.service.genericFormat.import.accidentalCatches")); - + GenericFormatFileResult importFileResult = importContext.getAccidentalCatchFileResult(); try (CsvConsumerForAccidentalCatch consumer = importContext.loadAccidentalCatches(true)) { for (ImportRow<AccidentalCatchRow> row : consumer) { @@ -989,6 +1040,10 @@ public class GenericFormatImportService extends GenericFormatServiceSupport { } } catch (IOException e) { throw new ApplicationTechnicalException("Could not close accidentalCatch.csv file", e); + } catch (ImportRuntimeException e) { + + importFileResult.addGlobalError(e.getMessage()); + } persitenceHelper.persistAccidentalBatches(); diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatReferentialImportResult.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatReferentialImportResult.java index 7b13241..8c972b9 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatReferentialImportResult.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatReferentialImportResult.java @@ -23,22 +23,20 @@ public class GenericFormatReferentialImportResult<E extends TuttiReferentialEnti private static final long serialVersionUID = 1L; - private final Map<String, E> entitiesAdded; + private Map<String, E> entitiesAdded; - private final Map<String, E> entitiesLinked; + private Map<String, E> entitiesLinked; - private final Map<String, String> idTranslationMap; + private Map<String, String> idTranslationMap; public GenericFormatReferentialImportResult(String filename) { - super(filename, 0, null); + super(filename); this.entitiesAdded = Collections.emptyMap(); this.entitiesLinked = Collections.emptyMap(); this.idTranslationMap = Collections.emptyMap(); } - public GenericFormatReferentialImportResult(String filename, int linesCount, ReferentialImportRequest<E, K> importRequest, ReferentialImportResult<E> importResult, Map<Long, String> rowsInError) { - - super(filename, linesCount, rowsInError); + public void flushResult(ReferentialImportRequest<E, K> importRequest, ReferentialImportResult<E> importResult) { Map<String, String> translationMap = new TreeMap<>(); @@ -101,15 +99,15 @@ public class GenericFormatReferentialImportResult<E extends TuttiReferentialEnti StringBuilder builder = new StringBuilder(); - Set<Map.Entry<Long, String>> errors = getErrorsEntries(); + Set<Map.Entry<Long, Set<String>>> errors = getErrorsEntries(); if (!errors.isEmpty()) { builder.append(String.format("%s lines in error:", errors.size())); - for (Map.Entry<Long, String> entry : errors) { + for (Map.Entry<Long, Set<String>> entry : errors) { Long lineNumber = entry.getKey(); - String error = entry.getValue(); - builder.append(String.format("\nline %s : \n%s", lineNumber, error)); + Set<String> error = entry.getValue(); + builder.append(String.format("\nline %s : \n%s", lineNumber, Joiner.on("\n\t").join(error))); } } diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatResultSupport.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatResultSupport.java index efe8e1e..3276cf6 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatResultSupport.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatResultSupport.java @@ -1,5 +1,6 @@ package fr.ifremer.tutti.service.genericformat; +import com.google.common.collect.ImmutableSet; import fr.ifremer.tutti.persistence.entities.data.Cruise; import fr.ifremer.tutti.persistence.entities.data.Program; import fr.ifremer.tutti.persistence.entities.data.SampleCategoryModel; @@ -11,6 +12,7 @@ import fr.ifremer.tutti.persistence.entities.referential.Vessel; import java.io.File; import java.io.Serializable; +import java.util.Collections; import java.util.Date; import java.util.Set; @@ -28,15 +30,21 @@ public abstract class GenericFormatResultSupport implements Serializable { private GenericFormatImportRequest importRequest; + private final Set<String> archiveLayoutErrors; + + private final Set<String> archiveFiles; + private final TuttiProtocol protocol; - private final GenericFormatReferentialImportResult<Gear, Integer> temporaryGearsImported; + private final GenericFormatFileResult sampleCategoryFileResult; + + private final GenericFormatReferentialImportResult<Gear, Integer> referentialTemporaryGearFileResult; - private final GenericFormatReferentialImportResult<Person, Integer> temporaryPersonsImported; + private final GenericFormatReferentialImportResult<Person, Integer> referentialTemporaryPersonFileResult; - private final GenericFormatReferentialImportResult<Species, Integer> temporarySpeciesImported; + private final GenericFormatReferentialImportResult<Species, Integer> referentialTemporarySpeciesFileResult; - private final GenericFormatReferentialImportResult<Vessel, String> temporaryVesselsImported; + private final GenericFormatReferentialImportResult<Vessel, String> referentialTemporaryVesselFileResult; private final GenericFormatFileResult surveyFileResult; @@ -54,23 +62,44 @@ public abstract class GenericFormatResultSupport implements Serializable { private final GenericFormatFileResult accidentalCatchFileResult; - protected GenericFormatResultSupport(GenericFormatImportContext validateFileContext) { + protected GenericFormatResultSupport(GenericFormatImportContext importContext) { - this.importRequest = validateFileContext.getImportRequest(); - this.protocol = validateFileContext.getImportedProtocol(); - this.temporaryGearsImported = validateFileContext.getImportedGears(); - this.temporaryPersonsImported = validateFileContext.getImportedPersons(); - this.temporarySpeciesImported = validateFileContext.getImportedSpecies(); - this.temporaryVesselsImported = validateFileContext.getImportedVessels(); - this.surveyFileResult = validateFileContext.getSurveyFileResult(); - this.gearCaracteristicFileResult = validateFileContext.getGearCaracteristicFileResult(); - this.operationFileResult = validateFileContext.getOperationFileResult(); - this.parameterFileResult = validateFileContext.getParameterFileResult(); - this.catchFileResult = validateFileContext.getCatchFileResult(); - this.marineLitterFileResult = validateFileContext.getMarineLitterFileResult(); - this.individualObservationFileResult = validateFileContext.getIndividualObservationFileResult(); - this.accidentalCatchFileResult = validateFileContext.getAccidentalCatchFileResult(); + this.importRequest = importContext.getImportRequest(); + this.protocol = importContext.getImportedProtocol(); + this.sampleCategoryFileResult = importContext.getSampleCategoryFileResult(); + this.referentialTemporaryGearFileResult = importContext.getReferentialTemporaryGearFileResult(); + this.referentialTemporaryPersonFileResult = importContext.getReferentialTemporaryPersonFileResult(); + this.referentialTemporarySpeciesFileResult = importContext.getReferentialTemporarySpeciesFileResult(); + this.referentialTemporaryVesselFileResult = importContext.getReferentialTemporaryVesselFileResult(); + this.surveyFileResult = importContext.getSurveyFileResult(); + this.gearCaracteristicFileResult = importContext.getGearCaracteristicFileResult(); + this.operationFileResult = importContext.getOperationFileResult(); + this.parameterFileResult = importContext.getParameterFileResult(); + this.catchFileResult = importContext.getCatchFileResult(); + this.marineLitterFileResult = importContext.getMarineLitterFileResult(); + this.individualObservationFileResult = importContext.getIndividualObservationFileResult(); + this.accidentalCatchFileResult = importContext.getAccidentalCatchFileResult(); + Set<String> archiveLayoutErrors1; + if (importContext.isArchiveLayoutValid()) { + archiveLayoutErrors1 = Collections.emptySet(); + } else { + archiveLayoutErrors1 = importContext.getArchiveLayoutErrors(); + } + this.archiveLayoutErrors = ImmutableSet.copyOf(archiveLayoutErrors1); + this.archiveFiles = importContext.getImportRequest().getArchive().getImportFiles(); + } + + public boolean isArchiveLayoutValid() { + return archiveLayoutErrors.isEmpty(); + } + + public Set<String> getArchiveLayoutErrors() { + return archiveLayoutErrors; + } + + public Set<String> getArchiveFiles() { + return archiveFiles; } public boolean isProtocolImported() { @@ -81,20 +110,24 @@ public abstract class GenericFormatResultSupport implements Serializable { return protocol; } - public GenericFormatReferentialImportResult<Gear, Integer> getTemporaryGearsImported() { - return temporaryGearsImported; + public GenericFormatFileResult getSampleCategoryFileResult() { + return sampleCategoryFileResult; + } + + public GenericFormatReferentialImportResult<Gear, Integer> getReferentialTemporaryGearFileResult() { + return referentialTemporaryGearFileResult; } - public GenericFormatReferentialImportResult<Person, Integer> getTemporaryPersonsImported() { - return temporaryPersonsImported; + public GenericFormatReferentialImportResult<Person, Integer> getReferentialTemporaryPersonFileResult() { + return referentialTemporaryPersonFileResult; } - public GenericFormatReferentialImportResult<Species, Integer> getTemporarySpeciesImported() { - return temporarySpeciesImported; + public GenericFormatReferentialImportResult<Species, Integer> getReferentialTemporarySpeciesFileResult() { + return referentialTemporarySpeciesFileResult; } - public GenericFormatReferentialImportResult<Vessel, String> getTemporaryVesselsImported() { - return temporaryVesselsImported; + public GenericFormatReferentialImportResult<Vessel, String> getReferentialTemporaryVesselFileResult() { + return referentialTemporaryVesselFileResult; } public GenericFormatFileResult getSurveyFileResult() { diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatValidateFileResult.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatValidateFileResult.java index 50ef390..21f8d3c 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatValidateFileResult.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/genericformat/GenericFormatValidateFileResult.java @@ -1,10 +1,5 @@ package fr.ifremer.tutti.service.genericformat; -import com.google.common.collect.ImmutableSet; - -import java.util.LinkedHashSet; -import java.util.Set; - /** * Created on 2/24/15. * @@ -15,17 +10,8 @@ public class GenericFormatValidateFileResult extends GenericFormatResultSupport private static final long serialVersionUID = 1L; - private final Set<String> archiveFormatErrors; - public GenericFormatValidateFileResult(GenericFormatValidateFileContext validateFileContext) { super(validateFileContext); - - this.archiveFormatErrors = new LinkedHashSet<>(); - - } - - public Set<String> getArchiveFormatErrors() { - return ImmutableSet.copyOf(archiveFormatErrors); } } diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/referential/ReferentialImportRequest.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/referential/ReferentialImportRequest.java index 80aac65..7f31441 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/referential/ReferentialImportRequest.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/referential/ReferentialImportRequest.java @@ -31,6 +31,7 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; import fr.ifremer.tutti.persistence.entities.referential.TuttiReferentialEntity; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -55,6 +56,10 @@ public class ReferentialImportRequest<E extends TuttiReferentialEntity, K> { private final Set<String> existingNaturalIds; + private final Set<K> newIds; + + private final Set<String> newNaturalIds; + private final Function<E, K> entityToIdFunction; public ReferentialImportRequest(List<E> existingEntities, Function<E, K> entityToIdFunction, Function<E, String> naturalIdFunction) { @@ -62,11 +67,14 @@ public class ReferentialImportRequest<E extends TuttiReferentialEntity, K> { this.entityToIdFunction = entityToIdFunction; this.existingEntitiesById = Maps.uniqueIndex(existingEntities, entityToIdFunction); this.existingNaturalIds = Sets.newHashSet(Iterables.transform(existingEntities, naturalIdFunction)); - + this.newIds = new HashSet<>(); + this.newNaturalIds = new HashSet<>(); } public void addEntityToAdd(E entityToAdd) { toAdd.add(entityToAdd); + K id = entityToIdFunction.apply(entityToAdd); + newIds.add(id); } public void addEntityToUpdate(E entityToUpdate) { @@ -118,7 +126,19 @@ public class ReferentialImportRequest<E extends TuttiReferentialEntity, K> { } public boolean addExistingNaturalId(String naturalId) { - return existingNaturalIds.add(naturalId); + boolean added = existingNaturalIds.add(naturalId); + if (added) { + newNaturalIds.add(naturalId); + } + return added; + } + + public boolean isNaturalIdAlreadyAdded(String naturalId) { + return newNaturalIds.contains(naturalId); + } + + public boolean isIdAlreadyAdded(K id) { + return newIds.contains(id); } public void removeExistingNaturalId(String naturalId) { diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/referential/consumer/CsvConsumerForTemporaryGear.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/referential/consumer/CsvConsumerForTemporaryGear.java index d723ad2..d9e1765 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/referential/consumer/CsvConsumerForTemporaryGear.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/referential/consumer/CsvConsumerForTemporaryGear.java @@ -30,8 +30,8 @@ public class CsvConsumerForTemporaryGear extends CsvComsumer<GearRow, GearModel> /** Logger. */ private static final Log log = LogFactory.getLog(CsvConsumerForTemporaryGear.class); - public CsvConsumerForTemporaryGear(Path file, char separator, boolean reportError) { - super(file, GearModel.forImport(separator), reportError); + public CsvConsumerForTemporaryGear(Path file, char separator, boolean failFast) { + super(file, GearModel.forImport(separator), failFast); } public void checkRow(ImportRow<GearRow> row, @@ -98,12 +98,20 @@ public class CsvConsumerForTemporaryGear extends CsvComsumer<GearRow, GearModel> addCheckError(row, new ApplicationBusinessException(t("tutti.service.referential.import.gear.error.idNotTemporary", id))); + }else if (requestResult.isIdAlreadyAdded(id)) { + + addCheckError(row, new ApplicationBusinessException(t("tutti.service.referential.import.gear.error.id.alreaydAdded"))); + } if (StringUtils.isBlank(name)) { addCheckError(row, new ApplicationBusinessException(t("tutti.service.referential.import.gear.error.noName"))); + } else if (requestResult.isNaturalIdAlreadyAdded(name)) { + + addCheckError(row, new ApplicationBusinessException(t("tutti.service.referential.import.gear.error.name.alreaydAdded"))); + } } diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/referential/consumer/CsvConsumerForTemporaryPerson.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/referential/consumer/CsvConsumerForTemporaryPerson.java index d4d4a93..24c2066 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/referential/consumer/CsvConsumerForTemporaryPerson.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/referential/consumer/CsvConsumerForTemporaryPerson.java @@ -97,12 +97,20 @@ public class CsvConsumerForTemporaryPerson extends CsvComsumer<PersonRow, Person addCheckError(row, new ApplicationBusinessException(t("tutti.service.referential.import.person.error.idNotTemporary", id))); + } else if (requestResult.isIdAlreadyAdded(id)) { + + addCheckError(row, new ApplicationBusinessException(t("tutti.service.referential.import.person.error.id.alreaydAdded"))); + } if (StringUtils.isBlank(name)) { addCheckError(row, new ApplicationBusinessException(t("tutti.service.referential.import.person.error.noName"))); + } else if (requestResult.isNaturalIdAlreadyAdded(name)) { + + addCheckError(row, new ApplicationBusinessException(t("tutti.service.referential.import.person.error.name.alreaydAdded"))); + } } diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/referential/consumer/CsvConsumerForTemporarySpecies.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/referential/consumer/CsvConsumerForTemporarySpecies.java index 0dc144f..f0008e3 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/referential/consumer/CsvConsumerForTemporarySpecies.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/referential/consumer/CsvConsumerForTemporarySpecies.java @@ -97,12 +97,20 @@ public class CsvConsumerForTemporarySpecies extends CsvComsumer<SpeciesRow, Spec addCheckError(row, new ApplicationBusinessException(t("tutti.service.referential.import.species.error.idNotTemporary", id))); + }else if (requestResult.isIdAlreadyAdded(id)) { + + addCheckError(row, new ApplicationBusinessException(t("tutti.service.referential.import.species.error.id.alreaydAdded"))); + } if (StringUtils.isBlank(name)) { addCheckError(row, new ApplicationBusinessException(t("tutti.service.referential.import.species.error.noName"))); + }else if (requestResult.isNaturalIdAlreadyAdded(name)) { + + addCheckError(row, new ApplicationBusinessException(t("tutti.service.referential.import.species.error.name.alreaydAdded"))); + } } diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/referential/consumer/CsvConsumerForTemporaryVessel.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/referential/consumer/CsvConsumerForTemporaryVessel.java index 0dec002..0399f6c 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/referential/consumer/CsvConsumerForTemporaryVessel.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/referential/consumer/CsvConsumerForTemporaryVessel.java @@ -98,12 +98,20 @@ public class CsvConsumerForTemporaryVessel extends CsvComsumer<VesselRow, Vessel addCheckError(row, new ApplicationBusinessException(t("tutti.service.referential.import.vessel.error.idNotTemporary", id))); + } else if (requestResult.isIdAlreadyAdded(id)) { + + addCheckError(row, new ApplicationBusinessException(t("tutti.service.referential.import.vessel.error.id.alreaydAdded"))); + } if (StringUtils.isBlank(internationalRegistrationCode)) { addCheckError(row, new ApplicationBusinessException(t("tutti.service.referential.import.vessel.error.noRegistrationCode"))); + } else if (requestResult.isNaturalIdAlreadyAdded(internationalRegistrationCode)) { + + addCheckError(row, new ApplicationBusinessException(t("tutti.service.referential.import.vessel.error.name.alreaydAdded"))); + } } diff --git a/tutti-service/src/main/resources/ftl/genericFormatValidateReport_fr.ftl b/tutti-service/src/main/resources/ftl/genericFormatImportReport_fr.ftl similarity index 79% copy from tutti-service/src/main/resources/ftl/genericFormatValidateReport_fr.ftl copy to tutti-service/src/main/resources/ftl/genericFormatImportReport_fr.ftl index e968ed3..9b6aff9 100644 --- a/tutti-service/src/main/resources/ftl/genericFormatValidateReport_fr.ftl +++ b/tutti-service/src/main/resources/ftl/genericFormatImportReport_fr.ftl @@ -24,9 +24,14 @@ <#assign errorsEntries = fileResult.errorsEntries> <ul> <li>Nom du fichier : ${fileResult.filename}</li> - <li>Nombre de lignes à importer : ${fileResult.linesCount}</li> + <#if !fileResult.imported??> + <li class="error">Fichier non importé</li> + </ul> + <#return> + </#if> +<li>Nombre de lignes à importer : ${fileResult.linesCount}</li> <#if errorsEntries?size != 0> - <li class="error">Nombre de lignes en erreurs : ${errorsEntries?size}</li> + <li class="error">Nombre de lignes en erreurs : ${errorsEntries?size}</li> </#if> </ul> <#if errorsEntries?size == 0> @@ -195,6 +200,11 @@ font-weight: bold; } + .error { + color: ${redColor}; + font-weight: bold; + } + h1, h4 { color: ${blueColor}; font-weight: bold; @@ -230,16 +240,11 @@ background: #fff; } - li.error { - color: ${redColor}; - font-weight: bold; - } - </style> </head> <body> -<h1>Rapport de validation avant import générique</h1> +<h1>Rapport de d'import générique</h1> <h2>Méta-données</h2> @@ -250,60 +255,64 @@ </ul> <h2>Format de l'archive</h2> -TODO -<h2 style="page-break-after:always"/> +<#if archiveLayoutValid> +<p>Le format de l'archive est correcte.</p> -<h2>Référentiels temporaires Engin</h2> -<#assign fileResult = temporaryGearsImported> -<#assign linesCount = fileResult.linesCount> -<#if linesCount == 0> -<p>Aucun engin temporaire à importer.</p> +<p>Liste des fichiers à valider:</p> +<ul> + <#list archiveFiles as file> + <li>${file}</li> + </#list> +</ul> <#else> +<p>Le format de l'archive n'est pas valide.</p> + +<p>Liste des erreurs détectées:</p> +<ul> + <#list archiveLayoutErrors as error> + <li class="error">${error}</li> + </#list> +</ul> +</#if> + +<h2 style="page-break-after:always"/> + +<h2>Référentiel temporaire Engin</h2> +<#assign fileResult = referentialTemporaryGearFileResult> +<@renderReferentialImportFile fileResult=fileResult/> +<#if fileResult.linesCount != 0> <p>Des engins temporaires ont été importés.</p> - <@renderReferentialImportFile fileResult=fileResult/> <@renderReferentialGearList title="Engins ajoutés" entries=fileResult.entitiesAddedEntries/> <@renderReferentialGearList title="Engins associés" entries=fileResult.entitiesLinkedEntries/> </#if> <h2 style="page-break-after:always"/> -<h2>Référentiels temporaires Personne</h2> -<#assign fileResult = temporaryPersonsImported> -<#assign linesCount = fileResult.linesCount> -<#if linesCount == 0> -<p>Aucune personne temporaire à importer.</p> -<#else> -<p>Des personnes temporaires ont été importées.</p> - <@renderReferentialImportFile fileResult=fileResult/> +<h2>Référentiel temporaire Personne</h2> +<#assign fileResult = referentialTemporaryPersonFileResult> +<@renderReferentialImportFile fileResult=fileResult/> +<#if fileResult.linesCount != 0><p>Des personnes temporaires ont été importées.</p> <@renderReferentialPersonList title="Personnes ajoutées" entries=fileResult.entitiesAddedEntries/> <@renderReferentialPersonList title="Personnes associées" entries=fileResult.entitiesLinkedEntries/> </#if> <h2 style="page-break-after:always"/> -<h2>Référentiels temporaires Espèce</h2> -<#assign fileResult = temporarySpeciesImported> -<#assign linesCount = fileResult.linesCount> -<#if linesCount == 0> -<p>Aucune espèce temporaire à importer.</p> -<#else> -<p>Des espèces temporaires ont été importées.</p> - <@renderReferentialImportFile fileResult=fileResult/> +<h2>Référentiel temporaire Espèce</h2> +<#assign fileResult = referentialTemporarySpeciesFileResult> +<@renderReferentialImportFile fileResult=fileResult/> +<#if fileResult.linesCount != 0><p>Des espèces temporaires ont été importées.</p> <@renderReferentialSpeciesList title="Espèces ajoutées" entries=fileResult.entitiesAddedEntries/> <@renderReferentialSpeciesList title="Espèces associées" entries=fileResult.entitiesLinkedEntries/> </#if> <h2 style="page-break-after:always"/> -<h2>Référentiels temporaires Navire</h2> -<#assign fileResult = temporaryVesselsImported> -<#assign linesCount = fileResult.linesCount> -<#if linesCount == 0> -<p>Aucun navire temporaire à importer.</p> -<#else> -<p>Des navires temporaires ont été importés.</p> - <@renderReferentialImportFile fileResult=fileResult/> +<h2>Référentiel temporaire Navire</h2> +<#assign fileResult = referentialTemporaryVesselFileResult> +<@renderReferentialImportFile fileResult=fileResult/> +<#if fileResult.linesCount != 0><p>Des navires temporaires ont été importés.</p> <@renderReferentialVesselList title="Navires ajoutés" entries=fileResult.entitiesAddedEntries/> <@renderReferentialVesselList title="Navires associés" entries=fileResult.entitiesLinkedEntries/> </#if> @@ -311,107 +320,90 @@ TODO <h2 style="page-break-after:always"/> <h2>Protocole</h2> -<#if protocolImported??> +<#if protocolImported> <p>Un protocol nommé ${protocol.name} a été importé et sélectionné pour l'import.</p> <#else> -<p>Pas de protocole utilisé/</p> +<p>Pas de protocole utilisé.</p> </#if> <h2 style="page-break-after:always"/> <h2>Fichier des campagnes</h2> <#assign fileResult = surveyFileResult> -<#assign linesCount = fileResult.linesCount> -<#if linesCount == 0> -<p>Aucune campagne à importer.</p> -<#else> -<p>Des campagnes ont été validées.</p> - <@renderImportFile fileResult=fileResult/> +<@renderImportFile fileResult=fileResult/> +<#if fileResult.linesCount != 0> +<p>Des campagnes ont été importées.</p> +<strong>TODO</strong> </#if> <h2 style="page-break-after:always"/> <h2>Fichier des mises en oeuvre d'engin</h2> <#assign fileResult = gearCaracteristicFileResult> -<#assign linesCount = fileResult.linesCount> -<#if linesCount == 0> -<p>Aucune mise en oeuvre d'engin à importer.</p> -<#else> -<p>Des mises en oeuvre d'engin ont été validées.</p> - <@renderImportFile fileResult=fileResult/> +<@renderImportFile fileResult=fileResult/> +<#if fileResult.linesCount != 0> +<p>Des mises en oeuvre d'engin ont été importées.</p> +<strong>TODO</strong> </#if> <h2 style="page-break-after:always"/> <h2>Fichier des traits</h2> <#assign fileResult = operationFileResult> -<#assign linesCount = fileResult.linesCount> -<#if linesCount == 0> -<p>Aucun trait à importer.</p> -<#else> -<p>Des traits ont été validés.</p> - <@renderImportFile fileResult=fileResult/> +<@renderImportFile fileResult=fileResult/> +<#if fileResult.linesCount != 0> +<p>Des traits ont été importés.</p> +<strong>TODO</strong> </#if> <h2 style="page-break-after:always"/> <h2>Fichier des caractéristiques de trait</h2> <#assign fileResult = parameterFileResult> -<#assign linesCount = fileResult.linesCount> -<#if linesCount == 0> -<p>Aucune caractéristique de trait à importer.</p> -<#else> -<p>Des caractéristiques de trait ont été validées.</p> - <@renderImportFile fileResult=fileResult/> +<@renderImportFile fileResult=fileResult/> +<#if fileResult.linesCount != 0> +<p>Des caractéristiques de trait ont été importées.</p> +<strong>TODO</strong> </#if> <h2 style="page-break-after:always"/> <h2>Fichier des lôts capture</h2> <#assign fileResult = catchFileResult> -<#assign linesCount = fileResult.linesCount> -<#if linesCount == 0> -<p>Aucun lôt capture à importer.</p> -<#else> -<p>Des lôts capture ont été validées.</p> - <@renderImportFile fileResult=fileResult/> +<@renderImportFile fileResult=fileResult/> +<#if fileResult.linesCount != 0> +<p>Des lôts capture ont été importées.</p> +<strong>TODO</strong> </#if> <h2 style="page-break-after:always"/> <h2>Fichier des macro déchets</h2> <#assign fileResult = marineLitterFileResult> -<#assign linesCount = fileResult.linesCount> -<#if linesCount == 0> -<p>Aucun macro déchet à importer.</p> -<#else> -<p>Des macro déchets ont été validés.</p> - <@renderImportFile fileResult=fileResult/> +<@renderImportFile fileResult=fileResult/> +<#if fileResult.linesCount != 0> +<p>Des macro déchets ont été importés.</p> +<strong>TODO</strong> </#if> <h2 style="page-break-after:always"/> <h2>Fichier des captures accidentelles</h2> <#assign fileResult = accidentalCatchFileResult> -<#assign linesCount = fileResult.linesCount> -<#if linesCount == 0> -<p>Aucune capture accidentelle à importer.</p> -<#else> -<p>Des captures accidentelles ont été validées.</p> - <@renderImportFile fileResult=fileResult/> +<@renderImportFile fileResult=fileResult/> +<#if fileResult.linesCount != 0> +<p>Des captures accidentelles ont été importées.</p> +<strong>TODO</strong> </#if> <h2 style="page-break-after:always"/> <h2>Fichier des observations individuelles</h2> <#assign fileResult = individualObservationFileResult> -<#assign linesCount = fileResult.linesCount> -<#if linesCount == 0> -<p>Aucune observation individuelle à importer.</p> -<#else> -<p>Des observations individuelles ont été validées.</p> - <@renderImportFile fileResult=fileResult/> +<@renderImportFile fileResult=fileResult/> +<#if fileResult.linesCount != 0> +<p>Des observations individuelles ont été importés.</p> +<strong>TODO</strong> </#if> - </body> </html> diff --git a/tutti-service/src/main/resources/ftl/genericFormatValidateReport_fr.ftl b/tutti-service/src/main/resources/ftl/genericFormatValidateReport_fr.ftl index e968ed3..672289f 100644 --- a/tutti-service/src/main/resources/ftl/genericFormatValidateReport_fr.ftl +++ b/tutti-service/src/main/resources/ftl/genericFormatValidateReport_fr.ftl @@ -20,18 +20,7 @@ #L% --> -<#macro renderImportFile fileResult> - <#assign errorsEntries = fileResult.errorsEntries> -<ul> - <li>Nom du fichier : ${fileResult.filename}</li> - <li>Nombre de lignes à importer : ${fileResult.linesCount}</li> - <#if errorsEntries?size != 0> - <li class="error">Nombre de lignes en erreurs : ${errorsEntries?size}</li> - </#if> -</ul> - <#if errorsEntries?size == 0> - <#return> - </#if> +<#macro renderErrors errorsEntries> <table> <tr> <th>Ligne</th> @@ -41,41 +30,57 @@ <tr> <td>${entry.key}</td> <td> - <pre>${entry.value}</pre> + <pre class="error"> + <#list entry.value as error> + ${error} + </#list> + </pre> </td> </tr> </#list> </table> </#macro> +<#macro renderImportFile fileResult> + <#assign errorsEntries = fileResult.errorsEntries> +<ul> + <li>Nom du fichier : <span class="bold">${fileResult.filename}</span></li> + <#if !fileResult.imported> + <li class="error">Fichier non importé</li> + </ul> + <#return> + </#if> +<li>Nombre de lignes à importer : ${fileResult.linesCount}</li> + <#if errorsEntries?size != 0> + <li class="error">Nombre de lignes en erreurs : ${errorsEntries?size}</li> + </#if> +</ul> + <#if errorsEntries?size == 0> + <#return> + </#if> + <@renderErrors errorsEntries=errorsEntries/> +</#macro> + <#macro renderReferentialImportFile fileResult> <#assign errorsEntries = fileResult.errorsEntries> <ul> <li>Nom du fichier : <span class="bold">${fileResult.filename}</span></li> - <li>Nombre de lignes à importer : ${fileResult.linesCount}</li> - <li>Nombre de lignes importées : ${fileResult.entitiesAddedEntries?size}</li> - <li>Nombre de lignes associées : ${fileResult.entitiesLinkedEntries?size}</li> + <#if !fileResult.imported> + <li class="error">Fichier non importé</li> + </ul> + <#return> + </#if> +<li>Nombre de lignes à importer : ${fileResult.linesCount}</li> +<li>Nombre de lignes importées : ${fileResult.entitiesAddedEntries?size}</li> +<li>Nombre de lignes associées : ${fileResult.entitiesLinkedEntries?size}</li> <#if errorsEntries?size != 0> - <li class="error">Nombre de lignes en erreurs : ${errorsEntries?size}</li> + <li class="error">Nombre de lignes en erreurs : ${errorsEntries?size}</li> </#if> </ul> <#if errorsEntries?size == 0> <#return> </#if> -<table> - <tr> - <th>Ligne</th> - <th>Erreur</th> - </tr> - <#list errorsEntries as entry> - <tr> - <td>${entry.key}</td> - <td> - <pre>${entry.value}</pre> - </td> - </tr> - </#list> -</table> + <@renderErrors errorsEntries=errorsEntries/> </#macro> <#macro renderReferentialGearList title entries> @@ -195,6 +200,11 @@ font-weight: bold; } + .error { + color: ${redColor}; + font-weight: bold; + } + h1, h4 { color: ${blueColor}; font-weight: bold; @@ -230,11 +240,6 @@ background: #fff; } - li.error { - color: ${redColor}; - font-weight: bold; - } - </style> </head> <body> @@ -250,60 +255,73 @@ </ul> <h2>Format de l'archive</h2> -TODO -<h2 style="page-break-after:always"/> +<#if archiveLayoutValid> +<p>Le format de l'archive est correcte.</p> -<h2>Référentiels temporaires Engin</h2> -<#assign fileResult = temporaryGearsImported> -<#assign linesCount = fileResult.linesCount> -<#if linesCount == 0> -<p>Aucun engin temporaire à importer.</p> +<p>Liste des fichiers à valider:</p> +<ul> + <#list archiveFiles as file> + <li>${file}</li> + </#list> +</ul> <#else> +<p>Le format de l'archive n'est pas valide.</p> + +<p>Liste des erreurs détectées:</p> +<ul> + <#list archiveLayoutErrors as error> + <li class="error">${error}</li> + </#list> +</ul> +</#if> + +<h2 style="page-break-after:always"/> + +<h2>Modèle de catégorisation</h2> +<#assign fileResult = sampleCategoryFileResult> +<@renderImportFile fileResult=fileResult/> +<#if fileResult.linesCount != 0> +<p>Le modèle de catégorisation est valide.</p> +</#if> + +<h2 style="page-break-after:always"/> + +<h2>Référentiel temporaire Engin</h2> +<#assign fileResult = referentialTemporaryGearFileResult> +<@renderReferentialImportFile fileResult=fileResult/> +<#if fileResult.linesCount != 0> <p>Des engins temporaires ont été importés.</p> - <@renderReferentialImportFile fileResult=fileResult/> <@renderReferentialGearList title="Engins ajoutés" entries=fileResult.entitiesAddedEntries/> <@renderReferentialGearList title="Engins associés" entries=fileResult.entitiesLinkedEntries/> </#if> <h2 style="page-break-after:always"/> -<h2>Référentiels temporaires Personne</h2> -<#assign fileResult = temporaryPersonsImported> -<#assign linesCount = fileResult.linesCount> -<#if linesCount == 0> -<p>Aucune personne temporaire à importer.</p> -<#else> -<p>Des personnes temporaires ont été importées.</p> - <@renderReferentialImportFile fileResult=fileResult/> +<h2>Référentiel temporaire Personne</h2> +<#assign fileResult = referentialTemporaryPersonFileResult> +<@renderReferentialImportFile fileResult=fileResult/> +<#if fileResult.linesCount != 0><p>Des personnes temporaires ont été importées.</p> <@renderReferentialPersonList title="Personnes ajoutées" entries=fileResult.entitiesAddedEntries/> <@renderReferentialPersonList title="Personnes associées" entries=fileResult.entitiesLinkedEntries/> </#if> <h2 style="page-break-after:always"/> -<h2>Référentiels temporaires Espèce</h2> -<#assign fileResult = temporarySpeciesImported> -<#assign linesCount = fileResult.linesCount> -<#if linesCount == 0> -<p>Aucune espèce temporaire à importer.</p> -<#else> -<p>Des espèces temporaires ont été importées.</p> - <@renderReferentialImportFile fileResult=fileResult/> +<h2>Référentiel temporaire Espèce</h2> +<#assign fileResult = referentialTemporarySpeciesFileResult> +<@renderReferentialImportFile fileResult=fileResult/> +<#if fileResult.linesCount != 0><p>Des espèces temporaires ont été importées.</p> <@renderReferentialSpeciesList title="Espèces ajoutées" entries=fileResult.entitiesAddedEntries/> <@renderReferentialSpeciesList title="Espèces associées" entries=fileResult.entitiesLinkedEntries/> </#if> <h2 style="page-break-after:always"/> -<h2>Référentiels temporaires Navire</h2> -<#assign fileResult = temporaryVesselsImported> -<#assign linesCount = fileResult.linesCount> -<#if linesCount == 0> -<p>Aucun navire temporaire à importer.</p> -<#else> -<p>Des navires temporaires ont été importés.</p> - <@renderReferentialImportFile fileResult=fileResult/> +<h2>Référentiel temporaire Navire</h2> +<#assign fileResult = referentialTemporaryVesselFileResult> +<@renderReferentialImportFile fileResult=fileResult/> +<#if fileResult.linesCount != 0><p>Des navires temporaires ont été importés.</p> <@renderReferentialVesselList title="Navires ajoutés" entries=fileResult.entitiesAddedEntries/> <@renderReferentialVesselList title="Navires associés" entries=fileResult.entitiesLinkedEntries/> </#if> @@ -311,106 +329,90 @@ TODO <h2 style="page-break-after:always"/> <h2>Protocole</h2> -<#if protocolImported??> +<#if protocolImported> <p>Un protocol nommé ${protocol.name} a été importé et sélectionné pour l'import.</p> <#else> -<p>Pas de protocole utilisé/</p> +<p>Pas de protocole utilisé.</p> </#if> <h2 style="page-break-after:always"/> <h2>Fichier des campagnes</h2> <#assign fileResult = surveyFileResult> -<#assign linesCount = fileResult.linesCount> -<#if linesCount == 0> -<p>Aucune campagne à importer.</p> -<#else> +<@renderImportFile fileResult=fileResult/> +<#if fileResult.linesCount != 0> <p>Des campagnes ont été validées.</p> - <@renderImportFile fileResult=fileResult/> +<strong>TODO</strong> </#if> <h2 style="page-break-after:always"/> <h2>Fichier des mises en oeuvre d'engin</h2> <#assign fileResult = gearCaracteristicFileResult> -<#assign linesCount = fileResult.linesCount> -<#if linesCount == 0> -<p>Aucune mise en oeuvre d'engin à importer.</p> -<#else> +<@renderImportFile fileResult=fileResult/> +<#if fileResult.linesCount != 0> <p>Des mises en oeuvre d'engin ont été validées.</p> - <@renderImportFile fileResult=fileResult/> +<strong>TODO</strong> </#if> <h2 style="page-break-after:always"/> <h2>Fichier des traits</h2> <#assign fileResult = operationFileResult> -<#assign linesCount = fileResult.linesCount> -<#if linesCount == 0> -<p>Aucun trait à importer.</p> -<#else> +<@renderImportFile fileResult=fileResult/> +<#if fileResult.linesCount != 0> <p>Des traits ont été validés.</p> - <@renderImportFile fileResult=fileResult/> +<strong>TODO</strong> </#if> <h2 style="page-break-after:always"/> <h2>Fichier des caractéristiques de trait</h2> <#assign fileResult = parameterFileResult> -<#assign linesCount = fileResult.linesCount> -<#if linesCount == 0> -<p>Aucune caractéristique de trait à importer.</p> -<#else> +<@renderImportFile fileResult=fileResult/> +<#if fileResult.linesCount != 0> <p>Des caractéristiques de trait ont été validées.</p> - <@renderImportFile fileResult=fileResult/> +<strong>TODO</strong> </#if> <h2 style="page-break-after:always"/> <h2>Fichier des lôts capture</h2> <#assign fileResult = catchFileResult> -<#assign linesCount = fileResult.linesCount> -<#if linesCount == 0> -<p>Aucun lôt capture à importer.</p> -<#else> +<@renderImportFile fileResult=fileResult/> +<#if fileResult.linesCount != 0> <p>Des lôts capture ont été validées.</p> - <@renderImportFile fileResult=fileResult/> +<strong>TODO</strong> </#if> <h2 style="page-break-after:always"/> <h2>Fichier des macro déchets</h2> <#assign fileResult = marineLitterFileResult> -<#assign linesCount = fileResult.linesCount> -<#if linesCount == 0> -<p>Aucun macro déchet à importer.</p> -<#else> +<@renderImportFile fileResult=fileResult/> +<#if fileResult.linesCount != 0> <p>Des macro déchets ont été validés.</p> - <@renderImportFile fileResult=fileResult/> +<strong>TODO</strong> </#if> <h2 style="page-break-after:always"/> <h2>Fichier des captures accidentelles</h2> <#assign fileResult = accidentalCatchFileResult> -<#assign linesCount = fileResult.linesCount> -<#if linesCount == 0> -<p>Aucune capture accidentelle à importer.</p> -<#else> +<@renderImportFile fileResult=fileResult/> +<#if fileResult.linesCount != 0> <p>Des captures accidentelles ont été validées.</p> - <@renderImportFile fileResult=fileResult/> +<strong>TODO</strong> </#if> <h2 style="page-break-after:always"/> <h2>Fichier des observations individuelles</h2> <#assign fileResult = individualObservationFileResult> -<#assign linesCount = fileResult.linesCount> -<#if linesCount == 0> -<p>Aucune observation individuelle à importer.</p> -<#else> +<@renderImportFile fileResult=fileResult/> +<#if fileResult.linesCount != 0> <p>Des observations individuelles ont été validées.</p> - <@renderImportFile fileResult=fileResult/> +<strong>TODO</strong> </#if> </body> diff --git a/tutti-service/src/test/java/fr/ifremer/tutti/service/genericformat/GenericFormatImportServiceTest.java b/tutti-service/src/test/java/fr/ifremer/tutti/service/genericformat/GenericFormatImportServiceTest.java index dba1fed..7ef8de3 100644 --- a/tutti-service/src/test/java/fr/ifremer/tutti/service/genericformat/GenericFormatImportServiceTest.java +++ b/tutti-service/src/test/java/fr/ifremer/tutti/service/genericformat/GenericFormatImportServiceTest.java @@ -40,7 +40,47 @@ public class GenericFormatImportServiceTest extends GenericFormatImportServiceTe } @Test - public void testValidateFile() throws IOException { + public void testValidateEmpty() throws IOException { + + ServiceDbResource.DataContext dataContext = getServiceDbResource().loadContext(PROGRAM_ID); + + File archiveFile = createArchive("empty.zip", "referential", "protocol", "empty"); + + GenericFormatImportConfiguration importConfiguration = new GenericFormatImportConfiguration(); + importConfiguration.setImportFile(archiveFile); + importConfiguration.setProgramId(dataContext.program.getId()); + + File reportFile = getServiceDbResource().getConfig().newTempFile("validateReportFile", ".pdf"); + importConfiguration.setReportFile(reportFile); + + int nbSteps = service.getValidateImportFileNbSteps(importConfiguration); + + ProgressionModel progressionModel = new ProgressionModel(); + progressionModel.setTotal(10000); + + if (log.isInfoEnabled()) { + log.info("Will validate file in " + nbSteps + " steps"); + } + + dbResource.setDestroyResources(false); + + service.validateImportFile(importConfiguration, progressionModel); + + File reportFile2 = getServiceDbResource().getConfig().newTempFile("validateReportFile", ".pdf"); + importConfiguration.setReportFile(reportFile2); + + service.validateImportFile(importConfiguration, progressionModel); + + if (log.isInfoEnabled()) { + log.info("Report file: " + reportFile); + log.info("Report file2: " + reportFile2); + } + } + + @Test + public void testValidateDefault() throws IOException { + + dbResource.setDestroyResources(false); ServiceDbResource.DataContext dataContext = getServiceDbResource().loadContext(PROGRAM_ID); @@ -50,6 +90,9 @@ public class GenericFormatImportServiceTest extends GenericFormatImportServiceTe importConfiguration.setImportFile(archiveFile); importConfiguration.setProgramId(dataContext.program.getId()); + File reportFile = getServiceDbResource().getConfig().newTempFile("validateReportFile", ".pdf"); + importConfiguration.setReportFile(reportFile); + int nbSteps = service.getValidateImportFileNbSteps(importConfiguration); ProgressionModel progressionModel = new ProgressionModel(); @@ -68,12 +111,17 @@ public class GenericFormatImportServiceTest extends GenericFormatImportServiceTe ServiceDbResource.DataContext dataContext = getServiceDbResource().loadContext(PROGRAM_ID); + dbResource.setDestroyResources(false); + File archiveFile = createArchive("empty.zip", "technical", "protocol", "default"); GenericFormatImportConfiguration importConfiguration = new GenericFormatImportConfiguration(); importConfiguration.setImportFile(archiveFile); importConfiguration.setProgramId(dataContext.program.getId()); + File reportFile = getServiceDbResource().getConfig().newTempFile("importReportFile", ".pdf"); + importConfiguration.setReportFile(reportFile); + int nbSteps = service.getImportProgramNbSteps(importConfiguration); ProgressionModel progressionModel = new ProgressionModel(); @@ -100,6 +148,9 @@ public class GenericFormatImportServiceTest extends GenericFormatImportServiceTe importConfiguration.setImportFile(archiveFile); importConfiguration.setProgramId(dataContext.program.getId()); + File reportFile = getServiceDbResource().getConfig().newTempFile("importReportFile", ".pdf"); + importConfiguration.setReportFile(reportFile); + int nbSteps = service.getImportProgramNbSteps(importConfiguration); ProgressionModel progressionModel = new ProgressionModel(); @@ -146,6 +197,8 @@ public class GenericFormatImportServiceTest extends GenericFormatImportServiceTe importConfiguration.setImportFile(archiveFile); importConfiguration.setProgramId(dataContext.program.getId()); + File reportFile = getServiceDbResource().getConfig().newTempFile("importReportFile", ".pdf"); + importConfiguration.setReportFile(reportFile); int nbSteps = service.getImportProgramNbSteps(importConfiguration); diff --git a/tutti-service/src/test/java/fr/ifremer/tutti/service/genericformat/GenericFormatImportServiceValidTest.java b/tutti-service/src/test/java/fr/ifremer/tutti/service/genericformat/GenericFormatImportServiceValidTest.java new file mode 100644 index 0000000..5734516 --- /dev/null +++ b/tutti-service/src/test/java/fr/ifremer/tutti/service/genericformat/GenericFormatImportServiceValidTest.java @@ -0,0 +1,133 @@ +package fr.ifremer.tutti.service.genericformat; + +import fr.ifremer.tutti.persistence.ProgressionModel; +import fr.ifremer.tutti.service.ServiceDbResource; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; + +/** + * Created on 2/26/15. + * + * @author Tony Chemit - chemit@codelutin.com + * @since XXX + */ +public class GenericFormatImportServiceValidTest extends GenericFormatImportServiceTestSupport { + + /** Logger. */ + private static final Log log = LogFactory.getLog(GenericFormatImportServiceTest.class); + + @Rule + public final ServiceDbResource dbResource = ServiceDbResource.writeDb("dbGenericFormatImport"); + + @Override + protected ServiceDbResource getServiceDbResource() { + return dbResource; + } + + @Test + public void testValidateLimitCases() throws IOException { + + dbResource.setDestroyResources(false); + + ServiceDbResource.DataContext dataContext = getServiceDbResource().loadContext(PROGRAM_ID); + + StringBuilder builder = new StringBuilder(); + + String programId = dataContext.program.getId(); + + // missing files in archive + { + + GenericFormatValidateFileResult result = doValidate(builder, "ArchiveLayoutNotValid", programId); + Assert.assertFalse(result.isArchiveLayoutValid()); + Assert.assertFalse(result.getSampleCategoryFileResult().isImported()); +// Assert.assertFalse(result.getSampleCategoryFileResult().isValid()); + Assert.assertFalse(result.getReferentialTemporaryGearFileResult().isImported()); +// Assert.assertFalse(result.getReferentialTemporaryGearFileResult().isValid()); + + } + + // sample categories file format is bad + { + + GenericFormatValidateFileResult result = doValidate(builder, "badSampleCategoryFormat", programId, "empty", "badSampleCategoryFormat"); + Assert.assertTrue(result.isArchiveLayoutValid()); + Assert.assertTrue(result.getSampleCategoryFileResult().isImported()); + Assert.assertFalse(result.getSampleCategoryFileResult().isValid()); + Assert.assertFalse(result.getReferentialTemporaryGearFileResult().isImported()); +// Assert.assertFalse(result.getReferentialTemporaryGearFileResult().isValid()); + + } + + // mismatch sample categories + { + + GenericFormatValidateFileResult result = doValidate(builder, "sampleCategoryFormatMismatch", programId, "empty", "sampleCategoryFormatMismatch"); + Assert.assertTrue(result.isArchiveLayoutValid()); + Assert.assertTrue(result.getSampleCategoryFileResult().isImported()); + Assert.assertFalse(result.getSampleCategoryFileResult().isValid()); + Assert.assertFalse(result.getReferentialTemporaryGearFileResult().isImported()); +// Assert.assertFalse(result.getReferentialTemporaryGearFileResult().isValid()); + + } + + // bad temporary referential gear format + { + + GenericFormatValidateFileResult result = doValidate(builder, "badTemporaryReferentialGearFormat", programId, "empty", "sampleCategory", "badTemporaryReferentialGearFormat"); + Assert.assertTrue(result.isArchiveLayoutValid()); + Assert.assertTrue(result.getSampleCategoryFileResult().isImported()); + Assert.assertTrue(result.getSampleCategoryFileResult().isValid()); + + Assert.assertTrue(result.getReferentialTemporaryGearFileResult().isImported()); + Assert.assertFalse(result.getReferentialTemporaryGearFileResult().isValid()); + + } + + // error while importing temporary referential gear + { + + GenericFormatValidateFileResult result = doValidate(builder, "badTemporaryReferentialGearImport", programId, "empty", "sampleCategory", "badTemporaryReferentialGearImport"); + Assert.assertTrue(result.isArchiveLayoutValid()); + Assert.assertTrue(result.getSampleCategoryFileResult().isImported()); + Assert.assertTrue(result.getSampleCategoryFileResult().isValid()); + Assert.assertTrue(result.getReferentialTemporaryGearFileResult().isImported()); + Assert.assertFalse(result.getReferentialTemporaryGearFileResult().isValid()); + + } + + if (log.isInfoEnabled()) { + log.info("Report files:" + builder.toString()); + } + + + } + + protected GenericFormatValidateFileResult doValidate(StringBuilder builder, String archivName, String programId, String... directoryies) throws IOException { + + File archiveFile = createArchive(archivName + ".zip", directoryies); + + GenericFormatImportConfiguration importConfiguration = new GenericFormatImportConfiguration(); + importConfiguration.setImportFile(archiveFile); + importConfiguration.setProgramId(programId); + + File reportFile = getServiceDbResource().getConfig().newTempFile(archivName, ".pdf"); + importConfiguration.setReportFile(reportFile); + + ProgressionModel progressionModel = new ProgressionModel(); + progressionModel.setTotal(10000); + + GenericFormatValidateFileResult result = service.validateImportFile(importConfiguration, progressionModel); + + builder.append("\n").append(archivName).append(" :\n evince ").append(reportFile); + + return result; + + } +} diff --git a/tutti-service/src/test/resources/genericFormat/technical/sampleCategory.csv b/tutti-service/src/test/resources/genericFormat/badSampleCategoryFormat/sampleCategory.csv similarity index 60% copy from tutti-service/src/test/resources/genericFormat/technical/sampleCategory.csv copy to tutti-service/src/test/resources/genericFormat/badSampleCategoryFormat/sampleCategory.csv index 2f2b57f..03212d1 100644 --- a/tutti-service/src/test/resources/genericFormat/technical/sampleCategory.csv +++ b/tutti-service/src/test/resources/genericFormat/badSampleCategoryFormat/sampleCategory.csv @@ -1,6 +1,6 @@ -code;order;caracteristic +Acode;order;caracteristic V_HV;0;1428 Class_Tri;1;198 Sexe;2;196 Maturite;3;174 -Age;4;1430 +Age;4;1430 \ No newline at end of file diff --git a/tutti-service/src/test/resources/genericFormat/technical/temporaryGears.csv b/tutti-service/src/test/resources/genericFormat/badTemporaryReferentialGearFormat/temporaryGears.csv similarity index 61% copy from tutti-service/src/test/resources/genericFormat/technical/temporaryGears.csv copy to tutti-service/src/test/resources/genericFormat/badTemporaryReferentialGearFormat/temporaryGears.csv index c96756a..63d1af9 100644 --- a/tutti-service/src/test/resources/genericFormat/technical/temporaryGears.csv +++ b/tutti-service/src/test/resources/genericFormat/badTemporaryReferentialGearFormat/temporaryGears.csv @@ -1,5 +1,5 @@ -id;name;label;scientificGear;toDelete +Aid;name;label;scientificGear;toDelete -1;Gear fishing name 1;Gear fishing label 1;N; -2;Gear fishing name 2;Gear fishing label 2;N; -3;Gear scientific name 3;Gear scientific label 3;Y; --4;Gear scientific name 4;Gear scientific label 4;Y; +-4;Gear scientific name 4;Gear scientific label 4;Y; \ No newline at end of file diff --git a/tutti-service/src/test/resources/genericFormat/badTemporaryReferentialGearImport/temporaryGears.csv b/tutti-service/src/test/resources/genericFormat/badTemporaryReferentialGearImport/temporaryGears.csv new file mode 100644 index 0000000..1439289 --- /dev/null +++ b/tutti-service/src/test/resources/genericFormat/badTemporaryReferentialGearImport/temporaryGears.csv @@ -0,0 +1,5 @@ +id;name;label;scientificGear;toDelete +-1;Gear fishing name 1;Gear fishing label 1;N; +-2;Gear fishing name 1;Gear fishing label 2;N; +-1;Gear scientific name 3;Gear scientific label 3;Y; +A;Gear scientific name 4;Gear scientific label 4;Y; \ No newline at end of file diff --git a/tutti-service/src/test/resources/genericFormat/default/accidentalCatch.csv b/tutti-service/src/test/resources/genericFormat/default/accidentalCatch.csv index 3fb8b64..c4d7071 100644 --- a/tutti-service/src/test/resources/genericFormat/default/accidentalCatch.csv +++ b/tutti-service/src/test/resources/genericFormat/default/accidentalCatch.csv @@ -12,4 +12,4 @@ Annee;Serie;Serie_Partielle;Code_Station;Id_Operation;Poche;Engin;Rang_Engin;Id_ 2014;Campagne EVHOE;1;S0981;156;1;GOV 36/47;1;100001;-5;Divers 2;gthy;1433;ID du PSFM - données - sans objet - Organisation des données de campagnes;294;CAM-EVHOE;294 2014;Campagne EVHOE;1;S0981;156;1;GOV 36/47;1;100001;-5;Divers 2;gthy;294;Hauteur de l'individu (H) - individu - totale - Mesure au mm par un observateur;45.0;CAM-EVHOE;45.0 2014;Campagne EVHOE;1;S0981;156;1;GOV 36/47;1;100001;-5;Divers 2;gthy;1460;Activité prévue - marée - totale - Déclaration d'un professionnel;GUD - Assistance;CAM-EVHOE;2408 -2014;Campagne EVHOE;1;S0981;156;1;GOV 36/47;1;100001;-5;Divers 2;gthy;102;Age - individu - otolithe droite - Lecture d'âge;45.0;CAM-EVHOE;45.0 +2014;Campagne EVHOE;1;S0981;156;1;GOV 36/47;1;100001;-5;Divers 2;gthy;102;Age - individu - otolithe droite - Lecture d'âge;45.0;CAM-EVHOE;45.0 \ No newline at end of file diff --git a/tutti-service/src/test/resources/genericFormat/default/individualObservation.csv b/tutti-service/src/test/resources/genericFormat/default/individualObservation.csv index fdb9391..b417869 100644 --- a/tutti-service/src/test/resources/genericFormat/default/individualObservation.csv +++ b/tutti-service/src/test/resources/genericFormat/default/individualObservation.csv @@ -23,4 +23,4 @@ Annee;Serie;Serie_Partielle;Code_Station;Id_Operation;Poche;Engin;Rang_Engin;Id_ 2014;Campagne EVHOE;1;S0981;156;1;GOV 36/47;1;100007;1972;Arnoglossus imperialis;;306;Longueur totale (LT) - individu - totale - Mesure au cm par un observateur;17.0;CAM-EVHOE;17.0 2014;Campagne EVHOE;1;S0981;156;1;GOV 36/47;1;100008;1972;Arnoglossus imperialis;;220;Poids - produit/lot - totale - Mesure par un observateur;0.009;CAM-EVHOE;0.009 2014;Campagne EVHOE;1;S0981;156;1;GOV 36/47;1;100008;1972;Arnoglossus imperialis;;1433;ID du PSFM - données - sans objet - Organisation des données de campagnes;306;CAM-EVHOE;306 -2014;Campagne EVHOE;1;S0981;156;1;GOV 36/47;1;100008;1972;Arnoglossus imperialis;;306;Longueur totale (LT) - individu - totale - Mesure au cm par un observateur;17.0;CAM-EVHOE;17.0 +2014;Campagne EVHOE;1;S0981;156;1;GOV 36/47;1;100008;1972;Arnoglossus imperialis;;306;Longueur totale (LT) - individu - totale - Mesure au cm par un observateur;17.0;CAM-EVHOE;17.0 \ No newline at end of file diff --git a/tutti-service/src/test/resources/genericFormat/default/marineLitter.csv b/tutti-service/src/test/resources/genericFormat/default/marineLitter.csv index ee90166..d78ded5 100644 --- a/tutti-service/src/test/resources/genericFormat/default/marineLitter.csv +++ b/tutti-service/src/test/resources/genericFormat/default/marineLitter.csv @@ -1,4 +1,4 @@ Annee;Serie;Serie_Partielle;Code_Station;Id_Operation;Poche;Engin;Rang_Engin;Categorie;Categorie_Taille;Nombre;Poids;Commentaire;Serie_Id;Categorie_Id;Categorie_Taille_Id 2014;Campagne EVHOE;1;S0981;156;1;GOV 36/47;1;L1a Sacs;X: UNK - catégorisation non réalisée;1;0.026;;CAM-EVHOE;2126;2163 2014;Campagne EVHOE;1;S0981;156;1;GOV 36/47;1;L1f Filets de pêche;X: UNK - catégorisation non réalisée;1;0.125;;CAM-EVHOE;2131;2163 -2014;Campagne EVHOE;1;S0981;156;1;GOV 36/47;1;L1i Cordes synthétiques / bandes de cerclage;X: UNK - catégorisation non réalisée;10;0.023;;CAM-EVHOE;2134;2163 +2014;Campagne EVHOE;1;S0981;156;1;GOV 36/47;1;L1i Cordes synthétiques / bandes de cerclage;X: UNK - catégorisation non réalisée;10;0.023;;CAM-EVHOE;2134;2163 \ No newline at end of file diff --git a/tutti-service/src/test/resources/genericFormat/default/operation.csv b/tutti-service/src/test/resources/genericFormat/default/operation.csv index dcba55b..a843157 100644 --- a/tutti-service/src/test/resources/genericFormat/default/operation.csv +++ b/tutti-service/src/test/resources/genericFormat/default/operation.csv @@ -1,3 +1,3 @@ Annee;Serie;Serie_Partielle;Code_Station;Id_Operation;Poche;Engin;Rang_Engin;Navire;DateDeb;LatDeb;LongDeb;DateFin;LatFin;LongFin;Duree;Strate;Sous_Strate;Localite;Validite_OP;Rectiligne;Distance;Saisisseur;Navire_Associe;Commentaire;Poids_Total;Poids_Total_Calcule;Poids_Total_Vrac;Poids_Total_Vrac_Calcule;Poids_Total_HorsVrac;Poids_Total_HorsVrac_Calcule;Poids_Total_Non_Trie;Poids_Total_Non_Trie_Calcule;Poids_Total_Tremis;Poids_Total_Tremis_Calcule;Poids_Total_Carroussel;Poids_Total_Car [...] 2014;Campagne EVHOE;1;S0820;1;1;GOV 36/47;1;868095;19/10/2014 02:00:00;;;19/10/2014 02:00:00;;;00;Strate Gn3;NA;NA;Y;N;-9;;;;177.471;Y;177.471;Y;0.0;Y;0.0;Y;169.7;N;205.1;N;176.213;Y;176.213;Y;176.213;Y;0.0;Y;0.0;Y;0.0;Y;1.258;Y;1.258;Y;1.258;Y;0.0;Y;0.0;Y;0.0;Y;0.68;Y;CAM-EVHOE;379;868095;57623;;;; -2014;Campagne EVHOE;1;S0981;156;1;GOV 36/47;1;868095;30/11/2014 01:00:00;;;30/11/2014 01:00:00;;;00;Strate Cc4w;NA;NA;Y;N;-9;;;;138.512;Y;138.512;Y;0.0;Y;0.0;Y;160.7;N;141.2;N;138.157;Y;138.157;Y;138.157;Y;0.0;Y;0.0;Y;0.0;Y;0.355;Y;0.355;Y;0.355;Y;0.0;Y;0.0;Y;0.0;Y;0.174;Y;CAM-EVHOE;379;868095;63356;;;; +2014;Campagne EVHOE;1;S0981;156;1;GOV 36/47;1;868095;30/11/2014 01:00:00;;;30/11/2014 01:00:00;;;00;Strate Cc4w;NA;NA;Y;N;-9;;;;138.512;Y;138.512;Y;0.0;Y;0.0;Y;160.7;N;141.2;N;138.157;Y;138.157;Y;138.157;Y;0.0;Y;0.0;Y;0.0;Y;0.355;Y;0.355;Y;0.355;Y;0.0;Y;0.0;Y;0.0;Y;0.174;Y;CAM-EVHOE;379;868095;63356;;;; \ No newline at end of file diff --git a/tutti-service/src/test/resources/genericFormat/default/survey.csv b/tutti-service/src/test/resources/genericFormat/default/survey.csv index 5b15941..c20bfc9 100644 --- a/tutti-service/src/test/resources/genericFormat/default/survey.csv +++ b/tutti-service/src/test/resources/genericFormat/default/survey.csv @@ -1,2 +1,2 @@ Annee;Serie;Serie_Partielle;Navire;Nombre_Poches;Pays;Zone_Etude;Campagne;Id_Sismer;Date_Deb_Campagne;Port_Deb_Campagne;Date_Fin_Campagne;Port_Fin_Campagne;Chef_Mission;Resp_Salle_Tri;Commentaire;Serie_Id;Navire_Id;Engin_Id;Port_Deb_Campagne_Id;Port_Fin_Campagne_Id;Chef_Mission_Id;Resp_Salle_Tri_Id -2014;Campagne EVHOE;1;868095;1;FRA;EVHOE - Golfe de Gascogne / Mer Celtique;Campagne EVHOE 2014;;17/10/2014 07:00:00;Brest;01/12/2014 07:00:00;Brest;Jean-Pierre LEAUTE|Michele SALAUN|Lionel PAWLOWSKI;Erwan DUHAMEL|Nicolas CAROFF;;CAM-EVHOE;868095;379|381|64;530;530;143|70|2332;128|59 +2014;Campagne EVHOE;1;868095;1;FRA;EVHOE - Golfe de Gascogne / Mer Celtique;Campagne EVHOE 2014;;17/10/2014 07:00:00;Brest;01/12/2014 07:00:00;Brest;Jean-Pierre LEAUTE|Michele SALAUN|Lionel PAWLOWSKI;Erwan DUHAMEL|Nicolas CAROFF;;CAM-EVHOE;868095;379|381|64;530;530;143|70|2332;128|59 \ No newline at end of file diff --git a/tutti-service/src/test/resources/genericFormat/empty/accidentalCatch.csv b/tutti-service/src/test/resources/genericFormat/empty/accidentalCatch.csv index d013e00..2a731a2 100644 --- a/tutti-service/src/test/resources/genericFormat/empty/accidentalCatch.csv +++ b/tutti-service/src/test/resources/genericFormat/empty/accidentalCatch.csv @@ -1 +1 @@ -Annee;Serie;Serie_Partielle;Code_Station;Id_Operation;Poche;Engin;Rang_Engin;Id_Lot;Code_Taxon;Nom_Scientifique;Commentaire;Code_PMFM;Libelle_PMFM;Valeur;Serie_Id;Valeur_Id +Annee;Serie;Serie_Partielle;Code_Station;Id_Operation;Poche;Engin;Rang_Engin;Id_Lot;Code_Taxon;Nom_Scientifique;Commentaire;Code_PMFM;Libelle_PMFM;Valeur;Serie_Id;Valeur_Id \ No newline at end of file diff --git a/tutti-service/src/test/resources/genericFormat/empty/catch.csv b/tutti-service/src/test/resources/genericFormat/empty/catch.csv index 0a20be7..afa0d72 100644 --- a/tutti-service/src/test/resources/genericFormat/empty/catch.csv +++ b/tutti-service/src/test/resources/genericFormat/empty/catch.csv @@ -1 +1 @@ -Annee;Serie;Serie_Partielle;Code_Station;Id_Operation;Poche;Engin;Rang_Engin;Code_Taxon;Code_Espece_Campagne;Nom_Scientifique;Benthos;Commentaire;V_HV;Num_Ordre_V_HV_H2;Tot_V_HV;Ech_V_HV;Type_Volume_Poids_V_HV;Unite_Volume_Poids_V_HV;Class_Tri;Num_Ordre_Class_Tri_H2;Tot_Class_Tri;Ech_Class_Tri;Type_Volume_Poids_Class_Tri;Unite_Volume_Poids_Class_Tri;Sexe;Num_Ordre_Sexe_H2;Tot_Sexe;Ech_Sexe;Type_Volume_Poids_Sexe;Unite_Volume_Poids_Sexe;Maturite;Num_Ordre_Maturite_H2;Tot_Maturite;Ech_Matu [...] +Annee;Serie;Serie_Partielle;Code_Station;Id_Operation;Poche;Engin;Rang_Engin;Code_Taxon;Code_Espece_Campagne;Nom_Scientifique;Benthos;Commentaire;V_HV;Num_Ordre_V_HV_H2;Tot_V_HV;Ech_V_HV;Type_Volume_Poids_V_HV;Unite_Volume_Poids_V_HV;Class_Tri;Num_Ordre_Class_Tri_H2;Tot_Class_Tri;Ech_Class_Tri;Type_Volume_Poids_Class_Tri;Unite_Volume_Poids_Class_Tri;Sexe;Num_Ordre_Sexe_H2;Tot_Sexe;Ech_Sexe;Type_Volume_Poids_Sexe;Unite_Volume_Poids_Sexe;Maturite;Num_Ordre_Maturite_H2;Tot_Maturite;Ech_Matu [...] \ No newline at end of file diff --git a/tutti-service/src/test/resources/genericFormat/empty/gearCaracteristics.csv b/tutti-service/src/test/resources/genericFormat/empty/gearCaracteristics.csv index 3256fe9..0c5e353 100644 --- a/tutti-service/src/test/resources/genericFormat/empty/gearCaracteristics.csv +++ b/tutti-service/src/test/resources/genericFormat/empty/gearCaracteristics.csv @@ -1 +1 @@ -Annee;Serie;Serie_Partielle;Engin;Rang_Engin;Code_PMFM;Libelle_PMFM;Valeur;Serie_Id;Engin_Id;Valeur_Id +Annee;Serie;Serie_Partielle;Engin;Rang_Engin;Code_PMFM;Libelle_PMFM;Valeur;Serie_Id;Engin_Id;Valeur_Id \ No newline at end of file diff --git a/tutti-service/src/test/resources/genericFormat/empty/individualObservation.csv b/tutti-service/src/test/resources/genericFormat/empty/individualObservation.csv index d013e00..2a731a2 100644 --- a/tutti-service/src/test/resources/genericFormat/empty/individualObservation.csv +++ b/tutti-service/src/test/resources/genericFormat/empty/individualObservation.csv @@ -1 +1 @@ -Annee;Serie;Serie_Partielle;Code_Station;Id_Operation;Poche;Engin;Rang_Engin;Id_Lot;Code_Taxon;Nom_Scientifique;Commentaire;Code_PMFM;Libelle_PMFM;Valeur;Serie_Id;Valeur_Id +Annee;Serie;Serie_Partielle;Code_Station;Id_Operation;Poche;Engin;Rang_Engin;Id_Lot;Code_Taxon;Nom_Scientifique;Commentaire;Code_PMFM;Libelle_PMFM;Valeur;Serie_Id;Valeur_Id \ No newline at end of file diff --git a/tutti-service/src/test/resources/genericFormat/empty/marineLitter.csv b/tutti-service/src/test/resources/genericFormat/empty/marineLitter.csv index e39c419..6335540 100644 --- a/tutti-service/src/test/resources/genericFormat/empty/marineLitter.csv +++ b/tutti-service/src/test/resources/genericFormat/empty/marineLitter.csv @@ -1 +1 @@ -Annee;Serie;Serie_Partielle;Code_Station;Id_Operation;Poche;Engin;Rang_Engin;Categorie;Categorie_Taille;Nombre;Poids;Commentaire;Serie_Id;Categorie_Id;Categorie_Taille_Id +Annee;Serie;Serie_Partielle;Code_Station;Id_Operation;Poche;Engin;Rang_Engin;Categorie;Categorie_Taille;Nombre;Poids;Commentaire;Serie_Id;Categorie_Id;Categorie_Taille_Id \ No newline at end of file diff --git a/tutti-service/src/test/resources/genericFormat/empty/operation.csv b/tutti-service/src/test/resources/genericFormat/empty/operation.csv index 1f909ab..db4f0b0 100644 --- a/tutti-service/src/test/resources/genericFormat/empty/operation.csv +++ b/tutti-service/src/test/resources/genericFormat/empty/operation.csv @@ -1 +1 @@ -Annee;Serie;Serie_Partielle;Code_Station;Id_Operation;Poche;Engin;Rang_Engin;Navire;DateDeb;LatDeb;LongDeb;DateFin;LatFin;LongFin;Duree;Strate;Sous_Strate;Localite;Validite_OP;Rectiligne;Distance;Saisisseur;Navire_Associe;Commentaire;Poids_Total;Poids_Total_Calcule;Poids_Total_Vrac;Poids_Total_Vrac_Calcule;Poids_Total_HorsVrac;Poids_Total_HorsVrac_Calcule;Poids_Total_Non_Trie;Poids_Total_Non_Trie_Calcule;Poids_Total_Tremis;Poids_Total_Tremis_Calcule;Poids_Total_Carroussel;Poids_Total_Car [...] +Annee;Serie;Serie_Partielle;Code_Station;Id_Operation;Poche;Engin;Rang_Engin;Navire;DateDeb;LatDeb;LongDeb;DateFin;LatFin;LongFin;Duree;Strate;Sous_Strate;Localite;Validite_OP;Rectiligne;Distance;Saisisseur;Navire_Associe;Commentaire;Poids_Total;Poids_Total_Calcule;Poids_Total_Vrac;Poids_Total_Vrac_Calcule;Poids_Total_HorsVrac;Poids_Total_HorsVrac_Calcule;Poids_Total_Non_Trie;Poids_Total_Non_Trie_Calcule;Poids_Total_Tremis;Poids_Total_Tremis_Calcule;Poids_Total_Carroussel;Poids_Total_Car [...] \ No newline at end of file diff --git a/tutti-service/src/test/resources/genericFormat/empty/parameter.csv b/tutti-service/src/test/resources/genericFormat/empty/parameter.csv index 32c61e5..4571ee6 100644 --- a/tutti-service/src/test/resources/genericFormat/empty/parameter.csv +++ b/tutti-service/src/test/resources/genericFormat/empty/parameter.csv @@ -1 +1 @@ -Annee;Serie;Serie_Partielle;Code_Station;Id_Operation;Poche;Code_PMFM;Libelle_PMFM;Valeur;Type;Serie_Id;Valeur_Id +Annee;Serie;Serie_Partielle;Code_Station;Id_Operation;Poche;Code_PMFM;Libelle_PMFM;Valeur;Type;Serie_Id;Valeur_Id \ No newline at end of file diff --git a/tutti-service/src/test/resources/genericFormat/empty/survey.csv b/tutti-service/src/test/resources/genericFormat/empty/survey.csv index 4537d3b..3b1b6c0 100644 --- a/tutti-service/src/test/resources/genericFormat/empty/survey.csv +++ b/tutti-service/src/test/resources/genericFormat/empty/survey.csv @@ -1 +1 @@ -Annee;Serie;Serie_Partielle;Navire;Nombre_Poches;Pays;Zone_Etude;Campagne;Id_Sismer;Date_Deb_Campagne;Port_Deb_Campagne;Date_Fin_Campagne;Port_Fin_Campagne;Chef_Mission;Resp_Salle_Tri;Commentaire;Serie_Id;Navire_Id;Engin_Id;Port_Deb_Campagne_Id;Port_Fin_Campagne_Id;Chef_Mission_Id;Resp_Salle_Tri_Id +Annee;Serie;Serie_Partielle;Navire;Nombre_Poches;Pays;Zone_Etude;Campagne;Id_Sismer;Date_Deb_Campagne;Port_Deb_Campagne;Date_Fin_Campagne;Port_Fin_Campagne;Chef_Mission;Resp_Salle_Tri;Commentaire;Serie_Id;Navire_Id;Engin_Id;Port_Deb_Campagne_Id;Port_Fin_Campagne_Id;Chef_Mission_Id;Resp_Salle_Tri_Id \ No newline at end of file diff --git a/tutti-service/src/test/resources/genericFormat/onlyCruise/survey.csv b/tutti-service/src/test/resources/genericFormat/onlyCruise/survey.csv index 8090b4c..a02010b 100644 --- a/tutti-service/src/test/resources/genericFormat/onlyCruise/survey.csv +++ b/tutti-service/src/test/resources/genericFormat/onlyCruise/survey.csv @@ -1,2 +1,2 @@ Annee;Serie;Serie_Partielle;Navire;Nombre_Poches;Pays;Zone_Etude;Campagne;Id_Sismer;Date_Deb_Campagne;Port_Deb_Campagne;Date_Fin_Campagne;Port_Fin_Campagne;Chef_Mission;Resp_Salle_Tri;Commentaire;Serie_Id;Navire_Id;Engin_Id;Port_Deb_Campagne_Id;Port_Fin_Campagne_Id;Chef_Mission_Id;Resp_Salle_Tri_Id -2014;Campagne EVHOE;1;868095;1;FRA;EVHOE - Golfe de Gascogne / Mer Celtique;Campagne EVHOE 2014;;17/10/2014 07:00:00;Brest;01/12/2014 07:00:00;Brest;Jean-Pierre LEAUTE|Michele SALAUN|Lionel PAWLOWSKI;Erwan DUHAMEL|Nicolas CAROFF;;CAM-EVHOE;868095;379|381|64|64;530;530;143|70|2332;128|59 +2014;Campagne EVHOE;1;868095;1;FRA;EVHOE - Golfe de Gascogne / Mer Celtique;Campagne EVHOE 2014;;17/10/2014 07:00:00;Brest;01/12/2014 07:00:00;Brest;Jean-Pierre LEAUTE|Michele SALAUN|Lionel PAWLOWSKI;Erwan DUHAMEL|Nicolas CAROFF;;CAM-EVHOE;868095;379|381|64|64;530;530;143|70|2332;128|59 \ No newline at end of file diff --git a/tutti-service/src/test/resources/genericFormat/technical/temporaryGears.csv b/tutti-service/src/test/resources/genericFormat/referentials/temporaryGears.csv similarity index 77% rename from tutti-service/src/test/resources/genericFormat/technical/temporaryGears.csv rename to tutti-service/src/test/resources/genericFormat/referentials/temporaryGears.csv index c96756a..24731af 100644 --- a/tutti-service/src/test/resources/genericFormat/technical/temporaryGears.csv +++ b/tutti-service/src/test/resources/genericFormat/referentials/temporaryGears.csv @@ -2,4 +2,4 @@ id;name;label;scientificGear;toDelete -1;Gear fishing name 1;Gear fishing label 1;N; -2;Gear fishing name 2;Gear fishing label 2;N; -3;Gear scientific name 3;Gear scientific label 3;Y; --4;Gear scientific name 4;Gear scientific label 4;Y; +-4;Gear scientific name 4;Gear scientific label 4;Y; \ No newline at end of file diff --git a/tutti-service/src/test/resources/genericFormat/technical/temporaryPersons.csv b/tutti-service/src/test/resources/genericFormat/referentials/temporaryPersons.csv similarity index 74% rename from tutti-service/src/test/resources/genericFormat/technical/temporaryPersons.csv rename to tutti-service/src/test/resources/genericFormat/referentials/temporaryPersons.csv index 3efb40c..372a462 100644 --- a/tutti-service/src/test/resources/genericFormat/technical/temporaryPersons.csv +++ b/tutti-service/src/test/resources/genericFormat/referentials/temporaryPersons.csv @@ -1,4 +1,4 @@ id;firstName;lastName;toDelete -1;Léo;Paul; -2;Paul;Valery; --3;Valery;François; +-3;Valery;François; \ No newline at end of file diff --git a/tutti-service/src/test/resources/genericFormat/technical/temporarySpecies.csv b/tutti-service/src/test/resources/genericFormat/referentials/temporarySpecies.csv similarity index 90% rename from tutti-service/src/test/resources/genericFormat/technical/temporarySpecies.csv rename to tutti-service/src/test/resources/genericFormat/referentials/temporarySpecies.csv index c912f20..16277a6 100644 --- a/tutti-service/src/test/resources/genericFormat/technical/temporarySpecies.csv +++ b/tutti-service/src/test/resources/genericFormat/referentials/temporarySpecies.csv @@ -6,4 +6,4 @@ id;name;toDelete -2;Divers 4; -7;Gelatineux; -8;MBECTOP; --3;Melange; +-3;Melange; \ No newline at end of file diff --git a/tutti-service/src/test/resources/genericFormat/technical/temporaryVessels.csv b/tutti-service/src/test/resources/genericFormat/referentials/temporaryVessels.csv similarity index 95% rename from tutti-service/src/test/resources/genericFormat/technical/temporaryVessels.csv rename to tutti-service/src/test/resources/genericFormat/referentials/temporaryVessels.csv index d1b7275..b8bc7d0 100644 --- a/tutti-service/src/test/resources/genericFormat/technical/temporaryVessels.csv +++ b/tutti-service/src/test/resources/genericFormat/referentials/temporaryVessels.csv @@ -2,4 +2,4 @@ id;name;internationalRegistrationCode;scientificVessel;toDelete #TEMP¿-1;Temporary fishing vessel name 1;International registration code F1;N; #TEMP¿-2;Temporary fishing vessel name 2;International registration code F2;N; #TEMP¿-3;Temporary scientific vessel name 3;International registration code S3;Y; -#TEMP¿-4;Temporary scientific vessel name 4;International registration code S4;Y; +#TEMP¿-4;Temporary scientific vessel name 4;International registration code S4;Y; \ No newline at end of file diff --git a/tutti-service/src/test/resources/genericFormat/technical/sampleCategory.csv b/tutti-service/src/test/resources/genericFormat/sampleCategory/sampleCategory.csv similarity index 87% copy from tutti-service/src/test/resources/genericFormat/technical/sampleCategory.csv copy to tutti-service/src/test/resources/genericFormat/sampleCategory/sampleCategory.csv index 2f2b57f..1958e32 100644 --- a/tutti-service/src/test/resources/genericFormat/technical/sampleCategory.csv +++ b/tutti-service/src/test/resources/genericFormat/sampleCategory/sampleCategory.csv @@ -3,4 +3,4 @@ V_HV;0;1428 Class_Tri;1;198 Sexe;2;196 Maturite;3;174 -Age;4;1430 +Age;4;1430 \ No newline at end of file diff --git a/tutti-service/src/test/resources/genericFormat/technical/sampleCategory.csv b/tutti-service/src/test/resources/genericFormat/sampleCategoryFormatMismatch/sampleCategory.csv similarity index 87% rename from tutti-service/src/test/resources/genericFormat/technical/sampleCategory.csv rename to tutti-service/src/test/resources/genericFormat/sampleCategoryFormatMismatch/sampleCategory.csv index 2f2b57f..f80708f 100644 --- a/tutti-service/src/test/resources/genericFormat/technical/sampleCategory.csv +++ b/tutti-service/src/test/resources/genericFormat/sampleCategoryFormatMismatch/sampleCategory.csv @@ -1,6 +1,6 @@ code;order;caracteristic -V_HV;0;1428 Class_Tri;1;198 +V_HV;0;1428 Sexe;2;196 Maturite;3;174 -Age;4;1430 +Age;4;1430 \ No newline at end of file -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.