[tutti] branch develop updated (d74f143 -> 9098e61)
This is an automated email from the git hooks/post-receive script. New change to branch develop in repository tutti. See http://git.codelutin.com/tutti.git from d74f143 refs-50 #5866: Campagne CAMANOC en cours : erreur à la génération du pdf à analyser new 725d2aa use internal csv separator for multiexport new 9098e61 fixes #5872: [PERFORMANCE] créer une base avec 100 traits pour tester les performances de l'application The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "adds" were already present in the repository and have only been added to this reference. Detailed log of new commits: commit 9098e61bb4cadf263ad7c4a77a1ea054ac499273 Author: Tony CHEMIT <chemit@codelutin.com> Date: Mon Sep 29 18:59:14 2014 +0200 fixes #5872: [PERFORMANCE] créer une base avec 100 traits pour tester les performances de l'application commit 725d2aa0536046143a5fdc3322292f086cb176c5 Author: Tony CHEMIT <chemit@codelutin.com> Date: Mon Sep 29 17:56:41 2014 +0200 use internal csv separator for multiexport Summary of changes: .../catches/multipost/MultiPostExportService.java | 31 ++-- .../catches/multipost/MultiPostImportService.java | 36 ++-- .../multipost/MultiPostImportServiceTest.java | 190 ++++++++++++++++++++- 3 files changed, 218 insertions(+), 39 deletions(-) -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.
This is an automated email from the git hooks/post-receive script. New commit to branch develop in repository tutti. See http://git.codelutin.com/tutti.git commit 725d2aa0536046143a5fdc3322292f086cb176c5 Author: Tony CHEMIT <chemit@codelutin.com> Date: Mon Sep 29 17:56:41 2014 +0200 use internal csv separator for multiexport --- .../catches/multipost/MultiPostExportService.java | 31 ++++++++----------- .../catches/multipost/MultiPostImportService.java | 36 ++++++++++------------ 2 files changed, 29 insertions(+), 38 deletions(-) diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/catches/multipost/MultiPostExportService.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/catches/multipost/MultiPostExportService.java index bb8b1a8..feab245 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/catches/multipost/MultiPostExportService.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/catches/multipost/MultiPostExportService.java @@ -69,9 +69,6 @@ import static org.nuiton.i18n.I18n.t; */ public class MultiPostExportService extends AbstractTuttiService { -// private static final Log log = -// LogFactory.getLog(TuttiMultiPostImportExportService.class); - public static final String BATCHES_KEY = "batchesKey"; protected static final String ATTACHMENTS_DIRECTORY = "attachments"; @@ -98,7 +95,7 @@ public class MultiPostExportService extends AbstractTuttiService { protected DecoratorService decoratorService; - protected char csvSeparator; + public static final char CSV_SEPARATOR = ';'; protected Map<String, CaracteristicQualitativeValue> sampleCategoryValueMap; @@ -108,8 +105,6 @@ public class MultiPostExportService extends AbstractTuttiService { persistenceService = getService(PersistenceService.class); decoratorService = getService(DecoratorService.class); - csvSeparator = context.getConfig().getCsvSeparator(); - SampleCategoryModel sampleCategoryModel = context.getSampleCategoryModel(); sampleCategoryValueMap = Maps.newTreeMap(); @@ -254,7 +249,7 @@ public class MultiPostExportService extends AbstractTuttiService { // export marine litters - MarineLitterRowModel marineLitterRowModel = new MarineLitterRowModel(csvSeparator); + MarineLitterRowModel marineLitterRowModel = new MarineLitterRowModel(CSV_SEPARATOR); File marineLitterFile = new File(directory, MARINE_LITTER_FILE); file2zip.add(marineLitterFile); export(marineLitterFile, @@ -269,7 +264,7 @@ public class MultiPostExportService extends AbstractTuttiService { weight.setTotalWeight(catchBatch.getMarineLitterTotalWeight()); exportOperation(weight, operation); - MarineLitterWeightRowModel weightModel = new MarineLitterWeightRowModel(csvSeparator); + MarineLitterWeightRowModel weightModel = new MarineLitterWeightRowModel(CSV_SEPARATOR); File weightFile = new File(directory, WEIGHTS_FILE); file2zip.add(weightFile); @@ -337,7 +332,7 @@ public class MultiPostExportService extends AbstractTuttiService { // export individual observations - IndividualObservationRowModel csvModel = new IndividualObservationRowModel(csvSeparator); + IndividualObservationRowModel csvModel = new IndividualObservationRowModel(CSV_SEPARATOR); File individualObservationFile = new File(directory, INDIVIDUAL_OBSERVATION_FILE); file2zip.add(individualObservationFile); export(individualObservationFile, @@ -347,7 +342,7 @@ public class MultiPostExportService extends AbstractTuttiService { // export caracteristics - CaracteristicRowModel caracteristicCsvModel = new CaracteristicRowModel(csvSeparator); + CaracteristicRowModel caracteristicCsvModel = new CaracteristicRowModel(CSV_SEPARATOR); File caracteristicFile = new File(directory, CARACTERISTIC_FILE); file2zip.add(caracteristicFile); export(caracteristicFile, @@ -360,7 +355,7 @@ public class MultiPostExportService extends AbstractTuttiService { FishingOperationRow foRow = new FishingOperationRow(); exportOperation(foRow, operation); - FishingOperationRowModel foRowModel = new FishingOperationRowModel(csvSeparator); + FishingOperationRowModel foRowModel = new FishingOperationRowModel(CSV_SEPARATOR); File weightFile = new File(directory, WEIGHTS_FILE); file2zip.add(weightFile); @@ -423,7 +418,7 @@ public class MultiPostExportService extends AbstractTuttiService { // export accidental catches - AccidentalCatchRowModel csvModel = new AccidentalCatchRowModel(csvSeparator); + AccidentalCatchRowModel csvModel = new AccidentalCatchRowModel(CSV_SEPARATOR); File directory = Files.createTempDir(); List<File> file2zip = Lists.newArrayList(); @@ -437,7 +432,7 @@ public class MultiPostExportService extends AbstractTuttiService { // export caracteristics - CaracteristicRowModel caracteristicCsvModel = new CaracteristicRowModel(csvSeparator); + CaracteristicRowModel caracteristicCsvModel = new CaracteristicRowModel(CSV_SEPARATOR); File caracteristicFile = new File(directory, CARACTERISTIC_FILE); file2zip.add(caracteristicFile); export(caracteristicFile, @@ -450,7 +445,7 @@ public class MultiPostExportService extends AbstractTuttiService { FishingOperationRow foRow = new FishingOperationRow(); exportOperation(foRow, operation); - FishingOperationRowModel foRowModel = new FishingOperationRowModel(csvSeparator); + FishingOperationRowModel foRowModel = new FishingOperationRowModel(CSV_SEPARATOR); File weightFile = new File(directory, WEIGHTS_FILE); file2zip.add(weightFile); @@ -477,9 +472,9 @@ public class MultiPostExportService extends AbstractTuttiService { List<CatchRow> rows, List<CatchFrequencyRow> frequencyRows, List<AttachmentRow> attachmentRows) { - CatchRowModel csvModel = new CatchRowModel(csvSeparator); - CatchFrequencyRowModel csvFrequencyModel = new CatchFrequencyRowModel(csvSeparator); - CatchWeightsRowModel catchWeightsModel = new CatchWeightsRowModel(csvSeparator); + CatchRowModel csvModel = new CatchRowModel(CSV_SEPARATOR); + CatchFrequencyRowModel csvFrequencyModel = new CatchFrequencyRowModel(CSV_SEPARATOR); + CatchWeightsRowModel catchWeightsModel = new CatchWeightsRowModel(CSV_SEPARATOR); File directory = Files.createTempDir(); List<File> file2zip = Lists.newArrayList(); @@ -636,7 +631,7 @@ public class MultiPostExportService extends AbstractTuttiService { List<File> file2zip, List<AttachmentRow> attachmentRows) { - AttachmentRowModel csvAttachmentModel = new AttachmentRowModel(csvSeparator); + AttachmentRowModel csvAttachmentModel = new AttachmentRowModel(CSV_SEPARATOR); File attachmentDirectory = new File(directory, ATTACHMENTS_DIRECTORY); ApplicationIOUtil.forceMkdir(attachmentDirectory, t("tutti.service.multipost.attachment.mkdir.error", attachmentDirectory)); diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/catches/multipost/MultiPostImportService.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/catches/multipost/MultiPostImportService.java index c5756fb..19a54b7 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/catches/multipost/MultiPostImportService.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/catches/multipost/MultiPostImportService.java @@ -117,8 +117,6 @@ public class MultiPostImportService extends AbstractTuttiService { protected DecoratorService decoratorService; - protected char csvSeparator; - protected SampleCategoryModel sampleCategoryModel; protected Map<String, CaracteristicQualitativeValue> sampleCategoryValueMap; @@ -129,8 +127,6 @@ public class MultiPostImportService extends AbstractTuttiService { persistenceService = getService(PersistenceService.class); decoratorService = getService(DecoratorService.class); - csvSeparator = context.getConfig().getCsvSeparator(); - sampleCategoryModel = context.getSampleCategoryModel(); sampleCategoryValueMap = Maps.newTreeMap(); @@ -171,7 +167,7 @@ public class MultiPostImportService extends AbstractTuttiService { n("tutti.service.multipost.uncompress.error")); //check operation - CatchWeightsRowModel weightsModel = new CatchWeightsRowModel(csvSeparator); + CatchWeightsRowModel weightsModel = new CatchWeightsRowModel(MultiPostExportService.CSV_SEPARATOR); checkSameOperation(tempDir, weightsModel, operation); final Map<String, Object> notImportedData = Maps.newHashMap(); @@ -240,7 +236,7 @@ public class MultiPostImportService extends AbstractTuttiService { n("tutti.service.multipost.import.batches.error")); try { - CatchRowModel csvModel = new CatchRowModel(csvSeparator, + CatchRowModel csvModel = new CatchRowModel(MultiPostExportService.CSV_SEPARATOR, speciesList); Import<CatchRow> importer = Import.newImport(csvModel, reader); @@ -330,7 +326,7 @@ public class MultiPostImportService extends AbstractTuttiService { try { - CatchFrequencyRowModel frequencyModel = new CatchFrequencyRowModel(csvSeparator, + CatchFrequencyRowModel frequencyModel = new CatchFrequencyRowModel(MultiPostExportService.CSV_SEPARATOR, caracteristics); Import<CatchFrequencyRow> importer = Import.newImport(frequencyModel, reader); @@ -441,7 +437,7 @@ public class MultiPostImportService extends AbstractTuttiService { n("tutti.service.multipost.uncompress.error")); //check operation - CatchWeightsRowModel weightsModel = new CatchWeightsRowModel(csvSeparator); + CatchWeightsRowModel weightsModel = new CatchWeightsRowModel(MultiPostExportService.CSV_SEPARATOR); checkSameOperation(tempDir, weightsModel, operation); Map<String, Object> notImportedData = Maps.newHashMap(); @@ -504,7 +500,7 @@ public class MultiPostImportService extends AbstractTuttiService { n("tutti.service.multipost.import.batches.error")); try { - CatchRowModel csvModel = new CatchRowModel(csvSeparator, + CatchRowModel csvModel = new CatchRowModel(MultiPostExportService.CSV_SEPARATOR, speciesList); Import<CatchRow> importer = Import.newImport(csvModel, reader); @@ -592,7 +588,7 @@ public class MultiPostImportService extends AbstractTuttiService { new File(file, FREQUENCIES_FILE), n("tutti.service.multipost.import.frequencies.error")); try { - CatchFrequencyRowModel frequencyModel = new CatchFrequencyRowModel(csvSeparator, + CatchFrequencyRowModel frequencyModel = new CatchFrequencyRowModel(MultiPostExportService.CSV_SEPARATOR, caracteristics); Import<CatchFrequencyRow> importer = Import.newImport(frequencyModel, reader); @@ -704,7 +700,7 @@ public class MultiPostImportService extends AbstractTuttiService { n("tutti.service.multipost.uncompress.error")); //check operation - MarineLitterWeightRowModel weightModel = new MarineLitterWeightRowModel(csvSeparator); + MarineLitterWeightRowModel weightModel = new MarineLitterWeightRowModel(MultiPostExportService.CSV_SEPARATOR); checkSameOperation(tempDir, weightModel, operation); String operationId = operation.getId(); @@ -766,7 +762,7 @@ public class MultiPostImportService extends AbstractTuttiService { try { - MarineLitterRowModel csvModel = new MarineLitterRowModel(csvSeparator, + MarineLitterRowModel csvModel = new MarineLitterRowModel(MultiPostExportService.CSV_SEPARATOR, categoryValues, sizeCategoryValues); @@ -873,7 +869,7 @@ public class MultiPostImportService extends AbstractTuttiService { //check operation FishingOperationRowModel operationModel = - new FishingOperationRowModel(csvSeparator); + new FishingOperationRowModel(MultiPostExportService.CSV_SEPARATOR); checkSameOperation(tempDir, operationModel, operation); TuttiDataContext dataContext = context.getDataContext(); @@ -916,7 +912,7 @@ public class MultiPostImportService extends AbstractTuttiService { Map<String, IndividualObservationBatch> batches = Maps.newLinkedHashMap(); - IndividualObservationRowModel csvModel = new IndividualObservationRowModel(csvSeparator, + IndividualObservationRowModel csvModel = new IndividualObservationRowModel(MultiPostExportService.CSV_SEPARATOR, speciesList, caracteristics); Reader reader = ApplicationIOUtil.newReader( @@ -959,7 +955,7 @@ public class MultiPostImportService extends AbstractTuttiService { protected void importIndividualObservationCaracteristics(File file, Map<String, IndividualObservationBatch> batches, List<Caracteristic> caracteristics) { - CaracteristicRowModel caracteristicModel = new CaracteristicRowModel(csvSeparator, + CaracteristicRowModel caracteristicModel = new CaracteristicRowModel(MultiPostExportService.CSV_SEPARATOR, caracteristics); Reader reader = ApplicationIOUtil.newReader( @@ -1025,7 +1021,7 @@ public class MultiPostImportService extends AbstractTuttiService { //check operation FishingOperationRowModel operationModel = - new FishingOperationRowModel(csvSeparator); + new FishingOperationRowModel(MultiPostExportService.CSV_SEPARATOR); checkSameOperation(tempDir, operationModel, operation); TuttiDataContext dataContext = context.getDataContext(); @@ -1081,7 +1077,7 @@ public class MultiPostImportService extends AbstractTuttiService { try { - AccidentalCatchRowModel csvModel = new AccidentalCatchRowModel(csvSeparator, + AccidentalCatchRowModel csvModel = new AccidentalCatchRowModel(MultiPostExportService.CSV_SEPARATOR, speciesList, genderValues, caracteristics, @@ -1124,7 +1120,7 @@ public class MultiPostImportService extends AbstractTuttiService { Map<String, AccidentalBatch> batches, List<Caracteristic> caracteristics) { - CaracteristicRowModel caracteristicModel = new CaracteristicRowModel(csvSeparator, + CaracteristicRowModel caracteristicModel = new CaracteristicRowModel(MultiPostExportService.CSV_SEPARATOR, caracteristics); Reader reader = ApplicationIOUtil.newReader( @@ -1249,7 +1245,7 @@ public class MultiPostImportService extends AbstractTuttiService { try { AttachmentRowModel attachmentModel = - new AttachmentRowModel(csvSeparator, new File(directory, ATTACHMENTS_DIRECTORY)); + new AttachmentRowModel(MultiPostExportService.CSV_SEPARATOR, new File(directory, ATTACHMENTS_DIRECTORY)); Import<AttachmentRow> importer = Import.newImport(attachmentModel, reader); try { @@ -1311,7 +1307,7 @@ public class MultiPostImportService extends AbstractTuttiService { try { AttachmentRowModel attachmentModel = - new AttachmentRowModel(csvSeparator, new File(directory, ATTACHMENTS_DIRECTORY)); + new AttachmentRowModel(MultiPostExportService.CSV_SEPARATOR, new File(directory, ATTACHMENTS_DIRECTORY)); Import<AttachmentRow> importer = Import.newImport(attachmentModel, reader); try { -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.
This is an automated email from the git hooks/post-receive script. New commit to branch develop in repository tutti. See http://git.codelutin.com/tutti.git commit 9098e61bb4cadf263ad7c4a77a1ea054ac499273 Author: Tony CHEMIT <chemit@codelutin.com> Date: Mon Sep 29 18:59:14 2014 +0200 fixes #5872: [PERFORMANCE] créer une base avec 100 traits pour tester les performances de l'application --- .../multipost/MultiPostImportServiceTest.java | 190 ++++++++++++++++++++- 1 file changed, 189 insertions(+), 1 deletion(-) diff --git a/tutti-service/src/test/java/fr/ifremer/tutti/service/catches/multipost/MultiPostImportServiceTest.java b/tutti-service/src/test/java/fr/ifremer/tutti/service/catches/multipost/MultiPostImportServiceTest.java index 2072358..0494bc2 100644 --- a/tutti-service/src/test/java/fr/ifremer/tutti/service/catches/multipost/MultiPostImportServiceTest.java +++ b/tutti-service/src/test/java/fr/ifremer/tutti/service/catches/multipost/MultiPostImportServiceTest.java @@ -1,10 +1,198 @@ package fr.ifremer.tutti.service.catches.multipost; +import fr.ifremer.tutti.persistence.entities.data.CatchBatch; +import fr.ifremer.tutti.persistence.entities.data.CatchBatchs; +import fr.ifremer.tutti.persistence.entities.data.FishingOperation; +import fr.ifremer.tutti.persistence.entities.data.FishingOperations; +import fr.ifremer.tutti.service.PersistenceService; +import fr.ifremer.tutti.service.ServiceDbResource; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.nuiton.jaxx.application.ApplicationIOUtil; +import org.nuiton.util.DateUtil; +import org.nuiton.util.FileUtil; + +import java.io.File; +import java.io.IOException; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.List; + +import static org.nuiton.i18n.I18n.n; + /** * Created on 9/29/14. * * @author Tony Chemit - chemit@codelutin.com - * @since XXX + * @since 3.8 */ public class MultiPostImportServiceTest { + + /** Logger. */ + private static final Log log = LogFactory.getLog(MultiPostImportServiceTest.class); + + @ClassRule + public static final ServiceDbResource dbResource = ServiceDbResource.writeDb("dbPerformance"); + + protected File dataDirectory; + + protected ServiceDbResource.DataContext dataContext; + + protected File speciesFile; + + protected File benthosFile; + + protected File marineLitterFile; + + @Before + public void setUp() throws Exception { + + dataDirectory = dbResource.getConfig().getDataDirectory(); + + dbResource.openDataContext(); + + dataContext = dbResource.loadContext("CAM-MEDITS", "100000", 0); + + speciesFile = FileUtil.getFileFromPaths(new File("src"), "test", "data", dbResource.getDbName(), "species"); + benthosFile = FileUtil.getFileFromPaths(new File("src"), "test", "data", dbResource.getDbName(), "benthos"); + marineLitterFile = FileUtil.getFileFromPaths(new File("src"), "test", "data", dbResource.getDbName(), "marineLitter"); + + } + + DateFormat df = new SimpleDateFormat("dd/MM/yyyy HH:mm:ss"); + + @Test + public void create100FishingOperations() throws IOException { + + MultiPostImportService multiPostImportService = dbResource.getServiceContext().getService(MultiPostImportService.class); + PersistenceService persistenceService = dbResource.getServiceContext().getService(PersistenceService.class); + + for (int operationNumber = 0; operationNumber < 100; operationNumber++) { + + // create a fishing operation + + FishingOperation fishingOperation = FishingOperations.newFishingOperation(); + + fishingOperation.setCruise(dataContext.cruise); + + String stationNumber = "MEDITS"; + Date date = DateUtil.createDate(1 + operationNumber, 10, 2014); + + if (log.isInfoEnabled()) { + log.info("Create fishingOperation " + operationNumber); + } + + fishingOperation.setStationNumber(stationNumber); + fishingOperation.setFishingOperationNumber(operationNumber); + fishingOperation.setMultirigAggregation("1"); + fishingOperation.setGearShootingStartDate(date); + fishingOperation.setGearShootingEndDate(date); + + fishingOperation = persistenceService.createFishingOperation(fishingOperation); + + CatchBatch catchBatch = CatchBatchs.newCatchBatch(); + catchBatch.setFishingOperation(fishingOperation); + persistenceService.createCatchBatch(catchBatch); + + // create import files + + File directory = dbResource.getServiceContext().getConfig().newTempFile("dbPerformance" + operationNumber); + + String operationDate = df.format(date); + + String operationNumberAsString = String.valueOf(operationNumber); + + { + if (log.isInfoEnabled()) { + log.info("Create species multiImport " + operationNumber); + } + + File speciesImportFile = copyImport(speciesFile, directory, "species", stationNumber, operationNumberAsString, operationDate); + + if (log.isInfoEnabled()) { + log.info("Import multiImport " + speciesImportFile); + } + + multiPostImportService.importSpecies(speciesImportFile, fishingOperation); + + } + + { + if (log.isInfoEnabled()) { + log.info("Create benthos multiImport " + operationNumber); + } + File benthosImportFile = copyImport(benthosFile, directory, "benthos", stationNumber, operationNumberAsString, operationDate); + + if (log.isInfoEnabled()) { + log.info("Import multiImport " + benthosImportFile); + } + + multiPostImportService.importBenthos(benthosImportFile, fishingOperation); + + } + + { + if (log.isInfoEnabled()) { + log.info("Create marineLitter multiImport " + operationNumber); + } + File marineLitterImportFile = copyImport(marineLitterFile, directory, "marineLitter", stationNumber, operationNumberAsString, operationDate); + + if (log.isInfoEnabled()) { + log.info("Import multiImport " + marineLitterImportFile); + } + + multiPostImportService.importMarineLitter(marineLitterImportFile, fishingOperation); + + } + + if (log.isInfoEnabled()) { + log.info("Import multiImport " + operationNumber); + } + + } + + } + + protected File copyImport(File incomingDirectory, + File directory, + String type, + String stationNumber, + String operationNumber, + String operationDate) throws IOException { + + FileUtils.forceMkdir(directory); + FileUtils.copyDirectoryToDirectory(incomingDirectory, directory); + + File targetDirectory = new File(directory, type); + + File weightsFile = new File(targetDirectory, "weights.csv"); + String weights = FileUtils.readFileToString(weightsFile) + .replace("$STATIONNUMBER$", stationNumber) + .replace("$OPERATIONNUMBER$", operationNumber) + .replace("$DATE$", operationDate); + + FileUtils.write(weightsFile, weights); + + File file = new File(directory, type + ".tutti" + StringUtils.capitalize(type)); + + List<File> files = new ArrayList<>(); + files.addAll(Arrays.asList(targetDirectory.listFiles())); + + ApplicationIOUtil.zip(targetDirectory, + file, + files, + n("tutti.service.multipost.export.error")); + + return file; + + } + } -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.
participants (1)
-
codelutin.com scm