March 2012 - Echobase-commits - mailman.cloud.codelutin.com

r380 - trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata
by tchemit＠users.forge.codelutin.com 19 Mar '12

19 Mar '12

Author: tchemit Date: 2012-03-19 18:54:32 +0100 (Mon, 19 Mar 2012) New Revision: 380 Url: http://forge.codelutin.com/repositories/revision/echobase/380 Log: redo something I have done at least 100 times, :( Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/AcousticDataImportService.java Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/AcousticDataImportService.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/AcousticDataImportService.java 2012-03-19 17:48:45 UTC (rev 379) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/AcousticDataImportService.java 2012-03-19 17:54:32 UTC (rev 380) @@ -476,7 +476,6 @@ CsvFileImportResult importResult) { /* - Latitude OK Longitude OK depthStart (surface ou bottom) OK @@ -488,8 +487,6 @@ Surface OK NumberOfSamplesRecorded OK NumberOfSamplesEchoIntegrated OK -minEIthr OK -maxEIthr OK */ String dataValue; String startMeta; @@ -543,14 +540,6 @@ dataValue = String.valueOf(row.getCellNasc()); createCellData(dao, cell, dataMetadatas, "NASC", dataValue, importResult); - // create minEIthr data - dataValue = String.valueOf(row.getCellMinEIthr()); - createCellData(dao, cell, dataMetadatas, "minEIthr", dataValue, importResult); - - // create maxEIthr data - dataValue = String.valueOf(row.getCellMaxEIthr()); - createCellData(dao, cell, dataMetadatas, "maxEIthr", dataValue, importResult); - // create Volume data dataValue = String.valueOf(row.getCellVolume()); createCellData(dao, cell, dataMetadatas, "Volume ",

1 0

r379 - trunk/echobase-tools/src/test/resources/csv/new
by tchemit＠users.forge.codelutin.com 19 Mar '12

19 Mar '12

Author: tchemit Date: 2012-03-19 18:48:45 +0100 (Mon, 19 Mar 2012) New Revision: 379 Url: http://forge.codelutin.com/repositories/revision/echobase/379 Log: fix references Modified: trunk/echobase-tools/src/test/resources/csv/new/input-DataMetadata.csv Modified: trunk/echobase-tools/src/test/resources/csv/new/input-DataMetadata.csv =================================================================== --- trunk/echobase-tools/src/test/resources/csv/new/input-DataMetadata.csv 2012-03-19 17:19:05 UTC (rev 378) +++ trunk/echobase-tools/src/test/resources/csv/new/input-DataMetadata.csv 2012-03-19 17:48:45 UTC (rev 379) @@ -34,5 +34,5 @@ "Formula";"Formula";"How was computed the sA value allocated to the cell/echotype combination?";;;;;;;;;;;;; "MeanXe";"MeanXe";"Mean acoustic scaling factor in (region) cells";;;;;;;0;10;;;999;"Mean"; "EsduFlag";"EsduFlag";"ESDU Flag: 0 discard, : keep";;"boolean";;;;;0;1;0;;999;"Mean"; -"minEIthr";;"minimum echo-integration threshold";;"dB ref. 1m-1";;;;;0;-120;0;;999999;; -"maxEIthr";;"maximum echo-integration threshold";;"dB ref. 1m-1";;;;;0;-120;0;;999999;; +"NumberOfSamplesRecorded";"NumberOfSamples";"Number of acoustic samples recorded in a cell";;;;;;;0;999999999;1;0;0;; +"NumberOfSamplesEchoIntegrated";"NumberOfSamples";"Number of acoustic samples echo-integrated in a cell";;;;;;;0;999999999;1;0;0;;

1 0

r378 - trunk/echobase-services/src/main/java/fr/ifremer/echobase/services
by tchemit＠users.forge.codelutin.com 19 Mar '12

19 Mar '12

Author: tchemit Date: 2012-03-19 18:19:05 +0100 (Mon, 19 Mar 2012) New Revision: 378 Url: http://forge.codelutin.com/repositories/revision/echobase/378 Log: fix NPE Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/ExportDbService.java Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/ExportDbService.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/ExportDbService.java 2012-03-19 17:07:09 UTC (rev 377) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/ExportDbService.java 2012-03-19 17:19:05 UTC (rev 378) @@ -129,10 +129,12 @@ MetaFilenameAware[] refEntries = EntitiesUtil.getReferenceEntries(getDbMeta()); + boolean withVoyages = model.getVoyageIds() != null; + if (computeSteps) { int nbstesp = refEntries.length; - if (model.getVoyageIds() != null) { + if (withVoyages) { nbstesp += model.getVoyageIds().length; } @@ -152,29 +154,31 @@ } } - MetaFilenameAware[] dataEntries = - EntitiesUtil.getDataEntries(getDbMeta()); - Multimap<EchoBaseEntityEnum, MetaFilenameAware> dataAssociationEntries = - EntitiesUtil.getDataAssociationsEntries(getDbMeta()); + if (withVoyages) { + MetaFilenameAware[] dataEntries = + EntitiesUtil.getDataEntries(getDbMeta()); + Multimap<EchoBaseEntityEnum, MetaFilenameAware> dataAssociationEntries = + EntitiesUtil.getDataAssociationsEntries(getDbMeta()); - ReplicateEntityVisitor visitor = ReplicateEntityVisitor.createVisitor( - exportService, - dataEntries, - dataAssociationEntries, - dir - ); + ReplicateEntityVisitor visitor = ReplicateEntityVisitor.createVisitor( + exportService, + dataEntries, + dataAssociationEntries, + dir + ); - try { - for (String voyageId : model.getVoyageIds()) { + try { + for (String voyageId : model.getVoyageIds()) { - model.incrementsProgression(); + model.incrementsProgression(); - Voyage voyage = getEntityById(Voyage.class, voyageId); - visitor.export(voyage); + Voyage voyage = getEntityById(Voyage.class, voyageId); + visitor.export(voyage); + } + } finally { + // clear visitor will close export files + visitor.clear(); } - } finally { - // clear visitor will close export files - visitor.clear(); } EchoBaseIOUtil.compressZipFile(zipFile, dir);

1 0

r377 - in trunk/echobase-services: . src/test/java/fr/ifremer/echobase/services
by tchemit＠users.forge.codelutin.com 19 Mar '12

19 Mar '12

Author: tchemit Date: 2012-03-19 18:07:09 +0100 (Mon, 19 Mar 2012) New Revision: 377 Url: http://forge.codelutin.com/repositories/revision/echobase/377 Log: pass a test in IT Added: trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/EmbeddedApplicationServiceIT.java Removed: trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/EmbeddedApplicationServiceTest.java Modified: trunk/echobase-services/pom.xml Modified: trunk/echobase-services/pom.xml =================================================================== --- trunk/echobase-services/pom.xml 2012-03-19 16:46:45 UTC (rev 376) +++ trunk/echobase-services/pom.xml 2012-03-19 17:07:09 UTC (rev 377) @@ -226,7 +226,7 @@ </executions> <configuration> <includes> - <include>**/*ImportServiceIT.java</include> + <include>**/*IT.java</include> </includes> </configuration> Copied: trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/EmbeddedApplicationServiceIT.java (from rev 376, trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/EmbeddedApplicationServiceTest.java) =================================================================== --- trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/EmbeddedApplicationServiceIT.java (rev 0) +++ trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/EmbeddedApplicationServiceIT.java 2012-03-19 17:07:09 UTC (rev 377) @@ -0,0 +1,69 @@ +/* + * #%L + * EchoBase :: Services + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2011 Ifremer, Codelutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ +package fr.ifremer.echobase.services; + +import fr.ifremer.echobase.services.configurations.EmbeddedApplicationConfiguration; +import fr.ifremer.echobase.services.importdata.AbstractImportDataServiceIT; +import org.junit.Assert; +import org.junit.Test; +import org.nuiton.util.FileUtil; + +import java.io.File; + +/** + * Test {@link EmbeddedApplicationService}. + * + * @author tchemit <chemit(a)codelutin.com> + * @since 0.2 + */ +public class EmbeddedApplicationServiceIT extends EchoBaseTestServiceSupport { + + protected FakeEchoBaseServiceContext initContext() { + return new FakeEchoBaseServiceContext( + AbstractImportDataServiceIT.IMPORT_DATA_ECHOBASE_CATCHES_AND_ACOUSTIC_AND_VOYAGE_RESULT); + } + + @Test + public void testCreateEmbeddedApplication() throws Exception { + EmbeddedApplicationConfiguration conf = new EmbeddedApplicationConfiguration(); + conf.setVoyageIds(getVoyageId()); + + File workingDirectory = new File(getTestDir(), "work-dir"); + FileUtil.createDirectoryIfNecessary(workingDirectory); + conf.setWorkingDirectory(workingDirectory); + File warLocation = new File(workingDirectory, "echobase.war"); + FileUtil.writeString(warLocation, "dummy!"); + conf.setWarLocation(warLocation); + conf.setFileName("echobase"); + EmbeddedApplicationService service = + getService(EmbeddedApplicationService.class); + + File zipFile = service.createEmbeddedApplication(conf); + + Assert.assertTrue(zipFile.exists()); + Assert.assertEquals(100f, conf.getProgression(), 1); + + // TODO Test that embedded db has exactly same number of data than in incoming db + } +} Property changes on: trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/EmbeddedApplicationServiceIT.java ___________________________________________________________________ Added: svn:keywords + Author Date Id Revision HeadURL Added: svn:eol-style + native Deleted: trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/EmbeddedApplicationServiceTest.java =================================================================== --- trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/EmbeddedApplicationServiceTest.java 2012-03-19 16:46:45 UTC (rev 376) +++ trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/EmbeddedApplicationServiceTest.java 2012-03-19 17:07:09 UTC (rev 377) @@ -1,69 +0,0 @@ -/* - * #%L - * EchoBase :: Services - * - * $Id$ - * $HeadURL$ - * %% - * Copyright (C) 2011 Ifremer, Codelutin - * %% - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * #L% - */ -package fr.ifremer.echobase.services; - -import fr.ifremer.echobase.services.configurations.EmbeddedApplicationConfiguration; -import fr.ifremer.echobase.services.importdata.AbstractImportDataServiceIT; -import org.junit.Assert; -import org.junit.Test; -import org.nuiton.util.FileUtil; - -import java.io.File; - -/** - * Test {@link EmbeddedApplicationService}. - * - * @author tchemit <chemit(a)codelutin.com> - * @since 0.2 - */ -public class EmbeddedApplicationServiceTest extends EchoBaseTestServiceSupport { - - protected FakeEchoBaseServiceContext initContext() { - return new FakeEchoBaseServiceContext( - AbstractImportDataServiceIT.IMPORT_DATA_ECHOBASE_CATCHES_AND_ACOUSTIC_AND_VOYAGE_RESULT); - } - - @Test - public void testCreateEmbeddedApplication() throws Exception { - EmbeddedApplicationConfiguration conf = new EmbeddedApplicationConfiguration(); - conf.setVoyageIds(getVoyageId()); - - File workingDirectory = new File(getTestDir(), "work-dir"); - FileUtil.createDirectoryIfNecessary(workingDirectory); - conf.setWorkingDirectory(workingDirectory); - File warLocation = new File(workingDirectory, "echobase.war"); - FileUtil.writeString(warLocation, "dummy!"); - conf.setWarLocation(warLocation); - conf.setFileName("echobase"); - EmbeddedApplicationService service = - getService(EmbeddedApplicationService.class); - - File zipFile = service.createEmbeddedApplication(conf); - - Assert.assertTrue(zipFile.exists()); - Assert.assertEquals(100f, conf.getProgression(), 1); - - // TODO Test that embedded db has exactly same number of data than in incoming db - } -}

1 0

r376 - in trunk/echobase-services/src: main/java/fr/ifremer/echobase/services test/java/fr/ifremer/echobase/services
by tchemit＠users.forge.codelutin.com 19 Mar '12

19 Mar '12

Author: tchemit Date: 2012-03-19 17:46:45 +0100 (Mon, 19 Mar 2012) New Revision: 376 Url: http://forge.codelutin.com/repositories/revision/echobase/376 Log: improve code for embedded create application + improve logs of impot db Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/EmbeddedApplicationService.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/ImportDbService.java trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/EmbeddedApplicationServiceTest.java Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/EmbeddedApplicationService.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/EmbeddedApplicationService.java 2012-03-19 16:42:51 UTC (rev 375) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/EmbeddedApplicationService.java 2012-03-19 16:46:45 UTC (rev 376) @@ -29,6 +29,7 @@ import fr.ifremer.echobase.EchoBaseTechnicalException; import fr.ifremer.echobase.EchoBaseTopiaRootContextFactory; import fr.ifremer.echobase.entities.EchoBaseUser; +import fr.ifremer.echobase.entities.EntitiesUtil; import fr.ifremer.echobase.entities.ExportQuery; import fr.ifremer.echobase.services.configurations.EmbeddedApplicationConfiguration; import fr.ifremer.echobase.services.configurations.ExportDbConfiguration; @@ -90,67 +91,130 @@ public File createEmbeddedApplication(EmbeddedApplicationConfiguration model) { - File warLocation = model.getWarLocation(); + int nbSteps = computeNbSteps(model); + model.setNbSteps(nbSteps); String fileName = model.getFileName(); - model.setNbSteps(5); - File tempDirectory = model.getWorkingDirectory(); + tempDirectory.deleteOnExit(); File zipFile = new File(tempDirectory, fileName + ".zip"); + model.setEmbeddedApplicationFile(zipFile); - tempDirectory.deleteOnExit(); - File dir = new File(tempDirectory, fileName); - model.setEmbeddedApplicationFile(zipFile); - if (log.isInfoEnabled()) { log.info("Creates embedded application to file [" + zipFile + "]"); } + + // export db to csv + File exportZipFile = exportDb(model, tempDirectory); + + // create static archive structure + createStaticArchiveStructure(model, dir); + + model.incrementsProgression(); + + // create h2 db in /db + TopiaContext rootContext = + new EchoBaseTopiaRootContextFactory().newEmbeddedDatabase(dir); + try { + + TopiaContext topiaContext = rootContext.beginTransaction(); + + model.incrementsProgression(); + + // create new service context (with new transaction) + EchoBaseServiceContext newServiceContext = + DefaultEchoBaseServiceContext.newContext(serviceContext, + topiaContext); + + EchoBaseUser admin = createUsers(newServiceContext); + + model.incrementsProgression(); + + copyExportQueries(newServiceContext, admin); + + model.incrementsProgression(); + + importDb(model, newServiceContext, exportZipFile, admin); + + } catch (Exception eee) { + throw new EchoBaseTechnicalException("Could not create h2 embedded database at "+dir,eee); + } finally { + try { + rootContext.closeContext(); + } catch (TopiaException eee) { + throw new EchoBaseTechnicalException("Could not close h2 embedded database at "+dir,eee); + } + } + + EchoBaseIOUtil.compressZipFile(zipFile, dir); + return zipFile; + } + + protected int computeNbSteps(EmbeddedApplicationConfiguration model) { + int nbSteps = 4; + + // export db steps + + nbSteps += EntitiesUtil.getReferenceEntries(getDbMeta()).length; + if (model.getVoyageIds() != null) { + nbSteps += model.getVoyageIds().length; + } + + // import db stesp + nbSteps += EntitiesUtil.getEntries(getDbMeta()).length; + return nbSteps; + } + + protected void createStaticArchiveStructure(EmbeddedApplicationConfiguration model, File dir) { + + + try { + File warLocation = model.getWarLocation(); + // create / FileUtil.createDirectoryIfNecessary(dir); // copy embedded files to / copyEmbeddedFiles(model, dir); - model.incrementsProgression(); - // copy war to / FileUtil.copy(warLocation, new File(dir, warLocation.getName())); + } catch (Exception eee) { + throw new EchoBaseTechnicalException("Could not create embedded zip structure at " + dir, eee); + } + } + protected File exportDb(EmbeddedApplicationConfiguration model, + File tempDirectory) { + + try { // create export zip file - File exportZipFile = exportDb(model, tempDirectory); + ExportDbConfiguration exportconfiguration = + new ExportDbConfiguration(model); + exportconfiguration.setFileName("echobase-export"); + File exportTempDir = new File(tempDirectory, "exportDb"); + exportconfiguration.setWorkingDirectory(exportTempDir); + exportconfiguration.setVoyageIds(model.getVoyageIds()); + ExportDbService exportDbService = + serviceContext.newService(ExportDbService.class); + exportDbService.exportPartialDb(exportconfiguration, false); + File exportZipFile = exportconfiguration.getExportFile(); if (log.isInfoEnabled()) { log.info("Export zip file = " + exportZipFile); } - - // create h2 db in /db - createH2Batabase(model, dir, true, exportZipFile); - - } catch (Exception eee) { + return exportZipFile; + } catch (IOException eee) { throw new EchoBaseTechnicalException( - "Could not create zip structure at location " + - dir, eee); + "Could not export db to zip", eee); } - EchoBaseIOUtil.compressZipFile(zipFile, dir); - return zipFile; - } - protected File exportDb(EmbeddedApplicationConfiguration model, - File tempDirectory) throws IOException { - ExportDbConfiguration exportconfiguration = new ExportDbConfiguration(model); - exportconfiguration.setFileName("echobase-export"); - exportconfiguration.setWorkingDirectory(new File(tempDirectory, "exportDb")); - exportconfiguration.setVoyageIds(model.getVoyageIds()); - - ExportDbService exportDbService = serviceContext.newService(ExportDbService.class); - exportDbService.exportPartialDb(exportconfiguration, false); - return exportconfiguration.getExportFile(); } protected void copyEmbeddedFiles(EmbeddedApplicationConfiguration model, @@ -161,8 +225,6 @@ String in = startFile.getInputFileName(); String out = startFile.getOutputFileName(); - model.incrementsProgression(); - InputStream inputStream = getClass().getResourceAsStream(EMBEDDED_PATH + in); try { @@ -171,7 +233,8 @@ if (log.isInfoEnabled()) { log.info("Copy configuration to " + in + " to " + outputFile); } - OutputStreamWriter outputStream = new OutputStreamWriter(new FileOutputStream(outputFile), Charsets.UTF_8); + OutputStreamWriter outputStream = new OutputStreamWriter( + new FileOutputStream(outputFile), Charsets.UTF_8); try { IOUtils.copy(inputStream, outputStream); } finally { @@ -183,91 +246,56 @@ } } - protected TopiaContext createH2Batabase(EmbeddedApplicationConfiguration model, - File zipDirectory, - boolean closeRootContext, - File exportZipFile - ) throws IOException, TopiaException, ExportQueryNameAlreadyExistException { + protected EchoBaseUser createUsers(EchoBaseServiceContext newServiceContext) { - model.incrementsProgression(); + // get user service from h2 db + UserService userService = + newServiceContext.newService(UserService.class); - // create new h2 db - TopiaContext rootContext = - new EchoBaseTopiaRootContextFactory().newEmbeddedDatabase(zipDirectory); + // create admin user + userService.createDefaultUsers(); - TopiaContext topiaContext = rootContext.beginTransaction(); - try { + // get admin from h2 db + EchoBaseUser admin = userService.getUserByEmail( + UserService.DEFAULT_ADMIN_EMAIL); + return admin; + } - model.incrementsProgression(); + protected void copyExportQueries(EchoBaseServiceContext newServiceContext, + EchoBaseUser admin) throws TopiaException, ExportQueryNameAlreadyExistException { - // create new service context (with new transaction) - EchoBaseServiceContext newServiceContext = - DefaultEchoBaseServiceContext.newContext(serviceContext, - topiaContext); + // get all export queries from application + List<ExportQuery> queries = getEntities(ExportQuery.class); - // get user service from h2 db - UserService userService = - newServiceContext.newService(UserService.class); + // replicate queries + getTransaction().replicateEntities( + newServiceContext.getTransaction(), queries); - // create admin user - userService.createDefaultUsers(); + // create export sql service from h2 db + ExportQueryService exportQueryService = + newServiceContext.newService(ExportQueryService.class); - // get admin from h2 db - EchoBaseUser admin = userService.getUserByEmail( - UserService.DEFAULT_ADMIN_EMAIL); + // get all queries from h2 db + queries = exportQueryService.getEntities(ExportQuery.class); - model.incrementsProgression(); - - // get export query service - ExportQueryService exportQueryService = - getService(ExportQueryService.class); - - // get all export queries from application - List<ExportQuery> queries = - exportQueryService.getEntities(ExportQuery.class); - - // remove link to application user - for (ExportQuery query : queries) { - query.setLastModifiedUser(null); - } - - // replicate queries - getTransaction().replicateEntities(topiaContext, queries); - - // create export sql service from h2 db - exportQueryService = - newServiceContext.newService(ExportQueryService.class); - - // get all queries from h2 db - queries = exportQueryService.getEntities(ExportQuery.class); - - // attach them to default created admin user in the db - for (ExportQuery query : queries) { - exportQueryService.createOrUpdate(query, admin); - } - - model.incrementsProgression(); - - importDb(model, newServiceContext, exportZipFile, admin); - - } finally { - if (closeRootContext) { - rootContext.closeContext(); - } + // attach them to default created admin user in the db + for (ExportQuery query : queries) { + exportQueryService.createOrUpdate(query, admin); } - return rootContext; } protected void importDb(EmbeddedApplicationConfiguration model, EchoBaseServiceContext newServiceContext, File exportZipFile, EchoBaseUser admin) throws IOException, TopiaException { - ImportDbConfiguration importConfiguration = new ImportDbConfiguration(model, getLocale()); + ImportDbConfiguration importConfiguration = + new ImportDbConfiguration(model, getLocale()); importConfiguration.getInput().setFile(exportZipFile); - importConfiguration.setWorkingDirectory(new File(model.getWorkingDirectory(), "importDb")); + File importTempDir = new File(model.getWorkingDirectory(), "importDb"); + importConfiguration.setWorkingDirectory(importTempDir); - ImportDbService importDbService = newServiceContext.newService(ImportDbService.class); - + ImportDbService importDbService = + newServiceContext.newService(ImportDbService.class); importDbService.importDb(importConfiguration, admin, false); } Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/ImportDbService.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/ImportDbService.java 2012-03-19 16:42:51 UTC (rev 375) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/ImportDbService.java 2012-03-19 16:46:45 UTC (rev 376) @@ -39,7 +39,6 @@ import org.nuiton.topia.TopiaException; import org.nuiton.topia.persistence.TopiaDAO; import org.nuiton.topia.persistence.TopiaEntity; -import org.nuiton.util.StringUtil; import org.nuiton.util.TimeLog; import org.nuiton.util.csv.Import; import org.nuiton.util.csv.ImportModel; @@ -86,14 +85,26 @@ ZipFile zipFile = new ZipFile(file); try { + List<String> missingEntries = Lists.newArrayList(); + Map<MetaFilenameAware, ZipEntry> entriestoConsume = discoverEntries( - file, zipFile); + file, zipFile, missingEntries); if (computeSteps) { int size = entriestoConsume.size(); - model.setNbSteps(size + 1); + model.setNbSteps(size + missingEntries.size() + 1); } + for (String missingEntry : missingEntries) { + + // skip thoses not found entries + model.incrementsProgression(); + + if (log.isInfoEnabled()) { + log.info("Skip not found entry " + missingEntry); + } + } + // consume all found entries for (Map.Entry<MetaFilenameAware, ZipEntry> entry : entriestoConsume.entrySet()) { @@ -137,10 +148,9 @@ } protected Map<MetaFilenameAware, ZipEntry> discoverEntries(File file, - ZipFile zipFile) { + ZipFile zipFile, + List<String> missingEntries) { - List<String> missingEntries = Lists.newArrayList(); - Map<MetaFilenameAware, ZipEntry> result = Maps.newLinkedHashMap(); @@ -157,10 +167,10 @@ result.put(entry, zipEntry); } } - if (!missingEntries.isEmpty() && log.isWarnEnabled()) { - log.warn("There is " + missingEntries.size() + " missing files " + - " in import " + file + "\n" + StringUtil.join(missingEntries, "\n", false)); - } +// if (!missingEntries.isEmpty() && log.isWarnEnabled()) { +// log.warn("There is " + missingEntries.size() + " missing files " + +// " in import " + file + "\n" + StringUtil.join(missingEntries, "\n", false)); +// } return result; } Modified: trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/EmbeddedApplicationServiceTest.java =================================================================== --- trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/EmbeddedApplicationServiceTest.java 2012-03-19 16:42:51 UTC (rev 375) +++ trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/EmbeddedApplicationServiceTest.java 2012-03-19 16:46:45 UTC (rev 376) @@ -62,5 +62,8 @@ File zipFile = service.createEmbeddedApplication(conf); Assert.assertTrue(zipFile.exists()); + Assert.assertEquals(100f, conf.getProgression(), 1); + + // TODO Test that embedded db has exactly same number of data than in incoming db } }

1 0

r375 - trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/configurations
by tchemit＠users.forge.codelutin.com 19 Mar '12

19 Mar '12

Author: tchemit Date: 2012-03-19 17:42:51 +0100 (Mon, 19 Mar 2012) New Revision: 375 Url: http://forge.codelutin.com/repositories/revision/echobase/375 Log: add javadoc Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/configurations/ProgressionModel.java Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/configurations/ProgressionModel.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/configurations/ProgressionModel.java 2012-03-19 16:00:01 UTC (rev 374) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/configurations/ProgressionModel.java 2012-03-19 16:42:51 UTC (rev 375) @@ -26,10 +26,10 @@ import java.io.Serializable; /** - * TODO + * Cotnract for a progression model while long service actions. * * @author tchemit <chemit(a)codelutin.com> - * @since TODO + * @since 0.3 */ public interface ProgressionModel extends Serializable {

1 0

r374 - in trunk/echobase-services/src: main/java/fr/ifremer/echobase main/java/fr/ifremer/echobase/services main/java/fr/ifremer/echobase/services/configurations main/java/fr/ifremer/echobase/services/csv main/java/fr/ifremer/echobase/services/exceptions main/java/fr/ifremer/echobase/services/importdata test/java/fr/ifremer/echobase/services test/java/fr/ifremer/echobase/services/importdata
by tchemit＠users.forge.codelutin.com 19 Mar '12

19 Mar '12

Author: tchemit Date: 2012-03-19 17:00:01 +0100 (Mon, 19 Mar 2012) New Revision: 374 Url: http://forge.codelutin.com/repositories/revision/echobase/374 Log: reformat codes Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/InputFile.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/ImportDbService.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/configurations/AbstractEchobaseActionConfiguration.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/AcousticDataImportModel.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/BiometrySampleImportModel.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/EchotypeImportModel.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/EchotypeImportModelRow.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/LengthAgeKeyImportModel.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/LengthAgeKeyImportModelRow.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/LengthWeightKeyImportModel.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/LengthWeightKeyImportModelRow.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/RegionCellAssociationImportModelRow.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/exceptions/ExportQueryNameAlreadyExistException.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/CommonDataImportService.java trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/EmbeddedApplicationServiceTest.java trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/importdata/AbstractImportDataServiceIT.java trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/importdata/ResultsEsduImportServiceIT.java Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/InputFile.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/InputFile.java 2012-03-19 15:58:51 UTC (rev 373) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/InputFile.java 2012-03-19 16:00:01 UTC (rev 374) @@ -89,6 +89,6 @@ } public boolean hasFile() { - return file!=null; + return file != null; } } Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/ImportDbService.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/ImportDbService.java 2012-03-19 15:58:51 UTC (rev 373) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/ImportDbService.java 2012-03-19 16:00:01 UTC (rev 374) @@ -72,8 +72,8 @@ /** * Import a complete db from the given zip file. * - * @param model model of the db import operation - * @param user user which perform the import + * @param model model of the db import operation + * @param user user which perform the import * @param computeSteps flag to compute stesp inside this method * @throws IOException if any io exception while import * @throws TopiaException if any pb while topia treatment Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/configurations/AbstractEchobaseActionConfiguration.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/configurations/AbstractEchobaseActionConfiguration.java 2012-03-19 15:58:51 UTC (rev 373) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/configurations/AbstractEchobaseActionConfiguration.java 2012-03-19 16:00:01 UTC (rev 374) @@ -23,8 +23,6 @@ */ package fr.ifremer.echobase.services.configurations; -import org.nuiton.util.StringUtil; - import java.io.IOException; import java.io.Serializable; Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/AcousticDataImportModel.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/AcousticDataImportModel.java 2012-03-19 15:58:51 UTC (rev 373) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/AcousticDataImportModel.java 2012-03-19 16:00:01 UTC (rev 374) @@ -41,7 +41,7 @@ public class AcousticDataImportModel extends CsvModelUtil.AbstractImportModel<AcousticDataImportModelRow> { -/* + /* ESDU Data Latitude MOVIES_EILayer\\cellset\\lat, OK @@ -71,7 +71,7 @@ NumberOfSamplesEchoIntegrated MOVIES_EILayer\\cellset\\ni minEIthr MOVIES_EILayer\cellset\thresholdlow maxEIthr MOVIES_EILayer\cellset\thresholdup - */ + */ public AcousticDataImportModel(char separator, Map<String, AcousticInstrument> instrumentsById) { super(separator); @@ -147,8 +147,8 @@ newIgnoredColumn("MOVIES_EILayer\\eilayer");//BN newMandatoryColumn("MOVIES_EILayer\\eilayer\\sa", AcousticDataImportModelRow.PROPERTY_CELL_NASC, CsvModelUtil.PRIMITIVE_FLOAT);//BO newIgnoredColumn("MOVIES_EILayer\\eilayer\\sv");//BP - newMandatoryColumn("MOVIES_EILayer\\eilayer\\ni", AcousticDataImportModelRow.PROPERTY_CELL_NUMBER_OF_SAMPLES_ECHO_INTEGRATED,CsvModelUtil.PRIMITIVE_INTEGER);//BQ - newMandatoryColumn("MOVIES_EILayer\\eilayer\\nt", AcousticDataImportModelRow.PROPERTY_CELL_NUMBER_OF_SAMPLES_RECORDED,CsvModelUtil.PRIMITIVE_INTEGER);//BR + newMandatoryColumn("MOVIES_EILayer\\eilayer\\ni", AcousticDataImportModelRow.PROPERTY_CELL_NUMBER_OF_SAMPLES_ECHO_INTEGRATED, CsvModelUtil.PRIMITIVE_INTEGER);//BQ + newMandatoryColumn("MOVIES_EILayer\\eilayer\\nt", AcousticDataImportModelRow.PROPERTY_CELL_NUMBER_OF_SAMPLES_RECORDED, CsvModelUtil.PRIMITIVE_INTEGER);//BR newIgnoredColumn("MOVIES_EILayer\\boterr");//BR newIgnoredColumn("MOVIES_EILayer\\boterr\\sa");//BT newIgnoredColumn("MOVIES_EILayer\\boterr\\ni");//BU Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/BiometrySampleImportModel.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/BiometrySampleImportModel.java 2012-03-19 15:58:51 UTC (rev 373) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/BiometrySampleImportModel.java 2012-03-19 16:00:01 UTC (rev 374) @@ -27,7 +27,6 @@ import fr.ifremer.echobase.entities.data.Sample; import fr.ifremer.echobase.entities.data.SampleData; import fr.ifremer.echobase.entities.references.SampleDataType; -import fr.ifremer.echobase.entities.references.SizeCategory; import fr.ifremer.echobase.entities.references.Species; import java.util.Map; @@ -40,7 +39,7 @@ */ public class BiometrySampleImportModel extends CsvModelUtil.AbstractImportModel<BiometrySampleImportModelRow> { - public BiometrySampleImportModel(char separator,Map<String, Operation> operationMap, + public BiometrySampleImportModel(char separator, Map<String, Operation> operationMap, Map<String, Species> speciesMap, Map<String, SampleDataType> sampleDataTypeMap) { super(separator); Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/EchotypeImportModel.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/EchotypeImportModel.java 2012-03-19 15:58:51 UTC (rev 373) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/EchotypeImportModel.java 2012-03-19 16:00:01 UTC (rev 374) @@ -23,9 +23,9 @@ */ package fr.ifremer.echobase.services.csv; +import fr.ifremer.echobase.entities.data.Echotype; import fr.ifremer.echobase.entities.data.Voyage; import fr.ifremer.echobase.entities.references.DepthStratum; -import fr.ifremer.echobase.entities.data.Echotype; import fr.ifremer.echobase.entities.references.Species; import java.util.Map; Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/EchotypeImportModelRow.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/EchotypeImportModelRow.java 2012-03-19 15:58:51 UTC (rev 373) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/EchotypeImportModelRow.java 2012-03-19 16:00:01 UTC (rev 374) @@ -23,10 +23,10 @@ */ package fr.ifremer.echobase.services.csv; +import fr.ifremer.echobase.entities.data.Echotype; +import fr.ifremer.echobase.entities.data.EchotypeImpl; import fr.ifremer.echobase.entities.data.Voyage; import fr.ifremer.echobase.entities.references.DepthStratum; -import fr.ifremer.echobase.entities.data.Echotype; -import fr.ifremer.echobase.entities.data.EchotypeImpl; import fr.ifremer.echobase.entities.references.Species; import java.io.Serializable; Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/LengthAgeKeyImportModel.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/LengthAgeKeyImportModel.java 2012-03-19 15:58:51 UTC (rev 373) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/LengthAgeKeyImportModel.java 2012-03-19 16:00:01 UTC (rev 374) @@ -23,8 +23,8 @@ */ package fr.ifremer.echobase.services.csv; +import fr.ifremer.echobase.entities.data.LengthAgeKey; import fr.ifremer.echobase.entities.data.Voyage; -import fr.ifremer.echobase.entities.data.LengthAgeKey; import fr.ifremer.echobase.entities.references.Species; import fr.ifremer.echobase.entities.references.Strata; Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/LengthAgeKeyImportModelRow.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/LengthAgeKeyImportModelRow.java 2012-03-19 15:58:51 UTC (rev 373) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/LengthAgeKeyImportModelRow.java 2012-03-19 16:00:01 UTC (rev 374) @@ -23,9 +23,9 @@ */ package fr.ifremer.echobase.services.csv; -import fr.ifremer.echobase.entities.data.Voyage; import fr.ifremer.echobase.entities.data.LengthAgeKey; import fr.ifremer.echobase.entities.data.LengthAgeKeyImpl; +import fr.ifremer.echobase.entities.data.Voyage; import fr.ifremer.echobase.entities.references.Species; import fr.ifremer.echobase.entities.references.Strata; Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/LengthWeightKeyImportModel.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/LengthWeightKeyImportModel.java 2012-03-19 15:58:51 UTC (rev 373) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/LengthWeightKeyImportModel.java 2012-03-19 16:00:01 UTC (rev 374) @@ -23,8 +23,8 @@ */ package fr.ifremer.echobase.services.csv; +import fr.ifremer.echobase.entities.data.LengthWeightKey; import fr.ifremer.echobase.entities.data.Voyage; -import fr.ifremer.echobase.entities.data.LengthWeightKey; import fr.ifremer.echobase.entities.references.SizeCategory; import fr.ifremer.echobase.entities.references.Species; Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/LengthWeightKeyImportModelRow.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/LengthWeightKeyImportModelRow.java 2012-03-19 15:58:51 UTC (rev 373) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/LengthWeightKeyImportModelRow.java 2012-03-19 16:00:01 UTC (rev 374) @@ -23,9 +23,9 @@ */ package fr.ifremer.echobase.services.csv; -import fr.ifremer.echobase.entities.data.Voyage; import fr.ifremer.echobase.entities.data.LengthWeightKey; import fr.ifremer.echobase.entities.data.LengthWeightKeyImpl; +import fr.ifremer.echobase.entities.data.Voyage; import fr.ifremer.echobase.entities.references.SizeCategory; import fr.ifremer.echobase.entities.references.Species; Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/RegionCellAssociationImportModelRow.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/RegionCellAssociationImportModelRow.java 2012-03-19 15:58:51 UTC (rev 373) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/RegionCellAssociationImportModelRow.java 2012-03-19 16:00:01 UTC (rev 374) @@ -34,7 +34,7 @@ public class RegionCellAssociationImportModelRow { protected Cell esduCell; - + protected Cell regionSurfaceCell; protected Cell regionBottomCell; Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/exceptions/ExportQueryNameAlreadyExistException.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/exceptions/ExportQueryNameAlreadyExistException.java 2012-03-19 15:58:51 UTC (rev 373) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/exceptions/ExportQueryNameAlreadyExistException.java 2012-03-19 16:00:01 UTC (rev 374) @@ -29,7 +29,7 @@ * @author tchemit <chemit(a)codelutin.com> * @since 0.3 */ -public class ExportQueryNameAlreadyExistException extends Exception{ +public class ExportQueryNameAlreadyExistException extends Exception { private static final long serialVersionUID = 1L; } Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/CommonDataImportService.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/CommonDataImportService.java 2012-03-19 15:58:51 UTC (rev 373) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/CommonDataImportService.java 2012-03-19 16:00:01 UTC (rev 374) @@ -43,7 +43,6 @@ import fr.ifremer.echobase.entities.references.MissionDAO; import fr.ifremer.echobase.entities.references.Vessel; import fr.ifremer.echobase.services.ImportException; -import fr.ifremer.echobase.services.exceptions.MissionNameAlreadyExistException; import fr.ifremer.echobase.services.configurations.CommonDataImportConfiguration; import fr.ifremer.echobase.services.csv.CsvFileImportResult; import fr.ifremer.echobase.services.csv.TransectImportModel; @@ -51,6 +50,7 @@ import fr.ifremer.echobase.services.csv.TransitImportModel; import fr.ifremer.echobase.services.csv.TransitImportModelRow; import fr.ifremer.echobase.services.csv.VoyageImportModel; +import fr.ifremer.echobase.services.exceptions.MissionNameAlreadyExistException; import org.nuiton.topia.TopiaException; import org.nuiton.util.csv.Import; import org.nuiton.util.csv.ImportModel; Modified: trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/EmbeddedApplicationServiceTest.java =================================================================== --- trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/EmbeddedApplicationServiceTest.java 2012-03-19 15:58:51 UTC (rev 373) +++ trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/EmbeddedApplicationServiceTest.java 2012-03-19 16:00:01 UTC (rev 374) @@ -32,9 +32,9 @@ import java.io.File; /** - * Test all fuctionnality of {@link EmbeddedApplicationService} + * Test {@link EmbeddedApplicationService}. * - * @author sletellier <letellier(a)codelutin.com> + * @author tchemit <chemit(a)codelutin.com> * @since 0.2 */ public class EmbeddedApplicationServiceTest extends EchoBaseTestServiceSupport { @@ -63,44 +63,4 @@ Assert.assertTrue(zipFile.exists()); } - -// @Test -// public void replicateVoyageEntity() throws Exception { -// -// String voyageId = getVoyageId(); -// -// // create new H2 dataBase in target/voyageId -// TopiaEntity voyageMocked = getEntityById(Voyage.class, voyageId); -// -// File testBaseDir = new File(getTestDir(), "replicated"); -// -// EmbeddedApplicationService service = -// getService(EmbeddedApplicationService.class); -// -// EmbeddedApplicationConfiguration conf = new EmbeddedApplicationConfiguration(); -// conf.setVoyageIds(voyageId); -// TopiaContext rootContextDest = service.createH2Batabase(conf, -// testBaseDir, -// false); -// -// // verify that replicated voyage is in new base -// TopiaContext transactionDest = null; -// try { -// transactionDest = rootContextDest.beginTransaction(); -// VoyageDAO voyageDAO = EchoBaseDAOHelper.getVoyageDAO(transactionDest); -// Voyage voyageReplicated = voyageDAO.findByTopiaId(voyageId); -// -// // assert voyage is in new base -// Assert.assertNotNull(voyageReplicated); -// -// // and same ids -// Assert.assertEquals(voyageMocked, voyageReplicated); -// -// // TODO sletellier 20111124 : try to test deph equality between entities -// } finally { -// if (transactionDest != null) { -// transactionDest.closeContext(); -// } -// } -// } } Modified: trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/importdata/AbstractImportDataServiceIT.java =================================================================== --- trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/importdata/AbstractImportDataServiceIT.java 2012-03-19 15:58:51 UTC (rev 373) +++ trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/importdata/AbstractImportDataServiceIT.java 2012-03-19 16:00:01 UTC (rev 374) @@ -55,11 +55,11 @@ public static final String IMPORT_DATA_ECHOBASE_OPERATION_RESULT = "/import-data/echobase-0.3-operation-result.h2.db.gz"; - public static final String IMPORT_DATA_ECHOBASE_CATCHES= "/import-data/echobase-0.3-catches.h2.db.gz"; + public static final String IMPORT_DATA_ECHOBASE_CATCHES = "/import-data/echobase-0.3-catches.h2.db.gz"; - public static final String IMPORT_DATA_ECHOBASE_CATCHES_AND_ACOUSTIC= "/import-data/echobase-0.3-catches-and-acoustic.h2.db.gz"; + public static final String IMPORT_DATA_ECHOBASE_CATCHES_AND_ACOUSTIC = "/import-data/echobase-0.3-catches-and-acoustic.h2.db.gz"; - public static final String IMPORT_DATA_ECHOBASE_CATCHES_AND_ACOUSTIC_AND_VOYAGE_RESULT= "/import-data/echobase-0.3-catches-and-acoustic-and-voyage-result.h2.db.gz"; + public static final String IMPORT_DATA_ECHOBASE_CATCHES_AND_ACOUSTIC_AND_VOYAGE_RESULT = "/import-data/echobase-0.3-catches-and-acoustic-and-voyage-result.h2.db.gz"; public static final String VESSEL_ID = "fr.ifremer.echobase.entities.references.Vessel#1323196672049#0.9790502711645855"; @@ -111,7 +111,7 @@ public static final int NB_ESDU_RESULT_BY_ECHOTYPE = 8; - public static final int NB_ESDU_RESULT_BY_ECHOTYPE_AND_SPECIES= 8; + public static final int NB_ESDU_RESULT_BY_ECHOTYPE_AND_SPECIES = 8; public void assertImportCommonData() throws TopiaException { assertNbEntities(Voyage.class, NB_VOYAGE); Modified: trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/importdata/ResultsEsduImportServiceIT.java =================================================================== --- trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/importdata/ResultsEsduImportServiceIT.java 2012-03-19 15:58:51 UTC (rev 373) +++ trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/importdata/ResultsEsduImportServiceIT.java 2012-03-19 16:00:01 UTC (rev 374) @@ -142,7 +142,6 @@ getImportPath("regionResults.csv")); - ResultsImportService service = getService(ResultsImportService.class);

1 0

r373 - in trunk: echobase-services/src/main/java/fr/ifremer/echobase/services echobase-services/src/main/java/fr/ifremer/echobase/services/configurations echobase-services/src/main/java/fr/ifremer/echobase/services/csv echobase-services/src/main/java/fr/ifremer/echobase/services/importdata echobase-services/src/test/java/fr/ifremer/echobase/services echobase-services/src/test/resources echobase-services/src/test/resources/import-data echobase-ui/src/main/java/fr/ifremer/echobase/ui/action
by tchemit＠users.forge.codelutin.com 19 Mar '12

19 Mar '12

Author: tchemit Date: 2012-03-19 16:58:51 +0100 (Mon, 19 Mar 2012) New Revision: 373 Url: http://forge.codelutin.com/repositories/revision/echobase/373 Log: redo embeddedApplication service using csv export-import (works much much better than hibernate replication) Added: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/ReplicateEntityVisitor.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/configurations/DefaultProgressionModel.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/configurations/ProgressionModel.java trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/ExportDbServiceTest.java Removed: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/EchoBaseImport.java Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/EchoBaseServiceSupport.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/EmbeddedApplicationService.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/ExportDbService.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/ImportDbService.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/configurations/AbstractEchobaseActionConfiguration.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/configurations/EmbeddedApplicationConfiguration.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/configurations/ExportDbConfiguration.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/configurations/ImportDbConfiguration.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/AcousticDataImportService.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/CatchesDataImportService.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/CommonDataImportService.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/OperationImportService.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/ResultsImportService.java trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/EmbeddedApplicationServiceTest.java trunk/echobase-services/src/test/resources/import-data/echobase-0.3-catches-and-acoustic-and-voyage-result.h2.db.gz trunk/echobase-services/src/test/resources/log4j.properties trunk/echobase-ui/src/main/java/fr/ifremer/echobase/ui/actions/importDb/Import.java Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/EchoBaseServiceSupport.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/EchoBaseServiceSupport.java 2012-03-19 15:55:36 UTC (rev 372) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/EchoBaseServiceSupport.java 2012-03-19 15:58:51 UTC (rev 373) @@ -74,6 +74,17 @@ } } + public final <E extends TopiaEntity> long countEntities(Class<E> entityType) { + Preconditions.checkNotNull(entityType); + try { + long result = getDAO(entityType).count(); + return result; + } catch (TopiaException eee) { + throw new EchoBaseTechnicalException( + "Could not count data for type " + entityType, eee); + } + } + public final <E extends TopiaEntity> E getEntityById(Class<E> entityType, String id) { Preconditions.checkNotNull(entityType); Preconditions.checkArgument(StringUtils.isNotEmpty(id)); Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/EmbeddedApplicationService.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/EmbeddedApplicationService.java 2012-03-19 15:55:36 UTC (rev 372) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/EmbeddedApplicationService.java 2012-03-19 15:58:51 UTC (rev 373) @@ -28,19 +28,17 @@ import fr.ifremer.echobase.EchoBaseIOUtil; import fr.ifremer.echobase.EchoBaseTechnicalException; import fr.ifremer.echobase.EchoBaseTopiaRootContextFactory; -import fr.ifremer.echobase.entities.EchoBaseEntityEnum; import fr.ifremer.echobase.entities.EchoBaseUser; -import fr.ifremer.echobase.entities.EntitiesUtil; import fr.ifremer.echobase.entities.ExportQuery; import fr.ifremer.echobase.services.configurations.EmbeddedApplicationConfiguration; +import fr.ifremer.echobase.services.configurations.ExportDbConfiguration; +import fr.ifremer.echobase.services.configurations.ImportDbConfiguration; import fr.ifremer.echobase.services.exceptions.ExportQueryNameAlreadyExistException; import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.topia.TopiaContext; import org.nuiton.topia.TopiaException; -import org.nuiton.topia.replication.TopiaReplicationService; -import org.nuiton.topia.replication.model.ReplicationModel; import org.nuiton.util.FileUtil; import java.io.File; @@ -123,8 +121,15 @@ // copy war to / FileUtil.copy(warLocation, new File(dir, warLocation.getName())); + // create export zip file + File exportZipFile = exportDb(model, tempDirectory); + + if (log.isInfoEnabled()) { + log.info("Export zip file = " + exportZipFile); + } + // create h2 db in /db - createH2Batabase(model, dir, true); + createH2Batabase(model, dir, true, exportZipFile); } catch (Exception eee) { throw new EchoBaseTechnicalException( @@ -136,8 +141,21 @@ return zipFile; } - protected void copyEmbeddedFiles(EmbeddedApplicationConfiguration model, File zipDirectory) throws IOException, URISyntaxException { + protected File exportDb(EmbeddedApplicationConfiguration model, + File tempDirectory) throws IOException { + ExportDbConfiguration exportconfiguration = new ExportDbConfiguration(model); + exportconfiguration.setFileName("echobase-export"); + exportconfiguration.setWorkingDirectory(new File(tempDirectory, "exportDb")); + exportconfiguration.setVoyageIds(model.getVoyageIds()); + ExportDbService exportDbService = serviceContext.newService(ExportDbService.class); + exportDbService.exportPartialDb(exportconfiguration, false); + return exportconfiguration.getExportFile(); + } + + protected void copyEmbeddedFiles(EmbeddedApplicationConfiguration model, + File zipDirectory) throws IOException, URISyntaxException { + for (EmbeddedFile startFile : EmbeddedFile.values()) { String in = startFile.getInputFileName(); @@ -167,7 +185,8 @@ protected TopiaContext createH2Batabase(EmbeddedApplicationConfiguration model, File zipDirectory, - boolean closeRootContext + boolean closeRootContext, + File exportZipFile ) throws IOException, TopiaException, ExportQueryNameAlreadyExistException { model.incrementsProgression(); @@ -229,18 +248,8 @@ model.incrementsProgression(); - // replicate referentiel to new h2 db - replicateReferentiel(topiaContext); + importDb(model, newServiceContext, exportZipFile, admin); - String[] voyageIds = model.getVoyageIds(); - - if (voyageIds != null) { - - model.incrementsProgression(); - - // replicate selected voyage to new h2 db - replicateData(topiaContext, voyageIds); - } } finally { if (closeRootContext) { rootContext.closeContext(); @@ -249,30 +258,17 @@ return rootContext; } - protected void replicateData(TopiaContext topiaContextDest, - String... voyagesIds) throws TopiaException { - TopiaReplicationService service = getTransaction().getService(TopiaReplicationService.class); - ReplicationModel replicationModel = service.prepare(EntitiesUtil.getDataTypes(), true, voyagesIds); - try { - service.doReplicate(replicationModel, topiaContextDest); - } catch (Exception eee) { - throw new EchoBaseTechnicalException( - "Failed to replicate referentiel : ", eee); - } - } + protected void importDb(EmbeddedApplicationConfiguration model, + EchoBaseServiceContext newServiceContext, + File exportZipFile, + EchoBaseUser admin) throws IOException, TopiaException { + ImportDbConfiguration importConfiguration = new ImportDbConfiguration(model, getLocale()); + importConfiguration.getInput().setFile(exportZipFile); + importConfiguration.setWorkingDirectory(new File(model.getWorkingDirectory(), "importDb")); - protected void replicateReferentiel(TopiaContext topiaContextDest) throws TopiaException { + ImportDbService importDbService = newServiceContext.newService(ImportDbService.class); - // Get all referenciel contracts - EchoBaseEntityEnum[] referencielClass = EntitiesUtil.getReferenceTypes(); - TopiaReplicationService service = getTransaction().getService(TopiaReplicationService.class); - ReplicationModel replicationModel = service.prepareForAll(referencielClass); - try { - service.doReplicate(replicationModel, topiaContextDest); - } catch (Exception eee) { - throw new EchoBaseTechnicalException( - "Failed to replicate referentiel : ", eee); - } + importDbService.importDb(importConfiguration, admin, false); } } Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/ExportDbService.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/ExportDbService.java 2012-03-19 15:55:36 UTC (rev 372) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/ExportDbService.java 2012-03-19 15:58:51 UTC (rev 373) @@ -23,8 +23,11 @@ */ package fr.ifremer.echobase.services; +import com.google.common.collect.Multimap; import fr.ifremer.echobase.EchoBaseIOUtil; +import fr.ifremer.echobase.entities.EchoBaseEntityEnum; import fr.ifremer.echobase.entities.EntitiesUtil; +import fr.ifremer.echobase.entities.data.Voyage; import fr.ifremer.echobase.entities.meta.AssociationMeta; import fr.ifremer.echobase.entities.meta.MetaFilenameAware; import fr.ifremer.echobase.entities.meta.TableMeta; @@ -92,8 +95,88 @@ exportService.exportDatas(meta, entryFile); } } + if (log.isInfoEnabled()) { + log.info("Export zip file = " + zipFile); + } + EchoBaseIOUtil.compressZipFile(zipFile, dir); + } + /** + * Export the partial db into csv files at the given location. + * + * @param model file name of the zip to create + * @param computeSteps flag to compute stesp inside this method + * @throws IOException if could not create or write files. + */ + public void exportPartialDb(ExportDbConfiguration model, + boolean computeSteps) throws IOException { + + String fileName = model.getFileName(); + + File tempDirectory = model.getWorkingDirectory(); + File dir = new File(tempDirectory, "echobase"); + FileUtil.createDirectoryIfNecessary(dir); + + if (log.isInfoEnabled()) { + log.info("Will export to " + dir); + } + + File zipFile = new File(tempDirectory, fileName + ".zip"); + model.setExportFile(zipFile); + + ExportService exportService = getService(ExportService.class); + + MetaFilenameAware[] refEntries = + EntitiesUtil.getReferenceEntries(getDbMeta()); + + if (computeSteps) { + int nbstesp = refEntries.length; + + if (model.getVoyageIds() != null) { + nbstesp += model.getVoyageIds().length; + } + + model.setNbSteps(nbstesp); + } + for (MetaFilenameAware entry : refEntries) { + + model.incrementsProgression(); + + File entryFile = new File(dir, entry.getFilename()); + if (entry instanceof AssociationMeta) { + AssociationMeta associationMeta = (AssociationMeta) entry; + exportService.exportDatas(associationMeta, entryFile); + } else { + TableMeta meta = (TableMeta) entry; + exportService.exportDatas(meta, entryFile); + } + } + + MetaFilenameAware[] dataEntries = + EntitiesUtil.getDataEntries(getDbMeta()); + Multimap<EchoBaseEntityEnum, MetaFilenameAware> dataAssociationEntries = + EntitiesUtil.getDataAssociationsEntries(getDbMeta()); + + ReplicateEntityVisitor visitor = ReplicateEntityVisitor.createVisitor( + exportService, + dataEntries, + dataAssociationEntries, + dir + ); + + try { + for (String voyageId : model.getVoyageIds()) { + + model.incrementsProgression(); + + Voyage voyage = getEntityById(Voyage.class, voyageId); + visitor.export(voyage); + } + } finally { + // clear visitor will close export files + visitor.clear(); + } + EchoBaseIOUtil.compressZipFile(zipFile, dir); } - } Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/ImportDbService.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/ImportDbService.java 2012-03-19 15:55:36 UTC (rev 372) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/ImportDbService.java 2012-03-19 15:58:51 UTC (rev 373) @@ -33,6 +33,7 @@ import fr.ifremer.echobase.entities.meta.MetaFilenameAware; import fr.ifremer.echobase.entities.meta.TableMeta; import fr.ifremer.echobase.services.configurations.ImportDbConfiguration; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.topia.TopiaException; @@ -73,11 +74,12 @@ * * @param model model of the db import operation * @param user user which perform the import + * @param computeSteps flag to compute stesp inside this method * @throws IOException if any io exception while import * @throws TopiaException if any pb while topia treatment */ public void importDb(ImportDbConfiguration model, - EchoBaseUser user) throws IOException, TopiaException { + EchoBaseUser user, boolean computeSteps) throws IOException, TopiaException { File file = model.getInput().getFile(); @@ -87,8 +89,10 @@ Map<MetaFilenameAware, ZipEntry> entriestoConsume = discoverEntries( file, zipFile); - int size = entriestoConsume.size(); - model.setNbSteps(size + 1); + if (computeSteps) { + int size = entriestoConsume.size(); + model.setNbSteps(size + 1); + } // consume all found entries for (Map.Entry<MetaFilenameAware, ZipEntry> entry : @@ -282,12 +286,14 @@ String topiaId = (String) row.get(TopiaEntity.TOPIA_ID); String[] associations = (String[]) row.get("target"); for (String association : associations) { - builder.append(String.format(updateString, topiaId, association)).append('\n'); - compt++; - if (compt % 10000 == 0) { - // flush it - getTransaction().executeSQL(builder.toString()); - builder = new StringBuilder(); + if (StringUtils.isNotEmpty(association)) { + builder.append(String.format(updateString, topiaId, association)).append('\n'); + compt++; + if (compt % 10000 == 0) { + // flush it + getTransaction().executeSQL(builder.toString()); + builder = new StringBuilder(); + } } } csvResult.incrementsNumberUpdated(); Added: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/ReplicateEntityVisitor.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/ReplicateEntityVisitor.java (rev 0) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/ReplicateEntityVisitor.java 2012-03-19 15:58:51 UTC (rev 373) @@ -0,0 +1,229 @@ +/* + * #%L + * EchoBase :: Services + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2011 - 2012 Ifremer, Codelutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ +package fr.ifremer.echobase.services; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; +import com.google.common.collect.Multimap; +import fr.ifremer.echobase.EchoBaseTechnicalException; +import fr.ifremer.echobase.csv.EntityExportContext; +import fr.ifremer.echobase.entities.EchoBaseEntityEnum; +import fr.ifremer.echobase.entities.meta.AssociationMeta; +import fr.ifremer.echobase.entities.meta.MetaFilenameAware; +import fr.ifremer.echobase.entities.meta.TableMeta; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.topia.TopiaException; +import org.nuiton.topia.persistence.EntityVisitor; +import org.nuiton.topia.persistence.TopiaEntity; +import org.nuiton.util.TimeLog; +import org.nuiton.util.csv.ExportModel; + +import java.io.File; +import java.io.FileNotFoundException; +import java.util.Collection; +import java.util.Map; + +/** + * entity visitor to export data to csv files. + * + * @author tchemit <chemit(a)codelutin.com> + * @since 0.3 + */ +public class ReplicateEntityVisitor implements EntityVisitor { + + /** Logger. */ + private static final Log log = + LogFactory.getLog(ReplicateEntityVisitor.class); + + public static final TimeLog TIME_LOG = + new TimeLog(ReplicateEntityVisitor.class); + + public static ReplicateEntityVisitor createVisitor(ExportService service, + MetaFilenameAware[] entityMetas, + Multimap<EchoBaseEntityEnum, MetaFilenameAware> associations, + File container) throws FileNotFoundException { + + Preconditions.checkNotNull(service); + Preconditions.checkNotNull(entityMetas); + Preconditions.checkNotNull(associations); + Preconditions.checkNotNull(container); + + Map<Class<?>, EntityExportContext> contexts = Maps.newHashMap(); + + for (MetaFilenameAware entityMeta : entityMetas) { + TableMeta meta = (TableMeta) entityMeta; + EchoBaseEntityEnum source = meta.getSource(); + + Collection<MetaFilenameAware> metaFilenameAwares = associations.get(source); + + ExportModel<TopiaEntity> model = service.buildForExport(meta, false); + + EntityExportContext exportContext = new EntityExportContext( + model, + meta, + container); + + // save both for contract and implementation with same context + contexts.put(source.getContract(), exportContext); + contexts.put(source.getImplementation(), exportContext); + + for (MetaFilenameAware metaFilenameAware : metaFilenameAwares) { + AssociationMeta associationMeta = (AssociationMeta) metaFilenameAware; + + ExportModel<TopiaEntity> associationModel = + service.buildForExport(associationMeta); + exportContext.addAssociationExportContext(associationMeta, + associationModel, + container); + } + } + + ReplicateEntityVisitor result = new ReplicateEntityVisitor(contexts); + return result; + } + + /** Export for simple entity. */ + protected final Map<Class<?>, EntityExportContext> entityExporters; + + public ReplicateEntityVisitor(Map<Class<?>, EntityExportContext> entityExporters) { + this.entityExporters = entityExporters; + } + + public void export(TopiaEntity entity) { + Preconditions.checkNotNull(entity); + long s1 = TimeLog.getTime(); + try { + entity.accept(this); + } catch (TopiaException e) { + throw new EchoBaseTechnicalException( + "Could not export entity " + entity.getTopiaId(), e); + } finally { + TIME_LOG.log(s1, "export::" + entity.getTopiaId()); + } + } + + @Override + public void start(TopiaEntity entity) { + try { + if (log.isDebugEnabled()) { + log.debug("Starts export of entity " + entity.getTopiaId()); + } + EntityExportContext entityExporter = + entityExporters.get(entity.getClass()); + entityExporter.setData(entity); + entityExporter.write(); + } catch (Exception e) { + throw new EchoBaseTechnicalException( + "Could not export entity " + entity, e); + } finally { + if (log.isDebugEnabled()) { + log.debug("Ends export of entity " + entity.getTopiaId()); + } + } + } + + @Override + public void end(TopiaEntity entity) { + try { + if (log.isDebugEnabled()) { + log.debug("Starts export of association of entity " + + entity.getTopiaId()); + } + EntityExportContext entityExporter = + entityExporters.get(entity.getClass()); + entityExporter.writeAssociations(); + } catch (Exception e) { + throw new EchoBaseTechnicalException( + "Could not export associations of entity " + entity, e); + } finally { + if (log.isDebugEnabled()) { + log.debug("Ends export of association of entity " + + entity.getTopiaId()); + } + } + } + + @Override + public void visit(TopiaEntity entity, String propertyName, + Class<?> type, Object value) { + // do nothing + } + + @Override + public void visit(TopiaEntity entity, + String propertyName, + Class<?> collectionType, + Class<?> type, + Object value) { + + if (TopiaEntity.class.isAssignableFrom(type) && + entityExporters.containsKey(type)) { + Collection<?> cValue = (Collection<?>) value; + + if (CollectionUtils.isNotEmpty(cValue)) { + + int i = 0; + for (Object currentValue : cValue) { + TopiaEntity currentEntity = (TopiaEntity) currentValue; + visit(entity, + propertyName, + type, + collectionType, + i++, + currentEntity + ); + + } + } + } + } + + @Override + public void visit(TopiaEntity entity, + String propertyName, + Class<?> collectionType, Class<?> type, + int index, + Object value) { + try { + ((TopiaEntity) value).accept(this); + } catch (TopiaException e) { + if (log.isErrorEnabled()) { + log.error("Can not visit entity " + value, e); + } + } + } + + @Override + public void clear() { + + // use at the end of visit (or later) + + for (EntityExportContext exportContext : entityExporters.values()) { + IOUtils.closeQuietly(exportContext); + } + } +} Property changes on: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/ReplicateEntityVisitor.java ___________________________________________________________________ Added: svn:keywords + Author Date Id Revision HeadURL Added: svn:eol-style + native Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/configurations/AbstractEchobaseActionConfiguration.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/configurations/AbstractEchobaseActionConfiguration.java 2012-03-19 15:55:36 UTC (rev 372) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/configurations/AbstractEchobaseActionConfiguration.java 2012-03-19 15:58:51 UTC (rev 373) @@ -34,58 +34,65 @@ * @author tchemit <chemit(a)codelutin.com> * @since 0.2 */ -public abstract class AbstractEchobaseActionConfiguration implements Serializable { +public abstract class AbstractEchobaseActionConfiguration implements Serializable, ProgressionModel { private static final long serialVersionUID = 1L; - private int nbSteps; + private final ProgressionModel progressionModel; - private float stepIncrement; + private Exception error; - private float progression; + protected AbstractEchobaseActionConfiguration() { + progressionModel = new DefaultProgressionModel(); + } - private long startTime; + protected AbstractEchobaseActionConfiguration(ProgressionModel progressionModel) { + this.progressionModel = progressionModel; + } - private long endTime; - - private Exception error; - - public final int getNbSteps() { - return nbSteps; + @Override + public final long getNbSteps() { + return progressionModel.getNbSteps(); } + @Override public final float getProgression() { - return progression; + return progressionModel.getProgression(); } + @Override public final void setProgression(float progression) { - this.progression = progression; + progressionModel.setProgression(progression); } + @Override public final void incrementsProgression() { - setProgression(progression + stepIncrement); + progressionModel.incrementsProgression(); } - public final void setNbSteps(int nbSteps) { - - this.nbSteps = nbSteps; - stepIncrement = 100f / nbSteps; + @Override + public final void setNbSteps(long nbSteps) { + progressionModel.setNbSteps(nbSteps); } + @Override public final long getStartTime() { - return startTime; + return progressionModel.getStartTime(); } + @Override public final void setStartTime(long startTime) { - this.startTime = startTime; + progressionModel.setStartTime(startTime); } + @Override public final long getEndTime() { - return endTime; + return progressionModel.getEndTime(); } + @Override public final void setEndTime(long endTime) { - this.endTime = endTime; + progressionModel.setEndTime(endTime); } public Exception getError() { @@ -96,10 +103,9 @@ this.error = error; } + @Override public String getActionTime() { - long sTime = getStartTime(); - long eTime = getEndTime(); - return StringUtil.convertTime(eTime - sTime); + return progressionModel.getActionTime(); } public void beginAction() { Added: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/configurations/DefaultProgressionModel.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/configurations/DefaultProgressionModel.java (rev 0) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/configurations/DefaultProgressionModel.java 2012-03-19 15:58:51 UTC (rev 373) @@ -0,0 +1,79 @@ +package fr.ifremer.echobase.services.configurations; + +import org.nuiton.util.StringUtil; + +/** + * Default progression model implementation. + * + * @author tchemit <chemit(a)codelutin.com> + * @since 0.3 + */ +public class DefaultProgressionModel implements ProgressionModel { + + private static final long serialVersionUID = 1L; + + private long nbSteps; + + private float stepIncrement; + + private float progression; + + private long startTime; + + private long endTime; + + @Override + public long getNbSteps() { + return nbSteps; + } + + @Override + public float getProgression() { + return progression; + } + + @Override + public void setProgression(float progression) { + this.progression = progression; + } + + @Override + public long getStartTime() { + return startTime; + } + + @Override + public void setStartTime(long startTime) { + this.startTime = startTime; + } + + @Override + public long getEndTime() { + return endTime; + } + + + @Override + public void setEndTime(long endTime) { + this.endTime = endTime; + } + + @Override + public final void incrementsProgression() { + setProgression(progression + stepIncrement); + } + + @Override + public final void setNbSteps(long nbSteps) { + + this.nbSteps = nbSteps; + stepIncrement = 100f / nbSteps; + } + + @Override + public String getActionTime() { + long sTime = getStartTime(); + long eTime = getEndTime(); + return StringUtil.convertTime(eTime - sTime); + } +} Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/configurations/EmbeddedApplicationConfiguration.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/configurations/EmbeddedApplicationConfiguration.java 2012-03-19 15:55:36 UTC (rev 372) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/configurations/EmbeddedApplicationConfiguration.java 2012-03-19 15:58:51 UTC (rev 373) @@ -82,7 +82,7 @@ return voyageIds; } - public void setVoyageIds(String[] voyageIds) { + public void setVoyageIds(String... voyageIds) { this.voyageIds = Arrays.copyOf(voyageIds, voyageIds.length); } Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/configurations/ExportDbConfiguration.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/configurations/ExportDbConfiguration.java 2012-03-19 15:55:36 UTC (rev 372) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/configurations/ExportDbConfiguration.java 2012-03-19 15:58:51 UTC (rev 373) @@ -44,6 +44,16 @@ private File exportFile; + /** Ids of voyages to push in embedded application (with all their datas). */ + private String[] voyageIds; + + public ExportDbConfiguration() { + } + + public ExportDbConfiguration(ProgressionModel progressionModel) { + super(progressionModel); + } + public String getFileName() { return fileName; } @@ -68,6 +78,14 @@ this.exportFile = exportFile; } + public String[] getVoyageIds() { + return voyageIds; + } + + public void setVoyageIds(String... voyageIds) { + this.voyageIds = voyageIds; + } + @Override public void destroy() throws IOException { if (workingDirectory != null) { Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/configurations/ImportDbConfiguration.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/configurations/ImportDbConfiguration.java 2012-03-19 15:55:36 UTC (rev 372) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/configurations/ImportDbConfiguration.java 2012-03-19 15:58:51 UTC (rev 373) @@ -44,11 +44,16 @@ protected File workingDirectory; - protected InputFile input; + protected final InputFile input; + public ImportDbConfiguration(ProgressionModel progressionModel, + Locale locale) { + super(progressionModel); + input = InputFile.newFile(l_(locale, "echobase.common.importDbFile")); + } + public ImportDbConfiguration(Locale locale) { - input = InputFile.newFile( - l_(locale, "echobase.common.importDbFile")); + input = InputFile.newFile(l_(locale, "echobase.common.importDbFile")); } public File getWorkingDirectory() { Added: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/configurations/ProgressionModel.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/configurations/ProgressionModel.java (rev 0) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/configurations/ProgressionModel.java 2012-03-19 15:58:51 UTC (rev 373) @@ -0,0 +1,55 @@ +/* + * #%L + * EchoBase :: Services + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2011 - 2012 Ifremer, Codelutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ +package fr.ifremer.echobase.services.configurations; + +import java.io.Serializable; + +/** + * TODO + * + * @author tchemit <chemit(a)codelutin.com> + * @since TODO + */ +public interface ProgressionModel extends Serializable { + + long getNbSteps(); + + float getProgression(); + + void setProgression(float progression); + + void incrementsProgression(); + + void setNbSteps(long nbSteps); + + long getStartTime(); + + void setStartTime(long startTime); + + long getEndTime(); + + void setEndTime(long endTime); + + String getActionTime(); +} Property changes on: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/configurations/ProgressionModel.java ___________________________________________________________________ Added: svn:keywords + Author Date Id Revision HeadURL Added: svn:eol-style + native Deleted: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/EchoBaseImport.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/EchoBaseImport.java 2012-03-19 15:55:36 UTC (rev 372) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/EchoBaseImport.java 2012-03-19 15:58:51 UTC (rev 373) @@ -1,137 +0,0 @@ -/* - * #%L - * EchoBase :: Services - * - * $Id$ - * $HeadURL$ - * %% - * Copyright (C) 2011 - 2012 Ifremer, Codelutin - * %% - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * #L% - */ -package fr.ifremer.echobase.services.csv; - -import com.google.common.base.Charsets; -import org.nuiton.util.StringUtil; -import org.nuiton.util.csv.Import; -import org.nuiton.util.csv.ImportModel; -import org.nuiton.util.csv.ImportRuntimeException; -import org.nuiton.util.csv.ImportableColumn; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.Reader; -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; - -import static org.nuiton.i18n.I18n._; - -/** - * This extends of the {@link Import} offers to allow case on header names. - * - * @author tchemit <chemit(a)codelutin.com> - * @since 0.3 - */ -public class EchoBaseImport<E> extends Import<E> { - - public static String[] getHeader(File file, char charSeprator) { - - try { - BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), Charsets.UTF_8.name())); - try { - String header = reader.readLine(); - String[] result = header.split(charSeprator + ""); - return result; - } finally { - reader.close(); - } - } catch (IOException e) { - throw new ImportRuntimeException("Could not obtain header of file " + file, e); - } - } - - public static <E> EchoBaseImport<E> newImport(ImportModel<E> model, - InputStream inputStream) { - return new EchoBaseImport<E>(model, inputStream); - } - - public static <E> EchoBaseImport<E> newImport(ImportModel<E> model, - Reader reader) { - return new EchoBaseImport<E>(model, reader); - } - - public EchoBaseImport(ImportModel<E> importModel, InputStream inputStream) { - super(importModel, inputStream); - } - - public EchoBaseImport(ImportModel<E> importModel, Reader reader) { - super(importModel, reader); - } - - @Override - protected void checkAllMandatoryHeadersArePresent(String[] headers) { - - List<String> csvHeaders = new ArrayList<String>(); - for (String header : headers) { - csvHeaders.add(header.toLowerCase()); - } -// Collections.addAll(csvHeaders, headers); - - List<String> mandatoryHeadersNames = new ArrayList<String>(); - for (ImportableColumn<E, ?> field : getAllMandatoryHeaders()) { -// mandatoryHeadersNames.add(field.getHeaderName()); - mandatoryHeadersNames.add(field.getHeaderName().toLowerCase()); - } - mandatoryHeadersNames.removeAll(csvHeaders); - - if (!mandatoryHeadersNames.isEmpty()) { - String validationMessage = - _("csv.import.error.missingMandatoryHeaders", - StringUtil.join(mandatoryHeadersNames, ", ", true)); - throw new ImportRuntimeException(validationMessage); - } - } - - @Override - protected void checkHeaderNamesAreAllKnown(String[] headers) { - List<String> csvHeaders = new ArrayList<String>(); - for (String header : headers) { - csvHeaders.add(header.toLowerCase()); - } -// Collections.addAll(csvHeaders, headers); - - for (ImportableColumn<E, ?> field : model.getColumnsForImport()) { - csvHeaders.remove(field.getHeaderName().toLowerCase()); -// csvHeaders.remove(field.getHeaderName()); - } - if (!csvHeaders.isEmpty()) { - List<String> validHeaderNames = new LinkedList<String>(); - for (ImportableColumn<E, ?> importableColumn : - model.getColumnsForImport()) { - validHeaderNames.add(importableColumn.getHeaderName()); - } - String validationMessage = - _("csv.import.error.unrecognizedHeaders", - StringUtil.join(csvHeaders, ", ", true), - StringUtil.join(validHeaderNames, ", ", true)); - throw new ImportRuntimeException(validationMessage); - } - } -} Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/AcousticDataImportService.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/AcousticDataImportService.java 2012-03-19 15:55:36 UTC (rev 372) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/AcousticDataImportService.java 2012-03-19 15:58:51 UTC (rev 373) @@ -28,6 +28,7 @@ import fr.ifremer.echobase.EchoBaseFunctions; import fr.ifremer.echobase.EchoBasePredicates; import fr.ifremer.echobase.InputFile; +import fr.ifremer.echobase.csv.EchoBaseImport; import fr.ifremer.echobase.entities.EchoBaseEntityEnum; import fr.ifremer.echobase.entities.EchoBaseUser; import fr.ifremer.echobase.entities.data.Cell; @@ -50,7 +51,6 @@ import fr.ifremer.echobase.services.csv.AcousticDataImportModel; import fr.ifremer.echobase.services.csv.AcousticDataImportModelRow; import fr.ifremer.echobase.services.csv.CsvFileImportResult; -import fr.ifremer.echobase.services.csv.EchoBaseImport; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/CatchesDataImportService.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/CatchesDataImportService.java 2012-03-19 15:55:36 UTC (rev 372) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/CatchesDataImportService.java 2012-03-19 15:58:51 UTC (rev 373) @@ -29,6 +29,7 @@ import fr.ifremer.echobase.EchoBaseFunctions; import fr.ifremer.echobase.EchoBasePredicates; import fr.ifremer.echobase.InputFile; +import fr.ifremer.echobase.csv.EchoBaseImport; import fr.ifremer.echobase.entities.EchoBaseEntityEnum; import fr.ifremer.echobase.entities.EchoBaseUser; import fr.ifremer.echobase.entities.data.Operation; @@ -51,7 +52,6 @@ import fr.ifremer.echobase.services.csv.BiometrySampleImportModel; import fr.ifremer.echobase.services.csv.BiometrySampleImportModelRow; import fr.ifremer.echobase.services.csv.CsvFileImportResult; -import fr.ifremer.echobase.services.csv.EchoBaseImport; import fr.ifremer.echobase.services.csv.SubSampleImportModel; import fr.ifremer.echobase.services.csv.SubSampleImportModelRow; import fr.ifremer.echobase.services.csv.TotalSampleImportModel; Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/CommonDataImportService.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/CommonDataImportService.java 2012-03-19 15:55:36 UTC (rev 372) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/CommonDataImportService.java 2012-03-19 15:58:51 UTC (rev 373) @@ -29,6 +29,7 @@ import fr.ifremer.echobase.EchoBaseFunctions; import fr.ifremer.echobase.EchoBaseTechnicalException; import fr.ifremer.echobase.InputFile; +import fr.ifremer.echobase.csv.EchoBaseImport; import fr.ifremer.echobase.entities.EchoBaseEntityEnum; import fr.ifremer.echobase.entities.EchoBaseUser; import fr.ifremer.echobase.entities.data.Transect; @@ -45,7 +46,6 @@ import fr.ifremer.echobase.services.exceptions.MissionNameAlreadyExistException; import fr.ifremer.echobase.services.configurations.CommonDataImportConfiguration; import fr.ifremer.echobase.services.csv.CsvFileImportResult; -import fr.ifremer.echobase.services.csv.EchoBaseImport; import fr.ifremer.echobase.services.csv.TransectImportModel; import fr.ifremer.echobase.services.csv.TransectImportModelRow; import fr.ifremer.echobase.services.csv.TransitImportModel; Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/OperationImportService.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/OperationImportService.java 2012-03-19 15:55:36 UTC (rev 372) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/OperationImportService.java 2012-03-19 15:58:51 UTC (rev 373) @@ -27,6 +27,7 @@ import com.google.common.collect.Maps; import fr.ifremer.echobase.EchoBaseFunctions; import fr.ifremer.echobase.InputFile; +import fr.ifremer.echobase.csv.EchoBaseImport; import fr.ifremer.echobase.entities.EchoBaseEntityEnum; import fr.ifremer.echobase.entities.EchoBaseUser; import fr.ifremer.echobase.entities.data.GearMetadataValue; @@ -46,7 +47,6 @@ import fr.ifremer.echobase.services.ImportException; import fr.ifremer.echobase.services.configurations.OperationImportConfiguration; import fr.ifremer.echobase.services.csv.CsvFileImportResult; -import fr.ifremer.echobase.services.csv.EchoBaseImport; import fr.ifremer.echobase.services.csv.GearMetadataValueImportModel; import fr.ifremer.echobase.services.csv.GearMetadataValueImportModelRow; import fr.ifremer.echobase.services.csv.OperationImportModel; Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/ResultsImportService.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/ResultsImportService.java 2012-03-19 15:55:36 UTC (rev 372) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/ResultsImportService.java 2012-03-19 15:58:51 UTC (rev 373) @@ -30,6 +30,7 @@ import fr.ifremer.echobase.EchoBaseFunctions; import fr.ifremer.echobase.EchoBasePredicates; import fr.ifremer.echobase.InputFile; +import fr.ifremer.echobase.csv.EchoBaseImport; import fr.ifremer.echobase.entities.EchoBaseEntityEnum; import fr.ifremer.echobase.entities.EchoBaseUser; import fr.ifremer.echobase.entities.data.Category; @@ -58,7 +59,6 @@ import fr.ifremer.echobase.services.csv.AcousticResultByEchotypeImportModel; import fr.ifremer.echobase.services.csv.AcousticResultByEchotypeImportModelRow; import fr.ifremer.echobase.services.csv.CsvFileImportResult; -import fr.ifremer.echobase.services.csv.EchoBaseImport; import fr.ifremer.echobase.services.csv.EchotypeImportModel; import fr.ifremer.echobase.services.csv.EchotypeImportModelRow; import fr.ifremer.echobase.services.csv.LengthAgeKeyImportModel; Modified: trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/EmbeddedApplicationServiceTest.java =================================================================== --- trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/EmbeddedApplicationServiceTest.java 2012-03-19 15:55:36 UTC (rev 372) +++ trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/EmbeddedApplicationServiceTest.java 2012-03-19 15:58:51 UTC (rev 373) @@ -23,15 +23,10 @@ */ package fr.ifremer.echobase.services; -import fr.ifremer.echobase.entities.EchoBaseDAOHelper; -import fr.ifremer.echobase.entities.data.Voyage; -import fr.ifremer.echobase.entities.data.VoyageDAO; import fr.ifremer.echobase.services.configurations.EmbeddedApplicationConfiguration; import fr.ifremer.echobase.services.importdata.AbstractImportDataServiceIT; import org.junit.Assert; import org.junit.Test; -import org.nuiton.topia.TopiaContext; -import org.nuiton.topia.persistence.TopiaEntity; import org.nuiton.util.FileUtil; import java.io.File; @@ -46,13 +41,13 @@ protected FakeEchoBaseServiceContext initContext() { return new FakeEchoBaseServiceContext( - AbstractImportDataServiceIT.IMPORT_DATA_ECHOBASE_OPERATION_RESULT); + AbstractImportDataServiceIT.IMPORT_DATA_ECHOBASE_CATCHES_AND_ACOUSTIC_AND_VOYAGE_RESULT); } @Test public void testCreateEmbeddedApplication() throws Exception { EmbeddedApplicationConfiguration conf = new EmbeddedApplicationConfiguration(); - conf.setVoyageIds(new String[]{getVoyageId()}); + conf.setVoyageIds(getVoyageId()); File workingDirectory = new File(getTestDir(), "work-dir"); FileUtil.createDirectoryIfNecessary(workingDirectory); @@ -69,43 +64,43 @@ Assert.assertTrue(zipFile.exists()); } - @Test - public void replicateVoyageEntity() throws Exception { - - String voyageId = getVoyageId(); - - // create new H2 dataBase in target/voyageId - TopiaEntity voyageMocked = getEntityById(Voyage.class, voyageId); - - File testBaseDir = new File(getTestDir(), "replicated"); - - EmbeddedApplicationService service = - getService(EmbeddedApplicationService.class); - - EmbeddedApplicationConfiguration conf = new EmbeddedApplicationConfiguration(); - conf.setVoyageIds(new String[]{voyageId}); - TopiaContext rootContextDest = service.createH2Batabase(conf, - testBaseDir, - false); - - // verify that replicated voyage is in new base - TopiaContext transactionDest = null; - try { - transactionDest = rootContextDest.beginTransaction(); - VoyageDAO voyageDAO = EchoBaseDAOHelper.getVoyageDAO(transactionDest); - Voyage voyageReplicated = voyageDAO.findByTopiaId(voyageId); - - // assert voyage is in new base - Assert.assertNotNull(voyageReplicated); - - // and same ids - Assert.assertEquals(voyageMocked, voyageReplicated); - - // TODO sletellier 20111124 : try to test deph equality between entities - } finally { - if (transactionDest != null) { - transactionDest.closeContext(); - } - } - } +// @Test +// public void replicateVoyageEntity() throws Exception { +// +// String voyageId = getVoyageId(); +// +// // create new H2 dataBase in target/voyageId +// TopiaEntity voyageMocked = getEntityById(Voyage.class, voyageId); +// +// File testBaseDir = new File(getTestDir(), "replicated"); +// +// EmbeddedApplicationService service = +// getService(EmbeddedApplicationService.class); +// +// EmbeddedApplicationConfiguration conf = new EmbeddedApplicationConfiguration(); +// conf.setVoyageIds(voyageId); +// TopiaContext rootContextDest = service.createH2Batabase(conf, +// testBaseDir, +// false); +// +// // verify that replicated voyage is in new base +// TopiaContext transactionDest = null; +// try { +// transactionDest = rootContextDest.beginTransaction(); +// VoyageDAO voyageDAO = EchoBaseDAOHelper.getVoyageDAO(transactionDest); +// Voyage voyageReplicated = voyageDAO.findByTopiaId(voyageId); +// +// // assert voyage is in new base +// Assert.assertNotNull(voyageReplicated); +// +// // and same ids +// Assert.assertEquals(voyageMocked, voyageReplicated); +// +// // TODO sletellier 20111124 : try to test deph equality between entities +// } finally { +// if (transactionDest != null) { +// transactionDest.closeContext(); +// } +// } +// } } Added: trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/ExportDbServiceTest.java =================================================================== --- trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/ExportDbServiceTest.java (rev 0) +++ trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/ExportDbServiceTest.java 2012-03-19 15:58:51 UTC (rev 373) @@ -0,0 +1,82 @@ +/* + * #%L + * EchoBase :: Services + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2011 - 2012 Ifremer, Codelutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ +package fr.ifremer.echobase.services; + +import fr.ifremer.echobase.services.configurations.ExportDbConfiguration; +import fr.ifremer.echobase.services.importdata.AbstractImportDataServiceIT; +import org.junit.Test; +import org.nuiton.util.FileUtil; + +import java.io.File; +import java.io.IOException; + +/** + * Test {@link ExportDbService}. + * + * @author tchemit <chemit(a)codelutin.com> + * @since 0.3 + */ +public class ExportDbServiceTest extends EchoBaseTestServiceSupport { + + protected FakeEchoBaseServiceContext initContext() { + return new FakeEchoBaseServiceContext( + AbstractImportDataServiceIT.IMPORT_DATA_ECHOBASE_CATCHES_AND_ACOUSTIC_AND_VOYAGE_RESULT); + } + + @Test + public void exportDb() throws IOException { + + ExportDbService service = + serviceContext.newService(ExportDbService.class); + + ExportDbConfiguration conf = new ExportDbConfiguration(); + + conf.setVoyageIds(getVoyageId()); + + File workingDirectory = new File(getTestDir(), "work-dir"); + FileUtil.createDirectoryIfNecessary(workingDirectory); + conf.setWorkingDirectory(workingDirectory); + conf.setFileName("echobase"); + + service.exportDb(conf); + } + + @Test + public void exportPartialDb() throws IOException { + + ExportDbService service = + serviceContext.newService(ExportDbService.class); + + ExportDbConfiguration conf = new ExportDbConfiguration(); + + conf.setVoyageIds(getVoyageId()); + + File workingDirectory = new File(getTestDir(), "work-dir"); + FileUtil.createDirectoryIfNecessary(workingDirectory); + conf.setWorkingDirectory(workingDirectory); + conf.setFileName("echobase"); + + service.exportPartialDb(conf, true); + } +} Property changes on: trunk/echobase-services/src/test/java/fr/ifremer/echobase/services/ExportDbServiceTest.java ___________________________________________________________________ Added: svn:keywords + Author Date Id Revision HeadURL Added: svn:eol-style + native Modified: trunk/echobase-services/src/test/resources/import-data/echobase-0.3-catches-and-acoustic-and-voyage-result.h2.db.gz =================================================================== (Binary files differ) Modified: trunk/echobase-services/src/test/resources/log4j.properties =================================================================== --- trunk/echobase-services/src/test/resources/log4j.properties 2012-03-19 15:55:36 UTC (rev 372) +++ trunk/echobase-services/src/test/resources/log4j.properties 2012-03-19 15:58:51 UTC (rev 373) @@ -30,4 +30,5 @@ # package level log4j.logger.fr.ifremer.echobase=INFO +#log4j.logger.fr.ifremer.echobase.services.ReplicateEntityVisitor=DEBUG log4j.logger.fr.ifremer.echobase.services.importdata.AcousticDataImportService=DEBUG Modified: trunk/echobase-ui/src/main/java/fr/ifremer/echobase/ui/actions/importDb/Import.java =================================================================== --- trunk/echobase-ui/src/main/java/fr/ifremer/echobase/ui/actions/importDb/Import.java 2012-03-19 15:55:36 UTC (rev 372) +++ trunk/echobase-ui/src/main/java/fr/ifremer/echobase/ui/actions/importDb/Import.java 2012-03-19 15:58:51 UTC (rev 373) @@ -65,7 +65,7 @@ model.getInput().getFileName()); } - service.importDb(model, getEchoBaseSession().getEchoBaseUser()); + service.importDb(model, getEchoBaseSession().getEchoBaseUser(), true); } @Override

1 0

r372 - in trunk/echobase-entities/src: main/java/fr/ifremer/echobase main/java/fr/ifremer/echobase/csv main/java/fr/ifremer/echobase/entities main/resources main/resources/i18n test/java/fr/ifremer/echobase/entities
by tchemit＠users.forge.codelutin.com 19 Mar '12

19 Mar '12

Author: tchemit Date: 2012-03-19 16:55:36 +0100 (Mon, 19 Mar 2012) New Revision: 372 Url: http://forge.codelutin.com/repositories/revision/echobase/372 Log: move csv neutral api to entity module Added: trunk/echobase-entities/src/main/java/fr/ifremer/echobase/csv/ trunk/echobase-entities/src/main/java/fr/ifremer/echobase/csv/AssociationExportContext.java trunk/echobase-entities/src/main/java/fr/ifremer/echobase/csv/EchoBaseExport.java trunk/echobase-entities/src/main/java/fr/ifremer/echobase/csv/EchoBaseImport.java trunk/echobase-entities/src/main/java/fr/ifremer/echobase/csv/EntityExportContext.java Modified: trunk/echobase-entities/src/main/java/fr/ifremer/echobase/EchoBaseFunctions.java trunk/echobase-entities/src/main/java/fr/ifremer/echobase/entities/EntitiesUtil.java trunk/echobase-entities/src/main/resources/i18n/echobase-entities_en_GB.properties trunk/echobase-entities/src/main/resources/i18n/echobase-entities_fr_FR.properties trunk/echobase-entities/src/main/resources/topia-h2.properties trunk/echobase-entities/src/test/java/fr/ifremer/echobase/entities/EntitiesUtilTest.java Modified: trunk/echobase-entities/src/main/java/fr/ifremer/echobase/EchoBaseFunctions.java =================================================================== --- trunk/echobase-entities/src/main/java/fr/ifremer/echobase/EchoBaseFunctions.java 2012-03-19 15:10:55 UTC (rev 371) +++ trunk/echobase-entities/src/main/java/fr/ifremer/echobase/EchoBaseFunctions.java 2012-03-19 15:55:36 UTC (rev 372) @@ -24,11 +24,13 @@ package fr.ifremer.echobase; import com.google.common.base.Function; +import fr.ifremer.echobase.entities.EchoBaseEntityEnum; import fr.ifremer.echobase.entities.data.DataProcessing; import fr.ifremer.echobase.entities.data.Echotype; import fr.ifremer.echobase.entities.data.Operation; import fr.ifremer.echobase.entities.data.Transect; import fr.ifremer.echobase.entities.data.Voyage; +import fr.ifremer.echobase.entities.meta.MetaFilenameAware; import fr.ifremer.echobase.entities.references.AcousticInstrument; import fr.ifremer.echobase.entities.references.AgeCategory; import fr.ifremer.echobase.entities.references.AreaOfOperation; @@ -303,6 +305,13 @@ } }; + public static final Function<MetaFilenameAware,EchoBaseEntityEnum> META_BY_SOURCE = new Function<MetaFilenameAware, EchoBaseEntityEnum>() { + @Override + public EchoBaseEntityEnum apply(MetaFilenameAware input) { + return input.getSource(); + } + }; + public static String getSpeciesCategoryKey(Species species, SizeCategory sizeCategory, SexCategory sexCategory) { String key = species.getBaracoudaCode(); if (sizeCategory != null) { Added: trunk/echobase-entities/src/main/java/fr/ifremer/echobase/csv/AssociationExportContext.java =================================================================== --- trunk/echobase-entities/src/main/java/fr/ifremer/echobase/csv/AssociationExportContext.java (rev 0) +++ trunk/echobase-entities/src/main/java/fr/ifremer/echobase/csv/AssociationExportContext.java 2012-03-19 15:55:36 UTC (rev 372) @@ -0,0 +1,110 @@ +/* + * #%L + * EchoBase :: Entities + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2011 - 2012 Ifremer, Codelutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ +package fr.ifremer.echobase.csv; + +import com.google.common.base.Charsets; +import com.google.common.base.Preconditions; +import fr.ifremer.echobase.entities.meta.AssociationMeta; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.topia.persistence.TopiaEntity; +import org.nuiton.util.csv.ExportModel; + +import java.io.Closeable; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.util.List; + +/** + * To export associations as csv files. + * + * @author tchemit <chemit(a)codelutin.com> + * @since 0.3 + */ +public class AssociationExportContext implements Closeable { + + /** Logger. */ + private static final Log log = + LogFactory.getLog(AssociationExportContext.class); + + /** association meta to export. */ + protected final AssociationMeta meta; + + /** Exporter object. */ + protected final EchoBaseExport export; + + /** Where to export datas. */ + protected final Writer writer; + + protected final File entryFile; + + public AssociationExportContext(ExportModel<TopiaEntity> model, + AssociationMeta meta, + File container, + List<TopiaEntity> data) throws FileNotFoundException { + Preconditions.checkNotNull(model); + Preconditions.checkNotNull(meta); + Preconditions.checkNotNull(container); + Preconditions.checkNotNull(data); + this.meta = meta; + + export = EchoBaseExport.newExport(model, data, true); + + entryFile = new File(container, meta.getFilename()); + if (log.isDebugEnabled()) { + log.debug("Creates AssociationExportContext::" + meta + " - " + entryFile.getName()); + } + writer = new OutputStreamWriter(new FileOutputStream(entryFile), + Charsets.UTF_8); + } + + @Override + public void close() throws IOException { + try { + if (export.isHeaderWritten()) { + + if (log.isInfoEnabled()) { + log.info("Export association " + meta + " to " + entryFile); + } + writer.flush(); + } else { + + // this file was not used, delete it + FileUtils.deleteQuietly(entryFile); + } + } finally { + IOUtils.closeQuietly(writer); + } + } + + public void write() throws Exception { + export.write(writer); + } +} Property changes on: trunk/echobase-entities/src/main/java/fr/ifremer/echobase/csv/AssociationExportContext.java ___________________________________________________________________ Added: svn:keywords + Author Date Id Revision HeadURL Added: svn:eol-style + native Added: trunk/echobase-entities/src/main/java/fr/ifremer/echobase/csv/EchoBaseExport.java =================================================================== --- trunk/echobase-entities/src/main/java/fr/ifremer/echobase/csv/EchoBaseExport.java (rev 0) +++ trunk/echobase-entities/src/main/java/fr/ifremer/echobase/csv/EchoBaseExport.java 2012-03-19 15:55:36 UTC (rev 372) @@ -0,0 +1,101 @@ +/* + * #%L + * EchoBase :: Entities + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2011 - 2012 Ifremer, Codelutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ +package fr.ifremer.echobase.csv; + +import org.nuiton.util.csv.Export; +import org.nuiton.util.csv.ExportModel; + +import java.io.File; +import java.io.IOException; +import java.io.Writer; +import java.nio.charset.Charset; + +/** + * Extends the {@link Export} classes to be able to generate only once + * the header. + * + * @author tchemit <chemit(a)codelutin.com> + * @since 0.3 + */ +public class EchoBaseExport<E> extends Export<E> { + + public static <E> EchoBaseExport<E> newExport(ExportModel<E> model, + Iterable<E> data, + boolean writeOnceHeader) { + return new EchoBaseExport<E>(model, data, writeOnceHeader); + } + + public static <E> void exportToWriter(ExportModel<E> model, + Iterable<E> data, + Writer writer, + boolean writeOnceHeader) throws Exception { + Export<E> exporter = newExport(model, data, writeOnceHeader); + exporter.write(writer); + } + + public static <E> void exportToFile(ExportModel<E> model, + Iterable<E> data, + File file, + Charset charset, + boolean writeOnceHeader) throws Exception { + Export<E> exporter = newExport(model, data, writeOnceHeader); + exporter.write(file, charset); + } + + public static <E> String exportToString(ExportModel<E> model, + Iterable<E> data, + Charset charset, + boolean writeOnceHeader) throws Exception { + Export<E> exporter = newExport(model, data, writeOnceHeader); + return exporter.toString(charset); + } + + protected final boolean writeOnceHeader; + + protected boolean headerWritten; + + public boolean isHeaderWritten() { + return headerWritten; + } + + protected EchoBaseExport(ExportModel<E> model, + Iterable<E> data, + boolean writeOnceHeader) { + super(model, data); + this.writeOnceHeader = writeOnceHeader; + } + + @Override + protected void writeHeader(Writer writer) throws IOException { + if (!writeOnceHeader || !headerWritten) { + + // no header generated, let's do it! + + super.writeHeader(writer); + + // mark it as written + headerWritten = true; + } + } +} Property changes on: trunk/echobase-entities/src/main/java/fr/ifremer/echobase/csv/EchoBaseExport.java ___________________________________________________________________ Added: svn:keywords + Author Date Id Revision HeadURL Added: svn:eol-style + native Copied: trunk/echobase-entities/src/main/java/fr/ifremer/echobase/csv/EchoBaseImport.java (from rev 369, trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/csv/EchoBaseImport.java) =================================================================== --- trunk/echobase-entities/src/main/java/fr/ifremer/echobase/csv/EchoBaseImport.java (rev 0) +++ trunk/echobase-entities/src/main/java/fr/ifremer/echobase/csv/EchoBaseImport.java 2012-03-19 15:55:36 UTC (rev 372) @@ -0,0 +1,137 @@ +/* + * #%L + * EchoBase :: Entities + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2011 - 2012 Ifremer, Codelutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ +package fr.ifremer.echobase.csv; + +import com.google.common.base.Charsets; +import org.nuiton.util.StringUtil; +import org.nuiton.util.csv.Import; +import org.nuiton.util.csv.ImportModel; +import org.nuiton.util.csv.ImportRuntimeException; +import org.nuiton.util.csv.ImportableColumn; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; + +import static org.nuiton.i18n.I18n._; + +/** + * This extends of the {@link Import} offers to allow case on header names. + * + * @author tchemit <chemit(a)codelutin.com> + * @since 0.3 + */ +public class EchoBaseImport<E> extends Import<E> { + + public static String[] getHeader(File file, char charSeprator) { + + try { + BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), Charsets.UTF_8.name())); + try { + String header = reader.readLine(); + String[] result = header.split(charSeprator + ""); + return result; + } finally { + reader.close(); + } + } catch (IOException e) { + throw new ImportRuntimeException("Could not obtain header of file " + file, e); + } + } + + public static <E> EchoBaseImport<E> newImport(ImportModel<E> model, + InputStream inputStream) { + return new EchoBaseImport<E>(model, inputStream); + } + + public static <E> EchoBaseImport<E> newImport(ImportModel<E> model, + Reader reader) { + return new EchoBaseImport<E>(model, reader); + } + + public EchoBaseImport(ImportModel<E> importModel, InputStream inputStream) { + super(importModel, inputStream); + } + + public EchoBaseImport(ImportModel<E> importModel, Reader reader) { + super(importModel, reader); + } + + @Override + protected void checkAllMandatoryHeadersArePresent(String[] headers) { + + List<String> csvHeaders = new ArrayList<String>(); + for (String header : headers) { + csvHeaders.add(header.toLowerCase()); + } +// Collections.addAll(csvHeaders, headers); + + List<String> mandatoryHeadersNames = new ArrayList<String>(); + for (ImportableColumn<E, ?> field : getAllMandatoryHeaders()) { +// mandatoryHeadersNames.add(field.getHeaderName()); + mandatoryHeadersNames.add(field.getHeaderName().toLowerCase()); + } + mandatoryHeadersNames.removeAll(csvHeaders); + + if (!mandatoryHeadersNames.isEmpty()) { + String validationMessage = + _("csv.import.error.missingMandatoryHeaders", + StringUtil.join(mandatoryHeadersNames, ", ", true)); + throw new ImportRuntimeException(validationMessage); + } + } + + @Override + protected void checkHeaderNamesAreAllKnown(String[] headers) { + List<String> csvHeaders = new ArrayList<String>(); + for (String header : headers) { + csvHeaders.add(header.toLowerCase()); + } +// Collections.addAll(csvHeaders, headers); + + for (ImportableColumn<E, ?> field : model.getColumnsForImport()) { + csvHeaders.remove(field.getHeaderName().toLowerCase()); +// csvHeaders.remove(field.getHeaderName()); + } + if (!csvHeaders.isEmpty()) { + List<String> validHeaderNames = new LinkedList<String>(); + for (ImportableColumn<E, ?> importableColumn : + model.getColumnsForImport()) { + validHeaderNames.add(importableColumn.getHeaderName()); + } + String validationMessage = + _("csv.import.error.unrecognizedHeaders", + StringUtil.join(csvHeaders, ", ", true), + StringUtil.join(validHeaderNames, ", ", true)); + throw new ImportRuntimeException(validationMessage); + } + } +} Property changes on: trunk/echobase-entities/src/main/java/fr/ifremer/echobase/csv/EchoBaseImport.java ___________________________________________________________________ Added: svn:keywords + Author Date Id Revision HeadURL Added: svn:eol-style + native Added: trunk/echobase-entities/src/main/java/fr/ifremer/echobase/csv/EntityExportContext.java =================================================================== --- trunk/echobase-entities/src/main/java/fr/ifremer/echobase/csv/EntityExportContext.java (rev 0) +++ trunk/echobase-entities/src/main/java/fr/ifremer/echobase/csv/EntityExportContext.java 2012-03-19 15:55:36 UTC (rev 372) @@ -0,0 +1,143 @@ +/* + * #%L + * EchoBase :: Entities + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2011 - 2012 Ifremer, Codelutin + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ +package fr.ifremer.echobase.csv; + +import com.google.common.base.Charsets; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import fr.ifremer.echobase.entities.meta.AssociationMeta; +import fr.ifremer.echobase.entities.meta.TableMeta; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.nuiton.topia.persistence.TopiaEntity; +import org.nuiton.util.csv.ExportModel; + +import java.io.Closeable; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.util.Collection; +import java.util.List; + +/** + * to export entity as csv files. + * + * @author tchemit <chemit(a)codelutin.com> + * @since 0.3 + */ +public class EntityExportContext implements Closeable { + + /** Logger. */ + private static final Log log = LogFactory.getLog(EntityExportContext.class); + + /** meta to export. */ + protected final TableMeta meta; + + /** Exporter object. */ + protected final EchoBaseExport export; + + /** Where to export datas. */ + protected final Writer writer; + + /** + * Unique list to store data to export. (will be shared with + * association export context.) + */ + protected final List<TopiaEntity> data; + + /** Association export context for this type of entity. */ + protected final Collection<AssociationExportContext> associationExportContexts; + + protected final File entryFile; + + public EntityExportContext(ExportModel<TopiaEntity> model, + TableMeta meta, + File container) throws FileNotFoundException { + Preconditions.checkNotNull(model); + Preconditions.checkNotNull(meta); + Preconditions.checkNotNull(container); + + this.meta = meta; + export = EchoBaseExport.newExport(model, data = Lists.newArrayList(), true); + entryFile = new File(container, meta.getFilename()); + if (log.isDebugEnabled()) { + log.debug("Creates EntityExportContext::" + meta + " - " + entryFile.getName()); + } + writer = new OutputStreamWriter(new FileOutputStream(entryFile), Charsets.UTF_8); + associationExportContexts = Lists.newArrayList(); + } + + public void addAssociationExportContext(AssociationMeta meta, + ExportModel<TopiaEntity> model, + File container) throws FileNotFoundException { + associationExportContexts.add( + new AssociationExportContext(model, + meta, + container, + data) + ); + } + + public void setData(TopiaEntity entity) { + data.clear(); + data.add(entity); + } + + @Override + public void close() throws IOException { + + try { + if (export.isHeaderWritten()) { + + if (log.isInfoEnabled()) { + log.info("Export table " + meta + " to " + entryFile); + } + writer.flush(); + } else { + // this file was not used, delete it + FileUtils.deleteQuietly(entryFile); + } + } finally { + IOUtils.closeQuietly(writer); + for (AssociationExportContext c : associationExportContexts) { + c.close(); + } + } + } + + public void write() throws Exception { + export.write(writer); + } + + public void writeAssociations() throws Exception { + for (AssociationExportContext c : associationExportContexts) { + c.write(); + } + } +} Property changes on: trunk/echobase-entities/src/main/java/fr/ifremer/echobase/csv/EntityExportContext.java ___________________________________________________________________ Added: svn:keywords + Author Date Id Revision HeadURL Added: svn:eol-style + native Modified: trunk/echobase-entities/src/main/java/fr/ifremer/echobase/entities/EntitiesUtil.java =================================================================== --- trunk/echobase-entities/src/main/java/fr/ifremer/echobase/entities/EntitiesUtil.java 2012-03-19 15:10:55 UTC (rev 371) +++ trunk/echobase-entities/src/main/java/fr/ifremer/echobase/entities/EntitiesUtil.java 2012-03-19 15:55:36 UTC (rev 372) @@ -24,6 +24,9 @@ package fr.ifremer.echobase.entities; import com.google.common.collect.Lists; +import com.google.common.collect.Multimap; +import com.google.common.collect.Multimaps; +import fr.ifremer.echobase.EchoBaseFunctions; import fr.ifremer.echobase.entities.meta.DbMeta; import fr.ifremer.echobase.entities.meta.MetaFilenameAware; import fr.ifremer.echobase.entities.meta.TableMeta; @@ -46,39 +49,7 @@ List<EchoBaseEntityEnum> result = Lists.newLinkedList(); - result.add(EchoBaseEntityEnum.Voyage); - result.add(EchoBaseEntityEnum.Transit); - result.add(EchoBaseEntityEnum.Transect); - result.add(EchoBaseEntityEnum.DataAcquisition); - result.add(EchoBaseEntityEnum.DataProcessing); - result.add(EchoBaseEntityEnum.Cell); - result.add(EchoBaseEntityEnum.Data); - - result.add(EchoBaseEntityEnum.Operation); - result.add(EchoBaseEntityEnum.OperationMetadataValue); - result.add(EchoBaseEntityEnum.GearMetadataValue); - result.add(EchoBaseEntityEnum.Sample); result.add(EchoBaseEntityEnum.SampleData); - - result.add(EchoBaseEntityEnum.Echotype); - result.add(EchoBaseEntityEnum.LengthAgeKey); - result.add(EchoBaseEntityEnum.LengthWeightKey); - result.add(EchoBaseEntityEnum.Category); - - result.add(EchoBaseEntityEnum.Result); - - return result.toArray(new EchoBaseEntityEnum[result.size()]); - } - - /** - * @return the entity types of data in correct order for replication - * purpose. - */ - public static EchoBaseEntityEnum[] getDataTypesForCopy() { - - List<EchoBaseEntityEnum> result = Lists.newLinkedList(); - - result.add(EchoBaseEntityEnum.SampleData); result.add(EchoBaseEntityEnum.Sample); result.add(EchoBaseEntityEnum.OperationMetadataValue); @@ -97,7 +68,6 @@ result.add(EchoBaseEntityEnum.Transect); result.add(EchoBaseEntityEnum.Transit); - result.add(EchoBaseEntityEnum.LengthAgeKey); result.add(EchoBaseEntityEnum.LengthWeightKey); @@ -107,57 +77,10 @@ } /** - * @return the entity types of reference in correct order for replication - * purpose. - */ - public static EchoBaseEntityEnum[] getReferenceTypes() { - List<EchoBaseEntityEnum> result = Lists.newLinkedList(); - result.add(EchoBaseEntityEnum.SexCategory); - result.add(EchoBaseEntityEnum.AgeCategory); - result.add(EchoBaseEntityEnum.SizeCategory); - result.add(EchoBaseEntityEnum.MeasureType); - result.add(EchoBaseEntityEnum.DataType); - result.add(EchoBaseEntityEnum.ReferenceDatumType); - result.add(EchoBaseEntityEnum.VesselType); - result.add(EchoBaseEntityEnum.EchotypeCategory); - result.add(EchoBaseEntityEnum.DepthStratum); - result.add(EchoBaseEntityEnum.Strata); - result.add(EchoBaseEntityEnum.CellMethod); - result.add(EchoBaseEntityEnum.AcousticInstrument); - result.add(EchoBaseEntityEnum.CategoryType); - result.add(EchoBaseEntityEnum.CategoryMeaning); - result.add(EchoBaseEntityEnum.Species); - result.add(EchoBaseEntityEnum.GearCharacteristic); - result.add(EchoBaseEntityEnum.ReferenceDatum); - result.add(EchoBaseEntityEnum.ReferencingMethod); - result.add(EchoBaseEntityEnum.GearCharacteristicValue); - result.add(EchoBaseEntityEnum.GearMetadata); - result.add(EchoBaseEntityEnum.Calibration); - result.add(EchoBaseEntityEnum.MeasurementMetadata); - result.add(EchoBaseEntityEnum.SampleDataType); - result.add(EchoBaseEntityEnum.SampleType); - result.add(EchoBaseEntityEnum.Vessel); - result.add(EchoBaseEntityEnum.DataMetadata); - result.add(EchoBaseEntityEnum.CellType); - result.add(EchoBaseEntityEnum.Impacte); - result.add(EchoBaseEntityEnum.Mission); - result.add(EchoBaseEntityEnum.OperationEvent); - result.add(EchoBaseEntityEnum.OperationMetadata); - result.add(EchoBaseEntityEnum.AreaOfOperation); - result.add(EchoBaseEntityEnum.CategoryRef); - result.add(EchoBaseEntityEnum.DataQuality); - result.add(EchoBaseEntityEnum.TSParameters); - result.add(EchoBaseEntityEnum.Gear); - result.add(EchoBaseEntityEnum.DataProtocol); - result.add(EchoBaseEntityEnum.SpeciesCategory); - return result.toArray(new EchoBaseEntityEnum[result.size()]); - } - - /** * @return the entity types of reference in correct order for copy * purpose (says import / export of all a database. */ - public static EchoBaseEntityEnum[] getReferenceTypesForCopy() { + public static EchoBaseEntityEnum[] getReferenceTypes() { List<EchoBaseEntityEnum> result = Lists.newLinkedList(); // with no dep @@ -210,12 +133,40 @@ List<MetaFilenameAware> entities = Lists.newArrayList(); List<MetaFilenameAware> associations = Lists.newArrayList(); - addEntries(dbMeta, entities, associations, getReferenceTypesForCopy()); - addEntries(dbMeta, entities, associations, getDataTypesForCopy()); + addEntries(dbMeta, entities, associations, getReferenceTypes()); + addEntries(dbMeta, entities, associations, getDataTypes()); entities.addAll(associations); return entities.toArray(new MetaFilenameAware[entities.size()]); } + public static MetaFilenameAware[] getReferenceEntries(DbMeta dbMeta) { + + List<MetaFilenameAware> entities = Lists.newArrayList(); + List<MetaFilenameAware> associations = Lists.newArrayList(); + + addEntries(dbMeta, entities, associations, getReferenceTypes()); + entities.addAll(associations); + return entities.toArray(new MetaFilenameAware[entities.size()]); + } + + public static MetaFilenameAware[] getDataEntries(DbMeta dbMeta) { + + List<MetaFilenameAware> entities = Lists.newArrayList(); + + addEntries(dbMeta, entities, null, getDataTypes()); + return entities.toArray(new MetaFilenameAware[entities.size()]); + } + + public static Multimap<EchoBaseEntityEnum, MetaFilenameAware> getDataAssociationsEntries(DbMeta dbMeta) { + + List<MetaFilenameAware> associations = Lists.newArrayList(); + + addEntries(dbMeta, null, associations, getDataTypes()); + + Multimap<EchoBaseEntityEnum, MetaFilenameAware> associationsBySource = Multimaps.index(associations, EchoBaseFunctions.META_BY_SOURCE); + return associationsBySource; + } + protected static void addEntries(DbMeta dbMeta, List<MetaFilenameAware> entities, List<MetaFilenameAware> associations, @@ -223,8 +174,12 @@ for (EchoBaseEntityEnum type : types) { TableMeta tableMeta = getTableMeta(dbMeta, type); - entities.add(tableMeta); - associations.addAll(tableMeta.getAssociations()); + if (entities != null) { + entities.add(tableMeta); + } + if (associations != null) { + associations.addAll(tableMeta.getAssociations()); + } } } Modified: trunk/echobase-entities/src/main/resources/i18n/echobase-entities_en_GB.properties =================================================================== --- trunk/echobase-entities/src/main/resources/i18n/echobase-entities_en_GB.properties 2012-03-19 15:10:55 UTC (rev 371) +++ trunk/echobase-entities/src/main/resources/i18n/echobase-entities_en_GB.properties 2012-03-19 15:55:36 UTC (rev 372) @@ -1,3 +1,5 @@ +csv.import.error.missingMandatoryHeaders= +csv.import.error.unrecognizedHeaders= echobase.common.AParameter= echobase.common.BParameter= echobase.common.TSParameters= Modified: trunk/echobase-entities/src/main/resources/i18n/echobase-entities_fr_FR.properties =================================================================== --- trunk/echobase-entities/src/main/resources/i18n/echobase-entities_fr_FR.properties 2012-03-19 15:10:55 UTC (rev 371) +++ trunk/echobase-entities/src/main/resources/i18n/echobase-entities_fr_FR.properties 2012-03-19 15:55:36 UTC (rev 372) @@ -1,3 +1,5 @@ +csv.import.error.missingMandatoryHeaders= +csv.import.error.unrecognizedHeaders= echobase.common.AParameter= echobase.common.BParameter= echobase.common.TSParameters= Modified: trunk/echobase-entities/src/main/resources/topia-h2.properties =================================================================== --- trunk/echobase-entities/src/main/resources/topia-h2.properties 2012-03-19 15:10:55 UTC (rev 371) +++ trunk/echobase-entities/src/main/resources/topia-h2.properties 2012-03-19 15:55:36 UTC (rev 372) @@ -24,9 +24,6 @@ hibernate.hbm2ddl.auto=update hibernate.show_sql=false -# Replication configuration -topia.service.replication=org.nuiton.topia.replication.TopiaReplicationServiceImpl - hibernate.dialect=org.hibernate.dialect.H2Dialect hibernate.connection.username=sa hibernate.connection.password=sa Modified: trunk/echobase-entities/src/test/java/fr/ifremer/echobase/entities/EntitiesUtilTest.java =================================================================== --- trunk/echobase-entities/src/test/java/fr/ifremer/echobase/entities/EntitiesUtilTest.java 2012-03-19 15:10:55 UTC (rev 371) +++ trunk/echobase-entities/src/test/java/fr/ifremer/echobase/entities/EntitiesUtilTest.java 2012-03-19 15:55:36 UTC (rev 372) @@ -24,25 +24,15 @@ package fr.ifremer.echobase.entities; import com.google.common.collect.Lists; -import com.google.common.collect.Sets; import fr.ifremer.echobase.entities.data.Voyage; import fr.ifremer.echobase.entities.references.Mission; import junit.framework.Assert; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.junit.Test; -import org.nuiton.topia.TopiaException; import org.nuiton.topia.persistence.TopiaEntity; -import org.nuiton.topia.persistence.TopiaEntityEnum; -import org.nuiton.topia.replication.TopiaReplicationModelBuilder; -import org.nuiton.topia.replication.model.ReplicationModel; -import org.nuiton.topia.replication.model.ReplicationNode; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.List; -import java.util.Set; /** * To test the class {@link EntitiesUtil}. @@ -52,9 +42,6 @@ */ public class EntitiesUtilTest { - /** Logger. */ - private static final Log log = LogFactory.getLog(EntitiesUtilTest.class); - @Test public void getReferenceTypes() { @@ -73,23 +60,6 @@ } @Test - public void getReferenceTypesForCopy() { - - List<EchoBaseEntityEnum> types = - Arrays.asList(EntitiesUtil.getReferenceTypesForCopy()); - - List<EchoBaseEntityEnum> expectedTypes = - Lists.newArrayList(getContractsOf(Mission.class.getPackage())); - - Assert.assertNotNull(types); - - for (EchoBaseEntityEnum type : expectedTypes) { - Assert.assertTrue("Missing type : " + type, types.contains(type)); - } - Assert.assertEquals(expectedTypes.size(), types.size()); - } - - @Test public void getDataTypes() { List<EchoBaseEntityEnum> types = @@ -97,24 +67,7 @@ List<EchoBaseEntityEnum> expectedTypes = Lists.newArrayList(getContractsOf(Voyage.class.getPackage())); - Assert.assertNotNull(types); - for (EchoBaseEntityEnum type : expectedTypes) { - Assert.assertTrue("Missing type : " + type, types.contains(type)); - } - Assert.assertEquals(expectedTypes.size(), types.size()); - } - - - @Test - public void getDataTypesForCopy() { - - List<EchoBaseEntityEnum> types = - Arrays.asList(EntitiesUtil.getDataTypesForCopy()); - - List<EchoBaseEntityEnum> expectedTypes = - Lists.newArrayList(getContractsOf(Voyage.class.getPackage())); - Assert.assertNotNull(types); for (EchoBaseEntityEnum type : expectedTypes) { @@ -123,90 +76,6 @@ Assert.assertEquals(expectedTypes.size(), types.size()); } - @Test - public void testGetReferentielCopyModel() throws TopiaException { - - EchoBaseEntityEnum[] types = EntitiesUtil.getReferenceTypesForCopy(); - Set<TopiaEntityEnum> dones = Sets.newHashSet(); - - ReplicationModel model = new ReplicationModel(types, false); - new TopiaReplicationModelBuilder().initModel(model, false); - List<ReplicationNode> order = model.getOrder(); - - StringBuilder buffer = new StringBuilder(); - for (ReplicationNode node : order) { - TopiaEntityEnum type = node.getContract(); - Assert.assertTrue(type + " was already treated.", dones.add(type)); - Collection<ReplicationNode> dependencies = node.getDependencies().values(); - for (ReplicationNode dependency : node.getDependencies().values()) { - TopiaEntityEnum dependencyType = dependency.getContract(); - Assert.assertTrue(dependencyType + - " was detected as a dependency of node " + - type + " but was not still registred", - dones.contains(dependencyType)); - } - Collection<ReplicationNode> assocations = node.getAssociations().values(); - for (ReplicationNode assocation : assocations) { - TopiaEntityEnum assocationType = assocation.getContract(); - Assert.assertTrue(assocationType + - " was detected as a association of node " + - type + " but was not still registred", - dones.contains(assocationType)); - } - buffer.append('\n'); - buffer.append(type); - buffer.append(" - "); - buffer.append(dependencies); - buffer.append(" - "); - buffer.append(assocations); - } - if (log.isInfoEnabled()) { - log.info(buffer.toString()); - } - } - - @Test - public void testGetDataCopyModel() throws TopiaException { - - EchoBaseEntityEnum[] types = EntitiesUtil.getDataTypesForCopy(); - Set<TopiaEntityEnum> dones = Sets.newHashSet(); - - ReplicationModel model = new ReplicationModel(types, false); - new TopiaReplicationModelBuilder().initModel(model, false); - List<ReplicationNode> order = model.getOrder(); - - StringBuilder buffer = new StringBuilder(); - for (ReplicationNode node : order) { - TopiaEntityEnum type = node.getContract(); - Assert.assertTrue(type + " was already treated.", dones.add(type)); - Collection<ReplicationNode> dependencies = node.getDependencies().values(); - for (ReplicationNode dependency : node.getDependencies().values()) { - TopiaEntityEnum dependencyType = dependency.getContract(); - Assert.assertTrue(dependencyType + - " was detected as a dependency of node " + - type + " but was not still registred", - dones.contains(dependencyType)); - } - Collection<ReplicationNode> assocations = node.getAssociations().values(); - for (ReplicationNode assocation : assocations) { - TopiaEntityEnum assocationType = assocation.getContract(); - Assert.assertTrue(assocationType + - " was detected as a association of node " + - type + " but was not still registred", - dones.contains(assocationType)); - } - buffer.append('\n'); - buffer.append(type); - buffer.append(" - "); - buffer.append(dependencies); - buffer.append(" - "); - buffer.append(assocations); - } - if (log.isInfoEnabled()) { - log.info(buffer.toString()); - } - } - /** * Used to get all contract of a package *

1 0

r371 - in trunk: . echobase-entities
by tchemit＠users.forge.codelutin.com 19 Mar '12

19 Mar '12

Author: tchemit Date: 2012-03-19 16:10:55 +0100 (Mon, 19 Mar 2012) New Revision: 371 Url: http://forge.codelutin.com/repositories/revision/echobase/371 Log: use last snpashot of nuiton-utils + nuiton-csv in entities module Modified: trunk/echobase-entities/pom.xml trunk/pom.xml Modified: trunk/echobase-entities/pom.xml =================================================================== --- trunk/echobase-entities/pom.xml 2012-03-19 07:18:36 UTC (rev 370) +++ trunk/echobase-entities/pom.xml 2012-03-19 15:10:55 UTC (rev 371) @@ -29,6 +29,11 @@ <artifactId>nuiton-utils</artifactId> </dependency> + <dependency> + <groupId>org.nuiton</groupId> + <artifactId>nuiton-csv</artifactId> + </dependency> + <!--dependency> <groupId>fr.ird</groupId> <artifactId>msaccess-importer</artifactId> Modified: trunk/pom.xml =================================================================== --- trunk/pom.xml 2012-03-19 07:18:36 UTC (rev 370) +++ trunk/pom.xml 2012-03-19 15:10:55 UTC (rev 371) @@ -125,7 +125,7 @@ <eugenePluginVersion>2.4.2</eugenePluginVersion> <topiaVersion>2.6.7</topiaVersion> - <nuitonUtilsVersion>2.4.4</nuitonUtilsVersion> + <nuitonUtilsVersion>2.4.6-SNAPSHOT</nuitonUtilsVersion> <nuitonI18nVersion>2.4.1</nuitonI18nVersion> <nuitonWebVersion>1.10</nuitonWebVersion> <msaccessImporterVersion>1.4.1</msaccessImporterVersion>

1 0