Author: tchemit Date: 2012-08-22 18:04:01 +0200 (Wed, 22 Aug 2012) New Revision: 2641 Url: http://nuiton.org/repositories/revision/topia/2641 Log: fixes 2266: Add some api about import / export in csv format (add some doc and fix api) Modified: branches/topia-2.6.x/topia-persistence/src/main/java/org/nuiton/topia/persistence/TopiaPersistenceHelper.java branches/topia-2.6.x/topia-persistence/src/main/java/org/nuiton/topia/persistence/csv/in/AbstractImportModel.java branches/topia-2.6.x/topia-persistence/src/main/java/org/nuiton/topia/persistence/csv/in/EntityAssociationImportModel.java branches/topia-2.6.x/topia-persistence/src/main/java/org/nuiton/topia/persistence/csv/in/ImportStrategy.java branches/topia-2.6.x/topia-persistence/src/main/java/org/nuiton/topia/persistence/csv/in/TopiaCsvImports.java Modified: branches/topia-2.6.x/topia-persistence/src/main/java/org/nuiton/topia/persistence/TopiaPersistenceHelper.java =================================================================== --- branches/topia-2.6.x/topia-persistence/src/main/java/org/nuiton/topia/persistence/TopiaPersistenceHelper.java 2012-08-22 10:47:37 UTC (rev 2640) +++ branches/topia-2.6.x/topia-persistence/src/main/java/org/nuiton/topia/persistence/TopiaPersistenceHelper.java 2012-08-22 16:04:01 UTC (rev 2641) @@ -38,4 +38,6 @@ <E extends TopiaEntity> TopiaDAO<E> getDAO(TopiaContext tx, Class<E> type); + <E extends TopiaEntity> TopiaDAO<E> getDAO(TopiaContext tx, T type); + } Modified: branches/topia-2.6.x/topia-persistence/src/main/java/org/nuiton/topia/persistence/csv/in/AbstractImportModel.java =================================================================== --- branches/topia-2.6.x/topia-persistence/src/main/java/org/nuiton/topia/persistence/csv/in/AbstractImportModel.java 2012-08-22 10:47:37 UTC (rev 2640) +++ branches/topia-2.6.x/topia-persistence/src/main/java/org/nuiton/topia/persistence/csv/in/AbstractImportModel.java 2012-08-22 16:04:01 UTC (rev 2641) @@ -30,6 +30,13 @@ import java.util.List; import java.util.Map; +/** + * Abstract import model which add the useful methdo about importing foreign keys. + * + * @param <E> type of entity to import + * @author tchemit <chemit@codelutin.com> + * @since 2.6.12 + */ public abstract class AbstractImportModel<E> extends org.nuiton.util.csv.ext.AbstractImportModel<E> { public AbstractImportModel(char separator) { Modified: branches/topia-2.6.x/topia-persistence/src/main/java/org/nuiton/topia/persistence/csv/in/EntityAssociationImportModel.java =================================================================== --- branches/topia-2.6.x/topia-persistence/src/main/java/org/nuiton/topia/persistence/csv/in/EntityAssociationImportModel.java 2012-08-22 10:47:37 UTC (rev 2640) +++ branches/topia-2.6.x/topia-persistence/src/main/java/org/nuiton/topia/persistence/csv/in/EntityAssociationImportModel.java 2012-08-22 16:04:01 UTC (rev 2641) @@ -32,7 +32,7 @@ import java.util.Map; /** - * A model to import associations of entities into csv files. + * A model to import associations of entities from csv files. * * @author tchemit <chemit@codelutin.com> * @since 2.6.12 Modified: branches/topia-2.6.x/topia-persistence/src/main/java/org/nuiton/topia/persistence/csv/in/ImportStrategy.java =================================================================== --- branches/topia-2.6.x/topia-persistence/src/main/java/org/nuiton/topia/persistence/csv/in/ImportStrategy.java 2012-08-22 10:47:37 UTC (rev 2640) +++ branches/topia-2.6.x/topia-persistence/src/main/java/org/nuiton/topia/persistence/csv/in/ImportStrategy.java 2012-08-22 16:04:01 UTC (rev 2641) @@ -11,7 +11,11 @@ import java.io.Reader; /** - * TODO + * Strategy to import some stuff. + * <p/> + * Implements it and then you can use it with helper methods + * {@link TopiaCsvImports#importTable(Reader, ImportStrategy, TableMeta, CsvImportResult)}, + * {@link TopiaCsvImports#importAssociation(Reader, ImportStrategy, AssociationMeta, CsvImportResult)}. * * @author tchemit <chemit@codelutin.com> * @since 2.6.12 @@ -20,17 +24,14 @@ ImportModelFactory<T> getModelFactory(); - <E extends TopiaEntity> void importTableFile(Import<E> importer, - TableMeta<T> meta, - Reader reader, - CsvImportResult<T> csvResult) throws TopiaException; + <E extends TopiaEntity> void importTable(Import<E> importer, + TableMeta<T> meta, + Reader reader, + CsvImportResult<T> csvResult) throws TopiaException; - <T extends TopiaEntityEnum> void importAssociationFile(AssociationMeta<T> meta, - ImportToMap importer, - CsvImportResult<T> csvResult) throws TopiaException; + void importAssociation(AssociationMeta<T> meta, + ImportToMap importer, + CsvImportResult<T> csvResult) throws TopiaException; - void importNMAssociationFile(AssociationMeta<T> meta, - ImportToMap importer, - CsvImportResult<T> csvResult) throws TopiaException; } Modified: branches/topia-2.6.x/topia-persistence/src/main/java/org/nuiton/topia/persistence/csv/in/TopiaCsvImports.java =================================================================== --- branches/topia-2.6.x/topia-persistence/src/main/java/org/nuiton/topia/persistence/csv/in/TopiaCsvImports.java 2012-08-22 10:47:37 UTC (rev 2640) +++ branches/topia-2.6.x/topia-persistence/src/main/java/org/nuiton/topia/persistence/csv/in/TopiaCsvImports.java 2012-08-22 16:04:01 UTC (rev 2641) @@ -29,7 +29,7 @@ import org.apache.commons.logging.LogFactory; import org.nuiton.topia.TopiaContext; import org.nuiton.topia.TopiaException; -import org.nuiton.topia.framework.TopiaContextImplementor; +import org.nuiton.topia.persistence.TopiaDAO; import org.nuiton.topia.persistence.TopiaEntity; import org.nuiton.topia.persistence.TopiaEntityEnum; import org.nuiton.topia.persistence.metadata.AssociationMeta; @@ -43,6 +43,7 @@ import java.io.File; import java.io.IOException; import java.io.Reader; +import java.util.Date; import java.util.List; import java.util.Map; import java.util.zip.ZipEntry; @@ -59,10 +60,95 @@ /** Logger. */ private static final Log log = LogFactory.getLog(TopiaCsvImports.class); + protected static final String UPDATE_ASSOCIATION = + "UPDATE %s SET %s = '%%s' WHERE topiaId ='%%s';"; + + protected static final String INSERT_ASSOCIATION = + "INSERT INTO %s (%s,%s) VALUES('%%s','%%s');"; + protected TopiaCsvImports() { // no instance of this helper } + /** + * Discover all files that can be imported (as a table or an association) from a zipfile. + * + * @param entryPrefix prefix where to find files in the zip + * @param possibleMetas list of possible meta to be imported + * @param zipFile zip file where to seek for csv files to import + * @param missingEntries to fill missing files + * @param <T> type of topia entity enum + * @param <M> type of data to import (table or association) + * @return the map of found files indexed by his meta + */ + public static <T extends TopiaEntityEnum, M extends MetaFilenameAware<T>> Map<M, ZipEntry> discoverEntries( + String entryPrefix, + Iterable<M> possibleMetas, + ZipFile zipFile, + List<String> missingEntries) { + + Map<M, ZipEntry> result = Maps.newLinkedHashMap(); + + // check that all mandatories + for (M entry : possibleMetas) { + String filename = entry.getFilename(); + ZipEntry zipEntry = zipFile.getEntry(entryPrefix + filename); + + if (zipEntry == null) { + missingEntries.add(filename); + } else { + result.put(entry, zipEntry); + } + } + return result; + } + + /** + * Discover all files that can be imported (as a table or an association) from a directory. + * + * @param possibleMetas list of possible meta to be imported + * @param directory directory where to seek for csv files to import + * @param missingEntries to fill missing files + * @param <T> type of topia entity enum + * @param <M> type of data to import (table or association) + * @return the map of found files indexed by his meta + */ + public static <T extends TopiaEntityEnum, M extends MetaFilenameAware<T>> Map<M, File> discoverEntries( + Iterable<M> possibleMetas, + File directory, + List<String> missingEntries) { + + Map<M, File> result = Maps.newLinkedHashMap(); + + // check that all mandatories + for (M entry : possibleMetas) { + String filename = entry.getFilename(); + File zipEntry = new File(directory, filename); + + if (zipEntry.exists()) { + result.put(entry, zipEntry); + } else { + missingEntries.add(filename); + } + } + return result; + } + + /** + * To import a table (given by his {@code meta}) from a reader and a strategy. + * <p/> + * Result of import can be stored in an optional csv result. + * + * @param reader where to read csv data + * @param importStrategy import strategy used to store csv data + * @param meta meta of the data + * @param csvResult optional csv result + * @param <T> type of entity enum + * @param <E> type of data + * @throws TopiaException if any db problem while storing datas + * @throws IOException if any pb while reading csv data + * @see ImportStrategy#importTable(Import, TableMeta, Reader, CsvImportResult) + */ public static <T extends TopiaEntityEnum, E extends TopiaEntity> void importTable(Reader reader, ImportStrategy<T> importStrategy, TableMeta<T> meta, @@ -75,20 +161,38 @@ ImportModel<E> model = importStrategy.getModelFactory().buildForImport(meta); Import<E> importer = Import.newImport(model, reader); - - importStrategy.importTableFile(importer, + try { + importStrategy.importTable(importer, meta, reader, csvResult); - - + } finally { + importer.close(); + } } - public static <T extends TopiaEntityEnum> void importAssociationFile(Reader reader, - ImportStrategy<T> importStrategy, - AssociationMeta<T> meta, - CsvImportResult<T> csvResult) throws IOException, TopiaException { + /** + * To import a association (given by his {@code meta}) from a reader and a strategy. + * <p/> + * Result of import can be stored in an optional csv result. + * + * @param reader where to read csv data + * @param importStrategy import strategy used to store csv data + * @param meta meta of the data + * @param csvResult optional csv result + * @param <T> type of entity enum + * @throws TopiaException if any db problem while storing datas + * @throws IOException if any pb while reading csv data + * @see ImportStrategy#importTable(Import, TableMeta, Reader, CsvImportResult) + */ + public static <T extends TopiaEntityEnum> void importAssociation(Reader reader, + ImportStrategy<T> importStrategy, + AssociationMeta<T> meta, + CsvImportResult<T> csvResult) throws IOException, TopiaException { + if (log.isInfoEnabled()) { + log.info("Will import " + meta); + } // load a association input ImportModelFactory<T> modelFactory = importStrategy.getModelFactory(); @@ -97,121 +201,80 @@ ImportToMap importer = ImportToMap.newImportToMap(model, reader, false); try { - if (modelFactory.isNMAssociationMeta(meta)) { - importStrategy.importAssociationFile(meta, importer, csvResult); - } else { - importStrategy.importNMAssociationFile(meta, importer, csvResult); - } + importStrategy.importAssociation(meta, importer, csvResult); + } finally { importer.close(); } } -// public static <T extends TopiaEntityEnum> void importTableFile(TopiaContextImplementor tx, -// ImportModelFactory<T> modelFactory, -// TableMeta<T> meta, -// Reader reader, -// CsvImportResult<T> csvResult) throws IOException, TopiaException { -// -// ImportModel<TopiaEntity> model = modelFactory.buildForImport(meta); -// Import<TopiaEntity> importer = Import.newImport(model, reader); -// -// try { -// TopiaDAO<TopiaEntity> dao = (TopiaDAO<TopiaEntity>) tx.getDAO(meta.getSource().getContract()); -// importEntityFile(dao, meta, importer, csvResult); -// } finally { -// importer.close(); -// } -// } + public static <T extends TopiaEntityEnum, E extends TopiaEntity> void importAllEntities(TopiaDAO<E> dao, + TableMeta<T> meta, + Import<E> importer, + CsvImportResult<T> csvResult) throws TopiaException { - public static <T extends TopiaEntityEnum> void importAssociationFile(TopiaContextImplementor tx, - ImportModelFactory<T> modelFactory, - AssociationMeta<T> meta, - Reader reader, - CsvImportResult<T> csvResult) throws IOException, TopiaException { + for (TopiaEntity entity : importer) { + Map<String, Object> properties = meta.prepareCreate( + entity, entity.getTopiaId()); + E entityToSave = dao.create(properties); - // load a association input - ImportModel<Map<String, Object>> model = - modelFactory.buildForImport(meta); - ImportToMap importer = ImportToMap.newImportToMap(model, reader, false); + meta.copy(entity, entityToSave); - try { - if (modelFactory.isNMAssociationMeta(meta)) { - importNMAssociationFile(tx, meta, importer, csvResult, 1000); - } else { - importAssociationFile(tx, meta, importer, csvResult, 1000); + if (csvResult != null) { + csvResult.incrementsNumberUpdated(); } - } finally { - importer.close(); } } - public static <T extends TopiaEntityEnum, M extends MetaFilenameAware<T>> Map<M, ZipEntry> discoverEntries( - String entryPrefix, - Iterable<M> entries, - ZipFile zipFile, - List<String> missingEntries) { + public static <T extends TopiaEntityEnum, E extends TopiaEntity> void importNotExistingEntities(TopiaDAO<E> dao, + TableMeta<T> meta, + Map<String, TopiaEntity> universe, + Import<E> importer, + CsvImportResult<T> csvResult) throws TopiaException { - Map<M, ZipEntry> result = Maps.newLinkedHashMap(); + for (TopiaEntity entity : importer) { - // check that all mandatories - for (M entry : entries) { - String filename = entry.getFilename(); - ZipEntry zipEntry = zipFile.getEntry(entryPrefix + filename); + String topiaId = entity.getTopiaId(); - if (zipEntry == null) { - missingEntries.add(filename); - } else { - result.put(entry, zipEntry); - } - } - return result; - } + Map<String, Object> properties = meta.prepareCreate(entity, null); + E existingEntity = dao.findByProperties(properties); + if (existingEntity == null) { - public static <T extends TopiaEntityEnum, M extends MetaFilenameAware<T>> Map<M, File> discoverEntries( - Iterable<M> entries, - File zipFile, - List<String> missingEntries) { + // new entity to create + E entityToSave = dao.create(properties); + String newTopiaId = entityToSave.getTopiaId(); + Date topiaCreateDate = entityToSave.getTopiaCreateDate(); + meta.copy(entity, entityToSave); + entityToSave.setTopiaId(newTopiaId); + entityToSave.setTopiaCreateDate(topiaCreateDate); - Map<M, File> result = Maps.newLinkedHashMap(); + if (log.isInfoEnabled()) { + log.info(String.format("Create entity [%s becomes %s] with naturalId %s", topiaId, newTopiaId, properties)); + } + universe.put(topiaId, entityToSave); - // check that all mandatories - for (M entry : entries) { - String filename = entry.getFilename(); - File zipEntry = new File(zipFile, filename); + if (csvResult != null) { + csvResult.incrementsNumberCreated(); + } + } else { + // existing entity, nothing to create + // just add a ref into universe to make translation possible by foreign keys - if (zipEntry.exists()) { - result.put(entry, zipEntry); - } else { - missingEntries.add(filename); + if (log.isDebugEnabled()) { + log.debug(String.format("Existing entity [%s] with naturalId %s, do not create anything", topiaId, properties)); + } + + universe.put(topiaId, existingEntity); } } - return result; } -// public static <T extends TopiaEntityEnum, E extends TopiaEntity> void importEntityFile(TopiaDAO<E> dao, -// TableMeta<T> meta, -// Import<E> importer, -// CsvImportResult<T> csvResult) throws TopiaException { -// -// for (TopiaEntity entity : importer) { -// -// Map<String, Object> properties = meta.prepareCreate( -// entity, entity.getTopiaId()); -// E entityToSave = dao.create(properties); -// -// meta.copy(entity, entityToSave); -// -// csvResult.incrementsNumberUpdated(); -// } -// } + public static <T extends TopiaEntityEnum> void importAssociation(TopiaContext tx, AssociationMeta<T> meta, + ImportToMap importer, + CsvImportResult<T> csvResult, + int nbRowBuffer) throws TopiaException { - public static <T extends TopiaEntityEnum> void importAssociationFile(TopiaContext tx, AssociationMeta<T> meta, - ImportToMap importer, - CsvImportResult<T> csvResult, - int nbRowBuffer) throws TopiaException { - T source = meta.getSource(); T target = meta.getTarget(); @@ -221,7 +284,7 @@ String sourceTableName = source.getContract().getSimpleName(); String table = targetTableName; - String updateString = String.format("UPDATE %s SET %s = '%%s' WHERE topiaId ='%%s';", table, sourceTableName); + String updateString = String.format(UPDATE_ASSOCIATION, table, sourceTableName); if (log.isDebugEnabled()) { log.debug("Will apply " + updateString); @@ -241,18 +304,20 @@ } } } - csvResult.incrementsNumberUpdated(); + if (csvResult != null) { + csvResult.incrementsNumberUpdated(); + } } if (builder.length() > 0) { tx.executeSQL(builder.toString()); } } - public static <T extends TopiaEntityEnum> void importNMAssociationFile(TopiaContext tx, - AssociationMeta<T> meta, - ImportToMap importer, - CsvImportResult<T> csvResult, - int nbRowBuffer) throws TopiaException { + public static <T extends TopiaEntityEnum> void importNMAssociation(TopiaContext tx, + AssociationMeta<T> meta, + ImportToMap importer, + CsvImportResult<T> csvResult, + int nbRowBuffer) throws TopiaException { T source = meta.getSource(); T target = meta.getTarget(); @@ -267,7 +332,7 @@ String table = TopiaEntityHelper.getNormalizedAssociationTableName( sourceTableName, targetTableName); - String updateString = String.format("INSERT INTO %s (%s,%s) VALUES('%%s','%%s');", table, sourceTableName, targetTableName); + String updateString = String.format(INSERT_ASSOCIATION, table, sourceTableName, targetTableName); if (log.isDebugEnabled()) { log.debug("Will apply " + updateString); @@ -287,7 +352,9 @@ } } } - csvResult.incrementsNumberUpdated(); + if (csvResult != null) { + csvResult.incrementsNumberUpdated(); + } } if (builder.length() > 0) { tx.executeSQL(builder.toString());