Tony CHEMIT pushed to branch develop at ultreiaio / ird-observe Commits: bc8730be by Tony Chemit at 2022-08-22T23:20:37+02:00 update pom - - - - - 62581c1b by Tony Chemit at 2022-08-22T23:20:37+02:00 downgrade m-site-p (still some velocity incompatibility) - - - - - 5b66ab55 by Tony Chemit at 2022-08-22T23:20:37+02:00 Persistence - generate application context schema names, instead of computing them. - - - - - 177b65d0 by Tony Chemit at 2022-08-22T23:20:37+02:00 Persistence - remove HibernateTopiaMigrationService - - - - - 1696b931 by Tony Chemit at 2022-08-22T23:20:37+02:00 Persistence - clean TopiaApplicationText - - - - - 11 changed files: - core/persistence/java/src/main/resources/META-INF/persistence/Observe/TopiaEntitySqlModel.json - pom.xml - toolkit/persistence/src/main/java/org/nuiton/topia/persistence/BeanTopiaConfiguration.java - − toolkit/persistence/src/main/java/org/nuiton/topia/persistence/HibernateTopiaMigrationService.java - toolkit/persistence/src/main/java/org/nuiton/topia/persistence/TopiaApplicationContext.java - toolkit/persistence/src/main/java/org/nuiton/topia/persistence/TopiaConfiguration.java - toolkit/persistence/src/main/java/org/nuiton/topia/persistence/internal/AbstractTopiaApplicationContext.java - toolkit/persistence/src/main/java/org/nuiton/topia/service/sql/model/TopiaEntitySqlModel.java - toolkit/plugin/src/main/java/fr/ird/observe/toolkit/maven/plugin/tck/GenerateTckArchive.java - toolkit/plugin/src/main/java/fr/ird/observe/toolkit/maven/plugin/tck/MigrateTckDatabases.java - toolkit/templates/src/main/java/org/nuiton/topia/templates/sql/EntitySqlDescriptorGenerator.java Changes: ===================================== core/persistence/java/src/main/resources/META-INF/persistence/Observe/TopiaEntitySqlModel.json ===================================== @@ -1,4 +1,16 @@ { + "schemaNames": [ + "common", + "ll_common", + "ll_landing", + "ll_logbook", + "ll_observation", + "ps_common", + "ps_landing", + "ps_localmarket", + "ps_logbook", + "ps_observation" + ], "replicationOrderForStandalone": [ "fr.ird.observe.entities.LastUpdateDate", "fr.ird.observe.entities.referential.common.Country", ===================================== pom.xml ===================================== @@ -23,7 +23,7 @@ <parent> <groupId>io.ultreia.maven</groupId> <artifactId>pom</artifactId> - <version>2022.80</version> + <version>2022.82</version> </parent> <groupId>fr.ird.observe</groupId> <artifactId>ird-observe</artifactId> @@ -96,6 +96,7 @@ <persistence.model.version>9.0</persistence.model.version> <!--persistence previous model version used for tck migration--> <persistence.previous.model.version>8.0</persistence.previous.model.version> + <plugin.version.site>4.0.0-M1</plugin.version.site> <tck.filename>tck-v${persistence.model.version}.zip</tck.filename> <tck.updated.filename>updated-${tck.filename}</tck.updated.filename> <tck.remote.path>/var/www/html/private/ird-observe/tck</tck.remote.path> ===================================== toolkit/persistence/src/main/java/org/nuiton/topia/persistence/BeanTopiaConfiguration.java ===================================== @@ -238,11 +238,6 @@ public class BeanTopiaConfiguration extends BeanJdbcConfiguration implements Top throw new TopiaMisconfigurationException("you must provide JDBC connection driver", this); } Map<String, String> hibernateExtraConfiguration = getHibernateExtraConfiguration(); - if (hibernateExtraConfiguration.containsKey(AvailableSettings.HBM2DDL_AUTO)) { - throw new TopiaMisconfigurationException("you must not use " + AvailableSettings.HBM2DDL_AUTO - + " configuration directive, if you want Hibernate to update the schema, use " + - HibernateTopiaMigrationService.class.getName(), this); - } for (Map.Entry<String, String> entry : hibernateExtraConfiguration.entrySet()) { String key = entry.getKey(); boolean entryIsOk = (key.startsWith("hibernate.") || key.startsWith("javax.persistence.")) ===================================== toolkit/persistence/src/main/java/org/nuiton/topia/persistence/HibernateTopiaMigrationService.java deleted ===================================== @@ -1,56 +0,0 @@ -package org.nuiton.topia.persistence; - -/* - * #%L - * ObServe Toolkit :: Persistence - * %% - * Copyright (C) 2008 - 2022 IRD, Ultreia.io - * %% - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public - * License along with this program. If not, see - * <http://www.gnu.org/licenses/gpl-3.0.html>. - * #L% - */ - - -import java.util.Map; - -public class HibernateTopiaMigrationService implements TopiaMigrationService { - - protected TopiaApplicationContext topiaApplicationContext; - - @Override - public void initTopiaService(TopiaApplicationContext<?> topiaApplicationContext, Map<String, String> serviceConfiguration) { - this.topiaApplicationContext = topiaApplicationContext; - } - - @Override - public String getSchemaVersion() { - throw new UnsupportedOperationException(); - } - - @Override - public void initOnCreateSchema() { - // do nothing - } - - @Override - public void runSchemaMigration() { - topiaApplicationContext.updateSchema(); - } - - @Override - public void close() { - - } -} ===================================== toolkit/persistence/src/main/java/org/nuiton/topia/persistence/TopiaApplicationContext.java ===================================== @@ -25,13 +25,14 @@ package org.nuiton.topia.persistence; import fr.ird.observe.entities.data.RootOpenableEntity; import fr.ird.observe.spi.referential.MissingReferentialRequest; import fr.ird.observe.spi.referential.MissingReferentialResult; +import fr.ird.observe.spi.referential.ReferentialIds; import fr.ird.observe.spi.service.ServiceContext; import io.ultreia.java4all.util.sql.SqlScript; -import io.ultreia.java4all.util.sql.SqlScriptConsumer; import org.nuiton.topia.persistence.internal.AbstractTopiaApplicationContext; import org.nuiton.topia.persistence.jdbc.JdbcHelper; import org.nuiton.topia.persistence.jdbc.JdbcHelperH2; import org.nuiton.topia.persistence.jdbc.JdbcHelperPostgres; +import org.nuiton.topia.persistence.security.SecurityScriptHelper; import org.nuiton.topia.persistence.support.TopiaServiceSupport; import org.nuiton.topia.service.sql.TopiaEntitySqlModelResource; import org.nuiton.topia.service.sql.TopiaSqlService; @@ -100,14 +101,20 @@ public interface TopiaApplicationContext<K extends TopiaPersistenceContext> exte return getConfiguration().getTopiaIdFactory(); } + TopiaIdFactoryForBulkSupport newIdFactoryForBulk(long timestamp); + + ReferentialIds getReferentialIds(Class<? extends TopiaEntity> dataType, String... ids); + + default SecurityScriptHelper newSecurityScriptHelper() { + return new SecurityScriptHelper(this); + } + TopiaSqlService getSqlService(); TopiaEntitySqlBlobIdsIterator newBlobIdsIterator(Path blobIdsPath); void executeSqlStatements(SqlScript content); - void executeSqlStatements(SqlScriptConsumer content); - TopiaMigrationService getMigrationService(); default void migrate() { @@ -140,16 +147,10 @@ public interface TopiaApplicationContext<K extends TopiaPersistenceContext> exte * @return all schema names used by persistent entities. * @since 3.0.1 */ - Set<String> getSchemaNames(); + List<String> getSchemaNames(); void createSchema(); - void showCreateSchema(); - - void updateSchema(); - - void dropSchema(); - /** * @return true is the application context is opened (ie not yet {@link #isClosed()}) */ ===================================== toolkit/persistence/src/main/java/org/nuiton/topia/persistence/TopiaConfiguration.java ===================================== @@ -75,7 +75,7 @@ public interface TopiaConfiguration extends JdbcConfiguration { * as keys of the map. * <p> * You <strong>MUST NOT</strong> pass {@link HibernateAvailableSettings#HBM2DDL_AUTO} since - * ToPIA will use validate. If you want use it, have a look at {@link HibernateTopiaMigrationService} + * ToPIA will use validate. * * @return a map containing hibernate configuration directives that ToPIA will use when it * will instantiate Hibernate. ===================================== toolkit/persistence/src/main/java/org/nuiton/topia/persistence/internal/AbstractTopiaApplicationContext.java ===================================== @@ -34,24 +34,17 @@ import fr.ird.observe.spi.referential.synchro.SynchronizeTaskType; import fr.ird.observe.spi.service.ServiceContext; import io.ultreia.java4all.util.TimeLog; import io.ultreia.java4all.util.sql.SqlScript; -import io.ultreia.java4all.util.sql.SqlScriptConsumer; import io.ultreia.java4all.util.sql.SqlScriptReader; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.hibernate.HibernateException; import org.hibernate.SessionFactory; import org.hibernate.boot.Metadata; import org.hibernate.cfg.AvailableSettings; import org.hibernate.cfg.Configuration; -import org.hibernate.tool.hbm2ddl.SchemaExport; -import org.hibernate.tool.hbm2ddl.SchemaUpdate; -import org.hibernate.tool.schema.TargetType; import org.nuiton.topia.persistence.TopiaApplicationContext; import org.nuiton.topia.persistence.TopiaConfiguration; import org.nuiton.topia.persistence.TopiaDao; -import org.nuiton.topia.persistence.TopiaEntity; import org.nuiton.topia.persistence.TopiaException; -import org.nuiton.topia.persistence.TopiaIdFactoryForBulkSupport; import org.nuiton.topia.persistence.TopiaMigrationService; import org.nuiton.topia.persistence.TopiaPersistenceContext; import org.nuiton.topia.persistence.TopiaService; @@ -59,7 +52,6 @@ import org.nuiton.topia.persistence.internal.support.TopiaServiceSupportImpl; import org.nuiton.topia.persistence.jdbc.JdbcHelper; import org.nuiton.topia.persistence.jdbc.JdbcHelperH2; import org.nuiton.topia.persistence.jdbc.JdbcHelperPostgres; -import org.nuiton.topia.persistence.security.SecurityScriptHelper; import org.nuiton.topia.persistence.support.TopiaServiceSupport; import org.nuiton.topia.persistence.util.TopiaUtil; import org.nuiton.topia.service.sql.TopiaSqlService; @@ -69,7 +61,6 @@ import java.io.File; import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; -import java.util.EnumSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -136,17 +127,11 @@ public abstract class AbstractTopiaApplicationContext<K extends TopiaPersistence protected abstract Map<String, Supplier<TopiaDao<?>>> createDaoMapping(); - public abstract TopiaIdFactoryForBulkSupport newIdFactoryForBulk(long timestamp); - @Override public final String getAuthenticationToken() { return authenticationToken; } - public String getPackageName() { - return getClass().getPackageName(); - } - protected void init() { configuration.check(); @@ -218,12 +203,6 @@ public abstract class AbstractTopiaApplicationContext<K extends TopiaPersistence } } - public abstract ReferentialIds getReferentialIds(Class<? extends TopiaEntity> dataType, String... ids); - - public final SecurityScriptHelper newSecurityScriptHelper() { - return new SecurityScriptHelper(this); - } - @Override public final TopiaSqlService getSqlService() { return getServices(TopiaSqlService.class).get(SQL_SERVICE_NAME); @@ -296,42 +275,6 @@ public abstract class AbstractTopiaApplicationContext<K extends TopiaPersistence } } - @Override - public final void showCreateSchema() { - try { - new SchemaExport().execute(EnumSet.of(TargetType.DATABASE, TargetType.STDOUT), SchemaExport.Action.CREATE, getHibernateProvider().getMetaData()); - } catch (HibernateException eee) { - throw new TopiaException(String.format("Could not show create schema for reason: %s", eee.getMessage()), eee); - } - - } - - @Override - public final void updateSchema() { - try { - EnumSet<TargetType> targetTypes = EnumSet.of(TargetType.DATABASE); - if (log.isDebugEnabled()) { - targetTypes.add(TargetType.STDOUT); - } - new SchemaUpdate().execute(targetTypes, getHibernateProvider().getMetaData()); - } catch (HibernateException eee) { - throw new TopiaException(String.format("Could not update schema for reason: %s", eee.getMessage()), eee); - } - } - - @Override - public final void dropSchema() { - try { - EnumSet<TargetType> targetTypes = EnumSet.of(TargetType.DATABASE); - if (log.isDebugEnabled()) { - targetTypes.add(TargetType.STDOUT); - } - new SchemaExport().execute(targetTypes, SchemaExport.Action.DROP, getHibernateProvider().getMetaData()); - } catch (HibernateException eee) { - throw new TopiaException(String.format("Could not drop schema for reason: %s", eee.getMessage()), eee); - } - } - @Override public void close() { @@ -379,7 +322,7 @@ public abstract class AbstractTopiaApplicationContext<K extends TopiaPersistence } @Override - public final Set<String> getSchemaNames() { + public final List<String> getSchemaNames() { return getSqlService().getModel().getSchemaNames(); } @@ -454,19 +397,7 @@ public abstract class AbstractTopiaApplicationContext<K extends TopiaPersistence SqlScriptReader scriptLocation = content.getLocation(); try (K persistenceContext = newPersistenceContext()) { persistenceContext.executeSqlScript(scriptLocation); - persistenceContext.commit(); - } - } catch (RuntimeException e) { - throw e; - } catch (Exception e) { - throw new TopiaException(e); - } - } - - public final void executeSqlStatements(SqlScriptConsumer content) { - try { - try (K persistenceContext = newPersistenceContext()) { - persistenceContext.executeSqlScript(content); + //FIXME-2022-08-22 this is a heart-break to force a commit, review this persistenceContext.commit(); } } catch (RuntimeException e) { @@ -511,8 +442,7 @@ public abstract class AbstractTopiaApplicationContext<K extends TopiaPersistence return null; } OneSideSqlRequest sqlRequest = OneSideSqlRequest.builder().addTasks(SynchronizeTaskType.ADD, missingIds).build(); - OneSideSqlResult sqlResult = newOneSideSqlResultBuilder(context, context.now()) - .build(sqlRequest); + OneSideSqlResult sqlResult = newOneSideSqlResultBuilder(context, context.now()).build(sqlRequest); Path scriptPath = temporaryDirectoryRoot.resolve(String.format("%s-%d.sql", getClass().getSimpleName(), System.nanoTime())); log.info(String.format("Generate script at: %s", scriptPath)); ===================================== toolkit/persistence/src/main/java/org/nuiton/topia/service/sql/model/TopiaEntitySqlModel.java ===================================== @@ -30,8 +30,6 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Set; -import java.util.TreeSet; /** * Describes the sql metadata model. @@ -43,6 +41,10 @@ import java.util.TreeSet; */ public class TopiaEntitySqlModel { + /** + * All schema names used. + */ + private final List<String> schemaNames; /** * Replication order for all entities not managed by any entry point.. */ @@ -52,7 +54,8 @@ public class TopiaEntitySqlModel { */ private final Map<String, TopiaEntitySqlDescriptor> descriptors; - public TopiaEntitySqlModel(List<String> replicationOrderForStandalone, Map<String, TopiaEntitySqlDescriptor> descriptors) { + public TopiaEntitySqlModel(List<String> schemaNames, List<String> replicationOrderForStandalone, Map<String, TopiaEntitySqlDescriptor> descriptors) { + this.schemaNames = Collections.unmodifiableList(Objects.requireNonNull(schemaNames)); this.replicationOrderForStandalone = Objects.requireNonNull(replicationOrderForStandalone); this.descriptors = Objects.requireNonNull(descriptors); } @@ -81,13 +84,8 @@ public class TopiaEntitySqlModel { return replicationOrderForStandalone; } - public Set<String> getSchemaNames() { - Set<String> result = new TreeSet<>(); - for (TopiaEntitySqlDescriptor entitySqlDescriptor : getDescriptors().values()) { - String schema = entitySqlDescriptor.getTable().getSchemaName(); - result.add(schema); - } - return Collections.unmodifiableSet(result); + public List<String> getSchemaNames() { + return schemaNames; } protected Map<String, TopiaEntitySqlDescriptor> getDescriptors() { ===================================== toolkit/plugin/src/main/java/fr/ird/observe/toolkit/maven/plugin/tck/GenerateTckArchive.java ===================================== @@ -30,7 +30,6 @@ import fr.ird.observe.toolkit.maven.plugin.PersistenceRunner; import io.ultreia.java4all.util.Version; import io.ultreia.java4all.util.Zips; import io.ultreia.java4all.util.sql.SqlScript; -import io.ultreia.java4all.util.sql.SqlScriptConsumer; import io.ultreia.java4all.util.sql.SqlScriptWriter; import org.nuiton.topia.persistence.TopiaApplicationContext; import org.nuiton.topia.persistence.jdbc.JdbcHelperH2; @@ -123,14 +122,14 @@ public class GenerateTckArchive extends PersistenceRunner { log.info(String.format("Load referential database for version %s", previousModelVersion)); - applicationContext.executeSqlStatements(SqlScriptConsumer.of(SqlScript.of(sources.resolve(previousModelVersion.getVersion()).resolve(DatabaseName.referential.name() + ".sql.gz")))); + applicationContext.executeSqlStatements(SqlScript.of(sources.resolve(previousModelVersion.getVersion()).resolve(DatabaseName.referential.name() + ".sql.gz"))); log.info(String.format("Migrate from version %s to %s", previousModelVersion, modelVersion)); applicationContext.migrate(); log.info(String.format("Load data script for version %s", modelVersion)); Path dataPath = exploded.resolve(modelVersion.getVersion()).resolve(DatabaseName.data.name() + ".sql"); - applicationContext.executeSqlStatements(SqlScriptConsumer.builder(dataPath).build()); + applicationContext.executeSqlStatements(SqlScript.of(dataPath)); log.info(String.format("Do backup to %s", backupFile)); backupSane(applicationContext, backupFile, false); ===================================== toolkit/plugin/src/main/java/fr/ird/observe/toolkit/maven/plugin/tck/MigrateTckDatabases.java ===================================== @@ -27,7 +27,6 @@ import fr.ird.observe.test.DatabaseName; import fr.ird.observe.toolkit.maven.plugin.PersistenceRunner; import io.ultreia.java4all.util.Version; import io.ultreia.java4all.util.sql.SqlScript; -import io.ultreia.java4all.util.sql.SqlScriptConsumer; import org.nuiton.topia.persistence.TopiaApplicationContext; import java.nio.file.Path; @@ -81,7 +80,7 @@ public class MigrateTckDatabases extends PersistenceRunner { try (TopiaApplicationContext<?> applicationContext = createTopiaApplicationContext(configuration)) { log.info(String.format("[%S] migrate from version %s to %s", tckDbName, previousModelVersion, modelVersion)); - applicationContext.executeSqlStatements(SqlScriptConsumer.of(SqlScript.of(dbPath.resolve(sqlFilename)))); + applicationContext.executeSqlStatements(SqlScript.of(dbPath.resolve(sqlFilename))); applicationContext.migrate(); log.info(String.format("[%S] backup to %s", tckDbName, backupFile)); @@ -97,7 +96,7 @@ public class MigrateTckDatabases extends PersistenceRunner { log.info(String.format("[%S] Check - loading new database with version %s", tckDbName, modelVersion)); - applicationContext.executeSqlStatements(SqlScriptConsumer.of(SqlScript.of(backupFile))); + applicationContext.executeSqlStatements(SqlScript.of(backupFile)); Path checkBackupFile = getTemporaryPath().resolve("check-backup-" + backupFile.toFile().getName().replace(".sql.gz", "-check.sql")); applicationContext.backup(checkBackupFile.toFile(), false); log.info(String.format("[%S] Check - backup to %s", tckDbName, checkBackupFile)); ===================================== toolkit/templates/src/main/java/org/nuiton/topia/templates/sql/EntitySqlDescriptorGenerator.java ===================================== @@ -394,7 +394,12 @@ public class EntitySqlDescriptorGenerator extends TopiaMetadataModelGeneratorSup modelCopyPlan = new TopiaEntitySqlCopyPlanModel(entryPointCopyPlans, standaloneCopyPlan); modelReplicatePlan = new TopiaEntitySqlReplicatePlanModel(replicatePlans); modelDeletePlan = new TopiaEntitySqlDeletePlanModel(typeDeletePlans); - sqlModel = new TopiaEntitySqlModel(replicationOrderWithStandalone, descriptors); + Set<String> schemaNames = new TreeSet<>(); + for (TopiaEntitySqlDescriptor entitySqlDescriptor : descriptors.values()) { + String schema = entitySqlDescriptor.getTable().getSchemaName(); + schemaNames.add(schema); + } + sqlModel = new TopiaEntitySqlModel(List.copyOf(schemaNames), replicationOrderWithStandalone, descriptors); LinkedHashMap<Class<? extends ReferentialDto>, List<String>> referentialBusinessProperties = new LinkedHashMap<>(); for (String entityName : replicationOrderWithStandalone) { if (!entityName.contains(".referential.")) { View it on GitLab: https://gitlab.com/ultreiaio/ird-observe/-/compare/685feca0c407582cdfb97e2dd... -- View it on GitLab: https://gitlab.com/ultreiaio/ird-observe/-/compare/685feca0c407582cdfb97e2dd... You're receiving this email because of your account on gitlab.com.