FINERACT-1498: Switching from Flyway to Liqubase migrations

This commit is contained in:
Arnold Galovics
2022-02-19 22:55:32 +01:00
committed by Aleksandar Vidakovic
parent 746c589a6e
commit 45bed0a05a
34 changed files with 30137 additions and 189 deletions
@@ -20,11 +20,12 @@ package org.apache.fineract.infrastructure.core.boot;
import org.apache.fineract.infrastructure.core.config.FineractProperties;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.flyway.FlywayAutoConfiguration;
import org.springframework.boot.autoconfigure.gson.GsonAutoConfiguration;
import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
import org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration;
import org.springframework.boot.autoconfigure.jdbc.JdbcTemplateAutoConfiguration;
import org.springframework.boot.autoconfigure.liquibase.LiquibaseAutoConfiguration;
import org.springframework.boot.autoconfigure.liquibase.LiquibaseProperties;
import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.ComponentScan;
@@ -38,11 +39,11 @@ import org.springframework.transaction.annotation.EnableTransactionManagement;
@Configuration
@EnableAutoConfiguration(exclude = { DataSourceAutoConfiguration.class, HibernateJpaAutoConfiguration.class,
DataSourceTransactionManagerAutoConfiguration.class, FlywayAutoConfiguration.class, GsonAutoConfiguration.class,
JdbcTemplateAutoConfiguration.class })
DataSourceTransactionManagerAutoConfiguration.class, GsonAutoConfiguration.class, JdbcTemplateAutoConfiguration.class,
LiquibaseAutoConfiguration.class })
@EnableTransactionManagement
@EnableWebSecurity
@EnableConfigurationProperties({ FineractProperties.class })
@EnableConfigurationProperties({ FineractProperties.class, LiquibaseProperties.class })
@ComponentScan(basePackages = "org.apache.fineract.**")
public abstract class AbstractApplicationConfiguration {
@@ -46,7 +46,7 @@ public class RoutingDataSource extends AbstractDataSource {
return determineTargetDataSource().getConnection();
}
private DataSource determineTargetDataSource() {
public DataSource determineTargetDataSource() {
return this.dataSourceServiceFactory.determineDataSourceService().retrieveDataSource();
}
@@ -1,169 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.fineract.infrastructure.core.service;
import static org.apache.fineract.infrastructure.core.domain.FineractPlatformTenantConnection.toJdbcUrl;
import static org.apache.fineract.infrastructure.core.domain.FineractPlatformTenantConnection.toProtocol;
import com.zaxxer.hikari.HikariConfig;
import java.util.List;
import java.util.Map;
import javax.annotation.PostConstruct;
import javax.sql.DataSource;
import org.apache.fineract.infrastructure.core.config.FineractProperties;
import org.apache.fineract.infrastructure.core.domain.FineractPlatformTenant;
import org.apache.fineract.infrastructure.core.domain.FineractPlatformTenantConnection;
import org.apache.fineract.infrastructure.security.service.TenantDetailsService;
import org.flywaydb.core.Flyway;
import org.flywaydb.core.api.FlywayException;
import org.flywaydb.core.internal.jdbc.DriverDataSource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.support.rowset.SqlRowSet;
import org.springframework.stereotype.Service;
/**
* A service that picks up on tenants that are configured to auto-update their specific schema on application startup.
*/
@Service
public class TenantDatabaseUpgradeService {
private static final Logger LOG = LoggerFactory.getLogger(TenantDatabaseUpgradeService.class);
private final HikariConfig hikariConfig;
private final TenantDetailsService tenantDetailsService;
protected final DataSource tenantDataSource;
protected final FineractProperties fineractProperties;
@Autowired
public TenantDatabaseUpgradeService(final HikariConfig hikariConfig, final TenantDetailsService detailsService,
@Qualifier("hikariTenantDataSource") final DataSource dataSource, final FineractProperties fineractProperties) {
this.hikariConfig = hikariConfig;
this.tenantDetailsService = detailsService;
this.tenantDataSource = dataSource;
this.fineractProperties = fineractProperties;
}
@PostConstruct
public void upgradeAllTenants() {
upgradeTenantDB();
final List<FineractPlatformTenant> tenants = this.tenantDetailsService.findAllTenants();
for (final FineractPlatformTenant tenant : tenants) {
final FineractPlatformTenantConnection connection = tenant.getConnection();
if (connection.isAutoUpdateEnabled()) {
String protocol = toProtocol(this.tenantDataSource);
String jdbcUrl = toJdbcUrl(protocol, connection.getSchemaServer(), connection.getSchemaServerPort(),
connection.getSchemaName(), connection.getSchemaConnectionParameters());
DriverDataSource source = new DriverDataSource(Thread.currentThread().getContextClassLoader(),
hikariConfig.getDriverClassName(), jdbcUrl, connection.getSchemaUsername(), connection.getSchemaPassword());
final Flyway flyway = Flyway.configure().dataSource(source).locations("sql/migrations/core_db").outOfOrder(true)
.placeholderReplacement(false).configuration(Map.of("flyway.table", "schema_version")) // FINERACT-979
.load();
// Should be removed later when all instances are stabilized
// :FINERACT-1008
repairFlywayVersionSkip(flyway.getConfiguration().getDataSource());
try {
flyway.repair();
flyway.migrate();
} catch (FlywayException e) {
String betterMessage = e.getMessage() + "; for Tenant DB URL: " + jdbcUrl + ", username: "
+ connection.getSchemaUsername();
throw new FlywayException(betterMessage, e);
}
}
}
}
/**
* Initializes, and if required upgrades (using Flyway) the Tenant DB itself.
*/
private void upgradeTenantDB() {
LOG.info("Upgrade tenant DB: {}:{}", fineractProperties.getTenant().getHost(), fineractProperties.getTenant().getPort());
LOG.info("- fineract.tenant.username: {}", fineractProperties.getTenant().getUsername());
LOG.info("- fineract.tenant.password: ****");
LOG.info("- fineract.tenant.parameters: {}", fineractProperties.getTenant().getParameters());
LOG.info("- fineract.tenant.timezone: {}", fineractProperties.getTenant().getTimezone());
LOG.info("- fineract.tenant.description: {}", fineractProperties.getTenant().getDescription());
LOG.info("- fineract.tenant.identifier: {}", fineractProperties.getTenant().getIdentifier());
LOG.info("- fineract.tenant.name: {}", fineractProperties.getTenant().getName());
final Flyway flyway = Flyway.configure().dataSource(tenantDataSource).locations("sql/migrations/list_db").outOfOrder(true)
// FINERACT-773
.placeholders(Map.of("fineract.tenant.host", fineractProperties.getTenant().getHost(), "fineract.tenant.port",
fineractProperties.getTenant().getPort().toString(), "fineract.tenant.username",
fineractProperties.getTenant().getUsername(), "fineract.tenant.password",
fineractProperties.getTenant().getPassword(), "fineract.tenant.parameters",
fineractProperties.getTenant().getParameters(), "fineract.tenant.timezone",
fineractProperties.getTenant().getTimezone(), "fineract.tenant.description",
fineractProperties.getTenant().getDescription(), "fineract.tenant.identifier",
fineractProperties.getTenant().getIdentifier(), "fineract.tenant.name", fineractProperties.getTenant().getName()))
.configuration(Map.of("flyway.table", "schema_version")) // FINERACT-979
.load();
// Should be removed later when all instances are stabilized
// :FINERACT-1008
repairFlywayVersionSkip(flyway.getConfiguration().getDataSource());
flyway.repair();
flyway.migrate();
}
private void repairFlywayVersionSkip(DataSource source) {
JdbcTemplate template = new JdbcTemplate(source);
LOG.info("repairFlywayVersionSkip: Check whether the version table is in old format ");
SqlRowSet ts = template.queryForRowSet("SHOW TABLES LIKE 'schema_version';");
if (ts.next()) {
SqlRowSet rs = template.queryForRowSet("SHOW COLUMNS FROM `schema_version` LIKE 'version_rank';");
if (rs.next()) {
LOG.info("repairFlywayVersionSkip: The schema_version table is in old format, executing repair ");
template.execute("CREATE TABLE `schema_version_history` ( `version_rank` int(11) NOT NULL, "
+ " `installed_rank` int(11) NOT NULL, `version` varchar(50) NOT NULL, `description` varchar(200) NOT NULL, "
+ "`type` varchar(20) NOT NULL, `script` varchar(1000) NOT NULL, `checksum` int(11) DEFAULT NULL, "
+ " `installed_by` varchar(100) NOT NULL, `installed_on` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, "
+ " `execution_time` int(11) NOT NULL, `success` tinyint(1) NOT NULL, PRIMARY KEY (`version`), "
+ " KEY `schema_version_vr_idx` (`version_rank`), KEY `schema_version_ir_idx` (`installed_rank`), "
+ " KEY `schema_version_s_idx` (`success`));");
template.execute("INSERT INTO schema_version_history select * from schema_version;");
template.execute("DROP TABLE schema_version;");
template.execute("CREATE TABLE `schema_version` ( `installed_rank` int(11) NOT NULL, "
+ " `version` varchar(50) DEFAULT NULL, `description` varchar(200) NOT NULL, `type` varchar(20) NOT NULL, "
+ " `script` varchar(1000) NOT NULL, `checksum` int(11) DEFAULT NULL, `installed_by` varchar(100) NOT NULL, "
+ " `installed_on` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, `execution_time` int(11) NOT NULL, "
+ " `success` tinyint(1) NOT NULL, PRIMARY KEY (`installed_rank`), KEY `flyway_schema_history_s_idx` (`success`) );");
template.execute("INSERT INTO schema_version (installed_rank, version, description, type, script, checksum, "
+ "installed_by, installed_on, execution_time, success) SELECT installed_rank, version, description, type, "
+ "script, checksum, installed_by, installed_on, execution_time, success FROM schema_version_history;");
template.execute("DROP TABLE schema_version_history;");
LOG.info("repairFlywayVersionSkip: The schema_version repair completed.");
} else {
LOG.info("repairFlywayVersionSkip: The schema_version table format is new, aborting repair");
}
} else {
LOG.info("repairFlywayVersionSkip: The schema_version table does not exist, aborting repair");
}
}
}
@@ -0,0 +1,63 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.fineract.infrastructure.core.service.database;
import static org.apache.commons.collections4.CollectionUtils.isNotEmpty;
import java.sql.SQLException;
import java.util.Collection;
import javax.sql.DataSource;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@Component
public class DatabaseIndependentQueryService implements DatabaseQueryService {
private final Collection<DatabaseQueryService> queryServices;
@Autowired
public DatabaseIndependentQueryService(Collection<DatabaseQueryService> queryServices) {
this.queryServices = queryServices;
}
private DatabaseQueryService choose(DataSource dataSource) {
try {
DatabaseQueryService result = null;
if (isNotEmpty(queryServices)) {
result = queryServices.stream().filter(qs -> qs.isSupported(dataSource)).findAny().orElse(null);
}
if (result == null) {
throw new IllegalStateException("DataSource not supported: " + dataSource.getConnection().getMetaData().getURL());
}
return result;
} catch (SQLException e) {
throw new RuntimeException("Error while trying to choose the proper query service", e);
}
}
@Override
public boolean isSupported(DataSource dataSource) {
return true;
}
@Override
public boolean isTablePresent(DataSource dataSource, String tableName) {
return choose(dataSource).isTablePresent(dataSource, tableName);
}
}
@@ -0,0 +1,28 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.fineract.infrastructure.core.service.database;
import javax.sql.DataSource;
public interface DatabaseQueryService {
boolean isSupported(DataSource dataSource);
boolean isTablePresent(DataSource dataSource, String tableName);
}
@@ -0,0 +1,23 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.fineract.infrastructure.core.service.database;
public enum DatabaseType {
MYSQL
}
@@ -0,0 +1,70 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.fineract.infrastructure.core.service.database;
import static org.apache.fineract.infrastructure.core.domain.FineractPlatformTenantConnection.toProtocol;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import javax.sql.DataSource;
import org.apache.fineract.infrastructure.core.service.RoutingDataSource;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@Component
public class DatabaseTypeResolver {
private final Map<String, DatabaseType> protocolMapping = Map.of("jdbc:mariadb", DatabaseType.MYSQL, "jdbc:mysql", DatabaseType.MYSQL);
private final Map<DataSource, DatabaseType> dataSourceCache = new ConcurrentHashMap<>();
private final RoutingDataSource routingDataSource;
@Autowired
public DatabaseTypeResolver(RoutingDataSource routingDataSource) {
this.routingDataSource = routingDataSource;
}
public boolean isMySQL() {
return isMySQL(routingDataSource);
}
public boolean isMySQL(DataSource dataSource) {
return DatabaseType.MYSQL.equals(databaseType(dataSource));
}
public DatabaseType databaseType() {
return databaseType(routingDataSource);
}
public DatabaseType databaseType(DataSource dataSource) {
DataSource unwrappedDataSource = dataSource;
if (dataSource instanceof RoutingDataSource) {
unwrappedDataSource = ((RoutingDataSource) dataSource).determineTargetDataSource();
}
return dataSourceCache.computeIfAbsent(unwrappedDataSource, (ds) -> {
String protocol = toProtocol(ds);
DatabaseType databaseType = protocolMapping.get(protocol);
if (databaseType == null) {
throw new IllegalArgumentException("The DataSource is not supported");
}
return databaseType;
});
}
}
@@ -0,0 +1,47 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.fineract.infrastructure.core.service.database;
import static java.lang.String.format;
import static org.apache.fineract.infrastructure.core.domain.FineractPlatformTenantConnection.toProtocol;
import java.util.Set;
import javax.sql.DataSource;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.support.rowset.SqlRowSet;
import org.springframework.stereotype.Component;
@Component
public class MySQLQueryService implements DatabaseQueryService {
private final Set<String> supportedProtocols = Set.of("jdbc:mariadb", "jdbc:mysql");
@Override
public boolean isSupported(DataSource dataSource) {
String protocol = toProtocol(dataSource);
return supportedProtocols.contains(protocol.toLowerCase());
}
@Override
public boolean isTablePresent(DataSource dataSource, String tableName) {
JdbcTemplate jdbcTemplate = new JdbcTemplate(dataSource);
SqlRowSet rs = jdbcTemplate.queryForRowSet(format("SHOW TABLES LIKE '%s'", tableName));
return rs.next();
}
}
@@ -0,0 +1,48 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.fineract.infrastructure.core.service.migration;
import java.util.Map;
import javax.sql.DataSource;
import org.apache.fineract.infrastructure.core.service.database.DatabaseType;
import org.apache.fineract.infrastructure.core.service.database.DatabaseTypeResolver;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@Component
public class DatabaseAwareMigrationContextProvider {
private final Map<DatabaseType, String> contextMapping = Map.of(DatabaseType.MYSQL, "mysql");
private final DatabaseTypeResolver databaseTypeResolver;
@Autowired
public DatabaseAwareMigrationContextProvider(DatabaseTypeResolver databaseTypeResolver) {
this.databaseTypeResolver = databaseTypeResolver;
}
public String provide(DataSource dataSource) {
DatabaseType databaseType = databaseTypeResolver.databaseType(dataSource);
String context = contextMapping.get(databaseType);
if (context == null) {
throw new IllegalStateException("Database is not supported");
}
return context;
}
}
@@ -0,0 +1,36 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.fineract.infrastructure.core.service.migration;
import java.sql.SQLException;
import liquibase.Liquibase;
import liquibase.exception.DatabaseException;
import liquibase.exception.LiquibaseException;
import liquibase.integration.spring.SpringLiquibase;
public class ExtendedSpringLiquibase extends SpringLiquibase {
public void changeLogSync() throws LiquibaseException {
try (Liquibase liquibase = createLiquibase(getDataSource().getConnection())) {
liquibase.changeLogSync(getContexts());
} catch (SQLException e) {
throw new DatabaseException(e);
}
}
}
@@ -0,0 +1,125 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.fineract.infrastructure.core.service.migration;
import static org.apache.commons.lang3.StringUtils.isNotBlank;
import com.google.common.base.Joiner;
import java.io.File;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import javax.sql.DataSource;
import org.springframework.boot.autoconfigure.liquibase.LiquibaseProperties;
import org.springframework.core.io.ResourceLoader;
public class ExtendedSpringLiquibaseBuilder {
private final Set<String> contexts = new HashSet<>();
private final Map<String, String> changeLogParameters = new HashMap<>();
private final boolean clearCheckSums;
private final String liquibaseSchema;
private final String liquibaseTableSpace;
private final String databaseChangeLogTable;
private final String databaseChangeLogLockTable;
private final boolean shouldRun;
private final String labels;
private final File rollbackFile;
private final boolean testRollbackOnUpdate;
private final String tag;
private String changeLog;
private ResourceLoader resourceLoader;
private String defaultSchema;
private boolean dropFirst;
private DataSource dataSource;
public ExtendedSpringLiquibaseBuilder(LiquibaseProperties liquibaseProperties) {
this.defaultSchema = liquibaseProperties.getDefaultSchema();
this.dropFirst = liquibaseProperties.isDropFirst();
if (liquibaseProperties.getParameters() != null) {
this.changeLogParameters.putAll(liquibaseProperties.getParameters());
}
this.changeLog = liquibaseProperties.getChangeLog();
if (isNotBlank(liquibaseProperties.getContexts())) {
this.contexts.add(liquibaseProperties.getContexts());
}
this.clearCheckSums = liquibaseProperties.isClearChecksums();
this.liquibaseSchema = liquibaseProperties.getLiquibaseSchema();
this.liquibaseTableSpace = liquibaseProperties.getLiquibaseTablespace();
this.databaseChangeLogTable = liquibaseProperties.getDatabaseChangeLogTable();
this.databaseChangeLogLockTable = liquibaseProperties.getDatabaseChangeLogLockTable();
this.shouldRun = liquibaseProperties.isEnabled();
this.labels = liquibaseProperties.getLabels();
this.rollbackFile = liquibaseProperties.getRollbackFile();
this.testRollbackOnUpdate = liquibaseProperties.isTestRollbackOnUpdate();
this.tag = liquibaseProperties.getTag();
}
public ExtendedSpringLiquibaseBuilder withDataSource(DataSource dataSource) {
this.dataSource = dataSource;
return this;
}
public ExtendedSpringLiquibaseBuilder withContext(String context) {
if (isNotBlank(context)) {
this.contexts.add(context);
}
return this;
}
public ExtendedSpringLiquibaseBuilder withContexts(String... contexts) {
return withContexts(Arrays.asList(contexts));
}
public ExtendedSpringLiquibaseBuilder withContexts(Collection<String> contexts) {
this.contexts.addAll(contexts);
return this;
}
public ExtendedSpringLiquibaseBuilder withResourceLoader(ResourceLoader resourceLoader) {
this.resourceLoader = resourceLoader;
return this;
}
public ExtendedSpringLiquibase build() {
ExtendedSpringLiquibase result = new ExtendedSpringLiquibase();
String liquibaseContexts = Joiner.on(",").join(contexts);
result.setContexts(liquibaseContexts);
result.setDataSource(dataSource);
result.setChangeLog(changeLog);
result.setResourceLoader(resourceLoader);
result.setDefaultSchema(defaultSchema);
result.setDropFirst(dropFirst);
result.setChangeLogParameters(changeLogParameters);
result.setClearCheckSums(clearCheckSums);
result.setLiquibaseSchema(liquibaseSchema);
result.setLiquibaseTablespace(liquibaseTableSpace);
result.setDatabaseChangeLogTable(databaseChangeLogTable);
result.setDatabaseChangeLogLockTable(databaseChangeLogLockTable);
result.setShouldRun(shouldRun);
result.setLabels(labels);
result.setRollbackFile(rollbackFile);
result.setTestRollbackOnUpdate(testRollbackOnUpdate);
result.setTag(tag);
return result;
}
}
@@ -0,0 +1,50 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.fineract.infrastructure.core.service.migration;
import javax.sql.DataSource;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.liquibase.LiquibaseProperties;
import org.springframework.core.env.Environment;
import org.springframework.core.io.ResourceLoader;
import org.springframework.stereotype.Component;
@Component
public class ExtendedSpringLiquibaseFactory {
private final LiquibaseProperties liquibaseProperties;
private final ResourceLoader resourceLoader;
private final Environment environment;
private final DatabaseAwareMigrationContextProvider databaseAwareMigrationContextProvider;
@Autowired
public ExtendedSpringLiquibaseFactory(LiquibaseProperties liquibaseProperties, ResourceLoader resourceLoader, Environment environment,
DatabaseAwareMigrationContextProvider databaseAwareMigrationContextProvider) {
this.liquibaseProperties = liquibaseProperties;
this.resourceLoader = resourceLoader;
this.environment = environment;
this.databaseAwareMigrationContextProvider = databaseAwareMigrationContextProvider;
}
public ExtendedSpringLiquibase create(DataSource dataSource, String... contexts) {
String databaseContext = databaseAwareMigrationContextProvider.provide(dataSource);
return new ExtendedSpringLiquibaseBuilder(liquibaseProperties).withDataSource(dataSource).withResourceLoader(resourceLoader)
.withContexts(contexts).withContexts(environment.getActiveProfiles()).withContext(databaseContext).build();
}
}
@@ -0,0 +1,26 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.fineract.infrastructure.core.service.migration;
public class SchemaUpgradeNeededException extends RuntimeException {
public SchemaUpgradeNeededException(String message) {
super(message);
}
}
@@ -0,0 +1,65 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.fineract.infrastructure.core.service.migration;
import static org.apache.fineract.infrastructure.core.domain.FineractPlatformTenantConnection.toJdbcUrl;
import static org.apache.fineract.infrastructure.core.domain.FineractPlatformTenantConnection.toProtocol;
import com.zaxxer.hikari.HikariDataSource;
import javax.sql.DataSource;
import org.apache.fineract.infrastructure.core.domain.FineractPlatformTenant;
import org.apache.fineract.infrastructure.core.domain.FineractPlatformTenantConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Component;
@Component
public class TenantDataSourceFactory {
private static final Logger LOG = LoggerFactory.getLogger(TenantDataSourceFactory.class);
private final HikariDataSource tenantDataSource;
@Autowired
public TenantDataSourceFactory(@Qualifier("hikariTenantDataSource") HikariDataSource tenantDataSource) {
this.tenantDataSource = tenantDataSource;
}
public DataSource create(FineractPlatformTenant tenant) {
HikariDataSource dataSource = new HikariDataSource();
dataSource.setDriverClassName(tenantDataSource.getDriverClassName());
dataSource.setDataSourceProperties(tenantDataSource.getDataSourceProperties());
dataSource.setMinimumIdle(tenantDataSource.getMinimumIdle());
dataSource.setMaximumPoolSize(tenantDataSource.getMaximumPoolSize());
dataSource.setIdleTimeout(tenantDataSource.getIdleTimeout());
dataSource.setConnectionTestQuery(tenantDataSource.getConnectionTestQuery());
FineractPlatformTenantConnection tenantConnection = tenant.getConnection();
dataSource.setUsername(tenantConnection.getSchemaUsername());
dataSource.setPassword(tenantConnection.getSchemaPassword());
String protocol = toProtocol(tenantDataSource);
String tenantJdbcUrl = toJdbcUrl(protocol, tenantConnection.getSchemaServer(), tenantConnection.getSchemaServerPort(),
tenantConnection.getSchemaName(), tenantConnection.getSchemaConnectionParameters());
LOG.debug("JDBC URL for tenant {} is {}", tenant.getTenantIdentifier(), tenantJdbcUrl);
dataSource.setJdbcUrl(tenantJdbcUrl);
return dataSource;
}
}
@@ -0,0 +1,83 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.fineract.infrastructure.core.service.migration;
import java.util.Map;
import java.util.Objects;
import javax.sql.DataSource;
import org.apache.fineract.infrastructure.core.service.database.DatabaseIndependentQueryService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.liquibase.LiquibaseProperties;
import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
import org.springframework.stereotype.Component;
@Component
public class TenantDatabaseStateVerifier {
private static final int TENANT_STORE_LATEST_FLYWAY_VERSION = 6;
private static final String TENANT_STORE_LATEST_FLYWAY_SCRIPT_NAME = "V6__add_unique_tenant_identifier.sql";
private static final int TENANT_STORE_LATEST_FLYWAY_SCRIPT_CHECKSUM = -43094919;
private static final int TENANT_LATEST_FLYWAY_VERSION = 392;
private static final String TENANT_LATEST_FLYWAY_SCRIPT_NAME = "V392__interest_recovery_conf_for_rescedule.sql";
private static final int TENANT_LATEST_FLYWAY_SCRIPT_CHECKSUM = 1102395052;
private final LiquibaseProperties liquibaseProperties;
private final DatabaseIndependentQueryService dbQueryService;
@Autowired
public TenantDatabaseStateVerifier(LiquibaseProperties liquibaseProperties, DatabaseIndependentQueryService dbQueryService) {
this.liquibaseProperties = liquibaseProperties;
this.dbQueryService = dbQueryService;
}
public boolean isFirstLiquibaseMigration(DataSource dataSource) {
boolean databaseChangelogTableExists = dbQueryService.isTablePresent(dataSource, "DATABASECHANGELOG");
return !databaseChangelogTableExists;
}
public boolean isFlywayPresent(DataSource dataSource) {
return dbQueryService.isTablePresent(dataSource, "schema_version");
}
public boolean isLiquibaseDisabled() {
return !liquibaseProperties.isEnabled();
}
public boolean isTenantStoreOnLatestUpgradableVersion(DataSource dataSource) {
return isOnLatestFlywayVersion(TENANT_STORE_LATEST_FLYWAY_VERSION, TENANT_STORE_LATEST_FLYWAY_SCRIPT_NAME,
TENANT_STORE_LATEST_FLYWAY_SCRIPT_CHECKSUM, dataSource);
}
public boolean isTenantOnLatestUpgradableVersion(DataSource dataSource) {
return isOnLatestFlywayVersion(TENANT_LATEST_FLYWAY_VERSION, TENANT_LATEST_FLYWAY_SCRIPT_NAME, TENANT_LATEST_FLYWAY_SCRIPT_CHECKSUM,
dataSource);
}
private boolean isOnLatestFlywayVersion(int version, String scriptName, int checksum, DataSource dataSource) {
Map<String, Object> paramMap = Map.of("latestFlywayVersion", version, "latestFlywayScriptName", scriptName,
"latestFlywayScriptChecksum", checksum);
NamedParameterJdbcTemplate jdbcTemplate = new NamedParameterJdbcTemplate(dataSource);
Integer result = jdbcTemplate.queryForObject(
"SELECT COUNT(script) FROM schema_version " + "WHERE version = :latestFlywayVersion "
+ "AND script = :latestFlywayScriptName " + "AND checksum = :latestFlywayScriptChecksum " + "AND success = 1",
paramMap, Integer.class);
return Objects.equals(result, 1);
}
}
@@ -0,0 +1,146 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.fineract.infrastructure.core.service.migration;
import static org.apache.commons.collections4.CollectionUtils.isNotEmpty;
import com.zaxxer.hikari.HikariDataSource;
import java.util.List;
import java.util.function.Function;
import javax.sql.DataSource;
import liquibase.exception.LiquibaseException;
import liquibase.integration.spring.SpringLiquibase;
import org.apache.fineract.infrastructure.core.config.FineractProperties;
import org.apache.fineract.infrastructure.core.domain.FineractPlatformTenant;
import org.apache.fineract.infrastructure.security.service.TenantDetailsService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Service;
/**
* A service that picks up on tenants that are configured to auto-update their specific schema on application startup.
*/
@Service
public class TenantDatabaseUpgradeService implements InitializingBean {
private static final Logger LOG = LoggerFactory.getLogger(TenantDatabaseUpgradeService.class);
private static final String TENANT_STORE_DB_CONTEXT = "tenant_store_db";
private static final String INITIAL_SWITCH_CONTEXT = "initial_switch";
private static final String TENANT_DB_CONTEXT = "tenant_db";
private final TenantDetailsService tenantDetailsService;
private final HikariDataSource tenantDataSource;
private final FineractProperties fineractProperties;
private final TenantDatabaseStateVerifier databaseStateVerifier;
private final ExtendedSpringLiquibaseFactory liquibaseFactory;
private final TenantDataSourceFactory tenantDataSourceFactory;
@Autowired
public TenantDatabaseUpgradeService(final TenantDetailsService detailsService,
@Qualifier("hikariTenantDataSource") final HikariDataSource tenantDataSource, final FineractProperties fineractProperties,
TenantDatabaseStateVerifier databaseStateVerifier, ExtendedSpringLiquibaseFactory liquibaseFactory,
TenantDataSourceFactory tenantDataSourceFactory) {
this.tenantDetailsService = detailsService;
this.tenantDataSource = tenantDataSource;
this.fineractProperties = fineractProperties;
this.databaseStateVerifier = databaseStateVerifier;
this.liquibaseFactory = liquibaseFactory;
this.tenantDataSourceFactory = tenantDataSourceFactory;
}
@Override
public void afterPropertiesSet() throws Exception {
if (databaseStateVerifier.isLiquibaseDisabled()) {
LOG.warn("Liquibase is disabled. Not upgrading any database.");
return;
}
try {
upgradeTenantStore();
upgradeIndividualTenants();
} catch (LiquibaseException e) {
throw new RuntimeException("Error while migrating the schema", e);
}
}
private void upgradeTenantStore() throws LiquibaseException {
LOG.warn("Upgrading tenant store DB at {}:{}", fineractProperties.getTenant().getHost(), fineractProperties.getTenant().getPort());
logTenantStoreDetails();
if (databaseStateVerifier.isFirstLiquibaseMigration(tenantDataSource)) {
ExtendedSpringLiquibase liquibase = liquibaseFactory.create(tenantDataSource, TENANT_STORE_DB_CONTEXT, INITIAL_SWITCH_CONTEXT);
applyInitialLiquibase(tenantDataSource, liquibase, "tenant store",
(ds) -> !databaseStateVerifier.isTenantStoreOnLatestUpgradableVersion(ds));
}
SpringLiquibase liquibase = liquibaseFactory.create(tenantDataSource, TENANT_STORE_DB_CONTEXT);
liquibase.afterPropertiesSet();
LOG.warn("Tenant store upgrade finished");
}
private void logTenantStoreDetails() {
LOG.info("- fineract.tenant.username: {}", fineractProperties.getTenant().getUsername());
LOG.info("- fineract.tenant.password: ****");
LOG.info("- fineract.tenant.parameters: {}", fineractProperties.getTenant().getParameters());
LOG.info("- fineract.tenant.timezone: {}", fineractProperties.getTenant().getTimezone());
LOG.info("- fineract.tenant.description: {}", fineractProperties.getTenant().getDescription());
LOG.info("- fineract.tenant.identifier: {}", fineractProperties.getTenant().getIdentifier());
LOG.info("- fineract.tenant.name: {}", fineractProperties.getTenant().getName());
}
private void upgradeIndividualTenants() throws LiquibaseException {
LOG.warn("Upgrading all tenants");
List<FineractPlatformTenant> tenants = tenantDetailsService.findAllTenants();
if (isNotEmpty(tenants)) {
for (FineractPlatformTenant tenant : tenants) {
upgradeIndividualTenant(tenant);
}
}
LOG.warn("Tenant upgrades have finished");
}
private void upgradeIndividualTenant(FineractPlatformTenant tenant) throws LiquibaseException {
LOG.info("Upgrade for tenant {} has started", tenant.getTenantIdentifier());
DataSource tenantDataSource = tenantDataSourceFactory.create(tenant);
if (databaseStateVerifier.isFirstLiquibaseMigration(tenantDataSource)) {
ExtendedSpringLiquibase liquibase = liquibaseFactory.create(tenantDataSource, TENANT_DB_CONTEXT, INITIAL_SWITCH_CONTEXT);
applyInitialLiquibase(tenantDataSource, liquibase, tenant.getTenantIdentifier(),
(ds) -> !databaseStateVerifier.isTenantOnLatestUpgradableVersion(ds));
}
SpringLiquibase tenantLiquibase = liquibaseFactory.create(tenantDataSource, TENANT_DB_CONTEXT);
tenantLiquibase.afterPropertiesSet();
LOG.info("Upgrade for tenant {} has finished", tenant.getTenantIdentifier());
}
private void applyInitialLiquibase(DataSource dataSource, ExtendedSpringLiquibase liquibase, String id,
Function<DataSource, Boolean> isUpgradableFn) throws LiquibaseException {
if (databaseStateVerifier.isFlywayPresent(dataSource)) {
if (isUpgradableFn.apply(dataSource)) {
LOG.warn("Cannot proceed with upgrading database {}", id);
LOG.warn("It seems the database doesn't have the latest schema changes applied until the 1.6 release");
throw new SchemaUpgradeNeededException("Make sure to upgrade to Fineract 1.6 first and then to a newer version");
}
LOG.warn("This is the first Liquibase migration for {}. We'll sync the changelog for you and then apply everything else", id);
liquibase.changeLogSync();
LOG.warn("Liquibase changelog sync is complete");
} else {
liquibase.afterPropertiesSet();
}
}
}
@@ -314,6 +314,9 @@ public class AppUserWritePlatformServiceJpaRepositoryImpl implements AppUserWrit
*/
private PlatformDataIntegrityException handleDataIntegrityIssues(final JsonCommand command, final Throwable realCause,
final Exception dve) {
// TODO: this needs to be fixed. The error condition should be independent from the underlying message and
// naming
// of the constraint
if (realCause.getMessage().contains("'username_org'")) {
final String username = command.stringValueOfParameterNamed("username");
final StringBuilder defaultMessageBuilder = new StringBuilder("User with username ").append(username)
@@ -322,6 +325,9 @@ public class AppUserWritePlatformServiceJpaRepositoryImpl implements AppUserWrit
username);
}
// TODO: this needs to be fixed. The error condition should be independent from the underlying message and
// naming
// of the constraint
if (realCause.getMessage().contains("'unique_self_client'")) {
return new PlatformDataIntegrityException("error.msg.user.self.service.user.already.exist",
"Self Service User Id is already created. Go to Admin->Users to edit or delete the self-service user.");