FINERACT-1724: Refactored datatable index and unique constraint handling + creating an index on fk columns for multirow datatables

This commit is contained in:
Arnold Galovics
2023-03-21 11:14:35 +01:00
committed by Arnold Gálovics
parent 66c3abcefb
commit 53018a14c8
17 changed files with 515 additions and 293 deletions
@@ -18,28 +18,24 @@
*/
package org.apache.fineract.infrastructure.dataqueries.handler;
import lombok.RequiredArgsConstructor;
import org.apache.fineract.commands.handler.NewCommandSourceHandler;
import org.apache.fineract.infrastructure.core.api.JsonCommand;
import org.apache.fineract.infrastructure.core.data.CommandProcessingResult;
import org.apache.fineract.infrastructure.dataqueries.service.ReadWriteNonCoreDataService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@Service
@RequiredArgsConstructor
public class CreateDatatableCommandHandler implements NewCommandSourceHandler {
private final ReadWriteNonCoreDataService writePlatformService;
@Autowired
public CreateDatatableCommandHandler(final ReadWriteNonCoreDataService writePlatformService) {
this.writePlatformService = writePlatformService;
}
@Transactional
@Override
public CommandProcessingResult processCommand(final JsonCommand command) {
return this.writePlatformService.createDatatable(command);
return writePlatformService.createDatatable(command);
}
}
@@ -18,28 +18,24 @@
*/
package org.apache.fineract.infrastructure.dataqueries.handler;
import lombok.RequiredArgsConstructor;
import org.apache.fineract.commands.handler.NewCommandSourceHandler;
import org.apache.fineract.infrastructure.core.api.JsonCommand;
import org.apache.fineract.infrastructure.core.data.CommandProcessingResult;
import org.apache.fineract.infrastructure.dataqueries.service.ReadWriteNonCoreDataService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@Service
@RequiredArgsConstructor
public class CreateDatatableEntryCommandHandler implements NewCommandSourceHandler {
private final ReadWriteNonCoreDataService writePlatformService;
@Autowired
public CreateDatatableEntryCommandHandler(final ReadWriteNonCoreDataService writePlatformService) {
this.writePlatformService = writePlatformService;
}
@Transactional
@Override
public CommandProcessingResult processCommand(final JsonCommand command) {
return this.writePlatformService.createNewDatatableEntry(command.entityName(), command.entityId(), command);
return writePlatformService.createNewDatatableEntry(command.entityName(), command.entityId(), command);
}
}
@@ -18,25 +18,21 @@
*/
package org.apache.fineract.infrastructure.dataqueries.handler;
import lombok.RequiredArgsConstructor;
import org.apache.fineract.commands.handler.NewCommandSourceHandler;
import org.apache.fineract.infrastructure.core.api.JsonCommand;
import org.apache.fineract.infrastructure.core.data.CommandProcessingResult;
import org.apache.fineract.infrastructure.core.data.CommandProcessingResultBuilder;
import org.apache.fineract.infrastructure.dataqueries.service.ReadWriteNonCoreDataService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@Service
@RequiredArgsConstructor
public class UpdateDatatableCommandHandler implements NewCommandSourceHandler {
private final ReadWriteNonCoreDataService writePlatformService;
@Autowired
public UpdateDatatableCommandHandler(final ReadWriteNonCoreDataService writePlatformService) {
this.writePlatformService = writePlatformService;
}
@Transactional
@Override
public CommandProcessingResult processCommand(final JsonCommand command) {
@@ -0,0 +1,33 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.fineract.infrastructure.dataqueries.service;
import org.springframework.stereotype.Component;
@Component
public class DatatableKeywordGenerator {
public String generateIndexName(String tableName, String columnName) {
return "idx_%s_%s".formatted(tableName, columnName);
}
public String generateUniqueKeyName(String tableName, String columnName) {
return "uk_%s_%s".formatted(tableName, columnName);
}
}
@@ -33,4 +33,8 @@ public interface GenericDataService {
String wrapSQL(String sql);
List<ResultsetColumnHeaderData> fillResultsetColumnHeaders(String datatable);
boolean isExplicitlyUnique(String datatable, String columnName);
boolean isExplicitlyIndexed(String datatable, String columnName);
}
@@ -25,7 +25,7 @@ import java.time.LocalDate;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
import javax.sql.DataSource;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.fineract.infrastructure.core.exception.PlatformDataIntegrityException;
import org.apache.fineract.infrastructure.core.service.database.DatabaseIndependentQueryService;
@@ -36,7 +36,6 @@ import org.apache.fineract.infrastructure.dataqueries.data.ResultsetColumnHeader
import org.apache.fineract.infrastructure.dataqueries.data.ResultsetColumnValueData;
import org.apache.fineract.infrastructure.dataqueries.data.ResultsetRowData;
import org.apache.fineract.infrastructure.dataqueries.exception.DatatableNotFoundException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.dao.DataAccessException;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.support.rowset.SqlRowSet;
@@ -45,19 +44,13 @@ import org.springframework.stereotype.Service;
@Service
@Slf4j
@RequiredArgsConstructor
public class GenericDataServiceImpl implements GenericDataService {
private final JdbcTemplate jdbcTemplate;
private final DataSource dataSource;
private final RoutingDataSource dataSource;
private final DatabaseIndependentQueryService databaseIndependentQueryService;
@Autowired
public GenericDataServiceImpl(final RoutingDataSource dataSource, final JdbcTemplate jdbcTemplate,
DatabaseIndependentQueryService databaseIndependentQueryService) {
this.dataSource = dataSource;
this.databaseIndependentQueryService = databaseIndependentQueryService;
this.jdbcTemplate = new JdbcTemplate(this.dataSource);
}
private final DatatableKeywordGenerator datatableKeywordGenerator;
@Override
public GenericResultsetData fillGenericResultSet(final String sql) {
@@ -228,11 +221,13 @@ public class GenericDataServiceImpl implements GenericDataService {
final boolean columnNullable = "YES".equalsIgnoreCase(isNullable) || "TRUE".equalsIgnoreCase(isNullable);
final boolean columnIsPrimaryKey = "PRI".equalsIgnoreCase(isPrimaryKey) || "TRUE".equalsIgnoreCase(isPrimaryKey);
final boolean columnIsUnique = checkUnique(datatable, columnName, indexDefinitions);
boolean columnIsIndexed = false;
if (!columnIsUnique) {
columnIsIndexed = checkIndexed(datatable, columnName, indexDefinitions);
}
// primary keys are automatically unique
final boolean columnIsUnique = columnIsPrimaryKey || isExplicitlyUnique(datatable, columnName, indexDefinitions);
// primary keys and unique constrained columns are automatically indexed
final boolean columnIsIndexed = columnIsPrimaryKey || columnIsUnique
|| isExplicitlyIndexed(datatable, columnName, indexDefinitions);
List<ResultsetColumnValueData> columnValues = new ArrayList<>();
String codeName = null;
@@ -240,7 +235,7 @@ public class GenericDataServiceImpl implements GenericDataService {
if ("varchar".equalsIgnoreCase(columnType) || "int".equalsIgnoreCase(columnType) || "integer".equalsIgnoreCase(columnType)) {
if (codePosition > 0) {
codeName = columnName.substring(0, codePosition);
columnValues = retreiveColumnValues(codeName);
columnValues = retrieveColumnValues(codeName);
}
}
@@ -251,16 +246,26 @@ public class GenericDataServiceImpl implements GenericDataService {
return columnHeaders;
}
private boolean checkUnique(String datatable, String columnName, List<IndexDetail> indexDefinitions) {
String keyNameToCheck = "uk_" + datatable + "_" + columnName;
private boolean isExplicitlyUnique(String datatable, String columnName, List<IndexDetail> indexDefinitions) {
String keyNameToCheck = datatableKeywordGenerator.generateUniqueKeyName(datatable, columnName);
return checkKeyPresent(keyNameToCheck, indexDefinitions);
}
private boolean checkIndexed(String datatable, String columnName, List<IndexDetail> indexDefinitions) {
String keyNameToCheck = "idx_" + datatable + "_" + columnName;
@Override
public boolean isExplicitlyUnique(String datatable, String columnName) {
return isExplicitlyUnique(datatable, columnName, getDatatableIndexData(datatable));
}
private boolean isExplicitlyIndexed(String datatable, String columnName, List<IndexDetail> indexDefinitions) {
String keyNameToCheck = datatableKeywordGenerator.generateIndexName(datatable, columnName);
return checkKeyPresent(keyNameToCheck, indexDefinitions);
}
@Override
public boolean isExplicitlyIndexed(String datatable, String columnName) {
return isExplicitlyIndexed(datatable, columnName, getDatatableIndexData(datatable));
}
private boolean checkKeyPresent(String keyNameToCheck, List<IndexDetail> indexDefinitions) {
for (IndexDetail indexDetail : indexDefinitions) {
if (indexDetail.getIndexName().equals(keyNameToCheck)) {
@@ -282,7 +287,7 @@ public class GenericDataServiceImpl implements GenericDataService {
/*
* Candidate for using caching there to get allowed 'column values' from code/codevalue tables
*/
private List<ResultsetColumnValueData> retreiveColumnValues(final String codeName) {
private List<ResultsetColumnValueData> retrieveColumnValues(final String codeName) {
final List<ResultsetColumnValueData> columnValues = new ArrayList<>();
@@ -143,6 +143,7 @@ public class ReadWriteNonCoreDataServiceImpl implements ReadWriteNonCoreDataServ
private final ColumnValidator columnValidator;
private final NamedParameterJdbcTemplate namedParameterJdbcTemplate;
private final SqlInjectionPreventerService preventSqlInjectionService;
private final DatatableKeywordGenerator datatableKeywordGenerator;
@Override
public List<DatatableData> retrieveDatatableNames(final String appTable) {
@@ -753,7 +754,7 @@ public class ReadWriteNonCoreDataServiceImpl implements ReadWriteNonCoreDataServ
}
}
if (unique) {
String uniqueKeyName = "uk_" + dataTableNameAlias + "_" + name;
String uniqueKeyName = datatableKeywordGenerator.generateUniqueKeyName(dataTableNameAlias, name);
constrainBuilder.append(", CONSTRAINT ").append(sqlGenerator.escape(uniqueKeyName)).append(" ")
.append("UNIQUE (" + sqlGenerator.escape(name) + ")");
}
@@ -854,11 +855,19 @@ public class ReadWriteNonCoreDataServiceImpl implements ReadWriteNonCoreDataServ
if (databaseTypeResolver.isMySQL()) {
sqlBuilder.append(" ENGINE=InnoDB DEFAULT CHARSET=UTF8MB4;");
}
log.debug("SQL:: {}", sqlBuilder.toString());
log.debug("SQL:: {}", sqlBuilder);
this.jdbcTemplate.execute(sqlBuilder.toString());
jdbcTemplate.execute(sqlBuilder.toString());
// create indexes
if (multiRow) {
createFkIndex(datatableName, fkColumnName);
} else {
/*
* in case of non-multirow, the primary key of the table is the FK and MySQL and PostgreSQL
* automatically puts an index onto it so no need to create it explicitly
*/
}
createIndexesForTable(datatableName, columns);
registerDatatable(datatableName, apptableName, entitySubType);
registerColumnCodeMapping(codeMappings);
@@ -884,6 +893,11 @@ public class ReadWriteNonCoreDataServiceImpl implements ReadWriteNonCoreDataServ
return new CommandProcessingResultBuilder().withCommandId(command.commandId()).withResourceIdAsString(datatableName).build();
}
private void createFkIndex(String datatableName, String fkColumnName) {
String indexName = datatableKeywordGenerator.generateIndexName(datatableName, fkColumnName);
createIndex(indexName, datatableName, fkColumnName);
}
private void createIndexesForTable(String datatableName, JsonArray columns) {
for (final JsonElement column : columns) {
createIndexForColumn(datatableName, column.getAsJsonObject());
@@ -896,8 +910,8 @@ public class ReadWriteNonCoreDataServiceImpl implements ReadWriteNonCoreDataServ
final Boolean indexed = column.has("indexed") ? column.get("indexed").getAsBoolean() : false;
if (indexed) {
if (!unique) {
String indexName = "idx_" + datatableName + "_" + name;
createIndexForColumnOnTable(indexName, datatableName, name);
String indexName = datatableKeywordGenerator.generateIndexName(datatableName, name);
createIndex(indexName, datatableName, name);
}
}
}
@@ -1072,7 +1086,7 @@ public class ReadWriteNonCoreDataServiceImpl implements ReadWriteNonCoreDataServ
}
if (unique) {
String uniqueKeyName = "uk_" + dataTableNameAlias + "_" + name;
String uniqueKeyName = datatableKeywordGenerator.generateUniqueKeyName(dataTableNameAlias, name);
constrainBuilder.append(",ADD CONSTRAINT ").append(sqlGenerator.escape(uniqueKeyName)).append(" ")
.append("UNIQUE (" + sqlGenerator.escape(name) + ")");
}
@@ -1277,7 +1291,7 @@ public class ReadWriteNonCoreDataServiceImpl implements ReadWriteNonCoreDataServ
sqlBuilder = sqlBuilder.deleteCharAt(indexOfFirstComma);
}
sqlBuilder.append(constrainBuilder);
this.jdbcTemplate.execute(sqlBuilder.toString());
jdbcTemplate.execute(sqlBuilder.toString());
createIndexesForTable(datatableName, addColumns);
registerColumnCodeMapping(codeMappings);
}
@@ -1302,7 +1316,7 @@ public class ReadWriteNonCoreDataServiceImpl implements ReadWriteNonCoreDataServ
}
sqlBuilder.append(constrainBuilder);
try {
this.jdbcTemplate.execute(sqlBuilder.toString());
jdbcTemplate.execute(sqlBuilder.toString());
deleteColumnCodeMapping(removeMappings);
registerColumnCodeMapping(codeMappings);
// update unique constraint
@@ -1310,6 +1324,7 @@ public class ReadWriteNonCoreDataServiceImpl implements ReadWriteNonCoreDataServ
// update indexes
updateIndexesForTable(datatableName, changeColumns, mapColumnNameDefinition);
} catch (final Exception e) {
log.error("Exception while modifying a datatable", e);
if (e.getMessage().contains("Error on rename")) {
throw new PlatformServiceUnavailableException("error.msg.datatable.column.update.not.allowed",
"One of the column name modification not allowed", e);
@@ -1380,8 +1395,8 @@ public class ReadWriteNonCoreDataServiceImpl implements ReadWriteNonCoreDataServ
String name = column.has("name") ? column.get("name").getAsString() : null;
String columnNewName = column.has("newName") ? column.get("newName").getAsString() : null;
final Boolean setUnique = column.has("unique") ? column.get("unique").getAsBoolean() : false;
final Boolean isAlreadyUnique = columnMetaData.getIsColumnUnique();
String uniqueKeyName = "uk_" + datatableName + "_" + name;
final Boolean isAlreadyUnique = genericDataService.isExplicitlyUnique(datatableName, name);
String uniqueKeyName = datatableKeywordGenerator.generateUniqueKeyName(datatableName, name);
if (isAlreadyUnique) {
if (!setUnique) {
@@ -1401,7 +1416,7 @@ public class ReadWriteNonCoreDataServiceImpl implements ReadWriteNonCoreDataServ
private void checkColumnRenameAndCreateUniqueConstraint(String datatableName, String name, String columnNewName, String constraintKey) {
if (columnNewName != null) {
// create constraint with new column name
String uniqueKeyName = "uk_" + datatableName + "_" + columnNewName;
String uniqueKeyName = datatableKeywordGenerator.generateUniqueKeyName(datatableName, columnNewName);
createUniqueConstraint(datatableName, columnNewName, uniqueKeyName);
} else {
// create constraint for column
@@ -1414,7 +1429,7 @@ public class ReadWriteNonCoreDataServiceImpl implements ReadWriteNonCoreDataServ
// drop existing constraint
dropUniqueConstraint(datatableName, existingConstraint);
// create constraint with new column name
String uniqueKeyName = "uk_" + datatableName + "_" + columnNewName;
String uniqueKeyName = datatableKeywordGenerator.generateUniqueKeyName(datatableName, columnNewName);
createUniqueConstraint(datatableName, columnNewName, uniqueKeyName);
}
}
@@ -1452,19 +1467,19 @@ public class ReadWriteNonCoreDataServiceImpl implements ReadWriteNonCoreDataServ
final Boolean setForUnique = column.has("unique") ? column.get("unique").getAsBoolean() : false;
final Boolean setForIndexed = column.has("indexed") ? column.get("indexed").getAsBoolean() : false;
if (!setForUnique) {
final Boolean isAlreadyIndexed = columnMetaData.getIsColumnIndexed();
String uniqueIndexName = "idx_" + datatableName + "_" + name;
final Boolean isAlreadyIndexed = genericDataService.isExplicitlyIndexed(datatableName, name);
String indexName = datatableKeywordGenerator.generateIndexName(datatableName, name);
if (isAlreadyIndexed) {
if (!setForIndexed) {
// drop index
dropIndex(datatableName, uniqueIndexName);
dropIndex(datatableName, indexName);
} else { // if column name changed
checkColumnRenameAndModifyIndex(datatableName, columnNewName, uniqueIndexName);
checkColumnRenameAndModifyIndex(datatableName, columnNewName, indexName);
}
} else {
if (setForIndexed) {
checkColumnRenameAndCreateIndex(datatableName, name, columnNewName, uniqueIndexName);
checkColumnRenameAndCreateIndex(datatableName, name, columnNewName, indexName);
}
}
@@ -1474,12 +1489,12 @@ public class ReadWriteNonCoreDataServiceImpl implements ReadWriteNonCoreDataServ
private void checkColumnRenameAndCreateIndex(String datatableName, String columnExistingName, String columnNewName, String indexName) {
if (columnNewName != null) {
String uniqueIndexName = "idx_" + datatableName + "_" + columnNewName;
String newIndexName = datatableKeywordGenerator.generateIndexName(datatableName, columnNewName);
// create index with new column name
createIndexForColumnOnTable(uniqueIndexName, datatableName, columnNewName);
createIndex(newIndexName, datatableName, columnNewName);
} else {
// create index with previous name
createIndexForColumnOnTable(indexName, datatableName, columnExistingName);
createIndex(indexName, datatableName, columnExistingName);
}
}
@@ -1488,16 +1503,17 @@ public class ReadWriteNonCoreDataServiceImpl implements ReadWriteNonCoreDataServ
// drop index with previous name
dropIndex(datatableName, existingIndex);
// create index with new name
String uniqueIndexName = "idx_" + datatableName + "_" + columnNewName;
createIndexForColumnOnTable(uniqueIndexName, datatableName, columnNewName);
String newIndexName = datatableKeywordGenerator.generateIndexName(datatableName, columnNewName);
createIndex(newIndexName, datatableName, columnNewName);
}
}
private void createIndexForColumnOnTable(String uniqueIndexName, String datatableName, String columnName) {
StringBuilder sqlIndexUpdateBuilder = new StringBuilder();
sqlIndexUpdateBuilder.append("CREATE INDEX ").append(sqlGenerator.escape(uniqueIndexName)).append(" ON ")
.append(sqlGenerator.escape(datatableName)).append(" (").append(sqlGenerator.escape(columnName)).append(");");
this.jdbcTemplate.execute(sqlIndexUpdateBuilder.toString());
private void createIndex(String indexName, String tableName, String columnName) {
String safeIndexName = sqlGenerator.escape(indexName);
String safeTableName = sqlGenerator.escape(tableName);
String safeColumnName = sqlGenerator.escape(columnName);
String sqlIndexUpdateBuilder = "CREATE INDEX %s ON %s (%s);".formatted(safeIndexName, safeTableName, safeColumnName);
jdbcTemplate.execute(sqlIndexUpdateBuilder);
}
private void dropIndex(String datatableName, String uniqueIndexName) {
@@ -1506,7 +1522,7 @@ public class ReadWriteNonCoreDataServiceImpl implements ReadWriteNonCoreDataServ
sqlIndexUpdateBuilder.append("ALTER TABLE ").append(sqlGenerator.escape(datatableName)).append(" ");
}
sqlIndexUpdateBuilder.append("DROP INDEX ").append(sqlGenerator.escape(uniqueIndexName)).append(";");
this.jdbcTemplate.execute(sqlIndexUpdateBuilder.toString());
jdbcTemplate.execute(sqlIndexUpdateBuilder.toString());
}
@Transactional
@@ -1662,6 +1678,7 @@ public class ReadWriteNonCoreDataServiceImpl implements ReadWriteNonCoreDataServ
}
@Override
@Transactional(readOnly = true)
public GenericResultsetData retrieveDataTableGenericResultSet(final String dataTableName, final Long appTableId, final String order,
final Long id) {
@@ -25,6 +25,7 @@ import org.apache.fineract.infrastructure.core.serialization.FromJsonHelper;
import org.apache.fineract.infrastructure.core.service.database.DatabaseSpecificSQLGenerator;
import org.apache.fineract.infrastructure.core.service.database.DatabaseTypeResolver;
import org.apache.fineract.infrastructure.dataqueries.data.DataTableValidator;
import org.apache.fineract.infrastructure.dataqueries.service.DatatableKeywordGenerator;
import org.apache.fineract.infrastructure.dataqueries.service.GenericDataService;
import org.apache.fineract.infrastructure.dataqueries.service.ReadWriteNonCoreDataService;
import org.apache.fineract.infrastructure.dataqueries.service.ReadWriteNonCoreDataServiceImpl;
@@ -48,9 +49,10 @@ public class DataQueriesAutoConfiguration {
final DatatableCommandFromApiJsonDeserializer fromApiJsonDeserializer,
final ConfigurationDomainService configurationDomainService, final CodeReadPlatformService codeReadPlatformService,
final DataTableValidator dataTableValidator, final ColumnValidator columnValidator,
final NamedParameterJdbcTemplate namedParameterJdbcTemplate, final SqlInjectionPreventerService preventSqlInjectionService) {
final NamedParameterJdbcTemplate namedParameterJdbcTemplate, final SqlInjectionPreventerService preventSqlInjectionService,
DatatableKeywordGenerator datatableKeywordGenerator) {
return new ReadWriteNonCoreDataServiceImpl(jdbcTemplate, databaseTypeResolver, sqlGenerator, context, fromJsonHelper,
genericDataService, fromApiJsonDeserializer, configurationDomainService, codeReadPlatformService, dataTableValidator,
columnValidator, namedParameterJdbcTemplate, preventSqlInjectionService);
columnValidator, namedParameterJdbcTemplate, preventSqlInjectionService, datatableKeywordGenerator);
}
}