Fixes to support MySQL 8 due to specific reserved keywords

.github/workflows/build-mysql.yml

@@ -0,0 +1,67 @@
+name: Fineract Gradle build - MySQL
+on: [push, pull_request]
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+
+    services:
+        mariad:
+            image: mysql:8.0
+            ports:
+                - 3306:3306
+            env:
+                MYSQL_ROOT_PASSWORD: mysql
+            options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3
+
+        mock-oauth2-server:
+          image: ghcr.io/navikt/mock-oauth2-server:0.4.0
+          ports:
+            - 9000:9000
+          env:
+            SERVER_PORT: 9000
+            JSON_CONFIG: '{ "interactiveLogin": true, "httpServer": "NettyWrapper", "tokenCallbacks": [ { "issuerId": "auth/realms/fineract", "tokenExpiry": 120, "requestMappings": [{ "requestParam": "scope", "match": "fineract", "claims": { "sub": "mifos", "scope": [ "test" ] } } ] } ] }'
+
+    env:
+        TZ: Asia/Kolkata
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set up JDK 17
+        uses: actions/setup-java@v3
+        with:
+          java-version: '17'
+          distribution: 'zulu'
+          cache: gradle
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 16
+      - name: Congfigure vega-cli
+        run: npm i -g vega-cli --unsafe
+      - name: Validate Gradle wrapper
+        uses: gradle/wrapper-validation-action@859c33240bd026ce8d5f711f5adcc65c2f8eafc1
+      - name: Verify MariaDB connection
+        run: |
+            while ! mysqladmin ping -h"127.0.0.1" -P3306 ; do
+                sleep 1
+            done
+      - name: Initialise databases
+        run: |
+            ./gradlew --no-daemon -q createMySQLDB -PdbName=fineract_tenants
+            ./gradlew --no-daemon -q createMySQLDB -PdbName=fineract_default
+      - name: Install additional software
+        run: |
+            sudo apt-get update
+            sudo apt-get install ghostscript graphviz -y
+
+      - name: Basic Auth Build & Test
+        run: ./gradlew --no-daemon -q --console=plain build test --fail-fast doc -x :twofactor-tests:test -x :oauth2-test:test -PdbType=mysql
+
+      - name: 2FA Build & Test
+        run: ./gradlew --no-daemon -q --console=plain :twofactor-tests:test --fail-fast -PdbType=mysql
+
+      - name: OAuth2 Build & Test
+        run: ./gradlew --no-daemon -q --console=plain :oauth2-tests:test --fail-fast -PdbType=mysql

fineract-provider/build.gradle

@@ -151,6 +151,7 @@ configurations {
 dependencies {
     driver 'org.mariadb.jdbc:mariadb-java-client:2.7.5'
     driver 'org.postgresql:postgresql:42.3.5'
+    driver 'mysql:mysql-connector-java:8.0.29'
 }
 
 URLClassLoader loader = GroovyObject.class.classLoader
@@ -159,7 +160,7 @@ configurations.driver.each {File file ->
 }
 
 task createDB {
-    description= "Creates the MySQL Database. Needs database name to be passed (like: -PdbName=someDBname)"
+    description= "Creates the MariaDB Database. Needs database name to be passed (like: -PdbName=someDBname)"
     doLast {
         def sql = Sql.newInstance( 'jdbc:mariadb://localhost:3306/', mysqlUser, mysqlPassword, 'org.mariadb.jdbc.Driver' )
         sql.execute( 'create database '+"`$dbName`" )
@@ -167,7 +168,7 @@ task createDB {
 }
 
 task dropDB {
-    description= "Drops the specified MySQL database. The database name has to be passed (like: -PdbName=someDBname)"
+    description= "Drops the specified MariaDB database. The database name has to be passed (like: -PdbName=someDBname)"
     doLast {
         def sql = Sql.newInstance( 'jdbc:mariadb://localhost:3306/', mysqlUser, mysqlPassword, 'org.mariadb.jdbc.Driver' )
         sql.execute( 'DROP DATABASE '+"`$dbName`")
@@ -190,6 +191,22 @@ task dropPGDB {
     }
 }
 
+task createMySQLDB {
+    description= "Creates the MySQL Database. Needs database name to be passed (like: -PdbName=someDBname)"
+    doLast {
+        def sql = Sql.newInstance( 'jdbc:mysql://localhost:3306/', mysqlUser, mysqlPassword, 'com.mysql.cj.jdbc.Driver' )
+        sql.execute( 'create database '+"`$dbName`" )
+    }
+}
+
+task dropMySQLDB {
+    description= "Drops the specified MySQL database. The database name has to be passed (like: -PdbName=someDBname)"
+    doLast {
+        def sql = Sql.newInstance( 'jdbc:mysql://localhost:3306/', mysqlUser, mysqlPassword, 'com.mysql.cj.jdbc.Driver' )
+        sql.execute( 'DROP DATABASE '+"`$dbName`")
+    }
+}
+
 task setBlankPassword {
     doLast {
         def sql = Sql.newInstance( 'jdbc:mariadb://localhost:3306/', mysqlUser, mysqlPassword, 'org.mariadb.jdbc.Driver' )

fineract-provider/src/main/java/org/apache/fineract/commands/service/AuditReadPlatformServiceImpl.java

@@ -425,7 +425,7 @@ public class AuditReadPlatformServiceImpl implements AuditReadPlatformService {
 
         sql = " select distinct(entity_name) as entityName from m_permission p ";
         sql += makercheckerCapabilityOnly(useType, currentUser);
-        sql += " order by (CASE WHEN grouping = 'datatable' THEN 'ZZZ' ELSE entity_name END), entity_name";
+        sql += " order by (CASE WHEN " + sqlGenerator.escape("grouping") + " = 'datatable' THEN 'ZZZ' ELSE entity_name END), entity_name";
         final EntityNamesMapper mapper2 = new EntityNamesMapper();
         final List<String> entityNames = this.jdbcTemplate.query(sql, mapper2); // NOSONAR
 

fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/database/DatabaseTypeResolver.java

@@ -29,7 +29,7 @@ import org.springframework.stereotype.Component;
 public class DatabaseTypeResolver implements InitializingBean {
 
     private final Map<String, DatabaseType> driverMapping = Map.of("org.mariadb.jdbc.Driver", DatabaseType.MYSQL, "com.mysql.jdbc.Driver",
-            DatabaseType.MYSQL, "org.postgresql.Driver", DatabaseType.POSTGRESQL);
+            DatabaseType.MYSQL, "com.mysql.cj.jdbc.Driver", DatabaseType.MYSQL, "org.postgresql.Driver", DatabaseType.POSTGRESQL);
 
     private final AtomicReference<DatabaseType> currentDatabaseType = new AtomicReference<>();
     private final HikariConfig hikariConfig;

fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/ReadWriteNonCoreDataServiceImpl.java

@@ -35,7 +35,9 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Stream;
 import javax.persistence.PersistenceException;
+import lombok.RequiredArgsConstructor;
 import org.apache.commons.lang3.BooleanUtils;
 import org.apache.commons.lang3.ObjectUtils;
 import org.apache.commons.lang3.StringUtils;
@@ -72,7 +74,6 @@ import org.apache.fineract.infrastructure.security.utils.SQLInjectionValidator;
 import org.apache.fineract.useradministration.domain.AppUser;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.dao.DataAccessException;
 import org.springframework.dao.DataIntegrityViolationException;
 import org.springframework.dao.EmptyResultDataAccessException;
@@ -86,6 +87,7 @@ import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 
 @Service
+@RequiredArgsConstructor
 public class ReadWriteNonCoreDataServiceImpl implements ReadWriteNonCoreDataService {
 
     private static final String DATATABLE_NAME_REGEX_PATTERN = "^[a-zA-Z][a-zA-Z0-9\\-_\\s]{0,48}[a-zA-Z0-9]$";
@@ -110,7 +112,7 @@ public class ReadWriteNonCoreDataServiceImpl implements ReadWriteNonCoreDataServ
     private final DatabaseSpecificSQLGenerator sqlGenerator;
     private final PlatformSecurityContext context;
     private final FromJsonHelper fromJsonHelper;
-    private final JsonParserHelper helper;
+    private final JsonParserHelper helper = new JsonParserHelper();
     private final GenericDataService genericDataService;
     private final DatatableCommandFromApiJsonDeserializer fromApiJsonDeserializer;
     private final ConfigurationDomainService configurationDomainService;
@@ -120,29 +122,6 @@ public class ReadWriteNonCoreDataServiceImpl implements ReadWriteNonCoreDataServ
     private final NamedParameterJdbcTemplate namedParameterJdbcTemplate;
     private final SqlInjectionPreventerService preventSqlInjectionService;
 
-    @Autowired(required = true)
-    public ReadWriteNonCoreDataServiceImpl(final JdbcTemplate jdbcTemplate, final NamedParameterJdbcTemplate namedParameterJdbcTemplate,
-            final PlatformSecurityContext context, final FromJsonHelper fromJsonHelper, final GenericDataService genericDataService,
-            final DatatableCommandFromApiJsonDeserializer fromApiJsonDeserializer, final CodeReadPlatformService codeReadPlatformService,
-            final ConfigurationDomainService configurationDomainService, final DataTableValidator dataTableValidator,
-            final ColumnValidator columnValidator, DatabaseTypeResolver databaseTypeResolver, DatabaseSpecificSQLGenerator sqlGenerator,
-            SqlInjectionPreventerService sqlInjectionPreventerService) {
-        this.databaseTypeResolver = databaseTypeResolver;
-        this.sqlGenerator = sqlGenerator;
-        this.jdbcTemplate = jdbcTemplate;
-        this.context = context;
-        this.fromJsonHelper = fromJsonHelper;
-        this.helper = new JsonParserHelper();
-        this.genericDataService = genericDataService;
-        this.fromApiJsonDeserializer = fromApiJsonDeserializer;
-        this.codeReadPlatformService = codeReadPlatformService;
-        this.configurationDomainService = configurationDomainService;
-        this.dataTableValidator = dataTableValidator;
-        this.columnValidator = columnValidator;
-        this.namedParameterJdbcTemplate = namedParameterJdbcTemplate;
-        this.preventSqlInjectionService = sqlInjectionPreventerService;
-    }
-
     @Override
     public List<DatatableData> retrieveDatatableNames(final String appTable) {
         Object[] params = new Object[] { this.context.authenticatedUser().getId() };
@@ -306,14 +285,16 @@ public class ReadWriteNonCoreDataServiceImpl implements ReadWriteNonCoreDataServ
         final String updatePermissionChecker = "'UPDATE_" + dataTableName + "_CHECKER'";
         final String deletePermission = "'DELETE_" + dataTableName + "'";
         final String deletePermissionChecker = "'DELETE_" + dataTableName + "_CHECKER'";
+        final List<String> escapedColumns = Stream.of("grouping", "code", "action_name", "entity_name", "can_maker_checker")
+                .map(sqlGenerator::escape).toList();
+        final String columns = String.join(", ", escapedColumns);
 
-        return "insert into m_permission (grouping, code, action_name, entity_name, can_maker_checker) values " + "('datatable', "
-                + createPermission + ", 'CREATE', '" + dataTableName + "', true)," + "('datatable', " + createPermissionChecker
-                + ", 'CREATE', '" + dataTableName + "', false)," + "('datatable', " + readPermission + ", 'READ', '" + dataTableName
-                + "', false)," + "('datatable', " + updatePermission + ", 'UPDATE', '" + dataTableName + "', true)," + "('datatable', "
-                + updatePermissionChecker + ", 'UPDATE', '" + dataTableName + "', false)," + "('datatable', " + deletePermission
-                + ", 'DELETE', '" + dataTableName + "', true)," + "('datatable', " + deletePermissionChecker + ", 'DELETE', '"
-                + dataTableName + "', false)";
+        return "insert into m_permission (" + columns + ") values " + "('datatable', " + createPermission + ", 'CREATE', '" + dataTableName
+                + "', true)," + "('datatable', " + createPermissionChecker + ", 'CREATE', '" + dataTableName + "', false),"
+                + "('datatable', " + readPermission + ", 'READ', '" + dataTableName + "', false)," + "('datatable', " + updatePermission
+                + ", 'UPDATE', '" + dataTableName + "', true)," + "('datatable', " + updatePermissionChecker + ", 'UPDATE', '"
+                + dataTableName + "', false)," + "('datatable', " + deletePermission + ", 'DELETE', '" + dataTableName + "', true),"
+                + "('datatable', " + deletePermissionChecker + ", 'DELETE', '" + dataTableName + "', false)";
 
     }
 
@@ -918,7 +899,7 @@ public class ReadWriteNonCoreDataServiceImpl implements ReadWriteNonCoreDataServ
      *            Name of data table
      * @param column
      *            JSON encoded array of column properties
-     * @see https://mifosforge.jira.com/browse/MIFOSX-1145
+     * @see <a href="https://mifosforge.jira.com/browse/MIFOSX-1145">MIFOSX-1145</a>
      **/
     private void removeNullValuesFromStringColumn(final String datatableName, final JsonObject column,
             final Map<String, ResultsetColumnHeaderData> mapColumnNameDefinition) {

fineract-provider/src/main/java/org/apache/fineract/infrastructure/survey/service/WriteSurveyServiceImpl.java

@@ -18,10 +18,13 @@
  */
 package org.apache.fineract.infrastructure.survey.service;
 
+import java.util.List;
+import java.util.stream.Stream;
+import lombok.RequiredArgsConstructor;
 import org.apache.fineract.infrastructure.core.api.JsonCommand;
 import org.apache.fineract.infrastructure.core.data.CommandProcessingResult;
+import org.apache.fineract.infrastructure.core.service.database.DatabaseSpecificSQLGenerator;
 import org.apache.fineract.infrastructure.dataqueries.service.ReadWriteNonCoreDataService;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 
@@ -29,15 +32,11 @@ import org.springframework.transaction.annotation.Transactional;
  * Created by Cieyou on 3/13/14.
  */
 @Service
+@RequiredArgsConstructor
 public class WriteSurveyServiceImpl implements WriteSurveyService {
 
     private final ReadWriteNonCoreDataService readWriteNonCoreDataService;
-
-    @Autowired(required = true)
-    WriteSurveyServiceImpl(final ReadWriteNonCoreDataService readWriteNonCoreDataService) {
-        this.readWriteNonCoreDataService = readWriteNonCoreDataService;
-
-    }
+    private final DatabaseSpecificSQLGenerator sqlGenerator;
 
     @Override
     @Transactional
@@ -59,14 +58,16 @@ public class WriteSurveyServiceImpl implements WriteSurveyService {
         final String updatePermissionChecker = "'UPDATE_" + dataTableName + "_CHECKER'";
         final String deletePermission = "'DELETE_" + dataTableName + "'";
         final String deletePermissionChecker = "'DELETE_" + dataTableName + "_CHECKER'";
+        final List<String> escapedColumns = Stream.of("grouping", "code", "action_name", "entity_name", "can_maker_checker")
+                .map(sqlGenerator::escape).toList();
+        final String columns = String.join(", ", escapedColumns);
 
-        return "insert into m_permission (grouping, code, action_name, entity_name, can_maker_checker) values " + "('datatable', "
-                + createPermission + ", 'CREATE', '" + dataTableName + "', false)," + "('datatable', " + createPermissionChecker
-                + ", 'CREATE', '" + dataTableName + "', false)," + "('datatable', " + readPermission + ", 'READ', '" + dataTableName
-                + "', false)," + "('datatable', " + updatePermission + ", 'UPDATE', '" + dataTableName + "', false)," + "('datatable', "
-                + updatePermissionChecker + ", 'UPDATE', '" + dataTableName + "', false)," + "('datatable', " + deletePermission
-                + ", 'DELETE', '" + dataTableName + "', false)," + "('datatable', " + deletePermissionChecker + ", 'DELETE', '"
-                + dataTableName + "', false)";
+        return "insert into m_permission (" + columns + ") values " + "('datatable', " + createPermission + ", 'CREATE', '" + dataTableName
+                + "', false)," + "('datatable', " + createPermissionChecker + ", 'CREATE', '" + dataTableName + "', false),"
+                + "('datatable', " + readPermission + ", 'READ', '" + dataTableName + "', false)," + "('datatable', " + updatePermission
+                + ", 'UPDATE', '" + dataTableName + "', false)," + "('datatable', " + updatePermissionChecker + ", 'UPDATE', '"
+                + dataTableName + "', false)," + "('datatable', " + deletePermission + ", 'DELETE', '" + dataTableName + "', false),"
+                + "('datatable', " + deletePermissionChecker + ", 'DELETE', '" + dataTableName + "', false)";
     }
 
     @Transactional

fineract-provider/src/main/resources/META-INF/orm.xml

@@ -28,6 +28,7 @@
 
     <persistence-unit-metadata>
         <persistence-unit-defaults>
+            <delimited-identifiers/>
             <entity-listeners>
                 <entity-listener
                     class="org.springframework.data.jpa.domain.support.AuditingEntityListener" />

integration-tests/build.gradle

@@ -31,6 +31,15 @@ apply from: 'dependencies.gradle'
 // enable when all tests are migrated
 tasks.cucumber.onlyIf {false}
 
+// Allow external drivers to be used for the tests without packaging it
+// mainly due to license incompatibilities
+configurations {
+    driver
+}
+dependencies {
+    driver 'mysql:mysql-connector-java:8.0.29'
+}
+
 cargo {
     containerId "tomcat9x"
 
@@ -41,7 +50,7 @@ cargo {
     }
 
     local {
-        logLevel = 'medium'
+        logLevel = 'low'
         // outputFile = file('build/output.log')
         installer {
             installConfiguration = configurations.tomcat
@@ -49,13 +58,20 @@ cargo {
             extractDir = file("$buildDir/tomcat")
         }
         startStopTimeout = 240000
+        sharedClasspath = configurations.driver
         containerProperties {
             def jvmArgs = '--add-exports=java.naming/com.sun.jndi.ldap=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.security=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.management/javax.management=ALL-UNNAMED --add-opens=java.naming/javax.naming=ALL-UNNAMED '
             if (project.hasProperty('localDebug')) {
                 jvmArgs += ' -agentlib:jdwp=transport=dt_socket,server=y,address=*:9000,suspend=n -Xmx2G -Duser.timezone=Asia/Kolkata '
             }
-            if (project.hasProperty('dbType') && 'postgresql'.equalsIgnoreCase(dbType)) {
-                jvmArgs += '-Dspring.datasource.hikari.driverClassName=org.postgresql.Driver -Dspring.datasource.hikari.jdbcUrl=jdbc:postgresql://localhost:5432/fineract_tenants -Dspring.datasource.hikari.username=root -Dspring.datasource.hikari.password=postgres -Dfineract.tenant.host=localhost -Dfineract.tenant.port=5432 -Dfineract.tenant.username=root -Dfineract.tenant.password=postgres'
+            if (project.hasProperty('dbType')) {
+                if ('postgresql'.equalsIgnoreCase(dbType)) {
+                    jvmArgs += '-Dspring.datasource.hikari.driverClassName=org.postgresql.Driver -Dspring.datasource.hikari.jdbcUrl=jdbc:postgresql://localhost:5432/fineract_tenants -Dspring.datasource.hikari.username=root -Dspring.datasource.hikari.password=postgres -Dfineract.tenant.host=localhost -Dfineract.tenant.port=5432 -Dfineract.tenant.username=root -Dfineract.tenant.password=postgres'
+                } else if ('mysql'.equalsIgnoreCase(dbType)) {
+                    jvmArgs += '-Dspring.datasource.hikari.driverClassName=com.mysql.cj.jdbc.Driver -Dspring.datasource.hikari.jdbcUrl=jdbc:mysql://localhost:3306/fineract_tenants -Dspring.datasource.hikari.username=root -Dspring.datasource.hikari.password=mysql -Dfineract.tenant.host=localhost -Dfineract.tenant.port=3306 -Dfineract.tenant.username=root -Dfineract.tenant.password=mysql'
+                } else {
+                    throw new GradleException('Provided dbType is not supported')
+                }
             } else {
                 jvmArgs += '-Dspring.datasource.hikari.driverClassName=org.mariadb.jdbc.Driver -Dspring.datasource.hikari.jdbcUrl=jdbc:mariadb://localhost:3306/fineract_tenants -Dspring.datasource.hikari.username=root -Dspring.datasource.hikari.password=mysql -Dfineract.tenant.host=localhost -Dfineract.tenant.port=3306 -Dfineract.tenant.username=root -Dfineract.tenant.password=mysql'
             }