FINERACT-982 - Completely ditch use of Drizzle JDBC Driver after all

.github/workflows/build-twofactor.yml

@@ -3,8 +3,17 @@ on: [push, pull_request]
 
 jobs:
   build:
-  # We need 18.04 for this - the 20.04 image contains MySQL 8.0, and the GitHub Action to install 5.7 is not permitted in the Apache repo.
+    runs-on: ubuntu-20.04
-    runs-on: ubuntu-18.04
+
+    services:
+        mariad:
+            image: mariadb:10.6
+            ports:
+                - 3306:3306
+            env:
+                MARIADB_ROOT_PASSWORD: mysql
+            options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3
+
     env:
         TZ: Asia/Kolkata
     steps:
@@ -26,10 +35,11 @@ jobs:
           distribution: 'zulu'
       - name: Validate Gradle wrapper
         uses: gradle/wrapper-validation-action@e6e38bacfdf1a337459f332974bb2327a31aaf4b
-      - name: Start up MySQL 5.7
+      - name: Verify MariaDB connection
         run: |
-          sudo /etc/init.d/mysql start
+            while ! mysqladmin ping -h"127.0.0.1" -P3306 ; do
-          echo "USE mysql;\nALTER USER 'root'@'localhost' IDENTIFIED BY 'mysql';\n" | mysql -u root -proot
+                sleep 1
+            done
       - name: Initialise databases
         run: |
             ./gradlew --no-daemon -q createDB -PdbName=fineract_tenants
@@ -39,4 +49,4 @@ jobs:
             sudo apt-get update
             sudo apt-get install ghostscript -y
       - name: Build & Test
-        run: ./gradlew --no-daemon -q --console=plain licenseMain licenseTest check build test --fail-fast doc -x :integration-tests:test -Ptwofactor=enabled
+        run: ./gradlew --no-daemon -q --console=plain build test --fail-fast -x :integration-tests:test -Ptwofactor=enabled

.github/workflows/build.yml

@@ -3,8 +3,16 @@ on: [push, pull_request]
 
 jobs:
   build:
-  # We need 18.04 for this - the 20.04 image contains MySQL 8.0, and the GitHub Action to install 5.7 is not permitted in the Apache repo.
+    runs-on: ubuntu-20.04
-    runs-on: ubuntu-18.04
+
+    services:
+        mariad:
+            image: mariadb:10.6
+            ports:
+                - 3306:3306
+            env:
+                MARIADB_ROOT_PASSWORD: mysql
+            options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3
     env:
         TZ: Asia/Kolkata
     steps:
@@ -26,10 +34,11 @@ jobs:
           distribution: 'zulu'
       - name: Validate Gradle wrapper
         uses: gradle/wrapper-validation-action@e6e38bacfdf1a337459f332974bb2327a31aaf4b
-      - name: Start up MySQL 5.7
+      - name: Verify MariaDB connection
         run: |
-          sudo /etc/init.d/mysql start
+            while ! mysqladmin ping -h"127.0.0.1" -P3306 ; do
-          echo "USE mysql;\nALTER USER 'root'@'localhost' IDENTIFIED BY 'mysql';\n" | mysql -u root -proot
+                sleep 1
+            done
       - name: Initialise databases
         run: |
             ./gradlew --no-daemon -q createDB -PdbName=fineract_tenants

Dockerfile

@@ -27,13 +27,9 @@ RUN ./gradlew --no-daemon -q -x rat -x compileTestJava -x test -x spotlessJavaCh
 WORKDIR /fineract/target
 RUN jar -xf /fineract/fineract-provider/build/libs/fineract-provider.jar
 
-# https://issues.apache.org/jira/browse/LEGAL-462
+# We download separately a JDBC driver (which not allowed to be included in Apache binary distribution)
-# https://issues.apache.org/jira/browse/FINERACT-762
-# We include an alternative JDBC driver (which is faster, but not allowed to be default in Apache distribution)
-# allowing implementations to switch the driver used by changing start-up parameters (for both tenants and each tenant DB)
-# The commented out lines in the docker-compose.yml illustrate how to do this.
 WORKDIR /fineract/target/BOOT-INF/libs
-RUN wget -q https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.23/mysql-connector-java-8.0.23.jar
+RUN wget -q https://downloads.mariadb.com/Connectors/java/connector-java-2.7.3/mariadb-java-client-2.7.3.jar
 
 # =========================================
 

README.md

@@ -21,17 +21,17 @@ If you are interested in contributing to this project, but perhaps don't quite k
 Requirements
 ============
 * Java >= 11 (OpenJDK JVM is tested by our CI on Travis)
-* MySQL 5.7
+* MariaDB 10.6
 
 You can run the required version of the database server in a container, instead of having to install it, like this:
 
-    docker run --name mysql-5.7 -p 3306:3306 -e MYSQL_ROOT_PASSWORD=mysql -d mysql:5.7
+    docker run --name mariadb-10.6 -p 3306:3306 -e MARIADB_ROOT_PASSWORD=mysql -d mariadb:10.6
 
 and stop and destroy it like this:
 
-    docker rm -f mysql-5.7
+    docker rm -f mariadb-10.6
 
-Beware that this database container database keeps its state inside the container and not on the host filesystem.  It is lost when you destroy (rm) this container.  This is typically fine for development.  See [Caveats: Where to Store Data on the database container documentation](https://hub.docker.com/_/mysql) re. how to make it persistent instead of ephemeral.
+Beware that this database container database keeps its state inside the container and not on the host filesystem.  It is lost when you destroy (rm) this container.  This is typically fine for development.  See [Caveats: Where to Store Data on the database container documentation](https://hub.docker.com/_/mariadb) re. how to make it persistent instead of ephemeral.
 
 Tomcat v9 is only required if you wish to deploy the Fineract WAR to a separate external servlet container.  Note that you do not require to install Tomcat to develop Fineract, or to run it in production if you use the self-contained JAR, which transparently embeds a servlet container using Spring Boot.  (Until FINERACT-730, Tomcat 7/8 were also supported, but now Tomcat 9 is required.)
 
@@ -48,8 +48,9 @@ Run the following commands:
 Instructions to build the JAR file
 ============
 1. Clone the repository or download and extract the archive file to your local directory.
-2. Run `./gradlew clean bootJar` to build a modern cloud native fully self contained JAR file which will be created at `build/libs` directory.
+2. Run `./gradlew clean bootJar` to build a modern cloud native fully self contained JAR file which will be created at `fineract-provider/build/libs` directory.
-3. Start it using `java -jar build/libs/fineract-provider.jar` (does not require external Tomcat)
+3. As we are not allowed to include a JDBC driver in the built JAR, download a JDBC driver of your choice. For example: `wget https://downloads.mariadb.com/Connectors/java/connector-java-2.7.3/mariadb-java-client-2.7.3.jar`
+4. Start the jar and pass the directory where you have downloaded the JDBC driver as loader.path, for example: `java -Dloader.path=. -jar fineract-provider/build/libs/fineract-provider.jar` (does not require external Tomcat)
 
 The tenants database connection details are configured [via environment variables (as with Docker container)](#instructions-to-run-using-docker-and-docker-compose), e.g. like this:
 
@@ -263,7 +264,7 @@ is used in development when running integration tests that use the Flyway librar
 driver is however not included in and distributed with the Fineract product and is not
 required to use the product.
 If you are developer and object to using the LGPL licensed Connector/J JDBC driver,
-simply do not run the integration tests that use the Flyway library.
+simply do not run the integration tests that use the Flyway library and/or use another JDBC driver.
 As discussed in [LEGAL-462](https://issues.apache.org/jira/browse/LEGAL-462), this project therefore
 complies with the [Apache Software Foundation third-party license policy](https://www.apache.org/legal/resolved.html).
 

build.gradle

@@ -119,7 +119,6 @@ allprojects  {
             dependency 'com.google.truth.extensions:truth-java8-extension:1.1.3'
             dependency 'org.apache.commons:commons-email:1.5'
             dependency 'commons-io:commons-io:2.11.0'
-            dependency 'org.drizzle.jdbc:drizzle-jdbc:1.4'
             dependency 'com.github.librepdf:openpdf:1.3.26'
             dependency 'org.mnode.ical4j:ical4j:3.1.1'
             dependency 'org.quartz-scheduler:quartz:2.3.2'
@@ -179,7 +178,7 @@ allprojects  {
             dependency "com.google.code.findbugs:jsr305:3.0.2"
             dependency "commons-codec:commons-codec:1.15"
 
-            dependency ('org.flywaydb:flyway-core:7.15.0') {
+            dependency ('org.flywaydb:flyway-core:8.1.0') {
                 // https://issues.apache.org/jira/browse/FINERACT-1172
                 // https://github.com/apache/fineract/pull/1355
                 // https://github.com/flyway/flyway/issues/2957

docker-compose.yml

@@ -22,8 +22,9 @@ version: '3.7'
 services:
   # Backend service
   fineractmysql:
-    image: mysql:5.7
+    image: mariadb:10.6
     volumes:
+      - ./fineract-db/server_collation.cnf:/etc/mysql/conf.d/server_collation.cnf
       - ./fineract-db/docker:/docker-entrypoint-initdb.d:Z,ro
     restart: always
     environment:
@@ -48,12 +49,12 @@ services:
       fineractmysql:
         condition: service_healthy
     environment:
-      - DRIVERCLASS_NAME=org.drizzle.jdbc.DrizzleDriver
+      - DRIVERCLASS_NAME=org.mariadb.jdbc.Driver
       - PROTOCOL=jdbc
       - node_id=1
-      - SUB_PROTOCOL=mysql:thin
+      - SUB_PROTOCOL=mariadb
-      - fineract_tenants_driver=org.drizzle.jdbc.DrizzleDriver
+      - fineract_tenants_driver=org.mariadb.jdbc.Driver
-      - fineract_tenants_url=jdbc:mysql:thin://fineractmysql:3306/fineract_tenants
+      - fineract_tenants_url=jdbc:mariadb://fineractmysql:3306/fineract_tenants
       - fineract_tenants_uid=root
       - fineract_tenants_pwd=skdcnwauicn2ucnaecasdsajdnizucawencascdca
       - FINERACT_DEFAULT_TENANTDB_HOSTNAME=fineractmysql
@@ -72,9 +73,3 @@ services:
     depends_on:
       fineract-server:
         condition: service_healthy
-
-# https://issues.apache.org/jira/browse/FINERACT-762
-# To use an altnerative JDBC driver (which is faster, but not allowed to be default in Apache distribution)
-# replace org.drizzle.jdbc.DrizzleDriver with com.mysql.jdbc.Driver in DRIVERCLASS_NAME and fineract_tenants_driver,
-# and remove ":thin:" from SUB_PROTOCOL and fineract_tenants_url.  Note that the mysql-connector-java-*.jar is already
-# bundled in the container by the Dockerfile, but just not used by default.

fineract-db/server_collation.cnf

@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+[client]
+default-character-set=utf8mb4
+
+[mysql]
+default-character-set=utf8mb4
+
+[mysqld]
+collation-server=utf8mb4_unicode_ci
+init-connect='SET NAMES utf8mb4'
+character-set-server=utf8mb4

fineract-provider/build.gradle

@@ -196,15 +196,8 @@ eclipse
 sourceSets.main.output.resourcesDir = sourceSets.main.java.outputDir
 sourceSets.test.output.resourcesDir = sourceSets.test.java.outputDir
 
-/* Exclude maria db related files for non dev builds */
 if (!(project.hasProperty('env') && project.getProperty('env') == 'dev')) {
     sourceSets {
-        main {
-            java {
-                exclude '**/ServerWithMariaDB*'
-                exclude '**/MariaDB4j*'
-            }
-        }
         test {
             java {
                 exclude '**/core/boot/tests/**'
@@ -224,7 +217,7 @@ configurations {
     driver
 }
 dependencies {
-    driver 'org.drizzle.jdbc:drizzle-jdbc:1.4'
+    driver 'org.mariadb.jdbc:mariadb-java-client:2.7.3'
 }
 
 URLClassLoader loader = GroovyObject.class.classLoader
@@ -235,7 +228,7 @@ configurations.driver.each {File file ->
 task createDB {
     description= "Creates the Database. Needs database name to be passed (like: -PdbName=someDBname)"
     doLast {
-        def sql = Sql.newInstance( 'jdbc:mysql:thin://localhost:3306/', mysqlUser, mysqlPassword, 'org.drizzle.jdbc.DrizzleDriver' )
+        def sql = Sql.newInstance( 'jdbc:mariadb://localhost:3306/', mysqlUser, mysqlPassword, 'org.mariadb.jdbc.Driver' )
         sql.execute( 'create database '+"`$dbName`" )
     }
 }
@@ -243,13 +236,13 @@ task createDB {
 task dropDB {
     description= "Drops the specified database. The database name has to be passed (like: -PdbName=someDBname)"
     doLast {
-        def sql = Sql.newInstance( 'jdbc:mysql:thin://localhost:3306/', mysqlUser, mysqlPassword, 'org.drizzle.jdbc.DrizzleDriver' )
+        def sql = Sql.newInstance( 'jdbc:mariadb://localhost:3306/', mysqlUser, mysqlPassword, 'org.mariadb.jdbc.Driver' )
         sql.execute( 'DROP DATABASE '+"`$dbName`")
     }
 }
 task setBlankPassword {
     doLast {
-        def sql = Sql.newInstance( 'jdbc:mysql:thin://localhost:3306/', mysqlUser, mysqlPassword, 'org.drizzle.jdbc.DrizzleDriver' )
+        def sql = Sql.newInstance( 'jdbc:mariadb://localhost:3306/', mysqlUser, mysqlPassword, 'org.mariadb.jdbc.Driver' )
         sql.execute('USE `fineract_tenants`')
         sql.execute('UPDATE fineract_tenants.tenants SET schema_server = \'localhost\', schema_server_port = \'3306\', schema_username = \'mifos\', schema_password = \'mysql\' WHERE id=1;')
     }
@@ -259,6 +252,10 @@ bootRun {
     jvmArgs = [
         "-Dspring.output.ansi.enabled=ALWAYS"
     ]
+
+    dependencies {
+        implementation 'org.mariadb.jdbc:mariadb-java-client:2.7.3'
+    }
 }
 
 springBoot {

fineract-provider/dependencies.gradle

@@ -54,7 +54,6 @@ dependencies {
             'org.apache.poi:poi-ooxml-schemas',
             'org.apache.tika:tika-core',
 
-            'org.drizzle.jdbc:drizzle-jdbc',
             'org.flywaydb:flyway-core',
 
             'com.github.librepdf:openpdf',

fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/domain/MySQLDictionaryCustom.java

@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.fineract.infrastructure.core.domain;
-
-import java.sql.Connection;
-import java.sql.SQLException;
-import org.apache.openjpa.jdbc.sql.BooleanRepresentationFactory;
-import org.apache.openjpa.jdbc.sql.MySQLDictionary;
-
-@SuppressWarnings("unchecked")
-public class MySQLDictionaryCustom extends MySQLDictionary {
-
-    public MySQLDictionaryCustom() {
-
-        this.supportsSubselect = true;
-        this.booleanRepresentation = BooleanRepresentationFactory.BOOLEAN;
-        this.supportsGetGeneratedKeys = false;
-        this.allowsAliasInBulkClause = true;
-        this.useWildCardForCount = true;
-    }
-
-    @Override
-    public void connectedConfiguration(Connection conn) throws SQLException {
-        super.connectedConfiguration(conn);
-        this.supportsSubselect = true;
-        this.supportsGetGeneratedKeys = false;
-        this.allowsAliasInBulkClause = true;
-        this.useWildCardForCount = true;
-    }
-}

fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/ResultsetColumnHeaderData.java

@@ -182,13 +182,17 @@ public final class ResultsetColumnHeaderData implements Serializable {
     }
 
     private boolean isAnyInteger() {
-        return isInt() || isSmallInt() || isTinyInt() || isMediumInt() || isBigInt() || isLong();
+        return isInt() || isInteger() || isSmallInt() || isTinyInt() || isMediumInt() || isBigInt() || isLong();
     }
 
     private boolean isInt() {
         return "int".equalsIgnoreCase(this.columnType);
     }
 
+    private boolean isInteger() {
+        return "integer".equalsIgnoreCase(this.columnType);
+    }
+
     private boolean isSmallInt() {
         return "smallint".equalsIgnoreCase(this.columnType);
     }

fineract-provider/src/main/resources/META-INF/persistence.xml

@@ -29,7 +29,6 @@
         <provider>org.apache.openjpa.persistence.PersistenceProviderImpl</provider>
         <properties>
             <property name="openjpa.Compatibility" value="QuotedNumbersInQueries=true"/>
-            <property name="openjpa.jdbc.DBDictionary" value="org.apache.fineract.infrastructure.core.domain.MySQLDictionaryCustom"/>
             <!--<property name="openjpa.InverseManager" value="true(Action=warn)"/> -->
             <property name="openjpa.Log" value="slf4j"/>
             <property name="openjpa.jdbc.MappingDefaults" value="ForeignKeyDeleteAction=CASCADE"/>

fineract-provider/src/main/resources/META-INF/spring/jdbc.properties

@@ -17,11 +17,11 @@
 # under the License.
 #
 
-DRIVERCLASS_NAME:org.drizzle.jdbc.DrizzleDriver
+DRIVERCLASS_NAME:org.mariadb.jdbc.Driver
 PROTOCOL:jdbc
-SUB_PROTOCOL:mysql:thin
+SUB_PROTOCOL:mariadb
 
-fineract_tenants_driver:org.drizzle.jdbc.DrizzleDriver
+fineract_tenants_driver:org.mariadb.jdbc.Driver
-fineract_tenants_url:jdbc:mysql:thin://localhost:3306/fineract_tenants
+fineract_tenants_url:jdbc:mariadb://localhost:3306/fineract_tenants
 fineract_tenants_uid:root
 fineract_tenants_pwd:mysql

fineract-provider/src/main/resources/sql/migrations/core_db/V342__topic_module_table.sql

@@ -38,8 +38,8 @@ CREATE TABLE IF NOT EXISTS `topic_subscriber` (
   CONSTRAINT `fk_topic_has_m_appuser_m_appuser1` FOREIGN KEY (`user_id`) REFERENCES `m_appuser` (`id`)
 ) ENGINE = InnoDB;
 
-INSERT INTO topic (enabled, entity_type, entity_id, member_type, title) SELECT true, 'OFFICE', o.id as entity_id, UPPER(r.name) as member_type, CONCAT(r.name, ' of ', o.name) as title FROM m_office o, m_role r WHERE o.parent_id IS NULL AND CONCAT(r.name, ' of ', o.name) NOT IN (SELECT title FROM topic);
+INSERT INTO topic (enabled, entity_type, entity_id, member_type, title) SELECT true, 'OFFICE', o.id as entity_id, UPPER(r.name) as member_type, CONCAT(r.name, ' of ', o.name) as title FROM m_office o, m_role r WHERE o.parent_id IS NULL AND CONCAT(r.name, ' of ', o.name) COLLATE utf8mb4_general_ci  NOT IN (SELECT title FROM topic);
 
-INSERT INTO topic (enabled, entity_type, entity_id, member_type, title) SELECT true, 'BRANCH', o.id as entity_id, UPPER(r.name) as member_type, CONCAT(r.name, ' of ', o.name) as title FROM m_office o, m_role r WHERE o.parent_id IS NOT NULL AND CONCAT(r.name, ' of ', o.name) NOT IN (SELECT title FROM topic);
+INSERT INTO topic (enabled, entity_type, entity_id, member_type, title) SELECT true, 'BRANCH', o.id as entity_id, UPPER(r.name) as member_type, CONCAT(r.name, ' of ', o.name) as title FROM m_office o, m_role r WHERE o.parent_id IS NOT NULL AND CONCAT(r.name, ' of ', o.name) COLLATE utf8mb4_general_ci NOT IN (SELECT title FROM topic);
 
-INSERT INTO topic_subscriber( user_id, topic_id, subscription_date ) SELECT u.id AS user_id, t.id AS topic_id, NOW() FROM topic t, m_appuser u, m_appuser_role ur, m_role r WHERE u.office_id = t.entity_id AND u.id = ur.appuser_id AND ur.role_id = r.id AND r.name = t.member_type AND NOT EXISTS (SELECT user_id, topic_id FROM topic_subscriber WHERE user_id = u.id AND topic_id = t.id);
+INSERT INTO topic_subscriber( user_id, topic_id, subscription_date ) SELECT u.id AS user_id, t.id AS topic_id, NOW() FROM topic t, m_appuser u, m_appuser_role ur, m_role r WHERE u.office_id = t.entity_id AND u.id = ur.appuser_id AND ur.role_id = r.id AND r.name COLLATE utf8mb4_general_ci  = t.member_type AND NOT EXISTS (SELECT user_id, topic_id FROM topic_subscriber WHERE user_id = u.id AND topic_id = t.id);

fineract-provider/src/main/resources/sql/migrations/core_db/V352__interop_init.sql

@@ -44,7 +44,7 @@ INSERT INTO `m_appuser`
 VALUES (NULL, 0, 1, NULL, @interop_username, 'Interop', 'User', '5787039480429368bf94732aacc771cd0a3ea02bcf504ffe1185ab94213bc63a',
                             'email@email.com', b'0', b'1', b'1', b'1', b'1', CURDATE(), 0, b'0');
 
-INSERT INTO `m_appuser_role` VALUES ((SELECT id FROM m_appuser WHERE username = @interop_username), 1);
+INSERT INTO `m_appuser_role` VALUES ((SELECT id FROM m_appuser WHERE username COLLATE utf8mb4_general_ci = @interop_username), 1);
 
 -- Interoperation permissions
 

kubernetes/fineract-server-deployment.yml

@@ -82,17 +82,17 @@ spec:
           periodSeconds: 1
         env:
         - name: DRIVERCLASS_NAME
-          value: org.drizzle.jdbc.DrizzleDriver
+          value: org.mariadb.jdbc.Driver
         - name: PROTOCOL
           value: jdbc
         - name: SUB_PROTOCOL
-          value: mysql:thin
+          value: mariadb
         - name: node_id
           value: 1
         - name: fineract_tenants_driver
-          value: org.drizzle.jdbc.DrizzleDriver
+          value: org.mariadb.jdbc.Driver
         - name: fineract_tenants_url
-          value: jdbc:mysql:thin://fineractmysql:3306/fineract_tenants
+          value: jdbc:mariadb://fineractmysql:3306/fineract_tenants
         - name: fineract_tenants_uid
           valueFrom:
             secretKeyRef: