diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker-mariadb.yml similarity index 100% rename from .github/workflows/build-docker.yml rename to .github/workflows/build-docker-mariadb.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build-mariadb.yml similarity index 100% rename from .github/workflows/build.yml rename to .github/workflows/build-mariadb.yml diff --git a/.github/workflows/build-mysql.yml b/.github/workflows/build-mysql.yml new file mode 100644 index 000000000..609133bfd --- /dev/null +++ b/.github/workflows/build-mysql.yml @@ -0,0 +1,67 @@ +name: Fineract Gradle build - MySQL +on: [push, pull_request] + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-20.04 + + services: + mariad: + image: mysql:8.0 + ports: + - 3306:3306 + env: + MYSQL_ROOT_PASSWORD: mysql + options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3 + + mock-oauth2-server: + image: ghcr.io/navikt/mock-oauth2-server:0.4.0 + ports: + - 9000:9000 + env: + SERVER_PORT: 9000 + JSON_CONFIG: '{ "interactiveLogin": true, "httpServer": "NettyWrapper", "tokenCallbacks": [ { "issuerId": "auth/realms/fineract", "tokenExpiry": 120, "requestMappings": [{ "requestParam": "scope", "match": "fineract", "claims": { "sub": "mifos", "scope": [ "test" ] } } ] } ] }' + + env: + TZ: Asia/Kolkata + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + java-version: '17' + distribution: 'zulu' + cache: gradle + - uses: actions/setup-node@v3 + with: + node-version: 16 + - name: Congfigure vega-cli + run: npm i -g vega-cli --unsafe + - name: Validate Gradle wrapper + uses: gradle/wrapper-validation-action@859c33240bd026ce8d5f711f5adcc65c2f8eafc1 + - name: Verify MariaDB connection + run: | + while ! mysqladmin ping -h"127.0.0.1" -P3306 ; do + sleep 1 + done + - name: Initialise databases + run: | + ./gradlew --no-daemon -q createMySQLDB -PdbName=fineract_tenants + ./gradlew --no-daemon -q createMySQLDB -PdbName=fineract_default + - name: Install additional software + run: | + sudo apt-get update + sudo apt-get install ghostscript graphviz -y + + - name: Basic Auth Build & Test + run: ./gradlew --no-daemon -q --console=plain build test --fail-fast doc -x :twofactor-tests:test -x :oauth2-test:test -PdbType=mysql + + - name: 2FA Build & Test + run: ./gradlew --no-daemon -q --console=plain :twofactor-tests:test --fail-fast -PdbType=mysql + + - name: OAuth2 Build & Test + run: ./gradlew --no-daemon -q --console=plain :oauth2-tests:test --fail-fast -PdbType=mysql diff --git a/fineract-provider/build.gradle b/fineract-provider/build.gradle index 8dd85a234..d6e38a943 100644 --- a/fineract-provider/build.gradle +++ b/fineract-provider/build.gradle @@ -151,6 +151,7 @@ configurations { dependencies { driver 'org.mariadb.jdbc:mariadb-java-client:2.7.5' driver 'org.postgresql:postgresql:42.3.5' + driver 'mysql:mysql-connector-java:8.0.29' } URLClassLoader loader = GroovyObject.class.classLoader @@ -159,7 +160,7 @@ configurations.driver.each {File file -> } task createDB { - description= "Creates the MySQL Database. Needs database name to be passed (like: -PdbName=someDBname)" + description= "Creates the MariaDB Database. Needs database name to be passed (like: -PdbName=someDBname)" doLast { def sql = Sql.newInstance( 'jdbc:mariadb://localhost:3306/', mysqlUser, mysqlPassword, 'org.mariadb.jdbc.Driver' ) sql.execute( 'create database '+"`$dbName`" ) @@ -167,7 +168,7 @@ task createDB { } task dropDB { - description= "Drops the specified MySQL database. The database name has to be passed (like: -PdbName=someDBname)" + description= "Drops the specified MariaDB database. The database name has to be passed (like: -PdbName=someDBname)" doLast { def sql = Sql.newInstance( 'jdbc:mariadb://localhost:3306/', mysqlUser, mysqlPassword, 'org.mariadb.jdbc.Driver' ) sql.execute( 'DROP DATABASE '+"`$dbName`") @@ -190,6 +191,22 @@ task dropPGDB { } } +task createMySQLDB { + description= "Creates the MySQL Database. Needs database name to be passed (like: -PdbName=someDBname)" + doLast { + def sql = Sql.newInstance( 'jdbc:mysql://localhost:3306/', mysqlUser, mysqlPassword, 'com.mysql.cj.jdbc.Driver' ) + sql.execute( 'create database '+"`$dbName`" ) + } +} + +task dropMySQLDB { + description= "Drops the specified MySQL database. The database name has to be passed (like: -PdbName=someDBname)" + doLast { + def sql = Sql.newInstance( 'jdbc:mysql://localhost:3306/', mysqlUser, mysqlPassword, 'com.mysql.cj.jdbc.Driver' ) + sql.execute( 'DROP DATABASE '+"`$dbName`") + } +} + task setBlankPassword { doLast { def sql = Sql.newInstance( 'jdbc:mariadb://localhost:3306/', mysqlUser, mysqlPassword, 'org.mariadb.jdbc.Driver' ) diff --git a/fineract-provider/src/main/java/org/apache/fineract/commands/service/AuditReadPlatformServiceImpl.java b/fineract-provider/src/main/java/org/apache/fineract/commands/service/AuditReadPlatformServiceImpl.java index c87cdfd4b..39a87ca24 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/commands/service/AuditReadPlatformServiceImpl.java +++ b/fineract-provider/src/main/java/org/apache/fineract/commands/service/AuditReadPlatformServiceImpl.java @@ -425,7 +425,7 @@ public class AuditReadPlatformServiceImpl implements AuditReadPlatformService { sql = " select distinct(entity_name) as entityName from m_permission p "; sql += makercheckerCapabilityOnly(useType, currentUser); - sql += " order by (CASE WHEN grouping = 'datatable' THEN 'ZZZ' ELSE entity_name END), entity_name"; + sql += " order by (CASE WHEN " + sqlGenerator.escape("grouping") + " = 'datatable' THEN 'ZZZ' ELSE entity_name END), entity_name"; final EntityNamesMapper mapper2 = new EntityNamesMapper(); final List entityNames = this.jdbcTemplate.query(sql, mapper2); // NOSONAR diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/database/DatabaseTypeResolver.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/database/DatabaseTypeResolver.java index b21bfb5ea..7062324df 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/database/DatabaseTypeResolver.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/database/DatabaseTypeResolver.java @@ -29,7 +29,7 @@ import org.springframework.stereotype.Component; public class DatabaseTypeResolver implements InitializingBean { private final Map driverMapping = Map.of("org.mariadb.jdbc.Driver", DatabaseType.MYSQL, "com.mysql.jdbc.Driver", - DatabaseType.MYSQL, "org.postgresql.Driver", DatabaseType.POSTGRESQL); + DatabaseType.MYSQL, "com.mysql.cj.jdbc.Driver", DatabaseType.MYSQL, "org.postgresql.Driver", DatabaseType.POSTGRESQL); private final AtomicReference currentDatabaseType = new AtomicReference<>(); private final HikariConfig hikariConfig; diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/ReadWriteNonCoreDataServiceImpl.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/ReadWriteNonCoreDataServiceImpl.java index 78a0213e1..7db6c1c20 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/ReadWriteNonCoreDataServiceImpl.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/ReadWriteNonCoreDataServiceImpl.java @@ -35,7 +35,9 @@ import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.stream.Stream; import javax.persistence.PersistenceException; +import lombok.RequiredArgsConstructor; import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -72,7 +74,6 @@ import org.apache.fineract.infrastructure.security.utils.SQLInjectionValidator; import org.apache.fineract.useradministration.domain.AppUser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.dao.DataAccessException; import org.springframework.dao.DataIntegrityViolationException; import org.springframework.dao.EmptyResultDataAccessException; @@ -86,6 +87,7 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @Service +@RequiredArgsConstructor public class ReadWriteNonCoreDataServiceImpl implements ReadWriteNonCoreDataService { private static final String DATATABLE_NAME_REGEX_PATTERN = "^[a-zA-Z][a-zA-Z0-9\\-_\\s]{0,48}[a-zA-Z0-9]$"; @@ -110,7 +112,7 @@ public class ReadWriteNonCoreDataServiceImpl implements ReadWriteNonCoreDataServ private final DatabaseSpecificSQLGenerator sqlGenerator; private final PlatformSecurityContext context; private final FromJsonHelper fromJsonHelper; - private final JsonParserHelper helper; + private final JsonParserHelper helper = new JsonParserHelper(); private final GenericDataService genericDataService; private final DatatableCommandFromApiJsonDeserializer fromApiJsonDeserializer; private final ConfigurationDomainService configurationDomainService; @@ -120,29 +122,6 @@ public class ReadWriteNonCoreDataServiceImpl implements ReadWriteNonCoreDataServ private final NamedParameterJdbcTemplate namedParameterJdbcTemplate; private final SqlInjectionPreventerService preventSqlInjectionService; - @Autowired(required = true) - public ReadWriteNonCoreDataServiceImpl(final JdbcTemplate jdbcTemplate, final NamedParameterJdbcTemplate namedParameterJdbcTemplate, - final PlatformSecurityContext context, final FromJsonHelper fromJsonHelper, final GenericDataService genericDataService, - final DatatableCommandFromApiJsonDeserializer fromApiJsonDeserializer, final CodeReadPlatformService codeReadPlatformService, - final ConfigurationDomainService configurationDomainService, final DataTableValidator dataTableValidator, - final ColumnValidator columnValidator, DatabaseTypeResolver databaseTypeResolver, DatabaseSpecificSQLGenerator sqlGenerator, - SqlInjectionPreventerService sqlInjectionPreventerService) { - this.databaseTypeResolver = databaseTypeResolver; - this.sqlGenerator = sqlGenerator; - this.jdbcTemplate = jdbcTemplate; - this.context = context; - this.fromJsonHelper = fromJsonHelper; - this.helper = new JsonParserHelper(); - this.genericDataService = genericDataService; - this.fromApiJsonDeserializer = fromApiJsonDeserializer; - this.codeReadPlatformService = codeReadPlatformService; - this.configurationDomainService = configurationDomainService; - this.dataTableValidator = dataTableValidator; - this.columnValidator = columnValidator; - this.namedParameterJdbcTemplate = namedParameterJdbcTemplate; - this.preventSqlInjectionService = sqlInjectionPreventerService; - } - @Override public List retrieveDatatableNames(final String appTable) { Object[] params = new Object[] { this.context.authenticatedUser().getId() }; @@ -306,14 +285,16 @@ public class ReadWriteNonCoreDataServiceImpl implements ReadWriteNonCoreDataServ final String updatePermissionChecker = "'UPDATE_" + dataTableName + "_CHECKER'"; final String deletePermission = "'DELETE_" + dataTableName + "'"; final String deletePermissionChecker = "'DELETE_" + dataTableName + "_CHECKER'"; + final List escapedColumns = Stream.of("grouping", "code", "action_name", "entity_name", "can_maker_checker") + .map(sqlGenerator::escape).toList(); + final String columns = String.join(", ", escapedColumns); - return "insert into m_permission (grouping, code, action_name, entity_name, can_maker_checker) values " + "('datatable', " - + createPermission + ", 'CREATE', '" + dataTableName + "', true)," + "('datatable', " + createPermissionChecker - + ", 'CREATE', '" + dataTableName + "', false)," + "('datatable', " + readPermission + ", 'READ', '" + dataTableName - + "', false)," + "('datatable', " + updatePermission + ", 'UPDATE', '" + dataTableName + "', true)," + "('datatable', " - + updatePermissionChecker + ", 'UPDATE', '" + dataTableName + "', false)," + "('datatable', " + deletePermission - + ", 'DELETE', '" + dataTableName + "', true)," + "('datatable', " + deletePermissionChecker + ", 'DELETE', '" - + dataTableName + "', false)"; + return "insert into m_permission (" + columns + ") values " + "('datatable', " + createPermission + ", 'CREATE', '" + dataTableName + + "', true)," + "('datatable', " + createPermissionChecker + ", 'CREATE', '" + dataTableName + "', false)," + + "('datatable', " + readPermission + ", 'READ', '" + dataTableName + "', false)," + "('datatable', " + updatePermission + + ", 'UPDATE', '" + dataTableName + "', true)," + "('datatable', " + updatePermissionChecker + ", 'UPDATE', '" + + dataTableName + "', false)," + "('datatable', " + deletePermission + ", 'DELETE', '" + dataTableName + "', true)," + + "('datatable', " + deletePermissionChecker + ", 'DELETE', '" + dataTableName + "', false)"; } @@ -918,7 +899,7 @@ public class ReadWriteNonCoreDataServiceImpl implements ReadWriteNonCoreDataServ * Name of data table * @param column * JSON encoded array of column properties - * @see https://mifosforge.jira.com/browse/MIFOSX-1145 + * @see MIFOSX-1145 **/ private void removeNullValuesFromStringColumn(final String datatableName, final JsonObject column, final Map mapColumnNameDefinition) { diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/survey/service/WriteSurveyServiceImpl.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/survey/service/WriteSurveyServiceImpl.java index b246aa426..06d553b48 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/survey/service/WriteSurveyServiceImpl.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/survey/service/WriteSurveyServiceImpl.java @@ -18,10 +18,13 @@ */ package org.apache.fineract.infrastructure.survey.service; +import java.util.List; +import java.util.stream.Stream; +import lombok.RequiredArgsConstructor; import org.apache.fineract.infrastructure.core.api.JsonCommand; import org.apache.fineract.infrastructure.core.data.CommandProcessingResult; +import org.apache.fineract.infrastructure.core.service.database.DatabaseSpecificSQLGenerator; import org.apache.fineract.infrastructure.dataqueries.service.ReadWriteNonCoreDataService; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -29,15 +32,11 @@ import org.springframework.transaction.annotation.Transactional; * Created by Cieyou on 3/13/14. */ @Service +@RequiredArgsConstructor public class WriteSurveyServiceImpl implements WriteSurveyService { private final ReadWriteNonCoreDataService readWriteNonCoreDataService; - - @Autowired(required = true) - WriteSurveyServiceImpl(final ReadWriteNonCoreDataService readWriteNonCoreDataService) { - this.readWriteNonCoreDataService = readWriteNonCoreDataService; - - } + private final DatabaseSpecificSQLGenerator sqlGenerator; @Override @Transactional @@ -59,14 +58,16 @@ public class WriteSurveyServiceImpl implements WriteSurveyService { final String updatePermissionChecker = "'UPDATE_" + dataTableName + "_CHECKER'"; final String deletePermission = "'DELETE_" + dataTableName + "'"; final String deletePermissionChecker = "'DELETE_" + dataTableName + "_CHECKER'"; + final List escapedColumns = Stream.of("grouping", "code", "action_name", "entity_name", "can_maker_checker") + .map(sqlGenerator::escape).toList(); + final String columns = String.join(", ", escapedColumns); - return "insert into m_permission (grouping, code, action_name, entity_name, can_maker_checker) values " + "('datatable', " - + createPermission + ", 'CREATE', '" + dataTableName + "', false)," + "('datatable', " + createPermissionChecker - + ", 'CREATE', '" + dataTableName + "', false)," + "('datatable', " + readPermission + ", 'READ', '" + dataTableName - + "', false)," + "('datatable', " + updatePermission + ", 'UPDATE', '" + dataTableName + "', false)," + "('datatable', " - + updatePermissionChecker + ", 'UPDATE', '" + dataTableName + "', false)," + "('datatable', " + deletePermission - + ", 'DELETE', '" + dataTableName + "', false)," + "('datatable', " + deletePermissionChecker + ", 'DELETE', '" - + dataTableName + "', false)"; + return "insert into m_permission (" + columns + ") values " + "('datatable', " + createPermission + ", 'CREATE', '" + dataTableName + + "', false)," + "('datatable', " + createPermissionChecker + ", 'CREATE', '" + dataTableName + "', false)," + + "('datatable', " + readPermission + ", 'READ', '" + dataTableName + "', false)," + "('datatable', " + updatePermission + + ", 'UPDATE', '" + dataTableName + "', false)," + "('datatable', " + updatePermissionChecker + ", 'UPDATE', '" + + dataTableName + "', false)," + "('datatable', " + deletePermission + ", 'DELETE', '" + dataTableName + "', false)," + + "('datatable', " + deletePermissionChecker + ", 'DELETE', '" + dataTableName + "', false)"; } @Transactional diff --git a/fineract-provider/src/main/resources/META-INF/orm.xml b/fineract-provider/src/main/resources/META-INF/orm.xml index 29ec044d6..1eb26c9cc 100644 --- a/fineract-provider/src/main/resources/META-INF/orm.xml +++ b/fineract-provider/src/main/resources/META-INF/orm.xml @@ -28,6 +28,7 @@ + diff --git a/integration-tests/build.gradle b/integration-tests/build.gradle index fdc79a14a..320ff5d83 100644 --- a/integration-tests/build.gradle +++ b/integration-tests/build.gradle @@ -31,6 +31,15 @@ apply from: 'dependencies.gradle' // enable when all tests are migrated tasks.cucumber.onlyIf {false} +// Allow external drivers to be used for the tests without packaging it +// mainly due to license incompatibilities +configurations { + driver +} +dependencies { + driver 'mysql:mysql-connector-java:8.0.29' +} + cargo { containerId "tomcat9x" @@ -41,7 +50,7 @@ cargo { } local { - logLevel = 'medium' + logLevel = 'low' // outputFile = file('build/output.log') installer { installConfiguration = configurations.tomcat @@ -49,13 +58,20 @@ cargo { extractDir = file("$buildDir/tomcat") } startStopTimeout = 240000 + sharedClasspath = configurations.driver containerProperties { def jvmArgs = '--add-exports=java.naming/com.sun.jndi.ldap=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.security=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.management/javax.management=ALL-UNNAMED --add-opens=java.naming/javax.naming=ALL-UNNAMED ' if (project.hasProperty('localDebug')) { jvmArgs += ' -agentlib:jdwp=transport=dt_socket,server=y,address=*:9000,suspend=n -Xmx2G -Duser.timezone=Asia/Kolkata ' } - if (project.hasProperty('dbType') && 'postgresql'.equalsIgnoreCase(dbType)) { - jvmArgs += '-Dspring.datasource.hikari.driverClassName=org.postgresql.Driver -Dspring.datasource.hikari.jdbcUrl=jdbc:postgresql://localhost:5432/fineract_tenants -Dspring.datasource.hikari.username=root -Dspring.datasource.hikari.password=postgres -Dfineract.tenant.host=localhost -Dfineract.tenant.port=5432 -Dfineract.tenant.username=root -Dfineract.tenant.password=postgres' + if (project.hasProperty('dbType')) { + if ('postgresql'.equalsIgnoreCase(dbType)) { + jvmArgs += '-Dspring.datasource.hikari.driverClassName=org.postgresql.Driver -Dspring.datasource.hikari.jdbcUrl=jdbc:postgresql://localhost:5432/fineract_tenants -Dspring.datasource.hikari.username=root -Dspring.datasource.hikari.password=postgres -Dfineract.tenant.host=localhost -Dfineract.tenant.port=5432 -Dfineract.tenant.username=root -Dfineract.tenant.password=postgres' + } else if ('mysql'.equalsIgnoreCase(dbType)) { + jvmArgs += '-Dspring.datasource.hikari.driverClassName=com.mysql.cj.jdbc.Driver -Dspring.datasource.hikari.jdbcUrl=jdbc:mysql://localhost:3306/fineract_tenants -Dspring.datasource.hikari.username=root -Dspring.datasource.hikari.password=mysql -Dfineract.tenant.host=localhost -Dfineract.tenant.port=3306 -Dfineract.tenant.username=root -Dfineract.tenant.password=mysql' + } else { + throw new GradleException('Provided dbType is not supported') + } } else { jvmArgs += '-Dspring.datasource.hikari.driverClassName=org.mariadb.jdbc.Driver -Dspring.datasource.hikari.jdbcUrl=jdbc:mariadb://localhost:3306/fineract_tenants -Dspring.datasource.hikari.username=root -Dspring.datasource.hikari.password=mysql -Dfineract.tenant.host=localhost -Dfineract.tenant.port=3306 -Dfineract.tenant.username=root -Dfineract.tenant.password=mysql' }