FINERACT-2166: Expose tenant connection pool metrics

docker-compose-web-app.yml

@@ -20,7 +20,7 @@ version: "3.8"
 services:
   # Frontend service
   community-app:
-    image: openmf/web-app:latest
+    image: openmf/web-app:master
     container_name: mifos-web-app
     restart: always
     ports:

fineract-core/src/main/java/org/apache/fineract/infrastructure/core/service/database/DataSourcePerTenantServiceFactory.java

@@ -23,46 +23,40 @@ import static org.apache.fineract.infrastructure.core.domain.FineractPlatformTen
 
 import com.zaxxer.hikari.HikariConfig;
 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+import io.micrometer.core.instrument.MeterRegistry;
 import javax.sql.DataSource;
+import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.fineract.infrastructure.core.config.FineractProperties;
+import org.apache.fineract.infrastructure.core.domain.FineractPlatformTenant;
 import org.apache.fineract.infrastructure.core.domain.FineractPlatformTenantConnection;
+import org.apache.fineract.infrastructure.core.service.database.metrics.TenantConnectionPoolMetricsTrackerFactory;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.context.ApplicationContext;
 import org.springframework.stereotype.Component;
 
 /**
- *
  * Factory class to get data source service based on the details stored in {@link FineractPlatformTenantConnection}
  * variable
- *
  */
 @Component
 @Slf4j
+@RequiredArgsConstructor
 public class DataSourcePerTenantServiceFactory {
 
     private final HikariConfig hikariConfig;
     private final FineractProperties fineractProperties;
     private final ApplicationContext context;
+    @Qualifier("hikariTenantDataSource")
     private final DataSource tenantDataSource;
     private final HikariDataSourceFactory hikariDataSourceFactory;
 
     private final DatabasePasswordEncryptor databasePasswordEncryptor;
-
-    public DataSourcePerTenantServiceFactory(@Qualifier("hikariTenantDataSource") DataSource tenantDataSource, HikariConfig hikariConfig,
-            FineractProperties fineractProperties, ApplicationContext context, HikariDataSourceFactory hikariDataSourceFactory,
-            DatabasePasswordEncryptor databasePasswordEncryptor) {
-        this.hikariConfig = hikariConfig;
-        this.fineractProperties = fineractProperties;
-        this.context = context;
-        this.tenantDataSource = tenantDataSource;
-        this.hikariDataSourceFactory = hikariDataSourceFactory;
-        this.databasePasswordEncryptor = databasePasswordEncryptor;
-    }
+    private final MeterRegistry meterRegistry;
 
     @SuppressFBWarnings(value = "SLF4J_SIGN_ONLY_FORMAT")
-    public DataSource createNewDataSourceFor(final FineractPlatformTenantConnection tenantConnection) {
+    public DataSource createNewDataSourceFor(FineractPlatformTenant tenant, FineractPlatformTenantConnection tenantConnection) {
         if (!databasePasswordEncryptor.isMasterPasswordHashValid(tenantConnection.getMasterPasswordHash())) {
             throw new IllegalArgumentException(
                     "Invalid master password on tenant connection %d.".formatted(tenantConnection.getConnectionId()));
@@ -103,6 +97,7 @@ public class DataSourcePerTenantServiceFactory {
 
         // https://github.com/brettwooldridge/HikariCP/wiki/MBean-(JMX)-Monitoring-and-Management
         config.setRegisterMbeans(true);
+        config.setMetricsTrackerFactory(new TenantConnectionPoolMetricsTrackerFactory(tenant.getTenantIdentifier(), meterRegistry));
 
         // https://github.com/brettwooldridge/HikariCP/wiki/MySQL-Configuration
         // These are the properties for each Tenant DB; the same configuration

fineract-core/src/main/java/org/apache/fineract/infrastructure/core/service/database/TomcatJdbcDataSourcePerTenantService.java

@@ -24,12 +24,12 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import javax.sql.DataSource;
+import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.fineract.infrastructure.core.domain.FineractPlatformTenant;
 import org.apache.fineract.infrastructure.core.domain.FineractPlatformTenantConnection;
 import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil;
 import org.apache.fineract.infrastructure.core.service.tenant.TenantDetailsService;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.context.ApplicationListener;
 import org.springframework.context.event.ContextRefreshedEvent;
@@ -43,22 +43,16 @@ import org.springframework.stereotype.Service;
  */
 @Slf4j
 @Service
+@RequiredArgsConstructor
 public class TomcatJdbcDataSourcePerTenantService implements RoutingDataSourceService, ApplicationListener<ContextRefreshedEvent> {
 
     private static final Map<Long, DataSource> TENANT_TO_DATA_SOURCE_MAP = new ConcurrentHashMap<>();
+    @Qualifier("hikariTenantDataSource")
     private final DataSource tenantDataSource;
     private final TenantDetailsService tenantDetailsService;
 
     private final DataSourcePerTenantServiceFactory dataSourcePerTenantServiceFactory;
 
-    @Autowired
-    public TomcatJdbcDataSourcePerTenantService(final @Qualifier("hikariTenantDataSource") DataSource tenantDataSource,
-            final DataSourcePerTenantServiceFactory dataSourcePerTenantServiceFactory, final TenantDetailsService tenantDetailsService) {
-        this.tenantDataSource = tenantDataSource;
-        this.dataSourcePerTenantServiceFactory = dataSourcePerTenantServiceFactory;
-        this.tenantDetailsService = tenantDetailsService;
-    }
-
     @Override
     public DataSource retrieveDataSource() {
         // default to tenant database datasource
@@ -71,7 +65,7 @@ public class TomcatJdbcDataSourcePerTenantService implements RoutingDataSourceSe
             // if tenantConnection information available switch to the
             // appropriate datasource for that tenant.
             actualDataSource = TENANT_TO_DATA_SOURCE_MAP.computeIfAbsent(tenantConnectionKey,
-                    (key) -> dataSourcePerTenantServiceFactory.createNewDataSourceFor(tenantConnection));
+                    (key) -> dataSourcePerTenantServiceFactory.createNewDataSourceFor(tenant, tenantConnection));
 
         }
 
@@ -91,7 +85,7 @@ public class TomcatJdbcDataSourcePerTenantService implements RoutingDataSourceSe
         final FineractPlatformTenantConnection tenantConnection = tenant.getConnection();
         Long tenantConnectionKey = tenantConnection.getConnectionId();
         TENANT_TO_DATA_SOURCE_MAP.computeIfAbsent(tenantConnectionKey, (key) -> {
-            DataSource tenantSpecificDataSource = dataSourcePerTenantServiceFactory.createNewDataSourceFor(tenantConnection);
+            DataSource tenantSpecificDataSource = dataSourcePerTenantServiceFactory.createNewDataSourceFor(tenant, tenantConnection);
             try (Connection connection = tenantSpecificDataSource.getConnection()) {
                 String url = connection.getMetaData().getURL();
                 log.debug("Established database connection with URL {}", url);

fineract-core/src/main/java/org/apache/fineract/infrastructure/core/service/database/metrics/TenantConnectionPoolMetricsTracker.java

@@ -0,0 +1,151 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.fineract.infrastructure.core.service.database.metrics;
+
+import com.zaxxer.hikari.metrics.IMetricsTracker;
+import com.zaxxer.hikari.metrics.PoolStats;
+import io.micrometer.core.instrument.Counter;
+import io.micrometer.core.instrument.Gauge;
+import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.core.instrument.Timer;
+import java.util.concurrent.TimeUnit;
+
+public class TenantConnectionPoolMetricsTracker implements IMetricsTracker {
+
+    public static final String HIKARI_METRIC_NAME_PREFIX = ".hikaricp";
+
+    private static final String METRIC_CATEGORY = "pool";
+    private static final String METRIC_NAME_WAIT = HIKARI_METRIC_NAME_PREFIX + ".connections.acquire";
+    private static final String METRIC_NAME_USAGE = HIKARI_METRIC_NAME_PREFIX + ".connections.usage";
+    private static final String METRIC_NAME_CONNECT = HIKARI_METRIC_NAME_PREFIX + ".connections.creation";
+
+    private static final String METRIC_NAME_TIMEOUT_RATE = HIKARI_METRIC_NAME_PREFIX + ".connections.timeout";
+    private static final String METRIC_NAME_TOTAL_CONNECTIONS = HIKARI_METRIC_NAME_PREFIX + ".connections";
+    private static final String METRIC_NAME_IDLE_CONNECTIONS = HIKARI_METRIC_NAME_PREFIX + ".connections.idle";
+    private static final String METRIC_NAME_ACTIVE_CONNECTIONS = HIKARI_METRIC_NAME_PREFIX + ".connections.active";
+    private static final String METRIC_NAME_PENDING_CONNECTIONS = HIKARI_METRIC_NAME_PREFIX + ".connections.pending";
+    private static final String METRIC_NAME_MAX_CONNECTIONS = HIKARI_METRIC_NAME_PREFIX + ".connections.max";
+    private static final String METRIC_NAME_MIN_CONNECTIONS = HIKARI_METRIC_NAME_PREFIX + ".connections.min";
+
+    private final Timer connectionObtainTimer;
+    private final Counter connectionTimeoutCounter;
+    private final Timer connectionUsage;
+    private final Timer connectionCreation;
+    private final Gauge totalConnectionGauge;
+    private final Gauge idleConnectionGauge;
+    private final Gauge activeConnectionGauge;
+    private final Gauge pendingConnectionGauge;
+    private final Gauge maxConnectionGauge;
+    private final Gauge minConnectionGauge;
+    private final MeterRegistry meterRegistry;
+    private final PoolStats poolStats;
+
+    public TenantConnectionPoolMetricsTracker(String tenantIdentifier, String poolName, PoolStats poolStats, MeterRegistry meterRegistry) {
+        // poolStats must be held with a 'strong reference' even though it is never referenced within this class
+        this.poolStats = poolStats; // DO NOT REMOVE
+
+        this.meterRegistry = meterRegistry;
+
+        String metricPrefix = "fineract.tenants." + tenantIdentifier;
+
+        this.connectionObtainTimer = Timer.builder(metricPrefix + METRIC_NAME_WAIT).description("Connection acquire time") //
+                .tags(METRIC_CATEGORY, poolName) //
+                .register(meterRegistry);
+
+        this.connectionCreation = Timer.builder(metricPrefix + METRIC_NAME_CONNECT).description("Connection creation time") //
+                .tags(METRIC_CATEGORY, poolName) //
+                .register(meterRegistry);
+
+        this.connectionUsage = Timer.builder(metricPrefix + METRIC_NAME_USAGE).description("Connection usage time") //
+                .tags(METRIC_CATEGORY, poolName) //
+                .register(meterRegistry);
+
+        this.connectionTimeoutCounter = Counter.builder(metricPrefix + METRIC_NAME_TIMEOUT_RATE)
+                .description("Connection timeout total count") //
+                .tags(METRIC_CATEGORY, poolName) //
+                .register(meterRegistry);
+
+        this.totalConnectionGauge = Gauge.builder(metricPrefix + METRIC_NAME_TOTAL_CONNECTIONS, poolStats, PoolStats::getTotalConnections)
+                .description("Total connections") //
+                .tags(METRIC_CATEGORY, poolName) //
+                .register(meterRegistry);
+
+        this.idleConnectionGauge = Gauge.builder(metricPrefix + METRIC_NAME_IDLE_CONNECTIONS, poolStats, PoolStats::getIdleConnections)
+                .description("Idle connections") //
+                .tags(METRIC_CATEGORY, poolName) //
+                .register(meterRegistry);
+
+        this.activeConnectionGauge = Gauge
+                .builder(metricPrefix + METRIC_NAME_ACTIVE_CONNECTIONS, poolStats, PoolStats::getActiveConnections)
+                .description("Active connections") //
+                .tags(METRIC_CATEGORY, poolName) //
+                .register(meterRegistry);
+
+        this.pendingConnectionGauge = Gauge.builder(metricPrefix + METRIC_NAME_PENDING_CONNECTIONS, poolStats, PoolStats::getPendingThreads)
+                .description("Pending threads") //
+                .tags(METRIC_CATEGORY, poolName) //
+                .register(meterRegistry);
+
+        this.maxConnectionGauge = Gauge.builder(metricPrefix + METRIC_NAME_MAX_CONNECTIONS, poolStats, PoolStats::getMaxConnections)
+                .description("Max connections") //
+                .tags(METRIC_CATEGORY, poolName) //
+                .register(meterRegistry);
+
+        this.minConnectionGauge = Gauge.builder(metricPrefix + METRIC_NAME_MIN_CONNECTIONS, poolStats, PoolStats::getMinConnections)
+                .description("Min connections") //
+                .tags(METRIC_CATEGORY, poolName) //
+                .register(meterRegistry);
+
+    }
+
+    @Override
+    public void recordConnectionAcquiredNanos(final long elapsedAcquiredNanos) {
+        connectionObtainTimer.record(elapsedAcquiredNanos, TimeUnit.NANOSECONDS);
+    }
+
+    @Override
+    public void recordConnectionUsageMillis(final long elapsedBorrowedMillis) {
+        connectionUsage.record(elapsedBorrowedMillis, TimeUnit.MILLISECONDS);
+    }
+
+    @Override
+    public void recordConnectionTimeout() {
+        connectionTimeoutCounter.increment();
+    }
+
+    @Override
+    public void recordConnectionCreatedMillis(long connectionCreatedMillis) {
+        connectionCreation.record(connectionCreatedMillis, TimeUnit.MILLISECONDS);
+    }
+
+    @Override
+    public void close() {
+        meterRegistry.remove(connectionObtainTimer);
+        meterRegistry.remove(connectionTimeoutCounter);
+        meterRegistry.remove(connectionUsage);
+        meterRegistry.remove(connectionCreation);
+        meterRegistry.remove(totalConnectionGauge);
+        meterRegistry.remove(idleConnectionGauge);
+        meterRegistry.remove(activeConnectionGauge);
+        meterRegistry.remove(pendingConnectionGauge);
+        meterRegistry.remove(maxConnectionGauge);
+        meterRegistry.remove(minConnectionGauge);
+    }
+}

fineract-core/src/main/java/org/apache/fineract/infrastructure/core/service/database/metrics/TenantConnectionPoolMetricsTrackerFactory.java

@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.core.service.database.metrics;
+
+import com.zaxxer.hikari.metrics.IMetricsTracker;
+import com.zaxxer.hikari.metrics.MetricsTrackerFactory;
+import com.zaxxer.hikari.metrics.PoolStats;
+import io.micrometer.core.instrument.MeterRegistry;
+import lombok.RequiredArgsConstructor;
+
+@RequiredArgsConstructor
+public class TenantConnectionPoolMetricsTrackerFactory implements MetricsTrackerFactory {
+
+    private final String tenantIdentifier;
+    private final MeterRegistry registry;
+
+    @Override
+    public IMetricsTracker create(String poolName, PoolStats poolStats) {
+        return new TenantConnectionPoolMetricsTracker(tenantIdentifier, poolName, poolStats, registry);
+    }
+}

fineract-provider/src/test/java/org/apache/fineract/infrastructure/core/DataSourcePerTenantServiceFactoryTest.java

@@ -87,6 +87,7 @@ public class DataSourcePerTenantServiceFactoryTest {
     public static final String MASTER_MASTER_PASSWORD = "fineract";
 
     public static final String MASTER_ENCRYPTION = "AES/CBC/PKCS5Padding";
+    public static final FineractPlatformTenant TENANT = new FineractPlatformTenant(1L, "", "", "", null);
 
     @Mock
     private FineractProperties fineractProperties;
@@ -183,7 +184,7 @@ public class DataSourcePerTenantServiceFactoryTest {
         given(fineractProperties.getMode()).willReturn(modeProperties);
 
         // when
-        DataSource dataSource = underTest.createNewDataSourceFor(defaultTenant.getConnection());
+        DataSource dataSource = underTest.createNewDataSourceFor(TENANT, defaultTenant.getConnection());
 
         // then
         assertNotNull(dataSource);
@@ -215,7 +216,7 @@ public class DataSourcePerTenantServiceFactoryTest {
         config.setMinPoolSize(minPoolSize);
 
         // when
-        DataSource dataSource = underTest.createNewDataSourceFor(defaultTenant.getConnection());
+        DataSource dataSource = underTest.createNewDataSourceFor(TENANT, defaultTenant.getConnection());
 
         // then
         assertNotNull(dataSource);
@@ -247,7 +248,7 @@ public class DataSourcePerTenantServiceFactoryTest {
         config.setMaxPoolSize(maxPoolSize);
 
         // when
-        DataSource dataSource = underTest.createNewDataSourceFor(defaultTenant.getConnection());
+        DataSource dataSource = underTest.createNewDataSourceFor(TENANT, defaultTenant.getConnection());
 
         // then
         assertNotNull(dataSource);
@@ -281,7 +282,7 @@ public class DataSourcePerTenantServiceFactoryTest {
         config.setMaxPoolSize(maxPoolSize);
 
         // when
-        DataSource dataSource = underTest.createNewDataSourceFor(defaultTenant.getConnection());
+        DataSource dataSource = underTest.createNewDataSourceFor(TENANT, defaultTenant.getConnection());
 
         // then
         assertNotNull(dataSource);
@@ -307,7 +308,7 @@ public class DataSourcePerTenantServiceFactoryTest {
         given(fineractProperties.getMode()).willReturn(modeProperties);
 
         // when
-        DataSource dataSource = underTest.createNewDataSourceFor(defaultTenant.getConnection());
+        DataSource dataSource = underTest.createNewDataSourceFor(TENANT, defaultTenant.getConnection());
 
         // then
         assertNotNull(dataSource);
@@ -333,7 +334,7 @@ public class DataSourcePerTenantServiceFactoryTest {
         given(fineractProperties.getMode()).willReturn(modeProperties);
 
         // when
-        DataSource dataSource = underTest.createNewDataSourceFor(defaultTenant.getConnection());
+        DataSource dataSource = underTest.createNewDataSourceFor(TENANT, defaultTenant.getConnection());
 
         // then
         assertNotNull(dataSource);