From e2cf138437fa0d92d6f6b8019243cca9eca11253 Mon Sep 17 00:00:00 2001 From: Aleksandar Vidakovic Date: Mon, 6 Nov 2023 15:11:16 +0100 Subject: [PATCH] FINERACT-1484: Micrometer Monitoring --- README.md | 10 +- .../org.apache.fineract.dependencies.gradle | 3 + config/docker/aws/etc/credentials | 24 + config/docker/compose/activemq.yml | 26 + config/docker/compose/fineract.yml | 31 + config/docker/compose/logging-loki.yml | 28 + config/docker/compose/mariadb.yml | 35 + config/docker/compose/observability.yml | 72 + config/docker/compose/postgresql.yml | 34 + config/docker/env/activemq.env | 21 + config/docker/env/aws.env | 30 + config/docker/env/cloudwatch.env | 23 + config/docker/env/debug.env | 21 + config/docker/env/fineract-common.env | 50 + config/docker/env/fineract-manager.env | 25 + config/docker/env/fineract-mariadb.env | 29 + config/docker/env/fineract-postgresql.env | 29 + config/docker/env/fineract-worker.env | 23 + config/docker/env/fineract.env | 20 + config/docker/env/kafka-client-msk.env | 34 + config/docker/env/kafka-client.env | 27 + config/docker/env/kafka-server.env | 27 + config/docker/env/mysql.env | 20 + config/docker/env/oltp.env | 22 + config/docker/env/postgresql.env | 25 + config/docker/env/prometheus.env | 20 + config/docker/env/tracing.env | 21 + .../grafana/dashboards/fineract-loki.json | 2431 +++++++++++++++++ .../dashboards/fineract-spring-boot.json | 1072 ++++++++ .../docker/grafana/datasources/datasource.yml | 95 + config/docker/grafana/etc/dashboards.yml | 29 + .../docker/mysql/conf.d}/server_collation.cnf | 0 .../01-databases.sql | 0 .../docker-entrypoint-initdb.d}/01-init.sh | 0 config/docker/prometheus/etc/prometheus.yml | 57 + config/docker/tempo/etc/tempo.yml | 74 + docker-compose-community-app.yml | 2 +- docker-compose-development.yml | 89 + docker-compose-postgresql-activemq.yml | 154 +- docker-compose-postgresql-kafka-msk.yml | 177 +- docker-compose-postgresql-kafka.yml | 166 +- docker-compose-postgresql.yml | 91 +- docker-compose.yml | 87 +- ...alEventConfigurationNotFoundException.java | 2 +- fineract-provider/dependencies.gradle | 6 + .../src/main/resources/application.properties | 35 +- ...entConfigurationValidationServiceTest.java | 2 +- 47 files changed, 4734 insertions(+), 565 deletions(-) create mode 100644 config/docker/aws/etc/credentials create mode 100644 config/docker/compose/activemq.yml create mode 100644 config/docker/compose/fineract.yml create mode 100644 config/docker/compose/logging-loki.yml create mode 100644 config/docker/compose/mariadb.yml create mode 100644 config/docker/compose/observability.yml create mode 100644 config/docker/compose/postgresql.yml create mode 100644 config/docker/env/activemq.env create mode 100644 config/docker/env/aws.env create mode 100644 config/docker/env/cloudwatch.env create mode 100644 config/docker/env/debug.env create mode 100644 config/docker/env/fineract-common.env create mode 100644 config/docker/env/fineract-manager.env create mode 100644 config/docker/env/fineract-mariadb.env create mode 100644 config/docker/env/fineract-postgresql.env create mode 100644 config/docker/env/fineract-worker.env create mode 100644 config/docker/env/fineract.env create mode 100644 config/docker/env/kafka-client-msk.env create mode 100644 config/docker/env/kafka-client.env create mode 100644 config/docker/env/kafka-server.env create mode 100644 config/docker/env/mysql.env create mode 100644 config/docker/env/oltp.env create mode 100644 config/docker/env/postgresql.env create mode 100644 config/docker/env/prometheus.env create mode 100644 config/docker/env/tracing.env create mode 100644 config/docker/grafana/dashboards/fineract-loki.json create mode 100644 config/docker/grafana/dashboards/fineract-spring-boot.json create mode 100644 config/docker/grafana/datasources/datasource.yml create mode 100644 config/docker/grafana/etc/dashboards.yml rename {fineract-db => config/docker/mysql/conf.d}/server_collation.cnf (100%) rename {fineract-db/docker => config/docker/mysql/docker-entrypoint-initdb.d}/01-databases.sql (100%) rename {fineract-db/docker/postgresql => config/docker/postgresql/docker-entrypoint-initdb.d}/01-init.sh (100%) create mode 100644 config/docker/prometheus/etc/prometheus.yml create mode 100644 config/docker/tempo/etc/tempo.yml create mode 100644 docker-compose-development.yml diff --git a/README.md b/README.md index de0a0c216..d715dac94 100644 --- a/README.md +++ b/README.md @@ -191,7 +191,8 @@ Now to run a new Fineract instance you can simply: 1. `git clone https://github.com/apache/fineract.git ; cd fineract` 1. for windows, use `git clone https://github.com/apache/fineract.git --config core.autocrlf=input ; cd fineract` 1. `./gradlew :fineract-provider:jibDockerBuild -x test` -1. `docker-compose up -d` +1. install the Loki log driver with `docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions` +1. `docker compose -f docker-compose-development.yml up -d` 1. fineract (back-end) is running at https://localhost:8443/fineract-provider/ 1. wait for https://localhost:8443/fineract-provider/actuator/health to return `{"status":"UP"}` 1. you must go to https://localhost:8443 and remember to accept the self-signed SSL certificate of the API once in your browser, otherwise you get a message that is rather misleading from the UI. @@ -206,6 +207,13 @@ _(Note that in previous versions, the `mysqlserver` environment variable used at `docker run` time did something similar; this has changed in [FINERACT-773](https://issues.apache.org/jira/browse/FINERACT-773)), and the `mysqlserver` environment variable is now no longer supported.)_ +If you need the Java Flight Recorder image then copy the file from the running fineract-development container: + +``` +docker container cp fineract-development:/tmp/fineract.jfr /tmp +``` + +NOTE: Download Azul Mission Control from here https://www.azul.com/products/components/azul-mission-control/ to analyze the Java Flight Recorder file. Connection pool configuration ============================= diff --git a/buildSrc/src/main/groovy/org.apache.fineract.dependencies.gradle b/buildSrc/src/main/groovy/org.apache.fineract.dependencies.gradle index fc5c8365a..e205b6d98 100644 --- a/buildSrc/src/main/groovy/org.apache.fineract.dependencies.gradle +++ b/buildSrc/src/main/groovy/org.apache.fineract.dependencies.gradle @@ -22,8 +22,11 @@ apply plugin: 'io.spring.dependency-management' // https://github.com/spring-gradle-plugins/dependency-management-plugin dependencyManagement { imports { + mavenBom 'io.micrometer:micrometer-bom:1.11.5' mavenBom 'org.springframework:spring-framework-bom:6.0.10' mavenBom 'org.springframework.boot:spring-boot-dependencies:3.1.1' + mavenBom 'io.awspring.cloud:spring-cloud-aws-dependencies:3.0.3' + mavenBom 'io.opentelemetry:opentelemetry-bom:1.31.0' mavenBom 'org.jetbrains.kotlin:kotlin-bom:1.8.22' mavenBom 'org.junit:junit-bom:5.9.3' mavenBom 'com.fasterxml.jackson:jackson-bom:2.15.2' diff --git a/config/docker/aws/etc/credentials b/config/docker/aws/etc/credentials new file mode 100644 index 000000000..f83f682c1 --- /dev/null +++ b/config/docker/aws/etc/credentials @@ -0,0 +1,24 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +[default] +aws_access_key_id=localstack +aws_secret_access_key=localstack +region=us-east-1 +output=json diff --git a/config/docker/compose/activemq.yml b/config/docker/compose/activemq.yml new file mode 100644 index 000000000..e60c40f93 --- /dev/null +++ b/config/docker/compose/activemq.yml @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +version: "3.8" + +services: + activemq: + image: symptoma/activemq:5.18.3 + ports: + - 6161:6161 + - 61616:61616 diff --git a/config/docker/compose/fineract.yml b/config/docker/compose/fineract.yml new file mode 100644 index 000000000..e4a3be880 --- /dev/null +++ b/config/docker/compose/fineract.yml @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +version: "3.8" + +services: + fineract: + image: fineract:latest + volumes: + - ../aws/etc/credentials:/etc/aws/credentials + healthcheck: + test: ["CMD", 'sh', '-c', 'echo -e "Checking for the availability of Fineract server deployment"; while ! nc -z "fineract-server" 8443; do sleep 1; printf "-"; done; echo -e " >> Fineract server has started";' ] + timeout: 10s + retries: 10 + env_file: + - ../env/fineract-common.env diff --git a/config/docker/compose/logging-loki.yml b/config/docker/compose/logging-loki.yml new file mode 100644 index 000000000..3718317bd --- /dev/null +++ b/config/docker/compose/logging-loki.yml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +x-logging: &default-logging + driver: loki + options: + loki-url: 'http://localhost:3100/api/prom/push' + loki-pipeline-stages: | + - multiline: + firstline: '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}' + max_wait_time: 3s + - regex: + expression: '^(?P\d{4}-\d{2}-\d{2}) (?P