FINERACT-1824

- Kafka support for remote batch jobs: cob
 - Kafka support for external events

fineract-core/src/main/java/org/apache/fineract/infrastructure/core/config/FineractProperties.java

@@ -19,9 +19,13 @@
 
 package org.apache.fineract.infrastructure.core.config;
 
+import java.util.Arrays;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import lombok.Getter;
 import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.boot.context.properties.ConfigurationProperties;
 
@@ -161,6 +165,7 @@ public class FineractProperties {
 
         private FineractRemoteJobMessageHandlerSpringEventsProperties springEvents;
         private FineractRemoteJobMessageHandlerJmsProperties jms;
+        private FineractRemoteJobMessageHandlerKafkaProperties kafka;
     }
 
     @Getter
@@ -185,6 +190,73 @@ public class FineractProperties {
         }
     }
 
+    @Getter
+    @Setter
+    public static class FineractRemoteJobMessageHandlerKafkaProperties {
+
+        private boolean enabled;
+        private String bootstrapServers;
+        private KafkaTopicProperties topic;
+        private KafkaConsumerProperties consumer;
+        private KafkaProperties producer;
+        private KafkaProperties admin;
+    }
+
+    @Getter
+    @Setter
+    public static class KafkaTopicProperties {
+
+        private boolean autoCreate;
+        private String name;
+        private int replicas;
+        private int partitions;
+    }
+
+    @Getter
+    @Setter
+    public static class KafkaConsumerProperties extends KafkaProperties {
+
+        private String groupId;
+    }
+
+    @Getter
+    @Setter
+    @Slf4j
+    public static class KafkaProperties {
+
+        private String extraPropertiesKeyValueSeparator;
+        private String extraPropertiesSeparator;
+        private String extraProperties;
+
+        public Map<String, String> getExtraPropertiesMap() {
+            Map<String, String> map = new HashMap<>();
+            if (StringUtils.isNotEmpty(getExtraProperties()) && validateSeparators()) {
+                String[] lines = StringUtils.split(getExtraProperties(), extraPropertiesSeparator);
+                Arrays.stream(lines).forEach(line -> {
+                    String[] keyAndValue = StringUtils.split(line, extraPropertiesKeyValueSeparator);
+                    if (keyAndValue.length == 2) {
+                        map.put(keyAndValue[0], keyAndValue[1]);
+                    } else {
+                        log.warn("Invalid property: {}", line);
+                    }
+
+                });
+            }
+            return map;
+        }
+
+        private boolean validateSeparators() {
+            boolean valid = (StringUtils.isNotEmpty(extraPropertiesSeparator) && extraPropertiesSeparator.length() == 1
+                    && StringUtils.isNotEmpty(extraPropertiesKeyValueSeparator) && extraPropertiesKeyValueSeparator.length() == 1
+                    && !extraPropertiesSeparator.equals(extraPropertiesKeyValueSeparator));
+            if (!valid) {
+                log.warn("Invalid KafkaProperties configuration, lineSeparator '{}' and keyValueSeparator '{}'", extraPropertiesSeparator,
+                        extraPropertiesKeyValueSeparator);
+            }
+            return valid;
+        }
+    }
+
     @Getter
     @Setter
     public static class FineractEventsProperties {
@@ -213,6 +285,7 @@ public class FineractProperties {
     public static class FineractExternalEventsProducerProperties {
 
         private FineractExternalEventsProducerJmsProperties jms;
+        private FineractExternalEventsProducerKafkaProperties kafka;
     }
 
     @Getter
@@ -235,6 +308,18 @@ public class FineractProperties {
         }
     }
 
+    @Getter
+    @Setter
+    public static class FineractExternalEventsProducerKafkaProperties {
+
+        private boolean enabled;
+        private String bootstrapServers;
+        private KafkaTopicProperties topic;
+        private KafkaProperties producer;
+        private KafkaProperties admin;
+        private int timeoutInSeconds;
+    }
+
     @Getter
     @Setter
     public static class FineractContentProperties {

fineract-core/src/main/java/org/apache/fineract/infrastructure/core/domain/FineractContext.java

@@ -22,10 +22,14 @@ import java.io.Serializable;
 import java.time.LocalDate;
 import java.util.HashMap;
 import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Getter;
+import lombok.extern.jackson.Jacksonized;
 import org.apache.fineract.infrastructure.businessdate.domain.BusinessDateType;
 
 @AllArgsConstructor
+@Jacksonized
+@Builder
 @Getter
 public class FineractContext implements Serializable {
 

fineract-core/src/main/java/org/apache/fineract/infrastructure/core/domain/FineractPlatformTenant.java

@@ -19,7 +19,11 @@
 package org.apache.fineract.infrastructure.core.domain;
 
 import java.io.Serializable;
+import lombok.Builder;
+import lombok.extern.jackson.Jacksonized;
 
+@Jacksonized
+@Builder
 public class FineractPlatformTenant implements Serializable {
 
     private final Long id;

fineract-core/src/main/java/org/apache/fineract/infrastructure/core/domain/FineractPlatformTenantConnection.java

@@ -21,13 +21,19 @@ package org.apache.fineract.infrastructure.core.domain;
 import java.io.Serializable;
 import java.sql.Connection;
 import javax.sql.DataSource;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Getter;
+import lombok.extern.jackson.Jacksonized;
 import org.apache.commons.lang3.StringUtils;
 
 /**
  * Holds Tenant's DB server connection connection details.
  */
 @Getter
+@Builder
+@AllArgsConstructor
+@Jacksonized
 public class FineractPlatformTenantConnection implements Serializable {
 
     private final Long connectionId;

fineract-core/src/main/java/org/apache/fineract/infrastructure/event/external/jobs/SendAsynchronousEventsTasklet.java

@@ -76,7 +76,8 @@ public class SendAsynchronousEventsTasklet implements Tasklet {
     }
 
     private boolean isDownstreamChannelEnabled() {
-        return fineractProperties.getEvents().getExternal().getProducer().getJms().isEnabled();
+        return fineractProperties.getEvents().getExternal().getProducer().getJms().isEnabled()
+                || fineractProperties.getEvents().getExternal().getProducer().getKafka().isEnabled();
     }
 
     private List<ExternalEventView> getQueuedEventsBatch() {

fineract-core/src/main/java/org/apache/fineract/infrastructure/event/external/producer/ExternalEventProducer.java

@@ -24,5 +24,13 @@ import org.apache.fineract.infrastructure.event.external.exception.Acknowledgeme
 
 public interface ExternalEventProducer {
 
+    /**
+     * Sends the created ExternalEvents
+     *
+     * @param partitions
+     *            is a Map<Long, List<byte[]>> partitions, the key here the id of the aggregated root. The value is list
+     *            of external events belong to the same key, serialized into byte array
+     * @throws AcknowledgementTimeoutException
+     */
     void sendEvents(Map<Long, List<byte[]>> partitions) throws AcknowledgementTimeoutException;
 }

fineract-core/src/main/java/org/apache/fineract/infrastructure/event/external/producer/NoopExternalEventEnabled.java

@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.event.external.producer;
+
+import org.apache.fineract.infrastructure.core.condition.PropertiesCondition;
+import org.apache.fineract.infrastructure.core.config.FineractProperties;
+
+public class NoopExternalEventEnabled extends PropertiesCondition {
+
+    @Override
+    protected boolean matches(FineractProperties properties) {
+        return (!getEffectiveJMSProperty(properties) && !getEffectiveKafkaProperty(properties));
+    }
+
+    private boolean getEffectiveJMSProperty(FineractProperties properties) {
+        if (properties.getEvents() == null || properties.getEvents().getExternal() == null
+                || properties.getEvents().getExternal().getProducer() == null
+                || properties.getEvents().getExternal().getProducer().getJms() == null) {
+            return false;
+        }
+        return properties.getEvents().getExternal().getProducer().getJms().isEnabled();
+    }
+
+    private boolean getEffectiveKafkaProperty(FineractProperties properties) {
+        if (properties.getEvents() == null || properties.getEvents().getExternal() == null
+                || properties.getEvents().getExternal().getProducer() == null
+                || properties.getEvents().getExternal().getProducer().getKafka() == null) {
+            return false;
+        }
+        return properties.getEvents().getExternal().getProducer().getKafka().isEnabled();
+    }
+
+}

fineract-core/src/main/java/org/apache/fineract/infrastructure/event/external/producer/NoopExternalEventProducer.java

@@ -22,11 +22,11 @@ import java.util.List;
 import java.util.Map;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.fineract.infrastructure.event.external.exception.AcknowledgementTimeoutException;
-import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Conditional;
 import org.springframework.stereotype.Component;
 
 @Component
-@ConditionalOnProperty(value = "fineract.events.external.producer.jms.enabled", havingValue = "false")
+@Conditional(NoopExternalEventEnabled.class)
 @Slf4j
 public class NoopExternalEventProducer implements ExternalEventProducer {
 

fineract-core/src/test/java/org/apache/fineract/infrastructure/core/config/FineractPropertiesTest.java

@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.core.config;
+
+import com.google.common.collect.Maps;
+import java.util.Map;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+class FineractPropertiesTest {
+
+    @Test
+    public void testKafkaProperties() {
+        // valid configs
+        testKafkaPropertiesParse("|", "=", null, Map.of());
+        testKafkaPropertiesParse("|", "=", "", Map.of());
+        testKafkaPropertiesParse("|", "=", "key1=value1", Map.of("key1", "value1"));
+        testKafkaPropertiesParse("|", "=", "key1=value1|key2=value2", Map.of("key1", "value1", "key2", "value2"));
+        testKafkaPropertiesParse(";", ":", "key1:value1;key2:value2", Map.of("key1", "value1", "key2", "value2"));
+
+        // invalid configs
+        testKafkaPropertiesParse("||", "=", "key1=value1", Map.of());
+        testKafkaPropertiesParse("|", "", "key1=value1", Map.of());
+        testKafkaPropertiesParse("", "", "key1=value1", Map.of());
+        testKafkaPropertiesParse("|", "=", "key1=value1=value2", Map.of());
+    }
+
+    private void testKafkaPropertiesParse(String lineSep, String keyValueSep, String property, Map<String, String> expected) {
+        FineractProperties.KafkaProperties kafkaProperties = new FineractProperties.KafkaProperties();
+        kafkaProperties.setExtraProperties(property);
+        kafkaProperties.setExtraPropertiesSeparator(lineSep);
+        kafkaProperties.setExtraPropertiesKeyValueSeparator(keyValueSep);
+        Assertions.assertTrue(Maps.difference(expected, kafkaProperties.getExtraPropertiesMap()).areEqual());
+    }
+
+}