FINERACT-1707 - S3 export and dependent service

- [x] S3 export function
- [x] list all available export type services
- [x] refactor export service
- [x] unit test for export name generator
- [x] s3 integration test
- [x] localstack integration for testing
This commit is contained in:
Janos Haber
2023-02-06 19:53:00 +01:00
committed by Arnold Gálovics
parent e08675ed54
commit d8fd8088b0
34 changed files with 1132 additions and 80 deletions
@@ -0,0 +1,30 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.fineract.integrationtests;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
@Retention(RetentionPolicy.RUNTIME)
@Target({ ElementType.TYPE, ElementType.METHOD })
@EnabledIfEnvironmentVariable(named = "GITHUB_ACTIONS", matches = "true")
public @interface CIOnly {}
@@ -0,0 +1,59 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.fineract.integrationtests.client;
import java.io.IOException;
import java.util.Map;
import okhttp3.MediaType;
import okhttp3.ResponseBody;
import org.apache.fineract.integrationtests.CIOnly;
import org.apache.fineract.integrationtests.common.Utils;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import retrofit2.Response;
/**
* Integration Test for /runreports/ API.
*
* @author Michael Vorburger.ch
*/
public class ReportExportTest extends IntegrationTest {
@BeforeEach
public void setup() {
Utils.initializeRESTAssured();
}
@Test
void runClientListingTableReportCSV() throws IOException {
Response<ResponseBody> result = okR(
fineract().reportsRun.runReportGetFile("Client Listing", Map.of("R_officeId", "1", "exportCSV", "true"), false));
assertThat(result.body().contentType()).isEqualTo(MediaType.parse("text/csv"));
assertThat(result.body().string()).contains("Office/Branch");
}
@Test
@CIOnly
void runClientListingTableReportS3() throws IOException {
Response<ResponseBody> result = okR(
fineract().reportsRun.runReportGetFile("Client Listing", Map.of("R_officeId", "1", "exportS3", "true"), false));
assertThat(result.code()).isEqualTo(204);
}
}
@@ -27,6 +27,7 @@ import org.apache.fineract.integrationtests.common.Utils;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import retrofit2.Response;
/**
* Integration Test for /runreports/ API.
@@ -51,6 +52,14 @@ public class ReportsTest extends IntegrationTest {
.getColumnName()).isEqualTo("Office/Branch");
}
@Test
void runClientListingTableReportCSV() throws IOException {
Response<ResponseBody> result = okR(
fineract().reportsRun.runReportGetFile("Client Listing", Map.of("R_officeId", "1", "exportCSV", "true"), false));
assertThat(result.body().contentType()).isEqualTo(MediaType.parse("text/csv"));
assertThat(result.body().string()).contains("Office/Branch");
}
@Test // see FINERACT-1306
void runReportCategory() throws IOException {
// Using raw OkHttp instead of Retrofit API here, because /runreports/reportCategoryList returns JSON Array -
@@ -96,9 +96,9 @@ public class GlobalConfigurationHelper {
// If any other column is modified by the integration test suite in
// the future, it needs to be reset here.
final Integer configDefaultId = (Integer) defaultGlobalConfiguration.get("id");
final Integer configDefaultValue = (Integer) defaultGlobalConfiguration.get("value");
final String configDefaultValue = String.valueOf(defaultGlobalConfiguration.get("value"));
updateValueForGlobalConfiguration(requestSpec, responseSpec, configDefaultId.toString(), configDefaultValue.toString());
updateValueForGlobalConfiguration(requestSpec, responseSpec, configDefaultId.toString(), configDefaultValue);
updateEnabledFlagForGlobalConfiguration(requestSpec, responseSpec, configDefaultId.toString(),
(Boolean) defaultGlobalConfiguration.get("enabled"));
changedNo++;
@@ -119,9 +119,9 @@ public class GlobalConfigurationHelper {
ArrayList<HashMap> expectedGlobalConfigurations = getAllDefaultGlobalConfigurations();
ArrayList<HashMap> actualGlobalConfigurations = getAllGlobalConfigurations(requestSpec, responseSpec);
// There are currently 48 global configurations.
Assertions.assertEquals(48, expectedGlobalConfigurations.size());
Assertions.assertEquals(48, actualGlobalConfigurations.size());
// There are currently 49 global configurations.
Assertions.assertEquals(49, expectedGlobalConfigurations.size());
Assertions.assertEquals(49, actualGlobalConfigurations.size());
for (int i = 0; i < expectedGlobalConfigurations.size(); i++) {
@@ -540,6 +540,14 @@ public class GlobalConfigurationHelper {
externalEventBatchSize.put("enabled", false);
externalEventBatchSize.put("trapDoor", false);
defaults.add(externalEventBatchSize);
HashMap<String, Object> reportExportS3FolderName = new HashMap<>();
reportExportS3FolderName.put("id", 54);
reportExportS3FolderName.put("name", "report-export-s3-folder-name");
reportExportS3FolderName.put("value", 0);
reportExportS3FolderName.put("enabled", false);
reportExportS3FolderName.put("trapDoor", false);
defaults.add(reportExportS3FolderName);
return defaults;
}