Adding ability to upload files on s3.
To enable file upload on S3 you need to enable the configuration setting in c_configuration table for s3. The S3 credentials (access_key, secret_key, bucket_name) in a m_external_services table. m_client table is normalised into m_client and m_image table. The relation between m_client and m_image is one-to-one for now. To persist the storage type(s3 or filesystem) a column had been added to m_document. MISOFX-69 Adding db migrations for s3 support MIFOSX-69 Adding method to return boolean value of S3 config MIFOSX-69 - Priti/Harsha - Refactoring Code, extracted interface for Documents upload MIFOSX-69 Refactored to add S3 Support for file upload. Modified saveDocs to throw Document Management Exception instead of IO exception MIFOSX-69 Adding S3 support for file upload MIFOSX-69 - Adding S3 support for Image upload MIFOSX-69 Adding factory to get instance of data store as per s3 configuration MIFOSX-69 Changing table s3_details to generic tabe for external services, adding column to store the storage type in m_documents table MIFOSX-69 Fetching S3 details from database MIFOSX-69 Adding document storage type while creating a document MIFOSX-69 Normalising client table to create an image table Refactoring ClientImage code to save client image in m_image table Refactored code to get document as file MIFOSX-69 Retreiving documents from S3 MIFOSX-69 Refactored ClientData to store ImageData MIFOSX-69 Retrieving images from S3, Modifying Base64 to encode from byte array instead of file in CLientImagesApiResource MIFOSX-69 Refactored code to delete client image MIFOSX-69 Deleting Images from S3 Refactored DocumentStore to return StoreType from string value MIFOSX-69 Refactored to move document deletion responsibility to storageType MIFOSX-69 Giving more appropriate name to getInstance methods of DocumentStoreFactory as per the instance creation criteria MIFOSX-69 Deleting Documents from S3 MIFOSX-69 Removing unnecessary s3_param from migration MIFOSX-69 Adding ability to update document and update doc store type MIFOSX-69 Renaming method to follow java conventions MIFOSX-69 Adding filename to error log for s3 file retrieval Adding data migrations and changing the version for migration file
This commit is contained in:
Harshac
+46
@@ -0,0 +1,46 @@
|
||||
package org.mifosplatform.infrastructure.configuration.domain;
|
||||
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.mifosplatform.useradministration.domain.Permission;
|
||||
import org.mifosplatform.useradministration.domain.PermissionRepository;
|
||||
import org.mockito.runners.MockitoJUnitRunner;
|
||||
import org.springframework.data.domain.Page;
|
||||
import org.springframework.data.domain.Pageable;
|
||||
import org.springframework.data.domain.Sort;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
@RunWith(MockitoJUnitRunner.class)
|
||||
public class ConfigurationDomainServiceJpaTest {
|
||||
|
||||
private final String S3 = "amazon-S3";
|
||||
private PermissionRepository permissionRepositoryMock = mock(PermissionRepository.class);;
|
||||
private GlobalConfigurationRepository globalConfigurationRepositoryMock = mock(GlobalConfigurationRepository.class);
|
||||
|
||||
@Test
|
||||
public void shouldReturnTrueIfAmazonS3ConfigurationIsEnabled(){
|
||||
when(globalConfigurationRepositoryMock.findOneByName(S3)).thenReturn(new GlobalConfigurationProperty(S3, true));
|
||||
ConfigurationDomainServiceJpa configurationDomainServiceJpa = new ConfigurationDomainServiceJpa(permissionRepositoryMock, globalConfigurationRepositoryMock);
|
||||
|
||||
boolean actualIsS3Enabled = configurationDomainServiceJpa.isAmazonS3Enabled();
|
||||
|
||||
assertTrue(actualIsS3Enabled);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldReturnFalseIfAmazonS3ConfigurationIsDisabled(){
|
||||
when(globalConfigurationRepositoryMock.findOneByName(S3)).thenReturn(new GlobalConfigurationProperty(S3, false));
|
||||
ConfigurationDomainServiceJpa configurationDomainServiceJpa = new ConfigurationDomainServiceJpa(permissionRepositoryMock, globalConfigurationRepositoryMock);
|
||||
|
||||
boolean actualIsS3Enabled = configurationDomainServiceJpa.isAmazonS3Enabled();
|
||||
|
||||
assertFalse(actualIsS3Enabled);
|
||||
}
|
||||
}
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
package org.mifosplatform.infrastructure.core.service;
|
||||
|
||||
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.mifosplatform.infrastructure.configuration.domain.ConfigurationDomainService;
|
||||
import org.mifosplatform.infrastructure.configuration.domain.ExternalServicesDomainService;
|
||||
import org.mockito.runners.MockitoJUnitRunner;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
|
||||
import static junit.framework.Assert.assertEquals;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
|
||||
@RunWith(MockitoJUnitRunner.class)
|
||||
public class DocumentStoreFactoryTest {
|
||||
|
||||
@Test
|
||||
public void shouldReturnS3DocumentStoreObjectIfS3Enabled(){
|
||||
S3ClientFactory s3ClientFactoryMock = mock(S3ClientFactory.class);
|
||||
when(s3ClientFactoryMock.getS3Client()).thenReturn(null);
|
||||
|
||||
ExternalServicesDomainService externalServicesDomainServiceMock = mock(ExternalServicesDomainService.class);
|
||||
when(externalServicesDomainServiceMock.getValue("s3_bucket_name")).thenReturn("bucket_name");
|
||||
|
||||
ConfigurationDomainService configurationDomainServiceMock = mock(ConfigurationDomainService.class);
|
||||
when(configurationDomainServiceMock.isAmazonS3Enabled()).thenReturn(true);
|
||||
|
||||
ApplicationContext applicationContextMock = mock(ApplicationContext.class);
|
||||
|
||||
when(applicationContextMock.getBean("configurationDomainServiceJpa", ConfigurationDomainService.class)).thenReturn(configurationDomainServiceMock);
|
||||
when(applicationContextMock.getBean("s3ClientFactory", S3ClientFactory.class)).thenReturn(s3ClientFactoryMock);
|
||||
when(applicationContextMock.getBean("externalServicesDomainService", ExternalServicesDomainService.class)).thenReturn(externalServicesDomainServiceMock);
|
||||
|
||||
|
||||
DocumentStoreFactory documentStoreFactory = new DocumentStoreFactory(applicationContextMock);
|
||||
assertEquals(documentStoreFactory.getInstanceFromConfiguration().getClass(),S3DocumentStore.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldReturnFileSystemDocumentStoreObjectIfS3IsDisabled(){
|
||||
ConfigurationDomainService configurationDomainService = mock(ConfigurationDomainService.class);
|
||||
when(configurationDomainService.isAmazonS3Enabled()).thenReturn(false);
|
||||
|
||||
ApplicationContext applicationContext = mock(ApplicationContext.class);
|
||||
when(applicationContext.getBean("configurationDomainServiceJpa", ConfigurationDomainService.class)).thenReturn(configurationDomainService);
|
||||
|
||||
DocumentStoreFactory documentStoreFactory = new DocumentStoreFactory(applicationContext);
|
||||
assertEquals(documentStoreFactory.getInstanceFromConfiguration().getClass(),FileSystemDocumentStore.class);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
package org.mifosplatform.infrastructure.core.service;
|
||||
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.mifosplatform.infrastructure.documentmanagement.data.DocumentData;
|
||||
import org.mifosplatform.infrastructure.documentmanagement.data.FileData;
|
||||
import org.mockito.runners.MockitoJUnitRunner;
|
||||
|
||||
import static junit.framework.Assert.assertEquals;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
|
||||
@RunWith(MockitoJUnitRunner.class)
|
||||
public class FileDocumentStoreTest {
|
||||
|
||||
@Test
|
||||
public void shouldRetrieveDocument(){
|
||||
FileSystemDocumentStore fileSystemDocumentStore = new FileSystemDocumentStore();
|
||||
DocumentData documentDataMock = mock(DocumentData.class);
|
||||
when(documentDataMock.fileName()).thenReturn("some_file");
|
||||
when(documentDataMock.fileName()).thenReturn("content_type");
|
||||
when(documentDataMock.fileLocation()).thenReturn("abcd");
|
||||
|
||||
FileData fileData = fileSystemDocumentStore.retrieveDocument(documentDataMock);
|
||||
|
||||
assertEquals(fileData.name(), documentDataMock.fileName());
|
||||
assertEquals(fileData.contentType(), documentDataMock.contentType());
|
||||
}
|
||||
}
|
||||
+135
@@ -0,0 +1,135 @@
|
||||
package org.mifosplatform.infrastructure.core.service;
|
||||
|
||||
import com.amazonaws.AmazonClientException;
|
||||
import com.amazonaws.AmazonServiceException;
|
||||
import com.amazonaws.services.s3.AmazonS3Client;
|
||||
import com.amazonaws.services.s3.model.*;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.mifosplatform.infrastructure.documentmanagement.command.DocumentCommand;
|
||||
import org.mifosplatform.infrastructure.documentmanagement.data.DocumentData;
|
||||
import org.mifosplatform.infrastructure.documentmanagement.data.FileData;
|
||||
import org.mifosplatform.infrastructure.documentmanagement.exception.DocumentManagementException;
|
||||
import org.mifosplatform.infrastructure.documentmanagement.exception.DocumentNotFoundException;
|
||||
import org.mockito.Matchers;
|
||||
import org.mockito.runners.MockitoJUnitRunner;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
|
||||
import static junit.framework.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.mockito.Mockito.*;
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
|
||||
@RunWith(MockitoJUnitRunner.class)
|
||||
public class S3DocumentStoreTest {
|
||||
|
||||
InputStream inputStream;
|
||||
AmazonS3Client s3ClientMock;
|
||||
DocumentCommand documentCommand;
|
||||
DocumentData documentDataMock;
|
||||
|
||||
@Test
|
||||
public void shouldGenerateParentDirectoryFromEntityDetails(){
|
||||
S3DocumentStore s3DocumentStore=new S3DocumentStore("bucketName", null);
|
||||
String actualParentDirectory = s3DocumentStore.generateFileParentDirectory("entity_type", 1234L);
|
||||
assertTrue(actualParentDirectory.startsWith("documents/entity_type/1234"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldGenerateClientImageDirectoryFromResourceId(){
|
||||
S3DocumentStore s3DocumentStore=new S3DocumentStore("bucketName", null);
|
||||
String actualParentDirectory = s3DocumentStore.generateClientImageParentDirectory(1234L);
|
||||
assertTrue(actualParentDirectory.startsWith("images/clients/1234"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldSaveADocumentToS3() throws IOException {
|
||||
initUpload();
|
||||
when(s3ClientMock.putObject(Matchers.<PutObjectRequest>anyObject())).thenReturn(new PutObjectResult());
|
||||
S3DocumentStore s3DocumentStore = new S3DocumentStore("bucketName", s3ClientMock);
|
||||
|
||||
s3DocumentStore.saveDocument(inputStream, documentCommand);
|
||||
|
||||
verify(s3ClientMock, times(1)).putObject(Matchers.<PutObjectRequest>anyObject());
|
||||
}
|
||||
|
||||
@Test(expected = DocumentManagementException.class)
|
||||
public void shouldCatchAmazonServiceExceptionWhenUnableToUploadDocumentToS3() throws IOException {
|
||||
initUpload();
|
||||
when(s3ClientMock.putObject(Matchers.<PutObjectRequest>anyObject())).thenThrow(new AmazonServiceException("serviceException"));
|
||||
|
||||
S3DocumentStore s3DocumentStore = new S3DocumentStore("bucketName", s3ClientMock);
|
||||
|
||||
s3DocumentStore.saveDocument(inputStream, documentCommand);
|
||||
}
|
||||
|
||||
@Test(expected = DocumentManagementException.class)
|
||||
public void shouldCatchAmazonClientExceptionWhenUnableToUploadDocumentToS3() throws IOException {
|
||||
initUpload();
|
||||
when(s3ClientMock.putObject(Matchers.<PutObjectRequest>anyObject())).thenThrow(new AmazonClientException("clientException"));
|
||||
|
||||
S3DocumentStore s3DocumentStore = new S3DocumentStore("bucketName", s3ClientMock);
|
||||
|
||||
s3DocumentStore.saveDocument(inputStream, documentCommand);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldRetrieveDocumentFromS3() throws IOException {
|
||||
initRetrieve();
|
||||
when(s3ClientMock.getObject(Matchers.<GetObjectRequest>anyObject())).thenReturn(new S3Object());
|
||||
S3DocumentStore s3DocumentStore = new S3DocumentStore("bucketName",s3ClientMock);
|
||||
|
||||
FileData fileData = s3DocumentStore.retrieveDocument(documentDataMock);
|
||||
|
||||
verify(s3ClientMock, times(1)).getObject(Matchers.<GetObjectRequest>anyObject());
|
||||
assertEquals(fileData.name(), "name");
|
||||
assertEquals(fileData.contentType(), "type");
|
||||
}
|
||||
|
||||
@Test(expected = DocumentNotFoundException.class)
|
||||
public void shouldCatchAmazonServiceExceptionWhenUnableToRetrieveDocumentFromS3() throws IOException {
|
||||
initRetrieve();
|
||||
when(s3ClientMock.getObject(Matchers.<GetObjectRequest>anyObject())).thenThrow(new AmazonServiceException("serviceException"));
|
||||
S3DocumentStore s3DocumentStore = new S3DocumentStore("bucketName",s3ClientMock);
|
||||
|
||||
s3DocumentStore.retrieveDocument(documentDataMock);
|
||||
}
|
||||
|
||||
@Test(expected = DocumentNotFoundException.class)
|
||||
public void shouldCatchAmazonClientExceptionWhenUnableToRetrieveDocumentFromS3() throws IOException {
|
||||
initRetrieve();
|
||||
when(s3ClientMock.getObject(Matchers.<GetObjectRequest>anyObject())).thenThrow(new AmazonClientException("clientException"));
|
||||
S3DocumentStore s3DocumentStore = new S3DocumentStore("bucketName",s3ClientMock);
|
||||
|
||||
s3DocumentStore.retrieveDocument(documentDataMock);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldDeleteImageFromS3() throws IOException {
|
||||
AmazonS3Client s3ClientMock = mock(AmazonS3Client.class);
|
||||
S3DocumentStore s3DocumentStore = new S3DocumentStore("bucketName",s3ClientMock);
|
||||
|
||||
s3DocumentStore.deleteImage(123L,"imagekey");
|
||||
|
||||
verify(s3ClientMock, times(1)).deleteObject(Matchers.<DeleteObjectRequest>anyObject());
|
||||
}
|
||||
|
||||
private void initRetrieve() {
|
||||
s3ClientMock = mock(AmazonS3Client.class);
|
||||
|
||||
documentDataMock = mock(DocumentData.class);
|
||||
when(documentDataMock.fileLocation()).thenReturn("path");
|
||||
when(documentDataMock.fileName()).thenReturn("name");
|
||||
when(documentDataMock.contentType()).thenReturn("type");
|
||||
}
|
||||
|
||||
|
||||
private void initUpload(){
|
||||
inputStream = S3DocumentStoreTest.class.getResourceAsStream("DummyString");
|
||||
s3ClientMock = mock(AmazonS3Client.class);
|
||||
documentCommand = new DocumentCommand(null, -1l, "parentEntityType", 1l, null, "someFileName", 3l, null, null, null);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user