OKHttp and compressed files
1. Initiate a request processing
import okhttp3.*; import .*; import ; import ; import ; import ; import ; import ; import ; import ; import ; import ; import ; public class ApiServiceCaller { private static final ExecutorService executor = (10, runnable -> { Thread thread = new Thread(runnable); ("ApiServiceCaller-Thread"); (true); return thread; }); private static final Logger logger = (()); private static final OkHttpClient client = new () .connectTimeout(5, ) .readTimeout(5, ) .connectionPool(new ConnectionPool(10, 5, )) .retryOnConnectionFailure(true) .build(); // Asynchronously call the external system API method public CompletableFuture<String> callExternalApi(String url, Map<String, String> params, String method) { return (() -> { try { Request request = buildRequest(url, params, method); return executeRequest(request); } catch (Exception e) { (, "Error building request or executing request", e); throw new RuntimeException("An error occurred while calling the API: " + url, e); } }, executor); } // Build GET request private Request buildGetRequest(String url, Map<String, String> params) { httpBuilder = (url).newBuilder(); if (params != null && !()) { (httpBuilder::addQueryParameter); } return new ().url(()).get().build(); } // Build POST request private Request buildPostRequest(String url, Map<String, String> params) throws IOException { RequestBody body = ( ("application/json"), new ().writeValueAsString(params) ); return new ().url(url).post(body).build(); } // General request construction method private Request buildRequest(String url, Map<String, String> params, String method) throws IOException { if ("GET".equalsIgnoreCase(method)) { return buildGetRequest(url, params); } else if ("POST".equalsIgnoreCase(method)) { return buildPostRequest(url, params); } else { throw new IllegalArgumentException("Unsupported method: " + method); } } // Execute the request and process the response private String executeRequest(Request request) throws IOException { try (Response response = (request).execute()) { if (() && () != null) { String responseBody = ().string(); ("Response received: " + responseBody); return responseBody; } else { ("Received an abnormal response code: " + ()); throw new RuntimeException("The API call failed, response code: " + ()); } } } // Methods for handling API calls with multiple different URLs and parameters public List<CompletableFuture<String>> callMultipleApis(List<ApiRequest> apiRequests) { ("Multiple APIs are being called..."); return () .map(request -> callExternalApi((), (), ())) .collect(()); } // Methods to efficiently process CompletableFuture results public void processApiResponses(List<CompletableFuture<String>> futures) { CompletableFuture<Void> allOf = ((new CompletableFuture[0])); (v -> (future -> { ((response, throwable) -> { if (throwable != null) { (, "Error handling future", throwable); ("An error occurred while handling future: " + ()); } else { ("Processing Response: " + response); (response); } return null; }); })); } // Main function, calling API public static void main(String[] args) { ApiServiceCaller apiServiceCaller = new ApiServiceCaller(); List<ApiRequest> apiRequests = new ArrayList<>(); (new ApiRequest("/api1", ("param1", "value1"), "GET")); (new ApiRequest("/api2", ("key", "value"), "POST")); (new ApiRequest("/api3", ("param3", "value3"), "GET")); ("Start calling API..."); List<CompletableFuture<String>> apiCalls = (apiRequests); (apiCalls); } // Unit test of ApiServiceCaller public static class ApiServiceCallerTest { @Test public void testCallExternalApi_getRequest() { ApiServiceCaller caller = new ApiServiceCaller(); CompletableFuture<String> responseFuture = ("/api1", ("param", "value"), "GET"); (() -> { String response = (10, ); (response); }); } @Test public void testCallExternalApi_postRequest() { ApiServiceCaller caller = new ApiServiceCaller(); CompletableFuture<String> responseFuture = ("/api1", ("key", "value"), "POST"); (() -> { String response = (10, ); (response); }); } @Test public void testCallMultipleApis() { ApiServiceCaller caller = new ApiServiceCaller(); List<ApiRequest> apiRequests = new ArrayList<>(); (new ApiRequest("/api1", ("param1", "value1"), "GET")); (new ApiRequest("/api2", ("key", "value"), "POST")); List<CompletableFuture<String>> responseFutures = (apiRequests); (2, ()); (future -> (() -> { String response = (10, ); (response); })); } } // Class used to save API request details public static class ApiRequest { private final String url; private final Map<String, String> params; private final String method; public ApiRequest(String url, Map<String, String> params, String method) { = url; = params; = method; } public String getUrl() { return url; } public Map<String, String> getParams() { return params; } public String getMethod() { return method; } } } // Ensure the actuator is closed gracefully().addShutdownHook(new Thread(() -> { try { ("Closing the actuator..."); (); if (!(5, )) { ("The executor did not terminate within the specified time."); (); } } catch (InterruptedException e) { (, "Closing process interrupted", e); (); } }));
2. Compress files
import .s3.AmazonS3; import .s3.AmazonS3ClientBuilder; import ..S3Object; import ..S3ObjectInputStream; import .*; import ; import ; import ; import ; import ; import ; import ; import ; public class S3DownloadAndCompress { private final AmazonS3 s3Client; private final ExecutorService executorService; public S3DownloadAndCompress(int threadPoolSize) { ("Initialize S3 client and executor service..."); this.s3Client = ().build(); = (threadPoolSize); } public ByteArrayOutputStream getCompressedFileStream(List<String> fileKeys, String bucketName) { ("Start the download and compression process..."); ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (ZipOutputStream zipOut = new ZipOutputStream(baos)) { List<CompletableFuture<Void>> futures = () .map(fileKey -> (() -> { ("Start downloading and zipping files: " + fileKey); downloadAndCompressFile(s3Client, bucketName, fileKey, zipOut); ("Complete download and compress files: " + fileKey); }, executorService)) .collect(()); CompletableFuture<Void> allDownloads = ((new CompletableFuture[0])); (); ("All files have been successfully downloaded and compressed."); } catch (IOException e) { ("An error occurred during downloading and compression: " + ()); (); } finally { ("Shut the actuator service..."); (); try { if (!(60, )) { ("The executor service failed to terminate within 60 seconds and is being forced to close..."); (); } } catch (InterruptedException e) { ("It is interrupted while waiting for the executor service to terminate, forced shutdown..."); (); } } if (() == 0) { ("Compressed file stream is empty."); } return baos; } public void saveCompressedFileToPath(ByteArrayOutputStream compressedStream, String targetPath) { if (compressedStream == null || () == 0) { ("The compressed file stream is empty and cannot be saved."); return; } try (FileOutputStream fos = new FileOutputStream(targetPath)) { (fos); ("The compressed file has been saved to: " + targetPath); } catch (IOException e) { ("An error occurred while saving the compressed file: " + ()); (); } } private void downloadAndCompressFile(AmazonS3 s3Client, String bucketName, String fileKey, ZipOutputStream zipOut) { synchronized (zipOut) { try (S3Object s3Object = (bucketName, fileKey); S3ObjectInputStream s3is = ()) { ("Download file from bucket: " + fileKey + "Bocket Name: " + bucketName); ZipEntry zipEntry = new ZipEntry(fileKey); (zipEntry); byte[] buffer = new byte[4096]; int length; while ((length = (buffer)) >= 0) { (buffer, 0, length); } (); ("document " + fileKey + "Added to zip."); } catch (IOException e) { ("An error occurred while downloading or compressing a file: " + fileKey + " - " + ()); (); } } } public static void main(String[] args) { ("Start S3DownloadAndCompress..."); int threadPoolSize = 10; // This can be configured as needed S3DownloadAndCompress downloader = new S3DownloadAndCompress(threadPoolSize); List<String> fileKeys = ("", "", ""); String bucketName = "your-bucket-name"; String targetPath = "compressed_files.zip"; ByteArrayOutputStream compressedStream = (fileKeys, bucketName); (compressedStream, targetPath); ("S3DownloadAndCompress is finished."); } }
import .s3.AmazonS3; import .s3.AmazonS3ClientBuilder; import .; import ..S3Object; import ..S3ObjectInputStream; import .; import .; import .; import ; import .*; import ; import ; import ; import ; import ; import ; import ; import ; import ; import ; import ; public class S3DownloadAndCompress { private final AmazonS3 s3Client; private final ExecutorService executorService; private final TransferManager transferManager; private final String defaultFileName = "default_filename.txt"; // Initialize Amazon S3 client and thread pool public S3DownloadAndCompress(int threadPoolSize) { ("Initialize S3 client and executor service..."); this.s3Client = ().build(); = (threadPoolSize); = ().withS3Client(s3Client).build(); ("S3 client and executor service initialization is completed."); } // Get the file list, compress it into a Zip file, and return the compressed file stream public ByteArrayOutputStream getCompressedFileStream(List<String> fileKeys, String bucketName) { ("Start the download and compression process..."); ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (ZipOutputStream zipOut = new ZipOutputStream(baos)) { List<CompletableFuture<Void>> futures = () .map(fileKey -> (() -> { ("Start downloading and zipping files: " + fileKey); downloadAndCompressFile(bucketName, fileKey, zipOut); ("Complete download and compress files: " + fileKey); }, executorService)) .collect(()); CompletableFuture<Void> allDownloads = ((new CompletableFuture[0])); (); ("All files have been successfully downloaded and compressed."); } catch (IOException e) { ("An error occurred during downloading and compression: " + ()); (); } finally { shutdownExecutorService(); } ("The compression process is completed, returning to the compressed file stream."); return baos; } // Save the compressed file stream to the specified path public void saveCompressedFileToPath(ByteArrayOutputStream compressedStream, String targetPath) { if (compressedStream == null || () == 0) { throw new IllegalArgumentException("The compressed file stream is empty and cannot be saved."); } ("Start save the compressed file to the path: " + targetPath); try (FileOutputStream fos = new FileOutputStream(targetPath)) { (fos); ("The compressed file has been saved to: " + targetPath); } catch (IOException e) { ("An error occurred while saving the compressed file: " + ()); (); } } // Download the specified file from S3 and save it to the target path public void downloadFileToPath(String bucketName, String fileKey, String targetPath) { ("Start downloading files from S3: " + fileKey + "To the path: " + targetPath); try { String resolvedFileKey = resolveFileKey(bucketName, fileKey); File targetFile = new File(targetPath); Download download = (bucketName, resolvedFileKey, targetFile); (); ("The file has been downloaded successfully: " + targetPath); } catch (Exception e) { ("An error occurred while downloading the file: " + ()); (); } } // Generate temporary access link for the specified file public URL generatePresignedUrl(String bucketName, String fileKey, int expirationMinutes) { ("Generate temporary link, file: " + fileKey + " Validity period: " + expirationMinutes + " minute"); try { String resolvedFileKey = resolveFileKey(bucketName, fileKey); Date expiration = new Date(() + expirationMinutes * 60 * 1000); GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(bucketName, resolvedFileKey) .withMethod() .withExpiration(expiration); URL url = (request); ("The generated temporary link: " + ()); return url; } catch (Exception e) { ("An error occurred while generating temporary link: " + ()); (); return null; } } // Use temporary link to download the file and save it to the specified path public void downloadFileFromPresignedUrl(URL presignedUrl, String targetPath) { ("Use temporary link to download file to path: " + targetPath); try (BufferedInputStream in = new BufferedInputStream(()); FileOutputStream fileOutputStream = new FileOutputStream(targetPath)) { byte[] dataBuffer = new byte[8192]; int bytesRead; while ((bytesRead = (dataBuffer, 0, 8192)) != -1) { (dataBuffer, 0, bytesRead); } ("The file has been successfully downloaded through the temporary link: " + targetPath); } catch (IOException e) { ("An error occurred while downloading a file through a temporary link: " + ()); (); } } // Use temporary link to get the input stream of the file public InputStream getFileStreamFromPresignedUrl(URL presignedUrl) { ("Get file streams through temporary links: " + presignedUrl); try { HttpURLConnection connection = (HttpURLConnection) (); ("GET"); InputStream inputStream = (); ("File stream was successfully obtained."); return inputStream; } catch (IOException e) { ("An error occurred while getting file stream through temporary link: " + ()); (); return null; } } // parse the file key name, if the file does not exist, return the default file name private String resolveFileKey(String bucketName, String fileKey) { ("Resolve file key name: " + fileKey); if ((bucketName, fileKey)) { ("The file exists: " + fileKey); return fileKey; } else { ("The file does not exist, use the default file name: " + defaultFileName); return defaultFileName; } } // Download the file from S3 and compress it into ZipOutputStream private void downloadAndCompressFile(String bucketName, String fileKey, ZipOutputStream zipOut) { ("Download and compress files from S3: " + fileKey); synchronized (zipOut) { try (S3Object s3Object = (bucketName, fileKey); S3ObjectInputStream s3is = ()) { ("Download file from bucket: " + fileKey + "Bocket Name: " + bucketName); ZipEntry zipEntry = new ZipEntry(fileKey); (zipEntry); byte[] buffer = new byte[8192]; int length; while ((length = (buffer)) >= 0) { (buffer, 0, length); } (); ("document " + fileKey + "Added to zip."); } catch (IOException e) { ("An error occurred while downloading or compressing a file: " + fileKey + " - " + ()); (); } } } // Turn off the actuator service private void shutdownExecutorService() { ("Shut the actuator service..."); try { (); if (!(60, )) { ("The executor service failed to terminate within 60 seconds and is being forced to close..."); (); ("Called shutdownNow() Forced shutdown of the actuator service。"); } } catch (InterruptedException e) { ("It is interrupted while waiting for the executor service to terminate, forced shutdown..."); (); ().interrupt(); } ("The executor service is closed."); } public static void main(String[] args) { ("Start S3DownloadAndCompress..."); int threadPoolSize = 10; // This can be configured as needed S3DownloadAndCompress downloader = new S3DownloadAndCompress(threadPoolSize); List<String> fileKeys = ("", "", ""); String bucketName = "your-bucket-name"; String targetPath = "compressed_files.zip"; // Download and compress the file and save it to the target path ("Start downloading and compressing files..."); (fileKeys, bucketName, targetPath); ("Download and compress the file to complete."); // Download directly to the specified path ("Start downloading files directly..."); (bucketName, "", "downloaded_file1.txt"); ("Download the file directly."); // Generate temporary links ("Start to generate temporary links..."); URL presignedUrl = (bucketName, "", 60); if (presignedUrl != null) { ("Access Temporary Links: " + presignedUrl); // Download to local via temporary link ("Download files through temporary links..."); (presignedUrl, "downloaded_from_presigned_url.txt"); ("Download the file through a temporary link."); // Get file stream ("Get file stream..."); InputStream fileStream = (presignedUrl); if (fileStream != null) { ("File stream was successfully obtained."); } } ("S3DownloadAndCompress is finished."); } }
3. File storage
1. Configuration
# Bucket 1 Configuration .=accessKey1 .=secretKey1 .=http://endpoint1 .=us-east-1 # Bucket 2 Configuration .=accessKey2 .=secretKey2 .=http://endpoint2 .=us-west-1
2. Entity Class
package .s3config; import ; import ; @Component @ConfigurationProperties public class BucketConfig { private String accessKey; private String secretKey; private String endpoint; private String region; // Getters and setters public String getAccessKey() { return accessKey; } public void setAccessKey(String accessKey) { = accessKey; } public String getSecretKey() { return secretKey; } public void setSecretKey(String secretKey) { = secretKey; } public String getEndpoint() { return endpoint; } public void setEndpoint(String endpoint) { = endpoint; } public String getRegion() { return region; } public void setRegion(String region) { = region; } }
3. Configuration class
package .s3config; import ; import ; import org.; import org.; import ; import ; @Component @ConfigurationProperties(prefix = "") public class BucketsConfig { private static final Logger logger = (); private Map<String, BucketConfig> bucketConfigs = new HashMap<>(); public Map<String, BucketConfig> getBucketConfigs() { return bucketConfigs; } public void setBucketConfigs(Map<String, BucketConfig> bucketConfigs) { = bucketConfigs; // Log to confirm if configurations are loaded correctly ("Bucket configurations loaded: {}", ()); } public BucketConfig getBucketConfig(String bucketName) { BucketConfig bucketConfig = (bucketName); if (bucketConfig == null) { throw new IllegalArgumentException("Invalid bucket name: " + bucketName); } return bucketConfig; } }
4. Initialize class
package .s3config; import ; import ; import .s3.AmazonS3; import .s3.AmazonS3ClientBuilder; import ; import ; import org.; import org.; import ; import ; @Component public class AmazonS3Config { private static final Logger logger = (); private final BucketsConfig bucketsConfig; private final Map<String, AmazonS3> amazonS3ClientsCache = new ConcurrentHashMap<>(); @Autowired public AmazonS3Config(BucketsConfig bucketsConfig) { = bucketsConfig; ("AmazonS3Config initialized with BucketsConfig"); } public AmazonS3 getAmazonS3Client(String bucketName) { // Check if client is already in cache if ((bucketName)) { ("Returning cached AmazonS3 client for bucket: {}", bucketName); return (bucketName); } // Get bucket configuration BucketConfig bucketConfig = (bucketName); // Ensure all required configurations are present if (() == null || () == null || () == null || () == null) { throw new IllegalArgumentException("Incomplete bucket configuration for: " + bucketName); } // Initialize AmazonS3 client BasicAWSCredentials awsCreds = new BasicAWSCredentials((), ()); AmazonS3 amazonS3 = () .withCredentials(new AWSStaticCredentialsProvider(awsCreds)) .withEndpointConfiguration( new ((), ())) .withPathStyleAccessEnabled(true) .build(); // Cache the client for future use (bucketName, amazonS3); ("AmazonS3 client created and cached for bucket: {}", bucketName); return amazonS3; } }
5. Get the object
package .s3config; import .s3.AmazonS3; import ; import ; import org.; import org.; import ; @Service public class S3Service { private static final Logger logger = (); private final AmazonS3Config amazonS3Config; @Autowired public S3Service(AmazonS3Config amazonS3Config) { this.amazonS3Config = amazonS3Config; ("S3Service initialized with AmazonS3Config"); } public void uploadFile(String bucketName, String key, File file) { AmazonS3 amazonS3 = amazonS3Config.getAmazonS3Client(bucketName); (bucketName, key, file); ("File uploaded to bucket: {}, key: {}", bucketName, key); } // Other operations }
6. Main program
package .s3config; import ; import ; import ; import ; import ; @SpringBootApplication @EnableConfigurationProperties() public class YourApplication { public static void main(String[] args) { (, args); } @Bean CommandLineRunner validateBucketsConfig(BucketsConfig bucketsConfig) { return args -> { ("Validating bucket configurations: " + ().keySet()); }; } }
7. Test class
package .s3config; import ; import ; import ; import ; import ; import static .*; @SpringBootTest @TestPropertySource("classpath:") public class BucketsConfigTest { @Autowired private BucketsConfig bucketsConfig; @Test public void testBucketsConfigLoaded() { assertNotNull(bucketsConfig, "BucketsConfig should not be null"); assertFalse(().isEmpty(), "Bucket configurations should not be empty"); assertTrue(().containsKey("bucket1"), "Bucket1 should be present in the configurations"); assertTrue(().containsKey("bucket2"), "Bucket2 should be present in the configurations"); } @Test public void testGetBucketConfig() { BucketConfig bucket1 = ("bucket1"); assertNotNull(bucket1, "BucketConfig for bucket1 should not be null"); assertEquals("accessKey1", ()); assertEquals("secretKey1", ()); assertEquals("http://endpoint1", ()); assertEquals("us-east-1", ()); } @Test public void testInvalidBucket() { Exception exception = assertThrows(, () -> { ("invalidBucket"); }); assertEquals("Invalid bucket name: invalidBucket", ()); } }
package .s3config; import .s3.AmazonS3; import ; import ; import ; import ; import ; import static .*; import static .*; @SpringBootTest @TestPropertySource("classpath:") public class AmazonS3ConfigTest { @Autowired private AmazonS3Config amazonS3Config; @MockBean private BucketsConfig bucketsConfig; @Test public void testGetAmazonS3Client() { // Mock the BucketConfig BucketConfig bucketConfig = new BucketConfig(); ("accessKey1"); ("secretKey1"); ("http://endpoint1"); ("us-east-1"); when(("bucket1")).thenReturn(bucketConfig); AmazonS3 s3Client = amazonS3Config.getAmazonS3Client("bucket1"); assertNotNull(s3Client, "AmazonS3 client should not be null"); // Verify that the client is cached AmazonS3 cachedClient = amazonS3Config.getAmazonS3Client("bucket1"); assertSame(s3Client, cachedClient, "Cached client should be the same instance"); } @Test public void testGetAmazonS3ClientInvalidBucket() { when(("invalidBucket")) .thenThrow(new IllegalArgumentException("Invalid bucket name: invalidBucket")); Exception exception = assertThrows(, () -> { amazonS3Config.getAmazonS3Client("invalidBucket"); }); assertEquals("Invalid bucket name: invalidBucket", ()); } }
package .s3config; import .s3.AmazonS3; import ; import ; import ; import ; import ; import static .*; import static .*; @SpringBootTest public class S3ServiceTest { @Mock private AmazonS3Config amazonS3Config; @Mock private AmazonS3 amazonS3; @InjectMocks private S3Service s3Service; @Test public void testUploadFile() { String bucketName = "bucket1"; String key = ""; File file = new File(""); when(amazonS3Config.getAmazonS3Client(bucketName)).thenReturn(amazonS3); (bucketName, key, file); verify(amazonS3Config, times(1)).getAmazonS3Client(bucketName); verify(amazonS3, times(1)).putObject(bucketName, key, file); } @Test public void testUploadFileWithInvalidBucket() { String bucketName = "invalidBucket"; String key = ""; File file = new File(""); when(amazonS3Config.getAmazonS3Client(bucketName)) .thenThrow(new IllegalArgumentException("Invalid bucket name: " + bucketName)); Exception exception = assertThrows(, () -> { (bucketName, key, file); }); assertEquals("Invalid bucket name: " + bucketName, ()); } }
8.rely
Make sure to be inAdd the following dependencies to:
<!-- AWS SDK --> <dependency> <groupId></groupId> <artifactId>aws-java-sdk-s3</artifactId> <version>1.12.100</version> </dependency> <!-- Spring Boot Starter --> <dependency> <groupId></groupId> <artifactId>spring-boot-starter</artifactId> </dependency> <!-- Spring Boot Configuration Processor --> <dependency> <groupId></groupId> <artifactId>spring-boot-configuration-processor</artifactId> <optional>true</optional> </dependency> <!-- Testing --> <dependency> <groupId></groupId> <artifactId>spring-boot-starter-test</artifactId> <scope>test</scope> </dependency> <!-- Mockito --> <dependency> <groupId></groupId> <artifactId>mockito-core</artifactId> <version>3.9.0</version> <scope>test</scope> </dependency>
This is the article about the use of OKHttp and compressed files in SpringBoot. For more relevant content on SpringBoot's OKHttp, please search for my previous articles or continue browsing the related articles below. I hope everyone will support me in the future!