From a6982024606810f43aa87e3f3a4f5011699fde34 Mon Sep 17 00:00:00 2001 From: Dongie Agnir Date: Thu, 24 Apr 2025 10:29:47 -0700 Subject: [PATCH 1/8] Add retry tests for request bodies This commit adds tests for implementations of `RequestBody` and `AsyncRequestBody` where the requests are retried to ensure that they send the same content for every attempt. --- test/s3-tests/pom.xml | 5 + .../s3/AsyncRequestBodyRetryTest.java | 241 ++++++++++++++++++ .../services/s3/BaseRequestBodyRetryTest.java | 228 +++++++++++++++++ .../awssdk/services/s3/ChunkedDecoder.java | 117 +++++++++ .../services/s3/SyncRequestBodyRetryTest.java | 216 ++++++++++++++++ test/s3-tests/src/test/resources/log4j2.xml | 36 +++ 6 files changed, 843 insertions(+) create mode 100644 test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/AsyncRequestBodyRetryTest.java create mode 100644 test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/BaseRequestBodyRetryTest.java create mode 100644 test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/ChunkedDecoder.java create mode 100644 test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/SyncRequestBodyRetryTest.java create mode 100644 test/s3-tests/src/test/resources/log4j2.xml diff --git a/test/s3-tests/pom.xml b/test/s3-tests/pom.xml index 9588bafc196..a6a306d6102 100644 --- a/test/s3-tests/pom.xml +++ b/test/s3-tests/pom.xml @@ -152,6 +152,11 @@ ${awsjavasdk.version} test + + com.github.tomakehurst + wiremock-jre8 + test + diff --git a/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/AsyncRequestBodyRetryTest.java b/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/AsyncRequestBodyRetryTest.java new file mode 100644 index 00000000000..195a30f994a --- /dev/null +++ b/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/AsyncRequestBodyRetryTest.java @@ -0,0 +1,241 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.stream.IntStream; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.http.SdkHttpConfigurationOption; +import software.amazon.awssdk.http.async.SdkAsyncHttpClient; +import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.retries.StandardRetryStrategy; +import software.amazon.awssdk.retries.api.BackoffStrategy; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.utils.AttributeMap; + +/** + * Tests to ensure different {@link AsyncRequestBody} implementations return the same data for every retry. + */ +public class AsyncRequestBodyRetryTest extends BaseRequestBodyRetryTest { + private static ExecutorService requestBodyExecutor; + private static SdkAsyncHttpClient netty; + private S3AsyncClient s3; + + @BeforeAll + public static void setup() throws IOException { + BaseRequestBodyRetryTest.setup(); + requestBodyExecutor = Executors.newSingleThreadExecutor(); + netty = NettyNioAsyncHttpClient.builder() + .buildWithDefaults(AttributeMap.builder() + .put(SdkHttpConfigurationOption.TRUST_ALL_CERTIFICATES, + true) + .build()); + } + + @BeforeEach + public void methodSetup() { + s3 = S3AsyncClient.builder() + .overrideConfiguration(o -> o.retryStrategy(StandardRetryStrategy.builder() + .maxAttempts(3) + .backoffStrategy(BackoffStrategy.retryImmediately()) + .build())) + .region(Region.US_WEST_2) + .endpointOverride(URI.create("https://localhost:" + wireMockServer.httpsPort())) + .httpClient(netty) + .forcePathStyle(true) + .build(); + } + + @AfterAll + public static void teardown() { + BaseRequestBodyRetryTest.teardown(); + netty.close(); + requestBodyExecutor.shutdown(); + } + + @ParameterizedTest + @MethodSource("retryTestCases") + void test_retries_allAttemptsSendSameBody(TestCase tc) throws IOException { + Assumptions.assumeFalse(tc.type == BodyType.BLOCKING_INPUTSTREAM, + "forBlockingInputStream does not support retrying"); + // all content is created the same way so this data should match what's in the RequestBody + byte[] referenceData = makeArrayOfSize(tc.size.getNumBytes()); + String expectedCrc32 = calculateCrc32(new ByteArrayInputStream(referenceData)); + + AsyncRequestBody body = makeRequestBody(tc); + + assertThatThrownBy(s3.putObject(r -> r.bucket("my-bucket").key("my-obj"), body)::join) + .hasCauseInstanceOf(S3Exception.class) + .matches(e -> { + S3Exception s3e = (S3Exception) e.getCause(); + return s3e.numAttempts() == 3 && s3e.statusCode() == 500; + }, "Should attempt total of 3 times"); + + List requestBodyChecksums = getRequestChecksums(); + assertThat(requestBodyChecksums.size()).isEqualTo(3); + + requestBodyChecksums.forEach(bodyChecksum -> assertThat(bodyChecksum).isEqualTo(expectedCrc32)); + } + + private static List retryTestCases() { + List testCases = new ArrayList<>(); + + for (BodyType type : BodyType.values()) { + for (BodySize size : BodySize.values()) { + testCases.add(new TestCase().type(type).size(size)); + } + } + + return testCases; + } + + private AsyncRequestBody makeRequestBody(TestCase tc) throws IOException { + int nBytes = tc.size.getNumBytes(); + switch (tc.type) { + case STRING: + return AsyncRequestBody.fromString(makeStringOfSize(nBytes), StandardCharsets.UTF_8); + case BYTES: + return AsyncRequestBody.fromBytes(makeArrayOfSize(nBytes)); + case BYTES_UNSAFE: + return AsyncRequestBody.fromBytesUnsafe(makeArrayOfSize(nBytes)); + case FILE: + return AsyncRequestBody.fromFile(testFiles.get(tc.size)); + case INPUTSTREAM: { + InputStream is = getMarkSupportedStreamOfSize(tc.size); + return AsyncRequestBody.fromInputStream(cfg -> cfg.inputStream(is) + .contentLength((long) nBytes) + .maxReadLimit(nBytes) + .executor(requestBodyExecutor)); + } + case REMAINING_BYTE_BUFFER: + // fall through + case REMAINING_BYTE_BUFFER_UNSAFE: + // fall through + case BYTE_BUFFER_UNSAFE: + // fall through + case BYTE_BUFFER: { + ByteBuffer byteBuffer = ByteBuffer.wrap(makeArrayOfSize(nBytes)); + switch (tc.type) { + case REMAINING_BYTE_BUFFER: + return AsyncRequestBody.fromRemainingByteBuffer(byteBuffer); + case REMAINING_BYTE_BUFFER_UNSAFE: + return AsyncRequestBody.fromRemainingByteBufferUnsafe(byteBuffer); + case BYTE_BUFFER_UNSAFE: + return AsyncRequestBody.fromByteBufferUnsafe(byteBuffer); + case BYTE_BUFFER: + return AsyncRequestBody.fromByteBuffer(byteBuffer); + default: + throw new RuntimeException("Unexpected type: " + tc.type); + } + } + case REMAINING_BYTE_BUFFERS: + // fall through + case REMAINING_BYTE_BUFFERS_UNSAFE: + // fall through + case BYTE_BUFFERS_UNSAFE: + // fall through + case BYTE_BUFFERS: { + byte[] bbContent = getDataSegment(); + int nSegments = nBytes / bbContent.length; + + ByteBuffer[] buffers = IntStream.range(0, nSegments) + .mapToObj(i -> ByteBuffer.wrap(bbContent)) + .toArray(ByteBuffer[]::new); + + switch (tc.type) { + case REMAINING_BYTE_BUFFERS: + return AsyncRequestBody.fromRemainingByteBuffers(buffers); + case REMAINING_BYTE_BUFFERS_UNSAFE: + return AsyncRequestBody.fromRemainingByteBuffersUnsafe(buffers); + case BYTE_BUFFERS_UNSAFE: + return AsyncRequestBody.fromByteBuffersUnsafe(buffers); + case BYTE_BUFFERS: + return AsyncRequestBody.fromByteBuffers(buffers); + default: + throw new RuntimeException("Unexpected type: " + tc.type); + } + } + default: + throw new RuntimeException("Unsupported body type: " + tc.type); + } + } + + private enum BodyType { + STRING, + + BYTES, + BYTES_UNSAFE, + + INPUTSTREAM, + BLOCKING_INPUTSTREAM, // Note: doesn't support retries, left out for testing + + BYTE_BUFFER, + BYTE_BUFFER_UNSAFE, + REMAINING_BYTE_BUFFER, + REMAINING_BYTE_BUFFER_UNSAFE, + + + BYTE_BUFFERS, + BYTE_BUFFERS_UNSAFE, + REMAINING_BYTE_BUFFERS, + REMAINING_BYTE_BUFFERS_UNSAFE, + + FILE; + } + + + private static class TestCase { + private BodyType type; + private BodySize size; + + public TestCase type(BodyType type) { + this.type = type; + return this; + } + + public TestCase size(BodySize size) { + this.size = size; + return this; + } + + @Override + public String toString() { + return "TestCase{" + + "type=" + type + + ", size=" + size + + '}'; + } + } +} diff --git a/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/BaseRequestBodyRetryTest.java b/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/BaseRequestBodyRetryTest.java new file mode 100644 index 00000000000..3f76cd59688 --- /dev/null +++ b/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/BaseRequestBodyRetryTest.java @@ -0,0 +1,228 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3; + +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.client.WireMock; +import com.github.tomakehurst.wiremock.stubbing.ServeEvent; +import com.github.tomakehurst.wiremock.verification.LoggedRequest; +import java.io.BufferedInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Base64; +import java.util.EnumMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.stream.Collectors; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import software.amazon.awssdk.checksums.DefaultChecksumAlgorithm; +import software.amazon.awssdk.checksums.SdkChecksum; +import software.amazon.awssdk.utils.Logger; + +/** + * Base class for testing sync and async request bodies under retries. + *

+ * All the test data is built up using the same random ASCII string so that regardless of form (e.g. String, byte array, file), + * the binary data is the same. + */ +public class BaseRequestBodyRetryTest { + private static final Logger LOG = Logger.loggerFor(BaseRequestBodyRetryTest.class); + + private static byte[] random1KbAsciiData; + + protected static final int KB = 1024; + protected static final int MB = KB * 1024; + protected static final Map testFiles = new EnumMap<>(BodySize.class); + protected static final WireMockServer wireMockServer = new WireMockServer(0, 0); + protected static final SdkChecksum crc32 = SdkChecksum.forAlgorithm(DefaultChecksumAlgorithm.CRC32); + + @BeforeAll + public static void setup() throws IOException { + wireMockServer.start(); + createTestFiles(); + } + + @AfterAll + public static void teardown() { + testFiles.values().forEach(tf -> { + try { + Files.delete(tf); + } catch (IOException e) { + LOG.warn(() -> "Could not delete test file " + tf.toAbsolutePath()); + } + }); + wireMockServer.stop(); + } + + @BeforeEach + public void resetWireMock() { + wireMockServer.resetAll(); + wireMockServer.stubFor(WireMock.put(WireMock.anyUrl()) + .willReturn(WireMock.aResponse().withStatus(500))); + } + + protected byte[] getDataSegment() { + return random1KbAsciiData; + } + + protected static String calculateCrc32(InputStream inputStream) throws IOException { + crc32.reset(); + + byte[] buff = new byte[4096]; + int read; + while ((read = inputStream.read(buff)) != -1) { + crc32.update(buff, 0, read); + } + + inputStream.close(); + return base64Encode(crc32.getChecksumBytes()); + } + + protected byte[] makeArrayOfSize(int size) { + byte[] segmentData = getDataSegment(); + int sourceLen = segmentData.length; + if (size % sourceLen != 0) { + throw new IllegalArgumentException("Must be multiple of " + sourceLen + " bytes"); + } + + byte[] array = new byte[size]; + int segments = size / sourceLen; + for (int i = 0; i < segments; i++) { + System.arraycopy(segmentData, 0, array, i * sourceLen, sourceLen); + } + + return array; + } + + protected InputStream getMarkSupportedStreamOfSize(BodySize size) throws IOException { + Path p = testFiles.get(size); + if (p == null) { + throw new RuntimeException("No file for size " + size); + } + + return new BufferedInputStream(Files.newInputStream(p)); + } + + protected String makeStringOfSize(int size) { + return new String(makeArrayOfSize(size), StandardCharsets.UTF_8); + } + + protected List getRequestChecksums() { + return requestBodies().stream() + .map(r -> { + if (isRequestChunked(r)) { + return getDecodedChecksum(r.getBody()); + } + crc32.reset(); + crc32.update(r.getBody()); + return crc32.getChecksumBytes(); + }).map(BaseRequestBodyRetryTest::base64Encode) + .collect(Collectors.toList()); + } + + private List requestBodies() { + return wireMockServer.getAllServeEvents().stream() + .filter(ServeEvent::getWasMatched) + .map(ServeEvent::getRequest) + .collect(Collectors.toList()); + } + + protected static byte[] getDecodedChecksum(byte[] chunkEncoded) { + crc32.reset(); + + ChunkedDecoder decoder = new ChunkedDecoder(crc32); + int updateLen = MB; + + int fullSegments = chunkEncoded.length / updateLen; + int offset = 0; + + for (; fullSegments > 0; fullSegments--, offset += updateLen) { + decoder.update(chunkEncoded, offset, updateLen); + } + decoder.update(chunkEncoded, offset, chunkEncoded.length - offset); + + return decoder.checksumBytes(); + } + + protected static String base64Encode(byte[] bytes) { + return Base64.getEncoder().encodeToString(bytes); + } + + private static void createTestFiles() throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(KB); + // ASCII A-Z + new Random().ints(KB, 'A', (byte) 'Z' + 1) + .forEach(i -> baos.write((byte) i)); + + random1KbAsciiData = baos.toByteArray(); + for (BodySize size : BodySize.values()) { + Path tempFile = Files.createTempFile(null, null); + OutputStream os = Files.newOutputStream(tempFile); + + for (int i = 0; i < size.getNumBytes() / KB; ++i) { + os.write(random1KbAsciiData); + } + + int remainder = size.getNumBytes() % KB; + + if (remainder > 0) { + os.write(random1KbAsciiData, 0, remainder); + } + + os.flush(); + os.close(); + + testFiles.put(size, tempFile); + } + } + + private boolean isRequestChunked(LoggedRequest request) { + return "aws-chunked".equalsIgnoreCase(request.getHeader("content-encoding")); + } + + protected enum BodySize { + SZ_128KB(128 * KB), + + SZ_4MB(4 * MB), + + SZ_8MB(8 * MB), + + SZ_16MB(16 * MB), + + SZ_32MB(32 * MB), + + SZ_64MB(64 * MB), + ; + + private final int bytes; + + BodySize(int bytes) { + this.bytes = bytes; + } + + public int getNumBytes() { + return bytes; + } + } +} diff --git a/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/ChunkedDecoder.java b/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/ChunkedDecoder.java new file mode 100644 index 00000000000..e71c4f47895 --- /dev/null +++ b/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/ChunkedDecoder.java @@ -0,0 +1,117 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3; + +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import software.amazon.awssdk.checksums.SdkChecksum; + +/** + * Simple decoder for decoding an 'aws-chunked' body to get the raw content's + * checksum. + *

+ * See + * https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html. + */ +public final class ChunkedDecoder { + + enum State { + READING_META, + READING_META_END, + READING_CONTENT, + READING_CONTENT_END, + READING_TRAILER, + ; + } + + private State currentState = State.READING_META; + + private final ByteBuffer buffer = ByteBuffer.allocate(8 * 1024 * 1024); // 8 MiB + private final SdkChecksum checksum; + private ByteArrayOutputStream metaLine = new ByteArrayOutputStream(); + private ByteArrayOutputStream rawContent = new ByteArrayOutputStream(); + private int remainingContent = 0; + + public ChunkedDecoder(SdkChecksum checksum) { + this.checksum = checksum; + } + + public void update(byte[] data, int offset, int length) { + buffer.put(data, offset, length); + buffer.flip(); + processBuffer(); + } + + public byte[] checksumBytes() { + return checksum.getChecksumBytes(); + } + + private void processBuffer() { + while (buffer.hasRemaining()) { + byte b = buffer.get(); + + switch (currentState) { + case READING_META: + if (b == '\r') { + currentState = State.READING_META_END; + String meta = new String(metaLine.toByteArray(), StandardCharsets.UTF_8); + int semiIdx = meta.indexOf(';'); + if (semiIdx > 0) { + meta = meta.substring(0, semiIdx); + } + remainingContent = Integer.parseInt(meta, 16); + } else { + metaLine.write(b); + } + break; + case READING_META_END: + if (b != '\n') { + throw new RuntimeException("Expected newline"); + } + // An empty chunk signifies the end of the content data. Only trailers if any, remain + if (remainingContent > 0) { + currentState = State.READING_CONTENT; + } else { + currentState = State.READING_TRAILER; + } + rawContent.reset(); + break; + case READING_CONTENT: + if (remainingContent > 0) { + checksum.update(b); + remainingContent -= 1; + } else if (b == '\r') { + currentState = State.READING_CONTENT_END; + } else { + throw new RuntimeException("Expected carriage return"); + } + break; + case READING_CONTENT_END: + if (b != '\n') { + throw new RuntimeException("Expected newline"); + } + currentState = State.READING_META; + metaLine.reset(); + break; + case READING_TRAILER: + // don't need it + break; + } + } + buffer.clear(); + } +} diff --git a/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/SyncRequestBodyRetryTest.java b/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/SyncRequestBodyRetryTest.java new file mode 100644 index 00000000000..c9e278159d8 --- /dev/null +++ b/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/SyncRequestBodyRetryTest.java @@ -0,0 +1,216 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.http.ContentStreamProvider; +import software.amazon.awssdk.http.SdkHttpClient; +import software.amazon.awssdk.http.SdkHttpConfigurationOption; +import software.amazon.awssdk.http.apache.ApacheHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.retries.StandardRetryStrategy; +import software.amazon.awssdk.retries.api.BackoffStrategy; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.utils.AttributeMap; +import software.amazon.awssdk.utils.FunctionalUtils; + +/** + * Tests to ensure different {@link RequestBody} implementations return the same data for every retry. + */ +public class SyncRequestBodyRetryTest extends BaseRequestBodyRetryTest { + private static SdkHttpClient apache; + private S3Client s3; + + @BeforeAll + public static void setup() throws IOException { + BaseRequestBodyRetryTest.setup(); + apache = ApacheHttpClient.builder() + .buildWithDefaults(AttributeMap.builder() + .put(SdkHttpConfigurationOption.TRUST_ALL_CERTIFICATES, true) + .build()); + } + + @BeforeEach + public void methodSetup() { + s3 = S3Client.builder() + .overrideConfiguration(o -> o.retryStrategy(StandardRetryStrategy.builder() + .maxAttempts(3) + .backoffStrategy(BackoffStrategy.retryImmediately()) + .build())) + .region(Region.US_WEST_2) + .endpointOverride(URI.create("https://localhost:" + wireMockServer.httpsPort())) + .forcePathStyle(true) + .httpClient(apache) + .build(); + } + + @AfterAll + public static void teardown() { + apache.close(); + BaseRequestBodyRetryTest.teardown(); + } + + @AfterEach + public void methodTeardown() { + s3.close(); + } + + @ParameterizedTest + @MethodSource("testCases") + void test_retries_allAttemptsSendSameBody(TestCase tc) throws IOException { + // fromInputStream sets an unconfigurable read limit of 128KiB so anything larger will fail to reset(). + Assumptions.assumeFalse(tc.type == BodyType.INPUTSTREAM && tc.size.getNumBytes() > 128 * KB, + "RequestBody.fromInputStream does not support retries for content larger than 128 KiB"); + + // all content is created the same way so this data should match what's in the RequestBody + byte[] referenceData = makeArrayOfSize(tc.size.getNumBytes()); + String expectedCrc32 = calculateCrc32(new ByteArrayInputStream(referenceData)); + + RequestBody testBody = makeRequestBody(tc); + + assertThatThrownBy(() -> { + s3.putObject(r -> r.bucket("my-bucket").key("my-obj"), testBody); + }).isInstanceOf(S3Exception.class) + .matches(e -> { + S3Exception s3e = (S3Exception) e; + return s3e.numAttempts() == 3 && s3e.statusCode() == 500; + }, "Should attempt total of 3 times"); + + List bodies = getRequestChecksums(); + assertThat(bodies.size()).isEqualTo(3); + + bodies.forEach(checksum -> assertThat(checksum).isEqualTo(expectedCrc32)); + } + + private static List testCases() { + List testCases = new ArrayList<>(); + for (BodyType type : BodyType.values()) { + for (BodySize size : BodySize.values()) { + testCases.add(new TestCase().size(size).type(type)); + } + } + + return testCases; + } + + private RequestBody makeRequestBody(TestCase tc) throws IOException { + switch (tc.type) { + case STRING: + return RequestBody.fromString(makeStringOfSize(tc.size.getNumBytes()), StandardCharsets.UTF_8); + case BYTES: + return RequestBody.fromBytes(makeArrayOfSize(tc.size.getNumBytes())); + case BYTE_BUFFER: + return RequestBody.fromByteBuffer(ByteBuffer.wrap(makeArrayOfSize(tc.size.getNumBytes()))); + case REMAINING_BYTE_BUFFER: + return RequestBody.fromRemainingByteBuffer(ByteBuffer.wrap(makeArrayOfSize(tc.size.getNumBytes()))); + case INPUTSTREAM: { + InputStream fileStream = getMarkSupportedStreamOfSize(tc.size); + return RequestBody.fromInputStream(fileStream, tc.size.getNumBytes()); + } + case CONTENT_PROVIDER: { + Path file = testFiles.get(tc.size); + return RequestBody.fromContentProvider(new TestContentSteamProvider(file), "text/plain"); + } + case CONTENT_PROVIDER_NO_LENGTH: { + Path file = testFiles.get(tc.size); + return RequestBody.fromContentProvider(new TestContentSteamProvider(file), tc.size.getNumBytes(), "text/plain"); + } + case FILE: + return RequestBody.fromFile(testFiles.get(tc.size)); + default: + throw new RuntimeException("Unsupported body type: " + tc.type); + } + } + + private static class TestCase { + private BodyType type; + private BodySize size; + + public TestCase size(BodySize size) { + this.size = size; + return this; + } + + public TestCase type(BodyType type) { + this.type = type; + return this; + } + + @Override + public String toString() { + return "TestCase{" + + "type=" + type + + ", size=" + size + + '}'; + } + } + + private enum BodyType { + BYTE_BUFFER, + + BYTES, + + CONTENT_PROVIDER, + + CONTENT_PROVIDER_NO_LENGTH, + + FILE, + + INPUTSTREAM, + + REMAINING_BYTE_BUFFER, + + STRING + } + + private static class TestContentSteamProvider implements ContentStreamProvider { + private final Path file; + private InputStream lastStream; + + public TestContentSteamProvider(Path file) { + this.file = file; + } + + @Override + public InputStream newStream() { + if (lastStream != null) { + FunctionalUtils.invokeSafely(lastStream::close); + } + lastStream = FunctionalUtils.invokeSafely(() -> Files.newInputStream(file)); + return lastStream; + } + } +} diff --git a/test/s3-tests/src/test/resources/log4j2.xml b/test/s3-tests/src/test/resources/log4j2.xml new file mode 100644 index 00000000000..82f3e09ef89 --- /dev/null +++ b/test/s3-tests/src/test/resources/log4j2.xml @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file From 42164e132f822bef23c781ede358ccdf3e86dfda Mon Sep 17 00:00:00 2001 From: Dongie Agnir Date: Fri, 25 Apr 2025 15:13:44 -0700 Subject: [PATCH 2/8] Add 0 byte test --- .../s3/AsyncRequestBodyRetryTest.java | 21 ++++++++++++------- .../services/s3/BaseRequestBodyRetryTest.java | 2 ++ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/AsyncRequestBodyRetryTest.java b/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/AsyncRequestBodyRetryTest.java index 195a30f994a..faa4413f44e 100644 --- a/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/AsyncRequestBodyRetryTest.java +++ b/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/AsyncRequestBodyRetryTest.java @@ -136,7 +136,8 @@ private AsyncRequestBody makeRequestBody(TestCase tc) throws IOException { InputStream is = getMarkSupportedStreamOfSize(tc.size); return AsyncRequestBody.fromInputStream(cfg -> cfg.inputStream(is) .contentLength((long) nBytes) - .maxReadLimit(nBytes) + // read limit has to be positive + .maxReadLimit(nBytes == 0 ? 1 : nBytes) .executor(requestBodyExecutor)); } case REMAINING_BYTE_BUFFER: @@ -167,12 +168,18 @@ private AsyncRequestBody makeRequestBody(TestCase tc) throws IOException { case BYTE_BUFFERS_UNSAFE: // fall through case BYTE_BUFFERS: { - byte[] bbContent = getDataSegment(); - int nSegments = nBytes / bbContent.length; - - ByteBuffer[] buffers = IntStream.range(0, nSegments) - .mapToObj(i -> ByteBuffer.wrap(bbContent)) - .toArray(ByteBuffer[]::new); + ByteBuffer[] buffers; + if (tc.size.getNumBytes() > 0) { + byte[] bbContent = getDataSegment(); + int nSegments = nBytes / bbContent.length; + buffers = IntStream.range(0, nSegments) + .mapToObj(i -> ByteBuffer.wrap(bbContent)) + .toArray(ByteBuffer[]::new); + } else { + // TODO: This is a workaround because you can't do AsyncRequestBody.fromByteBuffers(new ByteBuffer[0]); the + // subscriber is never signaled onComplete. See issue JAVA-8215. + buffers = new ByteBuffer[]{ ByteBuffer.allocate(0) }; + } switch (tc.type) { case REMAINING_BYTE_BUFFERS: diff --git a/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/BaseRequestBodyRetryTest.java b/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/BaseRequestBodyRetryTest.java index 3f76cd59688..8a85f3da44a 100644 --- a/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/BaseRequestBodyRetryTest.java +++ b/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/BaseRequestBodyRetryTest.java @@ -202,6 +202,8 @@ private boolean isRequestChunked(LoggedRequest request) { } protected enum BodySize { + SZ_0B(0), + SZ_128KB(128 * KB), SZ_4MB(4 * MB), From 4042893b8fbbf11b02d5c58ad120cd316c1554e9 Mon Sep 17 00:00:00 2001 From: Dongie Agnir Date: Tue, 29 Apr 2025 14:29:46 -0700 Subject: [PATCH 3/8] Try to reduce memory usage --- .../amazon/awssdk/services/s3/BaseRequestBodyRetryTest.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/BaseRequestBodyRetryTest.java b/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/BaseRequestBodyRetryTest.java index 8a85f3da44a..935a9202daa 100644 --- a/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/BaseRequestBodyRetryTest.java +++ b/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/BaseRequestBodyRetryTest.java @@ -212,9 +212,7 @@ protected enum BodySize { SZ_16MB(16 * MB), - SZ_32MB(32 * MB), - - SZ_64MB(64 * MB), + SZ_32MB(32 * MB) ; private final int bytes; From 544517e1b2042a5f83ef943fc2f6a1898cc1afea Mon Sep 17 00:00:00 2001 From: Dongie Agnir Date: Wed, 30 Apr 2025 10:48:08 -0700 Subject: [PATCH 4/8] Switch to plain Jetty server Unfortunately WireMock saves all the request bodies which increases memory usage. Use a plain Jetty server that doesn't save requests to avoid this. We don't rely on any WireWock request assertions so it works out. --- test/s3-tests/pom.xml | 9 +- .../s3/AsyncRequestBodyRetryTest.java | 6 +- .../services/s3/BaseRequestBodyRetryTest.java | 91 ++++++++++------ .../services/s3/SyncRequestBodyRetryTest.java | 6 +- .../amazon/awssdk/services/s3/TestServer.java | 100 ++++++++++++++++++ .../awssdk/services/s3/mock-keystore.jks | Bin 0 -> 2599 bytes 6 files changed, 169 insertions(+), 43 deletions(-) create mode 100644 test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/TestServer.java create mode 100644 test/s3-tests/src/test/resources/software/amazon/awssdk/services/s3/mock-keystore.jks diff --git a/test/s3-tests/pom.xml b/test/s3-tests/pom.xml index 068d8581639..5dcffbd2640 100644 --- a/test/s3-tests/pom.xml +++ b/test/s3-tests/pom.xml @@ -153,8 +153,13 @@ test - com.github.tomakehurst - wiremock-jre8 + org.eclipse.jetty + jetty-servlet + test + + + org.eclipse.jetty + jetty-server test diff --git a/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/AsyncRequestBodyRetryTest.java b/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/AsyncRequestBodyRetryTest.java index faa4413f44e..2201b54e264 100644 --- a/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/AsyncRequestBodyRetryTest.java +++ b/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/AsyncRequestBodyRetryTest.java @@ -54,7 +54,7 @@ public class AsyncRequestBodyRetryTest extends BaseRequestBodyRetryTest { private S3AsyncClient s3; @BeforeAll - public static void setup() throws IOException { + public static void setup() throws Exception { BaseRequestBodyRetryTest.setup(); requestBodyExecutor = Executors.newSingleThreadExecutor(); netty = NettyNioAsyncHttpClient.builder() @@ -72,14 +72,14 @@ public void methodSetup() { .backoffStrategy(BackoffStrategy.retryImmediately()) .build())) .region(Region.US_WEST_2) - .endpointOverride(URI.create("https://localhost:" + wireMockServer.httpsPort())) + .endpointOverride(URI.create("https://localhost:" + serverHttpsPort())) .httpClient(netty) .forcePathStyle(true) .build(); } @AfterAll - public static void teardown() { + public static void teardown() throws Exception { BaseRequestBodyRetryTest.teardown(); netty.close(); requestBodyExecutor.shutdown(); diff --git a/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/BaseRequestBodyRetryTest.java b/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/BaseRequestBodyRetryTest.java index 935a9202daa..0d3d59d14d2 100644 --- a/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/BaseRequestBodyRetryTest.java +++ b/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/BaseRequestBodyRetryTest.java @@ -15,10 +15,6 @@ package software.amazon.awssdk.services.s3; -import com.github.tomakehurst.wiremock.WireMockServer; -import com.github.tomakehurst.wiremock.client.WireMock; -import com.github.tomakehurst.wiremock.stubbing.ServeEvent; -import com.github.tomakehurst.wiremock.verification.LoggedRequest; import java.io.BufferedInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -27,17 +23,23 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; import java.util.Base64; import java.util.EnumMap; import java.util.List; import java.util.Map; import java.util.Random; -import java.util.stream.Collectors; +import javax.servlet.ServletInputStream; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import org.eclipse.jetty.http.HttpStatus; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import software.amazon.awssdk.checksums.DefaultChecksumAlgorithm; import software.amazon.awssdk.checksums.SdkChecksum; +import software.amazon.awssdk.checksums.spi.ChecksumAlgorithm; import software.amazon.awssdk.utils.Logger; /** @@ -54,17 +56,19 @@ public class BaseRequestBodyRetryTest { protected static final int KB = 1024; protected static final int MB = KB * 1024; protected static final Map testFiles = new EnumMap<>(BodySize.class); - protected static final WireMockServer wireMockServer = new WireMockServer(0, 0); protected static final SdkChecksum crc32 = SdkChecksum.forAlgorithm(DefaultChecksumAlgorithm.CRC32); + private static final ChecksumServlet checksumServlet = new ChecksumServlet(DefaultChecksumAlgorithm.CRC32); + private static final TestServer testServer = new TestServer(checksumServlet); + @BeforeAll - public static void setup() throws IOException { - wireMockServer.start(); + public static void setup() throws Exception { + testServer.start(); createTestFiles(); } @AfterAll - public static void teardown() { + public static void teardown() throws Exception { testFiles.values().forEach(tf -> { try { Files.delete(tf); @@ -72,14 +76,16 @@ public static void teardown() { LOG.warn(() -> "Could not delete test file " + tf.toAbsolutePath()); } }); - wireMockServer.stop(); + testServer.stop(); } @BeforeEach - public void resetWireMock() { - wireMockServer.resetAll(); - wireMockServer.stubFor(WireMock.put(WireMock.anyUrl()) - .willReturn(WireMock.aResponse().withStatus(500))); + public void resetServlet() { + checksumServlet.clearChecksums(); + } + + protected int serverHttpsPort() { + return testServer.getHttpsPort(); } protected byte[] getDataSegment() { @@ -129,23 +135,7 @@ protected String makeStringOfSize(int size) { } protected List getRequestChecksums() { - return requestBodies().stream() - .map(r -> { - if (isRequestChunked(r)) { - return getDecodedChecksum(r.getBody()); - } - crc32.reset(); - crc32.update(r.getBody()); - return crc32.getChecksumBytes(); - }).map(BaseRequestBodyRetryTest::base64Encode) - .collect(Collectors.toList()); - } - - private List requestBodies() { - return wireMockServer.getAllServeEvents().stream() - .filter(ServeEvent::getWasMatched) - .map(ServeEvent::getRequest) - .collect(Collectors.toList()); + return checksumServlet.requestChecksums(); } protected static byte[] getDecodedChecksum(byte[] chunkEncoded) { @@ -197,10 +187,6 @@ private static void createTestFiles() throws IOException { } } - private boolean isRequestChunked(LoggedRequest request) { - return "aws-chunked".equalsIgnoreCase(request.getHeader("content-encoding")); - } - protected enum BodySize { SZ_0B(0), @@ -225,4 +211,39 @@ public int getNumBytes() { return bytes; } } + + private static class ChecksumServlet extends HttpServlet { + private final List checksums = new ArrayList<>(); + private final SdkChecksum checksum; + + public ChecksumServlet(ChecksumAlgorithm checksumAlgorithm) { + checksum = SdkChecksum.forAlgorithm(checksumAlgorithm); + } + + public List requestChecksums() { + return checksums; + } + + public void clearChecksums() { + checksums.clear(); + } + + @Override + public void doPut(HttpServletRequest request, HttpServletResponse response) throws IOException { + ServletInputStream inputStream = request.getInputStream(); + + byte[] buff = new byte[4096]; + int read; + + checksum.reset(); + ChunkedDecoder decoder = new ChunkedDecoder(checksum); + while ((read = inputStream.read(buff)) != -1) { + decoder.update(buff, 0, read); + } + checksums.add(base64Encode(decoder.checksumBytes())); + + response.setStatus(HttpStatus.INTERNAL_SERVER_ERROR_500); + response.setContentLength(0); + } + } } diff --git a/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/SyncRequestBodyRetryTest.java b/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/SyncRequestBodyRetryTest.java index c9e278159d8..0a731e7e9ee 100644 --- a/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/SyncRequestBodyRetryTest.java +++ b/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/SyncRequestBodyRetryTest.java @@ -55,7 +55,7 @@ public class SyncRequestBodyRetryTest extends BaseRequestBodyRetryTest { private S3Client s3; @BeforeAll - public static void setup() throws IOException { + public static void setup() throws Exception { BaseRequestBodyRetryTest.setup(); apache = ApacheHttpClient.builder() .buildWithDefaults(AttributeMap.builder() @@ -71,14 +71,14 @@ public void methodSetup() { .backoffStrategy(BackoffStrategy.retryImmediately()) .build())) .region(Region.US_WEST_2) - .endpointOverride(URI.create("https://localhost:" + wireMockServer.httpsPort())) + .endpointOverride(URI.create("https://localhost:" + serverHttpsPort())) .forcePathStyle(true) .httpClient(apache) .build(); } @AfterAll - public static void teardown() { + public static void teardown() throws Exception { apache.close(); BaseRequestBodyRetryTest.teardown(); } diff --git a/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/TestServer.java b/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/TestServer.java new file mode 100644 index 00000000000..ec31554789e --- /dev/null +++ b/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/TestServer.java @@ -0,0 +1,100 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3; + +import javax.servlet.http.HttpServlet; +import org.eclipse.jetty.http.HttpVersion; +import org.eclipse.jetty.server.Connector; +import org.eclipse.jetty.server.HttpConfiguration; +import org.eclipse.jetty.server.HttpConnectionFactory; +import org.eclipse.jetty.server.SecureRequestCustomizer; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.SslConnectionFactory; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; +import org.eclipse.jetty.util.ssl.SslContextFactory; + +/** + * Simple test server that routes all requests to the given servlet. + */ +public class TestServer { + private final int httpPort; + private final int httpsPort; + private final HttpServlet servlet; + + private Server server; + private ServerConnector connector; + private ServerConnector sslConnector; + + public TestServer(int httpPort, int httpsPort, HttpServlet servlet) { + this.httpPort = httpPort; + this.httpsPort = httpsPort; + this.servlet = servlet; + } + + public TestServer(HttpServlet servlet) { + this(0, 0, servlet); + } + + public void start() throws Exception { + server = new Server(); + connector = new ServerConnector(server); + connector.setReuseAddress(true); + connector.setPort(httpPort); + + HttpConfiguration https = new HttpConfiguration(); + https.addCustomizer(new SecureRequestCustomizer()); + + SslContextFactory sslContextFactory = new SslContextFactory(); + sslContextFactory.setTrustAll(true); + sslContextFactory.setValidateCerts(false); + sslContextFactory.setNeedClientAuth(false); + sslContextFactory.setWantClientAuth(false); + sslContextFactory.setValidatePeerCerts(false); + sslContextFactory.setKeyStorePassword("password"); + sslContextFactory.setKeyStorePath(software.amazon.awssdk.services.s3.TestServer.class.getResource("mock-keystore.jks").toExternalForm()); + + sslConnector = new ServerConnector(server, + new SslConnectionFactory(sslContextFactory, + HttpVersion.HTTP_1_1.asString()), + new HttpConnectionFactory(https)); + sslConnector.setReuseAddress(true); + sslConnector.setPort(httpsPort); + + server.setConnectors(new Connector[] {connector, sslConnector}); + + ServletContextHandler context = new ServletContextHandler(server, "/", ServletContextHandler.SESSIONS); + context.addServlet(new ServletHolder(servlet), "/*"); + server.setHandler(context); + + server.start(); + } + + public void stop() throws Exception { + server.stop(); + sslConnector.stop(); + connector.stop(); + } + + public int getPort() { + return connector.getLocalPort(); + } + + public int getHttpsPort() { + return sslConnector.getLocalPort(); + } +} \ No newline at end of file diff --git a/test/s3-tests/src/test/resources/software/amazon/awssdk/services/s3/mock-keystore.jks b/test/s3-tests/src/test/resources/software/amazon/awssdk/services/s3/mock-keystore.jks new file mode 100644 index 0000000000000000000000000000000000000000..8570460ae70d27bce9d4a3b7d65305d31900a5d5 GIT binary patch literal 2599 zcmY+EcQ_l08pabM5j(`DR<%}1?0wAI7p+n>p*32qRvopDnN(A>YP6)N)mlM`Ek3!MQ&<#6UWIy(>o%*8^< zAS|@+%#e$9ySP&f{3ugJhefC@!1i$_C z#O>SlHl#MeESs}auRQbmoh=4NK&KrD3;NUb#!SYC@id;6E9NdqII)q5tvoxsb=W$H zm_X%>u-Oca-E{&u6&|6Et?iW#~cbJ8mqR8llo+OU|vcV+ib~a-|ML#U30cn43RZ> z;DMk*STH+bn|AHj(x9F9D^-y(fge(JH&OObkxVGYq<{;%6GPqDs`P}v+SD8mDap)2 zP0tC;4J4H>wI05%=FAn@zVN{h;enbL6j~5qe+GLUev=|yGUgvsTc_3XaJ_1F0%$ZK zR;3jYEs25Z)rH=dqe#J0R=&`7ZQa;~M+Ib;Y2c@&6#Cjta-)38tfXT51sRE)!XmhZ!^oPZvuWXX z)sEdx1a4l@^y<+imT|TE-o){;h?^`qSvWKM?$FBBdI`LYFsG|wrd@(iVKwGfVu#`_ z%`(U}?x8YB^P&aO=Jy({{korxmo-j4FvjdVH_837|{2y9Rfg>7oVrwE+5nCM=IsRNtBHX%?e8B21kJFDmb0N z@q8C3Gws1kkGYz4e%yOAiPP@+5)ET2Ue~X1W>HnZB)u#ski>GGsZ*`??MH?(T@;i}^J+t}$tkV^+f;C(MvvHlNH`e=`=zqj`Ca zN-J&`h}du5-<}Tx-C`45*}=;e=MNW5=gltI6?_U%3|x=m-0vHw@9=e`neZI9CU`B+ z3^oO+RoLP;5-q0VBa_zyWU2YTJWQi_%!t=gl7f6|&2#cyEZGxeXYXI!#+#e1scZ;R z;_jc5l9l@%FYc_-X+8aZKCbTL{1^Dp*S7QfPrkY%#V5N5%gXmU{4fPi{jpME%IYfZ zPl#=;atj63;JMBQyc}{Kzl)weREL@F-QG0RF0g(-b@7T^5Ki-Ms(69(!!gWNTKTgy z6r_Jzz46PvNVEbhE{~#^+KcsDvwGC+BY8Dv7o!hYGwbI zw9#0`gHJVr6_x-fH&s8ec~~RZIF*Zp?lta8S;PvAqQ3`OZ1f@YX$HJnak36$P_3l& zBgv~r+{}SmIuY8MdNl@9`O0iR4Zt29#&mj6hTHK%Z_(-Ndvc4B_R-DyzW3hW|>SjTo{aEsdNysirbVp zk8G4a#eRP@b%er`tJktJTX7O!Gn z?9wOVmG4l%7ss_e7i;=F4Si6P>C0EP5l+;=Z{)%z4FeR1ADo<1bt&c0#*Rm_4lmkk z^_*J8vjK$(JYh59LI&(f$)$&Ao7Gk)lV^S1m$Q_ zVo%QX*9r&EA1mCgoioA@st|&fr6gO7#^g9OWmBlOMEca5CN+^1e!+FWSa?zT}#b)QnG5)z^y!FMv6xg<+tSQ+(FO&n z(pIsisdv=Yo+IaP>HX28HAQ2(y8*vSgBAbS`#EUIRr=T=-p^h5VDphxFqW|Of9zs%5 zUX!LA5w5paQI!vY2Trv8q^(UVnk!(r{;rvqp7giWtwNRY8~TyJi$ql73=`g2!MXW>N z?7fupp~K0Z%{l@uOI<2-H(8^`mky%dPst((eS`!83TBe$U}O+t0)XJ&x2EV03#|sI q=d-GhaoY~JVP2ykcojD$Yb}F-2w?L0*x~^&loYv?%D@Q3<^CHWRopiK literal 0 HcmV?d00001 From 968955f2201cf5782ccd594cf3734aca408267b0 Mon Sep 17 00:00:00 2001 From: Dongie Agnir Date: Wed, 30 Apr 2025 13:29:39 -0700 Subject: [PATCH 5/8] Lower max size --- .../amazon/awssdk/services/s3/BaseRequestBodyRetryTest.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/BaseRequestBodyRetryTest.java b/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/BaseRequestBodyRetryTest.java index 0d3d59d14d2..40c52e73046 100644 --- a/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/BaseRequestBodyRetryTest.java +++ b/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/BaseRequestBodyRetryTest.java @@ -196,9 +196,7 @@ protected enum BodySize { SZ_8MB(8 * MB), - SZ_16MB(16 * MB), - - SZ_32MB(32 * MB) + SZ_16MB(16 * MB) ; private final int bytes; From 43f5ccac766b50bbdafbfa00fcc64358bde769c5 Mon Sep 17 00:00:00 2001 From: Dongie Agnir Date: Wed, 30 Apr 2025 14:31:53 -0700 Subject: [PATCH 6/8] Max 8MB --- .../amazon/awssdk/services/s3/BaseRequestBodyRetryTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/BaseRequestBodyRetryTest.java b/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/BaseRequestBodyRetryTest.java index 40c52e73046..ae7fb880258 100644 --- a/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/BaseRequestBodyRetryTest.java +++ b/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/BaseRequestBodyRetryTest.java @@ -195,8 +195,6 @@ protected enum BodySize { SZ_4MB(4 * MB), SZ_8MB(8 * MB), - - SZ_16MB(16 * MB) ; private final int bytes; From 22ee75ee4f1379a1c908448694199a536cbe40d5 Mon Sep 17 00:00:00 2001 From: Dongie Agnir Date: Wed, 30 Apr 2025 15:10:31 -0700 Subject: [PATCH 7/8] Force s3-tests to run at end --- test/s3-tests/pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/s3-tests/pom.xml b/test/s3-tests/pom.xml index 59a8b527f5f..a0cfae484f7 100644 --- a/test/s3-tests/pom.xml +++ b/test/s3-tests/pom.xml @@ -162,6 +162,12 @@ jetty-server test + + software.amazon.awssdk + bundle-sdk + ${project.version} + test + From d9c461e13ae6006abf8b1dfe895408d3a989da24 Mon Sep 17 00:00:00 2001 From: Dongie Agnir Date: Fri, 2 May 2025 18:58:31 +0000 Subject: [PATCH 8/8] Increase memory --- .../amazon/awssdk/services/s3/BaseRequestBodyRetryTest.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/BaseRequestBodyRetryTest.java b/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/BaseRequestBodyRetryTest.java index ae7fb880258..e51b6f350f5 100644 --- a/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/BaseRequestBodyRetryTest.java +++ b/test/s3-tests/src/test/java/software/amazon/awssdk/services/s3/BaseRequestBodyRetryTest.java @@ -195,6 +195,10 @@ protected enum BodySize { SZ_4MB(4 * MB), SZ_8MB(8 * MB), + + SZ_16(16 * MB), + + SZ_32(32 * MB), ; private final int bytes;