diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/traits/HttpChecksumTrait.java b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/traits/HttpChecksumTrait.java index f916bbc2bae5..ca37fffebcfa 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/traits/HttpChecksumTrait.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/traits/HttpChecksumTrait.java @@ -39,13 +39,19 @@ */ public class HttpChecksumTrait { + // TODO: run perf tests private static final Map CHECKSUM_ALGORITHM_PRIORITY = ImmutableMap.builder() .put(DefaultChecksumAlgorithm.CRC32C.algorithmId(), 1) .put(DefaultChecksumAlgorithm.CRC32.algorithmId(), 2) .put(DefaultChecksumAlgorithm.CRC64NVME.algorithmId(), 3) - .put(DefaultChecksumAlgorithm.SHA1.algorithmId(), 4) - .put(DefaultChecksumAlgorithm.SHA256.algorithmId(), 5) + .put(DefaultChecksumAlgorithm.XXHASH128.algorithmId(), 4) + .put(DefaultChecksumAlgorithm.XXHASH3.algorithmId(), 5) + .put(DefaultChecksumAlgorithm.XXHASH64.algorithmId(), 6) + .put(DefaultChecksumAlgorithm.SHA1.algorithmId(), 7) + .put(DefaultChecksumAlgorithm.SHA256.algorithmId(), 8) + .put(DefaultChecksumAlgorithm.SHA512.algorithmId(), 9) + .put(DefaultChecksumAlgorithm.MD5.algorithmId(), 10) .build(); private HttpChecksumTrait() { diff --git a/core/checksums/pom.xml b/core/checksums/pom.xml index 72c64d0d35af..22cbb768c52f 100644 --- a/core/checksums/pom.xml +++ b/core/checksums/pom.xml @@ -55,7 +55,7 @@ software.amazon.awssdk.crt aws-crt ${awscrt.version} - test + true diff --git a/core/checksums/src/main/java/software/amazon/awssdk/checksums/DefaultChecksumAlgorithm.java b/core/checksums/src/main/java/software/amazon/awssdk/checksums/DefaultChecksumAlgorithm.java index 017345c822b9..65a2940b8aa2 100644 --- a/core/checksums/src/main/java/software/amazon/awssdk/checksums/DefaultChecksumAlgorithm.java +++ b/core/checksums/src/main/java/software/amazon/awssdk/checksums/DefaultChecksumAlgorithm.java @@ -31,7 +31,11 @@ public final class DefaultChecksumAlgorithm { public static final ChecksumAlgorithm MD5 = of("MD5"); public static final ChecksumAlgorithm SHA256 = of("SHA256"); public static final ChecksumAlgorithm SHA1 = of("SHA1"); + public static final ChecksumAlgorithm SHA512 = of("SHA512"); public static final ChecksumAlgorithm CRC64NVME = of("CRC64NVME"); + public static final ChecksumAlgorithm XXHASH64 = of("XXHASH64"); + public static final ChecksumAlgorithm XXHASH3 = of("XXHASH3"); + public static final ChecksumAlgorithm XXHASH128 = of("XXHASH128"); private DefaultChecksumAlgorithm() { } diff --git a/core/checksums/src/main/java/software/amazon/awssdk/checksums/SdkChecksum.java b/core/checksums/src/main/java/software/amazon/awssdk/checksums/SdkChecksum.java index e11f6fd2cc8c..925a9db5ad11 100644 --- a/core/checksums/src/main/java/software/amazon/awssdk/checksums/SdkChecksum.java +++ b/core/checksums/src/main/java/software/amazon/awssdk/checksums/SdkChecksum.java @@ -18,9 +18,9 @@ import java.nio.ByteBuffer; import java.util.zip.Checksum; import software.amazon.awssdk.annotations.SdkProtectedApi; +import software.amazon.awssdk.checksums.internal.ChecksumProvider; import software.amazon.awssdk.checksums.internal.Crc32Checksum; import software.amazon.awssdk.checksums.internal.Crc64NvmeChecksum; -import software.amazon.awssdk.checksums.internal.CrcChecksumProvider; import software.amazon.awssdk.checksums.internal.DigestAlgorithm; import software.amazon.awssdk.checksums.internal.DigestAlgorithmChecksum; import software.amazon.awssdk.checksums.spi.ChecksumAlgorithm; @@ -38,17 +38,25 @@ public interface SdkChecksum extends Checksum { static SdkChecksum forAlgorithm(ChecksumAlgorithm algorithm) { switch (algorithm.algorithmId()) { case "CRC32C": - return CrcChecksumProvider.crc32cImplementation(); + return ChecksumProvider.crc32cImplementation(); case "CRC32": return new Crc32Checksum(); case "SHA1": return new DigestAlgorithmChecksum(DigestAlgorithm.SHA1); case "SHA256": return new DigestAlgorithmChecksum(DigestAlgorithm.SHA256); + case "SHA512": + return new DigestAlgorithmChecksum(DigestAlgorithm.SHA512); case "MD5": return new DigestAlgorithmChecksum(DigestAlgorithm.MD5); case "CRC64NVME": return new Crc64NvmeChecksum(); + case "XXHASH64": + return ChecksumProvider.xxHash64CrtImplementation(); + case "XXHASH3": + return ChecksumProvider.xxHash3CrtImplementation(); + case "XXHASH128": + return ChecksumProvider.xxHash128CrtImplementation(); default: throw new UnsupportedOperationException("Unsupported checksum algorithm: " + algorithm); } diff --git a/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/CrcChecksumProvider.java b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/ChecksumProvider.java similarity index 52% rename from core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/CrcChecksumProvider.java rename to core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/ChecksumProvider.java index 5debee9af894..32ab8d9abba3 100644 --- a/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/CrcChecksumProvider.java +++ b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/ChecksumProvider.java @@ -17,10 +17,16 @@ import java.util.zip.Checksum; import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.checksums.DefaultChecksumAlgorithm; import software.amazon.awssdk.checksums.SdkChecksum; +import software.amazon.awssdk.checksums.spi.ChecksumAlgorithm; +import software.amazon.awssdk.crt.checksums.CRC32C; +import software.amazon.awssdk.crt.checksums.CRC64NVME; +import software.amazon.awssdk.utils.ClassLoaderHelper; +import software.amazon.awssdk.utils.Lazy; /** - * Utility class providing implementations of CRC checksums, specifically CRC32C and CRC64NVME. + * Utility class providing implementations of checksums. * *

Supports the following implementations for CRC32C:

*
    @@ -29,23 +35,38 @@ *
  • SDK-based CRC32C (fallback)
    • *
* - *

Only supports CRT-based implementation for CRC64NVME (using AWS CRT library).

+ *

Supports CRT-based implementations for CRC64NVME and XXHASH algorithms (using AWS CRT library).

* *

For internal use only ({@link SdkInternalApi}).

*/ @SdkInternalApi -public final class CrcChecksumProvider { - +public final class ChecksumProvider { // Class paths for different CRC32C implementations private static final String CRT_CRC32C_CLASS_PATH = "software.amazon.awssdk.crt.checksums.CRC32C"; private static final String JAVA_CRC32C_CLASS_PATH = "java.util.zip.CRC32C"; private static final ConstructorCache CONSTRUCTOR_CACHE = new ConstructorCache(); private static final String CRT_CRC64NVME_PATH = "software.amazon.awssdk.crt.checksums.CRC64NVME"; + private static final String CRT_XXHASH_PATH = "software.amazon.awssdk.crt.checksums.XXHash"; private static final String CRT_MODULE = "software.amazon.awssdk.crt:aws-crt"; + private static Lazy isXxHashAvailable = checkCrtAvailability(CRT_XXHASH_PATH); + private static Lazy isCrc64NvmeAvailable = checkCrtAvailability(CRT_CRC64NVME_PATH); + private static Lazy isCrc32CAvailable = checkCrtAvailability(CRT_CRC32C_CLASS_PATH); + // Private constructor to prevent instantiation - private CrcChecksumProvider() { + private ChecksumProvider() { + } + + private static Lazy checkCrtAvailability(String fqcn) { + return new Lazy<>(() -> { + try { + ClassLoaderHelper.loadClass(fqcn, false); + } catch (ClassNotFoundException e) { + return false; + } + return true; + }); } /** @@ -74,14 +95,12 @@ public static SdkChecksum crc32cImplementation() { } static SdkChecksum createCrtCrc32C() { - return CONSTRUCTOR_CACHE.getConstructor(CRT_CRC32C_CLASS_PATH).map(constructor -> { - try { - Checksum checksumInstance = (Checksum) constructor.newInstance(); - return new CrcCloneOnMarkChecksum(checksumInstance); - } catch (ReflectiveOperationException e) { - throw new IllegalStateException("Failed to instantiate " + CRT_CRC32C_CLASS_PATH, e); - } - }).orElse(null); + if (!isCrc32CAvailable.getValue()) { + return null; + } + + Checksum checksumInstance = new CRC32C(); + return new CrcCloneOnMarkChecksum(checksumInstance); } /** @@ -96,16 +115,57 @@ static SdkChecksum createCrtCrc32C() { * @throws RuntimeException if the `CRC64NVME` implementation is not available. */ static SdkChecksum crc64NvmeCrtImplementation() { - return CONSTRUCTOR_CACHE.getConstructor(CRT_CRC64NVME_PATH).map(constructor -> { - try { - Checksum checksumInstance = (Checksum) constructor.newInstance(); - return new CrcCloneOnMarkChecksum(checksumInstance); - } catch (ReflectiveOperationException e) { - throw new IllegalStateException("Failed to instantiate " + CRT_CRC32C_CLASS_PATH, e); - } - }).orElseThrow(() -> new RuntimeException( - "Could not load " + CRT_CRC64NVME_PATH + ". Add dependency on '" + CRT_MODULE - + "' module to enable CRC64NVME feature.")); + + if (!isCrc64NvmeAvailable.getValue()) { + throw new RuntimeException( + "Could not load " + CRT_CRC64NVME_PATH + ". Add dependency on '" + CRT_MODULE + + "' module to enable CRC64NVME feature."); + } + + return new CrcCloneOnMarkChecksum(new CRC64NVME()); + } + + /** + * Creates an instance of the CRT-based XXHASH64 checksum using AWS's CRT library. + * + * @return An {@link SdkChecksum} instance for XXHASH64. + * @throws IllegalStateException if instantiation fails. + * @throws RuntimeException if the CRT implementation is not available. + */ + public static SdkChecksum xxHash64CrtImplementation() { + return crtXxHash(DefaultChecksumAlgorithm.XXHASH64); + } + + /** + * Creates an instance of the CRT-based XXHASH3 checksum using AWS's CRT library. + * + * @return An {@link SdkChecksum} instance for XXHASH3. + * @throws IllegalStateException if instantiation fails. + * @throws RuntimeException if the CRT implementation is not available. + */ + public static SdkChecksum xxHash3CrtImplementation() { + return crtXxHash(DefaultChecksumAlgorithm.XXHASH3); + } + + /** + * Creates an instance of the CRT-based XXHASH128 checksum using AWS's CRT library. + * + * @return An {@link SdkChecksum} instance for XXHASH128. + * @throws IllegalStateException if instantiation fails. + * @throws RuntimeException if the CRT implementation is not available. + */ + public static SdkChecksum xxHash128CrtImplementation() { + return crtXxHash(DefaultChecksumAlgorithm.XXHASH128); + } + + static SdkChecksum crtXxHash(ChecksumAlgorithm algorithm) { + if (!isXxHashAvailable.getValue()) { + throw new RuntimeException( + "Could not load " + CRT_XXHASH_PATH + ". Add dependency on '" + CRT_MODULE + + "' module."); + } + + return new XxHashChecksum(algorithm); } static SdkChecksum createJavaCrc32C() { @@ -117,5 +177,4 @@ static SdkChecksum createJavaCrc32C() { } }).orElse(null); } - } diff --git a/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Crc64NvmeChecksum.java b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Crc64NvmeChecksum.java index 76b0315c598b..9b705ec09426 100644 --- a/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Crc64NvmeChecksum.java +++ b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Crc64NvmeChecksum.java @@ -29,7 +29,7 @@ public final class Crc64NvmeChecksum implements SdkChecksum { private final SdkChecksum sdkChecksum; public Crc64NvmeChecksum() { - this.sdkChecksum = CrcChecksumProvider.crc64NvmeCrtImplementation(); + this.sdkChecksum = ChecksumProvider.crc64NvmeCrtImplementation(); } @Override diff --git a/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/DigestAlgorithm.java b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/DigestAlgorithm.java index 56d7b7517f35..a62dac22fa4a 100644 --- a/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/DigestAlgorithm.java +++ b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/DigestAlgorithm.java @@ -29,7 +29,8 @@ public enum DigestAlgorithm { SHA1("SHA-1"), MD5("MD5"), - SHA256("SHA-256") + SHA256("SHA-256"), + SHA512("SHA-512") ; private static final Supplier CLOSED_DIGEST = () -> { diff --git a/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/XxHashChecksum.java b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/XxHashChecksum.java new file mode 100644 index 000000000000..1300015ac2cc --- /dev/null +++ b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/XxHashChecksum.java @@ -0,0 +1,76 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.checksums.internal; + +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.checksums.DefaultChecksumAlgorithm; +import software.amazon.awssdk.checksums.SdkChecksum; +import software.amazon.awssdk.checksums.spi.ChecksumAlgorithm; +import software.amazon.awssdk.crt.checksums.XXHash; + +@SdkInternalApi +public final class XxHashChecksum implements SdkChecksum { + + private final XXHash xxHash; + + public XxHashChecksum(ChecksumAlgorithm algorithm) { + if (algorithm == DefaultChecksumAlgorithm.XXHASH64) { + xxHash = XXHash.newXXHash64(); + } else if (algorithm == DefaultChecksumAlgorithm.XXHASH3) { + xxHash = XXHash.newXXHash3_64(); + } else if (algorithm == DefaultChecksumAlgorithm.XXHASH128) { + xxHash = XXHash.newXXHash3_128(); + } else { + throw new UnsupportedOperationException("Unsupported algorithm: " + algorithm.algorithmId()); + } + } + + @Override + public void update(byte[] b) { + xxHash.update(b); + } + + @Override + public byte[] getChecksumBytes() { + return xxHash.digest(); + } + + + @Override + public void update(int b) { + xxHash.update(b); + } + + @Override + public void update(byte[] b, int off, int len) { + xxHash.update(b, off, len); + } + + @Override + public void reset() { + throw new UnsupportedOperationException("mark and reset is not supported"); + } + + @Override + public void mark(int readLimit) { + throw new UnsupportedOperationException("mark and reset is not supported"); + } + + @Override + public long getValue() { + throw new UnsupportedOperationException("Use getChecksumBytes() instead."); + } +} diff --git a/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/DefaultCRC32CChecksumTest.java b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/DefaultCRC32CChecksumTest.java index fd7ad8163191..9c6b9280a752 100644 --- a/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/DefaultCRC32CChecksumTest.java +++ b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/DefaultCRC32CChecksumTest.java @@ -22,7 +22,7 @@ public class DefaultCRC32CChecksumTest extends Crc32CChecksumTest { @BeforeEach public void setUp() { - sdkChecksum = CrcChecksumProvider.crc32cImplementation(); + sdkChecksum = ChecksumProvider.crc32cImplementation(); } diff --git a/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/DigestAlgorithmTest.java b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/DigestAlgorithmTest.java index 98978fa9f6a9..ac334d61ad88 100644 --- a/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/DigestAlgorithmTest.java +++ b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/DigestAlgorithmTest.java @@ -22,8 +22,12 @@ import java.security.MessageDigest; import java.util.ArrayList; import java.util.List; +import java.util.stream.Stream; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import software.amazon.awssdk.checksums.internal.DigestAlgorithm.CloseableMessageDigest; import software.amazon.awssdk.utils.BinaryUtils; @@ -47,31 +51,27 @@ void getDigest_returnsMessageDigest() { assertThat(digest.messageDigest()).isNotNull(); } - @Test - void digestAlgorithms_useCorrectImplementation() { + @ParameterizedTest + @MethodSource("digestAlgorithmTestCases") + void digestAlgorithms_useCorrectImplementation(DigestAlgorithm algorithm, String expectedBase64) { String input = "Hello, World!"; byte[] data = input.getBytes(StandardCharsets.UTF_8); - // Test SHA1 - CloseableMessageDigest sha1Digest = DigestAlgorithm.SHA1.getDigest(); - sha1Digest.messageDigest().update(data); - byte[] sha1Hash = sha1Digest.digest(); - assertThat(sha1Hash).isNotNull(); - assertThat(BinaryUtils.toBase64(sha1Hash)).isEqualTo("CgqfKmdylCVXq1NV12r0Qvj2XgE="); - - // Test MD5 - CloseableMessageDigest md5Digest = DigestAlgorithm.MD5.getDigest(); - md5Digest.messageDigest().update(data); - byte[] md5Hash = md5Digest.digest(); - assertThat(md5Hash).isNotNull(); - assertThat(BinaryUtils.toBase64(md5Hash)).isEqualTo("ZajifYh5KDgxtmS9i38K1A=="); - - // Test SHA256 - CloseableMessageDigest sha256Digest = DigestAlgorithm.SHA256.getDigest(); - sha256Digest.messageDigest().update(data); - byte[] sha256Hash = sha256Digest.digest(); - assertThat(sha256Hash).isNotNull(); - assertThat(BinaryUtils.toBase64(sha256Hash)).isEqualTo("3/1gIbsr1bCvZ2KQgJ7DpTGR3YHH9wpLKGiKNiGCmG8="); + CloseableMessageDigest digest = algorithm.getDigest(); + digest.messageDigest().update(data); + byte[] hash = digest.digest(); + + assertThat(hash).isNotNull(); + assertThat(BinaryUtils.toBase64(hash)).isEqualTo(expectedBase64); + } + + static Stream digestAlgorithmTestCases() { + return Stream.of( + Arguments.of(DigestAlgorithm.SHA1, "CgqfKmdylCVXq1NV12r0Qvj2XgE="), + Arguments.of(DigestAlgorithm.MD5, "ZajifYh5KDgxtmS9i38K1A=="), + Arguments.of(DigestAlgorithm.SHA256, "3/1gIbsr1bCvZ2KQgJ7DpTGR3YHH9wpLKGiKNiGCmG8="), + Arguments.of(DigestAlgorithm.SHA512, "N015SpXNz9izWZMYX++bo2jxYNja9DLQi6nx7R5avmzGkpHg+i/gAGpSVw7xjBne9OYXwzzlLvCm5fvjGMsDhw==") + ); } @Test diff --git a/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/Java9BasedCRC32CChecksumTest.java b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/Java9BasedCRC32CChecksumTest.java index e77232206a3a..a66c8d33637b 100644 --- a/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/Java9BasedCRC32CChecksumTest.java +++ b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/Java9BasedCRC32CChecksumTest.java @@ -24,7 +24,7 @@ public class Java9BasedCRC32CChecksumTest extends Crc32CChecksumTest { @BeforeEach public void setUp() { - sdkChecksum = CrcChecksumProvider.createJavaCrc32C(); + sdkChecksum = ChecksumProvider.createJavaCrc32C(); } } diff --git a/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/SdkImplmenetedCRC32CChecksumTest.java b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/SdkImplmenetedCRC32CChecksumTest.java index 70d6963eb0fa..84c402ed7a23 100644 --- a/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/SdkImplmenetedCRC32CChecksumTest.java +++ b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/SdkImplmenetedCRC32CChecksumTest.java @@ -21,6 +21,6 @@ public class SdkImplmenetedCRC32CChecksumTest extends Crc32CChecksumTest { @BeforeEach public void setUp() { - sdkChecksum = CrcChecksumProvider.createSdkBasedCrc32C(); + sdkChecksum = ChecksumProvider.createSdkBasedCrc32C(); } } diff --git a/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/XxHashChecksumTest.java b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/XxHashChecksumTest.java new file mode 100644 index 000000000000..fc8851018b06 --- /dev/null +++ b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/XxHashChecksumTest.java @@ -0,0 +1,82 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.checksums.internal; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.nio.charset.StandardCharsets; +import java.util.stream.Stream; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.awssdk.checksums.DefaultChecksumAlgorithm; +import software.amazon.awssdk.checksums.SdkChecksum; +import software.amazon.awssdk.checksums.spi.ChecksumAlgorithm; +import software.amazon.awssdk.utils.BinaryUtils; + +class XxHashChecksumTest { + + private static Stream xxHashAlgorithms() { + return Stream.of( + Arguments.of(DefaultChecksumAlgorithm.XXHASH64, "xQCwyRKzdtg="), + Arguments.of(DefaultChecksumAlgorithm.XXHASH3, "tqy52Eo4/3Q="), + Arguments.of(DefaultChecksumAlgorithm.XXHASH128, "c1H4mBL5c4K5HQWzHgTdfw==") + ); + } + + @ParameterizedTest + @MethodSource("xxHashAlgorithms") + void getChecksumBytes_withByteArray_returnsExpectedChecksum(ChecksumAlgorithm algorithm, String expectedBase64) { + SdkChecksum checksum = ChecksumProvider.crtXxHash(algorithm); + checksum.update("Hello world".getBytes(StandardCharsets.UTF_8)); + assertEquals(expectedBase64, BinaryUtils.toBase64(checksum.getChecksumBytes())); + } + + @ParameterizedTest + @MethodSource("xxHashAlgorithms") + void mark_throwsUnsupportedOperationException(ChecksumAlgorithm algorithm, String expectedBase64) { + SdkChecksum checksum = ChecksumProvider.crtXxHash(algorithm); + assertThrows(UnsupportedOperationException.class, () -> checksum.mark(1)); + } + + @ParameterizedTest + @MethodSource("xxHashAlgorithms") + void reset_throwsUnsupportedOperationException(ChecksumAlgorithm algorithm, String expectedBase64) { + SdkChecksum checksum = ChecksumProvider.crtXxHash(algorithm); + assertThrows(UnsupportedOperationException.class, () -> checksum.reset()); + } + + @ParameterizedTest + @MethodSource("xxHashAlgorithms") + void getChecksumBytes_withSingleByteUpdates_returnsExpectedChecksum(ChecksumAlgorithm algorithm, String expectedBase64) { + SdkChecksum checksum = ChecksumProvider.crtXxHash(algorithm); + byte[] bytes = "Hello world".getBytes(StandardCharsets.UTF_8); + for (byte b : bytes) { + checksum.update(b & 0xFF); + } + assertEquals(expectedBase64, BinaryUtils.toBase64(checksum.getChecksumBytes())); + } + + @ParameterizedTest + @MethodSource("xxHashAlgorithms") + void getChecksumBytes_withOffsetAndLength_returnsExpectedChecksum(ChecksumAlgorithm algorithm, String expectedBase64) { + SdkChecksum checksum = ChecksumProvider.crtXxHash(algorithm); + byte[] bytes = "Hello world".getBytes(StandardCharsets.UTF_8); + checksum.update(bytes, 0, bytes.length); + assertEquals(expectedBase64, BinaryUtils.toBase64(checksum.getChecksumBytes())); + } +} diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/useragent/BusinessMetricsUtils.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/useragent/BusinessMetricsUtils.java index 17d242939d23..162061e7cc94 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/useragent/BusinessMetricsUtils.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/useragent/BusinessMetricsUtils.java @@ -111,6 +111,27 @@ public static Optional resolveChecksumAlgorithmMetric(ChecksumAlgorithm if (algorithmId.equals(DefaultChecksumAlgorithm.SHA256.algorithmId())) { return Optional.of(BusinessMetricFeatureId.FLEXIBLE_CHECKSUMS_REQ_SHA256.value()); } + + if (algorithmId.equals(DefaultChecksumAlgorithm.SHA512.algorithmId())) { + return Optional.of(BusinessMetricFeatureId.FLEXIBLE_CHECKSUMS_REQ_SHA512.value()); + } + + if (algorithmId.equals(DefaultChecksumAlgorithm.XXHASH3.algorithmId())) { + return Optional.of(BusinessMetricFeatureId.FLEXIBLE_CHECKSUMS_REQ_XXHASH3.value()); + } + + if (algorithmId.equals(DefaultChecksumAlgorithm.XXHASH64.algorithmId())) { + return Optional.of(BusinessMetricFeatureId.FLEXIBLE_CHECKSUMS_REQ_XXHASH64.value()); + } + + if (algorithmId.equals(DefaultChecksumAlgorithm.XXHASH128.algorithmId())) { + return Optional.of(BusinessMetricFeatureId.FLEXIBLE_CHECKSUMS_REQ_XXHASH128.value()); + } + + if (algorithmId.equals(DefaultChecksumAlgorithm.MD5.algorithmId())) { + return Optional.of(BusinessMetricFeatureId.FLEXIBLE_CHECKSUMS_REQ_MD5.value()); + } + return Optional.empty(); } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/HttpChecksumUtils.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/HttpChecksumUtils.java index cb46be59b3c6..2069520d2807 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/HttpChecksumUtils.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/HttpChecksumUtils.java @@ -18,8 +18,12 @@ import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.CRC32; import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.CRC32C; import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.CRC64NVME; +import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.MD5; import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.SHA1; import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.SHA256; +import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.XXHASH128; +import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.XXHASH3; +import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.XXHASH64; import static software.amazon.awssdk.core.HttpChecksumConstant.HEADER_FOR_TRAILER_REFERENCE; import static software.amazon.awssdk.core.HttpChecksumConstant.HTTP_CHECKSUM_HEADER_PREFIX; import static software.amazon.awssdk.core.HttpChecksumConstant.SIGNING_METHOD; @@ -56,9 +60,14 @@ @SdkInternalApi public final class HttpChecksumUtils { private static final Logger log = Logger.loggerFor(HttpChecksumUtils.class); + private static final String CRT_CRC64NVME_PATH = "software.amazon.awssdk.crt.checksums.CRC64NVME"; + private static final String CRT_XXHASH_PATH = "software.amazon.awssdk.crt.checksums.XXHash"; private static final int CHECKSUM_BUFFER_SIZE = 16 * 1024; + /** + * Implementor notes: this exists for backwards compatibility reasons; don't add new checksum algos + */ private static final ImmutableMap NEW_CHECKSUM_TO_LEGACY = ImmutableMap.of( SHA256, Algorithm.SHA256, SHA1, Algorithm.SHA1, @@ -67,6 +76,9 @@ public final class HttpChecksumUtils { CRC64NVME, Algorithm.CRC64NVME ); + /** + * Implementor notes: this exists for backwards compatibility reasons; don't add new checksum algos + */ private static final ImmutableMap LEGACY_CHECKSUM_TO_NEW = ImmutableMap.of( Algorithm.SHA256, SHA256, Algorithm.SHA1, SHA1, @@ -75,18 +87,24 @@ public final class HttpChecksumUtils { Algorithm.CRC64NVME, CRC64NVME ); - private static Lazy isCrc64NvmeAvailable = new Lazy<>(() -> { - try { - ClassLoaderHelper.loadClass("software.amazon.awssdk.crt.checksums.CRC64NVME", false); - } catch (ClassNotFoundException e) { - return false; - } - return true; - }); + private static Lazy isCrc64NvmeAvailable = checkCrtAvailability(CRT_CRC64NVME_PATH); + + private static Lazy isXxHashAvailable = checkCrtAvailability(CRT_XXHASH_PATH); private HttpChecksumUtils() { } + private static Lazy checkCrtAvailability(String fqcn) { + return new Lazy<>(() -> { + try { + ClassLoaderHelper.loadClass(fqcn, false); + } catch (ClassNotFoundException e) { + return false; + } + return true; + }); + } + public static Algorithm toLegacyChecksumAlgorithm(ChecksumAlgorithm checksumAlgorithm) { return NEW_CHECKSUM_TO_LEGACY.get(checksumAlgorithm); } @@ -243,6 +261,13 @@ public static boolean isHttpChecksumCalculationNeeded(SdkHttpFullRequest.Builder hasFlexibleChecksumTrait && checksumSpecs.algorithmV2() != null; if (checksumAlgorithmSpecified) { + ChecksumAlgorithm checksumAlgorithm = checksumSpecs.algorithmV2(); + + // MD5 is not supported for flexible checksums (httpChecksum trait) + if (checksumAlgorithm.equals(MD5)) { + throw new IllegalArgumentException("MD5 is not supported. Please use a different checksum algorithm or provide a " + + "pre-calculated MD5 value."); + } return true; } @@ -325,18 +350,30 @@ public static Pair getAlgorithmChecksumValuePair(SdkH Optional firstMatchingHeader = sdkHttpResponse.firstMatchingHeader(httpChecksumHeader(checksumAlgorithm.algorithmId())); - if (firstMatchingHeader.isPresent()) { - if (checksumAlgorithm.equals(CRC64NVME) && !isCrc64NvmeAvailable.getValue()) { - log.debug(() -> "Skip CRC64NVME checksum validation because CRT is not on the classpath and CRC64NVME is " - + "not available"); - continue; - } + if (firstMatchingHeader.isPresent() && !shouldSkipAlgorithm(checksumAlgorithm)) { return Pair.of(checksumAlgorithm, firstMatchingHeader.get()); } } return null; } + private static boolean shouldSkipAlgorithm(ChecksumAlgorithm checksumAlgorithm) { + if (checksumAlgorithm.equals(CRC64NVME) && !isCrc64NvmeAvailable.getValue()) { + log.debug(() -> "Skip CRC64NVME checksum validation because CRT is not available"); + return true; + } + if ((checksumAlgorithm.equals(XXHASH64) || checksumAlgorithm.equals(XXHASH3) || + checksumAlgorithm.equals(XXHASH128)) && !isXxHashAvailable.getValue()) { + log.debug(() -> "Skip XXHASH checksum validation because CRT is not available"); + return true; + } + if (checksumAlgorithm.equals(MD5)) { + log.debug(() -> "Skip MD5 checksum validation because MD5 is not supported for flexible checksums"); + return true; + } + return false; + } + /** * * @param resolvedChecksumSpecs Resolved checksum specification for the operation. diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/useragent/BusinessMetricFeatureId.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/useragent/BusinessMetricFeatureId.java index 16ee2ff85315..b8952e541998 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/useragent/BusinessMetricFeatureId.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/useragent/BusinessMetricFeatureId.java @@ -52,6 +52,11 @@ public enum BusinessMetricFeatureId { FLEXIBLE_CHECKSUMS_REQ_WHEN_REQUIRED("a"), FLEXIBLE_CHECKSUMS_RES_WHEN_SUPPORTED("b"), FLEXIBLE_CHECKSUMS_RES_WHEN_REQUIRED("c"), + FLEXIBLE_CHECKSUMS_REQ_MD5("AE"), + FLEXIBLE_CHECKSUMS_REQ_SHA512("AF"), + FLEXIBLE_CHECKSUMS_REQ_XXHASH3("AG"), + FLEXIBLE_CHECKSUMS_REQ_XXHASH64("AH"), + FLEXIBLE_CHECKSUMS_REQ_XXHASH128("AI"), DDB_MAPPER("d"), BEARER_SERVICE_ENV_VARS("3"), CREDENTIALS_CODE("e"), diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/util/HttpChecksumUtilsTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/util/HttpChecksumUtilsTest.java index 31b00d68e64f..e3bad3eeac8f 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/util/HttpChecksumUtilsTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/util/HttpChecksumUtilsTest.java @@ -16,11 +16,13 @@ package software.amazon.awssdk.core.internal.util; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static software.amazon.awssdk.core.HttpChecksumConstant.HEADER_FOR_TRAILER_REFERENCE; import java.util.List; import java.util.Optional; import java.util.stream.Stream; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -148,4 +150,35 @@ void isHttpChecksumCalculationNeeded(boolean hasLegacyHttpChecksumTrait, private SdkHttpFullRequest.Builder createHttpRequestBuilder() { return SdkHttpFullRequest.builder().contentStreamProvider(RequestBody.fromString("test").contentStreamProvider()); } + + @Test + void isHttpChecksumCalculationNeeded_md5Algorithm_shouldThrowException() { + ExecutionAttributes executionAttributes = + ExecutionAttributes.builder() + .put(SdkExecutionAttribute.RESOLVED_CHECKSUM_SPECS, + ChecksumSpecs.builder().algorithmV2(DefaultChecksumAlgorithm.MD5).build()) + .put(SdkInternalExecutionAttribute.REQUEST_CHECKSUM_CALCULATION, + RequestChecksumCalculation.WHEN_SUPPORTED) + .build(); + + assertThatThrownBy(() -> HttpChecksumUtils.isHttpChecksumCalculationNeeded(createHttpRequestBuilder(), executionAttributes)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("MD5 is not supported"); + } + + @Test + void isHttpChecksumCalculationNeeded_md5AlgorithmWithExistingMd5Header_shouldReturnFalse() { + SdkHttpFullRequest.Builder request = createHttpRequestBuilder() + .putHeader("x-amz-checksum-md5", "PiWWCnnbxptnTNTsZ6csYg=="); + + ExecutionAttributes executionAttributes = + ExecutionAttributes.builder() + .put(SdkExecutionAttribute.RESOLVED_CHECKSUM_SPECS, + ChecksumSpecs.builder().algorithmV2(DefaultChecksumAlgorithm.MD5).build()) + .put(SdkInternalExecutionAttribute.REQUEST_CHECKSUM_CALCULATION, + RequestChecksumCalculation.WHEN_SUPPORTED) + .build(); + + assertThat(HttpChecksumUtils.isHttpChecksumCalculationNeeded(request, executionAttributes)).isFalse(); + } } diff --git a/pom.xml b/pom.xml index d02c787e93a1..a40f9678854c 100644 --- a/pom.xml +++ b/pom.xml @@ -133,7 +133,7 @@ 3.1.5 1.17.1 1.37 - 0.40.3 + 0.43.4 5.10.3 diff --git a/test/codegen-generated-classes-test/src/main/resources/codegen-resources/customresponsemetadata/service-2.json b/test/codegen-generated-classes-test/src/main/resources/codegen-resources/customresponsemetadata/service-2.json index 3893814d0b25..d7b16e280513 100644 --- a/test/codegen-generated-classes-test/src/main/resources/codegen-resources/customresponsemetadata/service-2.json +++ b/test/codegen-generated-classes-test/src/main/resources/codegen-resources/customresponsemetadata/service-2.json @@ -242,7 +242,7 @@ "requestChecksumRequired" : true, "requestAlgorithmMember": "ChecksumAlgorithm", "requestValidationModeMember" : "ChecksumMode", - "responseAlgorithms": ["crc64nvme","crc32", "crc32c", "sha256", "sha1"] + "responseAlgorithms": ["crc64nvme","crc32", "crc32c", "sha256", "sha512", "sha1", "xxhash64", "xxhash3", "xxhash128"] } }, "StreamingInputOperation":{ @@ -333,7 +333,7 @@ "output":{"shape":"ChecksumStructureWithStreaming"}, "httpChecksum" : { "requestValidationModeMember": "ChecksumMode", - "responseAlgorithms": ["CRC64NVME","CRC32C", "CRC32", "SHA1", "SHA256"] + "responseAlgorithms": ["CRC64NVME","CRC32C", "CRC32", "SHA1", "SHA256", "SHA512", "XXHASH64", "XXHASH3", "XXHASH128", "MD5"] } }, "GetOperationWithMapEndpointParam":{ @@ -357,7 +357,7 @@ "requestChecksumRequired": true, "requestAlgorithmMember": "ChecksumAlgorithm", "requestValidationModeMember": "ChecksumMode", - "responseAlgorithms": ["CRC64NVME","CRC32C", "CRC32", "SHA1", "SHA256"] + "responseAlgorithms": ["CRC64NVME","CRC32C", "CRC32", "SHA1", "SHA256", "SHA512", "XXHASH64", "XXHASH3", "XXHASH128", "MD5"] } }, "QueryParameterOperation":{ @@ -980,7 +980,12 @@ "CRC32C", "SHA1", "SHA256", - "CRC64NVME" + "SHA512", + "MD5", + "CRC64NVME", + "XXHASH64", + "XXHASH3", + "XXHASH128" ] }, "ChecksumMode":{ diff --git a/test/codegen-generated-classes-test/src/test/java/software/amazon/awssdk/services/FlexibleChecksumBusinessMetricTest.java b/test/codegen-generated-classes-test/src/test/java/software/amazon/awssdk/services/FlexibleChecksumBusinessMetricTest.java index 6766d6df048a..7e7794de62b2 100644 --- a/test/codegen-generated-classes-test/src/test/java/software/amazon/awssdk/services/FlexibleChecksumBusinessMetricTest.java +++ b/test/codegen-generated-classes-test/src/test/java/software/amazon/awssdk/services/FlexibleChecksumBusinessMetricTest.java @@ -17,6 +17,19 @@ import static org.assertj.core.api.Assertions.assertThat; import static software.amazon.awssdk.core.useragent.BusinessMetricCollection.METRIC_SEARCH_PATTERN; +import static software.amazon.awssdk.core.useragent.BusinessMetricFeatureId.FLEXIBLE_CHECKSUMS_REQ_CRC32; +import static software.amazon.awssdk.core.useragent.BusinessMetricFeatureId.FLEXIBLE_CHECKSUMS_REQ_CRC32C; +import static software.amazon.awssdk.core.useragent.BusinessMetricFeatureId.FLEXIBLE_CHECKSUMS_REQ_CRC64; +import static software.amazon.awssdk.core.useragent.BusinessMetricFeatureId.FLEXIBLE_CHECKSUMS_REQ_SHA1; +import static software.amazon.awssdk.core.useragent.BusinessMetricFeatureId.FLEXIBLE_CHECKSUMS_REQ_SHA256; +import static software.amazon.awssdk.core.useragent.BusinessMetricFeatureId.FLEXIBLE_CHECKSUMS_REQ_SHA512; +import static software.amazon.awssdk.core.useragent.BusinessMetricFeatureId.FLEXIBLE_CHECKSUMS_REQ_WHEN_REQUIRED; +import static software.amazon.awssdk.core.useragent.BusinessMetricFeatureId.FLEXIBLE_CHECKSUMS_REQ_WHEN_SUPPORTED; +import static software.amazon.awssdk.core.useragent.BusinessMetricFeatureId.FLEXIBLE_CHECKSUMS_REQ_XXHASH128; +import static software.amazon.awssdk.core.useragent.BusinessMetricFeatureId.FLEXIBLE_CHECKSUMS_REQ_XXHASH3; +import static software.amazon.awssdk.core.useragent.BusinessMetricFeatureId.FLEXIBLE_CHECKSUMS_REQ_XXHASH64; +import static software.amazon.awssdk.core.useragent.BusinessMetricFeatureId.FLEXIBLE_CHECKSUMS_RES_WHEN_REQUIRED; +import static software.amazon.awssdk.core.useragent.BusinessMetricFeatureId.FLEXIBLE_CHECKSUMS_RES_WHEN_SUPPORTED; import java.util.List; import java.util.stream.Stream; @@ -69,10 +82,10 @@ void when_noChecksumConfigurationIsSet_defaultConfigMetricsAreAdded() { String userAgent = getUserAgentFromLastRequest(); assertThat(userAgent) - .matches(METRIC_SEARCH_PATTERN.apply("Z")) - .matches(METRIC_SEARCH_PATTERN.apply("b")) - .doesNotMatch(METRIC_SEARCH_PATTERN.apply("a")) - .doesNotMatch(METRIC_SEARCH_PATTERN.apply("c")); + .matches(METRIC_SEARCH_PATTERN.apply(FLEXIBLE_CHECKSUMS_REQ_WHEN_SUPPORTED.value())) + .matches(METRIC_SEARCH_PATTERN.apply(FLEXIBLE_CHECKSUMS_RES_WHEN_SUPPORTED.value())) + .doesNotMatch(METRIC_SEARCH_PATTERN.apply(FLEXIBLE_CHECKSUMS_REQ_WHEN_REQUIRED.value())) + .doesNotMatch(METRIC_SEARCH_PATTERN.apply(FLEXIBLE_CHECKSUMS_RES_WHEN_REQUIRED.value())); } @ParameterizedTest @@ -93,11 +106,15 @@ void when_checksumAlgorithmIsUsed_correctMetricIsAdded(ChecksumAlgorithm algorit static Stream checksumAlgorithmTestCases() { return Stream.of( - Arguments.of(ChecksumAlgorithm.CRC32, "U"), - Arguments.of(ChecksumAlgorithm.CRC32_C, "V"), - Arguments.of(ChecksumAlgorithm.CRC64_NVME, "W"), - Arguments.of(ChecksumAlgorithm.SHA1, "X"), - Arguments.of(ChecksumAlgorithm.SHA256, "Y") + Arguments.of(ChecksumAlgorithm.CRC32, FLEXIBLE_CHECKSUMS_REQ_CRC32.value()), + Arguments.of(ChecksumAlgorithm.CRC32_C, FLEXIBLE_CHECKSUMS_REQ_CRC32C.value()), + Arguments.of(ChecksumAlgorithm.CRC64_NVME, FLEXIBLE_CHECKSUMS_REQ_CRC64.value()), + Arguments.of(ChecksumAlgorithm.SHA1, FLEXIBLE_CHECKSUMS_REQ_SHA1.value()), + Arguments.of(ChecksumAlgorithm.SHA256, FLEXIBLE_CHECKSUMS_REQ_SHA256.value()), + Arguments.of(ChecksumAlgorithm.SHA512, FLEXIBLE_CHECKSUMS_REQ_SHA512.value()), + Arguments.of(ChecksumAlgorithm.XXHASH64, FLEXIBLE_CHECKSUMS_REQ_XXHASH64.value()), + Arguments.of(ChecksumAlgorithm.XXHASH3, FLEXIBLE_CHECKSUMS_REQ_XXHASH3.value()), + Arguments.of(ChecksumAlgorithm.XXHASH128, FLEXIBLE_CHECKSUMS_REQ_XXHASH128.value()) ); } @@ -126,13 +143,21 @@ void when_checksumConfigurationIsSet_correctMetricIsAdded(RequestChecksumCalcula static Stream checksumConfigurationTestCases() { return Stream.of( Arguments.of(RequestChecksumCalculation.WHEN_SUPPORTED, - ResponseChecksumValidation.WHEN_SUPPORTED, "Z", "b"), + ResponseChecksumValidation.WHEN_SUPPORTED, + FLEXIBLE_CHECKSUMS_REQ_WHEN_SUPPORTED.value(), + FLEXIBLE_CHECKSUMS_RES_WHEN_SUPPORTED.value()), Arguments.of(RequestChecksumCalculation.WHEN_REQUIRED, - ResponseChecksumValidation.WHEN_REQUIRED, "a", "c"), + ResponseChecksumValidation.WHEN_REQUIRED, + FLEXIBLE_CHECKSUMS_REQ_WHEN_REQUIRED.value(), + FLEXIBLE_CHECKSUMS_RES_WHEN_REQUIRED.value()), Arguments.of(RequestChecksumCalculation.WHEN_REQUIRED, - ResponseChecksumValidation.WHEN_SUPPORTED, "a", "b"), + ResponseChecksumValidation.WHEN_SUPPORTED, + FLEXIBLE_CHECKSUMS_REQ_WHEN_REQUIRED.value(), + FLEXIBLE_CHECKSUMS_RES_WHEN_SUPPORTED.value()), Arguments.of(RequestChecksumCalculation.WHEN_SUPPORTED, - ResponseChecksumValidation.WHEN_REQUIRED, "Z", "c") + ResponseChecksumValidation.WHEN_REQUIRED, + FLEXIBLE_CHECKSUMS_REQ_WHEN_SUPPORTED.value(), + FLEXIBLE_CHECKSUMS_RES_WHEN_REQUIRED.value()) ); } @@ -169,37 +194,91 @@ static Stream checksumConfigurationWithAlgorithmTestCases() { return Stream.of( Arguments.of(RequestChecksumCalculation.WHEN_SUPPORTED, ResponseChecksumValidation.WHEN_SUPPORTED, - ChecksumAlgorithm.CRC32, "Z", "b", "U"), + ChecksumAlgorithm.CRC32, + FLEXIBLE_CHECKSUMS_REQ_WHEN_SUPPORTED.value(), + FLEXIBLE_CHECKSUMS_RES_WHEN_SUPPORTED.value(), + FLEXIBLE_CHECKSUMS_REQ_CRC32.value()), Arguments.of(RequestChecksumCalculation.WHEN_SUPPORTED, ResponseChecksumValidation.WHEN_SUPPORTED, - ChecksumAlgorithm.CRC32_C, "Z", "b", "V"), + ChecksumAlgorithm.CRC32_C, + FLEXIBLE_CHECKSUMS_REQ_WHEN_SUPPORTED.value(), + FLEXIBLE_CHECKSUMS_RES_WHEN_SUPPORTED.value(), + FLEXIBLE_CHECKSUMS_REQ_CRC32C.value()), Arguments.of(RequestChecksumCalculation.WHEN_SUPPORTED, ResponseChecksumValidation.WHEN_SUPPORTED, - ChecksumAlgorithm.SHA256, "Z", "b", "Y"), + ChecksumAlgorithm.SHA256, + FLEXIBLE_CHECKSUMS_REQ_WHEN_SUPPORTED.value(), + FLEXIBLE_CHECKSUMS_RES_WHEN_SUPPORTED.value(), + FLEXIBLE_CHECKSUMS_REQ_SHA256.value()), Arguments.of(RequestChecksumCalculation.WHEN_REQUIRED, ResponseChecksumValidation.WHEN_REQUIRED, - ChecksumAlgorithm.CRC32, "a", "c", "U"), + ChecksumAlgorithm.CRC32, + FLEXIBLE_CHECKSUMS_REQ_WHEN_REQUIRED.value(), + FLEXIBLE_CHECKSUMS_RES_WHEN_REQUIRED.value(), + FLEXIBLE_CHECKSUMS_REQ_CRC32.value()), Arguments.of(RequestChecksumCalculation.WHEN_REQUIRED, ResponseChecksumValidation.WHEN_REQUIRED, - ChecksumAlgorithm.CRC64_NVME, "a", "c", "W"), + ChecksumAlgorithm.CRC64_NVME, + FLEXIBLE_CHECKSUMS_REQ_WHEN_REQUIRED.value(), + FLEXIBLE_CHECKSUMS_RES_WHEN_REQUIRED.value(), + FLEXIBLE_CHECKSUMS_REQ_CRC64.value()), Arguments.of(RequestChecksumCalculation.WHEN_REQUIRED, ResponseChecksumValidation.WHEN_REQUIRED, - ChecksumAlgorithm.SHA1, "a", "c", "X"), + ChecksumAlgorithm.SHA1, + FLEXIBLE_CHECKSUMS_REQ_WHEN_REQUIRED.value(), + FLEXIBLE_CHECKSUMS_RES_WHEN_REQUIRED.value(), + FLEXIBLE_CHECKSUMS_REQ_SHA1.value()), + Arguments.of(RequestChecksumCalculation.WHEN_REQUIRED, + ResponseChecksumValidation.WHEN_REQUIRED, + ChecksumAlgorithm.SHA512, + FLEXIBLE_CHECKSUMS_REQ_WHEN_REQUIRED.value(), + FLEXIBLE_CHECKSUMS_RES_WHEN_REQUIRED.value(), + FLEXIBLE_CHECKSUMS_REQ_SHA512.value()), + Arguments.of(RequestChecksumCalculation.WHEN_REQUIRED, + ResponseChecksumValidation.WHEN_REQUIRED, + ChecksumAlgorithm.XXHASH64, + FLEXIBLE_CHECKSUMS_REQ_WHEN_REQUIRED.value(), + FLEXIBLE_CHECKSUMS_RES_WHEN_REQUIRED.value(), + FLEXIBLE_CHECKSUMS_REQ_XXHASH64.value()), + Arguments.of(RequestChecksumCalculation.WHEN_REQUIRED, + ResponseChecksumValidation.WHEN_REQUIRED, + ChecksumAlgorithm.XXHASH3, + FLEXIBLE_CHECKSUMS_REQ_WHEN_REQUIRED.value(), + FLEXIBLE_CHECKSUMS_RES_WHEN_REQUIRED.value(), + FLEXIBLE_CHECKSUMS_REQ_XXHASH3.value()), + Arguments.of(RequestChecksumCalculation.WHEN_REQUIRED, + ResponseChecksumValidation.WHEN_REQUIRED, + ChecksumAlgorithm.XXHASH128, + FLEXIBLE_CHECKSUMS_REQ_WHEN_REQUIRED.value(), + FLEXIBLE_CHECKSUMS_RES_WHEN_REQUIRED.value(), + FLEXIBLE_CHECKSUMS_REQ_XXHASH128.value()), Arguments.of(RequestChecksumCalculation.WHEN_REQUIRED, ResponseChecksumValidation.WHEN_SUPPORTED, - ChecksumAlgorithm.CRC32_C, "a", "b", "V"), + ChecksumAlgorithm.CRC32_C, + FLEXIBLE_CHECKSUMS_REQ_WHEN_REQUIRED.value(), + FLEXIBLE_CHECKSUMS_RES_WHEN_SUPPORTED.value(), + FLEXIBLE_CHECKSUMS_REQ_CRC32C.value()), Arguments.of(RequestChecksumCalculation.WHEN_REQUIRED, ResponseChecksumValidation.WHEN_SUPPORTED, - ChecksumAlgorithm.SHA256, "a", "b", "Y"), + ChecksumAlgorithm.SHA256, + FLEXIBLE_CHECKSUMS_REQ_WHEN_REQUIRED.value(), + FLEXIBLE_CHECKSUMS_RES_WHEN_SUPPORTED.value(), + FLEXIBLE_CHECKSUMS_REQ_SHA256.value()), Arguments.of(RequestChecksumCalculation.WHEN_SUPPORTED, ResponseChecksumValidation.WHEN_REQUIRED, - ChecksumAlgorithm.CRC64_NVME, "Z", "c", "W"), + ChecksumAlgorithm.CRC64_NVME, + FLEXIBLE_CHECKSUMS_REQ_WHEN_SUPPORTED.value(), + FLEXIBLE_CHECKSUMS_RES_WHEN_REQUIRED.value(), + FLEXIBLE_CHECKSUMS_REQ_CRC64.value()), Arguments.of(RequestChecksumCalculation.WHEN_SUPPORTED, ResponseChecksumValidation.WHEN_REQUIRED, - ChecksumAlgorithm.SHA1, "Z", "c", "X") + ChecksumAlgorithm.SHA1, + FLEXIBLE_CHECKSUMS_REQ_WHEN_SUPPORTED.value(), + FLEXIBLE_CHECKSUMS_RES_WHEN_REQUIRED.value(), + FLEXIBLE_CHECKSUMS_REQ_SHA1.value()) ); } diff --git a/test/codegen-generated-classes-test/src/test/java/software/amazon/awssdk/services/HttpChecksumCalculationTest.java b/test/codegen-generated-classes-test/src/test/java/software/amazon/awssdk/services/HttpChecksumCalculationTest.java index abd0ff077737..b7b56ed08b3b 100644 --- a/test/codegen-generated-classes-test/src/test/java/software/amazon/awssdk/services/HttpChecksumCalculationTest.java +++ b/test/codegen-generated-classes-test/src/test/java/software/amazon/awssdk/services/HttpChecksumCalculationTest.java @@ -16,8 +16,8 @@ package software.amazon.awssdk.services; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; -import static software.amazon.awssdk.auth.signer.S3SignerExecutionAttribute.ENABLE_CHUNKED_ENCODING; import static software.amazon.awssdk.core.HttpChecksumConstant.HTTP_CHECKSUM_HEADER_PREFIX; import io.reactivex.Flowable; @@ -27,6 +27,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -34,12 +35,9 @@ import org.mockito.Mockito; import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; import software.amazon.awssdk.core.HttpChecksumConstant; -import software.amazon.awssdk.core.SdkRequest; import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.core.checksums.RequestChecksumCalculation; -import software.amazon.awssdk.core.interceptor.Context; -import software.amazon.awssdk.core.interceptor.ExecutionAttributes; -import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; +import software.amazon.awssdk.core.exception.SdkException; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.http.ExecutableHttpRequest; import software.amazon.awssdk.http.HttpExecuteRequest; @@ -80,16 +78,6 @@ private ProtocolRestJsonAsyncClientBuilder initializeAsync(RequestChecksumCalcul .region(Region.US_WEST_2); } - private static final class EnableChunkedEncodingInterceptor implements ExecutionInterceptor { - public void beforeExecution(Context.BeforeExecution context, ExecutionAttributes executionAttributes) { - SdkRequest request = context.request(); - - if (request instanceof PutOperationWithChecksumRequest) { - executionAttributes.putAttributeIfAbsent(ENABLE_CHUNKED_ENCODING, true); - } - } - } - @BeforeEach public void setup() throws IOException { httpClient = Mockito.mock(SdkHttpClient.class); @@ -259,4 +247,54 @@ private SdkHttpRequest getAsyncRequest() { Mockito.verify(httpAsyncClient).execute(captor.capture()); return captor.getValue().request(); } + + @Test + public void syncMd5ChecksumCalculation_shouldThrowException() { + try (ProtocolRestJsonClient client = initializeSync(RequestChecksumCalculation.WHEN_SUPPORTED).build()) { + assertThatThrownBy(() -> client.putOperationWithChecksum( + PutOperationWithChecksumRequest.builder() + .checksumAlgorithm(ChecksumAlgorithm.MD5) + .build(), + RequestBody.fromString("Hello world"))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("MD5 is not supported"); + } + } + + @Test + public void asyncMd5ChecksumCalculation_shouldThrowException() { + try (ProtocolRestJsonAsyncClient client = initializeAsync(RequestChecksumCalculation.WHEN_SUPPORTED).build()) { + assertThatThrownBy(() -> client.putOperationWithChecksum( + PutOperationWithChecksumRequest.builder() + .checksumAlgorithm(ChecksumAlgorithm.MD5) + .build(), + AsyncRequestBody.fromString("Hello world")).join()) + .hasCauseInstanceOf(SdkException.class) + .hasMessageContaining("MD5 is not supported"); + } + } + + @Test + public void syncMd5ChecksumCalculation_userProvidedMd5Header_shouldSucceed() { + try (ProtocolRestJsonClient client = initializeSync(RequestChecksumCalculation.WHEN_SUPPORTED).build()) { + client.putOperationWithChecksum(r -> r.overrideConfiguration(o -> o.putHeader("x-amz-checksum-md5", "PiWWCnnbxptnTNTsZ6csYg==")), + RequestBody.fromString("Hello world")); + assertMd5HeaderPresent(getSyncRequest()); + } + } + + @Test + public void asyncMd5ChecksumCalculation_userProvidedMd5Header_shouldSucceed() { + try (ProtocolRestJsonAsyncClient client = initializeAsync(RequestChecksumCalculation.WHEN_SUPPORTED).build()) { + client.putOperationWithChecksum(r -> r.overrideConfiguration(o -> o.putHeader("x-amz-checksum-md5", "PiWWCnnbxptnTNTsZ6csYg==")), + AsyncRequestBody.fromString("Hello world")).join(); + assertMd5HeaderPresent(getAsyncRequest()); + } + } + + private void assertMd5HeaderPresent(SdkHttpRequest request) { + assertThat(request.firstMatchingHeader("x-amz-checksum-md5")) + .isPresent() + .hasValue("PiWWCnnbxptnTNTsZ6csYg=="); + } } \ No newline at end of file diff --git a/test/codegen-generated-classes-test/src/test/java/software/amazon/awssdk/services/HttpChecksumValidationTest.java b/test/codegen-generated-classes-test/src/test/java/software/amazon/awssdk/services/HttpChecksumValidationTest.java index facc376432d6..753f9de9956e 100644 --- a/test/codegen-generated-classes-test/src/test/java/software/amazon/awssdk/services/HttpChecksumValidationTest.java +++ b/test/codegen-generated-classes-test/src/test/java/software/amazon/awssdk/services/HttpChecksumValidationTest.java @@ -75,7 +75,11 @@ public class HttpChecksumValidationTest { .put("crc32c", "crUfeA==") .put("sha1", "e1AsOh9IyGCa4hLN+2Od7jlnP14=") .put("sha256", "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=") + .put("sha512", "t/eDuu2Cl/DbkXRiGE/08I5pwtXl95qUJgD5cl9Yzh8pwYE5v4CwbA//K900c4RS7PQMSIwip+PYDN9vnBwNRw==") .put("crc64nvme", "OOJZ0D8xKts=") + .put("xxhash64", "xQCwyRKzdtg=") + .put("xxhash3", "tqy52Eo4/3Q=") + .put("xxhash128", "c1H4mBL5c4K5HQWzHgTdfw==") .build(); private ProtocolRestJsonClient client; @@ -215,6 +219,30 @@ public void syncClientValidateStreamingResponse_noSupportedAlgorithmFoundInClien assertThat(CaptureChecksumValidationInterceptor.expectedAlgorithm).isNull(); } + @Test + public void syncClientValidateStreamingResponse_md5NotSupported_shouldSkipValidation() { + stubMd5Checksum(); + ResponseBytes responseBytes = + client.getOperationWithChecksum( + r -> r.checksumMode(ChecksumMode.ENABLED), + ResponseTransformer.toBytes()); + assertThat(responseBytes.asUtf8String()).isEqualTo("Hello world"); + assertThat(CaptureChecksumValidationInterceptor.checksumValidation).isEqualTo(ChecksumValidation.CHECKSUM_ALGORITHM_NOT_FOUND); + assertThat(CaptureChecksumValidationInterceptor.expectedAlgorithm).isNull(); + } + + @Test + public void asyncClientValidateStreamingResponse_md5NotSupported_shouldSkipValidation() { + stubMd5Checksum(); + ResponseBytes responseBytes = + asyncClient.getOperationWithChecksum( + r -> r.checksumMode(ChecksumMode.ENABLED), + AsyncResponseTransformer.toBytes()).join(); + assertThat(responseBytes.asUtf8String()).isEqualTo("Hello world"); + assertThat(CaptureChecksumValidationInterceptor.checksumValidation).isEqualTo(ChecksumValidation.CHECKSUM_ALGORITHM_NOT_FOUND); + assertThat(CaptureChecksumValidationInterceptor.expectedAlgorithm).isNull(); + } + @Test public void syncClientValidateStreamingResponseWithValidationFailed() { String expectedChecksum = "i9aeUg="; @@ -388,6 +416,11 @@ private void stubWithMultipleChecksums(String body, Map checksum stubFor(post(anyUrl()).willReturn(responseBuilder)); } + private void stubMd5Checksum() { + String md5Checksum = "PiWWCnnbxptnTNTsZ6csYg=="; + stubWithSingleChecksum("Hello world", md5Checksum, "md5"); + } + private void stubWithNoChecksum() { ResponseDefinitionBuilder responseBuilder = aResponse().withStatus(200) .withHeader("content-length", diff --git a/test/crt-unavailable-tests/src/main/resources/codegen-resources/service-2.json b/test/crt-unavailable-tests/src/main/resources/codegen-resources/service-2.json index 2a6c09074cd8..e7d6ee2636b9 100644 --- a/test/crt-unavailable-tests/src/main/resources/codegen-resources/service-2.json +++ b/test/crt-unavailable-tests/src/main/resources/codegen-resources/service-2.json @@ -242,7 +242,7 @@ "requestChecksumRequired" : true, "requestAlgorithmMember": "ChecksumAlgorithm", "requestValidationModeMember" : "ChecksumMode", - "responseAlgorithms": ["crc64nvme","crc32", "crc32c", "sha256", "sha1"] + "responseAlgorithms": ["crc64nvme","crc32", "crc32c", "sha256", "sha512", "sha1", "xxhash64", "xxhash3", "xxhash128"] } }, "StreamingInputOperation":{ @@ -333,7 +333,7 @@ "output":{"shape":"ChecksumStructureWithStreaming"}, "httpChecksum" : { "requestValidationModeMember": "ChecksumMode", - "responseAlgorithms": ["CRC64NVME","CRC32C", "CRC32", "SHA1", "SHA256"] + "responseAlgorithms": ["CRC64NVME","CRC32C", "CRC32", "SHA1", "SHA256", "SHA512", "XXHASH64", "XXHASH3", "XXHASH128"] } }, "GetOperationWithMapEndpointParam":{ @@ -357,7 +357,7 @@ "requestChecksumRequired": true, "requestAlgorithmMember": "ChecksumAlgorithm", "requestValidationModeMember": "ChecksumMode", - "responseAlgorithms": ["CRC64NVME","CRC32C", "CRC32", "SHA1", "SHA256"] + "responseAlgorithms": ["CRC64NVME","CRC32C", "CRC32", "SHA1", "SHA256", "SHA512", "XXHASH64", "XXHASH3", "XXHASH128"] } }, "QueryParameterOperation":{ @@ -980,7 +980,10 @@ "CRC32C", "SHA1", "SHA256", - "CRC64NVME" + "CRC64NVME", + "XXHASH64", + "XXHASH3", + "XXHASH128" ] }, "ChecksumMode":{ diff --git a/test/crt-unavailable-tests/src/test/java/software/amazon/awssdk/checksums/internal/CrtBasedChecksumTest.java b/test/crt-unavailable-tests/src/test/java/software/amazon/awssdk/checksums/internal/CrtBasedChecksumTest.java index e25b7de51433..e7da691317e4 100644 --- a/test/crt-unavailable-tests/src/test/java/software/amazon/awssdk/checksums/internal/CrtBasedChecksumTest.java +++ b/test/crt-unavailable-tests/src/test/java/software/amazon/awssdk/checksums/internal/CrtBasedChecksumTest.java @@ -35,7 +35,7 @@ void createCrc64WithoutCrtDependency(){ @Test void createCrtBased32CWithoutCrtDependency(){ - assertNull(CrcChecksumProvider.createCrtCrc32C()); + assertNull(ChecksumProvider.createCrtCrc32C()); } } \ No newline at end of file diff --git a/test/crt-unavailable-tests/src/test/java/software/amazon/awssdk/checksumtest/XxHashNotAvailableTest.java b/test/crt-unavailable-tests/src/test/java/software/amazon/awssdk/checksumtest/XxHashNotAvailableTest.java new file mode 100644 index 000000000000..6633941a4613 --- /dev/null +++ b/test/crt-unavailable-tests/src/test/java/software/amazon/awssdk/checksumtest/XxHashNotAvailableTest.java @@ -0,0 +1,177 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.checksumtest; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static org.mockito.ArgumentMatchers.any; + +import io.reactivex.Flowable; +import java.io.IOException; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Stream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.Mockito; +import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.checksums.ChecksumValidation; +import software.amazon.awssdk.core.interceptor.Context; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; +import software.amazon.awssdk.core.interceptor.SdkExecutionAttribute; +import software.amazon.awssdk.core.sync.ResponseTransformer; +import software.amazon.awssdk.http.ExecutableHttpRequest; +import software.amazon.awssdk.http.HttpExecuteResponse; +import software.amazon.awssdk.http.SdkHttpClient; +import software.amazon.awssdk.http.SdkHttpFullResponse; +import software.amazon.awssdk.http.async.AsyncExecuteRequest; +import software.amazon.awssdk.http.async.SdkAsyncHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.protocolrestjson.ProtocolRestJsonAsyncClient; +import software.amazon.awssdk.services.protocolrestjson.ProtocolRestJsonAsyncClientBuilder; +import software.amazon.awssdk.services.protocolrestjson.ProtocolRestJsonClient; +import software.amazon.awssdk.services.protocolrestjson.ProtocolRestJsonClientBuilder; +import software.amazon.awssdk.services.protocolrestjson.model.ChecksumMode; +import software.amazon.awssdk.services.protocolrestjson.model.OperationWithCustomRequestChecksumRequest; + +public class XxHashNotAvailableTest { + private SdkHttpClient httpClient; + private SdkAsyncHttpClient httpAsyncClient; + private ProtocolRestJsonAsyncClient asyncClient; + private ProtocolRestJsonClient client; + + @BeforeEach + public void setup() throws IOException { + httpClient = Mockito.mock(SdkHttpClient.class); + httpAsyncClient = Mockito.mock(SdkAsyncHttpClient.class); + client = initializeSync().build(); + asyncClient = initializeAsync().build(); + } + + private void stubResponse(SdkHttpFullResponse.Builder responseBuilder) throws IOException { + SdkHttpFullResponse successfulHttpResponse = responseBuilder + .statusCode(200) + .putHeader("Content-Length", "0") + .build(); + + ExecutableHttpRequest request = Mockito.mock(ExecutableHttpRequest.class); + Mockito.when(request.call()).thenReturn(HttpExecuteResponse.builder() + .response(successfulHttpResponse) + .build()); + Mockito.when(httpClient.prepareRequest(any())).thenReturn(request); + + Mockito.when(httpAsyncClient.execute(any())).thenAnswer(invocation -> { + AsyncExecuteRequest asyncExecuteRequest = invocation.getArgument(0, AsyncExecuteRequest.class); + asyncExecuteRequest.responseHandler().onHeaders(successfulHttpResponse); + asyncExecuteRequest.responseHandler().onStream(Flowable.empty()); + return CompletableFuture.completedFuture(null); + }); + } + + private ProtocolRestJsonAsyncClientBuilder initializeAsync() { + return ProtocolRestJsonAsyncClient.builder().httpClient(httpAsyncClient) + .credentialsProvider(AnonymousCredentialsProvider.create()) + .overrideConfiguration( + o -> o.addExecutionInterceptor(new CaptureChecksumValidationInterceptor())) + .region(Region.US_WEST_2); + } + + private ProtocolRestJsonClientBuilder initializeSync() { + return ProtocolRestJsonClient.builder().httpClient(httpClient) + .credentialsProvider(AnonymousCredentialsProvider.create()) + .overrideConfiguration( + o -> o.addExecutionInterceptor(new CaptureChecksumValidationInterceptor())) + .region(Region.US_WEST_2); + } + + @ParameterizedTest + @MethodSource("xxHashServiceAlgorithms") + public void asyncChecksumCalculation_xxHashNotAvailable_shouldThrowException( + software.amazon.awssdk.services.protocolrestjson.model.ChecksumAlgorithm serviceAlgorithm) throws IOException { + stubResponse(SdkHttpFullResponse.builder()); + assertThatThrownBy(() -> asyncClient.operationWithCustomRequestChecksum( + OperationWithCustomRequestChecksumRequest.builder() + .checksumAlgorithm(serviceAlgorithm) + .build()).join()) + .hasMessageContaining("Add dependency on 'software.amazon.awssdk.crt:aws-crt' module"); + } + + @ParameterizedTest + @MethodSource("xxHashServiceAlgorithms") + public void syncChecksumCalculation_xxHashNotAvailable_shouldThrowException( + software.amazon.awssdk.services.protocolrestjson.model.ChecksumAlgorithm serviceAlgorithm) throws IOException { + stubResponse(SdkHttpFullResponse.builder()); + assertThatThrownBy(() -> client.operationWithCustomRequestChecksum( + OperationWithCustomRequestChecksumRequest.builder() + .checksumAlgorithm(serviceAlgorithm) + .build())) + .hasMessageContaining("Add dependency on 'software.amazon.awssdk.crt:aws-crt' module"); + } + + @ParameterizedTest + @MethodSource("xxHashValidationHeaders") + public void syncChecksumValidation_onlyHasXxHash_shouldSkipValidation(String headerName) throws IOException { + stubResponse(SdkHttpFullResponse.builder().putHeader(headerName, "foobar")); + + client.getOperationWithChecksum( + r -> r.checksumMode(ChecksumMode.ENABLED), + ResponseTransformer.toBytes()); + + assertThat(CaptureChecksumValidationInterceptor.checksumValidation).isEqualTo(ChecksumValidation.CHECKSUM_ALGORITHM_NOT_FOUND); + assertThat(CaptureChecksumValidationInterceptor.expectedAlgorithm).isNull(); + } + + @ParameterizedTest + @MethodSource("xxHashValidationHeaders") + public void asyncChecksumValidation_onlyHasXxHash_shouldSkipValidation(String headerName) throws IOException { + stubResponse(SdkHttpFullResponse.builder().putHeader(headerName, "foobar")); + + asyncClient.getOperationWithChecksum( + r -> r.checksumMode(ChecksumMode.ENABLED), + AsyncResponseTransformer.toBytes()).join(); + + assertThat(CaptureChecksumValidationInterceptor.checksumValidation).isEqualTo(ChecksumValidation.CHECKSUM_ALGORITHM_NOT_FOUND); + assertThat(CaptureChecksumValidationInterceptor.expectedAlgorithm).isNull(); + } + + static Stream xxHashServiceAlgorithms() { + return Stream.of( + Arguments.of(software.amazon.awssdk.services.protocolrestjson.model.ChecksumAlgorithm.XXHASH64, "XXHASH64"), + Arguments.of(software.amazon.awssdk.services.protocolrestjson.model.ChecksumAlgorithm.XXHASH3, "XXHASH3"), + Arguments.of(software.amazon.awssdk.services.protocolrestjson.model.ChecksumAlgorithm.XXHASH128, "XXHASH128") + ); + } + + static Stream xxHashValidationHeaders() { + return Stream.of("x-amz-checksum-xxhash64", "x-amz-checksum-xxhash3", "x-amz-checksum-xxhash128"); + } + + private static class CaptureChecksumValidationInterceptor implements ExecutionInterceptor { + private static software.amazon.awssdk.checksums.spi.ChecksumAlgorithm expectedAlgorithm; + private static ChecksumValidation checksumValidation; + + @Override + public void afterExecution(Context.AfterExecution context, ExecutionAttributes executionAttributes) { + expectedAlgorithm = + executionAttributes.getOptionalAttribute(SdkExecutionAttribute.HTTP_CHECKSUM_VALIDATION_ALGORITHM_V2).orElse(null); + checksumValidation = + executionAttributes.getOptionalAttribute(SdkExecutionAttribute.HTTP_RESPONSE_CHECKSUM_VALIDATION).orElse(null); + } + } +}