Add additional checksum algorithms

codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/traits/HttpChecksumTrait.java

@@ -39,13 +39,19 @@
  */
 public class HttpChecksumTrait {
 
+    // TODO: run perf tests
     private static final Map<String, Integer> CHECKSUM_ALGORITHM_PRIORITY =
         ImmutableMap.<String, Integer>builder()
                     .put(DefaultChecksumAlgorithm.CRC32C.algorithmId(), 1)
                     .put(DefaultChecksumAlgorithm.CRC32.algorithmId(), 2)
                     .put(DefaultChecksumAlgorithm.CRC64NVME.algorithmId(), 3)
-                    .put(DefaultChecksumAlgorithm.SHA1.algorithmId(), 4)
-                    .put(DefaultChecksumAlgorithm.SHA256.algorithmId(), 5)
+                    .put(DefaultChecksumAlgorithm.XXHASH128.algorithmId(), 4)
+                    .put(DefaultChecksumAlgorithm.XXHASH3.algorithmId(), 5)
+                    .put(DefaultChecksumAlgorithm.XXHASH64.algorithmId(), 6)
+                    .put(DefaultChecksumAlgorithm.SHA1.algorithmId(), 7)
+                    .put(DefaultChecksumAlgorithm.SHA256.algorithmId(), 8)
+                    .put(DefaultChecksumAlgorithm.SHA512.algorithmId(), 9)
+                    .put(DefaultChecksumAlgorithm.MD5.algorithmId(), 10)
                     .build();
 
     private HttpChecksumTrait() {

core/checksums/pom.xml

@@ -55,7 +55,7 @@
             <groupId>software.amazon.awssdk.crt</groupId>
             <artifactId>aws-crt</artifactId>
             <version>${awscrt.version}</version>
-            <scope>test</scope>
+            <optional>true</optional>
         </dependency>
 
         <dependency>

core/checksums/src/main/java/software/amazon/awssdk/checksums/DefaultChecksumAlgorithm.java

@@ -31,7 +31,11 @@ public final class DefaultChecksumAlgorithm {
     public static final ChecksumAlgorithm MD5 = of("MD5");
     public static final ChecksumAlgorithm SHA256 = of("SHA256");
     public static final ChecksumAlgorithm SHA1 = of("SHA1");
+    public static final ChecksumAlgorithm SHA512 = of("SHA512");
     public static final ChecksumAlgorithm CRC64NVME = of("CRC64NVME");
+    public static final ChecksumAlgorithm XXHASH64 = of("XXHASH64");
+    public static final ChecksumAlgorithm XXHASH3 = of("XXHASH3");
+    public static final ChecksumAlgorithm XXHASH128 = of("XXHASH128");
 
     private DefaultChecksumAlgorithm() {
     }

core/checksums/src/main/java/software/amazon/awssdk/checksums/SdkChecksum.java

@@ -18,9 +18,9 @@
 import java.nio.ByteBuffer;
 import java.util.zip.Checksum;
 import software.amazon.awssdk.annotations.SdkProtectedApi;
+import software.amazon.awssdk.checksums.internal.ChecksumProvider;
 import software.amazon.awssdk.checksums.internal.Crc32Checksum;
 import software.amazon.awssdk.checksums.internal.Crc64NvmeChecksum;
-import software.amazon.awssdk.checksums.internal.CrcChecksumProvider;
 import software.amazon.awssdk.checksums.internal.DigestAlgorithm;
 import software.amazon.awssdk.checksums.internal.DigestAlgorithmChecksum;
 import software.amazon.awssdk.checksums.spi.ChecksumAlgorithm;
@@ -38,17 +38,25 @@ public interface SdkChecksum extends Checksum {
     static SdkChecksum forAlgorithm(ChecksumAlgorithm algorithm) {
         switch (algorithm.algorithmId()) {
             case "CRC32C":
-                return CrcChecksumProvider.crc32cImplementation();
+                return ChecksumProvider.crc32cImplementation();
             case "CRC32":
                 return new Crc32Checksum();
             case "SHA1":
                 return new DigestAlgorithmChecksum(DigestAlgorithm.SHA1);
             case "SHA256":
                 return new DigestAlgorithmChecksum(DigestAlgorithm.SHA256);
+            case "SHA512":
+                return new DigestAlgorithmChecksum(DigestAlgorithm.SHA512);
             case "MD5":
                 return new DigestAlgorithmChecksum(DigestAlgorithm.MD5);
             case "CRC64NVME":
                 return new Crc64NvmeChecksum();
+            case "XXHASH64":
+                return ChecksumProvider.xxHash64CrtImplementation();
+            case "XXHASH3":
+                return ChecksumProvider.xxHash3CrtImplementation();
+            case "XXHASH128":
+                return ChecksumProvider.xxHash128CrtImplementation();
             default:
                 throw new UnsupportedOperationException("Unsupported checksum algorithm: " + algorithm);
         }

core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/CrcChecksumProvider.java → core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/ChecksumProvider.java

@@ -17,10 +17,16 @@
 
 import java.util.zip.Checksum;
 import software.amazon.awssdk.annotations.SdkInternalApi;
+import software.amazon.awssdk.checksums.DefaultChecksumAlgorithm;
 import software.amazon.awssdk.checksums.SdkChecksum;
+import software.amazon.awssdk.checksums.spi.ChecksumAlgorithm;
+import software.amazon.awssdk.crt.checksums.CRC32C;
+import software.amazon.awssdk.crt.checksums.CRC64NVME;
+import software.amazon.awssdk.utils.ClassLoaderHelper;
+import software.amazon.awssdk.utils.Lazy;
 
 /**
- * Utility class providing implementations of CRC checksums, specifically CRC32C and CRC64NVME.
+ * Utility class providing implementations of checksums.
  *
  * <p>Supports the following implementations for CRC32C:</p>
  * <ul>
@@ -29,23 +35,38 @@
  *     <li>SDK-based CRC32C (fallback)</li>
  * </ul>
  *
- * <p>Only supports CRT-based implementation for CRC64NVME (using AWS CRT library).</p>
+ * <p>Supports CRT-based implementations for CRC64NVME and XXHASH algorithms (using AWS CRT library).</p>
  *
  * <p>For internal use only ({@link SdkInternalApi}).</p>
  */
 @SdkInternalApi
-public final class CrcChecksumProvider {
-
+public final class ChecksumProvider {
 
     // Class paths for different CRC32C implementations
     private static final String CRT_CRC32C_CLASS_PATH = "software.amazon.awssdk.crt.checksums.CRC32C";
     private static final String JAVA_CRC32C_CLASS_PATH = "java.util.zip.CRC32C";
     private static final ConstructorCache CONSTRUCTOR_CACHE = new ConstructorCache();
     private static final String CRT_CRC64NVME_PATH = "software.amazon.awssdk.crt.checksums.CRC64NVME";
+    private static final String CRT_XXHASH_PATH = "software.amazon.awssdk.crt.checksums.XXHash";
     private static final String CRT_MODULE = "software.amazon.awssdk.crt:aws-crt";
 
+    private static Lazy<Boolean> isXxHashAvailable = checkCrtAvailability(CRT_XXHASH_PATH);
+    private static Lazy<Boolean> isCrc64NvmeAvailable = checkCrtAvailability(CRT_CRC64NVME_PATH);
+    private static Lazy<Boolean> isCrc32CAvailable = checkCrtAvailability(CRT_CRC32C_CLASS_PATH);
+
     // Private constructor to prevent instantiation
-    private CrcChecksumProvider() {
+    private ChecksumProvider() {
+    }
+
+    private static Lazy<Boolean> checkCrtAvailability(String fqcn) {
+        return new Lazy<>(() -> {
+            try {
+                ClassLoaderHelper.loadClass(fqcn, false);
+            } catch (ClassNotFoundException e) {
+                return false;
+            }
+            return true;
+        });
     }
 
     /**
@@ -74,14 +95,12 @@ public static SdkChecksum crc32cImplementation() {
     }
 
     static SdkChecksum createCrtCrc32C() {
-        return CONSTRUCTOR_CACHE.getConstructor(CRT_CRC32C_CLASS_PATH).map(constructor -> {
-            try {
-                Checksum checksumInstance = (Checksum) constructor.newInstance();
-                return new CrcCloneOnMarkChecksum(checksumInstance);
-            } catch (ReflectiveOperationException e) {
-                throw new IllegalStateException("Failed to instantiate " + CRT_CRC32C_CLASS_PATH, e);
-            }
-        }).orElse(null);
+        if (!isCrc32CAvailable.getValue()) {
+            return null;
+        }
+
+        Checksum checksumInstance = new CRC32C();
+        return new CrcCloneOnMarkChecksum(checksumInstance);
     }
 
     /**
@@ -96,16 +115,57 @@ static SdkChecksum createCrtCrc32C() {
      * @throws RuntimeException if the `CRC64NVME` implementation is not available.
      */
     static SdkChecksum crc64NvmeCrtImplementation() {
-        return CONSTRUCTOR_CACHE.getConstructor(CRT_CRC64NVME_PATH).map(constructor -> {
-            try {
-                Checksum checksumInstance = (Checksum) constructor.newInstance();
-                return new CrcCloneOnMarkChecksum(checksumInstance);
-            } catch (ReflectiveOperationException e) {
-                throw new IllegalStateException("Failed to instantiate " + CRT_CRC32C_CLASS_PATH, e);
-            }
-        }).orElseThrow(() -> new RuntimeException(
-            "Could not load " + CRT_CRC64NVME_PATH + ". Add dependency on '" + CRT_MODULE
-            + "' module to enable CRC64NVME feature."));
+
+        if (!isCrc64NvmeAvailable.getValue()) {
+            throw new RuntimeException(
+                "Could not load " + CRT_CRC64NVME_PATH + ". Add dependency on '" + CRT_MODULE
+                + "' module to enable CRC64NVME feature.");
+        }
+
+        return new CrcCloneOnMarkChecksum(new CRC64NVME());
+    }
+
+    /**
+     * Creates an instance of the CRT-based XXHASH64 checksum using AWS's CRT library.
+     *
+     * @return An {@link SdkChecksum} instance for XXHASH64.
+     * @throws IllegalStateException if instantiation fails.
+     * @throws RuntimeException if the CRT implementation is not available.
+     */
+    public static SdkChecksum xxHash64CrtImplementation() {
+        return crtXxHash(DefaultChecksumAlgorithm.XXHASH64);
+    }
+
+    /**
+     * Creates an instance of the CRT-based XXHASH3 checksum using AWS's CRT library.
+     *
+     * @return An {@link SdkChecksum} instance for XXHASH3.
+     * @throws IllegalStateException if instantiation fails.
+     * @throws RuntimeException if the CRT implementation is not available.
+     */
+    public static SdkChecksum xxHash3CrtImplementation() {
+        return crtXxHash(DefaultChecksumAlgorithm.XXHASH3);
+    }
+
+    /**
+     * Creates an instance of the CRT-based XXHASH128 checksum using AWS's CRT library.
+     *
+     * @return An {@link SdkChecksum} instance for XXHASH128.
+     * @throws IllegalStateException if instantiation fails.
+     * @throws RuntimeException if the CRT implementation is not available.
+     */
+    public static SdkChecksum xxHash128CrtImplementation() {
+        return crtXxHash(DefaultChecksumAlgorithm.XXHASH128);
+    }
+
+    static SdkChecksum crtXxHash(ChecksumAlgorithm algorithm) {
+        if (!isXxHashAvailable.getValue()) {
+            throw new RuntimeException(
+                "Could not load " + CRT_XXHASH_PATH + ". Add dependency on '" + CRT_MODULE
+                + "' module.");
+        }
+
+        return new XxHashChecksum(algorithm);
     }
 
     static SdkChecksum createJavaCrc32C() {
@@ -117,5 +177,4 @@ static SdkChecksum createJavaCrc32C() {
             }
         }).orElse(null);
     }
-
 }

core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Crc64NvmeChecksum.java

@@ -29,7 +29,7 @@ public final class Crc64NvmeChecksum implements SdkChecksum {
     private final SdkChecksum sdkChecksum;
 
     public Crc64NvmeChecksum() {
-        this.sdkChecksum = CrcChecksumProvider.crc64NvmeCrtImplementation();
+        this.sdkChecksum = ChecksumProvider.crc64NvmeCrtImplementation();
     }
 
     @Override

core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/DigestAlgorithm.java

@@ -29,7 +29,8 @@ public enum DigestAlgorithm {
     SHA1("SHA-1"),
 
     MD5("MD5"),
-    SHA256("SHA-256")
+    SHA256("SHA-256"),
+    SHA512("SHA-512")
     ;
 
     private static final Supplier<MessageDigest> CLOSED_DIGEST = () -> {

core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/XxHashChecksum.java

@@ -0,0 +1,76 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.checksums.internal;
+
+import software.amazon.awssdk.annotations.SdkInternalApi;
+import software.amazon.awssdk.checksums.DefaultChecksumAlgorithm;
+import software.amazon.awssdk.checksums.SdkChecksum;
+import software.amazon.awssdk.checksums.spi.ChecksumAlgorithm;
+import software.amazon.awssdk.crt.checksums.XXHash;
+
+@SdkInternalApi
+public final class XxHashChecksum implements SdkChecksum {
+
+    private final XXHash xxHash;
+
+    public XxHashChecksum(ChecksumAlgorithm algorithm) {
+        if (algorithm == DefaultChecksumAlgorithm.XXHASH64) {
+            xxHash = XXHash.newXXHash64();
+        } else if (algorithm == DefaultChecksumAlgorithm.XXHASH3) {
+            xxHash = XXHash.newXXHash3_64();
+        } else if (algorithm == DefaultChecksumAlgorithm.XXHASH128) {
+            xxHash = XXHash.newXXHash3_128();
+        } else {
+            throw new UnsupportedOperationException("Unsupported algorithm: " + algorithm.algorithmId());
+        }
+    }
+
+    @Override
+    public void update(byte[] b) {
+        xxHash.update(b);
+    }
+
+    @Override
+    public byte[] getChecksumBytes() {
+        return xxHash.digest();
+    }
+
+
+    @Override
+    public void update(int b) {
+        xxHash.update(b);
+    }
+
+    @Override
+    public void update(byte[] b, int off, int len) {
+        xxHash.update(b, off, len);
+    }
+
+    @Override
+    public void reset() {
+        throw new UnsupportedOperationException("mark and reset is not supported");
+    }
+
+    @Override
+    public void mark(int readLimit) {
+        throw new UnsupportedOperationException("mark and reset is not supported");
+    }
+
+    @Override
+    public long getValue() {
+        throw new UnsupportedOperationException("Use getChecksumBytes() instead.");
+    }
+}

core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/DefaultCRC32CChecksumTest.java

@@ -22,7 +22,7 @@ public class DefaultCRC32CChecksumTest extends Crc32CChecksumTest {
 
     @BeforeEach
     public void setUp() {
-        sdkChecksum = CrcChecksumProvider.crc32cImplementation();
+        sdkChecksum = ChecksumProvider.crc32cImplementation();
     }