feat: Checksum reuse between retries in chunked streaming requests (#946)

* Changes required to reuse finalized checksums in chunked stremaing.

* Temporary github CI change to make logs more verbose for debugging.

* Temp workflow change.

* Undo temp GH workflow changes previously made for debugging purposes.

---------

Co-authored-by: Sichan Yoo <[email protected]>

Author: sichanyoo

Sources/ClientRuntime/Networking/Http/Middlewares/SignerMiddleware.swift

@@ -57,6 +57,11 @@ extension SignerMiddleware: ApplySigner {
             updatedSigningProperties.set(key: AttributeKey(name: "SignedBodyValue"), value: bodyValue)
         }
 
+        if case .stream(let stream) = request.body, stream.isEligibleForChunkedStreaming {
+            // Pass in context object via signing properties to reuse final checksum value in chunked streaming.
+            updatedSigningProperties.set(key: AttributeKey(name: "Context"), value: attributes)
+        }
+
         let signed = try await signer.signRequest(
             requestBuilder: request.toBuilder(),
             identity: identity,

Sources/SmithyChecksums/ChunkedReader.swift

@@ -26,13 +26,16 @@ public class ChunkedReader {
     private var checksumAlgorithm: ChecksumAlgorithm?
     private var checksum: (any Checksum)?
     private var checksumString: String?
+    private let context: Smithy.Context
+    private let cachedChecksum: String?
 
     init(
         stream: ReadableStream,
         signingConfig: SigningConfig,
         previousSignature: String,
         trailingHeaders: Headers,
-        checksumString: String? = nil // if nil, chunked encoding without checksum
+        checksumString: String? = nil, // if nil, chunked encoding without checksum
+        context: Smithy.Context
     ) throws {
         self.stream = stream
         self.signingConfig = signingConfig
@@ -52,7 +55,8 @@ public class ChunkedReader {
         } else {
             checksumAlgorithm = nil
         }
-
+        self.context = context
+        self.cachedChecksum = context.get(key: AttributeKey<String>(name: "ChunkedStreamCachedChecksum"))
         self.checksumAlgorithm = checksumAlgorithm // Enum for working with the checksum
         self.checksum = self.checksumAlgorithm?.createChecksum() // Create an instance of the Checksum
     }
@@ -67,10 +71,9 @@ public class ChunkedReader {
         let nextChunk = try await fetchNextChunk()
 
         if let chunk = nextChunk {
-            if let checksum = self.checksum {
+            if let checksum = self.checksum, cachedChecksum == nil {
                 try checksum.update(chunk: self.chunkBody)
             }
-
             self.chunk = chunk // Store the current chunk
             return true // Chunk processed successfully
         } else {
@@ -86,7 +89,15 @@ public class ChunkedReader {
         // Add a checksum if it is not nil
         if let checksum = self.checksum {
             let headerName = "x-amz-checksum-\(checksum.checksumName)"
-            let hashResult = try checksum.digest().toBase64String()
+            let hashResult: String!
+            // Use cached checksum from previous attempt if present.
+            // Allows request to fail if file payload has changed between retries, increasing durability for S3.
+            if let cachedChecksum {
+                hashResult = cachedChecksum
+            } else {
+                hashResult = try checksum.digest().toBase64String()
+                context.set(key: AttributeKey<String>(name: "ChunkedStreamCachedChecksum"), value: hashResult)
+            }
             self.updateTrailingHeader(name: headerName, value: hashResult)
         }
 

Sources/SmithyChecksums/ChunkedStream.swift

@@ -10,6 +10,7 @@ import struct SmithyHTTPAPI.Headers
 import struct Foundation.Data
 import AwsCommonRuntimeKit
 import class SmithyStreams.BufferedStream
+import class Smithy.Context
 
 /// Reads data from the input stream, chunks it, and passes it out through its output stream.
 ///
@@ -50,20 +51,21 @@ public class ChunkedStream: @unchecked Sendable {
         signingConfig: SigningConfig,
         previousSignature: String,
         trailingHeaders: Headers,
-        checksumString: String? = nil
+        checksumString: String? = nil,
+        context: Smithy.Context
     ) throws {
         self.inputStream = inputStream
         self.signingConfig = signingConfig
         self.previousSignature = previousSignature
         self.trailingHeaders = trailingHeaders
         self.checksumString = checksumString
-
         self.chunkedReader = try ChunkedReader(
             stream: self.inputStream,
             signingConfig: self.signingConfig,
             previousSignature: self.previousSignature,
             trailingHeaders: self.trailingHeaders,
-            checksumString: self.checksumString
+            checksumString: self.checksumString,
+            context: context
         )
     }
 }