feat(storage): Make PrecomputedChecksumsOption compatible with Options

google/cloud/storage/client.cc

@@ -140,10 +140,17 @@ ObjectWriteStream Client::WriteObjectImpl(
       connection_, request, std::move(response->upload_id),
       response->committed_size, std::move(response->metadata), buffer_size,
       internal::CreateHashFunction(request),
-      internal::HashValues{
-          request.GetOption<Crc32cChecksumValue>().value_or(""),
-          request.GetOption<MD5HashValue>().value_or(""),
-      },
+      [&]() {
+        auto crc = request.GetOption<Crc32cChecksumValue>().value_or("");
+        auto md5 = request.GetOption<MD5HashValue>().value_or("");
+        if ((crc.empty() || md5.empty()) &&
+            current.has<PrecomputedChecksumsOption>()) {
+          auto const& checksums = current.get<PrecomputedChecksumsOption>();
+          if (crc.empty()) crc = checksums.crc32c;
+          if (md5.empty()) md5 = checksums.md5;
+        }
+        return internal::HashValues{std::move(crc), std::move(md5)};
+      }(),
       internal::CreateHashValidator(request),
       request.GetOption<AutoFinalize>().value_or(
           AutoFinalizeConfig::kEnabled)));

google/cloud/storage/hashing_options.h

@@ -38,6 +38,17 @@ GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN
  * @see
  * https://sigops.org/s/conferences/hotos/2021/papers/hotos21-s01-hochschild.pdf
  */
+
+/**
+ * A structure to hold precomputed hashes.
+ *
+ * @ingroup storage-options
+ */
+struct PrecomputedChecksums {
+  std::string crc32c;
+  std::string md5;
+};
+
 struct MD5HashValue
     : public internal::ComplexOption<MD5HashValue, std::string> {
   using ComplexOption<MD5HashValue, std::string>::ComplexOption;

google/cloud/storage/internal/hash_function.cc

@@ -32,17 +32,24 @@ std::unique_ptr<HashFunction> CreateHashFunction(
     Crc32cChecksumValue const& crc32c_value,
     DisableCrc32cChecksum const& crc32c_disabled, MD5HashValue const& md5_value,
     DisableMD5Hash const& md5_disabled) {
+  auto const& options = google::cloud::internal::CurrentOptions();
   auto crc32c = std::unique_ptr<HashFunction>();
   auto crc32c_v = crc32c_value.value_or("");
+  auto md5 = std::unique_ptr<HashFunction>();
+  auto md5_v = md5_value.value_or("");
+
+  if ((crc32c_v.empty() || md5_v.empty()) &&
+      options.has<PrecomputedChecksumsOption>()) {
+    auto const& checksums = options.get<PrecomputedChecksumsOption>();
+    if (crc32c_v.empty()) crc32c_v = checksums.crc32c;
+    if (md5_v.empty()) md5_v = checksums.md5;
+  }
   if (!crc32c_v.empty()) {
     crc32c = std::make_unique<PrecomputedHashFunction>(
         HashValues{/*.crc32c=*/std::move(crc32c_v), /*md5=*/{}});
   } else if (!crc32c_disabled.value_or(false)) {
     crc32c = std::make_unique<Crc32cHashFunction>();
   }
-
-  auto md5 = std::unique_ptr<HashFunction>();
-  auto md5_v = md5_value.value_or("");
   if (!md5_v.empty()) {
     md5 = std::make_unique<PrecomputedHashFunction>(
         HashValues{/*.crc32c=*/{}, /*.md5=*/std::move(md5_v)});

google/cloud/storage/internal/hash_function_impl_test.cc

@@ -18,6 +18,7 @@
 #include "google/cloud/storage/internal/object_requests.h"
 #include "google/cloud/storage/testing/mock_hash_function.h"
 #include "google/cloud/storage/testing/upload_hash_cases.h"
+#include "google/cloud/options.h"
 #include "google/cloud/testing_util/status_matchers.h"
 #include <gmock/gmock.h>
 #include <memory>
@@ -450,10 +451,48 @@ TEST(HashFunctionImplTest, CreateHashFunctionInsertObjectMedia) {
   }
 }
 
+TEST(HashFunctionImplTest, CreateHashFunctionPrecomputedChecksumsOption) {
+  google::cloud::internal::OptionsSpan span(
+      google::cloud::Options{}.set<PrecomputedChecksumsOption>(
+          PrecomputedChecksums{"crc-from-options", "md5-from-options"}));
+
+  ResumableUploadRequest request("bucket", "object");
+  auto function = CreateHashFunction(request);
+  EXPECT_EQ(function->Finish().crc32c, "crc-from-options");
+  EXPECT_EQ(function->Finish().md5, "md5-from-options");
+}
+
+TEST(HashFunctionImplTest,
+     CreateHashFunctionPrecomputedChecksumsOptionPartial) {
+  google::cloud::internal::OptionsSpan span(
+      google::cloud::Options{}.set<PrecomputedChecksumsOption>(
+          PrecomputedChecksums{"crc-from-options", ""}));
+
+  ResumableUploadRequest request("bucket", "object");
+  auto function = CreateHashFunction(request);
+  EXPECT_EQ(function->Finish().crc32c, "crc-from-options");
+  // MD5 is disabled by default for uploads, so it will be empty
+  EXPECT_EQ(function->Finish().md5, "");
+}
+
+TEST(HashFunctionImplTest, CreateHashFunctionPrecedence) {
+  google::cloud::internal::OptionsSpan span(
+      google::cloud::Options{}.set<PrecomputedChecksumsOption>(
+          PrecomputedChecksums{"crc-from-options", "md5-from-options"}));
+
+  ResumableUploadRequest request("bucket", "object");
+  request.set_multiple_options(Crc32cChecksumValue("crc-from-request"),
+                               MD5HashValue("md5-from-request"));
+  auto function = CreateHashFunction(request);
+  // The variadic options provided directly to the request should take
+  // precedence
+  EXPECT_EQ(function->Finish().crc32c, "crc-from-request");
+  EXPECT_EQ(function->Finish().md5, "md5-from-request");
+}
+
 }  // namespace
 }  // namespace internal
 GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
 }  // namespace storage
 }  // namespace cloud
 }  // namespace google
-#include "google/cloud/internal/diagnostics_pop.inc"

google/cloud/storage/internal/rest/stub.cc

@@ -463,7 +463,8 @@ StatusOr<ObjectMetadata> RestStub::InsertObjectMedia(
   auto const settings =
       storage::internal::GetUploadChecksumSettings(request, options);
   if (!settings.md5 || !settings.crc32c || request.HasOption<MD5HashValue>() ||
-      request.HasOption<Crc32cChecksumValue>()) {
+      request.HasOption<Crc32cChecksumValue>() ||
+      options.has<PrecomputedChecksumsOption>()) {
     return InsertObjectMediaMultipart(context, options, request);
   }
 
@@ -718,6 +719,17 @@ StatusOr<CreateResumableUploadResponse> RestStub::CreateResumableUpload(
     resource["md5Hash"] = request.GetOption<MD5HashValue>().value();
   }
 
+  if (options.has<PrecomputedChecksumsOption>()) {
+    auto const& checksums = options.get<PrecomputedChecksumsOption>();
+    if (!request.HasOption<Crc32cChecksumValue>() &&
+        !checksums.crc32c.empty()) {
+      resource["crc32c"] = checksums.crc32c;
+    }
+    if (!request.HasOption<MD5HashValue>() && !checksums.md5.empty()) {
+      resource["md5Hash"] = checksums.md5;
+    }
+  }
+
   if (resource.empty()) {
     builder.AddQueryParameter("name", request.object_name());
   } else {

google/cloud/storage/options.h

@@ -15,6 +15,7 @@
 #ifndef GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_OPTIONS_H
 #define GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_OPTIONS_H
 
+#include "google/cloud/storage/hashing_options.h"
 #include "google/cloud/storage/idempotency_policy.h"
 #include "google/cloud/storage/retry_policy.h"
 #include "google/cloud/storage/version.h"
@@ -121,6 +122,19 @@ struct DownloadChecksumValidationOption {
   using Type = ChecksumAlgorithm;
 };
 
+/**
+ * Provide precomputed hashes for uploads and downloads.
+ *
+ * If set, the client will use these precomputed hashes instead of computing
+ * them locally. This is useful when the application has already computed the
+ * hash and wants to avoid recomputing it.
+ *
+ * @ingroup storage-options
+ */
+struct PrecomputedChecksumsOption {
+  using Type = PrecomputedChecksums;
+};
+
 /**
  * Configure the REST endpoint for the GCS client library.
  *
@@ -377,7 +391,7 @@ using ClientOptionList = ::google::cloud::OptionList<
     TransferStallTimeoutOption, RetryPolicyOption, BackoffPolicyOption,
     IdempotencyPolicyOption, CARootsFilePathOption,
     UploadChecksumValidationOption, DownloadChecksumValidationOption,
-    storage_experimental::HttpVersionOption,
+    PrecomputedChecksumsOption, storage_experimental::HttpVersionOption,
     storage_experimental::OTelSpanEnrichmentOption>;
 
 GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END

google/cloud/storage/parallel_upload.cc

@@ -43,10 +43,19 @@ class ParallelObjectWriteStreambuf : public ObjectWriteStreambuf {
             std::move(connection), request, std::move(upload_id),
             committed_size, std::move(metadata), max_buffer_size,
             CreateHashFunction(request),
-            internal::HashValues{
-                request.GetOption<Crc32cChecksumValue>().value_or(""),
-                request.GetOption<MD5HashValue>().value_or(""),
-            },
+            [&request]() {
+              auto const& current = google::cloud::internal::CurrentOptions();
+              auto crc = request.GetOption<Crc32cChecksumValue>().value_or("");
+              auto md5 = request.GetOption<MD5HashValue>().value_or("");
+              if ((crc.empty() || md5.empty()) &&
+                  current.has<PrecomputedChecksumsOption>()) {
+                auto const& checksums =
+                    current.get<PrecomputedChecksumsOption>();
+                if (crc.empty()) crc = checksums.crc32c;
+                if (md5.empty()) md5 = checksums.md5;
+              }
+              return internal::HashValues{std::move(crc), std::move(md5)};
+            }(),
             CreateHashValidator(request), AutoFinalizeConfig::kEnabled),
         state_(std::move(state)),
         stream_idx_(stream_idx) {}