Migrate async client to unified checksum options

.github/workflows/codeql-analysis.yml

@@ -72,7 +72,7 @@ jobs:
             --clean-after-build
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@54f647b7e1bb85c95cddabcd46b0c578ec92bc1a # v4
+        uses: github/codeql-action/init@7188fc363630916deb702c7fdcf4e481b751f97a # v4
         with:
           languages: ${{ matrix.language }}
 
@@ -92,4 +92,4 @@ jobs:
           cmake --build build/output
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@54f647b7e1bb85c95cddabcd46b0c578ec92bc1a # v4
+        uses: github/codeql-action/analyze@7188fc363630916deb702c7fdcf4e481b751f97a # v4

ci/bzlmod_tests/consumer-test/MODULE.bazel

@@ -1,5 +1,5 @@
 module(name = "bzlmod-consumer-test")
 
-bazel_dep(name = "google_cloud_cpp", version = "3.6.0")
+bazel_dep(name = "google_cloud_cpp", version = "3.7.0")
 bazel_dep(name = "googletest", version = "1.15.2")
 bazel_dep(name = "rules_cc", version = "0.1.1")

generator/ARCHITECTURE.md

@@ -0,0 +1,171 @@
+# Architecture: Compute Library Generation Pipeline
+
+This document explains the architecture and process used to generate C++ client
+libraries for Google Compute Engine in `google-cloud-cpp` using the Discovery
+Document `compute_public_google_rest_v1.json`.
+
+## Overview
+
+Unlike most Google Cloud services that use gRPC and have official Protobuf
+(`.proto`) service definitions, Google Compute Engine is a REST-only service
+defined via a Google API Discovery Document.
+
+Since the `google-cloud-cpp` generator (the C++ micro-generator) is built on top
+of Protobuf definitions, generating C++ code directly from REST Discovery JSON
+is not possible. To bridge this gap, the team implemented a **two-phase
+generation pipeline**:
+
+```mermaid
+graph TD
+    subgraph Phase 1: Discovery to Proto
+        A[compute_public_google_rest_v1.json] -->|Parse & Analyze| B(Codegen Tool)
+        C[generator_config.textproto] --> B
+        B -->|Generate| D["protos/google/cloud/compute/v1/internal/*.proto"]
+        B -->|Generate| E["google/cloud/compute/*_proto_export.h"]
+    end
+    subgraph Phase 2: Proto to C++
+        D -->|Inputs| F(Codegen Tool as protoc plugin)
+        C --> F
+        F -->|Generate| G["C++ Client Library google/cloud/compute/.../*.{h,cc}"]
+    end
+```
+
+______________________________________________________________________
+
+## Phase 1: Discovery to Proto (REST to Proto Translation)
+
+The first phase translates the REST Discovery JSON document into Protobuf
+definitions (`.proto` files).
+
+### Key Components
+
+- **Discovery Document**:
+  [discovery/compute_public_google_rest_v1.json](discovery/compute_public_google_rest_v1.json)
+  Contains all REST schema and resource definitions.
+- **Generator Configuration**:
+  [generator_config.textproto](generator_config.textproto) Contains a
+  `discovery_products` section that specifies the JSON file URL, operations
+  mappings, and service details.
+- **Generation Tool**: `//generator:google-cloud-cpp-codegen` run with
+  `--generate_discovery_protos=true`.
+
+### Process
+
+1. **Parse JSON & Extract Schemas**: The tool parses
+   `compute_public_google_rest_v1.json` and converts its `schemas` section into
+   `DiscoveryTypeVertex` graph nodes.
+1. **Resource Extraction**: It parses the `resources` section into
+   `DiscoveryResource` representation (methods, path templates, etc.).
+1. **Request/Response Synthesis**: In REST, method parameters are mapped to URI
+   path segments and query parameters. Protobuf, however, expects a single
+   request message. The tool synthesizes a request message for every REST method
+   (e.g. `GetAddressRequest` grouping `project`, `region`, `address` path
+   parameters and `fields` query parameters).
+1. **Dependency Analysis**: The tool traverses the `DiscoveryTypeVertex`
+   dependency graph to establish which types need to import others (i.e. to
+   generate correct `.proto` `import` statements).
+1. **Partitioning and common files**: Since Compute has thousands of types,
+   dumping them in one file is impractical.
+   - Types unique to a resource are generated inside that resource's `.proto`
+     file.
+   - Types shared across multiple resources are grouped into sequential common
+     files (e.g., `google/cloud/compute/v1/internal/common_000.proto`,
+     `common_001.proto`).
+1. **Writing Output**:
+   - Generates `.proto` files inside `protos/google/cloud/compute/v1/internal/`.
+   - Generates C++ export headers (`*_proto_export.h`) in public headers
+     directory (e.g.
+     `google/cloud/compute/addresses/v1/addresses_proto_export.h`). These
+     headers contain `IWYU pragma: export` directives targeting the C++ headers
+     compiled from `common_xxx.proto`, offering a cleaner public API without
+     leaking internal file layout.
+
+______________________________________________________________________
+
+## Phase 2: Proto to C++ (C++ Code Generation)
+
+In this phase, the generated `.proto` files are processed just like regular
+protobufs from `googleapis` to generate the C++ client libraries.
+
+### Key Components
+
+- **Generated Protos**: The outputs of Phase 1 in
+  `protos/google/cloud/compute/v1/internal/`.
+- **Generator Configuration**:
+  [generator_config.textproto](generator_config.textproto) In this phase, the
+  `rest_services` targets within `discovery_products` are processed.
+- **Generation Tool**: `//generator:google-cloud-cpp-codegen` run without
+  `--generate_discovery_protos`. Under the hood, this compiles the generator
+  code as a `protoc` plugin (`google-cloud-cpp-codegen`) and runs it.
+
+### Process
+
+1. The generator parses the `.proto` files using protobuf's
+   CommandLineInterface.
+1. For each configured service (e.g. `Addresses`), it generates:
+   - Public client interface (`addresses_client.h`, `addresses_client.cc`).
+   - Public connection interface (`addresses_connection.h`,
+     `addresses_connection.cc`).
+   - Internal stubs and decorators (`internal/addresses_rest_stub.h`,
+     `internal/addresses_rest_stub.cc`,
+     `internal/addresses_rest_metadata_decorator.cc`, etc.).
+   - Mocks (`mocks/mock_addresses_connection.h`).
+1. Because `generator_config.textproto` configures these services with
+   `generate_rest_transport: true` and `generate_grpc_transport: false`, the
+   generator emits REST-based stubs (`AddressesRestStub` using
+   `google::cloud::rest_internal::RestClient`) instead of gRPC stubs.
+
+______________________________________________________________________
+
+## Orchestration: `update_discovery_doc.sh`
+
+The transition between these phases is managed by the shell script
+[discovery/update_discovery_doc.sh](discovery/update_discovery_doc.sh).
+
+```mermaid
+sequenceDiagram
+    autonumber
+    actor Dev as Developer
+    participant Script as update_discovery_doc.sh
+    participant Web as googleapis.com
+    participant CB as cloudbuild/build.sh
+    participant Config as generator_config.textproto
+    participant Git as Git Repo
+
+    Dev->>Script: Run script
+    Script->>Web: Download latest compute Discovery JSON
+    Web-->>Script: Write to compute_public_google_rest_v1.json
+    Script->>Git: Commit updated Discovery JSON
+    Script->>CB: Run Phase 1: generate-libraries-pr (UPDATED_DISCOVERY_DOCUMENT=compute)
+    Note over CB: Runs Generator with --generate_discovery_protos=true
+    CB-->>Script: Write new .proto files under protos/
+
+    rect rgb(240, 240, 240)
+        Note over Script: If new resource protos are detected:
+        Script->>Config: Add rest_services definition using print_service_textproto
+        Script->>CB: Run Phase 2: generate-libraries-pr (without exit early)
+        Note over CB: Runs Generator to emit C++ code from Protos
+        CB-->>Script: Write C++ client library code under google/cloud/compute/
+        Script->>Script: Update service_dirs.cmake and service_dirs.bzl with new resource directories
+    end
+
+    Script->>Git: Commit updated config & C++ generated code
+```
+
+When a developer runs this script:
+
+1. It pulls the latest discovery document from the web.
+1. It runs the build system in Cloud Build (`generate-libraries-pr`) with
+   `UPDATED_DISCOVERY_DOCUMENT=compute`. This invokes
+   `ci/cloudbuild/builds/generate-libraries.sh`, which performs **Phase 1** and
+   writes the updated `.proto` files, then exits early.
+1. If new `.proto` files are detected:
+   - It dynamically updates `generator/generator_config.textproto` to add a new
+     `rest_services` entry for the new services.
+   - It re-runs the generator build, this time executing **Phase 2** to generate
+     the actual C++ client code.
+   - It inserts the new directories into C++ CMake and Bazel lists
+     (`google/cloud/compute/service_dirs.cmake` and
+     `google/cloud/compute/service_dirs.bzl`).
+1. It commits both the hand-crafted configuration updates and all generated code
+   (protos, C++ code).

generator/integration_tests/tests/golden_kitchen_sink_client_test.cc

@@ -331,9 +331,9 @@ TEST(GoldenKitchenSinkClientTest, AsyncStreamingReadWrite) {
         .WillOnce([] {
           Response response;
           response.set_response("test-only-response");
-          return make_ready_future(absl::make_optional(response));
+          return make_ready_future(std::make_optional(response));
         })
-        .WillOnce([] { return make_ready_future(absl::optional<Response>()); });
+        .WillOnce([] { return make_ready_future(std::optional<Response>()); });
     EXPECT_CALL(*stream, Finish).WillOnce([] {
       return make_ready_future(Status(StatusCode::kUnavailable, "try-again"));
     });

generator/integration_tests/tests/golden_kitchen_sink_option_defaults_test.cc

@@ -33,7 +33,7 @@ using ::google::cloud::golden_v1::GoldenKitchenSinkRetryPolicyOption;
 using ::google::cloud::testing_util::ScopedEnvironment;
 
 TEST(GoldenKitchenSinkDefaultOptions, DefaultEndpoint) {
-  auto env = ScopedEnvironment("GOLDEN_KITCHEN_SINK_ENDPOINT", absl::nullopt);
+  auto env = ScopedEnvironment("GOLDEN_KITCHEN_SINK_ENDPOINT", std::nullopt);
   Options options;
   auto updated_options = GoldenKitchenSinkDefaultOptions(options);
   EXPECT_EQ("goldenkitchensink.googleapis.com",
@@ -57,7 +57,7 @@ TEST(GoldenKitchenSinkDefaultOptions, OptionEndpoint) {
 }
 
 TEST(GoldenKitchenSinkDefaultOptions, UserProjectDefault) {
-  auto env = ScopedEnvironment("GOOGLE_CLOUD_CPP_USER_PROJECT", absl::nullopt);
+  auto env = ScopedEnvironment("GOOGLE_CLOUD_CPP_USER_PROJECT", std::nullopt);
   auto options = Options{};
   auto updated_options = GoldenKitchenSinkDefaultOptions(options);
   EXPECT_FALSE(updated_options.has<UserProjectOption>());
@@ -72,7 +72,7 @@ TEST(GoldenKitchenSinkDefaultOptions, UserProjectEnvVar) {
 }
 
 TEST(GoldenKitchenSinkDefaultOptions, UserProjectOptions) {
-  auto env = ScopedEnvironment("GOOGLE_CLOUD_CPP_USER_PROJECT", absl::nullopt);
+  auto env = ScopedEnvironment("GOOGLE_CLOUD_CPP_USER_PROJECT", std::nullopt);
   auto options = Options{}.set<UserProjectOption>("another-project");
   auto updated_options = GoldenKitchenSinkDefaultOptions(options);
   EXPECT_EQ("another-project", updated_options.get<UserProjectOption>());

generator/integration_tests/tests/golden_thing_admin_client_test.cc

@@ -599,7 +599,7 @@ TEST(GoldenThingAdminClientTest, SetIamPolicyUpdaterCancelled) {
   auto response = client.SetIamPolicy(
       expected_database, [etag_old](::google::iam::v1::Policy const& policy) {
         EXPECT_EQ(etag_old, policy.etag());
-        return absl::nullopt;
+        return std::nullopt;
       });
   ASSERT_THAT(response, Not(IsOk()));
   EXPECT_THAT(response, StatusIs(StatusCode::kCancelled));

generator/integration_tests/tests/mock_golden_kitchen_sink_stub.h

@@ -148,7 +148,7 @@ class MockStreamingReadRpc
           ::google::test::admin::database::v1::Response> {
  public:
   MOCK_METHOD(void, Cancel, (), (override));
-  MOCK_METHOD((absl::optional<Status>), Read,
+  MOCK_METHOD((std::optional<Status>), Read,
               (::google::test::admin::database::v1::Response*), (override));
   MOCK_METHOD(RpcMetadata, GetRequestMetadata, (), (const, override));
 };

generator/internal/descriptor_utils.cc

@@ -154,7 +154,7 @@ bool IsKnownIdempotentMethod(google::protobuf::MethodDescriptor const& m) {
 
 std::string DefaultIdempotencyFromHttpOperation(
     google::protobuf::MethodDescriptor const& method,
-    absl::optional<google::api::HttpRule> http_rule) {
+    std::optional<google::api::HttpRule> http_rule) {
   if (IsKnownIdempotentMethod(method)) return "kIdempotent";
   if (http_rule) {
     switch (http_rule->pattern_case()) {
@@ -495,26 +495,26 @@ std::string GetEffectiveServiceName(VarsDictionary const& vars,
 // If it does not exist, return a null optional.
 // Parses a command line argument in the form:
 // {"service_name_to_comments": "service_a=comment_a,service_b=comment_b"}.
-absl::optional<std::string> GetReplacementComment(VarsDictionary const& vars,
-                                                  absl::string_view name) {
+std::optional<std::string> GetReplacementComment(VarsDictionary const& vars,
+                                                 absl::string_view name) {
   auto service_name_to_comments = vars.find("service_name_to_comments");
   if (service_name_to_comments == vars.end()) {
-    return absl::nullopt;
+    return std::nullopt;
   }
   for (absl::string_view arg :
        absl::StrSplit(service_name_to_comments->second, ',')) {
     std::pair<absl::string_view, absl::string_view> p =
         absl::StrSplit(arg, absl::MaxSplits('=', 1));
     if (p.first == name) return std::string(p.second.data(), p.second.size());
   }
-  return absl::nullopt;
+  return std::nullopt;
 }
 
 VarsDictionary GetMethodVars(
     google::protobuf::ServiceDescriptor const& service,
     YAML::Node const& service_config,
     google::protobuf::MethodDescriptor const& method,
-    absl::optional<google::api::HttpRule> const& http_rule,
+    std::optional<google::api::HttpRule> const& http_rule,
     std::string const& grpc_stub_name,
     VarsDictionary const& idempotency_overrides,
     std::set<std::string> const& omitted_rpcs) {
@@ -892,7 +892,7 @@ std::map<std::string, VarsDictionary> CreateMethodVars(
   std::map<std::string, VarsDictionary> service_methods_vars;
   for (int i = 0; i < service.method_count(); i++) {
     auto const& method = *service.method(i);
-    absl::optional<google::api::HttpRule> http_rule;
+    std::optional<google::api::HttpRule> http_rule;
     if (method.options().HasExtension(google::api::http)) {
       http_rule = method.options().GetExtension(google::api::http);
     }

generator/internal/discovery_resource.cc

@@ -34,7 +34,7 @@ namespace {
 // that we need to introduce additional in the future if we come across other
 // LRO defining conventions.
 // https://cloud.google.com/compute/docs/regions-zones/global-regional-zonal-resources
-absl::optional<std::string> DetermineLongRunningOperationService(
+std::optional<std::string> DetermineLongRunningOperationService(
     nlohmann::json const& method_json, std::vector<std::string> const& params,
     std::set<std::string> const& operation_services,
     std::string const& resource_name) {
@@ -56,7 +56,7 @@ absl::optional<std::string> DetermineLongRunningOperationService(
     }
     return "GlobalOrganizationOperations";
   }
-  return absl::nullopt;
+  return std::nullopt;
 }
 
 }  // namespace

generator/internal/discovery_resource.h

@@ -103,7 +103,7 @@ class DiscoveryResource {
   nlohmann::json json_;
   std::map<std::string, DiscoveryTypeVertex*> request_types_;
   std::map<std::string, DiscoveryTypeVertex*> response_types_;
-  absl::optional<StatusOr<std::string>> service_api_version_;
+  std::optional<StatusOr<std::string>> service_api_version_;
 };
 
 }  // namespace generator_internal

generator/internal/discovery_type_vertex.cc

@@ -21,9 +21,9 @@
 #include "absl/strings/str_join.h"
 #include "absl/strings/str_replace.h"
 #include "absl/strings/str_split.h"
-#include "absl/types/optional.h"
 #include "google/protobuf/descriptor.pb.h"
 #include <cassert>
+#include <optional>
 #include <set>
 
 namespace google {
@@ -33,13 +33,13 @@ namespace {
 
 auto constexpr kInitialFieldNumber = 1;
 
-absl::optional<std::string> CheckForScalarType(nlohmann::json const& j) {
+std::optional<std::string> CheckForScalarType(nlohmann::json const& j) {
   std::string type = j.value("type", "");
   if (type == "string") return "string";
   if (type == "boolean") return "bool";
   if (type == "integer") return j.value("format", "int32");
   if (type == "number") return j.value("format", "float");
-  return absl::nullopt;
+  return std::nullopt;
 }
 
 }  // namespace

generator/internal/format_class_comments.cc

@@ -56,7 +56,7 @@ auto constexpr kFixedClientComment = R"""(
 std::string FormatClassCommentsFromServiceComments(
     google::protobuf::ServiceDescriptor const& service,
     std::string const& service_name,
-    absl::optional<std::string> const& replacement_comment) {
+    std::optional<std::string> const& replacement_comment) {
   google::protobuf::SourceLocation service_source_location;
   std::string formatted_comments;
   // Use the service descriptor to populate the service_source_location.

generator/internal/format_class_comments.h

@@ -37,7 +37,7 @@ namespace generator_internal {
 std::string FormatClassCommentsFromServiceComments(
     google::protobuf::ServiceDescriptor const& service,
     std::string const& service_name,
-    absl::optional<std::string> const& replacement_comment);
+    std::optional<std::string> const& replacement_comment);
 
 }  // namespace generator_internal
 }  // namespace cloud