Add support for Synthetics Private Locations for DDR in dd-sync-cli

datadog_sync/commands/shared/options.py

@@ -280,6 +280,13 @@ def click_config_file_provider(ctx: Context, opts: CustomOptionClass, value: Non
         "Disables progress bar.",
         cls=CustomOptionClass,
     ),
+    option(
+        "--datadog-host-override",
+        envvar=constants.DD_DATADOG_HOST_OVERRIDE,
+        required=False,
+        help="Optional CNAME override for the Datadog host used in DDR private location replication.",
+        cls=CustomOptionClass,
+    ),
 ]
 
 _storage_options = [

datadog_sync/constants.py

@@ -28,6 +28,7 @@
 DD_VERIFY_SSL_CERTIFICATES = "DD_VERIFY_SSL_CERTIFICATES"
 DD_ALLOW_PARTIAL_PERMISSIONS_ROLES = "DD_ALLOW_PARTIAL_PERMISSIONS_ROLES"
 DD_SYNC_JSON = "DD_SYNC_JSON"
+DD_DATADOG_HOST_OVERRIDE = "DD_DATADOG_HOST_OVERRIDE"
 
 LOCAL_STORAGE_TYPE = "local"
 S3_STORAGE_TYPE = "s3"

datadog_sync/model/synthetics_private_locations.py

@@ -4,6 +4,7 @@
 # Copyright 2019 Datadog, Inc.
 
 from __future__ import annotations
+import json
 import re
 
 from typing import TYPE_CHECKING, List, Dict, Optional, Tuple
@@ -14,7 +15,6 @@
 if TYPE_CHECKING:
     from datadog_sync.utils.custom_client import CustomClient
 
-
 class SyntheticsPrivateLocations(BaseResource):
     resource_type = "synthetics_private_locations"
     resource_config = ResourceConfig(
@@ -27,7 +27,9 @@ class SyntheticsPrivateLocations(BaseResource):
             "createdBy",
             "secrets",
             "config",
-            "result_encryption",
+            "ddr_metadata",
+            "pl_id",
+            "public_key_test",
         ],
         tagging_config=TaggingConfig(path="tags"),
         skip_resource_mapping=True,
@@ -48,7 +50,10 @@ async def import_resource(self, _id: Optional[str] = None, resource: Optional[Di
         if not self.pl_id_regex.match(import_id):
             raise SkipResource(import_id, self.resource_type, "Managed location.")
 
-        pl = await source_client.get(self.resource_config.base_path + f"/{import_id}")
+        pl = await source_client.get(
+            self.resource_config.base_path + f"/{import_id}",
+            params={"include_pl_info": "true"},
+        )
         self.config.state.set_source(self.resource_type, import_id, pl)
 
         return import_id, pl
@@ -61,14 +66,23 @@ async def pre_apply_hook(self) -> None:
 
     async def create_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]:
         destination_client = self.config.destination_client
+        source_pl = self.config.state.source[self.resource_type][_id]
 
-        resp = await destination_client.post(self.resource_config.base_path, resource)
+        if resource.get("metadata") is None:
+            resource.pop("metadata", None)
 
-        pl = resp["private_location"]
-        pl["config"] = resp.get("config")
-        pl["result_encryption"] = resp.get("result_encryption")
+        resource["ddr_metadata"] = {
+            "disaster_recovery": {
+                "source_pl_id": source_pl["pl_id"],
+                "source_name": _id,
+            }
+        }
+        resource["test_encryption_public_key"] = json.dumps(source_pl["public_key_test"])
+        if self.config.datadog_host_override:
+            resource["datadog_host_override"] = self.config.datadog_host_override
 
-        return _id, pl
+        resp = await destination_client.post(self.resource_config.base_path, resource)
+        return _id, resp["private_location"]
 
     async def update_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]:
         destination_client = self.config.destination_client

datadog_sync/model/synthetics_tests.py

@@ -80,7 +80,7 @@ class SyntheticsTests(BaseResource):
             "steps": [[]],
         },
         tagging_config=TaggingConfig(path="tags"),
-        skip_resource_mapping=True,
+        resource_mapping_key="metadata.disaster_recovery.source_public_id",
     )
     # Additional SyntheticsTests specific attributes
     browser_test_path: str = "/api/v1/synthetics/tests/browser/{}"
@@ -153,6 +153,17 @@ async def get_resources(self, client: CustomClient) -> List[Dict]:
         self.versions = await versions.get_resources(client)
         return resp["tests"]
 
+    async def map_existing_resources(self) -> None:
+        resp = await self.config.destination_client.get(
+            self.resource_config.base_path,
+            params=self.get_params,
+        )
+        self._existing_resources_map = {}
+        for resource in resp["tests"]:
+            key = self.get_resource_mapping_key(resource)
+            if key is not None:
+                self._existing_resources_map[key] = resource
+
     async def import_resource(self, _id: Optional[str] = None, resource: Optional[Dict] = None) -> Tuple[str, Dict]:
         source_client = self.config.source_client
         if _id:
@@ -322,6 +333,14 @@ async def create_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]:
         test_type = resource["type"]
         resource.pop("mobileApplicationsVersions", None)
 
+        # If a destination test already carries this source_public_id, adopt it into
+        # state and update, this prevents duplicate orphans when a prior sync's POST succeeded
+        # server-side but the client saw a 5xx and never persisted the destination id.
+        existing_key = self.get_resource_mapping_key(resource)
+        if existing_key and existing_key in self._existing_resources_map:
+            self.config.state.destination[self.resource_type][_id] = self._existing_resources_map[existing_key]
+            return await self.update_resource(_id, resource)
+
         # Force status to "paused" for new tests to prevent immediate execution
         # on destination during failover scenarios. Status can be manually changed after creation.
         resource["status"] = "paused"

datadog_sync/utils/configuration.py

@@ -75,6 +75,7 @@ class Configuration(object):
     backup_before_reset: bool
     show_progress_bar: bool
     allow_self_lockout: bool
+    datadog_host_override: Optional[str] = None
     emit_json: bool = False
     command: str = ""
     allow_partial_permissions_roles: List[str] = field(default_factory=list)
@@ -323,6 +324,7 @@ def build_config(cmd: Command, **kwargs: Optional[Any]) -> Configuration:
     if emit_json:
         show_progress_bar = False
     allow_self_lockout = kwargs.get("allow_self_lockout", False)
+    datadog_host_override = kwargs.get("datadog_host_override")
 
     # Parse allow_partial_permissions_roles
     allow_partial_permissions_roles = []
@@ -564,6 +566,7 @@ def build_config(cmd: Command, **kwargs: Optional[Any]) -> Configuration:
         backup_before_reset=backup_before_reset,
         show_progress_bar=show_progress_bar,
         allow_self_lockout=allow_self_lockout,
+        datadog_host_override=datadog_host_override,
         emit_json=emit_json,
         command=cmd.value,
         allow_partial_permissions_roles=allow_partial_permissions_roles,

tests/integration/cassettes/test_cli/TestCli.test_cleanup.yaml

@@ -1,4 +1,164 @@
 interactions:
+- request:
+    body: null
+    headers:
+      Content-Type:
+      - application/json
+    method: GET
+    uri: https://api.us5.datadoghq.com/api/v1/synthetics/tests?include_metadata=true
+  response:
+    body:
+      string: '{"tests": []}'
+    headers:
+      Content-Type:
+      - application/json
+    status:
+      code: 200
+      message: OK
+- request:
+    body: null
+    headers:
+      Content-Type:
+      - application/json
+    method: GET
+    uri: https://api.us5.datadoghq.com/api/v1/synthetics/tests?include_metadata=true
+  response:
+    body:
+      string: '{"tests": []}'
+    headers:
+      Content-Type:
+      - application/json
+    status:
+      code: 200
+      message: OK
+- request:
+    body: null
+    headers:
+      Content-Type:
+      - application/json
+    method: GET
+    uri: https://api.us5.datadoghq.com/api/v1/synthetics/tests?include_metadata=true
+  response:
+    body:
+      string: '{"tests": []}'
+    headers:
+      Content-Type:
+      - application/json
+    status:
+      code: 200
+      message: OK
+- request:
+    body: null
+    headers:
+      Content-Type:
+      - application/json
+    method: GET
+    uri: https://api.us5.datadoghq.com/api/v1/synthetics/tests?include_metadata=true
+  response:
+    body:
+      string: '{"tests": []}'
+    headers:
+      Content-Type:
+      - application/json
+    status:
+      code: 200
+      message: OK
+- request:
+    body: null
+    headers:
+      Content-Type:
+      - application/json
+    method: GET
+    uri: https://api.us5.datadoghq.com/api/v1/synthetics/tests?include_metadata=true
+  response:
+    body:
+      string: '{"tests": []}'
+    headers:
+      Content-Type:
+      - application/json
+    status:
+      code: 200
+      message: OK
+- request:
+    body: null
+    headers:
+      Content-Type:
+      - application/json
+    method: GET
+    uri: https://api.us5.datadoghq.com/api/v1/synthetics/tests?include_metadata=true
+  response:
+    body:
+      string: '{"tests": []}'
+    headers:
+      Content-Type:
+      - application/json
+    status:
+      code: 200
+      message: OK
+- request:
+    body: null
+    headers:
+      Content-Type:
+      - application/json
+    method: GET
+    uri: https://api.us5.datadoghq.com/api/v1/synthetics/tests?include_metadata=true
+  response:
+    body:
+      string: '{"tests": []}'
+    headers:
+      Content-Type:
+      - application/json
+    status:
+      code: 200
+      message: OK
+- request:
+    body: null
+    headers:
+      Content-Type:
+      - application/json
+    method: GET
+    uri: https://api.us5.datadoghq.com/api/v1/synthetics/tests?include_metadata=true
+  response:
+    body:
+      string: '{"tests": []}'
+    headers:
+      Content-Type:
+      - application/json
+    status:
+      code: 200
+      message: OK
+- request:
+    body: null
+    headers:
+      Content-Type:
+      - application/json
+    method: GET
+    uri: https://api.us5.datadoghq.com/api/v1/synthetics/tests?include_metadata=true
+  response:
+    body:
+      string: '{"tests": []}'
+    headers:
+      Content-Type:
+      - application/json
+    status:
+      code: 200
+      message: OK
+- request:
+    body: null
+    headers:
+      Content-Type:
+      - application/json
+    method: GET
+    uri: https://api.us5.datadoghq.com/api/v1/synthetics/tests?include_metadata=true
+  response:
+    body:
+      string: '{"tests": []}'
+    headers:
+      Content-Type:
+      - application/json
+    status:
+      code: 200
+      message: OK
 - request:
     body: null
     headers: