diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 59ee9e34..0c4d9c96 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,9 @@ # This product includes software developed at Datadog # (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. +# Validate supported hosts and generated consumer artifacts before merge. +# The uploader smoke catches launcher/runfiles failures that imports cannot expose. + name: CI on: @@ -126,6 +129,18 @@ jobs: shell: bash run: ./bazelw test //tools/... --test_output=errors + - name: Smoke default Python uploader launcher + timeout-minutes: 5 + shell: bash + run: | + ./bazelw run \ + //tools/tests/python/fixtures/generated_uploader:generated_uploader \ + -- \ + --dry-run \ + --allow-cached-payload-uploads \ + --workers=3 \ + --debug + - name: Build examples timeout-minutes: 20 shell: bash @@ -238,6 +253,18 @@ jobs: shell: bash run: ./bazelw test //tools/... --test_output=errors + - name: Smoke default Python uploader launcher + timeout-minutes: 5 + shell: bash + run: | + ./bazelw run \ + //tools/tests/python/fixtures/generated_uploader:generated_uploader \ + -- \ + --dry-run \ + --allow-cached-payload-uploads \ + --workers=3 \ + --debug + - name: Run Bazel tests (go companion module) shell: bash run: | diff --git a/README.md b/README.md index 34a10c50..32f47739 100644 --- a/README.md +++ b/README.md @@ -1349,17 +1349,26 @@ For a generic wrapper pattern, see [Other languages (without companion macro)](# - **Tracer/runtime with DD Test Optimization file-mode support** - Must honor `DD_TEST_OPTIMIZATION_MANIFEST_FILE` and `DD_TEST_OPTIMIZATION_PAYLOADS_IN_FILES` - **rules_go v0.51.0+** (for Go importpath inference) - This repository reads `GoInfo`/`GoArchive` providers when selecting per-module payloads - **DD_SITE format** - Accepts bare host, app/api-prefixed host, or full URL; leading/trailing ASCII whitespace is trimmed, then normalized to `https://api.` -- **Uploader tooling (per platform)** - Required for `bazel run //:dd_upload_payloads` - - **Linux**: `bash`, `curl`, `find`, `stat` (GNU), `awk`, and one of `md5sum` or `shasum` - - **macOS**: `bash` (3.2+), `curl`, `find`, `stat` (BSD), `awk`, and one of `md5` or `shasum` - - **Windows**: `powershell.exe` (Windows PowerShell 5.1+ or PowerShell 7+); the uploader uses .NET `HttpClient` and is intentionally PowerShell-only (no Git Bash dependency) +- **Default uploader tooling** - `Python 3.10+` on every platform, plus `bash` + on Linux/macOS or `powershell.exe` (Windows PowerShell 5.1+ or PowerShell 7+) + on Windows. The platform script only locates Python and Bazel runfiles; upload + behavior is shared. +- **Temporary legacy rollback tooling** - Required only when + `use_python_uploader = False`: Linux needs `bash`, `curl`, `find`, GNU `stat`, + `awk`, and one of `md5sum` or `shasum`; macOS needs `bash` 3.2+, `curl`, + `find`, BSD `stat`, `awk`, and one of `md5` or `shasum`; Windows needs + `powershell.exe` and uses .NET `HttpClient`. Optional tooling: -- **jq** (Linux/macOS) - Used to enrich test payloads with `context.json` and - split oversized test payloads. If missing, uploads proceed without enrichment; - payloads up to the 5,000,000-byte intake limit can still be sent intact, but - larger payloads fail because their `events` array cannot be partitioned. -- **python3** - Used for uploader payload schema validation and Unix telemetry metadata extraction. If missing, schema validation is skipped and telemetry files fail individually with a warning. +- **jq** (Linux/macOS, legacy rollback only) - Used by the legacy uploader to + enrich test payloads with `context.json` and split oversized test payloads. If + missing, legacy uploads proceed without enrichment; payloads up to the + 5,000,000-byte intake limit can still be sent intact, but larger payloads fail + because their `events` array cannot be partitioned. + +See the [parallel Python uploader](docs/Uploader_Reference.md#parallel-python-uploader-default) +for its shared cross-platform behavior. Set `use_python_uploader = False` only +for temporary rollback to the legacy Bash or PowerShell implementation. ### Contract gate checklist diff --git a/docs/Configuration_Reference.md b/docs/Configuration_Reference.md index ec4a9a91..99e286ef 100644 --- a/docs/Configuration_Reference.md +++ b/docs/Configuration_Reference.md @@ -332,6 +332,8 @@ Rule: `dd_payload_uploader(...)` | `keep_payloads` | bool | `False` | Keep payload files after successful upload | | `filter_prefix` | bool | `False` | Only upload files matching `span_events_*.json` or `coverage_*.json` | | `gzip_payloads` | bool | `False` | Gzip test payloads before upload | +| `workers` | int | `8` | Maximum independent payload-file workers in Python mode; `DD_TEST_OPTIMIZATION_WORKERS` and `--workers` override it at runtime | +| `use_python_uploader` | bool | `True` | Use the default cross-platform Python uploader; set to `False` only for temporary rollback to the legacy platform runtime | | `data` | label_list | `[]` | Data files to include (for example, `context.json` for enrichment) | | `expected_targets` | string_list | `[]` | Optional exact local labels expected in the matching BEP. Fresh and cached results jointly satisfy coverage; missing results are reported while other fresh outputs continue to upload | | `expected_targets_file` | label | unset | Optional schema-v1 exact-target file. Static and file inputs must match when both are non-empty; missing results do not block other fresh uploads | @@ -659,6 +661,7 @@ The doctor and/or uploader runtimes read these variables at `bazel run` time: | `DD_TEST_OPTIMIZATION_FILTER_PREFIX` | `0` uploads all payloads; `1` restricts to `span_events_*.json` / `coverage_*.json` | | `DD_TEST_OPTIMIZATION_DEBUG` | Enable verbose uploader logs | | `DD_TEST_OPTIMIZATION_GZIP` | Gzip test payloads before upload | +| `DD_TEST_OPTIMIZATION_WORKERS` | Override the maximum independent payload-file workers in Python mode; must be a positive integer | | `DD_TEST_OPTIMIZATION_MAX_WAIT_SEC` | Override uploader max wait | | `DD_TEST_OPTIMIZATION_QUIESCENT_SEC` | Override uploader quiescence wait | | `DD_TEST_OPTIMIZATION_MAX_DEPTH` | Limit payload discovery depth in large trees | @@ -671,7 +674,7 @@ The doctor and/or uploader runtimes read these variables at `bazel run` time: | `DD_TEST_OPTIMIZATION_REPORT_DIR` | Optional wrapper/report-script directory. CI wrappers write `doctor-report.json` plus `uploader-dry-run-report.json` without upload or `uploader-upload-report.json` with upload unless explicit report paths override them | | `DD_TEST_OPTIMIZATION_SUPPORT_BUNDLE` | Optional doctor or wrapper path for the redacted support diagnostics zip | | `DD_TEST_OPTIMIZATION_SUPPORT_BUNDLE_COLLECTOR` | Optional override for the support bundle collector script. Doctor targets provide this through runfiles; wrappers default to `create_support_bundle.py` beside the wrapper | -| `DD_TEST_OPTIMIZATION_PYTHON` | Optional Python interpreter used by wrapper support-bundle generation and helper scripts before falling back to `PYTHON`, `python3`, and `python` | +| `DD_TEST_OPTIMIZATION_PYTHON` | Python interpreter override before falling back to `PYTHON`, `python3`, and `python`; Python 3.10+ is required by the default uploader and optional only when `use_python_uploader = False` selects the legacy runtime | | `DD_TEST_OPTIMIZATION_ARTIFACT_SOURCE` | Artifact discovery source: `local`, `bep`, or `auto`. Recommended CI with zipped undeclared outputs should set `bep` | | `DD_TEST_OPTIMIZATION_REMOTE_ARTIFACTS` | Remote BEP artifact handling: `disabled`, `download`, or `required`. HTTP/HTTPS `outputs.zip` carriers can be staged natively when enabled | | `DD_TEST_OPTIMIZATION_ARTIFACT_STAGING_DIR` | Directory used for per-run staged BEP artifacts | @@ -687,6 +690,7 @@ Uploader CLI flags: |------|---------| | `--dry-run` | Enrich and validate discovered payloads without uploading or deleting files | | `--validate-enrichment` | Require key Git and Bazel tags to exist after enrichment, before either validation-only completion or upload | +| `--workers=` | Override the maximum independent payload-file workers in Python mode; takes precedence over `DD_TEST_OPTIMIZATION_WORKERS` and the rule attribute | | `--expected-enriched-tag=` | Add a required enriched tag; repeatable. Defaults cover `git.repository_url`, `git.commit.sha`, `bazel.target`, and `bazel.package`. Add `bazel.go.payload_selection` explicitly when a Go rollout must prove per-module selection | | `--bep-json=` | BEP JSON file from the matching Bazel test invocation; repeat for multiple invocations | | `--freshness-source=` | Freshness source: `auto`, `bep`, or `execution_log` | diff --git a/docs/Troubleshooting.md b/docs/Troubleshooting.md index a7d45cb9..a1826b33 100644 --- a/docs/Troubleshooting.md +++ b/docs/Troubleshooting.md @@ -46,6 +46,8 @@ in this release. | Doctor reports msgpack payloads | tracer is not in Bazel JSON file mode | Doctor failures | | Doctor reports missing Git or Bazel metadata | sync metadata context or sidecar metadata is absent | Doctor failures | | Uploaded tests miss Git or Bazel tags | run uploader dry-run enrichment validation | Uploader enrichment dry-run | +| Python uploader cannot start | Python 3.10+ discovery and override | Python uploader cannot resolve Python | +| Test payload split or `413` failure | enriched size, individual event size, fixed threshold | Python uploader split and `413` failures | | Upload network errors | credential mode (agentless vs EVP), intake reachability | Tests not uploading (network errors) | | CI failure requires log archaeology | archive the support bundle from the failing run | Collect diagnostic reports | | Module selection misses | `bazel query` for `module_*` targets and importpath/module label expectations | Per-module files not found | @@ -743,9 +745,70 @@ prove per-module selection. If this fails: 1. Ensure the uploader target has the right `data = ["@...//:test_optimization_context"]`. 2. Ensure `bazel_target_metadata.json` exists beside the payloads. -3. Ensure `jq` is available on Linux/macOS when using `--validate-enrichment`. +3. When using the temporary legacy Bash uploader, ensure `jq` is available on + Linux/macOS. Python uploader mode does not require `jq`. 4. Use `--expected-enriched-tag=` for repository-specific required tags. +## Python uploader cannot resolve Python + +**Symptom**: The generated uploader prints `Python 3.10 or newer was not found` +or reports that the discovered interpreter is older than 3.10. + +**Solution**: + +1. Confirm the uploader target does not explicitly set + `use_python_uploader = False`; that value selects the temporary legacy + rollback path. +2. Set `DD_TEST_OPTIMIZATION_PYTHON` to an explicit Python 3.10+ executable. + `PYTHON`, `python3`, and `python` are tried afterward, in that order. +3. On Windows, set the environment variable before invoking `bazel run`: + ```powershell + $env:DD_TEST_OPTIMIZATION_PYTHON = "C:\Python312\python.exe" + bazel run //:dd_upload_payloads -- --dry-run + ``` +4. On Linux/macOS: + ```bash + DD_TEST_OPTIMIZATION_PYTHON=/usr/local/bin/python3.12 \ + bazel run //:dd_upload_payloads -- --dry-run + ``` + +The launcher only resolves runfiles, the interpreter, and the generated config; +all enrichment and upload behavior remains in the shared Python runtime. + +## Python uploader split and `413` failures + +Python mode calculates the compact UTF-8 size after enrichment and before any +HTTP request. Test bodies larger than `4_718_592` bytes (4.5 MiB) are split +deterministically by event while preserving event order and all non-event +top-level fields. + +- `single_event_exceeds_payload_limit`: one enriched event cannot fit by + itself. Reduce the producing tracer/event metadata; increasing or overriding + the uploader threshold is intentionally unsupported. +- `test_payload_not_json_serializable` or `invalid_test_json`: the source or an + enriched value is not standards-compliant JSON. In particular, `NaN` and + positive/negative `Infinity` are rejected. +- `payload_limit_contract_mismatch` after HTTP `413`: every prepared chunk was + already at or below 4.5 MiB, so the backend limit or request contract differs + from the uploader contract. The uploader retains the source, stops later + chunks, does not retry `413`, and does not split adaptively. + +Reproduce preparation without backend traffic or deletion: + +```bash +bazel run //:dd_upload_payloads -- \ + --dry-run --allow-cached-payload-uploads --debug +``` + +```powershell +bazel run //:dd_upload_payloads -- ` + --dry-run --allow-cached-payload-uploads --debug +``` + +Inspect the final `split`, `requests`, and `cleanup` statistics and, when +configured, the schema-v1 JSON report. Do not retry a retained `413` payload +unchanged; first reconcile the configured 4.5 MiB contract with the backend. + ## Non-standard bazel-testlogs location **Symptom**: Uploader cannot find the `bazel-testlogs` directory. diff --git a/docs/Uploader_Reference.md b/docs/Uploader_Reference.md index 360cb0c7..21d8edf7 100644 --- a/docs/Uploader_Reference.md +++ b/docs/Uploader_Reference.md @@ -186,6 +186,36 @@ and runtime identity must agree. Missing, duplicate, mismatched, or incomplete runtime selections fail before payload discovery, cleanup, enrichment, or network access. +### Parallel Python uploader (default) + +The cross-platform Python uploader is the default. Use `uploader_kwargs` only +when choosing a non-default number of independent payload-file workers: + +```bzl +dd_test_optimization_targets( + name = "test_optimization", + sync_repo_name = "test_optimization_data", + uploader_kwargs = { + "workers": 8, + }, +) +``` + +This mode requires a host Python 3.10 or newer on Linux, macOS, and Windows at +`bazel run` time. The launcher resolves `DD_TEST_OPTIMIZATION_PYTHON`, then +`PYTHON`, then `python3`, then `python`. Each worker owns one payload file and +performs its enrichment, validation, preflight split at the conservative +`4_718_592`-byte (4.5 MiB) threshold, and upload/retries independently. Every +worker can process test, coverage, or telemetry payloads; chunks and other +derived requests belonging to one source remain sequential. Test and telemetry +JSON reject non-standard numbers (`NaN` and positive/negative `Infinity`) before +HTTP; coverage bodies remain opaque JSON/msgpack multipart parts. The +rule-level `workers` value defaults to `8`; `DD_TEST_OPTIMIZATION_WORKERS` +overrides it at runtime and `--workers=` has highest +precedence. Leave +`use_python_uploader` unset for the Python implementation. Set it to `False` +only for temporary rollback to the legacy Bash or PowerShell implementation. + If your repository is small, the same helper can live in the root package. In large monorepos, prefer `//tools/test_optimization` or another lightweight package to avoid loading unrelated root package wiring when running doctor or @@ -295,6 +325,7 @@ bazel run --config=test-optimization //:dd_upload_payloads | `DD_TEST_OPTIMIZATION_FILTER_PREFIX` | `0` | `0` uploads all payload files; set to `1` to only upload `span_events_*.json` or `coverage_*.json` | | `DD_TEST_OPTIMIZATION_DEBUG` | `0` | Set to `1` to enable verbose attempt, success, startTime, and runfile/CODEOWNERS resolution logging. Terminal test-upload failures always report the HTTP status, a bounded response body, and payload sizes. | | `DD_TEST_OPTIMIZATION_GZIP` | `0` | Set to `1` to gzip test payloads before upload (adds `Content-Encoding: gzip`) | +| `DD_TEST_OPTIMIZATION_WORKERS` | rule `workers` (`8`) | Override the maximum independent payload-file workers in Python mode; `--workers` has higher precedence | | `DD_TEST_OPTIMIZATION_MAX_WAIT_SEC` | `300` | Override max wait time for slow filesystems (NFS, network drives); set to `0` to skip waiting when no payloads are present | | `DD_TEST_OPTIMIZATION_QUIESCENT_SEC` | `10` | Override quiescence wait time | | `DD_TEST_OPTIMIZATION_MAX_DEPTH` | `0` (unlimited) | Limit `find` depth for large `bazel-testlogs` trees | @@ -679,11 +710,44 @@ payload discovery/quiescence before proceeding. ## Reliability -- HTTP requests use a 60-second timeout +### Python uploader + +- Python mode uses a 10-second connection timeout and a 60-second socket-I/O + timeout. +- Each logical request makes at most four total attempts: the initial attempt + plus up to three retries, normally separated by 2 seconds. +- Connection failures, timeouts, HTTP `408`, HTTP `429`, and HTTP `5xx` are + retryable. `Retry-After` is honored when the backend supplies it, with a + 60-second safety cap per retry so a response cannot stall a worker + indefinitely. +- Other HTTP `4xx` responses are terminal after the first attempt. For test + payloads, `413` indicates that the preventive split contract was violated. + Coverage and telemetry are not split, so their `413` failures use the + `upload_http_413` reason instead. No `413` is retried or triggers adaptive + splitting. +- JSON, gzip, telemetry, and multipart bodies are prepared once per logical + request and replayed byte-for-byte for every retry. +- Split test chunks are uploaded in event order and fail independently. After + partial success, the source is atomically replaced with only the failed + chunks so accepted events are not replayed; if every chunk fails or the + replacement cannot be written, the original source is retained. +- A telemetry source and its synthetic rule-facts request are also attempted + independently. After partial success, only the rejected prepared request is + retained for the next invocation, without regenerating augmentation or + replaying the accepted request. If retention fails, the original source is + kept and the final result reports a warning. +- The temporary legacy Bash/curl and PowerShell implementations remain + available only as an explicit opt-out rollback during the rollout window. + The normalized policy above is the default uploader contract on Linux, + macOS, and Windows. + +### Legacy Bash and PowerShell uploaders + +- HTTP requests use a 60-second timeout. - Failed requests are retried up to 3 times with a 2-second delay between - attempts + attempts. - Both transient errors (connection issues) and HTTP errors (4xx/5xx) trigger - retries + retries. - After enrichment, test payloads larger than 4,500,000 bytes are split by their top-level `events` array before compression and transport. Each part preserves the original top-level envelope and remains at or below the split @@ -705,7 +769,7 @@ payload discovery/quiescence before proceeding. characters of the response body, and the uncompressed, compressed, and transmitted byte counts. Response logging does not require debug mode. - Behavior is consistent across Linux/macOS (bash/curl) and Windows - (PowerShell-only runtime path; no Git Bash requirement) + (PowerShell-only runtime path; no Git Bash requirement). ## Metadata enrichment (`context.json`) @@ -716,9 +780,12 @@ payload discovery/quiescence before proceeding. 4. if multiple bundled contexts exist and no match is found, skip only the `context.json` merge for that payload and continue uploading 5. if no bundled context resolves, upload without context enrichment - When a `context.json` file is available, the uploader enriches each test - payload by merging all non-null keys from `context.json` into each event's - `content.meta` or `content.metrics`, and it also normalizes top-level - `metadata.*` runtime tags. + payload by merging its keys into each event's `content.meta` or + `content.metrics`, and it also normalizes top-level `metadata.*` runtime + tags. Numbers become metrics; strings and booleans become meta values; other + JSON values use their compact JSON representation. For legacy compatibility, + JSON `null` therefore becomes the meta string `"null"` rather than being + omitted. - Bazel sidecar metadata from `bazel_target_metadata.json` is merged separately. If a multi-context payload has no repo match, those Bazel sidecar tags remain and only the `context.json` merge is skipped. diff --git a/docs/python_parallel_uploader_migration_plan.md b/docs/python_parallel_uploader_migration_plan.md new file mode 100644 index 00000000..d6b3d646 --- /dev/null +++ b/docs/python_parallel_uploader_migration_plan.md @@ -0,0 +1,611 @@ + + +# Python Parallel Uploader: Implementation Guide and Tracker + +## Status + +The Python uploader is the default. The temporary `use_python_uploader = False` +rule value keeps the legacy Bash and PowerShell runtimes available as an +explicit rollback while the remaining removal gates in this document close. + +The implementation currently provides: + +- one dependency-free Python 3.10+ runtime for Linux, macOS, and Windows; +- a bounded pool of homogeneous file workers; +- one complete `enrich -> validate -> split -> upload -> cleanup` pipeline per + test file; +- equivalent complete pipelines for coverage and telemetry files; +- CODEOWNERS, context, schema, freshness, and telemetry planning before workers + start; +- preventive test-payload splitting at `4_718_592` bytes; +- retries, dry-run, debug logs, and final human/JSON statistics; +- small platform launchers that only resolve Python, config, and the bootstrap. + +This is an implementation milestone, not the end of the migration. Open +checkboxes are release gates, not deferred product ideas. + +## Goals + +Replace two functional uploader implementations with one portable runtime while +preserving the public uploader contract. The design must: + +1. Process independent source files concurrently. +2. Give one worker exclusive ownership of a file from dequeue through cleanup. +3. Let every worker process test, coverage, and telemetry payloads. +4. Parse CODEOWNERS and other invocation-wide inputs once before worker startup. +5. Split an enriched test body before HTTP when it exceeds 4.5 MiB. +6. Keep chunks from one source ordered, attempt every chunk independently, + and retain only failed chunks after a partial upload. +7. Retry only failures that can plausibly succeed without changing the body. +8. Exercise full preparation in dry-run without HTTP or source deletion. +9. Always produce useful terminal statistics for controlled completion. +10. Prefer standard-library code and explicit data flow over framework or queue + abstractions that are not required by the contract. + +## Non-goals + +- Parallelizing chunks from the same source file. +- Adding a second upload-worker pool or a prepared-payload queue. +- Adaptive splitting after HTTP `413`. +- Splitting coverage or telemetry payloads. +- Persisting resumable upload state. +- Providing a Bazel Python toolchain from the core module. +- Changing backend payload schemas or endpoint semantics. +- Keeping the legacy implementations indefinitely. + +## Why Python + +Recommended workflows already require host Python for the doctor and BEP +artifact staging. The uploader therefore reuses an existing operational +dependency instead of introducing a bundled Zig or other native binary. + +Runtime contract: + +- Python 3.10 or newer; +- standard library only; +- no public `rules_python` dependency from the core module; +- controlled exit code `2` with an actionable error when Python is unavailable. + +Interpreter lookup is identical in intent on Unix and Windows: + +1. `DD_TEST_OPTIMIZATION_PYTHON` +2. `PYTHON` +3. `python3` +4. `python` + +## Architecture + +```text +platform launcher + | + v +CLI + generated config + | + v +workspace lock + | + v +freshness/BEP staging + deterministic discovery + | + v +CODEOWNERS + contexts + schema + telemetry plan (once) + | + v +bounded queue + | + +--------+--------+ ... +--------+ + | | | + worker 1 worker 2 worker N + | | | + +-- each owns one file ----+ + load / enrich / validate + split when test > 4.5 MiB + prepare exact request body + upload derived requests in order + delete source after complete success + | + v +immutable FileResult values + | + v +coordinator aggregate + final report +``` + +There is no synchronization between file workers beyond queue ownership and +coordinator collection. A worker never waits for another file's enrichment, +split, upload, or cleanup. + +## Ownership Boundaries + +### Launcher + +The Bash and PowerShell Python launchers may only: + +- locate a compatible interpreter; +- locate `uploader_main.py` and the generated config through Bazel runfiles; +- export the launcher directory used by CODEOWNERS discovery; +- forward arguments and the child exit code. + +They must not contain discovery, enrichment, split, HTTP, retry, cleanup, or +report behavior. + +### Application + +The application owns the invocation lifecycle: + +- configuration diagnostics; +- workspace lock lifetime; +- expected-target and freshness preflight; +- BEP staging and owned staging cleanup; +- deterministic discovery and quiescence; +- resource loading; +- coordinator invocation; +- post-worker freshness validation; +- final report emission. + +### Coordinator + +The coordinator owns invocation-wide worker inputs and aggregation: + +- validate the snapshotted proxy configuration; +- build one immutable CODEOWNERS matcher; +- build one immutable telemetry plan; +- perform the warning-only API-key fingerprint check; +- create one shared read-only `WorkerRuntime`; +- create one HTTP transport per worker; +- run the bounded worker pool; +- convert results into one immutable `AggregateReport`. + +### Worker + +A worker owns exactly one `FileTask` at a time. Its complete responsibility is: + +1. create task-local temporary space; +2. dispatch by payload type; +3. read and prepare the source; +4. execute every derived request sequentially; +5. delete the source only after complete success; +6. clean task-local temporary files; +7. return one immutable `FileResult`. + +Workers do not mutate global counters and do not delete sources they have not +dequeued. + +### Reporting + +`AggregateReport` is the single source of truth for: + +- compact terminal statistics; +- detailed statistics JSON; +- the backward-compatible schema-v1 report. + +Report writing is atomic. A report-write failure is printed as a warning and +does not reinterpret completed uploads. + +## Pre-worker Flow + +The following order is intentional: + +1. Parse CLI, environment, and generated config. +2. Validate endpoints and static configuration. +3. Acquire the workspace lock. +4. Resolve expected targets and local testlogs. +5. Parse freshness inputs and optionally stage BEP artifacts. +6. Wait for a stable discovery snapshot. +7. Apply expected-target and freshness selection. +8. Validate credentials only when selected files require real upload. +9. Resolve contexts, schema, and telemetry facts through one runfiles snapshot. +10. Parse CODEOWNERS and build telemetry directives once. +11. Start at most `min(workers, files)` worker threads. + +The lock remains held through worker completion, staging cleanup, and final +reporting so a second uploader cannot race source deletion or staging cleanup. + +## Configuration Precedence + +Generated rule values are defaults. Environment overrides them, and explicit +CLI values override environment where a CLI option exists. + +| Behavior | Rule | Environment | CLI | +|---|---|---|---| +| worker limit | `workers` | `DD_TEST_OPTIMIZATION_WORKERS` | `--workers` | +| debug | `debug` | `DD_TEST_OPTIMIZATION_DEBUG` | `--debug` | +| dry-run | — | — | `--dry-run` | +| enrichment assertion | — | — | `--validate-enrichment` | +| retain sources | `keep_payloads` | `DD_TEST_OPTIMIZATION_KEEP_PAYLOADS` | — | +| prefix filter | `filter_prefix` | `DD_TEST_OPTIMIZATION_FILTER_PREFIX` | — | +| gzip tests | `gzip_payloads` | `DD_TEST_OPTIMIZATION_GZIP` | — | +| CODEOWNERS override | — | `DD_TEST_OPTIMIZATION_CODEOWNERS_FILE` | — | +| report path | — | `DD_TEST_OPTIMIZATION_UPLOADER_REPORT_JSON` | `--report-json` | + +Invalid booleans retain legacy truth semantics; numeric values, choices, URLs, +ports, and proxy URLs fail preflight when invalid. `--validate-enrichment` +requires `--dry-run`. + +## CODEOWNERS + +CODEOWNERS discovery occurs once before workers start. Lookup order preserves +the existing contract, including explicit override and common GitHub/GitLab +locations. + +The parser produces immutable compiled rules and preserves: + +- last matching rule wins; +- escaped whitespace and comments; +- GitLab section headers; +- explicit empty-owner rules; +- source candidate normalization; +- existing producer-owned tags; +- supported-event filtering. + +Workers share the matcher but keep a file-local source-to-match cache. They do +not mutate matcher rules or share per-file match state. + +## Worker Pool + +- The queue is bounded to apply producer backpressure. +- Threads are homogeneous and non-daemon. +- Each thread receives one private `HttpTransport`. +- Results are reordered to deterministic discovery order before reporting. +- Duplicate task IDs and mismatched result IDs are rejected. +- An unhandled file exception becomes a sanitized failed `FileResult`; it does + not terminate unrelated workers. +- `workers=1` is the sequential compatibility baseline. + +On `KeyboardInterrupt`, queued/unowned files are cancelled and retained. Files +already owned finish their current complete pipeline, all threads join, owned +temporary resources are cleaned, completed results are reported, and the +process exits `130`. + +## Test Payload Pipeline + +For each test JSON file, one worker performs: + +1. Validate suffix and optional `span_events_` prefix filter. +2. Parse strict JSON and require a non-empty `events` array. +3. Load optional Bazel sidecar metadata. +4. Select the correct repository context. +5. Enrich top-level metadata and individual events. +6. Apply CODEOWNERS using the shared matcher. +7. Run warning-only schema validation. +8. Optionally assert expected enriched tags in dry-run. +9. Serialize and split using the preventive size contract. +10. Optionally gzip each already-split chunk. +11. Validate or upload chunks sequentially. +12. Delete the source only if every chunk succeeds. + +### Preventive split contract + +`MAX_TEST_PAYLOAD_BYTES = 4_718_592` is the maximum compact UTF-8 JSON body +before gzip. The worker never sends a known-oversized test request. + +If the complete enriched body fits, one chunk is written. Otherwise the +splitter: + +- preserves every top-level field; +- partitions only the `events` array; +- preserves event order and exactly-once membership; +- greedily fills deterministic chunks up to the limit; +- rejects an envelope or individual event that cannot fit; +- writes every exact request body in task-local temporary space before HTTP. + +Chunks from one source are never parallelized. A failed chunk does not prevent +later chunks from being attempted. After partial success, the source is +atomically replaced with only the failed chunks so accepted events are not +replayed; if every chunk fails or persistence fails, the original is retained. + +HTTP `413` is a terminal `payload_limit_contract_mismatch` for test chunks. It +is not retried and does not trigger another split, because the preventive split +should already have made the request valid. + +## Coverage Pipeline + +Coverage accepts JSON and msgpack. The worker: + +1. verifies the source can be opened; +2. builds the fixed Datadog event + coverage multipart body in task-local space; +3. validates the exact `Content-Length` and media type; +4. reopens the same immutable body for every retry; +5. deletes the source only after a successful response. + +Coverage is not split. A `413` is terminal and identifies unsupported oversized +coverage rather than invoking test-split behavior. + +## Telemetry Pipeline + +Telemetry correlation is planned before workers start, but each source remains +owned by one worker. The source worker: + +1. parses strict JSON and validates required metadata; +2. applies any immutable environment/message directive; +3. rewrites provider tags when configured; +4. materializes the primary exact body in task-local space; +5. lets the selected anchor create any synthetic message batch; +6. validates or sends primary and synthetic requests sequentially; +7. deletes the source only after all requests succeed. + +Unchanged primary telemetry preserves its original bytes. Changed and +synthetic bodies use deterministic compact serialization. Telemetry is not +split, and `413` is terminal. + +## HTTP and Retry Contract + +Each worker-local standard-library transport owns: + +- system TLS verification; +- separate connect and socket-I/O timeouts; +- a snapshotted proxy and `NO_PROXY` configuration; +- redirect rejection so credentials remain on the configured intake host; +- bounded response excerpts; +- exact request body factories that reopen the same bytes per attempt; +- redacted task-scoped debug diagnostics. + +The normalized retry budget is four total attempts. + +Retry: + +- connection and timeout failures, except TLS certificate verification failure; +- HTTP `408`; +- HTTP `429`; +- HTTP `5xx`. + +Do not retry: + +- HTTP `2xx` success; +- permanent HTTP `4xx`, including `413`; +- TLS certificate verification failure; +- local request-preparation errors. + +`Retry-After` supports integer seconds and HTTP dates and is bounded to 60 +seconds. Otherwise the configured fixed delay is used. Dry-run performs no +sleep and creates no network connection. + +## Dry-run and Debug + +### Dry-run + +Dry-run uses the same discovery and per-file preparation code as upload mode, +including enrichment, schema checks, 4.5 MiB split, gzip, multipart spooling, +telemetry augmentation, and URL/header/body-length validation. + +It must perform: + +- zero backend requests; +- zero retry sleeps; +- zero source deletions; +- zero credential requirements when no HTTP request will occur. + +### Debug + +Debug adds diagnostics for effective config, redacted endpoints, runfiles, +freshness decisions, CODEOWNERS/context selection, queue lifecycle, worker +ownership, exact body/chunk sizes, HTTP attempts, retries, cleanup, and final +timing. + +Debug must never print API keys, authorization headers, full payload bodies, or +unbounded response bodies. + +## Cleanup and Outcome Rules + +- Delete a source only after all requests derived from it succeed. +- Keep sources on preparation, validation, split, transport, or HTTP failure; + after a partial split upload, replace the source with only failed chunks. +- `keep_payloads` suppresses deletion after success. +- Dry-run never deletes sources. +- Task/invocation temporary cleanup failure adds a warning without changing an + already-known upload result. +- BEP staging cleanup failure preserves completed counters, adds its own reason, + and changes an otherwise successful invocation to controlled failure. +- Failure of one source does not cancel other source files. + +Severity precedence is: + +1. interrupt (`130`); +2. preflight or lifecycle error (`2`, or the established freshness code); +3. one or more failed files (`1`); +4. success (`0`). + +## Final Statistics + +Every controlled completion prints a stable summary containing: + +- mode, result, exit code, configured workers, peak workers, elapsed time; +- discovered, eligible, processed, succeeded, failed, skipped, and cancelled + files; +- succeeded/failed/skipped counts for test, coverage, and telemetry; +- split files, chunks created/uploaded/failed, and oversized events; +- planned/attempted/succeeded/failed requests and retries; +- deleted and retained sources. + +Human output and JSON derive from the same aggregate. The schema-v1 report keeps +legacy fields and adds explicit concurrency, split, request, warning, and +failure sections. + +## Package Map + +| Module | Responsibility | +|---|---| +| `uploader_main.py` | minimal import bootstrap | +| `uploader_py/main.py` | startup, version check, top-level controlled exit | +| `config.py` | typed config and CLI/environment precedence | +| `application.py` | locked preflight, postflight, and report lifecycle | +| `coordinator.py` | shared inputs, worker execution, aggregation | +| `worker_pool.py` | bounded ownership and shutdown semantics | +| `file_worker.py` | complete per-type file pipelines | +| `splitting.py` | deterministic preventive test split | +| `transport.py` | exact HTTP requests and retry policy | +| `codeowners.py` | discovery, immutable parsing, matching | +| `enrichment.py` | context/sidecar/event enrichment | +| `telemetry.py` | immutable cross-file telemetry planning | +| `freshness.py` | BEP staging and current-invocation selection | +| `reporting.py` | aggregate counters and renderers | +| `topt_runtime/runfiles.py` | immutable cross-platform runfiles lookup | + +Small support modules contain focused models, endpoints, locking, temporary +directories, strict JSON, credentials, discovery, expected targets, and +resource loading. Avoid adding another layer unless it removes more complexity +than it introduces. + +## Implementation Tracker + +### Runtime foundations + +- [x] Add Python 3.10+ startup check and minimal bootstrap. +- [x] Add generated typed config and standard-library-only runtime package. +- [x] Add cross-platform immutable runfiles resolution. +- [x] Add endpoint, proxy, credential, strict-JSON, and redacted-log validation. +- [x] Add workspace lock and owned temporary-directory primitives. +- [x] Make schema validation importable without breaking its CLI. + +### Shared pre-worker state + +- [x] Port expected-target and freshness selection. +- [x] Reuse doctor BEP parsing/staging and preserve cleanup ownership. +- [x] Port context and telemetry-facts resource loading. +- [x] Detect, parse, and compile CODEOWNERS once. +- [x] Prove matcher reads are deterministic under concurrent workers. +- [x] Plan telemetry correlation once and emit immutable per-source directives. +- [x] Perform API-key fingerprint parity once before workers. + +### Complete file pipelines + +- [x] Implement one dispatcher supporting all payload types. +- [x] Implement test enrichment, validation, split, gzip, upload, and cleanup. +- [x] Implement exact coverage multipart preparation and upload. +- [x] Implement primary and synthetic telemetry preparation and upload. +- [x] Reopen immutable task-local bodies across retries. +- [x] Stop later derived requests after a terminal failure. +- [x] Delete a source only after complete success. +- [x] Exercise all preparation paths in dry-run. + +### Concurrency and reporting + +- [x] Add rule/environment/CLI worker configuration. +- [x] Add bounded queue and homogeneous non-daemon workers. +- [x] Give each thread one private HTTP transport. +- [x] Keep coordinator counters out of workers. +- [x] Preserve deterministic result order. +- [x] Implement interrupt drain/join/report behavior. +- [x] Emit final human and JSON statistics from one aggregate. + +### Bazel rollout + +- [x] Generate config and carry the Python runtime in runfiles. +- [x] Generate behavior-free Unix and Windows Python launchers. +- [x] Keep an explicit per-target `use_python_uploader` rollout switch. +- [x] Add generated-launcher dry-run smoke coverage to CI. +- [ ] Record Bash and PowerShell compatibility fixtures for all public modes. +- [ ] Pass Linux, macOS, and Windows parity lanes. +- [ ] Pass the sibling consumer fixture with local module overrides. +- [ ] Validate one representative real consumer workflow. +- [x] Make Python the default. +- [ ] Update `UPLOADER_VERSION` intentionally for release. +- [ ] Remove the temporary switch and legacy Bash/PowerShell runtimes. +- [ ] Remove obsolete jq/curl/gzip uploader prerequisites from docs. + +## Test Matrix + +Required automated coverage: + +- configuration precedence and invalid values; +- Python version and launcher resolution, including manifest-only runfiles and + paths with spaces; +- CODEOWNERS discovery, parser parity, last-match behavior, and concurrent reads; +- zero/one/multiple context selection and sidecar enrichment; +- strict JSON and schema warning behavior; +- exact split boundary at limit minus one, limit, and limit plus one; +- top-level preservation, event ordering, and oversized single event; +- partial split failure continuation, failed-only retry persistence, and safe + fallback to the original source when persistence is unavailable; +- gzip after split and byte-identical retry body; +- JSON/msgpack coverage multipart content and exact length; +- telemetry primary/synthetic ownership, independent failure handling, and + failed-request-only retry persistence; +- agentless and EVP headers/endpoints; +- transient retry matrix and terminal permanent errors/`413`; +- worker bound, exactly-once file ownership, deterministic order, and + `workers=1` baseline; +- interrupt, cleanup failure, keep-payloads, filter-prefix, and dry-run; +- final counters, schema-v1 compatibility, and secret redaction; +- real loopback all-protocol execution; +- generated Bazel launcher smoke on Linux, macOS, and Windows. + +Repository validation before changing the default: + +```bash +python3 -m unittest discover -s tools/tests/python -p 'test*_tools.py' +python3 tools/dev/lint_uploader_templates.py +./bazelw test //tools/... +./bazelw test //examples/... +``` + +Also run every companion-module command from `CONTRIBUTING.md` and the relevant +flow in `../rules_test_optimization_tests` with local overrides. + +## Acceptance Criteria + +- [x] One functional Python uploader implementation exists. +- [x] Platform launchers contain resolution only. +- [x] Every worker processes all three payload types. +- [x] One worker owns a source through complete cleanup. +- [x] CODEOWNERS and global telemetry state are prepared once. +- [x] Test split occurs before HTTP at exactly `4_718_592` bytes. +- [x] Chunks preserve top-level fields, order, and exactly-once events. +- [x] Partial split uploads retain only failed chunks without replaying accepted + events. +- [x] `413` is terminal and never triggers retry or adaptive split. +- [x] Dry-run performs full preparation with no HTTP, sleep, or deletion. +- [x] Debug is task-scoped and secret-safe. +- [x] Controlled completion prints final statistics. +- [x] Human and JSON counters share one aggregate. +- [x] Source deletion requires complete success. +- [x] `workers=4` demonstrates real bounded overlap in the loopback harness. +- [ ] Public Bash/PowerShell behavior is characterized and matched. +- [ ] Standard-library HTTP behavior passes all supported OS/proxy/TLS lanes. +- [ ] `workers=1` passes the complete cross-platform parity matrix. +- [ ] Linux, macOS, and Windows CI pass with the Python target. +- [ ] The sibling consumer fixture passes. +- [x] Python becomes the default. +- [ ] Legacy functional scripts are removed. + +## Performance Validation + +Measure with identical payload fixtures and backend behavior: + +- wall-clock duration; +- files and requests per second; +- peak active workers; +- retry count; +- peak resident memory; +- temporary bytes written; +- p50/p95 per-file latency. + +Compare `workers=1`, the legacy uploader, and at least `workers=2/4/8`. + +A local ARM64/Python 3.12 loopback benchmark at commit `123113e` measured the +real worker pipeline with identical fixtures and rotating execution order. For +48 small mixed test/coverage/telemetry files, median speedups over one worker +were `1.99x`, `3.83x`, and `6.14x` with 2, 4, and 8 workers. For eight test +payloads above the 4.5 MiB split threshold, the corresponding speedups were +`1.40x`, `1.50x`, and `1.51x`. Based on the network-bound large-batch result, +the default is `8`; consumers with split-heavy workloads or backend throttling +can override it to `4`. Continue validating memory, temporary storage, and +real-backend retry behavior on supported CI hosts. + +## Rollout and Rollback + +1. Compare dry-run outputs and loopback request captures against legacy. +2. Validate the default in representative consumers and supported CI platforms. +3. Remove legacy code promptly once the remaining gates close so two + implementations cannot drift. + +During rollout, rollback is the rule-level switch back to the legacy +executable. Payload files remain recoverable because failures and dry-run retain +sources. After legacy removal, rollback should be a source-control revert of the +default/removal change, not permanent dual maintenance. diff --git a/tools/core/BUILD.bazel b/tools/core/BUILD.bazel index 221c395e..e50a2d93 100644 --- a/tools/core/BUILD.bazel +++ b/tools/core/BUILD.bazel @@ -4,6 +4,9 @@ # This product includes software developed at Datadog # (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. +# Export host-side Test Optimization tools and their dependency-free runtimes. +# The uploader sources are runfiles because consumers execute them after Bazel tests. + exports_files( [ "bep_artifact_stage_helper.py", @@ -27,7 +30,21 @@ exports_files( "uploader_bash_runtime.sh.tpl", "uploader_powershell_runtime.ps1.tpl", "uploader_batch_runtime.bat.tpl", + "uploader_python_launcher.sh.tpl", + "uploader_python_launcher.ps1.tpl", "validate_payload_schema.py", ], visibility = ["//visibility:public"], ) + +filegroup( + name = "uploader_python_runtime", + srcs = [ + "uploader_main.py", + "validate_payload_schema.py", + ] + glob([ + "topt_runtime/*.py", + "uploader_py/*.py", + ]), + visibility = ["//visibility:public"], +) diff --git a/tools/core/test_optimization_uploader.bzl b/tools/core/test_optimization_uploader.bzl index 774db33a..7b85cded 100644 --- a/tools/core/test_optimization_uploader.bzl +++ b/tools/core/test_optimization_uploader.bzl @@ -6,8 +6,10 @@ """Uploader rule implementation for Datadog CI Visibility payloads. -This file generates platform-specific uploader entrypoints (Bash + PowerShell) -at analysis time and exposes them via a normal Bazel rule (`bazel run`). +This file generates platform-specific uploader entrypoints at analysis time +and exposes them via a normal Bazel rule (`bazel run`). During the migration +window it emits both the legacy Bash/PowerShell runtimes and the small launchers +for the shared Python implementation. Operational model: - Tests run hermetically and write JSON payloads to `TEST_UNDECLARED_OUTPUTS_DIR`. @@ -20,31 +22,20 @@ Operational model: - `bazel run //:dd_upload_payloads -- --dry-run --validate-enrichment` exercises the same enrichment path without uploading or deleting payloads. -Why generated scripts: -- Upload logic needs host-specific tooling (`bash/curl` on Unix, - PowerShell/.NET HttpClient on Windows). -- Keeping scripts generated from one Starlark source preserves parity while - avoiding separate hand-maintained script files. +Why generated launchers/configuration: +- Bazel must still select a native Unix or Windows entrypoint. +- The Python launchers only resolve a host interpreter, the bootstrap, and the + generated typed configuration; uploader behavior lives in `uploader_py`. +- The large legacy runtimes remain generated only for the temporary rollout + fallback and are not part of the target Python architecture. Developer navigation: - Starlark test helpers (manifest/CODEOWNERS parsing parity) near the top. - `_uploader_impl` builds both script templates and wires rule outputs. -- Script templates contain the runtime behavior for discovery, enrichment, - uploads, retries, and locking. +- `uploader_py` owns current runtime behavior; legacy generated scripts remain + here only as a rollout fallback. """ -# Usage pattern: -# bazel test //... || test_status=$?; test_status=${test_status:-0}; DD_API_KEY="$DD_API_KEY" DD_SITE="$DD_SITE" bazel run //:dd_upload_payloads; exit $test_status -# -# Key features: -# - Discovers all test.outputs/ directories in bazel-testlogs automatically -# - Supports sharded tests (shard_N_of_M/) and retries (run_N_of_M/) -# - Uploads test payloads to CI Test Cycle intake -# - Uploads coverage payloads to Code Coverage intake -# - Deletes payloads after successful upload (unless DD_TEST_OPTIMIZATION_KEEP_PAYLOADS=1) -# - Uses workspace-level lock to prevent concurrent uploaders -# - Enriches payloads with context.json metadata - load( "//tools/core:common_utils.bzl", "RULES_VERSION", @@ -652,8 +643,9 @@ resolve_runfile_manifest_powershell_for_tests = _resolve_runfile_manifest_powers def _uploader_impl(ctx): """Rule implementation that generates cross-platform uploader executables. - The generated scripts perform runtime payload discovery/enrichment/upload, - while this function stays analysis-time only (template rendering + runfiles). + Generated legacy scripts or Python launchers perform runtime execution, + while this function stays analysis-time only (configuration, templates, + and runfiles). """ # `_uploader_impl` is responsible for generating *all* runtime uploader @@ -662,8 +654,8 @@ def _uploader_impl(ctx): # # Responsibilities: # 1) collect rule attrs and data dependencies (context/schema/validator) - # 2) render Bash and PowerShell script templates with concrete runfile paths - # 3) emit platform launchers (`.sh`, `.ps1`, `.bat`) with consistent behavior + # 2) render rollout-only Bash and PowerShell legacy runtimes + # 3) emit small Python platform launchers (`.sh`, `.ps1`, `.bat`) # 4) return DefaultInfo exposing the correct executable for the target OS # # Keep template substitutions explicit and centralized. If new placeholders @@ -680,6 +672,9 @@ def _uploader_impl(ctx): keep_payloads = ctx.attr.keep_payloads filter_prefix_enabled = ctx.attr.filter_prefix gzip_payloads = ctx.attr.gzip_payloads + workers = ctx.attr.workers + if workers <= 0: + fail_with_prefix("test_optimization_uploader", "workers must be a positive integer") expected_targets_file = ctx.file.expected_targets_file if ctx.attr.runtime_selection and (ctx.attr.data or ctx.attr.expected_targets or expected_targets_file): fail_with_prefix("test_optimization_uploader", "runtime_selection cannot be combined with configured data, expected_targets, or expected_targets_file") @@ -727,6 +722,43 @@ def _uploader_impl(ctx): schema_validator_rloc = ctx.file._schema_validator.short_path if ctx.file._schema_validator else "" schema_validator_path = ctx.file._schema_validator.path if ctx.file._schema_validator else "" + # Generate the complete Python runtime contract now, while the legacy + # launchers remain published until parity gates are complete. + python_config = ctx.actions.declare_file(ctx.label.name + ".uploader_config.json") + ctx.actions.write( + output = python_config, + content = json.encode({ + "schema_version": 1, + "quiescent_sec": quiescent_sec, + "max_wait_sec": max_wait_sec, + "fail_on_error": fail_on_error, + "debug": debug, + "keep_payloads": keep_payloads, + "filter_prefix": filter_prefix_enabled, + "gzip_payloads": gzip_payloads, + "workers": workers, + "rules_version": RULES_VERSION, + "uploader_version": UPLOADER_VERSION, + "workspace_name": ctx.workspace_name, + "context_manifest_path": context_manifest.path, + "context_manifest_short_path": context_manifest.short_path, + "telemetry_facts_manifest_path": telemetry_facts_manifest.path, + "telemetry_facts_manifest_short_path": telemetry_facts_manifest.short_path, + "schema_json_path": schema_json_path, + "schema_json_short_path": schema_json_rloc, + "doctor_runtime_path": ctx.file._doctor_runtime.path, + "doctor_runtime_short_path": ctx.file._doctor_runtime.short_path, + "expected_targets": sorted(ctx.attr.expected_targets), + "expected_targets_file_path": ( + expected_targets_file.path if expected_targets_file else "" + ), + "expected_targets_file_short_path": ( + expected_targets_file.short_path if expected_targets_file else "" + ), + "runtime_selection": ctx.attr.runtime_selection, + }) + "\n", + ) + # High-level debug of rule inputs log_info("Generating uploader scripts (Option 2: TEST_UNDECLARED_OUTPUTS_DIR)") log_debug( @@ -858,6 +890,38 @@ def _uploader_impl(ctx): ) log_debug(debug, "outputs", "Declared outputs → bash='%s', ps='%s', bat='%s'" % (bash_file.basename, ps_file.basename, bat_file.basename)) + # The Python implementation is emitted in parallel during rollout. Its + # platform launchers only resolve Python, the bootstrap, and this target's + # generated config; all functional behavior lives in uploader_py. + python_launcher_substitutions = _tokenize_template_substitutions({ + "python_config_name": python_config.basename, + "python_config_path": python_config.path, + "python_config_rloc": python_config.short_path, + "python_main_path": ctx.file._python_main.path, + "python_main_rloc": ctx.file._python_main.short_path, + }) + python_bash_file = ctx.actions.declare_file(ctx.label.name + ".python.sh") + ctx.actions.expand_template( + template = ctx.file._python_bash_launcher_template, + output = python_bash_file, + substitutions = python_launcher_substitutions, + is_executable = True, + ) + python_ps_file = ctx.actions.declare_file(ctx.label.name + ".python.ps1") + ctx.actions.expand_template( + template = ctx.file._python_powershell_launcher_template, + output = python_ps_file, + substitutions = python_launcher_substitutions, + is_executable = False, + ) + python_bat_file = ctx.actions.declare_file(ctx.label.name + ".python.bat") + ctx.actions.expand_template( + template = ctx.file._batch_runtime_template, + output = python_bat_file, + substitutions = _tokenize_template_substitutions({"ps_name": python_ps_file.basename}), + is_executable = True, + ) + # ------------------------------------------------------------------ # Phase 5: Build runfiles set and choose platform-specific executable. # ------------------------------------------------------------------ @@ -870,22 +934,44 @@ def _uploader_impl(ctx): extra_files.append(ctx.file._schema_validator) runfiles = ctx.runfiles( files = [ - ps_file, - bat_file, - context_manifest, - telemetry_facts_manifest, - expected_targets, - ctx.file._bep_artifact_stage_helper, - ctx.file._doctor_runtime, - ] + ctx.files.data + ([expected_targets_file] if expected_targets_file else []) + extra_files, + ps_file, + bat_file, + python_bash_file, + python_ps_file, + python_bat_file, + python_config, + context_manifest, + telemetry_facts_manifest, + expected_targets, + ctx.file._bep_artifact_stage_helper, + ctx.file._doctor_runtime, + ctx.file._python_main, + ] + + ctx.files.data + + ctx.files._python_runtime + + ([expected_targets_file] if expected_targets_file else []) + + extra_files, ) log_debug(debug, "outputs", "Runfiles include %d data file(s) plus PowerShell and batch scripts" % len(ctx.files.data)) # Use target-platform constraints (ConstraintValueInfo) so executable # selection is analysis-time deterministic across host operating systems. is_windows = ctx.target_platform_has_constraint(ctx.attr._windows_constraint[platform_common.ConstraintValueInfo]) - executable = bat_file if is_windows else bash_file - return [DefaultInfo(files = depset([bash_file, ps_file, bat_file]), executable = executable, runfiles = runfiles)] + legacy_executable = bat_file if is_windows else bash_file + python_executable = python_bat_file if is_windows else python_bash_file + executable = python_executable if ctx.attr.use_python_uploader else legacy_executable + return [DefaultInfo( + files = depset([ + bash_file, + ps_file, + bat_file, + python_bash_file, + python_ps_file, + python_bat_file, + ]), + executable = executable, + runfiles = runfiles, + )] _dd_payload_uploader_rule = rule( implementation = _uploader_impl, @@ -898,6 +984,8 @@ _dd_payload_uploader_rule = rule( "keep_payloads": attr.bool(default = False, doc = "Keep payload files after successful upload (env: DD_TEST_OPTIMIZATION_KEEP_PAYLOADS)"), "filter_prefix": attr.bool(default = False, doc = "Boolean gate: only upload files matching span_events_*.json or coverage_*.json; telemetry uploads are always eligible (env: DD_TEST_OPTIMIZATION_FILTER_PREFIX)"), "gzip_payloads": attr.bool(default = False, doc = "Gzip test payloads before upload (env: DD_TEST_OPTIMIZATION_GZIP)"), + "workers": attr.int(default = 8, doc = "Maximum number of independent payload-file workers (env: DD_TEST_OPTIMIZATION_WORKERS, CLI: --workers)"), + "use_python_uploader": attr.bool(default = True, doc = "Use the default cross-platform Python uploader; set False only for temporary rollback to the legacy platform runtime."), # Optional files to place in runfiles (e.g., a generated context.json) "data": attr.label_list(allow_files = True, doc = "Data files to include in runfiles (e.g., context.json for enrichment)"), "expected_targets": attr.string_list(default = [], doc = "Optional local labels whose current-invocation outputs the uploader must account for."), @@ -908,6 +996,10 @@ _dd_payload_uploader_rule = rule( "_schema_validator": attr.label(default = "//tools/core:validate_payload_schema.py", allow_single_file = True), "_bep_artifact_stage_helper": attr.label(default = "//tools/core:bep_artifact_stage_helper.py", allow_single_file = True), "_doctor_runtime": attr.label(default = "//tools/core:test_optimization_doctor.py", allow_single_file = True), + "_python_runtime": attr.label(default = "//tools/core:uploader_python_runtime", allow_files = True), + "_python_main": attr.label(default = "//tools/core:uploader_main.py", allow_single_file = True), + "_python_bash_launcher_template": attr.label(default = "//tools/core:uploader_python_launcher.sh.tpl", allow_single_file = True), + "_python_powershell_launcher_template": attr.label(default = "//tools/core:uploader_python_launcher.ps1.tpl", allow_single_file = True), # Runtime templates (kept as standalone files, not inline Starlark strings) "_bash_runtime_template": attr.label(default = "//tools/core:uploader_bash_runtime.sh.tpl", allow_single_file = True), "_powershell_runtime_template": attr.label(default = "//tools/core:uploader_powershell_runtime.ps1.tpl", allow_single_file = True), diff --git a/tools/core/topt_runtime/__init__.py b/tools/core/topt_runtime/__init__.py new file mode 100644 index 00000000..cae5e313 --- /dev/null +++ b/tools/core/topt_runtime/__init__.py @@ -0,0 +1,10 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Dependency-free helpers shared by executable Test Optimization tools. + +This package keeps host-side infrastructure independent from uploader internals. +""" diff --git a/tools/core/topt_runtime/runfiles.py b/tools/core/topt_runtime/runfiles.py new file mode 100644 index 00000000..01e32f8c --- /dev/null +++ b/tools/core/topt_runtime/runfiles.py @@ -0,0 +1,291 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Resolve Bazel runfiles into one immutable lookup snapshot. + +Centralizing lookup keeps launchers portable and workers isolated from env changes. +""" + +from __future__ import annotations + +from dataclasses import dataclass +import os +from pathlib import Path +import re +from types import MappingProxyType +from typing import Iterable, Mapping + + +_WINDOWS_ABSOLUTE_RE = re.compile(r"^[A-Za-z]:/") + + +class RunfileResolutionError(RuntimeError): + """A requested runfile label is unsafe or cannot be resolved.""" + + +def runfile_candidates(raw: str) -> tuple[str, ...]: + """Return the legacy Bazel path variants for one safe logical runfile. + + Bazel may expose an external repository as ``external//...`` or + ``/...`` and Bzlmod may prefix main-repository entries with + ``_main/``. Leading ``../`` segments are a Bazel ``short_path`` convention; + parent traversal anywhere else is rejected. + """ + logical_path = raw.replace("\\", "/") + if logical_path.startswith("./"): + logical_path = logical_path[2:] + while logical_path.startswith("../"): + logical_path = logical_path[3:] + + parts = logical_path.split("/") + if ( + not logical_path + or logical_path.startswith("/") + or _WINDOWS_ABSOLUTE_RE.match(logical_path) + or ".." in parts + ): + raise RunfileResolutionError( + f"rejected suspicious runfile label: {raw!r}" + ) + + candidates = [logical_path] + if logical_path.startswith("external/"): + candidates.append(logical_path[len("external/") :]) + else: + candidates.append(f"external/{logical_path}") + if not logical_path.startswith("_main/"): + candidates.append(f"_main/{logical_path}") + return tuple(dict.fromkeys(candidates)) + + +def _manifest_line(line: str, *, first_line: bool) -> tuple[str, str] | None: + """Parse one Bazel manifest entry without splitting paths at spaces.""" + normalized = line.rstrip("\r\n") + if first_line: + normalized = normalized.lstrip("\ufeff") + if normalized.startswith(" "): + encoded = normalized[1:] + separator_index = encoded.find(" ") + if separator_index <= 0: + return None + key = _decode_manifest_field(encoded[:separator_index]) + path_text = _decode_manifest_field(encoded[separator_index + 1 :]) + if not path_text: + return None + return key.replace("\\", "/"), path_text + space_index = normalized.find(" ") + tab_index = normalized.find("\t") + indexes = [index for index in (space_index, tab_index) if index >= 0] + if not indexes: + return None + separator_index = min(indexes) + if separator_index <= 0: + return None + key = normalized[:separator_index] + path_text = normalized[separator_index + 1 :].strip() + if not path_text: + return None + return key.replace("\\", "/"), path_text + + +def _decode_manifest_field(value: str) -> str: + """Decode Bazel's escaped runfiles manifest field representation.""" + return value.replace(r"\s", " ").replace(r"\n", "\n").replace(r"\b", "\\") + + +def _load_manifest(path: Path | None) -> Mapping[str, Path]: + if path is None or not path.is_file(): + return MappingProxyType({}) + + entries: dict[str, Path] = {} + try: + with path.open("r", encoding="utf-8", errors="replace") as handle: + for line_number, line in enumerate(handle): + entry = _manifest_line(line, first_line=line_number == 0) + if entry is None: + continue + key, raw_value = entry + resolved_path = Path(raw_value) + if not resolved_path.is_absolute(): + resolved_path = path.parent / resolved_path + entries.setdefault(key, resolved_path) + except OSError as exc: + raise RunfileResolutionError( + f"failed to read runfiles manifest {path}: {type(exc).__name__}" + ) from exc + return MappingProxyType(entries) + + +def _unique_existing_directories(paths: Iterable[Path]) -> tuple[Path, ...]: + directories: list[Path] = [] + seen: set[Path] = set() + for raw_path in paths: + path = raw_path.resolve() + if path in seen or not path.is_dir(): + continue + seen.add(path) + directories.append(path) + return tuple(directories) + + +def _safe_workspace_name(raw: str) -> str | None: + normalized = raw.replace("\\", "/").strip("/") + if not normalized or _WINDOWS_ABSOLUTE_RE.match(normalized): + return None + if any(part in {"", ".", ".."} for part in normalized.split("/")): + return None + return normalized + + +@dataclass(frozen=True) +class RunfilesResolver: + """Read-only resolver safe to share with every uploader worker.""" + + cwd: Path + roots: tuple[Path, ...] + workspace_names: tuple[str, ...] + manifest_path: Path | None + manifest_entries: Mapping[str, Path] + + @classmethod + def from_environment( + cls, + *, + argv0: str | Path | None = None, + environ: Mapping[str, str] | None = None, + cwd: Path | None = None, + ) -> "RunfilesResolver": + """Snapshot all process-dependent lookup state once. + + ``environ`` is copied conceptually into immutable paths and strings; + later environment mutations therefore cannot affect worker lookups. + """ + environment = dict(os.environ if environ is None else environ) + working_directory = (cwd or Path.cwd()).resolve() + + root_candidates: list[Path] = [] + for variable in ("RUNFILES_DIR", "TEST_SRCDIR"): + raw_root = environment.get(variable, "") + if raw_root: + root = Path(raw_root) + root_candidates.append( + root if root.is_absolute() else working_directory / root + ) + + if argv0: + launcher = Path(argv0) + if not launcher.is_absolute(): + launcher = working_directory / launcher + root_candidates.extend( + ( + Path(f"{launcher}.runfiles"), + launcher.parent / f"{launcher.stem}.runfiles", + launcher.parent / f"{launcher.name}.runfiles", + ) + ) + for parent in (launcher.parent, *launcher.parents): + if parent.name.endswith(".runfiles"): + root_candidates.append(parent) + break + + workspace_candidates: list[str] = [] + for raw_name in ( + environment.get("TEST_WORKSPACE", ""), + environment.get("DD_TEST_OPTIMIZATION_RUNFILES_WORKSPACE", ""), + ): + safe_name = _safe_workspace_name(raw_name) + if safe_name is not None: + workspace_candidates.append(safe_name) + workspace_names = tuple(dict.fromkeys(workspace_candidates)) + raw_manifest = environment.get("RUNFILES_MANIFEST_FILE", "") + raw_manifest_path = Path(raw_manifest) if raw_manifest else None + manifest_path = ( + ( + raw_manifest_path + if raw_manifest_path.is_absolute() + else working_directory / raw_manifest_path + ).resolve() + if raw_manifest_path is not None + else None + ) + return cls( + cwd=working_directory, + roots=_unique_existing_directories(root_candidates), + workspace_names=workspace_names, + manifest_path=manifest_path, + manifest_entries=_load_manifest(manifest_path), + ) + + def resolve_file(self, raw_paths: str | Iterable[str]) -> Path: + """Resolve the first existing direct path or logical runfile. + + Direct filesystem paths are checked first for compatibility with rule + configuration values that point into Bazel's execroot. Missing absolute + paths are never reinterpreted as logical runfile labels. + """ + if isinstance(raw_paths, str): + requested = (raw_paths,) + else: + requested = tuple(raw_paths) + requested = tuple(raw for raw in requested if raw) + if not requested: + raise RunfileResolutionError("no runfile path was provided") + + logical_candidates: list[str] = [] + candidate_errors: list[RunfileResolutionError] = [] + for raw in requested: + direct = Path(raw) + direct_candidate = direct if direct.is_absolute() else self.cwd / direct + if direct_candidate.is_file(): + return direct_candidate.resolve() + if direct.is_absolute() or _WINDOWS_ABSOLUTE_RE.match(raw.replace("\\", "/")): + continue + try: + logical_candidates.extend(runfile_candidates(raw)) + except RunfileResolutionError as exc: + candidate_errors.append(exc) + + candidates = tuple(dict.fromkeys(logical_candidates)) + for root in self.roots: + for candidate in self._root_candidates(candidates): + resolved = root.joinpath(*candidate.split("/")) + if resolved.is_file(): + return resolved.resolve() + + manifest_match = self._resolve_manifest(candidates) + if manifest_match is not None: + return manifest_match + + if not candidates and candidate_errors: + raise candidate_errors[0] + rendered = ", ".join(repr(raw) for raw in requested) + raise RunfileResolutionError(f"runfile not found for: {rendered}") + + def _root_candidates(self, candidates: tuple[str, ...]) -> tuple[str, ...]: + expanded = list(candidates) + for workspace in self.workspace_names: + expanded.extend( + f"{workspace}/{candidate}" + for candidate in candidates + if not candidate.startswith(f"{workspace}/") + ) + return tuple(dict.fromkeys(expanded)) + + def _resolve_manifest(self, candidates: tuple[str, ...]) -> Path | None: + lookup_candidates = self._root_candidates(candidates) + for candidate in lookup_candidates: + matched_path = self.manifest_entries.get(candidate) + if matched_path is not None and matched_path.is_file(): + return matched_path.resolve() + + # Preserve the legacy fallback for manifests that add an unknown main + # repository prefix to otherwise valid keys. + for candidate in lookup_candidates: + suffix = f"/{candidate}" + for key, matched_path in self.manifest_entries.items(): + if key.endswith(suffix) and matched_path.is_file(): + return matched_path.resolve() + return None diff --git a/tools/core/uploader_bash_runtime.sh.tpl b/tools/core/uploader_bash_runtime.sh.tpl index 646aad92..a29ef47e 100644 --- a/tools/core/uploader_bash_runtime.sh.tpl +++ b/tools/core/uploader_bash_runtime.sh.tpl @@ -5021,9 +5021,9 @@ validate_enriched_payload_tags() { return 1 fi if (( DRY_RUN == 1 )); then - log "dry-run validated enriched test payload: $source_file" + dbg "dry-run validated enriched test payload: $source_file" else - log "validated enriched test payload: $source_file" + dbg "validated enriched test payload: $source_file" fi return 0 } diff --git a/tools/core/uploader_main.py b/tools/core/uploader_main.py new file mode 100644 index 00000000..7cad1101 --- /dev/null +++ b/tools/core/uploader_main.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Bootstrap the shared uploader package from a stable Bazel entrypoint. + +Keeping this file behavior-free lets both platform launchers invoke one runtime. +""" + +from uploader_py.main import main + + +raise SystemExit(main()) diff --git a/tools/core/uploader_powershell_runtime.ps1.tpl b/tools/core/uploader_powershell_runtime.ps1.tpl index 14d47fc7..c8647c96 100644 --- a/tools/core/uploader_powershell_runtime.ps1.tpl +++ b/tools/core/uploader_powershell_runtime.ps1.tpl @@ -4898,9 +4898,9 @@ function Test-EnrichedPayloadTags([string]$BodyPath, [string]$SourcePath) { return [bool]$false } if ($script:DryRun) { - Log "dry-run validated enriched test payload: $SourcePath" + Dbg "dry-run validated enriched test payload: $SourcePath" } else { - Log "validated enriched test payload: $SourcePath" + Dbg "validated enriched test payload: $SourcePath" } return [bool]$true } diff --git a/tools/core/uploader_py/__init__.py b/tools/core/uploader_py/__init__.py new file mode 100644 index 00000000..56d3a17b --- /dev/null +++ b/tools/core/uploader_py/__init__.py @@ -0,0 +1,14 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Expose the stable public surface of the cross-platform uploader package. + +The narrow package boundary prevents callers from depending on internal stages. +""" + +from .models import MAX_TEST_PAYLOAD_BYTES + +__all__ = ["MAX_TEST_PAYLOAD_BYTES"] diff --git a/tools/core/uploader_py/application.py b/tools/core/uploader_py/application.py new file mode 100644 index 00000000..51ac3502 --- /dev/null +++ b/tools/core/uploader_py/application.py @@ -0,0 +1,714 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Own the locked uploader lifecycle around independent file workers. + +Preflight, postflight, and reporting live here so workers only process files. +""" + +from __future__ import annotations + +from dataclasses import dataclass, replace +import logging +from pathlib import Path +import sys +import time +from typing import Callable, TextIO + +from topt_runtime.runfiles import RunfilesResolver + +from .config import ConfigError, UploaderConfig, validate_upload_credentials +from .coordinator import CoordinatorSettings, run_discovered_tasks +from .discovery import ( + DiscoveryError, + DiscoveryResult, + count_tasks_by_payload_type, + discover_file_tasks, + resolve_local_testlogs_root, + tests_executed, + wait_for_quiescence, +) +from .endpoints import EndpointSet +from .expected_targets import ( + ExpectedTargetsError, + ExpectedTargetsPlan, + load_expected_targets, + select_expected_outputs, +) +from .freshness import ( + FreshnessError, + FreshnessPlan, + FreshnessPreparation, + filter_discovery_for_freshness, + prepare_freshness, + validate_fresh_outputs_accounted, +) +from .locking import WorkspaceLock, WorkspaceLockError +from .logging_utils import redact_url +from .models import FileStatus, PayloadType +from .reporting import AggregateReport, LegacyReportContext, emit_report +from .resources import LoadedResources, ResourceError, ResourceInputs, load_resources +from .temporary import TemporaryDirectoryError +from .transport import HttpTransportError +from .worker_pool import WorkerPoolError + + +_CONTROLLED_ERRORS = ( + ConfigError, + DiscoveryError, + ExpectedTargetsError, + ResourceError, + FreshnessError, + TemporaryDirectoryError, + HttpTransportError, + WorkerPoolError, + WorkspaceLockError, +) + + +@dataclass(frozen=True) +class _ReportReason: + """Explicit reason fields carried from lifecycle decisions into reporting.""" + + code: str = "" + message: str = "" + next_steps: tuple[str, ...] = () + + +def run_uploader( + config: UploaderConfig, + *, + resolver: RunfilesResolver, + endpoints: EndpointSet, + logger: logging.Logger, + stream: TextIO | None = None, + transport_factory: Callable[[], object] | None = None, + clock: Callable[[], float] = time.monotonic, +) -> int: + """Execute one locked uploader invocation and release after reporting.""" + workspace_lock = WorkspaceLock(config.lock_workspace) + try: + return _run_uploader_with_lock( + config, + resolver=resolver, + endpoints=endpoints, + logger=logger, + stream=stream, + transport_factory=transport_factory, + clock=clock, + workspace_lock=workspace_lock, + ) + finally: + workspace_lock.release() + + +def _run_uploader_with_lock( + config: UploaderConfig, + *, + resolver: RunfilesResolver, + endpoints: EndpointSet, + logger: logging.Logger, + stream: TextIO | None, + transport_factory: Callable[[], object] | None, + clock: Callable[[], float], + workspace_lock: WorkspaceLock, +) -> int: + """Hold the workspace lock through preflight, cleanup, and reporting.""" + started = clock() + expected_targets_plan = ExpectedTargetsPlan() + freshness_plan = FreshnessPlan() + freshness_preparation: FreshnessPreparation | None = None + raw_discovery = _empty_discovery() + freshness_skipped_count = 0 + report: AggregateReport | None = None + report_reason: _ReportReason | None = None + workers_completed = False + + _log_invocation(config, endpoints, resolver, logger) + + try: + # The public wrapper keeps the lock through final report emission. + workspace_lock.acquire() + expected_targets_plan = load_expected_targets( + static_targets=config.rule.expected_targets, + expected_targets_file_paths=( + config.rule.expected_targets_file_path, + config.rule.expected_targets_file_short_path, + ), + resolver=resolver, + runtime_targets=config.runtime_expected_targets, + runtime_selection=config.rule.runtime_selection, + ) + resources = _load_resources(config, resolver, logger) + local_root = resolve_local_testlogs_root( + explicit=config.testlogs_dir, + workspace=config.workspace, + cwd=Path.cwd(), + ) + freshness_preparation = prepare_freshness( + config, + resolver=resolver, + local_testlogs_root=local_root, + expected_targets=expected_targets_plan.targets, + logger=logger, + ) + freshness_plan = freshness_preparation.plan + logger.debug( + "freshness prepared: selected_source=%s scan_roots=%s " + "staged_roots=%s eligible=%d cached=%d remote_only=%d", + freshness_plan.selected_source, + tuple(str(root.path) for root in freshness_preparation.scan_roots), + tuple(str(path) for path in freshness_preparation.staged_roots), + len(freshness_plan.eligible_outputs), + len(freshness_plan.cached_outputs), + len(freshness_plan.remote_only_outputs), + ) + all_expected_outputs_cached = _all_expected_outputs_cached( + expected_targets_plan, + freshness_plan, + ) + if ( + not freshness_preparation.scan_roots + and config.fail_on_error + and not all_expected_outputs_cached + ): + raise DiscoveryError( + "FAIL_ON_ERROR is set and no local or staged testlogs root was found" + ) + raw_discovery = _discover_payloads( + config, + freshness_preparation, + all_expected_outputs_cached=all_expected_outputs_cached, + logger=logger, + ) + + expected_discovery = select_expected_outputs( + raw_discovery, + expected_targets_plan, + allow_missing=freshness_plan.selected_source == "bep", + ) + freshness_selection = filter_discovery_for_freshness( + expected_discovery, + freshness_plan, + freshness_mode=config.freshness_mode, + ) + eligible_discovery = freshness_selection.discovery + freshness_skipped_outputs = freshness_selection.skipped_outputs + freshness_skipped_count = len(freshness_skipped_outputs) + for output_key in freshness_skipped_outputs: + logger.debug("freshness skipped output_key=%s", output_key) + _log_legacy_freshness_markers( + config, + freshness_plan, + freshness_skipped_outputs, + logger, + ) + if eligible_discovery.tasks: + validate_upload_credentials(config) + else: + logger.debug("credential validation skipped: no upload tasks") + report = run_discovered_tasks( + eligible_discovery, + settings=CoordinatorSettings.from_config(config), + endpoints=endpoints, + resources=resources, + logger=logger, + transport_factory=transport_factory, + clock=clock, + ) + workers_completed = True + report = replace( + report, + discovered_by_type=raw_discovery.discovered_by_type, + ) + if report.exit_code == 130: + report_reason = _ReportReason( + code="interrupted", + message="Uploader interrupted after active workers finished.", + next_steps=( + "Re-run the uploader to process retained and cancelled files.", + ), + ) + else: + try: + validate_fresh_outputs_accounted( + freshness_plan, + eligible_discovery, + report.results, + expected_targets=expected_targets_plan.targets, + fail_on_error=config.fail_on_error, + ) + except FreshnessError as exc: + logger.error("%s", exc) + report = replace(report, exit_code=exc.exit_code) + if "remote-only" in str(exc).lower(): + report_reason = _preflight_failure_reason( + config, + freshness_plan, + exc, + ) + else: + report_reason = _ReportReason( + code="fresh_output_without_payloads", + message=str(exc), + next_steps=( + "Inspect the fresh test.outputs payload directories.", + ), + ) + + if not raw_discovery.tasks: + if all_expected_outputs_cached: + report_reason = _ReportReason( + code="ok", + message=( + "All expected target outputs were cached; nothing was " + "uploaded." + ), + ) + elif ( + tests_executed(freshness_preparation.scan_roots) + and config.fail_on_error + ): + report = replace(report, exit_code=1) + report_reason = _ReportReason( + code="tests_ran_without_payloads", + message="Tests ran but no payload files were found.", + next_steps=( + "Check that DD_TEST_OPTIMIZATION_PAYLOADS_IN_FILES=true is set.", + ), + ) + except _CONTROLLED_ERRORS as exc: + exit_code = exc.exit_code if isinstance(exc, FreshnessError) else 2 + logger.error("%s", exc) + report = _empty_report( + config, + exit_code=exit_code, + elapsed_seconds=max(0.0, clock() - started), + discovery=raw_discovery, + ) + report_reason = _preflight_failure_reason( + config, + freshness_plan, + exc, + ) + finally: + report, report_reason = _cleanup_staging( + freshness_preparation, + report, + report_reason, + config=config, + discovery=raw_discovery, + started=started, + clock=clock, + logger=logger, + ) + + assert report is not None + # The coordinator measures only worker preparation and delivery. Final + # invocation statistics must also include locking, BEP staging, discovery + # quiescence, resource loading, and staging cleanup. + report = replace( + report, + elapsed_seconds=max(0.0, clock() - started), + ) + report_context = _legacy_context( + config, + freshness_plan=freshness_plan, + raw_discovery=raw_discovery, + staged_roots=( + freshness_preparation.staged_roots + if freshness_preparation is not None + else () + ), + freshness_skipped_count=freshness_skipped_count, + report_reason=report_reason, + ) + logger.debug( + "emitting final report: exit_code=%d results=%d warnings=%s", + report.exit_code, + len(report.results), + report.initialization_warning_codes, + ) + if workers_completed: + _log_legacy_result_markers(config, report, logger) + emit_report( + report, + stream=stream if stream is not None else sys.stdout, + report_json=config.report_json, + legacy_report_context=report_context, + ) + return report.exit_code + + +def _log_legacy_freshness_markers( + config: UploaderConfig, + freshness_plan: FreshnessPlan, + skipped_outputs: tuple[str, ...], + logger: logging.Logger, +) -> None: + """Keep stable freshness markers after selecting Python by default.""" + if freshness_plan.selected_source == "bep": + logger.info( + "freshness filtering enabled: source=bep files=%d " + "eligible_outputs=%d remote_only_outputs=%d", + len(config.bep_json_files), + len(freshness_plan.eligible_outputs), + len(freshness_plan.remote_only_outputs), + ) + elif freshness_plan.selected_source == "execution_log": + logger.info("freshness filtering enabled: source=execution_log") + elif config.freshness_mode == "disabled": + logger.info("freshness filtering disabled") + + skipped_or_cached_outputs = sorted( + set(skipped_outputs).union( + output_key for _label, output_key in freshness_plan.cached_outputs + ) + ) + for output_key in skipped_or_cached_outputs: + logger.info( + "skipping cached or non-current test output: %s (freshness selection)", + output_key, + ) + + +def _log_legacy_result_markers( + config: UploaderConfig, + report: AggregateReport, + logger: logging.Logger, +) -> None: + """Keep stable per-test result markers alongside the aggregate report.""" + successful_tests = tuple( + result + for result in report.results + if result.payload_type is PayloadType.TEST + and result.status is FileStatus.SUCCEEDED + ) + if config.validate_enrichment: + validation_prefix = "dry-run " if config.dry_run else "" + for result in successful_tests: + logger.debug( + "%svalidated enriched test payload: %s", + validation_prefix, + result.source_path, + ) + if config.dry_run: + logger.info("dry-run validated %d test payloads", len(successful_tests)) + else: + logger.info("uploaded %d test payloads", len(successful_tests)) + + +def _cleanup_staging( + freshness_preparation: FreshnessPreparation | None, + report: AggregateReport | None, + report_reason: _ReportReason | None, + *, + config: UploaderConfig, + discovery: DiscoveryResult, + started: float, + clock: Callable[[], float], + logger: logging.Logger, +) -> tuple[AggregateReport | None, _ReportReason | None]: + """Clean owned BEP staging without discarding completed upload results.""" + if freshness_preparation is None: + return report, report_reason + try: + freshness_preparation.cleanup() + except FreshnessError as exc: + logger.error("failed to clean BEP staging: %s", exc) + if report is None: + report = _empty_report( + config, + exit_code=2, + elapsed_seconds=max(0.0, clock() - started), + discovery=discovery, + ) + warning_codes = tuple( + dict.fromkeys( + report.initialization_warning_codes + ("staging_cleanup_failed",) + ) + ) + report = replace( + report, + exit_code=report.exit_code or 2, + initialization_warning_codes=warning_codes, + ) + if report.exit_code == 2 and ( + report_reason is None or report_reason.code in {"", "ok"} + ): + report_reason = _ReportReason( + code="staging_cleanup_failed", + message=str(exc), + next_steps=( + "Remove only the uploader-owned staging run directory.", + ), + ) + else: + logger.debug("BEP staging cleanup completed") + return report, report_reason + + +def _log_invocation( + config: UploaderConfig, + endpoints: EndpointSet, + resolver: RunfilesResolver, + logger: logging.Logger, +) -> None: + """Record the immutable inputs that explain the rest of the run.""" + logger.debug( + "effective config: mode=%s workers=%d validate_enrichment=%s " + "gzip=%s keep_payloads=%s filter_prefix=%s fail_on_error=%s " + "workspace=%s testlogs=%s freshness_source=%s freshness_mode=%s " + "artifact_source=%s remote_artifacts=%s proxy_configured=%s", + "dry-run" if config.dry_run else "upload", + config.workers, + config.validate_enrichment, + config.gzip_payloads, + config.keep_payloads, + config.filter_prefix, + config.fail_on_error, + config.workspace, + config.testlogs_dir or "auto", + config.freshness_source, + config.freshness_mode, + config.artifact_source, + config.remote_artifacts, + bool(config.proxy_environment), + ) + logger.debug( + "endpoint mode=%s tests=%s coverage=%s telemetry=%s", + "agentless" if endpoints.agentless else "evp", + redact_url(endpoints.test_url), + redact_url(endpoints.coverage_url), + redact_url(endpoints.telemetry_url), + ) + logger.debug( + "runfiles snapshot: cwd=%s roots=%s workspace_names=%s manifest=%s " + "manifest_entries=%d", + resolver.cwd, + tuple(str(path) for path in resolver.roots), + resolver.workspace_names, + resolver.manifest_path or "none", + len(resolver.manifest_entries), + ) + + +def _all_expected_outputs_cached( + expected: ExpectedTargetsPlan, + freshness: FreshnessPlan, +) -> bool: + expected_labels = frozenset(expected.targets) + cached_labels = frozenset(label for label, _key in freshness.cached_outputs) + return bool( + expected_labels + and cached_labels == expected_labels + and not freshness.eligible_outputs + and not freshness.remote_only_outputs + and not freshness.missing_output_labels + ) + + +def _discover_payloads( + config: UploaderConfig, + preparation: FreshnessPreparation, + *, + all_expected_outputs_cached: bool, + logger: logging.Logger, +) -> DiscoveryResult: + """Discover a stable source-file snapshot after freshness has authorized it.""" + selected_output_keys = { + output_key + for _label, output_key in preparation.plan.selected_artifact_outputs + } + for output_key in sorted(selected_output_keys): + logger.debug("freshness eligible output_key=%s", output_key) + + def scan() -> DiscoveryResult: + return discover_file_tasks( + preparation.scan_roots, + max_depth=config.max_depth, + staged_output_keys=selected_output_keys, + ) + + if all_expected_outputs_cached: + logger.debug("all expected target outputs were cached; skipping discovery wait") + return _empty_discovery() + if not preparation.scan_roots: + logger.debug("no local or staged testlogs roots were found") + return scan() + + quiescence = wait_for_quiescence( + scan, + quiescent_seconds=config.quiescent_sec, + max_wait_seconds=config.max_wait_sec, + ) + discovery = quiescence.discovery + logger.debug( + "discovery settled: reason=%s elapsed=%.2fs outputs=%d files=%d", + quiescence.reason, + quiescence.elapsed_seconds, + len(discovery.outputs), + len(discovery.tasks), + ) + return discovery + + +def _load_resources( + config: UploaderConfig, + resolver: RunfilesResolver, + logger: logging.Logger, +) -> LoadedResources: + """Resolve and log the read-only context shared by all workers.""" + resources = load_resources( + resolver, + ResourceInputs( + context_override=config.context_json, + context_manifest_paths=( + config.rule.context_manifest_path, + config.rule.context_manifest_short_path, + ), + telemetry_facts_manifest_paths=( + config.rule.telemetry_facts_manifest_path, + config.rule.telemetry_facts_manifest_short_path, + ), + schema_paths=( + config.rule.schema_json_path, + config.rule.schema_json_short_path, + ), + runtime_context_entries=config.runtime_context_entries, + runtime_selection=config.rule.runtime_selection, + workspace=config.workspace, + invocation_cwd=config.invocation_cwd, + ), + ) + logger.debug( + "resources resolved: primary_context=%s contexts=%d " + "telemetry_facts=%d schema=%s warnings=%s", + resources.primary_context_path or "none", + len(resources.context_plan.by_repo), + len(resources.telemetry_facts_paths), + "loaded" if resources.schema is not None else "absent", + resources.warning_codes, + ) + return resources + + +def _empty_discovery() -> DiscoveryResult: + return DiscoveryResult( + outputs=(), + tasks=(), + discovered_by_type=count_tasks_by_payload_type(()), + ) + + +def _empty_report( + config: UploaderConfig, + *, + exit_code: int, + elapsed_seconds: float, + discovery: DiscoveryResult, +) -> AggregateReport: + return AggregateReport.create( + dry_run=config.dry_run, + exit_code=exit_code, + configured_workers=config.workers, + worker_threads=0, + peak_active_workers=0, + elapsed_seconds=elapsed_seconds, + discovered_by_type=discovery.counts(), + results=(), + initialization_warning_codes=discovery.warning_codes, + ) + + +def _legacy_context( + config: UploaderConfig, + *, + freshness_plan: FreshnessPlan, + raw_discovery: DiscoveryResult, + staged_roots: tuple[Path, ...], + freshness_skipped_count: int, + report_reason: _ReportReason | None, +) -> LegacyReportContext: + final_reason = report_reason or _ReportReason() + return LegacyReportContext( + validate_enrichment=config.validate_enrichment, + artifact_source=config.artifact_source, + remote_artifacts=config.remote_artifacts, + freshness_source=config.freshness_source, + freshness_mode=config.freshness_mode, + allow_cached_payload_uploads=config.freshness_disabled_explicitly, + bep_files=tuple(str(path) for path in config.bep_json_files), + freshness_selected_source=freshness_plan.selected_source, + freshness_eligible_outputs=len(freshness_plan.eligible_outputs), + freshness_cached_outputs=len(freshness_plan.cached_outputs), + freshness_remote_only_outputs=len(freshness_plan.remote_only_outputs), + freshness_skipped_outputs=freshness_skipped_count, + freshness_missing_output_labels=len(freshness_plan.missing_output_labels), + staging_dir=str(config.artifact_staging_dir), + staged_testlogs_dirs=len(staged_roots), + selected_remote_artifacts=len(freshness_plan.selected_artifact_outputs), + staged_remote_artifacts=len(freshness_plan.staged_outputs), + remote_artifacts_ignored=len(freshness_plan.blocked_labels), + test_outputs_dirs=len(raw_discovery.outputs), + reason_code=final_reason.code, + reason=final_reason.message, + next_steps=final_reason.next_steps, + ) + + +def _preflight_failure_reason( + config: UploaderConfig, + plan: FreshnessPlan, + error: BaseException, +) -> _ReportReason: + message = str(error) + lower_message = message.lower() + if not config.bep_json_files and ( + config.freshness_source == "bep" + or config.artifact_source == "bep" + or config.freshness_mode == "required" + ): + return _ReportReason( + code="missing_bep_json", + message=( + "BEP freshness or artifact staging was required, but no BEP JSON " + "was configured." + ), + next_steps=( + "Pass --bep-json from the matching bazel test invocation.", + ), + ) + if plan.remote_only_outputs or "remote-only" in lower_message: + return _ReportReason( + code="bep_output_remote_only_without_downloader", + message=( + "BEP selected remote-only outputs that could not be materialized " + "locally." + ), + next_steps=( + "Enable --remote-artifacts=download with a downloader, or adjust " + "Bazel remote download settings.", + ), + ) + if plan.cached_outputs and not plan.eligible_outputs: + return _ReportReason( + code="target_cached_by_bazel", + message=( + "Cached Bazel outputs did not satisfy the requested freshness " + "contract." + ), + next_steps=( + "Use the BEP from the exact matching bazel test invocation.", + ), + ) + return _ReportReason( + code="upload_failed_unknown", + message=message, + next_steps=( + "Correct the uploader configuration or freshness inputs and retry.", + ), + ) diff --git a/tools/core/uploader_py/codeowners.py b/tools/core/uploader_py/codeowners.py new file mode 100644 index 00000000..fa1873ad --- /dev/null +++ b/tools/core/uploader_py/codeowners.py @@ -0,0 +1,511 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Discover and compile one immutable CODEOWNERS matcher per invocation. + +Parsing once avoids repeated I/O and makes concurrent worker lookups read-only. +""" + +from __future__ import annotations + +from dataclasses import dataclass +import os +from pathlib import Path +import re +from typing import Any, MutableMapping +from urllib.parse import unquote + +from .json_utils import strict_json_dumps + +CODEOWNERS_LOCATIONS = ( + "CODEOWNERS", + ".github/CODEOWNERS", + ".gitlab/CODEOWNERS", + "docs/CODEOWNERS", + ".docs/CODEOWNERS", +) +_VALID_PERCENT_ESCAPE_RE = re.compile(r"%[0-9A-Fa-f]{2}") +_DRIVE_PATH_RE = re.compile(r"^[A-Za-z]:/") + + +@dataclass(frozen=True) +class CodeOwnersRule: + """One compiled rule; tuple order preserves last-match-wins behavior.""" + + pattern: str + regex_text: str + regex: re.Pattern[str] + owners: tuple[str, ...] + + +@dataclass(frozen=True) +class CodeOwnersMatch: + """Result for one source, including explicit empty-owner matches.""" + + matched: bool + owners: tuple[str, ...] = () + candidate: str | None = None + + @property + def json_value(self) -> str | None: + if not self.owners: + return None + return strict_json_dumps( + self.owners, + ensure_ascii=False, + separators=(",", ":"), + ) + + +@dataclass(frozen=True) +class CodeOwnersMatcher: + """Read-only rules and repository roots shared by every worker.""" + + source_path: Path | None + workspace_root: str + context_workspace: str + rules: tuple[CodeOwnersRule, ...] = () + warnings: tuple[str, ...] = () + windows_paths: bool = os.name == "nt" + + @property + def enabled(self) -> bool: + return bool(self.rules) + + def match_source(self, source_path: str) -> CodeOwnersMatch: + """Return the first candidate hit; rules use last-match-wins.""" + for candidate in source_candidates( + source_path, + workspace_root=self.workspace_root, + context_workspace=self.context_workspace, + windows_paths=self.windows_paths, + ): + match = self._match_candidate(candidate) + if match.matched: + return CodeOwnersMatch(True, match.owners, candidate) + return CodeOwnersMatch(False) + + def _match_candidate(self, candidate: str) -> CodeOwnersMatch: + matched_rule: CodeOwnersRule | None = None + for rule in self.rules: + if rule.regex.search(candidate): + matched_rule = rule + if matched_rule is None: + return CodeOwnersMatch(False) + return CodeOwnersMatch(True, _dedupe(matched_rule.owners), candidate) + + +@dataclass(frozen=True) +class CodeOwnersEnrichmentStats: + scanned: int = 0 + enriched: int = 0 + skipped_existing: int = 0 + skipped_missing_source: int = 0 + skipped_unmatched: int = 0 + skipped_errors: int = 0 + + +def _decode_percent_path(value: str) -> str: + if "%" not in value or re.search(r"(?i)%00", value): + return value + if "%" in _VALID_PERCENT_ESCAPE_RE.sub("", value): + return value + try: + return unquote(value, encoding="utf-8", errors="strict") + except UnicodeError: + return value + + +def normalize_path_like(raw: str) -> str | None: + """Normalize producer, URI, runfiles, Unix, and Windows source paths.""" + value = raw + if value.startswith("file://"): + value = value[len("file://") :] + value = _decode_percent_path(value).replace("\\", "/") + value = re.sub(r"/{2,}", "/", value) + while value.startswith("./"): + value = value[2:] + if re.match(r"^/[A-Za-z]:/", value): + value = value[1:] + + absolute = value.startswith("/") + if absolute: + value = value[1:] + stack: list[str] = [] + for part in value.split("/"): + if part in {"", "."}: + continue + if part == "..": + if not stack: + return None + stack.pop() + continue + stack.append(part) + joined = "/".join(stack) + return f"/{joined}" if absolute else joined + + +def _strip_workspace_prefix( + path_value: str, + root_value: str, + *, + windows_paths: bool, +) -> str | None: + if not path_value or not root_value: + return None + path_normalized = normalize_path_like(path_value) + root_normalized = normalize_path_like(root_value) + if not path_normalized or not root_normalized: + return None + comparison_path = path_normalized.casefold() if windows_paths else path_normalized + comparison_root = root_normalized.casefold() if windows_paths else root_normalized + if comparison_path == comparison_root: + return "" + prefix = f"{comparison_root}/" + if comparison_path.startswith(prefix): + return path_normalized[len(root_normalized) + 1 :] + return None + + +def source_candidates( + source_path: str, + *, + workspace_root: str = "", + context_workspace: str = "", + windows_paths: bool = os.name == "nt", +) -> tuple[str, ...]: + """Return priority-ordered, repository-relative ownership candidates.""" + normalized = normalize_path_like(source_path) + if not normalized: + return () + candidates: list[str] = [] + + def add(candidate: str | None, *, derived: bool = False) -> None: + if not candidate: + return + if derived and ( + candidate.startswith("external/") + or candidate.startswith("_main/external/") + ): + return + normalized_candidate = normalize_path_like(candidate) + if not normalized_candidate: + return + normalized_candidate = normalized_candidate.lstrip("/") + if not normalized_candidate or normalized_candidate.startswith("bazel-out/"): + return + if normalized_candidate not in candidates: + candidates.append(normalized_candidate) + + add( + _strip_workspace_prefix( + normalized, + context_workspace, + windows_paths=windows_paths, + ) + ) + add( + _strip_workspace_prefix( + normalized, + workspace_root, + windows_paths=windows_paths, + ) + ) + + for expression in ( + r"/execroot/[^/]+/_main/(.+)$", + r"/execroot/[^/]+/(.+)$", + r"\.runfiles/_main/(.+)$", + r"\.runfiles/[^/]+/(.+)$", + ): + match = re.search(expression, normalized) + if match: + add(match.group(1), derived=True) + + if not normalized.startswith("/") and not _DRIVE_PATH_RE.match(normalized): + add(normalized) + return tuple(candidates) + + +def _glob_to_regex(pattern: str) -> str: + output: list[str] = [] + index = 0 + while index < len(pattern): + character = pattern[index] + if character == "\\": + if index + 1 < len(pattern): + output.append(re.escape(pattern[index + 1])) + index += 2 + else: + output.append(r"\\") + index += 1 + continue + if character == "*" and index + 1 < len(pattern) and pattern[index + 1] == "*": + if index + 2 < len(pattern) and pattern[index + 2] == "/": + output.append("(.*/)?") + index += 3 + else: + output.append(".*") + index += 2 + continue + if character == "*": + output.append("[^/]*") + index += 1 + continue + if character == "?": + output.append("[^/]") + index += 1 + continue + if character == "[": + class_regex, next_index = _character_class(pattern, index) + output.append(class_regex) + index = next_index + continue + output.append(re.escape(character)) + index += 1 + return "".join(output) + + +def _character_class(pattern: str, start: int) -> tuple[str, int]: + index = start + 1 + body: list[str] = [] + if index < len(pattern) and pattern[index] == "!": + body.append("^") + index += 1 + elif index < len(pattern) and pattern[index] == "^": + body.append(r"\^") + index += 1 + if index < len(pattern) and pattern[index] == "]": + body.append(r"\]") + index += 1 + + while index < len(pattern): + character = pattern[index] + if character == "]": + return f"[{''.join(body)}]", index + 1 + if character in {"\\", "^", "["}: + body.append(f"\\{character}") + else: + body.append(character) + index += 1 + return r"\[", start + 1 + + +def compile_pattern(pattern: str) -> tuple[str, re.Pattern[str]]: + """Compile one legacy-compatible CODEOWNERS glob.""" + anchored = pattern.startswith("/") + directory_only = pattern.endswith("/") + raw = pattern[1:] if anchored else pattern + raw = raw[:-1] if directory_only else raw + if not raw: + raise ValueError("empty CODEOWNERS pattern") + prefix = "^" if anchored or "/" in raw else "(^|.*/)" + suffix = "/.*$" if directory_only else "($|/.*)" + regex_text = f"{prefix}{_glob_to_regex(raw)}{suffix}" + return regex_text, re.compile(regex_text) + + +def _split_pattern_and_owners(line: str) -> tuple[str, str]: + pattern: list[str] = [] + escaped = False + for index, character in enumerate(line): + if escaped: + pattern.append(character) + escaped = False + continue + if character == "\\": + pattern.append(character) + escaped = True + continue + if character.isspace(): + return "".join(pattern), line[index:].lstrip() + pattern.append(character) + return "".join(pattern), "" + + +def _is_gitlab_section_header_pattern(pattern: str) -> bool: + if not re.fullmatch(r"\[[^][]+\]", pattern): + return False + inner = pattern[1:-1] + if any(character.isspace() for character in inner): + return True + if any(character in inner for character in "-!^\\"): + return False + if re.fullmatch(r"[A-Z0-9]+", inner): + return False + if len(inner) <= 3 and re.fullmatch(r"[A-Za-z0-9]+", inner): + return False + if re.fullmatch(r"[a-z0-9]+", inner): + return False + return True + + +def _is_gitlab_section_header_line(line: str) -> bool: + match = re.fullmatch(r"(\[[^][]+\])(?:\s+.*)?", line) + return bool(match and _is_gitlab_section_header_pattern(match.group(1))) + + +def parse_codeowners(text: str) -> tuple[tuple[CodeOwnersRule, ...], tuple[str, ...]]: + """Parse and compile once, skipping malformed rules with diagnostics.""" + rules: list[CodeOwnersRule] = [] + warnings: list[str] = [] + for line_number, raw_line in enumerate(text.splitlines(), start=1): + line = raw_line.lstrip().rstrip("\r") + if not line or line.startswith("#") or _is_gitlab_section_header_line(line): + continue + pattern, owners_raw = _split_pattern_and_owners(line) + if not pattern or _is_gitlab_section_header_pattern(pattern): + continue + owners_raw = owners_raw.rstrip() + if owners_raw.startswith("#"): + owners_raw = "" + else: + owners_raw = re.sub(r"\s#.*$", "", owners_raw).rstrip() + owners = tuple(token for token in owners_raw.split() if token) + try: + regex_text, compiled = compile_pattern(pattern) + except (ValueError, re.error) as exc: + warnings.append( + f"ignored invalid CODEOWNERS rule at line {line_number}: {type(exc).__name__}" + ) + continue + rules.append(CodeOwnersRule(pattern, regex_text, compiled, owners)) + return tuple(rules), tuple(warnings) + + +def load_codeowners_matcher( + *, + explicit_path: Path | None, + workspace_root: Path, + context_workspace: str = "", + cwd: Path | None = None, + launcher_directory: Path | None = None, + windows_paths: bool = os.name == "nt", +) -> CodeOwnersMatcher: + """Discover, read, and compile the invocation-wide matcher exactly once.""" + candidates: list[Path] = [] + if explicit_path is not None and explicit_path.is_file(): + candidates.append(explicit_path) + else: + for root in (Path(context_workspace) if context_workspace else None, workspace_root): + if root is not None: + candidates.extend(root / location for location in CODEOWNERS_LOCATIONS) + candidates.append((cwd or Path.cwd()) / "CODEOWNERS") + if launcher_directory is not None: + candidates.append(launcher_directory / "CODEOWNERS") + + source_path = next((path for path in candidates if path.is_file()), None) + if source_path is None: + return CodeOwnersMatcher( + source_path=None, + workspace_root=str(workspace_root), + context_workspace=context_workspace, + windows_paths=windows_paths, + ) + try: + text = source_path.read_text(encoding="utf-8-sig") + except (OSError, UnicodeError) as exc: + return CodeOwnersMatcher( + source_path=source_path, + workspace_root=str(workspace_root), + context_workspace=context_workspace, + warnings=(f"failed to read CODEOWNERS: {type(exc).__name__}",), + windows_paths=windows_paths, + ) + rules, warnings = parse_codeowners(text) + return CodeOwnersMatcher( + source_path=source_path.resolve(), + workspace_root=str(workspace_root), + context_workspace=context_workspace, + rules=rules, + warnings=warnings, + windows_paths=windows_paths, + ) + + +def _dedupe(owners: tuple[str, ...]) -> tuple[str, ...]: + return tuple(dict.fromkeys(owner for owner in owners if owner)) + + +def _event_source_path(event: dict[str, Any]) -> str | None: + content = event.get("content") + if not isinstance(content, dict): + return None + meta = content.get("meta") + if isinstance(meta, dict): + for key in ( + "test.source.file", + "test.source.path", + "source.file", + "source.path", + ): + value = meta.get(key) + if isinstance(value, str) and value: + return value + source = content.get("source") + if isinstance(source, dict): + for key in ("file", "path"): + value = source.get(key) + if isinstance(value, str) and value: + return value + return None + + +def enrich_payload_codeowners( + payload: dict[str, Any], + matcher: CodeOwnersMatcher, + *, + cache: MutableMapping[str, CodeOwnersMatch] | None = None, +) -> CodeOwnersEnrichmentStats: + """Mutate one worker-owned payload while preserving producer owners.""" + if not matcher.enabled: + return CodeOwnersEnrichmentStats() + events = payload.get("events") + if not isinstance(events, list): + return CodeOwnersEnrichmentStats() + local_cache = cache if cache is not None else {} + counters = { + "scanned": 0, + "enriched": 0, + "skipped_existing": 0, + "skipped_missing_source": 0, + "skipped_unmatched": 0, + "skipped_errors": 0, + } + + for event in events: + if not isinstance(event, dict) or event.get("type") == "span": + continue + counters["scanned"] += 1 + try: + content = event.get("content") + meta = content.get("meta") if isinstance(content, dict) else None + if isinstance(meta, dict) and "test.codeowners" in meta: + counters["skipped_existing"] += 1 + continue + source_path = _event_source_path(event) + if not source_path: + counters["skipped_missing_source"] += 1 + continue + match = local_cache.get(source_path) + if match is None: + match = matcher.match_source(source_path) + local_cache[source_path] = match + if not match.owners: + counters["skipped_unmatched"] += 1 + continue + if not isinstance(content, dict): + counters["skipped_errors"] += 1 + continue + if not isinstance(meta, dict): + meta = {} + content["meta"] = meta + meta["test.codeowners"] = match.json_value + counters["enriched"] += 1 + except (AttributeError, TypeError, ValueError): + counters["skipped_errors"] += 1 + + return CodeOwnersEnrichmentStats(**counters) diff --git a/tools/core/uploader_py/config.py b/tools/core/uploader_py/config.py new file mode 100644 index 00000000..c59bd3f0 --- /dev/null +++ b/tools/core/uploader_py/config.py @@ -0,0 +1,487 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Resolve rule, environment, and CLI settings into immutable configuration. + +One precedence boundary preserves the legacy interface across all worker threads. +""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +import json +import math +import os +from pathlib import Path +import re +from typing import Mapping, Sequence + +from .json_utils import strict_json_loads +from .models import DEFAULT_WORKERS + + +CONFIG_SCHEMA_VERSION = 1 +DEFAULT_EXPECTED_ENRICHED_TAGS = ( + "git.repository_url", + "git.commit.sha", + "bazel.target", + "bazel.package", +) +VALID_FRESHNESS_MODES = frozenset({"auto", "required", "optional", "disabled"}) +VALID_FRESHNESS_SOURCES = frozenset({"auto", "bep", "execution_log"}) +VALID_ARTIFACT_SOURCES = frozenset({"auto", "bep", "local"}) +VALID_REMOTE_ARTIFACT_MODES = frozenset({"disabled", "download", "required"}) +PROXY_ENVIRONMENT_NAMES = ( + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "http_proxy", + "https_proxy", + "no_proxy", +) +_NON_NEGATIVE_INTEGER_RE = re.compile(r"^[0-9]+$") +_POSITIVE_DECIMAL_RE = re.compile(r"^[+]?([0-9]+([.][0-9]*)?|[.][0-9]+)$") + + +class ConfigError(ValueError): + """The generated config, environment, or CLI is invalid.""" + + +@dataclass(frozen=True) +class RuleConfig: + """Analysis-time values written by the Bazel uploader rule.""" + + quiescent_sec: int = 10 + max_wait_sec: int = 300 + fail_on_error: bool = False + debug: bool = False + keep_payloads: bool = False + filter_prefix: bool = False + gzip_payloads: bool = False + workers: int = DEFAULT_WORKERS + rules_version: str = "" + uploader_version: str = "" + context_manifest_path: str = "" + context_manifest_short_path: str = "" + telemetry_facts_manifest_path: str = "" + telemetry_facts_manifest_short_path: str = "" + schema_json_path: str = "" + schema_json_short_path: str = "" + doctor_runtime_path: str = "" + doctor_runtime_short_path: str = "" + expected_targets: tuple[str, ...] = () + expected_targets_file_path: str = "" + expected_targets_file_short_path: str = "" + runtime_selection: bool = False + + +@dataclass(frozen=True) +class UploaderConfig: + """Fully resolved immutable configuration shared with workers.""" + + rule: RuleConfig + workspace: Path + lock_workspace: str + invocation_cwd: Path + launcher_directory: Path | None + dry_run: bool + validate_enrichment: bool + debug: bool + quiescent_sec: int + max_wait_sec: int + max_depth: int + fail_on_error: bool + keep_payloads: bool + filter_prefix: bool + gzip_payloads: bool + workers: int + expected_enriched_tags: tuple[str, ...] + runtime_expected_targets: tuple[str, ...] + runtime_context_entries: tuple[str, ...] + bep_json_files: tuple[Path, ...] + freshness_source: str + freshness_mode: str + freshness_disabled_explicitly: bool + execution_log_json: Path | None + artifact_source: str + remote_artifacts: str + artifact_staging_dir: Path + bep_artifact_downloader: Path | None + bep_artifact_downloader_timeout_sec: float + report_json: Path | None + testlogs_dir: Path | None + codeowners_file: Path | None + context_json: Path | None + api_key: str + site: str + agent_url: str + agentless_url: str + proxy_environment: tuple[tuple[str, str], ...] + ci: bool + + @property + def agentless(self) -> bool: + """Whether requests go directly to public intake.""" + return not self.agent_url + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="dd_upload_payloads", allow_abbrev=False) + parser.add_argument("--config", required=True, help=argparse.SUPPRESS) + parser.add_argument("--dry-run", action="store_true") + parser.add_argument("--validate-enrichment", action="store_true") + parser.add_argument("--expected-enriched-tag", action="append", default=[]) + parser.add_argument("--expected-target", action="append", default=[]) + parser.add_argument("--context-entry", action="append", default=[]) + parser.add_argument("--bep-json", action="append", default=[]) + parser.add_argument("--freshness-source") + parser.add_argument("--freshness-mode") + parser.add_argument("--allow-cached-payload-uploads", action="store_true") + parser.add_argument("--execution-log-json") + parser.add_argument("--execution-log-mode") + parser.add_argument("--artifact-source") + parser.add_argument("--remote-artifacts") + parser.add_argument("--artifact-staging-dir") + parser.add_argument("--bep-artifact-downloader") + parser.add_argument("--bep-artifact-downloader-timeout-sec") + parser.add_argument("--report-json") + parser.add_argument("--debug", action="store_true", default=None) + parser.add_argument("--workers") + return parser + + +def parse_uploader_config( + argv: Sequence[str], + *, + environ: Mapping[str, str] | None = None, + cwd: Path | None = None, +) -> UploaderConfig: + """Resolve generated config, environment, and CLI using public precedence.""" + env = dict(os.environ if environ is None else environ) + args = _parser().parse_args(list(argv)) + + def text_option( + cli_value: str | None, + environment_variable: str, + default: str = "", + ) -> str: + """Prefer an explicit CLI value to its environment fallback.""" + return cli_value if cli_value is not None else env.get(environment_variable) or default + + config_path = Path(args.config) + rule = load_rule_config(config_path) + invocation_cwd = Path(cwd or Path.cwd()).absolute() + lock_workspace = env.get("BUILD_WORKSPACE_DIRECTORY") or str(invocation_cwd) + workspace = Path(lock_workspace).resolve() + launcher_directory = _optional_path( + env.get("DD_TEST_OPTIMIZATION_UPLOADER_LAUNCHER_DIR", "") + ) + + quiescent_sec = _non_negative_integer( + "DD_TEST_OPTIMIZATION_QUIESCENT_SEC", + env.get("DD_TEST_OPTIMIZATION_QUIESCENT_SEC") or rule.quiescent_sec, + ) + max_wait_sec = _non_negative_integer( + "DD_TEST_OPTIMIZATION_MAX_WAIT_SEC", + env.get("DD_TEST_OPTIMIZATION_MAX_WAIT_SEC") or rule.max_wait_sec, + ) + max_depth = _non_negative_integer( + "DD_TEST_OPTIMIZATION_MAX_DEPTH", + env.get("DD_TEST_OPTIMIZATION_MAX_DEPTH") or 0, + ) + workers = _positive_integer( + "--workers/DD_TEST_OPTIMIZATION_WORKERS", + args.workers + if args.workers is not None + else env.get("DD_TEST_OPTIMIZATION_WORKERS") or rule.workers, + ) + + debug = ( + True + if args.debug is True + else _environment_bool(env, "DD_TEST_OPTIMIZATION_DEBUG", rule.debug) + ) + keep_payloads = _environment_bool( + env, "DD_TEST_OPTIMIZATION_KEEP_PAYLOADS", rule.keep_payloads + ) + filter_prefix = _environment_bool( + env, "DD_TEST_OPTIMIZATION_FILTER_PREFIX", rule.filter_prefix + ) + gzip_payloads = _environment_bool( + env, "DD_TEST_OPTIMIZATION_GZIP", rule.gzip_payloads + ) + + freshness_source_text = text_option( + args.freshness_source, + "DD_TEST_OPTIMIZATION_FRESHNESS_SOURCE", + "auto", + ) + freshness_source = _choice( + "--freshness-source/DD_TEST_OPTIMIZATION_FRESHNESS_SOURCE", + freshness_source_text, + VALID_FRESHNESS_SOURCES, + ) + freshness_mode_text = text_option(args.freshness_mode, "DD_TEST_OPTIMIZATION_FRESHNESS_MODE") + legacy_freshness_mode = text_option( + args.execution_log_mode, + "DD_TEST_OPTIMIZATION_EXECUTION_LOG_MODE", + ) + freshness_mode = _choice( + "--freshness-mode/DD_TEST_OPTIMIZATION_FRESHNESS_MODE", + freshness_mode_text or legacy_freshness_mode or "auto", + VALID_FRESHNESS_MODES, + ) + if args.allow_cached_payload_uploads: + freshness_mode = "disabled" + + artifact_source_text = text_option( + args.artifact_source, + "DD_TEST_OPTIMIZATION_ARTIFACT_SOURCE", + "local", + ) + artifact_source = _choice( + "--artifact-source/DD_TEST_OPTIMIZATION_ARTIFACT_SOURCE", + artifact_source_text, + VALID_ARTIFACT_SOURCES, + ) + remote_artifacts_text = text_option( + args.remote_artifacts, + "DD_TEST_OPTIMIZATION_REMOTE_ARTIFACTS", + "disabled", + ) + remote_artifacts = _choice( + "--remote-artifacts/DD_TEST_OPTIMIZATION_REMOTE_ARTIFACTS", + remote_artifacts_text, + VALID_REMOTE_ARTIFACT_MODES, + ) + downloader_timeout_text = text_option( + args.bep_artifact_downloader_timeout_sec, + "DD_TEST_OPTIMIZATION_BEP_ARTIFACT_DOWNLOADER_TIMEOUT_SEC", + "300", + ) + downloader_timeout = _positive_decimal( + "--bep-artifact-downloader-timeout-sec", + downloader_timeout_text, + ) + + staging_path_text = text_option( + args.artifact_staging_dir, + "DD_TEST_OPTIMIZATION_ARTIFACT_STAGING_DIR", + ) + artifact_staging_dir = ( + Path(staging_path_text) + if staging_path_text + else workspace / ".topt" / "bep-artifacts" + ) + if not artifact_staging_dir.is_absolute(): + artifact_staging_dir = workspace / artifact_staging_dir + + bep_json_paths: list[str] = [] + environment_bep_path = env.get("DD_TEST_OPTIMIZATION_BEP_JSON", "") + if environment_bep_path: + bep_json_paths.append(environment_bep_path) + bep_json_paths.extend(args.bep_json) + + expected_enriched_tags = ( + tuple(args.expected_enriched_tag) or DEFAULT_EXPECTED_ENRICHED_TAGS + ) + report_path_text = text_option( + args.report_json, + "DD_TEST_OPTIMIZATION_UPLOADER_REPORT_JSON", + ) + execution_log_path_text = text_option( + args.execution_log_json, + "DD_TEST_OPTIMIZATION_EXECUTION_LOG_JSON", + ) + downloader_path_text = text_option( + args.bep_artifact_downloader, + "DD_TEST_OPTIMIZATION_BEP_ARTIFACT_DOWNLOADER", + ) + + return UploaderConfig( + rule=rule, + workspace=workspace, + lock_workspace=lock_workspace, + invocation_cwd=invocation_cwd, + launcher_directory=launcher_directory, + dry_run=args.dry_run, + validate_enrichment=args.validate_enrichment, + debug=debug, + quiescent_sec=quiescent_sec, + max_wait_sec=max_wait_sec, + max_depth=max_depth, + fail_on_error=rule.fail_on_error, + keep_payloads=keep_payloads, + filter_prefix=filter_prefix, + gzip_payloads=gzip_payloads, + workers=workers, + expected_enriched_tags=expected_enriched_tags, + runtime_expected_targets=tuple(args.expected_target), + runtime_context_entries=tuple(args.context_entry), + bep_json_files=tuple(Path(path) for path in bep_json_paths), + freshness_source=freshness_source, + freshness_mode=freshness_mode, + freshness_disabled_explicitly=args.allow_cached_payload_uploads, + execution_log_json=_optional_path(execution_log_path_text), + artifact_source=artifact_source, + remote_artifacts=remote_artifacts, + artifact_staging_dir=artifact_staging_dir, + bep_artifact_downloader=_optional_path(downloader_path_text), + bep_artifact_downloader_timeout_sec=downloader_timeout, + report_json=_optional_path(report_path_text), + testlogs_dir=_optional_path(env.get("TESTLOGS_DIR", "")), + codeowners_file=_optional_path(env.get("DD_TEST_OPTIMIZATION_CODEOWNERS_FILE", "")), + context_json=_optional_path(env.get("DD_TEST_OPTIMIZATION_CONTEXT_JSON", "")), + api_key=env.get("DD_API_KEY", ""), + site=env.get("DD_SITE", "") or "datadoghq.com", + agent_url=env.get("DD_TEST_OPTIMIZATION_AGENT_URL", ""), + agentless_url=env.get("DD_TEST_OPTIMIZATION_AGENTLESS_URL", ""), + proxy_environment=tuple( + (name, env[name]) for name in PROXY_ENVIRONMENT_NAMES if env.get(name) + ), + ci=env.get("CI", "").strip().lower() not in {"", "0", "false", "no"}, + ) + + +def validate_upload_credentials(config: UploaderConfig) -> None: + """Validate credentials only after dry-run and endpoint mode are known.""" + if config.agentless and not config.dry_run and not config.api_key: + raise ConfigError("DD_API_KEY required for agentless uploads") + + +def load_rule_config(path: Path) -> RuleConfig: + """Load the small JSON config generated by the Bazel rule.""" + try: + raw = strict_json_loads(path.read_text(encoding="utf-8-sig")) + except FileNotFoundError as exc: + raise ConfigError(f"uploader config does not exist: {path}") from exc + except OSError as exc: + raise ConfigError(f"failed to read uploader config {path}: {exc}") from exc + except json.JSONDecodeError as exc: + raise ConfigError(f"invalid uploader config JSON in {path}: {exc}") from exc + if not isinstance(raw, dict): + raise ConfigError("uploader config must be a JSON object") + + schema_version = _json_integer(raw, "schema_version", CONFIG_SCHEMA_VERSION, minimum=1) + if schema_version != CONFIG_SCHEMA_VERSION: + raise ConfigError( + f"unsupported uploader config schema_version {schema_version}; " + f"expected {CONFIG_SCHEMA_VERSION}" + ) + # Launchers emit this key for schema compatibility; Python does not use it. + _json_string(raw, "workspace_name", "") + workers = _json_integer(raw, "workers", DEFAULT_WORKERS, minimum=1) + return RuleConfig( + quiescent_sec=_json_integer(raw, "quiescent_sec", 10, minimum=0), + max_wait_sec=_json_integer(raw, "max_wait_sec", 300, minimum=0), + fail_on_error=_json_boolean(raw, "fail_on_error", False), + debug=_json_boolean(raw, "debug", False), + keep_payloads=_json_boolean(raw, "keep_payloads", False), + filter_prefix=_json_boolean(raw, "filter_prefix", False), + gzip_payloads=_json_boolean(raw, "gzip_payloads", False), + workers=workers, + rules_version=_json_string(raw, "rules_version", ""), + uploader_version=_json_string(raw, "uploader_version", ""), + context_manifest_path=_json_string(raw, "context_manifest_path", ""), + context_manifest_short_path=_json_string(raw, "context_manifest_short_path", ""), + telemetry_facts_manifest_path=_json_string( + raw, "telemetry_facts_manifest_path", "" + ), + telemetry_facts_manifest_short_path=_json_string( + raw, "telemetry_facts_manifest_short_path", "" + ), + schema_json_path=_json_string(raw, "schema_json_path", ""), + schema_json_short_path=_json_string(raw, "schema_json_short_path", ""), + doctor_runtime_path=_json_string(raw, "doctor_runtime_path", ""), + doctor_runtime_short_path=_json_string( + raw, "doctor_runtime_short_path", "" + ), + expected_targets=_json_string_tuple(raw, "expected_targets"), + expected_targets_file_path=_json_string(raw, "expected_targets_file_path", ""), + expected_targets_file_short_path=_json_string( + raw, "expected_targets_file_short_path", "" + ), + runtime_selection=_json_boolean(raw, "runtime_selection", False), + ) + + +def _environment_bool(env: Mapping[str, str], name: str, fallback: bool) -> bool: + value = env.get(name) + if not value: + return fallback + return value.lower() in {"1", "true", "yes"} + + +def _non_negative_integer(name: str, value: object) -> int: + text = str(value) + if not _NON_NEGATIVE_INTEGER_RE.fullmatch(text): + raise ConfigError(f"{name} must be a non-negative integer, got: {text!r}") + return int(text) + + +def _positive_integer(name: str, value: object) -> int: + result = _non_negative_integer(name, value) + if result == 0: + raise ConfigError(f"{name} must be a positive integer, got: {str(value)!r}") + return result + + +def _positive_decimal(name: str, value: object) -> float: + text = str(value) + if not _POSITIVE_DECIMAL_RE.fullmatch(text): + raise ConfigError(f"{name} must be a finite number greater than zero") + try: + result = float(text) + except (TypeError, ValueError) as exc: + raise ConfigError(f"{name} must be a finite number greater than zero") from exc + if not math.isfinite(result) or result <= 0: + raise ConfigError(f"{name} must be a finite number greater than zero") + return result + + +def _choice(name: str, value: object, allowed: frozenset[str]) -> str: + normalized = str(value).lower() + if normalized not in allowed: + raise ConfigError( + f"{name} must be one of: {', '.join(sorted(allowed))}; got {value!r}" + ) + return normalized + + +def _optional_path(value: object) -> Path | None: + text = str(value) if value is not None else "" + return Path(text) if text else None + + +def _json_integer( + raw: Mapping[str, object], name: str, default: int, *, minimum: int +) -> int: + value = raw.get(name, default) + if isinstance(value, bool) or not isinstance(value, int) or value < minimum: + raise ConfigError(f"uploader config field {name!r} must be an integer >= {minimum}") + return value + + +def _json_boolean(raw: Mapping[str, object], name: str, default: bool) -> bool: + value = raw.get(name, default) + if not isinstance(value, bool): + raise ConfigError(f"uploader config field {name!r} must be a boolean") + return value + + +def _json_string(raw: Mapping[str, object], name: str, default: str) -> str: + value = raw.get(name, default) + if not isinstance(value, str): + raise ConfigError(f"uploader config field {name!r} must be a string") + return value + + +def _json_string_tuple(raw: Mapping[str, object], name: str) -> tuple[str, ...]: + value = raw.get(name, []) + if not isinstance(value, list) or any(not isinstance(item, str) for item in value): + raise ConfigError(f"uploader config field {name!r} must be a string array") + return tuple(value) diff --git a/tools/core/uploader_py/coordinator.py b/tools/core/uploader_py/coordinator.py new file mode 100644 index 00000000..2e782283 --- /dev/null +++ b/tools/core/uploader_py/coordinator.py @@ -0,0 +1,309 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Prepare shared inputs, run file workers, and aggregate immutable results. + +This boundary keeps invocation-wide state out of independent worker pipelines. +""" + +from __future__ import annotations + +from dataclasses import dataclass +import logging +from pathlib import Path +import time +from typing import Callable +import uuid + +from .codeowners import CodeOwnersMatcher, load_codeowners_matcher +from .config import UploaderConfig +from .credentials import check_api_key_fingerprint +from .discovery import DiscoveryResult +from .endpoints import EndpointSet +from .file_worker import WorkerRuntime, process_file +from .models import FileStatus +from .reporting import AggregateReport +from .resources import LoadedResources +from .telemetry import TelemetryPlan, build_telemetry_plan +from .temporary import invocation_temporary_directory +from .transport import HttpTransport, validate_proxy_environment +from .worker_pool import ( + WorkerPoolInterrupted, + WorkerPoolRun, + run_file_workers, +) + + +@dataclass(frozen=True) +class CoordinatorSettings: + """Worker-facing configuration with lifecycle-only options removed.""" + + workspace: Path + workers: int + dry_run: bool + validate_enrichment: bool + expected_enriched_tags: tuple[str, ...] + gzip_payloads: bool + keep_payloads: bool + filter_prefix: bool + rules_version: str + uploader_version: str + api_key: str + proxy_environment: tuple[tuple[str, str], ...] = () + codeowners_file: Path | None = None + invocation_cwd: Path | None = None + launcher_directory: Path | None = None + + @classmethod + def from_config(cls, config: UploaderConfig) -> "CoordinatorSettings": + return cls( + workspace=config.workspace, + workers=config.workers, + dry_run=config.dry_run, + validate_enrichment=config.validate_enrichment, + expected_enriched_tags=config.expected_enriched_tags, + gzip_payloads=config.gzip_payloads, + keep_payloads=config.keep_payloads, + filter_prefix=config.filter_prefix, + rules_version=config.rule.rules_version, + uploader_version=config.rule.uploader_version, + api_key=config.api_key, + proxy_environment=config.proxy_environment, + codeowners_file=config.codeowners_file, + invocation_cwd=config.invocation_cwd, + launcher_directory=config.launcher_directory, + ) + + +@dataclass(frozen=True) +class _WorkerInputs: + """Read-only resources prepared once before any worker starts.""" + + codeowners: CodeOwnersMatcher + telemetry: TelemetryPlan + warning_codes: tuple[str, ...] + + +def _prepare_worker_inputs( + discovery: DiscoveryResult, + *, + settings: CoordinatorSettings, + endpoints: EndpointSet, + resources: LoadedResources, + logger: logging.Logger | None, +) -> _WorkerInputs: + """Validate transport settings and snapshot shared enrichment resources.""" + validate_proxy_environment(settings.proxy_environment) + codeowners = load_codeowners_matcher( + explicit_path=settings.codeowners_file, + workspace_root=settings.workspace, + context_workspace=resources.context_workspace, + cwd=settings.invocation_cwd, + launcher_directory=settings.launcher_directory, + ) + telemetry = build_telemetry_plan( + discovery.tasks, + resources.telemetry_facts_paths, + primary_context=resources.primary_context, + ) + fingerprint = check_api_key_fingerprint( + resources.primary_context, + api_key=settings.api_key, + agentless=endpoints.agentless, + ) + fingerprint_warnings = ( + (fingerprint.warning_code,) if fingerprint.warning_code is not None else () + ) + warnings = tuple( + dict.fromkeys( + discovery.warning_codes + + resources.warning_codes + + codeowners.warnings + + telemetry.warning_codes + + fingerprint_warnings + ) + ) + if logger is not None: + logger.debug( + "pre-worker resources ready: files=%d contexts=%d " + "codeowners_file=%s codeowners_rules=%d telemetry_directives=%d", + len(discovery.tasks), + len(resources.context_plan.by_repo), + str(codeowners.source_path) if codeowners.source_path is not None else "none", + len(codeowners.rules), + len(telemetry.entries), + ) + if fingerprint.status == "mismatch": + logger.warning( + "warning_code=api_key_fingerprint_mismatch " + "DD_API_KEY mismatch between fetch and uploader" + ) + elif fingerprint.status == "evp_skipped": + logger.warning( + "warning_code=api_key_fingerprint_evp_skipped DD_API_KEY " + "fingerprint present but uploader is in EVP mode; check skipped" + ) + else: + logger.debug("DD_API_KEY fingerprint check status=%s", fingerprint.status) + for warning in (code for code in warnings if code not in fingerprint_warnings): + logger.warning("pre-worker warning_code=%s", warning) + return _WorkerInputs(codeowners, telemetry, warnings) + + +def run_discovered_tasks( + discovery: DiscoveryResult, + *, + settings: CoordinatorSettings, + endpoints: EndpointSet, + resources: LoadedResources, + logger: logging.Logger | None = None, + transport_factory: Callable[[], object] | None = None, + clock: Callable[[], float] = time.monotonic, + identifier_factory: Callable[[], str] = lambda: str(uuid.uuid4()), +) -> AggregateReport: + """Run already-authorized tasks; workers never mutate coordinator state.""" + started = clock() + worker_inputs = _prepare_worker_inputs( + discovery, + settings=settings, + endpoints=endpoints, + resources=resources, + logger=logger, + ) + + temporary_cleanup_errors: list[str] = [] + interrupted = False + cancelled = 0 + with invocation_temporary_directory( + on_cleanup_error=temporary_cleanup_errors.append, + ) as temporary_root: + if logger is not None: + logger.debug("invocation temporary root created: %s", temporary_root) + runtime = WorkerRuntime( + endpoints=endpoints, + invocation_temp_root=temporary_root, + context_plan=resources.context_plan, + codeowners_matcher=worker_inputs.codeowners, + runtime_id=identifier_factory(), + rules_version=settings.rules_version, + uploader_version=settings.uploader_version, + api_key=settings.api_key, + schema=resources.schema, + dry_run=settings.dry_run, + validate_enrichment=settings.validate_enrichment, + expected_enriched_tags=settings.expected_enriched_tags, + gzip_payloads=settings.gzip_payloads, + keep_payloads=settings.keep_payloads, + filter_prefix=settings.filter_prefix, + telemetry_session_id=identifier_factory(), + telemetry_plan=worker_inputs.telemetry, + logger=logger, + ) + create_transport = transport_factory or ( + lambda: HttpTransport( + proxy_environment=settings.proxy_environment, + logger=logger, + ) + ) + try: + worker_run: WorkerPoolRun = run_file_workers( + discovery.tasks, + workers=settings.workers, + runtime=runtime, + transport_factory=create_transport, + process_file=process_file, + logger=logger, + ) + except WorkerPoolInterrupted as exc: + worker_run = exc.run + interrupted = True + cancelled = exc.cancelled + + additional_warnings: tuple[str, ...] = () + if temporary_cleanup_errors: + additional_warnings += ("invocation_temp_cleanup_failed",) + if logger is not None: + logger.warning( + "invocation temporary cleanup failed: %s", + temporary_cleanup_errors[0], + ) + elif logger is not None: + logger.debug("invocation temporary root cleanup completed") + if interrupted: + additional_warnings += ("invocation_interrupted",) + if logger is not None: + logger.error( + "interrupted after completed=%d cancelled=%d", + len(worker_run.results), + cancelled, + ) + initialization_warnings = tuple( + dict.fromkeys(worker_inputs.warning_codes + additional_warnings) + ) + + if logger is not None: + _log_worker_results(worker_run, logger) + + exit_code = ( + 130 + if interrupted + else int(any(result.status is FileStatus.FAILED for result in worker_run.results)) + ) + report = AggregateReport.create( + dry_run=settings.dry_run, + exit_code=exit_code, + configured_workers=settings.workers, + worker_threads=worker_run.worker_threads, + peak_active_workers=worker_run.peak_active_workers, + elapsed_seconds=max(0.0, clock() - started), + discovered_by_type=discovery.counts(), + results=worker_run.results, + cancelled=cancelled, + initialization_warning_codes=initialization_warnings, + ) + if logger is not None: + stats = report.statistics() + logger.debug( + "coordinator completed exit_code=%d processed=%d cancelled=%d " + "requests_attempted=%d elapsed=%.3fs", + report.exit_code, + stats["files"]["processed"], + stats["files"]["cancelled"], + stats["requests"]["attempted"], + report.elapsed_seconds, + ) + return report + + +def _log_worker_results(worker_run: WorkerPoolRun, logger: logging.Logger) -> None: + """Emit task diagnostics after workers stop, keeping logging centralized.""" + for result in worker_run.results: + logger.debug( + "task=%s type=%s terminal_status=%s attempts=%d retries=%d", + result.task_id, + result.payload_type.value, + result.status.value, + result.requests_attempted, + result.retries, + ) + for warning_code in result.warning_codes: + log_method = ( + logger.debug if result.status is FileStatus.SKIPPED else logger.warning + ) + log_method( + "task=%s file=%s warning_code=%s", + result.task_id, + result.source_path, + warning_code, + ) + if result.status is FileStatus.FAILED: + logger.error( + "task=%s file=%s failure_code=%s detail=%s", + result.task_id, + result.source_path, + result.failure_code or "unknown", + result.failure_message or "none", + ) diff --git a/tools/core/uploader_py/credentials.py b/tools/core/uploader_py/credentials.py new file mode 100644 index 00000000..0dffaa57 --- /dev/null +++ b/tools/core/uploader_py/credentials.py @@ -0,0 +1,73 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Compare fetch and upload credential fingerprints without exposing secrets. + +The warning-only preflight detects configuration drift outside request processing. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Mapping + + +FINGERPRINT_ALPHABET = ( + "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + "-_:/.+@=#%~!$^*()[]{}<>?,;|\\\"'` " +) + + +@dataclass(frozen=True) +class ApiKeyFingerprintCheck: + """Safe pre-worker diagnostic derived from the primary context.""" + + status: str + warning_code: str | None = None + + +def api_key_fingerprint(api_key: str) -> str: + """Match the non-cryptographic sync-side FNV-1a-style fingerprint.""" + if not api_key: + return "" + value = 2_166_136_261 + alphabet_length = len(FINGERPRINT_ALPHABET) + for index, character in enumerate(api_key): + alphabet_index = FINGERPRINT_ALPHABET.find(character) + if alphabet_index < 0: + alphabet_index = alphabet_length + (index % 7) + value ^= alphabet_index + value = (value * 16_777_619) & 0xFFFFFFFF + return f"{value:08x}" + + +def check_api_key_fingerprint( + primary_context: Mapping[str, Any] | None, + *, + api_key: str, + agentless: bool, +) -> ApiKeyFingerprintCheck: + """Compare once before workers; mismatches remain warning-only.""" + expected = ( + primary_context.get("topt.api_key_fingerprint") + if primary_context is not None + else None + ) + if not isinstance(expected, str) or not expected: + return ApiKeyFingerprintCheck("absent") + if not agentless: + return ApiKeyFingerprintCheck( + "evp_skipped", + "api_key_fingerprint_evp_skipped", + ) + if not api_key: + return ApiKeyFingerprintCheck("api_key_unset") + if api_key_fingerprint(api_key) != expected: + return ApiKeyFingerprintCheck( + "mismatch", + "api_key_fingerprint_mismatch", + ) + return ApiKeyFingerprintCheck("match") diff --git a/tools/core/uploader_py/discovery.py b/tools/core/uploader_py/discovery.py new file mode 100644 index 00000000..9ab47933 --- /dev/null +++ b/tools/core/uploader_py/discovery.py @@ -0,0 +1,296 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Build a deterministic one-task-per-source plan from Bazel testlogs. + +Discovery happens before scheduling so workers never race filesystem traversal. +""" + +from __future__ import annotations + +from dataclasses import dataclass +import os +from pathlib import Path +import time +from typing import Callable, Iterable + +from .models import FileTask, PayloadType + + +_PAYLOAD_SUBDIRECTORIES = ( + (PayloadType.TEST, Path("payloads") / "tests"), + (PayloadType.COVERAGE, Path("payloads") / "coverage"), + (PayloadType.TELEMETRY, Path("payloads") / "telemetry"), +) +_PAYLOAD_SUFFIXES = frozenset({".json", ".msgpack"}) + + +class DiscoveryError(ValueError): + """Configured testlogs discovery cannot be performed safely.""" + + +@dataclass(frozen=True) +class ScanRoot: + path: Path + staged: bool = False + + +@dataclass(frozen=True) +class DiscoveredOutput: + path: Path + output_key: str + scan_root: Path + staged: bool + + +@dataclass(frozen=True) +class DiscoveryResult: + outputs: tuple[DiscoveredOutput, ...] + tasks: tuple[FileTask, ...] + discovered_by_type: tuple[tuple[PayloadType, int], ...] + warning_codes: tuple[str, ...] = () + + def counts(self) -> dict[PayloadType, int]: + return dict(self.discovered_by_type) + + +@dataclass(frozen=True) +class QuiescenceResult: + discovery: DiscoveryResult + reason: str + elapsed_seconds: float + + +def count_tasks_by_payload_type( + tasks: Iterable[FileTask], +) -> tuple[tuple[PayloadType, int], ...]: + """Count supported tasks once in the stable public payload-type order.""" + counts = {payload_type: 0 for payload_type in PayloadType} + for task in tasks: + if task.payload_type in counts: + counts[task.payload_type] += 1 + return tuple(counts.items()) + + +def resolve_local_testlogs_root( + *, + explicit: Path | None, + workspace: Path, + cwd: Path, +) -> Path | None: + """Resolve legacy TESTLOGS_DIR precedence without invoking Bazel recursively.""" + if explicit is not None: + if not explicit.is_dir(): + raise DiscoveryError(f"TESTLOGS_DIR is not a directory: {explicit}") + return explicit.resolve() + candidates = (workspace / "bazel-testlogs", cwd / "bazel-testlogs") + for candidate in candidates: + if candidate.is_dir(): + return candidate.resolve() + return None + + +def discover_file_tasks( + scan_roots: Iterable[ScanRoot], + *, + max_depth: int = 0, + staged_output_keys: Iterable[str] = (), +) -> DiscoveryResult: + """Discover sources once and assign stable IDs without reading payload bodies.""" + if max_depth < 0: + raise DiscoveryError("max_depth must be non-negative") + normalized_roots = _normalize_scan_roots(scan_roots) + selected_for_staging = frozenset( + key.replace("\\", "/").lstrip("/") for key in staged_output_keys if key + ) + outputs_by_key: dict[str, list[DiscoveredOutput]] = {} + for scan_root in normalized_roots: + for output_path in _find_test_outputs(scan_root.path, max_depth=max_depth): + output_key = output_path.relative_to(scan_root.path).as_posix() + outputs_by_key.setdefault(output_key, []).append( + DiscoveredOutput( + path=output_path, + output_key=output_key, + scan_root=scan_root.path, + staged=scan_root.staged, + ) + ) + + warnings: list[str] = [] + outputs: list[DiscoveredOutput] = [] + for output_key in sorted(outputs_by_key): + candidate_outputs = outputs_by_key[output_key] + if output_key in selected_for_staging: + staged_outputs = [output for output in candidate_outputs if output.staged] + if staged_outputs: + outputs.append(staged_outputs[0]) + continue + warnings.append("selected_staged_output_missing") + continue + outputs.append(candidate_outputs[0]) + + discovered_sources: list[tuple[Path, str, PayloadType, DiscoveredOutput]] = [] + seen_sources: set[str] = set() + for output in outputs: + for payload_type, relative_directory in _PAYLOAD_SUBDIRECTORIES: + payload_directory = output.path / relative_directory + if not payload_directory.is_dir(): + if payload_directory.is_symlink(): + warnings.append("payload_symlink_skipped") + continue + if not _safe_payload_directory(payload_directory, output.path): + warnings.append("payload_symlink_skipped") + continue + try: + sources = sorted( + payload_directory.iterdir(), + key=lambda item: item.name, + ) + except OSError: + warnings.append("payload_directory_unreadable") + continue + for source_path in sources: + if source_path.is_symlink(): + if source_path.suffix.lower() in _PAYLOAD_SUFFIXES: + warnings.append("payload_symlink_skipped") + continue + if not source_path.is_file() or source_path.suffix.lower() not in _PAYLOAD_SUFFIXES: + continue + source_key = str(source_path.resolve(strict=False)) + if source_key in seen_sources: + continue + seen_sources.add(source_key) + relative_source = source_path.relative_to(output.path).as_posix() + display_path = f"{output.output_key}/{relative_source}" + discovered_sources.append( + (source_path, display_path, payload_type, output) + ) + + tasks = tuple( + FileTask( + task_id=f"file-{index:06d}", + source_path=source_path, + display_path=display_path, + payload_type=payload_type, + test_outputs_dir=output.path, + output_key=output.output_key, + ) + for index, (source_path, display_path, payload_type, output) in enumerate( + discovered_sources, + start=1, + ) + ) + if max_depth > 0 and not outputs: + warnings.append("max_depth_may_be_too_shallow") + return DiscoveryResult( + outputs=tuple(outputs), + tasks=tasks, + discovered_by_type=count_tasks_by_payload_type(tasks), + warning_codes=tuple(dict.fromkeys(warnings)), + ) + + +def payload_latest_mtime(discovery: DiscoveryResult) -> float: + """Return the latest known source mtime, or zero when no source exists.""" + latest = 0.0 + for task in discovery.tasks: + try: + latest = max(latest, task.source_path.stat().st_mtime) + except OSError: + continue + return latest + + +def tests_executed(scan_roots: Iterable[ScanRoot]) -> bool: + """Detect Bazel test.log/test.xml markers without following directory links.""" + for scan_root in _normalize_scan_roots(scan_roots): + for _directory, _subdirectories, filenames in os.walk( + scan_root.path, + followlinks=False, + ): + if "test.log" in filenames or "test.xml" in filenames: + return True + return False + + +def wait_for_quiescence( + discover: Callable[[], DiscoveryResult], + *, + quiescent_seconds: int, + max_wait_seconds: int, + poll_seconds: float = 2.0, + clock: Callable[[], float] = time.time, + sleeper: Callable[[float], None] = time.sleep, +) -> QuiescenceResult: + """Refresh discovery until payload files settle or the wait budget expires.""" + if quiescent_seconds < 0 or max_wait_seconds < 0: + raise DiscoveryError("quiescence and maximum wait must be non-negative") + if poll_seconds <= 0: + raise DiscoveryError("poll_seconds must be positive") + started = clock() + while True: + current = discover() + now = clock() + elapsed = max(0.0, now - started) + if not current.tasks: + if max_wait_seconds == 0: + return QuiescenceResult(current, "no_payload_immediate", elapsed) + if elapsed >= max_wait_seconds: + return QuiescenceResult(current, "max_wait", elapsed) + else: + latest_mtime = payload_latest_mtime(current) + idle = max(0.0, now - latest_mtime) + if quiescent_seconds == 0 or idle >= quiescent_seconds: + return QuiescenceResult(current, "quiescent", elapsed) + if max_wait_seconds == 0 or elapsed >= max_wait_seconds: + return QuiescenceResult(current, "max_wait", elapsed) + remaining = max_wait_seconds - elapsed + sleeper(min(poll_seconds, remaining) if max_wait_seconds else poll_seconds) + + +def _normalize_scan_roots(scan_roots: Iterable[ScanRoot]) -> tuple[ScanRoot, ...]: + normalized: list[ScanRoot] = [] + seen: set[str] = set() + for scan_root in scan_roots: + if not scan_root.path.is_dir(): + raise DiscoveryError(f"testlogs scan root is not a directory: {scan_root.path}") + resolved = scan_root.path.resolve() + key = str(resolved) + if key in seen: + continue + seen.add(key) + normalized.append(ScanRoot(resolved, scan_root.staged)) + return tuple(normalized) + + +def _find_test_outputs(root: Path, *, max_depth: int) -> tuple[Path, ...]: + found: list[Path] = [] + for directory, subdirectories, _filenames in os.walk(root, followlinks=False): + current = Path(directory) + relative = current.relative_to(root) + depth = 0 if relative == Path(".") else len(relative.parts) + subdirectories.sort() + if current.name == "test.outputs": + found.append(current) + subdirectories.clear() + continue + if max_depth > 0 and depth >= max_depth: + subdirectories.clear() + return tuple(sorted(found, key=lambda path: path.as_posix())) + + +def _safe_payload_directory(path: Path, output: Path) -> bool: + """Accept only real payload directories contained by their test output.""" + current = output + for part in path.relative_to(output).parts: + current /= part + if current.is_symlink(): + return False + try: + path.resolve().relative_to(output.resolve()) + except (OSError, ValueError): + return False + return True diff --git a/tools/core/uploader_py/endpoints.py b/tools/core/uploader_py/endpoints.py new file mode 100644 index 00000000..de13c544 --- /dev/null +++ b/tools/core/uploader_py/endpoints.py @@ -0,0 +1,142 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Normalize and validate intake endpoints before workers start. + +One endpoint policy keeps agentless and EVP routing consistent across payload types. +""" + +from __future__ import annotations + +from dataclasses import dataclass +import re +from urllib.parse import SplitResult, urlsplit + +from .config import ConfigError, UploaderConfig + + +_VALID_HOSTNAME_RE = re.compile( + r"^[a-z0-9]([a-z0-9-]*[a-z0-9])?([.][a-z0-9]([a-z0-9-]*[a-z0-9])?)*$" +) +_INVALID_PERCENT_ESCAPE_RE = re.compile(r"%(?![0-9A-Fa-f]{2})") + + +@dataclass(frozen=True) +class EndpointSet: + """Resolved upload endpoints for one invocation.""" + + agentless: bool + site: str + test_url: str + coverage_url: str + telemetry_url: str + + +def normalize_dd_site(raw_site: str) -> str: + """Normalize legacy DD_SITE forms while rejecting unsafe host input.""" + site = raw_site.strip() + if not site: + return "datadoghq.com" + if "://" in site: + site = site.split("://", 1)[1] + site = site.split("/", 1)[0] + site = site.split("?", 1)[0] + site = site.split("#", 1)[0] + if site.lower().startswith("app."): + site = site[4:] + if site.lower().startswith("api."): + site = site[4:] + site = site.strip().lower() + + if not site: + raise ConfigError("DD_SITE resolved to an empty hostname") + if "@" in site: + raise ConfigError("DD_SITE must not include credentials/userinfo") + if ":" in site: + raise ConfigError("DD_SITE must be a hostname without an explicit port") + if site.startswith(".") or site.endswith(".") or ".." in site: + raise ConfigError("DD_SITE must be a valid hostname") + if not _VALID_HOSTNAME_RE.fullmatch(site): + raise ConfigError("DD_SITE contains unsupported hostname characters") + return site + + +def parse_http_url(raw_url: str) -> SplitResult: + """Parse one network-ready absolute HTTP(S) URL. + + urllib accepts some values during parsing that ``http.client`` rejects only + when it starts a request. Keeping this small preflight shared by endpoint + configuration and request preparation makes dry-run reject those values too. + """ + if not isinstance(raw_url, str) or not raw_url: + raise ValueError("HTTP URL must be a non-empty string") + if any(ord(character) < 33 or ord(character) > 126 for character in raw_url): + raise ValueError("HTTP URL must contain only printable ASCII characters") + if _INVALID_PERCENT_ESCAPE_RE.search(raw_url): + raise ValueError("HTTP URL contains an invalid percent escape") + + parsed = urlsplit(raw_url) + hostname = parsed.hostname or "" + # Accessing port performs urllib's numeric and range validation. + _ = parsed.port + if parsed.scheme.lower() not in {"http", "https"} or not hostname: + raise ValueError("HTTP URL must be absolute") + return parsed + + +def _validated_base_url(raw_url: str, variable_name: str) -> str: + """Validate an endpoint override without echoing sensitive URL components.""" + base = raw_url.rstrip("/") + try: + parsed = parse_http_url(base) + except (TypeError, ValueError): + parsed = None + if parsed is None: + raise ConfigError(f"{variable_name} must be an absolute HTTP(S) URL") + if parsed.username is not None or parsed.password is not None: + raise ConfigError(f"{variable_name} must not contain credentials/userinfo") + if parsed.query or parsed.fragment: + raise ConfigError(f"{variable_name} must not contain a query or fragment") + return base + + +def build_endpoints(config: UploaderConfig) -> EndpointSet: + """Build immutable agentless or EVP endpoint URLs.""" + site = normalize_dd_site(config.site) + if config.agentless: + if config.agentless_url: + base = _validated_base_url( + config.agentless_url, + "DD_TEST_OPTIMIZATION_AGENTLESS_URL", + ) + return EndpointSet( + agentless=True, + site=site, + test_url=f"{base}/api/v2/citestcycle", + coverage_url=f"{base}/api/v2/citestcov", + telemetry_url=f"{base}/api/v2/apmtelemetry", + ) + return EndpointSet( + agentless=True, + site=site, + test_url=f"https://citestcycle-intake.{site}/api/v2/citestcycle", + coverage_url=f"https://citestcov-intake.{site}/api/v2/citestcov", + telemetry_url=( + f"https://instrumentation-telemetry-intake.{site}/api/v2/apmtelemetry" + ), + ) + + base = _validated_base_url( + config.agent_url, + "DD_TEST_OPTIMIZATION_AGENT_URL", + ) + return EndpointSet( + agentless=False, + site=site, + test_url=f"{base}/evp_proxy/v2/api/v2/citestcycle", + coverage_url=f"{base}/evp_proxy/v2/api/v2/citestcov", + telemetry_url=f"{base}/telemetry/proxy/api/v2/apmtelemetry", + ) diff --git a/tools/core/uploader_py/enrichment.py b/tools/core/uploader_py/enrichment.py new file mode 100644 index 00000000..cd5c2ceb --- /dev/null +++ b/tools/core/uploader_py/enrichment.py @@ -0,0 +1,206 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Apply context, Bazel metadata, and CODEOWNERS to one test payload. + +Keeping enrichment pure and isolated makes concurrent transformations testable. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from types import MappingProxyType +from typing import Any, Mapping + +from .codeowners import ( + CodeOwnersEnrichmentStats, + CodeOwnersMatch, + CodeOwnersMatcher, + enrich_payload_codeowners, +) +from .json_utils import strict_json_dumps + + +TOP_LEVEL_EVENT_METADATA_KEYS = ( + "test", + "test_suite_end", + "test_module_end", + "test_session_end", +) +CONTEXT_EXCLUDED_EVENT_KEYS = frozenset({"topt.api_key_fingerprint"}) + + +@dataclass(frozen=True) +class ContextRecord: + repo_key: str + values: Mapping[str, Any] + + @classmethod + def create(cls, repo_key: str, values: Mapping[str, Any]) -> "ContextRecord": + return cls(repo_key, MappingProxyType(dict(values))) + + +@dataclass(frozen=True) +class ContextSelection: + values: Mapping[str, Any] | None + warning_code: str | None = None + + +@dataclass(frozen=True) +class ContextPlan: + """Invocation-wide contexts loaded once and selected per source file. + + Runtime-selected plans require the source sidecar to identify its repository; + static single-context plans retain their legacy fallback to the primary context. + """ + + primary: ContextRecord | None + by_repo: tuple[ContextRecord, ...] = () + override: bool = False + require_repo_match: bool = False + + def select(self, repo_key: str | None) -> ContextSelection: + if self.override: + return ContextSelection(self.primary.values if self.primary else None) + if not self.require_repo_match and len(self.by_repo) <= 1: + return ContextSelection(self.primary.values if self.primary else None) + if not repo_key: + return ContextSelection(None, "context_repo_metadata_missing") + for record in self.by_repo: + if record.repo_key == repo_key: + return ContextSelection(record.values) + return ContextSelection(None, "context_repo_not_found") + + +@dataclass(frozen=True) +class EnrichmentResult: + codeowners: CodeOwnersEnrichmentStats + warning_codes: tuple[str, ...] = () + + +def payload_repo_key(bazel_metadata: Mapping[str, Any] | None) -> str | None: + if bazel_metadata is None: + return None + value = bazel_metadata.get("bazel.test_optimization.repo_name") + return value if isinstance(value, str) and value else None + + +def enrich_test_payload( + payload: dict[str, Any], + *, + context_selection: ContextSelection, + bazel_metadata: Mapping[str, Any] | None, + runtime_id: str, + rules_version: str, + codeowners_matcher: CodeOwnersMatcher, + codeowners_cache: dict[str, CodeOwnersMatch] | None = None, +) -> EnrichmentResult: + """Apply the complete non-I/O enrichment sequence to one worker payload.""" + context = context_selection.values or {} + metadata = payload.get("metadata") + if not isinstance(metadata, dict): + metadata = {} + global_metadata = metadata.get("*") + if not isinstance(global_metadata, dict): + global_metadata = {} + + normalized_global_metadata = { + "runtime-id": _first_nonempty_string( + global_metadata.get("runtime-id"), + context.get("runtime-id"), + context.get("runtime.id"), + context.get("runtime_id"), + runtime_id, + ), + "language": _first_nonempty_string( + global_metadata.get("language"), + context.get("language"), + context.get("runtime.name"), + context.get("runtime_name"), + "bazel", + ), + "library_version": _first_nonempty_string( + global_metadata.get("library_version"), + rules_version, + ), + } + environment = _first_nonempty_string( + global_metadata.get("env"), + context.get("env"), + ) + if environment: + normalized_global_metadata["env"] = environment + + normalized_metadata: dict[str, Any] = {"*": normalized_global_metadata} + for key in TOP_LEVEL_EVENT_METADATA_KEYS: + if key in metadata and metadata[key] is not None: + normalized_metadata[key] = metadata[key] + payload["metadata"] = normalized_metadata + + events = payload.get("events") + if isinstance(events, list): + context_values = { + key: value + for key, value in context.items() + if key not in CONTEXT_EXCLUDED_EVENT_KEYS + } + for event in events: + if not isinstance(event, dict): + continue + content = event.get("content") + if not isinstance(content, dict): + content = {} + event["content"] = content + meta = content.get("meta") + if not isinstance(meta, dict): + meta = {} + content["meta"] = meta + metrics = content.get("metrics") + if not isinstance(metrics, dict): + metrics = {} + content["metrics"] = metrics + _merge_flat_metadata(meta, metrics, context_values) + if bazel_metadata is not None: + _merge_flat_metadata(meta, metrics, bazel_metadata) + + codeowners_stats = enrich_payload_codeowners( + payload, + codeowners_matcher, + cache=codeowners_cache, + ) + warnings = ( + (context_selection.warning_code,) + if context_selection.warning_code is not None + else () + ) + return EnrichmentResult(codeowners_stats, warnings) + + +def _first_nonempty_string(*values: Any) -> str: + for value in values: + if isinstance(value, str) and value: + return value + return "" + + +def _merge_flat_metadata( + meta: dict[str, Any], + metrics: dict[str, Any], + values: Mapping[str, Any], +) -> None: + for key, value in values.items(): + if isinstance(value, bool): + meta[key] = "true" if value else "false" + elif isinstance(value, (int, float)): + metrics[key] = value + elif isinstance(value, str): + meta[key] = value + else: + meta[key] = strict_json_dumps( + value, + ensure_ascii=False, + separators=(",", ":"), + ) diff --git a/tools/core/uploader_py/expected_targets.py b/tools/core/uploader_py/expected_targets.py new file mode 100644 index 00000000..dbefc49c --- /dev/null +++ b/tools/core/uploader_py/expected_targets.py @@ -0,0 +1,207 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Validate expected Bazel targets and select their discovered outputs. + +The pre-worker gate prevents unrelated or missing targets from entering the queue. +""" + +from __future__ import annotations + +from dataclasses import dataclass, replace +import json +from pathlib import Path, PurePosixPath +import re +from typing import Iterable + +from topt_runtime.runfiles import RunfileResolutionError, RunfilesResolver + +from .discovery import DiscoveryResult, count_tasks_by_payload_type +from .json_utils import strict_json_loads + + +class ExpectedTargetsError(ValueError): + """Expected target inputs do not identify a safe complete local set.""" + + +_BAZEL_TEST_ATTEMPT_DIRECTORY = re.compile( + r"(?:shard|run)_\d+_of_\d+|attempt_\d+" +) + + +@dataclass(frozen=True) +class ExpectedTargetsPlan: + targets: tuple[str, ...] = () + source: str = "discovery" + + +def load_expected_targets( + *, + static_targets: Iterable[str], + expected_targets_file_paths: Iterable[str], + resolver: RunfilesResolver, + runtime_targets: Iterable[str] = (), + runtime_selection: bool = False, +) -> ExpectedTargetsPlan: + """Merge generated and runtime targets into one validated selection.""" + static = tuple(_validate_label(label) for label in static_targets) + if len(static) != len(set(static)): + raise ExpectedTargetsError("static expected_targets contains duplicates") + candidates = tuple(path for path in expected_targets_file_paths if path) + configured = tuple(sorted(static)) + configured_source = "static" if static else "discovery" + if candidates: + try: + target_file = resolver.resolve_file(candidates) + except RunfileResolutionError as exc: + raise ExpectedTargetsError("expected_targets_file could not be resolved") from exc + dynamic = _load_target_file(target_file) + if static and set(static) != set(dynamic): + raise ExpectedTargetsError( + "static expected_targets and expected_targets_file contain different target sets" + ) + configured = tuple(sorted(static)) if static else dynamic + configured_source = "static_and_file" if static else "file" + + runtime = tuple(_validate_label(label) for label in runtime_targets) + if len(runtime) != len(set(runtime)): + raise ExpectedTargetsError( + "runtime --expected-target arguments contain duplicate target labels" + ) + runtime = tuple(sorted(runtime)) + if runtime: + if configured and set(configured) != set(runtime): + raise ExpectedTargetsError( + "configured and runtime expected targets contain different target sets" + ) + source = "runtime_and_configured" if configured else "runtime" + return ExpectedTargetsPlan(runtime, source) + if runtime_selection: + raise ExpectedTargetsError( + "runtime selection requires at least one --expected-target argument" + ) + return ExpectedTargetsPlan(configured, configured_source) + + +def select_expected_outputs( + discovery: DiscoveryResult, + plan: ExpectedTargetsPlan, + *, + allow_missing: bool = False, +) -> DiscoveryResult: + """Filter discovered outputs and stamp tasks with their expected target.""" + if not plan.targets: + return discovery + outputs_by_target: dict[str, list[str]] = {target: [] for target in plan.targets} + target_by_output_key: dict[str, str] = {} + for output in discovery.outputs: + matches = [ + target + for target in plan.targets + if _output_belongs_to_target(output.output_key, target) + ] + if len(matches) > 1: + most_specific = max(len(_target_output_parts(target)) for target in matches) + matches = [ + target + for target in matches + if len(_target_output_parts(target)) == most_specific + ] + if len(matches) > 1: + raise ExpectedTargetsError( + f"test.outputs path matches multiple expected targets: {output.output_key}" + ) + if not matches: + continue + target = matches[0] + outputs_by_target[target].append(output.output_key) + target_by_output_key[output.output_key] = target + + missing = tuple( + target for target in plan.targets if not outputs_by_target[target] + ) + if missing and not allow_missing: + raise ExpectedTargetsError( + "expected targets have no local test.outputs: " + ", ".join(missing) + ) + selected_outputs = tuple( + output + for output in discovery.outputs + if output.output_key in target_by_output_key + ) + selected_tasks = tuple( + replace(task, target_label=target_by_output_key[task.output_key or ""]) + for task in discovery.tasks + if (task.output_key or "") in target_by_output_key + ) + return DiscoveryResult( + outputs=selected_outputs, + tasks=selected_tasks, + discovered_by_type=count_tasks_by_payload_type(selected_tasks), + warning_codes=discovery.warning_codes, + ) + + +def _load_target_file(path: Path) -> tuple[str, ...]: + try: + value = strict_json_loads(path.read_bytes().decode("utf-8-sig")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ExpectedTargetsError("expected_targets_file is not valid JSON") from exc + if not isinstance(value, dict) or set(value) != {"schema_version", "targets"}: + raise ExpectedTargetsError( + "expected_targets_file must contain exactly schema_version and targets" + ) + if value["schema_version"] != 1: + raise ExpectedTargetsError("expected_targets_file schema_version must be 1") + targets = value["targets"] + if not isinstance(targets, list): + raise ExpectedTargetsError("expected_targets_file targets must be an array") + normalized = tuple(_validate_label(label) for label in targets) + if len(normalized) != len(set(normalized)): + raise ExpectedTargetsError("expected_targets_file contains duplicate labels") + if normalized != tuple(sorted(normalized)): + raise ExpectedTargetsError("expected_targets_file targets must be sorted") + return normalized + + +def _validate_label(value: object) -> str: + if not isinstance(value, str): + raise ExpectedTargetsError("expected target labels must be strings") + contains_control_character = any( + ord(character) <= 32 or ord(character) == 127 for character in value + ) + if value != value.strip() or contains_control_character: + raise ExpectedTargetsError(f"invalid expected target label: {value!r}") + if value.startswith("@") or not value.startswith("//"): + raise ExpectedTargetsError(f"expected target must be local: {value!r}") + body = value[2:] + if body.count(":") != 1: + raise ExpectedTargetsError(f"expected target must contain one colon: {value!r}") + package, target = body.split(":", 1) + if not target or "*" in value or "..." in value or "\\" in value: + raise ExpectedTargetsError(f"expected target must be fully expanded: {value!r}") + for component in tuple(filter(None, package.split("/"))) + tuple(target.split("/")): + if component in {".", "..", ""}: + raise ExpectedTargetsError(f"invalid expected target path: {value!r}") + if package.startswith("/") or package.endswith("/") or "//" in package: + raise ExpectedTargetsError(f"invalid expected target package: {value!r}") + return value + + +def _output_belongs_to_target(output_key: str, label: str) -> bool: + expected_parts = _target_output_parts(label) + output_parts = PurePosixPath(output_key.replace("\\", "/")).parts + if not output_parts or output_parts[-1] != "test.outputs": + return False + if output_parts[: len(expected_parts)] != expected_parts: + return False + attempt_parts = output_parts[len(expected_parts) : -1] + return all(_BAZEL_TEST_ATTEMPT_DIRECTORY.fullmatch(part) for part in attempt_parts) + + +def _target_output_parts(label: str) -> tuple[str, ...]: + package, target = label[2:].split(":", 1) + return tuple(filter(None, package.split("/"))) + tuple(target.split("/")) diff --git a/tools/core/uploader_py/file_worker.py b/tools/core/uploader_py/file_worker.py new file mode 100644 index 00000000..c23ab355 --- /dev/null +++ b/tools/core/uploader_py/file_worker.py @@ -0,0 +1,1279 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Run the complete isolated pipeline for one uploader source file. + +One worker owns enrichment through cleanup so files need no cross-worker sync. +""" + +from __future__ import annotations + +import copy +from dataclasses import dataclass, field, replace +import gzip +import json +import logging +from pathlib import Path +import stat +import tempfile +from typing import Any, Mapping, Protocol + +from validate_payload_schema import validate_payload as validate_schema_payload + +from .codeowners import CodeOwnersMatch, CodeOwnersMatcher +from .endpoints import EndpointSet +from .enrichment import ContextPlan, enrich_test_payload, payload_repo_key +from .json_utils import strict_json_dumps, strict_json_loads +from .models import ( + MAX_TEST_PAYLOAD_BYTES, + FileResult, + FileStatus, + FileTask, + PayloadType, +) +from .splitting import ( + PreparedTestChunk, + TestPayloadSplitError, + compact_json_bytes, + prepare_test_chunks, +) +from .telemetry import TelemetryDirective, TelemetryPlan +from .temporary import task_temporary_directory +from .transport import ( + HttpResult, + HttpTransportError, + PreparedMultipartBody, + prepare_coverage_multipart, + prepare_json_request, + prepare_spooled_multipart_request, +) + + +DEFAULT_HEADER_LANGUAGE = "bazel-starlark" +DEFAULT_HEADER_LANGUAGE_VERSION = "n/a" +DEFAULT_HEADER_LANGUAGE_INTERPRETER = "bazel-run" +_COVERAGE_EVENT_BODY = b'{"dummy":true}' +_TELEMETRY_RETRY_MARKER = "_dd_test_optimization_retry_without_augmentation" + + +class WorkerTransport(Protocol): + """Subset of the worker-local transport used by file processors.""" + + def post_json( + self, + url: str, + headers: Mapping[str, str], + body: bytes | Path, + *, + gzip_body: bool = False, + content_encoding: str | None = None, + ) -> HttpResult: ... + + def post_prepared_multipart( + self, + url: str, + headers: Mapping[str, str], + prepared: PreparedMultipartBody, + ) -> HttpResult: ... + + +@dataclass(frozen=True) +class WorkerRuntime: + """Invocation-wide read-only state shared by independent file workers.""" + + endpoints: EndpointSet + invocation_temp_root: Path + context_plan: ContextPlan + codeowners_matcher: CodeOwnersMatcher + runtime_id: str + rules_version: str + uploader_version: str + api_key: str = field(default="", repr=False) + schema: dict[str, Any] | None = field(default=None, repr=False, compare=False) + dry_run: bool = False + validate_enrichment: bool = False + expected_enriched_tags: tuple[str, ...] = () + gzip_payloads: bool = False + keep_payloads: bool = False + filter_prefix: bool = False + telemetry_session_id: str = "" + telemetry_plan: TelemetryPlan = field(default_factory=TelemetryPlan) + logger: logging.Logger | None = field(default=None, repr=False, compare=False) + + +@dataclass(frozen=True) +class _TestRequest: + """One bounded test chunk and its exact outbound representation.""" + + chunk: PreparedTestChunk + body: Path + content_encoding: str | None = None + + +@dataclass(frozen=True) +class _TelemetryRequest: + """One telemetry body paired with the headers derived from that body.""" + + body: Path + headers: dict[str, str] + + +def common_headers( + runtime: WorkerRuntime, + payload: Mapping[str, Any] | None = None, +) -> dict[str, str]: + """Build legacy-compatible Datadog headers from an enriched test body.""" + language = DEFAULT_HEADER_LANGUAGE + language_version = DEFAULT_HEADER_LANGUAGE_VERSION + language_interpreter = DEFAULT_HEADER_LANGUAGE_INTERPRETER + tracer_version = runtime.uploader_version + + if payload is not None: + metadata = payload.get("metadata") + global_metadata = metadata.get("*") if isinstance(metadata, dict) else None + if isinstance(global_metadata, dict): + language = _nonempty_string(global_metadata.get("language")) or language + tracer_version = ( + _nonempty_string(global_metadata.get("library_version")) + or tracer_version + ) + language_version = ( + _nonempty_string(global_metadata.get("language_version")) + or _nonempty_string(global_metadata.get("runtime_version")) + or language_version + ) + language_interpreter = ( + _nonempty_string(global_metadata.get("language_interpreter")) + or _nonempty_string(global_metadata.get("runtime_name")) + or language_interpreter + ) + + headers = { + "Datadog-Meta-Lang": language, + "Datadog-Meta-Lang-Version": language_version, + "Datadog-Meta-Lang-Interpreter": language_interpreter, + "Datadog-Meta-Tracer-Version": tracer_version, + "Accept": "application/json", + } + if runtime.endpoints.agentless: + headers["DD-API-KEY"] = runtime.api_key + return headers + + +def process_file( + task: FileTask, + runtime: WorkerRuntime, + transport: WorkerTransport, +) -> FileResult: + """Attach task-scoped transport diagnostics around one complete pipeline.""" + set_log_context = getattr(transport, "set_log_context", None) + clear_log_context = getattr(transport, "clear_log_context", None) + if callable(set_log_context): + set_log_context( + task.task_id, + task.payload_type.value, + task.display_path, + ) + try: + return _process_file_with_context(task, runtime, transport) + finally: + if callable(clear_log_context): + clear_log_context() + + +def _process_file_with_context( + task: FileTask, + runtime: WorkerRuntime, + transport: WorkerTransport, +) -> FileResult: + """Run one file's entire protocol without coordinating with other files.""" + _debug(runtime, task, "started complete file pipeline") + if _debug_enabled(runtime): + try: + _debug(runtime, task, f"source_bytes={task.source_path.stat().st_size}") + except OSError as exc: + _debug(runtime, task, f"source size unavailable: {type(exc).__name__}") + temporary_cleanup_errors: list[str] = [] + try: + with task_temporary_directory( + runtime.invocation_temp_root, + task.task_id, + on_cleanup_error=temporary_cleanup_errors.append, + ) as task_directory: + _debug(runtime, task, f"task temporary directory created: {task_directory}") + result = _dispatch_file( + task, + runtime, + transport, + task_directory, + ) + except HttpTransportError as exc: + _debug(runtime, task, "transport rejected a locally prepared request") + result = _failed( + task, + "request_preparation_failed", + type(exc).__name__, + ) + except OSError as exc: + _debug(runtime, task, f"file pipeline failed with {type(exc).__name__}") + result = _failed(task, "file_pipeline_io_failed", type(exc).__name__) + + if temporary_cleanup_errors: + if runtime.logger is not None: + runtime.logger.warning( + "task=%s type=%s file=%s task temporary cleanup failed: %s", + task.task_id, + task.payload_type.value, + task.display_path, + temporary_cleanup_errors[0], + ) + result = replace( + result, + warning_codes=tuple( + dict.fromkeys(result.warning_codes + ("task_temp_cleanup_failed",)) + ), + ) + else: + _debug(runtime, task, "task temporary cleanup completed") + return result + + +def _dispatch_file( + task: FileTask, + runtime: WorkerRuntime, + transport: WorkerTransport, + task_directory: Path, +) -> FileResult: + if task.payload_type is PayloadType.TEST: + return _process_test(task, runtime, transport, task_directory) + if task.payload_type is PayloadType.COVERAGE: + return _process_coverage(task, runtime, transport, task_directory) + if task.payload_type is PayloadType.TELEMETRY: + return _process_telemetry(task, runtime, transport, task_directory) + return _failed(task, "unsupported_payload_type", str(task.payload_type)) + + +def _process_test( + task: FileTask, + runtime: WorkerRuntime, + transport: WorkerTransport, + task_directory: Path, +) -> FileResult: + if task.source_path.suffix.lower() != ".json": + return _failed( + task, + "unsupported_test_payload_format", + "test payloads must be JSON", + ) + if runtime.filter_prefix and not task.source_path.name.startswith("span_events_"): + return _skipped(task, "prefix_filter") + + payload, read_failure = _read_json_object(task.source_path, "test") + if read_failure is not None: + return _failed(task, *read_failure) + assert payload is not None + events = payload.get("events") + if not isinstance(events, list) or not events: + return _skipped(task, "test_payload_without_events") + + warnings: list[str] = [] + failure = _enrich_and_validate_test(task, runtime, payload, warnings) + if failure is not None: + return failure + + try: + chunks = prepare_test_chunks(payload, task_directory) + except TestPayloadSplitError as exc: + return FileResult( + task_id=task.task_id, + source_path=task.display_path, + payload_type=task.payload_type, + status=FileStatus.FAILED, + events=_event_count(payload), + warning_codes=_unique_codes(warnings), + failure_code=exc.code, + failure_message=str(exc), + ) + requests = _prepare_test_requests( + task, + runtime, + chunks, + task_directory, + warnings, + ) + + test_result_fields = dict( + task_id=task.task_id, + source_path=task.display_path, + payload_type=task.payload_type, + events=_event_count(payload), + chunks_created=len(requests), + requests_planned=len(requests), + ) + headers = common_headers(runtime, payload) + if not runtime.endpoints.agentless: + headers["X-Datadog-EVP-Subdomain"] = "citestcycle-intake" + if runtime.dry_run: + for request in requests: + prepare_json_request( + runtime.endpoints.test_url, + headers, + request.body, + content_encoding=request.content_encoding, + ) + _debug(runtime, task, "dry-run completed without network or source cleanup") + return FileResult( + status=FileStatus.SUCCEEDED, + warning_codes=_unique_codes(warnings), + **test_result_fields, + ) + + requests_attempted = 0 + requests_succeeded = 0 + requests_failed = 0 + retries = 0 + failed_chunks: list[PreparedTestChunk] = [] + first_failure: tuple[str, str] | None = None + for request in requests: + chunk = request.chunk + _debug( + runtime, + task, + f"uploading test chunk {chunk.index}/{len(requests)} " + f"({chunk.size_bytes} bytes)", + ) + http_result = transport.post_json( + runtime.endpoints.test_url, + headers, + request.body, + content_encoding=request.content_encoding, + ) + requests_attempted += http_result.attempts + retries += http_result.retries + _debug( + runtime, + task, + f"test chunk {chunk.index} completed after " + f"{http_result.attempts} attempt(s)", + ) + if not http_result.succeeded: + failure_code, failure_message = _http_failure( + http_result, + payload_type=task.payload_type, + payload_limit_context=( + f"chunk={chunk.index}/{len(requests)} " + f"uncompressed_bytes={chunk.size_bytes} " + f"outbound_bytes={request.body.stat().st_size} " + f"threshold_bytes={MAX_TEST_PAYLOAD_BYTES}" + ), + ) + requests_failed += 1 + failed_chunks.append(chunk) + if first_failure is None: + first_failure = (failure_code, failure_message) + continue + requests_succeeded += 1 + + if first_failure is not None: + if requests_succeeded and not runtime.keep_payloads: + persisted = _persist_failed_test_chunks( + task.source_path, + payload, + failed_chunks, + ) + if persisted: + if runtime.logger is not None: + runtime.logger.info( + "task=%s type=%s file=%s retained %d failed test " + "chunk(s) for retry", + task.task_id, + task.payload_type.value, + task.display_path, + requests_failed, + ) + else: + warnings.append("failed_test_chunks_persist_failed") + if runtime.logger is not None: + runtime.logger.warning( + "task=%s type=%s file=%s failed to retain rejected test " + "chunks; keeping the original source", + task.task_id, + task.payload_type.value, + task.display_path, + ) + return FileResult( + status=FileStatus.FAILED, + chunks_uploaded=requests_succeeded, + chunks_failed=requests_failed, + requests_attempted=requests_attempted, + requests_succeeded=requests_succeeded, + requests_failed=requests_failed, + retries=retries, + warning_codes=_unique_codes(warnings), + failure_code=first_failure[0], + failure_message=first_failure[1], + **test_result_fields, + ) + + deleted, cleanup_warning = _cleanup_source(task.source_path, runtime.keep_payloads) + _debug(runtime, task, f"test cleanup completed source_deleted={deleted}") + final_warnings = list(warnings) + if cleanup_warning: + final_warnings.append(cleanup_warning) + return FileResult( + status=FileStatus.SUCCEEDED, + chunks_uploaded=len(requests), + requests_attempted=requests_attempted, + requests_succeeded=requests_succeeded, + retries=retries, + source_deleted=deleted, + warning_codes=_unique_codes(final_warnings), + **test_result_fields, + ) + + +def _enrich_and_validate_test( + task: FileTask, + runtime: WorkerRuntime, + payload: dict[str, Any], + warnings: list[str], +) -> FileResult | None: + """Enrich one mutable test payload and validate its prepared shape.""" + bazel_metadata, sidecar_warning = _load_bazel_metadata(task) + if sidecar_warning: + warnings.append(sidecar_warning) + repo_key = payload_repo_key(bazel_metadata) + context_selection = runtime.context_plan.select(repo_key) + if sidecar_warning: + sidecar_state = "invalid" + elif bazel_metadata is not None: + sidecar_state = "loaded" + else: + sidecar_state = "absent" + _debug( + runtime, + task, + "Bazel sidecar=%s context_repo=%s context_selected=%s context_warning=%s" + % ( + sidecar_state, + repr(repo_key[:256]) if repo_key is not None else "none", + "yes" if context_selection.values is not None else "no", + context_selection.warning_code or "none", + ), + ) + codeowners_cache: dict[str, CodeOwnersMatch] = {} + try: + enrichment = enrich_test_payload( + payload, + context_selection=context_selection, + bazel_metadata=bazel_metadata, + runtime_id=runtime.runtime_id, + rules_version=runtime.rules_version, + codeowners_matcher=runtime.codeowners_matcher, + codeowners_cache=codeowners_cache, + ) + except (TypeError, ValueError, OSError) as exc: + _debug(runtime, task, f"enrichment failed with {type(exc).__name__}") + return _failed(task, "test_enrichment_failed", type(exc).__name__) + + warnings.extend(enrichment.warning_codes) + _debug( + runtime, + task, + "enrichment completed events=%d codeowners_scanned=%d enriched=%d " + "existing=%d missing_source=%d unmatched=%d errors=%d" + % ( + _event_count(payload), + enrichment.codeowners.scanned, + enrichment.codeowners.enriched, + enrichment.codeowners.skipped_existing, + enrichment.codeowners.skipped_missing_source, + enrichment.codeowners.skipped_unmatched, + enrichment.codeowners.skipped_errors, + ), + ) + if _debug_enabled(runtime): + for source_path, match in codeowners_cache.items(): + _debug( + runtime, + task, + "CODEOWNERS source=%r candidate=%r matched=%s owner_count=%d" + % ( + source_path[:256], + match.candidate[:256] if match.candidate is not None else None, + match.matched, + len(match.owners), + ), + ) + try: + enriched_bytes = len(compact_json_bytes(payload)) + except TestPayloadSplitError as exc: + _debug(runtime, task, f"enriched_bytes unavailable: {exc.code}") + else: + _debug(runtime, task, f"enriched_bytes={enriched_bytes}") + + if runtime.schema is not None: + try: + validation = validate_schema_payload(payload, runtime.schema) + except (TypeError, ValueError, KeyError) as exc: + warnings.append("schema_validation_internal_error") + _debug(runtime, task, f"schema validation failed with {type(exc).__name__}") + else: + if not validation.valid: + warnings.append("schema_validation_failed") + _debug( + runtime, + task, + f"schema validation reported {len(validation.errors)} error(s)", + ) + if validation.warnings: + warnings.append("schema_validation_warning") + _debug(runtime, task, "warning-only schema validation completed") + + if runtime.validate_enrichment: + missing_tags = _missing_enriched_tags(payload, runtime.expected_enriched_tags) + if missing_tags: + return FileResult( + task_id=task.task_id, + source_path=task.display_path, + payload_type=task.payload_type, + status=FileStatus.FAILED, + events=_event_count(payload), + warning_codes=_unique_codes(warnings), + failure_code="enrichment_tags_missing", + failure_message=",".join(missing_tags), + ) + _debug(runtime, task, "expected enrichment tags validated") + return None + + +def _prepare_test_requests( + task: FileTask, + runtime: WorkerRuntime, + chunks: tuple[PreparedTestChunk, ...], + task_directory: Path, + warnings: list[str], +) -> tuple[_TestRequest, ...]: + """Choose JSON or deterministic gzip for every pre-split request body.""" + _debug( + runtime, + task, + f"split threshold_bytes={MAX_TEST_PAYLOAD_BYTES} chunks={len(chunks)}", + ) + for chunk in chunks: + _debug( + runtime, + task, + f"chunk={chunk.index}/{len(chunks)} bytes={chunk.size_bytes} " + f"events={chunk.event_count}", + ) + + requests: list[_TestRequest] = [] + for chunk in chunks: + if not runtime.gzip_payloads: + requests.append(_TestRequest(chunk, chunk.path)) + continue + gzip_path = task_directory / f"test_chunk_{chunk.index:04d}.json.gz" + try: + gzip_path.write_bytes(gzip.compress(chunk.path.read_bytes(), mtime=0)) + except OSError as exc: + warnings.append("gzip_preparation_failed") + requests.append(_TestRequest(chunk, chunk.path)) + _debug( + runtime, + task, + f"gzip preparation failed with {type(exc).__name__}; using JSON", + ) + else: + requests.append(_TestRequest(chunk, gzip_path, "gzip")) + + if runtime.gzip_payloads: + compressed = sum(request.content_encoding == "gzip" for request in requests) + _debug( + runtime, + task, + "gzip preparation completed compressed=%d fallback_json=%d" + % (compressed, len(requests) - compressed), + ) + return tuple(requests) + + +def _process_coverage( + task: FileTask, + runtime: WorkerRuntime, + transport: WorkerTransport, + task_directory: Path, +) -> FileResult: + suffix = task.source_path.suffix.lower() + if suffix not in {".json", ".msgpack"}: + return _failed( + task, + "unsupported_coverage_payload_format", + "coverage payloads must be JSON or msgpack", + ) + if runtime.filter_prefix and not task.source_path.name.startswith("coverage_"): + return _skipped(task, "prefix_filter") + try: + with task.source_path.open("rb") as handle: + handle.read(1) + except OSError as exc: + return _failed(task, "coverage_payload_read_failed", type(exc).__name__) + + coverage_result_fields = dict( + task_id=task.task_id, + source_path=task.display_path, + payload_type=task.payload_type, + requests_planned=1, + ) + coverage_filename = ( + "filecoveragex.msgpack" if suffix == ".msgpack" else "filecoveragex.json" + ) + coverage_content_type = ( + "application/msgpack" if suffix == ".msgpack" else "application/json" + ) + prepared = prepare_coverage_multipart( + task_directory / "coverage_multipart.body", + event_body=_COVERAGE_EVENT_BODY, + coverage_path=task.source_path, + coverage_filename=coverage_filename, + coverage_content_type=coverage_content_type, + ) + _debug( + runtime, + task, + f"prepared coverage multipart bytes={prepared.content_length} " + f"content_type={coverage_content_type}", + ) + headers = common_headers(runtime) + if not runtime.endpoints.agentless: + headers["X-Datadog-EVP-Subdomain"] = "citestcov-intake" + if runtime.dry_run: + prepare_spooled_multipart_request( + runtime.endpoints.coverage_url, + headers, + prepared, + ) + _debug( + runtime, + task, + f"dry-run prepared {prepared.content_length}-byte coverage multipart body", + ) + return FileResult(status=FileStatus.SUCCEEDED, **coverage_result_fields) + + http_result = transport.post_prepared_multipart( + runtime.endpoints.coverage_url, + headers, + prepared, + ) + _debug( + runtime, + task, + f"coverage request completed after {http_result.attempts} attempt(s)", + ) + if not http_result.succeeded: + failure_code, failure_message = _http_failure( + http_result, + payload_type=task.payload_type, + ) + return FileResult( + status=FileStatus.FAILED, + requests_attempted=http_result.attempts, + requests_failed=1, + retries=http_result.retries, + failure_code=failure_code, + failure_message=failure_message, + **coverage_result_fields, + ) + + deleted, cleanup_warning = _cleanup_source(task.source_path, runtime.keep_payloads) + _debug(runtime, task, f"coverage cleanup completed source_deleted={deleted}") + return FileResult( + status=FileStatus.SUCCEEDED, + requests_attempted=http_result.attempts, + requests_succeeded=1, + retries=http_result.retries, + source_deleted=deleted, + warning_codes=(cleanup_warning,) if cleanup_warning else (), + **coverage_result_fields, + ) + + +def _process_telemetry( + task: FileTask, + runtime: WorkerRuntime, + transport: WorkerTransport, + task_directory: Path, +) -> FileResult: + if task.source_path.suffix.lower() != ".json": + return _failed( + task, + "unsupported_telemetry_payload_format", + "telemetry payloads must be JSON", + ) + payload, source_bytes, read_failure = _read_json_object_with_raw( + task.source_path, + "telemetry", + ) + if read_failure is not None: + return _failed(task, *read_failure) + assert payload is not None + assert source_bytes is not None + + # A partially successful prior run stores the sole rejected request in the + # source file. The private marker prevents rule facts from being added a + # second time and is removed before the retry reaches the backend. + retry_without_augmentation = payload.pop(_TELEMETRY_RETRY_MARKER, False) is True + if retry_without_augmentation: + source_bytes = _compact_json_line(payload) + directive = TelemetryDirective() + else: + directive = runtime.telemetry_plan.directive_for(task.source_path) + warnings: list[str] = [] + source_body = _prepare_telemetry_source( + source_bytes, + payload, + directive, + runtime.telemetry_plan.provider_suffix, + task_directory, + warnings, + ) + metadata_failure = _telemetry_metadata_failure(payload) + if metadata_failure is not None: + return FileResult( + task_id=task.task_id, + source_path=task.display_path, + payload_type=task.payload_type, + status=FileStatus.FAILED, + warning_codes=_unique_codes(warnings), + failure_code="invalid_telemetry_metadata", + failure_message=metadata_failure, + ) + + requests: list[_TelemetryRequest] = [ + _TelemetryRequest(source_body, _telemetry_headers(runtime, payload)) + ] + if directive.create_synthetic: + synthetic = _build_synthetic_telemetry(payload, directive, warnings) + if synthetic is not None: + _rewrite_telemetry_provider_tags( + synthetic, + runtime.telemetry_plan.provider_suffix, + ) + synthetic_path = task_directory / "telemetry_synthetic.json" + synthetic_path.write_bytes(_compact_json_line(synthetic)) + requests.append( + _TelemetryRequest(synthetic_path, _telemetry_headers(runtime, synthetic)) + ) + _debug(runtime, task, f"prepared {len(requests)} telemetry request(s)") + if _debug_enabled(runtime): + for index, request in enumerate(requests, start=1): + try: + body_bytes = request.body.stat().st_size + except OSError as exc: + _debug( + runtime, + task, + f"telemetry request={index}/{len(requests)} size unavailable: " + f"{type(exc).__name__}", + ) + else: + _debug( + runtime, + task, + f"telemetry request={index}/{len(requests)} bytes={body_bytes}", + ) + + telemetry_result_fields = dict( + task_id=task.task_id, + source_path=task.display_path, + payload_type=task.payload_type, + requests_planned=len(requests), + ) + if runtime.dry_run: + for request in requests: + prepare_json_request( + runtime.endpoints.telemetry_url, + request.headers, + request.body, + ) + _debug(runtime, task, "dry-run completed telemetry preparation without network") + return FileResult( + status=FileStatus.SUCCEEDED, + warning_codes=_unique_codes(warnings), + **telemetry_result_fields, + ) + + requests_attempted = 0 + requests_succeeded = 0 + requests_failed = 0 + retries = 0 + failed_requests: list[_TelemetryRequest] = [] + first_failure: tuple[str, str] | None = None + for request in requests: + http_result = transport.post_json( + runtime.endpoints.telemetry_url, + request.headers, + request.body, + ) + requests_attempted += http_result.attempts + retries += http_result.retries + _debug( + runtime, + task, + "telemetry request completed after " + f"{http_result.attempts} attempt(s)", + ) + if not http_result.succeeded: + failure_code, failure_message = _http_failure( + http_result, + payload_type=task.payload_type, + ) + requests_failed += 1 + failed_requests.append(request) + if first_failure is None: + first_failure = (failure_code, failure_message) + continue + requests_succeeded += 1 + + if first_failure is not None: + if requests_succeeded and not runtime.keep_payloads: + if len(failed_requests) == 1 and _persist_failed_telemetry_request( + task.source_path, + failed_requests[0], + ): + if runtime.logger is not None: + runtime.logger.info( + "task=%s type=%s file=%s retained failed telemetry " + "request for retry", + task.task_id, + task.payload_type.value, + task.display_path, + ) + else: + warnings.append("failed_telemetry_request_persist_failed") + if runtime.logger is not None: + runtime.logger.warning( + "task=%s type=%s file=%s failed to retain rejected " + "telemetry request; keeping the original source", + task.task_id, + task.payload_type.value, + task.display_path, + ) + return FileResult( + status=FileStatus.FAILED, + requests_attempted=requests_attempted, + requests_succeeded=requests_succeeded, + requests_failed=requests_failed, + retries=retries, + warning_codes=_unique_codes(warnings), + failure_code=first_failure[0], + failure_message=first_failure[1], + **telemetry_result_fields, + ) + + deleted, cleanup_warning = _cleanup_source(task.source_path, runtime.keep_payloads) + _debug(runtime, task, f"telemetry cleanup completed source_deleted={deleted}") + if cleanup_warning: + warnings.append(cleanup_warning) + return FileResult( + status=FileStatus.SUCCEEDED, + requests_attempted=requests_attempted, + requests_succeeded=requests_succeeded, + retries=retries, + source_deleted=deleted, + warning_codes=_unique_codes(warnings), + **telemetry_result_fields, + ) + + +def _prepare_telemetry_source( + source_bytes: bytes, + payload: dict[str, Any], + directive: TelemetryDirective, + provider_suffix: str, + task_directory: Path, + warnings: list[str], +) -> Path: + changed = False + application = payload.get("application") + if directive.env_override: + if isinstance(application, dict): + application["env"] = directive.env_override + changed = True + else: + warnings.append("telemetry_env_normalization_skipped") + + if directive.append_messages: + messages = _decode_telemetry_messages(directive.messages_json) + items = payload.get("payload") + if messages is None or not isinstance(items, list): + warnings.append("telemetry_augmentation_skipped") + else: + items.extend(messages) + changed = True + + if provider_suffix: + _rewrite_telemetry_provider_tags(payload, provider_suffix) + # Preserve legacy serialization behavior whenever provider rewriting is + # enabled, even when this particular body has no matching series tag. + changed = True + + body_path = task_directory / "telemetry_body.json" + body_path.write_bytes(_compact_json_line(payload) if changed else source_bytes) + return body_path + + +def _telemetry_headers( + runtime: WorkerRuntime, + payload: Mapping[str, Any], +) -> dict[str, str]: + application = payload.get("application") + application_values = application if isinstance(application, dict) else {} + headers = { + "DD-Telemetry-API-Version": _nonempty_string(payload.get("api_version")), + "DD-Telemetry-Request-Type": _nonempty_string(payload.get("request_type")), + "DD-Session-ID": ( + _nonempty_string(payload.get("runtime_id")) + or runtime.telemetry_session_id + or runtime.runtime_id + ), + } + language = _nonempty_string(application_values.get("language_name")) + tracer_version = _nonempty_string(application_values.get("tracer_version")) + if language: + headers["DD-Client-Library-Language"] = language + if tracer_version: + headers["DD-Client-Library-Version"] = tracer_version + if runtime.endpoints.agentless: + headers["DD-API-KEY"] = runtime.api_key + return headers + + +def _telemetry_metadata_failure(payload: Mapping[str, Any]) -> str | None: + if not _nonempty_string(payload.get("api_version")): + return "missing or invalid api_version" + if not _nonempty_string(payload.get("request_type")): + return "missing or invalid request_type" + return None + + +def _decode_telemetry_messages(raw: bytes) -> list[Any] | None: + try: + value = strict_json_loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + return None + return copy.deepcopy(value) if isinstance(value, list) else None + + +def _build_synthetic_telemetry( + payload: Mapping[str, Any], + directive: TelemetryDirective, + warnings: list[str], +) -> dict[str, Any] | None: + messages = _decode_telemetry_messages(directive.messages_json) + application = payload.get("application") + if messages is None or not isinstance(application, dict): + warnings.append("telemetry_synthetic_skipped") + return None + synthetic = { + "api_version": payload.get("api_version"), + "request_type": "message-batch", + "runtime_id": payload.get("runtime_id"), + "seq_id": directive.synthetic_seq_id, + "tracer_time": directive.synthetic_timestamp, + "application": copy.deepcopy(application), + "host": copy.deepcopy(payload.get("host")), + "payload": messages, + } + if "debug" in payload: + synthetic["debug"] = copy.deepcopy(payload["debug"]) + return synthetic + + +def _rewrite_telemetry_provider_tags( + payload: dict[str, Any], + provider_suffix: str, +) -> None: + if not provider_suffix: + return + replacement = f"provider:bazel/{provider_suffix}" + + def rewrite_message(message: Any) -> None: + if not isinstance(message, dict): + return + request_type = message.get("request_type") + body = message.get("payload") + if request_type in {"generate-metrics", "distributions"}: + if not isinstance(body, dict): + return + series_items = body.get("series") + if not isinstance(series_items, list): + return + for series in series_items: + if not isinstance(series, dict): + continue + tags = series.get("tags") + if not isinstance(tags, list): + continue + for index, tag in enumerate(tags): + if tag == "provider:bazel": + tags[index] = replacement + elif request_type == "message-batch" and isinstance(body, list): + for child in body: + rewrite_message(child) + + rewrite_message(payload) + + +def _compact_json_line(payload: Mapping[str, Any]) -> bytes: + return ( + strict_json_dumps(payload, ensure_ascii=False, separators=(",", ":")) + "\n" + ).encode("utf-8") + + +def _load_bazel_metadata( + task: FileTask, +) -> tuple[dict[str, Any] | None, str | None]: + if task.test_outputs_dir is None: + return None, None + sidecar = task.test_outputs_dir / "bazel_target_metadata.json" + if not sidecar.is_file(): + return None, None + try: + sidecar_resolved = sidecar.resolve(strict=True) + output_resolved = task.test_outputs_dir.resolve(strict=True) + except OSError: + return None, "bazel_metadata_invalid" + if sidecar.is_symlink() or sidecar_resolved.parent != output_resolved: + return None, "bazel_metadata_unsafe" + payload, failure = _read_json_object(sidecar, "Bazel metadata") + if failure is not None: + return None, "bazel_metadata_invalid" + return payload, None + + +def _read_json_object( + path: Path, + label: str, +) -> tuple[dict[str, Any] | None, tuple[str, str] | None]: + value, _raw, failure = _read_json_object_with_raw(path, label) + return value, failure + + +def _read_json_object_with_raw( + path: Path, + label: str, +) -> tuple[ + dict[str, Any] | None, + bytes | None, + tuple[str, str] | None, +]: + try: + raw = path.read_bytes() + except OSError as exc: + return None, None, ( + f"{label.lower().replace(' ', '_')}_payload_read_failed", + type(exc).__name__, + ) + try: + value = strict_json_loads(raw.decode("utf-8-sig")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + return None, None, ( + f"invalid_{label.lower().replace(' ', '_')}_json", + type(exc).__name__, + ) + if not isinstance(value, dict): + return None, None, ( + f"invalid_{label.lower().replace(' ', '_')}", + f"{label} must be a JSON object", + ) + return value, raw, None + + +def _missing_enriched_tags( + payload: Mapping[str, Any], + expected_tags: tuple[str, ...], +) -> tuple[str, ...]: + events = payload.get("events") + event_objects = events if isinstance(events, list) else () + missing: list[str] = [] + for tag in expected_tags: + if not tag: + continue + present = False + for event in event_objects: + if not isinstance(event, dict): + continue + content = event.get("content") + if not isinstance(content, dict): + continue + meta = content.get("meta") + metrics = content.get("metrics") + if (isinstance(meta, dict) and tag in meta) or ( + isinstance(metrics, dict) and tag in metrics + ): + present = True + break + if not present: + missing.append(tag) + return tuple(missing) + + +def _cleanup_source(path: Path, keep_payloads: bool) -> tuple[bool, str | None]: + if keep_payloads: + return False, None + try: + path.unlink() + return True, None + except FileNotFoundError: + return False, "source_cleanup_missing" + except OSError: + try: + path.chmod(path.stat().st_mode | stat.S_IWUSR) + # POSIX unlink checks the directory, and Bazel output trees may be read-only. + parent = path.parent + parent.chmod(parent.stat().st_mode | stat.S_IWUSR) + path.unlink() + return True, None + except OSError: + return False, "source_cleanup_failed" + + +def _persist_failed_test_chunks( + source_path: Path, + payload: Mapping[str, Any], + failed_chunks: list[PreparedTestChunk], +) -> bool: + """Atomically replace a partially uploaded source with its failed events.""" + events = payload.get("events") + if not isinstance(events, list) or not failed_chunks: + return False + retry_payload = dict(payload) + retry_payload["events"] = [ + event + for chunk in failed_chunks + for event in events[chunk.event_start : chunk.event_end] + ] + try: + retry_body = compact_json_bytes(retry_payload) + except TestPayloadSplitError: + return False + return _atomic_replace_source(source_path, retry_body) + + +def _persist_failed_telemetry_request( + source_path: Path, + failed_request: _TelemetryRequest, +) -> bool: + """Retain one rejected prepared request without regenerating augmentation.""" + payload, failure = _read_json_object(failed_request.body, "telemetry retry") + if failure is not None or payload is None: + return False + payload[_TELEMETRY_RETRY_MARKER] = True + return _atomic_replace_source(source_path, _compact_json_line(payload)) + + +def _atomic_replace_source(source_path: Path, body: bytes) -> bool: + """Replace a payload beside itself, repairing Bazel output permissions once.""" + + parent = source_path.parent + for repair_permissions in (False, True): + if repair_permissions: + try: + parent.chmod(parent.stat().st_mode | stat.S_IWUSR) + source_path.chmod(source_path.stat().st_mode | stat.S_IWUSR) + except OSError: + pass + temporary_path: Path | None = None + try: + with tempfile.NamedTemporaryFile( + mode="wb", + prefix=f".{source_path.name}.retry.", + dir=parent, + delete=False, + ) as temporary: + temporary.write(body) + temporary_path = Path(temporary.name) + temporary_path.replace(source_path) + return True + except OSError: + if temporary_path is not None: + try: + temporary_path.unlink() + except OSError: + pass + return False + + +def _http_failure( + http_result: HttpResult, + *, + payload_type: PayloadType, + payload_limit_context: str | None = None, +) -> tuple[str, str]: + if http_result.status_code == 413: + if payload_type is PayloadType.TEST and payload_limit_context: + return ( + "payload_limit_contract_mismatch", + f"HTTP 413 after preventive split; {payload_limit_context}", + ) + return ( + "upload_http_413", + f"HTTP 413 for unsplit {payload_type.value} payload; " + f"{payload_type.value} splitting is not supported", + ) + if http_result.status_code is not None: + return "upload_http_error", f"HTTP {http_result.status_code}" + return "upload_transport_error", http_result.transport_error or "transport error" + + +def _event_count(payload: Mapping[str, Any]) -> int: + events = payload.get("events") + return len(events) if isinstance(events, list) else 0 + + +def _nonempty_string(value: Any) -> str: + return value if isinstance(value, str) and value else "" + + +def _unique_codes(codes: list[str]) -> tuple[str, ...]: + return tuple(dict.fromkeys(code for code in codes if code)) + + +def _failed(task: FileTask, code: str, message: str) -> FileResult: + return FileResult( + task_id=task.task_id, + source_path=task.display_path, + payload_type=task.payload_type, + status=FileStatus.FAILED, + failure_code=code, + failure_message=message, + ) + + +def _skipped(task: FileTask, code: str) -> FileResult: + return FileResult( + task_id=task.task_id, + source_path=task.display_path, + payload_type=task.payload_type, + status=FileStatus.SKIPPED, + warning_codes=(code,), + ) + + +def _debug(runtime: WorkerRuntime, task: FileTask, message: str) -> None: + if runtime.logger is not None: + runtime.logger.debug( + "task=%s type=%s file=%s %s", + task.task_id, + task.payload_type.value, + task.display_path, + message, + ) + + +def _debug_enabled(runtime: WorkerRuntime) -> bool: + return runtime.logger is not None and runtime.logger.isEnabledFor(logging.DEBUG) diff --git a/tools/core/uploader_py/freshness.py b/tools/core/uploader_py/freshness.py new file mode 100644 index 00000000..84b1ee90 --- /dev/null +++ b/tools/core/uploader_py/freshness.py @@ -0,0 +1,648 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Pre-worker BEP staging and freshness selection. + +The doctor remains the source of truth for the non-trivial BEP and artifact +carrier formats. This module loads that runtime once, snapshots its result in +immutable uploader models, and keeps all filesystem and policy decisions out +of worker threads. +""" + +from __future__ import annotations + +from dataclasses import dataclass, replace +import importlib.util +import json +import logging +from pathlib import Path +import sys +from types import ModuleType +from typing import Iterable + +from topt_runtime.runfiles import RunfileResolutionError, RunfilesResolver + +from .config import UploaderConfig +from .discovery import DiscoveryResult, ScanRoot, count_tasks_by_payload_type +from .json_utils import strict_json_loads +from .models import FileResult, FileStatus, FileTask + + +class FreshnessError(RuntimeError): + """Freshness or staging could not authorize a safe uploader run.""" + + def __init__(self, message: str, *, exit_code: int = 2) -> None: + super().__init__(message) + self.exit_code = exit_code + + +@dataclass(frozen=True) +class RemoteOutput: + label: str + output_key: str + artifact: str + + +@dataclass(frozen=True) +class FreshnessPlan: + """Immutable authorization snapshot used by discovery and postflight checks.""" + + selected_source: str = "none" + eligibility_enabled: bool = False + eligible_outputs: frozenset[tuple[str, str]] = frozenset() + cached_outputs: frozenset[tuple[str, str]] = frozenset() + remote_only_outputs: tuple[RemoteOutput, ...] = () + missing_output_labels: frozenset[str] = frozenset() + blocked_labels: frozenset[str] = frozenset() + selected_artifact_outputs: frozenset[tuple[str, str]] = frozenset() + staged_outputs: frozenset[tuple[str, str]] = frozenset() + warning_codes: tuple[str, ...] = () + + +@dataclass(frozen=True) +class FreshnessPreparation: + """Freshness plan plus any staged artifacts owned by this invocation.""" + + plan: FreshnessPlan + scan_roots: tuple[ScanRoot, ...] + staged_roots: tuple[Path, ...] = () + _doctor_runtime: ModuleType | None = None + _staged_artifacts: tuple[object, ...] = () + _staging_base_path: Path | None = None + + def cleanup(self) -> None: + """Remove only per-invocation staging roots owned by the doctor.""" + if not self._staged_artifacts: + return + assert self._doctor_runtime is not None + assert self._staging_base_path is not None + try: + self._doctor_runtime._cleanup_staged_bep_run_roots( + self._staged_artifacts, + staging_base=self._staging_base_path, + ) + except SystemExit as exc: + raise FreshnessError(_doctor_failure(self._doctor_runtime, exc)) from exc + + +@dataclass(frozen=True) +class FreshnessFilterResult: + discovery: DiscoveryResult + skipped_outputs: tuple[str, ...] = () + + +def prepare_freshness( + config: UploaderConfig, + *, + resolver: RunfilesResolver, + local_testlogs_root: Path | None, + expected_targets: Iterable[str] = (), + logger: logging.Logger | None = None, +) -> FreshnessPreparation: + """Parse freshness and optionally stage BEP artifacts before discovery.""" + expected_target_set = frozenset(expected_targets) + warning_codes: list[str] = [] + bep_configured = bool(config.bep_json_files) + staging_requested = config.artifact_source == "bep" or ( + config.artifact_source == "auto" and config.remote_artifacts != "disabled" + ) + if config.artifact_source == "bep" and not bep_configured: + raise FreshnessError( + "--artifact-source=bep requires --bep-json or " + "DD_TEST_OPTIMIZATION_BEP_JSON" + ) + + selected_source = _select_source(config, bep_configured, warning_codes) + bep_processing_required = selected_source == "bep" or staging_requested + doctor_runtime: ModuleType | None = None + staged_artifacts: tuple[object, ...] = () + staging_base_path: Path | None = None + selected_artifact_outputs: set[tuple[str, str]] = set() + blocked_labels: set[str] = set() + doctor_freshness = None + + if bep_processing_required: + if not bep_configured: + raise FreshnessError("BEP artifact staging requires a configured BEP JSON file") + try: + bep_files = tuple( + _resolve_runtime_file(path, workspace=config.workspace) + for path in config.bep_json_files + ) + except FreshnessError: + if config.freshness_mode != "optional" or staging_requested: + raise + warning_codes.append("bep_freshness_unavailable") + selected_source = "none" + bep_processing_required = False + bep_files = () + if bep_processing_required: + doctor_path = _resolve_doctor_runtime(config, resolver) + doctor_runtime = _load_doctor_runtime(doctor_path, warning_codes) + freshness_unavailable_is_error = config.freshness_mode != "optional" or staging_requested + try: + doctor_freshness = doctor_runtime._parse_bep_freshness( + list(bep_files), + unavailable_is_error=freshness_unavailable_is_error, + ) + if doctor_freshness is not None: + if staging_requested: + selected_artifact_outputs = set( + doctor_runtime._selected_bep_artifact_outputs( + doctor_freshness, + config.workspace, + config.remote_artifacts, + ) + ) + blocked_labels = set( + doctor_runtime._blocked_bep_artifact_labels( + doctor_freshness, + config.remote_artifacts, + ) + ) + staging_base_path = config.artifact_staging_dir.resolve() + staged_artifacts = tuple( + doctor_runtime._stage_bep_artifacts( + doctor_freshness, + workspace=config.workspace, + staging_dir=staging_base_path, + remote_artifacts=config.remote_artifacts, + downloader=( + str(config.bep_artifact_downloader) + if config.bep_artifact_downloader is not None + else "" + ), + downloader_timeout_sec=( + config.bep_artifact_downloader_timeout_sec + ), + ) + ) + doctor_runtime._apply_staged_bep_artifacts_to_freshness( + doctor_freshness, + list(staged_artifacts), + ) + except SystemExit as exc: + raise FreshnessError(_doctor_failure(doctor_runtime, exc)) from exc + except BaseException: + # Staging must not survive interrupts or unexpected doctor errors. + if staged_artifacts and staging_base_path is not None: + doctor_runtime._cleanup_staged_bep_run_roots( + staged_artifacts, + staging_base=staging_base_path, + ) + raise + + try: + plan = _build_plan( + config, + selected_source=selected_source, + doctor_freshness=doctor_freshness, + selected_artifact_outputs=selected_artifact_outputs, + blocked_labels=blocked_labels, + expected_targets=expected_target_set, + warning_codes=warning_codes, + staged_outputs={ + (str(item.label), str(item.output_key)) for item in staged_artifacts + }, + ) + except BaseException: + # Planning failures must obey the same ownership cleanup contract. + if staged_artifacts and doctor_runtime is not None and staging_base_path is not None: + doctor_runtime._cleanup_staged_bep_run_roots( + staged_artifacts, + staging_base=staging_base_path, + ) + raise + + scan_roots: list[ScanRoot] = [] + if local_testlogs_root is not None: + scan_roots.append(ScanRoot(local_testlogs_root)) + staged_roots = tuple( + sorted( + {Path(item.staging_root).resolve() for item in staged_artifacts}, + key=lambda path: path.as_posix(), + ) + ) + scan_roots.extend(ScanRoot(root, staged=True) for root in staged_roots) + if logger is not None: + for artifact in staged_artifacts: + logger.debug( + "BEP artifact staging selected output key: %s", + artifact.output_key, + ) + logger.debug( + "freshness ready: source=%s eligible=%d cached=%d remote_only=%d " + "staged=%d", + plan.selected_source, + len(plan.eligible_outputs), + len(plan.cached_outputs), + len(plan.remote_only_outputs), + len(plan.staged_outputs), + ) + for warning in plan.warning_codes: + logger.warning("preflight warning_code=%s", warning) + return FreshnessPreparation( + plan=plan, + scan_roots=tuple(scan_roots), + staged_roots=staged_roots, + _doctor_runtime=doctor_runtime, + _staged_artifacts=staged_artifacts, + _staging_base_path=staging_base_path, + ) + + +def filter_discovery_for_freshness( + discovery: DiscoveryResult, + plan: FreshnessPlan, + *, + freshness_mode: str, +) -> FreshnessFilterResult: + """Select current-invocation outputs and stamp worker tasks with labels.""" + selected_outputs = [] + task_label_by_output: dict[str, str] = {} + skipped_output_keys: list[str] = [] + tasks_by_output: dict[str, list[FileTask]] = {} + for task in discovery.tasks: + tasks_by_output.setdefault(task.output_key or "", []).append(task) + + for output in discovery.outputs: + tasks = tasks_by_output.get(output.output_key, []) + explicit_labels = { + task.target_label for task in tasks if task.target_label is not None + } + target_label = ( + sorted(explicit_labels)[0] + if len(explicit_labels) == 1 + else _read_target_label(output.path) + ) + if target_label: + task_label_by_output[output.output_key] = target_label + if target_label in plan.blocked_labels: + skipped_output_keys.append(output.output_key) + continue + if not plan.eligibility_enabled: + selected_outputs.append(output) + continue + if not target_label: + if plan.selected_source == "bep" and freshness_mode == "required": + raise FreshnessError( + "BEP required freshness cannot authorize " + f"{output.path} because bazel.target metadata is missing" + ) + skipped_output_keys.append(output.output_key) + continue + pair = (target_label, output.output_key) + if pair in plan.eligible_outputs: + selected_outputs.append(output) + continue + if ( + plan.selected_source == "bep" + and freshness_mode == "required" + and target_label in plan.missing_output_labels + ): + raise FreshnessError( + "BEP required freshness cannot authorize " + f"{output.path} because the fresh TestResult for {target_label} " + "did not contain a mappable test.outputs reference" + ) + skipped_output_keys.append(output.output_key) + + selected_keys = {output.output_key for output in selected_outputs} + selected_tasks = tuple( + replace( + task, + target_label=task.target_label + or task_label_by_output.get(task.output_key or ""), + ) + for task in discovery.tasks + if (task.output_key or "") in selected_keys + ) + filtered_discovery = DiscoveryResult( + outputs=tuple(selected_outputs), + tasks=selected_tasks, + discovered_by_type=count_tasks_by_payload_type(selected_tasks), + warning_codes=tuple( + dict.fromkeys( + discovery.warning_codes + + plan.warning_codes + + (("freshness_outputs_skipped",) if skipped_output_keys else ()) + ) + ), + ) + return FreshnessFilterResult(filtered_discovery, tuple(sorted(set(skipped_output_keys)))) + + +def validate_fresh_outputs_accounted( + plan: FreshnessPlan, + discovery: DiscoveryResult, + results: Iterable[FileResult], + *, + expected_targets: Iterable[str], + fail_on_error: bool, +) -> None: + """Preserve the legacy fail-on-error check after all workers finish.""" + if not fail_on_error or plan.selected_source != "bep": + return + non_skipped_task_ids = { + result.task_id + for result in results + if result.status is not FileStatus.SKIPPED + } + handled_pairs = { + (task.target_label, task.output_key) + for task in discovery.tasks + if task.task_id in non_skipped_task_ids + and task.target_label is not None + and task.output_key is not None + } + expected_target_set = frozenset(expected_targets) + if expected_target_set: + unhandled_outputs = sorted(plan.eligible_outputs.difference(handled_pairs)) + if unhandled_outputs: + label, output_key = unhandled_outputs[0] + raise FreshnessError( + "fresh expected test output produced no uploadable payloads: " + f"{label} {output_key}", + exit_code=1, + ) + remote_outputs = tuple( + output + for output in plan.remote_only_outputs + if output.label in expected_target_set + ) + if remote_outputs: + raise FreshnessError( + "BEP references remote-only test outputs for " + f"{remote_outputs[0].label}, but local test.outputs was not found" + ) + return + if plan.eligible_outputs and not handled_pairs: + raise FreshnessError( + f"BEP reported {len(plan.eligible_outputs)} fresh test output(s), " + "but none produced uploadable payloads", + exit_code=1, + ) + + +def _select_source( + config: UploaderConfig, + bep_configured: bool, + warning_codes: list[str], +) -> str: + if config.freshness_mode == "disabled": + if bep_configured: + warning_codes.append("freshness_disabled_bep_ignored") + if config.execution_log_json is not None: + warning_codes.append("freshness_disabled_execution_log_ignored") + return "none" + freshness_required = config.freshness_mode == "required" or ( + config.freshness_mode == "auto" and config.ci + ) + if config.freshness_source == "bep": + if bep_configured: + return "bep" + if freshness_required: + raise FreshnessError( + "BEP freshness filtering is required but no BEP JSON file was configured" + ) + warning_codes.append("bep_freshness_not_configured") + return "none" + if config.freshness_source == "execution_log": + if config.execution_log_json is not None: + return "execution_log" + if freshness_required: + raise FreshnessError( + "execution-log freshness filtering is required but no execution log " + "was configured" + ) + warning_codes.append("execution_log_freshness_not_configured") + return "none" + if bep_configured: + return "bep" + if config.execution_log_json is not None: + return "execution_log" + if freshness_required: + raise FreshnessError( + "freshness filtering is required in CI or required mode, but no BEP " + "or execution log was configured" + ) + warning_codes.append("freshness_not_configured") + return "none" + + +def _build_plan( + config: UploaderConfig, + *, + selected_source: str, + doctor_freshness: object | None, + selected_artifact_outputs: set[tuple[str, str]], + blocked_labels: set[str], + expected_targets: frozenset[str], + warning_codes: list[str], + staged_outputs: set[tuple[str, str]], +) -> FreshnessPlan: + if selected_source == "execution_log": + assert config.execution_log_json is not None + execution_log = _resolve_runtime_file( + config.execution_log_json, + workspace=config.workspace, + ) + eligible_outputs = _parse_execution_log(execution_log) + if expected_targets: + eligible_outputs = {pair for pair in eligible_outputs if pair[0] in expected_targets} + return FreshnessPlan( + selected_source="execution_log", + eligibility_enabled=True, + eligible_outputs=frozenset(eligible_outputs), + warning_codes=tuple(dict.fromkeys(warning_codes)), + ) + + if doctor_freshness is None: + return FreshnessPlan( + selected_source="none", + selected_artifact_outputs=frozenset(selected_artifact_outputs), + blocked_labels=frozenset(blocked_labels), + warning_codes=tuple(dict.fromkeys(warning_codes)), + ) + + eligible_outputs = set(doctor_freshness.eligible_outputs) + cached_outputs = set(doctor_freshness.cached_outputs) + remote_outputs = tuple( + RemoteOutput(item.label, item.output_key, item.artifact) + for item in doctor_freshness.remote_only_outputs + ) + missing_output_labels = set(doctor_freshness.missing_output_mappings) + if expected_targets: + eligible_outputs = {pair for pair in eligible_outputs if pair[0] in expected_targets} + cached_outputs = {pair for pair in cached_outputs if pair[0] in expected_targets} + remote_outputs = tuple(item for item in remote_outputs if item.label in expected_targets) + missing_output_labels.intersection_update(expected_targets) + covered_labels = { + label for label, _output_key in eligible_outputs.union(cached_outputs) + }.union(item.label for item in remote_outputs).union(missing_output_labels) + absent = sorted(expected_targets.difference(covered_labels)) + if absent: + raise FreshnessError( + "expected target output is neither fresh nor exclusively cached in " + f"BEP: {absent[0]} (no TestResult matched this target)" + ) + missing_expected = sorted(expected_targets.intersection(missing_output_labels)) + if missing_expected: + raise FreshnessError( + "expected target output is neither fresh nor exclusively cached in " + f"BEP: {missing_expected[0]} (the fresh TestResult did not contain " + "a mappable test.outputs reference)" + ) + + if remote_outputs: + if config.freshness_mode == "required" or config.remote_artifacts == "required": + first_remote_output = remote_outputs[0] + raise FreshnessError( + "BEP references remote-only test outputs for " + f"{first_remote_output.label}, but local test.outputs was not found" + ) + warning_codes.append("bep_remote_only_outputs_skipped") + + return FreshnessPlan( + selected_source=selected_source, + eligibility_enabled=selected_source == "bep", + eligible_outputs=frozenset(eligible_outputs), + cached_outputs=frozenset(cached_outputs), + remote_only_outputs=remote_outputs, + missing_output_labels=frozenset(missing_output_labels), + blocked_labels=frozenset(blocked_labels), + selected_artifact_outputs=frozenset(selected_artifact_outputs), + staged_outputs=frozenset( + pair for pair in eligible_outputs if pair in staged_outputs + ), + warning_codes=tuple(dict.fromkeys(warning_codes)), + ) + + +def _parse_execution_log(path: Path) -> set[tuple[str, str]]: + eligible: set[tuple[str, str]] = set() + for line_number, raw_line in enumerate(_stream_execution_log(path), start=1): + if not raw_line.strip(): + continue + try: + value = strict_json_loads(raw_line) + except json.JSONDecodeError as exc: + raise FreshnessError( + f"invalid execution log JSON in {path}:{line_number}: {exc}" + ) from exc + if not isinstance(value, dict) or value.get("mnemonic", "") != "TestRunner": + continue + runner = value.get("runner", "") + if value.get("cacheHit", False) is True or ( + isinstance(runner, str) and "cache hit" in runner.lower() + ): + continue + label = value.get("targetLabel") + if not isinstance(label, str) or not label: + continue + raw_outputs: list[object] = [] + listed = value.get("listedOutputs", []) + if isinstance(listed, list): + raw_outputs.extend(listed) + actual = value.get("actualOutputs", []) + if isinstance(actual, list): + for item in actual: + raw_outputs.append(item.get("path", "") if isinstance(item, dict) else "") + for raw_output in raw_outputs: + if not isinstance(raw_output, str): + continue + output_key = _execution_output_key(raw_output) + if output_key: + eligible.add((label, output_key)) + return eligible + + +def _stream_execution_log(path: Path) -> Iterable[str]: + """Yield one action record at a time so large logs stay memory-bounded.""" + try: + with path.open("r", encoding="utf-8-sig") as lines: + yield from lines + except (OSError, UnicodeError) as exc: + raise FreshnessError(f"failed to read execution log JSON {path}: {exc}") from exc + + +def _execution_output_key(raw: str) -> str: + normalized = raw.replace("\\", "/") + if "/testlogs/" in normalized: + normalized = normalized.rsplit("/testlogs/", 1)[1] + if "/test.outputs/" in normalized: + normalized = normalized.split("/test.outputs/", 1)[0] + "/test.outputs" + elif not normalized.endswith("/test.outputs"): + return "" + return normalized.removeprefix("./").lstrip("/") + + +def _read_target_label(output_dir: Path) -> str | None: + metadata_file = output_dir / "bazel_target_metadata.json" + try: + if metadata_file.is_symlink(): + return None + if metadata_file.resolve(strict=True).parent != output_dir.resolve(strict=True): + return None + value = strict_json_loads(metadata_file.read_text(encoding="utf-8-sig")) + except (OSError, UnicodeError, json.JSONDecodeError): + return None + label = value.get("bazel.target") if isinstance(value, dict) else None + return label if isinstance(label, str) and label else None + + +def _resolve_runtime_file(path: Path, *, workspace: Path) -> Path: + expanded = path.expanduser() + candidates = ( + (expanded if expanded.is_absolute() else workspace / expanded), + (expanded if expanded.is_absolute() else Path.cwd() / expanded), + ) + for candidate in candidates: + if candidate.is_file(): + return candidate.resolve() + raise FreshnessError(f"configured freshness file not found: {path}") + + +def _resolve_doctor_runtime( + config: UploaderConfig, + resolver: RunfilesResolver, +) -> Path: + candidates = tuple( + candidate + for candidate in ( + config.rule.doctor_runtime_path, + config.rule.doctor_runtime_short_path, + ) + if candidate + ) + if not candidates: + raise FreshnessError("generated uploader config has no doctor runtime") + try: + return resolver.resolve_file(candidates) + except RunfileResolutionError as exc: + raise FreshnessError("doctor runtime could not be resolved from runfiles") from exc + + +def _load_doctor_runtime(path: Path, warning_codes: list[str]) -> ModuleType: + name = f"_dd_topt_uploader_doctor_{id(warning_codes)}" + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise FreshnessError(f"doctor runtime is not importable: {path}") + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + try: + spec.loader.exec_module(module) + except BaseException: + sys.modules.pop(name, None) + raise + module._warn = lambda _message: warning_codes.append("bep_runtime_warning") + module._info = lambda _message: None + return module + + +def _doctor_failure(doctor_runtime: ModuleType, exc: SystemExit) -> str: + message = getattr(doctor_runtime, "_LAST_FAILURE_MESSAGE", "") + return message or f"doctor BEP runtime failed with exit code {exc.code}" diff --git a/tools/core/uploader_py/json_utils.py b/tools/core/uploader_py/json_utils.py new file mode 100644 index 00000000..9b631e3f --- /dev/null +++ b/tools/core/uploader_py/json_utils.py @@ -0,0 +1,33 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Encode and decode strict, interoperable JSON for uploader contracts. + +Central validation rejects non-standard values before they reach split or HTTP code. +""" + +from __future__ import annotations + +import json +from typing import Any + + +def _reject_non_finite_constant(value: str) -> None: + raise json.JSONDecodeError( + f"non-finite numeric constant {value!r} is not valid JSON", + value, + 0, + ) + + +def strict_json_loads(document: str | bytes | bytearray) -> Any: + """Parse RFC-compatible JSON and reject NaN/Infinity extensions.""" + return json.loads(document, parse_constant=_reject_non_finite_constant) + + +def strict_json_dumps(value: Any, **kwargs: Any) -> str: + """Serialize JSON while refusing non-finite floating-point values.""" + return json.dumps(value, allow_nan=False, **kwargs) diff --git a/tools/core/uploader_py/locking.py b/tools/core/uploader_py/locking.py new file mode 100644 index 00000000..36279e7e --- /dev/null +++ b/tools/core/uploader_py/locking.py @@ -0,0 +1,310 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Provide one cross-platform process lock per Bazel workspace. + +Serializing invocations prevents races over source deletion and staging cleanup. +""" + +from __future__ import annotations + +from contextlib import contextmanager +import hashlib +import os +from pathlib import Path +import tempfile +import time +from typing import BinaryIO, Callable, Iterator + + +DEFAULT_LOCK_ATTEMPTS = 3 +DEFAULT_LOCK_RETRY_SECONDS = 1.0 +DEFAULT_INCOMPLETE_LOCK_STALE_SECONDS = 30.0 + + +class WorkspaceLockError(RuntimeError): + """The uploader could not safely acquire its workspace lock.""" + + +def workspace_lock_name(workspace: str | Path) -> str: + """Return the legacy lock name for an exact workspace path string.""" + # Keep the legacy digest so Python and the Bash/PowerShell uploaders + # contend on the same lock during rollout. The digest only names a local + # lock; it is not used for a security purpose. + digest = hashlib.md5( + str(workspace).encode("utf-8"), + usedforsecurity=False, + ).hexdigest()[:8] + return f"dd_upload_payloads_{digest}.lock" + + +def _default_process_alive(pid: int) -> bool: + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + except OSError: + return False + return True + + +class WorkspaceLock: + """Own one uploader lock until explicitly released or context exit. + + Unix uses the legacy atomic directory plus PID metadata and serializes + acquisition/reclamation with a small advisory guard lock. Windows holds a + non-blocking byte-range lock for the lifetime of the process, matching the + lifetime semantics of PowerShell's exclusive file handle without requiring + .NET or third-party packages. + """ + + def __init__( + self, + workspace: str | Path, + *, + temp_root: Path | None = None, + attempts: int = DEFAULT_LOCK_ATTEMPTS, + retry_seconds: float = DEFAULT_LOCK_RETRY_SECONDS, + incomplete_stale_seconds: float = DEFAULT_INCOMPLETE_LOCK_STALE_SECONDS, + sleeper: Callable[[float], None] = time.sleep, + clock: Callable[[], float] = time.time, + process_alive: Callable[[int], bool] = _default_process_alive, + platform: str = os.name, + ) -> None: + if attempts <= 0: + raise ValueError("lock attempts must be positive") + if retry_seconds < 0: + raise ValueError("lock retry seconds must be non-negative") + self.workspace = str(workspace) + self.temp_root = Path(temp_root or tempfile.gettempdir()).resolve() + self.path = self.temp_root / workspace_lock_name(self.workspace) + self.attempts = attempts + self.retry_seconds = retry_seconds + self.incomplete_stale_seconds = incomplete_stale_seconds + self._sleeper = sleeper + self._clock = clock + self._process_alive = process_alive + self._platform = platform + self._owned = False + self._windows_lock_file: BinaryIO | None = None + + @property + def acquired(self) -> bool: + return self._owned + + def acquire(self) -> "WorkspaceLock": + if self._owned: + raise WorkspaceLockError(f"workspace lock is already acquired: {self.path}") + if not self.temp_root.is_dir(): + raise WorkspaceLockError( + f"temporary root for workspace lock is not a directory: {self.temp_root}" + ) + if self._platform == "nt": + self._acquire_windows() + else: + self._acquire_unix() + return self + + def release(self) -> None: + if not self._owned: + return + if self._platform == "nt": + self._release_windows() + else: + self._release_unix() + self._owned = False + + def __enter__(self) -> "WorkspaceLock": + return self.acquire() + + def __exit__(self, exc_type: object, exc: object, traceback: object) -> None: + self.release() + + def _acquire_unix(self) -> None: + # Stale inspection and removal must be one Python-process critical + # section. Otherwise a contender can replace the stale directory after + # inspection and have its newly acquired lock removed by this process. + with self._unix_guard(): + self._acquire_unix_guarded() + + def _acquire_unix_guarded(self) -> None: + last_detail = "lock remained unavailable" + for attempt in range(self.attempts): + try: + self.path.mkdir() + except FileExistsError: + lock_state = self._inspect_unix_lock() + if lock_state == "dead": + last_detail = "stale lock could not be removed" + self._remove_unix_lock_if_simple() + continue + if lock_state == "alive": + raise WorkspaceLockError( + f"another uploader is already running (lock: {self.path})" + ) + last_detail = "lock has fresh or unreadable PID metadata" + if attempt + 1 < self.attempts: + self._sleeper(self.retry_seconds) + continue + except OSError as exc: + raise WorkspaceLockError( + f"failed to create workspace lock {self.path}: {exc}" + ) from exc + + try: + pid_path = self.path / "pid" + with pid_path.open("x", encoding="ascii") as pid_file: + pid_file.write(f"{os.getpid()}\n") + except OSError as exc: + self._remove_unix_lock_if_simple() + raise WorkspaceLockError( + f"failed to initialize workspace lock metadata at {self.path / 'pid'}: {exc}" + ) from exc + self._owned = True + return + + raise WorkspaceLockError( + f"could not acquire workspace lock {self.path}: {last_detail}" + ) + + def _inspect_unix_lock(self) -> str: + """Return ``alive``, ``dead``, or ``incomplete`` for an existing lock.""" + if self.path.is_symlink() or not self.path.is_dir(): + return "incomplete" + try: + pid_text = (self.path / "pid").read_text(encoding="ascii").strip() + except (FileNotFoundError, OSError, UnicodeError): + return "dead" if self._incomplete_lock_is_stale() else "incomplete" + if not pid_text.isdecimal() or int(pid_text) <= 0: + return "dead" if self._incomplete_lock_is_stale() else "incomplete" + return "alive" if self._process_alive(int(pid_text)) else "dead" + + def _incomplete_lock_is_stale(self) -> bool: + try: + age = self._clock() - self.path.stat().st_mtime + except OSError: + return False + return age > self.incomplete_stale_seconds + + def _remove_unix_lock_if_simple(self) -> bool: + """Remove only the exact lock and expected PID file, never a tree.""" + if self.path.is_symlink() or not self.path.is_dir(): + return False + try: + if any(child.name != "pid" for child in self.path.iterdir()): + return False + (self.path / "pid").unlink(missing_ok=True) + self.path.rmdir() + return True + except FileNotFoundError: + return True + except OSError: + return False + + def _release_unix(self) -> None: + try: + with self._unix_guard(): + self._release_unix_guarded() + except WorkspaceLockError: + # Failure to enter the cleanup guard must not replace an already + # completed upload result. The PID directory remains recoverable as + # a stale lock on the next invocation. + return + + def _release_unix_guarded(self) -> None: + # Only an instance that successfully created the lock reaches this + # method. Refuse to remove it if PID metadata no longer identifies us. + if self.path.is_symlink() or not self.path.is_dir(): + return + try: + pid_text = (self.path / "pid").read_text(encoding="ascii").strip() + except (OSError, UnicodeError): + return + if pid_text != str(os.getpid()): + return + self._remove_unix_lock_if_simple() + + @contextmanager + def _unix_guard(self) -> Iterator[None]: + """Serialize Python lock lifecycle changes without changing lock parity.""" + try: + import fcntl + except ImportError as exc: # pragma: no cover - guarded by Unix CI + raise WorkspaceLockError("Unix locking support is unavailable") from exc + + guard_path = self.path.with_name(self.path.name + ".guard") + guard_file: BinaryIO | None = None + try: + guard_file = guard_path.open("a+b") + fcntl.flock(guard_file.fileno(), fcntl.LOCK_EX) + yield + except OSError as exc: + raise WorkspaceLockError( + f"failed to coordinate workspace lock {self.path}: {exc}" + ) from exc + finally: + if guard_file is not None: + try: + fcntl.flock(guard_file.fileno(), fcntl.LOCK_UN) + finally: + guard_file.close() + + def _acquire_windows(self) -> None: + try: + import msvcrt + except ImportError as exc: # pragma: no cover - guarded by Windows CI + raise WorkspaceLockError("Windows locking support is unavailable") from exc + + last_lock_error: OSError | None = None + for attempt in range(self.attempts): + lock_file: BinaryIO | None = None + try: + lock_file = self.path.open("a+b") + lock_file.seek(0, os.SEEK_END) + if lock_file.tell() == 0: + lock_file.write(b"\0") + lock_file.flush() + lock_file.seek(0) + msvcrt.locking(lock_file.fileno(), msvcrt.LK_NBLCK, 1) + except OSError as exc: + last_lock_error = exc + if lock_file is not None: + lock_file.close() + if attempt + 1 < self.attempts: + self._sleeper(self.retry_seconds) + continue + self._windows_lock_file = lock_file + self._owned = True + return + + raise WorkspaceLockError( + "another uploader is already running " + f"(lock: {self.path}): {last_lock_error}" + ) + + def _release_windows(self) -> None: + try: + import msvcrt + except ImportError: # pragma: no cover - guarded by Windows CI + return + lock_file = self._windows_lock_file + self._windows_lock_file = None + if lock_file is None: + return + try: + lock_file.seek(0) + msvcrt.locking(lock_file.fileno(), msvcrt.LK_UNLCK, 1) + finally: + lock_file.close() + try: + self.path.unlink() + except FileNotFoundError: + pass + except OSError: + # Another process may already have opened the persistent lock file. + pass diff --git a/tools/core/uploader_py/logging_utils.py b/tools/core/uploader_py/logging_utils.py new file mode 100644 index 00000000..05403afe --- /dev/null +++ b/tools/core/uploader_py/logging_utils.py @@ -0,0 +1,90 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Configure thread-safe uploader logging with explicit redaction. + +One logging boundary keeps debug useful without leaking credentials or payloads. +""" + +from __future__ import annotations + +import logging +import sys +from typing import Iterable, TextIO +from urllib.parse import urlsplit, urlunsplit + + +LOGGER_NAME = "dd-uploader" +_SENSITIVE_HEADER_NAMES = frozenset( + { + "authorization", + "cookie", + "dd-api-key", + "proxy-authorization", + "set-cookie", + } +) + + +class SecretRedactionFilter(logging.Filter): + """Replace known non-empty secrets after record interpolation.""" + + def __init__(self, secrets: Iterable[str]) -> None: + super().__init__() + self._secrets = tuple(sorted({value for value in secrets if value}, key=len, reverse=True)) + + def filter(self, record: logging.LogRecord) -> bool: + message = record.getMessage() + for secret in self._secrets: + message = message.replace(secret, "") + record.msg = message + record.args = () + return True + + +def configure_logging( + *, + debug: bool, + secrets: Iterable[str] = (), + stream: TextIO | None = None, +) -> logging.Logger: + """Configure the one process-wide uploader logger.""" + logger = logging.getLogger(LOGGER_NAME) + logger.handlers.clear() + logger.propagate = False + logger.setLevel(logging.DEBUG if debug else logging.INFO) + handler = logging.StreamHandler(stream if stream is not None else sys.stderr) + handler.setLevel(logging.DEBUG if debug else logging.INFO) + handler.setFormatter(logging.Formatter("[dd-uploader] %(levelname)s: %(message)s")) + handler.addFilter(SecretRedactionFilter(secrets)) + logger.addHandler(handler) + return logger + + +def redact_header_value(name: str, value: str) -> str: + """Hide authentication-bearing header values completely.""" + if name.lower() in _SENSITIVE_HEADER_NAMES: + return "" + return value + + +def redact_url(raw_url: str) -> str: + """Remove URL userinfo, query, and fragment while retaining routing context.""" + try: + parsed = urlsplit(raw_url) + if not parsed.scheme or not parsed.netloc: + return "" + host = parsed.hostname or "" + if not host: + return f"{parsed.scheme.lower()}://" + if ":" in host and not host.startswith("["): + host = f"[{host}]" + netloc = host + if parsed.port is not None: + netloc = f"{netloc}:{parsed.port}" + return urlunsplit((parsed.scheme.lower(), netloc, parsed.path, "", "")) + except (TypeError, ValueError): + return "" diff --git a/tools/core/uploader_py/main.py b/tools/core/uploader_py/main.py new file mode 100644 index 00000000..8f1f2cb9 --- /dev/null +++ b/tools/core/uploader_py/main.py @@ -0,0 +1,65 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Validate the host and enter the shared Python uploader runtime. + +Top-level error handling stays here so internal modules can use typed failures. +""" + +from __future__ import annotations + +import sys +from typing import Sequence + + +MINIMUM_PYTHON = (3, 10) + + +def python_version_is_supported(major: int, minor: int) -> bool: + """Return whether an interpreter satisfies the uploader runtime contract.""" + return (major, minor) >= MINIMUM_PYTHON + + +def main(argv: Sequence[str] | None = None) -> int: + """Validate startup and configuration before entering uploader preflight.""" + if not python_version_is_supported(sys.version_info.major, sys.version_info.minor): + print( + "[dd-uploader] error: Python 3.10 or newer is required " + f"(found {sys.version_info.major}.{sys.version_info.minor})", + file=sys.stderr, + ) + return 2 + + from .config import ConfigError, parse_uploader_config + from .endpoints import build_endpoints + from .logging_utils import configure_logging + from topt_runtime.runfiles import RunfileResolutionError, RunfilesResolver + + try: + config = parse_uploader_config(sys.argv[1:] if argv is None else argv) + endpoints = build_endpoints(config) + except ConfigError as exc: + print(f"[dd-uploader] error: {exc}", file=sys.stderr) + return 2 + + logger = configure_logging(debug=config.debug, secrets=(config.api_key,)) + try: + resolver = RunfilesResolver.from_environment(argv0=sys.argv[0]) + except RunfileResolutionError as exc: + logger.error("%s", exc) + return 2 + from .application import run_uploader + + try: + return run_uploader( + config, + resolver=resolver, + endpoints=endpoints, + logger=logger, + ) + except KeyboardInterrupt: + logger.error("interrupted") + return 130 diff --git a/tools/core/uploader_py/models.py b/tools/core/uploader_py/models.py new file mode 100644 index 00000000..03e4d718 --- /dev/null +++ b/tools/core/uploader_py/models.py @@ -0,0 +1,72 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Define immutable values exchanged by the coordinator and workers. + +Explicit result models replace shared counters and make aggregation deterministic. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from pathlib import Path + + +MAX_TEST_PAYLOAD_BYTES = 4_718_592 +DEFAULT_WORKERS = 8 + + +class PayloadType(str, Enum): + """Payload protocols supported by every uploader worker.""" + + TEST = "test" + COVERAGE = "coverage" + TELEMETRY = "telemetry" + + +class FileStatus(str, Enum): + """Terminal source-file outcomes returned to the coordinator.""" + + SUCCEEDED = "succeeded" + FAILED = "failed" + SKIPPED = "skipped" + + +@dataclass(frozen=True) +class FileTask: + """One source file owned by exactly one worker.""" + + task_id: str + source_path: Path + display_path: str + payload_type: PayloadType + test_outputs_dir: Path | None = None + output_key: str | None = None + target_label: str | None = None + + +@dataclass(frozen=True) +class FileResult: + """Immutable worker result aggregated by the coordinator.""" + + task_id: str + source_path: str + payload_type: PayloadType + status: FileStatus + events: int = 0 + chunks_created: int = 0 + chunks_uploaded: int = 0 + chunks_failed: int = 0 + requests_planned: int = 0 + requests_attempted: int = 0 + requests_succeeded: int = 0 + requests_failed: int = 0 + retries: int = 0 + source_deleted: bool = False + warning_codes: tuple[str, ...] = () + failure_code: str | None = None + failure_message: str | None = None diff --git a/tools/core/uploader_py/reporting.py b/tools/core/uploader_py/reporting.py new file mode 100644 index 00000000..dc58eadc --- /dev/null +++ b/tools/core/uploader_py/reporting.py @@ -0,0 +1,517 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Aggregate results and render deterministic human and JSON reports. + +One report model prevents terminal output and machine-readable counters from drifting. +""" + +from __future__ import annotations + +from collections import Counter +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterable, Mapping, TextIO + +from .json_utils import strict_json_dumps +from .models import FileResult, FileStatus, MAX_TEST_PAYLOAD_BYTES, PayloadType + + +@dataclass(frozen=True) +class LegacyReportContext: + """Pre-worker values needed to preserve every schema-v1 report section.""" + + validate_enrichment: bool = False + artifact_source: str = "local" + remote_artifacts: str = "disabled" + freshness_source: str = "auto" + freshness_mode: str = "auto" + allow_cached_payload_uploads: bool = False + bep_files: tuple[str, ...] = () + freshness_selected_source: str = "none" + freshness_eligible_outputs: int = 0 + freshness_cached_outputs: int = 0 + freshness_remote_only_outputs: int = 0 + freshness_skipped_outputs: int = 0 + freshness_missing_output_labels: int = 0 + staging_dir: str = "" + staged_testlogs_dirs: int = 0 + selected_remote_artifacts: int = 0 + staged_remote_artifacts: int = 0 + remote_artifacts_ignored: int = 0 + test_outputs_dirs: int = 0 + reason_code: str = "" + reason: str = "" + next_steps: tuple[str, ...] = () + + +@dataclass(frozen=True) +class AggregateReport: + """Single source of truth for terminal stdout and JSON statistics.""" + + dry_run: bool + exit_code: int + configured_workers: int + worker_threads: int + peak_active_workers: int + elapsed_seconds: float + discovered_by_type: tuple[tuple[PayloadType, int], ...] + results: tuple[FileResult, ...] + cancelled: int = 0 + initialization_warning_codes: tuple[str, ...] = () + + @classmethod + def create( + cls, + *, + dry_run: bool, + exit_code: int, + configured_workers: int, + worker_threads: int, + peak_active_workers: int, + elapsed_seconds: float, + discovered_by_type: Mapping[PayloadType, int], + results: Iterable[FileResult], + cancelled: int = 0, + initialization_warning_codes: Iterable[str] = (), + ) -> "AggregateReport": + if configured_workers <= 0: + raise ValueError("configured_workers must be positive") + integer_values = { + "exit_code": exit_code, + "worker_threads": worker_threads, + "peak_active_workers": peak_active_workers, + "cancelled": cancelled, + } + if any(value < 0 for value in integer_values.values()): + raise ValueError("aggregate integer counters must be non-negative") + if elapsed_seconds < 0: + raise ValueError("elapsed_seconds must be non-negative") + discovered = tuple( + (payload_type, int(discovered_by_type.get(payload_type, 0))) + for payload_type in PayloadType + ) + if any(count < 0 for _payload_type, count in discovered): + raise ValueError("discovered counters must be non-negative") + terminal_results = tuple(results) + task_ids = [result.task_id for result in terminal_results] + if len(task_ids) != len(set(task_ids)): + raise ValueError("aggregate results must contain unique task IDs") + if peak_active_workers > worker_threads: + raise ValueError("peak_active_workers cannot exceed worker_threads") + return cls( + dry_run=dry_run, + exit_code=exit_code, + configured_workers=configured_workers, + worker_threads=worker_threads, + peak_active_workers=peak_active_workers, + elapsed_seconds=elapsed_seconds, + discovered_by_type=discovered, + results=terminal_results, + cancelled=cancelled, + initialization_warning_codes=tuple(initialization_warning_codes), + ) + + @property + def mode(self) -> str: + return "dry-run" if self.dry_run else "upload" + + @property + def result_name(self) -> str: + if self.exit_code == 0: + return "success" + if any( + result.status is FileStatus.SUCCEEDED or result.requests_succeeded > 0 + for result in self.results + ): + return "partial_failure" + return "failure" + + def statistics(self) -> dict[str, Any]: + """Return the exact data model consumed by all report renderers.""" + succeeded = _status_count(self.results, FileStatus.SUCCEEDED) + failed = _status_count(self.results, FileStatus.FAILED) + skipped = _status_count(self.results, FileStatus.SKIPPED) + processed = len(self.results) + eligible = processed + self.cancelled + deleted = sum(int(result.source_deleted) for result in self.results) + discovered = sum(count for _payload_type, count in self.discovered_by_type) + type_summaries = { + payload_type.value: _payload_type_counts(self.results, payload_type) + for payload_type in PayloadType + } + failure_codes = Counter( + result.failure_code + for result in self.results + if result.failure_code is not None + ) + warning_codes = Counter(self.initialization_warning_codes) + warning_codes.update( + warning + for result in self.results + for warning in result.warning_codes + ) + return { + "mode": self.mode, + "result": self.result_name, + "exit_code": self.exit_code, + "elapsed_seconds": round(self.elapsed_seconds, 6), + "files": { + "discovered": discovered, + "eligible": eligible, + "processed": processed, + "succeeded": succeeded, + "failed": failed, + "skipped": skipped, + "cancelled": self.cancelled, + "deleted": deleted, + "retained": max(0, eligible - deleted), + }, + "payload_types": type_summaries, + "discovered_by_type": { + payload_type.value: count + for payload_type, count in self.discovered_by_type + }, + "concurrency": { + "workers": self.configured_workers, + "worker_threads": self.worker_threads, + "peak_active_workers": self.peak_active_workers, + }, + "splitting": { + "threshold_bytes": MAX_TEST_PAYLOAD_BYTES, + "source_files_split": sum( + int(result.chunks_created > 1) for result in self.results + ), + "chunks_created": sum(result.chunks_created for result in self.results), + "chunks_uploaded": sum(result.chunks_uploaded for result in self.results), + "chunks_failed": sum(result.chunks_failed for result in self.results), + "oversized_single_events": sum( + int(result.failure_code == "single_event_exceeds_payload_limit") + for result in self.results + ), + }, + "requests": { + "planned": sum(result.requests_planned for result in self.results), + "attempted": sum(result.requests_attempted for result in self.results), + "succeeded": sum(result.requests_succeeded for result in self.results), + "failed": sum(result.requests_failed for result in self.results), + "retries": sum(result.retries for result in self.results), + }, + "warnings": dict(sorted(warning_codes.items())), + "failures": dict(sorted(failure_codes.items())), + } + + def human_lines(self) -> tuple[str, ...]: + """Render a compact stable statistics block for CI logs.""" + stats = self.statistics() + files = stats["files"] + payload_types = stats["payload_types"] + splitting = stats["splitting"] + requests = stats["requests"] + concurrency = stats["concurrency"] + + def type_value(name: str) -> str: + counts = payload_types[name] + return f"{counts['succeeded']}/{counts['failed']}/{counts['skipped']}" + + return ( + ( + f"[dd-uploader] summary: mode={self.mode} result={self.result_name} " + f"exit_code={self.exit_code} workers={self.configured_workers} " + f"peak={concurrency['peak_active_workers']} " + f"elapsed={self.elapsed_seconds:.2f}s" + ), + ( + "[dd-uploader] files: " + f"discovered={files['discovered']} eligible={files['eligible']} " + f"processed={files['processed']} succeeded={files['succeeded']} " + f"failed={files['failed']} skipped={files['skipped']} " + f"cancelled={files['cancelled']}" + ), + ( + "[dd-uploader] types: " + f"tests={type_value('test')} coverage={type_value('coverage')} " + f"telemetry={type_value('telemetry')} (succeeded/failed/skipped)" + ), + ( + "[dd-uploader] split: " + f"files={splitting['source_files_split']} " + f"chunks_created={splitting['chunks_created']} " + f"chunks_uploaded={splitting['chunks_uploaded']} " + f"chunks_failed={splitting['chunks_failed']}" + ), + ( + "[dd-uploader] requests: " + f"planned={requests['planned']} attempted={requests['attempted']} " + f"succeeded={requests['succeeded']} failed={requests['failed']} " + f"retries={requests['retries']}" + ), + ( + "[dd-uploader] cleanup: " + f"deleted={files['deleted']} retained={files['retained']}" + ), + ) + + def schema_v1_report( + self, + context: LegacyReportContext, + ) -> dict[str, Any]: + """Preserve legacy fields and append explicit source/request sections.""" + stats = self.statistics() + status = "ok" if self.exit_code == 0 else "fail" + reason_code, reason, next_steps = _result_reason(self, context) + legacy_counts = _legacy_payload_type_counts(self) + failed_payloads = sum( + counts["failed"] for counts in legacy_counts.values() + ) + upload_attempted = not self.dry_run and stats["requests"]["attempted"] > 0 + successful_payloads = sum( + counts["processed"] for counts in legacy_counts.values() + ) + report_document = { + "schema_version": 1, + "tool": "dd-test-optimization-uploader", + "status": status, + "exit_code": self.exit_code, + "result": { + "status": status, + "reason_code": reason_code, + "reason": reason, + "next_steps": list(next_steps), + }, + "config": { + "dry_run": self.dry_run, + "validate_enrichment": context.validate_enrichment, + "artifact_source": context.artifact_source, + "remote_artifacts": context.remote_artifacts, + "freshness_source": context.freshness_source, + "freshness_mode": context.freshness_mode, + "allow_cached_payload_uploads": context.allow_cached_payload_uploads, + }, + "bep": { + "files": list(context.bep_files), + "freshness_selected_source": context.freshness_selected_source, + "eligible_outputs": context.freshness_eligible_outputs, + "cached_outputs": context.freshness_cached_outputs, + "remote_only_outputs": context.freshness_remote_only_outputs, + "skipped_outputs": context.freshness_skipped_outputs, + "missing_output_labels": context.freshness_missing_output_labels, + }, + "artifacts": { + "source": context.artifact_source, + "staging_dir": context.staging_dir, + "staged_testlogs_dirs": context.staged_testlogs_dirs, + "selected_remote_artifacts": context.selected_remote_artifacts, + "staged_remote_artifacts": context.staged_remote_artifacts, + "remote_artifacts_ignored": context.remote_artifacts_ignored, + }, + "upload": { + "attempted": upload_attempted, + "dry_run": self.dry_run, + "payloads_attempted": ( + successful_payloads + failed_payloads if upload_attempted else 0 + ), + "payloads_uploaded": successful_payloads if upload_attempted else 0, + "payloads_failed": failed_payloads, + }, + "payloads": { + "test_outputs_dirs": context.test_outputs_dirs, + "discovered": { + "tests": stats["discovered_by_type"]["test"], + "coverage": stats["discovered_by_type"]["coverage"], + "telemetry": stats["discovered_by_type"]["telemetry"], + }, + "tests": legacy_counts["tests"], + "coverage": legacy_counts["coverage"], + "telemetry": legacy_counts["telemetry"], + }, + "upload_failures": failed_payloads, + } + # New sections are copied verbatim from the same aggregate used by + # stdout so legacy and explicit source/request counters cannot drift. + for key in ( + "files", + "payload_types", + "discovered_by_type", + "concurrency", + "splitting", + "requests", + "warnings", + "failures", + ): + report_document[key] = stats[key] + return report_document + + +def write_statistics_json(path: Path, report: AggregateReport) -> None: + """Atomically write the aggregate statistics without partial JSON files.""" + _write_json(path, report.statistics()) + + +def write_schema_v1_report( + path: Path, + report: AggregateReport, + context: LegacyReportContext, +) -> None: + """Atomically write the backward-compatible public uploader report.""" + _write_json(path, report.schema_v1_report(context)) + + +def emit_report( + report: AggregateReport, + *, + stream: TextIO, + report_json: Path | None = None, + legacy_report_context: LegacyReportContext | None = None, +) -> None: + """Render stdout and optional JSON from one immutable aggregate report.""" + for line in report.human_lines(): + print(line, file=stream) + if report_json is None: + return + try: + if legacy_report_context is None: + write_statistics_json(report_json, report) + else: + write_schema_v1_report(report_json, report, legacy_report_context) + except OSError as exc: + print( + "[dd-uploader] warning: failed to write uploader report: " + f"{type(exc).__name__}", + file=stream, + ) + + +def _write_json(path: Path, payload: Mapping[str, Any]) -> None: + parent = path.parent + parent.mkdir(parents=True, exist_ok=True) + temporary = parent / f".{path.name}.tmp" + try: + temporary.write_text( + strict_json_dumps( + payload, + ensure_ascii=False, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + temporary.replace(path) + except OSError: + try: + temporary.unlink() + except OSError: + pass + raise + + +def _legacy_payload_type_counts( + report: AggregateReport, +) -> dict[str, dict[str, int]]: + """Recreate schema-v1 counters, where processed means successful.""" + counts_by_type: dict[str, dict[str, int]] = {} + for payload_type, legacy_key in ( + (PayloadType.TEST, "tests"), + (PayloadType.COVERAGE, "coverage"), + ): + type_results = tuple( + result + for result in report.results + if result.payload_type is payload_type + ) + counts_by_type[legacy_key] = { + "processed": _status_count(type_results, FileStatus.SUCCEEDED), + "failed": _status_count(type_results, FileStatus.FAILED), + "skipped": _status_count(type_results, FileStatus.SKIPPED), + } + + telemetry = tuple( + result + for result in report.results + if result.payload_type is PayloadType.TELEMETRY + ) + if report.dry_run: + telemetry_processed = sum( + result.requests_planned + for result in telemetry + if result.status is FileStatus.SUCCEEDED + ) + else: + telemetry_processed = sum(result.requests_succeeded for result in telemetry) + telemetry_failed = sum( + ( + result.requests_failed + if result.requests_failed > 0 + else int(result.status is FileStatus.FAILED) + ) + for result in telemetry + ) + counts_by_type["telemetry"] = { + "processed": telemetry_processed, + "failed": telemetry_failed, + "skipped": _status_count(telemetry, FileStatus.SKIPPED), + } + return counts_by_type + + +def _result_reason( + report: AggregateReport, + context: LegacyReportContext, +) -> tuple[str, str, tuple[str, ...]]: + if context.reason_code: + return context.reason_code, context.reason, context.next_steps + stats = report.statistics() + if context.test_outputs_dirs == 0: + return ( + "no_test_outputs_found", + "No local or staged test.outputs directories were found.", + ( + "Use --artifact-source=bep with the matching --bep-json, or " + "configure Bazel to materialize test outputs.", + ), + ) + if stats["files"]["discovered"] == 0: + return ( + "no_payload_json_found", + "Test output directories were found, but no payloads were available.", + ("Inspect TEST_UNDECLARED_OUTPUTS_DIR and outputs.zip.",), + ) + if report.exit_code != 0 and stats["requests"]["attempted"] > 0: + return ( + "upload_failed_http", + "One or more payload uploads failed.", + ("Check HTTP status diagnostics and Datadog credentials/site configuration.",), + ) + if report.exit_code != 0: + return ( + "payload_enrichment_failed", + "Dry-run or payload processing failed for at least one payload.", + ("Inspect uploader logs for the first payload validation failure.",), + ) + if report.dry_run: + return ( + "upload_skipped_dry_run", + "Dry-run completed successfully; real upload was not requested.", + ("Run again without --dry-run to send payloads.",), + ) + return "ok", "Uploader completed successfully.", () + + +def _payload_type_counts( + results: tuple[FileResult, ...], payload_type: PayloadType +) -> dict[str, int]: + type_results = tuple( + result for result in results if result.payload_type is payload_type + ) + return { + status.value: _status_count(type_results, status) + for status in FileStatus + } + + +def _status_count(results: Iterable[FileResult], status: FileStatus) -> int: + return sum(int(result.status is status) for result in results) diff --git a/tools/core/uploader_py/resources.py b/tools/core/uploader_py/resources.py new file mode 100644 index 00000000..99679185 --- /dev/null +++ b/tools/core/uploader_py/resources.py @@ -0,0 +1,359 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Resolve contexts, telemetry facts, and schema before workers start. + +Loading once removes runfiles I/O and mutable resource selection from workers. +""" + +from __future__ import annotations + +from dataclasses import dataclass +import json +from pathlib import Path +from typing import Any, Iterable + +from topt_runtime.runfiles import RunfileResolutionError, RunfilesResolver + +from .enrichment import ContextPlan, ContextRecord +from .json_utils import strict_json_loads + + +class ResourceError(ValueError): + """Runtime-selected resources are incomplete or inconsistent.""" + + +@dataclass(frozen=True) +class ResourceInputs: + """Runfile locations plus optional execution-time target context choices.""" + + context_override: Path | None = None + context_manifest_paths: tuple[str, ...] = () + telemetry_facts_manifest_paths: tuple[str, ...] = () + schema_paths: tuple[str, ...] = () + runtime_context_entries: tuple[str, ...] = () + runtime_selection: bool = False + workspace: Path | None = None + invocation_cwd: Path | None = None + + +@dataclass(frozen=True) +class LoadedResources: + """Validated read-only resources safe to share with every worker.""" + + context_plan: ContextPlan + primary_context: dict[str, Any] | None + primary_context_path: Path | None + telemetry_facts_paths: tuple[Path, ...] + schema: dict[str, Any] | None + warning_codes: tuple[str, ...] = () + + @property + def context_workspace(self) -> str: + if self.primary_context is None: + return "" + value = self.primary_context.get("ci.workspace_path") + return value if isinstance(value, str) else "" + + +@dataclass(frozen=True) +class _RuntimeContext: + """One validated context and its matching telemetry facts.""" + + repo_key: str + context_path: Path + context: dict[str, Any] + telemetry_path: Path + + +def load_resources( + resolver: RunfilesResolver, + inputs: ResourceInputs, +) -> LoadedResources: + """Resolve contexts, telemetry facts, and schema once with warning fallback.""" + warnings: list[str] = [] + override_path = _existing_file(inputs.context_override) + override_values = _load_json_object(override_path) if override_path else None + if inputs.context_override is not None and override_values is None: + warnings.append("context_override_invalid") + + runtime_contexts = _load_runtime_contexts( + inputs.runtime_context_entries, + workspace=inputs.workspace, + invocation_cwd=inputs.invocation_cwd, + ) + if inputs.runtime_selection and not runtime_contexts: + raise ResourceError( + "runtime selection requires at least one --context-entry argument" + ) + + context_records: list[ContextRecord] = [] + primary_context_path: Path | None = None + runtime_override_enabled = override_values is not None + if override_values is not None: + context_records.append( + ContextRecord.create("__runtime_override__", override_values) + ) + primary_context_path = override_path + elif runtime_contexts: + for runtime_context in runtime_contexts: + context_records.append( + ContextRecord.create( + runtime_context.repo_key, + runtime_context.context, + ) + ) + if primary_context_path is None: + primary_context_path = runtime_context.context_path + else: + context_manifest = _resolve_optional(resolver, inputs.context_manifest_paths) + if inputs.context_manifest_paths and context_manifest is None: + warnings.append("context_manifest_unresolved") + if context_manifest is not None: + seen_repo_keys: set[str] = set() + for repo_key, short_path, artifact_path in _context_manifest_entries( + context_manifest, + warnings, + ): + normalized_repo_key = repo_key.rsplit("+", 1)[-1] + if normalized_repo_key in seen_repo_keys: + warnings.append("context_manifest_duplicate_repo") + continue + context_path = _resolve_optional( + resolver, + (artifact_path, short_path), + ) + context_values = _load_json_object(context_path) if context_path else None + if context_values is None: + warnings.append("context_entry_invalid") + continue + seen_repo_keys.add(normalized_repo_key) + context_records.append( + ContextRecord.create(normalized_repo_key, context_values) + ) + if primary_context_path is None: + primary_context_path = context_path + + primary_context_record = context_records[0] if context_records else None + primary_context = ( + dict(primary_context_record.values) if primary_context_record else None + ) + context_plan = ContextPlan( + primary=primary_context_record, + by_repo=tuple(context_records), + override=runtime_override_enabled, + require_repo_match=inputs.runtime_selection, + ) + + telemetry_facts_paths: list[Path] = [] + if runtime_contexts: + telemetry_facts_paths.extend(item.telemetry_path for item in runtime_contexts) + else: + telemetry_manifest = _resolve_optional( + resolver, + inputs.telemetry_facts_manifest_paths, + ) + if inputs.telemetry_facts_manifest_paths and telemetry_manifest is None: + warnings.append("telemetry_facts_manifest_unresolved") + if telemetry_manifest is not None: + for short_path, artifact_path in _two_column_manifest_entries( + telemetry_manifest, + warnings, + "telemetry_facts_manifest_invalid", + ): + facts_path = _resolve_optional(resolver, (artifact_path, short_path)) + if facts_path is None: + warnings.append("telemetry_facts_entry_unresolved") + continue + telemetry_facts_paths.append(facts_path) + if runtime_override_enabled and primary_context_path is not None: + sibling = primary_context_path.parent / "telemetry_facts.json" + if sibling.is_file(): + telemetry_facts_paths.append(sibling.resolve()) + telemetry_facts_paths = list(dict.fromkeys(sorted(telemetry_facts_paths))) + + schema_path = _resolve_optional(resolver, inputs.schema_paths) + schema = _load_json_object(schema_path) if schema_path else None + if inputs.schema_paths and schema is None: + warnings.append("schema_invalid_or_unresolved") + return LoadedResources( + context_plan=context_plan, + primary_context=primary_context, + primary_context_path=primary_context_path, + telemetry_facts_paths=tuple(telemetry_facts_paths), + schema=schema, + warning_codes=tuple(dict.fromkeys(warnings)), + ) + + +def _context_manifest_entries( + path: Path, + warnings: list[str], +) -> tuple[tuple[str, str, str], ...]: + entries: list[tuple[str, str, str]] = [] + try: + lines = path.read_text(encoding="utf-8-sig").splitlines() + except (OSError, UnicodeError): + warnings.append("context_manifest_invalid") + return () + for line in lines: + if not line: + continue + fields = line.split("\t") + if len(fields) != 3 or not fields[0] or not (fields[1] or fields[2]): + warnings.append("context_manifest_invalid") + continue + entries.append((fields[0], fields[1], fields[2])) + return tuple(entries) + + +def _two_column_manifest_entries( + path: Path, + warnings: list[str], + warning_code: str, +) -> tuple[tuple[str, str], ...]: + entries: list[tuple[str, str]] = [] + try: + lines = path.read_text(encoding="utf-8-sig").splitlines() + except (OSError, UnicodeError): + warnings.append(warning_code) + return () + for line in lines: + if not line: + continue + fields = line.split("\t") + if len(fields) != 2 or not (fields[0] or fields[1]): + warnings.append(warning_code) + continue + entries.append((fields[0], fields[1])) + return tuple(entries) + + +def _resolve_optional( + resolver: RunfilesResolver, + candidates: Iterable[str], +) -> Path | None: + values = tuple(candidate for candidate in candidates if candidate) + if not values: + return None + try: + return resolver.resolve_file(values) + except RunfileResolutionError: + return None + + +def _existing_file(path: Path | None) -> Path | None: + return path.resolve() if path is not None and path.is_file() else None + + +def _load_json_object(path: Path | None) -> dict[str, Any] | None: + if path is None: + return None + try: + value = strict_json_loads(path.read_bytes().decode("utf-8-sig")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError): + return None + return value if isinstance(value, dict) else None + + +_APPARENT_REPO_NAME_CHARS = frozenset( + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._+-" +) + + +def _load_runtime_contexts( + entries: Iterable[str], + *, + workspace: Path | None, + invocation_cwd: Path | None, +) -> tuple[_RuntimeContext, ...]: + """Validate keyed contexts before discovery or worker startup.""" + contexts: list[_RuntimeContext] = [] + seen_repos: set[str] = set() + seen_paths: set[Path] = set() + for entry in entries: + repo_key, separator, raw_path = entry.partition("=") + if ( + not separator + or not raw_path + or not repo_key + or any(character not in _APPARENT_REPO_NAME_CHARS for character in repo_key) + ): + raise ResourceError( + "--context-entry must use =" + ) + if repo_key in seen_repos: + raise ResourceError( + f"duplicate --context-entry repository name: {repo_key!r}" + ) + context_path = _resolve_runtime_path( + raw_path, + workspace=workspace, + invocation_cwd=invocation_cwd, + ) + if context_path is None or context_path.name != "context.json": + raise ResourceError( + f"context entry for {repo_key!r} must reference an existing " + "regular context.json file" + ) + if context_path in seen_paths: + raise ResourceError( + f"duplicate --context-entry path for repository {repo_key!r}" + ) + telemetry_path = context_path.with_name("telemetry_facts.json") + if not telemetry_path.is_file(): + raise ResourceError( + f"context entry for {repo_key!r} is missing sibling telemetry_facts.json" + ) + telemetry_path = telemetry_path.resolve() + + context = _load_json_object(context_path) + telemetry = _load_json_object(telemetry_path) + service_name = context.get("service.name") if context is not None else None + runtime_name = context.get("runtime.name") if context is not None else None + valid = ( + context is not None + and context.get("topt.sync.repository_name") == repo_key + and isinstance(service_name, str) + and bool(service_name) + and isinstance(runtime_name, str) + and bool(runtime_name) + and telemetry is not None + and telemetry.get("schema_version") == 1 + and telemetry.get("service_name") == service_name + and telemetry.get("runtime_name") == runtime_name + and isinstance(telemetry.get("counts"), list) + and isinstance(telemetry.get("distributions"), list) + ) + if not valid: + raise ResourceError( + "runtime context/telemetry identity or schema mismatch for " + f"repository {repo_key!r}" + ) + seen_repos.add(repo_key) + seen_paths.add(context_path) + contexts.append( + _RuntimeContext(repo_key, context_path, context, telemetry_path) + ) + return tuple(sorted(contexts, key=lambda item: item.repo_key)) + + +def _resolve_runtime_path( + raw_path: str, + *, + workspace: Path | None, + invocation_cwd: Path | None, +) -> Path | None: + path = Path(raw_path).expanduser() + candidates = (path,) if path.is_absolute() else tuple( + root / path + for root in (invocation_cwd, workspace) + if root is not None + ) + for candidate in candidates: + if candidate.is_file(): + return candidate.resolve() + return None diff --git a/tools/core/uploader_py/splitting.py b/tools/core/uploader_py/splitting.py new file mode 100644 index 00000000..f9c1b03a --- /dev/null +++ b/tools/core/uploader_py/splitting.py @@ -0,0 +1,199 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Split enriched test JSON deterministically before any HTTP request. + +Preventive size enforcement avoids a known 413 while preserving event order. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Mapping + +from .json_utils import strict_json_dumps +from .models import MAX_TEST_PAYLOAD_BYTES + + +class TestPayloadSplitError(ValueError): + """A test payload cannot be safely represented as bounded chunks.""" + + def __init__(self, code: str, message: str) -> None: + super().__init__(message) + self.code = code + + +@dataclass(frozen=True) +class PreparedTestChunk: + """One exact compact JSON request body stored in task-local space.""" + + index: int + path: Path + size_bytes: int + event_start: int + event_end: int + + @property + def event_count(self) -> int: + return self.event_end - self.event_start + + +def compact_json_bytes(value: Any) -> bytes: + """Serialize using the one canonical uploader JSON representation.""" + try: + return strict_json_dumps( + value, + ensure_ascii=False, + separators=(",", ":"), + ).encode("utf-8") + except (TypeError, ValueError) as exc: + raise TestPayloadSplitError( + "test_payload_not_json_serializable", + f"test payload is not JSON serializable: {exc}", + ) from exc + + +def prepare_test_chunks( + payload: Mapping[str, Any], + task_directory: Path, + *, + limit_bytes: int = MAX_TEST_PAYLOAD_BYTES, +) -> tuple[PreparedTestChunk, ...]: + """Write exact bounded bodies before any caller may perform HTTP.""" + if isinstance(payload, dict): + payload_object = payload + else: + raise TestPayloadSplitError( + "invalid_test_payload", + "test payload must be a JSON object", + ) + if any(not isinstance(key, str) for key in payload_object): + raise TestPayloadSplitError( + "invalid_test_payload", + "test payload object keys must be strings", + ) + if isinstance(limit_bytes, bool) or not isinstance(limit_bytes, int) or limit_bytes <= 0: + raise ValueError("limit_bytes must be a positive integer") + events = payload_object.get("events") + if not isinstance(events, list) or not events: + raise TestPayloadSplitError( + "test_payload_without_events", + "test payload must contain a non-empty events array", + ) + if not task_directory.is_dir(): + raise TestPayloadSplitError( + "test_chunk_directory_unavailable", + f"test chunk directory does not exist: {task_directory}", + ) + + full_body = compact_json_bytes(payload_object) + if len(full_body) <= limit_bytes: + return (_write_chunk(task_directory, 1, full_body, 0, len(events)),) + + prefix, suffix = _event_array_frame(payload_object) + envelope_size = len(prefix) + len(suffix) + if envelope_size > limit_bytes: + raise TestPayloadSplitError( + "test_payload_envelope_exceeds_payload_limit", + f"test payload envelope is {envelope_size} bytes; limit is {limit_bytes}", + ) + + serialized_events = tuple(compact_json_bytes(event) for event in events) + for event_index, event_body in enumerate(serialized_events): + event_size = envelope_size + len(event_body) + if event_size > limit_bytes: + raise TestPayloadSplitError( + "single_event_exceeds_payload_limit", + ( + f"event index {event_index} requires {event_size} bytes with the " + f"payload envelope; limit is {limit_bytes}" + ), + ) + + event_ranges: list[tuple[int, int]] = [] + chunk_start = 0 + chunk_size = envelope_size + for event_index, event_body in enumerate(serialized_events): + separator_size = 0 if event_index == chunk_start else 1 + candidate_size = chunk_size + separator_size + len(event_body) + if candidate_size <= limit_bytes: + chunk_size = candidate_size + continue + event_ranges.append((chunk_start, event_index)) + chunk_start = event_index + chunk_size = envelope_size + len(event_body) + event_ranges.append((chunk_start, len(serialized_events))) + + chunks: list[PreparedTestChunk] = [] + for chunk_index, (event_start, event_end) in enumerate(event_ranges, start=1): + body = prefix + b",".join(serialized_events[event_start:event_end]) + suffix + if len(body) > limit_bytes: + raise AssertionError("internal split invariant violated: chunk exceeds byte limit") + chunks.append( + _write_chunk( + task_directory, + chunk_index, + body, + event_start, + event_end, + ) + ) + return tuple(chunks) + + +def _event_array_frame(payload: Mapping[str, Any]) -> tuple[bytes, bytes]: + fields_before_events: list[bytes] = [] + fields_after_events: list[bytes] = [] + destination_fields = fields_before_events + found_events = False + for key, value in payload.items(): + encoded_key = compact_json_bytes(key) + if key == "events": + found_events = True + destination_fields = fields_after_events + continue + encoded_entry = encoded_key + b":" + compact_json_bytes(value) + destination_fields.append(encoded_entry) + if not found_events: + raise TestPayloadSplitError( + "test_payload_without_events", + "test payload must contain a non-empty events array", + ) + + prefix = b"{" + if fields_before_events: + prefix += b",".join(fields_before_events) + b"," + prefix += compact_json_bytes("events") + b":[" + suffix = b"]" + if fields_after_events: + suffix += b"," + b",".join(fields_after_events) + suffix += b"}" + return prefix, suffix + + +def _write_chunk( + task_directory: Path, + index: int, + body: bytes, + event_start: int, + event_end: int, +) -> PreparedTestChunk: + path = task_directory / f"test_chunk_{index:04d}.json" + try: + path.write_bytes(body) + except OSError as exc: + raise TestPayloadSplitError( + "test_chunk_write_failed", + f"failed to write test chunk {index}: {exc}", + ) from exc + return PreparedTestChunk( + index=index, + path=path, + size_bytes=len(body), + event_start=event_start, + event_end=event_end, + ) diff --git a/tools/core/uploader_py/telemetry.py b/tools/core/uploader_py/telemetry.py new file mode 100644 index 00000000..b844808d --- /dev/null +++ b/tools/core/uploader_py/telemetry.py @@ -0,0 +1,374 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Build immutable telemetry directives for every discovered source. + +Planning once preserves cross-file correlation without synchronizing workers. +""" + +from __future__ import annotations + +from dataclasses import dataclass, replace +import json +from pathlib import Path +import time +from typing import Any, Callable, Iterable, Mapping + +from .json_utils import strict_json_dumps, strict_json_loads +from .models import FileTask, PayloadType + + +@dataclass(frozen=True) +class TelemetryDirective: + """Instructions that one telemetry file worker can execute independently.""" + + env_override: str = "" + messages_json: bytes = b"" + append_messages: bool = False + create_synthetic: bool = False + synthetic_seq_id: int = 0 + synthetic_timestamp: int = 0 + + +@dataclass(frozen=True) +class TelemetryPlan: + """Immutable lookup prepared once before worker threads are started.""" + + entries: tuple[tuple[str, TelemetryDirective], ...] = () + provider_suffix: str = "" + warning_codes: tuple[str, ...] = () + + def directive_for(self, source_path: Path) -> TelemetryDirective: + key = _path_key(source_path) + for planned_path, directive in self.entries: + if planned_path == key: + return directive + return TelemetryDirective() + + +@dataclass(frozen=True) +class _TelemetrySource: + """Identity needed to choose which tracer stream receives rule facts.""" + + path_key: str + service_name: str + language_name: str + runtime_id: str + seq_id: int | None + request_type: str + + +@dataclass(frozen=True) +class _TelemetryFacts: + """Rule-generated metrics waiting to be attached to a tracer stream.""" + + path_key: str + service_name: str + runtime_name: str + environment: str + counts: tuple[Any, ...] + distributions: tuple[Any, ...] + + +def build_telemetry_plan( + tasks: Iterable[FileTask], + facts_paths: Iterable[Path], + *, + primary_context: Mapping[str, Any] | None = None, + clock: Callable[[], float] = time.time, +) -> TelemetryPlan: + """Match rule facts to tracer streams without materializing outbound files.""" + warnings: list[str] = [] + sources = _load_sources(tasks, warnings) + provider_suffix = _provider_suffix(primary_context) + if not sources: + return TelemetryPlan( + provider_suffix=provider_suffix, + warning_codes=_unique_codes(warnings), + ) + + grouped_sources: dict[tuple[str, str], list[_TelemetrySource]] = {} + for source in sources: + grouped_sources.setdefault( + (source.service_name, source.language_name), [] + ).append(source) + + grouped_facts: dict[tuple[str, str], list[_TelemetryFacts]] = {} + for facts_path in sorted(facts_paths, key=lambda item: str(item)): + facts = _load_facts(facts_path, warnings) + if facts is None: + continue + matching_sources = [ + source for source in sources if source.service_name == facts.service_name + ] + if not matching_sources: + warnings.append("telemetry_facts_anchor_missing") + continue + languages = sorted({source.language_name for source in matching_sources}) + # Facts only identify a runtime. Use it as a language discriminator + # when one service emitted telemetry from multiple languages. + if len(languages) == 1: + language_name = languages[0] + else: + selected_languages = sorted( + { + source.language_name + for source in matching_sources + if source.language_name == facts.runtime_name + } + ) + if len(selected_languages) != 1: + warnings.append("telemetry_facts_language_ambiguous") + continue + language_name = selected_languages[0] + grouped_facts.setdefault((facts.service_name, language_name), []).append(facts) + + directives: dict[str, TelemetryDirective] = {} + for stream_key in sorted(grouped_facts): + group_sources = sorted( + grouped_sources.get(stream_key, ()), + key=lambda item: item.path_key, + ) + if not group_sources: + continue + facts_entries = sorted( + grouped_facts[stream_key], + key=lambda item: item.path_key, + ) + environments = sorted( + {facts.environment for facts in facts_entries if facts.environment} + ) + if len(environments) > 1: + warnings.append("telemetry_facts_env_conflict") + continue + env_override = environments[0] if environments else "" + if env_override: + for source in group_sources: + directives[source.path_key] = TelemetryDirective( + env_override=env_override + ) + + counts: list[Any] = [] + distributions: list[Any] = [] + for facts in facts_entries: + counts.extend(facts.counts) + distributions.extend(facts.distributions) + timestamp = int(clock()) + messages = _build_inner_messages(counts, distributions, timestamp) + if not messages: + continue + + streams: dict[str, list[_TelemetrySource]] = {} + for source in group_sources: + streams.setdefault(source.runtime_id, []).append(source) + # Prefer a stream that can accept the facts in an existing batch; + # otherwise synthesize one request beside the best available stream. + chosen_stream = max( + streams.values(), + key=lambda items: ( + any(item.request_type == "message-batch" for item in items), + _stream_best_path(items), + ), + ) + batch_sources = [ + source + for source in chosen_stream + if source.request_type == "message-batch" + ] + anchor_source = max( + batch_sources or chosen_stream, key=lambda item: item.path_key + ) + current_directive = directives.get(anchor_source.path_key, TelemetryDirective()) + encoded_messages = strict_json_dumps( + messages, + ensure_ascii=False, + separators=(",", ":"), + ).encode("utf-8") + if anchor_source.request_type == "message-batch": + directives[anchor_source.path_key] = replace( + current_directive, + messages_json=encoded_messages, + append_messages=True, + ) + else: + max_seq_id = max( + ( + source.seq_id + for source in chosen_stream + if source.seq_id is not None + ), + default=0, + ) + directives[anchor_source.path_key] = replace( + current_directive, + messages_json=encoded_messages, + create_synthetic=True, + synthetic_seq_id=max_seq_id + 1, + synthetic_timestamp=timestamp, + ) + + return TelemetryPlan( + entries=tuple(sorted(directives.items())), + provider_suffix=provider_suffix, + warning_codes=_unique_codes(warnings), + ) + + +def _load_sources( + tasks: Iterable[FileTask], warnings: list[str] +) -> tuple[_TelemetrySource, ...]: + sources: list[_TelemetrySource] = [] + telemetry_tasks = sorted( + (task for task in tasks if task.payload_type is PayloadType.TELEMETRY), + key=lambda task: str(task.source_path), + ) + for task in telemetry_tasks: + payload = _read_json_object(task.source_path) + if payload is None: + warnings.append("telemetry_plan_input_invalid") + continue + application = payload.get("application") + if not isinstance(application, dict): + continue + service_name = _string(application.get("service_name")) + language_name = _string(application.get("language_name")) + api_version = _string(payload.get("api_version")) + request_type = _string(payload.get("request_type")) + if not service_name or not language_name or not api_version or not request_type: + continue + seq_id = payload.get("seq_id") + sources.append( + _TelemetrySource( + path_key=_path_key(task.source_path), + service_name=service_name, + language_name=language_name, + runtime_id=_string(payload.get("runtime_id")), + seq_id=( + seq_id + if isinstance(seq_id, int) and not isinstance(seq_id, bool) + else None + ), + request_type=request_type, + ) + ) + return tuple(sources) + + +def _load_facts(path: Path, warnings: list[str]) -> _TelemetryFacts | None: + payload = _read_json_object(path) + if payload is None: + warnings.append("telemetry_facts_invalid") + return None + if not _string(payload.get("service_name")): + warnings.append("telemetry_facts_service_missing") + return None + counts = payload.get("counts") + distributions = payload.get("distributions") + return _TelemetryFacts( + path_key=_path_key(path), + service_name=_string(payload.get("service_name")), + runtime_name=_string(payload.get("runtime_name")), + environment=_string(payload.get("env")), + counts=tuple(counts) if isinstance(counts, list) else (), + distributions=( + tuple(distributions) if isinstance(distributions, list) else () + ), + ) + + +def _read_json_object(path: Path) -> dict[str, Any] | None: + try: + value = strict_json_loads(path.read_bytes().decode("utf-8-sig")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError): + return None + return value if isinstance(value, dict) else None + + +def _build_inner_messages( + counts: Iterable[Any], distributions: Iterable[Any], timestamp: int +) -> list[dict[str, Any]]: + count_series = [] + for fact in counts: + if not isinstance(fact, dict): + continue + name = _string(fact.get("name")) + if not name: + continue + tags = fact.get("tags") + count_series.append( + { + "metric": name, + "points": [[timestamp, fact.get("value")]], + "type": "count", + "tags": tags if isinstance(tags, list) else [], + "common": True, + "namespace": "civisibility", + } + ) + + distribution_series = [] + for fact in distributions: + if not isinstance(fact, dict): + continue + name = _string(fact.get("name")) + if not name: + continue + tags = fact.get("tags") + distribution_series.append( + { + "metric": name, + "points": [fact.get("value")], + "tags": tags if isinstance(tags, list) else [], + "common": True, + "namespace": "civisibility", + } + ) + + messages: list[dict[str, Any]] = [] + if count_series: + messages.append( + { + "request_type": "generate-metrics", + "payload": {"namespace": "civisibility", "series": count_series}, + } + ) + if distribution_series: + messages.append( + { + "request_type": "distributions", + "payload": {"namespace": "", "series": distribution_series}, + } + ) + return messages + + +def _stream_best_path(items: Iterable[_TelemetrySource]) -> str: + sources = tuple(items) + batches = sorted( + item.path_key for item in sources if item.request_type == "message-batch" + ) + return batches[-1] if batches else max(item.path_key for item in sources) + + +def _provider_suffix(context: Mapping[str, Any] | None) -> str: + if context is None: + return "" + return ( + _string(context.get("ci.provider.name")) + or _string(context.get("ci_provider_name")) + ).strip() + + +def _path_key(path: Path) -> str: + return str(path.resolve(strict=False)) + + +def _string(value: Any) -> str: + return value if isinstance(value, str) else "" + + +def _unique_codes(codes: Iterable[str]) -> tuple[str, ...]: + return tuple(dict.fromkeys(code for code in codes if code)) diff --git a/tools/core/uploader_py/temporary.py b/tools/core/uploader_py/temporary.py new file mode 100644 index 00000000..3cea30a8 --- /dev/null +++ b/tools/core/uploader_py/temporary.py @@ -0,0 +1,87 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Create owned temporary directories for an invocation and its tasks. + +Explicit ownership makes retry bodies isolated and cleanup failures reportable. +""" + +from __future__ import annotations + +from contextlib import contextmanager +from pathlib import Path +import tempfile +from typing import Callable, Iterator + + +class TemporaryDirectoryError(RuntimeError): + """The uploader could not create its owned temporary directory.""" + + +@contextmanager +def invocation_temporary_directory( + *, + temp_root: Path | None = None, + on_cleanup_error: Callable[[str], None] | None = None, +) -> Iterator[Path]: + """Create and best-effort clean one native invocation temporary root.""" + try: + owned_directory = tempfile.TemporaryDirectory( + prefix="dd_topt_payloads.", + dir=str(temp_root) if temp_root is not None else None, + ) + except OSError as exc: + location = ( + str(temp_root) + if temp_root is not None + else "the platform temporary directory" + ) + raise TemporaryDirectoryError( + f"failed to create uploader temporary directory under {location}: {exc}" + ) from exc + try: + yield Path(owned_directory.name) + finally: + _cleanup_owned_directory(owned_directory, on_cleanup_error) + + +@contextmanager +def task_temporary_directory( + invocation_root: Path, + task_id: str, + *, + on_cleanup_error: Callable[[str], None] | None = None, +) -> Iterator[Path]: + """Create and clean a task-local child without exposing source filenames.""" + safe_task_id = "".join( + ch if ch.isalnum() or ch in "-_" else "_" for ch in task_id + ) + prefix = f"task_{safe_task_id or 'unknown'}." + try: + owned_directory = tempfile.TemporaryDirectory( + prefix=prefix, + dir=str(invocation_root), + ) + except OSError as exc: + raise TemporaryDirectoryError( + f"failed to create task temporary directory under {invocation_root}: {exc}" + ) from exc + try: + yield Path(owned_directory.name) + finally: + _cleanup_owned_directory(owned_directory, on_cleanup_error) + + +def _cleanup_owned_directory( + owned_directory: tempfile.TemporaryDirectory, + on_cleanup_error: Callable[[str], None] | None, +) -> None: + """Keep cleanup failure from changing an already-known upload outcome.""" + try: + owned_directory.cleanup() + except OSError as exc: + if on_cleanup_error is not None: + on_cleanup_error(type(exc).__name__) diff --git a/tools/core/uploader_py/transport.py b/tools/core/uploader_py/transport.py new file mode 100644 index 00000000..fb9342e4 --- /dev/null +++ b/tools/core/uploader_py/transport.py @@ -0,0 +1,695 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Send exact reusable request bodies through one worker-local HTTP client. + +Per-worker transports provide concurrency without shared connection or retry state. +""" + +from __future__ import annotations + +import base64 +from contextlib import closing +from dataclasses import dataclass +from email.utils import parsedate_to_datetime +from functools import partial +import gzip +import http.client +import io +import logging +import math +from pathlib import Path +import secrets +import shutil +import ssl +import time +from typing import BinaryIO, Callable, Iterable, Mapping +from urllib.error import HTTPError, URLError +from urllib.parse import SplitResult, unquote +from urllib.request import ( + HTTPHandler, + HTTPRedirectHandler, + HTTPSHandler, + ProxyHandler, + Request, + build_opener, + proxy_bypass_environment, +) + +from .endpoints import parse_http_url +from .logging_utils import redact_url + + +DEFAULT_CONNECT_TIMEOUT_SECONDS = 10.0 +DEFAULT_REQUEST_TIMEOUT_SECONDS = 60.0 +DEFAULT_MAX_ATTEMPTS = 4 +DEFAULT_RETRY_DELAY_SECONDS = 2.0 +DEFAULT_MAX_RETRY_DELAY_SECONDS = 60.0 +DEFAULT_RESPONSE_LIMIT_BYTES = 2_000 +RETRYABLE_HTTP_STATUSES = frozenset({408, 429}) + + +class HttpTransportError(ValueError): + """The caller supplied an invalid URL, header, or request body.""" + + +@dataclass(frozen=True) +class HttpResult: + """Terminal outcome of one logical request, including all retry attempts.""" + + status_code: int | None + attempts: int + body_excerpt: bytes = b"" + body_truncated: bool = False + transport_error: str | None = None + retry_delays: tuple[float, ...] = () + + @property + def succeeded(self) -> bool: + return ( + self.transport_error is None + and self.status_code is not None + and 200 <= self.status_code < 300 + ) + + @property + def retries(self) -> int: + return max(0, self.attempts - 1) + + +@dataclass(frozen=True) +class PreparedMultipartBody: + """Exact task-local multipart body reusable across request attempts.""" + + path: Path + content_type: str + content_length: int + + +@dataclass(frozen=True) +class PreparedHttpRequest: + """A fully validated request that has not opened a network connection.""" + + url: str + headers: Mapping[str, str] + body_factory: Callable[[], BinaryIO] + + +class _RequestTimeoutHTTPConnection(http.client.HTTPConnection): + """Use a short connect timeout, then a separate socket I/O timeout.""" + + def __init__( + self, + host: str, + *, + timeout: float, + connect_timeout: float, + **kwargs: object, + ) -> None: + self._request_timeout = timeout + super().__init__(host, timeout=connect_timeout, **kwargs) + + def connect(self) -> None: + super().connect() + if self.sock is not None: + self.sock.settimeout(self._request_timeout) + + +class _RequestTimeoutHTTPSConnection(http.client.HTTPSConnection): + """HTTPS equivalent of :class:`_RequestTimeoutHTTPConnection`.""" + + def __init__( + self, + host: str, + *, + timeout: float, + connect_timeout: float, + **kwargs: object, + ) -> None: + self._request_timeout = timeout + super().__init__(host, timeout=connect_timeout, **kwargs) + + def connect(self) -> None: + super().connect() + if self.sock is not None: + self.sock.settimeout(self._request_timeout) + + +class _RequestTimeoutHTTPHandler(HTTPHandler): + def __init__(self, connect_timeout: float) -> None: + super().__init__() + self._connection = partial( + _RequestTimeoutHTTPConnection, + connect_timeout=connect_timeout, + ) + + def http_open(self, request: Request): + return self.do_open(self._connection, request) + + +class _RequestTimeoutHTTPSHandler(HTTPSHandler): + def __init__(self, connect_timeout: float, context: ssl.SSLContext) -> None: + super().__init__(context=context) + self._connection = partial( + _RequestTimeoutHTTPSConnection, + connect_timeout=connect_timeout, + ) + + def https_open(self, request: Request): + return self.do_open(self._connection, request, context=self._context) + + +class _NoRedirectHandler(HTTPRedirectHandler): + """Keep POST semantics and credentials on the configured intake host.""" + + def redirect_request(self, req, fp, code, msg, headers, newurl): + return None + + +class _SnapshotProxyHandler(ProxyHandler): + """Proxy handler whose bypass rules do not reread process environment.""" + + def __init__(self, proxies: Mapping[str, str], no_proxy: str) -> None: + self._no_proxy = no_proxy + super().__init__(dict(proxies)) + + def proxy_open(self, request: Request, proxy: str, proxy_type: str): + if request.host and self._bypass(request.host): + return None + + original_type = request.type + raw_proxy = proxy if "://" in proxy else f"{original_type}://{proxy}" + try: + parsed = _parsed_proxy_url(raw_proxy, original_type) + proxy_port = parsed.port + except (TypeError, ValueError): + raise URLError("invalid proxy configuration") + resolved_type = parsed.scheme or original_type + + host_port = parsed.hostname + if ":" in host_port and not host_port.startswith("["): + host_port = f"[{host_port}]" + if proxy_port is not None: + host_port = f"{host_port}:{proxy_port}" + if parsed.username is not None: + credentials = f"{unquote(parsed.username)}:{unquote(parsed.password or '')}" + encoded = base64.b64encode(credentials.encode("utf-8")).decode("ascii") + request.add_header("Proxy-authorization", f"Basic {encoded}") + + request.set_proxy(host_port, resolved_type) + if original_type == resolved_type or original_type == "https": + return None + return self.parent.open(request, timeout=request.timeout) + + def _bypass(self, host: str) -> bool: + if not self._no_proxy: + return False + return proxy_bypass_environment(host, {"no": self._no_proxy}) + + +def _proxy_configuration( + environment: Iterable[tuple[str, str]], +) -> tuple[dict[str, str], str]: + effective_values: dict[str, str] = {} + # Uppercase variants are supported, while lowercase variants take + # precedence when both are explicitly present. + items = tuple(environment) + for name, value in items: + if name in {"HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY"} and value: + effective_values[name.lower()] = value + for name, value in items: + if name in {"http_proxy", "https_proxy", "no_proxy"} and value: + effective_values[name] = value + proxies = { + scheme: effective_values[f"{scheme}_proxy"] + for scheme in ("http", "https") + if effective_values.get(f"{scheme}_proxy") + } + for scheme, proxy in proxies.items(): + try: + _parsed_proxy_url(proxy, scheme) + except (TypeError, ValueError) as exc: + raise HttpTransportError( + f"invalid {scheme.upper()} proxy configuration" + ) from exc + return proxies, effective_values.get("no_proxy", "") + + +def _parsed_proxy_url(raw_proxy: str, default_scheme: str) -> SplitResult: + """Parse the proxy forms accepted by standard proxy environment variables.""" + candidate = raw_proxy if "://" in raw_proxy else f"{default_scheme}://{raw_proxy}" + parsed = parse_http_url(candidate) + if parsed.scheme.lower() not in {"http", "https"}: + raise ValueError("proxy scheme must be HTTP(S)") + if parsed.path not in {"", "/"} or parsed.query or parsed.fragment: + raise ValueError("proxy URL must not contain a request path") + for credential in (parsed.username, parsed.password): + if credential is not None: + unquote(credential, encoding="utf-8", errors="strict") + return parsed + + +def validate_proxy_environment(environment: Iterable[tuple[str, str]]) -> None: + """Reject a known-invalid effective proxy before workers start.""" + _proxy_configuration(environment) + + +def _retryable_status(status_code: int) -> bool: + return status_code in RETRYABLE_HTTP_STATUSES or 500 <= status_code <= 599 + + +def _retry_after_seconds(value: str | None, now: float) -> float | None: + if not value: + return None + normalized = value.strip() + if normalized.isdecimal(): + try: + delay = float(normalized) + except (OverflowError, ValueError): + return None + return _bounded_retry_after_seconds(delay) + try: + parsed = parsedate_to_datetime(normalized) + except (TypeError, ValueError, OverflowError): + return None + if parsed.tzinfo is None: + return None + try: + delay = parsed.timestamp() - now + except (OSError, OverflowError, ValueError): + return None + return _bounded_retry_after_seconds(delay) + + +def _bounded_retry_after_seconds(delay: float) -> float | None: + """Keep server-controlled retry waits finite and operationally bounded.""" + if math.isnan(delay): + return None + if not math.isfinite(delay): + return DEFAULT_MAX_RETRY_DELAY_SECONDS if delay > 0 else 0.0 + return min(max(0.0, delay), DEFAULT_MAX_RETRY_DELAY_SECONDS) + + +def _bounded_response(stream, limit: int) -> tuple[bytes, bool]: + body = stream.read(limit + 1) + return body[:limit], len(body) > limit + + +def _validate_url(url: str) -> None: + try: + parsed = parse_http_url(url) + except (TypeError, ValueError) as exc: + raise HttpTransportError( + "upload URL must be an absolute HTTP(S) URL" + ) from exc + if parsed.username is not None or parsed.password is not None: + raise HttpTransportError("upload URL must not contain credentials/userinfo") + + +def _validated_headers(headers: Mapping[str, str]) -> dict[str, str]: + validated: dict[str, str] = {} + for name, value in headers.items(): + if not name or any(character in name for character in "\r\n:"): + raise HttpTransportError(f"invalid HTTP header name: {name!r}") + text_value = str(value) + if "\r" in text_value or "\n" in text_value: + raise HttpTransportError(f"invalid HTTP header value for {name!r}") + validated[name] = text_value + return validated + + +def _controlled_headers( + headers: Mapping[str, str], + *, + content_type: str, + content_length: int, + content_encoding: str | None = None, +) -> dict[str, str]: + controlled = _validated_headers(headers) + for name in tuple(controlled): + if name.lower() in {"content-type", "content-length", "content-encoding"}: + del controlled[name] + controlled["Content-Type"] = content_type + controlled["Content-Length"] = str(content_length) + if content_encoding: + controlled["Content-Encoding"] = content_encoding + return controlled + + +def prepare_json_request( + url: str, + headers: Mapping[str, str], + body: bytes | Path, + *, + gzip_body: bool = False, + content_encoding: str | None = None, +) -> PreparedHttpRequest: + """Validate and materialize everything needed for one JSON request.""" + _validate_url(url) + if gzip_body and content_encoding: + raise HttpTransportError( + "gzip_body and an existing content encoding cannot be combined" + ) + if isinstance(body, Path): + try: + if gzip_body: + compressed = gzip.compress(body.read_bytes(), mtime=0) + body_factory: Callable[[], BinaryIO] = ( + lambda: io.BytesIO(compressed) + ) + body_length = len(compressed) + else: + body_length = body.stat().st_size + with body.open("rb"): + pass + body_factory = lambda: body.open("rb") + except OSError as exc: + raise HttpTransportError( + f"failed to prepare JSON request body: {type(exc).__name__}" + ) from exc + else: + body_bytes = gzip.compress(body, mtime=0) if gzip_body else body + body_length = len(body_bytes) + body_factory = lambda: io.BytesIO(body_bytes) + + request_headers = _controlled_headers( + headers, + content_type="application/json", + content_length=body_length, + content_encoding="gzip" if gzip_body else content_encoding, + ) + return PreparedHttpRequest(url, request_headers, body_factory) + + +def prepare_spooled_multipart_request( + url: str, + headers: Mapping[str, str], + prepared: PreparedMultipartBody, +) -> PreparedHttpRequest: + """Validate a task-local multipart body without opening a connection.""" + _validate_url(url) + try: + actual_size = prepared.path.stat().st_size + with prepared.path.open("rb"): + pass + except OSError as exc: + raise HttpTransportError( + f"failed to read prepared multipart body: {type(exc).__name__}" + ) from exc + if actual_size != prepared.content_length: + raise HttpTransportError("prepared multipart body size changed") + request_headers = _controlled_headers( + headers, + content_type=prepared.content_type, + content_length=prepared.content_length, + ) + return PreparedHttpRequest( + url, + request_headers, + lambda: prepared.path.open("rb"), + ) + + +def _multipart_header( + boundary: str, + *, + field_name: str, + filename: str, + content_type: str, +) -> bytes: + for label, value in ( + ("field name", field_name), + ("filename", filename), + ("content type", content_type), + ): + if not value or any(character in value for character in '\r\n"'): + raise HttpTransportError(f"invalid multipart {label}: {value!r}") + return ( + f"--{boundary}\r\n" + f'Content-Disposition: form-data; name="{field_name}"; filename="{filename}"\r\n' + f"Content-Type: {content_type}\r\n\r\n" + ).encode("ascii") + + +def prepare_coverage_multipart( + output_path: Path, + *, + event_body: bytes, + coverage_path: Path, + coverage_filename: str, + coverage_content_type: str, +) -> PreparedMultipartBody: + """Spool the exact coverage request body into worker-owned temporary space.""" + boundary = f"dd-topt-{secrets.token_hex(16)}" + event_header = _multipart_header( + boundary, + field_name="event", + filename="fileevent.json", + content_type="application/json", + ) + coverage_header = _multipart_header( + boundary, + field_name="coveragex", + filename=coverage_filename, + content_type=coverage_content_type, + ) + closing_boundary = f"\r\n--{boundary}--\r\n".encode("ascii") + try: + with output_path.open("wb") as handle: + handle.write(event_header) + handle.write(event_body) + handle.write(b"\r\n") + handle.write(coverage_header) + with coverage_path.open("rb") as coverage: + shutil.copyfileobj(coverage, handle, length=64 * 1024) + handle.write(closing_boundary) + content_length = handle.tell() + except OSError as exc: + raise HttpTransportError( + f"failed to prepare coverage request body: {type(exc).__name__}" + ) from exc + return PreparedMultipartBody( + path=output_path, + content_type=f"multipart/form-data; boundary={boundary}", + content_length=content_length, + ) + + +class HttpTransport: + """One worker-local HTTP client with one explicit retry policy.""" + + def __init__( + self, + *, + proxy_environment: Iterable[tuple[str, str]] = (), + connect_timeout: float = DEFAULT_CONNECT_TIMEOUT_SECONDS, + request_timeout: float = DEFAULT_REQUEST_TIMEOUT_SECONDS, + max_attempts: int = DEFAULT_MAX_ATTEMPTS, + retry_delay: float = DEFAULT_RETRY_DELAY_SECONDS, + response_limit: int = DEFAULT_RESPONSE_LIMIT_BYTES, + sleeper: Callable[[float], None] = time.sleep, + clock: Callable[[], float] = time.time, + ssl_context: ssl.SSLContext | None = None, + logger: logging.Logger | None = None, + ) -> None: + if connect_timeout <= 0 or request_timeout <= 0: + raise ValueError("HTTP timeouts must be positive") + if max_attempts <= 0: + raise ValueError("HTTP max_attempts must be positive") + if retry_delay < 0 or response_limit <= 0: + raise ValueError("HTTP retry delay and response limit are invalid") + + self.connect_timeout = connect_timeout + self.request_timeout = request_timeout + self.max_attempts = max_attempts + self.retry_delay = retry_delay + self.response_limit = response_limit + self._sleeper = sleeper + self._clock = clock + self._ssl_context = ssl_context or ssl.create_default_context() + self._logger = logger + self._log_context = "" + proxies, no_proxy = _proxy_configuration(proxy_environment) + handlers = ( + _SnapshotProxyHandler(proxies, no_proxy), + _RequestTimeoutHTTPHandler(connect_timeout), + _RequestTimeoutHTTPSHandler(connect_timeout, self._ssl_context), + _NoRedirectHandler(), + ) + self._opener = build_opener(*handlers) + + def set_log_context( + self, + task_id: str, + payload_type: str, + source_path: str, + ) -> None: + """Set diagnostics for this worker-local transport's current file.""" + self._log_context = f"task={task_id} type={payload_type} file={source_path}" + + def clear_log_context(self) -> None: + """Clear task diagnostics before the worker dequeues another file.""" + self._log_context = "" + + def _debug(self, message: str, *args: object) -> None: + if self._logger is None: + return + if self._log_context: + self._logger.debug("%s " + message, self._log_context, *args) + else: + self._logger.debug(message, *args) + + @property + def verifies_tls(self) -> bool: + return ( + self._ssl_context.verify_mode == ssl.CERT_REQUIRED + and self._ssl_context.check_hostname + ) + + def post_json( + self, + url: str, + headers: Mapping[str, str], + body: bytes | Path, + *, + gzip_body: bool = False, + content_encoding: str | None = None, + ) -> HttpResult: + """POST an exact JSON body, optionally compressed with stdlib gzip.""" + prepared_request = prepare_json_request( + url, + headers, + body, + gzip_body=gzip_body, + content_encoding=content_encoding, + ) + return self._post(prepared_request) + + def post_prepared_multipart( + self, + url: str, + headers: Mapping[str, str], + prepared: PreparedMultipartBody, + ) -> HttpResult: + """POST a task-local multipart body, reopening it for every retry.""" + prepared_request = prepare_spooled_multipart_request( + url, + headers, + prepared, + ) + return self._post(prepared_request) + + def _post( + self, + prepared_request: PreparedHttpRequest, + ) -> HttpResult: + retry_delays: list[float] = [] + status_code: int | None = None + response_excerpt = b"" + excerpt_truncated = False + transport_error: str | None = None + + for attempt in range(1, self.max_attempts + 1): + retry_after: str | None = None + retryable = False + body_stream = prepared_request.body_factory() + self._debug( + "HTTP POST attempt=%d/%d url=%s", + attempt, + self.max_attempts, + redact_url(prepared_request.url), + ) + try: + http_request = Request( + prepared_request.url, + data=body_stream, + headers=dict(prepared_request.headers), + method="POST", + ) + try: + response = self._opener.open( + http_request, + timeout=self.request_timeout, + ) + except HTTPError as exc: + response = exc + with closing(response): + status_code = int(response.status) + response_excerpt, excerpt_truncated = _bounded_response( + response, + self.response_limit, + ) + transport_error = None + if 200 <= status_code < 300: + self._debug( + "HTTP POST succeeded attempt=%d status=%d", + attempt, + status_code, + ) + return HttpResult( + status_code=status_code, + attempts=attempt, + body_excerpt=response_excerpt, + body_truncated=excerpt_truncated, + retry_delays=tuple(retry_delays), + ) + self._debug( + "HTTP POST failed attempt=%d status=%d " + "body_excerpt=%r body_truncated=%s", + attempt, + status_code, + response_excerpt.decode("utf-8", errors="backslashreplace"), + excerpt_truncated, + ) + retryable = _retryable_status(status_code) + retry_after = response.headers.get("Retry-After") + except ( + URLError, + TimeoutError, + ConnectionError, + http.client.HTTPException, + ssl.SSLError, + ) as exc: + status_code = None + response_excerpt = b"" + excerpt_truncated = False + transport_cause = exc.reason if isinstance(exc, URLError) else exc + transport_error = type(transport_cause).__name__ + retryable = not isinstance( + transport_cause, ssl.SSLCertVerificationError + ) + finally: + body_stream.close() + + if not retryable or attempt >= self.max_attempts: + self._debug( + "HTTP POST terminal attempt=%d status=%s transport_error=%s", + attempt, + status_code, + transport_error or "none", + ) + return HttpResult( + status_code=status_code, + attempts=attempt, + body_excerpt=response_excerpt, + body_truncated=excerpt_truncated, + transport_error=transport_error, + retry_delays=tuple(retry_delays), + ) + + delay = _retry_after_seconds(retry_after, self._clock()) + if delay is None: + delay = self.retry_delay + retry_delays.append(delay) + self._debug( + "HTTP POST retry scheduled attempt=%d status=%s delay_seconds=%.3f", + attempt, + status_code, + delay, + ) + self._sleeper(delay) + + raise AssertionError("HTTP retry loop terminated without a result") diff --git a/tools/core/uploader_py/worker_pool.py b/tools/core/uploader_py/worker_pool.py new file mode 100644 index 00000000..455d6932 --- /dev/null +++ b/tools/core/uploader_py/worker_pool.py @@ -0,0 +1,286 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Schedule complete file pipelines on a bounded pool of homogeneous workers. + +The pool owns queueing and metrics while file workers remain mutually independent. +""" + +from __future__ import annotations + +from dataclasses import dataclass +import logging +from queue import Empty, Queue +import threading +import time +from typing import Callable, Iterable, Protocol, TypeVar + +from .models import FileResult, FileStatus, FileTask + + +RuntimeType = TypeVar("RuntimeType") +TransportType = TypeVar("TransportType") + + +class FileProcessor(Protocol[RuntimeType, TransportType]): + def __call__( + self, + task: FileTask, + runtime: RuntimeType, + transport: TransportType, + ) -> FileResult: ... + + +class WorkerPoolError(ValueError): + """The coordinator cannot safely construct the requested worker pool.""" + + +@dataclass(frozen=True) +class WorkerPoolRun: + """Deterministic results plus coordinator-owned concurrency observations.""" + + results: tuple[FileResult, ...] + worker_threads: int + peak_active_workers: int + + +class WorkerPoolInterrupted(KeyboardInterrupt): + """An interrupt after workers joined, carrying every completed result.""" + + def __init__(self, run: WorkerPoolRun, *, cancelled: int) -> None: + super().__init__("worker pool interrupted") + self.run = run + self.cancelled = cancelled + + +def run_file_workers( + tasks: Iterable[FileTask], + *, + workers: int, + runtime: RuntimeType, + transport_factory: Callable[[], TransportType], + process_file: FileProcessor[RuntimeType, TransportType], + logger: logging.Logger | None = None, +) -> WorkerPoolRun: + """Process each source exactly once through one complete worker pipeline. + + Results are returned in intake order for deterministic reports. The pool + owns aggregate concurrency metrics; file workers remain independent. + """ + if workers <= 0: + raise WorkerPoolError("workers must be positive") + planned_tasks = tuple(tasks) + task_ids = [task.task_id for task in planned_tasks] + if len(task_ids) != len(set(task_ids)): + raise WorkerPoolError("file task IDs must be unique") + if not planned_tasks: + return WorkerPoolRun((), 0, 0) + + worker_count = min(workers, len(planned_tasks)) + try: + worker_transports = tuple(transport_factory() for _ in range(worker_count)) + except Exception as exc: + raise WorkerPoolError( + f"failed to initialize worker transport: {type(exc).__name__}" + ) from exc + + task_queue: Queue[FileTask | None] = Queue(maxsize=max(1, worker_count * 2)) + result_queue: Queue[FileResult] = Queue() + activity_lock = threading.Lock() + shutdown_event = threading.Event() + active_workers = 0 + peak_active_workers = 0 + + def worker_loop(transport: TransportType) -> None: + nonlocal active_workers, peak_active_workers + if logger is not None: + logger.debug("worker=%s started", threading.current_thread().name) + while True: + try: + task = task_queue.get(timeout=0.1) + except Empty: + if shutdown_event.is_set(): + if logger is not None: + logger.debug( + "worker=%s stopped after interrupt", + threading.current_thread().name, + ) + return + continue + try: + if task is None: + if logger is not None: + logger.debug("worker=%s stopped", threading.current_thread().name) + return + if shutdown_event.is_set(): + if logger is not None: + logger.debug( + "task=%s type=%s file=%s cancelled before worker ownership", + task.task_id, + task.payload_type.value, + task.display_path, + ) + continue + task_started = time.monotonic() + if logger is not None: + logger.debug( + "task=%s type=%s file=%s dequeued worker=%s", + task.task_id, + task.payload_type.value, + task.display_path, + threading.current_thread().name, + ) + with activity_lock: + active_workers += 1 + peak_active_workers = max(peak_active_workers, active_workers) + try: + try: + result = process_file(task, runtime, transport) + if result.task_id != task.task_id: + raise WorkerPoolError( + "file processor returned a result for a different task" + ) + except Exception as exc: + # A broken file pipeline becomes that file's terminal + # result; it must not tear down unrelated workers. + result = FileResult( + task_id=task.task_id, + source_path=task.display_path, + payload_type=task.payload_type, + status=FileStatus.FAILED, + failure_code="unhandled_worker_exception", + failure_message=type(exc).__name__, + ) + finally: + with activity_lock: + active_workers -= 1 + result_queue.put(result) + if logger is not None: + logger.debug( + "task=%s type=%s file=%s worker=%s completed status=%s " + "elapsed=%.3fs", + task.task_id, + task.payload_type.value, + task.display_path, + threading.current_thread().name, + result.status.value, + max(0.0, time.monotonic() - task_started), + ) + finally: + task_queue.task_done() + + worker_threads = tuple( + threading.Thread( + target=worker_loop, + args=(transport,), + name=f"dd-uploader-worker-{index + 1}", + daemon=False, + ) + for index, transport in enumerate(worker_transports) + ) + started_threads: list[threading.Thread] = [] + + def stop_started_workers() -> None: + """Leave no queued work or live non-daemon thread after coordinator exit.""" + shutdown_event.set() + while True: + try: + task_queue.get_nowait() + except Empty: + break + else: + task_queue.task_done() + for started_thread in started_threads: + started_thread.join() + task_queue.join() + + try: + for thread in worker_threads: + thread.start() + started_threads.append(thread) + for task in planned_tasks: + task_queue.put(task) + if logger is not None: + logger.debug( + "task=%s type=%s file=%s enqueued", + task.task_id, + task.payload_type.value, + task.display_path, + ) + for _ in started_threads: + task_queue.put(None) + task_queue.join() + for thread in started_threads: + thread.join() + except KeyboardInterrupt: + if logger is not None: + logger.warning("interrupt received; draining unowned file tasks") + # Stop scheduling queued files, but let already-owned files finish + # their current pipeline. Their normal cleanup contract still applies; + # every not-yet-owned source remains untouched. Workers observe the + # shutdown event after finishing owned work, so no new stop markers are + # added here: some or all normal markers may already have been + # consumed when the interrupt arrives. + stop_started_workers() + interrupted_run = _collect_run( + planned_tasks, + result_queue, + worker_count=len(started_threads), + peak_active_workers=peak_active_workers, + require_complete=False, + ) + raise WorkerPoolInterrupted( + interrupted_run, + cancelled=len(planned_tasks) - len(interrupted_run.results), + ) from None + except Exception as exc: + stop_started_workers() + raise WorkerPoolError( + f"worker pool coordinator failed: {type(exc).__name__}" + ) from exc + + return _collect_run( + planned_tasks, + result_queue, + worker_count=worker_count, + peak_active_workers=peak_active_workers, + require_complete=True, + ) + + +def _collect_run( + planned_tasks: tuple[FileTask, ...], + result_queue: Queue[FileResult], + *, + worker_count: int, + peak_active_workers: int, + require_complete: bool, +) -> WorkerPoolRun: + """Drain terminal results and restore deterministic intake order.""" + completed_results: list[FileResult] = [] + while True: + try: + completed_results.append(result_queue.get_nowait()) + except Empty: + break + results_by_id = {result.task_id: result for result in completed_results} + if len(results_by_id) != len(completed_results): + raise WorkerPoolError("worker pool returned duplicate task results") + planned_ids = {task.task_id for task in planned_tasks} + if any(task_id not in planned_ids for task_id in results_by_id): + raise WorkerPoolError("worker pool returned a result for an unknown task") + if len(results_by_id) != len(planned_tasks): + if require_complete: + raise WorkerPoolError("worker pool did not return exactly one result per task") + return WorkerPoolRun( + results=tuple( + results_by_id[task.task_id] + for task in planned_tasks + if task.task_id in results_by_id + ), + worker_threads=worker_count, + peak_active_workers=peak_active_workers, + ) diff --git a/tools/core/uploader_python_launcher.ps1.tpl b/tools/core/uploader_python_launcher.ps1.tpl new file mode 100644 index 00000000..4497ffb2 --- /dev/null +++ b/tools/core/uploader_python_launcher.ps1.tpl @@ -0,0 +1,164 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +# Resolve the shared Python entrypoint and generated config from Bazel runfiles. +# Keeping upload behavior out of this launcher avoids a separate Windows runtime. + +param( + [Parameter(ValueFromRemainingArguments = $true)] + [string[]]$UploaderArgs +) + +$ErrorActionPreference = "Stop" +$script:LauncherPath = $MyInvocation.MyCommand.Path +$script:LauncherDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$script:LauncherBase = [System.IO.Path]::GetFileNameWithoutExtension($script:LauncherPath) +$script:BatchLauncherPath = Join-Path $script:LauncherDir "$script:LauncherBase.bat" +$env:DD_TEST_OPTIMIZATION_UPLOADER_LAUNCHER_DIR = $script:LauncherDir +$script:RunfilesManifests = @( + $env:RUNFILES_MANIFEST_FILE, + "$script:LauncherPath.runfiles_manifest", + "$script:BatchLauncherPath.runfiles_manifest", + (Join-Path $script:LauncherDir "$script:LauncherBase.runfiles_manifest"), + "$script:LauncherPath.runfiles\MANIFEST", + "$script:BatchLauncherPath.runfiles\MANIFEST", + (Join-Path $script:LauncherDir "$script:LauncherBase.runfiles\MANIFEST") +) + +foreach ($manifestCandidate in $script:RunfilesManifests) { + if ($manifestCandidate -and + (Test-Path -LiteralPath $manifestCandidate -PathType Leaf)) { + $env:RUNFILES_MANIFEST_FILE = (Resolve-Path -LiteralPath $manifestCandidate).Path + break + } +} + +function ConvertFrom-RunfilesManifestField { + param([string]$Value) + return $Value.Replace("\s", " ").Replace("\n", "`n").Replace("\b", "\") +} + +function Resolve-UploaderRunfile { + param( + [string]$DirectPath, + [string]$LogicalPath, + [string]$SiblingName = "" + ) + if ($SiblingName) { + $sibling = Join-Path $script:LauncherDir $SiblingName + if (Test-Path -LiteralPath $sibling -PathType Leaf) { + return (Resolve-Path -LiteralPath $sibling).Path + } + } + if ($DirectPath -and (Test-Path -LiteralPath $DirectPath -PathType Leaf)) { + return (Resolve-Path -LiteralPath $DirectPath).Path + } + + $normalized = $LogicalPath.Replace("\", "/") + while ($normalized.StartsWith("../")) { + $normalized = $normalized.Substring(3) + } + $logicalCandidates = [System.Collections.Generic.List[string]]::new() + [void]$logicalCandidates.Add($normalized) + if ($normalized.StartsWith("external/")) { + [void]$logicalCandidates.Add($normalized.Substring(9)) + } else { + [void]$logicalCandidates.Add("external/$normalized") + } + [void]$logicalCandidates.Add("_main/$normalized") + + $roots = @( + $env:RUNFILES_DIR, + $env:TEST_SRCDIR, + "$script:LauncherPath.runfiles", + "$script:BatchLauncherPath.runfiles", + (Join-Path $script:LauncherDir "$script:LauncherBase.runfiles") + ) + foreach ($root in $roots) { + if (-not $root -or -not (Test-Path -LiteralPath $root -PathType Container)) { + continue + } + foreach ($candidate in $logicalCandidates) { + $paths = @((Join-Path $root $candidate)) + if ($env:TEST_WORKSPACE) { + $paths += (Join-Path (Join-Path $root $env:TEST_WORKSPACE) $candidate) + } + foreach ($path in $paths) { + if (Test-Path -LiteralPath $path -PathType Leaf) { + return (Resolve-Path -LiteralPath $path).Path + } + } + } + } + + foreach ($manifest in $script:RunfilesManifests) { + if (-not $manifest -or -not (Test-Path -LiteralPath $manifest -PathType Leaf)) { + continue + } + foreach ($line in [System.IO.File]::ReadLines($manifest)) { + if ($line.StartsWith(" ")) { + $encoded = $line.Substring(1) + $separator = $encoded.IndexOf(" ") + if ($separator -le 0) { continue } + $key = ConvertFrom-RunfilesManifestField ` + $encoded.Substring(0, $separator) + $value = ConvertFrom-RunfilesManifestField ` + $encoded.Substring($separator + 1) + } else { + $separator = $line.IndexOf(" ") + if ($separator -le 0) { continue } + $key = $line.Substring(0, $separator) + $value = $line.Substring($separator + 1) + } + $key = $key.Replace("\", "/") + foreach ($candidate in $logicalCandidates) { + if (($key -eq $candidate -or $key.EndsWith("/$candidate")) -and + (Test-Path -LiteralPath $value -PathType Leaf)) { + return (Resolve-Path -LiteralPath $value).Path + } + } + } + } + throw "runfile could not be resolved: $LogicalPath" +} + +$python = $null +foreach ($candidate in @($env:DD_TEST_OPTIMIZATION_PYTHON, $env:PYTHON, "python3", "python")) { + if (-not $candidate) { continue } + $command = Get-Command $candidate -CommandType Application ` + -ErrorAction SilentlyContinue | Select-Object -First 1 + if (-not $command) { continue } + try { + & $command.Source -c ` + 'import sys; raise SystemExit(sys.version_info < (3, 10))' *> $null + } catch { + continue + } + if ($LASTEXITCODE -eq 0) { + $python = $command.Source + break + } +} +if (-not $python) { + [Console]::Error.WriteLine("[dd-uploader] error: Python 3.10 or newer was not found") + exit 2 +} + +try { + $mainPath = Resolve-UploaderRunfile ` + -DirectPath "__DDTPL_PYTHON_MAIN_PATH__" ` + -LogicalPath "__DDTPL_PYTHON_MAIN_RLOC__" + $configPath = Resolve-UploaderRunfile ` + -DirectPath "__DDTPL_PYTHON_CONFIG_PATH__" ` + -LogicalPath "__DDTPL_PYTHON_CONFIG_RLOC__" ` + -SiblingName "__DDTPL_PYTHON_CONFIG_NAME__" +} catch { + [Console]::Error.WriteLine("[dd-uploader] error: $($_.Exception.Message)") + exit 2 +} + +& $python $mainPath --config $configPath @UploaderArgs +exit $LASTEXITCODE diff --git a/tools/core/uploader_python_launcher.sh.tpl b/tools/core/uploader_python_launcher.sh.tpl new file mode 100644 index 00000000..933f2483 --- /dev/null +++ b/tools/core/uploader_python_launcher.sh.tpl @@ -0,0 +1,139 @@ +#!/usr/bin/env bash +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +# Resolve the shared Python entrypoint and generated config from Bazel runfiles. +# Keeping upload behavior out of this launcher avoids a separate Unix runtime. + +set -euo pipefail + +launcher_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +launcher_path="$launcher_dir/$(basename "${BASH_SOURCE[0]}")" +export DD_TEST_OPTIMIZATION_UPLOADER_LAUNCHER_DIR="$launcher_dir" +runfiles_manifests=( + "${RUNFILES_MANIFEST_FILE:-}" + "$launcher_path.runfiles_manifest" + "$launcher_path.runfiles/MANIFEST" +) + +if [[ -z "${RUNFILES_MANIFEST_FILE:-}" || ! -f "$RUNFILES_MANIFEST_FILE" ]]; then + for manifest_candidate in "${runfiles_manifests[@]}"; do + if [[ -f "$manifest_candidate" ]]; then + RUNFILES_MANIFEST_FILE="$manifest_candidate" + export RUNFILES_MANIFEST_FILE + break + fi + done +fi + +decode_runfiles_manifest_field() { + local decoded="$1" + decoded="${decoded//\\s/ }" + decoded="${decoded//\\n/$'\n'}" + decoded="${decoded//\\b/\\}" + printf '%s' "$decoded" +} + +resolve_runfile() { + local direct="$1" + local logical="$2" + local sibling_name="${3:-}" + local candidate root manifest line key value normalized + + if [[ -n "$sibling_name" && -f "$launcher_dir/$sibling_name" ]]; then + printf '%s\n' "$launcher_dir/$sibling_name" + return 0 + fi + if [[ -n "$direct" && -f "$direct" ]]; then + (cd "$(dirname "$direct")" && printf '%s/%s\n' "$(pwd -P)" "$(basename "$direct")") + return 0 + fi + + normalized="${logical//\\//}" + while [[ "$normalized" == ../* ]]; do + normalized="${normalized#../}" + done + local logical_candidates=("$normalized") + if [[ "$normalized" == external/* ]]; then + logical_candidates+=("${normalized#external/}") + else + logical_candidates+=("external/$normalized") + fi + logical_candidates+=("_main/$normalized") + + local roots=("${RUNFILES_DIR:-}" "${TEST_SRCDIR:-}" "$launcher_path.runfiles") + for root in "${roots[@]}"; do + [[ -n "$root" && -d "$root" ]] || continue + for candidate in "${logical_candidates[@]}"; do + for value in "$root/$candidate" "${TEST_WORKSPACE:+$root/$TEST_WORKSPACE/$candidate}"; do + if [[ -f "$value" ]]; then + printf '%s\n' "$value" + return 0 + fi + done + done + done + + for manifest in "${runfiles_manifests[@]}"; do + [[ -n "$manifest" && -f "$manifest" ]] || continue + while IFS= read -r line || [[ -n "$line" ]]; do + if [[ "${line:0:1}" == " " ]]; then + line="${line:1}" + [[ "$line" == *" "* ]] || continue + key="$(decode_runfiles_manifest_field "${line%% *}")" + value="$(decode_runfiles_manifest_field "${line#* }")" + else + [[ "$line" == *" "* ]] || continue + key="${line%% *}" + value="${line#* }" + fi + for candidate in "${logical_candidates[@]}"; do + if [[ "$key" == "$candidate" || "$key" == */"$candidate" ]]; then + [[ -f "$value" ]] || continue + printf '%s\n' "$value" + return 0 + fi + done + done <"$manifest" + done + return 1 +} + +python_bin="" +for candidate in "${DD_TEST_OPTIMIZATION_PYTHON:-}" "${PYTHON:-}" python3 python; do + [[ -n "$candidate" ]] || continue + if ! python_path="$(command -v "$candidate" 2>/dev/null)"; then + continue + fi + if "$python_path" -c \ + 'import sys; raise SystemExit(sys.version_info < (3, 10))' \ + >/dev/null 2>&1; then + python_bin="$python_path" + break + fi +done +if [[ -z "$python_bin" ]]; then + echo "[dd-uploader] error: Python 3.10 or newer was not found" >&2 + exit 2 +fi + +main_path="$(resolve_runfile \ + "__DDTPL_PYTHON_MAIN_PATH__" \ + "__DDTPL_PYTHON_MAIN_RLOC__" \ +)" || { + echo "[dd-uploader] error: uploader_main.py could not be resolved" >&2 + exit 2 +} +config_path="$(resolve_runfile \ + "__DDTPL_PYTHON_CONFIG_PATH__" \ + "__DDTPL_PYTHON_CONFIG_RLOC__" \ + "__DDTPL_PYTHON_CONFIG_NAME__" \ +)" || { + echo "[dd-uploader] error: generated uploader config could not be resolved" >&2 + exit 2 +} + +exec "$python_bin" "$main_path" --config "$config_path" "$@" diff --git a/tools/core/validate_payload_schema.py b/tools/core/validate_payload_schema.py index fcf06811..1aafcd7d 100644 --- a/tools/core/validate_payload_schema.py +++ b/tools/core/validate_payload_schema.py @@ -7,6 +7,9 @@ """Validate payload JSON against a practical JSON Schema subset. +The dependency-free API is shared by the CLI and uploader workers so dry-run +validation needs neither a subprocess nor per-thread schema copies. + Supported keywords include: `$ref`, `allOf`, `anyOf`, `if`/`then`/`else`, `type`, `const`, `enum`, `minimum`, `maximum`, `required`, `properties`, `patternProperties`, `additionalProperties`, `items`, and `additionalItems`. @@ -22,7 +25,7 @@ import os import re import sys -from typing import Any, Dict, List, Optional, Set +from typing import Any, Callable, Dict, List, NamedTuple, Optional, Set, Tuple DEFAULT_MAX_ERRORS = 20 _DEBUG_TRUTHY = {"1", "true", "yes", "on"} @@ -38,8 +41,21 @@ _UNSUPPORTED_POLICY_ENV = "DD_TEST_OPTIMIZATION_SCHEMA_UNSUPPORTED_KEYWORDS" +class ValidationResult(NamedTuple): + """Structured result returned by the importable validation API.""" + + errors: Tuple[str, ...] + warnings: Tuple[str, ...] + stats: Dict[str, int] + + @property + def valid(self) -> bool: + """Whether the payload satisfies the supported schema subset.""" + return not self.errors + + def _new_stats() -> Dict[str, int]: - """Internal helper for new stats behavior.""" + """Create counters isolated to one validation run.""" return { "nodes": 0, "refs": 0, @@ -57,34 +73,43 @@ def _new_stats() -> Dict[str, int]: def _reset_stats() -> None: - """Internal helper for reset stats behavior.""" + """Reset the legacy CLI statistics snapshot between invocations.""" global _STATS _STATS = _new_stats() def _debug_enabled() -> bool: - """Internal helper for debug enabled behavior.""" - val = os.getenv("DD_TEST_OPTIMIZATION_SCHEMA_DEBUG") - if val is None: - val = os.getenv("DD_TEST_OPTIMIZATION_DEBUG") - if val is None: + """Read the schema-specific debug switch with uploader fallback.""" + configured = os.getenv("DD_TEST_OPTIMIZATION_SCHEMA_DEBUG") + if configured is None: + configured = os.getenv("DD_TEST_OPTIMIZATION_DEBUG") + if configured is None: return False - return str(val).strip().lower() in _DEBUG_TRUTHY + return str(configured).strip().lower() in _DEBUG_TRUTHY def _debug(msg: str, debug: bool = False) -> None: - """Internal helper for debug behavior.""" + """Emit schema diagnostics only when explicit debug is enabled.""" if debug or _debug_enabled(): print(f"[schema-validator][dbg] {msg}", file=sys.stderr) def _stat_inc(stats: Dict[str, int], key: str, n: int = 1) -> None: - """Internal helper for stat inc behavior.""" + """Increment one validation counter.""" stats[key] = stats.get(key, 0) + n +def _format_stats(stats: Dict[str, int]) -> str: + """Render the stable debug summary shared by success and failure paths.""" + return ( + "stats: nodes={nodes} refs={refs} anyof={anyof} " + "anyof_matched={anyof_matched} anyof_failed={anyof_failed} " + "if={if} then={then} else={else} type_checks={type_checks}" + ).format(**stats) + + def _safe_size(path: str) -> Optional[int]: - """Internal helper for safe size behavior.""" + """Read a file size without turning diagnostics into a failure.""" try: return os.path.getsize(path) except OSError: @@ -92,14 +117,14 @@ def _safe_size(path: str) -> Optional[int]: def _format_size(size: Optional[int]) -> str: - """Internal helper for format size behavior.""" + """Render an optional byte count for debug output.""" if size is None: return "unknown" return str(size) def _ensure_readable_file(path: str, label: str) -> Optional[str]: - """Internal helper for ensure readable file behavior.""" + """Return an actionable error when a CLI input is not a regular file.""" if not os.path.exists(path): return f"{label} file does not exist: {path}" if not os.path.isfile(path): @@ -108,7 +133,7 @@ def _ensure_readable_file(path: str, label: str) -> Optional[str]: def _max_errors_from_env() -> int: - """Internal helper for max errors from env behavior.""" + """Resolve and validate the environment-controlled error limit.""" raw = os.getenv("DD_TEST_OPTIMIZATION_SCHEMA_MAX_ERRORS") if raw is None or raw.strip() == "": return DEFAULT_MAX_ERRORS @@ -122,7 +147,7 @@ def _max_errors_from_env() -> int: def _unsupported_policy_from_env() -> str: - """Internal helper for unsupported policy from env behavior.""" + """Resolve whether unsupported schema keywords fail or warn.""" raw = os.getenv(_UNSUPPORTED_POLICY_ENV, "error") mode = str(raw).strip().lower() if not mode: @@ -135,7 +160,7 @@ def _unsupported_policy_from_env() -> str: def _parse_args(argv: List[str]) -> argparse.Namespace: - """Internal helper for parse args behavior.""" + """Parse the stable standalone validator CLI.""" parser = argparse.ArgumentParser( prog = "validate_payload_schema.py", description = "Validate payload JSON against a schema JSON file.", @@ -152,7 +177,7 @@ def _parse_args(argv: List[str]) -> argparse.Namespace: def _sample_keys(value: Any, limit: int = 12) -> str: - """Internal helper for sample keys behavior.""" + """Render a bounded key sample without logging payload values.""" if not isinstance(value, dict): return "" keys = list(value.keys()) @@ -162,12 +187,12 @@ def _sample_keys(value: Any, limit: int = 12) -> str: def _is_number(value: Any) -> bool: - """Internal helper for is number behavior.""" + """Apply JSON number semantics, where booleans are not integers.""" return isinstance(value, (int, float)) and not isinstance(value, bool) def _is_type(value: Any, type_name: str) -> bool: - """Internal helper for is type behavior.""" + """Match one supported JSON Schema type name.""" if type_name == "object": return isinstance(value, dict) if type_name == "array": @@ -185,37 +210,41 @@ def _is_type(value: Any, type_name: str) -> bool: return False -def _resolve_ref(root: Dict[str, Any], ref: str, stats: Optional[Dict[str, int]] = None) -> Dict[str, Any]: - """Internal helper for resolve ref behavior.""" +def _resolve_ref( + root: Dict[str, Any], + ref: str, + stats: Optional[Dict[str, int]] = None, +) -> Dict[str, Any]: + """Resolve one local JSON Pointer reference within the loaded schema.""" if stats is not None: _stat_inc(stats, "refs") if not ref.startswith("#/"): raise ValueError(f"unsupported ref: {ref}") parts = ref[2:].split("/") - cur: Any = root + current: Any = root for part in parts: part = part.replace("~1", "/").replace("~0", "~") - if isinstance(cur, dict): - if part not in cur: + if isinstance(current, dict): + if part not in current: raise ValueError(f"ref segment not found: {part!r} in {ref}") - cur = cur[part] + current = current[part] continue - if isinstance(cur, list): + if isinstance(current, list): if not part.isdigit(): raise ValueError(f"ref segment is not a list index: {part!r} in {ref}") - idx = int(part) - if idx < 0 or idx >= len(cur): + index = int(part) + if index < 0 or index >= len(current): raise ValueError(f"ref index out of bounds: {part!r} in {ref}") - cur = cur[idx] + current = current[index] continue raise ValueError(f"ref traversal hit non-container at segment {part!r} in {ref}") - if not isinstance(cur, dict): + if not isinstance(current, dict): raise ValueError(f"ref did not resolve to an object: {ref}") - return cur + return current def _path_key(path: str, key: str) -> str: - """Internal helper for path key behavior.""" + """Append one safely quoted object key to a diagnostic JSON path.""" safe = key.replace("'", "\\'") return f"{path}['{safe}']" @@ -230,8 +259,9 @@ def _validate( stats: Optional[Dict[str, int]] = None, warned_unsupported: Optional[Set[str]] = None, unsupported_policy: str = "error", + warning_output: Optional[Callable[[str], None]] = None, ) -> None: - """Internal helper for validate behavior.""" + """Recursively validate one value while accumulating bounded diagnostics.""" if stats is None: stats = _new_stats() if warned_unsupported is None: @@ -247,14 +277,19 @@ def _validate( if keyword in schema: if unsupported_policy == "error": errors.append( - f"{path}: unsupported JSON Schema keyword '{keyword}' is not supported by this validator" + f"{path}: unsupported JSON Schema keyword '{keyword}' " + "is not supported by this validator" ) return if keyword not in warned_unsupported: - print( - f"warning: unsupported JSON Schema keyword '{keyword}' at {path} is ignored", - file=sys.stderr, + warning = ( + f"warning: unsupported JSON Schema keyword '{keyword}' " + f"at {path} is ignored" ) + if warning_output is None: + print(warning, file=sys.stderr) + else: + warning_output(warning) warned_unsupported.add(keyword) if "$ref" in schema: @@ -263,11 +298,17 @@ def _validate( except ValueError as exc: errors.append(f"{path}: {exc}") return - _validate(value, ref_schema, root, path, errors, max_errors, stats, warned_unsupported, unsupported_policy) + _validate( + value, ref_schema, root, path, errors, max_errors, + stats, warned_unsupported, unsupported_policy, warning_output, + ) return for subschema in schema.get("allOf", []): - _validate(value, subschema, root, path, errors, max_errors, stats, warned_unsupported, unsupported_policy) + _validate( + value, subschema, root, path, errors, max_errors, + stats, warned_unsupported, unsupported_policy, warning_output, + ) if len(errors) >= max_errors: return @@ -275,7 +316,10 @@ def _validate( _stat_inc(stats, "anyof") for subschema in schema["anyOf"]: sub_errors: List[str] = [] - _validate(value, subschema, root, path, sub_errors, max_errors, stats, warned_unsupported, unsupported_policy) + _validate( + value, subschema, root, path, sub_errors, max_errors, + stats, warned_unsupported, unsupported_policy, warning_output, + ) if not sub_errors: _stat_inc(stats, "anyof_matched") break @@ -287,25 +331,36 @@ def _validate( if "if" in schema: _stat_inc(stats, "if") cond_errors: List[str] = [] - _validate(value, schema["if"], root, path, cond_errors, max_errors, stats, warned_unsupported, unsupported_policy) + _validate( + value, schema["if"], root, path, cond_errors, max_errors, + stats, warned_unsupported, unsupported_policy, warning_output, + ) if not cond_errors: if "then" in schema: _stat_inc(stats, "then") - _validate(value, schema["then"], root, path, errors, max_errors, stats, warned_unsupported, unsupported_policy) + _validate( + value, schema["then"], root, path, errors, max_errors, + stats, warned_unsupported, unsupported_policy, warning_output, + ) else: if "else" in schema: _stat_inc(stats, "else") - _validate(value, schema["else"], root, path, errors, max_errors, stats, warned_unsupported, unsupported_policy) + _validate( + value, schema["else"], root, path, errors, max_errors, + stats, warned_unsupported, unsupported_policy, warning_output, + ) if "type" in schema: _stat_inc(stats, "type_checks") - type_spec = schema["type"] - if isinstance(type_spec, list): - ok = any(_is_type(value, t) for t in type_spec) + expected_types = schema["type"] + if isinstance(expected_types, list): + matches_type = any( + _is_type(value, expected_type) for expected_type in expected_types + ) else: - ok = _is_type(value, type_spec) - if not ok: - errors.append(f"{path}: expected type {type_spec}") + matches_type = _is_type(value, expected_types) + if not matches_type: + errors.append(f"{path}: expected type {expected_types}") return if "const" in schema and value != schema["const"]: @@ -338,47 +393,108 @@ def _validate( if len(errors) >= max_errors: return - props = schema.get("properties", {}) - pattern_props = schema.get("patternProperties", {}) + properties = schema.get("properties", {}) + pattern_properties = schema.get("patternProperties", {}) patterns = [] - for pattern, subschema in pattern_props.items(): + for pattern, subschema in pattern_properties.items(): try: patterns.append((re.compile(pattern), subschema)) except re.error as exc: errors.append(f"{path}: invalid patternProperties regex {pattern!r}: {exc}") return - for key, val in value.items(): + for key, property_value in value.items(): matched = False - if key in props: + if key in properties: matched = True - _validate(val, props[key], root, _path_key(path, key), errors, max_errors, stats, warned_unsupported, unsupported_policy) + _validate( + property_value, properties[key], root, _path_key(path, key), + errors, max_errors, stats, warned_unsupported, + unsupported_policy, warning_output, + ) for regex, subschema in patterns: if regex.search(key): matched = True - _validate(val, subschema, root, _path_key(path, key), errors, max_errors, stats, warned_unsupported, unsupported_policy) + _validate( + property_value, subschema, root, _path_key(path, key), + errors, max_errors, stats, warned_unsupported, + unsupported_policy, warning_output, + ) if not matched: - addl = schema.get("additionalProperties", True) - if addl is False: + additional_properties = schema.get("additionalProperties", True) + if additional_properties is False: errors.append(f"{path}: additional property '{key}' not allowed") - elif isinstance(addl, dict): - _validate(val, addl, root, _path_key(path, key), errors, max_errors, stats, warned_unsupported, unsupported_policy) + elif isinstance(additional_properties, dict): + _validate( + property_value, additional_properties, root, + _path_key(path, key), errors, max_errors, stats, + warned_unsupported, unsupported_policy, warning_output, + ) if isinstance(value, list): items = schema.get("items") if isinstance(items, dict): - for idx, item in enumerate(value): - _validate(item, items, root, f"{path}[{idx}]", errors, max_errors, stats, warned_unsupported, unsupported_policy) + for index, item in enumerate(value): + _validate( + item, items, root, f"{path}[{index}]", errors, max_errors, + stats, warned_unsupported, unsupported_policy, warning_output, + ) elif isinstance(items, list): - for idx, item in enumerate(value): - if idx < len(items): - _validate(item, items[idx], root, f"{path}[{idx}]", errors, max_errors, stats, warned_unsupported, unsupported_policy) + for index, item in enumerate(value): + if index < len(items): + _validate( + item, items[index], root, f"{path}[{index}]", + errors, max_errors, stats, warned_unsupported, + unsupported_policy, warning_output, + ) else: additional_items = schema.get("additionalItems", True) if additional_items is False: - errors.append(f"{path}[{idx}]: additional item not allowed") + errors.append( + f"{path}[{index}]: additional item not allowed" + ) elif isinstance(additional_items, dict): - _validate(item, additional_items, root, f"{path}[{idx}]", errors, max_errors, stats, warned_unsupported, unsupported_policy) + _validate( + item, additional_items, root, f"{path}[{index}]", + errors, max_errors, stats, warned_unsupported, + unsupported_policy, warning_output, + ) + + +def validate_payload( + payload: Any, + schema: Dict[str, Any], + *, + max_errors: int = DEFAULT_MAX_ERRORS, + unsupported_policy: str = "error", +) -> ValidationResult: + """Validate an already-decoded payload without process-global state. + + The schema is read-only during validation, so one loaded instance can be + shared safely by independent uploader threads. Errors, warnings, and stats + belong exclusively to this call. + """ + if max_errors <= 0: + raise ValueError("max_errors must be > 0") + if unsupported_policy not in {"error", "warn"}: + raise ValueError("unsupported_policy must be 'error' or 'warn'") + + errors: List[str] = [] + warnings: List[str] = [] + stats = _new_stats() + _validate( + payload, + schema, + schema, + "$", + errors, + max_errors, + stats, + set(), + unsupported_policy, + warnings.append, + ) + return ValidationResult(tuple(errors), tuple(warnings), dict(stats)) def main() -> int: @@ -425,8 +541,8 @@ def main() -> int: return 2 try: - with open(schema_path, "r", encoding="utf-8-sig") as f: - schema = json.load(f) + with open(schema_path, "r", encoding="utf-8-sig") as handle: + schema = json.load(handle) except (OSError, json.JSONDecodeError) as exc: print(f"error: failed to read schema: {exc}", file=sys.stderr) return 2 @@ -442,8 +558,8 @@ def main() -> int: _debug(f"schema required count: {len(schema.get('required', []))}", debug) try: - with open(payload_path, "r", encoding="utf-8-sig") as f: - payload = json.load(f) + with open(payload_path, "r", encoding="utf-8-sig") as handle: + payload = json.load(handle) except (OSError, json.JSONDecodeError) as exc: print(f"error: failed to read payload JSON: {exc}", file=sys.stderr) return 2 @@ -455,39 +571,32 @@ def main() -> int: if sample: _debug(f"payload key sample: {sample}", debug) - errors: List[str] = [] - stats = _new_stats() - warned_unsupported: Set[str] = set() _debug("validation start", debug) - _validate(payload, schema, schema, "$", errors, max_errors, stats, warned_unsupported, unsupported_policy) - _STATS = dict(stats) + result = validate_payload( + payload, + schema, + max_errors=max_errors, + unsupported_policy=unsupported_policy, + ) + errors = list(result.errors) + for warning in result.warnings: + print(warning, file=sys.stderr) + _STATS = dict(result.stats) if errors: print("schema validation failed:", file=sys.stderr) - for err in errors[:max_errors]: - print(f"- {err}", file=sys.stderr) + for error in errors[:max_errors]: + print(f"- {error}", file=sys.stderr) if len(errors) > max_errors: print(f"- ... and {len(errors) - max_errors} more", file=sys.stderr) _debug(f"validation result: failed ({len(errors)} error(s))", debug) if debug: - _debug( - "stats: nodes={nodes} refs={refs} anyof={anyof} anyof_matched={anyof_matched} " - "anyof_failed={anyof_failed} if={if} then={then} else={else} type_checks={type_checks}".format( - **_STATS - ), - debug, - ) + _debug(_format_stats(_STATS), debug) return 1 _debug("validation result: ok", debug) if debug: - _debug( - "stats: nodes={nodes} refs={refs} anyof={anyof} anyof_matched={anyof_matched} " - "anyof_failed={anyof_failed} if={if} then={then} else={else} type_checks={type_checks}".format( - **_STATS - ), - debug, - ) + _debug(_format_stats(_STATS), debug) return 0 diff --git a/tools/dev/lint_uploader_templates.py b/tools/dev/lint_uploader_templates.py index 733b679a..06df6f2c 100644 --- a/tools/dev/lint_uploader_templates.py +++ b/tools/dev/lint_uploader_templates.py @@ -5,23 +5,25 @@ # This product includes software developed at Datadog # (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. -"""Lint standalone uploader runtime template files.""" +"""Parse every generated uploader runtime and launcher template. + +Central linting keeps legacy and Python rollout entrypoints in the same CI gate. +""" from __future__ import annotations import argparse +from pathlib import Path +import re import subprocess import sys import tempfile -from pathlib import Path - -import re _TOKEN_RE = re.compile(r"__DDTPL_[A-Z0-9_]+__") def _repo_root() -> Path: - """Internal helper for repo root behavior.""" + """Find the checkout root so linting works from any current directory.""" here = Path(__file__).resolve().parent for candidate in [here] + list(here.parents): if (candidate / "MODULE.bazel").exists() or (candidate / ".git").exists(): @@ -30,22 +32,17 @@ def _repo_root() -> Path: def _normalize_bash_template_for_lint(template: str) -> str: - # Runtime templates carry __DDTPL_*__ tokens; replace them with deterministic - # literals so shellcheck parses render-equivalent syntax. - """Internal helper for normalize bash template for lint behavior.""" - normalized = _TOKEN_RE.sub("0", template) - return normalized + """Replace generated tokens with shellcheck-safe scalar values.""" + return _TOKEN_RE.sub("0", template) def _normalize_powershell_template_for_lint(template: str) -> str: - # Keep parser checks deterministic by replacing token placeholders with - # scalar literals. - """Internal helper for normalize powershell template for lint behavior.""" + """Replace generated tokens with PowerShell-parser-safe scalar values.""" return _TOKEN_RE.sub("0", template) def _lint_batch_template(template: str) -> None: - """Internal helper for lint batch template behavior.""" + """Check the small batch wrapper contract not covered by a parser.""" if "__DDTPL_PS_NAME__" not in template: raise RuntimeError("batch template missing __DDTPL_PS_NAME__ placeholder") normalized = _TOKEN_RE.sub("dd_upload_payloads.ps1", template).lower() @@ -58,20 +55,26 @@ def _lint_batch_template(template: str) -> None: def _run(cmd: list[str], cwd: Path) -> None: - """Internal helper for run behavior.""" + """Run one required linter and turn tool failures into useful diagnostics.""" try: - proc = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, check=False) + completed = subprocess.run( + cmd, + cwd=cwd, + capture_output=True, + text=True, + check=False, + ) except FileNotFoundError as exc: raise RuntimeError(f"required command not found: {cmd[0]}") from exc - if proc.returncode != 0: - stderr = proc.stderr.strip() - stdout = proc.stdout.strip() + if completed.returncode != 0: + stderr = completed.stderr.strip() + stdout = completed.stdout.strip() detail = stderr or stdout or "unknown error" raise RuntimeError(f"{' '.join(cmd)} failed: {detail}") def main() -> int: - """Run CLI entrypoint logic and return process exit code.""" + """Lint all rollout templates and return a process-compatible status.""" parser = argparse.ArgumentParser(description="Lint uploader runtime template files") parser.add_argument( "--skip-shellcheck", @@ -86,26 +89,27 @@ def main() -> int: args = parser.parse_args() repo = _repo_root() - bash_tpl = repo / "tools/core/uploader_bash_runtime.sh.tpl" - ps_tpl = repo / "tools/core/uploader_powershell_runtime.ps1.tpl" - batch_tpl = repo / "tools/core/uploader_batch_runtime.bat.tpl" - bash_template = _normalize_bash_template_for_lint(bash_tpl.read_text(encoding="utf-8")) - ps_template = _normalize_powershell_template_for_lint(ps_tpl.read_text(encoding="utf-8")) - batch_template = batch_tpl.read_text(encoding="utf-8") - - with tempfile.TemporaryDirectory(prefix="uploader_template_lint.") as tmp: - tmp_dir = Path(tmp) - bash_file = tmp_dir / "uploader_template.sh" - ps_file = tmp_dir / "uploader_template.ps1" - ps_parse_file = tmp_dir / "parse_template.ps1" - bash_file.write_text(bash_template, encoding="utf-8") - ps_file.write_text(ps_template, encoding="utf-8") - ps_parse_file.write_text( + bash_templates = ( + repo / "tools/core/uploader_bash_runtime.sh.tpl", + repo / "tools/core/uploader_python_launcher.sh.tpl", + ) + powershell_templates = ( + repo / "tools/core/uploader_powershell_runtime.ps1.tpl", + repo / "tools/core/uploader_python_launcher.ps1.tpl", + ) + batch_template_path = repo / "tools/core/uploader_batch_runtime.bat.tpl" + batch_template = batch_template_path.read_text(encoding="utf-8") + + with tempfile.TemporaryDirectory(prefix="uploader_template_lint.") as temporary: + temporary_root = Path(temporary) + powershell_parser_path = temporary_root / "parse_template.ps1" + powershell_parser_path.write_text( ( "param([string]$TemplatePath)\n" "$tokens = $null\n" "$errors = $null\n" - "[System.Management.Automation.Language.Parser]::ParseFile($TemplatePath, [ref]$tokens, [ref]$errors) | Out-Null\n" + "[System.Management.Automation.Language.Parser]::ParseFile(" + "$TemplatePath, [ref]$tokens, [ref]$errors) | Out-Null\n" "if ($errors -and $errors.Count -gt 0) {\n" " $errors | ForEach-Object { Write-Error $_ }\n" " exit 1\n" @@ -115,21 +119,37 @@ def main() -> int: ) if not args.skip_shellcheck: - _run(["shellcheck", "--severity=error", str(bash_file)], repo) + for index, template_path in enumerate(bash_templates): + bash_file = temporary_root / f"uploader_template_{index}.sh" + bash_file.write_text( + _normalize_bash_template_for_lint( + template_path.read_text(encoding="utf-8") + ), + encoding="utf-8", + ) + _run(["shellcheck", "--severity=error", str(bash_file)], repo) if not args.skip_powershell_parse: - _run( - [ - "pwsh", - "-NoProfile", - "-NonInteractive", - "-File", - str(ps_parse_file), - "-TemplatePath", - str(ps_file), - ], - repo, - ) + for index, template_path in enumerate(powershell_templates): + ps_file = temporary_root / f"uploader_template_{index}.ps1" + ps_file.write_text( + _normalize_powershell_template_for_lint( + template_path.read_text(encoding="utf-8") + ), + encoding="utf-8", + ) + _run( + [ + "pwsh", + "-NoProfile", + "-NonInteractive", + "-File", + str(powershell_parser_path), + "-TemplatePath", + str(ps_file), + ], + repo, + ) _lint_batch_template(batch_template) print("uploader template lint: ok") diff --git a/tools/tests/integration/run_bzlmod_go_integration.sh b/tools/tests/integration/run_bzlmod_go_integration.sh index c02463b8..c08d8454 100755 --- a/tools/tests/integration/run_bzlmod_go_integration.sh +++ b/tools/tests/integration/run_bzlmod_go_integration.sh @@ -539,7 +539,6 @@ run_bep_freshness_scenario() { --expected-enriched-tag=bazel.go.payload_selection ) >"$fresh_log" 2>&1 assert_log_contains "$fresh_log" "freshness filtering enabled: source=bep" "fresh BEP run did not select BEP freshness" - assert_log_contains "$fresh_log" "dry-run validated enriched test payload" "fresh BEP run did not validate enrichment" assert_log_matches "$fresh_log" "dry-run validated [1-9][0-9]* test payloads" "fresh BEP run did not validate any payloads" simulated_testlogs="$(simulate_bep_artifact_only_outputs "$ws_dir" "$fresh_bep")" diff --git a/tools/tests/integration/run_mock_server_tests.ps1 b/tools/tests/integration/run_mock_server_tests.ps1 index efac4c9f..92b3885f 100644 --- a/tools/tests/integration/run_mock_server_tests.ps1 +++ b/tools/tests/integration/run_mock_server_tests.ps1 @@ -1174,7 +1174,7 @@ filegroup( throw "multi-context uploader dry-run failed with exit code $dryRunExitCode`n$(Get-Content -LiteralPath $dryRunTranscript -Raw -ErrorAction SilentlyContinue)" } $dryRunOutput = Get-Content -LiteralPath $dryRunTranscript -Raw -Encoding UTF8 - if (-not $dryRunOutput.Contains("dry-run validated enriched test payload")) { + if ($dryRunOutput -notmatch "dry-run validated [1-9][0-9]* test payloads") { throw "multi-context uploader dry-run did not validate enriched test payloads" } if (-not $dryRunOutput.Contains("dry-run done")) { diff --git a/tools/tests/integration/run_mock_server_tests.sh b/tools/tests/integration/run_mock_server_tests.sh index 0c41ad23..0d670867 100755 --- a/tools/tests/integration/run_mock_server_tests.sh +++ b/tools/tests/integration/run_mock_server_tests.sh @@ -292,6 +292,12 @@ sh_test( dd_payload_uploader( name = "dd_upload_payloads", + use_python_uploader = False, +) + +dd_payload_uploader( + name = "dd_upload_payloads_python", + workers = 3, ) dd_payload_uploader( @@ -923,6 +929,62 @@ if grep -qiE "DD_API_KEY mismatch|API[ _-]?key mismatch" "$UPLOADER_LOG"; then exit 1 fi +# Exercise the default Python implementation through its real Bazel launcher and +# the same three protocol sources used by the legacy baseline above. Keep the +# source files so both implementations can run in one deterministic scenario. +PYTHON_UPLOAD_LOG_START="$(log_line_count)" +PYTHON_UPLOADER_LOG="$TMP_WS/uploader_python.log" +if ! TESTLOGS_DIR="$TESTLOGS_DIR" \ +BUILD_WORKSPACE_DIRECTORY="$WORKSPACE_FOR_UPLOADER" \ +DD_TEST_OPTIMIZATION_CODEOWNERS_FILE="$CODEOWNERS_FOR_UPLOADER" \ +DD_API_KEY=mock \ +DD_TEST_OPTIMIZATION_KEEP_PAYLOADS=1 \ +DD_TEST_OPTIMIZATION_AGENTLESS_URL="http://127.0.0.1:$PORT" \ +DD_TEST_OPTIMIZATION_MAX_WAIT_SEC=30 \ +DD_TEST_OPTIMIZATION_QUIESCENT_SEC=1 \ +DD_TEST_OPTIMIZATION_AGENT_URL= \ +"$BAZEL" "${BAZEL_FLAGS[@]}" run //:dd_upload_payloads_python \ + "${REPO_ENVS[@]}" -- --debug >"$PYTHON_UPLOADER_LOG" 2>&1; then + echo "error: default Python uploader command failed" + cat "$PYTHON_UPLOADER_LOG" || true + exit 1 +fi +if ! grep -q "summary: mode=upload result=success exit_code=0 workers=3" "$PYTHON_UPLOADER_LOG"; then + echo "error: default Python uploader did not emit the expected final statistics" + cat "$PYTHON_UPLOADER_LOG" || true + exit 1 +fi +LOG_FILE="$LOG_FILE" LOG_START="$PYTHON_UPLOAD_LOG_START" "$PYTHON" - <<'PY' +import json +import os +import sys + +expected = { + "/api/v2/citestcycle", + "/api/v2/citestcov", + "/api/v2/apmtelemetry", +} +start = int(os.environ["LOG_START"]) +paths = set() +with open(os.environ["LOG_FILE"], "r", encoding="utf-8") as handle: + for index, line in enumerate(handle): + if index < start: + continue + try: + record = json.loads(line) + except json.JSONDecodeError: + continue + path = record.get("path") + if isinstance(path, str): + paths.add(path) + +missing = sorted(expected - paths) +if missing: + print("error: default Python uploader missed protocol endpoints:", missing) + print("seen:", sorted(paths)) + sys.exit(1) +PY + # Scenario: CI defaults to cache-safe uploads. If no BEP or legacy execution log # is available, the uploader must fail closed unless the caller opts out explicitly. rm -rf "$WORKSPACE/.topt" @@ -4418,13 +4480,8 @@ DD_TEST_OPTIMIZATION_AGENT_URL= \ cat "$UPLOADER_DRY_RUN_LOG" || true exit 1 fi -if ! grep -q "dry-run validated enriched test payload" "$UPLOADER_DRY_RUN_LOG"; then - echo "error: uploader dry-run did not validate enriched test payloads" - cat "$UPLOADER_DRY_RUN_LOG" || true - exit 1 -fi -if ! grep -q "dry-run done" "$UPLOADER_DRY_RUN_LOG"; then - echo "error: uploader dry-run did not finish in dry-run mode" +if ! grep -q "summary: mode=dry-run result=success exit_code=0" "$UPLOADER_DRY_RUN_LOG"; then + echo "error: uploader did not report a successful enrichment dry-run" cat "$UPLOADER_DRY_RUN_LOG" || true exit 1 fi @@ -4682,7 +4739,7 @@ DD_TEST_OPTIMIZATION_AGENT_URL= \ exit 1 fi -if ! grep -q "no bundled context matched repo 'missing_runtime_repo'" "$UPLOADER_MULTI_CONTEXT_MISS_LOG"; then +if ! grep -q "warning_code=context_repo_not_found" "$UPLOADER_MULTI_CONTEXT_MISS_LOG"; then echo "error: missing expected warning for unmatched multi-context payload" cat "$UPLOADER_MULTI_CONTEXT_MISS_LOG" || true exit 1 @@ -6097,7 +6154,7 @@ PY # Scenario: when no tracer message-batch exists, the uploader should keep the # raw tracer telemetry files intact, normalize outbound env across the matched -# tracer set, and send one synthetic tracer-derived batch after the normal loop. +# tracer set, and send one synthetic tracer-derived batch from the anchor file. TELEMETRY_SYNTH_TESTLOGS="$TMP_WS/telemetry_synth_testlogs" TELEMETRY_SYNTH_DIR="$TELEMETRY_SYNTH_TESTLOGS/manual_telemetry_synth/test.outputs/payloads/telemetry" mkdir -p "$TELEMETRY_SYNTH_DIR" @@ -6182,11 +6239,11 @@ if len(telemetry_records) != 3: decoded = [json.loads(base64.b64decode(rec["body_b64"]).decode("utf-8")) for rec in telemetry_records] request_types = [payload.get("request_type") for payload in decoded] -if request_types != ["app-started", "app-closing", "message-batch"]: - print(f"error: synthetic scenario expected message-batch after raw tracer uploads, saw {request_types!r}") +if sorted(request_types) != ["app-closing", "app-started", "message-batch"]: + print(f"error: synthetic scenario expected one upload of each request type, saw {request_types!r}") sys.exit(1) -synthetic = decoded[-1] +synthetic = next(payload for payload in decoded if payload.get("request_type") == "message-batch") if synthetic.get("runtime_id") != "synthetic-runtime": print("error: synthetic telemetry should preserve runtime_id from tracer anchor") sys.exit(1) @@ -6207,7 +6264,7 @@ for message in synthetic.get("payload", []): if "git_requests.settings" not in metric_names or "known_tests.response_tests" not in metric_names: print(f"error: synthetic telemetry missing expected rule metrics: {metric_names!r}") sys.exit(1) -for raw_payload in decoded[:-1]: +for raw_payload in (payload for payload in decoded if payload is not synthetic): raw_app = raw_payload.get("application") or {} if raw_app.get("env") != "ci": print(f"error: raw tracer uploads should also rewrite application.env from facts, saw {raw_app.get('env')!r}") diff --git a/tools/tests/integration/run_workspace_go_integration.sh b/tools/tests/integration/run_workspace_go_integration.sh index 60d6ee01..8fe22203 100755 --- a/tools/tests/integration/run_workspace_go_integration.sh +++ b/tools/tests/integration/run_workspace_go_integration.sh @@ -521,7 +521,6 @@ run_bep_freshness_scenario() { --expected-enriched-tag=bazel.go.payload_selection ) >"$fresh_log" 2>&1 assert_log_contains "$fresh_log" "freshness filtering enabled: source=bep" "fresh BEP run did not select BEP freshness" - assert_log_contains "$fresh_log" "dry-run validated enriched test payload" "fresh BEP run did not validate enrichment" assert_log_matches "$fresh_log" "dry-run validated [1-9][0-9]* test payloads" "fresh BEP run did not validate any payloads" simulated_testlogs="$(simulate_bep_artifact_only_outputs "$ws_dir" "$fresh_bep")" diff --git a/tools/tests/python/BUILD.bazel b/tools/tests/python/BUILD.bazel index a7675310..c1a1f09f 100644 --- a/tools/tests/python/BUILD.bazel +++ b/tools/tests/python/BUILD.bazel @@ -4,6 +4,9 @@ # This product includes software developed at Datadog # (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. +# Run repository Python tooling under one runfiles-complete Bazel test target. +# Keeping the suite together makes host-side portability checks consistent. + load("@rules_shell//shell:sh_test.bzl", "sh_test") sh_test( @@ -12,48 +15,68 @@ sh_test( timeout = "moderate", srcs = ["run_python_tools_test.sh"], data = [ - "test_python_tools.py", - "test_rules_go_profile_tools.py", - "test_rules_go_variant_tools.py", + "fixtures/bep_cached_local.ndjson", "fixtures/bep_captured_bazelw_wrapper_cached.ndjson", "fixtures/bep_captured_bazelw_wrapper_fresh.ndjson", - "fixtures/bep_cached_local.ndjson", - "fixtures/bep_fresh_outputs_zip_file_uri.ndjson", "fixtures/bep_fresh_log_xml.ndjson", + "fixtures/bep_fresh_outputs_zip_file_uri.ndjson", "fixtures/bep_fresh_path_prefix.ndjson", "fixtures/bep_fresh_remote_bytestream.ndjson", "fixtures/bep_fresh_test_outputs_file_uri.ndjson", "fixtures/bep_snake_case_remote_cached.ndjson", - "//tools/tests/python/fixtures/generated_uploader:generated_uploader", + "test_python_tools.py", + "test_rules_go_profile_tools.py", + "test_rules_go_variant_tools.py", + "test_uploader_application_tools.py", + "test_uploader_codeowners_tools.py", + "test_uploader_coordinator_tools.py", + "test_uploader_discovery_tools.py", + "test_uploader_enrichment_tools.py", + "test_uploader_expected_targets_tools.py", + "test_uploader_file_worker_tools.py", + "test_uploader_freshness_tools.py", + "test_uploader_launcher_tools.py", + "test_uploader_reporting_tools.py", + "test_uploader_resources_tools.py", + "test_uploader_telemetry_tools.py", + "test_uploader_tools.py", + "test_uploader_transport_tools.py", + "test_uploader_worker_pool_tools.py", + "uploader_test_support.py", "//modules/go/tools/onboardingpins:rules_go_forks_gen.go", "//third_party/rules_go_orchestrion:registry.json", "//third_party/rules_go_orchestrion:workspace_runtime_profile", + "//tools/core:bep_artifact_stage_helper.py", "//tools/core:schemas/check_schema_parser_parity.py", "//tools/core:schemas/sync_agentless_schema.py", - "//tools/core:bep_artifact_stage_helper.py", "//tools/core:test_optimization_doctor.bzl", "//tools/core:test_optimization_doctor.py", "//tools/core:test_optimization_sync.bzl", + "//tools/core:test_optimization_uploader.bzl", "//tools/core:uploader_bash_runtime.sh.tpl", "//tools/core:uploader_batch_runtime.bat.tpl", "//tools/core:uploader_powershell_runtime.ps1.tpl", + "//tools/core:uploader_python_launcher.ps1.tpl", + "//tools/core:uploader_python_launcher.sh.tpl", + "//tools/core:uploader_python_runtime", "//tools/core:validate_payload_schema.py", - "//tools/test_optimization:ci_wrappers", - "//tools/test_optimization:create_support_bundle.py", - "//tools/test_optimization:render_report_summary.py", "//tools/dev:check_bazelversion_sync.py", "//tools/dev:check_module_versions.py", "//tools/dev:check_release_archive_contents.py", "//tools/dev:diff_rules_go_fork.py", - "//tools/dev:generate_rules_go_fork_maps.py", "//tools/dev:generate_rules_go_consumer_patch.py", + "//tools/dev:generate_rules_go_fork_maps.py", "//tools/dev:lint_uploader_templates.py", "//tools/dev:materialize_rules_go_fork.py", "//tools/dev:private_leak_public_denylist.txt", "//tools/dev:rules_go_fork_registry.py", "//tools/dev:verify_rules_go_profiles.py", "//tools/go:rules_go_forks.bzl", + "//tools/test_optimization:ci_wrappers", + "//tools/test_optimization:create_support_bundle.py", + "//tools/test_optimization:render_report_summary.py", "//tools/tests:integration/mock_dd_server.py", + "//tools/tests/python/fixtures/generated_uploader", ], ) diff --git a/tools/tests/python/fixtures/generated_uploader/BUILD.bazel b/tools/tests/python/fixtures/generated_uploader/BUILD.bazel index 189e0360..75e9c8c5 100644 --- a/tools/tests/python/fixtures/generated_uploader/BUILD.bazel +++ b/tools/tests/python/fixtures/generated_uploader/BUILD.bazel @@ -4,10 +4,15 @@ # This product includes software developed at Datadog # (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. +# Materialize the default uploader and both rollout implementations it carries. +# The fixture catches analysis/runfiles regressions that unit imports cannot expose. + load("//tools/core:test_optimization_uploader.bzl", "dd_payload_uploader") dd_payload_uploader( name = "generated_uploader", data = [], + max_wait_sec = 0, + quiescent_sec = 0, tags = ["manual"], ) diff --git a/tools/tests/python/test_python_tools.py b/tools/tests/python/test_python_tools.py index e092ebb5..135fa89d 100644 --- a/tools/tests/python/test_python_tools.py +++ b/tools/tests/python/test_python_tools.py @@ -5,7 +5,10 @@ # This product includes software developed at Datadog # (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. -"""Unit tests for repository Python tooling scripts.""" +"""Exercise repository-owned Python tools through one Bazel test target. + +The shared suite gives small developer scripts consistent runfiles and CI coverage. +""" from __future__ import annotations @@ -6352,6 +6355,59 @@ def test_lint_batch_template_requires_argument_forwarding(self) -> None: "exit /b %ERRORLEVEL%\n" ) + def test_main_lints_legacy_runtimes_and_python_launchers(self) -> None: + """Keep both rollout implementations covered by the lint entrypoint.""" + with tempfile.TemporaryDirectory() as tmp: + repo = Path(tmp) + core = repo / "tools" / "core" + core.mkdir(parents=True) + for name in ( + "uploader_bash_runtime.sh.tpl", + "uploader_python_launcher.sh.tpl", + ): + (core / name).write_text( + "#!/usr/bin/env bash\necho __DDTPL_VALUE__\n", + encoding="utf-8", + ) + for name in ( + "uploader_powershell_runtime.ps1.tpl", + "uploader_python_launcher.ps1.tpl", + ): + (core / name).write_text( + "Write-Output __DDTPL_VALUE__\n", + encoding="utf-8", + ) + (core / "uploader_batch_runtime.bat.tpl").write_text( + "@echo off\n" + "powershell.exe -File \"%SCRIPT_DIR%__DDTPL_PS_NAME__\" %*\n" + "exit /b %ERRORLEVEL%\n", + encoding="utf-8", + ) + + with mock.patch.object( + self.mod, + "_repo_root", + return_value=repo, + ), mock.patch.object(self.mod, "_run") as run, mock.patch.object( + sys, + "argv", + ["lint_uploader_templates.py"], + ): + self.assertEqual(0, self.mod.main()) + + shellcheck_calls = [ + call + for call in run.call_args_list + if call.args[0][0] == "shellcheck" + ] + powershell_calls = [ + call + for call in run.call_args_list + if call.args[0][0] == "pwsh" + ] + self.assertEqual(2, len(shellcheck_calls)) + self.assertEqual(2, len(powershell_calls)) + class RuntimeTemplateParityTests(unittest.TestCase): """Test case group covering RuntimeTemplateParityTests behaviors.""" @@ -6405,6 +6461,23 @@ def _extract_powershell_fingerprint_alphabet(powershell_text: str) -> str: raise AssertionError("unable to locate PowerShell fingerprint alphabet") return match.group(1).replace("''", "'") + @staticmethod + def _extract_python_fingerprint_alphabet(python_text: str) -> str: + """Extract the Python runtime alphabet without importing uploader code.""" + tree = ast.parse(python_text) + for node in tree.body: + if not isinstance(node, ast.Assign): + continue + if any( + isinstance(target, ast.Name) + and target.id == "FINGERPRINT_ALPHABET" + for target in node.targets + ): + value = ast.literal_eval(node.value) + if isinstance(value, str): + return value + raise AssertionError("unable to locate Python fingerprint alphabet") + @staticmethod def _write_bep_staging_smoke_fixture(root: Path) -> Path: """Create a BEP + outputs.zip fixture for generated uploader smoke tests.""" @@ -6596,6 +6669,7 @@ def _run_local_test_upload( responder, *, extra_args: tuple[str, ...] = (), + debug: bool = False, gzip_enabled: bool, keep_payloads: bool = True, read_only_parent: bool = False, @@ -6656,7 +6730,7 @@ def _run_local_test_upload( env = os.environ.copy() env.update({ "BUILD_WORKSPACE_DIRECTORY": str(root), - "DD_TEST_OPTIMIZATION_DEBUG": "0", + "DD_TEST_OPTIMIZATION_DEBUG": "1" if debug else "0", "DD_TEST_OPTIMIZATION_GZIP": "1" if gzip_enabled else "0", "DD_TEST_OPTIMIZATION_KEEP_PAYLOADS": "1" if keep_payloads else "0", "DD_TEST_OPTIMIZATION_MAX_WAIT_SEC": "0", @@ -6822,12 +6896,29 @@ def accept(_payload, _records): return 200, {} for runtime in ("Bash", "PowerShell"): - with self.subTest(runtime=runtime, result="valid"): + with self.subTest(runtime=runtime, result="valid-default-logging"): + result, records, retained_payload = self._run_local_test_upload( + runtime, + payload, + accept, + extra_args=("--validate-enrichment", "--expected-enriched-tag=event.id"), + gzip_enabled=True, + keep_payloads=False, + read_only_parent=True, + ) + output = result.stdout + result.stderr + self.assertEqual(0, result.returncode, output) + self.assertIsNone(retained_payload) + self.assertEqual(1, len(records), records) + self.assertNotIn("validated enriched test payload", output) + + with self.subTest(runtime=runtime, result="valid-debug-logging"): result, records, retained_payload = self._run_local_test_upload( runtime, payload, accept, extra_args=("--validate-enrichment", "--expected-enriched-tag=event.id"), + debug=True, gzip_enabled=True, keep_payloads=False, read_only_parent=True, @@ -7426,10 +7517,14 @@ def test_runtime_fingerprint_alphabet_matches_sync(self) -> None: powershell_text = _runfile("tools/core/uploader_powershell_runtime.ps1.tpl").read_text( encoding="utf-8" ) + python_text = _runfile("tools/core/uploader_py/credentials.py").read_text( + encoding="utf-8" + ) expected = self._extract_starlark_fingerprint_alphabet(sync_text) self.assertEqual(expected, self._extract_bash_fingerprint_alphabet(bash_text)) self.assertEqual(expected, self._extract_powershell_fingerprint_alphabet(powershell_text)) + self.assertEqual(expected, self._extract_python_fingerprint_alphabet(python_text)) def test_runtime_unknown_char_bucketing_matches_sync(self) -> None: """Validate runtime unknown char bucketing matches sync behavior.""" @@ -7437,8 +7532,12 @@ def test_runtime_unknown_char_bucketing_matches_sync(self) -> None: powershell_text = _runfile("tools/core/uploader_powershell_runtime.ps1.tpl").read_text( encoding="utf-8" ) + python_text = _runfile("tools/core/uploader_py/credentials.py").read_text( + encoding="utf-8" + ) self.assertIn("idx=$((alpha_len + (i % 7)))", bash_text) self.assertIn("$idx = $alphabet.Length + ($i % 7)", powershell_text) + self.assertIn("alphabet_length + (index % 7)", python_text) def test_bash_jq_avoids_reserved_label_variable(self) -> None: """Validate jq programs remain compatible with versions reserving `label`.""" @@ -7883,7 +7982,16 @@ def test_uploader_templates_declare_bep_artifact_helper_runfiles(self) -> None: self.assertIn("_doctor_runtime", rule_text) self.assertIn("bep_artifact_stage_helper.py", rule_text) self.assertIn("test_optimization_doctor.py", rule_text) - self.assertIn("files = depset([bash_file, ps_file, bat_file])", rule_text) + self.assertIn("files = depset([", rule_text) + for output_name in ( + "bash_file", + "ps_file", + "bat_file", + "python_bash_file", + "python_ps_file", + "python_bat_file", + ): + self.assertIn(output_name, rule_text) self.assertIn("--artifact-source=local|bep|auto", rule_text) self.assertIn("DD_TEST_OPTIMIZATION_ARTIFACT_SOURCE", rule_text) self.assertIn("--remote-artifacts=disabled|download|required", rule_text) diff --git a/tools/tests/python/test_uploader_application_tools.py b/tools/tests/python/test_uploader_application_tools.py new file mode 100644 index 00000000..668961e0 --- /dev/null +++ b/tools/tests/python/test_uploader_application_tools.py @@ -0,0 +1,682 @@ +#!/usr/bin/env python3 +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Exercise the whole uploader lifecycle without backend access. + +These tests isolate preflight, cleanup, and reporting regressions from HTTP behavior. +""" + +from __future__ import annotations + +from io import StringIO +import json +from pathlib import Path +import tempfile +import unittest +from unittest import mock + +from uploader_test_support import ( + add_uploader_runtime_to_path, + resolve_runfile as _runfile, +) + +add_uploader_runtime_to_path() + +from topt_runtime.runfiles import RunfilesResolver # noqa: E402 +from uploader_py.application import run_uploader # noqa: E402 +from uploader_py.config import parse_uploader_config # noqa: E402 +from uploader_py.endpoints import build_endpoints # noqa: E402 +from uploader_py.freshness import FreshnessError # noqa: E402 +from uploader_py.logging_utils import configure_logging # noqa: E402 +from uploader_py.reporting import AggregateReport # noqa: E402 + + +def _write_payload(output: Path, kind: str, name: str, body: object) -> Path: + path = output / "payloads" / kind / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(body), encoding="utf-8") + return path + + +class ApplicationTests(unittest.TestCase): + def _config( + self, + root: Path, + *, + dry_run: bool = True, + fail_on_error: bool = False, + expected_targets: tuple[str, ...] = (), + runtime_selection: bool = False, + allow_cached_payload_uploads: bool = True, + extra_arguments: tuple[str, ...] = (), + extra_environment: dict[str, str] | None = None, + ): + config_path = root / "uploader-config.json" + config_path.write_text( + json.dumps( + { + "schema_version": 1, + "quiescent_sec": 0, + "max_wait_sec": 0, + "fail_on_error": fail_on_error, + "expected_targets": expected_targets, + "runtime_selection": runtime_selection, + "workers": 3, + "rules_version": "rules-test", + "uploader_version": "uploader-test", + "workspace_name": "workspace", + "doctor_runtime_path": str( + _runfile("tools/core/test_optimization_doctor.py") + ), + } + ), + encoding="utf-8", + ) + environment = { + "BUILD_WORKSPACE_DIRECTORY": str(root), + "TESTLOGS_DIR": str(root / "bazel-testlogs"), + } + environment.update(extra_environment or {}) + arguments = ["--config", str(config_path)] + if dry_run: + arguments.append("--dry-run") + if allow_cached_payload_uploads: + arguments.append("--allow-cached-payload-uploads") + arguments.extend(extra_arguments) + return parse_uploader_config( + arguments, + environ=environment, + cwd=root, + ) + + def test_dry_run_prepares_all_types_prints_stats_and_writes_schema_v1(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + output = root / "bazel-testlogs" / "pkg" / "target" / "test.outputs" + sources = ( + _write_payload( + output, + "tests", + "events.json", + {"events": [{"content": {"meta": {"event.id": "1"}}}]}, + ), + _write_payload( + output, + "coverage", + "coverage.json", + {"files": []}, + ), + _write_payload( + output, + "telemetry", + "telemetry.json", + { + "api_version": "v2", + "request_type": "app-started", + "runtime_id": "runtime", + "application": { + "service_name": "service", + "language_name": "python", + }, + }, + ), + ) + report_path = root / "report.json" + config = self._config( + root, + extra_arguments=( + f"--report-json={report_path}", + "--validate-enrichment", + "--expected-enriched-tag=event.id", + ), + ) + stream = StringIO() + log_stream = StringIO() + + exit_code = run_uploader( + config, + resolver=RunfilesResolver.from_environment(cwd=root, environ={}), + endpoints=build_endpoints(config), + logger=configure_logging(debug=True, stream=log_stream), + stream=stream, + ) + + self.assertEqual(0, exit_code) + self.assertTrue(all(source.is_file() for source in sources)) + self.assertIn("files: discovered=3", stream.getvalue()) + report = json.loads(report_path.read_text(encoding="utf-8")) + self.assertEqual(1, report["schema_version"]) + self.assertEqual(3, report["files"]["succeeded"]) + self.assertEqual(3, report["requests"]["planned"]) + self.assertEqual(0, report["requests"]["attempted"]) + self.assertEqual("upload_skipped_dry_run", report["result"]["reason_code"]) + self.assertEqual( + str(config.artifact_staging_dir), + report["artifacts"]["staging_dir"], + ) + self.assertIn("task=file-000001", log_stream.getvalue()) + self.assertIn("freshness filtering disabled", log_stream.getvalue()) + self.assertIn( + "dry-run validated enriched test payload", + log_stream.getvalue(), + ) + self.assertIn("dry-run validated 1 test payloads", log_stream.getvalue()) + + normal_log_stream = StringIO() + exit_code = run_uploader( + config, + resolver=RunfilesResolver.from_environment(cwd=root, environ={}), + endpoints=build_endpoints(config), + logger=configure_logging(debug=False, stream=normal_log_stream), + stream=StringIO(), + ) + + self.assertEqual(0, exit_code) + self.assertNotIn( + "dry-run validated enriched test payload", + normal_log_stream.getvalue(), + ) + self.assertIn( + "dry-run validated 1 test payloads", + normal_log_stream.getvalue(), + ) + + def test_success_statistics_report_full_invocation_elapsed_time(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + output = root / "bazel-testlogs" / "pkg" / "target" / "test.outputs" + _write_payload( + output, + "tests", + "events.json", + {"events": [{"content": {}}]}, + ) + config = self._config(root) + stream = StringIO() + clock = mock.Mock(side_effect=(100.0, 108.0, 110.0, 125.0)) + + exit_code = run_uploader( + config, + resolver=RunfilesResolver.from_environment(cwd=root, environ={}), + endpoints=build_endpoints(config), + logger=configure_logging(debug=False, stream=StringIO()), + stream=stream, + clock=clock, + ) + + self.assertEqual(0, exit_code) + self.assertIn("elapsed=25.00s", stream.getvalue()) + self.assertEqual(4, clock.call_count) + + def test_agentless_upload_without_payloads_does_not_require_api_key(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + (root / "bazel-testlogs").mkdir() + report_path = root / "report.json" + config = self._config( + root, + dry_run=False, + extra_arguments=(f"--report-json={report_path}",), + ) + transport_factory = mock.Mock( + side_effect=AssertionError("transport must not be created") + ) + + exit_code = run_uploader( + config, + resolver=RunfilesResolver.from_environment(cwd=root, environ={}), + endpoints=build_endpoints(config), + logger=configure_logging(debug=False, stream=StringIO()), + stream=StringIO(), + transport_factory=transport_factory, + ) + + self.assertEqual(0, exit_code) + transport_factory.assert_not_called() + report = json.loads(report_path.read_text(encoding="utf-8")) + self.assertEqual(0, report["files"]["discovered"]) + self.assertEqual(0, report["requests"]["attempted"]) + + def test_agentless_upload_with_payload_requires_api_key_before_workers(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + output = root / "bazel-testlogs" / "pkg" / "target" / "test.outputs" + _write_payload( + output, + "tests", + "events.json", + {"events": [{"content": {}}]}, + ) + report_path = root / "report.json" + config = self._config( + root, + dry_run=False, + extra_arguments=(f"--report-json={report_path}",), + ) + transport_factory = mock.Mock( + side_effect=AssertionError("transport must not be created") + ) + + exit_code = run_uploader( + config, + resolver=RunfilesResolver.from_environment(cwd=root, environ={}), + endpoints=build_endpoints(config), + logger=configure_logging(debug=False, stream=StringIO()), + stream=StringIO(), + transport_factory=transport_factory, + ) + + self.assertEqual(2, exit_code) + transport_factory.assert_not_called() + report = json.loads(report_path.read_text(encoding="utf-8")) + self.assertEqual(1, report["files"]["discovered"]) + self.assertEqual(0, report["files"]["processed"]) + self.assertEqual(0, report["requests"]["attempted"]) + + def test_fail_on_error_reports_tests_without_payloads(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + testlogs = root / "bazel-testlogs" + marker = testlogs / "pkg" / "target" / "test.log" + marker.parent.mkdir(parents=True) + marker.write_text("ran", encoding="utf-8") + report_path = root / "report.json" + config = self._config( + root, + fail_on_error=True, + extra_arguments=(f"--report-json={report_path}",), + ) + + exit_code = run_uploader( + config, + resolver=RunfilesResolver.from_environment(cwd=root, environ={}), + endpoints=build_endpoints(config), + logger=configure_logging(debug=False, stream=StringIO()), + stream=StringIO(), + ) + + self.assertEqual(1, exit_code) + report = json.loads(report_path.read_text(encoding="utf-8")) + self.assertEqual("tests_ran_without_payloads", report["result"]["reason_code"]) + self.assertEqual(0, report["files"]["processed"]) + + def test_fail_on_error_all_cached_bep_without_testlogs_is_noop(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + bep = _runfile("tools/tests/python/fixtures/bep_cached_local.ndjson") + report_path = root / "report.json" + config = self._config( + root, + dry_run=False, + fail_on_error=True, + expected_targets=("//pkg:target",), + allow_cached_payload_uploads=False, + extra_arguments=( + "--freshness-source=bep", + "--freshness-mode=required", + f"--bep-json={bep}", + f"--report-json={report_path}", + ), + ) + transport_factory = mock.Mock( + side_effect=AssertionError("transport must not be created") + ) + log_stream = StringIO() + + with mock.patch( + "uploader_py.application.resolve_local_testlogs_root", + return_value=None, + ): + exit_code = run_uploader( + config, + resolver=RunfilesResolver.from_environment(cwd=root, environ={}), + endpoints=build_endpoints(config), + logger=configure_logging(debug=False, stream=log_stream), + stream=StringIO(), + transport_factory=transport_factory, + ) + + self.assertEqual(0, exit_code, log_stream.getvalue()) + transport_factory.assert_not_called() + self.assertIn( + "freshness filtering enabled: source=bep", + log_stream.getvalue(), + ) + self.assertIn( + "skipping cached or non-current test output", + log_stream.getvalue(), + ) + report = json.loads(report_path.read_text(encoding="utf-8")) + self.assertEqual("ok", report["result"]["reason_code"]) + self.assertEqual(1, report["bep"]["cached_outputs"]) + self.assertEqual(0, report["files"]["discovered"]) + self.assertEqual(0, report["requests"]["attempted"]) + + def test_all_cached_bep_skips_wait_and_stale_test_markers(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + stale_marker = root / "bazel-testlogs" / "pkg" / "old" / "test.log" + stale_marker.parent.mkdir(parents=True) + stale_marker.write_text("stale prior invocation", encoding="utf-8") + bep = _runfile("tools/tests/python/fixtures/bep_cached_local.ndjson") + report_path = root / "report.json" + config = self._config( + root, + dry_run=False, + fail_on_error=True, + expected_targets=("//pkg:target",), + allow_cached_payload_uploads=False, + extra_arguments=( + "--freshness-source=bep", + "--freshness-mode=required", + f"--bep-json={bep}", + f"--report-json={report_path}", + ), + ) + + with mock.patch( + "uploader_py.application.wait_for_quiescence", + side_effect=AssertionError("cached-only plan must not wait"), + ): + exit_code = run_uploader( + config, + resolver=RunfilesResolver.from_environment(cwd=root, environ={}), + endpoints=build_endpoints(config), + logger=configure_logging(debug=False, stream=StringIO()), + stream=StringIO(), + transport_factory=mock.Mock( + side_effect=AssertionError("transport must not be created") + ), + ) + + self.assertEqual(0, exit_code) + report = json.loads(report_path.read_text(encoding="utf-8")) + self.assertEqual("ok", report["result"]["reason_code"]) + self.assertEqual(1, report["bep"]["cached_outputs"]) + self.assertEqual(0, report["files"]["discovered"]) + self.assertEqual(0, report["requests"]["attempted"]) + + def test_partial_cached_coverage_does_not_short_circuit_remote_output(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + bep = _runfile( + "tools/tests/python/fixtures/bep_snake_case_remote_cached.ndjson" + ) + report_path = root / "report.json" + config = self._config( + root, + dry_run=False, + fail_on_error=True, + expected_targets=("//pkg:target", "//pkg:remote_only"), + allow_cached_payload_uploads=False, + extra_arguments=( + "--freshness-source=bep", + "--freshness-mode=optional", + f"--bep-json={bep}", + f"--report-json={report_path}", + ), + ) + + with mock.patch( + "uploader_py.application.resolve_local_testlogs_root", + return_value=None, + ): + exit_code = run_uploader( + config, + resolver=RunfilesResolver.from_environment(cwd=root, environ={}), + endpoints=build_endpoints(config), + logger=configure_logging(debug=False, stream=StringIO()), + stream=StringIO(), + ) + + self.assertEqual(2, exit_code) + report = json.loads(report_path.read_text(encoding="utf-8")) + self.assertEqual( + "bep_output_remote_only_without_downloader", + report["result"]["reason_code"], + ) + self.assertEqual(1, report["bep"]["cached_outputs"]) + self.assertEqual(1, report["bep"]["remote_only_outputs"]) + self.assertEqual(0, report["requests"]["attempted"]) + + def test_remote_only_expected_output_fails_with_existing_empty_testlogs(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + (root / "bazel-testlogs").mkdir() + bep = _runfile( + "tools/tests/python/fixtures/bep_snake_case_remote_cached.ndjson" + ) + report_path = root / "report.json" + config = self._config( + root, + dry_run=False, + fail_on_error=True, + expected_targets=("//pkg:target", "//pkg:remote_only"), + allow_cached_payload_uploads=False, + extra_arguments=( + "--freshness-source=bep", + "--freshness-mode=optional", + f"--bep-json={bep}", + f"--report-json={report_path}", + ), + ) + + exit_code = run_uploader( + config, + resolver=RunfilesResolver.from_environment(cwd=root, environ={}), + endpoints=build_endpoints(config), + logger=configure_logging(debug=False, stream=StringIO()), + stream=StringIO(), + transport_factory=mock.Mock( + side_effect=AssertionError("transport must not be created") + ), + ) + + self.assertEqual(2, exit_code) + report = json.loads(report_path.read_text(encoding="utf-8")) + self.assertEqual( + "bep_output_remote_only_without_downloader", + report["result"]["reason_code"], + ) + self.assertEqual(1, report["bep"]["remote_only_outputs"]) + self.assertEqual(0, report["requests"]["attempted"]) + + def test_runtime_selection_is_validated_before_discovery(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + config = self._config(root, runtime_selection=True) + + with mock.patch( + "uploader_py.application.resolve_local_testlogs_root", + side_effect=AssertionError("discovery must not start"), + ): + exit_code = run_uploader( + config, + resolver=RunfilesResolver.from_environment(cwd=root, environ={}), + endpoints=build_endpoints(config), + logger=configure_logging(debug=False, stream=StringIO()), + stream=StringIO(), + ) + + self.assertEqual(2, exit_code) + + def test_interrupted_worker_report_is_printed_and_written(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + output = root / "bazel-testlogs" / "pkg" / "target" / "test.outputs" + source = _write_payload( + output, + "tests", + "events.json", + {"events": [{"content": {}}]}, + ) + report_path = root / "report.json" + config = self._config( + root, + extra_arguments=(f"--report-json={report_path}",), + ) + stream = StringIO() + + def interrupted_report(discovery, *, settings, **_kwargs): + report = AggregateReport.create( + dry_run=settings.dry_run, + exit_code=130, + configured_workers=settings.workers, + worker_threads=1, + peak_active_workers=1, + elapsed_seconds=0.1, + discovered_by_type=discovery.counts(), + results=(), + cancelled=len(discovery.tasks), + initialization_warning_codes=("invocation_interrupted",), + ) + return report + + with mock.patch( + "uploader_py.application.run_discovered_tasks", + side_effect=interrupted_report, + ): + exit_code = run_uploader( + config, + resolver=RunfilesResolver.from_environment(cwd=root, environ={}), + endpoints=build_endpoints(config), + logger=configure_logging(debug=False, stream=StringIO()), + stream=stream, + ) + + self.assertEqual(130, exit_code) + self.assertTrue(source.exists()) + self.assertIn("exit_code=130", stream.getvalue()) + self.assertIn("cancelled=1", stream.getvalue()) + report = json.loads(report_path.read_text(encoding="utf-8")) + self.assertEqual("interrupted", report["result"]["reason_code"]) + self.assertEqual(1, report["files"]["cancelled"]) + + def test_staging_cleanup_failure_preserves_completed_worker_statistics(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + output = root / "bazel-testlogs" / "pkg" / "target" / "test.outputs" + _write_payload( + output, + "tests", + "events.json", + {"events": [{"content": {}}]}, + ) + report_path = root / "report.json" + config = self._config( + root, + extra_arguments=(f"--report-json={report_path}",), + ) + from uploader_py import application + + real_prepare = application.prepare_freshness + + def preparation_with_failed_cleanup(*args, **kwargs): + prepared = real_prepare(*args, **kwargs) + return mock.Mock( + plan=prepared.plan, + scan_roots=prepared.scan_roots, + staged_roots=prepared.staged_roots, + cleanup=mock.Mock( + side_effect=FreshnessError("simulated staging cleanup failure") + ), + ) + + with mock.patch( + "uploader_py.application.prepare_freshness", + side_effect=preparation_with_failed_cleanup, + ): + exit_code = run_uploader( + config, + resolver=RunfilesResolver.from_environment(cwd=root, environ={}), + endpoints=build_endpoints(config), + logger=configure_logging(debug=False, stream=StringIO()), + stream=StringIO(), + ) + + self.assertEqual(2, exit_code) + report = json.loads(report_path.read_text(encoding="utf-8")) + self.assertEqual(1, report["files"]["processed"]) + self.assertEqual(1, report["files"]["succeeded"]) + self.assertEqual(1, report["requests"]["planned"]) + self.assertEqual("staging_cleanup_failed", report["result"]["reason_code"]) + self.assertEqual(1, report["warnings"]["staging_cleanup_failed"]) + + def test_workspace_lock_is_held_until_final_report_is_emitted(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + (root / "bazel-testlogs").mkdir() + config = self._config(root) + events: list[str] = [] + locks = [] + + class RecordingLock: + def __init__(self, workspace: str) -> None: + self.workspace = workspace + self.acquired = False + locks.append(self) + + def acquire(self): + self.acquired = True + events.append("acquire") + return self + + def release(self) -> None: + events.append("release") + self.acquired = False + + def record_report(*_args, **_kwargs) -> None: + self.assertTrue(locks[0].acquired) + events.append("report") + + with mock.patch( + "uploader_py.application.WorkspaceLock", + RecordingLock, + ), mock.patch( + "uploader_py.application.emit_report", + side_effect=record_report, + ): + exit_code = run_uploader( + config, + resolver=RunfilesResolver.from_environment(cwd=root, environ={}), + endpoints=build_endpoints(config), + logger=configure_logging(debug=False, stream=StringIO()), + stream=StringIO(), + ) + + self.assertEqual(0, exit_code) + self.assertEqual(config.lock_workspace, locks[0].workspace) + self.assertEqual(["acquire", "report", "release"], events) + + events.clear() + + def fail_report(*_args, **_kwargs) -> None: + self.assertTrue(locks[-1].acquired) + events.append("report") + raise RuntimeError("report stream failed") + + with mock.patch( + "uploader_py.application.WorkspaceLock", + RecordingLock, + ), mock.patch( + "uploader_py.application.emit_report", + side_effect=fail_report, + ), self.assertRaisesRegex(RuntimeError, "report stream failed"): + run_uploader( + config, + resolver=RunfilesResolver.from_environment(cwd=root, environ={}), + endpoints=build_endpoints(config), + logger=configure_logging(debug=False, stream=StringIO()), + stream=StringIO(), + ) + + self.assertFalse(locks[-1].acquired) + self.assertEqual(["acquire", "report", "release"], events) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/tests/python/test_uploader_codeowners_tools.py b/tools/tests/python/test_uploader_codeowners_tools.py new file mode 100644 index 00000000..55d2772d --- /dev/null +++ b/tools/tests/python/test_uploader_codeowners_tools.py @@ -0,0 +1,364 @@ +#!/usr/bin/env python3 +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Characterize the invocation-wide Python CODEOWNERS matcher. + +Parity and concurrent-read coverage keep the shared matcher safe for all workers. +""" + +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor +from dataclasses import FrozenInstanceError +from pathlib import Path +import tempfile +import unittest + +from uploader_test_support import add_uploader_runtime_to_path + +add_uploader_runtime_to_path() + +from uploader_py.codeowners import ( # noqa: E402 + CodeOwnersMatcher, + enrich_payload_codeowners, + load_codeowners_matcher, + normalize_path_like, + parse_codeowners, + source_candidates, +) + + +CODEOWNERS_FIXTURE = r""" +[CoreTeam] +[Core Team] @org/section-space +* @org/default +[xy] @org/class-owner +[abc] @org/class-owner-abc +[A1B2C3] @org/class-owner-alnum-long +[ABCD] @org/class-owner-upper-long +[ABC] @org/class-owner-upper +[Abc] @org/class-owner-mixed +[Backend] @org/section-default +/manual/owned.cs @org/owned +/manual/unowned.cs +/manual/comment_only.cs # explicit empty-owner rule via inline comment +/manual/hash_owner.cs @org/team#chat +/manual/space\ owner.cs @org/space-owner +/manual/dir/ @org/dir-owner +/manual/literal\*.cs @org/literal-star +/manual/literal\?.cs @org/literal-question +/manual/literal\[ab\].cs @org/literal-brackets +/manual/duplicate_owners.cs @org/dedupe @org/dedupe @org/extra +/manual/last_match.cs @org/first +/manual/last_match.cs @org/second +/manual/override_empty.cs @org/will-be-overridden +/manual/override_empty.cs +/manual/file_scheme.cs @org/file-scheme +/manual/percent_slash.cs @org/percent-slash +/manual/dotnorm.cs @org/dotnorm +/external/local/file.cs @org/repo-external +/tracer/test/test-applications/integrations/Samples.XUnitTests/[Tt]estSuite.cs @DataDog/ci-app-libraries-dotnet +/manual/[z-a].cs @org/invalid-range +/manual/tab_sep.cs @org/tab-owner +""".lstrip() + + +def _fixture_matcher( + *, + workspace_root: str = "/workspace", + context_workspace: str = "", + windows_paths: bool = False, +) -> CodeOwnersMatcher: + rules, warnings = parse_codeowners(CODEOWNERS_FIXTURE) + return CodeOwnersMatcher( + source_path=Path("CODEOWNERS"), + workspace_root=workspace_root, + context_workspace=context_workspace, + rules=rules, + warnings=warnings, + windows_paths=windows_paths, + ) + + +class CodeOwnersMatcherTests(unittest.TestCase): + def test_existing_integration_fixture_cases_match_python(self) -> None: + matcher = _fixture_matcher() + checks = ( + ("manual/owned.cs", ("@org/owned",)), + ("manual/unowned.cs", ()), + ("manual/comment_only.cs", ()), + ("manual/hash_owner.cs", ("@org/team#chat",)), + ("manual/space owner.cs", ("@org/space-owner",)), + ("manual/dir/sub/file.cs", ("@org/dir-owner",)), + ("manual/literal*.cs", ("@org/literal-star",)), + ("manual/literal?.cs", ("@org/literal-question",)), + ("manual/literal[ab].cs", ("@org/literal-brackets",)), + ("manual/duplicate_owners.cs", ("@org/dedupe", "@org/extra")), + ("manual/last_match.cs", ("@org/second",)), + ("manual/override_empty.cs", ()), + ("file://manual/file_scheme.cs", ("@org/file-scheme",)), + ("manual%2Fpercent_slash.cs", ("@org/percent-slash",)), + ("./manual/sub/../dotnorm.cs", ("@org/dotnorm",)), + ("../manual/owned.cs", None), + ("manual%00owned.cs", ("@org/default",)), + ("manual%2Gbad.cs", ("@org/default",)), + ("/tmp/mock.runfiles/_main/manual/owned.cs", ("@org/owned",)), + ("/tmp/mock.runfiles/_main/external/rules_go/pkg/file.go", None), + ("/tmp/execroot/mock_ws/_main/manual/owned.cs", ("@org/owned",)), + ("manual/tab_sep.cs", ("@org/tab-owner",)), + ("x", ("@org/class-owner",)), + ("a", ("@org/class-owner-abc",)), + ("B", ("@org/class-owner-upper",)), + ("D", ("@org/class-owner-upper-long",)), + ("2", ("@org/class-owner-alnum-long",)), + ("b", ("@org/class-owner-mixed",)), + ("[Core", ("@org/default",)), + ("manual/invalid_range.cs", ("@org/default",)), + ("manual%5Cowned.cs", ("@org/owned",)), + ("/tmp/not-in-workspace/manual_external.cs", None), + ("/tmp/execroot/mock_ws/external/rules_go/pkg/file.go", None), + ("external/local/file.cs", ("@org/repo-external",)), + ( + "tracer/test/test-applications/integrations/" + "Samples.XUnitTests/TestSuite.cs", + ("@DataDog/ci-app-libraries-dotnet",), + ), + ("manual/z/file.cs", ("@org/default",)), + ) + for source, expected in checks: + with self.subTest(source=source): + match = matcher.match_source(source) + if expected is None: + self.assertFalse(match.matched) + else: + self.assertTrue(match.matched) + self.assertEqual(expected, match.owners) + self.assertEqual(1, len(matcher.warnings)) + self.assertIn("invalid CODEOWNERS rule", matcher.warnings[0]) + + def test_workspace_stripping_derived_paths_and_generated_paths(self) -> None: + self.assertEqual( + ("src/pkg/file.go",), + source_candidates( + "/workspace/src/pkg/file.go", + workspace_root="/workspace", + ), + ) + self.assertEqual( + ("manual/owned.cs", "_main/manual/owned.cs"), + source_candidates( + "/tmp/execroot/ws/_main/manual/owned.cs", + workspace_root="/workspace", + ), + ) + self.assertEqual( + (), + source_candidates( + "/tmp/execroot/ws/external/repo/file.go", + workspace_root="/workspace", + ), + ) + self.assertEqual( + (), + source_candidates("bazel-out/darwin-fastbuild/bin/generated.go"), + ) + self.assertEqual( + ("Src/File.cs",), + source_candidates( + r"c:\repo\Src\File.cs", + workspace_root="C:/Repo", + windows_paths=True, + ), + ) + + def test_normalization_is_safe_and_deterministic(self) -> None: + self.assertEqual("manual/owned.cs", normalize_path_like("manual%5Cowned.cs")) + self.assertEqual("C:/repo/file.cs", normalize_path_like("file:///C:/repo/file.cs")) + self.assertIsNone(normalize_path_like("../outside.cs")) + self.assertEqual("manual%00owned.cs", normalize_path_like("manual%00owned.cs")) + + def test_double_star_character_classes_and_last_match_wins(self) -> None: + rules, warnings = parse_codeowners( + "docs/**/test?.py @docs\n" + "docs/private/** @private\n" + "docs/private/generated/**\n" + "[!a-z] @upper\n" + ) + matcher = CodeOwnersMatcher(None, "", "", rules, warnings, False) + self.assertEqual(("@docs",), matcher.match_source("docs/test1.py").owners) + self.assertEqual(("@docs",), matcher.match_source("docs/a/test2.py").owners) + self.assertEqual(("@private",), matcher.match_source("docs/private/a.txt").owners) + self.assertEqual((), matcher.match_source("docs/private/generated/a.txt").owners) + self.assertEqual(("@upper",), matcher.match_source("Z").owners) + + def test_rules_and_matcher_are_immutable(self) -> None: + matcher = _fixture_matcher() + with self.assertRaises(FrozenInstanceError): + matcher.workspace_root = "changed" # type: ignore[misc] + with self.assertRaises(FrozenInstanceError): + matcher.rules[0].owners = ("changed",) # type: ignore[misc] + + +class CodeOwnersDiscoveryTests(unittest.TestCase): + def test_discovery_precedence_and_explicit_fallback(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + context = root / "context" + workspace = root / "workspace" + cwd = root / "cwd" + launcher = root / "launcher" + for directory in (context / ".github", workspace, cwd, launcher): + directory.mkdir(parents=True, exist_ok=True) + context_file = context / ".github" / "CODEOWNERS" + workspace_file = workspace / "CODEOWNERS" + cwd_file = cwd / "CODEOWNERS" + launcher_file = launcher / "CODEOWNERS" + context_file.write_text("* @context\n", encoding="utf-8") + workspace_file.write_text("* @workspace\n", encoding="utf-8") + cwd_file.write_text("* @cwd\n", encoding="utf-8") + launcher_file.write_text("* @launcher\n", encoding="utf-8") + + discovered = load_codeowners_matcher( + explicit_path=root / "missing-explicit", + workspace_root=workspace, + context_workspace=str(context), + cwd=cwd, + launcher_directory=launcher, + windows_paths=False, + ) + self.assertEqual(context_file.resolve(), discovered.source_path) + self.assertEqual(("@context",), discovered.match_source("any/file").owners) + + explicit = load_codeowners_matcher( + explicit_path=workspace_file, + workspace_root=workspace, + context_workspace=str(context), + cwd=cwd, + launcher_directory=launcher, + windows_paths=False, + ) + self.assertEqual(workspace_file.resolve(), explicit.source_path) + self.assertEqual(("@workspace",), explicit.match_source("any/file").owners) + + context_file.unlink() + workspace_file.unlink() + from_cwd = load_codeowners_matcher( + explicit_path=None, + workspace_root=workspace, + context_workspace=str(context), + cwd=cwd, + launcher_directory=launcher, + windows_paths=False, + ) + self.assertEqual(cwd_file.resolve(), from_cwd.source_path) + + cwd_file.unlink() + from_launcher = load_codeowners_matcher( + explicit_path=None, + workspace_root=workspace, + context_workspace=str(context), + cwd=cwd, + launcher_directory=launcher, + windows_paths=False, + ) + self.assertEqual(launcher_file.resolve(), from_launcher.source_path) + + def test_missing_or_unreadable_file_is_best_effort(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + matcher = load_codeowners_matcher( + explicit_path=root / "missing", + workspace_root=root, + cwd=root, + windows_paths=False, + ) + self.assertFalse(matcher.enabled) + self.assertIsNone(matcher.source_path) + + def test_matcher_is_loaded_once_then_shared_for_concurrent_reads(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + codeowners = root / "CODEOWNERS" + codeowners.write_text(CODEOWNERS_FIXTURE, encoding="utf-8") + matcher = load_codeowners_matcher( + explicit_path=codeowners, + workspace_root=root, + cwd=root, + windows_paths=False, + ) + codeowners.unlink() + sources = ["manual/owned.cs", "manual/default.cs"] * 100 + with ThreadPoolExecutor(max_workers=12) as executor: + results = list(executor.map(matcher.match_source, sources)) + + self.assertEqual(("@org/owned",), results[0].owners) + self.assertEqual(("@org/default",), results[1].owners) + self.assertTrue(all(result.matched for result in results)) + + +class CodeOwnersEnrichmentTests(unittest.TestCase): + def test_event_enrichment_preserves_existing_and_skips_spans(self) -> None: + matcher = _fixture_matcher() + payload = { + "events": [ + { + "type": "test", + "content": { + "meta": { + "test.source.file": "manual/owned.cs", + "test.codeowners": '["@existing"]', + } + }, + }, + { + "type": "span", + "content": {"meta": {"test.source.file": "manual/owned.cs"}}, + }, + { + "type": "test_module_end", + "content": {"source": {"path": "manual/owned.cs"}}, + }, + { + "type": "test", + "content": {"meta": {"test.source.path": "manual/unowned.cs"}}, + }, + {"type": "test", "content": {"meta": {}}}, + ] + } + cache = {} + stats = enrich_payload_codeowners(payload, matcher, cache=cache) + + events = payload["events"] + self.assertEqual('["@existing"]', events[0]["content"]["meta"]["test.codeowners"]) + self.assertNotIn("test.codeowners", events[1]["content"]["meta"]) + self.assertEqual( + '["@org/owned"]', + events[2]["content"]["meta"]["test.codeowners"], + ) + self.assertNotIn("test.codeowners", events[3]["content"]["meta"]) + self.assertEqual(4, stats.scanned) + self.assertEqual(1, stats.enriched) + self.assertEqual(1, stats.skipped_existing) + self.assertEqual(1, stats.skipped_missing_source) + self.assertEqual(1, stats.skipped_unmatched) + self.assertEqual(2, len(cache)) + + def test_disabled_matcher_is_a_noop(self) -> None: + payload = {"events": [{"type": "test", "content": {"meta": {}}}]} + stats = enrich_payload_codeowners( + payload, + CodeOwnersMatcher(None, "", ""), + ) + self.assertEqual(0, stats.scanned) + self.assertEqual( + {"events": [{"type": "test", "content": {"meta": {}}}]}, + payload, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/tests/python/test_uploader_coordinator_tools.py b/tools/tests/python/test_uploader_coordinator_tools.py new file mode 100644 index 00000000..957dbbb6 --- /dev/null +++ b/tools/tests/python/test_uploader_coordinator_tools.py @@ -0,0 +1,767 @@ +#!/usr/bin/env python3 +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Exercise mixed-protocol coordination over the real worker pool. + +These tests prove shared inputs and aggregation work across every payload type. +""" + +from __future__ import annotations + +from contextlib import contextmanager +from io import StringIO +import json +from pathlib import Path +import tempfile +import threading +import time +import unittest +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from unittest import mock + +from uploader_test_support import add_uploader_runtime_to_path + +add_uploader_runtime_to_path() + +from uploader_py.codeowners import load_codeowners_matcher # noqa: E402 +from uploader_py.coordinator import ( # noqa: E402 + CoordinatorSettings, + run_discovered_tasks, +) +from uploader_py.credentials import ( # noqa: E402 + api_key_fingerprint, + check_api_key_fingerprint, +) +from uploader_py.discovery import ScanRoot, discover_file_tasks # noqa: E402 +from uploader_py.endpoints import EndpointSet # noqa: E402 +from uploader_py.enrichment import ContextPlan, ContextRecord # noqa: E402 +from uploader_py.logging_utils import configure_logging # noqa: E402 +from uploader_py.models import FileResult, FileStatus # noqa: E402 +from uploader_py.reporting import emit_report # noqa: E402 +from uploader_py.resources import LoadedResources # noqa: E402 +from uploader_py.transport import ( # noqa: E402 + HttpResult, + HttpTransport, + HttpTransportError, +) +from uploader_py.worker_pool import WorkerPoolInterrupted, WorkerPoolRun # noqa: E402 + + +class _MixedTransport: + def __init__( + self, + barrier: threading.Barrier | None, + records: list[dict[str, object]], + lock: threading.Lock, + ) -> None: + self.barrier = barrier + self.records = records + self.lock = lock + + def _record(self, kind: str, body: bytes) -> HttpResult: + if self.barrier is not None: + self.barrier.wait(timeout=2) + with self.lock: + self.records.append({"kind": kind, "body": body}) + return HttpResult(200, 1) + + def post_json(self, _url, _headers, body, *, gzip_body=False, content_encoding=None): + raw = Path(body).read_bytes() + return self._record("json", raw) + + def post_prepared_multipart(self, _url, _headers, prepared): + return self._record("multipart", prepared.path.read_bytes()) + + +def _write_payload(output: Path, kind: str, name: str, value: object) -> Path: + path = output / "payloads" / kind / name + path.parent.mkdir(parents=True, exist_ok=True) + if isinstance(value, bytes): + path.write_bytes(value) + else: + path.write_text(json.dumps(value), encoding="utf-8") + return path + + +class CoordinatorTests(unittest.TestCase): + def test_invalid_proxy_is_rejected_before_dry_run_workers_start(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + with self.assertRaisesRegex( + HttpTransportError, + "invalid HTTP proxy configuration", + ): + run_discovered_tasks( + discover_file_tasks(()), + settings=CoordinatorSettings( + workspace=root, + workers=1, + dry_run=True, + validate_enrichment=False, + expected_enriched_tags=(), + gzip_payloads=False, + keep_payloads=False, + filter_prefix=False, + rules_version="rules-1", + uploader_version="uploader-1", + api_key="", + proxy_environment=( + ("http_proxy", "http://localhost:notaport"), + ), + ), + endpoints=EndpointSet( + True, + "datadoghq.com", + "https://test.invalid", + "https://coverage.invalid", + "https://telemetry.invalid", + ), + resources=LoadedResources( + ContextPlan(None), + {}, + None, + (), + None, + ), + ) + + def test_coordinator_passes_cwd_and_launcher_codeowners_fallbacks(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + workspace = root / "workspace" + invocation_cwd = root / "cwd" + launcher_directory = root / "launcher" + for directory in (workspace, invocation_cwd, launcher_directory): + directory.mkdir() + launcher_codeowners = launcher_directory / "CODEOWNERS" + launcher_codeowners.write_text("* @launcher\n", encoding="utf-8") + captured = [] + + def capture_matcher(**kwargs): + matcher = load_codeowners_matcher(**kwargs) + captured.append(matcher) + return matcher + + with mock.patch( + "uploader_py.coordinator.load_codeowners_matcher", + side_effect=capture_matcher, + ): + report = run_discovered_tasks( + discover_file_tasks(()), + settings=CoordinatorSettings( + workspace=workspace, + workers=1, + dry_run=True, + validate_enrichment=False, + expected_enriched_tags=(), + gzip_payloads=False, + keep_payloads=False, + filter_prefix=False, + rules_version="rules-1", + uploader_version="uploader-1", + api_key="", + invocation_cwd=invocation_cwd, + launcher_directory=launcher_directory, + ), + endpoints=EndpointSet( + True, + "datadoghq.com", + "https://test", + "https://coverage", + "https://telemetry", + ), + resources=LoadedResources( + ContextPlan(None), + {}, + None, + (), + None, + ), + identifier_factory=lambda: "stable-id", + ) + + self.assertEqual(0, report.exit_code) + self.assertEqual(1, len(captured)) + self.assertEqual(launcher_codeowners.resolve(), captured[0].source_path) + + def test_api_key_fingerprint_matches_sync_contract_and_warns_once(self) -> None: + self.assertEqual("c1a2b2aa", api_key_fingerprint("abc")) + self.assertEqual("43d28057", api_key_fingerprint("a&b")) + self.assertEqual( + "match", + check_api_key_fingerprint( + {"topt.api_key_fingerprint": api_key_fingerprint("secret")}, + api_key="secret", + agentless=True, + ).status, + ) + self.assertEqual( + "api_key_fingerprint_mismatch", + check_api_key_fingerprint( + {"topt.api_key_fingerprint": api_key_fingerprint("other")}, + api_key="secret", + agentless=True, + ).warning_code, + ) + self.assertEqual( + "api_key_fingerprint_evp_skipped", + check_api_key_fingerprint( + {"topt.api_key_fingerprint": api_key_fingerprint("other")}, + api_key="", + agentless=False, + ).warning_code, + ) + + def test_pre_worker_fingerprint_mismatch_is_warning_only_and_redacted(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + context = ContextRecord.create( + "repo", + {"topt.api_key_fingerprint": api_key_fingerprint("different-key")}, + ) + resources = LoadedResources( + ContextPlan(context, (context,)), + dict(context.values), + None, + (), + None, + ) + stream = StringIO() + secret = "upload-secret" + report = run_discovered_tasks( + discover_file_tasks(()), + settings=CoordinatorSettings( + workspace=root, + workers=2, + dry_run=True, + validate_enrichment=False, + expected_enriched_tags=(), + gzip_payloads=False, + keep_payloads=False, + filter_prefix=False, + rules_version="rules-1", + uploader_version="uploader-1", + api_key=secret, + ), + endpoints=EndpointSet( + True, + "datadoghq.com", + "https://test", + "https://coverage", + "https://telemetry", + ), + resources=resources, + logger=configure_logging(debug=True, secrets=(secret,), stream=stream), + identifier_factory=lambda: "stable-id", + ) + + self.assertEqual(0, report.exit_code) + self.assertIn( + "api_key_fingerprint_mismatch", + report.initialization_warning_codes, + ) + self.assertIn("DD_API_KEY mismatch", stream.getvalue()) + self.assertNotIn(secret, stream.getvalue()) + self.assertNotIn("different-key", stream.getvalue()) + + def test_interrupt_aggregates_completed_results_and_cancelled_files(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + output = root / "bazel-testlogs" / "pkg" / "test.outputs" + _write_payload(output, "tests", "one.json", {"events": [{"content": {}}]}) + _write_payload(output, "tests", "two.json", {"events": [{"content": {}}]}) + discovery = discover_file_tasks((ScanRoot(root / "bazel-testlogs"),)) + completed = FileResult( + task_id=discovery.tasks[0].task_id, + source_path=discovery.tasks[0].display_path, + payload_type=discovery.tasks[0].payload_type, + status=FileStatus.SUCCEEDED, + requests_planned=1, + ) + interrupted = WorkerPoolInterrupted( + WorkerPoolRun((completed,), 2, 1), + cancelled=1, + ) + + with mock.patch( + "uploader_py.coordinator.run_file_workers", + side_effect=interrupted, + ): + report = run_discovered_tasks( + discovery, + settings=CoordinatorSettings( + workspace=root, + workers=2, + dry_run=True, + validate_enrichment=False, + expected_enriched_tags=(), + gzip_payloads=False, + keep_payloads=False, + filter_prefix=False, + rules_version="rules-1", + uploader_version="uploader-1", + api_key="", + ), + endpoints=EndpointSet( + True, + "datadoghq.com", + "https://test", + "https://coverage", + "https://telemetry", + ), + resources=LoadedResources(ContextPlan(None), None, None, (), None), + identifier_factory=lambda: "stable-id", + ) + + stats = report.statistics() + self.assertEqual(130, report.exit_code) + self.assertEqual(1, stats["files"]["processed"]) + self.assertEqual(1, stats["files"]["cancelled"]) + self.assertEqual(2, stats["files"]["eligible"]) + self.assertIn( + "invocation_interrupted", + report.initialization_warning_codes, + ) + stream = StringIO() + emit_report(report, stream=stream) + self.assertIn("exit_code=130", stream.getvalue()) + self.assertIn("cancelled=1", stream.getvalue()) + + def test_invocation_temporary_cleanup_failure_is_warning_only(self) -> None: + @contextmanager + def cleanup_failure(*, on_cleanup_error): + with tempfile.TemporaryDirectory() as raw_root: + yield Path(raw_root) + on_cleanup_error("PermissionError") + + with tempfile.TemporaryDirectory() as raw_root, mock.patch( + "uploader_py.coordinator.invocation_temporary_directory", + cleanup_failure, + ): + root = Path(raw_root) + report = run_discovered_tasks( + discover_file_tasks(()), + settings=CoordinatorSettings( + workspace=root, + workers=2, + dry_run=True, + validate_enrichment=False, + expected_enriched_tags=(), + gzip_payloads=False, + keep_payloads=False, + filter_prefix=False, + rules_version="rules-1", + uploader_version="uploader-1", + api_key="", + ), + endpoints=EndpointSet( + True, + "datadoghq.com", + "https://test", + "https://coverage", + "https://telemetry", + ), + resources=LoadedResources(ContextPlan(None), None, None, (), None), + ) + + self.assertEqual(0, report.exit_code) + self.assertIn( + "invocation_temp_cleanup_failed", + report.initialization_warning_codes, + ) + + def test_workers_one_uploads_all_protocols_through_real_http(self) -> None: + class Handler(BaseHTTPRequestHandler): + records: list[dict[str, object]] = [] + + def do_POST(self): # noqa: N802 + length = int(self.headers.get("Content-Length", "0")) + body = self.rfile.read(length) + type(self).records.append( + { + "path": self.path, + "content_type": self.headers.get("Content-Type", ""), + "body": body, + } + ) + self.send_response(202) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, _format, *_args): + return + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + server.daemon_threads = True + server_thread = threading.Thread(target=server.serve_forever, daemon=True) + server_thread.start() + try: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + output = root / "bazel-testlogs" / "pkg" / "target" / "test.outputs" + sources = ( + _write_payload( + output, + "tests", + "events.json", + {"events": [{"content": {"name": "test"}}]}, + ), + _write_payload( + output, + "coverage", + "coverage.json", + {"files": []}, + ), + _write_payload( + output, + "telemetry", + "telemetry.json", + { + "api_version": "v2", + "request_type": "app-started", + "runtime_id": "runtime-a", + "application": { + "service_name": "service-a", + "language_name": "python", + }, + }, + ), + ) + discovery = discover_file_tasks((ScanRoot(root / "bazel-testlogs"),)) + resources = LoadedResources(ContextPlan(None), None, None, (), None) + base = f"http://127.0.0.1:{server.server_port}" + settings = CoordinatorSettings( + workspace=root, + workers=1, + dry_run=False, + validate_enrichment=False, + expected_enriched_tags=(), + gzip_payloads=False, + keep_payloads=False, + filter_prefix=False, + rules_version="rules-1", + uploader_version="uploader-1", + api_key="secret", + ) + + report = run_discovered_tasks( + discovery, + settings=settings, + endpoints=EndpointSet( + True, + "datadoghq.com", + f"{base}/tests", + f"{base}/coverage", + f"{base}/telemetry", + ), + resources=resources, + transport_factory=lambda: HttpTransport(max_attempts=1), + identifier_factory=lambda: "stable-id", + ) + + stats = report.statistics() + self.assertEqual(0, report.exit_code) + self.assertEqual(1, stats["concurrency"]["worker_threads"]) + self.assertEqual(1, stats["concurrency"]["peak_active_workers"]) + self.assertEqual(3, stats["files"]["succeeded"]) + self.assertEqual(3, stats["requests"]["attempted"]) + self.assertEqual( + ["/coverage", "/telemetry", "/tests"], + sorted(record["path"] for record in Handler.records), + ) + self.assertTrue( + any( + str(record["content_type"]).startswith("multipart/form-data") + for record in Handler.records + ) + ) + self.assertTrue(all(not source.exists() for source in sources)) + finally: + server.shutdown() + server.server_close() + server_thread.join(timeout=2) + + def test_real_http_requests_overlap_without_exceeding_worker_bound(self) -> None: + class Handler(BaseHTTPRequestHandler): + active = 0 + peak = 0 + calls = 0 + lock = threading.Lock() + + def do_POST(self): # noqa: N802 + length = int(self.headers.get("Content-Length", "0")) + self.rfile.read(length) + with self.lock: + type(self).active += 1 + type(self).peak = max(type(self).peak, type(self).active) + time.sleep(0.05) + with self.lock: + type(self).active -= 1 + type(self).calls += 1 + self.send_response(200) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, _format, *_args): + return + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + server.daemon_threads = True + server_thread = threading.Thread(target=server.serve_forever, daemon=True) + server_thread.start() + try: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + output = root / "bazel-testlogs" / "pkg" / "target" / "test.outputs" + sources = tuple( + _write_payload( + output, + "tests", + f"events-{index}.json", + {"events": [{"content": {"id": index}}]}, + ) + for index in range(8) + ) + discovery = discover_file_tasks((ScanRoot(root / "bazel-testlogs"),)) + resources = LoadedResources(ContextPlan(None), None, None, (), None) + url = f"http://127.0.0.1:{server.server_port}/upload" + settings = CoordinatorSettings( + workspace=root, + workers=4, + dry_run=False, + validate_enrichment=False, + expected_enriched_tags=(), + gzip_payloads=False, + keep_payloads=False, + filter_prefix=False, + rules_version="rules-1", + uploader_version="uploader-1", + api_key="secret", + ) + + report = run_discovered_tasks( + discovery, + settings=settings, + endpoints=EndpointSet(True, "datadoghq.com", url, url, url), + resources=resources, + transport_factory=HttpTransport, + ) + + self.assertEqual(0, report.exit_code) + self.assertEqual(8, Handler.calls) + self.assertGreater(Handler.peak, 1) + self.assertLessEqual(Handler.peak, 4) + self.assertEqual(4, report.peak_active_workers) + self.assertTrue(all(not source.exists() for source in sources)) + finally: + server.shutdown() + server.server_close() + server_thread.join(timeout=2) + + def test_three_protocols_run_concurrently_and_share_prebuilt_resources(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + output = root / "bazel-testlogs" / "pkg" / "target" / "test.outputs" + test_source = _write_payload( + output, + "tests", + "events.json", + { + "events": [ + {"content": {"meta": {"test.source.file": "src/a.py"}}} + ] + }, + ) + coverage_source = _write_payload( + output, + "coverage", + "coverage.json", + {"files": []}, + ) + telemetry_source = _write_payload( + output, + "telemetry", + "telemetry.json", + { + "api_version": "v2", + "request_type": "app-started", + "runtime_id": "runtime-a", + "application": { + "service_name": "service-a", + "language_name": "python", + }, + }, + ) + (root / "CODEOWNERS").write_text("src/*.py @team\n", encoding="utf-8") + discovery = discover_file_tasks((ScanRoot(root / "bazel-testlogs"),)) + context = ContextRecord.create("repo", {"git.commit.sha": "abc"}) + resources = LoadedResources( + context_plan=ContextPlan(context, (context,)), + primary_context=dict(context.values), + primary_context_path=None, + telemetry_facts_paths=(), + schema=None, + ) + records: list[dict[str, object]] = [] + record_lock = threading.Lock() + barrier = threading.Barrier(3) + settings = CoordinatorSettings( + workspace=root, + workers=3, + dry_run=False, + validate_enrichment=False, + expected_enriched_tags=(), + gzip_payloads=False, + keep_payloads=False, + filter_prefix=False, + rules_version="rules-1", + uploader_version="uploader-1", + api_key="secret", + ) + + report = run_discovered_tasks( + discovery, + settings=settings, + endpoints=EndpointSet( + True, + "datadoghq.com", + "https://test", + "https://coverage", + "https://telemetry", + ), + resources=resources, + transport_factory=lambda: _MixedTransport(barrier, records, record_lock), + identifier_factory=lambda: "stable-id", + ) + + stats = report.statistics() + self.assertEqual(0, report.exit_code) + self.assertEqual(3, stats["files"]["succeeded"]) + self.assertEqual(3, stats["concurrency"]["peak_active_workers"]) + self.assertEqual(3, stats["requests"]["attempted"]) + self.assertEqual({"json", "multipart"}, {record["kind"] for record in records}) + enriched_test = next( + json.loads(record["body"]) + for record in records + if record["kind"] == "json" and b'"events"' in record["body"] + ) + self.assertEqual( + '["@team"]', + enriched_test["events"][0]["content"]["meta"]["test.codeowners"], + ) + self.assertFalse(test_source.exists()) + self.assertFalse(coverage_source.exists()) + self.assertFalse(telemetry_source.exists()) + + stream = StringIO() + report_path = root / "report.json" + emit_report(report, stream=stream, report_json=report_path) + self.assertIn("[dd-uploader] summary: mode=upload", stream.getvalue()) + self.assertEqual(stats, json.loads(report_path.read_text(encoding="utf-8"))) + + def test_dry_run_uses_workers_but_never_calls_transport_or_deletes(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + output = root / "bazel-testlogs" / "pkg" / "test.outputs" + source = _write_payload( + output, + "tests", + "events.json", + {"events": [{"content": {}}]}, + ) + discovery = discover_file_tasks((ScanRoot(root / "bazel-testlogs"),)) + resources = LoadedResources(ContextPlan(None), None, None, (), None) + calls: list[dict[str, object]] = [] + settings = CoordinatorSettings( + workspace=root, + workers=4, + dry_run=True, + validate_enrichment=False, + expected_enriched_tags=(), + gzip_payloads=True, + keep_payloads=False, + filter_prefix=False, + rules_version="rules-1", + uploader_version="uploader-1", + api_key="", + ) + + report = run_discovered_tasks( + discovery, + settings=settings, + endpoints=EndpointSet( + True, + "datadoghq.com", + "https://test", + "https://coverage", + "https://telemetry", + ), + resources=resources, + transport_factory=lambda: _MixedTransport(None, calls, threading.Lock()), + identifier_factory=lambda: "stable-id", + ) + + stats = report.statistics() + self.assertEqual(0, report.exit_code) + self.assertEqual(1, stats["requests"]["planned"]) + self.assertEqual(0, stats["requests"]["attempted"]) + self.assertEqual([], calls) + self.assertTrue(source.exists()) + + def test_file_failures_are_logged_with_task_and_reason_code(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + output = root / "bazel-testlogs" / "pkg" / "test.outputs" + source = output / "payloads" / "tests" / "invalid.json" + source.parent.mkdir(parents=True) + source.write_text("[]", encoding="utf-8") + discovery = discover_file_tasks((ScanRoot(root / "bazel-testlogs"),)) + resources = LoadedResources(ContextPlan(None), None, None, (), None) + stream = StringIO() + settings = CoordinatorSettings( + workspace=root, + workers=1, + dry_run=True, + validate_enrichment=False, + expected_enriched_tags=(), + gzip_payloads=False, + keep_payloads=False, + filter_prefix=False, + rules_version="rules-1", + uploader_version="uploader-1", + api_key="", + ) + + report = run_discovered_tasks( + discovery, + settings=settings, + endpoints=EndpointSet( + True, + "datadoghq.com", + "https://test", + "https://coverage", + "https://telemetry", + ), + resources=resources, + logger=configure_logging(debug=True, stream=stream), + transport_factory=lambda: _MixedTransport( + None, + [], + threading.Lock(), + ), + identifier_factory=lambda: "stable-id", + ) + + self.assertEqual(1, report.exit_code) + logs = stream.getvalue() + self.assertIn("task=file-000001", logs) + self.assertIn("failure_code=invalid_test", logs) + self.assertIn("terminal_status=failed", logs) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/tests/python/test_uploader_discovery_tools.py b/tools/tests/python/test_uploader_discovery_tools.py new file mode 100644 index 00000000..94307c0c --- /dev/null +++ b/tools/tests/python/test_uploader_discovery_tools.py @@ -0,0 +1,302 @@ +#!/usr/bin/env python3 +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Verify deterministic testlogs discovery and task creation. + +Stable snapshots prevent duplicate ownership and nondeterministic reports. +""" + +from __future__ import annotations + +import os +from pathlib import Path +import tempfile +import unittest +from unittest import mock + +from uploader_test_support import add_uploader_runtime_to_path + +add_uploader_runtime_to_path() + +from uploader_py.discovery import ( # noqa: E402 + DiscoveryError, + ScanRoot, + discover_file_tasks, + payload_latest_mtime, + resolve_local_testlogs_root, + tests_executed, + wait_for_quiescence, +) +from uploader_py.models import PayloadType # noqa: E402 + + +def _payload(output: Path, kind: str, name: str, body: bytes = b"{}") -> Path: + destination = output / "payloads" / kind / name + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(body) + return destination + + +class DiscoveryTests(unittest.TestCase): + def test_builds_one_stable_task_per_direct_payload_file(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) / "bazel-testlogs" + output = root / "pkg" / "target" / "test.outputs" + _payload(output, "tests", "b.json") + _payload(output, "tests", "a.msgpack") + _payload(output, "coverage", "coverage.json") + _payload(output, "telemetry", "telemetry.json") + nested = output / "payloads" / "tests" / "nested" / "ignored.json" + nested.parent.mkdir() + nested.write_text("{}", encoding="utf-8") + (output / "payloads" / "tests" / "ignored.txt").write_text( + "ignored", encoding="utf-8" + ) + + discovery = discover_file_tasks((ScanRoot(root),)) + + self.assertEqual(1, len(discovery.outputs)) + self.assertEqual( + ["file-000001", "file-000002", "file-000003", "file-000004"], + [task.task_id for task in discovery.tasks], + ) + self.assertEqual( + [ + PayloadType.TEST, + PayloadType.TEST, + PayloadType.COVERAGE, + PayloadType.TELEMETRY, + ], + [task.payload_type for task in discovery.tasks], + ) + self.assertEqual( + "pkg/target/test.outputs/payloads/tests/a.msgpack", + discovery.tasks[0].display_path, + ) + self.assertEqual( + {PayloadType.TEST: 2, PayloadType.COVERAGE: 1, PayloadType.TELEMETRY: 1}, + discovery.counts(), + ) + + def test_staged_selected_output_suppresses_same_key_from_local_root(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + base = Path(raw_root) + local = base / "local" + staged = base / "staged" + key = "pkg/target/test.outputs" + local_source = _payload(local / key, "tests", "events.json", b"local") + staged_source = _payload(staged / key, "tests", "events.json", b"staged") + + discovery = discover_file_tasks( + (ScanRoot(local), ScanRoot(staged, staged=True)), + staged_output_keys=(key,), + ) + + self.assertEqual(1, len(discovery.tasks)) + self.assertEqual(staged_source.resolve(), discovery.tasks[0].source_path.resolve()) + self.assertNotEqual(local_source.resolve(), discovery.tasks[0].source_path.resolve()) + + def test_payload_symlink_is_never_scheduled(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + output = root / "pkg" / "target" / "test.outputs" + outside = root / "outside.json" + outside.write_text('{"events":[]}', encoding="utf-8") + link = output / "payloads" / "tests" / "linked.json" + link.parent.mkdir(parents=True) + try: + link.symlink_to(outside) + except OSError as exc: + self.skipTest(f"symlink creation unavailable: {exc}") + + discovery = discover_file_tasks((ScanRoot(root),)) + + self.assertEqual((), discovery.tasks) + self.assertIn("payload_symlink_skipped", discovery.warning_codes) + + def test_payload_directory_symlink_is_never_scheduled(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + output = root / "pkg" / "target" / "test.outputs" + outside = root / "outside" / "tests" + outside.mkdir(parents=True) + (outside / "events.json").write_text("{}", encoding="utf-8") + link = output / "payloads" / "tests" + link.parent.mkdir(parents=True) + try: + link.symlink_to(outside, target_is_directory=True) + except OSError as exc: + self.skipTest(f"directory symlink creation unavailable: {exc}") + + discovery = discover_file_tasks((ScanRoot(root),)) + + self.assertEqual((), discovery.tasks) + self.assertIn("payload_symlink_skipped", discovery.warning_codes) + + def test_unreadable_payload_directory_does_not_hide_valid_siblings(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + unreadable = root / "a" / "test.outputs" / "payloads" / "tests" + unreadable.mkdir(parents=True) + unreadable = unreadable.resolve() + valid_source = _payload( + root / "b" / "test.outputs", + "tests", + "events.json", + ) + original_iterdir = Path.iterdir + + def iterdir(path: Path): + if path == unreadable: + raise PermissionError("unreadable during discovery") + return original_iterdir(path) + + with mock.patch.object(Path, "iterdir", iterdir): + discovery = discover_file_tasks((ScanRoot(root),)) + + self.assertEqual( + (valid_source.resolve(),), + tuple(task.source_path.resolve() for task in discovery.tasks), + ) + self.assertIn("payload_directory_unreadable", discovery.warning_codes) + + def test_intermediate_payload_symlink_cannot_escape_test_outputs(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + output = root / "pkg" / "target" / "test.outputs" + outside = root / "outside" / "payloads" + (outside / "tests").mkdir(parents=True) + (outside / "tests" / "events.json").write_text("{}", encoding="utf-8") + output.mkdir(parents=True) + try: + (output / "payloads").symlink_to(outside, target_is_directory=True) + except OSError as exc: + self.skipTest(f"directory symlink creation unavailable: {exc}") + + discovery = discover_file_tasks((ScanRoot(root),)) + + self.assertEqual((), discovery.tasks) + self.assertIn("payload_symlink_skipped", discovery.warning_codes) + + def test_selected_output_never_falls_back_to_stale_local_when_staging_failed(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + key = "pkg/target/test.outputs" + _payload(root / key, "tests", "events.json", b"stale") + + discovery = discover_file_tasks( + (ScanRoot(root),), + staged_output_keys=(key,), + ) + + self.assertEqual((), discovery.tasks) + self.assertIn( + "selected_staged_output_missing", + discovery.warning_codes, + ) + + def test_max_depth_matches_find_style_root_relative_depth(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + output = root / "one" / "two" / "test.outputs" + _payload(output, "tests", "events.json") + + shallow = discover_file_tasks((ScanRoot(root),), max_depth=2) + sufficient = discover_file_tasks((ScanRoot(root),), max_depth=3) + + self.assertEqual((), shallow.tasks) + self.assertIn("max_depth_may_be_too_shallow", shallow.warning_codes) + self.assertEqual(1, len(sufficient.tasks)) + + def test_explicit_testlogs_precedence_and_invalid_override(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + explicit = root / "explicit" + workspace = root / "workspace" + cwd = root / "cwd" + explicit.mkdir() + (workspace / "bazel-testlogs").mkdir(parents=True) + (cwd / "bazel-testlogs").mkdir(parents=True) + + self.assertEqual( + explicit.resolve(), + resolve_local_testlogs_root( + explicit=explicit, + workspace=workspace, + cwd=cwd, + ), + ) + with self.assertRaisesRegex(DiscoveryError, "TESTLOGS_DIR"): + resolve_local_testlogs_root( + explicit=root / "missing", + workspace=workspace, + cwd=cwd, + ) + + def test_mtime_and_test_execution_markers_are_read_only_observations(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + output = root / "pkg" / "test.outputs" + source = _payload(output, "tests", "events.json") + os.utime(source, (123, 123)) + (output.parent / "test.log").write_text("ran", encoding="utf-8") + discovery = discover_file_tasks((ScanRoot(root),)) + + self.assertEqual(123, payload_latest_mtime(discovery)) + self.assertTrue(tests_executed((ScanRoot(root),))) + + def test_quiescence_refreshes_discovery_and_respects_wait_budget(self) -> None: + empty = discover_file_tasks(()) + calls = 0 + now = 100.0 + + def clock() -> float: + return now + + def sleep(delay: float) -> None: + nonlocal now + now += delay + + def discover(): + nonlocal calls + calls += 1 + return empty + + result = wait_for_quiescence( + discover, + quiescent_seconds=10, + max_wait_seconds=5, + poll_seconds=2, + clock=clock, + sleeper=sleep, + ) + + self.assertEqual("max_wait", result.reason) + self.assertEqual(5, result.elapsed_seconds) + self.assertGreaterEqual(calls, 3) + + def test_quiescence_can_proceed_immediately_for_old_payloads(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + output = root / "pkg" / "test.outputs" + source = _payload(output, "tests", "events.json") + os.utime(source, (10, 10)) + + result = wait_for_quiescence( + lambda: discover_file_tasks((ScanRoot(root),)), + quiescent_seconds=5, + max_wait_seconds=30, + clock=lambda: 100, + sleeper=lambda _delay: self.fail("old payload should be quiescent"), + ) + + self.assertEqual("quiescent", result.reason) + self.assertEqual(1, len(result.discovery.tasks)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/tests/python/test_uploader_enrichment_tools.py b/tools/tests/python/test_uploader_enrichment_tools.py new file mode 100644 index 00000000..847a4391 --- /dev/null +++ b/tools/tests/python/test_uploader_enrichment_tools.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Verify immutable context selection and per-file test enrichment. + +Focused coverage protects tag and metadata behavior independently from uploads. +""" + +from __future__ import annotations + +import unittest + +from uploader_test_support import add_uploader_runtime_to_path + +add_uploader_runtime_to_path() + +from uploader_py.codeowners import CodeOwnersMatcher, parse_codeowners # noqa: E402 +from uploader_py.enrichment import ( # noqa: E402 + ContextPlan, + ContextRecord, + ContextSelection, + enrich_test_payload, + payload_repo_key, +) + + +def _matcher() -> CodeOwnersMatcher: + rules, warnings = parse_codeowners("* @default\n/src/owned.py @src\n") + return CodeOwnersMatcher(None, "/workspace", "", rules, warnings, False) + + +class ContextPlanTests(unittest.TestCase): + def test_zero_one_override_and_multiple_context_selection(self) -> None: + empty = ContextPlan(None) + self.assertIsNone(empty.select(None).values) + + only = ContextRecord.create("repo-one", {"service": "one"}) + one = ContextPlan(only, (only,)) + self.assertEqual("one", one.select(None).values["service"]) + + two = ContextRecord.create("repo-two", {"service": "two"}) + multiple = ContextPlan(only, (only, two)) + self.assertEqual("two", multiple.select("repo-two").values["service"]) + self.assertEqual( + "context_repo_metadata_missing", + multiple.select(None).warning_code, + ) + self.assertEqual( + "context_repo_not_found", + multiple.select("missing").warning_code, + ) + + override = ContextPlan(only, (only, two), override=True) + self.assertEqual("one", override.select("missing").values["service"]) + + def test_context_records_are_immutable(self) -> None: + record = ContextRecord.create("repo", {"service": "value"}) + with self.assertRaises(TypeError): + record.values["service"] = "changed" # type: ignore[index] + + def test_repo_key_requires_nonempty_string(self) -> None: + self.assertEqual( + "repo", + payload_repo_key({"bazel.test_optimization.repo_name": "repo"}), + ) + self.assertIsNone(payload_repo_key({"bazel.test_optimization.repo_name": 1})) + self.assertIsNone(payload_repo_key(None)) + + +class TestPayloadEnrichmentTests(unittest.TestCase): + def test_top_level_context_sidecar_and_codeowners_order(self) -> None: + payload = { + "metadata": { + "*": {"language": "python", "env": "payload-env"}, + "test": {"keep": True}, + "unexpected": {"drop": True}, + }, + "events": [ + { + "type": "test", + "content": { + "meta": {"test.source.file": "src/owned.py"}, + "metrics": "invalid", + }, + }, + { + "type": "span", + "content": {"meta": {"test.source.file": "src/owned.py"}}, + }, + ], + } + context = ContextSelection( + ContextRecord.create( + "repo", + { + "runtime.id": "context-runtime", + "runtime.name": "context-language", + "env": "context-env", + "git.commit.sha": "abc", + "numeric": 7, + "enabled": True, + "nested": {"key": "value"}, + "topt.api_key_fingerprint": "must-not-upload", + }, + ).values + ) + sidecar = { + "bazel.target": "//pkg:test", + "numeric": 9, + "sidecar_bool": False, + } + result = enrich_test_payload( + payload, + context_selection=context, + bazel_metadata=sidecar, + runtime_id="fallback-runtime", + rules_version="1.2.3", + codeowners_matcher=_matcher(), + codeowners_cache={}, + ) + + self.assertEqual( + { + "*": { + "runtime-id": "context-runtime", + "language": "python", + "library_version": "1.2.3", + "env": "payload-env", + }, + "test": {"keep": True}, + }, + payload["metadata"], + ) + test_meta = payload["events"][0]["content"]["meta"] + test_metrics = payload["events"][0]["content"]["metrics"] + self.assertEqual("abc", test_meta["git.commit.sha"]) + self.assertEqual("//pkg:test", test_meta["bazel.target"]) + self.assertEqual("true", test_meta["enabled"]) + self.assertEqual("false", test_meta["sidecar_bool"]) + self.assertEqual('{"key":"value"}', test_meta["nested"]) + self.assertNotIn("topt.api_key_fingerprint", test_meta) + self.assertEqual(9, test_metrics["numeric"]) + self.assertEqual('["@src"]', test_meta["test.codeowners"]) + + span_meta = payload["events"][1]["content"]["meta"] + self.assertEqual("abc", span_meta["git.commit.sha"]) + self.assertNotIn("test.codeowners", span_meta) + self.assertEqual(1, result.codeowners.enriched) + + def test_missing_context_still_normalizes_payload(self) -> None: + payload = {"metadata": "invalid", "events": "invalid"} + result = enrich_test_payload( + payload, + context_selection=ContextSelection( + None, + "context_repo_metadata_missing", + ), + bazel_metadata=None, + runtime_id="runtime", + rules_version="rules", + codeowners_matcher=CodeOwnersMatcher(None, "", ""), + ) + self.assertEqual( + { + "*": { + "runtime-id": "runtime", + "language": "bazel", + "library_version": "rules", + } + }, + payload["metadata"], + ) + self.assertEqual(("context_repo_metadata_missing",), result.warning_codes) + + def test_producer_values_win_and_bazel_sidecar_overrides_context_tags(self) -> None: + payload = { + "metadata": { + "*": { + "runtime-id": "producer-runtime", + "language": "go", + "library_version": "producer-version", + "env": "producer-env", + } + }, + "events": [{"type": "test", "content": {"meta": {}}}], + } + enrich_test_payload( + payload, + context_selection=ContextSelection({"same": "context"}), + bazel_metadata={"same": "sidecar"}, + runtime_id="fallback", + rules_version="fallback", + codeowners_matcher=CodeOwnersMatcher(None, "", ""), + ) + self.assertEqual( + { + "runtime-id": "producer-runtime", + "language": "go", + "library_version": "producer-version", + "env": "producer-env", + }, + payload["metadata"]["*"], + ) + self.assertEqual("sidecar", payload["events"][0]["content"]["meta"]["same"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/tests/python/test_uploader_expected_targets_tools.py b/tools/tests/python/test_uploader_expected_targets_tools.py new file mode 100644 index 00000000..7282df08 --- /dev/null +++ b/tools/tests/python/test_uploader_expected_targets_tools.py @@ -0,0 +1,237 @@ +#!/usr/bin/env python3 +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Verify expected-target loading and discovered-output selection. + +These tests keep unrelated or malformed target data out of worker scheduling. +""" + +from __future__ import annotations + +import json +from pathlib import Path +import tempfile +import unittest + +from uploader_test_support import add_uploader_runtime_to_path + +add_uploader_runtime_to_path() + +from topt_runtime.runfiles import RunfilesResolver # noqa: E402 +from uploader_py.discovery import ScanRoot, discover_file_tasks # noqa: E402 +from uploader_py.expected_targets import ( # noqa: E402 + ExpectedTargetsError, + load_expected_targets, + select_expected_outputs, +) + + +def _payload(root: Path, target_path: str) -> None: + destination = root / target_path / "payloads" / "tests" / "events.json" + destination.parent.mkdir(parents=True) + destination.write_text('{"events":[{}]}', encoding="utf-8") + + +class ExpectedTargetsTests(unittest.TestCase): + def test_runtime_selection_requires_targets_and_rejects_mismatch(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + resolver = RunfilesResolver.from_environment(environ={}, cwd=root) + with self.assertRaisesRegex(ExpectedTargetsError, "at least one"): + load_expected_targets( + static_targets=(), + expected_targets_file_paths=(), + resolver=resolver, + runtime_selection=True, + ) + with self.assertRaisesRegex(ExpectedTargetsError, "different target sets"): + load_expected_targets( + static_targets=("//pkg:a",), + expected_targets_file_paths=(), + resolver=resolver, + runtime_targets=("//pkg:b",), + ) + + plan = load_expected_targets( + static_targets=(), + expected_targets_file_paths=(), + resolver=resolver, + runtime_targets=("//pkg:b", "//pkg:a"), + runtime_selection=True, + ) + self.assertEqual(("//pkg:a", "//pkg:b"), plan.targets) + self.assertEqual("runtime", plan.source) + + def test_schema_v1_file_selects_outputs_and_stamps_tasks(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + testlogs = root / "bazel-testlogs" + _payload(testlogs, "pkg/a/test.outputs") + _payload(testlogs, "pkg/b/shard_1_of_2/test.outputs") + _payload(testlogs, "pkg/unselected/test.outputs") + target_file = root / "targets.json" + target_file.write_text( + json.dumps( + { + "schema_version": 1, + "targets": ["//pkg:a", "//pkg:b"], + } + ), + encoding="utf-8", + ) + resolver = RunfilesResolver.from_environment(environ={}, cwd=root) + plan = load_expected_targets( + static_targets=(), + expected_targets_file_paths=(str(target_file),), + resolver=resolver, + ) + + selected = select_expected_outputs( + discover_file_tasks((ScanRoot(testlogs),)), + plan, + ) + + self.assertEqual("file", plan.source) + self.assertEqual(2, len(selected.tasks)) + self.assertEqual( + {"//pkg:a", "//pkg:b"}, + {task.target_label for task in selected.tasks}, + ) + + def test_static_and_dynamic_mismatch_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + target_file = root / "targets.json" + target_file.write_text( + '{"schema_version":1,"targets":["//pkg:b"]}', + encoding="utf-8", + ) + resolver = RunfilesResolver.from_environment(environ={}, cwd=root) + + with self.assertRaisesRegex(ExpectedTargetsError, "different target sets"): + load_expected_targets( + static_targets=("//pkg:a",), + expected_targets_file_paths=(str(target_file),), + resolver=resolver, + ) + + def test_output_matching_is_exact_except_for_bazel_attempt_directories(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + testlogs = root / "bazel-testlogs" + _payload(testlogs, "pkg/foo/test.outputs") + _payload(testlogs, "pkg/foo/shard_1_of_2/attempt_1/test.outputs") + _payload(testlogs, "pkg/foo/run_1_of_2/test.outputs") + _payload(testlogs, "pkg/foo/bar/test.outputs") + _payload(testlogs, "pkg/foo/not_a_bazel_attempt/test.outputs") + resolver = RunfilesResolver.from_environment(environ={}, cwd=root) + plan = load_expected_targets( + static_targets=("//pkg:foo", "//pkg/foo:bar"), + expected_targets_file_paths=(), + resolver=resolver, + ) + + selected = select_expected_outputs( + discover_file_tasks((ScanRoot(testlogs),)), + plan, + ) + + self.assertEqual( + { + "pkg/foo/test.outputs", + "pkg/foo/shard_1_of_2/attempt_1/test.outputs", + "pkg/foo/run_1_of_2/test.outputs", + "pkg/foo/bar/test.outputs", + }, + {output.output_key for output in selected.outputs}, + ) + labels_by_output = { + task.output_key: task.target_label for task in selected.tasks + } + self.assertEqual("//pkg:foo", labels_by_output["pkg/foo/test.outputs"]) + self.assertEqual( + "//pkg:foo", + labels_by_output["pkg/foo/shard_1_of_2/attempt_1/test.outputs"], + ) + self.assertEqual( + "//pkg:foo", + labels_by_output["pkg/foo/run_1_of_2/test.outputs"], + ) + self.assertEqual( + "//pkg/foo:bar", + labels_by_output["pkg/foo/bar/test.outputs"], + ) + self.assertNotIn( + "pkg/foo/not_a_bazel_attempt/test.outputs", + labels_by_output, + ) + + def test_exact_nested_target_wins_over_attempt_suffix_interpretation(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + testlogs = root / "bazel-testlogs" + output_key = "pkg/foo/shard_1_of_2/test.outputs" + _payload(testlogs, output_key) + resolver = RunfilesResolver.from_environment(environ={}, cwd=root) + plan = load_expected_targets( + static_targets=("//pkg:foo", "//pkg/foo:shard_1_of_2"), + expected_targets_file_paths=(), + resolver=resolver, + ) + + selected = select_expected_outputs( + discover_file_tasks((ScanRoot(testlogs),)), + plan, + allow_missing=True, + ) + + self.assertEqual(1, len(selected.tasks)) + self.assertEqual(output_key, selected.tasks[0].output_key) + self.assertEqual("//pkg/foo:shard_1_of_2", selected.tasks[0].target_label) + + def test_missing_expected_output_fails_unless_staging_can_supply_it(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + discovery = discover_file_tasks((ScanRoot(root),)) + resolver = RunfilesResolver.from_environment(environ={}, cwd=root) + plan = load_expected_targets( + static_targets=("//pkg:missing",), + expected_targets_file_paths=(), + resolver=resolver, + ) + + with self.assertRaisesRegex(ExpectedTargetsError, "no local test.outputs"): + select_expected_outputs(discovery, plan) + allowed = select_expected_outputs(discovery, plan, allow_missing=True) + self.assertEqual((), allowed.tasks) + + def test_rejects_unexpanded_external_and_unsorted_inputs(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + resolver = RunfilesResolver.from_environment(environ={}, cwd=root) + for label in ("@repo//pkg:test", "//pkg/...:all", "//pkg"): + with self.subTest(label=label), self.assertRaises(ExpectedTargetsError): + load_expected_targets( + static_targets=(label,), + expected_targets_file_paths=(), + resolver=resolver, + ) + target_file = root / "targets.json" + target_file.write_text( + '{"schema_version":1,"targets":["//pkg:b","//pkg:a"]}', + encoding="utf-8", + ) + with self.assertRaisesRegex(ExpectedTargetsError, "sorted"): + load_expected_targets( + static_targets=(), + expected_targets_file_paths=(str(target_file),), + resolver=resolver, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/tests/python/test_uploader_file_worker_tools.py b/tools/tests/python/test_uploader_file_worker_tools.py new file mode 100644 index 00000000..83fed1d2 --- /dev/null +++ b/tools/tests/python/test_uploader_file_worker_tools.py @@ -0,0 +1,725 @@ +#!/usr/bin/env python3 +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Exercise complete enrichment, split, upload, and cleanup for one file. + +The suite protects the invariant that one worker owns every derived request. +""" + +from __future__ import annotations + +from contextlib import contextmanager +import gzip +from io import StringIO +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from uploader_test_support import add_uploader_runtime_to_path + +add_uploader_runtime_to_path() + +from uploader_py.codeowners import CodeOwnersMatcher # noqa: E402 +from uploader_py.endpoints import EndpointSet # noqa: E402 +from uploader_py.enrichment import ContextPlan, ContextRecord # noqa: E402 +from uploader_py.file_worker import WorkerRuntime, common_headers, process_file # noqa: E402 +from uploader_py.logging_utils import configure_logging # noqa: E402 +from uploader_py.models import ( # noqa: E402 + MAX_TEST_PAYLOAD_BYTES, + FileStatus, + FileTask, + PayloadType, +) +from uploader_py.transport import HttpResult # noqa: E402 + + +class _FakeTransport: + def __init__(self, *results: HttpResult) -> None: + self.results = list(results) + self.json_calls: list[dict[str, object]] = [] + self.multipart_calls: list[dict[str, object]] = [] + + def post_json( + self, + url, + headers, + body, + *, + gzip_body=False, + content_encoding=None, + ): + self.json_calls.append( + { + "url": url, + "headers": dict(headers), + "body": Path(body).read_bytes() if isinstance(body, Path) else body, + "gzip_body": gzip_body, + "content_encoding": content_encoding, + } + ) + return self.results.pop(0) if self.results else HttpResult(200, 1) + + def post_prepared_multipart(self, url, headers, prepared): + self.multipart_calls.append( + { + "url": url, + "headers": dict(headers), + "body": prepared.path.read_bytes(), + "content_type": prepared.content_type, + "content_length": prepared.content_length, + } + ) + return self.results.pop(0) if self.results else HttpResult(200, 1) + + +def _runtime(root: Path, **changes: object) -> WorkerRuntime: + values: dict[str, object] = { + "endpoints": EndpointSet( + agentless=True, + site="datadoghq.com", + test_url="https://test.invalid/api/v2/citestcycle", + coverage_url="https://coverage.invalid/api/v2/citestcov", + telemetry_url="https://telemetry.invalid/api/v2/apmtelemetry", + ), + "invocation_temp_root": root, + "context_plan": ContextPlan( + ContextRecord.create( + "repo", + { + "git.repository_url": "https://example.invalid/repo.git", + "git.commit.sha": "abcdef", + "env": "ci", + }, + ), + ), + "codeowners_matcher": CodeOwnersMatcher(None, "", ""), + "runtime_id": "runtime-1", + "rules_version": "rules-1", + "uploader_version": "uploader-1", + "api_key": "secret", + } + values.update(changes) + return WorkerRuntime(**values) # type: ignore[arg-type] + + +def _test_task(source: Path, outputs: Path | None = None) -> FileTask: + return FileTask( + task_id="test-1", + source_path=source, + display_path="payloads/tests/events.json", + payload_type=PayloadType.TEST, + test_outputs_dir=outputs, + ) + + +def _large_test_payload(*markers: str) -> str: + return json.dumps( + { + "events": [ + {"content": {"meta": {"value": marker * 2_400_000}}} + for marker in markers + ] + } + ) + + +class FileWorkerTests(unittest.TestCase): + def test_non_finite_test_json_fails_before_enrichment_or_http(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + source = root / "events.json" + source.write_text( + '{"events":[{"content":{"metrics":{"invalid":NaN}}}]}', + encoding="utf-8", + ) + transport = _FakeTransport() + + result = process_file(_test_task(source), _runtime(root), transport) + + self.assertEqual(FileStatus.FAILED, result.status) + self.assertEqual("invalid_test_json", result.failure_code) + self.assertEqual([], transport.json_calls) + self.assertTrue(source.exists()) + + def test_task_temporary_cleanup_failure_preserves_successful_upload(self) -> None: + @contextmanager + def cleanup_failure(invocation_root, _task_id, *, on_cleanup_error): + directory = invocation_root / "simulated-task-temp" + directory.mkdir() + yield directory + on_cleanup_error("PermissionError") + + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + source = root / "events.json" + source.write_text('{"events":[{"content":{}}]}', encoding="utf-8") + transport = _FakeTransport(HttpResult(200, 1)) + + with patch( + "uploader_py.file_worker.task_temporary_directory", + cleanup_failure, + ): + result = process_file(_test_task(source), _runtime(root), transport) + + self.assertEqual(FileStatus.SUCCEEDED, result.status) + self.assertEqual(1, result.requests_succeeded) + self.assertTrue(result.source_deleted) + self.assertIn("task_temp_cleanup_failed", result.warning_codes) + + def test_debug_changes_only_logs_not_body_result_or_cleanup(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + sources: list[Path] = [] + transports: list[_FakeTransport] = [] + results = [] + logs: list[str] = [] + for debug in (False, True): + run_root = root / ("debug" if debug else "normal") + run_root.mkdir() + source = run_root / "events.json" + source.write_text( + '{"events":[{"content":{"meta":{}}}]}', + encoding="utf-8", + ) + stream = StringIO() + transport = _FakeTransport(HttpResult(200, 1)) + runtime = _runtime( + run_root, + keep_payloads=True, + logger=configure_logging( + debug=debug, + secrets=("secret",), + stream=stream, + ), + ) + sources.append(source) + transports.append(transport) + results.append(process_file(_test_task(source), runtime, transport)) + logs.append(stream.getvalue()) + + self.assertEqual(results[0], results[1]) + self.assertEqual(transports[0].json_calls, transports[1].json_calls) + self.assertTrue(all(source.exists() for source in sources)) + self.assertEqual("", logs[0]) + self.assertIn("source_bytes=", logs[1]) + self.assertIn("context_selected=yes", logs[1]) + self.assertIn("enriched_bytes=", logs[1]) + self.assertIn("threshold_bytes=4718592", logs[1]) + self.assertIn("chunk=1/1 bytes=", logs[1]) + self.assertIn("task temporary cleanup completed", logs[1]) + self.assertNotIn("secret", logs[1]) + + def test_test_worker_enriches_validates_splits_uploads_and_deletes_source(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + outputs = root / "test.outputs" + source_dir = outputs / "payloads" / "tests" + source_dir.mkdir(parents=True) + source = source_dir / "events.json" + source.write_text( + json.dumps( + { + "metadata": {"*": {"language": "go", "library_version": "1.2.3"}}, + "events": [ + {"content": {"meta": {"test.source.file": "src/a.go"}}}, + {"content": {"meta": {"test.source.file": "src/b.go"}}}, + ], + } + ), + encoding="utf-8", + ) + (outputs / "bazel_target_metadata.json").write_text( + json.dumps({"bazel.target": "//pkg:test", "bazel.package": "pkg"}), + encoding="utf-8", + ) + transport = _FakeTransport(HttpResult(202, 2, retry_delays=(2.0,))) + + result = process_file( + _test_task(source, outputs), + _runtime(root, gzip_payloads=True), + transport, + ) + + self.assertEqual(FileStatus.SUCCEEDED, result.status) + self.assertEqual(2, result.events) + self.assertEqual(1, result.chunks_created) + self.assertEqual(1, result.chunks_uploaded) + self.assertEqual(2, result.requests_attempted) + self.assertEqual(1, result.retries) + self.assertTrue(result.source_deleted) + self.assertFalse(source.exists()) + self.assertEqual(1, len(transport.json_calls)) + call = transport.json_calls[0] + self.assertFalse(call["gzip_body"]) + self.assertEqual("gzip", call["content_encoding"]) + self.assertEqual("secret", call["headers"]["DD-API-KEY"]) + self.assertEqual("go", call["headers"]["Datadog-Meta-Lang"]) + body = json.loads(gzip.decompress(call["body"])) + self.assertEqual("//pkg:test", body["events"][0]["content"]["meta"]["bazel.target"]) + self.assertEqual("abcdef", body["events"][1]["content"]["meta"]["git.commit.sha"]) + + def test_symlinked_bazel_metadata_is_never_merged(self) -> None: + if not hasattr(Path, "symlink_to"): + self.skipTest("symlinks are unavailable") + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + outputs = root / "test.outputs" + source = outputs / "payloads" / "tests" / "events.json" + source.parent.mkdir(parents=True) + source.write_text('{"events":[{"content":{"meta":{}}}]}', encoding="utf-8") + outside = root / "outside.json" + outside.write_text( + '{"secret":"must-not-upload","bazel.target":"//pkg:test"}', + encoding="utf-8", + ) + sidecar = outputs / "bazel_target_metadata.json" + try: + sidecar.symlink_to(outside) + except OSError as exc: + self.skipTest(f"symlinks are unavailable: {exc}") + transport = _FakeTransport(HttpResult(200, 1)) + + result = process_file( + _test_task(source, outputs), + _runtime(root, keep_payloads=True), + transport, + ) + + self.assertEqual(FileStatus.SUCCEEDED, result.status) + self.assertIn("bazel_metadata_unsafe", result.warning_codes) + body = json.loads(transport.json_calls[0]["body"]) + self.assertNotIn("secret", body["events"][0]["content"]["meta"]) + + def test_successful_upload_deletes_from_read_only_bazel_directory(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + source_dir = root / "test.outputs" / "payloads" / "tests" + source_dir.mkdir(parents=True) + source = source_dir / "events.json" + source.write_text('{"events":[{"content":{}}]}', encoding="utf-8") + source_dir.chmod(0o555) + + try: + result = process_file( + _test_task(source), + _runtime(root), + _FakeTransport(HttpResult(200, 1)), + ) + finally: + source_dir.chmod(0o755) + + self.assertEqual(FileStatus.SUCCEEDED, result.status) + self.assertTrue(result.source_deleted) + self.assertFalse(source.exists()) + + def test_preventive_split_happens_before_first_request(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + source = root / "events.json" + source.write_text(_large_test_payload("x", "y"), encoding="utf-8") + transport = _FakeTransport(HttpResult(200, 1), HttpResult(200, 1)) + + result = process_file(_test_task(source), _runtime(root), transport) + + self.assertEqual(FileStatus.SUCCEEDED, result.status) + self.assertEqual(2, result.chunks_created) + self.assertEqual(2, len(transport.json_calls)) + self.assertTrue( + all(len(call["body"]) <= MAX_TEST_PAYLOAD_BYTES for call in transport.json_calls) + ) + + def test_failed_middle_chunk_continues_and_persists_only_failed_events(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + source = root / "events.json" + source.write_text(_large_test_payload("x", "y", "z"), encoding="utf-8") + transport = _FakeTransport( + HttpResult(200, 1), + HttpResult(503, 4), + HttpResult(200, 1), + ) + + result = process_file(_test_task(source), _runtime(root), transport) + + self.assertEqual(FileStatus.FAILED, result.status) + self.assertEqual(3, result.chunks_created) + self.assertEqual(2, result.chunks_uploaded) + self.assertEqual(1, result.chunks_failed) + self.assertEqual(3, len(transport.json_calls)) + self.assertEqual(6, result.requests_attempted) + self.assertEqual(2, result.requests_succeeded) + self.assertEqual(1, result.requests_failed) + self.assertEqual(3, result.retries) + self.assertTrue(source.exists()) + retained = json.loads(source.read_text(encoding="utf-8")) + self.assertEqual(1, len(retained["events"])) + self.assertEqual( + "y" * 2_400_000, + retained["events"][0]["content"]["meta"]["value"], + ) + + retry_transport = _FakeTransport(HttpResult(200, 1)) + retry_result = process_file( + _test_task(source), + _runtime(root), + retry_transport, + ) + + self.assertEqual(FileStatus.SUCCEEDED, retry_result.status) + self.assertEqual(1, len(retry_transport.json_calls)) + self.assertFalse(source.exists()) + + def test_all_failed_chunks_keep_the_original_source(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + source = root / "events.json" + original = _large_test_payload("x", "y") + source.write_text(original, encoding="utf-8") + transport = _FakeTransport(HttpResult(503, 4), HttpResult(503, 4)) + + result = process_file(_test_task(source), _runtime(root), transport) + + self.assertEqual(FileStatus.FAILED, result.status) + self.assertEqual(0, result.chunks_uploaded) + self.assertEqual(2, result.chunks_failed) + self.assertEqual(2, len(transport.json_calls)) + self.assertEqual(original, source.read_text(encoding="utf-8")) + + def test_failed_chunk_persistence_error_keeps_the_original_source(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + source = root / "events.json" + original = _large_test_payload("x", "y") + source.write_text(original, encoding="utf-8") + transport = _FakeTransport(HttpResult(200, 1), HttpResult(503, 4)) + + with patch( + "uploader_py.file_worker.tempfile.NamedTemporaryFile", + side_effect=OSError("read-only directory"), + ): + result = process_file(_test_task(source), _runtime(root), transport) + + self.assertEqual(FileStatus.FAILED, result.status) + self.assertIn("failed_test_chunks_persist_failed", result.warning_codes) + self.assertEqual(original, source.read_text(encoding="utf-8")) + + def test_413_is_terminal_and_source_is_kept(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + source = root / "events.json" + source.write_text('{"events":[{"content":{}}]}', encoding="utf-8") + transport = _FakeTransport(HttpResult(413, 1, body_excerpt=b"too large")) + + result = process_file(_test_task(source), _runtime(root), transport) + + self.assertEqual(FileStatus.FAILED, result.status) + self.assertEqual("payload_limit_contract_mismatch", result.failure_code) + self.assertIn("threshold_bytes=4718592", result.failure_message) + self.assertEqual(1, result.requests_attempted) + self.assertEqual(0, result.retries) + self.assertTrue(source.exists()) + + def test_unsplit_payload_types_report_413_without_claiming_a_split(self) -> None: + cases = ( + (PayloadType.COVERAGE, "coverage.json", b'{"files":[]}'), + ( + PayloadType.TELEMETRY, + "telemetry.json", + ( + b'{"api_version":"v2","request_type":"app-started",' + b'"runtime_id":"runtime","application":{},"payload":[]}' + ), + ), + ) + for payload_type, filename, body in cases: + with self.subTest(payload_type=payload_type), tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + source = root / filename + source.write_bytes(body) + task = FileTask( + task_id=f"{payload_type.value}-413", + source_path=source, + display_path=f"payloads/{payload_type.value}/{filename}", + payload_type=payload_type, + ) + + result = process_file( + task, + _runtime(root), + _FakeTransport(HttpResult(413, 1, body_excerpt=b"too large")), + ) + + self.assertEqual(FileStatus.FAILED, result.status) + self.assertEqual("upload_http_413", result.failure_code) + self.assertIn(f"unsplit {payload_type.value} payload", result.failure_message) + self.assertNotIn("preventive split", result.failure_message) + self.assertEqual(1, result.requests_attempted) + self.assertEqual(0, result.retries) + self.assertTrue(source.exists()) + + def test_gzip_failure_warns_and_falls_back_to_exact_json(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + source = root / "events.json" + source.write_text('{"events":[{"content":{}}]}', encoding="utf-8") + transport = _FakeTransport(HttpResult(200, 1)) + logs = StringIO() + + with patch("uploader_py.file_worker.gzip.compress", side_effect=OSError): + result = process_file( + _test_task(source), + _runtime( + root, + gzip_payloads=True, + logger=configure_logging( + debug=True, + secrets=("secret",), + stream=logs, + ), + ), + transport, + ) + + self.assertEqual(FileStatus.SUCCEEDED, result.status) + self.assertIn("gzip_preparation_failed", result.warning_codes) + self.assertIsNone(transport.json_calls[0]["content_encoding"]) + self.assertEqual(1, len(json.loads(transport.json_calls[0]["body"])["events"])) + self.assertIn( + "gzip preparation completed compressed=0 fallback_json=1", + logs.getvalue(), + ) + + def test_dry_run_uses_same_preparation_without_network_or_cleanup(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + source = root / "events.json" + source.write_text( + '{"events":[{"content":{"meta":{}}}]}', + encoding="utf-8", + ) + transport = _FakeTransport() + runtime = _runtime( + root, + dry_run=True, + validate_enrichment=True, + expected_enriched_tags=("git.repository_url", "git.commit.sha"), + ) + + result = process_file(_test_task(source), runtime, transport) + + self.assertEqual(FileStatus.SUCCEEDED, result.status) + self.assertEqual(1, result.chunks_created) + self.assertEqual(1, result.requests_planned) + self.assertEqual(0, result.requests_attempted) + self.assertEqual([], transport.json_calls) + self.assertTrue(source.exists()) + + def test_upload_validates_enrichment_before_sending(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + source = root / "events.json" + source.write_text( + '{"events":[{"content":{"meta":{}}}]}', + encoding="utf-8", + ) + transport = _FakeTransport() + runtime = _runtime( + root, + validate_enrichment=True, + expected_enriched_tags=("git.repository_url", "git.commit.sha"), + ) + + result = process_file(_test_task(source), runtime, transport) + + self.assertEqual(FileStatus.SUCCEEDED, result.status) + self.assertEqual(1, result.requests_attempted) + self.assertEqual(1, len(transport.json_calls)) + self.assertFalse(source.exists()) + + def test_dry_run_validates_each_protocol_request_without_transport_calls(self) -> None: + cases = ( + ( + PayloadType.TEST, + "events.json", + b'{"events":[{"content":{}}]}', + ), + (PayloadType.COVERAGE, "coverage.json", b'{"files":[]}'), + ( + PayloadType.TELEMETRY, + "telemetry.json", + ( + b'{"api_version":"v2","request_type":"app-started",' + b'"runtime_id":"runtime","application":{},"payload":[]}' + ), + ), + ) + invalid_endpoints = EndpointSet( + agentless=True, + site="datadoghq.com", + test_url="not-an-absolute-url", + coverage_url="not-an-absolute-url", + telemetry_url="not-an-absolute-url", + ) + for payload_type, filename, body in cases: + with self.subTest(payload_type=payload_type), tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + source = root / filename + source.write_bytes(body) + task = FileTask( + task_id=f"dry-run-{payload_type.value}", + source_path=source, + display_path=f"payloads/{payload_type.value}/{filename}", + payload_type=payload_type, + ) + transport = _FakeTransport() + + result = process_file( + task, + _runtime( + root, + dry_run=True, + endpoints=invalid_endpoints, + ), + transport, + ) + + self.assertEqual(FileStatus.FAILED, result.status) + self.assertEqual("request_preparation_failed", result.failure_code) + self.assertEqual([], transport.json_calls) + self.assertEqual([], transport.multipart_calls) + self.assertTrue(source.exists()) + + def test_dry_run_validates_headers_without_transport_calls(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + source = root / "events.json" + source.write_text('{"events":[{"content":{}}]}', encoding="utf-8") + transport = _FakeTransport() + + result = process_file( + _test_task(source), + _runtime(root, dry_run=True, api_key="invalid\nheader"), + transport, + ) + + self.assertEqual(FileStatus.FAILED, result.status) + self.assertEqual("request_preparation_failed", result.failure_code) + self.assertEqual([], transport.json_calls) + self.assertTrue(source.exists()) + + def test_upload_enrichment_validation_fails_before_split_and_network(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + source = root / "events.json" + source.write_text('{"events":[{"content":{}}]}', encoding="utf-8") + transport = _FakeTransport() + + result = process_file( + _test_task(source), + _runtime( + root, + validate_enrichment=True, + expected_enriched_tags=("bazel.target",), + ), + transport, + ) + + self.assertEqual(FileStatus.FAILED, result.status) + self.assertEqual("enrichment_tags_missing", result.failure_code) + self.assertEqual([], transport.json_calls) + self.assertTrue(source.exists()) + + def test_empty_test_payload_and_prefix_filter_are_non_error_skips(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + empty = root / "span_events_empty.json" + empty.write_text('{"events":[]}', encoding="utf-8") + filtered = root / "custom.json" + filtered.write_text('{"events":[{"content":{}}]}', encoding="utf-8") + transport = _FakeTransport() + + empty_result = process_file(_test_task(empty), _runtime(root), transport) + filtered_result = process_file( + _test_task(filtered), + _runtime(root, filter_prefix=True), + transport, + ) + + self.assertEqual(FileStatus.SKIPPED, empty_result.status) + self.assertEqual(FileStatus.SKIPPED, filtered_result.status) + self.assertEqual([], transport.json_calls) + self.assertTrue(empty.exists()) + self.assertTrue(filtered.exists()) + + def test_coverage_worker_supports_json_and_msgpack_multipart(self) -> None: + for suffix, expected_type in ( + (".json", "application/json"), + (".msgpack", "application/msgpack"), + ): + with self.subTest(suffix=suffix), tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + source = root / f"coverage{suffix}" + source.write_bytes(b"coverage-body") + task = FileTask( + task_id="coverage-1", + source_path=source, + display_path=f"payloads/coverage/coverage{suffix}", + payload_type=PayloadType.COVERAGE, + ) + transport = _FakeTransport(HttpResult(200, 1)) + + result = process_file(task, _runtime(root, keep_payloads=True), transport) + + self.assertEqual(FileStatus.SUCCEEDED, result.status) + self.assertEqual(1, result.requests_succeeded) + self.assertTrue(source.exists()) + call = transport.multipart_calls[0] + self.assertIn(expected_type.encode("ascii"), call["body"]) + self.assertIn(b'{"dummy":true}', call["body"]) + self.assertEqual(len(call["body"]), call["content_length"]) + + def test_coverage_dry_run_spools_exact_body_without_transport(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + source = root / "coverage.json" + source.write_bytes(b'{"files":[]}') + task = FileTask( + task_id="coverage-dry-run", + source_path=source, + display_path="payloads/coverage/coverage.json", + payload_type=PayloadType.COVERAGE, + ) + transport = _FakeTransport() + + result = process_file(task, _runtime(root, dry_run=True), transport) + + self.assertEqual(FileStatus.SUCCEEDED, result.status) + self.assertEqual(1, result.requests_planned) + self.assertEqual([], transport.multipart_calls) + self.assertTrue(source.exists()) + + def test_evp_headers_and_common_header_defaults_are_protocol_specific(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + endpoints = EndpointSet( + agentless=False, + site="datadoghq.com", + test_url="https://agent.invalid/evp_proxy/v2/api/v2/citestcycle", + coverage_url="https://agent.invalid/evp_proxy/v2/api/v2/citestcov", + telemetry_url="https://agent.invalid/telemetry/proxy/api/v2/apmtelemetry", + ) + runtime = _runtime(root, endpoints=endpoints) + headers = common_headers(runtime) + self.assertNotIn("DD-API-KEY", headers) + self.assertEqual("bazel-starlark", headers["Datadog-Meta-Lang"]) + self.assertEqual("uploader-1", headers["Datadog-Meta-Tracer-Version"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/tests/python/test_uploader_freshness_tools.py b/tools/tests/python/test_uploader_freshness_tools.py new file mode 100644 index 00000000..6616f207 --- /dev/null +++ b/tools/tests/python/test_uploader_freshness_tools.py @@ -0,0 +1,394 @@ +#!/usr/bin/env python3 +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Verify freshness selection, BEP staging, and doctor-runtime reuse. + +This boundary prevents stale outputs from being authorized or staging from leaking. +""" + +from __future__ import annotations + +import json +from pathlib import Path +import tempfile +import unittest +from unittest import mock +import zipfile + +from uploader_test_support import ( + add_uploader_runtime_to_path, + resolve_runfile as _runfile, +) + +add_uploader_runtime_to_path() + +from topt_runtime.runfiles import RunfilesResolver # noqa: E402 +from uploader_py.config import parse_uploader_config # noqa: E402 +from uploader_py.discovery import ( # noqa: E402 + DiscoveryResult, + ScanRoot, + discover_file_tasks, +) +from uploader_py.expected_targets import ( # noqa: E402 + ExpectedTargetsPlan, + select_expected_outputs, +) +from uploader_py.freshness import ( # noqa: E402 + FreshnessError, + FreshnessPlan, + RemoteOutput, + filter_discovery_for_freshness, + prepare_freshness, + validate_fresh_outputs_accounted, +) +from uploader_py.models import PayloadType # noqa: E402 + + +def _payload(output: Path) -> None: + source = output / "payloads" / "tests" / "events.json" + source.parent.mkdir(parents=True, exist_ok=True) + source.write_text('{"events":[{"content":{}}]}', encoding="utf-8") + + +def _metadata(output: Path, label: str) -> None: + (output / "bazel_target_metadata.json").write_text( + json.dumps({"bazel.target": label}), + encoding="utf-8", + ) + + +class FreshnessTests(unittest.TestCase): + def test_optional_missing_bep_falls_back_to_local_discovery(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + testlogs = root / "bazel-testlogs" + testlogs.mkdir() + config = self._config( + root, + "--freshness-source=bep", + "--freshness-mode=optional", + "--bep-json=missing.ndjson", + ) + + prepared = prepare_freshness( + config, + resolver=RunfilesResolver.from_environment(cwd=root, environ={}), + local_testlogs_root=testlogs, + ) + + self.assertEqual("none", prepared.plan.selected_source) + self.assertFalse(prepared.plan.eligibility_enabled) + self.assertIn("bep_freshness_unavailable", prepared.plan.warning_codes) + self.assertEqual((ScanRoot(testlogs),), prepared.scan_roots) + + def test_remote_only_expected_output_fails_after_other_results(self) -> None: + plan = FreshnessPlan( + selected_source="bep", + eligibility_enabled=True, + remote_only_outputs=( + RemoteOutput("//pkg:remote", "pkg/remote/test.outputs", "remote"), + ), + ) + + with self.assertRaisesRegex(FreshnessError, "remote-only"): + validate_fresh_outputs_accounted( + plan, + DiscoveryResult(outputs=(), tasks=(), discovered_by_type={}), + (), + expected_targets=("//pkg:remote",), + fail_on_error=True, + ) + + def _config( + self, + root: Path, + *arguments: str, + environment: dict[str, str] | None = None, + ): + config_path = root / "uploader-config.json" + config_path.write_text( + json.dumps( + { + "schema_version": 1, + "workspace_name": "workspace", + "doctor_runtime_path": str( + _runfile("tools/core/test_optimization_doctor.py") + ), + "doctor_runtime_short_path": ( + "tools/core/test_optimization_doctor.py" + ), + } + ), + encoding="utf-8", + ) + values = ["--config", str(config_path), "--dry-run", *arguments] + return parse_uploader_config( + values, + environ={} if environment is None else environment, + cwd=root, + ) + + def test_execution_log_selects_only_fresh_output_and_stamps_label(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + testlogs = root / "bazel-testlogs" + fresh = testlogs / "pkg" / "fresh" / "test.outputs" + cached = testlogs / "pkg" / "cached" / "test.outputs" + for output, label in ( + (fresh, "//pkg:fresh"), + (cached, "//pkg:cached"), + ): + _payload(output) + _metadata(output, label) + execution_log = root / "execution.ndjson" + execution_log.write_text( + json.dumps( + { + "mnemonic": "TestRunner", + "cacheHit": False, + "targetLabel": "//pkg:fresh", + "listedOutputs": [ + "bazel-out/bin/testlogs/pkg/fresh/test.outputs/test.log" + ], + } + ) + + "\n" + + json.dumps( + { + "mnemonic": "TestRunner", + "cacheHit": True, + "targetLabel": "//pkg:cached", + "listedOutputs": [ + "bazel-out/bin/testlogs/pkg/cached/test.outputs/test.log" + ], + } + ) + + "\n", + encoding="utf-8", + ) + config = self._config( + root, + "--freshness-source=execution_log", + "--freshness-mode=required", + f"--execution-log-json={execution_log}", + ) + resolver = RunfilesResolver.from_environment(cwd=root, environ={}) + + with mock.patch.object( + Path, + "read_text", + side_effect=AssertionError("execution logs must be streamed"), + ): + prepared = prepare_freshness( + config, + resolver=resolver, + local_testlogs_root=testlogs, + ) + discovery = discover_file_tasks(prepared.scan_roots) + filtered = filter_discovery_for_freshness( + discovery, + prepared.plan, + freshness_mode=config.freshness_mode, + ) + + self.assertEqual("execution_log", prepared.plan.selected_source) + self.assertEqual(1, len(filtered.discovery.tasks)) + self.assertEqual("//pkg:fresh", filtered.discovery.tasks[0].target_label) + self.assertEqual(("pkg/cached/test.outputs",), filtered.skipped_outputs) + + def test_reuses_doctor_bep_parser_and_all_cached_target_is_noop(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + testlogs = root / "bazel-testlogs" + output = testlogs / "pkg" / "target" / "test.outputs" + _payload(output) + _metadata(output, "//pkg:target") + bep = _runfile("tools/tests/python/fixtures/bep_cached_local.ndjson") + config = self._config( + root, + "--freshness-source=bep", + "--freshness-mode=required", + f"--bep-json={bep}", + ) + resolver = RunfilesResolver.from_environment(cwd=root, environ={}) + prepared = prepare_freshness( + config, + resolver=resolver, + local_testlogs_root=testlogs, + expected_targets=("//pkg:target",), + ) + discovery = select_expected_outputs( + discover_file_tasks(prepared.scan_roots), + ExpectedTargetsPlan(("//pkg:target",), "static"), + allow_missing=True, + ) + filtered = filter_discovery_for_freshness( + discovery, + prepared.plan, + freshness_mode=config.freshness_mode, + ) + + self.assertEqual( + frozenset({("//pkg:target", "pkg/target/test.outputs")}), + prepared.plan.cached_outputs, + ) + self.assertEqual((), filtered.discovery.tasks) + prepared.cleanup() + + def test_required_bep_rejects_output_without_target_metadata(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + output = root / "pkg" / "target" / "test.outputs" + _payload(output) + discovery = discover_file_tasks((ScanRoot(root),)) + plan = FreshnessPlan( + selected_source="bep", + eligibility_enabled=True, + eligible_outputs=frozenset( + {("//pkg:target", "pkg/target/test.outputs")} + ), + ) + + with self.assertRaisesRegex(FreshnessError, "metadata is missing"): + filter_discovery_for_freshness( + discovery, + plan, + freshness_mode="required", + ) + + def test_freshness_does_not_follow_symlinked_target_metadata(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + output = root / "pkg" / "target" / "test.outputs" + _payload(output) + outside = root / "outside.json" + outside.write_text('{"bazel.target":"//pkg:target"}', encoding="utf-8") + try: + (output / "bazel_target_metadata.json").symlink_to(outside) + except OSError as exc: + self.skipTest(f"symlinks are unavailable: {exc}") + discovery = discover_file_tasks((ScanRoot(root),)) + plan = FreshnessPlan( + selected_source="bep", + eligibility_enabled=True, + eligible_outputs=frozenset( + {("//pkg:target", "pkg/target/test.outputs")} + ), + ) + + with self.assertRaisesRegex(FreshnessError, "metadata is missing"): + filter_discovery_for_freshness( + discovery, + plan, + freshness_mode="required", + ) + + def test_local_outputs_zip_is_staged_discovered_and_cleaned(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + carrier = ( + root + / "output-base" + / "testlogs" + / "pkg" + / "target" + / "test.outputs" + / "outputs.zip" + ) + carrier.parent.mkdir(parents=True) + with zipfile.ZipFile(carrier, "w") as archive: + archive.writestr( + "payloads/tests/events.json", + '{"events":[{"content":{}}]}', + ) + archive.writestr( + "bazel_target_metadata.json", + '{"bazel.target":"//pkg:target"}', + ) + bep = root / "bep.ndjson" + bep.write_text( + json.dumps( + { + "id": { + "testResult": { + "label": "//pkg:target", + "run": 1, + "shard": 1, + "attempt": 1, + } + }, + "testResult": { + "status": "PASSED", + "testActionOutput": [ + {"name": "outputs.zip", "uri": carrier.as_uri()} + ], + }, + } + ) + + "\n", + encoding="utf-8", + ) + config = self._config( + root, + "--artifact-source=bep", + "--freshness-source=bep", + "--freshness-mode=required", + f"--bep-json={bep}", + ) + resolver = RunfilesResolver.from_environment(cwd=root, environ={}) + + prepared = prepare_freshness( + config, + resolver=resolver, + local_testlogs_root=None, + expected_targets=("//pkg:target",), + ) + staged_roots = prepared.staged_roots + discovery = select_expected_outputs( + discover_file_tasks( + prepared.scan_roots, + staged_output_keys=("pkg/target/test.outputs",), + ), + ExpectedTargetsPlan(("//pkg:target",), "static"), + allow_missing=True, + ) + filtered = filter_discovery_for_freshness( + discovery, + prepared.plan, + freshness_mode="required", + ) + + self.assertEqual(1, len(filtered.discovery.tasks)) + self.assertEqual( + frozenset({("//pkg:target", "pkg/target/test.outputs")}), + prepared.plan.staged_outputs, + ) + self.assertTrue(all(path.is_dir() for path in staged_roots)) + prepared.cleanup() + self.assertTrue(all(not path.exists() for path in staged_roots)) + + def test_fail_on_error_requires_a_non_skipped_result_for_fresh_output(self) -> None: + plan = FreshnessPlan( + selected_source="bep", + eligibility_enabled=True, + eligible_outputs=frozenset({("//pkg:target", "pkg/target/test.outputs")}), + ) + empty = DiscoveryResult( + outputs=(), + tasks=(), + discovered_by_type=tuple((kind, 0) for kind in PayloadType), + ) + with self.assertRaisesRegex(FreshnessError, "none produced"): + validate_fresh_outputs_accounted( + plan, + empty, + (), + expected_targets=(), + fail_on_error=True, + ) +if __name__ == "__main__": + unittest.main() diff --git a/tools/tests/python/test_uploader_launcher_tools.py b/tools/tests/python/test_uploader_launcher_tools.py new file mode 100644 index 00000000..241fb962 --- /dev/null +++ b/tools/tests/python/test_uploader_launcher_tools.py @@ -0,0 +1,317 @@ +#!/usr/bin/env python3 +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Exercise behavior-free Python launchers across runfiles layouts. + +Manifest-only and spaced-path cases protect Bazel portability on every platform. +""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +import shlex +import shutil +import subprocess +import sys +import tempfile +import unittest + + +def _repo_file(relative: str) -> Path: + for parent in (Path(__file__).resolve().parent, *Path(__file__).resolve().parents): + candidate = parent / relative + if candidate.is_file(): + return candidate.resolve() + raise FileNotFoundError(relative) + + +def _render_direct_launcher(template: Path, destination: Path, root: Path) -> None: + main = root / "main.py" + config = root / "config.json" + main.write_text("# selected interpreter owns execution\n", encoding="utf-8") + config.write_text("{}", encoding="utf-8") + rendered = template.read_text(encoding="utf-8") + substitutions = { + "PYTHON_MAIN_PATH": str(main), + "PYTHON_MAIN_RLOC": "missing/main.py", + "PYTHON_CONFIG_PATH": str(config), + "PYTHON_CONFIG_RLOC": "missing/config.json", + "PYTHON_CONFIG_NAME": "missing-config.json", + } + for key, value in substitutions.items(): + rendered = rendered.replace(f"__DDTPL_{key}__", value) + destination.write_text(rendered, encoding="utf-8") + + +def _write_versioned_python_shims(root: Path) -> tuple[Path, Path]: + bin_dir = root / "bin" + bin_dir.mkdir() + marker = root / "selected-python.txt" + old_python = bin_dir / "python3" + old_python.write_text( + "#!/bin/sh\n" + 'if [ "${1:-}" = "-c" ]; then exit 1; fi\n' + "exit 97\n", + encoding="utf-8", + ) + compatible_python = bin_dir / "python" + compatible_python.write_text( + "#!/bin/sh\n" + 'if [ "${1:-}" = "-c" ]; then exit 0; fi\n' + f"printf '%s\\n' python > {shlex.quote(str(marker))}\n", + encoding="utf-8", + ) + old_python.chmod(0o755) + compatible_python.chmod(0o755) + return bin_dir, marker + + +class LauncherTests(unittest.TestCase): + def test_launchers_skip_an_unsupported_python_candidate(self) -> None: + if os.name == "nt": + self.skipTest("POSIX shim test") + powershell = shutil.which("pwsh") or shutil.which("powershell") + cases = [("bash", "uploader_python_launcher.sh.tpl")] + if powershell: + cases.append((powershell, "uploader_python_launcher.ps1.tpl")) + + for executable, template_name in cases: + with self.subTest(template=template_name), tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + launcher = root / template_name.removesuffix(".tpl") + _render_direct_launcher( + _repo_file(f"tools/core/{template_name}"), + launcher, + root, + ) + if executable == "bash": + launcher.chmod(0o755) + command = [str(launcher)] + else: + command = [ + executable, + "-NoProfile", + "-NonInteractive", + "-File", + str(launcher), + ] + bin_dir, marker = _write_versioned_python_shims(root) + environment = dict(os.environ) + environment.pop("DD_TEST_OPTIMIZATION_PYTHON", None) + environment.pop("PYTHON", None) + environment["PATH"] = os.pathsep.join( + (str(bin_dir), environment.get("PATH", "")) + ) + + completed = subprocess.run( + command, + cwd=root, + env=environment, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + timeout=10, + ) + + self.assertEqual(0, completed.returncode, completed.stderr) + self.assertEqual("python", marker.read_text(encoding="utf-8").strip()) + + def test_unix_launcher_supports_manifest_only_runfiles_and_space_paths(self) -> None: + if os.name == "nt": + self.skipTest("Unix launcher test") + template = _repo_file("tools/core/uploader_python_launcher.sh.tpl") + main = _repo_file("tools/core/uploader_main.py") + with tempfile.TemporaryDirectory(prefix="uploader launcher ") as raw_root: + root = Path(raw_root) + launcher_dir = root / "generated files" + launcher_dir.mkdir() + testlogs = root / "empty testlogs" + testlogs.mkdir() + launcher = launcher_dir / "uploader script.sh" + config = launcher_dir / "uploader config.json" + config.write_text( + json.dumps( + { + "schema_version": 1, + "quiescent_sec": 0, + "max_wait_sec": 0, + } + ), + encoding="utf-8", + ) + rendered = template.read_text(encoding="utf-8") + substitutions = { + "PYTHON_MAIN_PATH": "missing/direct/uploader_main.py", + "PYTHON_MAIN_RLOC": "repo/tools/core/uploader main.py", + "PYTHON_CONFIG_PATH": "missing/direct/config.json", + "PYTHON_CONFIG_RLOC": "repo/generated/config.json", + "PYTHON_CONFIG_NAME": config.name, + } + for key, value in substitutions.items(): + rendered = rendered.replace(f"__DDTPL_{key}__", value) + self.assertNotIn("__DDTPL_", rendered) + launcher.write_text(rendered, encoding="utf-8") + launcher.chmod(0o755) + manifest = Path(f"{launcher}.runfiles_manifest") + manifest.write_text( + f" repo/tools/core/uploader\\smain.py {main}\n", + encoding="utf-8", + ) + environment = { + "PATH": os.environ.get("PATH", ""), + "DD_TEST_OPTIMIZATION_PYTHON": sys.executable, + "BUILD_WORKSPACE_DIRECTORY": str(root), + "TESTLOGS_DIR": str(testlogs), + } + + completed = subprocess.run( + [ + str(launcher), + "--debug", + "--dry-run", + "--allow-cached-payload-uploads", + ], + cwd=root, + env=environment, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + timeout=10, + ) + + self.assertEqual(0, completed.returncode, completed.stderr) + self.assertIn("summary: mode=dry-run", completed.stdout) + self.assertIn(f"manifest={manifest.resolve()}", completed.stderr) + + def test_powershell_launcher_finds_batch_runfiles_manifest(self) -> None: + if os.name == "nt": + self.skipTest("covered by the generated uploader Windows CI smoke") + powershell = ( + shutil.which("pwsh") + or shutil.which("powershell.exe") + or shutil.which("powershell") + ) + if powershell is None: + self.skipTest("PowerShell is not installed") + template = _repo_file("tools/core/uploader_python_launcher.ps1.tpl") + main = _repo_file("tools/core/uploader_main.py") + with tempfile.TemporaryDirectory(prefix="uploader launcher ") as raw_root: + root = Path(raw_root) + fake_python = root / "fake python" + fake_python.write_text( + "#!/usr/bin/env sh\nprintf '%s\\n' \"$RUNFILES_MANIFEST_FILE\"\n", + encoding="utf-8", + ) + fake_python.chmod(0o755) + launcher_dir = root / "generated files" + launcher_dir.mkdir() + testlogs = root / "empty testlogs" + testlogs.mkdir() + launcher = launcher_dir / "uploader.python.ps1" + config = launcher_dir / "uploader config.json" + config.write_text( + json.dumps( + { + "schema_version": 1, + "quiescent_sec": 0, + "max_wait_sec": 0, + } + ), + encoding="utf-8", + ) + rendered = template.read_text(encoding="utf-8") + substitutions = { + "PYTHON_MAIN_PATH": "missing/direct/uploader_main.py", + "PYTHON_MAIN_RLOC": "repo/tools/core/uploader_main.py", + "PYTHON_CONFIG_PATH": "missing/direct/config.json", + "PYTHON_CONFIG_RLOC": "repo/generated/config.json", + "PYTHON_CONFIG_NAME": config.name, + } + for key, value in substitutions.items(): + rendered = rendered.replace(f"__DDTPL_{key}__", value) + self.assertNotIn("__DDTPL_", rendered) + launcher.write_text(rendered, encoding="utf-8") + manifest = launcher_dir / "uploader.python.bat.runfiles_manifest" + manifest_main = root / "runtime files" / "uploader main.py" + manifest_main.parent.mkdir() + manifest_main.write_bytes(main.read_bytes()) + encoded_main = ( + str(manifest_main).replace("\\", r"\b").replace(" ", r"\s") + ) + manifest.write_text( + " repo/tools/core/uploader_main.py " + f"{encoded_main}\n", + encoding="utf-8", + ) + environment = dict(os.environ) + for name in ("RUNFILES_DIR", "RUNFILES_MANIFEST_FILE", "TEST_SRCDIR"): + environment.pop(name, None) + environment.update( + { + "DD_TEST_OPTIMIZATION_PYTHON": str(fake_python), + "BUILD_WORKSPACE_DIRECTORY": str(root), + "TESTLOGS_DIR": str(testlogs), + } + ) + + completed = subprocess.run( + [ + powershell, + "-NoProfile", + "-NonInteractive", + "-File", + str(launcher), + "--dry-run", + "--allow-cached-payload-uploads", + ], + cwd=root, + env=environment, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + timeout=10, + ) + + self.assertEqual(0, completed.returncode, completed.stderr) + self.assertEqual( + manifest.resolve(), + Path(completed.stdout.strip()).resolve(), + ) + + def test_launchers_contain_resolution_only_not_uploader_behavior(self) -> None: + for relative in ( + "tools/core/uploader_python_launcher.sh.tpl", + "tools/core/uploader_python_launcher.ps1.tpl", + ): + text = _repo_file(relative).read_text(encoding="utf-8") + with self.subTest(relative=relative): + self.assertIn("python_main", text.lower()) + self.assertNotIn("citestcycle", text.lower()) + self.assertNotIn("codeowners", text.lower()) + self.assertNotIn("multipart", text.lower()) + self.assertNotIn("payloads/tests", text.lower()) + self.assertIn( + "DD_TEST_OPTIMIZATION_UPLOADER_LAUNCHER_DIR", + text, + ) + if relative.endswith(".ps1.tpl"): + self.assertIn( + "$script:BatchLauncherPath.runfiles_manifest", + text, + ) + self.assertIn("$env:RUNFILES_MANIFEST_FILE =", text) + else: + self.assertIn("export RUNFILES_MANIFEST_FILE", text) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/tests/python/test_uploader_reporting_tools.py b/tools/tests/python/test_uploader_reporting_tools.py new file mode 100644 index 00000000..1a7634be --- /dev/null +++ b/tools/tests/python/test_uploader_reporting_tools.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Verify final aggregation and both human and JSON renderers. + +Shared assertions keep public legacy fields and new statistics consistent. +""" + +from __future__ import annotations + +import json +from pathlib import Path +import tempfile +import unittest + +from uploader_test_support import add_uploader_runtime_to_path + +add_uploader_runtime_to_path() + +from uploader_py.models import FileResult, FileStatus, PayloadType # noqa: E402 +from uploader_py.reporting import ( # noqa: E402 + AggregateReport, + LegacyReportContext, + write_schema_v1_report, + write_statistics_json, +) + + +class ReportingTests(unittest.TestCase): + def test_one_aggregate_drives_all_source_request_and_split_counters(self) -> None: + results = ( + FileResult( + task_id="test-1", + source_path="test.json", + payload_type=PayloadType.TEST, + status=FileStatus.SUCCEEDED, + events=5, + chunks_created=2, + chunks_uploaded=2, + requests_planned=2, + requests_attempted=3, + requests_succeeded=2, + retries=1, + source_deleted=True, + warning_codes=("schema_validation_failed",), + ), + FileResult( + task_id="coverage-1", + source_path="coverage.json", + payload_type=PayloadType.COVERAGE, + status=FileStatus.FAILED, + requests_planned=1, + requests_attempted=4, + requests_failed=1, + retries=3, + failure_code="upload_http_error", + ), + FileResult( + task_id="telemetry-1", + source_path="telemetry.json", + payload_type=PayloadType.TELEMETRY, + status=FileStatus.SKIPPED, + ), + ) + report = AggregateReport.create( + dry_run=False, + exit_code=1, + configured_workers=4, + worker_threads=3, + peak_active_workers=3, + elapsed_seconds=12.34567, + discovered_by_type={ + PayloadType.TEST: 2, + PayloadType.COVERAGE: 2, + PayloadType.TELEMETRY: 2, + }, + results=results, + ) + + stats = report.statistics() + self.assertEqual("partial_failure", stats["result"]) + self.assertEqual( + { + "discovered": 6, + "eligible": 3, + "processed": 3, + "succeeded": 1, + "failed": 1, + "skipped": 1, + "cancelled": 0, + "deleted": 1, + "retained": 2, + }, + stats["files"], + ) + self.assertEqual(1, stats["splitting"]["source_files_split"]) + self.assertEqual(2, stats["splitting"]["chunks_created"]) + self.assertEqual( + {"planned": 3, "attempted": 7, "succeeded": 2, "failed": 1, "retries": 4}, + stats["requests"], + ) + self.assertEqual( + {"succeeded": 1, "failed": 0, "skipped": 0}, + stats["payload_types"]["test"], + ) + self.assertEqual({"upload_http_error": 1}, stats["failures"]) + self.assertEqual( + {"schema_validation_failed": 1}, + stats["warnings"], + ) + lines = report.human_lines() + self.assertIn("mode=upload result=partial_failure", lines[0]) + self.assertIn("tests=1/0/0", lines[2]) + self.assertIn("planned=3 attempted=7", lines[4]) + + def test_dry_run_summary_makes_zero_network_activity_explicit(self) -> None: + report = AggregateReport.create( + dry_run=True, + exit_code=0, + configured_workers=4, + worker_threads=1, + peak_active_workers=1, + elapsed_seconds=0.5, + discovered_by_type={PayloadType.TEST: 1}, + results=( + FileResult( + task_id="test-1", + source_path="test.json", + payload_type=PayloadType.TEST, + status=FileStatus.SUCCEEDED, + chunks_created=2, + requests_planned=2, + ), + ), + ) + + stats = report.statistics() + self.assertEqual("dry-run", stats["mode"]) + self.assertEqual(2, stats["requests"]["planned"]) + self.assertEqual(0, stats["requests"]["attempted"]) + self.assertIn("planned=2 attempted=0", report.human_lines()[4]) + + def test_json_writer_uses_the_same_statistics_model(self) -> None: + report = AggregateReport.create( + dry_run=False, + exit_code=0, + configured_workers=2, + worker_threads=0, + peak_active_workers=0, + elapsed_seconds=0, + discovered_by_type={}, + results=(), + ) + with tempfile.TemporaryDirectory() as raw_root: + output = Path(raw_root) / "nested" / "report.json" + write_statistics_json(output, report) + written = json.loads(output.read_text(encoding="utf-8")) + + self.assertEqual(report.statistics(), written) + self.assertEqual(0, written["files"]["processed"]) + self.assertEqual("success", written["result"]) + + def test_invalid_coordinator_counters_fail_before_rendering(self) -> None: + with self.assertRaisesRegex(ValueError, "peak_active_workers"): + AggregateReport.create( + dry_run=False, + exit_code=0, + configured_workers=2, + worker_threads=1, + peak_active_workers=2, + elapsed_seconds=0, + discovered_by_type={}, + results=(), + ) + + def test_schema_v1_keeps_legacy_telemetry_semantics_for_partial_delivery(self) -> None: + report = AggregateReport.create( + dry_run=False, + exit_code=1, + configured_workers=2, + worker_threads=1, + peak_active_workers=1, + elapsed_seconds=1, + discovered_by_type={PayloadType.TELEMETRY: 1}, + results=( + FileResult( + task_id="telemetry-1", + source_path="telemetry.json", + payload_type=PayloadType.TELEMETRY, + status=FileStatus.FAILED, + requests_planned=2, + requests_attempted=5, + requests_succeeded=1, + requests_failed=1, + retries=3, + failure_code="upload_http_error", + ), + ), + ) + context = LegacyReportContext(test_outputs_dirs=1) + public = report.schema_v1_report(context) + + self.assertEqual("partial_failure", report.result_name) + self.assertEqual(1, public["schema_version"]) + self.assertEqual("dd-test-optimization-uploader", public["tool"]) + self.assertEqual( + {"processed": 1, "failed": 1, "skipped": 0}, + public["payloads"]["telemetry"], + ) + self.assertEqual(2, public["upload"]["payloads_attempted"]) + self.assertEqual(1, public["upload"]["payloads_uploaded"]) + self.assertEqual(report.statistics()["requests"], public["requests"]) + + with tempfile.TemporaryDirectory() as raw_root: + output = Path(raw_root) / "report.json" + write_schema_v1_report(output, report, context) + written = json.loads(output.read_text(encoding="utf-8")) + self.assertEqual(public, written) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/tests/python/test_uploader_resources_tools.py b/tools/tests/python/test_uploader_resources_tools.py new file mode 100644 index 00000000..7fb20547 --- /dev/null +++ b/tools/tests/python/test_uploader_resources_tools.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Verify pre-worker context, telemetry-facts, and schema loading. + +These tests ensure shared resources are resolved once and selected deterministically. +""" + +from __future__ import annotations + +import json +from pathlib import Path +import tempfile +import unittest + +from uploader_test_support import add_uploader_runtime_to_path + +add_uploader_runtime_to_path() + +from topt_runtime.runfiles import RunfilesResolver # noqa: E402 +from uploader_py.resources import ResourceError, ResourceInputs, load_resources # noqa: E402 + + +def _json(path: Path, value: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value), encoding="utf-8") + + +class ResourceLoadingTests(unittest.TestCase): + def test_runtime_selection_loads_sorted_keyed_contexts_and_facts(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + entries = [] + for repo, service in (("repo-b", "service-b"), ("repo-a", "service-a")): + directory = root / repo + context = directory / "context.json" + facts = directory / "telemetry_facts.json" + _json( + context, + { + "topt.sync.repository_name": repo, + "service.name": service, + "runtime.name": "go", + }, + ) + _json( + facts, + { + "schema_version": 1, + "service_name": service, + "runtime_name": "go", + "counts": [], + "distributions": [], + }, + ) + entries.append(f"{repo}={context.relative_to(root)}") + + loaded = load_resources( + RunfilesResolver.from_environment(environ={}, cwd=root), + ResourceInputs( + runtime_context_entries=tuple(entries), + runtime_selection=True, + invocation_cwd=root, + ), + ) + + self.assertEqual( + ("repo-a", "repo-b"), + tuple(record.repo_key for record in loaded.context_plan.by_repo), + ) + self.assertEqual( + "service-a", + loaded.context_plan.select("repo-a").values["service.name"], + ) + self.assertEqual( + "context_repo_not_found", + loaded.context_plan.select("different-repo").warning_code, + ) + self.assertEqual("service-a", loaded.primary_context["service.name"]) + self.assertEqual( + ( + (root / "repo-a" / "telemetry_facts.json").resolve(), + (root / "repo-b" / "telemetry_facts.json").resolve(), + ), + loaded.telemetry_facts_paths, + ) + + def test_runtime_selection_fails_before_optional_resource_fallback(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + resolver = RunfilesResolver.from_environment(environ={}, cwd=root) + with self.assertRaisesRegex(ResourceError, "at least one --context-entry"): + load_resources( + resolver, + ResourceInputs(runtime_selection=True, invocation_cwd=root), + ) + + context = root / "context.json" + facts = root / "telemetry_facts.json" + _json( + context, + { + "topt.sync.repository_name": "different-repo", + "service.name": "service", + "runtime.name": "go", + }, + ) + _json( + facts, + { + "schema_version": 1, + "service_name": "service", + "runtime_name": "go", + "counts": [], + "distributions": [], + }, + ) + with self.assertRaisesRegex(ResourceError, "identity or schema mismatch"): + load_resources( + resolver, + ResourceInputs( + runtime_context_entries=(f"repo={context}",), + runtime_selection=True, + invocation_cwd=root, + ), + ) + + def test_manifest_loads_normalized_contexts_facts_and_schema_once(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + context_a = root / "context-a.json" + context_b = root / "context-b.json" + facts = root / "telemetry facts.json" + schema = root / "schema.json" + _json(context_a, {"env": "a", "ci.workspace_path": "/workspace"}) + _json(context_b, {"env": "b"}) + _json(facts, {"service_name": "service"}) + _json(schema, {"type": "object"}) + context_manifest = root / "contexts.manifest" + context_manifest.write_text( + f"canonical+repo_a\tmissing-a\t{context_a}\n" + f"repo_b\tmissing-b\t{context_b}\n", + encoding="utf-8", + ) + facts_manifest = root / "facts.manifest" + facts_manifest.write_text( + f"missing-facts\t{facts}\n", + encoding="utf-8", + ) + resolver = RunfilesResolver.from_environment(environ={}, cwd=root) + + loaded = load_resources( + resolver, + ResourceInputs( + context_manifest_paths=(str(context_manifest),), + telemetry_facts_manifest_paths=(str(facts_manifest),), + schema_paths=(str(schema),), + ), + ) + + self.assertEqual("a", loaded.context_plan.select("repo_a").values["env"]) + self.assertEqual("b", loaded.context_plan.select("repo_b").values["env"]) + self.assertEqual("/workspace", loaded.context_workspace) + self.assertEqual((facts.resolve(),), loaded.telemetry_facts_paths) + self.assertEqual({"type": "object"}, loaded.schema) + self.assertEqual((), loaded.warning_codes) + + def test_valid_override_wins_and_adds_sibling_telemetry_facts(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + override = root / "context.json" + sibling = root / "telemetry_facts.json" + _json(override, {"env": "override"}) + _json(sibling, {"service_name": "override-service"}) + resolver = RunfilesResolver.from_environment(environ={}, cwd=root) + + loaded = load_resources( + resolver, + ResourceInputs(context_override=override), + ) + + self.assertTrue(loaded.context_plan.override) + self.assertEqual("override", loaded.context_plan.select("anything").values["env"]) + self.assertEqual((sibling.resolve(),), loaded.telemetry_facts_paths) + + def test_invalid_optional_inputs_warn_and_disable_without_crashing(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + bad_override = root / "bad-context.json" + bad_override.write_text('{"env":NaN}', encoding="utf-8") + bad_schema = root / "bad-schema.json" + bad_schema.write_text("not-json", encoding="utf-8") + resolver = RunfilesResolver.from_environment(environ={}, cwd=root) + + loaded = load_resources( + resolver, + ResourceInputs( + context_override=bad_override, + context_manifest_paths=("missing.manifest",), + schema_paths=(str(bad_schema),), + ), + ) + + self.assertIsNone(loaded.context_plan.primary) + self.assertIsNone(loaded.schema) + self.assertIn("context_override_invalid", loaded.warning_codes) + self.assertIn("context_manifest_unresolved", loaded.warning_codes) + self.assertIn("schema_invalid_or_unresolved", loaded.warning_codes) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/tests/python/test_uploader_telemetry_tools.py b/tools/tests/python/test_uploader_telemetry_tools.py new file mode 100644 index 00000000..998d896f --- /dev/null +++ b/tools/tests/python/test_uploader_telemetry_tools.py @@ -0,0 +1,478 @@ +#!/usr/bin/env python3 +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Exercise telemetry planning and complete per-file processing. + +Coverage protects cross-file correlation without introducing worker synchronization. +""" + +from __future__ import annotations + +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from uploader_test_support import add_uploader_runtime_to_path + +add_uploader_runtime_to_path() + +from uploader_py.codeowners import CodeOwnersMatcher # noqa: E402 +from uploader_py.endpoints import EndpointSet # noqa: E402 +from uploader_py.enrichment import ContextPlan # noqa: E402 +from uploader_py.file_worker import WorkerRuntime, process_file # noqa: E402 +from uploader_py.models import FileStatus, FileTask, PayloadType # noqa: E402 +from uploader_py.telemetry import TelemetryPlan, build_telemetry_plan # noqa: E402 +from uploader_py.transport import HttpResult, prepare_json_request # noqa: E402 + + +class _Transport: + def __init__(self, *results: HttpResult) -> None: + self.results = list(results) + self.calls: list[dict[str, object]] = [] + + def post_json( + self, + url, + headers, + body, + *, + gzip_body=False, + content_encoding=None, + ): + self.calls.append( + { + "url": url, + "headers": dict(headers), + "body": Path(body).read_bytes(), + "gzip": gzip_body, + "content_encoding": content_encoding, + } + ) + return self.results.pop(0) if self.results else HttpResult(200, 1) + + +class _RetryingSourceMutationTransport: + """Simulate two transport attempts while the original source changes.""" + + def __init__(self, source: Path, replacement: bytes) -> None: + self.source = source + self.replacement = replacement + self.prepared_path: Path | None = None + self.declared_length = 0 + self.bodies: tuple[bytes, bytes] = () + + def post_json( + self, + url, + headers, + body, + *, + gzip_body=False, + content_encoding=None, + ): + self.prepared_path = Path(body) + request = prepare_json_request( + url, + headers, + self.prepared_path, + gzip_body=gzip_body, + content_encoding=content_encoding, + ) + self.declared_length = int(request.headers["Content-Length"]) + with request.body_factory() as first_attempt: + first_body = first_attempt.read() + self.source.write_bytes(self.replacement) + with request.body_factory() as second_attempt: + second_body = second_attempt.read() + self.bodies = (first_body, second_body) + return HttpResult(200, 2, retry_delays=(2.0,)) + + +def _task(source: Path) -> FileTask: + return FileTask( + task_id="telemetry-1", + source_path=source, + display_path=f"payloads/telemetry/{source.name}", + payload_type=PayloadType.TELEMETRY, + ) + + +def _runtime(root: Path, plan, *, dry_run: bool = False) -> WorkerRuntime: + return WorkerRuntime( + endpoints=EndpointSet( + agentless=True, + site="datadoghq.com", + test_url="https://test.invalid", + coverage_url="https://coverage.invalid", + telemetry_url="https://telemetry.invalid/api/v2/apmtelemetry", + ), + invocation_temp_root=root, + context_plan=ContextPlan(None), + codeowners_matcher=CodeOwnersMatcher(None, "", ""), + runtime_id="uploader-runtime", + rules_version="rules-1", + uploader_version="uploader-1", + api_key="secret", + telemetry_session_id="session-fallback", + telemetry_plan=plan, + dry_run=dry_run, + ) + + +def _synthetic_telemetry_fixture( + root: Path, +) -> tuple[Path, Path, FileTask, TelemetryPlan]: + """Create one source whose rule facts require a second telemetry request.""" + source = root / "app-started.json" + source.write_text( + json.dumps( + { + "api_version": "v2", + "request_type": "app-started", + "runtime_id": "runtime-a", + "seq_id": 7, + "application": { + "service_name": "service-a", + "language_name": "go", + "tracer_version": "1.0.0", + }, + "host": {"hostname": "builder"}, + "payload": {}, + } + ), + encoding="utf-8", + ) + facts = root / "facts.json" + facts.write_text( + json.dumps( + { + "service_name": "service-a", + "runtime_name": "go", + "distributions": [ + {"name": "duration", "value": [1, 2], "tags": []} + ], + } + ), + encoding="utf-8", + ) + task = _task(source) + return source, facts, task, build_telemetry_plan( + (task,), + (facts,), + clock=lambda: 456, + ) + + +class TelemetryWorkerTests(unittest.TestCase): + def test_non_finite_telemetry_json_fails_before_http(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + source = root / "telemetry.json" + source.write_text( + '{"api_version":"v2","request_type":"app-started",' + '"runtime_id":"runtime-a","application":{},"payload":NaN}', + encoding="utf-8", + ) + task = _task(source) + transport = _Transport() + + result = process_file( + task, + _runtime(root, build_telemetry_plan((task,), ())), + transport, + ) + + self.assertEqual(FileStatus.FAILED, result.status) + self.assertEqual("invalid_telemetry_json", result.failure_code) + self.assertEqual([], transport.calls) + self.assertTrue(source.exists()) + + def test_primary_body_is_task_local_and_stable_across_retry(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + source = root / "telemetry.json" + original = ( + b'\xef\xbb\xbf{\n "api_version": "v2",\n' + b' "request_type": "app-started",\n' + b' "runtime_id": "runtime-a",\n' + b' "application": {},\n "payload": []\n}\n' + ) + replacement = ( + b'{"api_version":"v2","request_type":"app-started",' + b'"runtime_id":"runtime-b","application":{},"payload":[1]}' + ) + source.write_bytes(original) + task = _task(source) + plan = build_telemetry_plan((task,), ()) + transport = _RetryingSourceMutationTransport(source, replacement) + + result = process_file(task, _runtime(root, plan), transport) + + self.assertEqual(FileStatus.SUCCEEDED, result.status) + self.assertEqual(2, result.requests_attempted) + self.assertEqual(1, result.retries) + self.assertEqual((original, original), transport.bodies) + self.assertEqual(len(original), transport.declared_length) + self.assertIsNotNone(transport.prepared_path) + self.assertNotEqual(source, transport.prepared_path) + self.assertEqual("telemetry_body.json", transport.prepared_path.name) + self.assertFalse(source.exists()) + + def test_message_batch_anchor_is_augmented_by_its_own_worker(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + source = root / "telemetry.json" + source.write_text( + json.dumps( + { + "api_version": "v2", + "request_type": "message-batch", + "runtime_id": "runtime-a", + "seq_id": 4, + "application": { + "service_name": "service-a", + "language_name": "python", + "tracer_version": "2.0.0", + "env": "none", + }, + "payload": [ + { + "request_type": "generate-metrics", + "payload": { + "series": [{"tags": ["provider:bazel"]}] + }, + } + ], + } + ), + encoding="utf-8", + ) + facts = root / "telemetry_facts.json" + facts.write_text( + json.dumps( + { + "service_name": "service-a", + "runtime_name": "python", + "env": "ci", + "counts": [ + { + "name": "event_created", + "value": 2, + "tags": ["provider:bazel"], + } + ], + } + ), + encoding="utf-8", + ) + task = _task(source) + plan = build_telemetry_plan( + (task,), + (facts,), + primary_context={"ci.provider.name": "github"}, + clock=lambda: 123.9, + ) + transport = _Transport(HttpResult(202, 1)) + + result = process_file(task, _runtime(root, plan), transport) + + self.assertEqual(FileStatus.SUCCEEDED, result.status) + self.assertEqual(1, result.requests_planned) + self.assertTrue(result.source_deleted) + body = json.loads(transport.calls[0]["body"]) + self.assertEqual("ci", body["application"]["env"]) + self.assertEqual(2, len(body["payload"])) + self.assertEqual( + ["provider:bazel/github"], + body["payload"][0]["payload"]["series"][0]["tags"], + ) + added = body["payload"][1]["payload"]["series"][0] + self.assertEqual([[123, 2]], added["points"]) + self.assertEqual(["provider:bazel/github"], added["tags"]) + headers = transport.calls[0]["headers"] + self.assertEqual("v2", headers["DD-Telemetry-API-Version"]) + self.assertEqual("message-batch", headers["DD-Telemetry-Request-Type"]) + self.assertEqual("runtime-a", headers["DD-Session-ID"]) + self.assertEqual("secret", headers["DD-API-KEY"]) + + def test_synthetic_failure_is_the_only_request_retried(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + source, facts, task, plan = _synthetic_telemetry_fixture(root) + transport = _Transport(HttpResult(200, 1), HttpResult(500, 4)) + + result = process_file(task, _runtime(root, plan), transport) + + self.assertEqual(FileStatus.FAILED, result.status) + self.assertEqual(2, result.requests_planned) + self.assertEqual(5, result.requests_attempted) + self.assertEqual(1, result.requests_succeeded) + self.assertEqual(1, result.requests_failed) + self.assertEqual(3, result.retries) + self.assertTrue(source.exists()) + synthetic = json.loads(transport.calls[1]["body"]) + self.assertEqual("message-batch", synthetic["request_type"]) + self.assertEqual(8, synthetic["seq_id"]) + self.assertEqual(456, synthetic["tracer_time"]) + self.assertEqual( + "message-batch", + transport.calls[1]["headers"]["DD-Telemetry-Request-Type"], + ) + + retry_plan = build_telemetry_plan((task,), (facts,), clock=lambda: 789) + retry_transport = _Transport(HttpResult(200, 1)) + retry_result = process_file( + task, + _runtime(root, retry_plan), + retry_transport, + ) + + self.assertEqual(FileStatus.SUCCEEDED, retry_result.status) + self.assertEqual(1, len(retry_transport.calls)) + retry_body = json.loads(retry_transport.calls[0]["body"]) + self.assertEqual("message-batch", retry_body["request_type"]) + self.assertEqual(8, retry_body["seq_id"]) + self.assertEqual(456, retry_body["tracer_time"]) + self.assertEqual(1, len(retry_body["payload"])) + self.assertNotIn( + "_dd_test_optimization_retry_without_augmentation", + retry_body, + ) + self.assertFalse(source.exists()) + + def test_telemetry_persistence_failure_keeps_original_source(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + source, _facts, task, plan = _synthetic_telemetry_fixture(root) + original = source.read_bytes() + transport = _Transport(HttpResult(200, 1), HttpResult(500, 4)) + + with patch( + "uploader_py.file_worker.tempfile.NamedTemporaryFile", + side_effect=OSError("read-only directory"), + ): + result = process_file(task, _runtime(root, plan), transport) + + self.assertEqual(FileStatus.FAILED, result.status) + self.assertIn( + "failed_telemetry_request_persist_failed", + result.warning_codes, + ) + self.assertEqual(original, source.read_bytes()) + + def test_source_failure_does_not_block_or_repeat_synthetic_request(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + source, facts, task, plan = _synthetic_telemetry_fixture(root) + transport = _Transport(HttpResult(500, 4), HttpResult(200, 1)) + + result = process_file(task, _runtime(root, plan), transport) + + self.assertEqual(FileStatus.FAILED, result.status) + self.assertEqual(2, len(transport.calls)) + self.assertEqual(1, result.requests_succeeded) + self.assertEqual(1, result.requests_failed) + self.assertTrue(source.exists()) + + retry_plan = build_telemetry_plan((task,), (facts,), clock=lambda: 789) + retry_transport = _Transport(HttpResult(200, 1)) + retry_result = process_file( + task, + _runtime(root, retry_plan), + retry_transport, + ) + + self.assertEqual(FileStatus.SUCCEEDED, retry_result.status) + self.assertEqual(1, len(retry_transport.calls)) + retry_body = json.loads(retry_transport.calls[0]["body"]) + self.assertEqual("app-started", retry_body["request_type"]) + self.assertNotIn( + "_dd_test_optimization_retry_without_augmentation", + retry_body, + ) + self.assertFalse(source.exists()) + + def test_dry_run_prepares_source_and_synthetic_without_transport(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + source = root / "telemetry.json" + source.write_text( + json.dumps( + { + "api_version": "v2", + "request_type": "app-started", + "application": { + "service_name": "service-a", + "language_name": "ruby", + }, + } + ), + encoding="utf-8", + ) + facts = root / "facts.json" + facts.write_text( + json.dumps( + { + "service_name": "service-a", + "counts": [{"name": "created", "value": 1}], + } + ), + encoding="utf-8", + ) + task = _task(source) + plan = build_telemetry_plan((task,), (facts,), clock=lambda: 1) + transport = _Transport() + + result = process_file(task, _runtime(root, plan, dry_run=True), transport) + + self.assertEqual(FileStatus.SUCCEEDED, result.status) + self.assertEqual(2, result.requests_planned) + self.assertEqual(0, result.requests_attempted) + self.assertEqual([], transport.calls) + self.assertTrue(source.exists()) + + def test_ambiguous_cross_language_facts_are_not_assigned(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + tasks = [] + for language in ("go", "python"): + source = root / f"{language}.json" + source.write_text( + json.dumps( + { + "api_version": "v2", + "request_type": "app-started", + "application": { + "service_name": "shared", + "language_name": language, + }, + } + ), + encoding="utf-8", + ) + tasks.append(_task(source)) + facts = root / "facts.json" + facts.write_text( + json.dumps( + { + "service_name": "shared", + "counts": [{"name": "created", "value": 1}], + } + ), + encoding="utf-8", + ) + + plan = build_telemetry_plan(tasks, (facts,)) + + self.assertEqual((), plan.entries) + self.assertIn("telemetry_facts_language_ambiguous", plan.warning_codes) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/tests/python/test_uploader_tools.py b/tools/tests/python/test_uploader_tools.py new file mode 100644 index 00000000..46e8ad69 --- /dev/null +++ b/tools/tests/python/test_uploader_tools.py @@ -0,0 +1,1284 @@ +#!/usr/bin/env python3 +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Characterize configuration, runfiles, locking, splitting, and startup. + +These foundation tests protect public compatibility below the upload pipelines. +""" + +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor +from dataclasses import FrozenInstanceError +import hashlib +import io +import json +import os +from pathlib import Path +import tempfile +import threading +import time +import unittest +from unittest import mock + +from uploader_test_support import ( + add_uploader_runtime_to_path, + resolve_runfile as _runfile, +) + +add_uploader_runtime_to_path() + +from uploader_py.config import ( # noqa: E402 + ConfigError, + DEFAULT_EXPECTED_ENRICHED_TAGS, + load_rule_config, + parse_uploader_config, + validate_upload_credentials, +) +from uploader_py.main import ( # noqa: E402 + main as uploader_main, + python_version_is_supported, +) +from uploader_py.endpoints import build_endpoints, normalize_dd_site # noqa: E402 +from uploader_py.logging_utils import ( # noqa: E402 + configure_logging, + redact_header_value, + redact_url, +) +from uploader_py.locking import ( # noqa: E402 + WorkspaceLock, + WorkspaceLockError, + workspace_lock_name, +) +from uploader_py.json_utils import strict_json_dumps, strict_json_loads # noqa: E402 +from uploader_py.models import ( # noqa: E402 + DEFAULT_WORKERS, + MAX_TEST_PAYLOAD_BYTES, + FileStatus, + FileTask, + PayloadType, +) +from uploader_py.splitting import ( # noqa: E402 + TestPayloadSplitError, + compact_json_bytes, + prepare_test_chunks, +) +from uploader_py.temporary import ( # noqa: E402 + TemporaryDirectoryError, + invocation_temporary_directory, + task_temporary_directory, +) +from topt_runtime.runfiles import ( # noqa: E402 + RunfileResolutionError, + RunfilesResolver, + runfile_candidates, +) +from validate_payload_schema import validate_payload # noqa: E402 + + +LEGACY_CLI_FLAGS = { + "--allow-cached-payload-uploads", + "--artifact-source", + "--artifact-staging-dir", + "--bep-artifact-downloader", + "--bep-artifact-downloader-timeout-sec", + "--bep-json", + "--dry-run", + "--execution-log-json", + "--execution-log-mode", + "--expected-enriched-tag", + "--freshness-mode", + "--freshness-source", + "--remote-artifacts", + "--report-json", + "--validate-enrichment", +} +NEW_CLI_FLAGS = {"--debug", "--workers"} +LEGACY_RUNTIME_ENVIRONMENT = { + "DD_TEST_OPTIMIZATION_AGENT_URL", + "DD_TEST_OPTIMIZATION_AGENTLESS_URL", + "DD_TEST_OPTIMIZATION_ARTIFACT_SOURCE", + "DD_TEST_OPTIMIZATION_ARTIFACT_STAGING_DIR", + "DD_TEST_OPTIMIZATION_BEP_ARTIFACT_DOWNLOADER", + "DD_TEST_OPTIMIZATION_BEP_ARTIFACT_DOWNLOADER_TIMEOUT_SEC", + "DD_TEST_OPTIMIZATION_BEP_JSON", + "DD_TEST_OPTIMIZATION_CODEOWNERS_FILE", + "DD_TEST_OPTIMIZATION_CONTEXT_JSON", + "DD_TEST_OPTIMIZATION_DEBUG", + "DD_TEST_OPTIMIZATION_EXECUTION_LOG_JSON", + "DD_TEST_OPTIMIZATION_EXECUTION_LOG_MODE", + "DD_TEST_OPTIMIZATION_FILTER_PREFIX", + "DD_TEST_OPTIMIZATION_FRESHNESS_MODE", + "DD_TEST_OPTIMIZATION_FRESHNESS_SOURCE", + "DD_TEST_OPTIMIZATION_GZIP", + "DD_TEST_OPTIMIZATION_KEEP_PAYLOADS", + "DD_TEST_OPTIMIZATION_MAX_DEPTH", + "DD_TEST_OPTIMIZATION_MAX_WAIT_SEC", + "DD_TEST_OPTIMIZATION_QUIESCENT_SEC", + "DD_TEST_OPTIMIZATION_REMOTE_ARTIFACTS", + "DD_TEST_OPTIMIZATION_UPLOADER_REPORT_JSON", +} +RULE_ATTRIBUTES = { + "quiescent_sec", + "max_wait_sec", + "fail_on_error", + "debug", + "keep_payloads", + "filter_prefix", + "gzip_payloads", + "workers", + "use_python_uploader", + "data", + "expected_targets", + "expected_targets_file", +} + + +class UploaderConfigTests(unittest.TestCase): + def setUp(self) -> None: + self.temp_dir = tempfile.TemporaryDirectory() + self.root = Path(self.temp_dir.name) + self.config_path = self.root / "uploader.config.json" + self.write_config() + + def tearDown(self) -> None: + self.temp_dir.cleanup() + + def write_config(self, **overrides: object) -> None: + body: dict[str, object] = { + "schema_version": 1, + "quiescent_sec": 10, + "max_wait_sec": 300, + "fail_on_error": False, + "debug": False, + "keep_payloads": False, + "filter_prefix": False, + "gzip_payloads": False, + "workers": 8, + "expected_targets": ["//pkg:test"], + } + body.update(overrides) + self.config_path.write_text(json.dumps(body), encoding="utf-8") + + def parse(self, *arguments: str, environ: dict[str, str] | None = None): + return parse_uploader_config( + ["--config", str(self.config_path), "--dry-run", *arguments], + environ={} if environ is None else environ, + cwd=self.root, + ) + + def test_rule_config_defaults_and_fixed_contracts(self) -> None: + rule = load_rule_config(self.config_path) + self.assertEqual(8, DEFAULT_WORKERS) + self.assertEqual(DEFAULT_WORKERS, rule.workers) + self.assertEqual(("//pkg:test",), rule.expected_targets) + self.assertEqual(4_718_592, MAX_TEST_PAYLOAD_BYTES) + + def test_rule_config_is_strictly_typed(self) -> None: + invalid_values = { + "schema_version": 2, + "quiescent_sec": -1, + "max_wait_sec": "300", + "fail_on_error": 1, + "workers": 0, + "workspace_name": 1, + "expected_targets": "//pkg:test", + } + for name, value in invalid_values.items(): + with self.subTest(name=name), self.assertRaises(ConfigError): + self.write_config(**{name: value}) + load_rule_config(self.config_path) + + def test_non_finite_numbers_are_rejected_as_non_standard_json(self) -> None: + for constant in ("NaN", "Infinity", "-Infinity"): + with self.subTest(constant=constant): + with self.assertRaises(json.JSONDecodeError): + strict_json_loads(f'{{"value":{constant}}}') + for value in (float("nan"), float("inf"), float("-inf")): + with self.subTest(value=value), self.assertRaises(ValueError): + strict_json_dumps({"value": value}) + + self.config_path.write_text( + '{"schema_version":1,"workers":NaN}', + encoding="utf-8", + ) + with self.assertRaisesRegex(ConfigError, "invalid uploader config JSON"): + load_rule_config(self.config_path) + + def test_cli_environment_and_rule_precedence(self) -> None: + self.write_config(debug=True, workers=2) + config = self.parse( + "--debug", + "--workers=7", + environ={ + "DD_TEST_OPTIMIZATION_DEBUG": "false", + "DD_TEST_OPTIMIZATION_WORKERS": "6", + }, + ) + self.assertTrue(config.debug) + self.assertEqual(7, config.workers) + + config = self.parse( + environ={ + "DD_TEST_OPTIMIZATION_DEBUG": "false", + "DD_TEST_OPTIMIZATION_WORKERS": "6", + } + ) + self.assertFalse(config.debug) + self.assertEqual(6, config.workers) + + config = self.parse() + self.assertTrue(config.debug) + self.assertEqual(2, config.workers) + + def test_workspace_lock_scope_preserves_legacy_unresolved_path(self) -> None: + actual_workspace = self.root / "actual-workspace" + actual_workspace.mkdir() + workspace_link = self.root / "workspace-link" + try: + workspace_link.symlink_to(actual_workspace, target_is_directory=True) + except (OSError, NotImplementedError) as exc: + self.skipTest(f"directory symlinks unavailable: {exc}") + + invocation_cwd = self.root / "invocation-cwd" + invocation_cwd.mkdir() + launcher_directory = self.root / "launcher" + launcher_directory.mkdir() + config = parse_uploader_config( + ["--config", str(self.config_path), "--dry-run"], + environ={ + "BUILD_WORKSPACE_DIRECTORY": str(workspace_link), + "DD_TEST_OPTIMIZATION_UPLOADER_LAUNCHER_DIR": str( + launcher_directory + ), + }, + cwd=invocation_cwd, + ) + + self.assertEqual(actual_workspace.resolve(), config.workspace) + self.assertEqual(str(workspace_link), config.lock_workspace) + self.assertEqual(invocation_cwd.absolute(), config.invocation_cwd) + self.assertEqual(launcher_directory, config.launcher_directory) + self.assertEqual( + workspace_lock_name(str(workspace_link)), + workspace_lock_name(config.lock_workspace), + ) + self.assertNotEqual( + workspace_lock_name(str(workspace_link)), + workspace_lock_name(config.workspace), + ) + + def test_legacy_boolean_normalization_is_preserved(self) -> None: + config = self.parse( + environ={ + "DD_TEST_OPTIMIZATION_KEEP_PAYLOADS": "YES", + "DD_TEST_OPTIMIZATION_FILTER_PREFIX": "unexpected", + "DD_TEST_OPTIMIZATION_GZIP": "1", + } + ) + self.assertTrue(config.keep_payloads) + self.assertFalse(config.filter_prefix) + self.assertTrue(config.gzip_payloads) + + def test_numeric_controls_and_worker_validation(self) -> None: + config = self.parse( + environ={ + "DD_TEST_OPTIMIZATION_QUIESCENT_SEC": "0", + "DD_TEST_OPTIMIZATION_MAX_WAIT_SEC": "0", + "DD_TEST_OPTIMIZATION_MAX_DEPTH": "12", + } + ) + self.assertEqual(0, config.quiescent_sec) + self.assertEqual(0, config.max_wait_sec) + self.assertEqual(12, config.max_depth) + + for arguments, environment in [ + (("--workers=0",), {}), + (("--workers=-1",), {}), + ((), {"DD_TEST_OPTIMIZATION_WORKERS": "many"}), + ((), {"DD_TEST_OPTIMIZATION_MAX_DEPTH": "-1"}), + ]: + with self.subTest(arguments=arguments, environment=environment): + with self.assertRaises(ConfigError): + self.parse(*arguments, environ=environment) + + def test_freshness_precedence_and_allow_cached_override(self) -> None: + config = self.parse( + "--freshness-mode=required", + "--execution-log-mode=optional", + environ={"DD_TEST_OPTIMIZATION_FRESHNESS_MODE": "disabled"}, + ) + self.assertEqual("required", config.freshness_mode) + + config = self.parse( + "--allow-cached-payload-uploads", + environ={"DD_TEST_OPTIMIZATION_FRESHNESS_MODE": "required"}, + ) + self.assertEqual("disabled", config.freshness_mode) + self.assertTrue(config.freshness_disabled_explicitly) + + config = self.parse( + environ={"DD_TEST_OPTIMIZATION_EXECUTION_LOG_MODE": "OPTIONAL"} + ) + self.assertEqual("optional", config.freshness_mode) + + def test_repeatable_values_and_expected_tag_defaults(self) -> None: + config = self.parse( + "--bep-json=cli-one.json", + "--bep-json", + "cli-two.json", + environ={"DD_TEST_OPTIMIZATION_BEP_JSON": "environment.json"}, + ) + self.assertEqual( + (Path("environment.json"), Path("cli-one.json"), Path("cli-two.json")), + config.bep_json_files, + ) + self.assertEqual(DEFAULT_EXPECTED_ENRICHED_TAGS, config.expected_enriched_tags) + + config = self.parse( + "--expected-enriched-tag=git.commit.sha", + "--expected-enriched-tag", + "bazel.target", + ) + self.assertEqual(("git.commit.sha", "bazel.target"), config.expected_enriched_tags) + + config = self.parse( + "--expected-target=//pkg:a", + "--expected-target", + "//pkg:b", + "--context-entry=repo-a=/tmp/a/context.json", + "--context-entry", + "repo-b=/tmp/b/context.json", + ) + self.assertEqual(("//pkg:a", "//pkg:b"), config.runtime_expected_targets) + self.assertEqual( + ("repo-a=/tmp/a/context.json", "repo-b=/tmp/b/context.json"), + config.runtime_context_entries, + ) + + def test_artifact_report_and_environment_paths(self) -> None: + config = self.parse( + "--artifact-source=BEP", + "--remote-artifacts=DOWNLOAD", + "--artifact-staging-dir=relative staging", + "--report-json=cli-report.json", + environ={ + "DD_TEST_OPTIMIZATION_UPLOADER_REPORT_JSON": "env-report.json", + "DD_TEST_OPTIMIZATION_EXECUTION_LOG_JSON": "execution.json", + "DD_TEST_OPTIMIZATION_BEP_ARTIFACT_DOWNLOADER": "fetch tool", + "TESTLOGS_DIR": "custom testlogs", + }, + ) + self.assertEqual("bep", config.artifact_source) + self.assertEqual("download", config.remote_artifacts) + self.assertEqual( + self.root.resolve() / "relative staging", + config.artifact_staging_dir, + ) + self.assertEqual(Path("cli-report.json"), config.report_json) + self.assertEqual(Path("execution.json"), config.execution_log_json) + self.assertEqual(Path("fetch tool"), config.bep_artifact_downloader) + self.assertEqual(Path("custom testlogs"), config.testlogs_dir) + + def test_positive_decimal_validation(self) -> None: + for accepted in (".5", "1", "1.", "+2.25"): + with self.subTest(accepted=accepted): + config = self.parse( + f"--bep-artifact-downloader-timeout-sec={accepted}" + ) + self.assertGreater(config.bep_artifact_downloader_timeout_sec, 0) + for rejected in ("0", "-1", "nan", "1e3", ""): + with self.subTest(rejected=rejected), self.assertRaises(ConfigError): + self.parse(f"--bep-artifact-downloader-timeout-sec={rejected}") + + def test_validate_enrichment_is_allowed_during_upload(self) -> None: + config = parse_uploader_config( + ["--config", str(self.config_path), "--validate-enrichment"], + environ={}, + cwd=self.root, + ) + + self.assertFalse(config.dry_run) + self.assertTrue(config.validate_enrichment) + + def test_unknown_and_abbreviated_arguments_exit_two(self) -> None: + for argument in ("--unknown", "--dry"): + with self.subTest(argument=argument): + with self.assertRaisesRegex(SystemExit, "2"): + with mock.patch("sys.stderr", io.StringIO()): + parse_uploader_config( + ["--config", str(self.config_path), argument], + environ={}, + cwd=self.root, + ) + + def test_credentials_are_validated_after_mode_resolution(self) -> None: + dry_run = self.parse() + validate_upload_credentials(dry_run) + self.assertEqual("datadoghq.com", dry_run.site) + + real_upload = parse_uploader_config( + ["--config", str(self.config_path)], environ={}, cwd=self.root + ) + with self.assertRaisesRegex(ConfigError, "DD_API_KEY"): + validate_upload_credentials(real_upload) + + evp = parse_uploader_config( + ["--config", str(self.config_path)], + environ={"DD_TEST_OPTIMIZATION_AGENT_URL": "http://localhost:8126"}, + cwd=self.root, + ) + validate_upload_credentials(evp) + + def test_ci_environment_false_markers_match_legacy_runtime(self) -> None: + for value in ("", "0", "false", "FALSE", " no "): + with self.subTest(value=value): + self.assertFalse(self.parse(environ={"CI": value}).ci) + for value in ("1", "true", "yes"): + with self.subTest(value=value): + self.assertTrue(self.parse(environ={"CI": value}).ci) + + def test_dd_site_normalization_and_endpoint_modes(self) -> None: + accepted = { + "": "datadoghq.com", + " APP.DatadogHQ.EU ": "datadoghq.eu", + "https://api.us5.datadoghq.com/path?query=yes#fragment": ( + "us5.datadoghq.com" + ), + } + for raw, expected in accepted.items(): + with self.subTest(raw=raw): + self.assertEqual(expected, normalize_dd_site(raw)) + + for raw in ( + "https://", + "user@example.com", + "example.com:443", + ".example.com", + "example..com", + "bad_name.example", + ): + with self.subTest(raw=raw), self.assertRaises(ConfigError): + normalize_dd_site(raw) + + direct = build_endpoints( + self.parse( + environ={ + "DD_SITE": "app.datadoghq.eu", + "DD_TEST_OPTIMIZATION_AGENTLESS_URL": "https://mock.invalid/root/", + } + ) + ) + self.assertTrue(direct.agentless) + self.assertEqual("https://mock.invalid/root/api/v2/citestcycle", direct.test_url) + self.assertEqual("datadoghq.eu", direct.site) + + evp = build_endpoints( + self.parse( + environ={ + "DD_TEST_OPTIMIZATION_AGENT_URL": "http://localhost:8126", + "DD_TEST_OPTIMIZATION_AGENTLESS_URL": "https://ignored.invalid", + } + ) + ) + self.assertFalse(evp.agentless) + self.assertEqual( + "http://localhost:8126/evp_proxy/v2/api/v2/citestcycle", + evp.test_url, + ) + + def test_endpoint_configuration_errors_do_not_expose_sensitive_urls(self) -> None: + marker = "SENSITIVE_URL_MARKER" + environments = ( + {"DD_SITE": f"https://user:{marker}@example.com"}, + { + "DD_TEST_OPTIMIZATION_AGENTLESS_URL": ( + f"https://user:{marker}@example.com" + ), + }, + { + "DD_TEST_OPTIMIZATION_AGENT_URL": ( + f"https://user:{marker}@example.com" + ), + }, + {"DD_TEST_OPTIMIZATION_AGENTLESS_URL": "not-an-absolute-url"}, + {"DD_TEST_OPTIMIZATION_AGENTLESS_URL": "http://localhost:notaport"}, + {"DD_TEST_OPTIMIZATION_AGENTLESS_URL": "http://localhost:65536"}, + { + "DD_TEST_OPTIMIZATION_AGENTLESS_URL": ( + "http://localhost/path with space" + ) + }, + {"DD_TEST_OPTIMIZATION_AGENTLESS_URL": "http://exa%mple.invalid"}, + {"DD_TEST_OPTIMIZATION_AGENT_URL": "http://localhost/%ZZ"}, + {"DD_TEST_OPTIMIZATION_AGENT_URL": "http://localhost:notaport"}, + ) + for environment in environments: + with self.subTest(environment=tuple(environment)): + with self.assertRaises(ConfigError) as caught: + build_endpoints(self.parse(environ=environment)) + self.assertNotIn(marker, str(caught.exception)) + + def test_proxy_environment_is_captured_immutably(self) -> None: + config = self.parse( + environ={ + "HTTPS_PROXY": "http://proxy.example", + "no_proxy": "localhost,127.0.0.1", + } + ) + self.assertEqual( + ( + ("HTTPS_PROXY", "http://proxy.example"), + ("no_proxy", "localhost,127.0.0.1"), + ), + config.proxy_environment, + ) + with self.assertRaises(FrozenInstanceError): + config.workers = 99 # type: ignore[misc] + + def test_file_tasks_are_immutable_and_typed(self) -> None: + task = FileTask( + task_id="0001", + source_path=Path("payload.json"), + display_path="payload.json", + payload_type=PayloadType.TEST, + ) + self.assertEqual(PayloadType.TEST, task.payload_type) + self.assertEqual("succeeded", FileStatus.SUCCEEDED.value) + with self.assertRaises(FrozenInstanceError): + task.task_id = "changed" # type: ignore[misc] + + +class UploaderTemporaryDirectoryTests(unittest.TestCase): + def test_cleanup_os_error_is_reported_without_replacing_body_outcome(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + owned_path = Path(raw_root) / "owned" + owned_path.mkdir() + + class FailedCleanup: + name = str(owned_path) + + def cleanup(self) -> None: + raise PermissionError("simulated cleanup failure") + + errors: list[str] = [] + with mock.patch( + "uploader_py.temporary.tempfile.TemporaryDirectory", + return_value=FailedCleanup(), + ): + with invocation_temporary_directory( + on_cleanup_error=errors.append, + ) as created: + self.assertEqual(owned_path, created) + + self.assertEqual(["PermissionError"], errors) + + def test_invocation_and_task_directories_support_complex_temp_roots(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + temp_root = Path(raw_root) / "temporary path ü" + temp_root.mkdir() + with invocation_temporary_directory(temp_root=temp_root) as invocation_root: + self.assertEqual(temp_root, invocation_root.parent) + with task_temporary_directory(invocation_root, "task:/ one") as task_root: + self.assertTrue(task_root.is_dir()) + self.assertNotIn(":", task_root.name) + self.assertFalse(task_root.exists()) + self.assertFalse(invocation_root.exists()) + + def test_temporary_root_creation_failure_is_actionable(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + not_a_directory = Path(raw_root) / "file" + not_a_directory.write_text("not a directory", encoding="utf-8") + with self.assertRaisesRegex( + TemporaryDirectoryError, "failed to create uploader temporary directory" + ): + with invocation_temporary_directory(temp_root=not_a_directory): + self.fail("temporary directory unexpectedly created") + + def test_task_directory_is_removed_after_failure(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + invocation_root = Path(raw_root) + task_root: Path | None = None + with self.assertRaisesRegex(RuntimeError, "simulated"): + with task_temporary_directory(invocation_root, "failure") as created: + task_root = created + raise RuntimeError("simulated") + self.assertIsNotNone(task_root) + self.assertFalse(task_root.exists()) + + def test_task_body_os_error_is_not_misclassified_as_temp_creation(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + with self.assertRaisesRegex(OSError, "source read failed"): + with task_temporary_directory(Path(raw_root), "source-read"): + raise OSError("source read failed") + + +class UploaderRunfilesTests(unittest.TestCase): + def test_direct_relative_path_is_resolved_from_snapshotted_cwd(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + cwd = Path(raw_root) / "working path ü" + cwd.mkdir() + expected = cwd / "direct payload.json" + expected.write_text("{}", encoding="utf-8") + resolver = RunfilesResolver.from_environment(environ={}, cwd=cwd) + + self.assertEqual(expected.resolve(), resolver.resolve_file("direct payload.json")) + + def test_runfiles_directory_supports_external_main_and_workspace_layouts(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + runfiles = Path(raw_root) / "tool.runfiles" + external = runfiles / "dependency" / "data" / "context.json" + main = runfiles / "_main" / "tools" / "schema.json" + workspace = runfiles / "workspace_name" / "pkg" / "facts.json" + for path in (external, main, workspace): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("{}", encoding="utf-8") + + resolver = RunfilesResolver.from_environment( + environ={ + "RUNFILES_DIR": str(runfiles), + "TEST_WORKSPACE": "workspace_name", + }, + cwd=Path(raw_root), + ) + self.assertEqual( + external.resolve(), + resolver.resolve_file("external/dependency/data/context.json"), + ) + self.assertEqual(main.resolve(), resolver.resolve_file("tools/schema.json")) + self.assertEqual(workspace.resolve(), resolver.resolve_file("pkg/facts.json")) + + def test_launcher_adjacent_runfiles_fallback_is_supported(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + launcher = root / "bin" / "dd_upload_payloads" + launcher.parent.mkdir() + runfile = Path(f"{launcher}.runfiles") / "_main" / "tools" / "config.json" + runfile.parent.mkdir(parents=True) + runfile.write_text("{}", encoding="utf-8") + + resolver = RunfilesResolver.from_environment( + argv0=launcher, + environ={}, + cwd=root, + ) + self.assertEqual(runfile.resolve(), resolver.resolve_file("tools/config.json")) + + def test_manifest_is_loaded_once_and_preserves_paths_with_spaces(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + exact = root / "actual files" / "context ü.json" + suffix = root / "actual files" / "telemetry facts.json" + exact.parent.mkdir() + exact.write_text("{}", encoding="utf-8") + suffix.write_text("{}", encoding="utf-8") + manifest = root / "MANIFEST" + manifest.write_text( + "\ufeffworkspace/pkg/context.json " + f"{exact}\nunknown-prefix/pkg/facts.json {suffix}\n", + encoding="utf-8", + ) + environment = { + "RUNFILES_MANIFEST_FILE": str(manifest), + "TEST_WORKSPACE": "workspace", + } + resolver = RunfilesResolver.from_environment( + environ=environment, + cwd=root, + ) + + # The resolver must not consult mutable process/environment state in workers. + environment.clear() + manifest.write_text("", encoding="utf-8") + self.assertEqual(exact.resolve(), resolver.resolve_file("pkg/context.json")) + self.assertEqual(suffix.resolve(), resolver.resolve_file("pkg/facts.json")) + + def test_manifest_decodes_bazel_escaped_entries(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + expected = root / "actual files" / "context.json" + expected.parent.mkdir() + expected.write_text("{}", encoding="utf-8") + encoded_path = ( + str(expected).replace("\\", r"\b").replace(" ", r"\s") + ) + manifest = root / "MANIFEST" + manifest.write_text( + f" workspace/pkg\\sname/context.json {encoded_path}\n", + encoding="utf-8", + ) + resolver = RunfilesResolver.from_environment( + environ={"RUNFILES_MANIFEST_FILE": str(manifest)}, + cwd=root, + ) + + self.assertEqual( + expected.resolve(), + resolver.resolve_file("workspace/pkg name/context.json"), + ) + + def test_short_path_normalization_and_suspicious_labels(self) -> None: + self.assertEqual( + ( + "repo/pkg/file.json", + "external/repo/pkg/file.json", + "_main/repo/pkg/file.json", + ), + runfile_candidates("../../repo/pkg/file.json"), + ) + for raw in ("pkg/../secret", "/absolute/missing", "C:\\absolute\\missing"): + with self.subTest(raw=raw), self.assertRaises(RunfileResolutionError): + runfile_candidates(raw) + + def test_unresolved_file_has_actionable_error(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + resolver = RunfilesResolver.from_environment(environ={}, cwd=Path(raw_root)) + with self.assertRaisesRegex(RunfileResolutionError, "runfile not found"): + resolver.resolve_file(("missing-one", "missing-two")) + + def test_unreadable_manifest_has_a_controlled_resolution_error(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + manifest = root / "MANIFEST" + manifest.write_text("workspace/file /actual/file\n", encoding="utf-8") + with mock.patch( + "topt_runtime.runfiles.Path.open", + side_effect=PermissionError("sensitive operating-system detail"), + ): + with self.assertRaisesRegex( + RunfileResolutionError, + "failed to read runfiles manifest.*PermissionError", + ): + RunfilesResolver.from_environment( + environ={"RUNFILES_MANIFEST_FILE": str(manifest)}, + cwd=root, + ) + + +class UploaderWorkspaceLockTests(unittest.TestCase): + def test_lock_name_matches_legacy_workspace_md5_contract(self) -> None: + with mock.patch( + "uploader_py.locking.hashlib.md5", + wraps=hashlib.md5, + ) as md5: + self.assertEqual( + "dd_upload_payloads_56512a07.lock", + workspace_lock_name("/workspace/example"), + ) + md5.assert_called_once_with( + b"/workspace/example", + usedforsecurity=False, + ) + + def test_same_workspace_contends_and_owner_cleanup_releases(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + first = WorkspaceLock("/workspace/same", temp_root=root, retry_seconds=0) + second = WorkspaceLock("/workspace/same", temp_root=root, retry_seconds=0) + with first: + self.assertTrue(first.acquired) + if os.name == "nt": + self.assertTrue(first.path.is_file()) + else: + self.assertTrue(first.path.is_dir()) + self.assertEqual( + str(os.getpid()), + (first.path / "pid").read_text().strip(), + ) + with self.assertRaisesRegex(WorkspaceLockError, "already running"): + second.acquire() + self.assertFalse(first.acquired) + self.assertFalse(first.path.exists()) + + with second: + self.assertTrue(second.acquired) + + def test_different_workspaces_can_run_at_the_same_time(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + with WorkspaceLock("/workspace/one", temp_root=root, retry_seconds=0) as first: + with WorkspaceLock("/workspace/two", temp_root=root, retry_seconds=0) as second: + self.assertNotEqual(first.path, second.path) + self.assertTrue(first.path.exists()) + self.assertTrue(second.path.exists()) + + @unittest.skipIf(os.name == "nt", "Unix stale-directory behavior") + def test_dead_pid_lock_is_reclaimed_without_waiting(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + waits: list[float] = [] + lock = WorkspaceLock( + "/workspace/stale", + temp_root=Path(raw_root), + sleeper=waits.append, + process_alive=lambda _pid: False, + ) + lock.path.mkdir() + (lock.path / "pid").write_text("999999\n", encoding="ascii") + with lock: + self.assertEqual(str(os.getpid()), (lock.path / "pid").read_text().strip()) + self.assertEqual([], waits) + + @unittest.skipIf(os.name == "nt", "Unix stale-directory behavior") + def test_fresh_malformed_lock_is_not_deleted(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + waits: list[float] = [] + lock = WorkspaceLock( + "/workspace/incomplete", + temp_root=Path(raw_root), + retry_seconds=0.25, + sleeper=waits.append, + clock=lambda: 105, + ) + lock.path.mkdir() + (lock.path / "pid").write_text("not-a-pid\n", encoding="ascii") + os.utime(lock.path, (100, 100)) + + with self.assertRaisesRegex(WorkspaceLockError, "PID metadata"): + lock.acquire() + self.assertTrue(lock.path.exists()) + self.assertEqual([0.25, 0.25], waits) + + @unittest.skipIf(os.name == "nt", "Unix stale-directory behavior") + def test_old_incomplete_lock_is_reclaimed_but_unexpected_tree_is_preserved(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + old = WorkspaceLock( + "/workspace/old-incomplete", + temp_root=root, + retry_seconds=0, + clock=lambda: 131, + ) + old.path.mkdir() + (old.path / "pid").write_text("broken", encoding="ascii") + os.utime(old.path, (100, 100)) + with old: + self.assertTrue(old.acquired) + + unsafe = WorkspaceLock( + "/workspace/unexpected-tree", + temp_root=root, + attempts=2, + retry_seconds=0, + process_alive=lambda _pid: False, + ) + unsafe.path.mkdir() + (unsafe.path / "pid").write_text("999999", encoding="ascii") + unexpected = unsafe.path / "do-not-delete" + unexpected.write_text("preserve", encoding="utf-8") + with self.assertRaises(WorkspaceLockError): + unsafe.acquire() + self.assertEqual("preserve", unexpected.read_text(encoding="utf-8")) + + @unittest.skipIf(os.name == "nt", "Unix stale-directory behavior") + def test_stale_reclamation_is_serialized_between_python_uploaders(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + first = WorkspaceLock( + "/workspace/stale-race", + temp_root=root, + attempts=2, + retry_seconds=0, + ) + second = WorkspaceLock( + "/workspace/stale-race", + temp_root=root, + attempts=2, + retry_seconds=0, + ) + first.path.mkdir() + (first.path / "pid").write_text("999999\n", encoding="ascii") + + start = threading.Barrier(3) + state_lock = threading.Lock() + outcomes: list[str] = [] + active_inspections = 0 + peak_inspections = 0 + original_inspect = WorkspaceLock._inspect_unix_lock + + def slow_inspect(lock: WorkspaceLock) -> str: + nonlocal active_inspections, peak_inspections + with state_lock: + active_inspections += 1 + peak_inspections = max(peak_inspections, active_inspections) + try: + time.sleep(0.05) + return original_inspect(lock) + finally: + with state_lock: + active_inspections -= 1 + + def contend(lock: WorkspaceLock) -> None: + start.wait() + try: + lock.acquire() + except WorkspaceLockError: + with state_lock: + outcomes.append("contended") + return + with state_lock: + outcomes.append("acquired") + try: + time.sleep(0.15) + finally: + lock.release() + + with mock.patch.object( + WorkspaceLock, + "_inspect_unix_lock", + slow_inspect, + ): + threads = ( + threading.Thread(target=contend, args=(first,)), + threading.Thread(target=contend, args=(second,)), + ) + for thread in threads: + thread.start() + start.wait() + for thread in threads: + thread.join(timeout=2) + + self.assertTrue(all(not thread.is_alive() for thread in threads)) + self.assertEqual(1, peak_inspections) + self.assertEqual(["acquired", "contended"], sorted(outcomes)) + + @unittest.skipUnless(os.name == "nt", "Windows byte-lock behavior") + def test_windows_lock_contention(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + first = WorkspaceLock("C:/workspace", temp_root=root, attempts=1) + second = WorkspaceLock("C:/workspace", temp_root=root, attempts=1) + with first: + with self.assertRaises(WorkspaceLockError): + second.acquire() + + +class UploaderSplittingTests(unittest.TestCase): + @staticmethod + def payload_with_exact_size(size_bytes: int) -> dict[str, object]: + payload: dict[str, object] = {"version": 1, "events": [{"value": ""}]} + base_size = len(compact_json_bytes(payload)) + if size_bytes < base_size: + raise ValueError("requested fixture is smaller than its JSON envelope") + payload["events"] = [{"value": "x" * (size_bytes - base_size)}] + actual_size = len(compact_json_bytes(payload)) + if actual_size != size_bytes: + raise AssertionError(f"fixture size mismatch: {actual_size} != {size_bytes}") + return payload + + def test_boundary_below_and_at_limit_remain_single_requests(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + for size in (MAX_TEST_PAYLOAD_BYTES - 1, MAX_TEST_PAYLOAD_BYTES): + with self.subTest(size=size): + payload = self.payload_with_exact_size(size) + chunks = prepare_test_chunks(payload, root) + self.assertEqual(1, len(chunks)) + self.assertEqual(size, chunks[0].size_bytes) + self.assertEqual(compact_json_bytes(payload), chunks[0].path.read_bytes()) + chunks[0].path.unlink() + + def test_one_byte_over_limit_splits_before_http(self) -> None: + events = [ + {"index": 0, "value": ""}, + {"index": 1, "value": ""}, + ] + payload = { + "meta": "kept", + "events": events, + "tail": 7, + } + base_size = len(compact_json_bytes(payload)) + events[0]["value"] = "a" * (MAX_TEST_PAYLOAD_BYTES + 1 - base_size) + self.assertEqual( + MAX_TEST_PAYLOAD_BYTES + 1, + len(compact_json_bytes(payload)), + ) + + with tempfile.TemporaryDirectory() as raw_root: + chunks = prepare_test_chunks(payload, Path(raw_root)) + raw_bodies = [chunk.path.read_bytes() for chunk in chunks] + decoded = [json.loads(body.decode("utf-8")) for body in raw_bodies] + self.assertGreater(len(chunks), 1) + self.assertTrue(all(chunk.size_bytes <= MAX_TEST_PAYLOAD_BYTES for chunk in chunks)) + self.assertEqual(["meta", "events", "tail"], list(decoded[0])) + self.assertTrue(all(body["meta"] == "kept" and body["tail"] == 7 for body in decoded)) + self.assertEqual(payload["events"], [event for body in decoded for event in body["events"]]) + for chunk, body in zip(chunks, raw_bodies): + expected = { + "meta": "kept", + "events": payload["events"][chunk.event_start : chunk.event_end], + "tail": 7, + } + self.assertEqual(compact_json_bytes(expected), body) + + def test_unicode_uses_utf8_bytes_not_character_count(self) -> None: + payload = {"events": [{"value": "€"}, {"value": "plain"}]} + character_count = len(compact_json_bytes(payload).decode("utf-8")) + byte_count = len(compact_json_bytes(payload)) + self.assertGreater(byte_count, character_count) + with tempfile.TemporaryDirectory() as raw_root: + chunks = prepare_test_chunks( + payload, + Path(raw_root), + limit_bytes=byte_count - 1, + ) + self.assertEqual(2, len(chunks)) + self.assertTrue(all(chunk.size_bytes <= byte_count - 1 for chunk in chunks)) + + def test_single_oversized_event_writes_no_chunk(self) -> None: + payload = self.payload_with_exact_size(MAX_TEST_PAYLOAD_BYTES + 1) + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + with self.assertRaises(TestPayloadSplitError) as captured: + prepare_test_chunks(payload, root) + self.assertEqual("single_event_exceeds_payload_limit", captured.exception.code) + self.assertEqual([], list(root.iterdir())) + + def test_invalid_or_empty_events_are_rejected(self) -> None: + invalid_payloads = ( + {}, + {"events": []}, + {"events": "not-an-array"}, + {1: "invalid-key", "events": [{"ok": True}]}, + ) + with tempfile.TemporaryDirectory() as raw_root: + for payload in invalid_payloads: + with self.subTest(payload=payload), self.assertRaises(TestPayloadSplitError): + prepare_test_chunks(payload, Path(raw_root)) + + def test_envelope_larger_than_custom_limit_is_actionable(self) -> None: + payload = {"large_meta": "x" * 100, "events": [{"ok": True}]} + with tempfile.TemporaryDirectory() as raw_root: + with self.assertRaises(TestPayloadSplitError) as captured: + prepare_test_chunks(payload, Path(raw_root), limit_bytes=20) + self.assertEqual( + "test_payload_envelope_exceeds_payload_limit", + captured.exception.code, + ) + + +class UploaderSchemaValidationTests(unittest.TestCase): + def test_importable_validation_returns_structured_result(self) -> None: + schema = { + "type": "object", + "required": ["ok"], + "properties": {"ok": {"type": "boolean"}}, + "additionalProperties": False, + } + valid = validate_payload({"ok": True}, schema) + invalid = validate_payload({"unexpected": 1}, schema) + + self.assertTrue(valid.valid) + self.assertEqual((), valid.errors) + self.assertFalse(invalid.valid) + self.assertIn("missing required property 'ok'", invalid.errors[0]) + self.assertGreater(valid.stats["nodes"], 0) + + def test_warn_policy_returns_warnings_without_writing_process_stderr(self) -> None: + stderr = io.StringIO() + with mock.patch("sys.stderr", stderr): + result = validate_payload( + "ok", + {"oneOf": [{"const": "ok"}]}, + unsupported_policy="warn", + ) + self.assertTrue(result.valid) + self.assertEqual("", stderr.getvalue()) + self.assertIn("unsupported JSON Schema keyword 'oneOf'", result.warnings[0]) + + def test_one_loaded_schema_is_safe_for_parallel_worker_validation(self) -> None: + schema = { + "type": "object", + "required": ["index"], + "properties": {"index": {"type": "integer", "minimum": 0}}, + "additionalProperties": False, + } + payloads = [{"index": index} for index in range(50)] + [{"index": -1}] + with ThreadPoolExecutor(max_workers=8) as executor: + results = list(executor.map(lambda value: validate_payload(value, schema), payloads)) + + self.assertTrue(all(result.valid for result in results[:-1])) + self.assertFalse(results[-1].valid) + self.assertTrue(all(result.stats["nodes"] == 2 for result in results)) + + def test_importable_validation_rejects_invalid_controls(self) -> None: + with self.assertRaisesRegex(ValueError, "max_errors"): + validate_payload({}, {}, max_errors=0) + with self.assertRaisesRegex(ValueError, "unsupported_policy"): + validate_payload({}, {}, unsupported_policy="ignore") + + +class UploaderLoggingTests(unittest.TestCase): + def test_debug_level_and_known_secret_redaction(self) -> None: + output = io.StringIO() + logger = configure_logging(debug=True, secrets=("secret-value",), stream=output) + logger.debug("request key=%s", "secret-value") + self.assertIn("DEBUG", output.getvalue()) + self.assertNotIn("secret-value", output.getvalue()) + self.assertIn("", output.getvalue()) + + def test_normal_mode_suppresses_debug(self) -> None: + output = io.StringIO() + logger = configure_logging(debug=False, stream=output) + logger.debug("hidden") + logger.info("visible") + self.assertNotIn("hidden", output.getvalue()) + self.assertIn("visible", output.getvalue()) + + def test_headers_and_urls_are_redacted(self) -> None: + self.assertEqual("", redact_header_value("DD-API-KEY", "abcd")) + self.assertEqual( + "application/json", + redact_header_value("Content-Type", "application/json"), + ) + self.assertEqual( + "https://example.com:8443/api/v2/citestcycle", + redact_url( + "HTTPS://user:password@example.com:8443/api/v2/citestcycle?token=secret#fragment" + ), + ) + self.assertEqual("", redact_url("not a URL")) + + +class UploaderContractCharacterizationTests(unittest.TestCase): + def test_supported_python_minimum_is_explicit(self) -> None: + self.assertFalse(python_version_is_supported(3, 9)) + self.assertTrue(python_version_is_supported(3, 10)) + self.assertTrue(python_version_is_supported(3, 14)) + + def test_legacy_runtimes_expose_the_recorded_cli_surface(self) -> None: + for relative_path in ( + "tools/core/uploader_bash_runtime.sh.tpl", + "tools/core/uploader_powershell_runtime.ps1.tpl", + ): + text = _runfile(relative_path).read_text(encoding="utf-8") + with self.subTest(runtime=relative_path): + for flag in LEGACY_CLI_FLAGS: + self.assertIn(flag, text) + + def test_legacy_runtimes_expose_the_recorded_environment_surface(self) -> None: + for relative_path in ( + "tools/core/uploader_bash_runtime.sh.tpl", + "tools/core/uploader_powershell_runtime.ps1.tpl", + ): + text = _runfile(relative_path).read_text(encoding="utf-8") + with self.subTest(runtime=relative_path): + for name in LEGACY_RUNTIME_ENVIRONMENT: + self.assertIn(name, text) + + def test_rule_exposes_the_recorded_attribute_surface(self) -> None: + text = _runfile("tools/core/test_optimization_uploader.bzl").read_text(encoding="utf-8") + for name in RULE_ATTRIBUTES: + self.assertIn(f'"{name}": attr.', text) + self.assertIn('"workers": attr.int(default = 8,', text) + self.assertIn('"use_python_uploader": attr.bool(default = True,', text) + + def test_python_parser_contains_legacy_and_new_options(self) -> None: + from uploader_py import config as config_module + + option_strings = { + option + for action in config_module._parser()._actions + for option in action.option_strings + } + self.assertTrue(LEGACY_CLI_FLAGS.issubset(option_strings)) + self.assertTrue(NEW_CLI_FLAGS.issubset(option_strings)) + self.assertIn("--help", option_strings) + self.assertIn("-h", option_strings) + + def test_bootstrap_is_intentionally_small(self) -> None: + text = _runfile("tools/core/uploader_main.py").read_text(encoding="utf-8") + self.assertIn("from uploader_py.main import main", text) + self.assertIn("raise SystemExit(main())", text) + self.assertNotIn("argparse", text) + self.assertNotIn("urllib", text) + + def test_python_entrypoint_runs_a_controlled_dry_run_noop(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + testlogs = root / "bazel-testlogs" + testlogs.mkdir() + config_path = root / "config.json" + config_path.write_text( + '{"schema_version":1,"quiescent_sec":0,"max_wait_sec":0}', + encoding="utf-8", + ) + stderr = io.StringIO() + stdout = io.StringIO() + with mock.patch.dict( + os.environ, + { + "BUILD_WORKSPACE_DIRECTORY": str(root), + "TESTLOGS_DIR": str(testlogs), + }, + clear=True, + ), mock.patch("sys.stderr", stderr), mock.patch("sys.stdout", stdout): + result = uploader_main( + [ + "--config", + str(config_path), + "--dry-run", + "--allow-cached-payload-uploads", + ] + ) + self.assertEqual(0, result) + self.assertIn("summary: mode=dry-run", stdout.getvalue()) + + def test_python_entrypoint_rejects_invalid_endpoint_during_dry_run(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + config_path = root / "config.json" + config_path.write_text( + '{"schema_version":1,"quiescent_sec":0,"max_wait_sec":0}', + encoding="utf-8", + ) + stderr = io.StringIO() + with mock.patch.dict( + os.environ, + { + "BUILD_WORKSPACE_DIRECTORY": str(root), + "DD_TEST_OPTIMIZATION_AGENTLESS_URL": ( + "http://localhost:notaport" + ), + }, + clear=True, + ), mock.patch("sys.stderr", stderr): + result = uploader_main( + ["--config", str(config_path), "--dry-run"] + ) + + self.assertEqual(2, result) + self.assertIn("absolute HTTP(S) URL", stderr.getvalue()) + + def test_python_entrypoint_handles_runfiles_manifest_failure(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + config_path = root / "config.json" + config_path.write_text( + '{"schema_version":1,"quiescent_sec":0,"max_wait_sec":0}', + encoding="utf-8", + ) + stderr = io.StringIO() + with mock.patch.dict( + os.environ, + {"BUILD_WORKSPACE_DIRECTORY": str(root)}, + clear=True, + ), mock.patch( + "topt_runtime.runfiles.RunfilesResolver.from_environment", + side_effect=RunfileResolutionError("manifest unavailable"), + ), mock.patch("sys.stderr", stderr): + result = uploader_main( + ["--config", str(config_path), "--dry-run"] + ) + + self.assertEqual(2, result) + self.assertIn("manifest unavailable", stderr.getvalue()) + self.assertNotIn("Traceback", stderr.getvalue()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/tests/python/test_uploader_transport_tools.py b/tools/tests/python/test_uploader_transport_tools.py new file mode 100644 index 00000000..0368bae7 --- /dev/null +++ b/tools/tests/python/test_uploader_transport_tools.py @@ -0,0 +1,490 @@ +#!/usr/bin/env python3 +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Exercise the dependency-free HTTP transport against loopback servers. + +Real requests protect retry, proxy, timeout, and exact-body behavior of stdlib HTTP. +""" + +from __future__ import annotations + +from contextlib import contextmanager +import gzip +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import io +from pathlib import Path +import ssl +import tempfile +import threading +import time +from typing import Iterator +import unittest +from unittest import mock +from urllib.error import HTTPError, URLError + +from uploader_test_support import add_uploader_runtime_to_path + +add_uploader_runtime_to_path() + +from uploader_py.transport import ( # noqa: E402 + DEFAULT_MAX_RETRY_DELAY_SECONDS, + HttpTransport, + HttpTransportError, + _retry_after_seconds, + prepare_coverage_multipart, +) +from uploader_py.logging_utils import configure_logging # noqa: E402 + + +class _RecordingHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_POST(self) -> None: + content_length = int(self.headers.get("Content-Length", "0")) + body = self.rfile.read(content_length) + self.server.records.append( # type: ignore[attr-defined] + { + "path": self.path, + "headers": {key.lower(): value for key, value in self.headers.items()}, + "body": body, + } + ) + response = self.server.responses.pop(0) # type: ignore[attr-defined] + delay = float(response.get("delay", 0)) + if delay: + time.sleep(delay) + response_body = response.get("body", b"ok") + self.send_response(int(response.get("status", 200))) + for name, value in response.get("headers", {}).items(): + self.send_header(name, value) + self.send_header("Content-Length", str(len(response_body))) + self.end_headers() + try: + self.wfile.write(response_body) + except (BrokenPipeError, ConnectionResetError): + pass + + def log_message(self, format: str, *args: object) -> None: + pass + + +class _Response(io.BytesIO): + def __init__(self, body: bytes = b"ok", *, status: int = 200) -> None: + super().__init__(body) + self.status = status + self.headers: dict[str, str] = {} + + +class _TlsReadFailure(_Response): + def read(self, _size: int = -1) -> bytes: + raise ssl.SSLEOFError(8, "TLS peer closed during response read") + + +@contextmanager +def _server(*responses: dict[str, object]) -> Iterator[tuple[ThreadingHTTPServer, str]]: + server = ThreadingHTTPServer(("127.0.0.1", 0), _RecordingHandler) + server.daemon_threads = True + server.records = [] # type: ignore[attr-defined] + server.responses = list(responses or ({"status": 200},)) # type: ignore[attr-defined] + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + host, port = server.server_address + yield server, f"http://{host}:{port}" + finally: + server.shutdown() + server.server_close() + thread.join(timeout=2) + + +class HttpTransportTests(unittest.TestCase): + def test_custom_agentless_and_evp_json_requests_preserve_body_and_headers(self) -> None: + with _server({"status": 202}, {"status": 204, "body": b""}) as (server, base): + transport = HttpTransport(max_attempts=1) + agentless_body = b'{"events":[{"ok":true}]}' + agentless = transport.post_json( + f"{base}/api/v2/citestcycle", + {"DD-API-KEY": "secret", "X-Datadog-Trace-Id": "123"}, + agentless_body, + ) + evp = transport.post_json( + f"{base}/evp_proxy/v2/api/v2/citestcycle", + {"X-Datadog-EVP-Subdomain": "citestcycle-intake"}, + b"{}", + ) + + self.assertTrue(agentless.succeeded) + self.assertTrue(evp.succeeded) + self.assertEqual(agentless_body, server.records[0]["body"]) # type: ignore[attr-defined] + self.assertEqual( + "secret", + server.records[0]["headers"]["dd-api-key"], # type: ignore[attr-defined] + ) + self.assertEqual( + "citestcycle-intake", + server.records[1]["headers"]["x-datadog-evp-subdomain"], # type: ignore[attr-defined] + ) + + def test_gzip_and_exact_content_length(self) -> None: + original = b'{"events":[{"value":"payload"}]}' + with _server({"status": 200}) as (server, base): + result = HttpTransport(max_attempts=1).post_json( + f"{base}/gzip", + {}, + original, + gzip_body=True, + ) + record = server.records[0] # type: ignore[attr-defined] + self.assertTrue(result.succeeded) + self.assertEqual("gzip", record["headers"]["content-encoding"]) + self.assertEqual(len(record["body"]), int(record["headers"]["content-length"])) + self.assertEqual(original, gzip.decompress(record["body"])) + + def test_precompressed_json_body_preserves_exact_bytes_and_encoding(self) -> None: + compressed = gzip.compress(b'{"events":[]}', mtime=0) + with _server({"status": 200}) as (server, base): + result = HttpTransport(max_attempts=1).post_json( + f"{base}/gzip", + {}, + compressed, + content_encoding="gzip", + ) + record = server.records[0] # type: ignore[attr-defined] + self.assertTrue(result.succeeded) + self.assertEqual("gzip", record["headers"]["content-encoding"]) + self.assertEqual(compressed, record["body"]) + + def test_retry_policy_honors_retry_after_and_caps_at_four_attempts(self) -> None: + waits: list[float] = [] + with _server( + {"status": 429, "headers": {"Retry-After": "3"}}, + {"status": 200}, + ) as (server, base): + result = HttpTransport(sleeper=waits.append).post_json(base, {}, b"{}") + self.assertTrue(result.succeeded) + self.assertEqual(2, result.attempts) + self.assertEqual((3.0,), result.retry_delays) + self.assertEqual([3.0], waits) + self.assertEqual(2, len(server.records)) # type: ignore[attr-defined] + + waits = [] + with _server(*({"status": 500} for _ in range(4))) as (server, base): + exhausted = HttpTransport(sleeper=waits.append).post_json(base, {}, b"{}") + self.assertFalse(exhausted.succeeded) + self.assertEqual(4, exhausted.attempts) + self.assertEqual(3, exhausted.retries) + self.assertEqual([2.0, 2.0, 2.0], waits) + self.assertEqual(4, len(server.records)) # type: ignore[attr-defined] + + def test_retry_after_delay_is_finite_and_capped(self) -> None: + self.assertEqual(3.0, _retry_after_seconds("3", 0.0)) + + excessive_values = ( + ("86400", 0.0), + ("Wed, 02 Jan 2030 00:00:00 GMT", 1_893_456_000.0), + ("9" * 400, 0.0), + ) + for value, now in excessive_values: + with self.subTest(value=value[:40]): + self.assertEqual( + DEFAULT_MAX_RETRY_DELAY_SECONDS, + _retry_after_seconds(value, now), + ) + + def test_each_retryable_http_status_retries_then_succeeds(self) -> None: + for status in (408, 500, 502, 503, 504): + with self.subTest(status=status), _server( + {"status": status}, + {"status": 200}, + ) as (server, base): + waits: list[float] = [] + result = HttpTransport(sleeper=waits.append).post_json( + base, + {}, + b"{}", + ) + + self.assertTrue(result.succeeded) + self.assertEqual(2, result.attempts) + self.assertEqual([2.0], waits) + self.assertEqual(2, len(server.records)) # type: ignore[attr-defined] + + def test_transient_connection_and_timeout_retry_then_succeed(self) -> None: + for first_error, error_name in ( + (URLError(ConnectionRefusedError("refused")), "ConnectionRefusedError"), + (TimeoutError("timed out"), "TimeoutError"), + ): + with self.subTest(error=error_name): + waits: list[float] = [] + transport = HttpTransport(sleeper=waits.append) + opener = mock.Mock() + opener.open.side_effect = [first_error, _Response()] + transport._opener = opener + + result = transport.post_json( + "http://localhost:8126/upload", + {}, + b"{}", + ) + + self.assertTrue(result.succeeded) + self.assertEqual(2, result.attempts) + self.assertEqual([2.0], waits) + self.assertEqual(2, opener.open.call_count) + + def test_tls_response_read_failures_retry_then_succeed(self) -> None: + failures = ( + _TlsReadFailure(), + HTTPError( + "https://localhost/upload", + 503, + "unavailable", + {}, + _TlsReadFailure(), + ), + ) + for failure in failures: + with self.subTest(response=type(failure).__name__): + waits: list[float] = [] + transport = HttpTransport(sleeper=waits.append) + opener = mock.Mock() + opener.open.side_effect = [failure, _Response()] + transport._opener = opener + + result = transport.post_json( + "https://localhost/upload", + {}, + b"{}", + ) + + self.assertTrue(result.succeeded) + self.assertEqual(2, result.attempts) + self.assertEqual([2.0], waits) + self.assertEqual(2, opener.open.call_count) + + def test_certificate_verification_failure_is_terminal(self) -> None: + waits: list[float] = [] + transport = HttpTransport(sleeper=waits.append) + opener = mock.Mock() + opener.open.side_effect = ssl.SSLCertVerificationError( + 1, + "certificate verify failed", + ) + transport._opener = opener + + result = transport.post_json( + "https://localhost/upload", + {}, + b"{}", + ) + + self.assertFalse(result.succeeded) + self.assertEqual(1, result.attempts) + self.assertEqual("SSLCertVerificationError", result.transport_error) + self.assertEqual([], waits) + + def test_debug_logs_attempt_and_retry_without_url_secrets(self) -> None: + stream = io.StringIO() + logger = configure_logging(debug=True, secrets=("api-secret",), stream=stream) + with _server( + {"status": 500, "body": b"backend api-secret\nretry"}, + {"status": 200}, + ) as (_server_instance, base): + transport = HttpTransport( + sleeper=lambda _delay: None, + logger=logger, + ) + transport.set_log_context("task-1", "test", "payloads/tests/events.json") + result = transport.post_json( + f"{base}/upload?signature=url-secret", + {"DD-API-KEY": "api-secret"}, + b"{}", + ) + + self.assertTrue(result.succeeded) + output = stream.getvalue() + self.assertIn("attempt=1/4", output) + self.assertIn("retry scheduled", output) + self.assertIn("task=task-1 type=test file=payloads/tests/events.json", output) + self.assertIn("succeeded attempt=2 status=200", output) + self.assertIn("body_excerpt='backend \\nretry'", output) + self.assertIn("body_truncated=False", output) + self.assertNotIn("url-secret", output) + self.assertNotIn("api-secret", output) + + def test_413_and_other_permanent_4xx_are_never_retried(self) -> None: + for status in (400, 401, 403, 404, 413): + with self.subTest(status=status), _server({"status": status}) as (server, base): + waits: list[float] = [] + result = HttpTransport(sleeper=waits.append).post_json(base, {}, b"{}") + self.assertFalse(result.succeeded) + self.assertEqual(status, result.status_code) + self.assertEqual(1, result.attempts) + self.assertEqual([], waits) + self.assertEqual(1, len(server.records)) # type: ignore[attr-defined] + + def test_gzip_request_body_is_byte_identical_across_retry(self) -> None: + original = b'{"events":[{"value":"payload"}]}' + with _server({"status": 500}, {"status": 200}) as (server, base): + result = HttpTransport(sleeper=lambda _delay: None).post_json( + f"{base}/gzip-retry", + {}, + original, + gzip_body=True, + ) + + self.assertTrue(result.succeeded) + self.assertEqual(2, result.attempts) + first, second = server.records # type: ignore[attr-defined] + self.assertEqual(first["body"], second["body"]) + self.assertEqual(original, gzip.decompress(first["body"])) + + def test_prepared_multipart_is_exact_and_reopened_for_retry(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + coverage = root / "coverage.json" + coverage.write_bytes(b'{"files":[]}') + prepared = prepare_coverage_multipart( + root / "multipart.body", + event_body=b'{"dummy":true}', + coverage_path=coverage, + coverage_filename="filecoveragex.json", + coverage_content_type="application/json", + ) + with _server({"status": 500}, {"status": 200}) as (server, base): + result = HttpTransport(sleeper=lambda _delay: None).post_prepared_multipart( + base, + {}, + prepared, + ) + + self.assertTrue(result.succeeded) + self.assertEqual(2, result.attempts) + first, second = server.records # type: ignore[attr-defined] + self.assertEqual(first["body"], second["body"]) + self.assertEqual(prepared.content_length, len(first["body"])) + self.assertEqual(prepared.content_type, first["headers"]["content-type"]) + self.assertIn(b'name="event"; filename="fileevent.json"', first["body"]) + self.assertIn(b'name="coveragex"; filename="filecoveragex.json"', first["body"]) + self.assertIn(b'{"files":[]}', first["body"]) + + def test_response_diagnostics_are_bounded(self) -> None: + stream = io.StringIO() + logger = configure_logging(debug=True, stream=stream) + with _server({"status": 400, "body": b"x" * 100}) as (_server_instance, base): + result = HttpTransport( + max_attempts=1, + response_limit=16, + logger=logger, + ).post_json( + base, + {}, + b"{}", + ) + self.assertEqual(b"x" * 16, result.body_excerpt) + self.assertTrue(result.body_truncated) + self.assertIn("body_excerpt='xxxxxxxxxxxxxxxx'", stream.getvalue()) + self.assertIn("body_truncated=True", stream.getvalue()) + + def test_request_timeout_is_reported_without_exposing_urls(self) -> None: + with _server({"status": 200, "delay": 0.2}) as (_server_instance, base): + result = HttpTransport( + max_attempts=1, + connect_timeout=0.05, + request_timeout=0.05, + ).post_json(f"{base}/slow?token=secret", {}, b"{}") + self.assertFalse(result.succeeded) + self.assertIsNone(result.status_code) + self.assertIn(result.transport_error, {"TimeoutError", "timeout"}) + self.assertNotIn("secret", result.transport_error or "") + + def test_http_proxy_and_no_proxy_use_snapshotted_configuration(self) -> None: + with _server({"status": 200}) as (proxy, proxy_base): + transport = HttpTransport( + max_attempts=1, + proxy_environment=( + ("HTTP_PROXY", "http://upper.invalid:1"), + ("http_proxy", proxy_base), + ), + ) + result = transport.post_json("http://origin.invalid/upload", {}, b"{}") + self.assertTrue(result.succeeded) + self.assertEqual( + "http://origin.invalid/upload", + proxy.records[0]["path"], # type: ignore[attr-defined] + ) + + with _server({"status": 200}) as (origin, origin_base): + with _server({"status": 502}) as (proxy, proxy_base): + bypassed = HttpTransport( + max_attempts=1, + proxy_environment=( + ("HTTP_PROXY", proxy_base), + ("NO_PROXY", "127.0.0.1"), + ), + ).post_json(f"{origin_base}/direct", {}, b"{}") + self.assertTrue(bypassed.succeeded) + self.assertEqual(1, len(origin.records)) # type: ignore[attr-defined] + self.assertEqual(0, len(proxy.records)) # type: ignore[attr-defined] + + def test_invalid_effective_proxy_is_rejected_before_any_request(self) -> None: + for proxy in ( + "http://localhost:notaport", + "http://localhost:65536", + "socks5://localhost:1080", + "http://localhost/proxy-path", + "http://user:%FF@localhost:8080", + ): + with self.subTest(proxy=proxy), self.assertRaisesRegex( + HttpTransportError, + "invalid HTTP proxy configuration", + ): + HttpTransport( + max_attempts=1, + proxy_environment=(("http_proxy", proxy),), + ) + + # Lowercase proxy variables retain their existing precedence over an + # ignored uppercase value, so only the effective proxy is validated. + HttpTransport( + max_attempts=1, + proxy_environment=( + ("HTTP_PROXY", "http://localhost:notaport"), + ("http_proxy", "http://localhost:8080"), + ), + ) + + def test_system_tls_verification_and_input_guards_are_enabled(self) -> None: + self.assertTrue(HttpTransport(max_attempts=1).verifies_tls) + with self.assertRaisesRegex(HttpTransportError, "must not contain credentials"): + HttpTransport(max_attempts=1).post_json( + "https://user:secret@example.test/upload", + {}, + b"{}", + ) + for url in ( + "http://localhost:notaport/upload", + "http://localhost:65536/upload", + "http://localhost/path with space", + "http://localhost/%ZZ", + "http://exa%mple.invalid/upload", + ): + with self.subTest(url=url), self.assertRaisesRegex( + HttpTransportError, + "absolute HTTP", + ): + HttpTransport(max_attempts=1).post_json(url, {}, b"{}") + with self.assertRaisesRegex(HttpTransportError, "header value"): + HttpTransport(max_attempts=1).post_json( + "https://example.test/upload", + {"X-Test": "ok\r\nInjected: yes"}, + b"{}", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/tests/python/test_uploader_worker_pool_tools.py b/tools/tests/python/test_uploader_worker_pool_tools.py new file mode 100644 index 00000000..26d0c11b --- /dev/null +++ b/tools/tests/python/test_uploader_worker_pool_tools.py @@ -0,0 +1,424 @@ +#!/usr/bin/env python3 +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Verify bounded concurrency and exactly-once ownership in the worker pool. + +Interrupt and ordering cases protect independent files from coordinator races. +""" + +from __future__ import annotations + +from pathlib import Path +import threading +import time +import unittest +from queue import Queue +from unittest import mock + +from uploader_test_support import add_uploader_runtime_to_path + +add_uploader_runtime_to_path() + +from uploader_py.models import FileResult, FileStatus, FileTask, PayloadType # noqa: E402 +from uploader_py.worker_pool import ( # noqa: E402 + WorkerPoolError, + WorkerPoolInterrupted, + run_file_workers, +) + + +def _tasks(count: int) -> tuple[FileTask, ...]: + return tuple( + FileTask( + task_id=f"{index:04d}", + source_path=Path(f"payload-{index}.json"), + display_path=f"payload-{index}.json", + payload_type=PayloadType.TEST, + ) + for index in range(count) + ) + + +class WorkerPoolTests(unittest.TestCase): + def test_bounded_task_queue_applies_backpressure(self) -> None: + first_started = threading.Event() + release_first = threading.Event() + blocked_put_observed = threading.Event() + completed_runs = [] + + class ObservedQueue(Queue): + def put(self, item, block=True, timeout=None): + if self.maxsize > 0 and self.full(): + blocked_put_observed.set() + return super().put(item, block=block, timeout=timeout) + + def processor(task, _runtime, _transport): + if task.task_id == "0000": + first_started.set() + release_first.wait(timeout=2) + return FileResult( + task_id=task.task_id, + source_path=task.display_path, + payload_type=task.payload_type, + status=FileStatus.SUCCEEDED, + ) + + def run_pool() -> None: + completed_runs.append( + run_file_workers( + _tasks(6), + workers=1, + runtime=object(), + transport_factory=object, + process_file=processor, + ) + ) + + with mock.patch("uploader_py.worker_pool.Queue", ObservedQueue): + runner = threading.Thread(target=run_pool) + runner.start() + try: + self.assertTrue(first_started.wait(timeout=2)) + self.assertTrue(blocked_put_observed.wait(timeout=2)) + self.assertTrue(runner.is_alive()) + finally: + release_first.set() + runner.join(timeout=3) + + self.assertFalse(runner.is_alive()) + self.assertEqual(1, len(completed_runs)) + self.assertEqual(6, len(completed_runs[0].results)) + + def test_keyboard_interrupt_drains_unowned_tasks_and_joins_workers(self) -> None: + tasks = _tasks(8) + processed: list[str] = [] + processed_lock = threading.Lock() + + def processor(task, _runtime, _transport): + with processed_lock: + processed.append(task.task_id) + time.sleep(0.03) + return FileResult( + task_id=task.task_id, + source_path=task.display_path, + payload_type=task.payload_type, + status=FileStatus.SUCCEEDED, + ) + + original_join = Queue.join + join_calls = 0 + + def interrupt_first_join(queue): + nonlocal join_calls + join_calls += 1 + if join_calls == 1: + raise KeyboardInterrupt + return original_join(queue) + + with mock.patch.object(Queue, "join", interrupt_first_join): + with self.assertRaises(WorkerPoolInterrupted) as raised: + run_file_workers( + tasks, + workers=2, + runtime=object(), + transport_factory=object, + process_file=processor, + ) + + interrupt = raised.exception + self.assertGreaterEqual(len(processed), 1) + self.assertLess(len(processed), len(tasks)) + self.assertEqual(len(processed), len(interrupt.run.results)) + self.assertEqual(len(tasks), len(interrupt.run.results) + interrupt.cancelled) + self.assertEqual( + sorted(processed), + sorted(result.task_id for result in interrupt.run.results), + ) + self.assertFalse( + any( + thread.name.startswith("dd-uploader-worker-") + for thread in threading.enumerate() + ) + ) + + def test_keyboard_interrupt_after_queue_completion_does_not_add_work(self) -> None: + tasks = _tasks(2) + + def processor(task, _runtime, _transport): + return FileResult( + task_id=task.task_id, + source_path=task.display_path, + payload_type=task.payload_type, + status=FileStatus.SUCCEEDED, + ) + + original_join = Queue.join + join_calls = 0 + + def interrupt_after_first_completion(queue): + nonlocal join_calls + join_calls += 1 + original_join(queue) + if join_calls == 1: + raise KeyboardInterrupt + + with mock.patch.object(Queue, "join", interrupt_after_first_completion): + with self.assertRaises(WorkerPoolInterrupted) as raised: + run_file_workers( + tasks, + workers=2, + runtime=object(), + transport_factory=object, + process_file=processor, + ) + + self.assertEqual(2, join_calls) + self.assertEqual(len(tasks), len(raised.exception.run.results)) + self.assertEqual(0, raised.exception.cancelled) + self.assertFalse( + any( + thread.name.startswith("dd-uploader-worker-") + for thread in threading.enumerate() + ) + ) + + def test_keyboard_interrupt_during_thread_start_joins_started_workers(self) -> None: + tasks = _tasks(4) + original_start = threading.Thread.start + start_calls = 0 + + def interrupt_second_start(thread): + nonlocal start_calls + start_calls += 1 + if start_calls == 2: + raise KeyboardInterrupt + return original_start(thread) + + with mock.patch.object(threading.Thread, "start", interrupt_second_start): + with self.assertRaises(WorkerPoolInterrupted) as raised: + run_file_workers( + tasks, + workers=2, + runtime=object(), + transport_factory=object, + process_file=lambda *_args: self.fail( + "no task should be owned during interrupted startup" + ), + ) + + self.assertEqual(1, raised.exception.run.worker_threads) + self.assertEqual(0, len(raised.exception.run.results)) + self.assertEqual(len(tasks), raised.exception.cancelled) + self.assertFalse( + any( + thread.name.startswith("dd-uploader-worker-") + for thread in threading.enumerate() + ) + ) + + def test_each_file_runs_its_complete_pipeline_on_one_worker(self) -> None: + tasks = _tasks(8) + steps: dict[str, list[tuple[str, int]]] = {task.task_id: [] for task in tasks} + state_lock = threading.Lock() + active = 0 + maximum_active = 0 + + def processor(task, _runtime, _transport): + nonlocal active, maximum_active + thread_id = threading.get_ident() + with state_lock: + active += 1 + maximum_active = max(maximum_active, active) + for step in ("enrich", "validate", "split", "send-1", "send-2"): + steps[task.task_id].append((step, thread_id)) + time.sleep(0.002) + with state_lock: + active -= 1 + return FileResult( + task_id=task.task_id, + source_path=task.display_path, + payload_type=task.payload_type, + status=FileStatus.SUCCEEDED, + chunks_created=2, + chunks_uploaded=2, + ) + + results = run_file_workers( + tasks, + workers=3, + runtime=object(), + transport_factory=object, + process_file=processor, + ).results + + self.assertEqual([task.task_id for task in tasks], [result.task_id for result in results]) + self.assertEqual(3, maximum_active) + for task in tasks: + task_steps = steps[task.task_id] + self.assertEqual( + ["enrich", "validate", "split", "send-1", "send-2"], + [step for step, _thread in task_steps], + ) + self.assertEqual(1, len({thread for _step, thread in task_steps})) + + def test_each_thread_owns_and_reuses_exactly_one_transport(self) -> None: + tasks = _tasks(12) + created: list[object] = [] + uses: list[tuple[int, int]] = [] + state_lock = threading.Lock() + first_wave = threading.Barrier(4) + + def transport_factory(): + transport = object() + created.append(transport) + return transport + + def processor(task, _runtime, transport): + if int(task.task_id) < 4: + first_wave.wait(timeout=2) + with state_lock: + uses.append((threading.get_ident(), id(transport))) + return FileResult( + task_id=task.task_id, + source_path=task.display_path, + payload_type=task.payload_type, + status=FileStatus.SUCCEEDED, + ) + + run_file_workers( + tasks, + workers=4, + runtime=object(), + transport_factory=transport_factory, + process_file=processor, + ) + self.assertEqual(4, len(created)) + transport_by_thread: dict[int, set[int]] = {} + for thread_id, transport_id in uses: + transport_by_thread.setdefault(thread_id, set()).add(transport_id) + self.assertTrue( + all( + len(transport_ids) == 1 + for transport_ids in transport_by_thread.values() + ) + ) + self.assertEqual({id(transport) for transport in created}, {item[1] for item in uses}) + + def test_one_file_exception_does_not_cancel_other_files(self) -> None: + tasks = _tasks(5) + + def processor(task, _runtime, _transport): + if task.task_id == "0002": + raise RuntimeError("secret details must not enter the result") + return FileResult( + task_id=task.task_id, + source_path=task.display_path, + payload_type=task.payload_type, + status=FileStatus.SUCCEEDED, + ) + + results = run_file_workers( + tasks, + workers=3, + runtime=object(), + transport_factory=object, + process_file=processor, + ).results + self.assertEqual(4, sum(result.status is FileStatus.SUCCEEDED for result in results)) + failure = results[2] + self.assertEqual(FileStatus.FAILED, failure.status) + self.assertEqual("unhandled_worker_exception", failure.failure_code) + self.assertEqual("RuntimeError", failure.failure_message) + + def test_empty_intake_creates_no_transports(self) -> None: + calls = 0 + + def transport_factory(): + nonlocal calls + calls += 1 + return object() + + results = run_file_workers( + (), + workers=4, + runtime=object(), + transport_factory=transport_factory, + process_file=lambda *_args: self.fail("processor should not run"), + ).results + self.assertEqual((), results) + self.assertEqual(0, calls) + + def test_workers_one_is_the_sequential_baseline(self) -> None: + tasks = _tasks(6) + observed: list[tuple[str, int]] = [] + + def processor(task, _runtime, _transport): + observed.append((task.task_id, threading.get_ident())) + return FileResult( + task_id=task.task_id, + source_path=task.display_path, + payload_type=task.payload_type, + status=FileStatus.SUCCEEDED, + ) + + results = run_file_workers( + tasks, + workers=1, + runtime=object(), + transport_factory=object, + process_file=processor, + ).results + self.assertEqual([task.task_id for task in tasks], [item[0] for item in observed]) + self.assertEqual(1, len({item[1] for item in observed})) + self.assertEqual([task.task_id for task in tasks], [result.task_id for result in results]) + + def test_pool_reports_peak_activity_from_coordinator_state(self) -> None: + barrier = threading.Barrier(3) + + def processor(task, _runtime, _transport): + barrier.wait(timeout=2) + return FileResult( + task_id=task.task_id, + source_path=task.display_path, + payload_type=task.payload_type, + status=FileStatus.SUCCEEDED, + ) + + run = run_file_workers( + _tasks(6), + workers=3, + runtime=object(), + transport_factory=object, + process_file=processor, + ) + + self.assertEqual(6, len(run.results)) + self.assertEqual(3, run.worker_threads) + self.assertEqual(3, run.peak_active_workers) + + def test_invalid_worker_setup_fails_before_threads_start(self) -> None: + with self.assertRaisesRegex(WorkerPoolError, "positive"): + run_file_workers( + _tasks(1), + workers=0, + runtime=object(), + transport_factory=object, + process_file=lambda *_args: self.fail("processor should not run"), + ) + duplicate = (_tasks(1)[0], _tasks(1)[0]) + with self.assertRaisesRegex(WorkerPoolError, "unique"): + run_file_workers( + duplicate, + workers=2, + runtime=object(), + transport_factory=object, + process_file=lambda *_args: self.fail("processor should not run"), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/tests/python/uploader_test_support.py b/tools/tests/python/uploader_test_support.py new file mode 100644 index 00000000..77782805 --- /dev/null +++ b/tools/tests/python/uploader_test_support.py @@ -0,0 +1,61 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Resolve uploader test runfiles and expose the runtime package for imports. + +One helper keeps direct, runfiles-tree, and manifest-only test startup consistent. +""" + +from __future__ import annotations + +import os +from pathlib import Path +import sys + + +def resolve_runfile(relative_path: str) -> Path: + """Resolve one test dependency across local and Bazel runfiles layouts.""" + test_source_root = os.environ.get("TEST_SRCDIR", "") + test_workspace = os.environ.get("TEST_WORKSPACE", "") + workspace = os.environ.get("BUILD_WORKSPACE_DIRECTORY", "") + candidates: list[Path] = [] + if test_source_root and test_workspace: + candidates.append(Path(test_source_root) / test_workspace / relative_path) + if test_source_root: + candidates.append(Path(test_source_root) / relative_path) + if workspace: + candidates.append(Path(workspace) / relative_path) + + test_directory = Path(__file__).resolve().parent + for parent in (test_directory, *test_directory.parents): + if (parent / "MODULE.bazel").exists() or (parent / ".git").exists(): + candidates.append(parent / relative_path) + break + for candidate in candidates: + if candidate.exists(): + return candidate.resolve() + + manifest_path = os.environ.get("RUNFILES_MANIFEST_FILE", "") + if manifest_path and Path(manifest_path).is_file(): + logical_paths = {relative_path} + if test_workspace: + logical_paths.add(f"{test_workspace}/{relative_path}") + with Path(manifest_path).open("r", encoding="utf-8") as manifest: + for line in manifest: + logical_path, separator, physical_path = ( + line.rstrip("\n").partition(" ") + ) + if separator and logical_path in logical_paths and physical_path: + return Path(physical_path) + raise FileNotFoundError(f"runfile not found: {relative_path}") + + +def add_uploader_runtime_to_path() -> Path: + """Make the uploader package importable and return its core directory.""" + core_directory = resolve_runfile("tools/core/uploader_main.py").parent + if str(core_directory) not in sys.path: + sys.path.insert(0, str(core_directory)) + return core_directory