diff --git a/.github/workflows/dependency-licenses.yml b/.github/workflows/dependency-licenses.yml index 24f2adcf..18896557 100644 --- a/.github/workflows/dependency-licenses.yml +++ b/.github/workflows/dependency-licenses.yml @@ -38,6 +38,12 @@ jobs: with: node-version: "24" + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version: "1.26.x" + cache: false + - name: Check licenses env: BASE_SHA: ${{ github.event.pull_request.base.sha }} @@ -70,6 +76,10 @@ jobs: uv-script) uv lock --script "$script" --check ;; npm) npm ci --ignore-scripts --no-audit --no-fund ;; cargo) cargo metadata --locked --format-version 1 > /dev/null ;; + go) + GOTOOLCHAIN=local GOWORK=off go mod tidy -diff + GOTOOLCHAIN=local GOWORK=off go mod verify + ;; *) echo "Unknown package manager: $manager" >&2; exit 1 ;; esac ) diff --git a/docs/development/dependency-licenses.md b/docs/development/dependency-licenses.md index 75dbc015..2389dc59 100644 --- a/docs/development/dependency-licenses.md +++ b/docs/development/dependency-licenses.md @@ -22,15 +22,19 @@ The checker searches the complete tracked Git tree, regardless of folder. It reads files from the committed base and head revisions, so untracked, ignored, and uncommitted files are outside the check. -Supported lockfiles are: +Supported dependency inventories are: - Python: `uv.lock` and uv script lockfiles (`*.py.lock`). - JavaScript: npm v2/v3 `package-lock.json`. - Rust: `Cargo.lock`. +- Go: `go.mod`, with `go.sum` required when it declares external modules. The inventory includes external packages declared directly by each project, including development, optional, and platform-specific direct dependencies. Transitive packages and first-party project or workspace packages are excluded. +For Go, requirements marked `// indirect` are transitive. Versioned `replace` +directives identify the replacement module that is checked; local-directory +replacements are repository-controlled source and are excluded. For a pull request, a package is checked when its locked name, version, source, or lockfile location is added or changed. Removed packages do not fail the @@ -38,24 +42,28 @@ check. The report still includes unchanged current direct dependencies so that every run provides a repository-wide view. Only additions and changes affect the status of an ordinary pull request. -Changed `pyproject.toml`, `package.json`, `Cargo.toml`, and PEP 723 inline-script -metadata must have a corresponding lockfile or belong to a declared locked -workspace. The workflow asks the native package manager to confirm that each -affected lock is current: +Changed `pyproject.toml`, `package.json`, `Cargo.toml`, `go.mod`, and PEP 723 +inline-script metadata must have the corresponding dependency files or belong +to a supported declared workspace. The workflow asks the native package manager +to confirm that each affected dependency inventory is current: - uv projects: `uv lock --check` - uv inline scripts: `uv lock --script SCRIPT --check` - npm projects: `npm ci --ignore-scripts --no-audit --no-fund` - Cargo projects: `cargo metadata --locked` +- Go modules: `go mod tidy -diff`, followed by `go mod verify` The npm command installs locked packages without lifecycle scripts. Cargo -metadata does not build project code. A compatible constraint edit does not -need to rewrite a lockfile when the native check accepts it. +metadata and the Go module commands do not build or run project code. Go +validation disables workspace discovery and automatic toolchain downloads so +that each module is checked independently with the CI-pinned toolchain. A +compatible constraint edit does not need to rewrite a lockfile when the native +check accepts it. The checker does not inventory vendored code, datasets, model weights, container or system packages, unsupported package managers, or dependencies -that are absent from a supported lockfile. A passing check is therefore not a -complete legal clearance. +that are absent from a supported dependency inventory. A passing check is +therefore not a complete legal clearance. ## Policy and unresolved results @@ -66,9 +74,12 @@ outages are unresolved rather than being treated as successful lookups. An unresolved addition or change fails the pull request; an unresolved unchanged dependency contributes no license to the report. -Public metadata comes from PyPI, npm, or crates.io. Private registries, Git -dependencies, and other unsupported source forms require reviewed, exact-source -clarification. +Public metadata comes from PyPI, npm, crates.io, or the stable deps.dev API. Go +license metadata covers exact public module versions known to proxy.golang.org; +when deps.dev reports multiple licenses, every reported license must satisfy the +policy because their relationship is unspecified. Private modules, private +registries, Git dependencies in other ecosystems, and other unsupported source +forms require reviewed, exact-source clarification. If metadata is incomplete, a maintainer can add a `[[clarification]]` entry with `ecosystem`, `name`, `version`, `source`, `license`, and an HTTPS `evidence` URL. @@ -86,8 +97,8 @@ itself from changes made in the same pull request. The workflow publishes the `Check dependency licenses` status and uploads a compact `dependency-licenses.json` artifact with three fields: -- `folders` maps every scanned project folder to the lockfiles inspected there - and its direct-dependency licenses. +- `folders` maps every scanned project folder to the dependency inventories + inspected there and its direct-dependency licenses. - `licenses` maps every unique reported license value to the project folders that depend on it. - `failures` maps affected project folders to the licenses that fail the check. @@ -96,9 +107,11 @@ compact `dependency-licenses.json` artifact with three fields: The report contains no package-level records and does not list transitive dependency licenses. -The lockfiles identify each direct dependency and its exact resolved version. -The reported license value comes from that version's package-registry metadata -or a reviewed policy clarification, not from the lockfile itself. +The dependency inventory identifies each direct dependency and its exact +resolved version. For Go, `go.mod` provides the version selection while `go.sum` +provides integrity hashes; `go.sum` is not treated as a lockfile or package +list. The reported license value comes from that version's public metadata or a +reviewed policy clarification, not from the dependency file itself. For a failure: diff --git a/scripts/check_dependency_licenses.py b/scripts/check_dependency_licenses.py index d8f93470..7096468c 100644 --- a/scripts/check_dependency_licenses.py +++ b/scripts/check_dependency_licenses.py @@ -12,6 +12,7 @@ import argparse import json +import shlex import subprocess import sys from concurrent.futures import ThreadPoolExecutor @@ -24,16 +25,19 @@ import tomllib POLICY_PATH = ".github/dependency-license-policy.toml" -LOCK_NAMES = {"uv.lock", "package-lock.json", "Cargo.lock"} +GO_MODULE_SOURCE = "https://proxy.golang.org" +LOCK_NAMES = {"uv.lock", "package-lock.json", "Cargo.lock", "go.mod"} MANIFEST_LOCKS = { "pyproject.toml": "uv.lock", "package.json": "package-lock.json", "Cargo.toml": "Cargo.lock", + "go.mod": "go.sum", } MANIFEST_MANAGERS = { "pyproject.toml": "uv", "package.json": "npm", "Cargo.toml": "cargo", + "go.mod": "go", } @@ -45,9 +49,74 @@ class Dependency: source: str +def go_mod_requirements(content: str) -> list[tuple[str, str, bool]]: + """Return effective external Go requirements and whether each is indirect.""" + requirements = [] + replacements: dict[tuple[str, str], tuple[str, str] | None] = {} + block = "" + for line_number, raw_line in enumerate(content.splitlines(), 1): + code, _, comment = raw_line.partition("//") + try: + lexer = shlex.shlex(code, posix=True, punctuation_chars="()=>") + lexer.commenters = "" + lexer.whitespace_split = True + tokens = list(lexer) + except ValueError as error: + raise ValueError(f"go.mod:{line_number}: {error}") from error + if not tokens: + continue + if block: + if tokens == [")"]: + block = "" + continue + directive, values = block, tokens + else: + directive, values = tokens[0], tokens[1:] + if values == ["("]: + if directive not in ("require", "replace"): + continue + block = directive + continue + if directive == "require": + if len(values) != 2: + raise ValueError( + f"go.mod:{line_number}: require needs a module and version" + ) + requirements.append((values[0], values[1], comment.strip() == "indirect")) + elif directive == "replace": + if "=>" not in values: + raise ValueError(f"go.mod:{line_number}: replace is missing =>") + separator = values.index("=>") + old, new = values[:separator], values[separator + 1 :] + if len(old) not in (1, 2) or len(new) not in (1, 2): + raise ValueError(f"go.mod:{line_number}: invalid replace directive") + replacements[(old[0], old[1] if len(old) == 2 else "")] = ( + (new[0], new[1]) if len(new) == 2 else None + ) + if block: + raise ValueError(f"go.mod: unterminated {block} block") + + result = [] + for name, version, indirect in requirements: + replacement = replacements.get((name, version), replacements.get((name, ""))) + if replacement is None and ( + (name, version) in replacements or (name, "") in replacements + ): + continue # A local replacement is repository-controlled source. + if replacement: + name, version = replacement + result.append((name, version, indirect)) + return result + + def inventory(path: str, content: str) -> set[Dependency]: """Include every locked external package, irrespective of groups or platform.""" result = set() + if Path(path).name == "go.mod": + return { + Dependency("go", name, version, GO_MODULE_SOURCE) + for name, version, _ in go_mod_requirements(content) + } if Path(path).name == "package-lock.json": lock = json.loads(content) if lock.get("lockfileVersion") not in (2, 3) or "packages" not in lock: @@ -102,6 +171,12 @@ def inventory(path: str, content: str) -> set[Dependency]: def direct_inventory(path: str, content: str) -> set[Dependency]: """Return locked packages declared directly by the project or script.""" + if Path(path).name == "go.mod": + return { + Dependency("go", name, version, GO_MODULE_SOURCE) + for name, version, indirect in go_mod_requirements(content) + if not indirect + } lock = ( json.loads(content) if Path(path).name == "package-lock.json" @@ -275,6 +350,29 @@ def package_license(dependency: Dependency) -> str: if info.get("dist", {}).get("tarball") != dependency.source: raise ValueError("registry tarball does not match locked source") return info.get("license") or "" + if dependency.ecosystem == "go": + if dependency.source != GO_MODULE_SOURCE: + raise ValueError( + "unsupported Go source; provide an exact-source clarification" + ) + info = fetch_json( + f"https://api.deps.dev/v3/systems/GO/packages/{name}/versions/{version}" + ) + key = info.get("versionKey", {}) + if ( + key.get("system") != "GO" + or key.get("name") != dependency.name + or key.get("version") != dependency.version + ): + raise ValueError("deps.dev response does not match requested Go module") + licenses = info.get("licenses", []) + if not isinstance(licenses, list) or any( + not isinstance(value, str) or not value.strip() for value in licenses + ): + raise ValueError("invalid deps.dev license metadata") + if len(licenses) == 1: + return licenses[0] + return " AND ".join(f"({value})" for value in licenses) if dependency.source not in ( "registry+https://github.com/rust-lang/crates.io-index", "sparse+https://index.crates.io/", @@ -365,6 +463,8 @@ def manifest_dependencies(name: str, data: dict) -> list: "target", ) ] + [data.get("workspace")] + if name == "go.mod": + raise AssertionError("Go module dependencies are parsed from source text") return [ data.get(key) for key in ( @@ -466,17 +566,24 @@ def check_manifest_coverage( if manifest.name not in MANIFEST_LOCKS: continue content = git_text(root, "show", f"{head}:{path}") - parse = json.loads if manifest.name == "package.json" else tomllib.loads - dependencies = manifest_dependencies(manifest.name, parse(content)) + if manifest.name == "go.mod": + parse = None + dependencies = [go_mod_requirements(content)] + else: + parse = json.loads if manifest.name == "package.json" else tomllib.loads + dependencies = manifest_dependencies(manifest.name, parse(content)) lock_name = MANIFEST_LOCKS[manifest.name] direct_lock = (manifest.parent / lock_name).as_posix() - previous_dependencies = ( - manifest_dependencies( + if path not in base_paths: + previous_dependencies = [] + elif manifest.name == "go.mod": + previous_dependencies = [ + go_mod_requirements(git_text(root, "show", f"{base}:{path}")) + ] + else: + previous_dependencies = manifest_dependencies( manifest.name, parse(git_text(root, "show", f"{base}:{path}")) ) - if path in base_paths - else [] - ) if ( not any(dependencies) and not any(previous_dependencies) @@ -508,6 +615,8 @@ def check_manifest_coverage( if manifest.name == "package.json" else tomllib.loads(workspace_content) ) + if manifest.name == "go.mod": + continue # Go workspaces still keep dependency files per module. if manifest.name == "package.json": members = workspace.get("workspaces", []) elif manifest.name == "pyproject.toml": diff --git a/tests/test_dependency_licenses.py b/tests/test_dependency_licenses.py index aaf4ab8c..b14cb28c 100644 --- a/tests/test_dependency_licenses.py +++ b/tests/test_dependency_licenses.py @@ -51,6 +51,13 @@ def uv_script(*dependencies): return f"# /// script\n# dependencies = {dependency_list}\n# ///\n" +def go_mod(*requirements): + lines = ["module example.com/root", "", "go 1.26.0", "", "require ("] + lines.extend(f"\t{requirement}" for requirement in requirements) + lines.append(")") + return "\n".join(lines) + "\n" + + def assert_equal(actual, expected): assert actual == expected @@ -91,6 +98,7 @@ def test_native_check_targets_for_direct_projects(): "npm", ), ("Cargo.toml", "Cargo.lock", '[dependencies]\nexample="1"', "cargo"), + ("go.mod", "go.sum", go_mod("example.com/module v1.0.0"), "go"), ]: targets = check_coverage( {}, {f"project/{manifest}": content, f"project/{lock}": "inventory"} @@ -108,6 +116,14 @@ def test_native_check_targets_for_changed_lock_only_and_no_changes(): check_coverage(before, {**before, "uv.lock": uv_lock("2.0")}), [{"directory": ".", "manager": "uv"}], ) + go_before = { + "go.mod": go_mod("example.com/module v1.0.0"), + "go.sum": "old sums", + } + assert_equal( + check_coverage(go_before, {**go_before, "go.sum": "new sums"}), + [{"directory": ".", "manager": "go"}], + ) def test_full_audit_validates_locked_projects_not_manifest_templates(): @@ -169,6 +185,7 @@ def test_added_dependency_manifest_requires_lockfile(): "new/pyproject.toml": '[project]\ndependencies=["example"]', "new/package.json": '{"dependencies":{"example":"1"}}', "new/Cargo.toml": '[dependencies]\nexample="1"', + "new/go.mod": go_mod("example.com/module v1.0.0"), } for path, content in manifests.items(): with pytest.raises(ValueError, match="no .* inventory"): @@ -211,6 +228,7 @@ def test_non_dependency_manifest_changes_do_not_require_new_inventory(): {"pyproject.toml": '[project]\nname="new"\ndependencies=["example"]'}, ) check_coverage({}, {"pyproject.toml": '[project]\nname="stdlib-only"'}) + check_coverage({}, {"go.mod": "module example.com/stdlib-only\ngo 1.26.0\n"}) def test_deleting_lockfile_with_remaining_dependencies_fails(): @@ -379,6 +397,65 @@ def test_cargo_sources_and_versions(): ) +def test_go_inventory_includes_external_requirements_and_applies_replacements(): + content = ( + go_mod( + "example.com/direct v1.0.0", + "example.com/indirect v2.0.0 // indirect", + "example.com/local v3.0.0", + ) + + """ +replace example.com/direct => example.com/fork v1.1.0 +replace example.com/local => ./local +""" + ) + dependencies = checker.inventory("project/go.mod", content) + assert_equal( + {(item.name, item.version, item.source) for item in dependencies}, + { + ("example.com/fork", "v1.1.0", checker.GO_MODULE_SOURCE), + ("example.com/indirect", "v2.0.0", checker.GO_MODULE_SOURCE), + }, + ) + assert_equal( + {item.name for item in checker.direct_inventory("project/go.mod", content)}, + {"example.com/fork"}, + ) + + +def test_go_inventory_handles_blocks_quotes_and_version_specific_replacements(): + content = '''module example.com/root +require "example.com/module" v1.0.0 +replace ( + example.com/module v0.9.0 => example.com/old-fork v0.9.1 + example.com/module v1.0.0 => example.com/current-fork v1.0.1 +) +''' + assert_equal( + {(item.name, item.version) for item in checker.inventory("go.mod", content)}, + {("example.com/current-fork", "v1.0.1")}, + ) + assert_equal( + { + (item.name, item.version) + for item in checker.inventory( + "go.mod", "require(\nexample.com/adjacent v1.2.3\n)\n" + ) + }, + {("example.com/adjacent", "v1.2.3")}, + ) + + +def test_malformed_go_inventory_fails_closed(): + for content in ( + "require example.com/module\n", + "replace example.com/module v1.0.0 example.com/fork v1.0.1\n", + "require (\nexample.com/module v1.0.0\n", + ): + with pytest.raises(ValueError): + checker.inventory("go.mod", content) + + def test_direct_inventory_excludes_transitive_dependencies(): uv_content = ( uv_lock() @@ -613,11 +690,62 @@ def test_cargo_reads_exact_version(): fetch.assert_called_once_with("https://crates.io/api/v1/crates/example/1.0") +def test_go_reads_exact_version_and_requires_every_reported_license(): + dependency = checker.Dependency( + "go", "example.com/module", "v1.2.3", checker.GO_MODULE_SOURCE + ) + with mock.patch.object( + checker, + "fetch_json", + return_value={ + "versionKey": { + "system": "GO", + "name": dependency.name, + "version": dependency.version, + }, + "licenses": ["Apache-2.0", "BSD-3-Clause"], + }, + ) as fetch: + result = checker.check_dependency(dependency, POLICY) + assert_true(result["passed"]) + assert_equal(result["license"], "(Apache-2.0) AND (BSD-3-Clause)") + fetch.assert_called_once_with( + "https://api.deps.dev/v3/systems/GO/packages/example.com%2Fmodule/versions/v1.2.3" + ) + + +def test_go_rejects_missing_or_mismatched_metadata(): + dependency = checker.Dependency( + "go", "example.com/module", "v1.2.3", checker.GO_MODULE_SOURCE + ) + for response in ( + { + "versionKey": { + "system": "GO", + "name": dependency.name, + "version": dependency.version, + }, + "licenses": [], + }, + { + "versionKey": { + "system": "GO", + "name": dependency.name, + "version": "v9.9.9", + }, + "licenses": ["MIT"], + }, + ): + with mock.patch.object(checker, "fetch_json", return_value=response): + assert_false(checker.check_dependency(dependency, POLICY)["passed"]) + + def test_unknown_sources_never_trigger_arbitrary_network_requests(): for ecosystem, source in [ ("pypi", '{"registry":"http://localhost/simple"}'), ("npm", "https://registry.npmjs.org.evil.test/pkg.tgz"), ("cargo", "git+https://example.org#abc"), + ("go", "https://proxy.example.org"), ]: with ( mock.patch.object(checker, "fetch_json") as fetch,