feat: add ValidatingAdmissionPolicy for team-specific hostname validation - #3598
Open
CasLubbers wants to merge 11 commits into
Open
feat: add ValidatingAdmissionPolicy for team-specific hostname validation#3598CasLubbers wants to merge 11 commits into
CasLubbers wants to merge 11 commits into
Conversation
Contributor
|
Comparison of Helm chart templating output: # raw/templates/resources.yaml
@@ (root level) @@
# admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy/apl-team-hostnames
! + one document added:
+ # Source: raw/templates/resources.yaml
+ apiVersion: admissionregistration.k8s.io/v1
+ kind: ValidatingAdmissionPolicy
+ metadata:
+ name: apl-team-hostnames
+ labels:
+ app: raw
+ app.kubernetes.io/instance: kubernetes-gateways-artifacts
+ app.kubernetes.io/managed-by: Helm
+ app.kubernetes.io/name: raw
+ app.kubernetes.io/part-of: otomi
+ app.kubernetes.io/version: "0.2.3"
+ helm.sh/chart: raw-0.2.3
+ spec:
+ variables:
+ - name: reserved
+ expression: "{ 'alertmanager-admin.dev.linode-apl.net': 'admin-po-alertmanager', 'alertmanager-demo.dev.linode-apl.net': 'demo-po-alertmanager', 'alertmanager-dev.dev.linode-apl.net': 'dev-po-alertmanager', 'alertmanager.dev.linode-apl.net': '', 'api.dev.linode-apl.net': '', 'argocd.dev.linode-apl.net': '', 'auth.dev.linode-apl.net': '', 'console.dev.linode-apl.net': '', 'gitea.dev.linode-apl.net': '', 'grafana-admin.dev.linode-apl.net': 'admin-po-grafana', 'grafana-demo.dev.linode-apl.net': 'demo-po-grafana', 'grafana-dev.dev.linode-apl.net': 'dev-po-grafana', 'grafana.dev.linode-apl.net': '', 'harbor.dev.linode-apl.net': '', 'keycloak.dev.linode-apl.net': '', 'kubeflow-pipelines.dev.linode-apl.net': '', 'prometheus.dev.linode-apl.net': '', 'tekton-admin.dev.linode-apl.net': 'tekton-admin', 'tekton-demo.dev.linode-apl.net': 'tekton-demo', 'tekton-dev.dev.linode-apl.net': 'tekton-dev', 'tekton.dev.linode-apl.net': '', 'tty.dev.linode-apl.net': '' }"
+ - name: hostnames
+ expression: |
+ request.resource.resource == 'listenersets'
+ ? object.spec.listeners.map(l, has(l.hostname) ? l.hostname : '')
+ : (has(object.spec.hostnames) ? object.spec.hostnames : [])
+
+ failurePolicy: Fail
+ validations:
+ - expression: "size(variables.hostnames) > 0 && !variables.hostnames.exists(h, h == \"\")"
+ message: "every hostname must be set: an empty hostname binds the whole cluster domain"
+ - expression: |
+ variables.hostnames.all(h,
+ !h.startsWith('*.')
+ || !('dev.linode-apl.net' == h.substring(2) || 'dev.linode-apl.net'.endsWith('.' + h.substring(2)))
+ )
+
+ message: "a wildcard hostname may not cover dev.linode-apl.net"
+ - expression: |
+ variables.hostnames.all(h,
+ !(h in variables.reserved) || object.metadata.name == variables.reserved[h]
+ )
+
+ message: "this hostname is served by a platform app and cannot be rebound"
+ matchConstraints:
+ resourceRules:
+ - resources:
+ - httproutes
+ - listenersets
+ apiGroups:
+ - gateway.networking.k8s.io
+ apiVersions:
+ - "*"
+ operations:
+ - CREATE
+ - UPDATE
@@ (root level) @@
# admissionregistration.k8s.io/v1/ValidatingAdmissionPolicyBinding/apl-team-hostnames
! + one document added:
+ # Source: raw/templates/resources.yaml
+ apiVersion: admissionregistration.k8s.io/v1
+ kind: ValidatingAdmissionPolicyBinding
+ metadata:
+ name: apl-team-hostnames
+ labels:
+ app: raw
+ app.kubernetes.io/instance: kubernetes-gateways-artifacts
+ app.kubernetes.io/managed-by: Helm
+ app.kubernetes.io/name: raw
+ app.kubernetes.io/part-of: otomi
+ app.kubernetes.io/version: "0.2.3"
+ helm.sh/chart: raw-0.2.3
+ spec:
+ matchResources:
+ namespaceSelector:
+ matchLabels:
+ type: team
+ policyName: apl-team-hostnames
+ validationActions:
+ - Deny
# oauth2-proxy/templates/configmap.yaml
@@ data.oauth2_proxy.cfg @@
! ± value change in multiline text (one insert, one deletion)
# Defaults
email_domains = [ "*" ]
# Upon successful token verification, oauth2-proxy returns 200 and K8s gateway passes the request to the app route
[five lines unchanged)]
scope = "openid email profile"
oidc_issuer_url = "https://keycloak.dev.linode-apl.net/realms/otomi"
code_challenge_method = "S256"
insecure_oidc_allow_unverified_email = true
- cookie_domains = ["console.dev.linode-apl.net","auth.dev.linode-apl.net","api.dev.linode-apl.net","tty.dev.linode-apl.net","gitea.dev.linode-apl.net","keycloak.dev.linode-apl.net","harbor.dev.linode-apl.net","argocd.dev.linode-apl.net","tekton.dev.linode-apl.net","kubeflow-pipelines.dev.linode-apl.net","alertmanager.dev.linode-apl.net","grafana.dev.linode-apl.net","prometheus.dev.linode-apl.net","oauth2-proxy.istio-system.svc.cluster.local","alertmanager-admin.dev.linode-apl.net","alertmanager-demo.dev.linode-apl.net","alertmanager-dev.dev.linode-apl.net","grafana-admin.dev.linode-apl.net","grafana-demo.dev.linode-apl.net","grafana-dev.dev.linode-apl.net","tekton-admin.dev.linode-apl.net","tekton-demo.dev.linode-apl.net","tekton-dev.dev.linode-apl.net"]
+ cookie_domains = ["console.dev.linode-apl.net","auth.dev.linode-apl.net","alertmanager.dev.linode-apl.net","argocd.dev.linode-apl.net","tty.dev.linode-apl.net","gitea.dev.linode-apl.net","grafana.dev.linode-apl.net","harbor.dev.linode-apl.net","keycloak.dev.linode-apl.net","kubeflow-pipelines.dev.linode-apl.net","tekton.dev.linode-apl.net","api.dev.linode-apl.net","prometheus.dev.linode-apl.net","oauth2-proxy.istio-system.svc.cluster.local","alertmanager-admin.dev.linode-apl.net","alertmanager-demo.dev.linode-apl.net","alertmanager-dev.dev.linode-apl.net","grafana-admin.dev.linode-apl.net","grafana-demo.dev.linode-apl.net","grafana-dev.dev.linode-apl.net","tekton-admin.dev.linode-apl.net","tekton-demo.dev.linode-apl.net","tekton-dev.dev.linode-apl.net"]
# lax is enough for the redirect-based OIDC login and blocks cross-site delivery
cookie_samesite = "lax"
# set to the same as keycloak realm's accessCodeLifespan
cookie_refresh = "1m"
[three lines unchanged)]
pass_authorization_header = true
skip_auth_routes = "/manifest.json"
silence_ping_logging = true
custom_templates_dir = "/etc/oauth2-proxy"
# oauth2-proxy/templates/deployment.yaml
# rabbitmq-cluster-operator/templates/messaging-topology-operator/validating-webhook-configuration.yaml
# values-repo.yaml
|
Contributor
There was a problem hiding this comment.
Pull request overview
Adds a Kubernetes ValidatingAdmissionPolicy/binding to enforce that team namespaces can only bind hostnames within the cluster domain that follow the team-specific convention, preventing teams from creating hostnames that resemble platform hostnames.
Changes:
- Add a
ValidatingAdmissionPolicy+ValidatingAdmissionPolicyBindingtargetingHTTPRouteandListenerSetresources in namespaces labeledtype: team. - Implement CEL validations to (1) require explicit hostnames and (2) constrain in-cluster-domain hostnames to
<name>-<teamId>.<domainSuffix>with additional wildcard safeguards. - Add templating helpers to build a CEL-safe regex from
cluster.domainSuffix.
Suppressed comments (1)
values/kubernetes-gateways/kubernetes-gateways-raw.gotmpl:121
- The regex string in
variables.patterncurrently uses\.{{ $domainSuffixRe }}(single backslash before the dot). In CEL string literals, the dot should be escaped as\\.(i.e., change\.→\\.) so the regex matches a literal.; this matches how CEL regexes are written elsewhere in the repo (e.g., Gateway API CRD validations use\\.).
- name: pattern
expression: |
'^[a-z0-9]([-a-z0-9]*[a-z0-9])?-' + variables.teamId + '\.{{ $domainSuffixRe }}$'
💡 Add a code-review agent skill or configure MCP servers for context-aware, tailored reviews. Learn more in the docs.
Comment on lines
+133
to
+137
| variables.hostnames.all(h, | ||
| h.startsWith('*.') | ||
| ? !'{{ $domainSuffix }}'.endsWith(h.substring(2)) | ||
| : (!h.endsWith('.{{ $domainSuffix }}') || h.matches(variables.pattern)) | ||
| ) |
Comment on lines
+5
to
+6
| {{- /* Dots escaped so the suffix can be used inside a CEL regex. */}} | ||
| {{- $domainSuffixRe := $domainSuffix | replace "." "\\." }} |
Contributor
Author
|
routes used for testing: # ok-1: an ordinary team hostname
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: vaptest-ok-1
namespace: team-demo
spec:
hostnames: ["vaptest-demo.caslubbers-5067-1.dev-akamai-apl.net"]
rules:
- backendRefs: [{name: dummy, port: 8080}]
---
# ok-2: an external alias, outside the cluster domain
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: vaptest-ok-2
namespace: team-demo
spec:
hostnames: ["app.example.com"]
rules:
- backendRefs: [{name: dummy, port: 8080}]
---
# ok-3: a name with no team prefix - allowed by the deny-list, e.g. headlamp
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: vaptest-ok-3
namespace: team-demo
spec:
hostnames: ["headlamp.caslubbers-5067-1.dev-akamai-apl.net"]
rules:
- backendRefs: [{name: dummy, port: 8080}]
---
# deny-1: the console - the originally reported attack
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: vaptest-deny-1
namespace: team-demo
spec:
hostnames: ["console.caslubbers-5067-1.dev-akamai-apl.net"]
rules:
- backendRefs: [{name: dummy, port: 8080}]
---
# deny-2: harbor
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: vaptest-deny-2
namespace: team-demo
spec:
hostnames: ["harbor.caslubbers-5067-1.dev-akamai-apl.net"]
rules:
- backendRefs: [{name: dummy, port: 8080}]
---
# deny-3: the team's own tekton hostname, under a name the team chose
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: vaptest-deny-3
namespace: team-demo
spec:
hostnames: ["tekton-demo.caslubbers-5067-1.dev-akamai-apl.net"]
rules:
- backendRefs: [{name: dummy, port: 8080}]
---
# deny-4: same for grafana
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: vaptest-deny-4
namespace: team-demo
spec:
hostnames: ["grafana-demo.caslubbers-5067-1.dev-akamai-apl.net"]
rules:
- backendRefs: [{name: dummy, port: 8080}]
---
# deny-5: a wildcard covering the cluster domain
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: vaptest-deny-5
namespace: team-demo
spec:
hostnames: ["*.caslubbers-5067-1.dev-akamai-apl.net"]
rules:
- backendRefs: [{name: dummy, port: 8080}]
---
# deny-6: no hostnames at all - would bind the whole wildcard listener
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: vaptest-deny-6
namespace: team-demo
spec:
rules:
- backendRefs: [{name: dummy, port: 8080}]outcome: httproute.gateway.networking.k8s.io/vaptest-ok-1 created
httproute.gateway.networking.k8s.io/vaptest-ok-2 created
httproute.gateway.networking.k8s.io/vaptest-ok-3 created
Error from server (Invalid): error when creating "STDIN": httproutes.gateway.networking.k8s.io "vaptest-deny-1" is forbidden: ValidatingAdmissionPolicy 'apl-team-hostnames' with binding 'apl-team-hostnames' denied request: this hostname is served by a platform app and cannot be rebound
Error from server (Invalid): error when creating "STDIN": httproutes.gateway.networking.k8s.io "vaptest-deny-2" is forbidden: ValidatingAdmissionPolicy 'apl-team-hostnames' with binding 'apl-team-hostnames' denied request: this hostname is served by a platform app and cannot be rebound
Error from server (Invalid): error when creating "STDIN": httproutes.gateway.networking.k8s.io "vaptest-deny-3" is forbidden: ValidatingAdmissionPolicy 'apl-team-hostnames' with binding 'apl-team-hostnames' denied request: this hostname is served by a platform app and cannot be rebound
Error from server (Invalid): error when creating "STDIN": httproutes.gateway.networking.k8s.io "vaptest-deny-4" is forbidden: ValidatingAdmissionPolicy 'apl-team-hostnames' with binding 'apl-team-hostnames' denied request: this hostname is served by a platform app and cannot be rebound
Error from server (Invalid): error when creating "STDIN": httproutes.gateway.networking.k8s.io "vaptest-deny-5" is forbidden: ValidatingAdmissionPolicy 'apl-team-hostnames' with binding 'apl-team-hostnames' denied request: a wildcard hostname may not cover caslubbers-5067-1.dev-akamai-apl.net
Error from server (Invalid): error when creating "STDIN": httproutes.gateway.networking.k8s.io "vaptest-deny-6" is forbidden: ValidatingAdmissionPolicy 'apl-team-hostnames' with binding 'apl-team-hostnames' denied request: every hostname must be set: an empty hostname binds the whole cluster domain |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
📌 Summary
Add ValidatingAdmissionPolicy so that teams cannot create hostnames that look like platform hosts
🔍 Reviewer Notes
🧹 Checklist