diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index 780cf8b9c6..15198a891f 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -225,6 +225,35 @@ Envoy Gateway is already installed by Skaffold (the `envoy-gateway` Helm release service for the proxy; klipper-lb binds it to hostPort 80, reachable via the `8080:80` load balancer port mapping. +### BackendTLSPolicy (end-to-end TLS) + +To enable end-to-end TLS between the Gateway proxy and the gateway pod, add +BackendTLSPolicy values to the Helm install: + +```bash +helm upgrade --install openshell deploy/helm/openshell \ + --set grpcRoute.enabled=true \ + --set grpcRoute.backendTLSPolicy.enabled=true \ + --set server.tls.enableMtls=false \ + ... +``` + +This requires `server.tls.enableMtls=false` because ingress proxies cannot +present client certificates to the backend. The certgen hook creates a backend +CA ConfigMap from the server Secret's `ca.crt` key. With cert-manager, a +separate post-install Job polls for the cert-manager-issued certificate (up to +`pkiInitJob.timeoutSeconds`); with built-in PKI the ConfigMap is created in the +same pre-install hook. The ConfigMap is reconciled on every upgrade so CA +rotations propagate automatically. + +Key Helm values: +- `grpcRoute.backendTLSPolicy.enabled`: create the BackendTLSPolicy resource +- `grpcRoute.backendTLSPolicy.caCertificateConfigMapName`: override ConfigMap name +- `grpcRoute.backendTLSPolicy.hostname`: override backend validation hostname +- `server.tls.enableMtls`: must be `false` for BackendTLSPolicy +- `pkiInitJob.timeoutSeconds`: polling duration for cert-manager mode +- `pkiInitJob.failOnTimeout`: fail install if cert-manager times out + ### Keycloak OIDC One-time setup — only needed once per cluster lifetime: diff --git a/architecture/gateway.md b/architecture/gateway.md index 8fc28dfb21..b1f6c6341b 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -757,6 +757,25 @@ requested present -> generate and write. This guards continuity across restarts and upgrades while still recovering cleanly if an operator deletes everything and starts over. +When `grpcRoute.backendTLSPolicy.enabled=true`, the certgen hook also creates a +`ConfigMap` containing the CA certificate (`ca.crt`) used by the Gateway proxy +to validate the backend pod's TLS certificate. The CA is always read from the +authoritative server Secret (not the in-memory bundle) so that enabling +BackendTLSPolicy on an existing release uses the CA that actually signed the +server certificate. The ConfigMap is reconciled on every hook run: if the CA +changes (rotation, re-issue), the ConfigMap is updated in place. In built-in PKI +mode the ConfigMap is created in the same pre-install hook. In cert-manager +mode, a separate post-install/post-upgrade hook Job polls for the cert-manager- +issued server Secret and then creates or updates the ConfigMap, because +cert-manager Certificate resources are regular release objects applied after +pre-install hooks. + +The `server.tls.enableMtls` value controls whether the gateway requires client +certificates. When `enableMtls` is `false`, the gateway runs HTTPS-only without +client certificate verification (use OIDC for identity instead). BackendTLSPolicy +requires `enableMtls=false` because the ingress proxy cannot present client +certificates to the backend. + Operators who manage TLS PKI with cert-manager enable `certManager.enabled`; cert-manager takes precedence over built-in TLS generation and the chart still renders the JWT-only hook. Operators who pre-create all TLS and JWT Secrets can diff --git a/crates/openshell-server/src/certgen.rs b/crates/openshell-server/src/certgen.rs index 1f3ffca02d..764ceba98c 100644 --- a/crates/openshell-server/src/certgen.rs +++ b/crates/openshell-server/src/certgen.rs @@ -24,7 +24,7 @@ use clap::Args; use k8s_openapi::ByteString; -use k8s_openapi::api::core::v1::Secret; +use k8s_openapi::api::core::v1::{ConfigMap, Secret}; use kube::Client; use kube::api::{Api, ObjectMeta, PostParams}; use miette::{IntoDiagnostic, Result, WrapErr}; @@ -78,6 +78,33 @@ pub struct CertgenArgs { /// For local debugging. #[arg(long)] dry_run: bool, + + /// Name of a `ConfigMap` to create containing the CA certificate (key: ca.crt) + /// for `BackendTLSPolicy` backend validation. The CA is always read from the + /// authoritative server Secret: --server-secret-name in full PKI mode, + /// --backend-ca-source-secret in --jwt-only mode. + #[arg(long, value_name = "NAME")] + backend_ca_configmap_name: Option, + + /// Name of an existing `Secret` containing a ca.crt key to populate the + /// backend CA `ConfigMap` from. Required with --jwt-only when + /// --backend-ca-configmap-name is set (typically the server TLS `Secret` + /// created by cert-manager). + #[arg(long, value_name = "NAME", requires = "backend_ca_configmap_name")] + backend_ca_source_secret: Option, + + /// Maximum time in seconds to poll for the backend CA source `Secret` when + /// using cert-manager. Defaults to 90 seconds. The Helm chart sets this to + /// (Job activeDeadlineSeconds - 30) to leave margin for `ConfigMap` creation. + #[arg(long, value_name = "SECONDS", default_value = "90")] + backend_ca_poll_timeout_seconds: u64, + + /// Fail with an error if the backend CA source `Secret` is not found within + /// the polling timeout. When false (default), the hook succeeds with a warning + /// and the `ConfigMap` is not created. The Helm chart sets this based on + /// pkiInitJob.failOnTimeout. + #[arg(long)] + backend_ca_fail_on_timeout: bool, } pub async fn run(args: CertgenArgs) -> Result<()> { @@ -97,7 +124,13 @@ pub async fn run(args: CertgenArgs) -> Result<()> { run_local(dir, &args.server_sans) } else { let bundle = generate_pki(&args.server_sans)?; - run_kubernetes(&args, &bundle).await + run_kubernetes(&args, &bundle).await?; + + if let Some(ref cm_name) = args.backend_ca_configmap_name { + create_backend_ca_configmap_if_needed(&args, cm_name).await?; + } + + Ok(()) } } @@ -293,6 +326,165 @@ async fn create_tls_secrets( Ok(()) } +fn extract_ca_from_secret(secret: &Secret, name: &str) -> Result { + let data = secret + .data + .as_ref() + .ok_or_else(|| miette::miette!("secret {name} has no data"))?; + let ca = data + .get("ca.crt") + .ok_or_else(|| miette::miette!("secret {name} has no ca.crt key"))?; + String::from_utf8(ca.0.clone()) + .into_diagnostic() + .wrap_err("ca.crt is not valid UTF-8") +} + +async fn create_backend_ca_configmap_if_needed( + args: &CertgenArgs, + configmap_name: &str, +) -> Result<()> { + let namespace = args + .namespace + .as_deref() + .ok_or_else(|| miette::miette!("--namespace is required (or set POD_NAMESPACE)"))?; + + let client = Client::try_default() + .await + .into_diagnostic() + .wrap_err("failed to construct Kubernetes client for backend CA ConfigMap")?; + let secret_api: Api = Api::namespaced(client.clone(), namespace); + let cm_api: Api = Api::namespaced(client, namespace); + + // Resolve the CA from the authoritative server Secret rather than the + // in-memory bundle so upgrades that enable BackendTLSPolicy on an + // existing release use the CA that actually signed the server cert. + let ca_pem = if let Some(source_secret) = &args.backend_ca_source_secret { + let poll_timeout = std::time::Duration::from_secs(args.backend_ca_poll_timeout_seconds); + let poll_interval = std::time::Duration::from_secs(2); + let start = std::time::Instant::now(); + + let secret = loop { + match secret_api + .get_opt(source_secret) + .await + .into_diagnostic() + .wrap_err_with(|| format!("failed to read secret {source_secret}"))? + { + Some(secret) => break secret, + None if start.elapsed() >= poll_timeout => { + let msg = format!( + "Backend CA source secret {source_secret} not found after {timeout_secs}s; \ + ConfigMap {configmap_name} not created. This is expected if cert-manager \ + is still issuing the certificate.", + timeout_secs = poll_timeout.as_secs() + ); + if args.backend_ca_fail_on_timeout { + return Err(miette::miette!( + "{msg} Install failed due to --backend-ca-fail-on-timeout." + )); + } + warn!( + secret = %source_secret, + configmap = %configmap_name, + timeout_secs = poll_timeout.as_secs(), + "{msg} Run helm upgrade after the TLS secret exists or the BackendTLSPolicy \ + will remain non-functional until the ConfigMap is created manually." + ); + return Ok(()); + } + None => { + if start.elapsed().as_secs().is_multiple_of(10) { + info!( + secret = %source_secret, + elapsed_secs = start.elapsed().as_secs(), + "Waiting for cert-manager to issue TLS certificate..." + ); + } + tokio::time::sleep(poll_interval).await; + } + } + }; + + info!( + secret = %source_secret, + elapsed_secs = start.elapsed().as_secs(), + "cert-manager TLS certificate found." + ); + extract_ca_from_secret(&secret, source_secret)? + } else if let Some(server_secret) = &args.server_secret_name { + let secret = secret_api + .get(server_secret) + .await + .into_diagnostic() + .wrap_err_with(|| format!("failed to read server secret {server_secret}"))?; + extract_ca_from_secret(&secret, server_secret)? + } else { + return Err(miette::miette!( + "--backend-ca-source-secret or --server-secret-name is required \ + with --backend-ca-configmap-name" + )); + }; + + // Reconcile: create or update so the backend CA stays current across + // CA rotations and upgrades. + if let Some(existing) = cm_api + .get_opt(configmap_name) + .await + .into_diagnostic() + .wrap_err_with(|| format!("failed to read configmap {configmap_name}"))? + { + let up_to_date = existing + .data + .as_ref() + .and_then(|d| d.get("ca.crt")) + .map(String::as_str) + == Some(&ca_pem); + if up_to_date { + info!( + namespace = %namespace, + configmap = %configmap_name, + "Backend CA ConfigMap is up-to-date, skipping." + ); + return Ok(()); + } + let mut updated = existing; + updated.data = Some(BTreeMap::from([("ca.crt".to_string(), ca_pem)])); + cm_api + .replace(configmap_name, &PostParams::default(), &updated) + .await + .into_diagnostic() + .wrap_err_with(|| format!("failed to update configmap {configmap_name}"))?; + info!( + namespace = %namespace, + configmap = %configmap_name, + "Backend CA ConfigMap updated with current CA." + ); + return Ok(()); + } + + let configmap = ConfigMap { + metadata: ObjectMeta { + name: Some(configmap_name.to_string()), + ..Default::default() + }, + data: Some(BTreeMap::from([("ca.crt".to_string(), ca_pem)])), + ..Default::default() + }; + + cm_api + .create(&PostParams::default(), &configmap) + .await + .into_diagnostic() + .wrap_err_with(|| format!("failed to create configmap {configmap_name}"))?; + + info!( + namespace = %namespace, + configmap = %configmap_name, + "Backend CA ConfigMap created." + ); + Ok(()) +} + fn tls_secret(name: &str, crt_pem: &str, key_pem: &str, ca_pem: &str) -> Secret { let mut data = BTreeMap::new(); data.insert( diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index c11d895618..9b01b749a4 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -2314,6 +2314,59 @@ mod tests { .expect("complete package-generated TLS paths may not exist before certificate generation"); } + #[test] + fn generate_certs_backend_ca_configmap_flags_parse() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _g1 = EnvVarGuard::remove("OPENSHELL_DB_URL"); + let _g2 = EnvVarGuard::remove("POD_NAMESPACE"); + + let cli = Cli::try_parse_from([ + "openshell-gateway", + "generate-certs", + "--namespace", + "openshell", + "--jwt-only", + "--jwt-secret-name", + "openshell-jwt-keys", + "--backend-ca-configmap-name", + "openshell-backend-ca", + "--backend-ca-source-secret", + "openshell-server-tls", + ]) + .expect("backend CA ConfigMap flags should parse with --jwt-only"); + + assert!(matches!( + cli.command, + Some(super::Commands::GenerateCerts(_)) + )); + } + + #[test] + fn generate_certs_backend_ca_source_secret_requires_configmap_name() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _g1 = EnvVarGuard::remove("OPENSHELL_DB_URL"); + let _g2 = EnvVarGuard::remove("POD_NAMESPACE"); + + let err = Cli::try_parse_from([ + "openshell-gateway", + "generate-certs", + "--namespace", + "openshell", + "--jwt-only", + "--jwt-secret-name", + "openshell-jwt-keys", + "--backend-ca-source-secret", + "openshell-server-tls", + ]) + .expect_err("--backend-ca-source-secret should require --backend-ca-configmap-name"); + + assert_eq!(err.kind(), clap::error::ErrorKind::MissingRequiredArgument); + } + #[test] fn bare_invocation_with_no_db_url_parses_for_runtime_defaults() { // db_url is Option at the clap level so subcommand parsing diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 92bd2a9db8..3c32e24b67 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -56,6 +56,9 @@ helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version -backend-ca` when empty. The certgen hook auto-creates this: with pkiInitJob (default), immediately on install/upgrade; with cert-manager, the hook polls for pkiInitJob.timeoutSeconds seconds waiting for cert-manager to issue the server certificate, then creates the ConfigMap. A single install usually succeeds; if cert-manager takes longer, increase pkiInitJob.timeoutSeconds. By default (pkiInitJob.failOnTimeout=true), the install fails if the timeout is reached; set failOnTimeout=false to allow the install to succeed and run `helm upgrade` after the certificate is issued. | +| grpcRoute.backendTLSPolicy.enabled | bool | `false` | Create a BackendTLSPolicy resource for end-to-end TLS between the Gateway proxy and the OpenShell gateway pod. The traffic flow is: client → HTTPS → Gateway (terminate) → TLS (re-encrypt) → gateway pod. Requires server.disableTls=false and server.tls.enableMtls=false. The certgen hook auto-creates the backend CA ConfigMap. | +| grpcRoute.backendTLSPolicy.hostname | string | `""` | Hostname the Gateway proxy validates against the backend's TLS certificate SAN. Defaults to the service FQDN (`..svc.cluster.local`) when empty, which matches the SAN included by both cert-manager and the pkiInitJob. | | grpcRoute.enabled | bool | `false` | Create a Gateway API GRPCRoute for the gateway service. | | grpcRoute.gateway.className | string | `"eg"` | GatewayClass to reference. Envoy Gateway installs one named "eg". | | grpcRoute.gateway.create | bool | `false` | When true, a Gateway resource is created in the release namespace. Set to false and provide name/namespace to attach to a pre-existing Gateway. | @@ -204,8 +210,10 @@ discovery endpoint or its TLS CA. | openshiftRoute.enabled | bool | `false` | Create an OpenShift Route with TLS passthrough. | | openshiftRoute.host | string | `""` | Hostname for the Route. Must match a SAN on the gateway's server cert. | | pkiInitJob.enabled | bool | `true` | Run a pre-install/pre-upgrade Job that creates gateway and client mTLS Secrets. When certManager.enabled=true, cert-manager owns TLS and this same hook runs in JWT-only mode even if pkiInitJob.enabled remains true. | +| pkiInitJob.failOnTimeout | bool | `true` | Fail the helm install/upgrade if cert-manager does not issue the certificate within the polling timeout. When true (default), the install fails immediately if the timeout is reached, providing clear feedback that BackendTLSPolicy is non-functional. When false, the hook succeeds with a warning and you can run `helm upgrade` after cert-manager issues the certificate to create the backend CA ConfigMap. If you set this to false and see "TLS error: Secret is not supplied by SDS" when connecting to the gateway, check if the TLS secret exists and run `helm upgrade` to create the ConfigMap. | | pkiInitJob.serverDnsNames | list | `[]` | Extra DNS SANs to append to the server certificate. | | pkiInitJob.serverIpAddresses | list | `[]` | Extra IP SANs to append to the server certificate. | +| pkiInitJob.timeoutSeconds | int | `120` | Maximum time in seconds for the certgen hook to poll for cert-manager certificates. When using cert-manager with BackendTLSPolicy, the hook polls for this many seconds waiting for the certificate to be issued, then creates the backend CA ConfigMap. The Job deadline is set to (timeoutSeconds + 30) to allow time for ConfigMap creation and cleanup. Increase this if cert-manager takes longer than 120 seconds to issue certificates. | | podAnnotations | object | `{}` | Extra annotations to add to the gateway pod. | | podLabels | object | `{}` | Extra labels to add to the gateway pod. | | podLifecycle.terminationGracePeriodSeconds | int | `5` | Grace period, in seconds, before Kubernetes terminates the gateway pod. | @@ -286,8 +294,9 @@ discovery endpoint or its TLS CA. | server.sandboxNamespace | string | `""` | Namespace where sandbox pods are created. Defaults to the Helm release namespace (.Release.Namespace) when left empty. | | server.telemetryEnabled | bool | `true` | Enable anonymous OpenShell telemetry from the gateway and the sandbox supervisors it launches. | | server.tls.certSecretName | string | `"openshell-server-tls"` | K8s secret (type kubernetes.io/tls) with tls.crt and tls.key for the server. | -| server.tls.clientCaSecretName | string | `"openshell-server-client-ca"` | K8s secret with ca.crt for client certificate verification (mTLS). Set to "" to disable mTLS and run HTTPS-only (use OIDC for auth instead). Do not set to null; omit the key to use the default secret name above. | +| server.tls.clientCaSecretName | string | `"openshell-server-client-ca"` | K8s secret with ca.crt for client certificate verification (mTLS). Only used when enableMtls is true. Set to "" to disable client certificate verification for HTTPS-only mode. | | server.tls.clientTlsSecretName | string | `"openshell-client-tls"` | K8s secret mounted into sandbox pods for mTLS to the server. | +| server.tls.enableMtls | bool | `true` | Enable mTLS client certificate authentication. When false, the gateway runs HTTPS-only without requiring client certificates (use OIDC for auth instead). Must be false when using BackendTLSPolicy because ingress proxies cannot present client certificates to the backend. | | server.workspaceDefaultStorageSize | string | `""` | Default storage size for the workspace PVC in sandbox pods. Uses Kubernetes quantity syntax (e.g. "2Gi", "10Gi", "500Mi"). Empty = built-in default (2Gi). | | server.workspaceStorageClass | string | `""` | Kubernetes StorageClass for the workspace PVC in sandbox pods. Empty (default) = omit storageClassName, using the cluster's default StorageClass. Set this on clusters with no default StorageClass, otherwise the workspace PVC stays Pending and the sandbox never starts. | | service.healthPort | int | `8081` | Gateway health service port. | diff --git a/deploy/helm/openshell/README.md.gotmpl b/deploy/helm/openshell/README.md.gotmpl index cf8677741e..6e6f2012fc 100644 --- a/deploy/helm/openshell/README.md.gotmpl +++ b/deploy/helm/openshell/README.md.gotmpl @@ -56,6 +56,9 @@ helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version -backend-ca` when empty. The certgen hook auto-creates this: + # with pkiInitJob (default), immediately on install/upgrade; with + # cert-manager, the hook polls for pkiInitJob.timeoutSeconds seconds + # waiting for cert-manager to issue the server certificate, then creates the + # ConfigMap. A single install usually succeeds; if cert-manager takes longer, + # increase pkiInitJob.timeoutSeconds. By default (pkiInitJob.failOnTimeout=true), + # the install fails if the timeout is reached; set failOnTimeout=false to allow + # the install to succeed and run `helm upgrade` after the certificate is issued. + caCertificateConfigMapName: "" + # -- Hostname the Gateway proxy validates against the backend's TLS + # certificate SAN. Defaults to the service FQDN + # (`..svc.cluster.local`) when empty, which matches + # the SAN included by both cert-manager and the pkiInitJob. + hostname: "" # OpenShift Route with TLS passthrough. The gateway terminates its own # TLS/mTLS; the router only forwards based on SNI, so it never sees plaintext diff --git a/docs/kubernetes/ingress.mdx b/docs/kubernetes/ingress.mdx index 51284465d9..85d12fc8d9 100644 --- a/docs/kubernetes/ingress.mdx +++ b/docs/kubernetes/ingress.mdx @@ -120,8 +120,9 @@ helm upgrade --install openshell \ --set grpcRoute.gateway.listener.port=443 \ --set 'grpcRoute.gateway.listener.tls.certificateRefs[0].name=openshell-ingress-tls' \ --set server.disableTls=true \ - --set server.oidc.issuer=https:// \ - --set 'grpcRoute.hostnames[0]=' + --set server.oidc.issuer=https://keycloak.example.com/realms/openshell \ + --set server.oidc.audience=openshell-cli \ + --set 'grpcRoute.hostnames[0]=gateway.example.com' ``` Keep the certificate Secret in the release namespace. Referencing a Secret in another namespace requires a `ReferenceGrant`. @@ -129,12 +130,82 @@ Keep the certificate Secret in the release namespace. Referencing a Secret in an ### Register over HTTPS ```shell -openshell gateway add https:// --name production --oidc-issuer https:// +openshell gateway add https://gateway.example.com \ + --name production \ + --oidc-issuer https://keycloak.example.com/realms/openshell \ + --oidc-client-id openshell-cli openshell status ``` See [Authentication](/kubernetes/setup) for OIDC issuer, audience, and roles configuration. +## End-to-end TLS (BackendTLSPolicy) + +As an alternative to the plaintext backend path above, the chart can create a `BackendTLSPolicy` that tells the Gateway proxy to re-encrypt traffic when connecting to the OpenShell gateway pod: + +```text +client → HTTPS → Gateway (terminate TLS) → TLS (re-encrypt) → openshell gateway pod +``` + +This keeps TLS on the gateway pod rather than disabling it with `server.disableTls=true`. The Gateway proxy validates the backend's certificate against a CA ConfigMap that the certgen hook auto-creates. + +BackendTLSPolicy is a standard Gateway API resource. It is supported on OpenShift 4.22+ (via the OpenShift gateway controller) and on other platforms where the Gateway API implementation supports it (check your controller's documentation). + +### Install with e2e TLS + +The certgen hook automatically creates the backend CA ConfigMap when `backendTLSPolicy` is enabled: + +```shell +helm upgrade --install openshell \ + oci://ghcr.io/nvidia/openshell/helm-chart \ + --version \ + --namespace openshell \ + --set server.tls.enableMtls=false \ + --set grpcRoute.enabled=true \ + --set grpcRoute.gateway.create=true \ + --set grpcRoute.gateway.className=eg \ + --set grpcRoute.gateway.listener.protocol=HTTPS \ + --set grpcRoute.gateway.listener.port=443 \ + --set 'grpcRoute.gateway.listener.tls.certificateRefs[0].name=openshell-ingress-tls' \ + --set grpcRoute.backendTLSPolicy.enabled=true \ + --set server.oidc.issuer=https://keycloak.example.com/realms/openshell \ + --set server.oidc.audience=openshell-cli \ + --set 'grpcRoute.hostnames[0]=gateway.example.com' +``` + +Note that `server.disableTls` is **not** set — the gateway pod continues to serve TLS — but `server.tls.enableMtls=false` disables mTLS client certificate authentication because the Gateway proxy cannot present a client certificate to the backend. The chart will fail the install if you try to enable both `grpcRoute.backendTLSPolicy.enabled=true` and `server.tls.enableMtls=true` simultaneously. The BackendTLSPolicy hostname defaults to the service FQDN, which matches the SAN on the server certificate. Use OIDC for authentication (configured via `server.oidc.issuer`). + +The example above uses the default `pkiInitJob` for TLS, which creates the backend CA ConfigMap immediately. If using cert-manager instead (`--set certManager.enabled=true`), the Certificate resources are regular release objects, and a separate post-install/post-upgrade Job (`-certgen-backend-ca`) polls for up to 120 seconds waiting for cert-manager to issue the server certificate, then creates the backend CA ConfigMap. This means a single `helm install` is sufficient in most cases. + +If cert-manager takes longer than 120 seconds to issue certificates, increase the polling timeout with `--set pkiInitJob.timeoutSeconds=`. The hook polls for exactly this many seconds. For example, `timeoutSeconds=180` polls for 180 seconds. By default (`pkiInitJob.failOnTimeout=true`), the install fails if the timeout is reached, providing clear feedback that the BackendTLSPolicy is non-functional. + +### Troubleshooting + +If you see the error `remote connection failure, transport failure reason: TLS error: Secret is not supplied by SDS` when connecting through the Gateway: + +1. Check if the backend CA ConfigMap exists: + + ```shell + kubectl get configmap -backend-ca -n + ``` + +2. If the ConfigMap is missing, verify the TLS secret exists: + + ```shell + kubectl get secret -server-tls -n + ``` + +3. If the secret exists but the ConfigMap doesn't, run `helm upgrade` to create it: + + ```shell + helm upgrade oci://ghcr.io/nvidia/openshell/helm-chart \ + --reuse-values --namespace + ``` + +This situation can occur if you set `pkiInitJob.failOnTimeout=false` and cert-manager issued the certificate after the hook timed out. + +For OpenShift 4.22+, see [OpenShift](/kubernetes/openshift#end-to-end-tls-openshift-422) for platform-specific instructions including Gateway and GatewayClass setup. + ## SSH Relay Sandbox SSH uses the gateway endpoint registered with the CLI. No separate Helm SSH host or port values are required. diff --git a/docs/kubernetes/managing-certificates.mdx b/docs/kubernetes/managing-certificates.mdx index c4cb07f57e..d97c1e24be 100644 --- a/docs/kubernetes/managing-certificates.mdx +++ b/docs/kubernetes/managing-certificates.mdx @@ -66,7 +66,7 @@ By default, cert-manager issues both the server and client certificates from a self-signed CA the chart creates — this rotates automatically, but the server certificate is still not publicly trusted. `certManager.serverIssuerRef` overrides the `issuerRef` on the server `Certificate` resource to point at a -real `Issuer` or `ClusterIssuer` instead, for example an ACME issuer: +real `Issuer` or `ClusterIssuer` instead, for example a LetsEncrypt/ACME issuer: ```shell helm upgrade --install openshell \ diff --git a/docs/kubernetes/openshift.mdx b/docs/kubernetes/openshift.mdx index 43e7d0338b..cfae001a38 100644 --- a/docs/kubernetes/openshift.mdx +++ b/docs/kubernetes/openshift.mdx @@ -87,13 +87,123 @@ openshell gateway add http://127.0.0.1:8080 --local --name openshift openshell status ``` -## Production: expose externally with a real certificate +## Options for end-to-end TLS -The steps above run the gateway over plaintext HTTP for quick evaluation. For -a real deployment, cert-manager can issue the gateway's server certificate -from a real Issuer or ClusterIssuer (for example, an ACME issuer), and an -OpenShift Route with TLS passthrough exposes it externally while the gateway -keeps terminating its own TLS and mTLS. +The steps above run the gateway over plaintext HTTP for quick evaluation. For production deployments, choose one of the approaches below based on your OpenShift version and preferences. + +### End-to-end TLS using Gateway API and BackendTLSPolicy (OpenShift 4.22+) + +OpenShift 4.22 and later support `BackendTLSPolicy` in the Gateway API, enabling end-to-end TLS between the OpenShift router and the OpenShell gateway pod. The traffic flow is: + +```text +client → HTTPS → OpenShift Gateway (terminate TLS) → TLS (re-encrypt) → openshell gateway pod +``` + +This removes the requirement to run the gateway with `server.disableTls=true`. The OpenShift router terminates client-facing TLS at the listener and re-encrypts when connecting to the backend service, validating the backend's certificate against a CA you provide. + +#### Prerequisites + +- OpenShift 4.22+ cluster with the Gateway API enabled +- cert-manager installed (recommended) or the built-in pkiInitJob for server certificates +- A `GatewayClass` registered for the OpenShift gateway controller + +#### Create the GatewayClass + +If your cluster does not already have an OpenShift GatewayClass, create one: + +```shell +oc apply -f - <<'EOF' +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: openshift-default +spec: + controllerName: openshift.io/gateway-controller/v1 +EOF +``` + +#### Create the Gateway + +Create a Gateway resource in the `openshift-ingress` namespace. Replace `` with your cluster's route hostname (typically a wildcard like `*.openshell-ingress-gw.example.com`): + +```shell +oc apply -f - <<'EOF' +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: openshell-gateway + namespace: openshift-ingress +spec: + gatewayClassName: openshift-default + listeners: + - name: grpc + hostname: "" + port: 443 + protocol: HTTPS + tls: + mode: Terminate + certificateRefs: + - name: + kind: Secret + allowedRoutes: + namespaces: + from: Selector + selector: + matchLabels: + kubernetes.io/metadata.name: openshell +EOF +``` + +The listener TLS Secret should contain the certificate for the external hostname. + +#### Install with e2e TLS + +Install the chart with the GRPCRoute and BackendTLSPolicy enabled. The certgen hook automatically creates the backend CA ConfigMap from the generated PKI bundle: + +```shell +helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart \ + --version \ + --namespace openshell \ + --set podSecurityContext.fsGroup=null \ + --set securityContext.runAsUser=null \ + --set server.tls.enableMtls=false \ + --set grpcRoute.enabled=true \ + --set grpcRoute.gateway.name=openshell-gateway \ + --set grpcRoute.gateway.namespace=openshift-ingress \ + --set 'grpcRoute.hostnames[0]=gateway.example.com' \ + --set grpcRoute.backendTLSPolicy.enabled=true \ + --set server.oidc.issuer=https://keycloak.example.com/realms/openshell \ + --set server.oidc.audience=openshell-cli +``` + +| Override | Reason | +|---|---| +| `podSecurityContext.fsGroup=null` / `securityContext.runAsUser=null` | Let OpenShift's SCC admission assign UIDs. | +| `server.tls.enableMtls=false` | Disable mTLS client certificate authentication. BackendTLSPolicy only validates the server certificate; the ingress proxy cannot present a client certificate to the backend. Use OIDC for authentication instead. | +| `grpcRoute.enabled=true` | Create a GRPCRoute pointing at the external Gateway. | +| `grpcRoute.gateway.name` / `namespace` | Reference the Gateway created above in `openshift-ingress`. | +| `grpcRoute.backendTLSPolicy.enabled=true` | Create a BackendTLSPolicy for TLS re-encryption to the gateway pod. The certgen hook auto-creates the backend CA ConfigMap. The Gateway proxy validates the backend certificate against the service FQDN, which is already in the default server certificate SANs. | +| `grpcRoute.hostnames` | External hostname for the GRPCRoute. This goes on the Gateway listener certificate, not the backend certificate. | + +Note that `server.disableTls` is **not** set — the gateway pod serves TLS over HTTPS without requiring client certificates. Use OIDC for authentication (see [Access Control](/kubernetes/access-control)). + +**Using cert-manager instead of pkiInitJob:** Add `--set certManager.enabled=true` to the install command. The default `certManager.serverDnsNames` already includes the service FQDN needed for BackendTLSPolicy validation. The Certificate resources are regular release objects, and a separate post-install/post-upgrade Job (`-certgen-backend-ca`) polls for up to 120 seconds waiting for cert-manager to issue the server certificate, then creates the backend CA ConfigMap. A single `helm install` is sufficient in most cases. + +If cert-manager takes longer than 120 seconds to issue certificates, increase the polling timeout with `--set pkiInitJob.timeoutSeconds=`. The hook polls for exactly this many seconds. For example, `timeoutSeconds=180` polls for 180 seconds. By default (`pkiInitJob.failOnTimeout=true`), the install fails if the timeout is reached, providing clear feedback that the BackendTLSPolicy is non-functional. + +#### Register over HTTPS + +```shell +openshell gateway add https://gateway.example.com \ + --name openshift \ + --oidc-issuer https://keycloak.example.com/realms/openshell \ + --oidc-client-id openshell-cli +openshell status +``` + +### End-to-end TLS using pass-through Route (all OpenShift versions) + +For OpenShift versions prior to 4.22, or when you prefer Route-based ingress, cert-manager can issue the gateway's server certificate from a real Issuer or ClusterIssuer (for example, a LetsEncrypt/ACME issuer), and an OpenShift Route with TLS passthrough exposes it externally while the gateway keeps terminating its own TLS and mTLS. Install cert-manager and configure a working `ClusterIssuer` first — see [Managing Certificates](/kubernetes/managing-certificates) for the @@ -110,13 +220,13 @@ helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart \ --set securityContext.runAsUser=null \ --set server.disableTls=false \ --set certManager.enabled=true \ - --set certManager.serverIssuerRef.name= \ + --set certManager.serverIssuerRef.name=letsencrypt-prod \ --set certManager.serverIssuerRef.kind=ClusterIssuer \ - --set certManager.serverDnsNames[0]= \ + --set certManager.serverDnsNames[0]=gateway.example.com \ --set openshiftRoute.enabled=true \ - --set openshiftRoute.host= \ - --set server.oidc.issuer= \ - --set server.oidc.audience= + --set openshiftRoute.host=gateway.example.com \ + --set server.oidc.issuer=https://keycloak.example.com/realms/openshell \ + --set server.oidc.audience=openshell-cli ``` | Override | Reason | @@ -129,9 +239,10 @@ Register the gateway with the CLI over OIDC. Remote gateways authenticate CLI users via OIDC, not mTLS — see [Access Control](/kubernetes/access-control): ```shell -openshell gateway add https:// \ +openshell gateway add https://gateway.example.com \ --name openshift \ - --oidc-issuer + --oidc-issuer https://keycloak.example.com/realms/openshell \ + --oidc-client-id openshell-cli openshell gateway login openshift ``` diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index e2d48087cd..f9a3f00321 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -522,6 +522,9 @@ compute_driver = "kubernetes" [openshell.gateway.tls] cert_path = "/etc/openshell-tls/server/tls.crt" key_path = "/etc/openshell-tls/server/tls.key" +# client_ca_path is only rendered when server.tls.enableMtls is true (the +# default). When enableMtls is false — required for BackendTLSPolicy — the +# gateway runs HTTPS-only and this line is omitted by Helm. client_ca_path = "/etc/openshell-tls/client-ca/ca.crt" # When cert-manager serverIssuerRef is configured, these are populated by Helm: # external_cert_path = "/etc/openshell-tls/server-external/tls.crt" diff --git a/mise.lock b/mise.lock index 522084b559..a5e38be2c2 100644 --- a/mise.lock +++ b/mise.lock @@ -134,6 +134,18 @@ checksum = "sha256:b8514ed7552e148b0a032114f745118dcb801791adafafeaf9935e4bfb0ed url = "https://github.com/mozilla/sccache/releases/download/v0.16.0/sccache-v0.16.0-x86_64-pc-windows-msvc.zip" url_api = "https://api.github.com/repos/mozilla/sccache/releases/assets/452060720" +[[tools."github:mozilla/sccache"]] +version = "0.16.0" +backend = "github:mozilla/sccache" + +[tools."github:mozilla/sccache".options] +asset_pattern = "sccache-v*x86_64*linux*.tar.gz" + +[tools."github:mozilla/sccache"."platforms.linux-x64"] +checksum = "sha256:aec995a83ad3dff3d14b6314e08858b7b73d35ca85a5bcf3d3a9ec07dee35588" +url = "https://github.com/mozilla/sccache/releases/download/v0.16.0/sccache-v0.16.0-x86_64-unknown-linux-musl.tar.gz" +url_api = "https://api.github.com/repos/mozilla/sccache/releases/assets/452060682" + [[tools."github:nextest-rs/nextest"]] version = "cargo-nextest-0.9.143" backend = "github:nextest-rs/nextest" @@ -156,12 +168,6 @@ url = "https://github.com/nextest-rs/nextest/releases/download/cargo-nextest-0.9 url_api = "https://api.github.com/repos/nextest-rs/nextest/releases/assets/501885637" provenance = "github-attestations" -[tools."github:nextest-rs/nextest"."platforms.windows-arm64"] -checksum = "sha256:58c1637ba2396e6c556aa0092f9aa4388695594b8ddda5a4b8b39212574678ce" -url = "https://github.com/nextest-rs/nextest/releases/download/cargo-nextest-0.9.143/cargo-nextest-0.9.143-aarch64-pc-windows-msvc.zip" -url_api = "https://api.github.com/repos/nextest-rs/nextest/releases/assets/501886128" -provenance = "github-attestations" - [tools."github:nextest-rs/nextest"."platforms.windows-x64"] checksum = "sha256:c670ba18e8731fd2eff33a47af33a0fa53d1afa6d0678344e82dc6f8fc7344ac" url = "https://github.com/nextest-rs/nextest/releases/download/cargo-nextest-0.9.143/cargo-nextest-0.9.143-x86_64-pc-windows-msvc.zip" @@ -451,13 +457,11 @@ backend = "aqua:astral-sh/uv" [tools.uv."platforms.linux-arm64"] checksum = "sha256:55bd1c1c10ec8b95a8c184f5e18b566703c6ab105f0fc118aaa4d748aabf28e4" url = "https://github.com/astral-sh/uv/releases/download/0.10.12/uv-aarch64-unknown-linux-musl.tar.gz" -url_api = "https://api.github.com/repos/astral-sh/uv/releases/assets/377491942" provenance = "github-attestations" [tools.uv."platforms.linux-x64"] checksum = "sha256:adccf40b5d1939a5e0093081ec2307ea24235adf7c2d96b122c561fa37711c46" url = "https://github.com/astral-sh/uv/releases/download/0.10.12/uv-x86_64-unknown-linux-musl.tar.gz" -url_api = "https://api.github.com/repos/astral-sh/uv/releases/assets/377491998" provenance = "github-attestations" [tools.uv."platforms.macos-arm64"] diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 360bf28ee4..577b801a83 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -446,6 +446,31 @@ label, supervisor env vars `OPENSHELL_K8S_SA_TOKEN_FILE` and `OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET`, plus both the projected `openshell-sa-token` volume and the `spiffe-workload-api` CSI volume. +If `grpcRoute.backendTLSPolicy.enabled=true`, the Gateway proxy validates the +backend pod's TLS certificate against a CA in a ConfigMap. Check that the +ConfigMap exists and contains the correct CA, that `enableMtls` is disabled, +and that the BackendTLSPolicy resource is present: + +```bash +kubectl -n openshell get backendtlspolicy +kubectl -n openshell get configmap openshell-backend-ca -o yaml +helm -n openshell get values openshell | grep -E 'backendTLSPolicy|enableMtls|failOnTimeout|timeoutSeconds|caCertificateConfigMapName' +``` + +If the ConfigMap is missing after a cert-manager install, the post-install +certgen hook may have timed out waiting for cert-manager to issue the server +certificate. Check the certgen Job logs: + +```bash +kubectl -n openshell get jobs | grep certgen +kubectl -n openshell logs job/openshell-certgen-backend-ca +``` + +Increase `pkiInitJob.timeoutSeconds` and run `helm upgrade` to retry. If the +Gateway proxy reports `TLS error: Secret is not supplied by SDS` or similar +backend TLS errors, the ConfigMap CA likely does not match the server +certificate CA — verify both are from the same issuer. + Check the image references currently used by the gateway deployment: ```bash