From 7606a1e5eba0ee931ef2e39b495f19a10bb5071e Mon Sep 17 00:00:00 2001 From: Thomas Michael Date: Mon, 7 Sep 2026 15:54:25 +0200 Subject: [PATCH 1/2] documenation about to test on local openshift (crc) --- docs/deploy-local-openshift.md | 276 ++++++++++++++++++ scripts/init-cluster.sh | 76 +++-- scripts/local-openshift/helm/gop-rbac.yaml | 18 ++ scripts/local-openshift/helm/gop-values.yaml | 29 ++ scripts/local-openshift/manifest/gop-job.yaml | 28 ++ .../local-openshift/manifest/gop-rbac.yaml | 18 ++ 6 files changed, 415 insertions(+), 30 deletions(-) create mode 100644 docs/deploy-local-openshift.md create mode 100644 scripts/local-openshift/helm/gop-rbac.yaml create mode 100644 scripts/local-openshift/helm/gop-values.yaml create mode 100644 scripts/local-openshift/manifest/gop-job.yaml create mode 100644 scripts/local-openshift/manifest/gop-rbac.yaml diff --git a/docs/deploy-local-openshift.md b/docs/deploy-local-openshift.md new file mode 100644 index 000000000..e074f3dba --- /dev/null +++ b/docs/deploy-local-openshift.md @@ -0,0 +1,276 @@ +# GOP Deployment from Local Docker Image to Local OpenShift (CRC) + +This guide provides a step-by-step walkthrough for deploying the local Docker image `local/gop:latest` to a local +OpenShift environment (CodeReady Containers / OpenShift Local) using the internal OpenShift Image Registry. + +--- + +## 1. Prerequisites Check + +- **OpenShift Local (CRC):** Running (`crc status`) +- **OpenShift CLI (`oc`):** Installed and operational +- **Docker / Podman:** Running with the local image `local/gop:latest` + +### Recommend + +Start openshift with more cpu and memory! + +### CRC should use different default ports, because K3d has to use 80/443. + +```bash + crc config set ingress-http-port 8880 + crc config set ingress-https-port 8843 +``` + +### Start with more resources, because CRC Argocd needs more cpu and memory. + +```bash +crc start --cpus 6 --memory 16384 --disk-size 80 +``` + +Verify local image: + +```bash +docker images | grep gop +``` + +--- + +## 2. Step 1: Enable the Internal OpenShift Image Registry & Expose Route + +In OpenShift Local (CRC), the internal Image Registry is often not exposed via an external route by default. This must +be enabled once with administrator privileges. + +### 2.1 Log in as `kubeadmin` + +Retrieve the password via CRC (if not already known): + +```bash +crc console --credentials +``` + +Log in with admin rights: + +```bash +oc login -u kubeadmin -p https://api.crc.testing:6443 +``` + +### 2.2 Configure Image Registry Operator + +For local test clusters (CRC), set the storage to `emptyDir` and the operator to `Managed`: + +```bash +oc patch configs.imageregistry.operator.openshift.io/cluster --type merge -p '{"spec":{"managementState":"Managed","storage":{"emptyDir":{}}}}' +``` + +### 2.3 Expose the Default Route for External Access + +```bash +oc patch configs.imageregistry.operator.openshift.io/cluster --type merge -p '{"spec":{"defaultRoute":true}}' +``` + +### 2.4 Verify Registry Route + +```bash +oc get route default-route -n openshift-image-registry +``` + +The registry host URL typically resolves to: `default-route-openshift-image-registry.apps-crc.testing`. + +--- + +## 3. Step 2: Create Project / Namespace for GOP + +Create a new project in the OpenShift cluster (can be executed as `developer` or `kubeadmin`): + +```bash +# Optional: Switch to developer user +oc login -u developer -p developer https://api.crc.testing:6443 + +# Create new project +oc new-project gop +``` + +--- + +## 4. Step 3: Authenticate Docker with OpenShift Registry + +To allow Docker to push the local image into the cluster, authenticate using the current OpenShift session token: + +```bash +docker login -u $(oc whoami) -p $(oc whoami -t) default-route-openshift-image-registry.apps-crc.testing +``` + +> **Note on SSL/TLS Certificate Errors (x509: certificate signed by unknown authority):** +> Add the registry domain to `insecure-registries` in Docker Desktop under **Settings** → **Docker Engine**: +> ```json +> { +> "insecure-registries": [ +> "default-route-openshift-image-registry.apps-crc.testing" +> ] +> } +> ``` +> Then click *Apply & restart*. + +--- + +## 5. Step 4: Tag & Push Local Image to OpenShift + +Tag the local image with the registry URL and target project (`gop`), then push: + +```bash +# Tag image +docker tag local/gop:latest default-route-openshift-image-registry.apps-crc.testing/gop/gop:latest + +# Push image to OpenShift Registry +docker push default-route-openshift-image-registry.apps-crc.testing/gop/gop:latest +``` + +### Verify ImageStream in Cluster + +After pushing, OpenShift automatically creates an `ImageStream`: + +```bash +oc get is -n gop +oc describe is gop -n gop +``` + +--- + +## 6. Step 5: Configure ServiceAccount, RBAC & OpenShift SCCs + +GOP operates as an orchestrator job that provisions tools (SCM-Manager, Argo CD, Vault, etc.) across various namespaces. +By default, OpenShift blocks containers using root groups (`fsGroup: 0`) under the `restricted-v2` SCC. We therefore set +up the permissions and pre-create the required tool namespaces with the `anyuid` SCC. + +### 5.1 Create ServiceAccount & ClusterRoleBinding for GOP + +Manifest definition is located +in [scripts/local-openshift/manifest/gop-rbac.yaml](../scripts/local-openshift/manifest/gop-rbac.yaml): + +```bash +# Apply manifest (as kubeadmin) +oc apply -f scripts/local-openshift/manifest/gop-rbac.yaml +``` + +### 5.2 Assign OpenShift SCC `anyuid` to GOP ServiceAccount + +Allows the GOP installer pod itself to start: + +```bash +oc adm policy add-scc-to-user anyuid -z gop-sa -n gop +``` + +### 5.3 Pre-create Tool Namespaces & Assign `anyuid` SCC + +Ensures that tools deployed by GOP (e.g. SCM-Manager with `fsGroup: 0`) can start without security constraint errors: + +```bash +# 1. Pre-create namespaces for initial tools +oc create namespace scm-manager || true +oc create namespace argocd || true +oc create namespace vault || true + +# 2. Grant anyuid SCC to all ServiceAccounts in these namespaces +oc adm policy add-scc-to-group anyuid system:serviceaccounts:scm-manager +oc adm policy add-scc-to-group anyuid system:serviceaccounts:argocd +oc adm policy add-scc-to-group anyuid system:serviceaccounts:vault +``` + +> **Note for additional tools (e.g. Monitoring / Prometheus):** +> When activating additional components later, simply execute the same commands for the new namespace: +> `oc create namespace monitoring || true` +> `oc adm policy add-scc-to-group anyuid system:serviceaccounts:monitoring` + +### 5.4 Clean Up Previous Failed Jobs (if any) + +```bash +oc delete job -l app.kubernetes.io/name=gop-helm -n gop || true +oc delete job gop-installer-job -n gop || true +``` + +--- + +## 7. Step 6: Run GOP in OpenShift Cluster + +Two execution options are available: + +### Option A: Installation via Helm (Path A with `gop-values.yaml`) + +Configuration file is located +at [scripts/local-openshift/helm/gop-values.yaml](../scripts/local-openshift/helm/gop-values.yaml): + +```bash +helm upgrade -i gop oci://ghcr.io/cloudogu/gop-helm -n gop -f scripts/local-openshift/helm/gop-values.yaml +``` + +Stream live installer logs: + +```bash +oc logs -f -l app.kubernetes.io/name=gop-helm -n gop +``` + +--- + +### Option B: Direct Manifest via OpenShift Job (Path B with `gop-job.yaml`) + +Job manifest is located +at [scripts/local-openshift/manifest/gop-job.yaml](../scripts/local-openshift/manifest/gop-job.yaml): + +```bash +# Clean up prior installer job (if present) +oc delete job gop-installer-job -n gop || true + +# Apply manifest and start job +oc apply -f scripts/local-openshift/manifest/gop-job.yaml +``` + +Stream live installer logs: + +```bash +oc logs -f job/gop-installer-job -n gop +``` + +--- + +## 8. Step 7: Access Installed Tools & Routes + +Once the GOP installer job finishes with status `Completed`, the deployed tools are accessible via OpenShift Routes. + +### 8.1 List All Created Routes + +```bash +oc get routes -A +``` + +### 8.2 Default URLs with `baseUrl: http://apps-crc.testing` + +* **SCM-Manager:** `http://scmm.apps-crc.testing` +* **Argo CD:** `http://argocd.apps-crc.testing` +* **Vault:** `http://vault.apps-crc.testing` +* **Grafana / Metrics:** `http://grafana.apps-crc.testing` + +**Default Credentials:** + +* **Username:** `admin` +* **Password:** `admin` (or the configured value in `gop-values.yaml`) + +--- + +## 9. Troubleshooting & Common Commands + +- **Rerun GOP Job (Helm):** + ```bash + oc delete job -l app.kubernetes.io/name=gop-helm -n gop + helm upgrade -i gop oci://ghcr.io/cloudogu/gop-helm -n gop -f scripts/local-openshift/helm/gop-values.yaml + ``` +- **Rerun GOP Job (Manifest):** + ```bash + oc delete job gop-installer-job -n gop + oc apply -f scripts/local-openshift/manifest/gop-job.yaml + ``` +- **Update Image after Local Code Changes:** + ```bash + docker tag local/gop:latest default-route-openshift-image-registry.apps-crc.testing/gop/gop:latest + docker push default-route-openshift-image-registry.apps-crc.testing/gop/gop:latest + ``` diff --git a/scripts/init-cluster.sh b/scripts/init-cluster.sh index 6421573d3..7e8f03e77 100755 --- a/scripts/init-cluster.sh +++ b/scripts/init-cluster.sh @@ -2,9 +2,9 @@ # See https://github.com/rancher/k3d/releases # This variable is also read in Jenkinsfile -K3D_VERSION=5.8.3 +K3D_VERSION=5.9.0 # When updating please also adapt in Dockerfile, vars.tf and Config.groovy -K8S_VERSION=1.35.3 +K8S_VERSION=1.36.4 K3S_VERSION="rancher/k3s:v${K8S_VERSION}-k3s1" set -o errexit @@ -13,12 +13,12 @@ set -o pipefail function main() { readParameters "$@" - + [[ $TRACE == true ]] && set -x; - + # Install k3d if necessary if ! command -v k3d >/dev/null 2>&1; then - echo The GitOps playground uses k3d, which is not found on the PATH. + echo The GitOps playground uses k3d, which is not found on the PATH. installK3d else ACTUAL_K3D_VERSION="$(k3d --version | grep k3d | sed 's/k3d version v\(.*\)/\1/')" @@ -62,13 +62,28 @@ function createCluster() { fi fi + # Ensure loopback alias exists when using a non-default loopback IP (e.g. 127.0.0.2) + # This avoids port conflicts with CRC/OpenShift which binds *:80/*:443 on the default 127.0.0.1 + if [[ "${BIND_INGRESS_HOST}" != "127.0.0.1" && "${BIND_INGRESS_HOST}" =~ ^127\. ]]; then + if ! ifconfig lo0 | grep -q "${BIND_INGRESS_HOST}"; then + echo "Adding loopback alias ${BIND_INGRESS_HOST} to lo0 (requires sudo)..." + sudo ifconfig lo0 alias "${BIND_INGRESS_HOST}" + else + echo "Loopback alias ${BIND_INGRESS_HOST} already configured." + fi + fi + HOST_PORT_RANGE='8010-65535' + DOCKER_SOCK_PATH="/var/run/docker.sock" + if [[ -S "$HOME/.orbstack/run/docker.sock" ]]; then + DOCKER_SOCK_PATH="$HOME/.orbstack/run/docker.sock" + fi K3D_ARGS=( # Allow services to bind to portBindings < 30000 > 32xxx # This makes is easier to match for example --bind-registry-port=0 on ci or use lower ports for development "--k3s-arg=--kube-apiserver-arg=service-node-port-range=${HOST_PORT_RANGE}@server:*" # Used by Jenkins Agents pods - '-v /var/run/docker.sock:/var/run/docker.sock@server:*' + "-v ${DOCKER_SOCK_PATH}:/var/run/docker.sock@server:*" # Allows for finding out the GID of the docker group in order to allow the Jenkins agents pod to access docker socket '-v /etc/group:/etc/group@server:*' # Persists the cache of Jenkins agents pods for faster builds @@ -78,7 +93,7 @@ function createCluster() { # Disable traefik (we roll our own ingress-controller) '--k3s-arg=--disable=traefik@server:*' ) - + REGISTRIES="" if [[ -n "$DOCKER_IO_REGISTRY_MIRROR" ]]; then REGISTRIES=$(cat <> Help screen" echo echo " | --cluster-name=STRING >> Set your preferred cluster name to install k3d. Defaults to 'gitops-playground'." - + echo " | --bind-localhost=BOOLEAN >> Bind the k3d container to host network. Exposes all k8s nodePorts to localhost. Defaults to false." - echo " | --bind-ingress-host=STRING >> Bind the ingress controller to this local ip. Defaults to 127.0.0.1." + echo " | --bind-ingress-host=STRING >> Bind the ingress controller to this local ip. Defaults to 127.0.0.2 (avoids port conflict with CRC/OpenShift)." echo " | --bind-ingress-port=INT >> Bind the ingress controller to this port. Defaults to 80. Set to - to disable." echo " | --bind-registry-port=INT >> Specify a custom port for the container registry to bind to localhost port. Only use this when port 30000 is blocked and --bind-localhost=true. Defaults to 30000 (default used by the playground)." echo " | --bind-ports=STRING >> A comma separated list of additional port bindings like 443:443,9090:9090. Ignored when --bind-localhost." - + echo " | --docker-io-registry-mirror=STRING >> the hostname of a registry that mirrors DockerHub. Useful when encountering rate limits" echo echo " -x | --trace >> Debug + Show each command executed (set -x)" @@ -213,7 +228,7 @@ function confirm() { # shellcheck disable=SC2145 # - the line break between args is intended here! printf "%s\n" "${@:-Are you sure? [y/N]} " - + read -r response case "$response" in [yY][eE][sS] | [yY]) @@ -233,7 +248,7 @@ get_longopt_value(){ # or # 2='--expected' # 3='value' - + # check $2 has the form --longopt=value VALUE=$(echo "$2" | sed -e 's/^[^=]*=//') if [ -z "$VALUE" ]; then @@ -249,6 +264,7 @@ get_longopt_value(){ readParameters() { CLUSTER_NAME=gitops-playground BIND_LOCALHOST=false + # Use 127.0.0.2 to avoid port conflict with CRC/OpenShift which binds *:80/*:443 BIND_INGRESS_HOST="127.0.0.1" BIND_INGRESS_PORT="80" BIND_INGRESS_HTTPS_PORT="443" @@ -264,7 +280,7 @@ readParameters() { -x | --trace ) TRACE=true; shift ;; --bind-localhost) BIND_LOCALHOST=true; shift ;; --cluster-name*) CLUSTER_NAME=$(get_longopt_value "--cluster-name" "$@") - # Allow passing portBindings with and without '=' + # Allow passing portBindings with and without '=' if [[ "$1" == *"="* ]]; then shift; else shift 2; fi ;; --bind-ingress-port*) BIND_INGRESS_PORT=$(get_longopt_value "--bind-ingress-port" "$@") if [[ "$1" == *"="* ]]; then shift; else shift 2; fi ;; @@ -272,11 +288,11 @@ readParameters() { if [[ "$1" == *"="* ]]; then shift; else shift 2; fi ;; --bind-ingress-https-port*) BIND_INGRESS_HTTPS_PORT=$(get_longopt_value "--bind-ingress-https-port" "$@") if [[ "$1" == *"="* ]]; then shift; else shift 2; fi ;; - --bind-registry-port*) BIND_REGISTRY_PORT=$(get_longopt_value "--bind-registry-port" "$@") + --bind-registry-port*) BIND_REGISTRY_PORT=$(get_longopt_value "--bind-registry-port" "$@") if [[ "$1" == *"="* ]]; then shift; else shift 2; fi ;; - --bind-ports*) BIND_PORTS=$(get_longopt_value "--bind-ports" "$@"); + --bind-ports*) BIND_PORTS=$(get_longopt_value "--bind-ports" "$@"); if [[ "$1" == *"="* ]]; then shift; else shift 2; fi ;; - --docker-io-registry-mirror*) DOCKER_IO_REGISTRY_MIRROR=$(get_longopt_value "--docker-io-registry-mirror" "$@"); + --docker-io-registry-mirror*) DOCKER_IO_REGISTRY_MIRROR=$(get_longopt_value "--docker-io-registry-mirror" "$@"); if [[ "$1" == *"="* ]]; then shift; else shift 2; fi ;; --) shift; break ;; *) break ;; diff --git a/scripts/local-openshift/helm/gop-rbac.yaml b/scripts/local-openshift/helm/gop-rbac.yaml new file mode 100644 index 000000000..14124115d --- /dev/null +++ b/scripts/local-openshift/helm/gop-rbac.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: gop-sa + namespace: gop +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: gop-sa-cluster-admin-binding +subjects: + - kind: ServiceAccount + name: gop-sa + namespace: gop +roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io diff --git a/scripts/local-openshift/helm/gop-values.yaml b/scripts/local-openshift/helm/gop-values.yaml new file mode 100644 index 000000000..595e6aeea --- /dev/null +++ b/scripts/local-openshift/helm/gop-values.yaml @@ -0,0 +1,29 @@ +# gop-values.yaml - Configuration for GOP Helm Chart on OpenShift Local (CRC) + +# Container image from the internal OpenShift Image Registry +image: + repository: image-registry.openshift-image-registry.svc:5000/gop/gop + tag: latest + pullPolicy: Always + +# Use pre-created ServiceAccount with assigned permissions (cluster-admin + SCC) +serviceAccount: + create: false + name: gop-sa + +logLevel: trace + +# Direct CLI flags passed to the GOP container (takes highest precedence) +extraArgs: + - "--profile=full" + - "--openshift" + - "--base-url=http://apps-crc.testing" + # Optional: Uncomment if all tools should run in a single shared namespace: + # - "--namespace=gop" + +config: + # yaml-language-server: $schema=https://raw.githubusercontent.com/cloudogu/gitops-playground/refs/heads/main/docs/configuration.schema.json + application: + "yes": false # strange, but toggle issue with picocli + password: "admin" + insecure: true diff --git a/scripts/local-openshift/manifest/gop-job.yaml b/scripts/local-openshift/manifest/gop-job.yaml new file mode 100644 index 000000000..0c8a0d8b3 --- /dev/null +++ b/scripts/local-openshift/manifest/gop-job.yaml @@ -0,0 +1,28 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: gop-installer-job + namespace: gop +spec: + # Retain job after completion to preserve logs + backoffLimit: 0 + template: + metadata: + name: gop-installer + labels: + app: gop-installer + spec: + serviceAccountName: gop-sa + containers: + - name: gop-container + image: image-registry.openshift-image-registry.svc:5000/gop/gop:latest + imagePullPolicy: Always + args: + - "--yes=true" + - "--profile=full" + - "-x" + - "--openshift" + - "--base-url=http://apps-crc.testing" + # Optional: Uncomment if all tools should run in a single shared namespace: + # - "--namespace=gop" + restartPolicy: Never diff --git a/scripts/local-openshift/manifest/gop-rbac.yaml b/scripts/local-openshift/manifest/gop-rbac.yaml new file mode 100644 index 000000000..14124115d --- /dev/null +++ b/scripts/local-openshift/manifest/gop-rbac.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: gop-sa + namespace: gop +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: gop-sa-cluster-admin-binding +subjects: + - kind: ServiceAccount + name: gop-sa + namespace: gop +roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io From c0029068a12a12c9150f379b38847b22bbbab202 Mon Sep 17 00:00:00 2001 From: Marco Droll Date: Wed, 9 Sep 2026 13:46:55 +0200 Subject: [PATCH 2/2] bump k3s version identically as the used version in the init-cluster.sh --- .../java/com/cloudogu/gitops/config/Config.java | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/cloudogu/gitops/config/Config.java b/src/main/java/com/cloudogu/gitops/config/Config.java index ae2b4b007..3fa56e6ec 100644 --- a/src/main/java/com/cloudogu/gitops/config/Config.java +++ b/src/main/java/com/cloudogu/gitops/config/Config.java @@ -182,10 +182,8 @@ @Setter public class Config { - // When updating please also update in Dockerfile - public static final String HELM_IMAGE = "ghcr.io/cloudogu/helm:4.2.1-1"; // When updating please also adapt in Dockerfile, vars.tf and init-cluster.sh - public static final String K8S_VERSION = "1.36.2"; + public static final String K8S_VERSION = "1.36.4"; public static final String DEFAULT_ADMIN_USER = "admin"; // Generated once when Config is initialized and intentionally shared by all Config instances in the JVM. @@ -1043,9 +1041,9 @@ public enum VaultMode { @JsonCreator public static VaultMode fromExternalValue(String value) { return Arrays.stream(values()) - .filter(mode -> mode.externalValue.equalsIgnoreCase(value)) - .findFirst() - .orElseThrow(() -> new IllegalArgumentException("Unknown Vault mode: " + value)); + .filter(mode -> mode.externalValue.equalsIgnoreCase(value)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException("Unknown Vault mode: " + value)); } @JsonValue @@ -1073,7 +1071,7 @@ public static Config fromMap(Map map) { public Map toMap() { return objectMapper.convertValue( - this, new TypeReference>() { + this, new TypeReference<>() { } ); } @@ -1096,8 +1094,8 @@ public List changeProperties( BeanDescription beanDesc, List beanProperties) { return beanProperties.stream() - .filter(writer -> writer.getAnnotation(JsonPropertyDescription.class) != null) - .toList(); + .filter(writer -> writer.getAnnotation(JsonPropertyDescription.class) != null) + .toList(); } })); return mapper;