From 552d3dd28986bc2f602e7d4fff6c800118a1555d Mon Sep 17 00:00:00 2001 From: Ryan D McGuire Date: Mon, 10 Aug 2026 17:06:34 -0400 Subject: [PATCH] add kubeconform --- images/node-agent/Dockerfile | 72 +++++++++++++++++++++++++++++++++++- images/node-agent/README.md | 40 ++++++++++++++++++-- images/node-agent/test.sh | 61 ++++++++++++++++++++++++++++-- 3 files changed, 166 insertions(+), 7 deletions(-) diff --git a/images/node-agent/Dockerfile b/images/node-agent/Dockerfile index d4f8752..0f3ca27 100644 --- a/images/node-agent/Dockerfile +++ b/images/node-agent/Dockerfile @@ -14,7 +14,7 @@ RUN apk add --no-cache \ coreutils findutils diffutils gawk grep sed \ file less patch tree \ curl wget \ - jq \ + jq yamllint \ bind-tools iputils-ping netcat-openbsd socat rsync \ git git-lfs openssh-client-default \ ripgrep fd \ @@ -35,6 +35,76 @@ RUN arch="$(apk --print-arch | sed -e 's/x86_64/amd64/' -e 's/aarch64/arm64/')" && chmod 0755 /usr/local/bin/yq \ && yq --version +# kubeconform validates manifests against the Kubernetes and CRD JSON schemas. Not +# packaged for Alpine either, so same treatment as yq: current upstream release, run it +# in the same layer so a truncated download fails the build. +# +# ALL schemas are VENDORED, which is the whole point of this layer. The yamllint MCP +# server in 50W/kube-manifests runs this image with NO network egress. +# +# Note that kubeconform's `-schema-location default` is a REMOTE URL +# (raw.githubusercontent.com), so it is useless here — with no egress it fails for +# EVERY kind, including built-ins like ConfigMap, and reports a download error rather +# than anything about the manifest. Both schema sets below are therefore local, and +# consumers must NOT pass `default`: +# builtin/ yannh/kubernetes-json-schema, master-standalone-strict (~62 MB / 1505) +# crds/ datreeio/CRDs-catalog, only the groups this cluster uses (~18 MB / 229) +# +# The CRD half matters most: without a local HelmRelease schema, a corrupted Flux +# HelmRelease — the exact thing this is here to catch — comes back "skipped" rather +# than "invalid". The full datree catalog is ~211 MB / 4177 schemas; sparse checkout +# means the other 4000 blobs are never downloaded. To cover a new group, add it to +# $groups and rebuild. +# +# Groups with NO published schema anywhere (toolhive.stacklok.dev, kagent.dev, +# openclaw.rocks, inference.llmkube.dev) are absent by necessity; kubeconform is +# invoked with -ignore-missing-schemas so those resources report as "skipped". +# +# Set BEFORE the RUN below so the build-time verification uses the same value consumers +# do. Used by the yamllint MCP shim in 50W/kube-manifests (toolhive/cm-yamllint-mcp.yaml). +ENV KUBECONFORM_SCHEMAS=/usr/local/share/kubeconform + +# $groups is word-split on purpose — it is a list of directory names, and both +# `sparse-checkout set` and the `for` loop want it expanded into separate arguments. +# hadolint ignore=SC2086 +RUN groups="acid.zalan.do ceph.rook.io cert-manager.io trust.cert-manager.io \ + cilium.io external-secrets.io gateway.networking.k8s.io \ + grafana.integreatly.org helm.toolkit.fluxcd.io jetstream.nats.io \ + k8s.keycloak.org k8up.io kustomize.config.k8s.io \ + kustomize.toolkit.fluxcd.io metallb.io monitoring.coreos.com \ + monitoring.grafana.com networking.istio.io objectbucket.io \ + operator.victoriametrics.com security.istio.io \ + source.toolkit.fluxcd.io telemetry.istio.io valkey.io" \ + && arch="$(apk --print-arch | sed -e 's/x86_64/amd64/' -e 's/aarch64/arm64/')" \ + && curl -fsSL \ + "https://github.com/yannh/kubeconform/releases/latest/download/kubeconform-linux-${arch}.tar.gz" \ + | tar -xz -C /usr/local/bin kubeconform \ + && chmod 0755 /usr/local/bin/kubeconform \ + && kubeconform -v \ + && mkdir -p "${KUBECONFORM_SCHEMAS}/crds" "${KUBECONFORM_SCHEMAS}/builtin" \ + && git clone --depth 1 --filter=blob:none --sparse \ + https://github.com/datreeio/CRDs-catalog.git /tmp/crds \ + && git -C /tmp/crds sparse-checkout set $groups \ + && for g in $groups; do cp -a "/tmp/crds/$g" "${KUBECONFORM_SCHEMAS}/crds/"; done \ + && rm -rf /tmp/crds \ + && git clone --depth 1 --filter=blob:none --sparse \ + https://github.com/yannh/kubernetes-json-schema.git /tmp/kjs \ + && git -C /tmp/kjs sparse-checkout set master-standalone-strict \ + && cp -a /tmp/kjs/master-standalone-strict "${KUBECONFORM_SCHEMAS}/builtin/" \ + && rm -rf /tmp/kjs \ + # Prove the vendored schemas actually resolve, for a CRD kind AND a built-in kind. + # `-schema-location default` is deliberately absent: it is a remote URL, so + # including it would let this check pass on the network the build happens to have + # and still fail in the egress-less pod. A missing or renamed schema directory + # would otherwise only surface at run time as a silent "skipped", which is exactly + # the failure this layer exists to prevent. + && { printf 'apiVersion: helm.toolkit.fluxcd.io/v2\nkind: HelmRelease\nmetadata:\n name: t\nspec:\n interval: 1h\n chart:\n spec:\n chart: c\n sourceRef:\n kind: HelmRepository\n name: r\n'; \ + printf -- '---\napiVersion: v1\nkind: ConfigMap\nmetadata:\n name: c\n'; } \ + | kubeconform -strict -summary -verbose \ + -schema-location "${KUBECONFORM_SCHEMAS}/builtin/master-standalone-strict/{{.ResourceKind}}{{.KindSuffix}}.json" \ + -schema-location "${KUBECONFORM_SCHEMAS}/crds/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json" \ + - + # Defaults that keep non-interactive agent shells from hanging or failing: # * pagers off — git/kubectl/helm otherwise block on `less` with no TTY # * no git prompts — a private remote without creds fails fast instead of waiting diff --git a/images/node-agent/README.md b/images/node-agent/README.md index 9610fb9..5a5c26d 100644 --- a/images/node-agent/README.md +++ b/images/node-agent/README.md @@ -23,7 +23,8 @@ Base: `node:22-alpine` · ~400 MB uncompressed (`kubectl` and `helm` are about | Shell | `bash`, GNU `coreutils`, `findutils`, `diffutils`, `gawk`, `grep`, `sed` | | Text & search | `jq`, `yq`, `ripgrep` (`rg`), `fd`, `less`, `file`, `patch`, `tree` | | Network | `curl`, `wget`, `dig`/`nslookup`/`host` (bind-tools), `nc`, `socat`, `rsync`, `ping`, `openssl` | -| Kubernetes | `helm`, `kubectl` | +| Kubernetes | `helm`, `kubectl`, `kubeconform` (+ vendored schemas) | +| Linting | `yamllint`, `kubeconform` | | Source control | `git`, `git-lfs`, `ssh`/`scp`/`sftp` | | Archives | `tar`, `gzip`, `xz`, `zip`, `unzip` | | System | `ps`/`top`/`free` (procps-ng), `make`, `su-exec`, `tini`, `tzdata`, `ca-certificates` | @@ -32,8 +33,40 @@ Base: `node:22-alpine` · ~400 MB uncompressed (`kubectl` and `helm` are about Versions track Alpine's repositories and upstream releases at build time — nothing is pinned. Run ` --version` in the image for what a given tag actually shipped. -`yq` comes from upstream releases because Alpine does not package it; everything -else is an Alpine package. +`yq` and `kubeconform` come from upstream releases because Alpine does not package +them; everything else is an Alpine package. + +### Schemas for `kubeconform` + +Both schema sets are **vendored** under `$KUBECONFORM_SCHEMAS`, because the pod that +uses them runs with **no network egress**: + +| Subdirectory | Source | Size | +| --- | --- | --- | +| `builtin/master-standalone-strict/` | [yannh/kubernetes-json-schema](https://github.com/yannh/kubernetes-json-schema) | ~62 MB / 1505 | +| `crds//` | [datree CRDs-catalog](https://github.com/datreeio/CRDs-catalog), only the groups the 50W cluster uses | ~18 MB / 229 | + +```sh +kubeconform -strict -ignore-missing-schemas \ + -schema-location "$KUBECONFORM_SCHEMAS/builtin/master-standalone-strict/{{.ResourceKind}}{{.KindSuffix}}.json" \ + -schema-location "$KUBECONFORM_SCHEMAS/crds/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json" \ + manifest.yaml +``` + +**Do not add `-schema-location default`.** It is a remote +`raw.githubusercontent.com` URL, so with no egress it fails for *every* kind — +`ConfigMap` included — returning a DNS error rather than a verdict on the manifest. +That failure is invisible on a networked machine, which is why `test.sh` runs the +schema checks in their own `--network none` container. + +Why bother: a CRD `kubeconform` has no schema for is reported **`Skipped`**, not +`Invalid`, so a corrupted Flux `HelmRelease` would otherwise sail through. Only the +groups in use are kept; the full catalog is ~211 MB / 4177 schemas. The list lives in +the `Dockerfile` — add a group there and rebuild. + +Groups with no published schema anywhere (`toolhive.stacklok.dev`, `kagent.dev`, +`openclaw.rocks`, `inference.llmkube.dev`) are absent by necessity, so resources of +those kinds report as `Skipped`. Read the summary line — a skip is not a pass. ## Environment @@ -42,6 +75,7 @@ else is an Alpine package. | `PAGER`, `GIT_PAGER` | `cat` | `git`, `kubectl` and `helm` otherwise block on `less` when there is no TTY | | `GIT_TERMINAL_PROMPT` | `0` | a remote needing credentials fails fast instead of hanging on a prompt | | `HELM_CACHE_HOME`, `HELM_CONFIG_HOME`, `HELM_DATA_HOME` | under `/tmp` | lets `helm` run with a read-only root filesystem | +| `KUBECONFORM_SCHEMAS` | `/usr/local/share/kubeconform` | root of the vendored `builtin/` and `crds/` schema trees, so consumers do not hardcode the path | `ENTRYPOINT` is `tini -- docker-entrypoint.sh`, so orphaned grandchildren of a shell command get reaped. `CMD` is `bash`. diff --git a/images/node-agent/test.sh b/images/node-agent/test.sh index 9014a82..4cf80a6 100755 --- a/images/node-agent/test.sh +++ b/images/node-agent/test.sh @@ -42,9 +42,9 @@ gnu() { } echo "-- present on PATH" -for t in bash sh curl wget jq yq helm kubectl git git-lfs ssh scp dig nslookup host \ - nc socat rsync ping rg fd tar gzip xz zip unzip less file patch tree make \ - ps top free tini su-exec node npm npx openssl base64 env xargs; do +for t in bash sh curl wget jq yq yamllint kubeconform helm kubectl git git-lfs ssh scp \ + dig nslookup host nc socat rsync ping rg fd tar gzip xz zip unzip less file \ + patch tree make ps top free tini su-exec node npm npx openssl base64 env xargs; do have "$t" done @@ -63,6 +63,8 @@ works 'fd finds' bash -c 'fd -1 . /tmp >/dev/null' works 'dig runs' dig -v works 'openssl runs' openssl version works 'node runs' node -e 'process.exit(0)' +works 'yamllint flags bad' bash -c '! printf "a: [\n" | yamllint - >/dev/null 2>&1' +works 'yamllint passes ok' bash -c 'printf "a: 1\n" | yamllint -d relaxed -' echo "-- deployment contract" [ "$(id -u)" = 1000 ] && ok 'runs as uid 1000' || bad "runs as uid $(id -u), want 1000" @@ -82,6 +84,59 @@ echo exit "$fail" INNER +# The vendored schemas get their OWN container run, with --network none. +# +# This is not paranoia — it is the check that caught the bug. kubeconform's +# `-schema-location default` is a REMOTE raw.githubusercontent.com URL, so a +# kubeconform test run on a networked machine passes while the real pod (deny-all +# NetworkPolicy) fails on EVERY kind, ConfigMap included, with a DNS error instead of a +# verdict. Only --network none distinguishes "resolves locally" from "quietly +# downloaded it". Note `default` is absent below for the same reason. +echo "==> checking vendored schemas resolve with NO network" +docker run --rm --interactive \ + --user 1000:1000 --read-only --tmpfs /tmp:rw,size=64m --cap-drop ALL \ + --network none --env HOME=/tmp \ + "$REF" bash -s <<'INNER' +set -uo pipefail +fail=0 +ok() { printf ' \033[32mok\033[0m %s\n' "$1"; } +bad() { printf ' \033[31mFAIL\033[0m %s\n' "$1"; fail=1; } + +[ -d "${KUBECONFORM_SCHEMAS:-/nonexistent}" ] \ + && ok "schema root ${KUBECONFORM_SCHEMAS}" \ + || bad "schema root ${KUBECONFORM_SCHEMAS:-unset} missing" +[ -f "${KUBECONFORM_SCHEMAS}/crds/helm.toolkit.fluxcd.io/helmrelease_v2.json" ] \ + && ok 'HelmRelease v2 CRD schema vendored' \ + || bad 'HelmRelease v2 CRD schema missing' +[ -f "${KUBECONFORM_SCHEMAS}/builtin/master-standalone-strict/configmap-v1.json" ] \ + && ok 'built-in schemas vendored' \ + || bad 'built-in schemas missing' + +kc() { + kubeconform -strict -summary -ignore-missing-schemas \ + -schema-location "${KUBECONFORM_SCHEMAS}/builtin/master-standalone-strict/{{.ResourceKind}}{{.KindSuffix}}.json" \ + -schema-location "${KUBECONFORM_SCHEMAS}/crds/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json" - +} +hr() { printf 'apiVersion: helm.toolkit.fluxcd.io/v2\nkind: HelmRelease\nmetadata:\n name: t\nspec:\n interval: 1h\n chart:\n spec:\n chart: c\n %s:\n kind: HelmRepository\n name: r\n' "$1"; } + +if hr sourceRef | kc >/dev/null 2>&1; then ok 'valid HelmRelease passes' +else bad "valid HelmRelease rejected: $(hr sourceRef | kc 2>&1 | tr '\n' ' ')"; fi +# The corruption this exists to catch: a renamed key that is still perfectly valid YAML. +if hr sourceRelease | kc >/dev/null 2>&1; then bad 'renamed key was ACCEPTED' +else ok 'renamed key is rejected'; fi +if printf 'apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: c\n' | kc >/dev/null 2>&1 +then ok 'built-in kind validates offline' +else bad 'built-in kind failed offline'; fi +# A CRD with no published schema must SKIP, not error, or every toolhive/kagent CR fails. +if printf 'apiVersion: toolhive.stacklok.dev/v1beta1\nkind: MCPServer\nmetadata:\n name: x\n' | kc >/dev/null 2>&1 +then ok 'unknown CRD skips cleanly' +else bad 'unknown CRD errored instead of skipping'; fi + +echo +[ "$fail" = 0 ] && echo "schema checks passed" || echo "FAILURES above" +exit "$fail" +INNER + # The default CMD should land in a usable shell rather than the node REPL. Pass no # arguments so the image's own CMD runs, and feed the script on stdin — the base # image's docker-entrypoint.sh rewrites a leading "-c" into a `node` invocation.