feat(litellm): deploy LiteLLM Proxy for OpenAI + Anthropic routing

.gitignore

@@ -5,6 +5,7 @@
 /.pi-subagents/
 /.private/
 /.remember/
+/.superpowers/
 /.task/
 /.worktrees/
 

kubernetes/apps/litellm-operator/kustomization.yaml

@@ -0,0 +1,11 @@
+---
+# yaml-language-server: $schema=https://json.schemastore.org/kustomization
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./namespace.yaml
+  - ./litellm-operator/ks.yaml
+components:
+  - ../../components/common
+transformers:
+  - ./transformers

kubernetes/apps/litellm-operator/litellm-operator/app/helmrelease.yaml

@@ -0,0 +1,12 @@
+---
+# yaml-language-server: $schema=https://crd.kantai.xyz/helm.toolkit.fluxcd.io/helmrelease_v2.json
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: litellm-operator
+spec:
+  interval: 1h
+  chartRef:
+    kind: OCIRepository
+    name: litellm-operator
+  values: {}

kubernetes/apps/litellm-operator/litellm-operator/app/kustomization.yaml

@@ -0,0 +1,8 @@
+---
+# yaml-language-server: $schema=https://json.schemastore.org/kustomization
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./ocirepository.yaml
+  - ./helmrelease.yaml
+  - ./mutatingadmissionpolicy.yaml

kubernetes/apps/litellm-operator/litellm-operator/app/mutatingadmissionpolicy.yaml

@@ -0,0 +1,127 @@
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingAdmissionPolicy
+metadata:
+  name: litellm-securitycontext
+spec:
+  failurePolicy: Fail
+  reinvocationPolicy: Never
+  matchConstraints:
+    resourceRules:
+      - apiGroups: [""]
+        apiVersions: ["v1"]
+        operations: ["CREATE"]
+        resources: ["pods"]
+  matchConditions:
+    - name: is-litellm-proxy-pod
+      expression: >-
+        has(object.metadata.labels) &&
+        object.metadata.labels[?'app.kubernetes.io/managed-by'].orValue('') == 'litellm-operator'
+  mutations:
+    - patchType: ApplyConfiguration
+      applyConfiguration:
+        expression: >-
+          Object{
+            spec: Object.spec{
+              securityContext: Object.spec.securityContext{
+                runAsNonRoot: true,
+                runAsUser: 1000,
+                runAsGroup: 1000,
+                fsGroup: 1000,
+                fsGroupChangePolicy: "OnRootMismatch",
+                seccompProfile: Object.spec.securityContext.seccompProfile{
+                  type: "RuntimeDefault"
+                }
+              }
+            }
+          }
+    - patchType: JSONPatch
+      jsonPatch:
+        expression: >-
+          lists.range(object.spec.containers.size()).map(i,
+            JSONPatch{
+              op: "add",
+              path: "/spec/containers/" + string(i) + "/securityContext",
+              value: Object.spec.containers.securityContext{
+                allowPrivilegeEscalation: false,
+                readOnlyRootFilesystem: true,
+                capabilities: Object.spec.containers.securityContext.capabilities{
+                  drop: ["ALL"]
+                }
+              }
+            })
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingAdmissionPolicyBinding
+metadata:
+  name: litellm-securitycontext-binding
+spec:
+  policyName: litellm-securitycontext
+  matchResources:
+    namespaceSelector:
+      matchLabels:
+        kubernetes.io/metadata.name: litellm
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingAdmissionPolicy
+metadata:
+  name: litellm-init-db
+spec:
+  failurePolicy: Fail
+  reinvocationPolicy: Never
+  matchConstraints:
+    resourceRules:
+      - apiGroups: [""]
+        apiVersions: ["v1"]
+        operations: ["CREATE"]
+        resources: ["pods"]
+  matchConditions:
+    - name: is-litellm-proxy-pod
+      expression: >-
+        has(object.metadata.labels) &&
+        object.metadata.labels[?'app.kubernetes.io/managed-by'].orValue('') == 'litellm-operator'
+  mutations:
+    - patchType: ApplyConfiguration
+      applyConfiguration:
+        expression: >-
+          Object{
+            spec: Object.spec{
+              initContainers: [
+                Object.spec.initContainers{
+                  name: "init-db",
+                  image: "ghcr.io/home-operations/postgres-init:18.4.0@sha256:ebd9d30add17acdf935d73eb004758c7dfd9388aaa605fda70ad74378ab92aec",
+                  envFrom: [
+                    Object.spec.initContainers.envFrom{
+                      secretRef: Object.spec.initContainers.envFrom.secretRef{name: "litellm-db"}
+                    },
+                    Object.spec.initContainers.envFrom{
+                      secretRef: Object.spec.initContainers.envFrom.secretRef{name: "litellm-initdb"}
+                    }
+                  ],
+                  securityContext: Object.spec.initContainers.securityContext{
+                    allowPrivilegeEscalation: false,
+                    readOnlyRootFilesystem: true,
+                    runAsNonRoot: true,
+                    runAsUser: 1000,
+                    capabilities: Object.spec.initContainers.securityContext.capabilities{
+                      drop: ["ALL"]
+                    },
+                    seccompProfile: Object.spec.initContainers.securityContext.seccompProfile{
+                      type: "RuntimeDefault"
+                    }
+                  }
+                }
+              ]
+            }
+          }
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingAdmissionPolicyBinding
+metadata:
+  name: litellm-init-db-binding
+spec:
+  policyName: litellm-init-db
+  matchResources:
+    namespaceSelector:
+      matchLabels:
+        kubernetes.io/metadata.name: litellm

kubernetes/apps/litellm-operator/litellm-operator/app/ocirepository.yaml

@@ -0,0 +1,19 @@
+---
+# yaml-language-server: $schema=https://crd.kantai.xyz/source.toolkit.fluxcd.io/ocirepository_v1.json
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: OCIRepository
+metadata:
+  name: litellm-operator
+spec:
+  interval: 1h
+  layerSelector:
+    mediaType: application/vnd.cncf.helm.chart.content.v1.tar+gzip
+    operation: copy
+  ref:
+    tag: 0.0.15
+  url: oci://ghcr.io/home-operations/charts/litellm-operator
+  verify:
+    provider: cosign
+    matchOIDCIdentity:
+      - issuer: ^https://token.actions.githubusercontent.com$
+        subject: ^https://github.com/home-operations/litellm-operator/.*$

kubernetes/apps/litellm-operator/litellm-operator/ks.yaml

@@ -0,0 +1,24 @@
+---
+# yaml-language-server: $schema=https://crd.kantai.xyz/kustomize.toolkit.fluxcd.io/kustomization_v1.json
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: &app litellm-operator
+spec:
+  healthChecks:
+    - apiVersion: helm.toolkit.fluxcd.io/v2
+      kind: HelmRelease
+      name: litellm-operator
+      namespace: litellm-operator
+  path: ./kubernetes/apps/litellm-operator/litellm-operator/app
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  interval: 1h
+  retryInterval: 2m
+  timeout: 10m
+  postBuild:
+    substitute:
+      APP: *app

kubernetes/apps/litellm-operator/namespace.yaml

@@ -0,0 +1,8 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: .invalid
+  labels:
+    pod-security.kubernetes.io/enforce: restricted
+    pod-security.kubernetes.io/enforce-version: latest

kubernetes/apps/litellm-operator/transformers/kustomization.yaml

@@ -0,0 +1,7 @@
+---
+# yaml-language-server: $schema=https://json.schemastore.org/kustomization
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: litellm-operator
+resources:
+  - ../../../transformers

kubernetes/apps/litellm/kustomization.yaml

@@ -0,0 +1,11 @@
+---
+# yaml-language-server: $schema=https://json.schemastore.org/kustomization
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./namespace.yaml
+  - ./litellm/ks.yaml
+components:
+  - ../../components/common
+transformers:
+  - ./transformers

kubernetes/apps/litellm/litellm/app/externalsecret.yaml

@@ -0,0 +1,130 @@
+---
+# yaml-language-server: $schema=https://crd.kantai.xyz/external-secrets.io/externalsecret_v1.json
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: litellm
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: onepassword
+  target:
+    name: litellm
+  dataFrom:
+    - extract:
+        key: litellm
+---
+# yaml-language-server: $schema=https://crd.kantai.xyz/external-secrets.io/externalsecret_v1.json
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: litellm-db
+spec:
+  refreshInterval: "0"
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: onepassword
+  target:
+    name: litellm-db
+    template:
+      data:
+        DATABASE_USERNAME: litellm
+        DATABASE_PASSWORD: "{{ .DB_PASSWORD }}"
+        DATABASE_URL: "postgresql://litellm:{{ .DB_PASSWORD }}@pg18vc-rw.database.svc.cluster.local:5432/litellm"
+        DATABASE_URL_READ_REPLICA: "postgresql://litellm:{{ .DB_PASSWORD }}@pg18vc-ro.database.svc.cluster.local:5432/litellm"
+        INIT_POSTGRES_DBNAME: litellm
+        INIT_POSTGRES_HOST: pg18vc-rw.database.svc.cluster.local
+        INIT_POSTGRES_USER: litellm
+        INIT_POSTGRES_PASS: "{{ .DB_PASSWORD }}"
+  dataFrom:
+    - sourceRef:
+        generatorRef:
+          apiVersion: generators.external-secrets.io/v1alpha1
+          kind: Password
+          name: password32
+      rewrite:
+        - regexp:
+            source: "password"
+            target: "DB_PASSWORD"
+---
+# yaml-language-server: $schema=https://crd.kantai.xyz/external-secrets.io/externalsecret_v1.json
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: litellm-initdb
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: onepassword
+  target:
+    name: litellm-initdb
+  data:
+    - secretKey: INIT_POSTGRES_SUPER_PASS
+      remoteRef:
+        key: cnpg-pg18vc/password
+---
+# yaml-language-server: $schema=https://crd.kantai.xyz/external-secrets.io/externalsecret_v1.json
+# Master-key rotation: delete this Secret (litellm-keys) to force ExternalSecret to regenerate
+# PROXY_MASTER_KEY — Stakater Reloader (podAnnotations."reloader.stakater.com/auto") restarts
+# the Deployment automatically on the change, no manual restart step needed. Do NOT call the
+# proxy's POST /key/regenerate endpoint WITH new_master_key — that path is for deployments
+# without a salt key; calling it here would re-encrypt already-salt-encrypted provider
+# credentials under a key the proxy never decrypts with, bricking them. (Plain
+# /key/regenerate WITHOUT new_master_key, for rotating a *virtual* key, is unrelated and
+# fine — virtual keys are hashed, not encrypted.)
+# LITELLM_SALT_KEY lives in the separate litellm-salt Secret below specifically so it can
+# never be rotated by this procedure — see https://docs.litellm.ai/docs/proxy/master_key_rotations.
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: litellm-keys
+spec:
+  refreshInterval: "0"
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: onepassword
+  target:
+    name: litellm-keys
+    template:
+      data:
+        # litellm requires the master key to start with "sk-"
+        PROXY_MASTER_KEY: "sk-{{ .MASTER_KEY_RAW }}"
+  dataFrom:
+    - sourceRef:
+        generatorRef:
+          apiVersion: generators.external-secrets.io/v1alpha1
+          kind: Password
+          name: password32
+      rewrite:
+        - regexp:
+            source: "password"
+            target: "MASTER_KEY_RAW"
+---
+# yaml-language-server: $schema=https://crd.kantai.xyz/external-secrets.io/externalsecret_v1.json
+# LITELLM_SALT_KEY encrypts stored provider credentials and must NEVER be regenerated/rotated
+# once set. Kept in its own Secret, separate from litellm-keys, specifically so that deleting
+# litellm-keys to rotate the master key (see comment there) can never also regenerate this.
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: litellm-salt
+spec:
+  refreshInterval: "0"
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: onepassword
+  target:
+    name: litellm-salt
+    template:
+      data:
+        LITELLM_SALT_KEY: "{{ .SALT_KEY_RAW }}"
+  dataFrom:
+    - sourceRef:
+        generatorRef:
+          apiVersion: generators.external-secrets.io/v1alpha1
+          kind: Password
+          name: password32
+      rewrite:
+        - regexp:
+            source: "password"
+            target: "SALT_KEY_RAW"

kubernetes/apps/litellm/litellm/app/grafanadashboard.yaml

@@ -0,0 +1,15 @@
+---
+# yaml-language-server: $schema=https://crd.kantai.xyz/grafana.integreatly.org/grafanadashboard_v1beta1.json
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: litellm
+spec:
+  allowCrossNamespaceImport: true
+  instanceSelector:
+    matchLabels:
+      grafana.internal/instance: grafana
+  datasources:
+    - datasourceName: prometheus
+      inputName: DS_PROMETHEUS
+  url: https://grafana.com/api/dashboards/24965/revisions/2/download