From 0258e9aeb4408915d5573c92580d69511bf6ffb8 Mon Sep 17 00:00:00 2001 From: Jean-Francois Roy Date: Thu, 6 Aug 2026 10:57:59 -0700 Subject: [PATCH] feat(litellm): deploy using home-ops operator --- .gitignore | 1 + .../apps/litellm-operator/kustomization.yaml | 11 ++ .../litellm-operator/app/helmrelease.yaml | 12 ++ .../litellm-operator/app/kustomization.yaml | 8 ++ .../app/mutatingadmissionpolicy.yaml | 127 +++++++++++++++++ .../litellm-operator/app/ocirepository.yaml | 19 +++ .../litellm-operator/litellm-operator/ks.yaml | 24 ++++ .../apps/litellm-operator/namespace.yaml | 8 ++ .../transformers/kustomization.yaml | 7 + kubernetes/apps/litellm/kustomization.yaml | 11 ++ .../litellm/litellm/app/externalsecret.yaml | 130 ++++++++++++++++++ .../litellm/litellm/app/grafanadashboard.yaml | 15 ++ .../litellm/litellm/app/kustomization.yaml | 12 ++ .../litellm/litellm/app/litellmmodels.yaml | 92 +++++++++++++ .../litellm/litellm/app/litellmproxy.yaml | 101 ++++++++++++++ .../litellm/app/objectbucketclaim.yaml | 11 ++ .../litellm/litellm/app/prometheusrule.yaml | 36 +++++ .../litellm/litellm/app/servicemonitor.yaml | 27 ++++ kubernetes/apps/litellm/litellm/ks.yaml | 22 +++ kubernetes/apps/litellm/namespace.yaml | 8 ++ .../litellm/transformers/kustomization.yaml | 7 + 21 files changed, 689 insertions(+) create mode 100644 kubernetes/apps/litellm-operator/kustomization.yaml create mode 100644 kubernetes/apps/litellm-operator/litellm-operator/app/helmrelease.yaml create mode 100644 kubernetes/apps/litellm-operator/litellm-operator/app/kustomization.yaml create mode 100644 kubernetes/apps/litellm-operator/litellm-operator/app/mutatingadmissionpolicy.yaml create mode 100644 kubernetes/apps/litellm-operator/litellm-operator/app/ocirepository.yaml create mode 100644 kubernetes/apps/litellm-operator/litellm-operator/ks.yaml create mode 100644 kubernetes/apps/litellm-operator/namespace.yaml create mode 100644 kubernetes/apps/litellm-operator/transformers/kustomization.yaml create mode 100644 kubernetes/apps/litellm/kustomization.yaml create mode 100644 kubernetes/apps/litellm/litellm/app/externalsecret.yaml create mode 100644 kubernetes/apps/litellm/litellm/app/grafanadashboard.yaml create mode 100644 kubernetes/apps/litellm/litellm/app/kustomization.yaml create mode 100644 kubernetes/apps/litellm/litellm/app/litellmmodels.yaml create mode 100644 kubernetes/apps/litellm/litellm/app/litellmproxy.yaml create mode 100644 kubernetes/apps/litellm/litellm/app/objectbucketclaim.yaml create mode 100644 kubernetes/apps/litellm/litellm/app/prometheusrule.yaml create mode 100644 kubernetes/apps/litellm/litellm/app/servicemonitor.yaml create mode 100644 kubernetes/apps/litellm/litellm/ks.yaml create mode 100644 kubernetes/apps/litellm/namespace.yaml create mode 100644 kubernetes/apps/litellm/transformers/kustomization.yaml diff --git a/.gitignore b/.gitignore index 343f164dcc..6d5e41d465 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ /.pi-subagents/ /.private/ /.remember/ +/.superpowers/ /.task/ /.worktrees/ diff --git a/kubernetes/apps/litellm-operator/kustomization.yaml b/kubernetes/apps/litellm-operator/kustomization.yaml new file mode 100644 index 0000000000..cf2db2e21a --- /dev/null +++ b/kubernetes/apps/litellm-operator/kustomization.yaml @@ -0,0 +1,11 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/kustomization +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./namespace.yaml + - ./litellm-operator/ks.yaml +components: + - ../../components/common +transformers: + - ./transformers diff --git a/kubernetes/apps/litellm-operator/litellm-operator/app/helmrelease.yaml b/kubernetes/apps/litellm-operator/litellm-operator/app/helmrelease.yaml new file mode 100644 index 0000000000..c5f4a5bff4 --- /dev/null +++ b/kubernetes/apps/litellm-operator/litellm-operator/app/helmrelease.yaml @@ -0,0 +1,12 @@ +--- +# yaml-language-server: $schema=https://crd.kantai.xyz/helm.toolkit.fluxcd.io/helmrelease_v2.json +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: litellm-operator +spec: + interval: 1h + chartRef: + kind: OCIRepository + name: litellm-operator + values: {} diff --git a/kubernetes/apps/litellm-operator/litellm-operator/app/kustomization.yaml b/kubernetes/apps/litellm-operator/litellm-operator/app/kustomization.yaml new file mode 100644 index 0000000000..16e5e9a7f5 --- /dev/null +++ b/kubernetes/apps/litellm-operator/litellm-operator/app/kustomization.yaml @@ -0,0 +1,8 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/kustomization +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./ocirepository.yaml + - ./helmrelease.yaml + - ./mutatingadmissionpolicy.yaml diff --git a/kubernetes/apps/litellm-operator/litellm-operator/app/mutatingadmissionpolicy.yaml b/kubernetes/apps/litellm-operator/litellm-operator/app/mutatingadmissionpolicy.yaml new file mode 100644 index 0000000000..fe6597551d --- /dev/null +++ b/kubernetes/apps/litellm-operator/litellm-operator/app/mutatingadmissionpolicy.yaml @@ -0,0 +1,127 @@ +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingAdmissionPolicy +metadata: + name: litellm-securitycontext +spec: + failurePolicy: Fail + reinvocationPolicy: Never + matchConstraints: + resourceRules: + - apiGroups: [""] + apiVersions: ["v1"] + operations: ["CREATE"] + resources: ["pods"] + matchConditions: + - name: is-litellm-proxy-pod + expression: >- + has(object.metadata.labels) && + object.metadata.labels[?'app.kubernetes.io/managed-by'].orValue('') == 'litellm-operator' + mutations: + - patchType: ApplyConfiguration + applyConfiguration: + expression: >- + Object{ + spec: Object.spec{ + securityContext: Object.spec.securityContext{ + runAsNonRoot: true, + runAsUser: 1000, + runAsGroup: 1000, + fsGroup: 1000, + fsGroupChangePolicy: "OnRootMismatch", + seccompProfile: Object.spec.securityContext.seccompProfile{ + type: "RuntimeDefault" + } + } + } + } + - patchType: JSONPatch + jsonPatch: + expression: >- + lists.range(object.spec.containers.size()).map(i, + JSONPatch{ + op: "add", + path: "/spec/containers/" + string(i) + "/securityContext", + value: Object.spec.containers.securityContext{ + allowPrivilegeEscalation: false, + readOnlyRootFilesystem: true, + capabilities: Object.spec.containers.securityContext.capabilities{ + drop: ["ALL"] + } + } + }) +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingAdmissionPolicyBinding +metadata: + name: litellm-securitycontext-binding +spec: + policyName: litellm-securitycontext + matchResources: + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: litellm +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingAdmissionPolicy +metadata: + name: litellm-init-db +spec: + failurePolicy: Fail + reinvocationPolicy: Never + matchConstraints: + resourceRules: + - apiGroups: [""] + apiVersions: ["v1"] + operations: ["CREATE"] + resources: ["pods"] + matchConditions: + - name: is-litellm-proxy-pod + expression: >- + has(object.metadata.labels) && + object.metadata.labels[?'app.kubernetes.io/managed-by'].orValue('') == 'litellm-operator' + mutations: + - patchType: ApplyConfiguration + applyConfiguration: + expression: >- + Object{ + spec: Object.spec{ + initContainers: [ + Object.spec.initContainers{ + name: "init-db", + image: "ghcr.io/home-operations/postgres-init:18.4.0@sha256:ebd9d30add17acdf935d73eb004758c7dfd9388aaa605fda70ad74378ab92aec", + envFrom: [ + Object.spec.initContainers.envFrom{ + secretRef: Object.spec.initContainers.envFrom.secretRef{name: "litellm-db"} + }, + Object.spec.initContainers.envFrom{ + secretRef: Object.spec.initContainers.envFrom.secretRef{name: "litellm-initdb"} + } + ], + securityContext: Object.spec.initContainers.securityContext{ + allowPrivilegeEscalation: false, + readOnlyRootFilesystem: true, + runAsNonRoot: true, + runAsUser: 1000, + capabilities: Object.spec.initContainers.securityContext.capabilities{ + drop: ["ALL"] + }, + seccompProfile: Object.spec.initContainers.securityContext.seccompProfile{ + type: "RuntimeDefault" + } + } + } + ] + } + } +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingAdmissionPolicyBinding +metadata: + name: litellm-init-db-binding +spec: + policyName: litellm-init-db + matchResources: + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: litellm diff --git a/kubernetes/apps/litellm-operator/litellm-operator/app/ocirepository.yaml b/kubernetes/apps/litellm-operator/litellm-operator/app/ocirepository.yaml new file mode 100644 index 0000000000..be453a0456 --- /dev/null +++ b/kubernetes/apps/litellm-operator/litellm-operator/app/ocirepository.yaml @@ -0,0 +1,19 @@ +--- +# yaml-language-server: $schema=https://crd.kantai.xyz/source.toolkit.fluxcd.io/ocirepository_v1.json +apiVersion: source.toolkit.fluxcd.io/v1 +kind: OCIRepository +metadata: + name: litellm-operator +spec: + interval: 1h + layerSelector: + mediaType: application/vnd.cncf.helm.chart.content.v1.tar+gzip + operation: copy + ref: + tag: 0.0.15 + url: oci://ghcr.io/home-operations/charts/litellm-operator + verify: + provider: cosign + matchOIDCIdentity: + - issuer: ^https://token.actions.githubusercontent.com$ + subject: ^https://github.com/home-operations/litellm-operator/.*$ diff --git a/kubernetes/apps/litellm-operator/litellm-operator/ks.yaml b/kubernetes/apps/litellm-operator/litellm-operator/ks.yaml new file mode 100644 index 0000000000..c9dc958d36 --- /dev/null +++ b/kubernetes/apps/litellm-operator/litellm-operator/ks.yaml @@ -0,0 +1,24 @@ +--- +# yaml-language-server: $schema=https://crd.kantai.xyz/kustomize.toolkit.fluxcd.io/kustomization_v1.json +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app litellm-operator +spec: + healthChecks: + - apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + name: litellm-operator + namespace: litellm-operator + path: ./kubernetes/apps/litellm-operator/litellm-operator/app + prune: true + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 1h + retryInterval: 2m + timeout: 10m + postBuild: + substitute: + APP: *app diff --git a/kubernetes/apps/litellm-operator/namespace.yaml b/kubernetes/apps/litellm-operator/namespace.yaml new file mode 100644 index 0000000000..6e7e563653 --- /dev/null +++ b/kubernetes/apps/litellm-operator/namespace.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: .invalid + labels: + pod-security.kubernetes.io/enforce: restricted + pod-security.kubernetes.io/enforce-version: latest diff --git a/kubernetes/apps/litellm-operator/transformers/kustomization.yaml b/kubernetes/apps/litellm-operator/transformers/kustomization.yaml new file mode 100644 index 0000000000..c0434c999a --- /dev/null +++ b/kubernetes/apps/litellm-operator/transformers/kustomization.yaml @@ -0,0 +1,7 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/kustomization +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: litellm-operator +resources: + - ../../../transformers diff --git a/kubernetes/apps/litellm/kustomization.yaml b/kubernetes/apps/litellm/kustomization.yaml new file mode 100644 index 0000000000..5f34158cda --- /dev/null +++ b/kubernetes/apps/litellm/kustomization.yaml @@ -0,0 +1,11 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/kustomization +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./namespace.yaml + - ./litellm/ks.yaml +components: + - ../../components/common +transformers: + - ./transformers diff --git a/kubernetes/apps/litellm/litellm/app/externalsecret.yaml b/kubernetes/apps/litellm/litellm/app/externalsecret.yaml new file mode 100644 index 0000000000..28c5640f39 --- /dev/null +++ b/kubernetes/apps/litellm/litellm/app/externalsecret.yaml @@ -0,0 +1,130 @@ +--- +# yaml-language-server: $schema=https://crd.kantai.xyz/external-secrets.io/externalsecret_v1.json +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: litellm +spec: + secretStoreRef: + kind: ClusterSecretStore + name: onepassword + target: + name: litellm + dataFrom: + - extract: + key: litellm +--- +# yaml-language-server: $schema=https://crd.kantai.xyz/external-secrets.io/externalsecret_v1.json +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: litellm-db +spec: + refreshInterval: "0" + secretStoreRef: + kind: ClusterSecretStore + name: onepassword + target: + name: litellm-db + template: + data: + DATABASE_USERNAME: litellm + DATABASE_PASSWORD: "{{ .DB_PASSWORD }}" + DATABASE_URL: "postgresql://litellm:{{ .DB_PASSWORD }}@pg18vc-rw.database.svc.cluster.local:5432/litellm" + DATABASE_URL_READ_REPLICA: "postgresql://litellm:{{ .DB_PASSWORD }}@pg18vc-ro.database.svc.cluster.local:5432/litellm" + INIT_POSTGRES_DBNAME: litellm + INIT_POSTGRES_HOST: pg18vc-rw.database.svc.cluster.local + INIT_POSTGRES_USER: litellm + INIT_POSTGRES_PASS: "{{ .DB_PASSWORD }}" + dataFrom: + - sourceRef: + generatorRef: + apiVersion: generators.external-secrets.io/v1alpha1 + kind: Password + name: password32 + rewrite: + - regexp: + source: "password" + target: "DB_PASSWORD" +--- +# yaml-language-server: $schema=https://crd.kantai.xyz/external-secrets.io/externalsecret_v1.json +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: litellm-initdb +spec: + secretStoreRef: + kind: ClusterSecretStore + name: onepassword + target: + name: litellm-initdb + data: + - secretKey: INIT_POSTGRES_SUPER_PASS + remoteRef: + key: cnpg-pg18vc/password +--- +# yaml-language-server: $schema=https://crd.kantai.xyz/external-secrets.io/externalsecret_v1.json +# Master-key rotation: delete this Secret (litellm-keys) to force ExternalSecret to regenerate +# PROXY_MASTER_KEY — Stakater Reloader (podAnnotations."reloader.stakater.com/auto") restarts +# the Deployment automatically on the change, no manual restart step needed. Do NOT call the +# proxy's POST /key/regenerate endpoint WITH new_master_key — that path is for deployments +# without a salt key; calling it here would re-encrypt already-salt-encrypted provider +# credentials under a key the proxy never decrypts with, bricking them. (Plain +# /key/regenerate WITHOUT new_master_key, for rotating a *virtual* key, is unrelated and +# fine — virtual keys are hashed, not encrypted.) +# LITELLM_SALT_KEY lives in the separate litellm-salt Secret below specifically so it can +# never be rotated by this procedure — see https://docs.litellm.ai/docs/proxy/master_key_rotations. +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: litellm-keys +spec: + refreshInterval: "0" + secretStoreRef: + kind: ClusterSecretStore + name: onepassword + target: + name: litellm-keys + template: + data: + # litellm requires the master key to start with "sk-" + PROXY_MASTER_KEY: "sk-{{ .MASTER_KEY_RAW }}" + dataFrom: + - sourceRef: + generatorRef: + apiVersion: generators.external-secrets.io/v1alpha1 + kind: Password + name: password32 + rewrite: + - regexp: + source: "password" + target: "MASTER_KEY_RAW" +--- +# yaml-language-server: $schema=https://crd.kantai.xyz/external-secrets.io/externalsecret_v1.json +# LITELLM_SALT_KEY encrypts stored provider credentials and must NEVER be regenerated/rotated +# once set. Kept in its own Secret, separate from litellm-keys, specifically so that deleting +# litellm-keys to rotate the master key (see comment there) can never also regenerate this. +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: litellm-salt +spec: + refreshInterval: "0" + secretStoreRef: + kind: ClusterSecretStore + name: onepassword + target: + name: litellm-salt + template: + data: + LITELLM_SALT_KEY: "{{ .SALT_KEY_RAW }}" + dataFrom: + - sourceRef: + generatorRef: + apiVersion: generators.external-secrets.io/v1alpha1 + kind: Password + name: password32 + rewrite: + - regexp: + source: "password" + target: "SALT_KEY_RAW" diff --git a/kubernetes/apps/litellm/litellm/app/grafanadashboard.yaml b/kubernetes/apps/litellm/litellm/app/grafanadashboard.yaml new file mode 100644 index 0000000000..ffdb8d3ec9 --- /dev/null +++ b/kubernetes/apps/litellm/litellm/app/grafanadashboard.yaml @@ -0,0 +1,15 @@ +--- +# yaml-language-server: $schema=https://crd.kantai.xyz/grafana.integreatly.org/grafanadashboard_v1beta1.json +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaDashboard +metadata: + name: litellm +spec: + allowCrossNamespaceImport: true + instanceSelector: + matchLabels: + grafana.internal/instance: grafana + datasources: + - datasourceName: prometheus + inputName: DS_PROMETHEUS + url: https://grafana.com/api/dashboards/24965/revisions/2/download diff --git a/kubernetes/apps/litellm/litellm/app/kustomization.yaml b/kubernetes/apps/litellm/litellm/app/kustomization.yaml new file mode 100644 index 0000000000..13de59593d --- /dev/null +++ b/kubernetes/apps/litellm/litellm/app/kustomization.yaml @@ -0,0 +1,12 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/kustomization +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./externalsecret.yaml + - ./objectbucketclaim.yaml + - ./litellmproxy.yaml + - ./litellmmodels.yaml + - ./servicemonitor.yaml + - ./prometheusrule.yaml + - ./grafanadashboard.yaml diff --git a/kubernetes/apps/litellm/litellm/app/litellmmodels.yaml b/kubernetes/apps/litellm/litellm/app/litellmmodels.yaml new file mode 100644 index 0000000000..22cebd1c7c --- /dev/null +++ b/kubernetes/apps/litellm/litellm/app/litellmmodels.yaml @@ -0,0 +1,92 @@ +--- +# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/litellm.home-operations.com/litellmmodel_v1alpha1.json +apiVersion: litellm.home-operations.com/v1alpha1 +kind: LiteLLMModel +metadata: + name: claude-sonnet-5 +spec: + proxyRef: litellm + modelName: claude-sonnet-5 + params: + model: anthropic/claude-sonnet-5 + apiKeyRef: + name: litellm + key: ANTHROPIC_API_KEY + additional: + rust: true +--- +# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/litellm.home-operations.com/litellmmodel_v1alpha1.json +apiVersion: litellm.home-operations.com/v1alpha1 +kind: LiteLLMModel +metadata: + name: claude-opus-5 +spec: + proxyRef: litellm + modelName: claude-opus-5 + params: + model: anthropic/claude-opus-5 + apiKeyRef: + name: litellm + key: ANTHROPIC_API_KEY + additional: + rust: true +--- +# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/litellm.home-operations.com/litellmmodel_v1alpha1.json +apiVersion: litellm.home-operations.com/v1alpha1 +kind: LiteLLMModel +metadata: + name: claude-fable-5 +spec: + proxyRef: litellm + modelName: claude-fable-5 + params: + model: anthropic/claude-fable-5 + apiKeyRef: + name: litellm + key: ANTHROPIC_API_KEY + additional: + rust: true +--- +# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/litellm.home-operations.com/litellmmodel_v1alpha1.json +apiVersion: litellm.home-operations.com/v1alpha1 +kind: LiteLLMModel +metadata: + # Kubernetes object names can't contain dots — spec.modelName (below) keeps the real + # dotted name clients call. + name: gpt-5-6-sol +spec: + proxyRef: litellm + modelName: gpt-5.6-sol + params: + model: openai/gpt-5.6-sol + apiKeyRef: + name: litellm + key: OPENAI_API_KEY +--- +# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/litellm.home-operations.com/litellmmodel_v1alpha1.json +apiVersion: litellm.home-operations.com/v1alpha1 +kind: LiteLLMModel +metadata: + name: gpt-5-6-terra +spec: + proxyRef: litellm + modelName: gpt-5.6-terra + params: + model: openai/gpt-5.6-terra + apiKeyRef: + name: litellm + key: OPENAI_API_KEY +--- +# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/litellm.home-operations.com/litellmmodel_v1alpha1.json +apiVersion: litellm.home-operations.com/v1alpha1 +kind: LiteLLMModel +metadata: + name: gpt-5-6-luna +spec: + proxyRef: litellm + modelName: gpt-5.6-luna + params: + model: openai/gpt-5.6-luna + apiKeyRef: + name: litellm + key: OPENAI_API_KEY diff --git a/kubernetes/apps/litellm/litellm/app/litellmproxy.yaml b/kubernetes/apps/litellm/litellm/app/litellmproxy.yaml new file mode 100644 index 0000000000..736f236ad7 --- /dev/null +++ b/kubernetes/apps/litellm/litellm/app/litellmproxy.yaml @@ -0,0 +1,101 @@ +--- +# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/litellm.home-operations.com/litellmproxy_v1alpha1.json +apiVersion: litellm.home-operations.com/v1alpha1 +kind: LiteLLMProxy +metadata: + name: litellm +spec: + # renovate: datasource=docker depName=ghcr.io/berriai/litellm versioning=loose + image: ghcr.io/berriai/litellm:v1.98.0@sha256:20b5044b619055374061a6d5b7b08754cad75aeabbf82ddf4f69cc0cf80ddaf4 + replicas: 1 + applyMode: api + podAnnotations: + reloader.stakater.com/auto: "true" + apiAccess: + masterKeyRef: + name: litellm-keys + key: PROXY_MASTER_KEY + route: + hostnames: + - litellm.kantai.xyz + parentRefs: + - name: envoy-internal + namespace: network + callbacks: + callbacks: ["prometheus", "s3_v2"] + generalSettings: + master_key: os.environ/PROXY_MASTER_KEY + database_connection_pool_limit: 10 + proxy_batch_write_at: 60 + litellmSettings: + s3_callback_params: + s3_bucket_name: os.environ/S3_BUCKET_NAME + s3_region_name: os.environ/S3_REGION_NAME + s3_endpoint_url: os.environ/S3_ENDPOINT_URL + s3_use_virtual_hosted_style: true + request_timeout: 600 + set_verbose: false + json_logs: true + env: + - name: LITELLM_LOG + value: ERROR + - name: LITELLM_MODE + value: PRODUCTION + - name: TZ + value: America/Los_Angeles + - name: S3_BUCKET_NAME + value: litellm-s3 + - name: S3_REGION_NAME + value: us-west-1 + - name: S3_ENDPOINT_URL + value: https://s3.kantai.xyz + - name: PROXY_BASE_URL + value: https://litellm.kantai.xyz + - name: GENERIC_AUTHORIZATION_ENDPOINT + value: https://pid.kantai.xyz/authorize + - name: GENERIC_CLIENT_ID + value: a0b9bcc2-6e1d-4a6f-99fc-b431666aac89 + - name: GENERIC_TOKEN_ENDPOINT + value: https://pid.kantai.xyz/api/oidc/token + - name: GENERIC_USERINFO_ENDPOINT + value: https://pid.kantai.xyz/api/oidc/userinfo + - name: GENERIC_SCOPE + value: "openid profile email" + - name: GENERIC_USER_ID_ATTRIBUTE + value: sub + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: litellm-db + key: DATABASE_URL + - name: DATABASE_URL_READ_REPLICA + valueFrom: + secretKeyRef: + name: litellm-db + key: DATABASE_URL_READ_REPLICA + envFrom: + - secretRef: + name: litellm + - secretRef: + name: litellm-keys + - secretRef: + name: litellm-salt + - secretRef: + name: litellm-s3-obc + volumes: + - name: tmp + emptyDir: + sizeLimit: 500Mi + - name: cache + emptyDir: + sizeLimit: 500Mi + - name: npm + emptyDir: + sizeLimit: 500Mi + volumeMounts: + - name: tmp + mountPath: /tmp + - name: cache + mountPath: /.cache + - name: npm + mountPath: /.npm diff --git a/kubernetes/apps/litellm/litellm/app/objectbucketclaim.yaml b/kubernetes/apps/litellm/litellm/app/objectbucketclaim.yaml new file mode 100644 index 0000000000..947f564f61 --- /dev/null +++ b/kubernetes/apps/litellm/litellm/app/objectbucketclaim.yaml @@ -0,0 +1,11 @@ +# yaml-language-server: $schema=https://crd.kantai.xyz/objectbucket.io/objectbucketclaim_v1alpha1.json +--- +apiVersion: objectbucket.io/v1alpha1 +kind: ObjectBucketClaim +metadata: + name: litellm-s3-obc + annotations: + kustomize.toolkit.fluxcd.io/prune: disabled +spec: + bucketName: litellm-s3 + storageClassName: ceph-bucket diff --git a/kubernetes/apps/litellm/litellm/app/prometheusrule.yaml b/kubernetes/apps/litellm/litellm/app/prometheusrule.yaml new file mode 100644 index 0000000000..f45fd813c8 --- /dev/null +++ b/kubernetes/apps/litellm/litellm/app/prometheusrule.yaml @@ -0,0 +1,36 @@ +--- +# yaml-language-server: $schema=https://crd.kantai.xyz/monitoring.coreos.com/prometheusrule_v1.json +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + labels: + app.kubernetes.io/component: litellm + app.kubernetes.io/instance: litellm + app.kubernetes.io/name: litellm + name: litellm +spec: + groups: + - name: litellm.rules + rules: + - alert: LitellmDown + annotations: + description: litellm proxy has disappeared from Prometheus target discovery. + summary: litellm proxy is down. + expr: | + absent(up{job=~".*litellm.*"} == 1) + for: 5m + labels: + severity: critical + - alert: LitellmHighFailureRate + annotations: + description: "litellm proxy failure rate is {{ $value | humanizePercentage }} over the last 15 minutes." + summary: litellm proxy is failing a high proportion of requests. + expr: | + ( + sum(rate(litellm_proxy_failed_requests_metric_total[15m])) + / + sum(rate(litellm_proxy_total_requests_metric_total[15m])) + ) > 0.1 + for: 10m + labels: + severity: warning diff --git a/kubernetes/apps/litellm/litellm/app/servicemonitor.yaml b/kubernetes/apps/litellm/litellm/app/servicemonitor.yaml new file mode 100644 index 0000000000..0b2eb80d72 --- /dev/null +++ b/kubernetes/apps/litellm/litellm/app/servicemonitor.yaml @@ -0,0 +1,27 @@ +--- +# yaml-language-server: $schema=https://crd.kantai.xyz/monitoring.coreos.com/servicemonitor_v1.json +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + app.kubernetes.io/name: litellm + name: litellm +spec: + selector: + matchLabels: + app.kubernetes.io/name: litellm + app.kubernetes.io/instance: litellm + namespaceSelector: + matchNames: + - litellm + endpoints: + - port: http + path: /metrics/ + scheme: http + interval: 30s + scrapeTimeout: 10s + authorization: + type: Bearer + credentials: + name: litellm-keys + key: PROXY_MASTER_KEY diff --git a/kubernetes/apps/litellm/litellm/ks.yaml b/kubernetes/apps/litellm/litellm/ks.yaml new file mode 100644 index 0000000000..fe5f45e606 --- /dev/null +++ b/kubernetes/apps/litellm/litellm/ks.yaml @@ -0,0 +1,22 @@ +--- +# yaml-language-server: $schema=https://crd.kantai.xyz/kustomize.toolkit.fluxcd.io/kustomization_v1.json +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app litellm +spec: + dependsOn: + - name: litellm-operator + namespace: litellm-operator + path: ./kubernetes/apps/litellm/litellm/app + prune: true + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 1h + retryInterval: 2m + timeout: 10m + postBuild: + substitute: + APP: *app diff --git a/kubernetes/apps/litellm/namespace.yaml b/kubernetes/apps/litellm/namespace.yaml new file mode 100644 index 0000000000..6e7e563653 --- /dev/null +++ b/kubernetes/apps/litellm/namespace.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: .invalid + labels: + pod-security.kubernetes.io/enforce: restricted + pod-security.kubernetes.io/enforce-version: latest diff --git a/kubernetes/apps/litellm/transformers/kustomization.yaml b/kubernetes/apps/litellm/transformers/kustomization.yaml new file mode 100644 index 0000000000..b5efab0980 --- /dev/null +++ b/kubernetes/apps/litellm/transformers/kustomization.yaml @@ -0,0 +1,7 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/kustomization +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: litellm +resources: + - ../../../transformers