diff --git a/ai/generative-ai-service/litellm-gateway/files/README.md b/ai/generative-ai-service/litellm-gateway/files/README.md index 2acb6e2a7..41afa0ca3 100644 --- a/ai/generative-ai-service/litellm-gateway/files/README.md +++ b/ai/generative-ai-service/litellm-gateway/files/README.md @@ -48,8 +48,8 @@ Without Postgres, comment out `database_url` in `general_settings` — the gatew `model_list` exposes three kinds of models behind identical OpenAI semantics: -1. **OCI on-demand models** (`oci/xai.grok-4`, `oci/meta.llama-4-scout...`) through LiteLLM's native OCI provider, signed with your API key. -2. **Imported models on a Dedicated AI Cluster.** OCI serves these behind its OpenAI-compatible endpoint (`.../20231130/actions/v1`), authenticated with a plain OCI GenAI API key (`sk-...`) where the *model name is the DAC endpoint OCID*. The `qwen3-dac` entry shows the pattern — swap in your own endpoint OCID via `OCI_DAC_MODEL`. Run `examples/03_dac_imported_model.py` to see both the gateway route and the direct call. +1. **OCI on-demand models** (`oci/xai.grok-4`, `oci/meta.llama-3.3-70b-instruct`, ...) through LiteLLM's native OCI provider, signed with your API key. +2. **Imported models on a Dedicated AI Cluster.** OCI serves these behind its OpenAI-compatible endpoint (`.../20231130/actions/v1`), authenticated with a plain OCI GenAI API key (`sk-...`) where the *model name is the DAC endpoint OCID*. The `gpt-oss-120b-dac` entry shows the pattern — swap in your own endpoint OCID via `OCI_DAC_MODEL`. Run `examples/03_dac_imported_model.py` to see both the gateway route and the direct call. 3. **External providers** (OpenAI, Anthropic) — uncomment the entries and set keys. Mint a virtual key for a team (works for **all** models, with budget and rate limits): @@ -58,7 +58,7 @@ Mint a virtual key for a team (works for **all** models, with budget and rate li curl -X POST http://localhost:4000/key/generate \ -H "Authorization: Bearer $LITELLM_MASTER_KEY" \ -H "Content-Type: application/json" \ - -d '{"key_alias": "team-alpha", "max_budget": 50.0, "rpm_limit": 100, "models": ["auto", "grok-4-fast", "llama-4-scout", "qwen3-dac"]}' + -d '{"key_alias": "team-alpha", "max_budget": 50.0, "rpm_limit": 100, "models": ["auto", "grok-4-fast", "llama-3.3-70b", "gpt-oss-120b-dac"]}' ``` Try it: `python examples/01_basic_chat.py` @@ -69,12 +69,12 @@ Applications call `model="auto"`. The complexity router (LiteLLM ≥ 1.94) score | Tier | Routed to | Typical request | |------|-----------|-----------------| -| SIMPLE | `llama-4-scout` | short factual questions | +| SIMPLE | `llama-3.3-70b` | short factual questions | | MEDIUM | `grok-4-fast` | everyday tasks | | COMPLEX | `grok-4` | long, multi-part, code-heavy | | REASONING | `grok-4-fast-reasoning` | step-by-step logic | -The serving model is returned in the `x-litellm-model` response header. Requests can also *constrain* routing with tags (`enable_tag_filtering`), e.g. `tags: ["oci"]` to guarantee data never leaves OCI even when external providers are configured. +The serving model is returned in the `x-litellm-model-name` response header (the response body's `model` field echoes the alias, e.g. `auto`). Requests can also *constrain* routing with tags (`enable_tag_filtering`), e.g. `tags: ["oci"]` to guarantee data never leaves OCI even when external providers are configured. Try it: `python examples/02_auto_routing.py` diff --git a/ai/generative-ai-service/litellm-gateway/files/config/config.yaml b/ai/generative-ai-service/litellm-gateway/files/config/config.yaml index af33f047f..1ad2f84c1 100644 --- a/ai/generative-ai-service/litellm-gateway/files/config/config.yaml +++ b/ai/generative-ai-service/litellm-gateway/files/config/config.yaml @@ -88,7 +88,7 @@ model_list: # DAC endpoint OCID*. LiteLLM therefore talks to it with the standard # `openai/` provider - no OCI signing needed. # ========================================================================= - - model_name: qwen3-dac + - model_name: gpt-oss-120b-dac litellm_params: # OCI_DAC_MODEL must be the full string "openai/" # (LiteLLM only resolves os.environ/ when it is the entire value). @@ -97,7 +97,7 @@ model_list: api_key: os.environ/OCI_GENAI_API_KEY tags: ["oci", "dac", "imported"] model_info: - description: "Imported model on a Dedicated AI Cluster via the OpenAI-compatible endpoint." + description: "gpt-oss-120b hosted on a Dedicated AI Cluster via the OpenAI-compatible endpoint." # ========================================================================= # 3. External providers - same gateway, same virtual keys @@ -109,11 +109,13 @@ model_list: # api_key: os.environ/OPENAI_API_KEY # tags: ["external"] - # - model_name: claude-sonnet - # litellm_params: - # model: anthropic/claude-sonnet-5 - # api_key: os.environ/ANTHROPIC_API_KEY - # tags: ["external"] + - model_name: claude-sonnet + litellm_params: + model: anthropic/claude-sonnet-5 + api_key: os.environ/ANTHROPIC_API_KEY + tags: ["external"] + model_info: + description: "Anthropic Claude Sonnet 5 via the Claude API (needs a Console API key with credits)." # ========================================================================= # Stage 2 - Automatic routing. @@ -131,6 +133,28 @@ model_list: MEDIUM: grok-4-fast COMPLEX: grok-4 REASONING: grok-4-fast-reasoning + # --------------------------------------------------------------- + # Router upgrade path (uncomment ONE level; default is the free, + # deterministic keyword scorer — <1ms, $0, auditable): + # + # (a) Teach the scorer your domain vocabulary (still <1ms, $0): + # custom_technical_keywords: ["terraform", "disaster recovery", "rpo", "rto", "failover"] + # + # (b) Deterministic overrides — force a tier on keyword match: + # keyword_tier_rules: + # - keywords: ["disaster recovery", "architecture review"] + # tier: COMPLEX + # + # (c) LLM-judged routing: a small model classifies each request + # ("judge the intellectual difficulty, not the length") with + # structured output. Adds ~0.5s + one cheap call per request; + # falls back to the keyword scorer on timeout/error, and the + # judge's spend is attributed to the calling virtual key. + # classifier_type: llm + # classifier_llm_config: + # model: llama-3.3-70b # judge model — must be in model_list + # timeout_ms: 3000 + # --------------------------------------------------------------- complexity_router_default_model: grok-4-fast router_settings: diff --git a/ai/generative-ai-service/litellm-gateway/files/examples/01_basic_chat.py b/ai/generative-ai-service/litellm-gateway/files/examples/01_basic_chat.py index f0468e354..0afb2e033 100644 --- a/ai/generative-ai-service/litellm-gateway/files/examples/01_basic_chat.py +++ b/ai/generative-ai-service/litellm-gateway/files/examples/01_basic_chat.py @@ -34,9 +34,9 @@ print("Tokens:", response.usage.total_tokens) # Streaming -print("\n[llama-4-scout, streaming] ", end="", flush=True) +print("\n[llama-3.3-70b, streaming] ", end="", flush=True) stream = client.chat.completions.create( - model="llama-4-scout", + model="llama-3.3-70b", messages=[{"role": "user", "content": "Write a haiku about Oracle Cloud."}], stream=True, ) diff --git a/ai/generative-ai-service/litellm-gateway/files/examples/02_auto_routing.py b/ai/generative-ai-service/litellm-gateway/files/examples/02_auto_routing.py index 3d22159eb..b99fbbc40 100644 --- a/ai/generative-ai-service/litellm-gateway/files/examples/02_auto_routing.py +++ b/ai/generative-ai-service/litellm-gateway/files/examples/02_auto_routing.py @@ -6,7 +6,8 @@ tier per request - token count, code presence, reasoning markers and multi-step patterns are scored in sub-millisecond time (see the `auto` entry in config/config.yaml). The model that actually served the request -comes back in the `x-litellm-model` response header. +comes back in the `x-litellm-model-name` response header (the body's +`model` field echoes the alias "auto"). python 02_auto_routing.py """ @@ -20,18 +21,29 @@ api_key=os.getenv("GATEWAY_API_KEY", "sk-change-me-admin-key"), ) +# Prompts verified against the classifier's default weights/boundaries +# (litellm 1.95.0) so each one lands in its intended tier: the scorer keys on +# keyword signals (code terms, technical terms, explicit reasoning markers), +# not on how hard the task *feels* — a prompt with no keyword signals scores +# SIMPLE no matter how elaborate it reads. PROMPTS = { "simple": "What is the capital of Sweden?", + "medium": ( + "Write a SQL query that returns the top ten customers by total order " + "value in the last quarter." + ), "complex": ( - "Design a multi-region disaster recovery architecture for a bank on OCI. " - "Cover RPO/RTO targets, data replication between Frankfurt and Zurich, " - "failover automation, and how you would test it quarterly. " - "Then write Terraform pseudocode for the DNS failover piece." + "Implement a Python function that queries our orders database with SQL, " + "handles connection errors with retry logic, and exposes the result " + "through a REST API endpoint. The architecture is distributed " + "microservices on Kubernetes with strict latency and throughput " + "requirements. Refactor for performance and optimize the query." ), "reasoning": ( - "A farmer has 17 sheep. All but 9 run away, then he buys twice as many as " - "remain, and sells a third of the total. Reason step by step: how many " - "sheep does he have?" + "Think through this step by step and explain your reasoning: " + "A farmer has 17 sheep. All but 9 run away, then he buys twice as many " + "as remain, and sells a third of the total. Analyze this carefully, " + "break down each stage, and conclude with the final count." ), } @@ -41,9 +53,10 @@ messages=[{"role": "user", "content": prompt}], max_tokens=200, ) - routed_to = raw.headers.get("x-litellm-model", "?") + routed_to = raw.headers.get("x-litellm-model-name", "?") + cost = raw.headers.get("x-litellm-response-cost", "?") response = raw.parse() - print(f"[{label:9s}] routed to: {routed_to}") + print(f"[{label:9s}] routed to: {routed_to} (cost ${cost})") print(f" {response.choices[0].message.content[:120]!r}...\n") # You can also pin deployments with tags instead of full auto-routing diff --git a/ai/generative-ai-service/litellm-gateway/files/examples/03_dac_imported_model.py b/ai/generative-ai-service/litellm-gateway/files/examples/03_dac_imported_model.py index cad6a78a4..e4559ffde 100644 --- a/ai/generative-ai-service/litellm-gateway/files/examples/03_dac_imported_model.py +++ b/ai/generative-ai-service/litellm-gateway/files/examples/03_dac_imported_model.py @@ -5,7 +5,7 @@ Models imported into OCI GenAI (e.g. Qwen 3 from Hugging Face) and hosted on a DAC are served behind OCI's OpenAI-compatible endpoint. Two ways in: - A) Through the gateway (recommended) - the `qwen3-dac` entry in + A) Through the gateway (recommended) - the `gpt-oss-120b-dac` entry in config.yaml maps to the DAC endpoint; clients don't need to know any OCIDs or OCI keys. @@ -28,14 +28,14 @@ ) response = gateway.chat.completions.create( - model="qwen3-dac", + model="gpt-oss-120b-dac", messages=[{"role": "user", "content": "Say hello from a Dedicated AI Cluster."}], max_tokens=100, ) print("[via gateway]", response.choices[0].message.content) # --- B) Direct against the OCI OpenAI-compatible endpoint -------------------- -# (this is exactly what the gateway's `qwen3-dac` entry does internally) +# (this is exactly what the gateway's `gpt-oss-120b-dac` entry does internally) if os.getenv("OCI_GENAI_API_KEY"): direct = OpenAI( base_url=os.environ["OCI_COMPAT_API_BASE"], diff --git a/ai/generative-ai-service/litellm-gateway/files/examples/04_guardrails.py b/ai/generative-ai-service/litellm-gateway/files/examples/04_guardrails.py index 3cb8e273a..7d037d29f 100644 --- a/ai/generative-ai-service/litellm-gateway/files/examples/04_guardrails.py +++ b/ai/generative-ai-service/litellm-gateway/files/examples/04_guardrails.py @@ -27,19 +27,24 @@ GUARDRAILS = {"guardrails": ["oci-guardrails"]} # opt-in per request -# 1) PII gets masked before the model ever sees it +# 1) PII gets masked before the model ever sees it. Asking the model to +# repeat the contact details back makes the masking *visible*: the reply +# contains [EMAIL_REDACTED] / [TELEPHONE_NUMBER_REDACTED] where the real +# data was, proving the model never received it. response = client.chat.completions.create( model="grok-4-fast", messages=[ { "role": "user", - "content": "Draft a short meeting invite and send it to anna.svensson@example.com, " - "phone +46 70 123 45 67.", + "content": "First, quote verbatim the email address and phone number exactly as " + "they appear in this message, each on its own line labelled 'Email:' " + "and 'Phone:'. Do this before anything else. Then draft a one-line " + "meeting invite. Contact: anna.svensson@example.com, +46 70 123 45 67.", } ], extra_body=GUARDRAILS, ) -print("[pii-mask] model saw the masked prompt; reply:") +print("[pii-mask] model repeats what it received (note the redacted placeholders):") print(response.choices[0].message.content[:300], "\n") # 2) Prompt injection gets blocked with HTTP 400