Update litellm-gateway: new models

ai/generative-ai-service/litellm-gateway/files/README.md

@@ -48,8 +48,8 @@ Without Postgres, comment out `database_url` in `general_settings` — the gatew
 
 `model_list` exposes three kinds of models behind identical OpenAI semantics:
 
-1. **OCI on-demand models** (`oci/xai.grok-4`, `oci/meta.llama-4-scout...`) through LiteLLM's native OCI provider, signed with your API key.
-2. **Imported models on a Dedicated AI Cluster.** OCI serves these behind its OpenAI-compatible endpoint (`.../20231130/actions/v1`), authenticated with a plain OCI GenAI API key (`sk-...`) where the *model name is the DAC endpoint OCID*. The `qwen3-dac` entry shows the pattern — swap in your own endpoint OCID via `OCI_DAC_MODEL`. Run `examples/03_dac_imported_model.py` to see both the gateway route and the direct call.
+1. **OCI on-demand models** (`oci/xai.grok-4`, `oci/meta.llama-3.3-70b-instruct`, ...) through LiteLLM's native OCI provider, signed with your API key.
+2. **Imported models on a Dedicated AI Cluster.** OCI serves these behind its OpenAI-compatible endpoint (`.../20231130/actions/v1`), authenticated with a plain OCI GenAI API key (`sk-...`) where the *model name is the DAC endpoint OCID*. The `gpt-oss-120b-dac` entry shows the pattern — swap in your own endpoint OCID via `OCI_DAC_MODEL`. Run `examples/03_dac_imported_model.py` to see both the gateway route and the direct call.
 3. **External providers** (OpenAI, Anthropic) — uncomment the entries and set keys.
 
 Mint a virtual key for a team (works for **all** models, with budget and rate limits):
@@ -58,7 +58,7 @@ Mint a virtual key for a team (works for **all** models, with budget and rate li
 curl -X POST http://localhost:4000/key/generate \
   -H "Authorization: Bearer $LITELLM_MASTER_KEY" \
   -H "Content-Type: application/json" \
-  -d '{"key_alias": "team-alpha", "max_budget": 50.0, "rpm_limit": 100, "models": ["auto", "grok-4-fast", "llama-4-scout", "qwen3-dac"]}'
+  -d '{"key_alias": "team-alpha", "max_budget": 50.0, "rpm_limit": 100, "models": ["auto", "grok-4-fast", "llama-3.3-70b", "gpt-oss-120b-dac"]}'
 ```
 
 Try it: `python examples/01_basic_chat.py`
@@ -69,12 +69,12 @@ Applications call `model="auto"`. The complexity router (LiteLLM ≥ 1.94) score
 
 | Tier | Routed to | Typical request |
 |------|-----------|-----------------|
-| SIMPLE | `llama-4-scout` | short factual questions |
+| SIMPLE | `llama-3.3-70b` | short factual questions |
 | MEDIUM | `grok-4-fast` | everyday tasks |
 | COMPLEX | `grok-4` | long, multi-part, code-heavy |
 | REASONING | `grok-4-fast-reasoning` | step-by-step logic |
 
-The serving model is returned in the `x-litellm-model` response header. Requests can also *constrain* routing with tags (`enable_tag_filtering`), e.g. `tags: ["oci"]` to guarantee data never leaves OCI even when external providers are configured.
+The serving model is returned in the `x-litellm-model-name` response header (the response body's `model` field echoes the alias, e.g. `auto`). Requests can also *constrain* routing with tags (`enable_tag_filtering`), e.g. `tags: ["oci"]` to guarantee data never leaves OCI even when external providers are configured.
 
 Try it: `python examples/02_auto_routing.py`
 

ai/generative-ai-service/litellm-gateway/files/config/config.yaml

@@ -88,7 +88,7 @@ model_list:
   # DAC endpoint OCID*. LiteLLM therefore talks to it with the standard
   # `openai/` provider - no OCI signing needed.
   # =========================================================================
-  - model_name: qwen3-dac
+  - model_name: gpt-oss-120b-dac
     litellm_params:
       # OCI_DAC_MODEL must be the full string "openai/<DAC endpoint OCID>"
       # (LiteLLM only resolves os.environ/ when it is the entire value).
@@ -97,7 +97,7 @@ model_list:
       api_key: os.environ/OCI_GENAI_API_KEY
       tags: ["oci", "dac", "imported"]
     model_info:
-      description: "Imported model on a Dedicated AI Cluster via the OpenAI-compatible endpoint."
+      description: "gpt-oss-120b hosted on a Dedicated AI Cluster via the OpenAI-compatible endpoint."
 
   # =========================================================================
   # 3. External providers - same gateway, same virtual keys
@@ -109,11 +109,13 @@ model_list:
   #     api_key: os.environ/OPENAI_API_KEY
   #     tags: ["external"]
 
-  # - model_name: claude-sonnet
-  #   litellm_params:
-  #     model: anthropic/claude-sonnet-5
-  #     api_key: os.environ/ANTHROPIC_API_KEY
-  #     tags: ["external"]
+  - model_name: claude-sonnet
+    litellm_params:
+      model: anthropic/claude-sonnet-5
+      api_key: os.environ/ANTHROPIC_API_KEY
+      tags: ["external"]
+    model_info:
+      description: "Anthropic Claude Sonnet 5 via the Claude API (needs a Console API key with credits)."
 
   # =========================================================================
   # Stage 2 - Automatic routing.
@@ -131,6 +133,28 @@ model_list:
           MEDIUM: grok-4-fast
           COMPLEX: grok-4
           REASONING: grok-4-fast-reasoning
+        # ---------------------------------------------------------------
+        # Router upgrade path (uncomment ONE level; default is the free,
+        # deterministic keyword scorer — <1ms, $0, auditable):
+        #
+        # (a) Teach the scorer your domain vocabulary (still <1ms, $0):
+        # custom_technical_keywords: ["terraform", "disaster recovery", "rpo", "rto", "failover"]
+        #
+        # (b) Deterministic overrides — force a tier on keyword match:
+        # keyword_tier_rules:
+        #   - keywords: ["disaster recovery", "architecture review"]
+        #     tier: COMPLEX
+        #
+        # (c) LLM-judged routing: a small model classifies each request
+        #     ("judge the intellectual difficulty, not the length") with
+        #     structured output. Adds ~0.5s + one cheap call per request;
+        #     falls back to the keyword scorer on timeout/error, and the
+        #     judge's spend is attributed to the calling virtual key.
+        # classifier_type: llm
+        # classifier_llm_config:
+        #   model: llama-3.3-70b      # judge model — must be in model_list
+        #   timeout_ms: 3000
+        # ---------------------------------------------------------------
       complexity_router_default_model: grok-4-fast
 
 router_settings:

ai/generative-ai-service/litellm-gateway/files/examples/01_basic_chat.py

@@ -34,9 +34,9 @@
 print("Tokens:", response.usage.total_tokens)
 
 # Streaming
-print("\n[llama-4-scout, streaming] ", end="", flush=True)
+print("\n[llama-3.3-70b, streaming] ", end="", flush=True)
 stream = client.chat.completions.create(
-    model="llama-4-scout",
+    model="llama-3.3-70b",
     messages=[{"role": "user", "content": "Write a haiku about Oracle Cloud."}],
     stream=True,
 )

ai/generative-ai-service/litellm-gateway/files/examples/02_auto_routing.py

@@ -6,7 +6,8 @@
 tier per request - token count, code presence, reasoning markers and
 multi-step patterns are scored in sub-millisecond time (see the `auto`
 entry in config/config.yaml). The model that actually served the request
-comes back in the `x-litellm-model` response header.
+comes back in the `x-litellm-model-name` response header (the body's
+`model` field echoes the alias "auto").
 
     python 02_auto_routing.py
 """
@@ -20,18 +21,29 @@
     api_key=os.getenv("GATEWAY_API_KEY", "sk-change-me-admin-key"),
 )
 
+# Prompts verified against the classifier's default weights/boundaries
+# (litellm 1.95.0) so each one lands in its intended tier: the scorer keys on
+# keyword signals (code terms, technical terms, explicit reasoning markers),
+# not on how hard the task *feels* — a prompt with no keyword signals scores
+# SIMPLE no matter how elaborate it reads.
 PROMPTS = {
     "simple": "What is the capital of Sweden?",
+    "medium": (
+        "Write a SQL query that returns the top ten customers by total order "
+        "value in the last quarter."
+    ),
     "complex": (
-        "Design a multi-region disaster recovery architecture for a bank on OCI. "
-        "Cover RPO/RTO targets, data replication between Frankfurt and Zurich, "
-        "failover automation, and how you would test it quarterly. "
-        "Then write Terraform pseudocode for the DNS failover piece."
+        "Implement a Python function that queries our orders database with SQL, "
+        "handles connection errors with retry logic, and exposes the result "
+        "through a REST API endpoint. The architecture is distributed "
+        "microservices on Kubernetes with strict latency and throughput "
+        "requirements. Refactor for performance and optimize the query."
     ),
     "reasoning": (
-        "A farmer has 17 sheep. All but 9 run away, then he buys twice as many as "
-        "remain, and sells a third of the total. Reason step by step: how many "
-        "sheep does he have?"
+        "Think through this step by step and explain your reasoning: "
+        "A farmer has 17 sheep. All but 9 run away, then he buys twice as many "
+        "as remain, and sells a third of the total. Analyze this carefully, "
+        "break down each stage, and conclude with the final count."
     ),
 }
 
@@ -41,9 +53,10 @@
         messages=[{"role": "user", "content": prompt}],
         max_tokens=200,
     )
-    routed_to = raw.headers.get("x-litellm-model", "?")
+    routed_to = raw.headers.get("x-litellm-model-name", "?")
+    cost = raw.headers.get("x-litellm-response-cost", "?")
     response = raw.parse()
-    print(f"[{label:9s}] routed to: {routed_to}")
+    print(f"[{label:9s}] routed to: {routed_to}  (cost ${cost})")
     print(f"            {response.choices[0].message.content[:120]!r}...\n")
 
 # You can also pin deployments with tags instead of full auto-routing

ai/generative-ai-service/litellm-gateway/files/examples/03_dac_imported_model.py

@@ -5,7 +5,7 @@
 Models imported into OCI GenAI (e.g. Qwen 3 from Hugging Face) and hosted
 on a DAC are served behind OCI's OpenAI-compatible endpoint. Two ways in:
 
-  A) Through the gateway (recommended) - the `qwen3-dac` entry in
+  A) Through the gateway (recommended) - the `gpt-oss-120b-dac` entry in
      config.yaml maps to the DAC endpoint; clients don't need to know
      any OCIDs or OCI keys.
 
@@ -28,14 +28,14 @@
 )
 
 response = gateway.chat.completions.create(
-    model="qwen3-dac",
+    model="gpt-oss-120b-dac",
     messages=[{"role": "user", "content": "Say hello from a Dedicated AI Cluster."}],
     max_tokens=100,
 )
 print("[via gateway]", response.choices[0].message.content)
 
 # --- B) Direct against the OCI OpenAI-compatible endpoint --------------------
-# (this is exactly what the gateway's `qwen3-dac` entry does internally)
+# (this is exactly what the gateway's `gpt-oss-120b-dac` entry does internally)
 if os.getenv("OCI_GENAI_API_KEY"):
     direct = OpenAI(
         base_url=os.environ["OCI_COMPAT_API_BASE"],

ai/generative-ai-service/litellm-gateway/files/examples/04_guardrails.py

@@ -27,19 +27,24 @@
 
 GUARDRAILS = {"guardrails": ["oci-guardrails"]}  # opt-in per request
 
-# 1) PII gets masked before the model ever sees it
+# 1) PII gets masked before the model ever sees it. Asking the model to
+#    repeat the contact details back makes the masking *visible*: the reply
+#    contains [EMAIL_REDACTED] / [TELEPHONE_NUMBER_REDACTED] where the real
+#    data was, proving the model never received it.
 response = client.chat.completions.create(
     model="grok-4-fast",
     messages=[
         {
             "role": "user",
-            "content": "Draft a short meeting invite and send it to anna.svensson@example.com, "
-                       "phone +46 70 123 45 67.",
+            "content": "First, quote verbatim the email address and phone number exactly as "
+                       "they appear in this message, each on its own line labelled 'Email:' "
+                       "and 'Phone:'. Do this before anything else. Then draft a one-line "
+                       "meeting invite. Contact: anna.svensson@example.com, +46 70 123 45 67.",
         }
     ],
     extra_body=GUARDRAILS,
 )
-print("[pii-mask] model saw the masked prompt; reply:")
+print("[pii-mask] model repeats what it received (note the redacted placeholders):")
 print(response.choices[0].message.content[:300], "\n")
 
 # 2) Prompt injection gets blocked with HTTP 400