Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 23 additions & 5 deletions Autotests/import_knowledge/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ The integration tests start and remove the container themselves through `scripts

## Unit: scripts/import_knowledge.sh

The script reads `EMBEDDING_PROVIDER`, runs `import-knowledge` (or `import-knowledge --local`), and writes a per-provider sentinel under `CHROMA_DB_PATH` so a second start skips the import. The stub on PATH records how `import-knowledge` was called.
The script reads `EMBEDDING_PROVIDER` and `EMBEDDING_MODEL`, runs `import-knowledge --provider openai|asicloud [--model <model>]` (or `import-knowledge --local`), and writes a per-provider sentinel under `CHROMA_DB_PATH` so a second start skips the import. The stub on PATH records how `import-knowledge` was called.

### 1. test_local_runs_import_and_writes_sentinel

Expand All @@ -49,7 +49,7 @@ Local provider runs the import and leaves the local sentinel behind.

OpenAI with `OPENAI_API_KEY` set runs the import and leaves the OpenAI sentinel.

- Checks: exit 0; the stub was called without `--local`; `.import-kb.openai.done` exists.
- Checks: exit 0; the stub was called with `--provider openai`; `.import-kb.openai.done` exists.

### 3. test_openai_without_key_exit1

Expand Down Expand Up @@ -99,23 +99,41 @@ When `import-knowledge` exits non-zero, no sentinel is written, so the next star

- Checks: non-zero exit; the stub was called; `.import-kb.local.done` is absent.

### 11. test_asicloud_with_key_runs_import_and_writes_sentinel

ASICloud with `ASI_API_KEY` set runs the import and leaves the ASICloud sentinel.

- Checks: exit 0; the stub was called with `--provider asicloud`; `.import-kb.asicloud.done` exists.

### 12. test_asicloud_without_key_exit1

ASICloud without a key stops before importing.

- Checks: exit 1; stderr says `ASI_API_KEY is required`; the stub was never called.

### 13. test_embedding_model_is_passed_to_the_import

`EMBEDDING_MODEL` reaches the import as `--model`.

- Checks: exit 0; the stub was called with `--provider asicloud --model BAAI/bge-base-en-v1.5`.

## Integration: container startup

They launch the image through `scripts/omega`, so the real entrypoint runs (nginx, env scrub, import gating).

### 11. test_entrypoint_imports_when_enabled
### 14. test_entrypoint_imports_when_enabled

With `IMPORT_KB_ON_START=1` the entrypoint starts the import on boot.

- Checks: `[import-kb] Running` appears in the container log within 180 s.

### 12. test_entrypoint_skips_when_disabled
### 15. test_entrypoint_skips_when_disabled

With `IMPORT_KB_ON_START=0` the entrypoint never touches import-kb.

- Checks: after a 25 s window no `[import-kb]` line appears in the log.

### 13. test_local_real_import_runs
### 16. test_local_real_import_runs

A real local import runs and lands in the same `chroma_db` the agent reads from.

Expand Down
32 changes: 31 additions & 1 deletion Autotests/import_knowledge/test_import_knowledge.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,9 @@ def env(tmp_path):
e["CHROMA_DB_PATH"] = str(chroma)
e.pop("IMPORT_KB_FORCE", None)
e.pop("OPENAI_API_KEY", None)
e.pop("ASI_API_KEY", None)
e.pop("EMBEDDING_PROVIDER", None)
e.pop("EMBEDDING_MODEL", None)
return {"env": e, "marker": marker, "chroma": chroma}


Expand Down Expand Up @@ -62,7 +64,7 @@ def test_openai_with_key_runs_import_and_writes_sentinel(env):
r = run(env["env"])
assert r.returncode == 0, r.stderr
assert env["marker"].exists()
assert env["marker"].read_text().strip() == ""
assert env["marker"].read_text().strip() == "--provider openai"
assert (env["chroma"] / ".import-kb.openai.done").exists()


Expand Down Expand Up @@ -165,3 +167,31 @@ def test_failed_import_does_not_write_sentinel(env):
assert r.returncode != 0
assert env["marker"].exists()
assert not (env["chroma"] / ".import-kb.local.done").exists()


def test_asicloud_with_key_runs_import_and_writes_sentinel(env):
env["env"]["EMBEDDING_PROVIDER"] = "ASICloud"
env["env"]["ASI_API_KEY"] = "dummy-key"
r = run(env["env"])
assert r.returncode == 0, r.stderr
assert env["marker"].read_text().strip() == "--provider asicloud"
assert (env["chroma"] / ".import-kb.asicloud.done").exists()


def test_asicloud_without_key_exit1(env):
env["env"]["EMBEDDING_PROVIDER"] = "ASICloud"
r = run(env["env"])
assert r.returncode == 1
assert "ASI_API_KEY is required" in r.stderr
assert not env["marker"].exists()


def test_embedding_model_is_passed_to_the_import(env):
env["env"]["EMBEDDING_PROVIDER"] = "ASICloud"
env["env"]["ASI_API_KEY"] = "dummy-key"
env["env"]["EMBEDDING_MODEL"] = "BAAI/bge-base-en-v1.5"
r = run(env["env"])
assert r.returncode == 0, r.stderr
assert env["marker"].read_text().strip() == (
"--provider asicloud --model BAAI/bge-base-en-v1.5"
)
1 change: 1 addition & 0 deletions Autotests/run_mandatory
Original file line number Diff line number Diff line change
Expand Up @@ -41,3 +41,4 @@ unit/test_fileio_verified_writes.py
unit/test_fileio_verified_deletes.py
unit/test_helper_parsing.py
unit/test_openclaw_unit.py
import_knowledge/test_import_knowledge.py
45 changes: 44 additions & 1 deletion Autotests/test_openai_runtime_embeddings.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,18 +3,23 @@
import types
from pathlib import Path

import pytest


REPO_ROOT = Path(__file__).resolve().parents[1]
RAG_MODULE_PATH = REPO_ROOT / "src" / "rag.py"
MEMORY_METTA_PATH = REPO_ROOT / "src" / "memory.metta"


def load_rag_module(monkeypatch, config=None, expected_model="text-embedding-3-large"):
def load_rag_module(monkeypatch, config=None, expected_model="text-embedding-3-large",
error=None):
created_clients = []
settings = {"GATEWAY_URL": "http://gateway:8080", **(config or {})}

class FakeEmbeddings:
def create(self, *, model, input):
if error is not None:
raise error
assert model == expected_model
assert input == ["runtime probe"]
return types.SimpleNamespace(
Expand Down Expand Up @@ -72,6 +77,44 @@ def test_runtime_embedding_uses_the_configured_provider_and_model(monkeypatch):
assert clients[0].base_url == "http://gateway:8080/asicloud/"


def test_runtime_asicloud_without_model_uses_the_asicloud_default(monkeypatch):
rag, clients = load_rag_module(
monkeypatch,
config={"embeddingprovider": "ASICloud"},
expected_model="WhereIsAI/UAE-Large-V1",
)

assert rag.cloud_embed("runtime probe") == [0.1, 0.2, 0.3]
assert clients[0].base_url == "http://gateway:8080/asicloud/"


def test_runtime_empty_model_falls_back_to_the_provider_default(monkeypatch):
rag, _ = load_rag_module(
monkeypatch,
config={"embeddingprovider": "ASICloud", "embeddingModel": ""},
expected_model="WhereIsAI/UAE-Large-V1",
)

assert rag.cloud_embed("runtime probe") == [0.1, 0.2, 0.3]


def test_runtime_embedding_failure_logs_the_provider_error(monkeypatch):
rag, _ = load_rag_module(
monkeypatch,
config={"embeddingprovider": "ASICloud"},
error=Exception("Error code: 400 - {'error': 'Model not found'}"),
)
logged = []
rag.logger = types.SimpleNamespace(
error=lambda message, *args, **kwargs: logged.append(message)
)

with pytest.raises(RuntimeError):
rag.cloud_embed("runtime probe")

assert any("Model not found" in message for message in logged)


def test_memory_metta_routes_openai_embeddings_to_rag_wrapper():
memory_metta = MEMORY_METTA_PATH.read_text(encoding="utf-8")

Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ After start go to https://webchat.quakenet.org/ to communicate with the agent. J

If you are running Omega without Docker and would like to load it with preset knowledge, follow these steps:

1. Set EMBEDDING_PROVIDER in your environment. It can be set to either OpenAI or Local. OpenAI embeddings also require OPENAI_API_KEY to be set in your environment.
1. Set EMBEDDING_PROVIDER in your environment. It can be set to OpenAI, ASICloud or Local. OpenAI embeddings also require OPENAI_API_KEY, and ASICloud embeddings require ASI_API_KEY to be set in your environment.

2. Run:
```
Expand Down
5 changes: 3 additions & 2 deletions config/config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,9 @@ maxHistory: 30000
chromaDbPath: "./chroma_db"
# `Local` (Python-side model) or the id of a provider serving OpenAI-compatible embeddings: `OpenAI`, `ASICloud`
embeddingprovider: Local
# Model asked of a non-`Local` embeddingprovider
embeddingModel: "text-embedding-3-large"
# Model asked of a non-`Local` embeddingprovider; empty means the provider default:
# `text-embedding-3-large` for `OpenAI`, `WhereIsAI/UAE-Large-V1` for `ASICloud`
embeddingModel: ""
# Enable authenticated operator-triggered /memory-export commands (disabled by default).
memoryExportEnabled: false

Expand Down
2 changes: 1 addition & 1 deletion docs/reference-configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ This reads a command-line override via `argk` (`name=value` on the MeTTa command
| `memoryDirectory` | `./repos/Omega/memory` | Directory containing persistent memory files such as `history.metta`. |
| `chromaDbPath` | `./chroma_db` | ChromaDB persistence directory used for memory backup and restore. |
| `embeddingprovider` | `Local` | `Local` (Python-side model), or the id of a provider that serves an OpenAI-compatible `/embeddings` endpoint — `OpenAI` and `ASICloud` are known to. The gateway supplies that provider's key. |
| `embeddingModel` | `text-embedding-3-large` | Model asked of a non-`Local` `embeddingprovider`. |
| `embeddingModel` | empty | Model asked of a non-`Local` `embeddingprovider`. Empty means the provider default: `text-embedding-3-large` for `OpenAI`, `WhereIsAI/UAE-Large-V1` for `ASICloud`. |

## Channels (`src/channels.metta`, `initChannels`)

Expand Down
2 changes: 1 addition & 1 deletion docs/tutorial-01-teaching-memories.md
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ From `src/memory.metta`:
- `maxRecallItems` — how many items `query` returns (default 20).
- `maxEpisodeRecallLines` — how many lines `episodes` returns (default 20).
- `maxHistory` — characters of history fed back into the prompt (default 30000).
- `embeddingprovider` — `OpenAI` or `Local`.
- `embeddingprovider` — `OpenAI`, `ASICloud` or `Local`.

Change any of these by editing the `configure` calls in `initMemory` or passing command-line overrides — see [reference-configuration.md](./reference-configuration.md).

Expand Down
7 changes: 4 additions & 3 deletions entrypoint.sh
Original file line number Diff line number Diff line change
Expand Up @@ -45,11 +45,12 @@ if [[ "${IMPORT_KB_ON_START}" == "1" ]]; then
fi

MEMORY_PORTABILITY_PYTHON='import os
import sys
from config import init_config
from memory_export import create_memory_store
from memory_portability import MemoryTransfer

init_config([])
init_config(sys.argv[1:])
transfer = MemoryTransfer(
transfer_dir="/memory-transfer",
store=create_memory_store(),
Expand All @@ -70,13 +71,13 @@ export MEMORY_PORTABILITY_PYTHON
export PYTHONPATH="${OMEGA_DIR}:${OMEGA_DIR}/src${PYTHONPATH:+:${PYTHONPATH}}"

export MEMORY_PORTABILITY_OPERATION=recover
su nobody -s /bin/sh -c 'exec python3 -c "$MEMORY_PORTABILITY_PYTHON"' \
su nobody -s /bin/sh -c 'exec python3 -c "$MEMORY_PORTABILITY_PYTHON" "$@"' sh "$@" \
|| { echo "Memory import recovery failed. Aborting startup." >&2; exit 1; }

if [[ -n "${MEMORY_IMPORT_FILE:-}" ]]; then
echo "memory_portability: importing ${MEMORY_IMPORT_FILE}"
export MEMORY_PORTABILITY_OPERATION=import
su nobody -s /bin/sh -c 'exec python3 -c "$MEMORY_PORTABILITY_PYTHON"' \
su nobody -s /bin/sh -c 'exec python3 -c "$MEMORY_PORTABILITY_PYTHON" "$@"' sh "$@" \
|| { echo "Memory import failed. Aborting startup." >&2; exit 1; }
echo "memory_portability: import complete"
fi
Expand Down
16 changes: 16 additions & 0 deletions src/embedding_models.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
DEFAULT_MODELS = {
"openai": "text-embedding-3-large",
"asicloud": "WhereIsAI/UAE-Large-V1",
}

DIMENSIONS = {
"text-embedding-3-large": 3072,
"WhereIsAI/UAE-Large-V1": 1024,
}


def embedding_model(provider, configured=None):
configured = str(configured or "").strip()
if configured:
return configured
return DEFAULT_MODELS.get(str(provider).casefold(), DEFAULT_MODELS["openai"])
29 changes: 28 additions & 1 deletion src/memory_export.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from pathlib import Path

from config import config_get_by_key
from embedding_models import DEFAULT_MODELS, DIMENSIONS, embedding_model
from helper import omega_version, projectRootDirectory
from src.logger import get_logger

Expand Down Expand Up @@ -33,6 +34,31 @@ def _resolve_chroma_path() -> Path:
return Path(str(configured)).expanduser().resolve()


def _cloud_embedder(provider, model):
def embed(texts):
from import_knowledge.import_knowledge import embed_batch, init_embeddings

init_embeddings(mode=provider.casefold(), model_name=model)
return embed_batch(texts)

return embed


def _embedding_options():
provider = str(config_get_by_key("embeddingprovider", "Local")).strip()
if provider.casefold() == "local":
return {}
model = embedding_model(provider, config_get_by_key("embeddingModel", ""))
return {
"embed_batch": _cloud_embedder(provider, model),
"embedding_profile": {
"provider": provider,
"model": model,
"vector_dimension": DIMENSIONS.get(model),
},
}


def create_memory_store():
"""Build an import-kb store from Omega's effective configuration."""
from memory_portability.storage import MemoryStore
Expand All @@ -41,6 +67,7 @@ def create_memory_store():
memory_dir=_resolve_memory_dir(),
chroma_path=_resolve_chroma_path(),
collection_name="memories",
**_embedding_options(),
)


Expand All @@ -50,7 +77,7 @@ def _get_transfer():
from memory_portability import MemoryTransfer

embedding_provider = str(config_get_by_key("embeddingprovider", "Local")).strip()
if embedding_provider.casefold() not in {"local", "openai"}:
if embedding_provider.casefold() not in {"local", *DEFAULT_MODELS}:
raise ValueError(f"Unsupported embedding provider: {embedding_provider!r}")

os.environ["EMBEDDING_PROVIDER"] = embedding_provider
Expand Down
5 changes: 3 additions & 2 deletions src/rag.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,12 @@
from lib_llm_ext import initLocalEmbedding, useLocalEmbedding
from src.logger import get_logger
from config import config_get_by_key
from embedding_models import embedding_model

logger = get_logger(__name__)

# --- Constants -----------------------------------------------------------

EMBEDDING_MODEL = "text-embedding-3-large"
COLLECTION_NAME = "memories"
TOP_K = 5
MIN_CHUNK_CHARS = 100
Expand Down Expand Up @@ -140,8 +140,8 @@ def _chunk_markdown(text, filename):

def cloud_embed_batch(texts):
"""Embed a list of texts via an OpenAI-compatible API. Returns list of float vectors."""
model = config_get_by_key("embeddingModel", EMBEDDING_MODEL)
provider = str(config_get_by_key("embeddingprovider", "OpenAI"))
model = embedding_model(provider, config_get_by_key("embeddingModel", ""))
proxy_url = config_get_by_key("GATEWAY_URL")
if proxy_url:
client = openai.OpenAI(base_url=f"{proxy_url.rstrip('/')}/{provider.lower()}/", api_key="unused")
Expand All @@ -150,6 +150,7 @@ def cloud_embed_batch(texts):
try:
resp = client.embeddings.create(model=model, input=texts)
except Exception as e:
logger.error(f"Embedding request failed: provider={provider} model={model}: {e}")
raise RuntimeError(f"Embedding request failed: {e}") from e
return [item.embedding for item in resp.data]

Expand Down
Loading