diff --git a/nodes/src/nodes/store_astra/IGlobal.py b/nodes/src/nodes/store_astra/IGlobal.py index 393e24102f..0823c18827 100644 --- a/nodes/src/nodes/store_astra/IGlobal.py +++ b/nodes/src/nodes/store_astra/IGlobal.py @@ -24,49 +24,36 @@ # ------------------------------------------------------------------------------ # This class controls the data shared between all threads for the task # ------------------------------------------------------------------------------ -from rocketlib import OPEN_MODE -from ai.common.transform import IGlobalTransform +from typing import Any, Dict +from ai.common.store import StoreGlobalBase -class IGlobal(IGlobalTransform): - def beginGlobal(self): - # Are we in config mode or some other mode? - if self.IEndpoint.endpoint.openMode == OPEN_MODE.CONFIG: - # We are going to get a call to configureService but - # we don't actually need to load the driver for that - pass - else: - # Import store definition - even though - from .astra_db import Store - - # Declare store - self.store: Store | None = None - # Get our bag - bag = self.IEndpoint.endpoint.bag +class IGlobal(StoreGlobalBase): + serverName: str = 'astra' - # Get the passed configuration - connConfig = self.getConnConfig() + def _open_store(self, logical_type: str, conn_config: Dict[str, Any], bag: Dict[str, Any]): + """Return the driver's Store, imported lazily so config mode never loads the driver.""" + from .astra_db import Store - # Get the configuration - self.store = Store(self.glb.logicalType, connConfig, bag) - collection = getattr(self.store, 'collection_name', getattr(self.store, 'collection', '')) + return Store(logical_type, conn_config, bag) - # Prefer cloud endpoint if present; otherwise fall back to host[:port]; otherwise logical type - if hasattr(self.store, 'api_endpoint') and self.store.api_endpoint: - identifier = self.store.api_endpoint.rstrip('/') - elif hasattr(self.store, 'host') and getattr(self.store, 'host', ''): - port = getattr(self.store, 'port', None) - identifier = f'{self.store.host}:{port}' if port else self.store.host - else: - identifier = self.glb.logicalType + def _sub_key(self) -> str: + """Return the transform sub-key: endpoint-or-host/collection.""" + store = self.store + collection = getattr(store, 'collection_name', getattr(store, 'collection', '')) - subKey = f'{identifier}/{collection}' + # Prefer cloud endpoint if present; otherwise fall back to host[:port]; otherwise logical type + if getattr(store, 'api_endpoint', ''): + identifier = store.api_endpoint.rstrip('/') + elif getattr(store, 'host', ''): + port = getattr(store, 'port', None) + identifier = f'{store.host}:{port}' if port else store.host + else: + identifier = self.glb.logicalType - # Call the base - super().beginGlobal(subKey) - return + return f'{identifier}/{collection}' - def endGlobal(self): - # Release the index and embeddings - self.store = None + def _probe_connection(self, config: Dict[str, Any]) -> None: + """Astra validates the collection name inside Store.__init__; no save-time probe here.""" + return diff --git a/nodes/src/nodes/store_astra/IInstance.py b/nodes/src/nodes/store_astra/IInstance.py index f9a9124645..9a3391fc29 100644 --- a/nodes/src/nodes/store_astra/IInstance.py +++ b/nodes/src/nodes/store_astra/IInstance.py @@ -24,61 +24,9 @@ # ------------------------------------------------------------------------------ # This class controls the data for each thread of the task # ------------------------------------------------------------------------------ -from rocketlib import Entry -from typing import List from .IGlobal import IGlobal -from ai.common.schema import Doc, Question -from ai.common.transform import IInstanceTransform +from ai.common.store import StoreInstanceBase -class IInstance(IInstanceTransform): +class IInstance(StoreInstanceBase): IGlobal: IGlobal - - def writeQuestions(self, question: Question): - """ - Take a question, perform a search, and writes the results as documents. - """ - # Check if store exists - if self.IGlobal.store is None: - raise Exception('No document store') - - # Dispatch to the search handler - self.IGlobal.store.dispatchSearch(self, question) - - def writeDocuments(self, documents: List[Doc]): - """ - Take a list of documents and adds them to the vector store. - - Any chunks in the database that have the same object id will - be removed - """ - # Check if store exists - if self.IGlobal.store is None: - raise Exception('No document store') - - # Add the document chunks - self.IGlobal.store.addChunks(documents) - - def renderObject(self, object: Entry): - """ - Output the document text to the writeText lane. - """ - - def callback(text: str) -> None: - self.instance.sendText(text) - - # Check if store exists - if self.IGlobal.store is None: - raise Exception('No document store') - - # If we do not have a vectorize flag, or we have not vectorized - # it, allow the next driver to render - if not object.hasVectorBatchId or not object.vectorBatchId: - return - - # Render the data on this object from the store and - # send it to the renderData function - self.IGlobal.store.render(objectId=object.objectId, callback=callback) - - # Stop right here - self.preventDefault() diff --git a/nodes/src/nodes/store_astra/README.md b/nodes/src/nodes/store_astra/README.md index 9378b9d578..caa3f5fea1 100644 --- a/nodes/src/nodes/store_astra/README.md +++ b/nodes/src/nodes/store_astra/README.md @@ -1,6 +1,6 @@ # store_astra -A RocketRide store node that persists embedded documents in DataStax Astra DB and retrieves them by semantic or keyword search. +A RocketRide store node that persists embedded documents in DataStax Astra DB and retrieves them by semantic or keyword search, and exposes search/upsert/delete as agent-callable tools. ## What it does @@ -78,6 +78,20 @@ Set `api_endpoint` to the database's Data API URL and `application_token` to the --- +## Agent tools + +When wired to an agent, the node exposes three tools via `VectorStoreToolMixin`. Each tool is named `.` (defaults: `astra.search`, `astra.upsert`, `astra.delete`). + +| Tool | Key inputs | Description | +| -------- | ------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------ | +| `search` | `query` (required); `top_k` (default 10, max 100); `filter` (optional dict, keys `objectId`/`nodeId`/`parent` are honored) | Semantic search over stored documents; returns content, metadata, and score per result. Falls back to keyword search if semantic search fails. | +| `upsert` | `documents` array, each with `content` and `object_id`; optional `metadata`, `embedding`, and `embedding_model` | Add or update documents. Embeddings are computed automatically via the bound embedding provider, or pre-computed vectors can be supplied. | +| `delete` | `object_ids` (non-empty string array) | Hard-delete documents by object ID. Returns `deleted_count`. | + +Tool calls run on the control plane and do not flow through the pipeline's embedding lanes. Semantic search in the `search` tool and automatic embedding in `upsert` require an embedding provider bound to the node (the `all.embedding` block in its parameters). Without one, those calls return `{"success": false, "error": ...}`. + +--- + diff --git a/nodes/src/nodes/store_astra/services.json b/nodes/src/nodes/store_astra/services.json index 4e23ab6c6f..93438134a3 100644 --- a/nodes/src/nodes/store_astra/services.json +++ b/nodes/src/nodes/store_astra/services.json @@ -13,13 +13,13 @@ // Required: // Class type of the node - what it does // - "classType": ["store"], + "classType": ["store", "tool"], // // Required: // Capabilities are flags that change the behavior of the underlying // engine // - "capabilities": [], + "capabilities": ["invoke"], // // Optional: // Register is either filter, endpoint or ignored if not specified. If the @@ -139,14 +139,16 @@ "title": "Local test server", "api_endpoint": "http://localhost:8080", "application_token": "test-token", - "collection": "ROCKETRIDE" + "collection": "ROCKETRIDE", + "serverName": "astra" }, "cloud": { "mode": "cloud", "title": "Astra DB cloud server", "api_endpoint": "", "application_token": "", - "collection": "ROCKETRIDE" + "collection": "ROCKETRIDE", + "serverName": "astra" } } }, @@ -167,6 +169,13 @@ "title": "Application Token", "description": "Enter the server API application token" }, + "astra_db.serverName": { + "type": "string", + "title": "Tool Server Name", + "description": "Namespace for agent-facing tool names, e.g. 'astra' exposes tools as astra.search / astra.upsert / astra.delete. Change this when running multiple Astra DB nodes in the same pipeline so their tool names do not collide.", + "default": "astra", + "minLength": 1 + }, "astra_db.provider": { "type": "string", "const": "astra_db", @@ -190,7 +199,7 @@ { "section": "Pipe", "title": "Astra DB Vector Store", - "properties": ["astra_db.api_endpoint", "astra_db.application_token", "vector.score", "vector.collection"] + "properties": ["astra_db.api_endpoint", "astra_db.application_token", "vector.score", "vector.collection", "astra_db.serverName"] }, { "section": "Transform", diff --git a/nodes/src/nodes/store_atlas/IGlobal.py b/nodes/src/nodes/store_atlas/IGlobal.py index ddccca83a2..2850a778d6 100644 --- a/nodes/src/nodes/store_atlas/IGlobal.py +++ b/nodes/src/nodes/store_atlas/IGlobal.py @@ -24,57 +24,36 @@ # ------------------------------------------------------------------------------ # This class controls the data shared between all threads for the task # ------------------------------------------------------------------------------ -from rocketlib import OPEN_MODE -from ai.common.transform import IGlobalTransform -from rocketlib import warning -from ai.common.config import Config - - -class IGlobal(IGlobalTransform): - def beginGlobal(self): - # Are we in config mode or some other mode? - if self.IEndpoint.endpoint.openMode == OPEN_MODE.CONFIG: - # We are going to get a call to configureService but - # we don't actually need to load the driver for that - # Validate configuration only during config mode - self.validateConfig() - pass - else: - # Declare store - from . import getStore +import re +from typing import Any, Dict - # Get our bag - bag = self.IEndpoint.endpoint.bag +from ai.common.store import StoreGlobalBase +from rocketlib import warning - # Get the passed configuration - connConfig = self.getConnConfig() - # Get the configuration - Store = getStore() - self.store = Store(self.glb.logicalType, connConfig, bag) +class IGlobal(StoreGlobalBase): + serverName: str = 'atlas' - # Get the info about our store - database = self.store.database - collection = self.store.collection - host = self.store.host + def _open_store(self, logical_type: str, conn_config: Dict[str, Any], bag: Dict[str, Any]): + """Return the driver's Store, resolved lazily via the package hook so config mode never loads the driver.""" + from . import getStore - # Format it into a subKey - subKey = f'{host}/{database}/{collection}' + Store = getStore() + return Store(logical_type, conn_config, bag) - # Call the base - super().beginGlobal(subKey) - return + def _sub_key(self) -> str: + """Return the transform sub-key: host/database/collection.""" + return f'{self.store.host}/{self.store.database}/{self.store.collection}' - def validateConfig(self): + def _probe_connection(self, config: Dict[str, Any]) -> None: """ - Validate MongoDB config at save-time with optional connection testing. + Validate MongoDB config at save-time with lightweight format checks. - Performs lightweight format validation by default. - No syntaxOnly arg per engine expectations. + Performs format validation only (no network call): API key present, + host matches the Atlas ``mongodb+srv`` URI shape, and database/collection + names respect MongoDB naming restrictions. """ try: - # Get config - config = Config.getNodeConfig(self.glb.logicalType, self.glb.connConfig) host = config.get('host') api_key = config.get('apikey') database = config.get('database') @@ -93,11 +72,12 @@ def validateConfig(self): host = host.strip() # Basic MongoDB URI format validation using regex - import re - mongodb_uri_pattern = r'^mongodb\+srv://[^:]+:[^@]+@[a-zA-Z0-9\-]+\.[a-zA-Z0-9]+\.mongodb\.net/\?.*' if not re.match(mongodb_uri_pattern, host): - warning("Host must be a valid MongoDB URI (e.g., 'mongodb+srv://cluster.example.mongodb.net')") + warning( + 'Host must be a valid MongoDB SRV URI ' + "(e.g., 'mongodb+srv://user:pass@cluster.abc12.mongodb.net/?retryWrites=true')" + ) return # Database name validation - format check only @@ -140,7 +120,3 @@ def validateConfig(self): except Exception as e: msg = str(e) warning(msg) - - def endGlobal(self): - # Release the database connection - self.store = None diff --git a/nodes/src/nodes/store_atlas/IInstance.py b/nodes/src/nodes/store_atlas/IInstance.py index 7dff5f0324..9a3391fc29 100644 --- a/nodes/src/nodes/store_atlas/IInstance.py +++ b/nodes/src/nodes/store_atlas/IInstance.py @@ -24,57 +24,9 @@ # ------------------------------------------------------------------------------ # This class controls the data for each thread of the task # ------------------------------------------------------------------------------ -from rocketlib import Entry -from typing import List from .IGlobal import IGlobal -from ai.common.schema import Doc, Question -from ai.common.transform import IInstanceTransform +from ai.common.store import StoreInstanceBase -class IInstance(IInstanceTransform): +class IInstance(StoreInstanceBase): IGlobal: IGlobal - - def writeQuestions(self, question: Question): - """ - Take a question, perform a search, and writes the results as documents. - """ - if self.IGlobal.store is None: - raise Exception('No document store') - # Dispatch to the search handler - self.IGlobal.store.dispatchSearch(self, question) - - def writeDocuments(self, documents: List[Doc]): - """ - Take a list of documents and adds them to the document store. - - Any documents in the database that have the same object id will - be removed - """ - if self.IGlobal.store is None: - raise Exception('No document store') - # Add the document chunks - self.IGlobal.store.addChunks(documents) - - def renderObject(self, object: Entry): - """ - Output the document text to the writeText lane. - """ - - def callback(text: str) -> None: - self.instance.sendText(text) - - # Check it - if self.IGlobal.store is None: - raise Exception('No document store') - - # If we do not have a vectorize flag, or we have not vectorized - # it, allow the next driver to render - if not object.hasVectorBatchId or not object.vectorBatchId: - return - - # Render the data on this object from the store and - # send it to the renderData function - self.IGlobal.store.render(objectId=object.objectId, callback=callback) - - # Stop right here - self.preventDefault() diff --git a/nodes/src/nodes/store_atlas/README.md b/nodes/src/nodes/store_atlas/README.md index de95c6800b..5a455d41bc 100644 --- a/nodes/src/nodes/store_atlas/README.md +++ b/nodes/src/nodes/store_atlas/README.md @@ -1,6 +1,6 @@ # store_atlas -A RocketRide vector store node that stores embedded document chunks in MongoDB Atlas and retrieves them by semantic or keyword search. +A RocketRide vector store node that stores embedded document chunks in MongoDB Atlas and retrieves them by semantic or keyword search, and exposes search/upsert/delete as agent-callable tools. ## What it does @@ -53,6 +53,20 @@ Documents can be marked deleted or active in place via `meta.isDeleted`. Filters --- +## Agent tools + +When wired to an agent, the node exposes three tools via `VectorStoreToolMixin`. Each tool is named `.` (defaults: `atlas.search`, `atlas.upsert`, `atlas.delete`). + +| Tool | Key inputs | Description | +| -------- | ------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------ | +| `search` | `query` (required); `top_k` (default 10, max 100); `filter` (optional dict, keys `objectId`/`nodeId`/`parent` are honored) | Semantic search over stored documents; returns content, metadata, and score per result. Falls back to keyword search if semantic search fails. | +| `upsert` | `documents` array, each with `content` and `object_id`; optional `metadata`, `embedding`, and `embedding_model` | Add or update documents. Embeddings are computed automatically via the bound embedding provider, or pre-computed vectors can be supplied. | +| `delete` | `object_ids` (non-empty string array) | Hard-delete documents by object ID. Returns `deleted_count`. | + +Tool calls run on the control plane and do not flow through the pipeline's embedding lanes. Semantic search in the `search` tool and automatic embedding in `upsert` require an embedding provider bound to the node (the `all.embedding` block in its parameters). Without one, those calls return `{"success": false, "error": ...}`. + +--- + diff --git a/nodes/src/nodes/store_atlas/services.json b/nodes/src/nodes/store_atlas/services.json index 12d804cc43..9f92340484 100644 --- a/nodes/src/nodes/store_atlas/services.json +++ b/nodes/src/nodes/store_atlas/services.json @@ -13,13 +13,13 @@ // Required: // Class type of the node - what it does // - "classType": ["store"], + "classType": ["store", "tool"], // // Required: // Capabilities are flags that change the behavior of the underlying // engine // - "capabilities": [], + "capabilities": ["invoke"], // // Optional: // Register is either filter, endpoint or ignored if not specified. If the @@ -80,7 +80,9 @@ "default": "default", // Defines profiles used with the "profile": key "profiles": { - "default": {} + "default": { + "serverName": "atlas" + } } }, // @@ -96,6 +98,13 @@ "vector.database": { "default": "rocketride_db" }, + "atlas.serverName": { + "type": "string", + "title": "Tool Server Name", + "description": "Namespace for agent-facing tool names, e.g. 'atlas' exposes tools as atlas.search / atlas.upsert / atlas.delete. Change this when running multiple Atlas nodes in the same pipeline so their tool names do not collide.", + "default": "atlas", + "minLength": 1 + }, "atlas.provider": { "type": "string", "const": "atlas", @@ -124,7 +133,7 @@ { "section": "Pipe", "title": "MongoDB Atlas Vector Store", - "properties": ["vector.cloud.host", "vector.apikey", "vector.score", "vector.database", "vector.collection"] + "properties": ["vector.cloud.host", "vector.apikey", "vector.score", "vector.database", "vector.collection", "atlas.serverName"] }, { "section": "Transform", diff --git a/nodes/src/nodes/store_chroma/IGlobal.py b/nodes/src/nodes/store_chroma/IGlobal.py index 4789aab5bd..500eeb8b7f 100644 --- a/nodes/src/nodes/store_chroma/IGlobal.py +++ b/nodes/src/nodes/store_chroma/IGlobal.py @@ -24,90 +24,24 @@ # ------------------------------------------------------------------------------ # This class controls the data shared between all threads for the task # ------------------------------------------------------------------------------ -from rocketlib import OPEN_MODE, warning -from ai.common.config import Config -from ai.common.transform import IGlobalTransform +from typing import Any, Dict +from ai.common.store import StoreGlobalBase -class IGlobal(IGlobalTransform): - serverName: str = 'chroma' - - def beginGlobal(self): - if self.IEndpoint.endpoint.openMode == OPEN_MODE.CONFIG: - # We are going to get a call to configureService but - # we don't actually need to load the driver for that - pass - else: - # Import store definition - even though - from .chroma import Store - - # Declare store - self.store: Store | None = None - - # Get our bag - bag = self.IEndpoint.endpoint.bag - - # Get the passed configuration - connConfig = self.getConnConfig() - - # Resolve the namespace used for agent-facing tool names - # (chroma.search/upsert/delete). Read from the merged config - # so it honors both profile defaults and user overrides. - cfg = Config.getNodeConfig(self.glb.logicalType, connConfig) - resolved_name = cfg.get('serverName') if isinstance(cfg, dict) or hasattr(cfg, 'get') else None - if isinstance(resolved_name, str) and resolved_name.strip(): - self.serverName = resolved_name.strip() - - # Create the loader - self.store = Store(self.glb.logicalType, connConfig, bag) - # Wire an embedder for the control-plane tool path (search/upsert). - # Mirrors autopipe pattern: only instantiate when embedding config present. - self._tool_embedding = None - self.embed_query = None - self.embed_model_name = None - - try: - embed_provider, embed_config = Config.getMultiProviderConfig('embedding', connConfig) - except Exception: - embed_provider, embed_config = None, None - - if embed_provider: - try: - from ai.common.embedding import getEmbedding as _getEmbedding - - self._tool_embedding = _getEmbedding(embed_provider, embed_config, bag) - except Exception as exc: # noqa: BLE001 - warning(f'{self.glb.logicalType}: tool path embedder unavailable: {exc}') - self._tool_embedding = None - - if self._tool_embedding is not None: - from ai.common.schema import Question as _Question, QuestionText as _QuestionText - - def _embed_query(text: str, _emb=self._tool_embedding) -> list: - qt = _QuestionText(text=text) - q = _Question() - q.questions = [qt] - _emb.encodeQuestion(q) - return list(qt.embedding or []) - - self.embed_query = _embed_query - self.embed_model_name = getattr(self._tool_embedding, '_model', None) +class IGlobal(StoreGlobalBase): + serverName: str = 'chroma' - # Get the info about our store - collection = self.store.collection - host = self.store.host - port = self.store.port + def _open_store(self, logical_type: str, conn_config: Dict[str, Any], bag: Dict[str, Any]): + """Return the driver's Store, imported lazily so config mode never loads the driver.""" + from .chroma import Store - # Format it into a subKey - subKey = f'{host}/{port}/{collection}' + return Store(logical_type, conn_config, bag) - # Call the base - super().beginGlobal(subKey) - return + def _sub_key(self) -> str: + """Return the transform sub-key: host/port/collection.""" + return f'{self.store.host}/{self.store.port}/{self.store.collection}' - def endGlobal(self): - self._tool_embedding = None - self.embed_query = None - self.embed_model_name = None - self.store = None + def _probe_connection(self, config: Dict[str, Any]) -> None: + """Chroma has no save-time probe; nothing to validate here.""" + return diff --git a/nodes/src/nodes/store_chroma/IInstance.py b/nodes/src/nodes/store_chroma/IInstance.py index 13a8ac0316..9a3391fc29 100644 --- a/nodes/src/nodes/store_chroma/IInstance.py +++ b/nodes/src/nodes/store_chroma/IInstance.py @@ -24,54 +24,9 @@ # ------------------------------------------------------------------------------ # This class controls the data for each thread of the task # ------------------------------------------------------------------------------ -from rocketlib import Entry -from typing import List from .IGlobal import IGlobal -from ai.common.schema import Doc, Question -from ai.common.store import VectorStoreToolMixin -from ai.common.transform import IInstanceTransform +from ai.common.store import StoreInstanceBase -class IInstance(VectorStoreToolMixin, IInstanceTransform): +class IInstance(StoreInstanceBase): IGlobal: IGlobal - - def writeQuestions(self, question: Question): - """ - Take a question, performs a search, and writes the results as documents. - """ - # Dispatch to the search handler - self.IGlobal.store.dispatchSearch(self, question) - - def writeDocuments(self, documents: List[Doc]): - """ - Take a list of documents and adds them to the vector store. - - Any chunks in the database that have the same object id will - be removed - """ - # Add the document chunks - self.IGlobal.store.addChunks(documents) - - def renderObject(self, object: Entry): - """ - Output all the document text to the writeText lane. - """ - - def callback(text: str) -> None: - self.instance.sendText(text) - - # Check it - if self.IGlobal.store is None: - raise Exception('No document store') - - # If we do not have a vectorize flag, or we have not vectorized - # it, allow the next driver to render - if not object.hasVectorBatchId or not object.vectorBatchId: - return - - # Render the data on this object from the store and - # send it to the renderData function - self.IGlobal.store.render(objectId=object.objectId, callback=callback) - - # Stop right here - self.preventDefault() diff --git a/nodes/src/nodes/store_milvus/IGlobal.py b/nodes/src/nodes/store_milvus/IGlobal.py index 10b6a73b51..db44801ca9 100644 --- a/nodes/src/nodes/store_milvus/IGlobal.py +++ b/nodes/src/nodes/store_milvus/IGlobal.py @@ -26,19 +26,32 @@ # ------------------------------------------------------------------------------ import os import re -from rocketlib import OPEN_MODE, warning -from ai.common.config import Config -from ai.common.transform import IGlobalTransform +from typing import Any, Dict +from ai.common.store import StoreGlobalBase +from rocketlib import warning -class IGlobal(IGlobalTransform): + +class IGlobal(StoreGlobalBase): """Global interface for the Milvus node.""" + serverName: str = 'milvus' + # Precompiled rules for Milvus collection names NAME_PATTERN = re.compile(r'^[A-Za-z_][A-Za-z0-9_]{0,254}$') RULES_TEXT = 'Rules: 1) length 1–255, 2) Letters (A–Z, a–z), digits (0–9), and underscores only, 3) start with a letter or underscore, 4) subsequent characters may be letters, digits, or underscores.' - def validateConfig(self): + def _open_store(self, logical_type: str, conn_config: Dict[str, Any], bag: Dict[str, Any]): + """Return the driver's Store, imported lazily so config mode never loads the driver.""" + from .milvus import Store + + return Store(logical_type, conn_config, bag) + + def _sub_key(self) -> str: + """Return the transform sub-key: host/port/collection.""" + return f'{self.store.host}/{self.store.port}/{self.store.collection}' + + def _probe_connection(self, config: Dict[str, Any]) -> None: """Validate Milvus configuration at save-time with a minimal probe.""" try: from depends import depends # type: ignore @@ -50,7 +63,6 @@ def validateConfig(self): from pymilvus import connections, utility, exceptions as milvus_exc # Read config - config = Config.getNodeConfig(self.glb.logicalType, self.glb.connConfig) host_raw = config.get('host', '').strip() token = (config.get('apikey') or '').strip() # cloud # Use port exactly as provided by UI (let SDK validate) @@ -93,48 +105,6 @@ def validateConfig(self): warning(str(e)) return - def beginGlobal(self): - """Initialize global resources for the Milvus node.""" - # Are we in config mode or some other mode? - if self.IEndpoint.endpoint.openMode == OPEN_MODE.CONFIG: - # We are going to get a call to configureService but - # we don't actually need to load the driver for that - pass - else: - # Import store definition - even though - from .milvus import Store - - # Declare store - self.store: Store | None = None - - # Get our bag - bag = self.IEndpoint.endpoint.bag - - # The parameters are in different places based on whether we - # are a filter or an endpoint. If we are a filter, they are in - # glb.connConfig, otherwise the are in IEndpoint.endpoint.parameters - connConfig = self.getConnConfig() - - # Get the configuration - self.store = Store(self.glb.logicalType, connConfig, bag) - - # Get the info about our store - collection = self.store.collection - host = self.store.host - port = self.store.port - - # Format it into a subKey - subKey = f'{host}/{port}/{collection}' - - # Call the base - super().beginGlobal(subKey) - return - - def endGlobal(self): - """Release global resources for the Milvus node.""" - # Release the index and embeddings - self.store = None - def _format_error(self, exc: Exception) -> str: """Return concise error string for Milvus/driver exceptions.""" try: diff --git a/nodes/src/nodes/store_milvus/IInstance.py b/nodes/src/nodes/store_milvus/IInstance.py index c866e30d7b..9a3391fc29 100644 --- a/nodes/src/nodes/store_milvus/IInstance.py +++ b/nodes/src/nodes/store_milvus/IInstance.py @@ -24,53 +24,9 @@ # ------------------------------------------------------------------------------ # This class controls the data for each thread of the task # ------------------------------------------------------------------------------ -from rocketlib import Entry -from typing import List from .IGlobal import IGlobal -from ai.common.schema import Doc, Question -from ai.common.transform import IInstanceTransform +from ai.common.store import StoreInstanceBase -class IInstance(IInstanceTransform): +class IInstance(StoreInstanceBase): IGlobal: IGlobal - - def writeQuestions(self, question: Question): - """ - Take a question, performs a search, and writes the results as documents. - """ - # Dispatch to the search handler - self.IGlobal.store.dispatchSearch(self, question) - - def writeDocuments(self, documents: List[Doc]): - """ - Take a list of documents and adds them to the vector store. - - Any chunks in the database that have the same object id will - be removed - """ - # Add the document chunks - self.IGlobal.store.addChunks(documents) - - def renderObject(self, object: Entry): - """ - Output all the document text to the writeText lane. - """ - - def callback(text: str) -> None: - self.instance.sendText(text) - - # Check it - if self.IGlobal.store is None: - raise Exception('No document store') - - # If we do not have a vectorize flag, or we have not vectorized - # it, allow the next driver to render - if not object.hasVectorBatchId or not object.vectorBatchId: - return - - # Render the data on this object from the store and - # send it to the renderData function - self.IGlobal.store.render(objectId=object.objectId, callback=callback) - - # Stop right here - self.preventDefault() diff --git a/nodes/src/nodes/store_milvus/README.md b/nodes/src/nodes/store_milvus/README.md index 65a062747e..01e1839185 100644 --- a/nodes/src/nodes/store_milvus/README.md +++ b/nodes/src/nodes/store_milvus/README.md @@ -1,6 +1,6 @@ # store_milvus -A RocketRide store node that persists embedded documents in a Milvus vector database and retrieves them by semantic or keyword similarity search. +A RocketRide store node that persists embedded documents in a Milvus vector database and retrieves them by semantic or keyword similarity search, and exposes search/upsert/delete as agent-callable tools. ## What it does @@ -82,6 +82,20 @@ metric, and an `STL_SORT` scalar index on `id`. --- +## Agent tools + +When wired to an agent, the node exposes three tools via `VectorStoreToolMixin`. Each tool is named `.` (defaults: `milvus.search`, `milvus.upsert`, `milvus.delete`). + +| Tool | Key inputs | Description | +| -------- | ------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------ | +| `search` | `query` (required); `top_k` (default 10, max 100); `filter` (optional dict, keys `objectId`/`nodeId`/`parent` are honored) | Semantic search over stored documents; returns content, metadata, and score per result. Falls back to keyword search if semantic search fails. | +| `upsert` | `documents` array, each with `content` and `object_id`; optional `metadata`, `embedding`, and `embedding_model` | Add or update documents. Embeddings are computed automatically via the bound embedding provider, or pre-computed vectors can be supplied. | +| `delete` | `object_ids` (non-empty string array) | Hard-delete documents by object ID. Returns `deleted_count`. | + +Tool calls run on the control plane and do not flow through the pipeline's embedding lanes. Semantic search in the `search` tool and automatic embedding in `upsert` require an embedding provider bound to the node (the `all.embedding` block in its parameters). Without one, those calls return `{"success": false, "error": ...}`. + +--- + ## Search behavior **Semantic search** requires the question to carry an embedding (bind the pipeline to an diff --git a/nodes/src/nodes/store_milvus/services.json b/nodes/src/nodes/store_milvus/services.json index 357026e576..128cede5c3 100644 --- a/nodes/src/nodes/store_milvus/services.json +++ b/nodes/src/nodes/store_milvus/services.json @@ -13,13 +13,13 @@ // Required: // Class type of the node - what it does // - "classType": ["store"], + "classType": ["store", "tool"], // // Required: // Capabilities are flags that change the behavior of the underlying // engine // - "capabilities": [], + "capabilities": ["invoke"], // // Optional: // Register is either filter, endpoint or ignored if not specified. If the @@ -140,7 +140,8 @@ "title": "Your own Milvus server", "host": "localhost", "port": 19530, - "collection": "ROCKETRIDE" + "collection": "ROCKETRIDE", + "serverName": "milvus" }, "cloud": { "mode": "cloud", @@ -148,7 +149,8 @@ "host": "", "port": 443, "apikey": "", - "collection": "ROCKETRIDE" + "collection": "ROCKETRIDE", + "serverName": "milvus" } } }, @@ -171,13 +173,20 @@ "vector.local.port": { "default": 19530 }, + "milvus.serverName": { + "type": "string", + "title": "Tool Server Name", + "description": "Namespace for agent-facing tool names, e.g. 'milvus' exposes tools as milvus.search / milvus.upsert / milvus.delete. Change this when running multiple Milvus nodes in the same pipeline so their tool names do not collide.", + "default": "milvus", + "minLength": 1 + }, "milvus.cloud": { "object": "cloud", - "properties": ["vector.cloud.host", "vector.cloud.port", "vector.apikey", "vector.score", "vector.collection"] + "properties": ["vector.cloud.host", "vector.cloud.port", "vector.apikey", "vector.score", "vector.collection", "milvus.serverName"] }, "milvus.local": { "object": "local", - "properties": ["vector.local.host", "vector.local.port", "vector.score", "vector.collection"] + "properties": ["vector.local.host", "vector.local.port", "vector.score", "vector.collection", "milvus.serverName"] }, "milvus.profile": { "title": "Type of Milvus host", diff --git a/nodes/src/nodes/store_pinecone/IGlobal.py b/nodes/src/nodes/store_pinecone/IGlobal.py index d3f81e4c1d..c6401ffc75 100644 --- a/nodes/src/nodes/store_pinecone/IGlobal.py +++ b/nodes/src/nodes/store_pinecone/IGlobal.py @@ -27,99 +27,28 @@ import os import re +from typing import Any, Dict -from .pinecone import Store - -from rocketlib import OPEN_MODE, warning -from ai.common.transform import IGlobalTransform -from ai.common.config import Config +from ai.common.store import StoreGlobalBase +from rocketlib import warning HTTP_BODY_MARKER = 'http response body:' -class IGlobal(IGlobalTransform): - # Class attributes - properly defined for IDE support - store: 'Store | None' = None +class IGlobal(StoreGlobalBase): serverName: str = 'pinecone' - def beginGlobal(self): - """ - Initialize Pinecone store connection and set up global resources. - - Creates store instance and configures subkey for the base transform. - """ - # Are we in config mode or some other mode? - if self.IEndpoint.endpoint.openMode == OPEN_MODE.CONFIG: - # We are going to get a call to configureService but - # we don't actually need to load the driver for that - pass - else: - # Import store definition - even though - from .pinecone import Store - - # Get our bag - bag = self.IEndpoint.endpoint.bag - - # Get the passed configuration - connConfig = self.getConnConfig() - - # Resolve the namespace used for agent-facing tool names - # (pinecone.search/upsert/delete). Read from the merged config - # so it honors both profile defaults and user overrides. - cfg = Config.getNodeConfig(self.glb.logicalType, connConfig) - resolved_name = cfg.get('serverName') if isinstance(cfg, dict) or hasattr(cfg, 'get') else None - if isinstance(resolved_name, str) and resolved_name.strip(): - self.serverName = resolved_name.strip() - - # Get the configuration - self.store = Store(self.glb.logicalType, connConfig, bag) - - # Wire an embedder for the control-plane tool path (search/upsert). - # Mirrors autopipe pattern: only instantiate when embedding config present. - self._tool_embedding = None - self.embed_query = None - self.embed_model_name = None - - try: - embed_provider, embed_config = Config.getMultiProviderConfig('embedding', connConfig) - except Exception: - embed_provider, embed_config = None, None - - if embed_provider: - try: - from ai.common.embedding import getEmbedding as _getEmbedding - - self._tool_embedding = _getEmbedding(embed_provider, embed_config, bag) - except Exception as exc: # noqa: BLE001 - warning(f'{self.glb.logicalType}: tool path embedder unavailable: {exc}') - self._tool_embedding = None + def _open_store(self, logical_type: str, conn_config: Dict[str, Any], bag: Dict[str, Any]): + """Return the driver's Store, imported lazily so config mode never loads the driver.""" + from .pinecone import Store - if self._tool_embedding is not None: - from ai.common.schema import Question as _Question, QuestionText as _QuestionText + return Store(logical_type, conn_config, bag) - def _embed_query(text: str, _emb=self._tool_embedding) -> list: - qt = _QuestionText(text=text) - q = _Question() - q.questions = [qt] - _emb.encodeQuestion(q) - return list(qt.embedding or []) + def _sub_key(self) -> str: + """Return the transform sub-key: host/port/collection.""" + return f'{self.store.host}/{self.store.port}/{self.store.collection}' - self.embed_query = _embed_query - self.embed_model_name = getattr(self._tool_embedding, '_model', None) - - # Get the info about our store - collection = self.store.collection - host = self.store.host - port = self.store.port - - # Format it into a subKey - subKey = f'{host}/{port}/{collection}' - - # Call the base - super().beginGlobal(subKey) - return - - def validateConfig(self): + def _probe_connection(self, config: Dict[str, Any]) -> None: """ Validate the configuration for Pinecone vector store. @@ -136,8 +65,6 @@ def validateConfig(self): # Use HTTP client for validation to surface structured ApiException with status/body from pinecone import Pinecone - # Get config - config = Config.getNodeConfig(self.glb.logicalType, self.glb.connConfig) apikey = config.get('apikey') collection = config.get('collection') mode = config.get('mode') # pod-based or serverless-dense @@ -225,12 +152,3 @@ def validateConfig(self): else: warning(error_str.strip()) return - - def endGlobal(self): - """ - Clean up global resources and release Pinecone store connection. - """ - self._tool_embedding = None - self.embed_query = None - self.embed_model_name = None - self.store = None diff --git a/nodes/src/nodes/store_pinecone/IInstance.py b/nodes/src/nodes/store_pinecone/IInstance.py index 13a8ac0316..9a3391fc29 100644 --- a/nodes/src/nodes/store_pinecone/IInstance.py +++ b/nodes/src/nodes/store_pinecone/IInstance.py @@ -24,54 +24,9 @@ # ------------------------------------------------------------------------------ # This class controls the data for each thread of the task # ------------------------------------------------------------------------------ -from rocketlib import Entry -from typing import List from .IGlobal import IGlobal -from ai.common.schema import Doc, Question -from ai.common.store import VectorStoreToolMixin -from ai.common.transform import IInstanceTransform +from ai.common.store import StoreInstanceBase -class IInstance(VectorStoreToolMixin, IInstanceTransform): +class IInstance(StoreInstanceBase): IGlobal: IGlobal - - def writeQuestions(self, question: Question): - """ - Take a question, performs a search, and writes the results as documents. - """ - # Dispatch to the search handler - self.IGlobal.store.dispatchSearch(self, question) - - def writeDocuments(self, documents: List[Doc]): - """ - Take a list of documents and adds them to the vector store. - - Any chunks in the database that have the same object id will - be removed - """ - # Add the document chunks - self.IGlobal.store.addChunks(documents) - - def renderObject(self, object: Entry): - """ - Output all the document text to the writeText lane. - """ - - def callback(text: str) -> None: - self.instance.sendText(text) - - # Check it - if self.IGlobal.store is None: - raise Exception('No document store') - - # If we do not have a vectorize flag, or we have not vectorized - # it, allow the next driver to render - if not object.hasVectorBatchId or not object.vectorBatchId: - return - - # Render the data on this object from the store and - # send it to the renderData function - self.IGlobal.store.render(objectId=object.objectId, callback=callback) - - # Stop right here - self.preventDefault() diff --git a/nodes/src/nodes/store_postgres/IGlobal.py b/nodes/src/nodes/store_postgres/IGlobal.py index f84db5eff2..ae811cc313 100644 --- a/nodes/src/nodes/store_postgres/IGlobal.py +++ b/nodes/src/nodes/store_postgres/IGlobal.py @@ -26,50 +26,29 @@ # ------------------------------------------------------------------------------ import os import re -from rocketlib import OPEN_MODE, warning -from ai.common.transform import IGlobalTransform -from ai.common.config import Config +from typing import Any, Dict + +from ai.common.store import StoreGlobalBase +from rocketlib import warning # Valid unquoted PostgreSQL identifier: letter/underscore start, alphanumeric/underscore body, max 63 chars VALID_TABLE = re.compile(r'^[A-Za-z_][A-Za-z0-9_]{0,62}$') -class IGlobal(IGlobalTransform): - def beginGlobal(self): - # Are we in config mode or some other mode? - if self.IEndpoint.endpoint.openMode == OPEN_MODE.CONFIG: - # We are going to get a call to configureService but - # we don't actually need to load the driver for that - pass - else: - # Import store definition - even though - from .postgres import Store - - # Declare store - self.store: Store | None = None - - # Get our bag - bag = self.IEndpoint.endpoint.bag +class IGlobal(StoreGlobalBase): + serverName: str = 'postgres' - # Get the passed configuration - connConfig = self.getConnConfig() + def _open_store(self, logical_type: str, conn_config: Dict[str, Any], bag: Dict[str, Any]): + """Return the driver's Store, imported lazily so config mode never loads the driver.""" + from .postgres import Store - # Get the configuration - self.store = Store(self.glb.logicalType, connConfig, bag) + return Store(logical_type, conn_config, bag) - # Get the info about our store - collection = self.store.collection - host = self.store.host - port = self.store.port + def _sub_key(self) -> str: + """Return the transform sub-key: host/port/collection.""" + return f'{self.store.host}/{self.store.port}/{self.store.collection}' - # Format it into a subKey - subKey = f'{host}/{port}/{collection}' - - # Call the base - super().beginGlobal(subKey) - return - - def validateConfig(self): + def _probe_connection(self, config: Dict[str, Any]) -> None: """ Validate PostgreSQL vector store config with fast, read-only probes. @@ -79,14 +58,13 @@ def validateConfig(self): - Surface raw provider errors without truncation """ try: - cfg = Config.getNodeConfig(self.glb.logicalType, self.glb.connConfig) - host = (cfg.get('host') or '').strip() - port = cfg.get('port') - user = cfg.get('user') - password = cfg.get('password') - database = (cfg.get('database') or 'postgres').strip() + host = (config.get('host') or '').strip() + port = config.get('port') + user = config.get('user') + password = config.get('password') + database = (config.get('database') or 'postgres').strip() # Support legacy key 'collection' if UI still sends it - table = ((cfg.get('table') or cfg.get('collection')) or '').strip() + table = ((config.get('table') or config.get('collection')) or '').strip() # Table name validation: unquoted identifier rules, total ≤63 chars if not VALID_TABLE.fullmatch(table or ''): @@ -129,10 +107,6 @@ def validateConfig(self): except Exception as e: warning(_format_error(e)) - def endGlobal(self): - # Release the index and embeddings - self.store = None - def _format_error(e: Exception) -> str: """Concise error string for psycopg2 exceptions. diff --git a/nodes/src/nodes/store_postgres/IInstance.py b/nodes/src/nodes/store_postgres/IInstance.py index 6fb840276d..9a3391fc29 100644 --- a/nodes/src/nodes/store_postgres/IInstance.py +++ b/nodes/src/nodes/store_postgres/IInstance.py @@ -24,57 +24,9 @@ # ------------------------------------------------------------------------------ # This class controls the data for each thread of the task # ------------------------------------------------------------------------------ -from rocketlib import Entry -from typing import List from .IGlobal import IGlobal -from ai.common.schema import Doc, Question -from ai.common.transform import IInstanceTransform +from ai.common.store import StoreInstanceBase -class IInstance(IInstanceTransform): +class IInstance(StoreInstanceBase): IGlobal: IGlobal - - def writeQuestions(self, question: Question): - """ - Take a question, performs a search, and writes the results as documents. - """ - # Dispatch to the search handler - if self.IGlobal.store is None: - raise Exception('No document store') - self.IGlobal.store.dispatchSearch(self, question) - - def writeDocuments(self, documents: List[Doc]): - """ - Take a list of documents and adds them to the vector store. - - Any chunks in the database that have the same object id will - be removed - """ - # Add the document chunks - if self.IGlobal.store is None: - raise Exception('No document store') - self.IGlobal.store.addChunks(documents) - - def renderObject(self, object: Entry): - """ - Output all the document text to the writeText lane. - """ - - def callback(text: str) -> None: - self.instance.sendText(text) - - # Check it - if self.IGlobal.store is None: - raise Exception('No document store') - - # If we do not have a vectorize flag, or we have not vectorized - # it, allow the next driver to render - if not object.hasVectorBatchId or not object.vectorBatchId: - return - - # Render the data on this object from the store and - # send it to the renderData function - self.IGlobal.store.render(objectId=object.objectId, callback=callback) - - # Stop right here - self.preventDefault() diff --git a/nodes/src/nodes/store_postgres/README.md b/nodes/src/nodes/store_postgres/README.md index 341e77b12e..7c465c4e11 100644 --- a/nodes/src/nodes/store_postgres/README.md +++ b/nodes/src/nodes/store_postgres/README.md @@ -1,6 +1,6 @@ # store_postgres -A RocketRide store node that keeps document embeddings in PostgreSQL with the pgvector extension and retrieves them by semantic or keyword search. +A RocketRide store node that keeps document embeddings in PostgreSQL with the pgvector extension and retrieves them by semantic or keyword search, and exposes search/upsert/delete as agent-callable tools. ## What it does @@ -98,6 +98,20 @@ if the extension is missing or the connection fails. --- +## Agent tools + +When wired to an agent, the node exposes three tools via `VectorStoreToolMixin`. Each tool is named `.` (defaults: `postgres.search`, `postgres.upsert`, `postgres.delete`). + +| Tool | Key inputs | Description | +| -------- | ------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------ | +| `search` | `query` (required); `top_k` (default 10, max 100); `filter` (optional dict, keys `objectId`/`nodeId`/`parent` are honored) | Semantic search over stored documents; returns content, metadata, and score per result. Falls back to keyword search if semantic search fails. | +| `upsert` | `documents` array, each with `content` and `object_id`; optional `metadata`, `embedding`, and `embedding_model` | Add or update documents. Embeddings are computed automatically via the bound embedding provider, or pre-computed vectors can be supplied. | +| `delete` | `object_ids` (non-empty string array) | Hard-delete documents by object ID. Returns `deleted_count`. | + +Tool calls run on the control plane and do not flow through the pipeline's embedding lanes. Semantic search in the `search` tool and automatic embedding in `upsert` require an embedding provider bound to the node (the `all.embedding` block in its parameters). Without one, those calls return `{"success": false, "error": ...}`. + +--- + diff --git a/nodes/src/nodes/store_postgres/services.json b/nodes/src/nodes/store_postgres/services.json index 973295f8ec..9a909cf673 100644 --- a/nodes/src/nodes/store_postgres/services.json +++ b/nodes/src/nodes/store_postgres/services.json @@ -13,13 +13,13 @@ // Required: // Class type of the node - what it does // - "classType": ["store"], + "classType": ["store", "tool"], // // Required: // Capabilities are flags that change the behavior of the underlying // engine // - "capabilities": [], + "capabilities": ["invoke"], // // Optional: // Register is either filter, endpoint or ignored if not specified. If the @@ -142,7 +142,8 @@ "database": "rocketride", "collection": "rocketride", "score": 0.5, - "similarity": "cosine" + "similarity": "cosine", + "serverName": "postgres" } } }, @@ -195,9 +196,16 @@ "default": "cosine", "enum": ["cosine", "l2", "inner_product"] }, + "postgres.serverName": { + "type": "string", + "title": "Tool Server Name", + "description": "Namespace for agent-facing tool names, e.g. 'postgres' exposes tools as postgres.search / postgres.upsert / postgres.delete. Change this when running multiple Postgres nodes in the same pipeline so their tool names do not collide.", + "default": "postgres", + "minLength": 1 + }, "postgres.local": { "object": "local", - "properties": ["vector.local.host", "vector.local.port", "vector.local.user", "vector.local.password", "vector.local.database", "vector.collection", "vector.score", "vector.similarity"] + "properties": ["vector.local.host", "vector.local.port", "vector.local.user", "vector.local.password", "vector.local.database", "vector.collection", "vector.score", "vector.similarity", "postgres.serverName"] }, "postgres.profile": { "hidden": true, diff --git a/nodes/src/nodes/store_qdrant/IGlobal.py b/nodes/src/nodes/store_qdrant/IGlobal.py index d35407ebe8..9a5735251d 100644 --- a/nodes/src/nodes/store_qdrant/IGlobal.py +++ b/nodes/src/nodes/store_qdrant/IGlobal.py @@ -26,9 +26,9 @@ # ------------------------------------------------------------------------------ import os import re -from ai.common.config import Config -from ai.common.transform import IGlobalTransform -from rocketlib import OPEN_MODE +from typing import Any, Dict + +from ai.common.store import StoreGlobalBase from rocketlib import warning @@ -36,85 +36,23 @@ QDRANT_COLLECTION_RE = re.compile(r'^[A-Za-z0-9._-]{1,255}$') -class IGlobal(IGlobalTransform): +class IGlobal(StoreGlobalBase): serverName: str = 'qdrant' - def beginGlobal(self): - # Are we in config mode or some other mode? - if self.IEndpoint.endpoint.openMode == OPEN_MODE.CONFIG: - # We are going to get a call to configureService but - # we don't actually need to load the driver for that - pass - else: - # Import store definition - even though - from .qdrant import Store - - # Declare store - self.store: Store | None = None - - # Get our bag - bag = self.IEndpoint.endpoint.bag - - # Get the passed configuration - connConfig = self.getConnConfig() - - # Resolve the namespace used for agent-facing tool names - # (qdrant.search/upsert/delete). Read from the merged config - # so it honors both profile defaults and user overrides. - cfg = Config.getNodeConfig(self.glb.logicalType, connConfig) - resolved_name = cfg.get('serverName') if isinstance(cfg, dict) or hasattr(cfg, 'get') else None - if isinstance(resolved_name, str) and resolved_name.strip(): - self.serverName = resolved_name.strip() - - # Get the configuration - self.store = Store(self.glb.logicalType, connConfig, bag) - - # Wire an embedder for the control-plane tool path (search/upsert). - # Mirrors autopipe pattern: only instantiate when embedding config present. - self._tool_embedding = None - self.embed_query = None - self.embed_model_name = None - - try: - embed_provider, embed_config = Config.getMultiProviderConfig('embedding', connConfig) - except Exception: - embed_provider, embed_config = None, None - - if embed_provider: - try: - from ai.common.embedding import getEmbedding as _getEmbedding - - self._tool_embedding = _getEmbedding(embed_provider, embed_config, bag) - except Exception as exc: # noqa: BLE001 - warning(f'{self.glb.logicalType}: tool path embedder unavailable: {exc}') - self._tool_embedding = None - - if self._tool_embedding is not None: - from ai.common.schema import Question as _Question, QuestionText as _QuestionText - - def _embed_query(text: str, _emb=self._tool_embedding) -> list: - qt = _QuestionText(text=text) - q = _Question() - q.questions = [qt] - _emb.encodeQuestion(q) - return list(qt.embedding or []) + def _open_store(self, logical_type: str, conn_config: Dict[str, Any], bag: Dict[str, Any]): + """Return the driver's Store, imported lazily so config mode never loads the driver.""" + from .qdrant import Store - self.embed_query = _embed_query - self.embed_model_name = getattr(self._tool_embedding, '_model', None) + return Store(logical_type, conn_config, bag) - # Get the info about our store - collection = self.store.collection - host = self.store.host - port = self.store.port + def _sub_key(self) -> str: + """Return the transform sub-key: host/port/collection.""" + collection = self.store.collection + host = self.store.host + port = self.store.port + return f'{host}/{port}/{collection}' - # Format it into a subKey - subKey = f'{host}/{port}/{collection}' - - # Call the base - super().beginGlobal(subKey) - return - - def validateConfig(self): + def _probe_connection(self, config: Dict[str, Any]) -> None: """ Validate Qdrant config at save-time with a fast SDK probe. @@ -124,8 +62,6 @@ def validateConfig(self): - Surface concise SDK/HTTP errors via a unified formatter """ try: - # Load configuration - config = Config.getNodeConfig(self.glb.logicalType, self.glb.connConfig) host = (config.get('host', '')).strip() port = config.get('port') apikey = (config.get('apikey', '')).strip() @@ -178,12 +114,6 @@ def validateConfig(self): except Exception as e: warning(_format_error(e)) - def endGlobal(self): - self._tool_embedding = None - self.embed_query = None - self.embed_model_name = None - self.store = None - def _format_error(e: Exception) -> str: """Concise error string for qdrant's httpx-based exceptions. diff --git a/nodes/src/nodes/store_qdrant/IInstance.py b/nodes/src/nodes/store_qdrant/IInstance.py index dabcffe40e..9a3391fc29 100644 --- a/nodes/src/nodes/store_qdrant/IInstance.py +++ b/nodes/src/nodes/store_qdrant/IInstance.py @@ -24,54 +24,9 @@ # ------------------------------------------------------------------------------ # This class controls the data for each thread of the task # ------------------------------------------------------------------------------ -from rocketlib import Entry -from typing import List from .IGlobal import IGlobal -from ai.common.schema import Doc, Question -from ai.common.store import VectorStoreToolMixin -from ai.common.transform import IInstanceTransform +from ai.common.store import StoreInstanceBase -class IInstance(VectorStoreToolMixin, IInstanceTransform): +class IInstance(StoreInstanceBase): IGlobal: IGlobal - - def writeQuestions(self, question: Question): - """ - Take a question, perform a search, and writes the results as documents. - """ - # Dispatch to the search handler - self.IGlobal.store.dispatchSearch(self, question) - - def writeDocuments(self, documents: List[Doc]): - """ - Take a list of documents and adds them to the vector store. - - Any chunks in the database that have the same object id will - be removed - """ - # Add the document chunks - self.IGlobal.store.addChunks(documents) - - def renderObject(self, object: Entry): - """ - Output the document text to the writeText lane. - """ - - def callback(text: str) -> None: - self.instance.sendText(text) - - # Check it - if self.IGlobal.store is None: - raise Exception('No document store') - - # If we do not have a vectorize flag, or we have not vectorized - # it, allow the next driver to render - if not object.hasVectorBatchId or not object.vectorBatchId: - return - - # Render the data on this object from the store and - # send it to the renderData function - self.IGlobal.store.render(objectId=object.objectId, callback=callback) - - # Stop right here - self.preventDefault() diff --git a/nodes/src/nodes/store_weaviate/IGlobal.py b/nodes/src/nodes/store_weaviate/IGlobal.py index 3665cfd88f..02606fb291 100644 --- a/nodes/src/nodes/store_weaviate/IGlobal.py +++ b/nodes/src/nodes/store_weaviate/IGlobal.py @@ -21,11 +21,14 @@ # SOFTWARE. # ============================================================================= +# ------------------------------------------------------------------------------ +# This class controls the data shared between all threads for the task +# ------------------------------------------------------------------------------ import os import re -from ai.common.config import Config -from ai.common.transform import IGlobalTransform -from rocketlib import OPEN_MODE +from typing import Any, Dict + +from ai.common.store import StoreGlobalBase from rocketlib import warning @@ -33,42 +36,20 @@ WEAVIATE_COLLECTION_RE = re.compile(r'^[A-Z][_0-9A-Za-z]*$') -class IGlobal(IGlobalTransform): - def beginGlobal(self): - # Are we in config mode or some other mode? - if self.IEndpoint.endpoint.openMode == OPEN_MODE.CONFIG: - # We are going to get a call to configureService but - # we don't actually need to load the driver for that - pass - else: - # Import store definition - even though - from .weaviate import Store - - # Declare store - self.store: Store | None = None - - # Get our bag - bag = self.IEndpoint.endpoint.bag +class IGlobal(StoreGlobalBase): + serverName: str = 'weaviate' - # Get the passed configuration - connConfig = self.getConnConfig() + def _open_store(self, logical_type: str, conn_config: Dict[str, Any], bag: Dict[str, Any]): + """Return the driver's Store, imported lazily so config mode never loads the driver.""" + from .weaviate import Store - # Get the configuration - self.store = Store(self.glb.logicalType, connConfig, bag) + return Store(logical_type, conn_config, bag) - # Get the info about our store - collection = self.store.collection - host = self.store.host - port = self.store.port + def _sub_key(self) -> str: + """Return the transform sub-key: host/port/collection.""" + return f'{self.store.host}/{self.store.port}/{self.store.collection}' - # Format it into a subKey - subKey = f'{host}/{port}/{collection}' - - # Call the base - super().beginGlobal(subKey) - return - - def validateConfig(self): + def _probe_connection(self, config: Dict[str, Any]) -> None: """ Validate Weaviate config at save-time with a fast, SDK-driven probe. @@ -78,8 +59,6 @@ def validateConfig(self): - Minimal timeouts; SDK/HTTP exceptions surfaced without truncation """ try: - # Get config - config = Config.getNodeConfig(self.glb.logicalType, self.glb.connConfig) host = (config.get('host', '')).strip() port = config.get('port') grpc_port = config.get('grpc_port') @@ -153,10 +132,6 @@ def validateConfig(self): except Exception as e: warning(_format_error(e)) - def endGlobal(self): - # Release the index and embeddings - self.store = None - def _format_error(e: Exception) -> str: """Concise error string for weaviate's httpx-based exceptions. diff --git a/nodes/src/nodes/store_weaviate/IInstance.py b/nodes/src/nodes/store_weaviate/IInstance.py index ce139e0d28..9a3391fc29 100644 --- a/nodes/src/nodes/store_weaviate/IInstance.py +++ b/nodes/src/nodes/store_weaviate/IInstance.py @@ -21,53 +21,12 @@ # SOFTWARE. # ============================================================================= -from rocketlib import Entry -from typing import List +# ------------------------------------------------------------------------------ +# This class controls the data for each thread of the task +# ------------------------------------------------------------------------------ from .IGlobal import IGlobal -from ai.common.schema import Doc, Question -from ai.common.transform import IInstanceTransform +from ai.common.store import StoreInstanceBase -class IInstance(IInstanceTransform): +class IInstance(StoreInstanceBase): IGlobal: IGlobal - - def writeQuestions(self, question: Question): - """ - Take a question, performs a search, and writes the results as documents. - """ - # Dispatch to the search handler - self.IGlobal.store.dispatchSearch(self, question) - - def writeDocuments(self, documents: List[Doc]): - """ - Take a list of documents and adds them to the vector store. - - Any chunks in the database that have the same object id will - be removed - """ - # Add the document chunks - self.IGlobal.store.addChunks(documents) - - def renderObject(self, object: Entry): - """ - Output all the document text to the writeText lane. - """ - - def callback(text: str) -> None: - self.instance.sendText(text) - - # Check it - if self.IGlobal.store is None: - raise Exception('No document store') - - # If we do not have a vectorize flag, or we have not vectorized - # it, allow the next driver to render - if not object.hasVectorBatchId or not object.vectorBatchId: - return - - # Render the data on this object from the store and - # send it to the renderData function - self.IGlobal.store.render(objectId=object.objectId, callback=callback) - - # Stop right here - self.preventDefault() diff --git a/nodes/src/nodes/store_weaviate/README.md b/nodes/src/nodes/store_weaviate/README.md index 1a4a6b15d5..e2039898f4 100644 --- a/nodes/src/nodes/store_weaviate/README.md +++ b/nodes/src/nodes/store_weaviate/README.md @@ -1,6 +1,6 @@ # store_weaviate -A RocketRide store node that persists embedded document chunks in a Weaviate instance and retrieves them by semantic or keyword search. +A RocketRide store node that persists embedded document chunks in a Weaviate instance and retrieves them by semantic or keyword search, and exposes search/upsert/delete as agent-callable tools. ## What it does @@ -60,6 +60,20 @@ The preconfig default profile is `cloud`. The cloud profile exposes host, API ke --- +## Agent tools + +When wired to an agent, the node exposes three tools via `VectorStoreToolMixin`. Each tool is named `.` (defaults: `weaviate.search`, `weaviate.upsert`, `weaviate.delete`). + +| Tool | Key inputs | Description | +| -------- | ------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------ | +| `search` | `query` (required); `top_k` (default 10, max 100); `filter` (optional dict, keys `objectId`/`nodeId`/`parent` are honored) | Semantic search over stored documents; returns content, metadata, and score per result. Falls back to keyword search if semantic search fails. | +| `upsert` | `documents` array, each with `content` and `object_id`; optional `metadata`, `embedding`, and `embedding_model` | Add or update documents. Embeddings are computed automatically via the bound embedding provider, or pre-computed vectors can be supplied. | +| `delete` | `object_ids` (non-empty string array) | Hard-delete documents by object ID. Returns `deleted_count`. | + +Tool calls run on the control plane and do not flow through the pipeline's embedding lanes. Semantic search in the `search` tool and automatic embedding in `upsert` require an embedding provider bound to the node (the `all.embedding` block in its parameters). Without one, those calls return `{"success": false, "error": ...}`. + +--- + ## Search behavior - **Semantic search** runs a `near_vector` query with the question's embedding. The question must carry an embedding (bind an embedding node), and a non-zero result offset is not supported. When the requested limit is 10 or less, the node queries with a limit of 25. diff --git a/nodes/src/nodes/store_weaviate/services.json b/nodes/src/nodes/store_weaviate/services.json index 9ca8a0dc9a..4634c76835 100644 --- a/nodes/src/nodes/store_weaviate/services.json +++ b/nodes/src/nodes/store_weaviate/services.json @@ -13,13 +13,13 @@ // Required: // Class type of the node - what it does // - "classType": ["store"], + "classType": ["store", "tool"], // // Required: // Capabilities are flags that change the behavior of the underlying // engine // - "capabilities": [], + "capabilities": ["invoke"], // // Optional: // Register is either filter, endpoint or ignored if not specified. If the @@ -140,7 +140,8 @@ "title": "Your own Weaviate server", "host": "localhost", "port": 8080, - "collection": "ROCKETRIDE" + "collection": "ROCKETRIDE", + "serverName": "weaviate" }, "cloud": { "mode": "cloud", @@ -148,7 +149,8 @@ "host": "", "port": 443, "apikey": "", - "collection": "ROCKETRIDE" + "collection": "ROCKETRIDE", + "serverName": "weaviate" } } }, @@ -174,13 +176,20 @@ "vector.local.grpc_port": { "default": 50051 }, + "weaviate.serverName": { + "type": "string", + "title": "Tool Server Name", + "description": "Namespace for agent-facing tool names, e.g. 'weaviate' exposes tools as weaviate.search / weaviate.upsert / weaviate.delete. Change this when running multiple Weaviate nodes in the same pipeline so their tool names do not collide.", + "default": "weaviate", + "minLength": 1 + }, "weaviate.cloud": { "object": "cloud", - "properties": ["vector.cloud.host", "vector.apikey", "vector.score", "vector.collection"] + "properties": ["vector.cloud.host", "vector.apikey", "vector.score", "vector.collection", "weaviate.serverName"] }, "weaviate.local": { "object": "local", - "properties": ["vector.local.host", "vector.local.port", "vector.local.grpc_port", "vector.score", "vector.collection"] + "properties": ["vector.local.host", "vector.local.port", "vector.local.grpc_port", "vector.score", "vector.collection", "weaviate.serverName"] }, "weaviate.profile": { "title": "Type of Weaviate host", diff --git a/nodes/test/test_vectordb_tool_mixin.py b/nodes/test/test_vectordb_tool_mixin.py index 9a0c3e1cb6..d798180212 100644 --- a/nodes/test/test_vectordb_tool_mixin.py +++ b/nodes/test/test_vectordb_tool_mixin.py @@ -3,7 +3,7 @@ # Copyright (c) 2026 Aparavi Software AG # ============================================================================= -"""Unit tests for the VectorStoreToolMixin (packages/ai/src/ai/common/store.py). +"""Unit tests for the VectorStoreToolMixin (packages/ai/src/ai/common/store/document_store.py). These tests load ``store.py`` in isolation by stubbing its heavy imports (``rocketlib``, ``ai.common.schema``) so the module can be exercised without @@ -31,7 +31,7 @@ from typing import Any, Callable, Dict, List, Optional _ROOT = Path(__file__).resolve().parents[2] -_STORE_PY = _ROOT / 'packages' / 'ai' / 'src' / 'ai' / 'common' / 'store.py' +_STORE_PY = _ROOT / 'packages' / 'ai' / 'src' / 'ai' / 'common' / 'store' / 'document_store.py' _STUB_MODULE_NAMES = ( 'rocketlib', @@ -39,6 +39,7 @@ 'ai.common', 'ai.common.schema', 'ai.common.store', + 'ai.common.store.document_store', ) @@ -188,9 +189,13 @@ def _install_stubs() -> None: schema_mod.QuestionType = _StubQuestionType schema_mod.Answer = _StubAnswer + store_pkg = types.ModuleType('ai.common.store') + store_pkg.__path__ = [] # mark as package so document_store's ``..schema`` resolves + sys.modules['ai'] = ai_pkg sys.modules['ai.common'] = common_pkg sys.modules['ai.common.schema'] = schema_mod + sys.modules['ai.common.store'] = store_pkg @contextmanager @@ -209,13 +214,13 @@ def _scoped_stubs() -> Iterator[None]: def _load_store_module() -> types.ModuleType: with _scoped_stubs(): - # Load store.py as a submodule of the stubbed ``ai.common`` package so - # its ``from .schema import ...`` relative import resolves against the - # stub already installed in sys.modules. - spec = importlib.util.spec_from_file_location('ai.common.store', _STORE_PY) + # Load store/document_store.py as a submodule of the stubbed + # ``ai.common`` package so its ``from ..schema import ...`` relative + # import resolves against the stub already installed in sys.modules. + spec = importlib.util.spec_from_file_location('ai.common.store.document_store', _STORE_PY) assert spec is not None and spec.loader is not None module = importlib.util.module_from_spec(spec) - sys.modules['ai.common.store'] = module + sys.modules['ai.common.store.document_store'] = module spec.loader.exec_module(module) return module diff --git a/packages/ai/src/ai/common/store/__init__.py b/packages/ai/src/ai/common/store/__init__.py new file mode 100644 index 0000000000..d20d8d3cbe --- /dev/null +++ b/packages/ai/src/ai/common/store/__init__.py @@ -0,0 +1,51 @@ +# ============================================================================= +# RocketRide Engine +# ============================================================================= +# MIT License +# Copyright (c) 2026 Aparavi Software AG +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# ============================================================================= + +"""Base classes and driver interface for vector store nodes. + +``store_qdrant`` is the reference the IGlobal/IInstance abstraction was +extracted from; the other ``store_*`` drivers still carry their own copies and +migrate onto these bases one at a time. The store interface itself +(``DocumentStoreBase``, ``getStore``) and the agent-tool mixin +(``VectorStoreToolMixin``) live in ``document_store`` and are re-exported here so +``from ai.common.store import ...`` keeps working after the module became a +package. +""" + +from .document_store import ( + DocumentStoreBase, + VectorStoreToolMixin, + getStore, +) +from .store_global_base import StoreGlobalBase +from .store_instance_base import StoreInstanceBase + +__all__ = [ + 'DocumentStoreBase', + 'StoreGlobalBase', + 'StoreInstanceBase', + 'VectorStoreToolMixin', + 'getStore', +] diff --git a/packages/ai/src/ai/common/store.py b/packages/ai/src/ai/common/store/document_store.py similarity index 99% rename from packages/ai/src/ai/common/store.py rename to packages/ai/src/ai/common/store/document_store.py index f3b7aaebfe..f01c7acf87 100644 --- a/packages/ai/src/ai/common/store.py +++ b/packages/ai/src/ai/common/store/document_store.py @@ -4,7 +4,7 @@ from abc import abstractmethod, ABC from typing import List, Callable, Dict, Any, Optional, Tuple from rocketlib import IInstanceBase, tool_function, warning -from .schema import Doc, DocFilter, DocMetadata, Question, QuestionText, QuestionType, Answer +from ..schema import Doc, DocFilter, DocMetadata, Question, QuestionText, QuestionType, Answer class DocumentStoreBase(ABC): diff --git a/packages/ai/src/ai/common/store/store_global_base.py b/packages/ai/src/ai/common/store/store_global_base.py new file mode 100644 index 0000000000..7abe67cfba --- /dev/null +++ b/packages/ai/src/ai/common/store/store_global_base.py @@ -0,0 +1,179 @@ +# ============================================================================= +# RocketRide Engine +# ============================================================================= +# MIT License +# Copyright (c) 2026 Aparavi Software AG +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# ============================================================================= + +"""Abstract IGlobal layer shared by every vector store node. + +Vector stores differ in transport (Qdrant REST/gRPC, Weaviate, Pinecone, ...) +and in how they validate a collection name, but they share a lifecycle: skip the +driver in CONFIG mode, resolve the tool namespace, open the store, wire an +optional embedder for the control-plane tool path, and register a transform key. +That lifecycle lives here; the driver-specific pieces live in the hooks below. + +A concrete node implements ``_open_store``, ``_sub_key`` and ``_probe_connection`` +and inherits the rest. See ``nodes/src/nodes/store_qdrant/IGlobal.py`` for a +worked example. +""" + +from abc import ABC, abstractmethod +from typing import Any, Dict, Optional + +from rocketlib import OPEN_MODE, warning + +from ai.common.config import Config +from ai.common.transform import IGlobalTransform + +from .document_store import DocumentStoreBase + + +class StoreGlobalBase(IGlobalTransform, ABC): + """Abstract base for the IGlobal layer of any vector store node.""" + + # Namespace prefix for agent-facing tool names (``.search`` ...). + # Overridden by the driver and, when present, by the merged node config. + serverName: str = '' + + # The live store, opened in beginGlobal and read by StoreInstanceBase. + store: Optional[DocumentStoreBase] = None + + # Embedder wiring for the control-plane tool path (search/upsert). Bound in + # beginGlobal only when an embedding sub-block is configured. + embed_query = None + embed_model_name: Optional[str] = None + + # ------------------------------------------------------------------ + # Abstract interface — every store driver must implement these + # ------------------------------------------------------------------ + + @abstractmethod + def _open_store(self, logical_type: str, conn_config: Dict[str, Any], bag: Dict[str, Any]) -> DocumentStoreBase: + """Instantiate and return the driver's ``DocumentStoreBase`` for this node. + + Called once per pipeline start (outside CONFIG mode). Must raise if the + store cannot be constructed so the failure surfaces at pipeline start. + """ + + @abstractmethod + def _sub_key(self) -> str: + """Return the driver-specific transform sub-key (e.g. ``host/port/collection``). + + Read from ``self.store`` after it is opened. Folded into the transform + key that tracks which objects a given store has already ingested. + """ + + @abstractmethod + def _probe_connection(self, config: Dict[str, Any]) -> None: + """Save-time connectivity/format probe. Report problems with ``warning()``. + + Must not raise: this runs while the user is still editing the node, so a + bad host or collection name should produce a readable warning, not a + stack trace. + """ + + # ------------------------------------------------------------------ + # Lifecycle — identical across vector stores + # ------------------------------------------------------------------ + + def beginGlobal(self) -> None: + """Open the store and wire the tool-path embedder, unless in CONFIG mode.""" + # CONFIG mode only needs configureService; the driver is not loaded. + if self.IEndpoint.endpoint.openMode == OPEN_MODE.CONFIG: + return + + self.store = None + bag = self.IEndpoint.endpoint.bag + connConfig = self.getConnConfig() + + # Resolve the namespace used for agent-facing tool names from the merged + # config so it honors both profile defaults and user overrides. + cfg = Config.getNodeConfig(self.glb.logicalType, connConfig) + resolved_name = cfg.get('serverName') if hasattr(cfg, 'get') else None + if isinstance(resolved_name, str) and resolved_name.strip(): + self.serverName = resolved_name.strip() + + # Open the driver's store, then wire an optional embedder for the tool path. + # Pass the raw connConfig, not the merged cfg above: each driver's Store + # re-resolves it via Config.getNodeConfig, so the profile merge stays in one place. + self.store = self._open_store(self.glb.logicalType, connConfig, bag) + self._wire_embedding(connConfig, bag) + + # Register the transform key so re-ingest can skip unchanged objects. + super().beginGlobal(self._sub_key()) + + def _wire_embedding(self, connConfig: Dict[str, Any], bag: Dict[str, Any]) -> None: + """Bind ``embed_query``/``embed_model_name`` for the control-plane tool path. + + Tool invocations do not flow through the data-lane embedding filters, so + the search/upsert tools need their own embedder. Only instantiate one + when an embedding sub-block is present; otherwise leave the hooks unset + and the tools fall back to keyword search. + """ + self._tool_embedding = None + self.embed_query = None + self.embed_model_name = None + + try: + embed_provider, embed_config = Config.getMultiProviderConfig('embedding', connConfig) + except Exception: + embed_provider, embed_config = None, None + + if embed_provider: + try: + from ai.common.embedding import getEmbedding as _getEmbedding + + self._tool_embedding = _getEmbedding(embed_provider, embed_config, bag) + except Exception as exc: # noqa: BLE001 + warning(f'{self.glb.logicalType}: tool path embedder unavailable: {exc}') + self._tool_embedding = None + + if self._tool_embedding is not None: + from ai.common.schema import Question as _Question, QuestionText as _QuestionText + + def _embed_query(text: str, _emb=self._tool_embedding) -> list: + qt = _QuestionText(text=text) + q = _Question() + q.questions = [qt] + _emb.encodeQuestion(q) + return list(qt.embedding or []) + + self.embed_query = _embed_query + self.embed_model_name = getattr(self._tool_embedding, '_model', None) + + def endGlobal(self) -> None: + """Drop the store and the tool-path embedder, then clear the transform key.""" + self._tool_embedding = None + self.embed_query = None + self.embed_model_name = None + self.store = None + + # Let the transform layer clear TRANFORM_KEY_TAG_NAME on teardown. + super().endGlobal() + + def validateConfig(self) -> None: + """Save-time validation: probe the server so the user sees errors immediately.""" + try: + config = Config.getNodeConfig(self.glb.logicalType, self.glb.connConfig) + self._probe_connection(config) + except Exception as e: + warning(str(e)) diff --git a/packages/ai/src/ai/common/store/store_instance_base.py b/packages/ai/src/ai/common/store/store_instance_base.py new file mode 100644 index 0000000000..1a15c5f132 --- /dev/null +++ b/packages/ai/src/ai/common/store/store_instance_base.py @@ -0,0 +1,97 @@ +# ============================================================================= +# RocketRide Engine +# ============================================================================= +# MIT License +# Copyright (c) 2026 Aparavi Software AG +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# ============================================================================= + +"""Abstract IInstance layer shared by every vector store node. + +The three pipeline lane handlers are byte-identical across the store drivers: +questions run a search, documents are written as chunks, and rendering rehydrates +an object's chunks. They all delegate to ``self.IGlobal.store`` (a +``DocumentStoreBase``), so they live here once. Subclassing +``VectorStoreToolMixin`` also gives every node the ``search``/``upsert``/``delete`` +agent tools uniformly. + +A concrete node inherits this class with no per-driver overrides. See +``nodes/src/nodes/store_qdrant/IInstance.py``. +""" + +from abc import ABC +from typing import List + +from rocketlib import Entry + +from ai.common.schema import Doc, Question +from ai.common.transform import IInstanceTransform + +from .document_store import VectorStoreToolMixin +from .store_global_base import StoreGlobalBase + + +class StoreInstanceBase(VectorStoreToolMixin, IInstanceTransform, ABC): + """Abstract base for the IInstance layer of any vector store node.""" + + IGlobal: StoreGlobalBase + + def writeQuestions(self, question: Question) -> None: + """Take a question, perform a search, and write the results as documents.""" + # Check it + if self.IGlobal.store is None: + raise Exception('No document store') + + # Dispatch to the search handler + self.IGlobal.store.dispatchSearch(self, question) + + def writeDocuments(self, documents: List[Doc]) -> None: + """Take a list of documents and add them to the vector store. + + Any chunks in the database that have the same object id will be removed. + """ + # Check it + if self.IGlobal.store is None: + raise Exception('No document store') + + # Add the document chunks + self.IGlobal.store.addChunks(documents) + + def renderObject(self, object: Entry) -> None: + """Output the document text to the writeText lane.""" + + def callback(text: str) -> None: + self.instance.sendText(text) + + # Check it + if self.IGlobal.store is None: + raise Exception('No document store') + + # If we do not have a vectorize flag, or we have not vectorized + # it, allow the next driver to render + if not object.hasVectorBatchId or not object.vectorBatchId: + return + + # Render the data on this object from the store and + # send it to the renderData function + self.IGlobal.store.render(objectId=object.objectId, callback=callback) + + # Stop right here + self.preventDefault() diff --git a/packages/ai/tests/ai/common/store/__init__.py b/packages/ai/tests/ai/common/store/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/ai/tests/ai/common/store/test_store_base.py b/packages/ai/tests/ai/common/store/test_store_base.py new file mode 100644 index 0000000000..b917dcc7d1 --- /dev/null +++ b/packages/ai/tests/ai/common/store/test_store_base.py @@ -0,0 +1,249 @@ +""" +Unit tests for ai.common.store.store_global_base.StoreGlobalBase and +ai.common.store.store_instance_base.StoreInstanceBase. + +The base classes are ABCs with three abstract driver hooks (``_open_store``, +``_sub_key``, ``_probe_connection``). Tests use a concrete ``_TestableGlobal`` +that supplies a fake store, then drive the shared lifecycle with a stubbed +``IEndpoint``/``glb`` and monkeypatched ``Config``/embedding so no engine or +real driver is needed. + +The behaviors pinned here are the ones a regression in this shared file could +silently break across all eight store nodes at once — including the two +review-driven fixes (the lane ``store is None`` guards and ``endGlobal`` +clearing the transform key), which otherwise have no test. +""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest +from rocketlib import OPEN_MODE + +import ai.common.store.store_global_base as sgb +from ai.common.store.store_global_base import StoreGlobalBase +from ai.common.store.store_instance_base import StoreInstanceBase + + +# --------------------------------------------------------------------------- +# Fakes / test subclasses +# --------------------------------------------------------------------------- + + +class _FakeStore: + """Minimal stand-in for a DocumentStoreBase with the attrs _sub_key reads.""" + + def __init__(self): + self.host = 'Host' + self.port = 5555 + self.collection = 'Coll' + self.calls = [] + + def dispatchSearch(self, instance, question): + self.calls.append(('search', question)) + + def addChunks(self, documents): + self.calls.append(('add', documents)) + + def render(self, objectId, callback): + self.calls.append(('render', objectId)) + + +class _TestableGlobal(StoreGlobalBase): + """Concrete StoreGlobalBase: fake store, deterministic sub-key, no-op probe.""" + + serverName = 'defaultname' + + def __init__(self, store): + self._store_to_return = store + self._conn_config = {} + self.opened = False + + def getConnConfig(self): + # Bypass the filter/endpoint config plumbing; tests inject directly. + return self._conn_config + + def _open_store(self, logical_type, conn_config, bag): + self.opened = True + return self._store_to_return + + def _sub_key(self) -> str: + return f'{self.store.host}/{self.store.port}/{self.store.collection}' + + def _probe_connection(self, config) -> None: + return + + +class _TestableInstance(StoreInstanceBase): + """Concrete StoreInstanceBase with no per-driver overrides (the common case).""" + + +# --------------------------------------------------------------------------- +# Fixtures / helpers +# --------------------------------------------------------------------------- + + +@pytest.fixture +def warnings(monkeypatch): + """Capture rocketlib.warning calls made from the base module.""" + captured = [] + monkeypatch.setattr(sgb, 'warning', lambda msg: captured.append(msg)) + return captured + + +def _make(open_mode=OPEN_MODE.SOURCE, store=None): + g = _TestableGlobal(store=store if store is not None else _FakeStore()) + g.IEndpoint = SimpleNamespace(endpoint=SimpleNamespace(openMode=open_mode, bag={})) + g.glb = SimpleNamespace(logicalType='teststore', connConfig={}) + return g + + +def _patch_config(monkeypatch, node_config, embedding=(None, None)): + """Stub Config.getNodeConfig (serverName source) and getMultiProviderConfig.""" + monkeypatch.setattr(sgb.Config, 'getNodeConfig', staticmethod(lambda lt, cc: node_config)) + monkeypatch.setattr(sgb.Config, 'getMultiProviderConfig', staticmethod(lambda kind, cc: embedding)) + + +# --------------------------------------------------------------------------- +# 1. CONFIG mode short-circuits before the driver is loaded +# --------------------------------------------------------------------------- + + +def test_config_mode_returns_before_open_store(monkeypatch): + """CONFIG open mode returns before _open_store; store stays None (driver never imported).""" + _patch_config(monkeypatch, {}) + g = _make(open_mode=OPEN_MODE.CONFIG) + g.beginGlobal() + assert g.opened is False + assert g.store is None + + +# --------------------------------------------------------------------------- +# 2. serverName resolution from merged config +# --------------------------------------------------------------------------- + + +def test_server_name_taken_from_config_when_present(monkeypatch): + """A non-blank string serverName in the merged config overrides the class default.""" + _patch_config(monkeypatch, {'serverName': 'custom'}) + g = _make() + g.beginGlobal() + assert g.serverName == 'custom' + + +@pytest.mark.parametrize('node_config', [{}, {'serverName': ''}, {'serverName': ' '}, {'serverName': 123}]) +def test_server_name_default_survives_when_absent_blank_or_non_string(monkeypatch, node_config): + """The class default survives when serverName is missing, blank, or not a string.""" + _patch_config(monkeypatch, node_config) + g = _make() + g.beginGlobal() + assert g.serverName == 'defaultname' + + +# --------------------------------------------------------------------------- +# 3. _wire_embedding +# --------------------------------------------------------------------------- + + +def test_wire_embedding_leaves_query_none_without_embedding_block(monkeypatch): + """With no embedding provider, embed_query stays None (tools fall back to keyword).""" + _patch_config(monkeypatch, {}, embedding=(None, None)) + g = _make() + g.beginGlobal() + assert g.embed_query is None + assert g.embed_model_name is None + + +def test_wire_embedding_warns_instead_of_raising_when_getembedding_throws(monkeypatch, warnings): + """If getEmbedding raises, beginGlobal warns and leaves embed_query None rather than failing.""" + _patch_config(monkeypatch, {}, embedding=('provider', {'k': 'v'})) + + import ai.common.embedding as emb + + def _boom(provider, config, bag): + raise RuntimeError('no model here') + + monkeypatch.setattr(emb, 'getEmbedding', _boom) + + g = _make() + g.beginGlobal() # must not raise + assert g.embed_query is None + assert any('tool path embedder unavailable' in w for w in warnings) + + +# --------------------------------------------------------------------------- +# 4. transform key format +# --------------------------------------------------------------------------- + + +def test_transform_key_format(monkeypatch): + """TRANFORM_KEY_TAG_NAME is f'{logicalType}://{sub_key.lower()}/status'.""" + _patch_config(monkeypatch, {}) + g = _make() + g.beginGlobal() + # sub_key = 'Host/5555/Coll' -> lowercased in the key + assert g.TRANFORM_KEY_TAG_NAME == 'teststore://host/5555/coll/status' + + +# --------------------------------------------------------------------------- +# 5. endGlobal clears store, embedder AND the transform key +# --------------------------------------------------------------------------- + + +def test_end_global_clears_state_and_transform_key(monkeypatch): + """The endGlobal teardown drops store/embed_query/embed_model_name and clears the transform key.""" + _patch_config(monkeypatch, {}) + g = _make() + g.beginGlobal() + assert g.TRANFORM_KEY_TAG_NAME # set by beginGlobal + + g.endGlobal() + assert g.store is None + assert g.embed_query is None + assert g.embed_model_name is None + assert g.TRANFORM_KEY_TAG_NAME == '' + + +# --------------------------------------------------------------------------- +# 6. lane handlers guard a missing store +# --------------------------------------------------------------------------- + + +def _instance_with_store(store): + inst = _TestableInstance.__new__(_TestableInstance) + inst.IGlobal = SimpleNamespace(store=store) + return inst + + +def test_write_questions_raises_when_store_missing(): + """The writeQuestions lane raises 'No document store' when the store is absent.""" + inst = _instance_with_store(None) + with pytest.raises(Exception, match='No document store'): + inst.writeQuestions(SimpleNamespace()) + + +def test_write_documents_raises_when_store_missing(): + """The writeDocuments lane raises 'No document store' when the store is absent.""" + inst = _instance_with_store(None) + with pytest.raises(Exception, match='No document store'): + inst.writeDocuments([]) + + +def test_render_object_raises_when_store_missing(): + """The renderObject lane raises 'No document store' when the store is absent.""" + inst = _instance_with_store(None) + with pytest.raises(Exception, match='No document store'): + inst.renderObject(SimpleNamespace(hasVectorBatchId=True, vectorBatchId='b', objectId='o')) + + +def test_lane_handlers_delegate_to_store_when_present(): + """With a store present, the three lanes delegate to it.""" + store = _FakeStore() + inst = _instance_with_store(store) + + inst.writeQuestions(SimpleNamespace()) + inst.writeDocuments([{'x': 1}]) + + kinds = [c[0] for c in store.calls] + assert kinds == ['search', 'add']