feat: add TOS object store support via OpenDAL

docs/src/guide/object_store.md

@@ -218,3 +218,33 @@ ds = lance.dataset(
 | `oss_secret_access_key` | Access key secret used for OSS authentication. Optional if credentials are provided by environment. |
 | `oss_region` | OSS region (for example, `cn-hangzhou`). Optional. |
 | `oss_security_token` | Security token for temporary credentials (STS). Optional. |
+
+## Volcengine TOS Configuration
+
+TOS credentials can be set in the environment variables `TOS_ACCESS_KEY_ID`,
+`TOS_SECRET_ACCESS_KEY`, `TOS_ENDPOINT`, `TOS_REGION`, and `TOS_SECURITY_TOKEN`.
+Lance also accepts the corresponding `VOLCENGINE_` environment variable prefix.
+Alternatively, credentials can be passed as parameters to the `storage_options`
+parameter; explicit `storage_options` override environment variables:
+
+```python
+import lance
+ds = lance.dataset(
+    "tos://bucket/path",
+    storage_options={
+        "tos_endpoint": "https://tos-cn-beijing.volces.com",
+        "tos_region": "cn-beijing",
+        "tos_access_key_id": "my-access-key",
+        "tos_secret_access_key": "my-secret-key",
+        "tos_security_token": "my-session-token",
+    }
+)
+```
+
+| Key | Description |
+|-----|-------------|
+| `tos_endpoint` | TOS endpoint. Required (for example, `https://tos-cn-beijing.volces.com`). |
+| `tos_region` | TOS signing region (for example, `cn-beijing`). Optional. |
+| `tos_access_key_id` | Access key ID used for TOS authentication. Optional if credentials are provided by environment. |
+| `tos_secret_access_key` | Secret access key used for TOS authentication. Optional if credentials are provided by environment. |
+| `tos_security_token` | Security token for temporary credentials. Optional. |

rust/lance-io/Cargo.toml

@@ -76,6 +76,8 @@ azure = ["object_store/azure", "dep:opendal", "opendal/services-azblob", "openda
 oss = ["dep:opendal", "opendal/services-oss", "dep:object_store_opendal"]
 tencent = ["dep:opendal", "opendal/services-cos", "dep:object_store_opendal"]
 huggingface = ["dep:opendal", "opendal/services-huggingface", "dep:object_store_opendal"]
+tos = ["dep:opendal", "opendal/services-tos", "dep:object_store_opendal"]
+tos-test = ["tos"]
 test-util = []
 
 [lints]

rust/lance-io/src/object_store.rs

@@ -36,7 +36,7 @@ use super::local::LocalObjectReader;
 use crate::uring::{UringCurrentThreadReader, UringReader};
 #[cfg(any(feature = "aws", feature = "azure", feature = "gcp"))]
 pub(crate) mod dynamic_credentials;
-#[cfg(any(feature = "oss", feature = "huggingface"))]
+#[cfg(any(feature = "oss", feature = "huggingface", feature = "tos"))]
 pub(crate) mod dynamic_opendal;
 mod list_retry;
 pub mod providers;
@@ -61,7 +61,15 @@ pub const DEFAULT_LOCAL_IO_PARALLELISM: usize = 8;
 pub const DEFAULT_CLOUD_IO_PARALLELISM: usize = 64;
 
 const DEFAULT_LOCAL_BLOCK_SIZE: usize = 4 * 1024; // 4KB block size
-#[cfg(any(feature = "aws", feature = "gcp", feature = "azure"))]
+#[cfg(any(
+    feature = "aws",
+    feature = "gcp",
+    feature = "azure",
+    feature = "oss",
+    feature = "tencent",
+    feature = "huggingface",
+    feature = "tos",
+))]
 const DEFAULT_CLOUD_BLOCK_SIZE: usize = 64 * 1024; // 64KB block size
 
 pub static DEFAULT_MAX_IOP_SIZE: std::sync::LazyLock<u64> = std::sync::LazyLock::new(|| {

rust/lance-io/src/object_store/providers.rs

@@ -33,6 +33,8 @@ pub mod oss;
 pub mod shared_memory;
 #[cfg(feature = "tencent")]
 pub mod tencent;
+#[cfg(feature = "tos")]
+pub mod tos;
 
 #[async_trait::async_trait]
 pub trait ObjectStoreProvider: std::fmt::Debug + Sync + Send {
@@ -95,6 +97,7 @@ pub struct ObjectStoreRegistryStats {
 /// - `s3+ddb`: An S3 object store with DynamoDB for metadata.
 /// - `az`: An Azure Blob Storage object store.
 /// - `gs`: A Google Cloud Storage object store.
+/// - `tos`: A Volcengine TOS object store.
 ///
 /// Use [`Self::empty()`] to create an empty registry, with no providers registered.
 ///
@@ -330,6 +333,8 @@ impl Default for ObjectStoreRegistry {
         providers.insert("cos".into(), Arc::new(tencent::TencentStoreProvider));
         #[cfg(feature = "huggingface")]
         providers.insert("hf".into(), Arc::new(huggingface::HuggingfaceStoreProvider));
+        #[cfg(feature = "tos")]
+        providers.insert("tos".into(), Arc::new(tos::TosStoreProvider));
         Self {
             providers: RwLock::new(providers),
             active_stores: RwLock::new(HashMap::new()),