Feat/rapid 6800 batch url upload

src/rapidata/rapidata_client/benchmark/participant/_participant.py

@@ -2,7 +2,7 @@
 
 from concurrent.futures import ThreadPoolExecutor, as_completed
 import time
-from tqdm import tqdm
+from tqdm.auto import tqdm
 
 from rapidata.rapidata_client.config import logger
 from rapidata.rapidata_client.config.rapidata_config import rapidata_config

src/rapidata/rapidata_client/config/rapidata_config.py

@@ -1,7 +1,6 @@
 from pydantic import BaseModel, Field
 
 from rapidata.rapidata_client.config.logging_config import LoggingConfig
-from rapidata.rapidata_client.config.order_config import OrderConfig
 from rapidata.rapidata_client.config.upload_config import UploadConfig
 
 
@@ -15,8 +14,6 @@ class RapidataConfig(BaseModel):
         enableBetaFeatures (bool): Whether to enable beta features. Defaults to False.
         upload (UploadConfig): The configuration for the upload process.
             Such as the maximum number of worker threads for processing media paths and the maximum number of retries for failed uploads.
-        order (OrderConfig): The configuration for the order process.
-            Such as the minimum number of datapoints required so that an automatic validationset gets created if no recommended was found.
         logging (LoggingConfig): The configuration for the logging process.
             Such as the logging level and the logging file.
 
@@ -29,7 +26,6 @@ class RapidataConfig(BaseModel):
 
     enableBetaFeatures: bool = False
     upload: UploadConfig = Field(default_factory=UploadConfig)
-    order: OrderConfig = Field(default_factory=OrderConfig)
     logging: LoggingConfig = Field(default_factory=LoggingConfig)
 
 

src/rapidata/rapidata_client/config/upload_config.py

@@ -1,6 +1,6 @@
 from pathlib import Path
 import shutil
-from pydantic import BaseModel, Field, field_validator
+from pydantic import BaseModel, ConfigDict, Field, field_validator
 from rapidata.rapidata_client.config import logger
 
 
@@ -11,19 +11,29 @@ class UploadConfig(BaseModel):
     Attributes:
         maxWorkers (int): The maximum number of worker threads for concurrent uploads. Defaults to 25.
         maxRetries (int): The maximum number of retries for failed uploads. Defaults to 3.
-        cacheUploads (bool): Enable/disable upload caching. Defaults to True.
+        cacheToDisk (bool): Enable disk-based caching for file uploads. If False, uses in-memory cache only. Defaults to True.
+            Note: URL assets are always cached in-memory regardless of this setting.
+            Caching cannot be disabled entirely as it's required for the two-step upload flow.
         cacheTimeout (float): Cache operation timeout in seconds. Defaults to 0.1.
         cacheLocation (Path): Directory for cache storage. Defaults to ~/.cache/rapidata/upload_cache.
-            This is immutable
+            This is immutable. Only used for file uploads when cacheToDisk=True.
         cacheShards (int): Number of cache shards for parallel access. Defaults to 128.
             Higher values improve concurrency but increase file handles. Must be positive.
-            This is immutable
+            This is immutable. Only used for file uploads when cacheToDisk=True.
+        enableBatchUpload (bool): Enable batch URL uploading (two-step process). Defaults to True.
+        batchSize (int): Number of URLs per batch (10-500). Defaults to 100.
+        batchPollInterval (float): Polling interval in seconds. Defaults to 0.5.
     """
 
+    model_config = ConfigDict(validate_assignment=True)
+
     maxWorkers: int = Field(default=25)
     maxRetries: int = Field(default=3)
-    cacheUploads: bool = Field(default=True)
-    cacheTimeout: float = Field(default=0.1)
+    cacheToDisk: bool = Field(
+        default=True,
+        description="Enable disk-based caching for file uploads. URLs are always cached in-memory.",
+    )
+    cacheTimeout: float = Field(default=1)
     cacheLocation: Path = Field(
         default=Path.home() / ".cache" / "rapidata" / "upload_cache",
         frozen=True,
@@ -32,6 +42,14 @@ class UploadConfig(BaseModel):
         default=128,
         frozen=True,
     )
+    batchSize: int = Field(
+        default=1000,
+        description="Number of URLs per batch (100-5000)",
+    )
+    batchPollInterval: float = Field(
+        default=0.5,
+        description="Polling interval in seconds",
+    )
 
     @field_validator("maxWorkers")
     @classmethod
@@ -54,6 +72,13 @@ def validate_cache_shards(cls, v: int) -> int:
             )
         return v
 
+    @field_validator("batchSize")
+    @classmethod
+    def validate_batch_size(cls, v: int) -> int:
+        if v < 100:
+            raise ValueError("batchSize must be at least 100")
+        return v
+
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
         self._migrate_cache()