Datasets API v2 - SDK (#795)

encord/orm/dataset.py

@@ -1051,3 +1051,20 @@ class DatasetDataLongPolling(BaseDTO):
 @dataclasses.dataclass(frozen=True)
 class DatasetLinkItems:
     pass
+
+
+class CreateDatasetPayload(BaseDTO):
+    title: str
+    description: Optional[str]
+
+    create_backing_folder: bool  # this creates a legacy "mirror" dataset and it's backing folder in one go
+
+    # only for analytics, to know if customers are
+    # using depreciated EncordUserClient.create_private_dataset
+    # this is only place which should pass legacy_call=True
+    legacy_call: bool  # this field will be removed soon
+
+
+class CreateDatasetResponseV2(BaseDTO):
+    dataset_uuid: UUID
+    backing_folder_uuid: Optional[UUID] = None  # a 'not None' indicates a legacy "mirror" dataset was created

encord/user_client.py

@@ -56,7 +56,9 @@
 from encord.orm.cloud_integration import CloudIntegration, GetCloudIntegrationsResponse
 from encord.orm.dataset import (
     DEFAULT_DATASET_ACCESS_SETTINGS,
+    CreateDatasetPayload,
     CreateDatasetResponse,
+    CreateDatasetResponseV2,
     DatasetAccessSettings,
     DatasetInfo,
     DatasetUserRole,
@@ -94,7 +96,8 @@
 )
 from encord.orm.project import Project as OrmProject
 from encord.orm.project_with_user_role import ProjectWithUserRole
-from encord.orm.storage import ListFoldersParams, ListItemsParams, StorageItemType
+from encord.orm.storage import CreateStorageFolderPayload, ListFoldersParams, ListItemsParams, StorageItemType
+from encord.orm.storage import StorageFolder as OrmStorageFolder
 from encord.project import Project
 from encord.storage import FoldersSortBy, StorageFolder, StorageItem
 from encord.utilities.client_utilities import (
@@ -199,6 +202,34 @@ def get_ontology(self, ontology_hash: str) -> Ontology:
         )
         return Ontology._from_api_payload(ontology_with_user_role, self._api_client)
 
+    def __create_dataset(
+        self,
+        title: str,
+        description: Optional[str],
+        storage_location: StorageLocation,
+        create_backing_folder: bool,
+        legacy_call: bool,
+    ) -> CreateDatasetResponse:
+        res_dataset = self._api_client.post(
+            "datasets",
+            params=None,
+            payload=CreateDatasetPayload(
+                title=title,
+                description=description,
+                create_backing_folder=create_backing_folder,
+                legacy_call=legacy_call,
+            ),
+            result_type=CreateDatasetResponseV2,
+        )
+
+        return CreateDatasetResponse(
+            title=title,
+            storage_location=storage_location,
+            dataset_hash=str(res_dataset.dataset_uuid),
+            user_hash="fields withdrawn for compliance reasons",
+            backing_folder_uuid=res_dataset.backing_folder_uuid,
+        )
+
     @deprecated("0.1.104", alternative=".create_dataset")
     def create_private_dataset(
         self,
@@ -209,7 +240,14 @@ def create_private_dataset(
         """
         DEPRECATED - please use `create_dataset` instead.
         """
-        return self.create_dataset(dataset_title, dataset_type, dataset_description)
+
+        return self.__create_dataset(
+            title=dataset_title,
+            description=dataset_description,
+            storage_location=dataset_type,
+            create_backing_folder=True,
+            legacy_call=True,
+        )
 
     def create_dataset(
         self,
@@ -229,17 +267,14 @@ def create_dataset(
         Returns:
             CreateDatasetResponse
         """
-        dataset = {
-            "title": dataset_title,
-            "type": dataset_type,
-            "create_backing_folder": create_backing_folder,
-        }
-
-        if dataset_description:
-            dataset["description"] = dataset_description
 
-        result = self._querier.basic_setter(OrmDataset, uid=None, payload=dataset)
-        return CreateDatasetResponse.from_dict(result)
+        return self.__create_dataset(
+            title=dataset_title,
+            description=dataset_description,
+            storage_location=dataset_type,
+            create_backing_folder=create_backing_folder,
+            legacy_call=False,
+        )
 
     def get_datasets(
         self,