Revert "Feat: Using fsspec to download files" (#380)

This reverts commit 719bae2.

Author: tchaton

README.md

@@ -217,8 +217,9 @@ Additionally, you can inject client connection settings for [S3](https://boto3.a
 from litdata import StreamingDataset
 
 storage_options = {
-    "key": "your_access_key_id",
-    "secret": "your_secret_access_key",
+    "endpoint_url": "your_endpoint_url",
+    "aws_access_key_id": "your_access_key_id",
+    "aws_secret_access_key": "your_secret_access_key",
 }
 
 dataset = StreamingDataset('s3://my-bucket/my-data', storage_options=storage_options)
@@ -263,47 +264,33 @@ for batch in val_dataloader:
 
  
 
-The StreamingDataset supports reading optimized datasets from common cloud providers.
+The StreamingDataset supports reading optimized datasets from common cloud providers. 
 
 ```python
 import os
 import litdata as ld
 
 # Read data from AWS S3
 aws_storage_options={
-    "key": os.environ['AWS_ACCESS_KEY_ID'],
-    "secret": os.environ['AWS_SECRET_ACCESS_KEY'],
+    "AWS_ACCESS_KEY_ID": os.environ['AWS_ACCESS_KEY_ID'],
+    "AWS_SECRET_ACCESS_KEY": os.environ['AWS_SECRET_ACCESS_KEY'],
 }
 dataset = ld.StreamingDataset("s3://my-bucket/my-data", storage_options=aws_storage_options)
 
 # Read data from GCS
 gcp_storage_options={
-    "token": {
-      # dumped from cat ~/.config/gcloud/application_default_credentials.json 
-        "account": "",
-        "client_id": "your_client_id",
-        "client_secret": "your_client_secret",
-        "quota_project_id": "your_quota_project_id",
-        "refresh_token": "your_refresh_token",
-        "type": "authorized_user",
-        "universe_domain": "googleapis.com",
-    }
+    "project": os.environ['PROJECT_ID'],
 }
 dataset = ld.StreamingDataset("gs://my-bucket/my-data", storage_options=gcp_storage_options)
 
 # Read data from Azure
 azure_storage_options={
-    "account_name": "azure_account_name",
-    "account_key": os.environ['AZURE_ACCOUNT_ACCESS_KEY']
+    "account_url": f"https://{os.environ['AZURE_ACCOUNT_NAME']}.blob.core.windows.net",
+    "credential": os.environ['AZURE_ACCOUNT_ACCESS_KEY']
 }
 dataset = ld.StreamingDataset("azure://my-bucket/my-data", storage_options=azure_storage_options)
 ```
 
-- For more details on which storage options are supported, please refer to:
-  - [AWS S3 storage options](https://github.com/fsspec/s3fs/blob/main/s3fs/core.py#L176)
-  - [GCS storage options](https://github.com/fsspec/gcsfs/blob/main/gcsfs/core.py#L154)
-  - [Azure storage options](https://github.com/fsspec/adlfs/blob/main/adlfs/spec.py#L124)
-
 </details>  
 
 <details>

requirements.txt

@@ -2,7 +2,5 @@ torch
 lightning-utilities
 filelock
 numpy
-# boto3
+boto3
 requests
-fsspec
-fsspec[s3] # aws s3

requirements/extras.txt

@@ -5,5 +5,3 @@ pyarrow
 tqdm
 lightning-sdk ==0.1.17 # Must be pinned to ensure compatibility
 google-cloud-storage
-fsspec[gs] # google cloud storage
-fsspec[abfs] # azure blob

src/litdata/constants.py

@@ -85,4 +85,3 @@
 _TIME_FORMAT = "%Y-%m-%d_%H-%M-%S.%fZ"
 _IS_IN_STUDIO = bool(os.getenv("LIGHTNING_CLOUD_PROJECT_ID", None)) and bool(os.getenv("LIGHTNING_CLUSTER_ID", None))
 _ENABLE_STATUS = bool(int(os.getenv("ENABLE_STATUS_REPORT", "0")))
-_SUPPORTED_CLOUD_PROVIDERS = ["s3", "gs", "azure", "abfs"]