Skip to content

Commit

Permalink
fix flags setting bug (#91)
Browse files Browse the repository at this point in the history
Fixing a bug in #90, didn't set the
flag correctly so cache was not disabled, so xet cp download didn't get
the maximum speedup.

Confirmed after this fix, downloading 137.9 GB data reaching 4.5 Gbps
(bottlenecked by disk) in us-west, matching the previously [benchmarked
results](https://www.notion.so/xethub/xet-cp-download-benchmark-655e0d5fa4ea45e5817fee6f99ad8b06).
```
(.env) ubuntu@ip-10-0-27-153:~/data/pyxet/python/pyxet$ time xet cp -r -p 3 xet://XetHub/Llama2/main/models/llama-2-70b-chat ~/data
...
real    4m6.189s
user    2m2.321s 
sys     4m58.232s

```
  • Loading branch information
seanses authored Sep 1, 2023
1 parent af05769 commit 62216de
Show file tree
Hide file tree
Showing 2 changed files with 3 additions and 3 deletions.
4 changes: 2 additions & 2 deletions python/pyxet/pyxet/file_system.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import json
import sys
from urllib.parse import urlparse
from enum import Enum
from enum import IntEnum

import fsspec
import os
Expand Down Expand Up @@ -32,7 +32,7 @@ def open(file_url, mode="rb", **kwargs):
fs = XetFS()
return fs._open(file_url, mode=mode, **kwargs)

class XetFSOpenFlags(Enum):
class XetFSOpenFlags(IntEnum):
FILE_FLAG_NO_BUFFERING = 0x20000000

class XetFS(fsspec.spec.AbstractFileSystem):
Expand Down
2 changes: 1 addition & 1 deletion python/pyxet/pyxet/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ def _single_file_copy(src_fs, src_path, dest_fs, dest_path,

# Fasttrack for downloading a file to local
if src_fs.protocol == "xet" and dest_fs.protocol == "file":
with src_fs.open(src_path, "rb", {"flags": XetFSOpenFlags.FILE_FLAG_NO_BUFFERING}) as source_file:
with src_fs.open(src_path, "rb", flags=XetFSOpenFlags.FILE_FLAG_NO_BUFFERING) as source_file:
source_file.get(dest_path)
else:
with src_fs.open(src_path, "rb") as source_file:
Expand Down

0 comments on commit 62216de

Please sign in to comment.