Skip to content

Commit

Permalink
fix: delete files
Browse files Browse the repository at this point in the history
  • Loading branch information
ervandagadzhanyan committed Oct 15, 2024
1 parent 32ab5b9 commit ccaaed9
Show file tree
Hide file tree
Showing 5 changed files with 34 additions and 15 deletions.
6 changes: 4 additions & 2 deletions assets/assets/routers/files_router.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import minio
import sqlalchemy.orm
import sqlalchemy_filters.exceptions
from badgerdoc_storage import storage as bd_storage

from assets import db, exceptions, schemas, utils

Expand Down Expand Up @@ -148,16 +149,17 @@ async def delete_files(
)
continue

storage = bd_storage.get_storage(x_current_tenant)
f_name = file.original_name
minio_file_name = "files/" + str(file_id)
f_original_ext = file.original_ext
minio_ = utils.minio_utils.delete_one_from_minio(
minio_ = utils.minio_utils.delete_one_from_storage(
bucket_name, minio_file_name, storage
)
minio_originals = 1
if f_original_ext:
minio_origin_file_name = "files/origins/" + str(file_id)
minio_originals = utils.minio_utils.delete_one_from_minio(
minio_originals = utils.minio_utils.delete_one_from_storage(
bucket_name, minio_origin_file_name, storage
)
db_ = db.service.delete_file_from_db(session, file_id)
Expand Down
21 changes: 15 additions & 6 deletions assets/assets/utils/minio_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -236,15 +236,16 @@ def upload_thumbnail(
return True


def delete_one_from_minio(bucket: str, obj: str, client: minio.Minio) -> bool:
def delete_one_from_storage(
bucket: str, obj: str, storage: bd_storage.BadgerDocStorage
) -> bool:
try:
objects = client.list_objects(bucket, obj, recursive=True)
names = [a.object_name for a in objects]
if not names:
objects = storage.list_objects(obj)
if not objects:
logger_.error(f"{obj} does not exist in bucket {bucket}")
return False
for name in names:
client.remove_object(bucket, name)
for name in objects:
remove_recursive(name, storage)
except urllib3.exceptions.MaxRetryError as e:
logger_.error(f"Connection error - detail: {e}")
return False
Expand All @@ -255,6 +256,14 @@ def delete_one_from_minio(bucket: str, obj: str, client: minio.Minio) -> bool:
return True


def remove_recursive(path: str, storage: bd_storage.BadgerDocStorage) -> None:
if not path.endswith("/"):
storage.remove(path)
return None
for _path in storage.list_objects(path):
remove_recursive(_path, storage)


def stream_minio(
path: str, bucket: str, storage: minio.Minio
) -> urllib3.response.HTTPResponse:
Expand Down
4 changes: 3 additions & 1 deletion assets/tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,9 @@ def client_app_main(
)
app.dependency_overrides[tenant] = lambda: setup_tenant

with patch.object(minio_utils, "delete_one_from_minio", return_value=True):
with patch.object(
minio_utils, "delete_one_from_storage", return_value=True
):
client = TestClient(app)
yield client

Expand Down
10 changes: 5 additions & 5 deletions assets/tests/test_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
update_file_status,
)
from assets.schemas import FileProcessingStatus
from assets.utils.minio_utils import check_bucket, delete_one_from_minio
from assets.utils.minio_utils import check_bucket, delete_one_from_storage


@pytest.fixture
Expand Down Expand Up @@ -63,18 +63,18 @@ def test_check_bucket_positive(minio_mock_exists_bucket_true):
check_bucket(random_name, minio_mock_exists_bucket_true)


def test_delete_one_from_minio(minio_mock_exists_bucket_true):
with patch("tests.test_helpers.delete_one_from_minio") as mock_:
def test_delete_one_from_storage(minio_mock_exists_bucket_true):
with patch("tests.test_helpers.delete_one_from_storage") as mock_:
mock_.side_effect = [True, False]
random_name = uuid.uuid4().hex
minio_mock_exists_bucket_true.fput_object(
random_name, "testfile", Mock()
)
x = delete_one_from_minio(
x = delete_one_from_storage(
random_name, "testfile", minio_mock_exists_bucket_true
)
assert x
y = delete_one_from_minio(
y = delete_one_from_storage(
random_name, "testfile", minio_mock_exists_bucket_true
)
assert not y
Expand Down
8 changes: 7 additions & 1 deletion lib/badgerdoc_storage/src/badgerdoc_storage/storage.py
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,13 @@ def create_tenant_dir(self) -> bool:
return True

def remove(self, file: str) -> None:
raise NotImplementedError("Method not implemented")
bucket_name = self.__get_bucket_name()
try:
self.s3_resource.Object(bucket_name, file).delete()
except ClientError as err:
raise BadgerDocStorageError(
f"Unable to delete file: {file}"
) from err


class BadgerDocAzureStorage:
Expand Down

0 comments on commit ccaaed9

Please sign in to comment.