Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Handle checksums on copy #242

Merged
merged 7 commits into from
May 3, 2023
Merged

Handle checksums on copy #242

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
bfec8ea
Handle checksums on copy
rapkyt Jun 7, 2022
81a4c33
Support flexible checksum copy
rapkyt Jun 17, 2022
18f992a
Add changelog for checksum support when copying in parts.
jonathan343 Apr 12, 2023
9540bb0
Ran linter.
jonathan343 Apr 12, 2023
45a0041
Merge branch 'develop' of https://github.com/boto/s3transfer into s3-…
jonathan343 Apr 14, 2023
888300c
Added functional tests related to ChecksumAlgorithm support.
jonathan343 Apr 19, 2023
13a1aa4
Fixed typo & linted code.
jonathan343 Apr 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 13 additions & 2 deletions s3transfer/copies.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -352,7 +352,8 @@ def _main(
{'Etag': etag_value, 'PartNumber': part_number}

This value can be appended to a list to be used to complete
the multipart upload.
the multipart upload. if checksum is in the response, it will be included
in such dictionary
rapkyt marked this conversation as resolved.
Show resolved Hide resolved
"""
response = client.upload_part_copy(
CopySource=copy_source,
Expand All @@ -365,4 +366,14 @@ def _main(
for callback in callbacks:
callback(bytes_transferred=size)
etag = response['CopyPartResult']['ETag']
return {'ETag': etag, 'PartNumber': part_number}
part = {'ETag': etag, 'PartNumber': part_number}
checksums = [
'ChecksumCRC32',
'ChecksumCRC32C',
'ChecksumSHA1',
'ChecksumSHA256',
]
for checksum in checksums:
if checksum in response['CopyPartResult']:
part[checksum] = response['CopyPartResult'][checksum]
return part
rapkyt marked this conversation as resolved.
Show resolved Hide resolved
120 changes: 120 additions & 0 deletions tests/unit/test_copies.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,10 @@ def setUp(self):
self.upload_id = 'myuploadid'
self.part_number = 1
self.result_etag = 'my-etag'
self.checksum_crc32 = 'my-checksum_crc32'
self.checksum_crc32c = 'my-checksum_crc32c'
self.checksum_sha1 = 'my-checksum_sha1'
self.checksum_sha256 = 'my-checksum_sha256'

def get_copy_task(self, **kwargs):
default_kwargs = {
Expand Down Expand Up @@ -133,6 +137,122 @@ def test_main(self):
)
self.stubber.assert_no_pending_responses()

def test_main_checksum_crc32(self):
rapkyt marked this conversation as resolved.
Show resolved Hide resolved
self.stubber.add_response(
'upload_part_copy',
service_response={
'CopyPartResult': {
'ETag': self.result_etag,
'ChecksumCRC32': self.checksum_crc32,
}
},
expected_params={
'Bucket': self.bucket,
'Key': self.key,
'CopySource': self.copy_source,
'UploadId': self.upload_id,
'PartNumber': self.part_number,
'CopySourceRange': self.copy_source_range,
},
)
task = self.get_copy_task()
self.assertEqual(
task(),
{
'PartNumber': self.part_number,
'ETag': self.result_etag,
'ChecksumCRC32': self.checksum_crc32,
},
)
self.stubber.assert_no_pending_responses()

def test_main_checksum_crc32c(self):
self.stubber.add_response(
'upload_part_copy',
service_response={
'CopyPartResult': {
'ETag': self.result_etag,
'ChecksumCRC32C': self.checksum_crc32c,
}
},
expected_params={
'Bucket': self.bucket,
'Key': self.key,
'CopySource': self.copy_source,
'UploadId': self.upload_id,
'PartNumber': self.part_number,
'CopySourceRange': self.copy_source_range,
},
)
task = self.get_copy_task()
self.assertEqual(
task(),
{
'PartNumber': self.part_number,
'ETag': self.result_etag,
'ChecksumCRC32C': self.checksum_crc32c,
},
)
self.stubber.assert_no_pending_responses()

def test_main_checksum_sha1(self):
self.stubber.add_response(
'upload_part_copy',
service_response={
'CopyPartResult': {
'ETag': self.result_etag,
'ChecksumSHA1': self.checksum_sha1,
}
},
expected_params={
'Bucket': self.bucket,
'Key': self.key,
'CopySource': self.copy_source,
'UploadId': self.upload_id,
'PartNumber': self.part_number,
'CopySourceRange': self.copy_source_range,
},
)
task = self.get_copy_task()
self.assertEqual(
task(),
{
'PartNumber': self.part_number,
'ETag': self.result_etag,
'ChecksumSHA1': self.checksum_sha1,
},
)
self.stubber.assert_no_pending_responses()

def test_main_checksum_sha256(self):
self.stubber.add_response(
'upload_part_copy',
service_response={
'CopyPartResult': {
'ETag': self.result_etag,
'ChecksumSHA256': self.checksum_sha256,
}
},
expected_params={
'Bucket': self.bucket,
'Key': self.key,
'CopySource': self.copy_source,
'UploadId': self.upload_id,
'PartNumber': self.part_number,
'CopySourceRange': self.copy_source_range,
},
)
task = self.get_copy_task()
self.assertEqual(
task(),
{
'PartNumber': self.part_number,
'ETag': self.result_etag,
'ChecksumSHA256': self.checksum_sha256,
},
)
self.stubber.assert_no_pending_responses()

def test_extra_args(self):
self.extra_args['RequestPayer'] = 'requester'
self.stubber.add_response(
Expand Down