diff --git a/README-Android.md b/README-Android.md index 68b655e..f98195c 100644 --- a/README-Android.md +++ b/README-Android.md @@ -1,3 +1,13 @@ +Version 3.23.9 +New features: +1. Allow you set\get\list\delete bucket inventory configuration +2. Added client side encryption feature +3. Allow you config AbortIncompleteMultipartUpload in LifecycleConfiguration + +Third-party dependence: +1. Replace okio 3.5.0 with okio 2.10.0 +2. Replace okhttp 4.11.0 with okhttp 4.10.0 +----------------------------------------------------------------------------------- Version 3.23.5 New features: 1. Allow you put Object in two Buckets by calling putObjectInTwoBucket diff --git a/README-Java.md b/README-Java.md index 9e53ca6..7800ca4 100644 --- a/README-Java.md +++ b/README-Java.md @@ -1,3 +1,13 @@ +Version 3.23.9 +New features: +1. Allow you set\get\list\delete bucket inventory configuration +2. Added client side encryption feature +3. Allow you config AbortIncompleteMultipartUpload in LifecycleConfiguration + +Third-party dependence: +1. Replace okio 3.5.0 with okio 2.10.0 +2. Replace okhttp 4.11.0 with okhttp 4.10.0 +----------------------------------------------------------------------------------- Version 3.23.5 New features: 1. Allow you put Object in two Buckets by calling putObjectInTwoBucket diff --git a/README.MD b/README.MD index dbec4c6..d58773f 100644 --- a/README.MD +++ b/README.MD @@ -1,3 +1,13 @@ +Version 3.23.9 +New features: +1. Allow you set\get\list\delete bucket inventory configuration +2. Added client side encryption feature +3. Allow you config AbortIncompleteMultipartUpload in LifecycleConfiguration + +Third-party dependence: +1. Replace okio 3.5.0 with okio 2.10.0 +2. Replace okhttp 4.11.0 with okhttp 4.10.0 +----------------------------------------------------------------------------------- Version 3.23.5 New features: 1. Allow you put Object in two Buckets by calling putObjectInTwoBucket diff --git a/README_CN.MD b/README_CN.MD index 75700c8..31609bd 100644 --- a/README_CN.MD +++ b/README_CN.MD @@ -1,3 +1,13 @@ +Version 3.23.9 +New features: +1. 新增配置桶清单接口 +2. 新增客户端加密 +3. 支持在生命周期规则中配置碎片过期时间 + +Third-party dependence: +1. 使用 okio 3.5.0 替代 okio 2.10.0 +2. 使用 okhttp 4.11.0 替代 okhttp 4.10.0 +----------------------------------------------------------------------------------- Version 3.23.5 New features: 1. 新增双写桶能力 diff --git a/app/src/main/java/com/obs/services/AbstractBucketClient.java b/app/src/main/java/com/obs/services/AbstractBucketClient.java index 6681a33..68fbef2 100644 --- a/app/src/main/java/com/obs/services/AbstractBucketClient.java +++ b/app/src/main/java/com/obs/services/AbstractBucketClient.java @@ -53,6 +53,12 @@ import com.obs.services.model.SetBucketRequestPaymentRequest; import com.obs.services.model.SetBucketStoragePolicyRequest; import com.obs.services.model.SetBucketVersioningRequest; +import com.obs.services.model.inventory.SetInventoryConfigurationRequest; +import com.obs.services.model.inventory.GetInventoryConfigurationRequest; +import com.obs.services.model.inventory.DeleteInventoryConfigurationRequest; +import com.obs.services.model.inventory.ListInventoryConfigurationRequest; +import com.obs.services.model.inventory.GetInventoryConfigurationResult; +import com.obs.services.model.inventory.ListInventoryConfigurationResult; public abstract class AbstractBucketClient extends AbstractDeprecatedBucketClient { /* @@ -917,4 +923,64 @@ public HeaderResponse action() throws ServiceException { } }); } + + @Override + public HeaderResponse setInventoryConfiguration(SetInventoryConfigurationRequest request) throws ObsException { + ServiceUtils.assertParameterNotNull(request, "request is null"); + ServiceUtils.assertParameterNotNull(request.getBucketName(), "bucketName is null"); + ServiceUtils.assertParameterNotNull(request.getInventoryConfiguration(), "inventoryConfiguration is null"); + + return this.doActionWithResult("setInventoryConfiguration", request.getBucketName(), + new ActionCallbackWithResult() { + @Override + public HeaderResponse action() throws ServiceException { + return AbstractBucketClient.this.setInventoryConfigurationImpl(request); + } + }); + } + + @Override + public GetInventoryConfigurationResult getInventoryConfiguration(GetInventoryConfigurationRequest request) throws ObsException { + ServiceUtils.assertParameterNotNull(request, "request is null"); + ServiceUtils.assertParameterNotNull2(request.getBucketName(), "bucketName is null"); + ServiceUtils.assertParameterNotNull2(request.getConfigurationId(), "configurationId is null"); + + return this.doActionWithResult("getInventoryConfiguration", request.getBucketName(), + new ActionCallbackWithResult() { + @Override + public GetInventoryConfigurationResult action() throws ServiceException { + return AbstractBucketClient.this.getInventoryConfigurationImpl(request); + } + }); + } + + @Override + public ListInventoryConfigurationResult listInventoryConfiguration(ListInventoryConfigurationRequest request) throws ObsException { + ServiceUtils.assertParameterNotNull(request, "request is null"); + ServiceUtils.assertParameterNotNull2(request.getBucketName(), "bucketName is null"); + + return this.doActionWithResult("listInventoryConfiguration", request.getBucketName(), + new ActionCallbackWithResult() { + @Override + public ListInventoryConfigurationResult action() throws ServiceException { + return AbstractBucketClient.this.listInventoryConfigurationImpl(request); + } + }); + } + + @Override + public HeaderResponse deleteInventoryConfiguration(DeleteInventoryConfigurationRequest request) throws ObsException { + ServiceUtils.assertParameterNotNull(request, "request is null"); + ServiceUtils.assertParameterNotNull2(request.getBucketName(), "bucketName is null"); + ServiceUtils.assertParameterNotNull2(request.getConfigurationId(), "configurationId is null"); + + return this.doActionWithResult("deleteInventoryConfiguration", request.getBucketName(), + new ActionCallbackWithResult() { + @Override + public HeaderResponse action() throws ServiceException { + return AbstractBucketClient.this.deleteInventoryConfigurationImpl(request); + } + }); + + } } diff --git a/app/src/main/java/com/obs/services/IObsClient.java b/app/src/main/java/com/obs/services/IObsClient.java index 626d004..71b0a72 100644 --- a/app/src/main/java/com/obs/services/IObsClient.java +++ b/app/src/main/java/com/obs/services/IObsClient.java @@ -125,6 +125,12 @@ import com.obs.services.model.WebsiteConfiguration; import com.obs.services.model.crr.GetCrrProgressRequest; import com.obs.services.model.crr.GetCrrProgressResult; +import com.obs.services.model.inventory.SetInventoryConfigurationRequest; +import com.obs.services.model.inventory.GetInventoryConfigurationRequest; +import com.obs.services.model.inventory.DeleteInventoryConfigurationRequest; +import com.obs.services.model.inventory.ListInventoryConfigurationRequest; +import com.obs.services.model.inventory.GetInventoryConfigurationResult; +import com.obs.services.model.inventory.ListInventoryConfigurationResult; import com.obs.services.model.select.SelectObjectRequest; import com.obs.services.model.select.SelectObjectResult; import com.obs.services.model.ObjectTagResult; @@ -2293,6 +2299,14 @@ UploadPartResult uploadPart(String bucketName, String objectKey, String uploadId */ HeaderResponse deleteBucketDirectColdAccess(BaseBucketRequest request) throws ObsException; + HeaderResponse setInventoryConfiguration(SetInventoryConfigurationRequest request) throws ObsException; + + GetInventoryConfigurationResult getInventoryConfiguration(GetInventoryConfigurationRequest request) throws ObsException; + + ListInventoryConfigurationResult listInventoryConfiguration(ListInventoryConfigurationRequest request) throws ObsException; + + HeaderResponse deleteInventoryConfiguration(DeleteInventoryConfigurationRequest request) throws ObsException; + /** * Close ObsClient and release connection resources. * diff --git a/app/src/main/java/com/obs/services/crypto/CTRCipherGenerator.java b/app/src/main/java/com/obs/services/crypto/CTRCipherGenerator.java new file mode 100644 index 0000000..ce07fcb --- /dev/null +++ b/app/src/main/java/com/obs/services/crypto/CTRCipherGenerator.java @@ -0,0 +1,304 @@ +/** + * Copyright 2019 Huawei Technologies Co.,Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed + * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.obs.services.crypto; + +import com.obs.log.ILogger; +import com.obs.log.LoggerBuilder; +import com.obs.services.exception.ObsException; +import com.obs.services.internal.utils.ServiceUtils; + +import java.io.BufferedInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.security.InvalidAlgorithmParameterException; +import java.security.InvalidKeyException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import javax.crypto.BadPaddingException; +import javax.crypto.Cipher; +import javax.crypto.CipherInputStream; +import javax.crypto.IllegalBlockSizeException; +import javax.crypto.NoSuchPaddingException; +import javax.crypto.spec.IvParameterSpec; +import javax.crypto.spec.SecretKeySpec; + +public class CTRCipherGenerator { + public static final String ENCRYPTED_ALGORITHM = "AES256-Ctr/iv_base64/NoPadding"; + public static final int CRYPTO_KEY_BYTES_LEN = 32; + public static final int CRYPTO_IV_BYTES_LEN = 16; + private String masterKeyInfo; + private byte[] cryptoIvBytes; + private byte[] cryptoKeyBytes; + + private boolean needSha256; + + private SecureRandom secureRandom; + + protected static final String AES_ALGORITHM = "AES/CTR/NoPadding"; + + static int sha256BufferLen = 65536; + + public SecureRandom getSecureRandom() { + return secureRandom; + } + + public void setSecureRandom(SecureRandom secureRandom) { + this.secureRandom = secureRandom; + } + + public String getMasterKeyInfo() { + return masterKeyInfo; + } + + public void setMasterKeyInfo(String masterKeyInfo) { + this.masterKeyInfo = masterKeyInfo; + } + + public byte[] getCryptoIvBytes() { + return cryptoIvBytes; + } + + public byte[] getRandomCryptoIvBytes() { + return getRandomBytes(CRYPTO_IV_BYTES_LEN); + } + + public void setCryptoIvBytes(byte[] cryptoIvBytes) { + this.cryptoIvBytes = cryptoIvBytes; + } + + public byte[] getCryptoKeyBytes() { + return cryptoKeyBytes; + } + + public byte[] getRandomCryptoKeyBytes() { + return getRandomBytes(CRYPTO_KEY_BYTES_LEN); + } + public byte[] getRandomBytes(int randomBytesLen) { + byte[] randomBytes; + randomBytes = new byte[randomBytesLen]; + secureRandom.nextBytes(randomBytes); + return randomBytes; + } + + public void setCryptoKeyBytes(byte[] cryptoKeyBytes) { + this.cryptoKeyBytes = cryptoKeyBytes; + } + + public boolean isNeedSha256() { + return needSha256; + } + + public void setNeedSha256(boolean needSha256) { + this.needSha256 = needSha256; + } + + public CTRCipherGenerator( + String masterKeyInfo, + byte[] cryptoIvBytes, + byte[] cryptoKeyBytes, + boolean needSha256, + SecureRandom secureRandom) { + this.masterKeyInfo = masterKeyInfo; + this.cryptoIvBytes = cryptoIvBytes; + this.cryptoKeyBytes = cryptoKeyBytes; + this.needSha256 = needSha256; + this.secureRandom = secureRandom; + } + + public CTRCipherGenerator( + String masterKeyInfo, byte[] cryptoKeyBytes, boolean needSha256, SecureRandom secureRandom) { + this.cryptoKeyBytes = cryptoKeyBytes; + this.masterKeyInfo = masterKeyInfo; + this.secureRandom = secureRandom; + this.needSha256 = needSha256; + } + + public CipherInputStream getAES256DecryptedStream( + InputStream ciphertextInput, byte[] object_CryptoIvBytes, byte[] object_CryptoKeyBytes) + throws NoSuchPaddingException, NoSuchAlgorithmException, InvalidAlgorithmParameterException, + InvalidKeyException { + SecretKeySpec keySpec = new SecretKeySpec(object_CryptoKeyBytes, "AES"); + IvParameterSpec ivSpec = new IvParameterSpec(object_CryptoIvBytes); + Cipher cipher = Cipher.getInstance(AES_ALGORITHM); + cipher.init(Cipher.DECRYPT_MODE, keySpec, ivSpec); + return new CipherInputStream(ciphertextInput, cipher); + } + + public CipherInputStream getAES256EncryptedStream( + InputStream plaintextInput, byte[] object_CryptoIvBytes, byte[] object_CryptoKeyBytes) + throws NoSuchPaddingException, NoSuchAlgorithmException, InvalidAlgorithmParameterException, + InvalidKeyException { + SecretKeySpec keySpec = new SecretKeySpec(object_CryptoKeyBytes, "AES"); + IvParameterSpec ivSpec = new IvParameterSpec(object_CryptoIvBytes); + Cipher cipher = Cipher.getInstance(AES_ALGORITHM); + cipher.init(Cipher.ENCRYPT_MODE, keySpec, ivSpec); + + return new CipherInputStream(plaintextInput, cipher); + } + + public static String getBase64Info(byte[] cryptoInfo) { + return ServiceUtils.toBase64(cryptoInfo); + } + + public static byte[] getBytesFromBase64(String cryptoInfo) throws UnsupportedEncodingException { + return ServiceUtils.fromBase64(cryptoInfo); + } + + public static byte[] getAESEncryptedBytes( + byte[] plainText, int plainTextOffset, int plainTextLength, byte[] aesKeyBytes, byte[] aesIvBytes) + throws IllegalBlockSizeException, BadPaddingException, InvalidAlgorithmParameterException, + InvalidKeyException, NoSuchPaddingException, NoSuchAlgorithmException { + SecretKeySpec keySpec = new SecretKeySpec(aesKeyBytes, "AES"); + IvParameterSpec ivSpec = new IvParameterSpec(aesIvBytes); + Cipher cipher = Cipher.getInstance(AES_ALGORITHM); + cipher.init(Cipher.ENCRYPT_MODE, keySpec, ivSpec); + return cipher.doFinal(plainText, plainTextOffset, plainTextLength); + } + + public void setBase64AES256Key(String cryptoKeyBase64) throws UnsupportedEncodingException, ObsException { + byte[] keyBytes = ServiceUtils.fromBase64(cryptoKeyBase64); + if (keyBytes.length != CRYPTO_KEY_BYTES_LEN) { + throw new ObsException("cryptoKeyBytes.length must be " + CRYPTO_KEY_BYTES_LEN); + } + cryptoKeyBytes = keyBytes; + } + + public void setBase64AES256Iv(String cryptoIvBase64) throws UnsupportedEncodingException, ObsException { + byte[] ivBytes = ServiceUtils.fromBase64(cryptoIvBase64); + if (ivBytes.length != CRYPTO_IV_BYTES_LEN) { + throw new ObsException("cryptoIvBytes.length must be " + CRYPTO_IV_BYTES_LEN); + } else { + cryptoIvBytes = ivBytes; + } + } + + class SHA256Info { + protected String sha256ForPlainText; + protected String sha256ForAESEncrypted; + + public String getSha256ForPlainText() { + return sha256ForPlainText; + } + + public void setSha256ForPlainText(String sha256ForPlainText) { + this.sha256ForPlainText = sha256ForPlainText; + } + + public String getSha256ForAESEncrypted() { + return sha256ForAESEncrypted; + } + + public void setSha256ForAESEncrypted(String sha256ForAESEncrypted) { + this.sha256ForAESEncrypted = sha256ForAESEncrypted; + } + + public SHA256Info(String sha256ForPlainText, String sha256ForAESEncrypted) { + this.sha256ForPlainText = sha256ForPlainText; + this.sha256ForAESEncrypted = sha256ForAESEncrypted; + } + } + + public static byte[] getFileSha256Bytes(String filePath) throws IOException, NoSuchAlgorithmException { + MessageDigest fileSha256Digest = MessageDigest.getInstance("SHA-256"); + try (FileInputStream fileInputStream = new FileInputStream(filePath)) { + byte[] buffer = new byte[sha256BufferLen]; + int bytesRead; + while ((bytesRead = fileInputStream.read(buffer, 0, buffer.length)) != -1) { + // 更新文件的sha256 + fileSha256Digest.update(buffer, 0, bytesRead); + } + } + return fileSha256Digest.digest(); + } + + public SHA256Info computeSHA256HashAES( + InputStream plainTextStream, + byte[] object_CryptoIvBytes, + byte[] object_CryptoKeyBytes, + boolean needPlainTextSha256) + throws NoSuchAlgorithmException, IOException, ObsException { + BufferedInputStream bis = null; + try { + SecretKeySpec keySpec = new SecretKeySpec(object_CryptoKeyBytes, "AES"); + IvParameterSpec ivSpec = new IvParameterSpec(object_CryptoIvBytes); + Cipher cipher = Cipher.getInstance(AES_ALGORITHM); + cipher.init(Cipher.ENCRYPT_MODE, keySpec, ivSpec); + bis = new BufferedInputStream(plainTextStream); + MessageDigest plainTextSha256 = MessageDigest.getInstance("SHA-256"); + MessageDigest encryptedInfoSha256 = MessageDigest.getInstance("SHA-256"); + byte[] buffer = new byte[sha256BufferLen]; + int bytesRead; + while ((bytesRead = bis.read(buffer, 0, buffer.length)) != -1) { + // 加密 + byte[] encryptedData = cipher.update(buffer, 0, bytesRead); + // 计算加密块的sha256 + encryptedInfoSha256.update(encryptedData, 0, bytesRead); + if (needPlainTextSha256) { + // 计算明文的sha256 + plainTextSha256.update(buffer, 0, bytesRead); + } + } + + SHA256Info sha256Info = new SHA256Info("", ""); + byte[] encryptedInfoSha256Bytes = encryptedInfoSha256.digest(); + // 转为16进制 + StringBuilder sha256Builder = new StringBuilder(); + for (byte aB : encryptedInfoSha256Bytes) { + sha256Builder.append(String.format("%02x", aB)); + } + sha256Info.setSha256ForAESEncrypted(sha256Builder.toString()); + + if (needPlainTextSha256) { + byte[] plainTextSha256Bytes = plainTextSha256.digest(); + sha256Builder.setLength(0); + for (byte aB : plainTextSha256Bytes) { + sha256Builder.append(String.format("%02x", aB)); + } + sha256Info.setSha256ForPlainText(sha256Builder.toString()); + } + return sha256Info; + } catch (Exception e) { + throw ServiceUtils.changeFromException(e); + } finally { + if (bis != null) { + try { + bis.close(); + } catch (IOException e) { + if (log.isWarnEnabled()) { + log.warn("close failed.", e); + } + } + } + } + } + + public static int getSha256BufferLen() { + return sha256BufferLen; + } + + public static void setSha256BufferLen(int sha256BufferLen) { + CTRCipherGenerator.sha256BufferLen = sha256BufferLen; + } + + public static final String ENCRYPTED_ALGORITHM_META_NAME = "encrypted-algorithm"; + public static final String ENCRYPTED_START_META_NAME = "encrypted-start"; + public static final String MASTER_KEY_INFO_META_NAME = "master-key-info"; + public static final String PLAINTEXT_SHA_256_META_NAME = "plaintext-sha256"; + public static final String PLAINTEXT_CONTENT_LENGTH_META_NAME = "plaintext-content-length"; + public static final String ENCRYPTED_SHA_256_META_NAME = "encrypted-sha256"; + private static final ILogger log = LoggerBuilder.getLogger(CTRCipherGenerator.class); +} diff --git a/app/src/main/java/com/obs/services/crypto/CryptoObsClient.java b/app/src/main/java/com/obs/services/crypto/CryptoObsClient.java new file mode 100644 index 0000000..394e036 --- /dev/null +++ b/app/src/main/java/com/obs/services/crypto/CryptoObsClient.java @@ -0,0 +1,455 @@ +/** + * Copyright 2019 Huawei Technologies Co.,Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed + * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.obs.services.crypto; + +import static com.obs.services.crypto.CTRCipherGenerator.ENCRYPTED_ALGORITHM_META_NAME; +import static com.obs.services.crypto.CTRCipherGenerator.ENCRYPTED_START_META_NAME; +import static com.obs.services.crypto.CTRCipherGenerator.getBase64Info; +import static com.obs.services.crypto.CtrRSACipherGenerator.ENCRYPTED_AES_KEY_META_NAME; + +import com.obs.log.ILogger; +import com.obs.log.LoggerBuilder; +import com.obs.services.IObsCredentialsProvider; +import com.obs.services.ObsClient; +import com.obs.services.ObsConfiguration; +import com.obs.services.crypto.CTRCipherGenerator.SHA256Info; +import com.obs.services.exception.ObsException; +import com.obs.services.internal.Constants; +import com.obs.services.internal.ObsConstraint; +import com.obs.services.internal.ProgressManager; +import com.obs.services.internal.ServiceException; +import com.obs.services.internal.SimpleProgressManager; +import com.obs.services.internal.io.ProgressInputStream; +import com.obs.services.internal.trans.NewTransResult; +import com.obs.services.internal.utils.JSONChange; +import com.obs.services.internal.utils.ServiceUtils; +import com.obs.services.model.AccessControlList; +import com.obs.services.model.AuthTypeEnum; +import com.obs.services.model.GetObjectRequest; +import com.obs.services.model.ObjectMetadata; +import com.obs.services.model.ObsObject; +import com.obs.services.model.PutObjectRequest; +import com.obs.services.model.PutObjectResult; +import com.obs.services.model.StorageClassEnum; +import com.obs.services.model.fs.ObsFSAttribute; +import com.obs.services.model.fs.ObsFSFile; +import com.obs.services.model.fs.ReadFileResult; + +import okhttp3.Response; + +import java.io.BufferedInputStream; +import java.io.Closeable; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.charset.StandardCharsets; +import java.security.InvalidAlgorithmParameterException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.util.Objects; +import javax.crypto.BadPaddingException; +import javax.crypto.IllegalBlockSizeException; +import javax.crypto.NoSuchPaddingException; + +public class CryptoObsClient extends ObsClient { + private CTRCipherGenerator ctrCipherGenerator; + private static final ILogger log = LoggerBuilder.getLogger(CryptoObsClient.class); + + public CryptoObsClient(String endPoint, CTRCipherGenerator ctrCipherGenerator) { + super(endPoint); + this.ctrCipherGenerator = ctrCipherGenerator; + } + + public CryptoObsClient(ObsConfiguration config, CTRCipherGenerator ctrCipherGenerator) { + super(config); + this.ctrCipherGenerator = ctrCipherGenerator; + } + + public CryptoObsClient(String accessKey, String secretKey, String endPoint, CTRCipherGenerator ctrCipherGenerator) { + super(accessKey, secretKey, endPoint); + this.ctrCipherGenerator = ctrCipherGenerator; + } + + public CryptoObsClient( + String accessKey, String secretKey, ObsConfiguration config, CTRCipherGenerator ctrCipherGenerator) { + super(accessKey, secretKey, config); + this.ctrCipherGenerator = ctrCipherGenerator; + } + + public CryptoObsClient( + String accessKey, + String secretKey, + String securityToken, + String endPoint, + CTRCipherGenerator ctrCipherGenerator) { + super(accessKey, secretKey, securityToken, endPoint); + this.ctrCipherGenerator = ctrCipherGenerator; + } + + public CryptoObsClient( + String accessKey, + String secretKey, + String securityToken, + ObsConfiguration config, + CTRCipherGenerator ctrCipherGenerator) { + super(accessKey, secretKey, securityToken, config); + this.ctrCipherGenerator = ctrCipherGenerator; + } + + public CryptoObsClient(IObsCredentialsProvider provider, String endPoint, CTRCipherGenerator ctrCipherGenerator) { + super(provider, endPoint); + this.ctrCipherGenerator = ctrCipherGenerator; + } + + public CryptoObsClient( + IObsCredentialsProvider provider, ObsConfiguration config, CTRCipherGenerator ctrCipherGenerator) { + super(provider, config); + this.ctrCipherGenerator = ctrCipherGenerator; + } + + @Override + public PutObjectResult putObject(final PutObjectRequest request) throws ObsException { + ServiceUtils.assertParameterNotNull(request, "PutObjectRequest is null"); + ServiceUtils.assertParameterNotNull2(request.getBucketName(), "bucketName is null"); + ServiceUtils.assertParameterNotNull2(request.getObjectKey(), "objectKey is null"); + + return this.doActionWithResult( + "putObject", + request.getBucketName(), + new ActionCallbackWithResult() { + @Override + public PutObjectResult action() throws ServiceException { + if (null != request.getInput() && null != request.getFile()) { + throw new ServiceException("Both input and file are set, only one is allowed"); + } + return CryptoObsClient.this.putObjectImpl(request); + } + }); + } + + @Override + protected ObsFSFile putObjectImpl(PutObjectRequest request) throws ServiceException { + TransResult result = null; + Response response; + boolean isExtraAclPutRequired; + AccessControlList acl = request.getAcl(); + NewTransResult newTransResult; + if (request.getMetadata() == null) { + request.setMetadata(new ObjectMetadata()); + } + try { + if (this.ctrCipherGenerator != null) { + // 获取AES密钥和初始值 + byte[] object_CryptoIvBytes = getOrGenerateCryptoIvBytes(); + byte[] object_CryptoKeyBytes = getOrGenerateCryptoKeyBytes(); + // 计算加密前后数据的sha256值 + if (request.getFile() != null && ctrCipherGenerator.isNeedSha256()) { + request.getMetadata() + .addUserMetadata( + CTRCipherGenerator.PLAINTEXT_CONTENT_LENGTH_META_NAME, + String.valueOf(request.getFile().length())); + try (FileInputStream fileInputStream = new FileInputStream(request.getFile())) { + String content_sha256_header = Constants.OBS_HEADER_PREFIX + Constants.CONTENT_SHA256; + if (request.getUserHeaders().containsKey(content_sha256_header)) { + // 用户已设置明文数据的sha256.直接到设置自定义加密元数据中 + request.getMetadata() + .addUserMetadata( + CTRCipherGenerator.PLAINTEXT_SHA_256_META_NAME, + request.getUserHeaders().get(content_sha256_header)); + // 计算加密后的数据的sha256 + SHA256Info sha256Info = + ctrCipherGenerator.computeSHA256HashAES( + fileInputStream, object_CryptoIvBytes, object_CryptoKeyBytes, false); + + // 设置自定义加密元数据中的密文数据的sha256 + request.getMetadata() + .addUserMetadata( + CTRCipherGenerator.ENCRYPTED_SHA_256_META_NAME, + sha256Info.getSha256ForAESEncrypted()); + // 设置头域中的sha256用于验证数据完整性 + request.addUserHeaders(content_sha256_header, sha256Info.getSha256ForAESEncrypted()); + } else { + // 计算文件sha256和加密后的sha256 + SHA256Info sha256Info = + ctrCipherGenerator.computeSHA256HashAES( + fileInputStream, object_CryptoIvBytes, object_CryptoKeyBytes, true); + // 设置自定义加密元数据中的明文数据的sha256 + request.getMetadata() + .addUserMetadata( + CTRCipherGenerator.PLAINTEXT_SHA_256_META_NAME, + sha256Info.getSha256ForPlainText()); + // 设置自定义加密元数据中的密文数据的sha256 + request.getMetadata() + .addUserMetadata( + CTRCipherGenerator.ENCRYPTED_SHA_256_META_NAME, + sha256Info.getSha256ForAESEncrypted()); + // 设置头域中的sha256用于验证数据完整性 + request.addUserHeaders(content_sha256_header, sha256Info.getSha256ForAESEncrypted()); + } + request.setInput(new FileInputStream(request.getFile())); + } catch (FileNotFoundException e) { + throw new IllegalArgumentException("File doesn't exist"); + } catch (IOException e) { + throw new ServiceException(e); + } + } + + // 设置加密数据流 + request.setInput( + ctrCipherGenerator.getAES256EncryptedStream( + request.getInput(), object_CryptoIvBytes, object_CryptoKeyBytes)); + ObjectMetadata objectMetadata = request.getMetadata(); + // 设置加密信息的自定义头域 + objectMetadata.addUserMetadata(ENCRYPTED_START_META_NAME, getBase64Info(object_CryptoIvBytes)); + if (ctrCipherGenerator.getMasterKeyInfo() != null) { + objectMetadata.addUserMetadata( + CTRCipherGenerator.MASTER_KEY_INFO_META_NAME, ctrCipherGenerator.getMasterKeyInfo()); + } + if (this.ctrCipherGenerator instanceof CtrRSACipherGenerator) { + // 附件加密算法元数据信息 + objectMetadata.addUserMetadata( + ENCRYPTED_ALGORITHM_META_NAME, CtrRSACipherGenerator.ENCRYPTED_ALGORITHM); + // rsa 加密aesKey + CtrRSACipherGenerator ctrRSACipherGenerator = (CtrRSACipherGenerator) ctrCipherGenerator; + byte[] rsaEncryptedAESKey = ctrRSACipherGenerator.RSAEncrypted(object_CryptoKeyBytes); + // 将加密后的aesKey附加到元数据 + objectMetadata.addUserMetadata( + ENCRYPTED_AES_KEY_META_NAME, ServiceUtils.toBase64(rsaEncryptedAESKey)); + } else { + // 附件加密算法元数据信息 + objectMetadata.addUserMetadata( + ENCRYPTED_ALGORITHM_META_NAME, CTRCipherGenerator.ENCRYPTED_ALGORITHM); + } + } + // 后面和普通上传一致 + result = this.transPutObjectRequest(request); + isExtraAclPutRequired = !prepareRESTHeaderAcl(request.getBucketName(), result.getHeaders(), acl); + if (request.getCallback() != null) { + ServiceUtils.assertParameterNotNull(request.getCallback().getCallbackUrl(), "callbackUrl is null"); + ServiceUtils.assertParameterNotNull(request.getCallback().getCallbackBody(), "callbackBody is null"); + result.getHeaders() + .put( + (this.getProviderCredentials().getLocalAuthType(request.getBucketName()) + != AuthTypeEnum.OBS + ? Constants.V2_HEADER_PREFIX + : Constants.OBS_HEADER_PREFIX) + + Constants.CommonHeaders.CALLBACK, + ServiceUtils.toBase64( + JSONChange.objToJson(request.getCallback()).getBytes(StandardCharsets.UTF_8))); + } + // todo prepareRESTHeaderAcl 也会操作头域，下次重构可以将其合并 + newTransResult = transObjectRequestWithResult(result, request); + response = performRequest(newTransResult, true, false, false, false); + } catch (InvalidAlgorithmParameterException + | NoSuchPaddingException + | NoSuchAlgorithmException + | InvalidKeyException + | IllegalBlockSizeException + | BadPaddingException e) { + throw new ServiceException(e); + } finally { + if (result != null && result.getBody() != null && request.isAutoClose()) { + if (result.getBody() instanceof Closeable) { + ServiceUtils.closeStream((Closeable) result.getBody()); + } + } + } + + ObsFSFile ret = + new ObsFSFile( + request.getBucketName(), + request.getObjectKey(), + response.header(Constants.CommonHeaders.ETAG), + response.header(this.getIHeaders(request.getBucketName()).versionIdHeader()), + StorageClassEnum.getValueFromCode( + response.header(this.getIHeaders(request.getBucketName()).storageClassHeader())), + this.getObjectUrl(request.getBucketName(), request.getObjectKey(), request.getIsIgnorePort())); + if (request.getCallback() != null) { + try { + ret.setCallbackResponseBody(Objects.requireNonNull(response.body()).byteStream()); + } catch (Exception e) { + throw new ServiceException(e); + } + } + setHeadersAndStatus(ret, response); + if (isExtraAclPutRequired && acl != null) { + try { + putAclImpl(request.getBucketName(), request.getObjectKey(), acl, null, request.isRequesterPays()); + } catch (Exception e) { + log.warn("Try to set object acl error", e); + } + } + return ret; + } + + @Override + public ObsObject getObject(final GetObjectRequest request) throws ObsException { + ServiceUtils.assertParameterNotNull(request, "GetObjectRequest is null"); + ServiceUtils.assertParameterNotNull2(request.getObjectKey(), "objectKey is null"); + return this.doActionWithResult( + "getObject", + request.getBucketName(), + new ActionCallbackWithResult() { + @Override + public ObsObject action() throws ServiceException { + return CryptoObsClient.this.getObjectImpl(request); + } + }); + } + + @Override + protected ObsObject getObjectImpl(GetObjectRequest request) throws ServiceException { + Response response; + TransResult result = this.transGetObjectRequest(request); + if (request.getRequestParameters() != null) { + result.getParams().putAll(request.getRequestParameters()); + } + response = + performRestGet( + request.getBucketName(), + request.getObjectKey(), + result.getParams(), + result.getHeaders(), + request.getUserHeaders(), + false, + request.isEncodeHeaders()); + + ObsFSAttribute objMetadata = + this.getObsFSAttributeFromResponse(request.getBucketName(), response, request.isEncodeHeaders()); + + ReadFileResult obsObject = new ReadFileResult(); + obsObject.setObjectKey(request.getObjectKey()); + obsObject.setBucketName(request.getBucketName()); + obsObject.setMetadata(objMetadata); + + // 该接口是下载对象，需要将流返回给客户（调用方），我们不能关闭这个流 + + if (ctrCipherGenerator != null) { + String encryptedAlgorithm = + (String) + objMetadata + .getOriginalHeaders() + .get(Constants.OBS_HEADER_META_PREFIX + ENCRYPTED_ALGORITHM_META_NAME); + String encryptedStart = + (String) + objMetadata + .getOriginalHeaders() + .get(Constants.OBS_HEADER_META_PREFIX + ENCRYPTED_START_META_NAME); + + if (isValidEncryptedAlgorithm(encryptedAlgorithm)) { + byte[] cryptoKeyBytes = ctrCipherGenerator.getCryptoKeyBytes(); + if (encryptedAlgorithm.equals(CtrRSACipherGenerator.ENCRYPTED_ALGORITHM)) { + // 如果是rsa加密方式 + if (!(ctrCipherGenerator instanceof CtrRSACipherGenerator)) { + throw new ServiceException( + "wrong CipherGenerator ,need " + CtrRSACipherGenerator.class.getSimpleName()); + } else { + try { + CtrRSACipherGenerator ctrRSACipherGenerator = (CtrRSACipherGenerator) ctrCipherGenerator; + String aesEncryptedKey = + (String) + objMetadata + .getOriginalHeaders() + .get( + Constants.OBS_HEADER_META_PREFIX + + ENCRYPTED_AES_KEY_META_NAME); + // 解密rsa加密后的主密钥 + cryptoKeyBytes = + ctrRSACipherGenerator.RSADecrypted(ServiceUtils.fromBase64(aesEncryptedKey)); + } catch (NoSuchPaddingException + | IllegalBlockSizeException + | UnsupportedEncodingException + | NoSuchAlgorithmException + | BadPaddingException + | InvalidKeyException e) { + throw new ServiceException(e); + } + } + } + try { + byte[] iv = CTRCipherGenerator.getBytesFromBase64(encryptedStart); + + // 设置解密流 + obsObject.setObjectContent( + ctrCipherGenerator.getAES256DecryptedStream( + response.body().byteStream(), iv, cryptoKeyBytes)); + } catch (UnsupportedEncodingException + | InvalidAlgorithmParameterException + | NoSuchPaddingException + | NoSuchAlgorithmException + | InvalidKeyException e) { + throw new ServiceException(e); + } + } else { + log.warn("no encrypted-algorithm metadata received"); + obsObject.setObjectContent(response.body().byteStream()); + } + } else { + log.warn("CipherGenerator is null"); + obsObject.setObjectContent(response.body().byteStream()); + } + if (request.getProgressListener() != null) { + ProgressManager progressManager = + new SimpleProgressManager( + objMetadata.getContentLength(), + 0, + request.getProgressListener(), + request.getProgressInterval() > 0 + ? request.getProgressInterval() + : ObsConstraint.DEFAULT_PROGRESS_INTERVAL); + obsObject.setObjectContent(new ProgressInputStream(obsObject.getObjectContent(), progressManager)); + } + + int readBufferSize = + obsProperties.getIntProperty(ObsConstraint.READ_BUFFER_SIZE, ObsConstraint.DEFAULT_READ_BUFFER_STREAM); + if (readBufferSize > 0) { + obsObject.setObjectContent(new BufferedInputStream(obsObject.getObjectContent(), readBufferSize)); + } + return obsObject; + } + + public boolean isValidEncryptedAlgorithm(String encryptedAlgorithm) { + return encryptedAlgorithm.equals(CtrRSACipherGenerator.ENCRYPTED_ALGORITHM) + || encryptedAlgorithm.equals(CTRCipherGenerator.ENCRYPTED_ALGORITHM); + } + + protected byte[] getOrGenerateCryptoIvBytes() { + if (ctrCipherGenerator.getCryptoIvBytes() != null) { + log.info("get user-set AES iv"); + return ctrCipherGenerator.getCryptoIvBytes(); + } else { + log.info("get random AES iv"); + return ctrCipherGenerator.getRandomCryptoIvBytes(); + } + } + + protected byte[] getOrGenerateCryptoKeyBytes() { + if (ctrCipherGenerator.getCryptoKeyBytes() != null) { + log.info("get user-set AES key"); + return ctrCipherGenerator.getCryptoKeyBytes(); + } else { + log.info("get random AES key"); + return ctrCipherGenerator.getRandomCryptoKeyBytes(); + } + } + + public CTRCipherGenerator getCtrCipherGenerator() { + return ctrCipherGenerator; + } + + public void setCtrCipherGenerator(CTRCipherGenerator ctrCipherGenerator) { + this.ctrCipherGenerator = ctrCipherGenerator; + } +} diff --git a/app/src/main/java/com/obs/services/crypto/CtrRSACipherGenerator.java b/app/src/main/java/com/obs/services/crypto/CtrRSACipherGenerator.java new file mode 100644 index 0000000..be6c6b5 --- /dev/null +++ b/app/src/main/java/com/obs/services/crypto/CtrRSACipherGenerator.java @@ -0,0 +1,126 @@ +/** + * Copyright 2019 Huawei Technologies Co.,Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed + * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.obs.services.crypto; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; +import java.security.InvalidKeyException; +import java.security.KeyFactory; +import java.security.NoSuchAlgorithmException; +import java.security.PrivateKey; +import java.security.PublicKey; +import java.security.SecureRandom; +import java.security.spec.InvalidKeySpecException; +import java.security.spec.PKCS8EncodedKeySpec; +import java.security.spec.X509EncodedKeySpec; +import java.util.Base64; +import javax.crypto.BadPaddingException; +import javax.crypto.Cipher; +import javax.crypto.IllegalBlockSizeException; +import javax.crypto.NoSuchPaddingException; + +public class CtrRSACipherGenerator extends CTRCipherGenerator { + public static final String ENCRYPTED_AES_KEY_META_NAME = "encrypted-object-key"; + public static final String ENCRYPTED_ALGORITHM = "AES256-Ctr/RSA-Object-Key/NoPadding"; + private PrivateKey privateKey; + private PublicKey publicKey; + protected static final String RSA_ALGORITHM = "RSA"; + + public CtrRSACipherGenerator( + String masterKeyInfo, + boolean needSha256, + SecureRandom secureRandom, + PrivateKey privateKey, + PublicKey publicKey) + throws NoSuchAlgorithmException, InvalidKeySpecException { + super(masterKeyInfo, null, null, needSha256, secureRandom); + this.privateKey = privateKey; + this.publicKey = publicKey; + } + + public static PrivateKey importPKCS8PrivateKey(String filePath) + throws IOException, NoSuchAlgorithmException, InvalidKeySpecException { + StringBuilder sb = new StringBuilder(); + + try (BufferedReader br = new BufferedReader(new FileReader(filePath))) { + String line; + boolean startKey = false; + while ((line = br.readLine()) != null) { + if (line.startsWith("-----BEGIN PRIVATE KEY-----")) { + startKey = true; + } else if (line.startsWith("-----END PRIVATE KEY-----")) { + startKey = false; + } else if (startKey) { + sb.append(line); + } + } + } + + byte[] keyBytes = Base64.getDecoder().decode(sb.toString()); + PKCS8EncodedKeySpec spec = new PKCS8EncodedKeySpec(keyBytes); + KeyFactory kf = KeyFactory.getInstance(RSA_ALGORITHM); + return kf.generatePrivate(spec); + } + + public static PublicKey importPublicKey(String filename) + throws IOException, NoSuchAlgorithmException, InvalidKeySpecException { + StringBuilder sb = new StringBuilder(); + + try (BufferedReader br = new BufferedReader(new FileReader(filename))) { + String line; + boolean startKey = false; + while ((line = br.readLine()) != null) { + if (line.startsWith("-----BEGIN PUBLIC KEY-----")) { + startKey = true; + } else if (line.startsWith("-----END PUBLIC KEY-----")) { + startKey = false; + } else if (startKey) { + sb.append(line); + } + } + } + + byte[] keyBytes = Base64.getDecoder().decode(sb.toString()); + X509EncodedKeySpec spec = new X509EncodedKeySpec(keyBytes); + KeyFactory kf = KeyFactory.getInstance(RSA_ALGORITHM); + return kf.generatePublic(spec); + } + + public byte[] RSAEncrypted(byte[] plaintext) + throws NoSuchPaddingException, NoSuchAlgorithmException, InvalidKeyException, IllegalBlockSizeException, + BadPaddingException { + Cipher cipher = Cipher.getInstance(RSA_ALGORITHM); + // 加密 + cipher.init(Cipher.ENCRYPT_MODE, publicKey); + return cipher.doFinal(plaintext); + } + + public byte[] RSADecrypted(byte[] cipherInfo) + throws NoSuchPaddingException, NoSuchAlgorithmException, InvalidKeyException, IllegalBlockSizeException, + BadPaddingException { + Cipher cipher = Cipher.getInstance(RSA_ALGORITHM); + // 解密 + cipher.init(Cipher.DECRYPT_MODE, privateKey); + return cipher.doFinal(cipherInfo); + } + + public void setPrivateKey(PrivateKey privateKey) { + this.privateKey = privateKey; + } + + public void setPublicKey(PublicKey publicKey) { + this.publicKey = publicKey; + } +} diff --git a/app/src/main/java/com/obs/services/internal/Constants.java b/app/src/main/java/com/obs/services/internal/Constants.java index 7766f19..c1b9a80 100644 --- a/app/src/main/java/com/obs/services/internal/Constants.java +++ b/app/src/main/java/com/obs/services/internal/Constants.java @@ -213,7 +213,7 @@ public static class ObsRequestParams { public static final TimeZone GMT_TIMEZONE = TimeZone.getTimeZone("GMT"); - public static final String OBS_SDK_VERSION = "3.23.3"; + public static final String OBS_SDK_VERSION = "3.23.9"; public static final String USER_AGENT_VALUE = "obs-sdk-java/" + Constants.OBS_SDK_VERSION; @@ -262,6 +262,8 @@ public static class ObsRequestParams { public static final String POSIX = "POSIX"; + public static final String CONTENT_SHA256 = "content-sha256"; + public static final long MAX_PART_SIZE = 5 * 1024 * 1024 * 1024L; public static final long MIN_PART_SIZE = 100 * 1024L; @@ -287,7 +289,7 @@ public static class ObsRequestParams { "storagepolicy", "storageclass", "requestpayment", "versions", "versioning", "versionid", "uploads", "uploadid", "partnumber", "website", "notification", "lifecycle", "deletebucket", "delete", "cors", "restore", "tagging", "replication", "metadata", "encryption", "directcoldaccess", "mirrorrefresh", - "mirrorbacktosource", "obsbucketalias", "obsalias", "replication_progress", + "mirrorbacktosource", "obsbucketalias", "obsalias", "replication_progress", "inventory", /** * File System API */ diff --git a/app/src/main/java/com/obs/services/internal/IConvertor.java b/app/src/main/java/com/obs/services/internal/IConvertor.java index 33f0c15..742298f 100644 --- a/app/src/main/java/com/obs/services/internal/IConvertor.java +++ b/app/src/main/java/com/obs/services/internal/IConvertor.java @@ -36,6 +36,7 @@ import com.obs.services.model.WebsiteConfiguration; import com.obs.services.model.fs.FSStatusEnum; import com.obs.services.model.ObjectTagResult; +import com.obs.services.model.inventory.InventoryConfiguration; public interface IConvertor { @@ -89,4 +90,6 @@ String transBucketNotificationConfiguration(BucketNotificationConfiguration buck String transBucketDirectColdAccess(BucketDirectColdAccess access) throws ServiceException; + String transBucketInventoryConfiguration(InventoryConfiguration inventoryConfiguration) throws ServiceException; + } \ No newline at end of file diff --git a/app/src/main/java/com/obs/services/internal/RestConnectionService.java b/app/src/main/java/com/obs/services/internal/RestConnectionService.java index b0a27b2..39dfc2d 100644 --- a/app/src/main/java/com/obs/services/internal/RestConnectionService.java +++ b/app/src/main/java/com/obs/services/internal/RestConnectionService.java @@ -36,6 +36,8 @@ import okhttp3.Request; import okhttp3.RequestBody; +import static com.obs.services.internal.utils.ServiceUtils.getLoggableInfo; + public class RestConnectionService { private static final ILogger log = LoggerBuilder.getLogger(RestConnectionService.class); @@ -210,7 +212,7 @@ protected String addRequestParametersToUrlPath(String urlPath, Map> requestHeaders = requestInfo.getRequest().headers().toMultimap(); + //遍历requestHeaders并打印所有可以打印的元素 + for (Map.Entry> entry : requestHeaders.entrySet()) { + String key = entry.getKey(); + if(isInfoLoggable(key)) { + List values = entry.getValue(); + for (String value : values) { + log.debug("Headers: " + key + " = " + value); + } + } + } RetryController retryController = new RetryController(new RetryCounter( obsProperties.getIntProperty(ObsConstraint.HTTP_RETRY_MAX, @@ -702,7 +717,18 @@ private Request addBaseHeaders(Request request, String bucketName, boolean doSig builder.header(CommonHeaders.USER_AGENT, Constants.USER_AGENT_VALUE); request = builder.build(); } - log.debug("Request Headers: " + request.headers().toString().replace("\n", "|")); + + Map> requestHeaders = request.headers().toMultimap(); + //遍历requestHeaders并打印所有可以打印的元素 + for (Map.Entry> entry : requestHeaders.entrySet()) { + String key = entry.getKey(); + if(isInfoLoggable(key)) { + List values = entry.getValue(); + for (String value : values) { + log.debug("Request Headers: " + key + " = " + value); + } + } + } return request; } @@ -784,6 +810,9 @@ protected Request authorizeHttpRequest(Request request, String bucketName, Strin providerCredentials = this.getProviderCredentials(); } else { providerCredentials.setAuthType(this.getProviderCredentials().getLocalAuthType(bucketName)); + LinkedHashMap localAuthType = new LinkedHashMap<>(); + localAuthType.put(bucketName, providerCredentials.getAuthType()); + providerCredentials.setLocalAuthType(localAuthType); } if (isProviderCredentialsInValid(providerCredentials)) { if (log.isInfoEnabled()) { @@ -1025,7 +1054,8 @@ protected void renameMetadataKeys(String bucketName, Request.Builder builder, Ma for (Map.Entry entry : convertedMetadata.entrySet()) { builder.addHeader(entry.getKey(), entry.getValue()); if (log.isDebugEnabled()) { - log.debug("Added metadata to connection: " + entry.getKey() + "=" + entry.getValue()); + log.debug("Added metadata to connection: " + entry.getKey() + "=" + + getLoggableInfo(entry.getKey(), entry.getValue())); } } } @@ -1055,7 +1085,7 @@ protected void addRequestHeadersToConnection(String bucketName, Request.Builder } builder.addHeader(key, value); if (log.isDebugEnabled()) { - log.debug("Added request header to connection: " + key + "=" + value); + log.debug("Added request header to connection: " + key + "=" + getLoggableInfo(key, value)); } } } diff --git a/app/src/main/java/com/obs/services/internal/V2BucketConvertor.java b/app/src/main/java/com/obs/services/internal/V2BucketConvertor.java index 45dbbb0..f05138f 100644 --- a/app/src/main/java/com/obs/services/internal/V2BucketConvertor.java +++ b/app/src/main/java/com/obs/services/internal/V2BucketConvertor.java @@ -41,6 +41,7 @@ import com.obs.services.model.TopicConfiguration; import com.obs.services.model.fs.FSStatusEnum; import com.obs.services.model.ObjectTagResult; +import com.obs.services.model.inventory.InventoryConfiguration; public abstract class V2BucketConvertor implements IConvertor { @@ -322,4 +323,37 @@ public String transBucketDirectColdAccess(BucketDirectColdAccess access) throws throw new ServiceException("Failed to build XML document for Tagging", e); } } + + @Override + public String transBucketInventoryConfiguration(InventoryConfiguration inventoryConfiguration) throws ServiceException { + try { + OBSXMLBuilder builder = OBSXMLBuilder.create("InventoryConfiguration"); + + builder.e("Id").t(inventoryConfiguration.getConfigurationId()); + builder.e("IsEnabled").t(inventoryConfiguration.getEnabled().toString()); + + if(!inventoryConfiguration.getObjectPrefix().equals("")) { + OBSXMLBuilder filter = builder.e("Filter"); + filter.e("Prefix").t(inventoryConfiguration.getObjectPrefix()); + } + OBSXMLBuilder destination = builder.e("Destination"); + destination.e("Format").t(inventoryConfiguration.getInventoryFormat()); + destination.e("Bucket").t(inventoryConfiguration.getDestinationBucket()); + if(!inventoryConfiguration.getInventoryPrefix().equals("")) { + destination.e("Prefix").t(inventoryConfiguration.getInventoryPrefix()); + } + OBSXMLBuilder schedule = builder.e("Schedule"); + schedule.e("Frequency").t(inventoryConfiguration.getFrequency()); + builder.e("IncludedObjectVersions").t(inventoryConfiguration.getIncludedObjectVersions()); + if(!inventoryConfiguration.getOptionalFields().isEmpty()) { + OBSXMLBuilder optionalFields = builder.e("OptionalFields"); + for(String optionalField:inventoryConfiguration.getOptionalFields()) { + optionalFields.e("Field").t(optionalField); + } + } + return builder.up().asString(); + } catch (Exception e) { + throw new ServiceException("Failed to build XML document for InventoryConfiguration", e); + } + } } diff --git a/app/src/main/java/com/obs/services/internal/V2Convertor.java b/app/src/main/java/com/obs/services/internal/V2Convertor.java index bd0d9ad..63a8469 100644 --- a/app/src/main/java/com/obs/services/internal/V2Convertor.java +++ b/app/src/main/java/com/obs/services/internal/V2Convertor.java @@ -163,6 +163,11 @@ public String transLifecycleConfiguration(LifecycleConfiguration config) throws noncurrentVersionBuilder.elem("NoncurrentDays").t( rule.getNoncurrentVersionExpiration().getDays().toString()); } + if (rule.getAbortIncompleteMultipartUpload().getDaysAfterInitiation() > 0) { + OBSXMLBuilder abortIncompleteMultipartUpload = b.elem("AbortIncompleteMultipartUpload"); + abortIncompleteMultipartUpload.elem("DaysAfterInitiation") + .t(String.valueOf(rule.getAbortIncompleteMultipartUpload().getDaysAfterInitiation())); + } } return builder.asString(); } catch (ParserConfigurationException e) { diff --git a/app/src/main/java/com/obs/services/internal/handler/XmlResponsesSaxParser.java b/app/src/main/java/com/obs/services/internal/handler/XmlResponsesSaxParser.java index b0b81cd..25fc2b3 100644 --- a/app/src/main/java/com/obs/services/internal/handler/XmlResponsesSaxParser.java +++ b/app/src/main/java/com/obs/services/internal/handler/XmlResponsesSaxParser.java @@ -72,6 +72,7 @@ import com.obs.services.model.fs.DirSummary; import com.obs.services.model.fs.FolderContentSummary; import com.obs.services.model.fs.ListContentSummaryFsResult; +import com.obs.services.model.inventory.InventoryConfiguration; import org.xml.sax.InputSource; import org.xml.sax.XMLReader; import org.xml.sax.helpers.DefaultHandler; @@ -2140,6 +2141,57 @@ public void endElement(String name, String content) { } + public static class InventoryConfigurationsHandler extends DefaultXmlHandler { + private ArrayList inventoryConfigurations = new ArrayList<>(); + + private InventoryConfiguration inventoryConfiguration = new InventoryConfiguration(); + + private String prefix; + + public ArrayList getInventoryConfigurations() { + return inventoryConfigurations; + } + @Override + public void startElement(String name) {} + @Override + public void endElement(String name, String content) { + if("Id".equals(name)) { + inventoryConfiguration.setConfigurationId(content); + } + if("IsEnabled".equals(name)) { + inventoryConfiguration.setEnabled(Boolean.valueOf(content)); + } + if("Prefix".equals(name)) { + prefix = content; + } + if("Filter".equals(name)) { + inventoryConfiguration.setObjectPrefix(prefix); + } + if("Format".equals(name)) { + inventoryConfiguration.setInventoryFormat(content); + } + if("Bucket".equals(name)) { + inventoryConfiguration.setDestinationBucket(content); + } + if("Destination".equals(name)) { + inventoryConfiguration.setInventoryPrefix(prefix); + } + if("Frequency".equals(name)) { + inventoryConfiguration.setFrequency(content); + } + if("IncludedObjectVersions".equals(name)) { + inventoryConfiguration.setIncludedObjectVersions(content); + } + if("Field".equals(name)) { + inventoryConfiguration.getOptionalFields().add(content); + } + if("InventoryConfiguration".equals(name)) { + inventoryConfigurations.add(inventoryConfiguration); + inventoryConfiguration = new InventoryConfiguration(); + } + } + } + public static class BucketNotificationConfigurationHandler extends DefaultXmlHandler { private BucketNotificationConfiguration bucketNotificationConfiguration = new BucketNotificationConfiguration(); @@ -2247,7 +2299,12 @@ public void startNoncurrentVersionTransition() { latestRule.getNoncurrentVersionTransitions() .add(((LifecycleConfiguration.NoncurrentVersionTransition) latestTimeEvent)); } - + public void startAbortIncompleteMultipartUpload() { + latestRule.setAbortIncompleteMultipartUpload(config.new AbortIncompleteMultipartUpload()); + } + public void endDaysAfterInitiation(String content) { + latestRule.getAbortIncompleteMultipartUpload().setDaysAfterInitiation(Integer.parseInt(content)); + } public void endStorageClass(String content) { LifecycleConfiguration.setStorageClass(latestTimeEvent, StorageClassEnum.getValueFromCode(content)); } diff --git a/app/src/main/java/com/obs/services/internal/security/LimitedTimeSecurityKey.java b/app/src/main/java/com/obs/services/internal/security/LimitedTimeSecurityKey.java index 979160b..351a1b1 100644 --- a/app/src/main/java/com/obs/services/internal/security/LimitedTimeSecurityKey.java +++ b/app/src/main/java/com/obs/services/internal/security/LimitedTimeSecurityKey.java @@ -63,7 +63,8 @@ public boolean willSoonExpire() { public static Date getUtcTime() { Calendar calendar = Calendar.getInstance(); int offset = calendar.get(Calendar.ZONE_OFFSET); - calendar.add(Calendar.MILLISECOND, -offset); + int dstOffset = calendar.get(Calendar.DST_OFFSET); + calendar.add(Calendar.MILLISECOND, -(offset + dstOffset)); return calendar.getTime(); } diff --git a/app/src/main/java/com/obs/services/internal/service/ObsBucketBaseService.java b/app/src/main/java/com/obs/services/internal/service/ObsBucketBaseService.java index f786bb3..82232e1 100644 --- a/app/src/main/java/com/obs/services/internal/service/ObsBucketBaseService.java +++ b/app/src/main/java/com/obs/services/internal/service/ObsBucketBaseService.java @@ -20,6 +20,7 @@ import java.util.IdentityHashMap; import java.util.List; import java.util.Map; +import java.util.ArrayList; import com.obs.log.ILogger; import com.obs.log.LoggerBuilder; @@ -57,6 +58,13 @@ import com.obs.services.model.fs.GetBucketFSStatusResult; import com.obs.services.model.fs.SetBucketFSStatusRequest; +import com.obs.services.model.inventory.InventoryConfiguration; +import com.obs.services.model.inventory.SetInventoryConfigurationRequest; +import com.obs.services.model.inventory.GetInventoryConfigurationRequest; +import com.obs.services.model.inventory.DeleteInventoryConfigurationRequest; +import com.obs.services.model.inventory.ListInventoryConfigurationRequest; +import com.obs.services.model.inventory.GetInventoryConfigurationResult; +import com.obs.services.model.inventory.ListInventoryConfigurationResult; import okhttp3.Response; public abstract class ObsBucketBaseService extends RequestConvertor { @@ -433,4 +441,71 @@ protected BucketVersioningConfiguration getBucketVersioningImpl(BaseBucketReques setHeadersAndStatus(ret, response); return ret; } + + protected HeaderResponse setInventoryConfigurationImpl(SetInventoryConfigurationRequest request) throws ServiceException { + Map requestParams = new HashMap<>(); + requestParams.put(SpecialParamEnum.INVENTORY.getOriginalStringCode(), ""); + requestParams.put(SpecialParamEnum.ID.getOriginalStringCode(), request.getInventoryConfiguration().getConfigurationId()); + String xml = this.getIConvertor(request.getBucketName()).transBucketInventoryConfiguration(request.getInventoryConfiguration()); + Map headers = new HashMap<>(); + transRequestPaymentHeaders(request, headers, this.getIHeaders(request.getBucketName())); + NewTransResult result = transRequest(request); + result.setParams(requestParams); + headers.put(CommonHeaders.CONTENT_TYPE, Mimetypes.MIMETYPE_TEXT_PLAIN); + result.setHeaders(headers); + result.setBody(createRequestBody(Mimetypes.MIMETYPE_TEXT_PLAIN, xml)); + Response response = performRequest(result); + return build(response); + } + + protected GetInventoryConfigurationResult getInventoryConfigurationImpl(GetInventoryConfigurationRequest request) throws ServiceException { + Map requestParams = new HashMap<>(); + requestParams.put(SpecialParamEnum.INVENTORY.getOriginalStringCode(), ""); + requestParams.put(SpecialParamEnum.ID.getOriginalStringCode(), request.getConfigurationId()); + + NewTransResult newTransResult = transRequest(request); + newTransResult.setParams(requestParams); + Response httpResponse = performRequest(newTransResult, true,false,false,false); + this.verifyResponseContentType(httpResponse); + + GetInventoryConfigurationResult result = new GetInventoryConfigurationResult(); + List configurations = getXmlResponseSaxParser().parse(new HttpMethodReleaseInputStream(httpResponse), + XmlResponsesSaxParser.InventoryConfigurationsHandler.class, false).getInventoryConfigurations(); + if(configurations == null || configurations.isEmpty()) { + log.warn("No configuration got, config id is :" + request.getConfigurationId()); + } else { + result.setInventoryConfiguration(configurations.get(0)); + } + setHeadersAndStatus(result, httpResponse); + return result; + } + + protected ListInventoryConfigurationResult listInventoryConfigurationImpl(ListInventoryConfigurationRequest request) throws ServiceException { + Map requestParams = new HashMap<>(); + requestParams.put(SpecialParamEnum.INVENTORY.getOriginalStringCode(), ""); + + NewTransResult newTransResult = transRequest(request); + newTransResult.setParams(requestParams); + Response httpResponse = performRequest(newTransResult, true,false,false,false); + this.verifyResponseContentType(httpResponse); + + ListInventoryConfigurationResult result = new ListInventoryConfigurationResult(); + List configurations = getXmlResponseSaxParser().parse(new HttpMethodReleaseInputStream(httpResponse), + XmlResponsesSaxParser.InventoryConfigurationsHandler.class, false).getInventoryConfigurations(); + result.setInventoryConfigurations(configurations); + setHeadersAndStatus(result, httpResponse); + return result; + } + + protected HeaderResponse deleteInventoryConfigurationImpl(DeleteInventoryConfigurationRequest request) throws ServiceException { + Map requestParams = new HashMap<>(); + requestParams.put(SpecialParamEnum.INVENTORY.getOriginalStringCode(), ""); + requestParams.put(SpecialParamEnum.ID.getOriginalStringCode(), request.getConfigurationId()); + + NewTransResult newTransResult = transRequest(request); + newTransResult.setParams(requestParams); + Response httpResponse = performRequest(newTransResult, true,false,false,false); + + return build(httpResponse); + } } diff --git a/app/src/main/java/com/obs/services/internal/service/ObsObjectBaseService.java b/app/src/main/java/com/obs/services/internal/service/ObsObjectBaseService.java index 7286016..d42acec 100644 --- a/app/src/main/java/com/obs/services/internal/service/ObsObjectBaseService.java +++ b/app/src/main/java/com/obs/services/internal/service/ObsObjectBaseService.java @@ -29,10 +29,12 @@ import com.obs.services.internal.io.HttpMethodReleaseInputStream; import com.obs.services.internal.io.ProgressInputStream; import com.obs.services.internal.trans.NewTransResult; +import com.obs.services.internal.utils.JSONChange; import com.obs.services.internal.utils.Mimetypes; import com.obs.services.internal.utils.RestUtils; import com.obs.services.internal.utils.ServiceUtils; import com.obs.services.model.AccessControlList; +import com.obs.services.model.AuthTypeEnum; import com.obs.services.model.CopyObjectRequest; import com.obs.services.model.CopyObjectResult; import com.obs.services.model.DeleteObjectRequest; @@ -68,10 +70,12 @@ import java.io.BufferedInputStream; import java.io.Closeable; import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.text.ParseException; import java.util.Date; import java.util.HashMap; import java.util.Map; +import java.util.Objects; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -92,7 +96,7 @@ protected boolean doesObjectExistImpl(GetObjectMetadataRequest request) throws S boolean doesObjectExist = false; try { Response response = performRestHead(request.getBucketName(), request.getObjectKey(), params, headers, - new HashMap<>(), request.isEncodeHeaders()); + request.getUserHeaders(), request.isEncodeHeaders()); if (200 == response.code()) { doesObjectExist = true; } @@ -114,9 +118,18 @@ protected ObsFSFile putObjectImpl(PutObjectRequest request) throws ServiceExcept try { result = this.transPutObjectRequest(request); isExtraAclPutRequired = !prepareRESTHeaderAcl(request.getBucketName(), result.getHeaders(), acl); + if (request.getCallback() != null) { + ServiceUtils.assertParameterNotNull(request.getCallback().getCallbackUrl(), + "callbackUrl is null"); + ServiceUtils.assertParameterNotNull(request.getCallback().getCallbackBody(), + "callbackBody is null"); + result.getHeaders().put((this.getProviderCredentials().getLocalAuthType(request.getBucketName()) != AuthTypeEnum.OBS + ? Constants.V2_HEADER_PREFIX : Constants.OBS_HEADER_PREFIX) + CommonHeaders.CALLBACK, + ServiceUtils.toBase64(JSONChange.objToJson(request.getCallback()).getBytes(StandardCharsets.UTF_8))); + } // todo prepareRESTHeaderAcl 也会操作头域，下次重构可以将其合并 newTransResult = transObjectRequestWithResult(result, request); - response = performRequest(newTransResult); + response = performRequest(newTransResult, true, false, false, false); } finally { if (result != null && result.getBody() != null && request.isAutoClose()) { if (result.getBody() instanceof Closeable) { @@ -131,7 +144,13 @@ protected ObsFSFile putObjectImpl(PutObjectRequest request) throws ServiceExcept StorageClassEnum.getValueFromCode(response.header(this.getIHeaders(request.getBucketName()) .storageClassHeader())), this.getObjectUrl(request.getBucketName(), request.getObjectKey(), request.getIsIgnorePort())); - + if(request.getCallback() != null) { + try { + ret.setCallbackResponseBody(Objects.requireNonNull(response.body()).byteStream()); + } catch (Exception e) { + throw new ServiceException(e); + } + } setHeadersAndStatus(ret, response); if (isExtraAclPutRequired && acl != null) { try { @@ -444,7 +463,7 @@ protected String getObjectUrl(String bucketName, String objectKey, boolean isIgn + RestUtils.uriEncode(objectKey, false); } - private ObsFSAttribute getObsFSAttributeFromResponse(String bucketName, Response response, boolean needDecode) { + protected ObsFSAttribute getObsFSAttributeFromResponse(String bucketName, Response response, boolean needDecode) { Date lastModifiedDate = null; String lastModified = response.header(CommonHeaders.LAST_MODIFIED); if (lastModified != null) { diff --git a/app/src/main/java/com/obs/services/internal/utils/ServiceUtils.java b/app/src/main/java/com/obs/services/internal/utils/ServiceUtils.java index 2634f82..9d94289 100644 --- a/app/src/main/java/com/obs/services/internal/utils/ServiceUtils.java +++ b/app/src/main/java/com/obs/services/internal/utils/ServiceUtils.java @@ -48,7 +48,9 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; +import java.util.HashSet; import java.util.IdentityHashMap; import java.util.List; import java.util.Locale; @@ -235,11 +237,11 @@ public static Map cleanRestMetadataMapV2(Map met } else if (Constants.ALLOWED_RESPONSE_HTTP_HEADER_METADATA_NAMES .contains(key.toLowerCase(Locale.getDefault()))) { if (log.isDebugEnabled()) { - log.debug("Leaving HTTP header item unchanged: " + key + "=" + value); + log.debug("Leaving HTTP header item unchanged: " + key + "=" + getLoggableInfo(key, value)); } } else { if (log.isDebugEnabled()) { - log.debug("Ignoring metadata item: " + key + "=" + value); + log.debug("Ignoring metadata item: " + key + "=" + getLoggableInfo(key, value)); } continue; } @@ -443,7 +445,7 @@ public static boolean isBucketNameValidDNSName(String bucketName) { String[] fragments = bucketName.split("\\."); for (String fragment : fragments) { - if (Pattern.matches("^-.*", fragment) || Pattern.matches(".*-$", fragment) + if (Pattern.matches("^-.*", fragment) || Pattern.matches("^.{0,62}-$", fragment) || Pattern.matches("^$", fragment)) { return false; } @@ -535,10 +537,10 @@ public static ObsException changeFromException(Exception e) { public static ObsException changeFromServiceException(ServiceException se) { ObsException exception; if (se.getResponseCode() < 0) { - exception = new ObsException("OBS servcie Error Message. " + se.getMessage(), se.getCause()); + exception = new ObsException("OBS service Error Message. " + se.getMessage(), se.getCause()); } else { exception = new ObsException( - (se.getMessage() != null ? "Error message:" + se.getMessage() : "") + "OBS servcie Error Message.", + (se.getMessage() != null ? "Error message:" + se.getMessage() : "") + "OBS service Error Message.", se.getXmlMessage(), se.getCause()); exception.setErrorCode(se.getErrorCode()); exception.setErrorMessage(se.getErrorMessage() == null ? se.getMessage() : se.getErrorMessage()); @@ -732,4 +734,41 @@ public static boolean deleteFileIgnoreException(File file) { return true; } + //所有可以打印的Metadata和ResponseHeader + public static final HashSet LoggableResponseHeader = new HashSet<>(Arrays.asList( + "content-type","content-length","content-language","expires","origin", + "cache-control","content-disposition","content-encoding","access-control-request-method", + "access-control-request-headers","x-default-storage-class","location","date", + "range","host","if-modified-since","if-unmodified-since","if-match","if-none-match", + "last-modified", "content-range", "accept-encoding" + )); + public static final HashSet LoggableObsMetadata = new HashSet<>(Arrays.asList("Origin", + "Access-Control-Request-Headers","x-obs-server-side-encryption-customer-algorithm","x-obs-expiration", + "x-obs-website-redirect-location","x-obs-version-id","Access-Control-Allow-Origin", + "Access-Control-Allow-Headers","Access-Control-Max-Age","Access-Control-Allow-Methods", + "Access-Control-Expose-Headers","x-obs-server-side-encryption","x-obs-server-side-data-encryption", + "x-obs-server-side-encryption-customer-algorithm", "x-obs-storage-class","x-obs-restore", + "x-obs-object-type","x-obs-next-append-position","x-obs-uploadId","x-obs-object-lock-mode", + "x-obs-object-lock-retain-until-date")); + + public static final HashSet LoggableAmzMetadata = new HashSet<>(Arrays.asList( + "x-amz-server-side-encryption-customer-algorithm","x-amz-expiration", + "x-amz-website-redirect-location","x-amz-version-id","x-amz-server-side-encryption", + "x-amz-server-side-data-encryption", "x-amz-server-side-encryption-customer-algorithm", + "x-amz-storage-class","x-amz-restore","x-amz-object-type","x-amz-next-append-position", + "x-amz-uploadId","x-amz-object-lock-mode","x-amz-object-lock-retain-until-date")); + + public static final String LoggableInfo = "******"; + public static boolean isInfoLoggable(String infoKey) { + return LoggableObsMetadata.contains(infoKey) + || LoggableResponseHeader.contains(infoKey) + || LoggableAmzMetadata.contains(infoKey); + } + public static String getLoggableInfo(String infoKey,String infoVal) { + if(isInfoLoggable(infoKey)) { + return infoVal; + }else { + return LoggableInfo; + } + } } diff --git a/app/src/main/java/com/obs/services/internal/xml/OBSXMLBuilder.java b/app/src/main/java/com/obs/services/internal/xml/OBSXMLBuilder.java index de5e8e8..1bc4db3 100644 --- a/app/src/main/java/com/obs/services/internal/xml/OBSXMLBuilder.java +++ b/app/src/main/java/com/obs/services/internal/xml/OBSXMLBuilder.java @@ -81,6 +81,10 @@ protected static Document createDocumentImpl(String name, String namespaceURI, b protected static Document parseDocumentImpl(InputSource inputSource, boolean enableExternalEntities, boolean isNamespaceAware) throws ParserConfigurationException, SAXException, IOException { DocumentBuilderFactory factory = OBSXMLBuilder.findDocumentBuilderFactory(); + factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); + factory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false); + factory.setFeature("http://xml.org/sax/features/external-general-entities", false); + factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false); factory.setNamespaceAware(isNamespaceAware); enableOrDisableExternalEntityParsing(factory, enableExternalEntities); DocumentBuilder builder = factory.newDocumentBuilder(); diff --git a/app/src/main/java/com/obs/services/model/BucketTagInfo.java b/app/src/main/java/com/obs/services/model/BucketTagInfo.java index 96a3e09..a07b5d5 100644 --- a/app/src/main/java/com/obs/services/model/BucketTagInfo.java +++ b/app/src/main/java/com/obs/services/model/BucketTagInfo.java @@ -146,6 +146,14 @@ public boolean equals(Object obj) { } return true; } + + @Override + public String toString() { + return "Tag [" + + "key='" + key + '\'' + + ", value='" + value + '\'' + + ']'; + } } /** @@ -209,6 +217,11 @@ public Tag removeTagByKey(String key) { } return null; } + + @Override + public String toString() { + return "TagSet [" + tags + "]"; + } } /** diff --git a/app/src/main/java/com/obs/services/model/LifecycleConfiguration.java b/app/src/main/java/com/obs/services/model/LifecycleConfiguration.java index c99e5a0..fcb0c3a 100644 --- a/app/src/main/java/com/obs/services/model/LifecycleConfiguration.java +++ b/app/src/main/java/com/obs/services/model/LifecycleConfiguration.java @@ -584,6 +584,22 @@ public String toString() { } + public class AbortIncompleteMultipartUpload { + public AbortIncompleteMultipartUpload() { + //default invalid value + daysAfterInitiation = -1; + } + + public int getDaysAfterInitiation() { + return daysAfterInitiation; + } + + public void setDaysAfterInitiation(int daysAfterInitiation) { + this.daysAfterInitiation = daysAfterInitiation; + } + + protected int daysAfterInitiation; + } /** * Bucket lifecycle rule */ @@ -604,6 +620,19 @@ public class Rule { protected List noncurrentVersionTransitions; + public AbortIncompleteMultipartUpload getAbortIncompleteMultipartUpload() { + if(abortIncompleteMultipartUpload == null) { + abortIncompleteMultipartUpload = new AbortIncompleteMultipartUpload(); + } + return abortIncompleteMultipartUpload; + } + + public void setAbortIncompleteMultipartUpload( + AbortIncompleteMultipartUpload abortIncompleteMultipartUpload) { + this.abortIncompleteMultipartUpload = abortIncompleteMultipartUpload; + } + + protected AbortIncompleteMultipartUpload abortIncompleteMultipartUpload; /** * No-argument constructor */ @@ -914,6 +943,7 @@ public boolean equals(Object obj) { public String toString() { return "Rule [id=" + id + ", prefix=" + prefix + ", enabled=" + enabled + ", expiration=" + expiration + ", noncurrentVersionExpiration=" + noncurrentVersionExpiration + ", transitions=" + transitions + + ", tagSet=" + tagSet + ", noncurrentVersionTransitions=" + noncurrentVersionTransitions + "]"; } diff --git a/app/src/main/java/com/obs/services/model/PutObjectRequest.java b/app/src/main/java/com/obs/services/model/PutObjectRequest.java index 524c683..19d511c 100644 --- a/app/src/main/java/com/obs/services/model/PutObjectRequest.java +++ b/app/src/main/java/com/obs/services/model/PutObjectRequest.java @@ -39,6 +39,8 @@ public class PutObjectRequest extends PutObjectBasicRequest { private long progressInterval = ObsConstraint.DEFAULT_PROGRESS_INTERVAL; + private Callback callback; + public PutObjectRequest() { } @@ -274,6 +276,13 @@ public void setProgressInterval(long progressInterval) { this.progressInterval = progressInterval; } + public Callback getCallback() { + return callback; + } + + public void setCallback(Callback callback) { + this.callback = callback; + } @Override public String toString() { return "PutObjectRequest [file=" + file + ", input=" + input + ", metadata=" + metadata diff --git a/app/src/main/java/com/obs/services/model/PutObjectResult.java b/app/src/main/java/com/obs/services/model/PutObjectResult.java index 5d9a2b7..6acea2f 100644 --- a/app/src/main/java/com/obs/services/model/PutObjectResult.java +++ b/app/src/main/java/com/obs/services/model/PutObjectResult.java @@ -14,6 +14,7 @@ package com.obs.services.model; +import java.io.InputStream; import java.util.Map; /** @@ -33,6 +34,8 @@ public class PutObjectResult extends HeaderResponse { private String objectUrl; + private InputStream callbackResponseBody; + public PutObjectResult(String bucketName, String objectKey, String etag, String versionId, StorageClassEnum storageClass, String objectUrl) { super(); @@ -110,6 +113,14 @@ public String getObjectUrl() { return objectUrl; } + + public InputStream getCallbackResponseBody() { + return callbackResponseBody; + } + + public void setCallbackResponseBody(InputStream callbackResponseBody) { + this.callbackResponseBody = callbackResponseBody; + } @Override public String toString() { return "PutObjectResult [bucketName=" + bucketName + ", objectKey=" + objectKey + ", etag=" + etag diff --git a/app/src/main/java/com/obs/services/model/SpecialParamEnum.java b/app/src/main/java/com/obs/services/model/SpecialParamEnum.java index 3870703..d2a8c6f 100644 --- a/app/src/main/java/com/obs/services/model/SpecialParamEnum.java +++ b/app/src/main/java/com/obs/services/model/SpecialParamEnum.java @@ -184,7 +184,11 @@ public enum SpecialParamEnum { /** * Set, obtain, or delete the Custom Domain Name of a Bucket */ - CUSTOMDOMAIN("customdomain"); + CUSTOMDOMAIN("customdomain"), + + ID("id"), + + INVENTORY("inventory"); /** * Specify the corresponding code in the database and the external code. diff --git a/app/src/main/java/com/obs/services/model/inventory/DeleteInventoryConfigurationRequest.java b/app/src/main/java/com/obs/services/model/inventory/DeleteInventoryConfigurationRequest.java new file mode 100644 index 0000000..f51e246 --- /dev/null +++ b/app/src/main/java/com/obs/services/model/inventory/DeleteInventoryConfigurationRequest.java @@ -0,0 +1,28 @@ +package com.obs.services.model.inventory; + +import com.obs.services.model.BaseBucketRequest; +import com.obs.services.model.HttpMethodEnum; + +public class DeleteInventoryConfigurationRequest extends BaseBucketRequest { + { + httpMethod = HttpMethodEnum.DELETE; + } + + public String getConfigurationId() { + if(configurationId == null) { + configurationId = ""; + } + return configurationId; + } + + public void setConfigurationId(String configurationId) { + this.configurationId = configurationId; + } + + public DeleteInventoryConfigurationRequest(String bucketName, String configurationId) { + super(bucketName); + this.configurationId = configurationId; + } + + protected String configurationId; +} diff --git a/app/src/main/java/com/obs/services/model/inventory/GetInventoryConfigurationRequest.java b/app/src/main/java/com/obs/services/model/inventory/GetInventoryConfigurationRequest.java new file mode 100644 index 0000000..937efa0 --- /dev/null +++ b/app/src/main/java/com/obs/services/model/inventory/GetInventoryConfigurationRequest.java @@ -0,0 +1,29 @@ +package com.obs.services.model.inventory; + +import com.obs.services.model.BaseBucketRequest; +import com.obs.services.model.HttpMethodEnum; + +public class GetInventoryConfigurationRequest extends BaseBucketRequest { + + { + httpMethod = HttpMethodEnum.GET; + } + + public String getConfigurationId() { + if(configurationId == null) { + configurationId = ""; + } + return configurationId; + } + + public void setConfigurationId(String configurationId) { + this.configurationId = configurationId; + } + + public GetInventoryConfigurationRequest(String bucketName, String configurationId) { + super(bucketName); + this.configurationId = configurationId; + } + + protected String configurationId; +} diff --git a/app/src/main/java/com/obs/services/model/inventory/GetInventoryConfigurationResult.java b/app/src/main/java/com/obs/services/model/inventory/GetInventoryConfigurationResult.java new file mode 100644 index 0000000..ff36e64 --- /dev/null +++ b/app/src/main/java/com/obs/services/model/inventory/GetInventoryConfigurationResult.java @@ -0,0 +1,18 @@ +package com.obs.services.model.inventory; + +import com.obs.services.model.HeaderResponse; + +public class GetInventoryConfigurationResult extends HeaderResponse { + protected InventoryConfiguration inventoryConfiguration; + + public InventoryConfiguration getInventoryConfiguration() { + if (inventoryConfiguration == null) { + this.inventoryConfiguration = new InventoryConfiguration(); + } + return inventoryConfiguration; + } + + public void setInventoryConfiguration(InventoryConfiguration inventoryConfiguration) { + this.inventoryConfiguration = inventoryConfiguration; + } +} diff --git a/app/src/main/java/com/obs/services/model/inventory/InventoryConfiguration.java b/app/src/main/java/com/obs/services/model/inventory/InventoryConfiguration.java new file mode 100644 index 0000000..9d83c69 --- /dev/null +++ b/app/src/main/java/com/obs/services/model/inventory/InventoryConfiguration.java @@ -0,0 +1,175 @@ +package com.obs.services.model.inventory; + +import java.util.ArrayList; +import java.util.Objects; + +public class InventoryConfiguration { + public String getConfigurationId() { + if(configurationId == null) { + configurationId = ""; + } + return configurationId; + } + + public void setConfigurationId(String configurationId) { + this.configurationId = configurationId; + } + + public Boolean getEnabled() { + if(isEnabled == null) { + isEnabled = true; + } + return isEnabled; + } + + public void setEnabled(Boolean enabled) { + isEnabled = enabled; + } + + public String getObjectPrefix() { + if(objectPrefix == null) { + objectPrefix = ""; + } + return objectPrefix; + } + + public void setObjectPrefix(String objectPrefix) { + this.objectPrefix = objectPrefix; + } + + public String getFrequency() { + if(frequency == null) { + frequency = ""; + } + return frequency; + } + + public void setFrequency(String frequency) { + this.frequency = frequency; + } + + public String getInventoryFormat() { + if(inventoryFormat == null) { + inventoryFormat = ""; + } + return inventoryFormat; + } + + public void setInventoryFormat(String inventoryFormat) { + this.inventoryFormat = inventoryFormat; + } + + public String getDestinationBucket() { + if(destinationBucket == null) { + destinationBucket = ""; + } + return destinationBucket; + } + + public void setDestinationBucket(String destinationBucket) { + this.destinationBucket = destinationBucket; + } + + public String getInventoryPrefix() { + if(inventoryPrefix == null) { + inventoryPrefix = ""; + } + return inventoryPrefix; + } + + public void setInventoryPrefix(String inventoryPrefix) { + this.inventoryPrefix = inventoryPrefix; + } + + public String getIncludedObjectVersions() { + if(includedObjectVersions == null) { + includedObjectVersions = ""; + } + return includedObjectVersions; + } + + public void setIncludedObjectVersions(String includedObjectVersions) { + this.includedObjectVersions = includedObjectVersions; + } + + public ArrayList getOptionalFields() { + if(optionalFields == null) { + optionalFields = new ArrayList<>(); + } + return optionalFields; + } + + public void setOptionalFields(ArrayList optionalFields) { + this.optionalFields = optionalFields; + } + + public InventoryConfiguration() {} + public InventoryConfiguration(String configurationId, Boolean isEnabled, String frequency, String inventoryFormat, String destinationBucket, String includedObjectVersions) { + this.configurationId = configurationId; + this.isEnabled = isEnabled; + this.frequency = frequency; + this.inventoryFormat = inventoryFormat; + this.destinationBucket = destinationBucket; + this.includedObjectVersions = includedObjectVersions; + } + + @Override + public int hashCode() { + return Objects.hash(configurationId, isEnabled, objectPrefix, frequency, inventoryFormat, destinationBucket, inventoryPrefix, includedObjectVersions, optionalFields); + } + + @Override + public boolean equals(Object that) { + if(this == that) { + return true; + }else if(that instanceof InventoryConfiguration) { + InventoryConfiguration thatConfig = (InventoryConfiguration)that; + return configurationId.equals(thatConfig.getConfigurationId()) + && isEnabled.equals(thatConfig.getEnabled()) + && objectPrefix.equals(thatConfig.getObjectPrefix()) + && frequency.equals(thatConfig.getFrequency()) + && inventoryFormat.equals(thatConfig.getInventoryFormat()) + && destinationBucket.equals(thatConfig.getDestinationBucket()) + && inventoryPrefix.equals(thatConfig.getInventoryPrefix()) + && includedObjectVersions.equals(thatConfig.getIncludedObjectVersions()) + && optionalFields.equals(thatConfig.getOptionalFields()); + + }else { + return false; + } + } + + protected String configurationId; + protected Boolean isEnabled; + protected String objectPrefix; + protected String frequency; + protected String inventoryFormat; + protected String destinationBucket; + protected String inventoryPrefix; + protected String includedObjectVersions; + protected ArrayList optionalFields; + + public static class FrequencyOptions { + public static final String DAILY = "Daily"; + public static final String WEEKLY = "Weekly"; + } + + public static class InventoryFormatOptions { + public static final String CSV = "CSV"; + } + + public static class IncludedObjectVersionsOptions { + public static final String ALL = "All"; + public static final String CURRENT = "Current"; + } + + public static class OptionalFieldOptions { + public static final String SIZE = "Size"; + public static final String LAST_MODIFIED_DATE = "LastModifiedDate"; + public static final String STORAGE_CLASS = "StorageClass"; + public static final String ETAG = "ETag"; + public static final String IS_MULTIPART_UPLOADED = "IsMultipartUploaded"; + public static final String REPLICATION_STATUS = "ReplicationStatus"; + public static final String ENCRYPTION_STATUS = "EncryptionStatus"; + } +} diff --git a/app/src/main/java/com/obs/services/model/inventory/ListInventoryConfigurationRequest.java b/app/src/main/java/com/obs/services/model/inventory/ListInventoryConfigurationRequest.java new file mode 100644 index 0000000..e950cde --- /dev/null +++ b/app/src/main/java/com/obs/services/model/inventory/ListInventoryConfigurationRequest.java @@ -0,0 +1,15 @@ +package com.obs.services.model.inventory; + +import com.obs.services.model.BaseBucketRequest; +import com.obs.services.model.HttpMethodEnum; + +public class ListInventoryConfigurationRequest extends BaseBucketRequest { + + { + httpMethod = HttpMethodEnum.GET; + } + + public ListInventoryConfigurationRequest(String bucketName) { + super(bucketName); + } +} diff --git a/app/src/main/java/com/obs/services/model/inventory/ListInventoryConfigurationResult.java b/app/src/main/java/com/obs/services/model/inventory/ListInventoryConfigurationResult.java new file mode 100644 index 0000000..be60ae4 --- /dev/null +++ b/app/src/main/java/com/obs/services/model/inventory/ListInventoryConfigurationResult.java @@ -0,0 +1,21 @@ +package com.obs.services.model.inventory; + +import com.obs.services.model.HeaderResponse; + +import java.util.List; +import java.util.ArrayList; + +public class ListInventoryConfigurationResult extends HeaderResponse { + protected List inventoryConfigurations; + + public List getInventoryConfigurations() { + if(inventoryConfigurations == null) { + inventoryConfigurations = new ArrayList<>(); + } + return inventoryConfigurations; + } + + public void setInventoryConfigurations(List inventoryConfigurations) { + this.inventoryConfigurations = inventoryConfigurations; + } +} diff --git a/app/src/main/java/com/obs/services/model/inventory/SetInventoryConfigurationRequest.java b/app/src/main/java/com/obs/services/model/inventory/SetInventoryConfigurationRequest.java new file mode 100644 index 0000000..e6b63ce --- /dev/null +++ b/app/src/main/java/com/obs/services/model/inventory/SetInventoryConfigurationRequest.java @@ -0,0 +1,31 @@ +package com.obs.services.model.inventory; + +import com.obs.services.model.BaseBucketRequest; +import com.obs.services.model.HttpMethodEnum; + +import java.util.ArrayList; + +public class SetInventoryConfigurationRequest extends BaseBucketRequest { + + { + httpMethod = HttpMethodEnum.PUT; + } + + protected InventoryConfiguration inventoryConfiguration; + + public InventoryConfiguration getInventoryConfiguration() { + if(inventoryConfiguration == null) { + inventoryConfiguration = new InventoryConfiguration(); + } + return inventoryConfiguration; + } + + public void setInventoryConfiguration(InventoryConfiguration inventoryConfiguration) { + this.inventoryConfiguration = inventoryConfiguration; + } + + public SetInventoryConfigurationRequest(String bucketName, InventoryConfiguration inventoryConfiguration) { + super(bucketName); + this.inventoryConfiguration = inventoryConfiguration; + } +} diff --git a/pom-android.xml b/pom-android.xml index 256b414..339ead9 100644 --- a/pom-android.xml +++ b/pom-android.xml @@ -1,10 +1,10 @@ + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 com.huaweicloud esdk-obs-android - 3.23.5 + 3.23.9 jar OBS SDK for Android @@ -16,6 +16,7 @@ + com.jamesmurty.utils java-xmlbuilder @@ -30,14 +31,14 @@ com.squareup.okhttp3 okhttp - 4.10.0 + 4.11.0 com.squareup.okio okio - 2.10.0 + 3.5.0 - + com.fasterxml.jackson.core jackson-core @@ -53,10 +54,20 @@ jackson-annotations 2.13.3 + app/src/main/java/com + + + + app/src/main/resource + + log4j2.xml + + + app/src/test/java @@ -108,17 +119,17 @@ - + org.apache.maven.plugins maven-jar-plugin - okio/* + okio/* - + org.apache.maven.plugins maven-assembly-plugin diff --git a/pom-java-optimization.xml b/pom-java-optimization.xml index e8fedc6..377c817 100644 --- a/pom-java-optimization.xml +++ b/pom-java-optimization.xml @@ -1,194 +1,196 @@ - 4.0.0 + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + 4.0.0 - com.huaweicloud - esdk-obs-java-optimised - 3.23.5 - jar + com.huaweicloud + esdk-obs-java-optimised + 3.23.9 + jar OBS SDK for Java Optimised The OBS SDK for Java used for accessing Object Storage Service - - UTF-8 - + + UTF-8 + - - - com.jamesmurty.utils - java-xmlbuilder - 1.3 - - - net.iharder - base64 - - - - - com.squareup.okhttp3 - okhttp - 3.14.9 - - - com.squareup.okio - okio - 1.17.5 - - - - com.fasterxml.jackson.core - jackson-core - 2.13.3 - - - com.fasterxml.jackson.core - jackson-databind - 2.13.4.1 - - - com.fasterxml.jackson.core - jackson-annotations - 2.13.3 - - - - org.apache.logging.log4j - log4j-core - 2.18.0 - - - org.apache.logging.log4j - log4j-api - 2.18.0 - - + + + com.jamesmurty.utils + java-xmlbuilder + 1.3 + + + net.iharder + base64 + + + + + com.squareup.okhttp3 + okhttp + 3.14.9 + + + com.squareup.okio + okio + 1.17.5 + - - app/src/main/java - - app/src/test/java - - - org.apache.maven.plugins - maven-clean-plugin - - - - logs - - - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - - UTF-8 - 1.8 - 1.8 - - - - compile - - - - - - + + com.fasterxml.jackson.core + jackson-core + 2.13.3 + + + com.fasterxml.jackson.core + jackson-databind + 2.13.4.1 + + + com.fasterxml.jackson.core + jackson-annotations + 2.13.2 + - - org.apache.maven.plugins - maven-shade-plugin - 3.2.1 - - - package - - shade - - - - - - - com.fasterxml.jackson.core:jackson-core:jar: - com.fasterxml.jackson.core:jackson-databind:jar: - com.fasterxml.jackson.core:jackson-annotations:jar: - org.apache.logging.log4j:log4j-core:jar: - org.apache.logging.log4j:log4j-api:jar: - - - - - com.jamesmurty.utils - com.obs.shade.jamesmurty.utils - - - okhttp3 - com.obs.shade.okhttp3 - - - okio - com.obs.shade.okio - - - - - - - - + + org.apache.logging.log4j + log4j-core + 2.18.0 + + + org.apache.logging.log4j + log4j-api + 2.18.0 + + + + + app/src/main/java + + + app/src/main/resource + + log4j2.xml + + + + + app/src/test/java + + + org.apache.maven.plugins + maven-clean-plugin + + + + logs + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + UTF-8 + 1.8 + 1.8 + + + + compile + + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.1 + + + package + + shade + + + + + + + com.fasterxml.jackson.core:jackson-core:jar: + com.fasterxml.jackson.core:jackson-databind:jar: + com.fasterxml.jackson.core:jackson-annotations:jar: + org.apache.logging.log4j:log4j-core:jar: + org.apache.logging.log4j:log4j-api:jar: + + + + + com.jamesmurty.utils + com.obs.shade.jamesmurty.utils + + + okhttp3 + com.obs.shade.okhttp3 + + + okio + com.obs.shade.okio + + + + + + + + diff --git a/pom-java.xml b/pom-java.xml index 6625538..a27629d 100644 --- a/pom-java.xml +++ b/pom-java.xml @@ -4,7 +4,7 @@ com.huaweicloud esdk-obs-java - 3.23.5 + 3.23.9 jar OBS SDK for Java @@ -29,12 +29,12 @@ com.squareup.okhttp3 okhttp - 4.10.0 + 4.11.0 com.squareup.okio okio - 2.10.0 + 3.5.0 @@ -89,13 +89,14 @@ app/src/main/java - app/src/test/java diff --git a/pom.xml b/pom.xml index 364aefc..f8bd487 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.huaweicloud esdk-obs-java - 3.23.5 + 3.23.9 jar OBS SDK for Java @@ -30,12 +30,12 @@ com.squareup.okhttp3 okhttp - 4.10.0 + 4.11.0 com.squareup.okio okio - 2.10.0 + 3.5.0 @@ -82,13 +82,14 @@ app/src/main/java - app/src/test/java