diff --git a/README-Android.md b/README-Android.md index b45965d..b3756d1 100644 --- a/README-Android.md +++ b/README-Android.md @@ -1,3 +1,21 @@ +Version 3.24.8 +Resolved issues: +1. PutObject、Getobject、GetObjectMetadata、UploadPart、AppendObject、CopyObject、CopyPart、CompeleMultiUploadPart now supports crc64 checksum. +2. UploadFile can be cancelled and aborted now. +3. Allow you set okhttp's EventListenerFactory to profile each stage of a http request，not set by default. +4. Fixed the issue that client encryption is available only in obs protocal and add some check logic when encryption algrithm is null. +5. Optimised the logic of set progress listener when using uploadFile. +6. Optimised some log info format. +7. Added some logic to compatible with Android 7.0 when using DateTimeFormatter. + +Third-party dependence: +1. Replace okio 3.8.0 with okio 3.6.0 +2. Replace log4j-core 2.20.0 with log4j-core 2.18.0 +3. Replace jackson-core 2.15.4 with jackson-core 2.15.2 +4. Replace jackson-databind 2.15.4 with jackson-databind 2.15.2 +5. Replace jackson-annotations 2.15.4 with jackson-annotations 2.15.2 +6. Replace log4j-api 2.20.0 with log4j-api 2.18.0 +----------------------------------------------------------------------------------- Version 3.24.3 Resolved issues: 1. Optimized log info of some exception stack diff --git a/README-Java.md b/README-Java.md index d724f2c..0b759d3 100644 --- a/README-Java.md +++ b/README-Java.md @@ -1,3 +1,21 @@ +Version 3.24.8 +Resolved issues: +1. PutObject、Getobject、GetObjectMetadata、UploadPart、AppendObject、CopyObject、CopyPart、CompeleMultiUploadPart now supports crc64 checksum. +2. UploadFile can be cancelled and aborted now. +3. Allow you set okhttp's EventListenerFactory to profile each stage of a http request，not set by default. +4. Fixed the issue that client encryption is available only in obs protocal and add some check logic when encryption algrithm is null. +5. Optimised the logic of set progress listener when using uploadFile. +6. Optimised some log info format. +7. Added some logic to compatible with Android 7.0 when using DateTimeFormatter. + +Third-party dependence: +1. Replace okio 3.8.0 with okio 3.6.0 +2. Replace log4j-core 2.20.0 with log4j-core 2.18.0 +3. Replace jackson-core 2.15.4 with jackson-core 2.15.2 +4. Replace jackson-databind 2.15.4 with jackson-databind 2.15.2 +5. Replace jackson-annotations 2.15.4 with jackson-annotations 2.15.2 +6. Replace log4j-api 2.20.0 with log4j-api 2.18.0 +----------------------------------------------------------------------------------- Version 3.24.3 Resolved issues: 1. Optimized log info of some exception stack diff --git a/README.MD b/README.MD index afdba28..089726c 100644 --- a/README.MD +++ b/README.MD @@ -1,3 +1,21 @@ +Version 3.24.8 +Resolved issues: +1. PutObject、Getobject、GetObjectMetadata、UploadPart、AppendObject、CopyObject、CopyPart、CompeleMultiUploadPart now supports crc64 checksum. +2. UploadFile can be cancelled and aborted now. +3. Allow you set okhttp's EventListenerFactory to profile each stage of a http request，not set by default. +4. Fixed the issue that client encryption is available only in obs protocal and add some check logic when encryption algrithm is null. +5. Optimised the logic of set progress listener when using uploadFile. +6. Optimised some log info format. +7. Added some logic to compatible with Android 7.0 when using DateTimeFormatter. + +Third-party dependence: +1. Replace okio 3.8.0 with okio 3.6.0 +2. Replace log4j-core 2.20.0 with log4j-core 2.18.0 +3. Replace jackson-core 2.15.4 with jackson-core 2.15.2 +4. Replace jackson-databind 2.15.4 with jackson-databind 2.15.2 +5. Replace jackson-annotations 2.15.4 with jackson-annotations 2.15.2 +6. Replace log4j-api 2.20.0 with log4j-api 2.18.0 +----------------------------------------------------------------------------------- Version 3.24.3 Resolved issues: 1. Optimized log info of some exception stack diff --git a/README_CN.MD b/README_CN.MD index b5ccca1..44d159b 100644 --- a/README_CN.MD +++ b/README_CN.MD @@ -1,3 +1,21 @@ +Version 3.24.8 +Resolved issues: +1. PutObject、Getobject、GetObjectMetadata、UploadPart、AppendObject、CopyObject、CopyPart、CompeleMultiUploadPart支持crc64校验 +2. 断点续传上传支持暂停、取消 +3. 支持设置okhttp的EventListenerFactory，用于统计http请求各阶段耗时，默认关闭 +4. 修复客户端加密只能在obs协议下使用的问题，增加加密算法为null时的判断 +5. 优化断点续传上传时的进度条设置逻辑 +6. 优化部分日志打印格式 +7. 使用DateTimeFormatter时兼容Android 7.0 + +Third-party dependence: +1. 使用 okio 3.8.0 替代 okio 3.6.0 +2. 使用 log4j-core 2.20.0 替代 log4j-core 2.18.0 +3. 使用 jackson-core 2.15.4 替代 jackson-core 2.15.2 +4. 使用 jackson-databind 2.15.4 替代 jackson-databind 2.15.2 +5. 使用 jackson-annotations 2.15.4 替代 jackson-annotations 2.15.2 +6. 使用 log4j-api 2.20.0 替代 log4j-api 2.18.0 +----------------------------------------------------------------------------------- Version 3.24.3 Resolved issues: 1. 优化某些堆栈的日志打印 diff --git a/app/src/main/java/com/obs/log/LoggerBuilder.java b/app/src/main/java/com/obs/log/LoggerBuilder.java index 5bb2f89..a605abd 100644 --- a/app/src/main/java/com/obs/log/LoggerBuilder.java +++ b/app/src/main/java/com/obs/log/LoggerBuilder.java @@ -39,8 +39,8 @@ static class GetLoggerHolder { try { loggerClass = Class.forName("java.util.logging.Logger"); getLoggerClass = GetLoggerHolder.loggerClass.getMethod("getLogger", String.class); - } catch (NoSuchMethodException | SecurityException | ClassNotFoundException | - NoClassDefFoundError exx) { + } catch (NoSuchMethodException | SecurityException | ClassNotFoundException + | NoClassDefFoundError exx) { ILOG.warning(exx.getMessage()); } } diff --git a/app/src/main/java/com/obs/services/AbstractClient.java b/app/src/main/java/com/obs/services/AbstractClient.java index b290937..d96f1a6 100644 --- a/app/src/main/java/com/obs/services/AbstractClient.java +++ b/app/src/main/java/com/obs/services/AbstractClient.java @@ -68,7 +68,8 @@ protected void init(String accessKey, String secretKey, String securityToken, Ob if (this.isAuthTypeNegotiation()) { this.getProviderCredentials().setIsAuthTypeNegotiation(true); } - this.initHttpClient(config.getHttpDispatcher(), config.getCustomizedDnsImpl(), config.getHostnameVerifier()); + this.initHttpClient(config.getHttpDispatcher(), config.getCustomizedDnsImpl(), config.getHostnameVerifier(), + config.getEventListenerFactory()); OBSXMLBuilder.setXmlDocumentBuilderFactoryClass(config.getXmlDocumentBuilderFactoryClass()); reqBean.setRespTime(new Date()); reqBean.setResultCode(Constants.RESULTCODE_SUCCESS); diff --git a/app/src/main/java/com/obs/services/IObsClientAsync.java b/app/src/main/java/com/obs/services/IObsClientAsync.java new file mode 100644 index 0000000..293c8ee --- /dev/null +++ b/app/src/main/java/com/obs/services/IObsClientAsync.java @@ -0,0 +1,12 @@ +package com.obs.services; + +import com.obs.services.internal.task.UploadFileTask; +import com.obs.services.model.CompleteMultipartUploadResult; +import com.obs.services.model.TaskCallback; +import com.obs.services.model.UploadFileRequest; + +public interface IObsClientAsync { + UploadFileTask uploadFileAsync( + UploadFileRequest uploadFileRequest, + TaskCallback completeCallback); +} diff --git a/app/src/main/java/com/obs/services/ObsClientAsync.java b/app/src/main/java/com/obs/services/ObsClientAsync.java new file mode 100644 index 0000000..2548563 --- /dev/null +++ b/app/src/main/java/com/obs/services/ObsClientAsync.java @@ -0,0 +1,159 @@ +package com.obs.services; + +import com.obs.log.ILogger; +import com.obs.log.LoggerBuilder; +import com.obs.services.internal.task.UploadFileTask; +import com.obs.services.model.CompleteMultipartUploadResult; +import com.obs.services.model.TaskCallback; +import com.obs.services.model.UploadFileRequest; + +import java.io.IOException; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +public class ObsClientAsync extends ObsClient implements IObsClientAsync { + /** + * Constructor + * + * @param endPoint OBS endpoint + */ + public ObsClientAsync(String endPoint) { + super(endPoint); + } + + /** + * Constructor + * + * @param config Configuration parameters of ObsClient + */ + public ObsClientAsync(ObsConfiguration config) { + super(config); + } + + /** + * Constructor + * + * @param accessKey AK in the access key + * @param secretKey SK in the access key + * @param endPoint OBS endpoint + */ + public ObsClientAsync(String accessKey, String secretKey, String endPoint) { + super(accessKey, secretKey, endPoint); + } + + /** + * Constructor + * + * @param accessKey AK in the access key + * @param secretKey SK in the access key + * @param config Configuration parameters of ObsClient + */ + public ObsClientAsync(String accessKey, String secretKey, ObsConfiguration config) { + super(accessKey, secretKey, config); + } + + /** + * Constructor + * + * @param accessKey AK in the temporary access key + * @param secretKey SK in the temporary access key + * @param securityToken Security token + * @param endPoint OBS endpoint + */ + public ObsClientAsync(String accessKey, String secretKey, String securityToken, String endPoint) { + super(accessKey, secretKey, securityToken, endPoint); + } + + /** + * Constructor + * + * @param accessKey AK in the temporary access key + * @param secretKey SK in the temporary access key + * @param securityToken Security token + * @param config Configuration parameters of ObsClient + */ + public ObsClientAsync(String accessKey, String secretKey, String securityToken, ObsConfiguration config) { + super(accessKey, secretKey, securityToken, config); + } + + public ObsClientAsync(IObsCredentialsProvider provider, String endPoint) { + super(provider, endPoint); + } + + public ObsClientAsync(IObsCredentialsProvider provider, ObsConfiguration config) { + super(provider, config); + } + + private static final ILogger log = LoggerBuilder.getLogger(ObsClientAsync.class); + private ExecutorService asyncClientExecutorService; + private static final int DEFAULT_CLIENT_EXECUTOR_SERVICE_SIZE = 128; + + private int queryInterval = 1000; + + @Override + public void close() throws IOException { + log.warn("ObsClientAsync closing"); + try { + // finishing all task + getExecutorService().shutdown(); + log.warn("ObsClientAsync closed"); + } catch (Exception e) { + log.warn("ObsClientAsync close failed, detail:", e); + } + super.close(); + } + + private static final String ASYNC_CLIENT_EXECUTOR_SERVICE_THREAD_NAME = "async-client-thread"; + protected ExecutorService getExecutorService() { + if (asyncClientExecutorService == null) { + asyncClientExecutorService = Executors.newFixedThreadPool(DEFAULT_CLIENT_EXECUTOR_SERVICE_SIZE, + r -> new Thread(r, ASYNC_CLIENT_EXECUTOR_SERVICE_THREAD_NAME)); + } + return asyncClientExecutorService; + } + + public void setExecutorService(ExecutorService service) { + if (asyncClientExecutorService != null) { // wait for all task finish + asyncClientExecutorService.shutdown(); + while (!asyncClientExecutorService.isTerminated()) { + try { + Thread.sleep(queryInterval); + } catch (InterruptedException e) { + log.warn("ObsClientAsync setExecutorService failed, detail:", e); + } + } + } + asyncClientExecutorService = service; + } + + public int getQueryInterval() { + return queryInterval; + } + + public void setQueryInterval(int queryInterval) { + this.queryInterval = queryInterval; + } + + /** + * @param uploadFileRequest + * @param completeCallback + * @return + */ + @Override + public UploadFileTask uploadFileAsync( + UploadFileRequest uploadFileRequest, + TaskCallback completeCallback) { + log.debug("start uploadFileAsync"); + if (uploadFileRequest.getCancelHandler() != null) { + uploadFileRequest.getCancelHandler().resetCancelStatus(); + } + UploadFileTask uploadFileTask = + new UploadFileTask(this, uploadFileRequest.getBucketName(), uploadFileRequest, completeCallback); + Future future = getExecutorService().submit((Callable) uploadFileTask); + + uploadFileTask.setResultFuture(future); + return uploadFileTask; + } +} diff --git a/app/src/main/java/com/obs/services/ObsConfiguration.java b/app/src/main/java/com/obs/services/ObsConfiguration.java index 49d1ca8..2b7ba8a 100644 --- a/app/src/main/java/com/obs/services/ObsConfiguration.java +++ b/app/src/main/java/com/obs/services/ObsConfiguration.java @@ -24,6 +24,7 @@ import okhttp3.Dispatcher; import okhttp3.Dns; +import okhttp3.EventListener; import java.security.SecureRandom; @@ -104,6 +105,7 @@ public class ObsConfiguration implements Cloneable { private HostnameVerifier hostnameVerifier; private String xmlDocumentBuilderFactoryClass; + private EventListener.Factory eventListenerFactory; /** * Constructor @@ -939,4 +941,15 @@ public SecureRandom getSecureRandom() { public void setSecureRandom(SecureRandom secureRandom) { this.secureRandom = secureRandom; } + + public EventListener.Factory getEventListenerFactory() + { + return eventListenerFactory; + } + + public void setEventListenerFactory(EventListener.Factory eventListenerFactory) + { + this.eventListenerFactory = eventListenerFactory; + } + } diff --git a/app/src/main/java/com/obs/services/crypto/CryptoObsClient.java b/app/src/main/java/com/obs/services/crypto/CryptoObsClient.java index b2afe24..2433c0a 100644 --- a/app/src/main/java/com/obs/services/crypto/CryptoObsClient.java +++ b/app/src/main/java/com/obs/services/crypto/CryptoObsClient.java @@ -339,16 +339,20 @@ protected ObsObject getObjectImpl(GetObjectRequest request) throws ServiceExcept // 该接口是下载对象，需要将流返回给客户（调用方），我们不能关闭这个流 if (ctrCipherGenerator != null) { + String headerMetaPrefix = + this.getProviderCredentials() != null && + this.getProviderCredentials().getLocalAuthType(request.getBucketName()) != AuthTypeEnum.OBS + ? Constants.V2_HEADER_META_PREFIX : Constants.OBS_HEADER_META_PREFIX; String encryptedAlgorithm = (String) objMetadata .getOriginalHeaders() - .get(Constants.OBS_HEADER_META_PREFIX + ENCRYPTED_ALGORITHM_META_NAME); + .get(headerMetaPrefix + ENCRYPTED_ALGORITHM_META_NAME); String encryptedStart = (String) objMetadata .getOriginalHeaders() - .get(Constants.OBS_HEADER_META_PREFIX + ENCRYPTED_START_META_NAME); + .get(headerMetaPrefix + ENCRYPTED_START_META_NAME); if (isValidEncryptedAlgorithm(encryptedAlgorithm)) { byte[] cryptoKeyBytes = ctrCipherGenerator.getCryptoKeyBytes(); @@ -365,7 +369,7 @@ protected ObsObject getObjectImpl(GetObjectRequest request) throws ServiceExcept objMetadata .getOriginalHeaders() .get( - Constants.OBS_HEADER_META_PREFIX + headerMetaPrefix + ENCRYPTED_AES_KEY_META_NAME); // 解密rsa加密后的主密钥 cryptoKeyBytes = @@ -425,8 +429,8 @@ protected ObsObject getObjectImpl(GetObjectRequest request) throws ServiceExcept } public boolean isValidEncryptedAlgorithm(String encryptedAlgorithm) { - return encryptedAlgorithm.equals(CtrRSACipherGenerator.ENCRYPTED_ALGORITHM) - || encryptedAlgorithm.equals(CTRCipherGenerator.ENCRYPTED_ALGORITHM); + return encryptedAlgorithm != null && (encryptedAlgorithm.equals(CtrRSACipherGenerator.ENCRYPTED_ALGORITHM) + || encryptedAlgorithm.equals(CTRCipherGenerator.ENCRYPTED_ALGORITHM)); } protected byte[] getOrGenerateCryptoIvBytes() { diff --git a/app/src/main/java/com/obs/services/exception/ObsException.java b/app/src/main/java/com/obs/services/exception/ObsException.java index a450d3e..ab9ca6c 100644 --- a/app/src/main/java/com/obs/services/exception/ObsException.java +++ b/app/src/main/java/com/obs/services/exception/ObsException.java @@ -14,6 +14,7 @@ package com.obs.services.exception; +import java.util.Locale; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -64,17 +65,32 @@ public ObsException(String message, String xmlMessage, Throwable cause) { @Override public String toString() { - String myString = super.toString(); + StringBuilder myString = new StringBuilder(super.toString()); if (responseCode != -1) { - myString += " -- ResponseCode: " + responseCode + ", ResponseStatus: " + responseStatus; + myString.append(" -- ResponseCode: ") + .append(responseCode) + .append(", ResponseStatus: ") + .append(responseStatus); } if (isParsedFromXmlMessage()) { - myString += ", XML Error Message: " + xmlMessage; + myString.append(", XML Error Message: ").append(xmlMessage); } else if (errorRequestId != null) { - myString += ", RequestId: " + errorRequestId + ", HostId: " + errorHostId; + myString.append(", RequestId: ").append(errorRequestId).append(", HostId: ").append(errorHostId); } - return myString; + // 遍历Map的entry,打印所有报错相关头域 + Map headers = getResponseHeaders(); + if (headers != null) { + for (Map.Entry header : headers.entrySet()) { + if (header.getKey().toLowerCase(Locale.ROOT).contains("error")) { + myString.append(", ErrorHeaderKey: ") + .append(header.getKey()) + .append(", ErrorHeaderValue: ") + .append(header.getValue()); + } + } + } + return myString.toString(); } private boolean isParsedFromXmlMessage() { diff --git a/app/src/main/java/com/obs/services/internal/Constants.java b/app/src/main/java/com/obs/services/internal/Constants.java index 048009b..c3f1cd0 100644 --- a/app/src/main/java/com/obs/services/internal/Constants.java +++ b/app/src/main/java/com/obs/services/internal/Constants.java @@ -60,7 +60,9 @@ public static class CommonHeaders { public static final String CONTENT_DISPOSITION = "Content-Disposition"; - public static final String HASH_CRC64ECMA = "hash-crc64ecma"; + public static final String HASH_CRC64ECMA = "checksum-crc64ecma"; + + public static final String INVALID_CRC_64 = "InvalidCRC64"; public static final String CONTENT_ENCODING = "Content-Encoding"; @@ -213,7 +215,7 @@ public static class ObsRequestParams { public static final TimeZone GMT_TIMEZONE = TimeZone.getTimeZone("GMT"); - public static final String OBS_SDK_VERSION = "3.24.3"; + public static final String OBS_SDK_VERSION = "3.24.8"; public static final String USER_AGENT_VALUE = "obs-sdk-java/" + Constants.OBS_SDK_VERSION; diff --git a/app/src/main/java/com/obs/services/internal/DownloadResumableClient.java b/app/src/main/java/com/obs/services/internal/DownloadResumableClient.java index af59fac..abfd8f2 100644 --- a/app/src/main/java/com/obs/services/internal/DownloadResumableClient.java +++ b/app/src/main/java/com/obs/services/internal/DownloadResumableClient.java @@ -14,7 +14,23 @@ package com.obs.services.internal; -import java.io.Closeable; +import com.obs.log.ILogger; +import com.obs.log.LoggerBuilder; +import com.obs.services.AbstractClient; +import com.obs.services.exception.ObsException; +import com.obs.services.internal.io.ProgressInputStream; +import com.obs.services.internal.utils.CRC64; +import com.obs.services.internal.utils.CRC64InputStream; +import com.obs.services.internal.utils.SecureObjectInputStream; +import com.obs.services.internal.utils.ServiceUtils; +import com.obs.services.model.DownloadFileRequest; +import com.obs.services.model.DownloadFileResult; +import com.obs.services.model.GetObjectMetadataRequest; +import com.obs.services.model.GetObjectRequest; +import com.obs.services.model.MonitorableProgressListener; +import com.obs.services.model.ObjectMetadata; +import com.obs.services.model.ObsObject; + import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -30,26 +46,15 @@ import java.util.LinkedList; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import com.obs.log.ILogger; -import com.obs.log.LoggerBuilder; -import com.obs.services.AbstractClient; -import com.obs.services.exception.ObsException; -import com.obs.services.internal.io.ProgressInputStream; -import com.obs.services.internal.utils.SecureObjectInputStream; -import com.obs.services.internal.utils.ServiceUtils; -import com.obs.services.model.DownloadFileRequest; -import com.obs.services.model.DownloadFileResult; -import com.obs.services.model.GetObjectMetadataRequest; -import com.obs.services.model.GetObjectRequest; -import com.obs.services.model.MonitorableProgressListener; -import com.obs.services.model.ObjectMetadata; -import com.obs.services.model.ObsObject; +import static com.obs.services.internal.Constants.CommonHeaders.HASH_CRC64ECMA; +import static com.obs.services.internal.Constants.CommonHeaders.INVALID_CRC_64; public class DownloadResumableClient { @@ -146,7 +151,6 @@ private DownloadFileResult downloadCheckPoint(DownloadFileRequest downloadFileRe // 并发下载分片 DownloadResult downloadResult = this.download(downloadCheckPoint, downloadFileRequest); checkDownloadResult(downloadFileRequest, downloadCheckPoint, downloadResult); - // 重命名临时文件 renameTo(downloadFileRequest.getTempDownloadFile(), downloadFileRequest.getDownloadFile()); @@ -154,9 +158,59 @@ private DownloadFileResult downloadCheckPoint(DownloadFileRequest downloadFileRe if (downloadFileRequest.isEnableCheckpoint()) { ServiceUtils.deleteFileIgnoreException(downloadFileRequest.getCheckpointFile()); } + tryGetCombinedCRC64(downloadFileRequest, downloadCheckPoint, downloadFileResult); return downloadFileResult; } + private String tryGetRequestID(DownloadFileResult downloadFileResult) { + if (downloadFileResult == null) { + return ""; + } + ObjectMetadata objectMetadata = downloadFileResult.getObjectMetadata(); + if (objectMetadata == null) { + return ""; + } + Map responseHeaders = objectMetadata.getResponseHeaders(); + if (responseHeaders == null) { + return ""; + } + return (String) responseHeaders.get("request-id"); + } + private void tryGetCombinedCRC64(DownloadFileRequest downloadFileRequest, DownloadCheckPoint downloadCheckPoint, + DownloadFileResult downloadFileResult) throws ServiceException { + + if (downloadFileRequest.isNeedCalculateCRC64() && downloadCheckPoint.isAllCompleted) { + try { + CRC64 crc64Combined = new CRC64(downloadCheckPoint.downloadParts.get(0).crc64); + for (int i = 1; i < downloadCheckPoint.downloadParts.size(); ++i) { + DownloadPart downloadPartI = downloadCheckPoint.downloadParts.get(i); + crc64Combined.combineWithAnotherCRC64 + (downloadPartI.crc64, downloadPartI.end - downloadPartI.offset + 1L); + } + downloadFileResult.setCombinedCRC64(crc64Combined); + String sdkCalculatedCRC64 = crc64Combined.toString(); + String serverReturnedCRC64 = (String) downloadFileResult. + getObjectMetadata().getResponseHeaders().get(HASH_CRC64ECMA); + if (!sdkCalculatedCRC64.equals(serverReturnedCRC64)) { + String errorInfo = "downloadedObject: " + downloadFileRequest.getObjectKey() + + " 's CRC64 and server returned CRC64 don't match! " + + "Please deleted downloadedFile and try again. " + + "sdk calculated downloadFile CRC64 is " + sdkCalculatedCRC64 + + " server returned CRC64 is " + serverReturnedCRC64 + + " server requestID is " + tryGetRequestID(downloadFileResult); + ServiceException serviceException = new ServiceException(errorInfo); + serviceException.setErrorCode(INVALID_CRC_64); + throw serviceException; + } + } catch (ServiceException serviceException) { + log.error("tryGetCombinedCRC64 for downloadFile failed, exception:", serviceException); + throw serviceException; + } catch (Throwable t) { + log.error("tryGetCombinedCRC64 for downloadFile failed, throwable:", t); + throw ServiceUtils.changeFromThrowable(t); + } + } + } private void checkDownloadResult(DownloadFileRequest downloadFileRequest, DownloadCheckPoint downloadCheckPoint, DownloadResult downloadResult) throws Exception { @@ -275,6 +329,18 @@ private DownloadResult download(final DownloadCheckPoint downloadCheckPoint, ? downloadFileRequest.getProgressInterval() : ObsConstraint.DEFAULT_PROGRESS_INTERVAL); } + if (downloadFileRequest.getTaskNum() == 1) { + for (Task task : unfinishedTasks) { + task.setProgressManager(progressManager); + taskResults.add(task.call()); + } + downloadResult.setPartResults(taskResults); + if (progressManager != null) { + progressManager.progressEnd(); + } + return downloadResult; + } + ExecutorService service = Executors.newFixedThreadPool(downloadFileRequest.getTaskNum()); for (Task task : unfinishedTasks) { task.setProgressManager(progressManager); @@ -368,6 +434,9 @@ public PartResultDown call() throws Exception { ObsObject object = obsClient.getObject(getObjectRequest); content = object.getObjectContent(); + if (downloadFileRequest.isNeedCalculateCRC64()) { + content = new CRC64InputStream(content); + } byte[] buffer = new byte[ObsConstraint.DEFAULT_CHUNK_SIZE]; int bytesOffset; if (this.progressManager != null) { @@ -380,7 +449,11 @@ public PartResultDown call() throws Exception { output.write(buffer, 0, bytesOffset); } } - downloadCheckPoint.update(partIndex, true, downloadFileRequest.getTempDownloadFile()); + CRC64 partCrc64 = null; + if (downloadFileRequest.isNeedCalculateCRC64()) { + partCrc64 = ((CRC64InputStream) content).getCrc64(); + } + downloadCheckPoint.update(partIndex, true, downloadFileRequest.getTempDownloadFile(), partCrc64); } catch (ObsException e) { if (e.getResponseCode() >= 300 && e.getResponseCode() < 500 && e.getResponseCode() != 408) { downloadCheckPoint.isAbort = true; @@ -560,6 +633,7 @@ static class DownloadCheckPoint implements Serializable { public TmpFileStatus tmpFileStatus; ArrayList downloadParts; public transient volatile boolean isAbort = false; + public transient volatile boolean isAllCompleted = true; @Override public int hashCode() { @@ -668,10 +742,12 @@ public boolean isValid(String tmpFilePath, ObjectMetadata objectMetadata) { * @param tmpFilePath * @throws IOException */ - public synchronized void update(int index, boolean completed, String tmpFilePath) throws IOException { + public synchronized void update(int index, boolean completed, String tmpFilePath, CRC64 crc64) throws IOException { downloadParts.get(index).isCompleted = completed; File tmpfile = new File(tmpFilePath); this.tmpFileStatus.lastModified = new Date(tmpfile.lastModified()); + isAllCompleted = (isAllCompleted && completed); + downloadParts.get(index).crc64 = crc64; } /** @@ -794,6 +870,7 @@ static class DownloadPart implements Serializable { public long offset; // 分片起始位置 public long end; // 分片片结束位置 public boolean isCompleted; // 该分片下载是否完成 + public CRC64 crc64; // 分片的crc64，用于合并后进行crc64校验 @Override public int hashCode() { @@ -803,6 +880,7 @@ public int hashCode() { result = prime * result + (isCompleted ? 0 : 8); result = prime * result + (int) (end ^ (end >>> 32)); result = prime * result + (int) (offset ^ (offset >>> 32)); + result = prime * result + ((crc64 == null) ? 0 : crc64.hashCode()); return result; } @@ -947,6 +1025,5 @@ public List getPartResults() { public void setPartResults(List partResults) { this.partResults = partResults; } - } } diff --git a/app/src/main/java/com/obs/services/internal/ProgressManager.java b/app/src/main/java/com/obs/services/internal/ProgressManager.java index 6c88f10..c0920f8 100644 --- a/app/src/main/java/com/obs/services/internal/ProgressManager.java +++ b/app/src/main/java/com/obs/services/internal/ProgressManager.java @@ -31,6 +31,7 @@ static class BytesUnit { } } + private boolean endFlag = true; protected final long totalBytes; protected long startCheckpoint; protected long lastCheckpoint; @@ -78,4 +79,12 @@ protected List createCurrentInstantaneousBytes(long bytes, long now) public abstract void progressEnd(); protected abstract void doProgressChanged(int bytes); + + public boolean isEndFlag() { + return endFlag; + } + + public void setEndFlag(boolean endFlag) { + this.endFlag = endFlag; + } } diff --git a/app/src/main/java/com/obs/services/internal/RestConnectionService.java b/app/src/main/java/com/obs/services/internal/RestConnectionService.java index ee6f3e4..cc50f75 100644 --- a/app/src/main/java/com/obs/services/internal/RestConnectionService.java +++ b/app/src/main/java/com/obs/services/internal/RestConnectionService.java @@ -34,6 +34,7 @@ import com.obs.services.model.HttpMethodEnum; import okhttp3.Dispatcher; import okhttp3.Dns; +import okhttp3.EventListener; import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.RequestBody; @@ -57,10 +58,12 @@ public class RestConnectionService { protected volatile ProviderCredentials credentials; - protected void initHttpClient(Dispatcher httpDispatcher, Dns customizedDnsImpl, HostnameVerifier hostnameVerifier) { + protected void initHttpClient(Dispatcher httpDispatcher, Dns customizedDnsImpl, HostnameVerifier hostnameVerifier, + EventListener.Factory eventListenerFactory) { OkHttpClient.Builder builder = RestUtils.initHttpClientBuilder(obsProperties, keyManagerFactory, - trustManagerFactory, httpDispatcher, customizedDnsImpl, hostnameVerifier, credentials.getSecureRandom()); + trustManagerFactory, httpDispatcher, customizedDnsImpl, eventListenerFactory, hostnameVerifier, + credentials.getSecureRandom()); if (this.obsProperties.getBoolProperty(ObsConstraint.PROXY_ISABLE, true)) { String proxyHostAddress = this.obsProperties.getStringProperty(ObsConstraint.PROXY_HOST, null); diff --git a/app/src/main/java/com/obs/services/internal/RestStorageService.java b/app/src/main/java/com/obs/services/internal/RestStorageService.java index e313bda..d5b467f 100644 --- a/app/src/main/java/com/obs/services/internal/RestStorageService.java +++ b/app/src/main/java/com/obs/services/internal/RestStorageService.java @@ -25,6 +25,7 @@ import com.obs.services.internal.security.ProviderCredentialThreadContext; import com.obs.services.internal.security.ProviderCredentials; import com.obs.services.internal.trans.NewTransResult; +import com.obs.services.internal.utils.CallCancelHandler; import com.obs.services.internal.utils.IAuthentication; import com.obs.services.internal.utils.JSONChange; import com.obs.services.internal.utils.Mimetypes; @@ -55,6 +56,8 @@ import static com.obs.services.internal.utils.ServiceUtils.getLoggableInfo; import static com.obs.services.internal.utils.ServiceUtils.isInfoLoggable; +import static com.obs.services.internal.utils.ServiceUtils.tokenMasked; +import static com.obs.services.internal.utils.ServiceUtils.messageMasked; public abstract class RestStorageService extends RestConnectionService { private static final ILogger log = LoggerBuilder.getLogger(RestStorageService.class); @@ -378,7 +381,7 @@ protected Response performRequest(Request request, Map requestPa boolean doSignature, boolean isOEF, boolean needEncode) throws ServiceException { RequestInfo requestInfo = new RequestInfo(request, new InterfaceLogBean("performRequest", "", "")); try { - tryRequest(requestParameters, bucketName, doSignature, isOEF, requestInfo, needEncode); + tryRequest(requestParameters, bucketName, doSignature, isOEF, requestInfo, needEncode, null); } catch (Throwable t) { throw this.handleThrowable(bucketName, requestInfo.getRequest(), requestInfo.getResponse(), requestInfo.getCall(), t, needEncode); @@ -409,7 +412,7 @@ protected Response performRequest(NewTransResult result, boolean needSignature, RequestInfo requestInfo = new RequestInfo(request, new InterfaceLogBean("performRequest", "", "")); try { tryRequest(result.getParams(), result.getBucketName(), needSignature, - isOEF, requestInfo, result.isEncodeHeaders()); + isOEF, requestInfo, result.isEncodeHeaders(), result.getCancelHandler()); } catch (Throwable t) { throw this.handleThrowable(result.getBucketName(), requestInfo.getRequest(), requestInfo.getResponse(), requestInfo.getCall(), t, result.isEncodeHeaders()); @@ -421,14 +424,15 @@ protected Response performRequest(NewTransResult result, boolean needSignature, log.info(requestInfo.getReqBean()); } Response response = requestInfo.getResponse(); - if (autoRelease) { + if (autoRelease && response != null) { response.close(); } return response; } private void tryRequest(Map requestParameters, String bucketName, boolean doSignature, - boolean isOEF, RequestInfo requestInfo, boolean needEncode) throws Exception { + boolean isOEF, RequestInfo requestInfo, boolean needEncode, + CallCancelHandler cancelHandler) throws Exception { requestInfo.setRequest(initRequest(bucketName, requestInfo.getRequest(), needEncode)); log.debug("Performing " + requestInfo.getRequest().method() @@ -462,9 +466,16 @@ private void tryRequest(Map requestParameters, String bucketName retryController.setWasRecentlyRedirected(false); } - requestInfo.setCall(httpClient.newCall(requestInfo.getRequest())); - requestInfo.setResponse(executeRequest(requestInfo.getCall(), + Call okhttpCall = httpClient.newCall(requestInfo.getRequest()); + requestInfo.setCall(okhttpCall); + if (cancelHandler != null) { + cancelHandler.setCall(okhttpCall); + } + requestInfo.setResponse(executeRequest(okhttpCall, requestInfo.getRequest(), retryController)); + if (cancelHandler != null) { + cancelHandler.removeFinishedCall(okhttpCall); + } if (null == requestInfo.getResponse()) { continue; } @@ -582,6 +593,10 @@ private Response executeRequest(Call call, try { semaphore.acquire(); + if (log.isDebugEnabled()) { + long acquireTime = System.currentTimeMillis(); + log.debug("semaphore.acquire cost " + (acquireTime - start) + " ms, acquire time:" + acquireTime); + } return call.execute(); } catch (UnrecoverableIOException e) { if (retryController.getLastException() != null) { @@ -666,13 +681,15 @@ private String createErrorMessageForSignatureDoesNotMatch(String xmlMessage, Str if (!xmlMessage.contains("SignatureDoesNotMatch")) { return message; } - int startStringToSign = xmlMessage.lastIndexOf("") + "".length(); + int startStringToSign = xmlMessage.lastIndexOf(""); if(startStringToSign < 0){ return message; } + startStringToSign += "".length(); int endStringToSign = xmlMessage.lastIndexOf(""); StringBuilder ErrorStringToSignMessage = new StringBuilder(); + tokenMasked(stringToSignToReturn); ErrorStringToSignMessage.append("your local StringToSign is (between\"---\"):\n---\n"); ErrorStringToSignMessage.append(stringToSignToReturn); ErrorStringToSignMessage.append("\n---\nPlease compare it to Server's StringToSign (between\"---\"):\n---\n"); @@ -685,7 +702,7 @@ private String readResponseMessage(Response response) { String xmlMessage = null; try { if (response.body() != null) { - xmlMessage = response.body().string(); + xmlMessage = messageMasked(response.body().string()); } } catch (IOException e) { log.warn("read response body failed.", e); @@ -909,7 +926,7 @@ protected Request authorizeHttpRequest(Request request, String bucketName, Strin stringToSignToReturn.append(iauthentication.getStringToSign()); } log.debug("StringToSign ('|' is a newline): " - + iauthentication.getStringToSign().replace('\n', '|')); + + messageMasked(iauthentication.getStringToSign().replace('\n', '|'))); String authorizationString = iauthentication.getAuthorization(); builder.header(CommonHeaders.AUTHORIZATION, authorizationString); diff --git a/app/src/main/java/com/obs/services/internal/UploadResumableClient.java b/app/src/main/java/com/obs/services/internal/UploadResumableClient.java index 36e26b5..c5111d6 100644 --- a/app/src/main/java/com/obs/services/internal/UploadResumableClient.java +++ b/app/src/main/java/com/obs/services/internal/UploadResumableClient.java @@ -14,42 +14,48 @@ package com.obs.services.internal; +import com.obs.log.ILogger; +import com.obs.log.LoggerBuilder; +import com.obs.services.AbstractClient; +import com.obs.services.exception.ObsException; +import com.obs.services.internal.utils.CRC64; +import com.obs.services.internal.utils.CRC64InputStream; +import com.obs.services.internal.utils.SecureObjectInputStream; +import com.obs.services.internal.utils.ServiceUtils; +import com.obs.services.model.AbortMultipartUploadRequest; +import com.obs.services.model.CompleteMultipartUploadRequest; +import com.obs.services.model.CompleteMultipartUploadResult; +import com.obs.services.model.HeaderResponse; +import com.obs.services.model.InitiateMultipartUploadRequest; +import com.obs.services.model.InitiateMultipartUploadResult; +import com.obs.services.model.PartEtag; +import com.obs.services.model.UploadFileRequest; +import com.obs.services.model.UploadPartRequest; +import com.obs.services.model.UploadPartResult; + import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStream; import java.io.ObjectOutputStream; import java.io.Serializable; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import com.obs.log.ILogger; -import com.obs.log.LoggerBuilder; -import com.obs.services.AbstractClient; -import com.obs.services.exception.ObsException; -import com.obs.services.internal.io.ProgressInputStream; -import com.obs.services.internal.utils.SecureObjectInputStream; -import com.obs.services.internal.utils.ServiceUtils; -import com.obs.services.model.AbortMultipartUploadRequest; -import com.obs.services.model.CompleteMultipartUploadRequest; -import com.obs.services.model.CompleteMultipartUploadResult; -import com.obs.services.model.HeaderResponse; -import com.obs.services.model.InitiateMultipartUploadRequest; -import com.obs.services.model.InitiateMultipartUploadResult; -import com.obs.services.model.PartEtag; -import com.obs.services.model.UploadFileRequest; -import com.obs.services.model.UploadPartRequest; -import com.obs.services.model.UploadPartResult; +import static com.obs.services.internal.Constants.CommonHeaders.HASH_CRC64ECMA; +import static com.obs.services.internal.Constants.CommonHeaders.INVALID_CRC_64; public class UploadResumableClient { @@ -104,6 +110,33 @@ protected HeaderResponse abortMultipartUpload(String uploadId, UploadFileRequest return this.obsClient.abortMultipartUpload(request); } + private void abortUploadFileTaskIfCanceledAndAborted(UploadCheckPoint uploadCheckPoint, UploadFileRequest uploadFileRequest) { + if (uploadFileRequest.getCancelHandler() != null + && uploadFileRequest.getCancelHandler().isCancelled() + && uploadFileRequest.isNeedAbortUploadFileAfterCancel()) { + log.error("aborted uploadFileRequest after canceled"); + this.abortMultipartUploadSilent(uploadCheckPoint.uploadID, uploadFileRequest); + ServiceUtils.deleteFileIgnoreException(uploadFileRequest.getCheckpointFile()); + } + } + + private CRC64 tryGetCombinedCRC64(UploadCheckPoint uploadCheckPoint) { + if (!uploadCheckPoint.partCRC64s.isEmpty()) { + try { + CRC64 crc64Combined = new CRC64(uploadCheckPoint.partCRC64s.get(1)); + for (int i = 2; i <= uploadCheckPoint.partCRC64s.size(); ++i) { + crc64Combined.combineWithAnotherCRC64( + uploadCheckPoint.partCRC64s.get(i), + uploadCheckPoint.uploadParts.get(i - 1).size); + } + return crc64Combined; + } catch (Throwable t) { + log.error("tryGetCombinedCRC64 for uploadFile failed, throwable:", t); + throw ServiceUtils.changeFromThrowable(t); + } + } + return null; + } private CompleteMultipartUploadResult uploadFileCheckPoint(UploadFileRequest uploadFileRequest) throws Exception { UploadCheckPoint uploadCheckPoint = new UploadCheckPoint(); if (uploadFileRequest.isEnableCheckpoint()) { @@ -126,11 +159,18 @@ private CompleteMultipartUploadResult uploadFileCheckPoint(UploadFileRequest upl if (uploadCheckPoint.isDeleteUploadRecordFile) { ServiceUtils.deleteFileIgnoreException(uploadFileRequest.getCheckpointFile()); } + } else { + abortUploadFileTaskIfCanceledAndAborted(uploadCheckPoint, uploadFileRequest); } throw partResult.getException(); } } + if (uploadFileRequest.getCancelHandler() != null && uploadFileRequest.getCancelHandler().isCancelled()) { + log.warn("uploadFileRequest is canceled"); + abortUploadFileTaskIfCanceledAndAborted(uploadCheckPoint, uploadFileRequest); + throw new ObsException("uploadFileRequest is canceled, no completeMultipartUploadRequest is sent"); + } // 合并多段 CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest( uploadFileRequest.getBucketName(), uploadFileRequest.getObjectKey(), uploadCheckPoint.uploadID, @@ -140,7 +180,15 @@ private CompleteMultipartUploadResult uploadFileCheckPoint(UploadFileRequest upl completeMultipartUploadRequest.setIsIgnorePort(uploadFileRequest.getIsIgnorePort()); completeMultipartUploadRequest.setCallback(uploadFileRequest.getCallback()); completeMultipartUploadRequest.setUserHeaders(uploadFileRequest.getUserHeaders()); + completeMultipartUploadRequest.setCancelHandler(uploadFileRequest.getCancelHandler()); + CRC64 crc64Combined = tryGetCombinedCRC64(uploadCheckPoint); + if (crc64Combined != null) { + completeMultipartUploadRequest.setUserHeaders(new HashMap<>(uploadFileRequest.getUserHeaders())); + completeMultipartUploadRequest.addUserHeaders( + obsClient.getRestHeaderPrefix(uploadFileRequest.getBucketName()) + HASH_CRC64ECMA, + crc64Combined.toString()); + } try { CompleteMultipartUploadResult result = this.obsClient .completeMultipartUpload(completeMultipartUploadRequest); @@ -155,6 +203,8 @@ private CompleteMultipartUploadResult uploadFileCheckPoint(UploadFileRequest upl if (e.getResponseCode() >= 300 && e.getResponseCode() < 500 && e.getResponseCode() != 408) { this.abortMultipartUploadSilent(uploadCheckPoint.uploadID, uploadFileRequest); ServiceUtils.deleteFileIgnoreException(uploadFileRequest.getCheckpointFile()); + } else { + abortUploadFileTaskIfCanceledAndAborted(uploadCheckPoint, uploadFileRequest); } } throw e; @@ -278,8 +328,24 @@ public PartResult call() throws Exception { PartResult tr = null; UploadPart uploadPart = uploadCheckPoint.uploadParts.get(partIndex); tr = new PartResult(partIndex + 1, uploadPart.offset, uploadPart.size); + boolean uploadFileCanceled = (uploadFileRequest.getCancelHandler() != null && uploadFileRequest.getCancelHandler().isCancelled()); + + if (uploadFileCanceled) { + String errorInfo = String.format( + Locale.ROOT, + "Task %d:%s upload part %d canceled.", id, "upload" + id, partIndex + 1); + log.warn(errorInfo); + ObsException e = new ObsException(errorInfo); + tr.setException(e); + tr.setFailed(true); + return tr; + } if (!uploadCheckPoint.isAbort) { - InputStream input = null; + CRC64InputStream crc64InputStream = null; + String failedInfoSuffix = String.format( + Locale.ROOT, + ", task %d:%s upload part %d failed.", + id, "upload" + id, partIndex + 1); try { UploadPartRequest uploadPartRequest = new UploadPartRequest(); uploadPartRequest.setBucketName(uploadFileRequest.getBucketName()); @@ -289,28 +355,28 @@ public PartResult call() throws Exception { uploadPartRequest.setPartNumber(uploadPart.partNumber); uploadPartRequest.setRequesterPays(uploadFileRequest.isRequesterPays()); uploadPartRequest.setUserHeaders(uploadFileRequest.getUserHeaders()); - - if (this.progressManager == null) { - uploadPartRequest.setFile(new File(uploadFileRequest.getUploadFile())); - uploadPartRequest.setOffset(uploadPart.offset); - } else { - input = new FileInputStream(uploadFileRequest.getUploadFile()); - long offset = uploadPart.offset; - long skipByte = input.skip(offset); - if (offset < skipByte) { - log.error(String.format( - Locale.ROOT, - "The actual number of skipped bytes (%d) is less than expected (%d): ", skipByte, - offset)); - } + uploadPartRequest.setCancelHandler(uploadFileRequest.getCancelHandler()); + uploadPartRequest.setNeedCalculateCRC64(uploadFileRequest.isNeedCalculateCRC64()); + uploadPartRequest.setFile(new File(uploadFileRequest.getUploadFile())); + uploadPartRequest.setOffset(uploadPart.offset); + if (uploadFileRequest.isNeedStreamCalculateCRC64() && !uploadFileRequest.isNeedCalculateCRC64()) { + uploadPartRequest.setInput(new FileInputStream(uploadFileRequest.getUploadFile())); + long skipByte = uploadPartRequest.getInput().skip(uploadPart.offset); + crc64InputStream = new CRC64InputStream(uploadPartRequest.getInput()); + uploadPartRequest.setInput(crc64InputStream); + log.info("CRC64InputStream Skip " + skipByte + " bytes; offset : " + uploadPart.offset); + } + if (this.progressManager != null) { // TODO no md5 - uploadPartRequest.setInput(new ProgressInputStream(input, this.progressManager, false)); + this.progressManager.setEndFlag(false); + uploadPartRequest.setProgressManager(this.progressManager); } UploadPartResult result = obsClient.uploadPart(uploadPartRequest); - + tryCRC64StreamCheck(crc64InputStream, result, failedInfoSuffix); PartEtag partEtag = new PartEtag(result.getEtag(), result.getPartNumber()); - uploadCheckPoint.update(partIndex, partEtag, true); + CRC64 partCRC64 = result.getClientCalculatedCRC64(); + uploadCheckPoint.update(partIndex, partEtag, true, partCRC64); tr.setFailed(false); if (uploadFileRequest.isEnableCheckpoint()) { @@ -343,8 +409,8 @@ public PartResult call() throws Exception { e); } } finally { - if (null != input) { - input.close(); + if (null != crc64InputStream) { + crc64InputStream.close(); } } } else { @@ -352,7 +418,49 @@ public PartResult call() throws Exception { } return tr; } + private String tryGetRequestID(UploadPartResult uploadPartResult) { + if (uploadPartResult == null) { + return ""; + } + Map responseHeaders = uploadPartResult.getResponseHeaders(); + if (responseHeaders == null) { + return ""; + } + return (String) responseHeaders.get("request-id"); + } + public void tryCRC64StreamCheck(CRC64InputStream crc64InputStream, UploadPartResult result, String failedInfoSuffix) { + if (uploadFileRequest.isNeedStreamCalculateCRC64() && !uploadFileRequest.isNeedCalculateCRC64()) { + String errorInfo = null; + if (crc64InputStream != null) { + result.setClientCalculatedCRC64(crc64InputStream.getCrc64()); + String sdkCalculatedCRC64 = result.getClientCalculatedCRC64().toString(); + String serverCRC64 = (String) result.getResponseHeaders().get(HASH_CRC64ECMA); + if (serverCRC64 == null) { + errorInfo = + "UploadPartResult.getResponseHeaders() doesn't contains " + HASH_CRC64ECMA + + ", crc64 check failed" + failedInfoSuffix + + " server requestID is " + tryGetRequestID(result); + + } else if (!sdkCalculatedCRC64.equals(serverCRC64)) { + errorInfo = "UploadPart CRC64 mismatch! " + + "sdk calculated downloadFile CRC64 is " + sdkCalculatedCRC64 + + " server returned CRC64 is " + serverCRC64 + + " server requestID is " + tryGetRequestID(result) + + failedInfoSuffix; + } + } else { + errorInfo = "Crc64InputStream is null, crc64 not set" + + " server requestID is " + tryGetRequestID(result) + failedInfoSuffix; + } + if (errorInfo != null) { + log.error(errorInfo); + ObsException obsException = new ObsException(errorInfo); + obsException.setErrorCode(INVALID_CRC_64); + throw obsException; + } + } + } public void setProgressManager(ProgressManager progressManager) { this.progressManager = progressManager; } @@ -368,6 +476,7 @@ private void prepare(UploadFileRequest uploadFileRequest, UploadCheckPoint uploa uploadCheckPoint.uploadParts = splitUploadFile(uploadCheckPoint.uploadFileStatus.size, uploadFileRequest.getPartSize()); uploadCheckPoint.partEtags = new ArrayList(); + uploadCheckPoint.partCRC64s = new ConcurrentHashMap<>(); InitiateMultipartUploadRequest initiateUploadRequest = new InitiateMultipartUploadRequest( uploadFileRequest.getBucketName(), uploadFileRequest.getObjectKey()); @@ -382,6 +491,7 @@ private void prepare(UploadFileRequest uploadFileRequest, UploadCheckPoint uploa initiateUploadRequest.setEncodingType(uploadFileRequest.getEncodingType()); initiateUploadRequest.setIsEncodeHeaders(uploadFileRequest.isEncodeHeaders()); initiateUploadRequest.setUserHeaders(uploadFileRequest.getUserHeaders()); + initiateUploadRequest.setCancelHandler(uploadFileRequest.getCancelHandler()); InitiateMultipartUploadResult initiateUploadResult = this.obsClient .initiateMultipartUpload(initiateUploadRequest); @@ -519,9 +629,12 @@ public synchronized void record(String checkPointFile) throws IOException { * @param partETag * @param completed */ - public synchronized void update(int partIndex, PartEtag partETag, boolean completed) { + public synchronized void update(int partIndex, PartEtag partETag, boolean completed, CRC64 partCRC64) { partEtags.add(partETag); uploadParts.get(partIndex).isCompleted = completed; + if (partCRC64 != null) { + partCRC64s.put(partETag.getPartNumber(), partCRC64); + } } /** @@ -561,6 +674,7 @@ public int hashCode() { result = prime * result + ((objectKey == null) ? 0 : objectKey.hashCode()); result = prime * result + ((bucketName == null) ? 0 : bucketName.hashCode()); result = prime * result + ((partEtags == null) ? 0 : partEtags.hashCode()); + result = prime * result + ((partCRC64s == null) ? 0 : partCRC64s.hashCode()); result = prime * result + ((uploadFile == null) ? 0 : uploadFile.hashCode()); result = prime * result + ((uploadFileStatus == null) ? 0 : uploadFileStatus.hashCode()); result = prime * result + ((uploadID == null) ? 0 : uploadID.hashCode()); @@ -590,6 +704,7 @@ private void assign(UploadCheckPoint tmp) { this.uploadID = tmp.uploadID; this.uploadParts = tmp.uploadParts; this.partEtags = tmp.partEtags; + this.partCRC64s = tmp.partCRC64s; } public int md5; @@ -600,6 +715,7 @@ private void assign(UploadCheckPoint tmp) { public String uploadID; public ArrayList uploadParts; public ArrayList partEtags; + public ConcurrentHashMap partCRC64s; public transient volatile boolean isAbort = false; public transient volatile boolean isDeleteUploadRecordFile = true; } diff --git a/app/src/main/java/com/obs/services/internal/handler/XmlResponsesSaxParser.java b/app/src/main/java/com/obs/services/internal/handler/XmlResponsesSaxParser.java index 25fc2b3..a7b025f 100644 --- a/app/src/main/java/com/obs/services/internal/handler/XmlResponsesSaxParser.java +++ b/app/src/main/java/com/obs/services/internal/handler/XmlResponsesSaxParser.java @@ -982,6 +982,8 @@ public void endElement(String name, String elementText) { public static class CopyObjectResultHandler extends DefaultXmlHandler { private String etag; + private String crc64; + private Date lastModified; public Date getLastModified() { @@ -992,6 +994,10 @@ public String getETag() { return etag; } + public String getCRC64() { + return crc64; + } + @Override public void endElement(String name, String elementText) { if (name.equals("LastModified")) { @@ -1004,6 +1010,8 @@ public void endElement(String name, String elementText) { } } else if (name.equals("ETag")) { etag = elementText; + } else if (name.equals("CRC64")) { + crc64 = elementText; } } } @@ -1646,12 +1654,14 @@ public static class CopyPartResultHandler extends SimpleHandler { private String etag; + private String crc64; + public CopyPartResultHandler(XMLReader xr) { super(xr); } public CopyPartResult getCopyPartResult(int partNumber) { - CopyPartResult result = new CopyPartResult(partNumber, etag, lastModified); + CopyPartResult result = new CopyPartResult(partNumber, etag, lastModified, crc64); return result; } @@ -1666,6 +1676,9 @@ public void endLastModified(String content) { public void endETag(String content) { this.etag = content; } + public void endCRC64(String content) { + this.crc64 = content; + } } diff --git a/app/src/main/java/com/obs/services/internal/io/ProgressInputStream.java b/app/src/main/java/com/obs/services/internal/io/ProgressInputStream.java index 67ea1ec..a6b807f 100644 --- a/app/src/main/java/com/obs/services/internal/io/ProgressInputStream.java +++ b/app/src/main/java/com/obs/services/internal/io/ProgressInputStream.java @@ -31,7 +31,7 @@ public class ProgressInputStream extends FilterInputStream { private boolean endFlag; public ProgressInputStream(InputStream in, ProgressManager progressManager) { - this(in, progressManager, true); + this(in, progressManager, progressManager.isEndFlag()); } public ProgressInputStream(InputStream in, ProgressManager progressManager, boolean endFlag) { diff --git a/app/src/main/java/com/obs/services/internal/service/AbstractRequestConvertor.java b/app/src/main/java/com/obs/services/internal/service/AbstractRequestConvertor.java index 259f94a..f907268 100644 --- a/app/src/main/java/com/obs/services/internal/service/AbstractRequestConvertor.java +++ b/app/src/main/java/com/obs/services/internal/service/AbstractRequestConvertor.java @@ -23,6 +23,7 @@ import com.obs.services.internal.RestStorageService; import com.obs.services.internal.ServiceException; import com.obs.services.internal.trans.NewTransResult; +import com.obs.services.internal.utils.CRC64; import com.obs.services.internal.utils.Mimetypes; import com.obs.services.internal.utils.ServiceUtils; import com.obs.services.model.AuthTypeEnum; @@ -48,9 +49,9 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.TreeMap; -import java.util.Locale; public abstract class AbstractRequestConvertor extends RestStorageService { private static final ILogger log = LoggerBuilder.getLogger("com.obs.services.ObsClient"); @@ -64,6 +65,7 @@ protected static class TransResult { private Map userHeaders = new HashMap<>(); private RequestBody body; + private CRC64 calculatedCrc64; TransResult(Map headers) { this(headers, null, null); @@ -112,6 +114,15 @@ public void setBody(RequestBody body) { public RequestBody getBody() { return body; } + + public CRC64 getCalculatedCrc64() { + return calculatedCrc64; + } + + public void setCalculatedCrc64(CRC64 calculatedCrc64) { + this.calculatedCrc64 = calculatedCrc64; + } + } /** diff --git a/app/src/main/java/com/obs/services/internal/service/ObsMultipartObjectService.java b/app/src/main/java/com/obs/services/internal/service/ObsMultipartObjectService.java index bad7234..d05263c 100644 --- a/app/src/main/java/com/obs/services/internal/service/ObsMultipartObjectService.java +++ b/app/src/main/java/com/obs/services/internal/service/ObsMultipartObjectService.java @@ -60,6 +60,7 @@ protected InitiateMultipartUploadResult initiateMultipartUploadImpl(InitiateMult this.prepareRESTHeaderAcl(request.getBucketName(), result.getHeaders(), request.getAcl()); NewTransResult newTransResult = transObjectRequestWithResult(result, request); + newTransResult.setCancelHandler(request.getCancelHandler()); Response response = performRequest(newTransResult, true, false, false, false); this.verifyResponseContentType(response); @@ -107,6 +108,7 @@ protected CompleteMultipartUploadResult completeMultipartUploadImpl(CompleteMult transResult.setParams(requestParams); transResult.setHeaders(headers); transResult.setBody(createRequestBody(Mimetypes.MIMETYPE_XML, xml)); + transResult.setCancelHandler(request.getCancelHandler()); Response response = performRequest(transResult, true, false, false, false); @@ -240,6 +242,7 @@ protected UploadPartResult uploadPartImpl(UploadPartRequest request) throws Serv try { result = this.transUploadPartRequest(request); NewTransResult newTransResult = transObjectRequestWithResult(result, request); + newTransResult.setCancelHandler(request.getCancelHandler()); response = performRequest(newTransResult); } finally { if (result != null && result.getBody() != null && request.isAutoClose()) { @@ -248,9 +251,14 @@ protected UploadPartResult uploadPartImpl(UploadPartRequest request) throws Serv } } UploadPartResult ret = new UploadPartResult(); - ret.setEtag(response.header(CommonHeaders.ETAG)); ret.setPartNumber(request.getPartNumber()); - setHeadersAndStatus(ret, response); + if (result != null) { + ret.setClientCalculatedCRC64(result.getCalculatedCrc64()); + } + if (response != null) { + ret.setEtag(response.header(CommonHeaders.ETAG)); + setHeadersAndStatus(ret, response); + } return ret; } diff --git a/app/src/main/java/com/obs/services/internal/service/ObsObjectBaseService.java b/app/src/main/java/com/obs/services/internal/service/ObsObjectBaseService.java index d42acec..737ca90 100644 --- a/app/src/main/java/com/obs/services/internal/service/ObsObjectBaseService.java +++ b/app/src/main/java/com/obs/services/internal/service/ObsObjectBaseService.java @@ -69,7 +69,6 @@ import java.io.BufferedInputStream; import java.io.Closeable; -import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.text.ParseException; import java.util.Date; @@ -94,9 +93,8 @@ protected boolean doesObjectExistImpl(GetObjectMetadataRequest request) throws S params.put(ObsRequestParams.VERSION_ID, request.getVersionId()); } boolean doesObjectExist = false; - try { - Response response = performRestHead(request.getBucketName(), request.getObjectKey(), params, headers, - request.getUserHeaders(), request.isEncodeHeaders()); + try (Response response = performRestHead(request.getBucketName(), request.getObjectKey(), params, headers, + request.getUserHeaders(), request.isEncodeHeaders())) { if (200 == response.code()) { doesObjectExist = true; } @@ -251,22 +249,20 @@ protected Object getObjectImpl(GetObjectMetadataRequest request) throws ServiceE // pmd error message: CloseResource - Ensure that resources like this // InputStream object are closed after use // 该接口是下载对象，需要将流返回给客户（调用方），我们不能关闭这个流 - InputStream input = response.body().byteStream(); // NOPMD + obsObject.setObjectContent(response.body().byteStream()); // NOPMD if (getRequest.getProgressListener() != null) { ProgressManager progressManager = new SimpleProgressManager(objMetadata.getContentLength(), 0, getRequest.getProgressListener(), getRequest.getProgressInterval() > 0 ? getRequest.getProgressInterval() : ObsConstraint.DEFAULT_PROGRESS_INTERVAL); - input = new ProgressInputStream(input, progressManager); + obsObject.setObjectContent(new ProgressInputStream(obsObject.getObjectContent(), progressManager)); } int readBufferSize = obsProperties.getIntProperty(ObsConstraint.READ_BUFFER_SIZE, ObsConstraint.DEFAULT_READ_BUFFER_STREAM); if (readBufferSize > 0) { - input = new BufferedInputStream(input, readBufferSize); + obsObject.setObjectContent(new BufferedInputStream(obsObject.getObjectContent(), readBufferSize)); } - - obsObject.setObjectContent(input); return obsObject; } @@ -370,7 +366,7 @@ protected CopyObjectResult copyObjectImpl(CopyObjectRequest request) throws Serv response.header(this.getIHeaders(request.getBucketName()).versionIdHeader()), response.header(this.getIHeaders(request.getBucketName()).copySourceVersionIdHeader()), StorageClassEnum.getValueFromCode(response.header( - this.getIHeaders(request.getBucketName()).storageClassHeader()))); + this.getIHeaders(request.getBucketName()).storageClassHeader())), handler.getCRC64()); setHeadersAndStatus(copyRet, response); if (isExtraAclPutRequired && acl != null) { diff --git a/app/src/main/java/com/obs/services/internal/service/ObsObjectService.java b/app/src/main/java/com/obs/services/internal/service/ObsObjectService.java index 8d51edc..50d28da 100644 --- a/app/src/main/java/com/obs/services/internal/service/ObsObjectService.java +++ b/app/src/main/java/com/obs/services/internal/service/ObsObjectService.java @@ -153,6 +153,7 @@ protected AppendObjectResult appendObjectImpl(AppendObjectRequest request) throw } } } + ret.setClientCalculatedCRC64(result.getCalculatedCrc64()); return ret; } diff --git a/app/src/main/java/com/obs/services/internal/service/RequestConvertor.java b/app/src/main/java/com/obs/services/internal/service/RequestConvertor.java index 5ff42fa..607e3cb 100644 --- a/app/src/main/java/com/obs/services/internal/service/RequestConvertor.java +++ b/app/src/main/java/com/obs/services/internal/service/RequestConvertor.java @@ -16,6 +16,7 @@ import com.obs.log.ILogger; import com.obs.log.LoggerBuilder; +import com.obs.services.exception.ObsException; import com.obs.services.internal.Constants; import com.obs.services.internal.Constants.CommonHeaders; import com.obs.services.internal.Constants.ObsRequestParams; @@ -27,6 +28,7 @@ import com.obs.services.internal.ServiceException; import com.obs.services.internal.SimpleProgressManager; import com.obs.services.internal.io.ProgressInputStream; +import com.obs.services.internal.utils.CRC64; import com.obs.services.internal.utils.Mimetypes; import com.obs.services.internal.utils.RestUtils; import com.obs.services.internal.utils.ServiceUtils; @@ -74,10 +76,10 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Map.Entry; import java.util.Set; -import java.util.Locale; public abstract class RequestConvertor extends AclHeaderConvertor { private static final ILogger log = LoggerBuilder.getLogger("com.obs.services.ObsClient"); @@ -322,6 +324,41 @@ protected RestoreObjectStatus transRestoreObjectResultToRestoreObjectStatus(Rest return ret; } + protected CRC64 tryAddCrc64ForPutObjectRequest(PutObjectRequest request, + IHeaders iheaders, Map headers) { + HashMap userHeaders = request.getUserHeaders(); + String crc64HeaderKey = iheaders.headerPrefix() + CommonHeaders.HASH_CRC64ECMA; + String crc64UserHeaderValue; + if (userHeaders != null && null != (crc64UserHeaderValue = userHeaders.get(crc64HeaderKey))) { + log.warn("userHeader contains " + crc64HeaderKey + + ", value is " + crc64UserHeaderValue + ", sdk crc64 calculation skipped."); + } else if (!request.isNeedCalculateCRC64()) { + log.debug("Request not set 'needCalculateCRC64'"); + } else if (request.getFile() != null) { + try { + CRC64 crc64 = CRC64.fromFile(request.getFile()); + if (request instanceof AppendObjectRequest && + ((AppendObjectRequest) request).getCrc64BeforeAppend() != null) { + AppendObjectRequest appendObjectRequest = (AppendObjectRequest) request; + String crc64BeforeAppend = appendObjectRequest.getCrc64BeforeAppend(); + long crc64BeforeAppendValue = CRC64.fromString(crc64BeforeAppend); + crc64.setValue(CRC64.combine(crc64BeforeAppendValue, crc64.getValue(), request.getFile().length())); + } + long crc64InLong = crc64.getValue(); + String crc64InUnsignedString = crc64.toString(); + headers.put(crc64HeaderKey, crc64InUnsignedString); + log.info("crc64InLong:" + crc64InLong + ", crc64InUnsignedString:" + crc64InUnsignedString); + return crc64; + } catch (IOException | NumberFormatException e) { + throw new ObsException("Failed to calculate crc64, Error :", e); + } + } else { + log.error("sdk crc64 calculation is valid only when " + + "PutObjectRequest.getFile() is not null."); + } + return null; + } + protected TransResult transPutObjectRequest(PutObjectRequest request) throws ServiceException { Map headers = new HashMap(); IHeaders iheaders = this.getIHeaders(request.getBucketName()); @@ -357,6 +394,8 @@ protected TransResult transPutObjectRequest(PutObjectRequest request) throws Ser long contentLengthValue = contentLength == null ? -1L : Long.parseLong(contentLength.toString()); + CRC64 crc64 = tryAddCrc64ForPutObjectRequest(request, iheaders, headers); + if (request.getFile() != null) { if (Mimetypes.MIMETYPE_OCTET_STREAM.equals(contentType)) { contentType = Mimetypes.getInstance().getMimetype(request.getFile()); @@ -366,7 +405,7 @@ protected TransResult transPutObjectRequest(PutObjectRequest request) throws Ser try { request.setInput(new FileInputStream(request.getFile())); } catch (FileNotFoundException e) { - throw new IllegalArgumentException("File doesnot exist"); + throw new IllegalArgumentException("File does not exist"); } contentLengthValue = getContentLengthFromFile(request, contentLengthValue, fileSize); @@ -390,7 +429,9 @@ protected TransResult transPutObjectRequest(PutObjectRequest request) throws Ser : new RepeatableRequestEntity(request.getInput(), contentTypeStr, contentLengthValue, this.obsProperties); - return new TransResult(headers, body); + TransResult transResult = new TransResult(headers, body); + transResult.setCalculatedCrc64(crc64); + return transResult; } private long getContentLengthFromFile(PutObjectRequest request, long contentLengthValue, long fileSize) { @@ -896,6 +937,32 @@ protected TransResult transGetContentSummaryFs(ContentSummaryFsRequest contentSu return new TransResult(headers, params, null); } + protected CRC64 tryAddCrc64ForUploadPartRequest(UploadPartRequest request, IHeaders iheaders, Map headers) { + HashMap userHeaders = request.getUserHeaders(); + String crc64HeaderKey = iheaders.headerPrefix() + CommonHeaders.HASH_CRC64ECMA; + String crc64UserHeaderValue; + if (userHeaders != null && null != (crc64UserHeaderValue = userHeaders.get(crc64HeaderKey))) { + log.warn("userHeader contains " + crc64HeaderKey + ", value is " + crc64UserHeaderValue + + ", sdk crc64 calculation skipped."); + } else if (!request.isNeedCalculateCRC64()) { + log.debug("UploadPartRequest not set 'needCalculateCRC64'"); + } else if (request.getFile() != null) { + try (FileInputStream fileInputStream = new FileInputStream(request.getFile())) { + CRC64 crc64 = CRC64.fromInputStream(fileInputStream, request.getOffset(), request.getPartSize()); + long crc64InLong = crc64.getValue(); + String crc64InUnsignedString = crc64.toString(); + headers.put(crc64HeaderKey, crc64InUnsignedString); + log.info("crc64InLong:" + crc64InLong + ", crc64InUnsignedString:" + crc64InUnsignedString); + return crc64; + } catch (IOException e) { + throw new ServiceException("Failed to calculate crc64, Error :", e); + } + } else { + log.error("sdk crc64 calculation is valid only when " + + "PutObjectRequest.getFile() is not null."); + } + return null; + } protected TransResult transUploadPartRequest(UploadPartRequest request) throws ServiceException { Map params = new HashMap(); params.put(ObsRequestParams.PART_NUMBER, String.valueOf(request.getPartNumber())); @@ -912,6 +979,7 @@ protected TransResult transUploadPartRequest(UploadPartRequest request) throws S this.transSseCHeaders(request.getSseCHeader(), headers, iheaders); long contentLength = -1L; + CRC64 crc64 = tryAddCrc64ForUploadPartRequest(request, iheaders, headers); if (null != request.getFile()) { long fileSize = request.getFile().length(); long offset = (request.getOffset() >= 0 && request.getOffset() < fileSize) ? request.getOffset() : 0; @@ -939,8 +1007,11 @@ protected TransResult transUploadPartRequest(UploadPartRequest request) throws S } } - if (request.getInput() != null && request.getProgressListener() != null) { - ProgressManager progressManager = new SimpleProgressManager(contentLength, 0, request.getProgressListener(), + if (request.getInput() != null && request.getProgressManager() != null) { + request.setInput(new ProgressInputStream(request.getInput(), request.getProgressManager())); + } else if (request.getInput() != null && request.getProgressListener() != null) { + ProgressManager progressManager = + new SimpleProgressManager(contentLength, 0, request.getProgressListener(), request.getProgressInterval() > 0 ? request.getProgressInterval() : ObsConstraint.DEFAULT_PROGRESS_INTERVAL); request.setInput(new ProgressInputStream(request.getInput(), progressManager)); @@ -954,7 +1025,9 @@ protected TransResult transUploadPartRequest(UploadPartRequest request) throws S } RequestBody body = request.getInput() == null ? null : new RepeatableRequestEntity(request.getInput(), contentType, contentLength, this.obsProperties); - return new TransResult(headers, params, body); + TransResult transResult = new TransResult(headers, params, body); + transResult.setCalculatedCrc64(crc64); + return transResult; } protected void transSseKmsHeaders(SseKmsHeader kmsHeader, Map headers, IHeaders iheaders, diff --git a/app/src/main/java/com/obs/services/internal/task/AbstractTaskCallable.java b/app/src/main/java/com/obs/services/internal/task/AbstractTaskCallable.java new file mode 100644 index 0000000..65d72be --- /dev/null +++ b/app/src/main/java/com/obs/services/internal/task/AbstractTaskCallable.java @@ -0,0 +1,45 @@ +/** + * Copyright 2019 Huawei Technologies Co.,Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed + * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.obs.services.internal.task; + +import com.obs.services.AbstractClient; +import com.obs.services.model.HeaderResponse; + +import java.util.concurrent.Callable; + +public abstract class AbstractTaskCallable implements Callable { + private AbstractClient obsClient; + private String bucketName; + + public AbstractTaskCallable(AbstractClient obsClient, String bucketName) { + this.obsClient = obsClient; + this.bucketName = bucketName; + } + + public AbstractClient getObsClient() { + return obsClient; + } + + public void setObsClient(AbstractClient obsClient) { + this.obsClient = obsClient; + } + + public String getBucketName() { + return bucketName; + } + + public void setBucketName(String bucketName) { + this.bucketName = bucketName; + } +} diff --git a/app/src/main/java/com/obs/services/internal/task/UploadFileTask.java b/app/src/main/java/com/obs/services/internal/task/UploadFileTask.java new file mode 100644 index 0000000..e209afe --- /dev/null +++ b/app/src/main/java/com/obs/services/internal/task/UploadFileTask.java @@ -0,0 +1,104 @@ +package com.obs.services.internal.task; + +import static com.obs.services.internal.utils.ServiceUtils.changeFromThrowable; + +import com.obs.log.ILogger; +import com.obs.log.LoggerBuilder; +import com.obs.services.AbstractClient; +import com.obs.services.exception.ObsException; +import com.obs.services.model.CompleteMultipartUploadResult; +import com.obs.services.model.TaskCallback; +import com.obs.services.model.UploadFileRequest; + +import java.util.Optional; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +public class UploadFileTask extends AbstractTaskCallable { + private UploadFileRequest taskRequest; + private TaskCallback completeCallback; + + private Future resultFuture; + + private static final ILogger log = LoggerBuilder.getLogger(UploadFileTask.class); + + public UploadFileTask( + AbstractClient obsClient, + String bucketName, + UploadFileRequest taskRequest, + TaskCallback completeCallback) { + super(obsClient, bucketName); + this.taskRequest = taskRequest; + this.completeCallback = completeCallback; + } + + public Optional getResult() { + try { + Object result = resultFuture.get(); + if (result instanceof CompleteMultipartUploadResult) { + return Optional.of((CompleteMultipartUploadResult) result); + } else { + String errorMsg = "UploadFileTask Error, result is " + + (result != null ? "not instance of CompleteMultipartUploadResult!" : "null"); + errorMsg += (taskRequest.getCancelHandler() != null && taskRequest.getCancelHandler().isCancelled()) ? + ", uploadFileRequest is canceled." : ""; + log.error(errorMsg); + return Optional.empty(); + } + } catch (InterruptedException | ExecutionException e) { + log.error("UploadFileTask Error:" , e); + return Optional.empty(); + } + } + + public void setResultFuture(Future future) { + resultFuture = future; + } + + public boolean cancel() { + if (taskRequest.getCancelHandler() != null) { + taskRequest.getCancelHandler().cancel(); + return true; + } else { + String errorInfo = "UploadFileTask Cancel Error: CancelHandler is null, can not cancel!"; + log.error(errorInfo); + return false; + } + } + + protected CompleteMultipartUploadResult uploadFileWithCallBack() { + try { + CompleteMultipartUploadResult uploadFileResult = getObsClient().uploadFile(taskRequest); + completeCallback.onSuccess(uploadFileResult); + return uploadFileResult; + } catch (ObsException e) { + completeCallback.onException(e, taskRequest); + } catch (Throwable t) { + completeCallback.onException(changeFromThrowable(t), taskRequest); + } + return null; + } + + public boolean isTaskFinished() { + return resultFuture.isDone(); + } + + public void waitUntilFinished() { + try { + resultFuture.get(); + } catch (Throwable t) { + log.warn("UploadFileTask waitUntilFinished Error:", t); + } + } + + /** + * Computes a result, or throws an exception if unable to do so. + * + * @return computed result + * @throws Exception if unable to compute a result + */ + @Override + public Object call() throws Exception { + return uploadFileWithCallBack(); + } +} diff --git a/app/src/main/java/com/obs/services/internal/trans/NewTransResult.java b/app/src/main/java/com/obs/services/internal/trans/NewTransResult.java index bcdccb5..e773c2d 100644 --- a/app/src/main/java/com/obs/services/internal/trans/NewTransResult.java +++ b/app/src/main/java/com/obs/services/internal/trans/NewTransResult.java @@ -1,5 +1,6 @@ package com.obs.services.internal.trans; +import com.obs.services.internal.utils.CallCancelHandler; import com.obs.services.model.HttpMethodEnum; import okhttp3.RequestBody; @@ -17,6 +18,8 @@ public class NewTransResult { private boolean encodeHeaders = false; private boolean encodeUrl = true; + protected CallCancelHandler cancelHandler; + public NewTransResult() { } @@ -105,4 +108,11 @@ public boolean isEncodeUrl() { public void setEncodeUrl(boolean encodeUrl) { this.encodeUrl = encodeUrl; } + public CallCancelHandler getCancelHandler() { + return cancelHandler; + } + + public void setCancelHandler(CallCancelHandler cancelHandler) { + this.cancelHandler = cancelHandler; + } } \ No newline at end of file diff --git a/app/src/main/java/com/obs/services/internal/utils/AccessLoggerUtils.java b/app/src/main/java/com/obs/services/internal/utils/AccessLoggerUtils.java index 5f6d140..9a85e06 100644 --- a/app/src/main/java/com/obs/services/internal/utils/AccessLoggerUtils.java +++ b/app/src/main/java/com/obs/services/internal/utils/AccessLoggerUtils.java @@ -42,7 +42,7 @@ private static String getLogPrefix() { } return new StringBuilder().append(stacktrace.getClassName()).append("|").append(stacktrace.getMethodName()) - .append("|").append(stacktrace.getLineNumber()).append("|").toString(); + .append("|line:").append(stacktrace.getLineNumber()).append("|").toString(); } private static StringBuilder getLog() { diff --git a/app/src/main/java/com/obs/services/internal/utils/CRC64.java b/app/src/main/java/com/obs/services/internal/utils/CRC64.java new file mode 100644 index 0000000..866468f --- /dev/null +++ b/app/src/main/java/com/obs/services/internal/utils/CRC64.java @@ -0,0 +1,392 @@ +package com.obs.services.internal.utils; + +import com.obs.log.ILogger; +import com.obs.log.LoggerBuilder; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.Serializable; +import java.util.Arrays; +import java.util.zip.Checksum; + +public class CRC64 implements Checksum, Serializable { + private static final long serialVersionUID = 5684087791018577616L; + private static final ILogger log = LoggerBuilder.getLogger(CRC64.class); + + private static final long POLY = (long) 0xc96c5795d7870f42L; // ECMA-182 + + /* CRC64 calculation table. */ + private static final long[][] CRC64_TABLE; + + /* Current CRC value. */ + private long value; + + /** + * Initialize with another CRC64's value. + * + * @param origin: + * original crc64 + */ + public CRC64(CRC64 origin) { + this.value = origin.value; + } + + /** + * Initialize with a value of zero. + */ + public CRC64() { + this.value = 0; + } + + /** + * Initialize with a custom CRC value. + * + * @param value + */ + public CRC64(long value) { + this.value = value; + } + + /** + * Initialize by calculating the CRC of the given byte blocks. + * + * @param b + * block of bytes + * @param len + * number of bytes to process + */ + public CRC64(byte[] b, int len) { + this.value = 0; + update(b, len); + } + + /** + * Initialize by calculating the CRC of the given byte blocks. + * + * @param b + * block of bytes + * @param off + * starting offset of the byte block + * @param len + * number of bytes to process + */ + public CRC64(byte[] b, int off, int len) { + this.value = 0; + update(b, off, len); + } + + /** + * Calculate the CRC64 of the given file's content. + * + * @param f + * @return new {@link CRC64} instance initialized to the file's CRC value + * @throws IOException + * in case the {@link FileInputStream#read(byte[])} method fails + */ + public static CRC64 fromFile(File f) throws IOException { + return fromInputStream(new FileInputStream(f)); + } + + /** + * Calculate the CRC64 of the given {@link InputStream} until the end of the + * stream has been reached. + * + * @param in + * the stream will be closed automatically + * @return new {@link CRC64} instance initialized to the {@link InputStream}'s CRC value + * @throws IOException + * in case the {@link InputStream#read(byte[])} method fails + */ + public static CRC64 fromInputStream(InputStream in) throws IOException { + try { + CRC64 crc = new CRC64(); + byte[] b = new byte[4096]; + int l; + + while ((l = in.read(b)) != -1) { + crc.update(b, l); + } + + return crc; + + } finally { + in.close(); + } + } + + /** + * Calculate the CRC64 of the given {@link InputStream} from offset to offset + sizeToReadTotal. + * + * @param in + * the stream will be closed automatically + * @param offset + * the start offset in stream in at which the data is read. + * @param sizeToReadTotal + * the maximum number of bytes to read. + * @return new {@link CRC64} instance initialized to the {@link InputStream}'s CRC value + * @throws IOException + * in case the {@link InputStream#read(byte[])} method fails + */ + public static CRC64 fromInputStream(InputStream in, long offset, long sizeToReadTotal) throws IOException { + try { + long skippedSize = in.skip(offset); + if (skippedSize != offset) { + String errorInfo = + "Failed to skip the input stream to the specified offset:" + + offset + + ". actual skip size is " + + skippedSize; + log.error(errorInfo); + throw new IOException(errorInfo); + } + CRC64 crc = new CRC64(); + int bufferSize = 4096; + byte[] b = new byte[bufferSize]; + int l; + long sizeToRead = Long.min(sizeToReadTotal, bufferSize); + while (sizeToRead > 0 && (l = in.read(b, 0, (int) sizeToRead)) != -1) { + crc.update(b, l); + sizeToReadTotal -= l; + sizeToRead = Long.min(sizeToReadTotal, bufferSize); + } + return crc; + + } finally { + in.close(); + } + } + + /** + * Get long representation of current CRC64 value. + */ + public long getValue() { + return this.value; + } + + /** + * Set long representation of current CRC64 value. + */ + public void setValue(long value) { + this.value = value; + } + + /** + * Update CRC64 with new byte block. + */ + public void update(byte[] b, int len) { + this.update(b, 0, len); + } + + /** + * Update CRC64 with new byte block. + */ + public void update(byte[] b, int off, int len) { + this.value = ~this.value; + + /* fast middle processing, 8 bytes (aligned!) per loop */ + + int idx = off; + while (len >= 8) { + value = + CRC64_TABLE[7][(int) (value & 0xff ^ (b[idx] & 0xff))] + ^ CRC64_TABLE[6][(int) ((value >>> 8) & 0xff ^ (b[idx + 1] & 0xff))] + ^ CRC64_TABLE[5][(int) ((value >>> 16) & 0xff ^ (b[idx + 2] & 0xff))] + ^ CRC64_TABLE[4][(int) ((value >>> 24) & 0xff ^ (b[idx + 3] & 0xff))] + ^ CRC64_TABLE[3][(int) ((value >>> 32) & 0xff ^ (b[idx + 4] & 0xff))] + ^ CRC64_TABLE[2][(int) ((value >>> 40) & 0xff ^ (b[idx + 5] & 0xff))] + ^ CRC64_TABLE[1][(int) ((value >>> 48) & 0xff ^ (b[idx + 6] & 0xff))] + ^ CRC64_TABLE[0][(int) ((value >>> 56) ^ b[idx + 7] & 0xff)]; + idx += 8; + len -= 8; + } + + /* process remaining bytes (can't be larger than 8) */ + while (len > 0) { + value = CRC64_TABLE[0][(int) ((this.value ^ b[idx]) & 0xff)] ^ (this.value >>> 8); + idx++; + len--; + } + + this.value = ~this.value; + } + + public void update(int b) { + this.update(new byte[] {(byte) b}, 0, 1); + } + + public void reset() { + this.value = 0; + } + + // dimension of GF(2) vectors (length of CRC) + private static final int GF2_DIM = 64; + + private static long gf2MatrixTimes(long[] mat, long vec) { + long sum = 0L; + int idx = 0; + while (vec != 0) { + if ((vec & 1) == 1) { + sum ^= mat[idx]; + } + vec >>>= 1; + idx++; + } + return sum; + } + + private static void gf2MatrixSquare(long[] square, long[] mat) { + for (int n = 0; n < GF2_DIM; n++) { + square[n] = gf2MatrixTimes(mat, mat[n]); + } + } + + /* + * calculate the CRC-64 of two sequential blocks and set it to this.value, + * where this.value is the CRC-64 of the first block, + * anotherCRC64.value is the CRC-64 of the second block, and len2 is the + * length of the second block. + */ + public void combineWithAnotherCRC64(CRC64 anotherCRC64, long len2) { + this.value = combine(this.value, anotherCRC64.value, len2); + } + + private static final long[] EVEN_SQUARE; + private static final long[] ODD_SQUARE; + + static { + /* + * Nested tables as described by Mark Adler + */ + CRC64_TABLE = new long[8][256]; + + for (int n = 0; n < 256; n++) { + long crc = n; + for (int k = 0; k < 8; k++) { + if ((crc & 1) == 1) { + crc = (crc >>> 1) ^ POLY; + } else { + crc = (crc >>> 1); + } + } + CRC64_TABLE[0][n] = crc; + } + + /* generate nested CRC table for future slice-by-8 lookup */ + for (int n = 0; n < 256; n++) { + long crc = CRC64_TABLE[0][n]; + for (int k = 1; k < 8; k++) { + crc = CRC64_TABLE[0][(int) (crc & 0xff)] ^ (crc >>> 8); + CRC64_TABLE[k][n] = crc; + } + } + + int n; + long row; + EVEN_SQUARE = new long[GF2_DIM]; // even-power-of-two zeros operator + ODD_SQUARE = new long[GF2_DIM]; // odd-power-of-two zeros operator + // put operator for one zero bit in odd + ODD_SQUARE[0] = POLY; // CRC-64 polynomial + row = 1; + for (n = 1; n < GF2_DIM; n++) { + ODD_SQUARE[n] = row; + row <<= 1; + } + // put operator for two zero bits in even + gf2MatrixSquare(EVEN_SQUARE, ODD_SQUARE); + // put operator for four zero bits in odd + gf2MatrixSquare(ODD_SQUARE, EVEN_SQUARE); + } + + /* + * Return the CRC-64 of two sequential blocks, where crc1 is the CRC-64 of + * the first block, crc2 is the CRC-64 of the second block, and len2 is the + * length of the second block. + */ + public static long combine(long crc1, long crc2, long len2) { + // degenerate case. + if (len2 == 0) { + return crc1; + } + + long[] even = Arrays.copyOf(EVEN_SQUARE, EVEN_SQUARE.length); + long[] odd = Arrays.copyOf(ODD_SQUARE, ODD_SQUARE.length); + // apply len2 zeros to crc1 (first square will put the operator for one + // zero byte, eight zero bits, in even) + do { + // apply zeros operator for this bit of len2 + gf2MatrixSquare(even, odd); + if ((len2 & 1) == 1) { + crc1 = gf2MatrixTimes(even, crc1); + } + len2 >>>= 1; + + // if no more bits set, then done + if (len2 == 0) { + break; + } + + // another iteration of the loop with odd and even swapped + gf2MatrixSquare(odd, even); + if ((len2 & 1) == 1) { + crc1 = gf2MatrixTimes(odd, crc1); + } + len2 >>>= 1; + + // if no more bits set, then done + } while (len2 != 0); + // return combined crc. + crc1 ^= crc2; + return crc1; + } + + /** + * Returns a string representation of the object. In general, the + * {@code toString} method returns a string that + * "textually represents" this object. The result should + * be a concise but informative representation that is easy for a + * person to read. + * It is recommended that all subclasses override this method. + *

+ * The {@code toString} method for class {@code Object} + * returns a string consisting of the name of the class of which the + * object is an instance, the at-sign character `{@code @}', and + * the unsigned hexadecimal representation of the hash code of the + * object. In other words, this method returns a string equal to the + * value of: + *

+ * 
+     * getClass().getName() + '@' + Integer.toHexString(hashCode())
+     *
+ * + * @return a string representation of the object. + */ + @Override + public String toString() { + return Long.toUnsignedString(this.value); + } + + public static String toString(long value) { + return Long.toUnsignedString(value); + } + + public static long fromString(String crcString) throws NumberFormatException { + return Long.parseUnsignedLong(crcString); + } + + @Override + public int hashCode() { + return Long.hashCode(this.value); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } else if (obj instanceof CRC64) { + return ((CRC64) obj).value == this.value; + } + return false; + } +} diff --git a/app/src/main/java/com/obs/services/internal/utils/CRC64InputStream.java b/app/src/main/java/com/obs/services/internal/utils/CRC64InputStream.java new file mode 100644 index 0000000..f338c65 --- /dev/null +++ b/app/src/main/java/com/obs/services/internal/utils/CRC64InputStream.java @@ -0,0 +1,92 @@ +package com.obs.services.internal.utils; + +import java.io.IOException; +import java.io.InputStream; + +public class CRC64InputStream extends InputStream { + private final InputStream inputStream; + + public CRC64 getCrc64() { + return crc64; + } + + private final CRC64 crc64; + + public CRC64InputStream(InputStream inputStream) { + this.inputStream = inputStream; + this.crc64 = new CRC64(); + } + + /** + * Reads the next byte of data from the input stream. The value byte is + * returned as an int in the range 0 to + * 255. If no byte is available because the end of the stream + * has been reached, the value -1 is returned. This method + * blocks until input data is available, the end of the stream is detected, + * or an exception is thrown. + * + *

A subclass must provide an implementation of this method. + * + * @return the next byte of data, or -1 if the end of the + * stream is reached. + * @throws IOException if an I/O error occurs. + */ + @Override + public int read() throws IOException { + int byteRead = this.inputStream.read(); + if (byteRead != -1) { + crc64.update(byteRead); + } + return byteRead; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + int bytesRead = inputStream.read(b, off, len); + if (bytesRead != -1) { + crc64.update(b, off, bytesRead); + } + return bytesRead; + } + @Override + public long skip(long n) throws IOException { + byte[] buffer = new byte[512]; + long totalSkippedBytes = 0L; + long skippedBytes; + long remainBytesToSkip; + while (totalSkippedBytes < n) { + remainBytesToSkip = n - totalSkippedBytes; + skippedBytes = + read(buffer, 0, remainBytesToSkip < buffer.length ? (int) remainBytesToSkip : buffer.length); + if (skippedBytes == -1) { + return totalSkippedBytes; + } + totalSkippedBytes += skippedBytes; + } + return totalSkippedBytes; + } + @Override + public int available() throws IOException { + return inputStream.available(); + } + @Override + public synchronized void mark(int readlimit) { + inputStream.mark(readlimit); + } + @Override + public boolean markSupported() { + return inputStream.markSupported(); + } + + /** + * Closes this input stream and releases any system resources associated + * with the stream. + * + * @throws IOException if an I/O error occurs. + */ + @Override + public void close() throws IOException { + inputStream.close(); + super.close(); + } +} diff --git a/app/src/main/java/com/obs/services/internal/utils/CallCancelHandler.java b/app/src/main/java/com/obs/services/internal/utils/CallCancelHandler.java new file mode 100644 index 0000000..aef3be0 --- /dev/null +++ b/app/src/main/java/com/obs/services/internal/utils/CallCancelHandler.java @@ -0,0 +1,79 @@ +package com.obs.services.internal.utils; + +import com.obs.log.ILogger; +import com.obs.log.LoggerBuilder; +import com.obs.services.exception.ObsException; + +import okhttp3.Call; + +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +public class CallCancelHandler +{ + private final AtomicInteger maxCancelQueueCapacity = new AtomicInteger(0); + protected static final ILogger log = LoggerBuilder.getLogger(CallCancelHandler.class); + protected AtomicBoolean isCancelled = new AtomicBoolean(false); + + protected ConcurrentLinkedQueue calls = new ConcurrentLinkedQueue<>(); + + /** + * cancel + * + */ + public void cancel() { + isCancelled.set(true); + calls.forEach( + call -> { + if (call != null && !call.isCanceled()) { + call.cancel(); + } + }); + calls.clear(); + } + + public boolean isCancelled() { + return isCancelled.get(); + } + + public void setCall(Call call) { + if (this.isCancelled.get()) { + String msg = "transport is cancelled"; + if (call == null) { + msg += ", call is null"; + } else if(call.request() == null) { + msg += ", call's request is null"; + } else { + msg += (", url :" + call.request().url()); + } + log.warn(msg); + throw new ObsException("transport is cancelled by cancelHandler"); + } + if (calls.size() >= maxCancelQueueCapacity.get()) { + log.debug( + this.getClass().getName() + + "'s calls Capacity is full. cancel may not working! " + + "try adjust it by setMaxCallCapacity"); + } else { + this.calls.add(call); + } + } + + public void removeFinishedCall(Call call) { + calls.remove(call); + } + + public void resetCancelStatus() { + calls.clear(); + isCancelled.set(false); + } + + public int getMaxCallCapacity() { + return maxCancelQueueCapacity.get(); + } + + public void setMaxCallCapacity(int maxCancelQueueCapacity) { + this.maxCancelQueueCapacity.set(maxCancelQueueCapacity); + } +} diff --git a/app/src/main/java/com/obs/services/internal/utils/OkhttpCallProfiler.java b/app/src/main/java/com/obs/services/internal/utils/OkhttpCallProfiler.java new file mode 100644 index 0000000..73cbe44 --- /dev/null +++ b/app/src/main/java/com/obs/services/internal/utils/OkhttpCallProfiler.java @@ -0,0 +1,591 @@ +package com.obs.services.internal.utils; + +import com.obs.log.ILogger; +import com.obs.log.LoggerBuilder; +import okhttp3.Call; +import okhttp3.Connection; +import okhttp3.EventListener; +import okhttp3.Handshake; +import okhttp3.HttpUrl; +import okhttp3.Protocol; +import okhttp3.Request; +import okhttp3.Response; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.Proxy; +import java.util.HashMap; +import java.util.List; +import java.util.function.Consumer; + +public class OkhttpCallProfiler extends EventListener { + private final Consumer profiler; + private final HashMap progressStartTime; + private static final ILogger log = LoggerBuilder.getLogger("com.obs.services.ObsClient"); + private static final String ProFileTimeUnit = "ms"; + static protected boolean isEnabled = true; + + public OkhttpCallProfiler(Consumer profiler) { + this.profiler = profiler; + progressStartTime = new HashMap<>(); + } + + public OkhttpCallProfiler() { + this.profiler = log::debug; + progressStartTime = new HashMap<>(); + } + + /** + * @param call + */ + @Override + public void callEnd(Call call) { + if (!isEnabled) { + return; + } + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append(" callEnd "); + appendProgressTime("call", stringBuilder); + appendDetailForCall(call, stringBuilder); + profiler.accept(stringBuilder); + } + + /** + * @param call + * @param ioe + */ + @Override + public void callFailed(Call call, IOException ioe) { + if (!isEnabled) { + return; + } + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append(" callFailed "); + appendProgressTime("call", stringBuilder); + appendDetailIOException(ioe, stringBuilder); + appendDetailForCall(call, stringBuilder); + profiler.accept(stringBuilder); + } + + /** + * @param call + */ + @Override + public void callStart(Call call) { + if (!isEnabled) { + return; + } + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append(" callStart "); + appendProgressTime("call", stringBuilder); + appendDetailForCall(call, stringBuilder, true); + profiler.accept(stringBuilder); + } + + /** + * @param call + */ + @Override + public void canceled(Call call) { + if (!isEnabled) { + return; + } + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append(" call canceled."); + appendProgressTime("call", stringBuilder); + appendDetailForCall(call, stringBuilder); + profiler.accept(stringBuilder); + } + + /** + * @param call + * @param inetSocketAddress + * @param proxy + * @param protocol + */ + @Override + public void connectEnd(Call call, InetSocketAddress inetSocketAddress, Proxy proxy, Protocol protocol) { + if (!isEnabled) { + return; + } + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append(" connectEnd "); + appendProgressTime("connect", stringBuilder); + appendDetailInetSocketAddress(inetSocketAddress, stringBuilder); + appendDetailProxy(proxy, stringBuilder); + appendDetailProtocol(protocol, stringBuilder); + appendDetailForCall(call, stringBuilder); + profiler.accept(stringBuilder); + } + + /** + * @param call + * @param inetSocketAddress + * @param proxy + * @param protocol + * @param ioe + */ + @Override + public void connectFailed( + Call call, InetSocketAddress inetSocketAddress, Proxy proxy, Protocol protocol, IOException ioe) { + if (!isEnabled) { + return; + } + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append(" connectFailed "); + appendProgressTime("connect", stringBuilder); + appendDetailInetSocketAddress(inetSocketAddress, stringBuilder); + appendDetailProxy(proxy, stringBuilder); + appendDetailProtocol(protocol, stringBuilder); + appendDetailIOException(ioe, stringBuilder); + appendDetailForCall(call, stringBuilder); + profiler.accept(stringBuilder); + } + + /** + * @param call + * @param inetSocketAddress + * @param proxy + */ + @Override + public void connectStart(Call call, InetSocketAddress inetSocketAddress, Proxy proxy) { + if (!isEnabled) { + return; + } + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append(" connectStart "); + appendProgressTime("connect", stringBuilder); + appendDetailInetSocketAddress(inetSocketAddress, stringBuilder); + appendDetailProxy(proxy, stringBuilder); + appendDetailForCall(call, stringBuilder); + profiler.accept(stringBuilder); + } + + /** + * @param call + * @param connection + */ + @Override + public void connectionAcquired(Call call, Connection connection) { + if (!isEnabled) { + return; + } + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append(" connectionAcquired "); + appendProgressTime("connection", stringBuilder); + appendDetailConnection(connection, stringBuilder); + appendDetailForCall(call, stringBuilder); + profiler.accept(stringBuilder); + } + + /** + * @param call + * @param connection + */ + @Override + public void connectionReleased(Call call, Connection connection) { + if (!isEnabled) { + return; + } + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append(" connectionReleased "); + appendProgressTime("connection", stringBuilder); + appendDetailConnection(connection, stringBuilder); + appendDetailForCall(call, stringBuilder); + profiler.accept(stringBuilder); + } + + /** + * @param call + * @param domainName + * @param inetAddressList + */ + @Override + public void dnsEnd(Call call, String domainName, List inetAddressList) { + if (!isEnabled) { + return; + } + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append(" dnsEnd "); + appendProgressTime("dns", stringBuilder); + appendDetailDns(domainName, inetAddressList, stringBuilder); + appendDetailForCall(call, stringBuilder); + profiler.accept(stringBuilder); + } + + /** + * @param call + * @param domainName + */ + @Override + public void dnsStart(Call call, String domainName) { + if (!isEnabled) { + return; + } + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append(" dnsStart "); + appendProgressTime("dns", stringBuilder); + appendDetailDns(domainName, null, stringBuilder); + appendDetailForCall(call, stringBuilder); + profiler.accept(stringBuilder); + } + + /** + * @param call + * @param url + * @param proxies + */ + @Override + public void proxySelectEnd(Call call, HttpUrl url, List proxies) { + if (!isEnabled) { + return; + } + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append(" proxySelectEnd "); + appendProgressTime("proxySelect", stringBuilder); + stringBuilder.append(" url{"); + stringBuilder.append(url); + stringBuilder.append("} "); + stringBuilder.append(" proxies{"); + stringBuilder.append(proxies); + stringBuilder.append("} "); + appendDetailForCall(call, stringBuilder); + profiler.accept(stringBuilder); + } + + /** + * @param call + * @param url + */ + @Override + public void proxySelectStart(Call call, HttpUrl url) { + if (!isEnabled) { + return; + } + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append(" proxySelectStart "); + appendProgressTime("proxySelect", stringBuilder); + stringBuilder.append(" url{"); + stringBuilder.append(url); + stringBuilder.append("} "); + appendDetailForCall(call, stringBuilder); + profiler.accept(stringBuilder); + } + + /** + * @param call + * @param byteCount + */ + @Override + public void requestBodyEnd(Call call, long byteCount) { + if (!isEnabled) { + return; + } + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append(" requestBodyEnd "); + appendProgressTime("requestBody", stringBuilder); + appendProgressTime("request", stringBuilder); + stringBuilder.append(" byteCount{"); + stringBuilder.append(byteCount); + stringBuilder.append("} "); + appendDetailForCall(call, stringBuilder); + profiler.accept(stringBuilder); + } + + /** + * @param call + */ + @Override + public void requestBodyStart(Call call) { + if (!isEnabled) { + return; + } + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append(" requestBodyStart "); + appendProgressTime("requestBody", stringBuilder); + appendDetailForCall(call, stringBuilder); + profiler.accept(stringBuilder); + } + + /** + * @param call + * @param ioe + */ + @Override + public void requestFailed(Call call, IOException ioe) { + if (!isEnabled) { + return; + } + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append(" requestFailed "); + appendProgressTime("request", stringBuilder); + appendDetailIOException(ioe, stringBuilder); + appendDetailForCall(call, stringBuilder); + profiler.accept(stringBuilder); + } + + /** + * @param call + * @param request + */ + @Override + public void requestHeadersEnd(Call call, Request request) { + if (!isEnabled) { + return; + } + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append(" requestHeadersEnd "); + appendProgressTime("requestHeaders", stringBuilder); + appendDetailRequestHeaders(request, stringBuilder); + appendDetailForCall(call, stringBuilder); + profiler.accept(stringBuilder); + } + + /** + * @param call + */ + @Override + public void requestHeadersStart(Call call) { + if (!isEnabled) { + return; + } + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append(" requestHeadersStart "); + appendProgressTime("request", stringBuilder); + appendProgressTime("requestHeaders", stringBuilder); + appendDetailForCall(call, stringBuilder); + profiler.accept(stringBuilder); + } + + /** + * @param call + * @param byteCount + */ + @Override + public void responseBodyEnd(Call call, long byteCount) { + if (!isEnabled) { + return; + } + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append(" responseBodyEnd "); + appendProgressTime("responseBody", stringBuilder); + appendProgressTime("response", stringBuilder); + stringBuilder.append(" byteCount{"); + stringBuilder.append(byteCount); + stringBuilder.append("} "); + appendDetailForCall(call, stringBuilder); + profiler.accept(stringBuilder); + } + + /** + * @param call + */ + @Override + public void responseBodyStart(Call call) { + if (!isEnabled) { + return; + } + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append(" responseBodyStart "); + appendProgressTime("responseBody", stringBuilder); + appendDetailForCall(call, stringBuilder); + profiler.accept(stringBuilder); + } + + /** + * @param call + * @param ioe + */ + @Override + public void responseFailed(Call call, IOException ioe) { + if (!isEnabled) { + return; + } + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append(" responseFailed "); + appendProgressTime("response", stringBuilder); + appendDetailIOException(ioe, stringBuilder); + appendDetailForCall(call, stringBuilder); + profiler.accept(stringBuilder); + } + + /** + * @param call + * @param response + */ + @Override + public void responseHeadersEnd(Call call, Response response) { + if (!isEnabled) { + return; + } + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append(" responseHeadersEnd "); + appendProgressTime("responseHeaders", stringBuilder); + appendDetailResponseHeaders(response, stringBuilder); + appendDetailForCall(call, stringBuilder); + profiler.accept(stringBuilder); + } + + /** + * @param call + */ + @Override + public void responseHeadersStart(Call call) { + if (!isEnabled) { + return; + } + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append(" responseHeadersStart "); + appendProgressTime("response", stringBuilder); + appendProgressTime("responseHeaders", stringBuilder); + appendDetailForCall(call, stringBuilder); + profiler.accept(stringBuilder); + } + + /** + * @param call + * @param handshake + */ + @Override + public void secureConnectEnd(Call call, Handshake handshake) { + if (!isEnabled) { + return; + } + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append(" secureConnectEnd "); + appendProgressTime("secureConnect", stringBuilder); + appendDetailHandshake(handshake, stringBuilder); + appendDetailForCall(call, stringBuilder); + profiler.accept(stringBuilder); + } + + /** + * @param call + */ + @Override + public void secureConnectStart(Call call) { + if (!isEnabled) { + return; + } + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append(" secureConnectStart "); + appendProgressTime("secureConnect", stringBuilder); + appendDetailForCall(call, stringBuilder); + profiler.accept(stringBuilder); + } + + protected void appendProgressTime(String progressName, StringBuilder stringBuilder) { + Long startTime = this.progressStartTime.get(progressName); + if (startTime == null) { + startTime = System.currentTimeMillis(); + this.progressStartTime.put(progressName, startTime); + stringBuilder.append(progressName); + stringBuilder.append(" start time: "); + stringBuilder.append(startTime); + stringBuilder.append(ProFileTimeUnit); + stringBuilder.append(" "); + } else { + long currentTime = System.currentTimeMillis(); + long costTime = currentTime - startTime; + this.progressStartTime.remove(progressName, startTime); + stringBuilder.append(progressName); + stringBuilder.append(" cost time: "); + stringBuilder.append(costTime); + stringBuilder.append(ProFileTimeUnit); + stringBuilder.append(" end time: "); + stringBuilder.append(currentTime); + stringBuilder.append(ProFileTimeUnit); + stringBuilder.append(" "); + } + } + + protected void appendDetailForCall(Call call, StringBuilder stringBuilder) { + appendDetailForCall(call, stringBuilder, false); + } + protected void appendDetailForCall(Call call, StringBuilder stringBuilder, boolean needDetail) { + stringBuilder.append("call {"); + if (needDetail) { + stringBuilder.append("url:"); + stringBuilder.append(call.request().url()); + stringBuilder.append(" method:"); + stringBuilder.append(call.request().method()); + stringBuilder.append(" "); + } + stringBuilder.append("hash:"); + stringBuilder.append(System.identityHashCode(call)); + stringBuilder.append("} "); + } + + protected void appendDetailInetSocketAddress(InetSocketAddress inetSocketAddress, StringBuilder stringBuilder) { + stringBuilder.append(" InetSocketAddress {"); + stringBuilder.append(inetSocketAddress); + stringBuilder.append("} "); + } + + protected void appendDetailProxy(Proxy proxy, StringBuilder stringBuilder) { + stringBuilder.append(" Proxy {type:"); + stringBuilder.append(proxy.type()); + stringBuilder.append(" address:"); + stringBuilder.append(proxy.address()); + stringBuilder.append("} "); + } + + protected void appendDetailProtocol(Protocol protocol, StringBuilder stringBuilder) { + stringBuilder.append(" Protocol{"); + stringBuilder.append(protocol); + stringBuilder.append("} "); + } + + protected void appendDetailConnection(Connection connection, StringBuilder stringBuilder) { + stringBuilder.append(" "); + stringBuilder.append(connection); + stringBuilder.append(" "); + } + + protected void appendDetailDns(String domainName, List inetAddressList, StringBuilder stringBuilder) { + stringBuilder.append(" domainName:"); + stringBuilder.append(domainName); + stringBuilder.append(" inetAddressList{"); + stringBuilder.append(inetAddressList); + stringBuilder.append("} "); + } + + protected void appendDetailRequestHeaders(Request request, StringBuilder stringBuilder) { + stringBuilder.append(" requestHeaders{"); + stringBuilder.append(request.headers()); + stringBuilder.append("} "); + } + + protected void appendDetailResponseHeaders(Response response, StringBuilder stringBuilder) { + stringBuilder.append(" responseHeaders{"); + stringBuilder.append(response.headers()); + stringBuilder.append("} "); + } + + protected void appendDetailHandshake(Handshake handshake, StringBuilder stringBuilder) { + stringBuilder.append(" handshake{"); + stringBuilder.append(handshake); + stringBuilder.append("} "); + } + + protected void appendDetailIOException(IOException ioe, StringBuilder stringBuilder) { + try (StringWriter stringWriter = new StringWriter(); + PrintWriter printWriter = new PrintWriter(stringWriter)) { + ioe.printStackTrace(printWriter); + stringBuilder.append(stringWriter); + } catch (IOException ignore) { + stringBuilder.append("appendDetailIOException failed"); + } + } + + public static boolean isEnabled() { + return isEnabled; + } + + public static void setEnabled(boolean enabled) { + isEnabled = enabled; + } +} diff --git a/app/src/main/java/com/obs/services/internal/utils/RestUtils.java b/app/src/main/java/com/obs/services/internal/utils/RestUtils.java index 642b90a..a01767a 100644 --- a/app/src/main/java/com/obs/services/internal/utils/RestUtils.java +++ b/app/src/main/java/com/obs/services/internal/utils/RestUtils.java @@ -29,12 +29,12 @@ import okhttp3.Credentials; import okhttp3.Dispatcher; import okhttp3.Dns; +import okhttp3.EventListener; import okhttp3.OkHttpClient; import okhttp3.Protocol; import okhttp3.Request; import okhttp3.Response; import okhttp3.Route; -import org.jetbrains.annotations.NotNull; import javax.net.SocketFactory; import javax.net.ssl.HostnameVerifier; @@ -333,7 +333,8 @@ public Socket createSocket(InetAddress address, int port, InetAddress localAddre public static OkHttpClient.Builder initHttpClientBuilder(ObsProperties obsProperties, KeyManagerFactory keyManagerFactory, TrustManagerFactory trustManagerFactory, - Dispatcher httpDispatcher, Dns customizedDnsImpl, HostnameVerifier userHostnameVerifier, SecureRandom secureRandom) { + Dispatcher httpDispatcher, Dns customizedDnsImpl, EventListener.Factory eventListenerFactory, + HostnameVerifier userHostnameVerifier, SecureRandom secureRandom) { List protocols = new ArrayList(2); protocols.add(Protocol.HTTP_1_1); @@ -384,6 +385,10 @@ public static OkHttpClient.Builder initHttpClientBuilder(ObsProperties obsProper .hostnameVerifier(hostnameVerifier) .dns(dns); + if (eventListenerFactory != null) { + builder.eventListenerFactory(eventListenerFactory); + } + int socketReadBufferSize = obsProperties.getIntProperty(ObsConstraint.SOCKET_READ_BUFFER_SIZE, -1); int socketWriteBufferSize = obsProperties.getIntProperty(ObsConstraint.SOCKET_WRITE_BUFFER_SIZE, -1); @@ -501,9 +506,8 @@ public DefaultObsDns() { * @return * @throws UnknownHostException */ - @NotNull @Override - public List lookup(@NotNull String hostname) throws UnknownHostException { + public List lookup(String hostname) throws UnknownHostException { List adds = Dns.SYSTEM.lookup(hostname); log.info("internet host address:" + adds); return adds; diff --git a/app/src/main/java/com/obs/services/internal/utils/SecureObjectInputStream.java b/app/src/main/java/com/obs/services/internal/utils/SecureObjectInputStream.java index ec164e4..a55c76c 100644 --- a/app/src/main/java/com/obs/services/internal/utils/SecureObjectInputStream.java +++ b/app/src/main/java/com/obs/services/internal/utils/SecureObjectInputStream.java @@ -33,7 +33,16 @@ public final class SecureObjectInputStream extends ObjectInputStream { "com.obs.services.internal.UploadResumableClient$UploadPart", "com.obs.services.internal.DownloadResumableClient$DownloadCheckPoint", "com.obs.services.internal.DownloadResumableClient$DownloadPart", - "com.obs.services.internal.DownloadResumableClient$ObjectStatus")); + "com.obs.services.internal.DownloadResumableClient$ObjectStatus", + "com.obs.services.internal.utils.CRC64", + "java.util.concurrent.ConcurrentHashMap", + "[Ljava.util.concurrent.ConcurrentHashMap$Segment;", + "java.util.concurrent.ConcurrentHashMap$Segment", + "java.util.concurrent.locks.ReentrantLock", + "java.util.concurrent.locks.ReentrantLock$NonfairSync", + "java.util.concurrent.locks.ReentrantLock$Sync", + "java.util.concurrent.locks.AbstractQueuedSynchronizer", + "java.util.concurrent.locks.AbstractOwnableSynchronizer")); public SecureObjectInputStream() throws IOException, SecurityException { super(); diff --git a/app/src/main/java/com/obs/services/internal/utils/ServiceUtils.java b/app/src/main/java/com/obs/services/internal/utils/ServiceUtils.java index 7edfed1..d9f2659 100644 --- a/app/src/main/java/com/obs/services/internal/utils/ServiceUtils.java +++ b/app/src/main/java/com/obs/services/internal/utils/ServiceUtils.java @@ -74,7 +74,16 @@ public class ServiceUtils { private static Pattern pattern = Pattern .compile("^((2[0-4]\\d|25[0-5]|[01]?\\d\\d?)\\.){3}(2[0-4]\\d|25[0-5]|[01]?\\d\\d?)$"); - private static DateTimeFormatter rfc822DateTimeFormatter = DateTimeFormatter.ofPattern(RFC_822_TIME_PARSER_STRING, Locale.US).withZone(ZoneId.of("GMT")); + private static DateTimeFormatter rfc822DateTimeFormatter = null; + + static { + try { + Class.forName("java.time.format.DateTimeFormatter"); + rfc822DateTimeFormatter = DateTimeFormatter.ofPattern(RFC_822_TIME_PARSER_STRING, Locale.US).withZone(ZoneId.of("GMT")); + } catch (Throwable e) { + log.error("failed to load java.time.format.DateTimeFormatter", e); + } + } public static boolean isValid(String s) { return s != null && !s.equals(""); @@ -154,7 +163,7 @@ public static String formatIso8601MidnightDate(Date date) { public static Date parseRfc822Date(String dateString) throws ParseException { try { return Date.from(ZonedDateTime.parse(dateString, rfc822DateTimeFormatter).toInstant()); - } catch (Exception e) { + } catch (Throwable e) { log.warn("parseRfc822Date with DateTimeFormatter failed, using SimpleDateFormat instead, error detail :", e); SimpleDateFormat rfc822TimeParser = new SimpleDateFormat(RFC_822_TIME_PARSER_STRING, Locale.US); rfc822TimeParser.setTimeZone(Constants.GMT_TIMEZONE); @@ -165,7 +174,7 @@ public static Date parseRfc822Date(String dateString) throws ParseException { public static String formatRfc822Date(Date date) { try { return rfc822DateTimeFormatter.format(date.toInstant()); - } catch (Exception e) { + } catch (Throwable e) { log.warn("formatRfc822Date with DateTimeFormatter failed, using SimpleDateFormat instead, error detail :", e); SimpleDateFormat rfc822TimeParser = new SimpleDateFormat(RFC_822_TIME_PARSER_STRING, Locale.US); rfc822TimeParser.setTimeZone(Constants.GMT_TIMEZONE); @@ -294,7 +303,7 @@ public static Map cleanUserMetadata(Map original } public static void cleanListMetadata(Map originalHeaders, boolean decodeHeaders, - Map userMetadata, String key) + Map userMetadata, String key) throws UnsupportedEncodingException { List cleanedValue = new ArrayList<>(); for (Object v : (List) originalHeaders.get(key)) { @@ -547,14 +556,22 @@ public static ObsException changeFromException(Exception e) { } } + public static ObsException changeFromThrowable(Throwable t) { + boolean isObsException = t instanceof ObsException; + if (!isObsException) { + return new ObsException(t.getMessage(), t); + } else { + return (ObsException) t; + } + } public static ObsException changeFromServiceException(ServiceException se) { ObsException exception; if (se.getResponseCode() < 0) { - exception = new ObsException("OBS service Error Message. " + se.getMessage(), se.getCause()); + exception = new ObsException("OBS service Error Message. " + se.getMessage(), se); } else { exception = new ObsException( (se.getMessage() != null ? "Error message:" + se.getMessage() : "") + "OBS service Error Message.", - se.getXmlMessage(), se.getCause()); + se.getXmlMessage(), se); exception.setErrorCode(se.getErrorCode()); exception.setErrorMessage(se.getErrorMessage() == null ? se.getMessage() : se.getErrorMessage()); exception.setErrorRequestId(se.getErrorRequestId()); @@ -784,4 +801,43 @@ public static String getLoggableInfo(String infoKey,String infoVal) { return LoggableInfo; } } + + public static void tokenMasked(StringBuilder info) { + int indexOfToken; + if (info != null && (indexOfToken = info.indexOf("security-token:")) != -1) { + int start = indexOfToken + "security-token:".length(); + int end = info.indexOf("\n", start); + if(end == -1) { + end = info.indexOf("|", start); + } + if(end == -1) { + end = info.length(); + } + if(start < end) { + info.replace(start, end, LoggableInfo); + } + } + } + public static void bytesMasked(StringBuilder info) { + int startStringToSign = info.lastIndexOf(""); + if (startStringToSign > 0) { + int endStringToSign = info.lastIndexOf(""); + if (endStringToSign > 0) { + startStringToSign += "".length(); + if(startStringToSign < endStringToSign) { + info.replace(startStringToSign, endStringToSign, LoggableInfo); + } + } + } + } + public static String messageMasked(String info) { + if (info == null || !info.contains("security-token:")) { + return info; + } else { + StringBuilder stringBuilder = new StringBuilder(info); + tokenMasked(stringBuilder); + bytesMasked(stringBuilder); + return stringBuilder.toString(); + } + } } diff --git a/app/src/main/java/com/obs/services/internal/xml/OBSXMLBuilder.java b/app/src/main/java/com/obs/services/internal/xml/OBSXMLBuilder.java index 33bb6e7..ec1cb28 100644 --- a/app/src/main/java/com/obs/services/internal/xml/OBSXMLBuilder.java +++ b/app/src/main/java/com/obs/services/internal/xml/OBSXMLBuilder.java @@ -337,9 +337,14 @@ public String asString() throws TransformerException { } Transformer transformer = tf.newTransformer(); transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes"); - StringWriter writer = new StringWriter(); - transformer.transform(new DOMSource(this.getDocument()), new StreamResult(writer)); - return writer.getBuffer().toString().replaceAll("|\r", ""); + try (StringWriter writer = new StringWriter()) { + transformer.transform(new DOMSource(this.getDocument()), new StreamResult(writer)); + return writer.getBuffer().toString().replaceAll("|\r", ""); + } catch (IOException e) { + log.error("Transformer.transform failed, detail:", e); + throw new TransformerException(e); + } + } public int hashCode() throws UnsupportedOperationException { diff --git a/app/src/main/java/com/obs/services/model/AppendObjectRequest.java b/app/src/main/java/com/obs/services/model/AppendObjectRequest.java index 45b1eb4..6822652 100644 --- a/app/src/main/java/com/obs/services/model/AppendObjectRequest.java +++ b/app/src/main/java/com/obs/services/model/AppendObjectRequest.java @@ -24,6 +24,8 @@ public class AppendObjectRequest extends PutObjectRequest { protected long position; + protected String crc64BeforeAppend; + { this.httpMethod = HttpMethodEnum.POST; } @@ -54,4 +56,11 @@ public void setPosition(long position) { this.position = position; } + public String getCrc64BeforeAppend() { + return crc64BeforeAppend; + } + + public void setCrc64BeforeAppend(String crc64BeforeAppend) { + this.crc64BeforeAppend = crc64BeforeAppend; + } } diff --git a/app/src/main/java/com/obs/services/model/AppendObjectResult.java b/app/src/main/java/com/obs/services/model/AppendObjectResult.java index bfa12d9..b38efde 100644 --- a/app/src/main/java/com/obs/services/model/AppendObjectResult.java +++ b/app/src/main/java/com/obs/services/model/AppendObjectResult.java @@ -14,6 +14,8 @@ package com.obs.services.model; +import com.obs.services.internal.utils.CRC64; + /** * Response to an appendable upload request * @@ -32,6 +34,8 @@ public class AppendObjectResult extends HeaderResponse { private String objectUrl; + private CRC64 clientCalculatedCRC64; + public AppendObjectResult(String bucketName, String objectKey, String etag, long nextPosition, StorageClassEnum storageClass, String objectUrl) { this.bucketName = bucketName; @@ -96,10 +100,19 @@ public String getObjectUrl() { return objectUrl; } + public CRC64 getClientCalculatedCRC64() { + return clientCalculatedCRC64; + } + + public void setClientCalculatedCRC64(CRC64 clientCalculatedCRC64) { + this.clientCalculatedCRC64 = clientCalculatedCRC64; + } + @Override public String toString() { return "AppendObjectResult [bucketName=" + bucketName + ", objectKey=" + objectKey + ", etag=" + etag - + ", nextPosition=" + nextPosition + ", storageClass=" + storageClass + ", objectUrl=" + objectUrl + + ", nextPosition=" + nextPosition + ", storageClass=" + storageClass + ", objectUrl=" + objectUrl + + ", clientCalculatedCRC64=" + clientCalculatedCRC64 + "]"; } diff --git a/app/src/main/java/com/obs/services/model/CopyObjectResult.java b/app/src/main/java/com/obs/services/model/CopyObjectResult.java index cb1abe7..8b4b723 100644 --- a/app/src/main/java/com/obs/services/model/CopyObjectResult.java +++ b/app/src/main/java/com/obs/services/model/CopyObjectResult.java @@ -24,6 +24,8 @@ public class CopyObjectResult extends HeaderResponse { private String etag; + private String crc64; + private Date lastModified; private String versionId; @@ -33,12 +35,13 @@ public class CopyObjectResult extends HeaderResponse { private StorageClassEnum storageClass; public CopyObjectResult(String etag, Date lastModified, String versionId, String copySourceVersionId, - StorageClassEnum storageClass) { + StorageClassEnum storageClass, String crc64) { this.etag = etag; this.lastModified = ServiceUtils.cloneDateIgnoreNull(lastModified); this.versionId = versionId; this.copySourceVersionId = copySourceVersionId; this.storageClass = storageClass; + this.crc64 = crc64; } /** @@ -86,6 +89,15 @@ public StorageClassEnum getObjectStorageClass() { return storageClass; } + /** + * Obtain the crc64 of the destination object. + * + * @return crc64 value of the destination object + */ + public String getCRC64() { + return crc64; + } + @Override public String toString() { return "CopyObjectResult [etag=" + etag + ", lastModified=" + lastModified + ", versionId=" + versionId diff --git a/app/src/main/java/com/obs/services/model/CopyPartResult.java b/app/src/main/java/com/obs/services/model/CopyPartResult.java index 358b201..b2c506a 100644 --- a/app/src/main/java/com/obs/services/model/CopyPartResult.java +++ b/app/src/main/java/com/obs/services/model/CopyPartResult.java @@ -27,11 +27,13 @@ public class CopyPartResult extends HeaderResponse { private String etag; private Date lastModified; + private String crc64; - public CopyPartResult(int partNumber, String etag, Date lastModified) { + public CopyPartResult(int partNumber, String etag, Date lastModified, String crc64) { this.partNumber = partNumber; this.etag = etag; this.lastModified = ServiceUtils.cloneDateIgnoreNull(lastModified); + this.crc64 = crc64; } /** @@ -52,6 +54,15 @@ public String getEtag() { return etag; } + /** + * Obtain the crc64 of the copied part. + * + * @return crc64 of the copied part + */ + public String getCrc64() { + return crc64; + } + /** * Obtain the last modification time of the to-be-copied part. * @@ -63,7 +74,8 @@ public Date getLastModified() { @Override public String toString() { - return "CopyPartResult [partNumber=" + partNumber + ", etag=" + etag + ", lastModified=" + lastModified + "]"; + return "CopyPartResult [partNumber=" + partNumber + ", etag=" + etag + ", lastModified=" + lastModified + + ", crc64=" + crc64 + "]"; } } diff --git a/app/src/main/java/com/obs/services/model/DownloadFileRequest.java b/app/src/main/java/com/obs/services/model/DownloadFileRequest.java index d0e1268..d190e65 100644 --- a/app/src/main/java/com/obs/services/model/DownloadFileRequest.java +++ b/app/src/main/java/com/obs/services/model/DownloadFileRequest.java @@ -51,6 +51,8 @@ public class DownloadFileRequest extends BaseObjectRequest { private long ttl; + private boolean needCalculateCRC64 = false; + /** * Constructor * @@ -520,6 +522,23 @@ public void setTtl(long ttl) { } + /** + * @return + * Whether you need sdk to calculate CRC64 value and compare it with CRC64 returned by server + */ + public boolean isNeedCalculateCRC64() { + return needCalculateCRC64; + } + + /** + * @param needCalculateCRC64 + * Whether you need sdk to calculate CRC64 value and compare it with CRC64 returned by server + */ + public void setNeedCalculateCRC64(boolean needCalculateCRC64) { + this.needCalculateCRC64 = needCalculateCRC64; + } + + @Override public String toString() { return "DownloadFileRequest [bucketName=" + bucketName + ", objectKey=" + objectKey + ", downloadFile=" diff --git a/app/src/main/java/com/obs/services/model/DownloadFileResult.java b/app/src/main/java/com/obs/services/model/DownloadFileResult.java index e248806..028cb41 100644 --- a/app/src/main/java/com/obs/services/model/DownloadFileResult.java +++ b/app/src/main/java/com/obs/services/model/DownloadFileResult.java @@ -14,10 +14,14 @@ package com.obs.services.model; +import com.obs.services.internal.utils.CRC64; + /** * Response to a file download request */ public class DownloadFileResult { + private CRC64 crc64Combined; + /** * Obtain object properties. * @@ -39,9 +43,17 @@ public void setObjectMetadata(ObjectMetadata objectMetadata) { private ObjectMetadata objectMetadata; + + public CRC64 getCombinedCRC64() { + return crc64Combined; + } + public void setCombinedCRC64(CRC64 crc64Combined) { + this.crc64Combined = crc64Combined; + } + @Override public String toString() { - return "DownloadFileResult [objectMetadata=" + objectMetadata + "]"; + return "DownloadFileResult [objectMetadata=" + objectMetadata + ". crc64Combined=" + crc64Combined + "]"; } } diff --git a/app/src/main/java/com/obs/services/model/GenericRequest.java b/app/src/main/java/com/obs/services/model/GenericRequest.java index 894f741..e5571c8 100644 --- a/app/src/main/java/com/obs/services/model/GenericRequest.java +++ b/app/src/main/java/com/obs/services/model/GenericRequest.java @@ -14,6 +14,8 @@ package com.obs.services.model; +import com.obs.services.internal.utils.CallCancelHandler; + import java.util.HashMap; /** @@ -111,4 +113,13 @@ public void setRequesterPays(boolean isRequesterPays) { public String toString() { return "GenericRequest [isRequesterPays=" + isRequesterPays + "]"; } + public CallCancelHandler getCancelHandler() { + return cancelHandler; + } + + public void setCancelHandler(CallCancelHandler cancelHandler) { + this.cancelHandler = cancelHandler; + } + + protected CallCancelHandler cancelHandler; } diff --git a/app/src/main/java/com/obs/services/model/ModifyObjectRequest.java b/app/src/main/java/com/obs/services/model/ModifyObjectRequest.java index ae65280..f2e02d2 100644 --- a/app/src/main/java/com/obs/services/model/ModifyObjectRequest.java +++ b/app/src/main/java/com/obs/services/model/ModifyObjectRequest.java @@ -112,4 +112,22 @@ public ModifyObjectRequest(String bucketName, String objectKey, InputStream inpu this.input = input; this.position = position; } + + /** + * @return Whether you need sdk to calculate CRC64 value and check it by adding to header + */ + @Override + public boolean isNeedCalculateCRC64() { + // not supported in this API + return false; + } + + /** + * @param needCalculateCRC64 Whether you need sdk to calculate CRC64 value and check it by adding to header + */ + @Override + public void setNeedCalculateCRC64(boolean needCalculateCRC64) throws IllegalArgumentException { + // not supported in this API + throw new IllegalArgumentException("not supported in this API"); + } } diff --git a/app/src/main/java/com/obs/services/model/ObjectTaggingRequest.java b/app/src/main/java/com/obs/services/model/ObjectTaggingRequest.java index cc01912..0139515 100644 --- a/app/src/main/java/com/obs/services/model/ObjectTaggingRequest.java +++ b/app/src/main/java/com/obs/services/model/ObjectTaggingRequest.java @@ -12,7 +12,7 @@ * specific language governing permissions and limitations under the License. */ - package com.obs.services.model; +package com.obs.services.model; /** * Request for setting tags for a object diff --git a/app/src/main/java/com/obs/services/model/PutObjectRequest.java b/app/src/main/java/com/obs/services/model/PutObjectRequest.java index 19d511c..33f5d75 100644 --- a/app/src/main/java/com/obs/services/model/PutObjectRequest.java +++ b/app/src/main/java/com/obs/services/model/PutObjectRequest.java @@ -41,6 +41,8 @@ public class PutObjectRequest extends PutObjectBasicRequest { private Callback callback; + private boolean needCalculateCRC64 = false; + public PutObjectRequest() { } @@ -283,6 +285,21 @@ public Callback getCallback() { public void setCallback(Callback callback) { this.callback = callback; } + + /** + * @return Whether you need sdk to calculate CRC64 value and add it to header + */ + public boolean isNeedCalculateCRC64() { + return needCalculateCRC64; + } + + /** + * @param needCalculateCRC64 + * Whether you need sdk to calculate CRC64 value and add it to header + */ + public void setNeedCalculateCRC64(boolean needCalculateCRC64) { + this.needCalculateCRC64 = needCalculateCRC64; + } @Override public String toString() { return "PutObjectRequest [file=" + file + ", input=" + input + ", metadata=" + metadata diff --git a/app/src/main/java/com/obs/services/model/S3Object.java b/app/src/main/java/com/obs/services/model/S3Object.java index 94cfd36..733d83e 100644 --- a/app/src/main/java/com/obs/services/model/S3Object.java +++ b/app/src/main/java/com/obs/services/model/S3Object.java @@ -14,6 +14,8 @@ package com.obs.services.model; +import com.obs.services.internal.utils.CRC64InputStream; + import java.io.InputStream; @Deprecated @@ -38,6 +40,8 @@ public class S3Object { protected InputStream objectContent; + protected CRC64InputStream objectContentWithCRC64; + public String getBucketName() { return bucketName; } @@ -68,6 +72,12 @@ public void setMetadata(ObjectMetadata metadata) { public InputStream getObjectContent() { return objectContent; } + public CRC64InputStream getObjectContentWithCRC64() { + if (objectContentWithCRC64 == null) { + objectContentWithCRC64 = new CRC64InputStream(objectContent); + } + return objectContentWithCRC64; + } public void setObjectContent(InputStream objectContent) { this.objectContent = objectContent; diff --git a/app/src/main/java/com/obs/services/model/UploadFileRequest.java b/app/src/main/java/com/obs/services/model/UploadFileRequest.java index d3d81e2..9ea8e58 100644 --- a/app/src/main/java/com/obs/services/model/UploadFileRequest.java +++ b/app/src/main/java/com/obs/services/model/UploadFileRequest.java @@ -45,6 +45,12 @@ public class UploadFileRequest extends PutObjectBasicRequest { private long progressInterval = ObsConstraint.DEFAULT_PROGRESS_INTERVAL; + private boolean needAbortUploadFileAfterCancel = false; + + private boolean needCalculateCRC64 = false; + + private boolean needStreamCalculateCRC64 = false; + /** * Constructor * @@ -428,6 +434,44 @@ public void setCallback(Callback callback) { this.callback = callback; } + public boolean isNeedAbortUploadFileAfterCancel() { + return needAbortUploadFileAfterCancel; + } + + public void setNeedAbortUploadFileAfterCancel(boolean needAbortUploadFileAfterCancel) { + this.needAbortUploadFileAfterCancel = needAbortUploadFileAfterCancel; + } + + /** + * @return Whether you need sdk to calculate CRC64 value and add it to header + */ + public boolean isNeedCalculateCRC64() { + return needCalculateCRC64; + } + + /** + * @param needCalculateCRC64 + * Whether you need sdk to calculate CRC64 value and add it to header + */ + public void setNeedCalculateCRC64(boolean needCalculateCRC64) { + this.needCalculateCRC64 = needCalculateCRC64; + } + + + /** + * @return Whether you need sdk to calculate CRC64 value by stream, priority is lower than needCalculateCRC64 + */ + public boolean isNeedStreamCalculateCRC64() { + return needStreamCalculateCRC64; + } + + /** + * @param needStreamCalculateCRC64 + * Whether you need sdk to calculate CRC64 value by stream, priority is lower than needCalculateCRC64 + */ + public void setNeedStreamCalculateCRC64(boolean needStreamCalculateCRC64) { + this.needStreamCalculateCRC64 = needStreamCalculateCRC64; + } @Override public String toString() { return "UploadFileRequest [bucketName=" + bucketName + ", objectKey=" + objectKey + ", partSize=" + partSize diff --git a/app/src/main/java/com/obs/services/model/UploadPartRequest.java b/app/src/main/java/com/obs/services/model/UploadPartRequest.java index 50a0c60..eb67a83 100644 --- a/app/src/main/java/com/obs/services/model/UploadPartRequest.java +++ b/app/src/main/java/com/obs/services/model/UploadPartRequest.java @@ -18,6 +18,7 @@ import java.io.InputStream; import com.obs.services.internal.ObsConstraint; +import com.obs.services.internal.ProgressManager; /** * Parameters in a part upload request @@ -50,6 +51,8 @@ public class UploadPartRequest extends AbstractMultipartRequest { private ProgressListener progressListener; private long progressInterval = ObsConstraint.DEFAULT_PROGRESS_INTERVAL; + private boolean needCalculateCRC64 = false; + private ProgressManager progressManager; public UploadPartRequest() { } @@ -369,6 +372,36 @@ public void setProgressInterval(long progressInterval) { this.progressInterval = progressInterval; } + /** + * @return Whether you need sdk to calculate CRC64 value and add it to header + */ + public boolean isNeedCalculateCRC64() { + return needCalculateCRC64; + } + + /** + * @param needCalculateCRC64 + * Whether you need sdk to calculate CRC64 value and add it to header + */ + public void setNeedCalculateCRC64(boolean needCalculateCRC64) { + this.needCalculateCRC64 = needCalculateCRC64; + } + + /** + * @return progressManager + * get progressManager which receives progress, progressManager priority is higher than ProgressListener + */ + public ProgressManager getProgressManager() { + return progressManager; + } + + /** + * @param progressManager + * set progressManager to receive progress, progressManager priority is higher than ProgressListener + */ + public void setProgressManager(ProgressManager progressManager) { + this.progressManager = progressManager; + } @Override public String toString() { return "UploadPartRequest [uploadId=" + this.getUploadId() + ", bucketName=" + this.getBucketName() diff --git a/app/src/main/java/com/obs/services/model/UploadPartResult.java b/app/src/main/java/com/obs/services/model/UploadPartResult.java index 7225ed8..73867e0 100644 --- a/app/src/main/java/com/obs/services/model/UploadPartResult.java +++ b/app/src/main/java/com/obs/services/model/UploadPartResult.java @@ -14,6 +14,8 @@ package com.obs.services.model; +import com.obs.services.internal.utils.CRC64; + /** * Response to a part upload request */ @@ -22,6 +24,8 @@ public class UploadPartResult extends HeaderResponse { private String etag; + private CRC64 clientCalculatedCRC64; + /** * Obtain the part number. * @@ -48,8 +52,17 @@ public void setEtag(String objEtag) { this.etag = objEtag; } + + public CRC64 getClientCalculatedCRC64() { + return clientCalculatedCRC64; + } + + public void setClientCalculatedCRC64(CRC64 clientCalculatedCRC64) { + this.clientCalculatedCRC64 = clientCalculatedCRC64; + } @Override public String toString() { - return "UploadPartResult [partNumber=" + partNumber + ", etag=" + etag + "]"; + return "UploadPartResult [partNumber=" + partNumber + ", etag=" + etag + + ", clientCalculatedCRC64=" + clientCalculatedCRC64 + "]"; } } diff --git a/app/src/main/java/com/obs/services/model/fs/NewFileRequest.java b/app/src/main/java/com/obs/services/model/fs/NewFileRequest.java index 42a9cea..cdf4758 100644 --- a/app/src/main/java/com/obs/services/model/fs/NewFileRequest.java +++ b/app/src/main/java/com/obs/services/model/fs/NewFileRequest.java @@ -77,4 +77,22 @@ public NewFileRequest(String bucketName, String objectKey) { this.objectKey = objectKey; } + + /** + * @return Whether you need sdk to calculate CRC64 value and check it by adding to header + */ + @Override + public boolean isNeedCalculateCRC64() { + // not supported in this API + return false; + } + + /** + * @param needCalculateCRC64 Whether you need sdk to calculate CRC64 value and check it by adding to header + */ + @Override + public void setNeedCalculateCRC64(boolean needCalculateCRC64) throws IllegalArgumentException { + // not supported in this API + throw new IllegalArgumentException("not supported in this API"); + } } diff --git a/app/src/main/java/com/obs/services/model/fs/WriteFileRequest.java b/app/src/main/java/com/obs/services/model/fs/WriteFileRequest.java index f964565..ee38de6 100644 --- a/app/src/main/java/com/obs/services/model/fs/WriteFileRequest.java +++ b/app/src/main/java/com/obs/services/model/fs/WriteFileRequest.java @@ -109,4 +109,23 @@ public WriteFileRequest(String bucketName, String objectKey, InputStream input, this(bucketName, objectKey, input); this.position = position; } + + + /** + * @return Whether you need sdk to calculate CRC64 value and check it by adding to header + */ + @Override + public boolean isNeedCalculateCRC64() { + // not supported in this API + return false; + } + + /** + * @param needCalculateCRC64 Whether you need sdk to calculate CRC64 value and check it by adding to header + */ + @Override + public void setNeedCalculateCRC64(boolean needCalculateCRC64) throws IllegalArgumentException { + // not supported in this API + throw new IllegalArgumentException("not supported in this API"); + } } diff --git a/pom-android.xml b/pom-android.xml index d6a4b6c..e73ac41 100644 --- a/pom-android.xml +++ b/pom-android.xml @@ -4,7 +4,7 @@ com.huaweicloud esdk-obs-android - 3.24.3 + 3.24.8 jar OBS SDK for Android @@ -24,23 +24,23 @@ com.squareup.okio okio - 3.6.0 + 3.8.0 com.fasterxml.jackson.core jackson-core - 2.15.2 + 2.15.4 com.fasterxml.jackson.core jackson-databind - 2.15.2 + 2.15.4 com.fasterxml.jackson.core jackson-annotations - 2.15.2 + 2.15.4 diff --git a/pom-java-optimization.xml b/pom-java-optimization.xml index fe410d6..b680c05 100644 --- a/pom-java-optimization.xml +++ b/pom-java-optimization.xml @@ -4,7 +4,7 @@ com.huaweicloud esdk-obs-java-optimised - 3.24.3 + 3.24.8 jar OBS SDK for Java Optimised @@ -29,28 +29,28 @@ com.fasterxml.jackson.core jackson-core - 2.15.2 + 2.15.4 com.fasterxml.jackson.core jackson-databind - 2.15.2 + 2.15.4 com.fasterxml.jackson.core jackson-annotations - 2.15.2 + 2.15.4 org.apache.logging.log4j log4j-core - 2.18.0 + 2.20.0 org.apache.logging.log4j log4j-api - 2.18.0 + 2.20.0 diff --git a/pom-java.xml b/pom-java.xml index 5ae9b6f..e1613c1 100644 --- a/pom-java.xml +++ b/pom-java.xml @@ -4,7 +4,7 @@ com.huaweicloud esdk-obs-java - 3.24.3 + 3.24.8 jar OBS SDK for Java @@ -23,37 +23,36 @@ com.squareup.okio okio - 3.6.0 + 3.8.0 com.fasterxml.jackson.core jackson-core - 2.15.2 + 2.15.4 com.fasterxml.jackson.core jackson-databind - 2.15.2 + 2.15.4 com.fasterxml.jackson.core jackson-annotations - 2.15.2 + 2.15.4 org.apache.logging.log4j log4j-core - 2.18.0 + 2.20.0 org.apache.logging.log4j log4j-api - 2.18.0 + 2.20.0 - org.mockito mockito-core diff --git a/pom.xml b/pom.xml index 60f9cc5..35bdeda 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.huaweicloud esdk-obs-java - 3.24.3 + 3.24.8 jar OBS SDK for Java @@ -23,34 +23,34 @@ com.squareup.okio okio - 3.6.0 + 3.8.0 com.fasterxml.jackson.core jackson-core - 2.15.2 + 2.15.4 com.fasterxml.jackson.core jackson-databind - 2.15.2 + 2.15.4 com.fasterxml.jackson.core jackson-annotations - 2.15.2 + 2.15.4 org.apache.logging.log4j log4j-core - 2.18.0 + 2.20.0 org.apache.logging.log4j log4j-api - 2.18.0 + 2.20.0