From c5e07dd7de6d2f2d6d65deda67966e348e3b551e Mon Sep 17 00:00:00 2001 From: zhuxiaolong37 Date: Tue, 12 Dec 2023 16:15:13 +0800 Subject: [PATCH 1/6] TimeoutServiceClient adds custom threads --- .../com/aliyun/oss/ClientConfiguration.java | 9 ++++ .../oss/common/comm/TimeoutServiceClient.java | 14 ++++-- .../common/comm/TimeoutServiceClientTest.java | 50 +++++++++++++++++++ 3 files changed, 68 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/aliyun/oss/ClientConfiguration.java b/src/main/java/com/aliyun/oss/ClientConfiguration.java index 69c8dbc8..db0685a5 100644 --- a/src/main/java/com/aliyun/oss/ClientConfiguration.java +++ b/src/main/java/com/aliyun/oss/ClientConfiguration.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.Map; import java.util.Date; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import javax.net.ssl.HostnameVerifier; @@ -125,6 +126,8 @@ public class ClientConfiguration { private boolean verifyObjectStrict = true; + private ThreadPoolExecutor executor = null; + public ClientConfiguration() { super(); AppendDefaultExcludeList(this.cnameExcludeList); @@ -971,5 +974,11 @@ public boolean isVerifyObjectStrict() { return verifyObjectStrict; } + public ThreadPoolExecutor getExecutor() { + return executor; + } + public void setExecutor(ThreadPoolExecutor executor) { + this.executor = executor; + } } diff --git a/src/main/java/com/aliyun/oss/common/comm/TimeoutServiceClient.java b/src/main/java/com/aliyun/oss/common/comm/TimeoutServiceClient.java index 3c956998..4ff1834c 100644 --- a/src/main/java/com/aliyun/oss/common/comm/TimeoutServiceClient.java +++ b/src/main/java/com/aliyun/oss/common/comm/TimeoutServiceClient.java @@ -50,12 +50,16 @@ public class TimeoutServiceClient extends DefaultServiceClient { public TimeoutServiceClient(ClientConfiguration config) { super(config); + if(config.getExecutor() == null){ + int processors = Runtime.getRuntime().availableProcessors(); + executor = new ThreadPoolExecutor(processors * 5, processors * 10, 60L, TimeUnit.SECONDS, + new ArrayBlockingQueue(processors * 100), Executors.defaultThreadFactory(), + new ThreadPoolExecutor.CallerRunsPolicy()); + executor.allowCoreThreadTimeOut(true); + } else { + executor = config.getExecutor(); + } - int processors = Runtime.getRuntime().availableProcessors(); - executor = new ThreadPoolExecutor(processors * 5, processors * 10, 60L, TimeUnit.SECONDS, - new ArrayBlockingQueue(processors * 100), Executors.defaultThreadFactory(), - new ThreadPoolExecutor.CallerRunsPolicy()); - executor.allowCoreThreadTimeOut(true); } @Override diff --git a/src/test/java/com/aliyun/oss/common/comm/TimeoutServiceClientTest.java b/src/test/java/com/aliyun/oss/common/comm/TimeoutServiceClientTest.java index 689c2a25..e57d05be 100644 --- a/src/test/java/com/aliyun/oss/common/comm/TimeoutServiceClientTest.java +++ b/src/test/java/com/aliyun/oss/common/comm/TimeoutServiceClientTest.java @@ -20,12 +20,19 @@ package com.aliyun.oss.common.comm; import com.aliyun.oss.ClientConfiguration; +import com.aliyun.oss.ClientException; import com.aliyun.oss.HttpMethod; import com.aliyun.oss.internal.OSSConstants; import org.junit.Test; import java.io.ByteArrayInputStream; import java.net.URI; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertEquals; public class TimeoutServiceClientTest { @Test @@ -60,4 +67,47 @@ public void testTimeoutServiceClient() { } } + + @Test + public void testTimeoutServiceClientWithThread() { + int processors = Runtime.getRuntime().availableProcessors(); + ThreadPoolExecutor executor = new ThreadPoolExecutor(processors * 20, processors * 30, 60L, TimeUnit.SECONDS, + new ArrayBlockingQueue(processors * 200), Executors.defaultThreadFactory(), + new ThreadPoolExecutor.CallerRunsPolicy()); + + ClientConfiguration config = new ClientConfiguration(); + config.setRequestTimeoutEnabled(true); + config.setExecutor(executor); + + // cover partial codes + try { + String content = "test content"; + byte[] contentBytes = content.getBytes(OSSConstants.DEFAULT_CHARSET_NAME); + ByteArrayInputStream contentStream = new ByteArrayInputStream(contentBytes); + + RequestMessage request = new RequestMessage(null, null); + request.setEndpoint(new URI("http://localhost")); + request.setMethod(HttpMethod.GET); + request.setContent(contentStream); + request.setContentLength(contentBytes.length); + ExecutionContext context = new ExecutionContext(); + + TimeoutServiceClient client = new TimeoutServiceClient(config); + client.sendRequest(request, context); + client.shutdown(); + } catch (ClientException e) { + e.printStackTrace(); + assertEquals(e.getErrorCode(), "SocketException"); + } catch (Exception e) { + e.printStackTrace(); + } + + try { + TimeoutServiceClient client = new TimeoutServiceClient(config); + client.shutdown(); + } catch (Exception e) { + e.printStackTrace(); + } + } + } From 1016fada3450128d689cab876a77176077e32f38 Mon Sep 17 00:00:00 2001 From: zhuxiaolong37 Date: Tue, 1 Aug 2023 15:30:19 +0800 Subject: [PATCH 2/6] Added certificate creation date and other fields to the getBucketCname interface --- .../aliyun/oss/internal/ResponseParsers.java | 4 ++ .../aliyun/oss/model/CnameConfiguration.java | 37 ++++++++++ .../common/parser/ResponseParsersTest.java | 68 +++++++++++++++++++ .../oss/integrationtests/BucketCnameTest.java | 53 +++++++++++++++ 4 files changed, 162 insertions(+) diff --git a/src/main/java/com/aliyun/oss/internal/ResponseParsers.java b/src/main/java/com/aliyun/oss/internal/ResponseParsers.java index 20be996d..85d7254f 100644 --- a/src/main/java/com/aliyun/oss/internal/ResponseParsers.java +++ b/src/main/java/com/aliyun/oss/internal/ResponseParsers.java @@ -3611,6 +3611,10 @@ public static List parseGetBucketCname(InputStream responseB cname.setCertType(CnameConfiguration.CertType.parse(certElem.getChildText("Type"))); cname.setCertStatus(CnameConfiguration.CertStatus.parse(certElem.getChildText("Status"))); cname.setCertId(certElem.getChildText("CertId")); + cname.setCreationDate(certElem.getChildText("CreationDate")); + cname.setFingerprint(certElem.getChildText("Fingerprint")); + cname.setValidStartDate(certElem.getChildText("ValidStartDate")); + cname.setValidEndDate(certElem.getChildText("ValidEndDate")); } cnames.add(cname); diff --git a/src/main/java/com/aliyun/oss/model/CnameConfiguration.java b/src/main/java/com/aliyun/oss/model/CnameConfiguration.java index f70dff2e..5162cd2e 100644 --- a/src/main/java/com/aliyun/oss/model/CnameConfiguration.java +++ b/src/main/java/com/aliyun/oss/model/CnameConfiguration.java @@ -188,6 +188,38 @@ public void setPurgeCdnCache(Boolean purgeCdnCache) { public void setCertId(String certId) { this.certId = certId; } + public String getCreationDate() { + return creationDate; + } + + public void setCreationDate(String creationDate) { + this.creationDate = creationDate; + } + + public String getFingerprint() { + return fingerprint; + } + + public void setFingerprint(String fingerprint) { + this.fingerprint = fingerprint; + } + + public String getValidStartDate() { + return validStartDate; + } + + public void setValidStartDate(String validStartDate) { + this.validStartDate = validStartDate; + } + + public String getValidEndDate() { + return validEndDate; + } + + public void setValidEndDate(String validEndDate) { + this.validEndDate = validEndDate; + } + @Override public String toString() { return "CnameConfiguration [domain=" + domain + ", status=" + status + ", lastMofiedTime=" + lastMofiedTime @@ -202,4 +234,9 @@ public String toString() { private CertStatus certStatus; private CertType certType; private String certId; + private String creationDate; + private String fingerprint; + private String validStartDate; + private String validEndDate; + } diff --git a/src/test/java/com/aliyun/oss/common/parser/ResponseParsersTest.java b/src/test/java/com/aliyun/oss/common/parser/ResponseParsersTest.java index ed19295a..88a814d1 100644 --- a/src/test/java/com/aliyun/oss/common/parser/ResponseParsersTest.java +++ b/src/test/java/com/aliyun/oss/common/parser/ResponseParsersTest.java @@ -20,6 +20,7 @@ import java.io.ByteArrayInputStream; import java.io.InputStream; import java.io.UnsupportedEncodingException; +import java.text.ParseException; import java.util.*; /** @@ -1500,6 +1501,73 @@ public void testGetBucketCnameResponseParser() { } } + @Test + public void testGetBucketCnameWithCreationDate() { + String respBody = null; + InputStream instream = null; + Date dt = new Date(); + DateUtil.formatIso8601Date(dt); + + respBody = "" + + "\n" + + " targetbucket\n" + + " testowner\n" + + " \n" + + " example.com\n" + + " 2021-09-15T02:35:07.000Z\n" + + " Enabled\n" + + " \n" + + " CAS\n" + + " 493****-cn-hangzhou\n" + + " Enabled\n" + + " Wed, 15 Sep 2021 02:35:06 GMT\n" + + " DE:01:CF:EC:7C:A7:98:CB:D8:6E:FB:1D:97:EB:A9:64:1D:4E:**:**\n" + + " Wed, 12 Apr 2023 10:14:51 GMT\n" + + " Mon, 4 May 2048 10:14:51 GMT\n" + + " \n" + + " \n" + + " \n" + + " example.org\n" + + " 2021-09-15T02:34:58.000Z\n" + + " Enabled\n" + + " \n" + + " \n" + + " example.edu\n" + + " 2021-09-15T02:50:34.000Z\n" + + " Enabled\n" + + " \n" + + ""; + + try { + instream = new ByteArrayInputStream(respBody.getBytes("utf-8")); + } catch (UnsupportedEncodingException e) { + Assert.fail("UnsupportedEncodingException"); + } + + try { + ResponseMessage responseMessage = new ResponseMessage(null);; + responseMessage.setContent(instream); + ResponseParsers.GetBucketCnameResponseParser parser = new ResponseParsers.GetBucketCnameResponseParser(); + List result = parser.parse(responseMessage); + Assert.assertEquals(result.size(), 3); + Assert.assertEquals(result.get(0).getDomain(), "example.com"); + Assert.assertEquals(result.get(0).getLastMofiedTime(), DateUtil.parseIso8601Date("2021-09-15T02:35:07.000Z")); + Assert.assertEquals(result.get(0).getStatus(), CnameConfiguration.CnameStatus.Enabled); + Assert.assertEquals(result.get(0).getCertType(), CnameConfiguration.CertType.CAS); + Assert.assertEquals(result.get(0).getStatus(), CnameConfiguration.CnameStatus.Enabled); + Assert.assertEquals(result.get(0).getCertId(), "493****-cn-hangzhou"); + Assert.assertEquals(result.get(0).getCreationDate(), "Wed, 15 Sep 2021 02:35:06 GMT"); + Assert.assertEquals(result.get(0).getFingerprint(), "DE:01:CF:EC:7C:A7:98:CB:D8:6E:FB:1D:97:EB:A9:64:1D:4E:**:**"); + Assert.assertEquals(result.get(0).getValidStartDate(), "Wed, 12 Apr 2023 10:14:51 GMT"); + Assert.assertEquals(result.get(0).getValidEndDate(), "Mon, 4 May 2048 10:14:51 GMT"); + } catch (ResponseParseException e) { + Assert.fail("UnsupportedEncodingException"); + } catch (ParseException e) { + e.printStackTrace(); + } + + } + @Test public void testGetBucketReplicationResponseParser() { String respBody = null; diff --git a/src/test/java/com/aliyun/oss/integrationtests/BucketCnameTest.java b/src/test/java/com/aliyun/oss/integrationtests/BucketCnameTest.java index 897a4255..83d9a0a6 100644 --- a/src/test/java/com/aliyun/oss/integrationtests/BucketCnameTest.java +++ b/src/test/java/com/aliyun/oss/integrationtests/BucketCnameTest.java @@ -254,4 +254,57 @@ public void testBucketCnameToken() { Assert.assertEquals("CnameTokenNotFound", e.getErrorCode()); } } + + @Ignore + @Test + public void testGetBucketCname() { + + final String bucketName = "normal-get-bucket-cname"; + Date curDate; + + try { + ossClient.createBucket(bucketName); + + // set multi cname + for (String domain : domains) { + CreateBucketCnameTokenRequest createBucketCnameTokenRequest = new CreateBucketCnameTokenRequest(bucketName); + createBucketCnameTokenRequest.setDomain(domain); + ossClient.createBucketCnameToken(createBucketCnameTokenRequest); + + AddBucketCnameRequest request = new AddBucketCnameRequest(bucketName); + request.setDomain(domain); + ossClient.addBucketCname(request); + } + + waitForCacheExpiration(5); + + curDate = new Date(System.currentTimeMillis()); + + List cnames = ossClient.getBucketCname(bucketName); + Assert.assertEquals(cnames.size(), domains.length); + for (int i = 0; i < cnames.size(); i++) { + System.out.println(cnames.get(i)); + Assert.assertEquals(cnames.get(i).getDomain(), domains[i]); + Assert.assertEquals(cnames.get(i).getStatus(), CnameConfiguration.CnameStatus.Enabled); + Assert.assertEquals(cnames.get(i).getLastMofiedTime().getYear(), curDate.getYear()); + Assert.assertEquals(cnames.get(i).getLastMofiedTime().getMonth(), curDate.getMonth()); + Assert.assertEquals(cnames.get(i).getLastMofiedTime().getDay(), curDate.getDay()); + } + + for (String domain : domains) { + DeleteBucketCnameRequest req = new DeleteBucketCnameRequest(bucketName); + req.setDomain(domain); + ossClient.deleteBucketCname(req); + } + + cnames = ossClient.getBucketCname(bucketName); + Assert.assertEquals(cnames.size(), 0); + + } catch (Exception e) { + e.printStackTrace(); + Assert.fail(e.getMessage()); + } finally { + ossClient.deleteBucket(bucketName); + } + } } From e9671aea61aa70a343809aeaf95554c792da2be5 Mon Sep 17 00:00:00 2001 From: zhuxiaolong37 Date: Wed, 20 Dec 2023 18:06:06 +0800 Subject: [PATCH 3/6] Added archive direct read --- src/main/java/com/aliyun/oss/OSS.java | 21 ++++++ src/main/java/com/aliyun/oss/OSSClient.java | 10 +++ .../oss/common/parser/RequestMarshallers.java | 18 +++++ .../oss/internal/OSSBucketOperation.java | 27 +++++++ .../oss/internal/RequestParameters.java | 1 + .../aliyun/oss/internal/ResponseParsers.java | 32 +++++++++ .../aliyun/oss/internal/SignParameters.java | 2 +- .../GetBucketArchiveDirectReadResult.java | 15 ++++ .../PutBucketArchiveDirectReadRequest.java | 22 ++++++ .../BucketArchiveDirectReadSample.java | 34 +++++++++ .../common/parser/RequestMarshallersTest.java | 33 +++++++++ .../common/parser/ResponseParsersTest.java | 72 +++++++++++++++++++ .../BucketArchiveDirectReadTest.java | 33 +++++++++ 13 files changed, 319 insertions(+), 1 deletion(-) create mode 100644 src/main/java/com/aliyun/oss/model/GetBucketArchiveDirectReadResult.java create mode 100644 src/main/java/com/aliyun/oss/model/PutBucketArchiveDirectReadRequest.java create mode 100644 src/samples/BucketArchiveDirectReadSample.java create mode 100644 src/test/java/com/aliyun/oss/integrationtests/BucketArchiveDirectReadTest.java diff --git a/src/main/java/com/aliyun/oss/OSS.java b/src/main/java/com/aliyun/oss/OSS.java index 2da7367e..5cc5dc99 100644 --- a/src/main/java/com/aliyun/oss/OSS.java +++ b/src/main/java/com/aliyun/oss/OSS.java @@ -5200,4 +5200,25 @@ public UdfApplicationLog getUdfApplicationLog(GetUdfApplicationLogRequest getUdf * If any errors occurred in OSS while processing the request. */ public VoidResult writeGetObjectResponse(WriteGetObjectResponseRequest writeGetObjectResponseRequest) throws OSSException, ClientException; + + + /** + * create archive direct Reading + * + * @param request request + * @return void + * @throws OSSException OSSException + * @throws ClientException ClientException + */ + VoidResult putBucketArchiveDirectRead(PutBucketArchiveDirectReadRequest request) throws OSSException, ClientException; + + /** + * get archive direct Reading + * + * @param bucketName + * @return + * @throws OSSException OSSException + * @throws ClientException ClientException + */ + GetBucketArchiveDirectReadResult getBucketArchiveDirectRead(String bucketName) throws OSSException, ClientException; } diff --git a/src/main/java/com/aliyun/oss/OSSClient.java b/src/main/java/com/aliyun/oss/OSSClient.java index 9cc06eb4..3e6e5b55 100644 --- a/src/main/java/com/aliyun/oss/OSSClient.java +++ b/src/main/java/com/aliyun/oss/OSSClient.java @@ -1997,6 +1997,16 @@ public VoidResult writeGetObjectResponse(WriteGetObjectResponseRequest writeGetO return this.objectOperation.writeGetObjectResponse(writeGetObjectResponseRequest); } + @Override + public VoidResult putBucketArchiveDirectRead(PutBucketArchiveDirectReadRequest request) throws OSSException, ClientException { + return this.bucketOperation.putBucketArchiveDirectRead(request); + } + + @Override + public GetBucketArchiveDirectReadResult getBucketArchiveDirectRead(String bucketName) throws OSSException, ClientException { + return this.bucketOperation.getBucketArchiveDirectRead(new GenericRequest(bucketName)); + } + @Override public void shutdown() { try { diff --git a/src/main/java/com/aliyun/oss/common/parser/RequestMarshallers.java b/src/main/java/com/aliyun/oss/common/parser/RequestMarshallers.java index 35e732a9..a8b45396 100644 --- a/src/main/java/com/aliyun/oss/common/parser/RequestMarshallers.java +++ b/src/main/java/com/aliyun/oss/common/parser/RequestMarshallers.java @@ -102,6 +102,7 @@ public final class RequestMarshallers { public static final DoMetaQueryRequestMarshaller doMetaQueryRequestMarshaller = new DoMetaQueryRequestMarshaller(); public static final SetBucketCallbackPolicyRequestMarshaller setBucketCallbackPolicyRequestMarshaller = new SetBucketCallbackPolicyRequestMarshaller(); public static final AsyncProcessObjectRequestMarshaller asyncProcessObjectRequestMarshaller = new AsyncProcessObjectRequestMarshaller(); + public static final PutBucketArchiveDirectReadRequestMarshaller putBucketArchiveDirectReadRequestMarshaller = new PutBucketArchiveDirectReadRequestMarshaller(); public interface RequestMarshaller extends Marshaller { @@ -1942,6 +1943,23 @@ public byte[] marshall(AsyncProcessObjectRequest request) { } + public static final class PutBucketArchiveDirectReadRequestMarshaller implements RequestMarshaller2 { + @Override + public byte[] marshall(PutBucketArchiveDirectReadRequest input) { + StringBuffer xmlBody = new StringBuffer(); + xmlBody.append(""); + xmlBody.append(input.getEnabled()); + xmlBody.append(""); + byte[] rawData = null; + try { + rawData = xmlBody.toString().getBytes(DEFAULT_CHARSET_NAME); + } catch (UnsupportedEncodingException e) { + throw new ClientException("Unsupported encoding " + e.getMessage(), e); + } + return rawData; + } + } + private static enum EscapedChar { // "\r" RETURN(" "), diff --git a/src/main/java/com/aliyun/oss/internal/OSSBucketOperation.java b/src/main/java/com/aliyun/oss/internal/OSSBucketOperation.java index ff314df7..d465f877 100644 --- a/src/main/java/com/aliyun/oss/internal/OSSBucketOperation.java +++ b/src/main/java/com/aliyun/oss/internal/OSSBucketOperation.java @@ -2194,4 +2194,31 @@ public VoidResult deleteBucketCallbackPolicy(GenericRequest genericRequest) thro return doOperation(request, requestIdResponseParser, bucketName, null); } + + public VoidResult putBucketArchiveDirectRead(PutBucketArchiveDirectReadRequest putBucketArchiveDirectReadRequest) throws OSSException, ClientException { + assertParameterNotNull(putBucketArchiveDirectReadRequest, "putBucketArchiveDirectReadRequest"); + String bucketName = putBucketArchiveDirectReadRequest.getBucketName(); + assertParameterNotNull(bucketName, "bucketName"); + ensureBucketNameValid(bucketName); + Map params = new HashMap(); + params.put(ARCHIVE_DIRECT_READ, null); + byte[] rawContent = putBucketArchiveDirectReadRequestMarshaller.marshall(putBucketArchiveDirectReadRequest); + RequestMessage request = new OSSRequestMessageBuilder(getInnerClient()).setEndpoint(getEndpoint()) + .setMethod(HttpMethod.PUT).setBucket(bucketName).setParameters(params) + .setOriginalRequest(putBucketArchiveDirectReadRequest).setInputSize(rawContent.length).setInputStream(new ByteArrayInputStream(rawContent)).build(); + return doOperation(request, requestIdResponseParser, bucketName, null, true); + } + + public GetBucketArchiveDirectReadResult getBucketArchiveDirectRead(GenericRequest genericRequest) throws OSSException, ClientException { + assertParameterNotNull(genericRequest, "genericRequest"); + String bucketName = genericRequest.getBucketName(); + assertParameterNotNull(bucketName, "bucketName"); + ensureBucketNameValid(bucketName); + Map params = new HashMap(); + params.put(ARCHIVE_DIRECT_READ, null); + RequestMessage request = new OSSRequestMessageBuilder(getInnerClient()).setEndpoint(getEndpoint()) + .setMethod(HttpMethod.GET).setBucket(bucketName).setParameters(params) + .setOriginalRequest(genericRequest).build(); + return doOperation(request, getBucketArchiveDirectReadResponseParser, bucketName, null, true); + } } diff --git a/src/main/java/com/aliyun/oss/internal/RequestParameters.java b/src/main/java/com/aliyun/oss/internal/RequestParameters.java index 5b5c74d6..e5979572 100644 --- a/src/main/java/com/aliyun/oss/internal/RequestParameters.java +++ b/src/main/java/com/aliyun/oss/internal/RequestParameters.java @@ -159,5 +159,6 @@ public final class RequestParameters { public static final String REGIONS = "regions"; public static final String X_OSS_ASYNC_PROCESS = "x-oss-async-process"; public static final String WRITE_GET_OBJECT_RESPONSE = "x-oss-write-get-object-response"; + public static final String ARCHIVE_DIRECT_READ = "bucketArchiveDirectRead"; } diff --git a/src/main/java/com/aliyun/oss/internal/ResponseParsers.java b/src/main/java/com/aliyun/oss/internal/ResponseParsers.java index 85d7254f..b1c6ed4b 100644 --- a/src/main/java/com/aliyun/oss/internal/ResponseParsers.java +++ b/src/main/java/com/aliyun/oss/internal/ResponseParsers.java @@ -140,6 +140,8 @@ public final class ResponseParsers { public static final DescribeRegionsResponseParser describeRegionsResponseParser = new DescribeRegionsResponseParser(); public static final GetBucketCallbackPolicyResponseParser getBucketCallbackPolicyResponseParser = new GetBucketCallbackPolicyResponseParser(); public static final AsyncProcessObjectResponseParser asyncProcessObjectResponseParser = new AsyncProcessObjectResponseParser(); + public static final GetBucketArchiveDirectReadResponseParser getBucketArchiveDirectReadResponseParser = new GetBucketArchiveDirectReadResponseParser(); + public static Long parseLongWithDefault(String defaultValue){ if(defaultValue == null || "".equals(defaultValue)){ @@ -4304,4 +4306,34 @@ private AsyncProcessObjectResult parseAsyncProcessObject(InputStream inputStream } } } + + public static final class GetBucketArchiveDirectReadResponseParser implements ResponseParser { + @Override + public GetBucketArchiveDirectReadResult parse(ResponseMessage response) throws ResponseParseException { + try { + GetBucketArchiveDirectReadResult result = parseGetArchiveDirectRead(response.getContent()); + setResultParameter(result, response); + return result; + } finally { + safeCloseResponse(response); + } + } + private GetBucketArchiveDirectReadResult parseGetArchiveDirectRead(InputStream inputStream) throws ResponseParseException { + GetBucketArchiveDirectReadResult getBucketArchiveDirectReadResult = new GetBucketArchiveDirectReadResult(); + if (inputStream == null) { + return getBucketArchiveDirectReadResult; + } + try { + Element root = getXmlRootElement(inputStream); + if (root.getChildText("Enabled") != null) { + getBucketArchiveDirectReadResult.setEnabled(Boolean.valueOf(root.getChildText("Enabled"))); + } + return getBucketArchiveDirectReadResult; + } catch (JDOMParseException e) { + throw new ResponseParseException(e.getPartialDocument() + ": " + e.getMessage(), e); + } catch (Exception e) { + throw new ResponseParseException(e.getMessage(), e); + } + } + } } diff --git a/src/main/java/com/aliyun/oss/internal/SignParameters.java b/src/main/java/com/aliyun/oss/internal/SignParameters.java index f673aa2a..a21460c1 100644 --- a/src/main/java/com/aliyun/oss/internal/SignParameters.java +++ b/src/main/java/com/aliyun/oss/internal/SignParameters.java @@ -43,6 +43,6 @@ public class SignParameters { SUBRESOURCE_WORM_ID, SUBRESOURCE_WORM_EXTEND, SUBRESOURCE_CALLBACK, SUBRESOURCE_CALLBACK_VAR, SUBRESOURCE_DIR, SUBRESOURCE_RENAME, SUBRESOURCE_DIR_DELETE, SUBRESOURCE_TRANSFER_ACCELERATION, X_OSS_AC_SOURCE_IP, X_OSS_AC_SUBNET_MASK, X_OSS_AC_VPC_ID, X_OSS_AC_FORWARD_ALLOW, META_QUERY, SUBRESOURCE_RESOURCE_GROUP, - SUBRESOURCE_REGION_LIST, X_OSS_ASYNC_PROCESS, WRITE_GET_OBJECT_RESPONSE}); + SUBRESOURCE_REGION_LIST, X_OSS_ASYNC_PROCESS, WRITE_GET_OBJECT_RESPONSE, ARCHIVE_DIRECT_READ}); } diff --git a/src/main/java/com/aliyun/oss/model/GetBucketArchiveDirectReadResult.java b/src/main/java/com/aliyun/oss/model/GetBucketArchiveDirectReadResult.java new file mode 100644 index 00000000..d7c41772 --- /dev/null +++ b/src/main/java/com/aliyun/oss/model/GetBucketArchiveDirectReadResult.java @@ -0,0 +1,15 @@ +package com.aliyun.oss.model; + +public class GetBucketArchiveDirectReadResult extends GenericResult { + private boolean enabled; + public boolean getEnabled() { + return enabled; + } + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + public GetBucketArchiveDirectReadResult withEnabled(boolean enabled) { + this.enabled = enabled; + return this; + } +} \ No newline at end of file diff --git a/src/main/java/com/aliyun/oss/model/PutBucketArchiveDirectReadRequest.java b/src/main/java/com/aliyun/oss/model/PutBucketArchiveDirectReadRequest.java new file mode 100644 index 00000000..1dc3f701 --- /dev/null +++ b/src/main/java/com/aliyun/oss/model/PutBucketArchiveDirectReadRequest.java @@ -0,0 +1,22 @@ +package com.aliyun.oss.model; + +public class PutBucketArchiveDirectReadRequest extends GenericRequest { + private boolean enabled = false; + public PutBucketArchiveDirectReadRequest(String bucketName) { + super(bucketName); + } + public PutBucketArchiveDirectReadRequest(String bucketName, boolean enabled) { + super(bucketName); + this.enabled = enabled; + } + public boolean getEnabled() { + return enabled; + } + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + public PutBucketArchiveDirectReadRequest withEnabled(boolean enabled) { + this.enabled = enabled; + return this; + } +} diff --git a/src/samples/BucketArchiveDirectReadSample.java b/src/samples/BucketArchiveDirectReadSample.java new file mode 100644 index 00000000..0a05ca7a --- /dev/null +++ b/src/samples/BucketArchiveDirectReadSample.java @@ -0,0 +1,34 @@ +package samples; +import com.aliyun.oss.ClientException; +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSClientBuilder; +import com.aliyun.oss.OSSException; +import com.aliyun.oss.model.*; +public class BucketArchiveDirectReadSample { + private static String endpoint = "*** Provide OSS endpoint ***"; + private static String accessKeyId = "*** Provide your AccessKeyId ***"; + private static String accessKeySecret = "*** Provide your AccessKeySecret ***"; + private static String bucketName = "*** Provide bucket name ***"; + public static void main(String[] args) { + OSS ossClient = new OSSClientBuilder().build(endpoint, accessKeyId, accessKeySecret); + try { + // Creating Archive Direct Reading + PutBucketArchiveDirectReadRequest readRequest = new PutBucketArchiveDirectReadRequest(bucketName, true); + ossClient.putBucketArchiveDirectRead(readRequest); + // Obtain Archive Direct Reading + GetBucketArchiveDirectReadResult result = ossClient.getBucketArchiveDirectRead(bucketName); + System.out.println(result.getEnabled()); + } catch (OSSException oe) { + System.out.println("Error Message: " + oe.getErrorMessage()); + System.out.println("Error Code: " + oe.getErrorCode()); + System.out.println("Request ID: " + oe.getRequestId()); + System.out.println("Host ID: " + oe.getHostId()); + } catch (ClientException ce) { + System.out.println("Error Message: " + ce.getMessage()); + } finally { + if(ossClient != null){ + ossClient.shutdown(); + } + } + } +} diff --git a/src/test/java/com/aliyun/oss/common/parser/RequestMarshallersTest.java b/src/test/java/com/aliyun/oss/common/parser/RequestMarshallersTest.java index d11c7cf6..7d6574b6 100644 --- a/src/test/java/com/aliyun/oss/common/parser/RequestMarshallersTest.java +++ b/src/test/java/com/aliyun/oss/common/parser/RequestMarshallersTest.java @@ -1596,4 +1596,37 @@ public void testPutLifeCycleFilterObjectSizeThanRequestMarshaller() { Assert.assertEquals("key", root.getChild("Rule").getChild("Filter").getChildren("Not").get(0).getChild("Tag").getChildText("Key")); Assert.assertEquals("value", root.getChild("Rule").getChild("Filter").getChildren("Not").get(0).getChild("Tag").getChildText("Value")); } + + @Test + public void testArchiveDirectRead() { + String bucketName = "testBucket-archiveDirectRead"; + try { + PutBucketArchiveDirectReadRequest readRequest = new PutBucketArchiveDirectReadRequest(bucketName, true); + byte[] data = putBucketArchiveDirectReadRequestMarshaller.marshall(readRequest); + ByteArrayInputStream is = new ByteArrayInputStream(data); + SAXBuilder builder = new SAXBuilder(); + Document doc = null; + doc = builder.build(is); + Element root = doc.getRootElement(); + Assert.assertEquals("true", root.getChildText("Enabled")); + } catch (JDOMException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } + try { + PutBucketArchiveDirectReadRequest readRequest = new PutBucketArchiveDirectReadRequest(bucketName, false); + byte[] data = putBucketArchiveDirectReadRequestMarshaller.marshall(readRequest); + ByteArrayInputStream is = new ByteArrayInputStream(data); + SAXBuilder builder = new SAXBuilder(); + Document doc = null; + doc = builder.build(is); + Element root = doc.getRootElement(); + Assert.assertEquals("false", root.getChildText("Enabled")); + } catch (JDOMException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } + } } diff --git a/src/test/java/com/aliyun/oss/common/parser/ResponseParsersTest.java b/src/test/java/com/aliyun/oss/common/parser/ResponseParsersTest.java index 88a814d1..42c50cef 100644 --- a/src/test/java/com/aliyun/oss/common/parser/ResponseParsersTest.java +++ b/src/test/java/com/aliyun/oss/common/parser/ResponseParsersTest.java @@ -5728,4 +5728,76 @@ public void testParseGetBucketLifecycleFilterObjectSizeThan() { Assert.assertEquals(rules.get(0).getFilter().getObjectSizeLessThan(), Long.valueOf(String.valueOf(64000))); } + @Test + public void testArchiveDirectReadResponseParser() { + String respBody = "" + + "\n" + + " true\n" + + ""; + + InputStream instream = null; + try { + instream = new ByteArrayInputStream(respBody.getBytes("utf-8")); + } catch (UnsupportedEncodingException e) { + Assert.fail("UnsupportedEncodingException"); + } + + GetBucketArchiveDirectReadResult result = null; + try { + ResponseMessage response = new ResponseMessage(null); + response.setContent(instream); + ResponseParsers.GetBucketArchiveDirectReadResponseParser parser = new ResponseParsers.GetBucketArchiveDirectReadResponseParser(); + result = parser.parse(response); + } catch (ResponseParseException e) { + Assert.fail("parse archive direct read response body fail!"); + } + + Assert.assertEquals(true, result.getEnabled()); + + + respBody = "" + + "\n" + + " false\n" + + ""; + + try { + instream = new ByteArrayInputStream(respBody.getBytes("utf-8")); + } catch (UnsupportedEncodingException e) { + Assert.fail("UnsupportedEncodingException"); + } + + result = null; + try { + ResponseMessage response = new ResponseMessage(null); + response.setContent(instream); + ResponseParsers.GetBucketArchiveDirectReadResponseParser parser = new ResponseParsers.GetBucketArchiveDirectReadResponseParser(); + result = parser.parse(response); + } catch (ResponseParseException e) { + Assert.fail("parse archive direct read response body fail!"); + } + Assert.assertEquals(false, result.getEnabled()); + + + + respBody = "" + + "\n" + + ""; + + try { + instream = new ByteArrayInputStream(respBody.getBytes("utf-8")); + } catch (UnsupportedEncodingException e) { + Assert.fail("UnsupportedEncodingException"); + } + + result = null; + try { + ResponseMessage response = new ResponseMessage(null); + response.setContent(instream); + ResponseParsers.GetBucketArchiveDirectReadResponseParser parser = new ResponseParsers.GetBucketArchiveDirectReadResponseParser(); + result = parser.parse(response); + } catch (ResponseParseException e) { + Assert.fail("parse archive direct read response body fail!"); + } + Assert.assertEquals(false, result.getEnabled()); + } } diff --git a/src/test/java/com/aliyun/oss/integrationtests/BucketArchiveDirectReadTest.java b/src/test/java/com/aliyun/oss/integrationtests/BucketArchiveDirectReadTest.java new file mode 100644 index 00000000..1ac86e1c --- /dev/null +++ b/src/test/java/com/aliyun/oss/integrationtests/BucketArchiveDirectReadTest.java @@ -0,0 +1,33 @@ +package com.aliyun.oss.integrationtests; + +import com.aliyun.oss.OSSException; +import com.aliyun.oss.model.GetBucketArchiveDirectReadResult; +import com.aliyun.oss.model.PutBucketArchiveDirectReadRequest; +import com.aliyun.oss.model.VoidResult; +import junit.framework.Assert; +import org.junit.Test; +public class BucketArchiveDirectReadTest extends TestBase { + @Test + public void testBucketArchiveDirectRead() { + try { + PutBucketArchiveDirectReadRequest readRequest = new PutBucketArchiveDirectReadRequest(bucketName, true); + VoidResult result = ossClient.putBucketArchiveDirectRead(readRequest); + Assert.assertEquals(200, result.getResponse().getStatusCode()); + GetBucketArchiveDirectReadResult getResult = ossClient.getBucketArchiveDirectRead(bucketName); + Assert.assertEquals(true, getResult.getEnabled()); + } catch (OSSException e) { + Assert.assertEquals("NoSuchArchiveDirectReadConfiguration", e.getErrorCode()); + } catch (Exception e1) { + Assert.fail(e1.getMessage()); + } + try { + PutBucketArchiveDirectReadRequest readRequest = new PutBucketArchiveDirectReadRequest(bucketName); + VoidResult result = ossClient.putBucketArchiveDirectRead(readRequest); + Assert.assertEquals(200, result.getResponse().getStatusCode()); + GetBucketArchiveDirectReadResult getResult = ossClient.getBucketArchiveDirectRead(bucketName); + Assert.assertEquals(false, getResult.getEnabled()); + } catch (Exception e1) { + Assert.fail(e1.getMessage()); + } + } +} \ No newline at end of file From 8b53cb1050629aa601d8c654adc278d509e10611 Mon Sep 17 00:00:00 2001 From: zhuxiaolong37 Date: Wed, 10 Jan 2024 15:44:05 +0800 Subject: [PATCH 4/6] Fix the Oss CheckPoint deserialization vulnerability --- .../internal/CheckPointObjectInputStream.java | 21 +++++++++++ .../oss/internal/OSSDownloadOperation.java | 2 +- .../ObjectDownloadFileTest.java | 35 +++++++++++++++++++ 3 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 src/main/java/com/aliyun/oss/internal/CheckPointObjectInputStream.java create mode 100644 src/test/java/com/aliyun/oss/integrationtests/ObjectDownloadFileTest.java diff --git a/src/main/java/com/aliyun/oss/internal/CheckPointObjectInputStream.java b/src/main/java/com/aliyun/oss/internal/CheckPointObjectInputStream.java new file mode 100644 index 00000000..a906aae3 --- /dev/null +++ b/src/main/java/com/aliyun/oss/internal/CheckPointObjectInputStream.java @@ -0,0 +1,21 @@ +package com.aliyun.oss.internal; + +import java.io.*; + +public class CheckPointObjectInputStream extends ObjectInputStream { + + public CheckPointObjectInputStream(InputStream in) throws IOException { + super(in); + } + + @Override + protected Class resolveClass(ObjectStreamClass desc) throws IOException, + ClassNotFoundException { + if (!desc.getName().equals(OSSDownloadOperation.DownloadCheckPoint.class.getName())) { + throw new InvalidClassException( + "Unauthorized deserialization attempt", + desc.getName()); + } + return super.resolveClass(desc); + } +} diff --git a/src/main/java/com/aliyun/oss/internal/OSSDownloadOperation.java b/src/main/java/com/aliyun/oss/internal/OSSDownloadOperation.java index b4fe9afb..e4a6fc8f 100644 --- a/src/main/java/com/aliyun/oss/internal/OSSDownloadOperation.java +++ b/src/main/java/com/aliyun/oss/internal/OSSDownloadOperation.java @@ -89,7 +89,7 @@ static class DownloadCheckPoint implements Serializable { */ public synchronized void load(String cpFile) throws IOException, ClassNotFoundException { FileInputStream fileIn = new FileInputStream(cpFile); - ObjectInputStream in = new ObjectInputStream(fileIn); + CheckPointObjectInputStream in = new CheckPointObjectInputStream(fileIn); DownloadCheckPoint dcp = (DownloadCheckPoint) in.readObject(); assign(dcp); in.close(); diff --git a/src/test/java/com/aliyun/oss/integrationtests/ObjectDownloadFileTest.java b/src/test/java/com/aliyun/oss/integrationtests/ObjectDownloadFileTest.java new file mode 100644 index 00000000..e94a30e6 --- /dev/null +++ b/src/test/java/com/aliyun/oss/integrationtests/ObjectDownloadFileTest.java @@ -0,0 +1,35 @@ +package com.aliyun.oss.integrationtests; + +import com.aliyun.oss.model.DownloadFileRequest; +import com.aliyun.oss.model.DownloadFileResult; +import org.junit.Test; +import java.io.*; +public class ObjectDownloadFileTest extends TestBase implements Serializable { + + @Test + public void testObjectDownloadFileCheckPoint() throws IOException { + String objectName = "testObjectDownloadFileCheckPoint.txt"; + String filePath = "D:\\"; + + FileOutputStream fileOut = new FileOutputStream(filePath+objectName+".dcp"); + ObjectOutputStream outStream = new ObjectOutputStream(fileOut); + outStream.writeObject(this); + outStream.close(); + fileOut.close(); + + File file = createSampleFile(objectName, 1024 * 500); + ossClient.putObject(bucketName, objectName, new FileInputStream(file)); + + DownloadFileRequest downloadFileRequest = new DownloadFileRequest(bucketName, objectName); + downloadFileRequest.setDownloadFile(filePath+objectName); + downloadFileRequest.setPartSize(4 * 1024 * 1024); + downloadFileRequest.setTaskNum(1); + downloadFileRequest.setEnableCheckpoint(true); + + try { + DownloadFileResult result = ossClient.downloadFile(downloadFileRequest); + } catch (Throwable throwable) { + throwable.printStackTrace(); + } + } +} From da7c8157a1a09921c2e57c2001536747aa9130af Mon Sep 17 00:00:00 2001 From: zhuxiaolong37 Date: Fri, 12 Jan 2024 17:58:22 +0800 Subject: [PATCH 5/6] Fix the issue of not being able to load resource files in different locale --- .../java/com/aliyun/oss/common/utils/ResourceManager.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/aliyun/oss/common/utils/ResourceManager.java b/src/main/java/com/aliyun/oss/common/utils/ResourceManager.java index 80ba8a67..d187a903 100644 --- a/src/main/java/com/aliyun/oss/common/utils/ResourceManager.java +++ b/src/main/java/com/aliyun/oss/common/utils/ResourceManager.java @@ -30,7 +30,11 @@ public class ResourceManager { private ResourceBundle bundle; ResourceManager(String baseName, Locale locale) { - this.bundle = ResourceBundle.getBundle(baseName, locale); + try{ + this.bundle = ResourceBundle.getBundle(baseName, locale); + } catch (Exception e){ + this.bundle = ResourceBundle.getBundle(baseName); + } } public static ResourceManager getInstance(String baseName) { From e489d26d584fe2f4961bac47e8ff5144ed32a69c Mon Sep 17 00:00:00 2001 From: zhuxiaolong37 Date: Mon, 18 Mar 2024 17:36:52 +0800 Subject: [PATCH 6/6] Debug mode adds signature log --- src/main/java/com/aliyun/oss/internal/SignUtils.java | 2 ++ src/main/java/com/aliyun/oss/internal/SignV2Utils.java | 2 ++ .../java/com/aliyun/oss/internal/signer/OSSV1Signer.java | 3 +++ .../java/com/aliyun/oss/internal/signer/OSSV2Signer.java | 3 +++ .../java/com/aliyun/oss/internal/signer/OSSV4Signer.java | 9 +++++---- 5 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/aliyun/oss/internal/SignUtils.java b/src/main/java/com/aliyun/oss/internal/SignUtils.java index e3037619..dad70efa 100644 --- a/src/main/java/com/aliyun/oss/internal/SignUtils.java +++ b/src/main/java/com/aliyun/oss/internal/SignUtils.java @@ -40,6 +40,7 @@ import com.aliyun.oss.common.comm.RequestMessage; import com.aliyun.oss.common.utils.HttpHeaders; import com.aliyun.oss.common.utils.HttpUtil; +import com.aliyun.oss.common.utils.LogUtils; import com.aliyun.oss.model.GeneratePresignedUrlRequest; public class SignUtils { @@ -225,6 +226,7 @@ public static String buildCanonicalizedResource(String resourcePath, Map rawAdditionalHeaderNames = SignV2Utils.buildRawAdditionalHeaderNames(request.getHeaders().keySet(), signerParams.getAdditionalHeaderNames()); String canonicalString = SignV2Utils.buildCanonicalString(request.getMethod().toString(), canonicalResource, request, rawAdditionalHeaderNames); + LogUtils.getLog().debug("stringToSign: "+canonicalString); String signature = new HmacSHA256Signature().computeSignature(secretAccessKey, canonicalString); request.addParameter(OSS_SIGNATURE, signature); diff --git a/src/main/java/com/aliyun/oss/internal/signer/OSSV4Signer.java b/src/main/java/com/aliyun/oss/internal/signer/OSSV4Signer.java index c85b438d..0e7595e7 100644 --- a/src/main/java/com/aliyun/oss/internal/signer/OSSV4Signer.java +++ b/src/main/java/com/aliyun/oss/internal/signer/OSSV4Signer.java @@ -4,10 +4,7 @@ import com.aliyun.oss.common.auth.Credentials; import com.aliyun.oss.common.auth.ServiceSignature; import com.aliyun.oss.common.comm.RequestMessage; -import com.aliyun.oss.common.utils.BinaryUtil; -import com.aliyun.oss.common.utils.HttpHeaders; -import com.aliyun.oss.common.utils.HttpUtil; -import com.aliyun.oss.common.utils.StringUtils; +import com.aliyun.oss.common.utils.*; import com.aliyun.oss.internal.OSSHeaders; import com.aliyun.oss.internal.SignParameters; @@ -227,7 +224,9 @@ private String buildAuthorization(String signature) { @Override protected void addAuthorizationHeader(RequestMessage request) { String canonicalRequest = buildCanonicalRequest(request); + LogUtils.getLog().debug("canonicalString: "+canonicalRequest); String stringToSign = buildStringToSign(canonicalRequest); + LogUtils.getLog().debug("stringToSign: "+stringToSign); byte[] signingKey = buildSigningKey(); String signature = buildSignature(signingKey, stringToSign); String authorization = buildAuthorization(signature); @@ -283,11 +282,13 @@ public void presign(RequestMessage request) throws ClientException { // sign String canonicalRequest = buildCanonicalRequest(request); + LogUtils.getLog().debug("canonicalString: "+canonicalRequest); //String stringToSign = buildStringToSign(canonicalRequest); String stringToSign = OSS4_HMAC_SHA256 + SignParameters.NEW_LINE + getDateTime() + SignParameters.NEW_LINE + buildScope() + SignParameters.NEW_LINE + BinaryUtil.toHex(BinaryUtil.calculateSha256(canonicalRequest.getBytes(StringUtils.UTF8))); + LogUtils.getLog().debug("stringToSign: "+stringToSign); byte[] signingKey = buildSigningKey(); String signature = buildSignature(signingKey, stringToSign);