diff --git a/Lib/Azure Storage Client Library/AZSClient.xcodeproj/project.pbxproj b/Lib/Azure Storage Client Library/AZSClient.xcodeproj/project.pbxproj index 301445e..c8f437b 100644 --- a/Lib/Azure Storage Client Library/AZSClient.xcodeproj/project.pbxproj +++ b/Lib/Azure Storage Client Library/AZSClient.xcodeproj/project.pbxproj @@ -47,6 +47,10 @@ 69A278081C24A6A000981FF0 /* AZSAccountSasTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 69A278071C24A6A000981FF0 /* AZSAccountSasTests.m */; }; 69BE84101C223F2B00AF170E /* AZSIPRange.h in Headers */ = {isa = PBXBuildFile; fileRef = 69BE840E1C223F2B00AF170E /* AZSIPRange.h */; settings = {ATTRIBUTES = (Public, ); }; }; 69BE84111C223F2B00AF170E /* AZSIPRange.m in Sources */ = {isa = PBXBuildFile; fileRef = 69BE840F1C223F2B00AF170E /* AZSIPRange.m */; }; + 69F68DAE1C653EA70056D9E3 /* AZSStreamDownloadBuffer.h in Headers */ = {isa = PBXBuildFile; fileRef = 69F68DAD1C653EA70056D9E3 /* AZSStreamDownloadBuffer.h */; }; + 69F68DB01C653F3F0056D9E3 /* AZSStreamDownloadBuffer.m in Sources */ = {isa = PBXBuildFile; fileRef = 69F68DAF1C653F3F0056D9E3 /* AZSStreamDownloadBuffer.m */; }; + 69F68DB21C6550C10056D9E3 /* AZSBlobInputStream.h in Headers */ = {isa = PBXBuildFile; fileRef = 69F68DB11C6550C10056D9E3 /* AZSBlobInputStream.h */; }; + 69F68DB41C6942330056D9E3 /* AZSBlobInputStream.m in Sources */ = {isa = PBXBuildFile; fileRef = 69F68DB31C6942330056D9E3 /* AZSBlobInputStream.m */; }; 69FF6EBF1B83C56200AF4FDE /* AZSLeaseTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 69FF6EBE1B83C56200AF4FDE /* AZSLeaseTests.m */; }; B01734011B0A59BE00E5807C /* AZSCloudBlob.m in Sources */ = {isa = PBXBuildFile; fileRef = B01734001B0A59BE00E5807C /* AZSCloudBlob.m */; }; B01734041B0B04A500E5807C /* AZSBlobRequestXML.m in Sources */ = {isa = PBXBuildFile; fileRef = B01734031B0B04A500E5807C /* AZSBlobRequestXML.m */; }; @@ -81,7 +85,7 @@ B05A0E801B126592005DCF06 /* AZSCloudBlockBlobTests.m in Sources */ = {isa = PBXBuildFile; fileRef = B05A0E7F1B126592005DCF06 /* AZSCloudBlockBlobTests.m */; }; B05DEAF81AF006A30023B955 /* AZSBlobRequestOptions.m in Sources */ = {isa = PBXBuildFile; fileRef = B05DEAF71AF006A30023B955 /* AZSBlobRequestOptions.m */; }; B05EC6E01B27F47F0048EDD4 /* AZSBlobOutputStream.m in Sources */ = {isa = PBXBuildFile; fileRef = B05EC6DF1B27F47F0048EDD4 /* AZSBlobOutputStream.m */; }; - B05EC6E21B28E63F0048EDD4 /* AZSBlobOutputStreamTests.m in Sources */ = {isa = PBXBuildFile; fileRef = B05EC6E11B28E63F0048EDD4 /* AZSBlobOutputStreamTests.m */; }; + B05EC6E21B28E63F0048EDD4 /* AZSBlobStreamTests.m in Sources */ = {isa = PBXBuildFile; fileRef = B05EC6E11B28E63F0048EDD4 /* AZSBlobStreamTests.m */; }; B06CA9E81B748083001B7303 /* AZSTestRetry.m in Sources */ = {isa = PBXBuildFile; fileRef = B06CA9E71B748083001B7303 /* AZSTestRetry.m */; }; B07ED56D1AE6CE6A0012E8C1 /* AZSBlobRequestFactory.m in Sources */ = {isa = PBXBuildFile; fileRef = B07ED56C1AE6CE6A0012E8C1 /* AZSBlobRequestFactory.m */; }; B07ED57E1AE829FC0012E8C1 /* AZSSharedKeyBlobAuthenticationHandler.m in Sources */ = {isa = PBXBuildFile; fileRef = B07ED57D1AE829FC0012E8C1 /* AZSSharedKeyBlobAuthenticationHandler.m */; }; @@ -189,6 +193,10 @@ 69A278071C24A6A000981FF0 /* AZSAccountSasTests.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = AZSAccountSasTests.m; sourceTree = ""; }; 69BE840E1C223F2B00AF170E /* AZSIPRange.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = AZSIPRange.h; sourceTree = ""; }; 69BE840F1C223F2B00AF170E /* AZSIPRange.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = AZSIPRange.m; sourceTree = ""; }; + 69F68DAD1C653EA70056D9E3 /* AZSStreamDownloadBuffer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = AZSStreamDownloadBuffer.h; sourceTree = ""; }; + 69F68DAF1C653F3F0056D9E3 /* AZSStreamDownloadBuffer.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = AZSStreamDownloadBuffer.m; sourceTree = ""; }; + 69F68DB11C6550C10056D9E3 /* AZSBlobInputStream.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = AZSBlobInputStream.h; sourceTree = ""; }; + 69F68DB31C6942330056D9E3 /* AZSBlobInputStream.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = AZSBlobInputStream.m; sourceTree = ""; }; 69FF6EBE1B83C56200AF4FDE /* AZSLeaseTests.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = AZSLeaseTests.m; sourceTree = ""; }; B01733FF1B0A59BE00E5807C /* AZSCloudBlob.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = AZSCloudBlob.h; sourceTree = ""; }; B01734001B0A59BE00E5807C /* AZSCloudBlob.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = AZSCloudBlob.m; sourceTree = ""; }; @@ -245,7 +253,7 @@ B05DEAF71AF006A30023B955 /* AZSBlobRequestOptions.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = AZSBlobRequestOptions.m; sourceTree = ""; }; B05EC6DE1B27F47F0048EDD4 /* AZSBlobOutputStream.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = AZSBlobOutputStream.h; sourceTree = ""; }; B05EC6DF1B27F47F0048EDD4 /* AZSBlobOutputStream.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = AZSBlobOutputStream.m; sourceTree = ""; }; - B05EC6E11B28E63F0048EDD4 /* AZSBlobOutputStreamTests.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = AZSBlobOutputStreamTests.m; sourceTree = ""; }; + B05EC6E11B28E63F0048EDD4 /* AZSBlobStreamTests.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = AZSBlobStreamTests.m; sourceTree = ""; }; B06CA9E71B748083001B7303 /* AZSTestRetry.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = AZSTestRetry.m; sourceTree = ""; }; B07ED56B1AE6CE6A0012E8C1 /* AZSBlobRequestFactory.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = AZSBlobRequestFactory.h; sourceTree = ""; }; B07ED56C1AE6CE6A0012E8C1 /* AZSBlobRequestFactory.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = AZSBlobRequestFactory.m; sourceTree = ""; }; @@ -329,6 +337,21 @@ /* End PBXFrameworksBuildPhase section */ /* Begin PBXGroup section */ + 953A1AC11ECA7AAF005C57E2 /* ServiceProperties */ = { + isa = PBXGroup; + children = ( + 57E472281DF73E7000B343AB /* AZSServiceProperties.m */, + 57E4722A1DF7423A00B343AB /* AZSServiceProperties.h */, + 57F37C011E1F09BB00FF130F /* AZSLoggingProperties.h */, + 57F37C021E1F1D3400FF130F /* AZSMetricsProperties.h */, + 57F37C031E1F290A00FF130F /* AZSLoggingProperties.m */, + 57F37C051E1F292800FF130F /* AZSMetricsProperties.m */, + 57F37C0A1E2022B900FF130F /* AZSCorsRule.h */, + 57F37C0B1E2022D300FF130F /* AZSCorsRule.m */, + ); + name = ServiceProperties; + sourceTree = ""; + }; B01AF6021AE0718F009A2022 /* Blob */ = { isa = PBXGroup; children = ( @@ -338,14 +361,12 @@ B09BDEA61C1A4787004D2F94 /* AZSCloudBlobDirectory.m */, B01AF5F31AE06DDC009A2022 /* AZSCloudBlobClient.h */, B01AF5F41AE06DDC009A2022 /* AZSCloudBlobClient.m */, - BE4510691A9D254600C3F971 /* AZSCloudBlockBlob.h */, - BE45106A1A9D254600C3F971 /* AZSCloudBlockBlob.m */, B01AF5F01AE053BE009A2022 /* AZSCloudBlobContainer.h */, B01AF5F11AE053BE009A2022 /* AZSCloudBlobContainer.m */, - B05DEAF61AF006A30023B955 /* AZSBlobRequestOptions.h */, - B05DEAF71AF006A30023B955 /* AZSBlobRequestOptions.m */, B01733FF1B0A59BE00E5807C /* AZSCloudBlob.h */, B01734001B0A59BE00E5807C /* AZSCloudBlob.m */, + B05DEAF61AF006A30023B955 /* AZSBlobRequestOptions.h */, + B05DEAF71AF006A30023B955 /* AZSBlobRequestOptions.m */, B05A0E6F1B0BF308005DCF06 /* AZSBlockListItem.h */, B05A0E701B0BF308005DCF06 /* AZSBlockListItem.m */, BE7E3E4D1B165F6F00BC96B6 /* AZSBlobContainerProperties.h */, @@ -354,10 +375,14 @@ BE7E3E521B18D80A00BC96B6 /* AZSBlobProperties.m */, BE7E3E551B18D82100BC96B6 /* AZSCopyState.h */, BE7E3E561B18D82100BC96B6 /* AZSCopyState.m */, + 69F68DB11C6550C10056D9E3 /* AZSBlobInputStream.h */, + 69F68DB31C6942330056D9E3 /* AZSBlobInputStream.m */, B05EC6DE1B27F47F0048EDD4 /* AZSBlobOutputStream.h */, B05EC6DF1B27F47F0048EDD4 /* AZSBlobOutputStream.m */, B058A4951B55628700BB0F57 /* AZSBlobUploadHelper.h */, B058A4961B55628700BB0F57 /* AZSBlobUploadHelper.m */, + BE4510691A9D254600C3F971 /* AZSCloudBlockBlob.h */, + BE45106A1A9D254600C3F971 /* AZSCloudBlockBlob.m */, B0DD6B5E1C208105004B3A7D /* AZSCloudPageBlob.h */, B0DD6B5F1C208105004B3A7D /* AZSCloudPageBlob.m */, B0DD6B641C209175004B3A7D /* AZSCloudAppendBlob.h */, @@ -396,6 +421,8 @@ B01AF60F1AE098CB009A2022 /* AZSExecutor.m */, B01AF6111AE099C5009A2022 /* AZSRequestOptions.h */, B01AF6121AE099C5009A2022 /* AZSRequestOptions.m */, + 69F68DAD1C653EA70056D9E3 /* AZSStreamDownloadBuffer.h */, + 69F68DAF1C653F3F0056D9E3 /* AZSStreamDownloadBuffer.m */, ); name = Executor; sourceTree = ""; @@ -446,6 +473,7 @@ BE4510451A9D252300C3F971 /* AZSClient */ = { isa = PBXGroup; children = ( + 953A1AC11ECA7AAF005C57E2 /* ServiceProperties */, BEC4476D1B751FBA00111ADA /* Constants */, B07ED56E1AE713CA0012E8C1 /* Auth */, B01AF6041AE08C50009A2022 /* Executor */, @@ -491,14 +519,6 @@ BE90F9041B4EDF7300CD278B /* AZSUriQueryBuilder.m */, BE4510461A9D252300C3F971 /* Supporting Files */, B0432F5C1CE3B05D00FF4E5A /* AZSULLRange.h */, - 57E472281DF73E7000B343AB /* AZSServiceProperties.m */, - 57E4722A1DF7423A00B343AB /* AZSServiceProperties.h */, - 57F37C011E1F09BB00FF130F /* AZSLoggingProperties.h */, - 57F37C021E1F1D3400FF130F /* AZSMetricsProperties.h */, - 57F37C031E1F290A00FF130F /* AZSLoggingProperties.m */, - 57F37C051E1F292800FF130F /* AZSMetricsProperties.m */, - 57F37C0A1E2022B900FF130F /* AZSCorsRule.h */, - 57F37C0B1E2022D300FF130F /* AZSCorsRule.m */, ); name = AZSClient; path = "Azure Storage Client Library"; @@ -529,10 +549,10 @@ B02EF9931B14F10B005F1DE2 /* AZSBlobTestBase.m */, B04A084A1B274C7100998078 /* AZSTestHelpers.h */, B04A084B1B274C7100998078 /* AZSTestHelpers.m */, - B05EC6E11B28E63F0048EDD4 /* AZSBlobOutputStreamTests.m */, B06CA9E71B748083001B7303 /* AZSTestRetry.m */, 693220411C16439500B07D98 /* AZSTestSemaphore.h */, 693220421C16439500B07D98 /* AZSTestSemaphore.m */, + B05EC6E11B28E63F0048EDD4 /* AZSBlobStreamTests.m */, B082D1441BA9E0D200A39C18 /* AZSBlobSASTests.m */, B09BDEA91C1A4BB5004D2F94 /* AZSCloudBlobDirectoryTests.m */, B0DD6B621C2087C5004B3A7D /* AZSCloudPageBlobTests.m */, @@ -584,6 +604,7 @@ B082D1851BB0D29300A39C18 /* AZSCloudBlobContainer.h in Headers */, B082D1861BB0D29700A39C18 /* AZSBlobRequestOptions.h in Headers */, B082D1871BB0D29E00A39C18 /* AZSCloudBlob.h in Headers */, + 69F68DAE1C653EA70056D9E3 /* AZSStreamDownloadBuffer.h in Headers */, B082D1881BB0D2A100A39C18 /* AZSBlockListItem.h in Headers */, B082D1891BB0D2A400A39C18 /* AZSBlobContainerProperties.h in Headers */, B082D18A1BB0D2A600A39C18 /* AZSBlobProperties.h in Headers */, @@ -607,6 +628,7 @@ B082D1921BB0D2D100A39C18 /* AZSResultSegment.h in Headers */, B082D1931BB0D2D400A39C18 /* AZSContinuationToken.h in Headers */, B082D1941BB0D2D600A39C18 /* AZSAccessCondition.h in Headers */, + 69F68DB21C6550C10056D9E3 /* AZSBlobInputStream.h in Headers */, B082D1951BB0D2D900A39C18 /* AZSRetryInfo.h in Headers */, 57746E391E97FADB007F9CB9 /* AZSMetricsProperties.h in Headers */, 57E4722B1DF7423A00B343AB /* AZSServiceProperties.h in Headers */, @@ -782,6 +804,7 @@ 57F37C061E1F292800FF130F /* AZSMetricsProperties.m in Sources */, BE90F9011B3DEC7900CD278B /* AZSNavigationUtil.m in Sources */, B01AF5F21AE053BE009A2022 /* AZSCloudBlobContainer.m in Sources */, + 69F68DB01C653F3F0056D9E3 /* AZSStreamDownloadBuffer.m in Sources */, B058A4971B55628700BB0F57 /* AZSBlobUploadHelper.m in Sources */, B01AF5FB1AE0715D009A2022 /* AZSStorageCredentials.m in Sources */, 69A278041C24A4DB00981FF0 /* AZSSharedAccessAccountParameters.m in Sources */, @@ -795,6 +818,7 @@ B07ED5811AE85E1B0012E8C1 /* AZSUtil.m in Sources */, B07ED56D1AE6CE6A0012E8C1 /* AZSBlobRequestFactory.m in Sources */, B07ED58A1AE9AEDF0012E8C1 /* AZSAccessCondition.m in Sources */, + 69F68DB41C6942330056D9E3 /* AZSBlobInputStream.m in Sources */, BE7E3E5F1B1F9AEB00BC96B6 /* AZSRequestFactory.m in Sources */, 57F37C041E1F290A00FF130F /* AZSLoggingProperties.m in Sources */, ); @@ -814,7 +838,7 @@ B01B6C2A1C24ADEA004D7CFE /* AZSCloudAppendBlobTests.m in Sources */, B04A084C1B274C7100998078 /* AZSTestHelpers.m in Sources */, B05A0E7E1B1264E8005DCF06 /* AZSCloudBlobClientTests.m in Sources */, - B05EC6E21B28E63F0048EDD4 /* AZSBlobOutputStreamTests.m in Sources */, + B05EC6E21B28E63F0048EDD4 /* AZSBlobStreamTests.m in Sources */, B06CA9E81B748083001B7303 /* AZSTestRetry.m in Sources */, BE7E3E641B277DB100BC96B6 /* AZSNoOpAuthenticationHandler.m in Sources */, B09BDEAA1C1A4BB5004D2F94 /* AZSCloudBlobDirectoryTests.m in Sources */, diff --git a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSBlobInputStream.h b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSBlobInputStream.h new file mode 100644 index 0000000..c32088a --- /dev/null +++ b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSBlobInputStream.h @@ -0,0 +1,63 @@ +// ----------------------------------------------------------------------------------------- +// +// Copyright 2015 Microsoft Corporation +// +// Licensed under the MIT License; +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://spdx.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------------------- + +#import +#import "AZSMacros.h" + +AZS_ASSUME_NONNULL_BEGIN + +@class AZSCloudBlob; +@class AZSAccessCondition; +@class AZSBlobRequestOptions; +@class AZSStreamDownloadBuffer; +@class AZSOperationContext; + +/** An AZSBlobInputStream is used to read data from a blob on the service. + + The AZSBlobInputStream class inherits from NSInputStream, and is designed to be used similarly. + To create an AZSBlobInputStream instance, call the createInputStream method on an instance of AZSCloudBlob. + Just like a regular Input stream, you can set a delegate to be called on stream events, and then schedule the stream in a runloop. + See https://developer.apple.com/library/mac/documentation/Cocoa/Conceptual/Streams/Streams.html for more details about how to use streams. + */ +@interface AZSBlobInputStream : NSInputStream + +-(instancetype)initWithBlob:(AZSCloudBlob *)blob accessCondition:(AZSAccessCondition *)accessCondition requestOptions:(AZSNullable AZSBlobRequestOptions *)requestOptions operationContext:(AZSOperationContext *)operationContext AZS_DESIGNATED_INITIALIZER; + +// NSStream methods and properties: +@property (readonly) NSStreamStatus streamStatus; +@property (readonly, copy, AZSNullable) NSError *streamError; +@property (assign, AZSNullable) id delegate; +@property (strong, readonly) AZSStreamDownloadBuffer *downloadBuffer; + +-(void)open; +-(void)close; + +-(AZSNullable id)propertyForKey:(NSString *)key; +-(BOOL)setProperty:(AZSNullable id)property forKey:(NSString *)key; + +-(void)scheduleInRunLoop:(NSRunLoop *)runLoop forMode:(NSString *)mode; +-(void)removeFromRunLoop:(NSRunLoop *)runLoop forMode:(NSString *)mode; + +// NSInputStream methods and properties: +@property (readonly) BOOL hasBytesAvailable; + +-(NSInteger)read:(uint8_t *)buffer maxLength:(NSUInteger)length; +-(BOOL)getBuffer:(uint8_t * __AZSNullable * __nonnull)buffer length:(NSUInteger *)length; + +@end + +AZS_ASSUME_NONNULL_END \ No newline at end of file diff --git a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSBlobInputStream.m b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSBlobInputStream.m new file mode 100644 index 0000000..ccabcd6 --- /dev/null +++ b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSBlobInputStream.m @@ -0,0 +1,399 @@ +// ----------------------------------------------------------------------------------------- +// +// Copyright 2015 Microsoft Corporation +// +// Licensed under the MIT License; +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://spdx.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------------------- + +#import "AZSBlobInputStream.h" +#import "AZSBlobProperties.h" +#import "AZSBlobRequestFactory.h" +#import "AZSBlobResponseParser.h" +#import "AZSCloudBlobClient.h" +#import "AZSCloudBlockBlob.h" +#import "AZSBlockListItem.h" +#import "AZSBlobRequestOptions.h" +#import "AZSErrors.h" +#import "AZSExecutor.h" +#import "AZSRequestResult.h" +#import "AZSResponseParser.h" +#import "AZSStorageCommand.h" +#import "AZSStreamDownloadBuffer.h" +#import "AZSOperationContext.h" + +@interface AZSBlobInputStream() + +@property BOOL isStreamOpen; +@property BOOL isReading; +@property BOOL isStreamClosing; +@property BOOL isStreamClosed; +@property CFRunLoopSourceRef runLoopSource; +@property (strong) NSMutableArray *runLoopsRegistered; +@property BOOL waitingOnCaller; +@property BOOL hasStreamOpenEventFired; +@property BOOL hasStreamErrorEventFired; +@property BOOL hasStreamEndEventFired; +@property (strong) AZSCloudBlob *underlyingBlob; +@property (copy, readonly) void(^beginDownload)(); + +-(instancetype)init; + +// This method should never be called. It is only here to comply with subclassing requirements. +-(instancetype)initWithFileAtPath:(NSString *)path; + +// This method should never be called. It is only here to comply with subclassing requirements. +-(instancetype)initWithData:(NSData *)data AZS_DESIGNATED_INITIALIZER; + +// This method should never be called. It is only here to comply with subclassing requirements. +-(instancetype)initWithURL:(NSURL *)url AZS_DESIGNATED_INITIALIZER; + +@end + +// These methods need to be here because the runloop expects them, but we don't have any work to be done. +void AZSBlobInputStreamRunLoopSourceScheduleRoutine (void *info, CFRunLoopRef rl, CFStringRef mode) {} +void AZSBlobInputStreamRunLoopSourceCancelRoutine (void *info, CFRunLoopRef rl, CFStringRef mode) {} + +void AZSBlobInputStreamRunLoopSourcePerformRoutine (void *info) +{ + AZSBlobInputStream *stream = (__bridge AZSBlobInputStream *)info; + [stream.downloadBuffer.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"Perform Routine called."]; + BOOL fireStreamOpenEvent = NO; + BOOL hasBytesAvailable = NO; + BOOL fireStreamErrorEvent = NO; + BOOL downloadComplete = NO; + + // Syncronize access to the various constants. + // TODO: determine if synchronizing is needed. + @synchronized(stream) { + if (stream.isStreamOpen && !stream.hasStreamOpenEventFired) { + fireStreamOpenEvent = YES; + } + + if (stream.streamError && !stream.hasStreamErrorEventFired) { + fireStreamErrorEvent = YES; + } + + if (stream.isStreamClosing || stream.isStreamClosed || !stream.isStreamOpen) { + return; + } + + if (stream.hasBytesAvailable && !stream.waitingOnCaller && !stream.isReading) { + hasBytesAvailable = YES; + } + + if (!stream.hasBytesAvailable && stream.downloadBuffer.downloadComplete) { + downloadComplete = YES; + } + } + + if (fireStreamErrorEvent) { + if ([stream.delegate respondsToSelector:@selector(stream:handleEvent:)]) { + stream.hasStreamErrorEventFired = YES; + [stream.delegate stream:stream handleEvent:NSStreamEventErrorOccurred]; + } + } else if (fireStreamOpenEvent) { + if ([stream.delegate respondsToSelector:@selector(stream:handleEvent:)]) { + stream.hasStreamOpenEventFired = YES; + [stream.delegate stream:stream handleEvent:NSStreamEventOpenCompleted]; + } + } else if (hasBytesAvailable) { + if ([stream.delegate respondsToSelector:@selector(stream:handleEvent:)]) { + @synchronized(stream) { + stream.waitingOnCaller = YES; + } + [stream.delegate stream:stream handleEvent:NSStreamEventHasBytesAvailable]; + } + } + else if (downloadComplete) { + if ([stream.delegate respondsToSelector:@selector(stream:handleEvent:)]) { + [stream.delegate stream:stream handleEvent:NSStreamEventEndEncountered]; + } + } +} + +@implementation AZSBlobInputStream + +@synthesize delegate = _delegate; +@synthesize hasBytesAvailable = _hasBytesAvailable; + +-(instancetype)init +{ + self = [self initWithData:[NSData alloc]]; + return nil; +} + +-(instancetype)initWithFileAtPath:(NSString *)path +{ + self = [self initWithData:[NSData alloc]]; + return nil; +} + +-(instancetype)initWithData:(NSData *)data +{ + self = [super initWithData:data]; + return nil; +} + +-(instancetype)initWithURL:(NSURL *)url +{ + self = [super initWithURL:url]; + return nil; +} + +-(instancetype)initWithBlob:(AZSCloudBlob *)blob accessCondition:(AZSAccessCondition *)accessCondition requestOptions:(AZSNullable AZSBlobRequestOptions *)requestOptions operationContext:(AZSOperationContext *)operationContext +{ + // A designated initializer must make a super call to a designated initializer of the super class. + self = [super initWithData:[[NSData alloc] init]]; + if (self) { + _underlyingBlob = blob; + _downloadBuffer = [[AZSStreamDownloadBuffer alloc] initWithInputStream:self maxSizeToBuffer:requestOptions.maximumDownloadBufferSize calculateMD5:!requestOptions.disableContentMD5Validation runLoopForDownload:nil operationContext:operationContext fireEventBlock:^() { + [self fireStreamEvent]; + } setHasBytesAvailable:^() { + _hasBytesAvailable = YES; + }]; + + _isStreamOpen = NO; + _isStreamClosing = NO; + _isStreamClosed = NO; + _isReading = NO; + CFRunLoopSourceContext context = + {0, (__bridge void *)(self), NULL, NULL, NULL, NULL, NULL, + &AZSBlobInputStreamRunLoopSourceScheduleRoutine, + AZSBlobInputStreamRunLoopSourceCancelRoutine, + AZSBlobInputStreamRunLoopSourcePerformRoutine}; + _runLoopSource = CFRunLoopSourceCreate(NULL, 0, &context); + _runLoopsRegistered = [NSMutableArray arrayWithCapacity:1]; + _waitingOnCaller = NO; + _delegate = self; + _hasStreamOpenEventFired = NO; + _hasStreamErrorEventFired = NO; + + __weak AZSStreamDownloadBuffer *weakBuffer = _downloadBuffer; + _beginDownload = ^() { + AZSStorageCommand *command = [blob downloadCommandWithRange:AZSULLMakeRange(0, 0) AccessCondition:accessCondition requestOptions:requestOptions operationContext:operationContext]; + [AZSExecutor ExecuteWithStorageCommand:command requestOptions:requestOptions operationContext:operationContext downloadBuffer:weakBuffer completionHandler:^(NSError *error, id result) { + weakBuffer.streamError = error; + }]; + + return; + }; + } + + return self; +} + +-(void)setDelegate:(id)delegate +{ + _delegate = delegate ?: self; +} + +-(id)delegate +{ + return _delegate; +} + +- (void)stream:(NSStream *)stream handleEvent:(NSStreamEvent)eventCode +{ + // No default impementation if the user doesn't provide a delegate. +} + +-(void)fireStreamEvent +{ + [self.downloadBuffer.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"Fire stream event called."]; + CFRunLoopSourceSignal(self.runLoopSource); + + // TODO: confirm this. I don't know if we have to wake up all runloops here. + for (NSRunLoop *runLoop in self.runLoopsRegistered) { + CFRunLoopWakeUp([runLoop getCFRunLoop]); + } +} + +-(void)open +{ + [self.downloadBuffer.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"Called open."]; + @synchronized (self) { + self.isStreamOpen = YES; + } + [self fireStreamEvent]; + + if (self.beginDownload) { + @synchronized (self) { + if (self.beginDownload) { + self.beginDownload(); + _beginDownload = nil; + } + } + } + + // TODO: Should opening a closed stream be an error? +} + +- (BOOL)getBuffer:(uint8_t * __AZSNullable *)buffer length:(NSUInteger *)length +{ + BOOL success = NO; + @synchronized(self) { + self.waitingOnCaller = NO; + self.isReading = YES; + } + + if (self.isStreamOpen) { + [self.downloadBuffer processDataWithProcess:^long(uint8_t *buf, long len) { + *buffer = buf; + *length = len; + return len; + }]; + + success = YES; + } + + [self fireStreamEvent]; + + @synchronized (self) { + self.isReading = NO; + } + + return success; +} + +-(NSInteger)read:(uint8_t *)buffer maxLength:(NSUInteger)length +{ + if (!length) { + return 0; + } + + // TODO: Investigate whether or not we need to set self.waitingOnCaller to NO if this method is called with zero bytes. + // TODO: Should these be read/write locked or are reads atomic? + @synchronized(self) { + self.waitingOnCaller = NO; + self.isReading = YES; + } + + if (self.streamError) { + return -1; + } + + // TODO: if length > MAX_INT32, dataCopied can overflow + __block NSInteger dataCopied = 0; + if (self.isStreamOpen) { + [self.downloadBuffer processDataWithProcess:^long (uint8_t * buf, long amount) { + dataCopied = MIN(amount, length); + memcpy(buffer, buf, dataCopied); + return dataCopied; + }]; + + if (self.downloadBuffer.currentLength <= 0) { + _hasBytesAvailable = NO; + } + } + else { + dataCopied = 0; + } + + [self fireStreamEvent]; + @synchronized(self) { + self.isReading = NO; + } + + return dataCopied; +} + +-(void)close +{ + @synchronized (self) { + self.isStreamClosing = YES; + self.isStreamOpen = NO; + + [self.downloadBuffer.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"Called close."]; + + self.isStreamClosed = YES; + self.downloadBuffer.streamClosed = YES; + self.isStreamClosing = NO; + } +} + +// TODO: NSStreamStatusError +// What was this todo for? +-(NSStreamStatus) streamStatus +{ + [self.downloadBuffer.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"Stream status requested."]; + if (self.streamError) { + return NSStreamStatusError; + } + + if (self.isStreamClosed) { + return NSStreamStatusClosed; + } + + if (self.isStreamClosing) { + return NSStreamStatusAtEnd; + } + + if (self.isReading) { + return NSStreamStatusReading; + } + + if (self.isStreamOpen) { + return NSStreamStatusOpen; + } + + return NSStreamStatusNotOpen; +} + +// TODO: Do we need this? Or can we just use the autogenerated _streamError? +-(NSError *) streamError +{ + return self.downloadBuffer.streamError; +} + +-(void)scheduleInRunLoop:runLoop forMode:(NSString *)mode +{ + [self.downloadBuffer.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"Schedule in run loop requested."]; + CFRunLoopRef cfRunLoop = [runLoop getCFRunLoop]; + CFRunLoopAddSource(cfRunLoop, self.runLoopSource, (__bridge CFStringRef)(mode) /*kCFRunLoopDefaultMode*/); + + @synchronized(self) { + if (![self.runLoopsRegistered containsObject:runLoop]) { + [self.runLoopsRegistered addObject:runLoop]; + } + } +} + +-(void)removeFromRunLoop:runLoop forMode:(NSString *)mode +{ + [self.downloadBuffer.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"Remove from run loop requested."]; + // TODO: Call CFBridgingRelease(self). + CFRunLoopRef cfRunLoop = [runLoop getCFRunLoop]; + CFRunLoopRemoveSource(cfRunLoop, self.runLoopSource, (__bridge CFStringRef)(mode) /*kCFRunLoopDefaultMode*/); + + @synchronized(self) { + [self.runLoopsRegistered removeObject:runLoop]; + } +} + +-(id)propertyForKey:(NSString *)key +{ + [self.downloadBuffer.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"Property for key requested. Key = %@", key]; + + // TODO: Decide if we want to support NSStreamFileCurrentOffsetKey. (Note that not all blobs are files.) + return nil; +} + +-(BOOL)setProperty:(id)property forKey:(NSString *)key +{ + [self.downloadBuffer.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"SetProperty for key requested. Key = %@", key]; + + // TODO: Decide if we want to support NSStreamFileCurrentOffsetKey. (Note that not all blobs are files.) + return NO; +} + +@end \ No newline at end of file diff --git a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSBlobOutputStream.m b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSBlobOutputStream.m index 3bbbbd5..cd3c3e0 100644 --- a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSBlobOutputStream.m +++ b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSBlobOutputStream.m @@ -122,7 +122,6 @@ void AZSBlobOutputStreamRunLoopSourcePerformRoutine (void *info) @implementation AZSBlobOutputStream @synthesize delegate = _delegate; -@synthesize hasSpaceAvailable = _hasSpaceAvailable; -(instancetype)initToMemory { @@ -245,7 +244,7 @@ -(void)open -(NSInteger)write:(const uint8_t *)buffer maxLength:(NSUInteger)length { - // TODO: Investigate whether or not we need to set self.waitingOnCaller to NO if this method is called with zero bytes. + // TODO: Investigate whether or not we need to set self.waitingOnCaller to NO if this method is called with zero bytes, or if it returns zero bytes. @synchronized(self) { self.waitingOnCaller = NO; @@ -259,7 +258,7 @@ -(NSInteger)write:(const uint8_t *)buffer maxLength:(NSUInteger)length NSInteger dataCopied = 0; if (self.isStreamOpen) { - dataCopied = [self.blobUploadHelper write:buffer maxLength:length completionHandler:^{ + dataCopied = [self.blobUploadHelper write:buffer length:length completionHandler:^{ [self fireStreamEvent]; }]; } @@ -277,16 +276,13 @@ -(void)close [self.blobUploadHelper.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"Called close."]; self.isStreamClosing = YES; self.isStreamOpen = NO; - if (![self.blobUploadHelper closeWithCompletionHandler:^{ + [self.blobUploadHelper closeWithCompletionHandler:^{ + self.isStreamClosed = YES; + self.isStreamClosing = NO; [self fireStreamEvent]; - }]) - { - // TODO: Error. - } - - self.isStreamClosed = YES; - self.isStreamClosing = NO; - [self fireStreamEvent]; + } uploadBufferCompletionHandler:^{ + [self fireStreamEvent]; + }]; } // TODO: NSStreamStatusError @@ -316,12 +312,16 @@ -(NSStreamStatus) streamStatus return NSStreamStatusNotOpen; } -// TODO: Do we need this? Or can we just use the autogenerated _streamError? -(NSError *) streamError { return self.blobUploadHelper.streamingError; } +-(BOOL) hasSpaceAvailable +{ + return self.blobUploadHelper.hasSpaceAvailable; +} + -(void)scheduleInRunLoop:runLoop forMode:(NSString *)mode { [self.blobUploadHelper.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"Schedule in run loop requested."]; diff --git a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSBlobRequestFactory.h b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSBlobRequestFactory.h index de32a22..997a2ca 100644 --- a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSBlobRequestFactory.h +++ b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSBlobRequestFactory.h @@ -56,9 +56,9 @@ +(NSMutableURLRequest *) abortCopyBlobWithCopyId:(NSString *)copyId accessCondition:(AZSAccessCondition *)accessCondition urlComponents:(NSURLComponents *)urlComponents timeout:(NSTimeInterval)timeout operationContext:(AZSOperationContext *)operationContext; // Block blobs -+(NSMutableURLRequest *) putBlockBlobWithLength:(NSUInteger)length blobProperties:(AZSBlobProperties *)blobPropertes contentMD5:(NSString *)contentMD5 cloudMetadata:(NSMutableDictionary *)cloudMetadata AccessCondition:(AZSAccessCondition *)accessCondition urlComponents:(NSURLComponents *)urlComponents timeout:(NSTimeInterval)timeout operationContext:(AZSOperationContext *)operationContext; ++(NSMutableURLRequest *) putBlockBlobWithLength:(NSUInteger)length blobProperties:(AZSBlobProperties *)blobPropertes contentMD5:(NSString *)contentMD5 cloudMetadata:(NSMutableDictionary *)cloudMetadata accessCondition:(AZSAccessCondition *)accessCondition urlComponents:(NSURLComponents *)urlComponents timeout:(NSTimeInterval)timeout operationContext:(AZSOperationContext *)operationContext; +(NSMutableURLRequest *) putBlockWithLength:(NSUInteger)length blockID:(NSString *)blockID contentMD5:(NSString *)contentMD5 accessCondition:(AZSAccessCondition *)accessCondition urlComponents:(NSURLComponents *)urlComponents timeout:(NSTimeInterval)timeout operationContext:(AZSOperationContext *)operationContext; -+(NSMutableURLRequest *) putBlockListWithLength:(NSUInteger)length blobProperties:(AZSBlobProperties *)blobPropertes contentMD5:(NSString *)contentMD5 cloudMetadata:(NSMutableDictionary *)cloudMetadata AccessCondition:(AZSAccessCondition *)accessCondition urlComponents:(NSURLComponents *)urlComponents timeout:(NSTimeInterval)timeout operationContext:(AZSOperationContext *)operationContext; ++(NSMutableURLRequest *) putBlockListWithLength:(NSUInteger)length blobProperties:(AZSBlobProperties *)blobPropertes contentMD5:(NSString *)contentMD5 cloudMetadata:(NSMutableDictionary *)cloudMetadata accessCondition:(AZSAccessCondition *)accessCondition urlComponents:(NSURLComponents *)urlComponents timeout:(NSTimeInterval)timeout operationContext:(AZSOperationContext *)operationContext; +(NSMutableURLRequest *) getBlockListWithBlockListFilter:(AZSBlockListFilter)blockListFilter snapshotTime:(NSString *)snapshotTime accessCondition:(AZSAccessCondition *)accessCondition urlComponents:(NSURLComponents *)urlComponents timeout:(NSTimeInterval)timeout operationContext:(AZSOperationContext *)operationContext; // Page blobs diff --git a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSBlobRequestFactory.m b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSBlobRequestFactory.m index e1494b4..96f70a9 100644 --- a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSBlobRequestFactory.m +++ b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSBlobRequestFactory.m @@ -183,7 +183,7 @@ +(NSMutableURLRequest *) downloadServicePropertiesWithUrlComponents:(NSURLCompon return request; } -+(NSMutableURLRequest *) putBlockBlobWithLength:(NSUInteger)length blobProperties:(AZSBlobProperties *)blobProperties contentMD5:(NSString *)contentMD5 cloudMetadata:(NSMutableDictionary *)cloudMetadata AccessCondition:(AZSAccessCondition *)accessCondition urlComponents:(NSURLComponents *)urlComponents timeout:(NSTimeInterval)timeout operationContext:(AZSOperationContext *)operationContext ++(NSMutableURLRequest *) putBlockBlobWithLength:(NSUInteger)length blobProperties:(AZSBlobProperties *)blobProperties contentMD5:(NSString *)contentMD5 cloudMetadata:(NSMutableDictionary *)cloudMetadata accessCondition:(AZSAccessCondition *)accessCondition urlComponents:(NSURLComponents *)urlComponents timeout:(NSTimeInterval)timeout operationContext:(AZSOperationContext *)operationContext { // TODO: IOS 8 - update this to use urlComponents.queryItems NSMutableURLRequest *request = [AZSRequestFactory putRequestWithUrlComponents:urlComponents timeout:timeout]; @@ -213,7 +213,7 @@ +(NSMutableURLRequest *) putBlockWithLength:(NSUInteger)length blockID:(NSString return request; } -+(NSMutableURLRequest *) putBlockListWithLength:(NSUInteger)length blobProperties:(AZSBlobProperties *)blobProperties contentMD5:(NSString *)contentMD5 cloudMetadata:(NSMutableDictionary *)cloudMetadata AccessCondition:(AZSAccessCondition *)accessCondition urlComponents:(NSURLComponents *)urlComponents timeout:(NSTimeInterval)timeout operationContext:(AZSOperationContext *)operationContext ++(NSMutableURLRequest *) putBlockListWithLength:(NSUInteger)length blobProperties:(AZSBlobProperties *)blobProperties contentMD5:(NSString *)contentMD5 cloudMetadata:(NSMutableDictionary *)cloudMetadata accessCondition:(AZSAccessCondition *)accessCondition urlComponents:(NSURLComponents *)urlComponents timeout:(NSTimeInterval)timeout operationContext:(AZSOperationContext *)operationContext { urlComponents.percentEncodedQuery = [AZSRequestFactory appendToQuery:urlComponents.percentEncodedQuery stringToAppend:AZSCQueryCompBlockList]; // TODO: IOS 8 - update this to use urlComponents.queryItems diff --git a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSBlobRequestOptions.h b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSBlobRequestOptions.h index 0e92477..d5b1974 100644 --- a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSBlobRequestOptions.h +++ b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSBlobRequestOptions.h @@ -49,8 +49,9 @@ AZS_ASSUME_NONNULL_BEGIN */ @property BOOL absorbConditionalErrorsOnRetry; -// TODO: Implement logic to upload a blob as a single Put Blob call if below the below threshold. -//@property NSInteger *singleBlobUploadThreshold; +/** If the size in bytes is below this threshold, the blob would be uploaded as a single Put Blob call */ +// TOASK should there be an upper limit on this? +@property NSUInteger singleBlobUploadThreshold; /** Initializes a new AZSBlobRequestOptions object. Once the object is initialized, individual properties can be set.*/ diff --git a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSBlobRequestOptions.m b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSBlobRequestOptions.m index dbe1f75..12075d5 100644 --- a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSBlobRequestOptions.m +++ b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSBlobRequestOptions.m @@ -24,6 +24,7 @@ @interface AZSBlobRequestOptions() BOOL _disableContentMD5ValidationSet; BOOL _parallelismFactorSet; BOOL _absorbConditionalErrorsOnRetrySet; + BOOL _singleBlobUploadThresholdSet; } @end @@ -35,6 +36,7 @@ @implementation AZSBlobRequestOptions @synthesize disableContentMD5Validation = _disableContentMD5Validation; @synthesize parallelismFactor = _parallelismFactor; @synthesize absorbConditionalErrorsOnRetry = _absorbConditionalErrorsOnRetry; +@synthesize singleBlobUploadThreshold = _singleBlobUploadThreshold; -(instancetype)init { @@ -52,6 +54,8 @@ -(instancetype)init _parallelismFactorSet = NO; _absorbConditionalErrorsOnRetry = NO; _absorbConditionalErrorsOnRetrySet = NO; + _singleBlobUploadThreshold = 4*1024*1024; + _singleBlobUploadThresholdSet = NO; } return self; @@ -95,6 +99,11 @@ -(instancetype)applyDefaultsFromOptions:(AZSBlobRequestOptions *)sourceOptions { self.absorbConditionalErrorsOnRetry = sourceOptions.absorbConditionalErrorsOnRetry; } + + if (sourceOptions->_singleBlobUploadThresholdSet) + { + self.singleBlobUploadThreshold = sourceOptions.singleBlobUploadThreshold; + } } return self; @@ -168,4 +177,15 @@ -(void)setAbsorbConditionalErrorsOnRetry:(BOOL)absorbConditionalErrorsOnRetry _absorbConditionalErrorsOnRetrySet = YES; } +-(NSUInteger)singleBlobUploadThreshold +{ + return _singleBlobUploadThreshold; +} + +-(void)setSingleBlobUploadThreshold:(NSUInteger)singleBlobUploadThreshold +{ + _singleBlobUploadThreshold = singleBlobUploadThreshold; + _singleBlobUploadThresholdSet = YES; +} + @end diff --git a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSBlobUploadHelper.h b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSBlobUploadHelper.h index 5cb95b3..d68a584 100644 --- a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSBlobUploadHelper.h +++ b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSBlobUploadHelper.h @@ -38,9 +38,9 @@ AZS_ASSUME_NONNULL_BEGIN -(instancetype)initToBlockBlob:(AZSCloudBlockBlob *)blockBlob accessCondition:(AZSNullable AZSAccessCondition *)accessCondition requestOptions:(AZSNullable AZSBlobRequestOptions *)requestOptions operationContext:(AZSNullable AZSOperationContext *)operationContext completionHandler:(void (^ __AZSNullable)(NSError* __AZSNullable))completionHandler AZS_DESIGNATED_INITIALIZER; -(instancetype)initToPageBlob:(AZSCloudPageBlob *)pageBlob totalBlobSize:(AZSNullable NSNumber *)totalBlobSize initialSequenceNumber:(AZSNullable NSNumber *)initialSequenceNumber accessCondition:(AZSAccessCondition *)accessCondition requestOptions:(AZSBlobRequestOptions *)requestOptions operationContext:(AZSOperationContext *)operationContext completionHandler:(void (^ __AZSNullable)(NSError * __AZSNullable))completionHandler AZS_DESIGNATED_INITIALIZER; -(instancetype)initToAppendBlob:(AZSCloudAppendBlob *)appendBlob createNew:(BOOL)createNew accessCondition:(AZSNullable AZSAccessCondition *)accessCondition requestOptions:(AZSNullable AZSBlobRequestOptions *)requestOptions operationContext:(AZSNullable AZSOperationContext *)operationContext completionHandler:(void (^ __AZSNullable)(NSError* __AZSNullable))completionHandler AZS_DESIGNATED_INITIALIZER; --(NSInteger)write:(const uint8_t *)buffer maxLength:(NSUInteger)length completionHandler:(void(^)())completionHandler; +-(NSInteger)write:(const uint8_t *)buffer length:(NSUInteger)length completionHandler:(void(^)())completionHandler; -(void)openWithCompletionHandler:(void(^)(BOOL))completionHandler; --(BOOL)closeWithCompletionHandler:(void(^)())completionHandler; +-(BOOL)closeWithCompletionHandler:(void(^)())completionHandler uploadBufferCompletionHandler:(void (^)())uploadBufferCompletionHandler; -(BOOL)hasSpaceAvailable; -(BOOL)allDataUploaded; -(void)stream:(NSStream *)stream handleEvent:(NSStreamEvent)eventCode; diff --git a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSBlobUploadHelper.m b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSBlobUploadHelper.m index 4441e97..c7b65ee 100644 --- a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSBlobUploadHelper.m +++ b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSBlobUploadHelper.m @@ -35,7 +35,6 @@ @interface AZSBlobUploadHelper() @property (strong) AZSCloudBlob *underlyingBlob; @property (strong) NSMutableData *dataBuffer; -@property dispatch_semaphore_t blockUploadSemaphore; @property (strong) NSMutableArray *blockIDs; @property NSUInteger chunksTotal; @property NSUInteger chunksUploaded; @@ -49,6 +48,8 @@ @interface AZSBlobUploadHelper() @property BOOL createNew; @property NSNumber *totalPageBlobSize; @property NSNumber *initialPageBlobSequenceNumber; +@property (strong) NSCondition *maxUploadsCondition; +@property BOOL closeCalled; @end @@ -75,7 +76,7 @@ -(instancetype)initToBlockBlob:(AZSCloudBlockBlob *)blockBlob accessCondition:(A _dataBuffer = [NSMutableData dataWithCapacity:AZSCMaxBlockSize]; //TODO: This should be user-settable. _blockIDs = [NSMutableArray arrayWithCapacity:10]; _maxOpenUploads = requestOptions.parallelismFactor; - _blockUploadSemaphore = dispatch_semaphore_create(self.maxOpenUploads); + _maxUploadsCondition = [[NSCondition alloc] init]; _streamWaiting = NO; _uploadLock = [[NSObject alloc] init]; _accessCondition = accessCondition ?: [[AZSAccessCondition alloc] init]; @@ -88,6 +89,7 @@ -(instancetype)initToBlockBlob:(AZSCloudBlockBlob *)blockBlob accessCondition:(A } _streamingError = nil; _createNew = NO; + _closeCalled = NO; } return self; } @@ -101,7 +103,7 @@ -(instancetype)initToPageBlob:(AZSCloudPageBlob *)pageBlob totalBlobSize:(NSNumb _blobType = AZSBlobTypePageBlob; _dataBuffer = [NSMutableData dataWithCapacity:AZSCMaxBlockSize]; //TODO: This should be user-settable. _maxOpenUploads = requestOptions.parallelismFactor; - _blockUploadSemaphore = dispatch_semaphore_create(self.maxOpenUploads); + _maxUploadsCondition = [[NSCondition alloc] init]; _streamWaiting = NO; _uploadLock = [[NSObject alloc] init]; _accessCondition = accessCondition ?: [[AZSAccessCondition alloc] init]; @@ -113,12 +115,14 @@ -(instancetype)initToPageBlob:(AZSCloudPageBlob *)pageBlob totalBlobSize:(NSNumb CC_MD5_Init(&_md5Context); } _streamingError = nil; + _createNew = NO; if (totalBlobSize) { _createNew = YES; _totalPageBlobSize = totalBlobSize; _initialPageBlobSequenceNumber = initialSequenceNumber; } + _closeCalled = NO; } return self; } @@ -132,7 +136,7 @@ -(instancetype)initToAppendBlob:(AZSCloudAppendBlob *)appendBlob createNew:(BOOL _blobType = AZSBlobTypeAppendBlob; _dataBuffer = [NSMutableData dataWithCapacity:AZSCMaxBlockSize]; //TODO: This should be the user-settable. _maxOpenUploads = 1; //TODO: Investigate if this should always be 1, or if we should use the value in requestOptions.parallelismFactor. - _blockUploadSemaphore = dispatch_semaphore_create(self.maxOpenUploads); + _maxUploadsCondition = [[NSCondition alloc] init]; _streamWaiting = NO; _uploadLock = [[NSObject alloc] init]; _accessCondition = accessCondition ?: [[AZSAccessCondition alloc] init]; @@ -145,6 +149,7 @@ -(instancetype)initToAppendBlob:(AZSCloudAppendBlob *)appendBlob createNew:(BOOL } _streamingError = nil; _createNew = createNew; + _closeCalled = NO; } return self; } @@ -169,42 +174,50 @@ -(BOOL)hasSpaceAvailable } } --(NSInteger)write:(const uint8_t *)buffer maxLength:(NSUInteger)maxLength completionHandler:(void(^)())completionHandler +-(NSInteger)write:(const uint8_t *)buffer length:(NSUInteger)length completionHandler:(void(^)())completionHandler { - if (self.streamingError) - { + if (self.streamingError) { return -1; } + if (!self.hasSpaceAvailable || self.closeCalled) { + return 0; + } + // TODO: Make this configurable. NSUInteger maxSizePerBlock = AZSCMaxBlockSize; int bytesCopied = 0; - while (bytesCopied < maxLength) - { - NSUInteger bytesToAppend = MIN(maxLength - bytesCopied, maxSizePerBlock - [self.dataBuffer length]); + // TODO: If length > MAX_INT32 this is an infinite loop (because length is unsigned) + while (bytesCopied < length) { + NSUInteger bytesToAppend = MIN(length - bytesCopied, maxSizePerBlock - [self.dataBuffer length]); [self.dataBuffer appendBytes:(buffer + bytesCopied) length:bytesToAppend]; bytesCopied += bytesToAppend; - if (maxSizePerBlock == [self.dataBuffer length]) - { + if (maxSizePerBlock == [self.dataBuffer length]) { [self uploadBufferWithCompletionHandler:completionHandler]; } } - return maxLength; + return length; } // TODO: Consider having this method fail, rather than block, in the cannot-acquire-semaphore case. -(BOOL) uploadBufferWithCompletionHandler:(void(^)())completionHandler { [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"Uploading buffer, buffer size = %ld", (unsigned long)[self.dataBuffer length]]; - dispatch_semaphore_wait(self.blockUploadSemaphore, DISPATCH_TIME_FOREVER); - @synchronized(self) - { - self.chunksTotal++; + + [self.maxUploadsCondition lock]; + while (self.chunksTotal - self.chunksUploaded >= self.maxOpenUploads) { + [self.maxUploadsCondition unlock]; + // Spin until there's an available slot to upload. + [[NSRunLoop currentRunLoop] runUntilDate:[NSDate dateWithTimeIntervalSinceNow:0.1]]; + [self.maxUploadsCondition lock]; } + self.chunksTotal++; + [self.maxUploadsCondition unlock]; + NSData *blockData = self.dataBuffer; self.dataBuffer = [NSMutableData dataWithCapacity:AZSCMaxBlockSize]; @@ -227,36 +240,37 @@ -(BOOL) uploadBufferWithCompletionHandler:(void(^)())completionHandler { self.streamingError = error; } - @synchronized(self) - { - self.chunksUploaded++; - } - dispatch_semaphore_signal(self.blockUploadSemaphore); + [self.maxUploadsCondition lock]; + self.chunksUploaded++; + [self.maxUploadsCondition unlock]; completionHandler(); }]; break; } case AZSBlobTypePageBlob: { - NSUInteger currentOffset = 0; - @synchronized(self) { - currentOffset = self.blobOffset; - self.blobOffset = self.blobOffset + [blockData length]; + if ([blockData length] % AZSCPageSize != 0) { + self.streamingError = [NSError errorWithDomain:AZSErrorDomain code:AZSEInvalidArgument userInfo:@{NSLocalizedDescriptionKey:@"Page blob uploads must be 512-byte aligned."}]; } - - AZSCloudPageBlob *blob = (AZSCloudPageBlob *)self.underlyingBlob; - [blob uploadPagesWithData:blockData startOffset:[NSNumber numberWithUnsignedInteger:currentOffset] contentMD5:nil accessCondition:self.accessCondition requestOptions:self.requestOptions operationContext:self.operationContext completionHandler:^(NSError * _Nullable error) { - if (error) - { - self.streamingError = error; + else { + NSUInteger currentOffset = 0; + @synchronized(self) { + currentOffset = self.blobOffset; + self.blobOffset = self.blobOffset + [blockData length]; } - @synchronized(self) - { + + AZSCloudPageBlob *blob = (AZSCloudPageBlob *)self.underlyingBlob; + [blob uploadPagesWithData:blockData startOffset:[NSNumber numberWithUnsignedInteger:currentOffset] contentMD5:nil accessCondition:self.accessCondition requestOptions:self.requestOptions operationContext:self.operationContext completionHandler:^(NSError * _Nullable error) { + if (error) + { + self.streamingError = error; + } + [self.maxUploadsCondition lock]; self.chunksUploaded++; - } - dispatch_semaphore_signal(self.blockUploadSemaphore); - completionHandler(); - }]; + [self.maxUploadsCondition unlock]; + completionHandler(); + }]; + } break; } case AZSBlobTypeAppendBlob: @@ -272,7 +286,6 @@ -(BOOL) uploadBufferWithCompletionHandler:(void(^)())completionHandler { // TODO: improve this error self.streamingError = [NSError errorWithDomain:AZSErrorDomain code:AZSEOutputStreamError userInfo:nil]; - dispatch_semaphore_signal(self.blockUploadSemaphore); completionHandler(); } @@ -287,7 +300,7 @@ -(BOOL) uploadBufferWithCompletionHandler:(void(^)())completionHandler [blob appendBlockWithData:blockData contentMD5:nil accessCondition:self.accessCondition requestOptions:self.requestOptions operationContext:self.operationContext completionHandler:^(NSError * _Nullable error, NSNumber * _Nonnull appendOffset) { if (error) { - // check for stuff + // check for stuff TOASK clarification? if (self.requestOptions.absorbConditionalErrorsOnRetry && (error.userInfo[AZSCHttpStatusCode] == [NSNumber numberWithInt:412]) && ([error.userInfo[AZSCXmlCode] isEqualToString:@"AppendPositionConditionNotMet"] || [error.userInfo[AZSCXmlCode] isEqualToString:@"MaxBlobSizeConditionNotMet"]) && (self.operationContext.requestResults.count - currentResultsCount > 1)) { @@ -298,11 +311,9 @@ -(BOOL) uploadBufferWithCompletionHandler:(void(^)())completionHandler self.streamingError = error; } } - @synchronized(self) - { - self.chunksUploaded++; - } - dispatch_semaphore_signal(self.blockUploadSemaphore); + [self.maxUploadsCondition lock]; + self.chunksUploaded++; + [self.maxUploadsCondition unlock]; completionHandler(); }]; } @@ -320,40 +331,70 @@ -(BOOL) uploadBufferWithCompletionHandler:(void(^)())completionHandler -(BOOL)allDataUploaded { BOOL allDataUploaded = NO; - @synchronized(self) - { - allDataUploaded = self.chunksTotal == self.chunksUploaded; - } + [self.maxUploadsCondition lock]; + allDataUploaded = self.chunksTotal == self.chunksUploaded; + [self.maxUploadsCondition unlock]; return allDataUploaded; } --(BOOL)closeWithCompletionHandler:(void (^)())completionHandler +-(BOOL)closeWithCompletionHandler:(void (^)())completionHandler uploadBufferCompletionHandler:(void (^)())uploadBufferCompletionHandler { - // TODO: If there have been no put block calls yet, just call put blob, rather than put block / put block list. + self.closeCalled = YES; if ((!self.streamingError) && (self.dataBuffer.length > 0)) { - if (![self uploadBufferWithCompletionHandler:completionHandler]) + [self.maxUploadsCondition lock]; + NSInteger chunksTotal = self.chunksTotal; + [self.maxUploadsCondition unlock]; + + if ((self.blobType == AZSBlobTypeBlockBlob) && (chunksTotal == 0) && (self.dataBuffer.length <= self.requestOptions.singleBlobUploadThreshold)) { - return NO; + AZSCloudBlockBlob *blockBlob = (AZSCloudBlockBlob *)self.underlyingBlob; + [self.maxUploadsCondition lock]; + self.chunksTotal = 1; + [self.maxUploadsCondition unlock]; + // uploadFromData calls put blob internally if the data buffer is less than the single blob upload threshold. + [blockBlob uploadFromData:self.dataBuffer accessCondition:self.accessCondition requestOptions:self.requestOptions operationContext:self.operationContext completionHandler:^(NSError * _Nullable error) { + if (error) + { + self.streamingError = error; + } + [self.maxUploadsCondition lock]; + self.chunksUploaded = 1; + [self.maxUploadsCondition unlock]; + uploadBufferCompletionHandler(); + completionHandler(self.streamingError); + }]; + return YES; + } + else + { + if (![self uploadBufferWithCompletionHandler:uploadBufferCompletionHandler]) + { + return NO; + } } } - BOOL finished = NO; - @synchronized(self) + // If there's an error, abort. TOASK This needs to be checked again after the loop confirming that all blocks are updated exits. + if (self.streamingError) { - finished = self.chunksTotal == self.chunksUploaded; + completionHandler(self.streamingError); + return NO; } + BOOL finished = NO; + [self.maxUploadsCondition lock]; + finished = self.chunksTotal == self.chunksUploaded; while (!finished) { // Spin until all blocks have been uploaded. + [self.maxUploadsCondition unlock]; [[NSRunLoop currentRunLoop] runUntilDate:[NSDate dateWithTimeIntervalSinceNow:0.1]]; - @synchronized(self) - { - finished = self.chunksTotal == self.chunksUploaded; - } + [self.maxUploadsCondition lock]; + finished = self.chunksTotal == self.chunksUploaded; } - + [self.maxUploadsCondition unlock]; + if (self.requestOptions.storeBlobContentMD5) { unsigned char md5Bytes[CC_MD5_DIGEST_LENGTH]; @@ -364,37 +405,31 @@ -(BOOL)closeWithCompletionHandler:(void (^)())completionHandler switch (self.blobType) { case AZSBlobTypeBlockBlob: { - dispatch_semaphore_t blockListSemaphore = dispatch_semaphore_create(0); - AZSCloudBlockBlob *blob = (AZSCloudBlockBlob *)self.underlyingBlob; [blob uploadBlockListFromArray:self.blockIDs accessCondition:self.accessCondition requestOptions:self.requestOptions operationContext:self.operationContext completionHandler:^(NSError * error) { if (!self.streamingError && error) { self.streamingError = error; } - - dispatch_semaphore_signal(blockListSemaphore); + completionHandler(self.streamingError); }]; - - dispatch_semaphore_wait(blockListSemaphore, DISPATCH_TIME_FOREVER); break; } case AZSBlobTypePageBlob: { if (self.requestOptions.storeBlobContentMD5) { - dispatch_semaphore_t setPropertiesSemaphore = dispatch_semaphore_create(0); - [self.underlyingBlob uploadPropertiesWithAccessCondition:self.accessCondition requestOptions:self.requestOptions operationContext:self.operationContext completionHandler:^(NSError * _Nullable error) { if (!self.streamingError && error) { self.streamingError = error; } - - dispatch_semaphore_signal(setPropertiesSemaphore); + completionHandler(self.streamingError); }]; - - dispatch_semaphore_wait(setPropertiesSemaphore, DISPATCH_TIME_FOREVER); + } + else + { + completionHandler(self.streamingError); } break; } @@ -402,19 +437,19 @@ -(BOOL)closeWithCompletionHandler:(void (^)())completionHandler { if (self.requestOptions.storeBlobContentMD5) { - dispatch_semaphore_t setPropertiesSemaphore = dispatch_semaphore_create(0); - [self.underlyingBlob uploadPropertiesWithAccessCondition:self.accessCondition requestOptions:self.requestOptions operationContext:self.operationContext completionHandler:^(NSError * _Nullable error) { if (!self.streamingError && error) { self.streamingError = error; } - - dispatch_semaphore_signal(setPropertiesSemaphore); + completionHandler(self.streamingError); }]; - - dispatch_semaphore_wait(setPropertiesSemaphore, DISPATCH_TIME_FOREVER); - } break; + } + else + { + completionHandler(self.streamingError); + } + break; } default: @@ -436,7 +471,7 @@ -(void)writeFromStreamCallbackWithStream:(NSInputStream *)inputStream; len = [inputStream read:buf maxLength:AZSCKilobyte]; if (len) { - [self write:buf maxLength:len completionHandler:^{ + [self write:buf length:len completionHandler:^{ [self writeFromStreamCallbackWithStream:inputStream]; }]; } @@ -448,8 +483,10 @@ -(void)writeFromStreamCallbackWithStream:(NSInputStream *)inputStream; - (void)stream:(NSStream *)stream handleEvent:(NSStreamEvent)eventCode { NSInputStream *inputStream = (NSInputStream *)stream; - switch (eventCode) { + switch (eventCode) + { case NSStreamEventHasBytesAvailable: + { // TODO: Stop reading if there was a error in uploading the blob? Not sure if this is possible. @synchronized(self.uploadLock) { @@ -461,7 +498,7 @@ - (void)stream:(NSStream *)stream handleEvent:(NSStreamEvent)eventCode len = [inputStream read:buf maxLength:AZSCKilobyte]; // The 0 and -1 case should be handled by the EndEncountered and ErrorOccurred events. if (len > 0) { - [self write:buf maxLength:len completionHandler:^{ + [self write:buf length:len completionHandler:^{ [self writeFromStreamCallbackWithStream:inputStream]; }]; } @@ -472,30 +509,32 @@ - (void)stream:(NSStream *)stream handleEvent:(NSStreamEvent)eventCode } } break; + } case NSStreamEventEndEncountered: - // Note that the below method is syncronous for the time being. + { [self closeWithCompletionHandler:^{ + if (self.completionHandler) + { + self.completionHandler(self.streamingError); + } + } uploadBufferCompletionHandler:^{ ; }]; - if (self.completionHandler) - { - self.completionHandler(self.streamingError); - } break; + } case NSStreamEventErrorOccurred: { NSError *error = inputStream.streamError; [self.operationContext logAtLevel:AZSLogLevelError withMessage:@"Error in stream callback. Error code = %ld, error domain = %@, error userinfo = %@", (long)error.code, error.domain, error.userInfo]; - // Note that the below method is syncronous for the time being. [self closeWithCompletionHandler:^{ + if (self.completionHandler) + { + self.completionHandler(error); + } + } uploadBufferCompletionHandler:^{ ; }]; - - if (self.completionHandler) - { - self.completionHandler(error); - } break; } default: @@ -554,4 +593,4 @@ -(void)openWithCompletionHandler:(void (^)(BOOL))completionHandler } } -@end \ No newline at end of file +@end diff --git a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSCloudAppendBlob.m b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSCloudAppendBlob.m index a947fef..1ef0342 100644 --- a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSCloudAppendBlob.m +++ b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSCloudAppendBlob.m @@ -210,16 +210,24 @@ -(void)runBlobUploadFromStreamWithContainer:(AZSAppendBlobUploadFromStreamInputC { @autoreleasepool { NSRunLoop *runLoopForUpload = [NSRunLoop currentRunLoop]; - BOOL __block blobFinished = NO; + __block BOOL blobFinished = NO; + __block NSError *error; // The blob will already exist in this case. - AZSBlobUploadHelper *blobUploadHelper = [[AZSBlobUploadHelper alloc] initToAppendBlob:inputContainer.targetBlob createNew:NO accessCondition:inputContainer.accessCondition requestOptions:inputContainer.blobRequestOptions operationContext:inputContainer.operationContext completionHandler:^(NSError * error) { + AZSBlobUploadHelper *blobUploadHelper = [[AZSBlobUploadHelper alloc] initToAppendBlob:inputContainer.targetBlob createNew:NO accessCondition:inputContainer.accessCondition requestOptions:inputContainer.blobRequestOptions operationContext:inputContainer.operationContext completionHandler:^(NSError * err) { + error = err; + blobFinished = YES; [inputContainer.sourceStream close]; [inputContainer.sourceStream removeFromRunLoop:runLoopForUpload forMode:NSDefaultRunLoopMode]; - blobFinished = YES; if (inputContainer.completionHandler) { - inputContainer.completionHandler(error); + // We want to do this on the global async queue because the current thread was created to perform the 'upload from + // stream' call only, and is not being managed. This is needed to spin the runloop, but is not ideal for the resulting + // user callback. + dispatch_async(dispatch_get_global_queue( DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^(void) + { + inputContainer.completionHandler(error); + }); } }]; diff --git a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSCloudBlob.h b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSCloudBlob.h index 77ad512..ec08ee4 100644 --- a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSCloudBlob.h +++ b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSCloudBlob.h @@ -24,8 +24,11 @@ AZS_ASSUME_NONNULL_BEGIN @class AZSCloudBlobContainer; @class AZSAccessCondition; +@class AZSBlobInputStream; @class AZSBlobRequestOptions; +@class AZSStreamDownloadBuffer; @class AZSOperationContext; +@class AZSStorageCommand; @class AZSStorageUri; @class AZSCloudBlobClient; @class AZSCopyState; @@ -55,6 +58,7 @@ AZS_ASSUME_NONNULL_BEGIN /** The snapshot time of this snapshot of the blob. If nil, this AZSCloudBlob object does not represent a snapshot.*/ @property (copy, AZSNullable) NSString *snapshotTime; +// TODO: Remove? //@property (nonatomic, strong) AZSCloudBlobDirectory *parent; /** The metadata on this blob.*/ @@ -460,6 +464,7 @@ AZS_ASSUME_NONNULL_BEGIN - (void)renewLeaseWithAccessCondition:(AZSNullable AZSAccessCondition *)accessCondition requestOptions:(AZSNullable AZSBlobRequestOptions *)requestOptions operationContext:(AZSNullable AZSOperationContext *)operationContext completionHandler:(void (^)(NSError* __AZSNullable))completionHandler; +(void)updateEtagAndLastModifiedWithResponse:(NSHTTPURLResponse *)response properties:(AZSBlobProperties *)properties updateLength:(BOOL)updateLength; +-(AZSStorageCommand *)downloadCommandWithRange:(AZSULLRange)range AccessCondition:(AZSAccessCondition *)accessCondition requestOptions:(AZSBlobRequestOptions *)requestOptions operationContext:(AZSOperationContext *)operationContext; /** Downloads contents of a blob to an NSData object. @@ -655,6 +660,29 @@ AZS_ASSUME_NONNULL_BEGIN */ -(void)abortAsyncCopyWithCopyId:(NSString *)copyId accessCondition:(AZSNullable AZSAccessCondition *)accessCondition requestOptions:(AZSNullable AZSBlobRequestOptions *)requestOptions operationContext:(AZSNullable AZSOperationContext *)operationContext completionHandler:(void (^)(NSError * __AZSNullable))completionHandler; + +/** Creates an input stream that is capable of reading from the blob. + + This method returns an instance of AZSBlobInputStream. The caller can then assign a delegate and schedule the stream in a runloop + (similar to any other NSInputStream.) See AZSBlobInputStream documentation for details. + + @returns The created AZSBlobInputStream, capable of reading from this blob. + */ +- (AZSBlobInputStream *)createInputStream; + +/** Creates an input stream that is capable of reading from the blob. + + This method returns an instance of AZSBlobInputStream. The caller can then assign a delegate and schedule the stream in a runloop + (similar to any other NSInputStream.) See AZSBlobInputStream documentation for details. + + @param accessCondition The access condition for the request. + @param requestOptions The options to use for the request. + @param operationContext The operation context to use for the call. + + @returns The created AZSBlobInputStream, capable of reading from this blob. + */ +- (AZSBlobInputStream *)createInputStreamWithAccessCondition:(AZSNullable AZSAccessCondition *)accessCondition requestOptions:(AZSNullable AZSBlobRequestOptions *)requestOptions operationContext:(AZSNullable AZSOperationContext *)operationContext; + // TODO: Delete If Exists... somehow we missed that one. @end diff --git a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSCloudBlob.m b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSCloudBlob.m index 6b1cdbd..9a0e548 100644 --- a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSCloudBlob.m +++ b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSCloudBlob.m @@ -38,6 +38,7 @@ #import "AZSSharedAccessSignatureHelper.h" #import "AZSStorageCredentials.h" #import "AZSBlobResponseParser.h" +#import "AZSBlobInputStream.h" @interface AZSCloudBlob() @@ -145,77 +146,17 @@ -(void)downloadToStream:(NSOutputStream *)targetStream AZSULLrange:(AZSULLRange) { operationContext = [[AZSOperationContext alloc] init]; } - AZSBlobRequestOptions *modifiedOptions = [[AZSBlobRequestOptions copyOptions:requestOptions] applyDefaultsFromOptions:self.client.defaultRequestOptions]; - AZSStorageCommand * command = [[AZSStorageCommand alloc] initWithStorageCredentials:self.client.credentials storageUri:self.storageUri calculateResponseMD5:!(modifiedOptions.disableContentMD5Validation) operationContext:operationContext]; - command.allowedStorageLocation = AZSAllowedStorageLocationPrimaryOrSecondary; - [command setBuildRequest:^ NSMutableURLRequest * (NSURLComponents *urlComponents, NSTimeInterval timeout, AZSOperationContext *operationContext) - { - return [AZSBlobRequestFactory getBlobWithSnapshotTime:self.snapshotTime range:range getRangeContentMD5:modifiedOptions.useTransactionalMD5 accessCondition:accessCondition urlComponents:urlComponents timeout:timeout operationContext:operationContext]; - }]; - - [command setAuthenticationHandler:self.client.authenticationHandler]; - - __block NSString *desiredContentMD5 = nil; - - [command setPreProcessResponse:^id(NSHTTPURLResponse * urlResponse, AZSRequestResult * requestResult, AZSOperationContext * operationContext) { - NSError *error = [AZSResponseParser preprocessResponseWithResponse:urlResponse requestResult:requestResult operationContext:operationContext]; - if (error) - { - return error; - } - - AZSBlobProperties *parsedProperties = [AZSBlobResponseParser getBlobPropertiesWithResponse:urlResponse operationContext:operationContext error:&error]; - if (error) - { - return error; - } - - if (parsedProperties.blobType != AZSBlobTypeUnspecified && parsedProperties.blobType != self.properties.blobType) - { - return error = [NSError errorWithDomain:AZSErrorDomain code:AZSEInvalidArgument userInfo:@{NSLocalizedDescriptionKey:@"Blob type on the local object does not match blob type on the service."}]; - } - - if (modifiedOptions.useTransactionalMD5 && !modifiedOptions.disableContentMD5Validation && !parsedProperties.contentMD5) - { - NSMutableDictionary *userInfo = [NSMutableDictionary dictionary]; - NSError *storageError = [NSError errorWithDomain:AZSErrorDomain code:AZSEMD5Mismatch userInfo:userInfo]; - return storageError; - } - - desiredContentMD5 = parsedProperties.contentMD5; - - if (range.length > 0) - { - // If it's a range get, don't update the contentMD5 on the blob's properties. - parsedProperties.contentMD5 = self.properties.contentMD5; - } - - self.properties = parsedProperties; - self.blobCopyState = [AZSBlobResponseParser getCopyStateWithResponse:urlResponse]; - self.metadata = [AZSBlobResponseParser getMetadataWithResponse:urlResponse]; - - return nil; - }]; + AZSBlobRequestOptions *modifiedOptions = [[AZSBlobRequestOptions copyOptions:requestOptions] applyDefaultsFromOptions:self.client.defaultRequestOptions]; + AZSStorageCommand *command = [self downloadCommandWithRange:range AccessCondition:accessCondition requestOptions:modifiedOptions operationContext:operationContext]; [command setDestinationStream:targetStream]; - [command setPostProcessResponse:^id(NSHTTPURLResponse *response, AZSRequestResult *requestResult, NSOutputStream *outputStream, AZSOperationContext *operationContext, NSError **error) { - if (desiredContentMD5 && !modifiedOptions.disableContentMD5Validation) - { - if ([desiredContentMD5 compare:requestResult.calculatedResponseMD5 options:NSLiteralSearch] != NSOrderedSame) - { - *error = [NSError errorWithDomain:AZSErrorDomain code:AZSEMD5Mismatch userInfo:nil]; - } - } - return nil; - }]; - - [AZSExecutor ExecuteWithStorageCommand:command requestOptions:modifiedOptions operationContext:operationContext completionHandler:^(NSError *error, id result) + [AZSExecutor ExecuteWithStorageCommand:command requestOptions:modifiedOptions operationContext:operationContext downloadBuffer:nil completionHandler:^(NSError *error, id result) { completionHandler(error); }]; + return; - } -(void)deleteWithCompletionHandler:(void (^)(NSError*))completionHandler @@ -782,6 +723,73 @@ +(void)updateEtagAndLastModifiedWithResponse:(NSHTTPURLResponse *)response prope } } +-(AZSStorageCommand *)downloadCommandWithRange:(AZSULLRange)range AccessCondition:(AZSAccessCondition *)accessCondition requestOptions:(AZSBlobRequestOptions *)requestOptions operationContext:(AZSOperationContext *)operationContext +{ + AZSStorageCommand *command = [[AZSStorageCommand alloc] initWithStorageCredentials:self.client.credentials storageUri:self.storageUri calculateResponseMD5:!(requestOptions.disableContentMD5Validation) operationContext:operationContext]; + command.allowedStorageLocation = AZSAllowedStorageLocationPrimaryOrSecondary; + [command setBuildRequest:^ NSMutableURLRequest * (NSURLComponents *urlComponents, NSTimeInterval timeout, AZSOperationContext *operationContext) + { + return [AZSBlobRequestFactory getBlobWithSnapshotTime:self.snapshotTime range:range getRangeContentMD5:requestOptions.useTransactionalMD5 accessCondition:accessCondition urlComponents:urlComponents timeout:timeout operationContext:operationContext]; + }]; + + [command setAuthenticationHandler:self.client.authenticationHandler]; + + __block NSString *desiredContentMD5 = nil; + + [command setPreProcessResponse:^id(NSHTTPURLResponse * urlResponse, AZSRequestResult * requestResult, AZSOperationContext * operationContext) { + NSError *error = [AZSResponseParser preprocessResponseWithResponse:urlResponse requestResult:requestResult operationContext:operationContext]; + if (error) + { + return error; + } + + AZSBlobProperties *parsedProperties = [AZSBlobResponseParser getBlobPropertiesWithResponse:urlResponse operationContext:operationContext error:&error]; + if (error) + { + return error; + } + + if (parsedProperties.blobType != AZSBlobTypeUnspecified && parsedProperties.blobType != self.properties.blobType) + { + return error = [NSError errorWithDomain:AZSErrorDomain code:AZSEInvalidArgument userInfo:@{NSLocalizedDescriptionKey:@"Blob type on the local object does not match blob type on the service."}]; + } + + if (requestOptions.useTransactionalMD5 && !requestOptions.disableContentMD5Validation && !parsedProperties.contentMD5) + { + NSMutableDictionary *userInfo = [NSMutableDictionary dictionary]; + NSError *storageError = [NSError errorWithDomain:AZSErrorDomain code:AZSEMD5Mismatch userInfo:userInfo]; + return storageError; + } + + desiredContentMD5 = parsedProperties.contentMD5; + + if (range.length > 0) + { + // If it's a range get, don't update the contentMD5 on the blob's properties. + parsedProperties.contentMD5 = self.properties.contentMD5; + } + + self.properties = parsedProperties; + self.blobCopyState = [AZSBlobResponseParser getCopyStateWithResponse:urlResponse]; + self.metadata = [AZSBlobResponseParser getMetadataWithResponse:urlResponse]; + + return nil; + }]; + + [command setPostProcessResponse:^id(NSHTTPURLResponse *response, AZSRequestResult *requestResult, NSOutputStream *outputStream, AZSOperationContext *operationContext, NSError **error) { + if (desiredContentMD5 && !requestOptions.disableContentMD5Validation) + { + if ([desiredContentMD5 compare:requestResult.calculatedResponseMD5 options:NSLiteralSearch] != NSOrderedSame) + { + *error = [NSError errorWithDomain:AZSErrorDomain code:AZSEMD5Mismatch userInfo:nil]; + } + } + return nil; + }]; + + return command; +} + -(void)downloadToDataWithCompletionHandler:(void (^)(NSError *, NSData *))completionHandler { [self downloadToDataWithAccessCondition:nil requestOptions:nil operationContext:nil completionHandler:completionHandler]; @@ -939,4 +947,21 @@ -(void)abortAsyncCopyWithCopyId:(NSString *)copyId accessCondition:(AZSAccessCon return; } +- (AZSBlobInputStream *)createInputStream +{ + return [self createInputStreamWithAccessCondition:nil requestOptions:nil operationContext:nil]; +} + +- (AZSBlobInputStream *)createInputStreamWithAccessCondition:(AZSNullable AZSAccessCondition *)accessCondition requestOptions:(AZSNullable AZSBlobRequestOptions *)requestOptions operationContext:(AZSNullable AZSOperationContext *)operationContext +{ + if (!operationContext) + { + operationContext = [[AZSOperationContext alloc] init]; + } + + // TODO: Check access conditions properly (download attributes if necessary.). Also for UploadFromStream. + AZSBlobRequestOptions *modifiedOptions = [[AZSBlobRequestOptions copyOptions:requestOptions] applyDefaultsFromOptions:self.client.defaultRequestOptions]; + return [[AZSBlobInputStream alloc] initWithBlob:self accessCondition:(AZSAccessCondition *)accessCondition requestOptions:modifiedOptions operationContext:operationContext]; +} + @end \ No newline at end of file diff --git a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSCloudBlockBlob.m b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSCloudBlockBlob.m index b19b1f2..44867f8 100644 --- a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSCloudBlockBlob.m +++ b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSCloudBlockBlob.m @@ -15,6 +15,7 @@ // // ----------------------------------------------------------------------------------------- +#import "AZSBlobProperties.h" #import "AZSCloudBlockBlob.h" #import "AZSStorageCommand.h" #import "AZSBlobRequestFactory.h" @@ -24,10 +25,12 @@ #import "AZSBlobRequestOptions.h" #import "AZSBlobRequestXML.h" #import "AZSBlobResponseParser.h" +#import "AZSBlobInputStream.h" #import "AZSBlobOutputStream.h" #import "AZSResponseParser.h" #import "AZSBlobUploadHelper.h" #import "AZSUtil.h" +#import "AZSEnums.h" #import "AZSErrors.h" #import "AZSStorageUri.h" #import "AZSBlobProperties.h" @@ -95,14 +98,24 @@ -(void)runBlobUploadFromStreamWithContainer:(AZSBlobUploadFromStreamInputContain @autoreleasepool { NSRunLoop *runLoopForUpload = [NSRunLoop currentRunLoop]; BOOL __block blobFinished = NO; - AZSBlobUploadHelper *blobUploadHelper = [[AZSBlobUploadHelper alloc] initToBlockBlob:inputContainer.targetBlob accessCondition:inputContainer.accessCondition requestOptions:inputContainer.blobRequestOptions operationContext:inputContainer.operationContext completionHandler:^(NSError * error) { + __block NSError *error; + AZSBlobUploadHelper *blobUploadHelper = [[AZSBlobUploadHelper alloc] initToBlockBlob:inputContainer.targetBlob accessCondition:inputContainer.accessCondition requestOptions:inputContainer.blobRequestOptions operationContext:inputContainer.operationContext completionHandler:^(NSError * err) { + error = err; + blobFinished = YES; [inputContainer.sourceStream close]; [inputContainer.sourceStream removeFromRunLoop:runLoopForUpload forMode:NSDefaultRunLoopMode]; - blobFinished = YES; + if (inputContainer.completionHandler) { - inputContainer.completionHandler(error); + // We want to do this on the global async queue because the current thread was created to perform the 'upload from + // stream' call only, and is not being managed. This is needed to spin the runloop, but is not ideal for the resulting + // user callback. + dispatch_async(dispatch_get_global_queue( DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^(void) + { + inputContainer.completionHandler(error); + }); } + }]; [inputContainer.sourceStream setDelegate:blobUploadHelper]; @@ -120,6 +133,7 @@ -(void)runBlobUploadFromStreamWithContainer:(AZSBlobUploadFromStreamInputContain runLoopSuccess = [runLoopForUpload runMode:NSDefaultRunLoopMode beforeDate:loopUntil]; } } + } } @@ -157,8 +171,16 @@ -(void)uploadFromData:(NSData *)sourceData completionHandler:(void (^)(NSError * -(void)uploadFromData:(NSData *)sourceData accessCondition:(AZSAccessCondition *)accessCondition requestOptions:(AZSBlobRequestOptions *)requestOptions operationContext:(AZSOperationContext *)operationContext completionHandler:(void (^)(NSError *))completionHandler { - NSInputStream *sourceStream = [NSInputStream inputStreamWithData:sourceData]; - [self uploadFromStream:sourceStream accessCondition:accessCondition requestOptions:requestOptions operationContext:operationContext completionHandler:completionHandler]; + AZSBlobRequestOptions *modifiedOptions = [[AZSBlobRequestOptions copyOptions:requestOptions] applyDefaultsFromOptions:self.client.defaultRequestOptions]; + if (sourceData.length <= modifiedOptions.singleBlobUploadThreshold) + { + [self putBlobFromData:sourceData accessCondition:accessCondition requestOptions:modifiedOptions operationContext:operationContext completionHandler:completionHandler]; + } + else + { + NSInputStream *sourceStream = [NSInputStream inputStreamWithData:sourceData]; + [self uploadFromStream:sourceStream accessCondition:accessCondition requestOptions:requestOptions operationContext:operationContext completionHandler:completionHandler]; + } } -(void)uploadFromText:(NSString *)textToUpload completionHandler:(void (^)(NSError *))completionHandler @@ -194,34 +216,23 @@ -(void)uploadFromFileWithURL:(NSURL *)fileURL accessCondition:(AZSAccessConditio [self uploadFromStream:sourceStream accessCondition:accessCondition requestOptions:requestOptions operationContext:operationContext completionHandler:completionHandler]; } -// TODO: Convert the below method into an explicit 'put blob' call. -/* --(void)uploadFromData:(NSData *)sourceData accessCondition:(AZSAccessCondition *)accessCondition requestOptions:(AZSBlobRequestOptions *)requestOptions operationContext:(AZSOperationContext *)operationContext completionHandler:(void (^)(NSError *))completionHandler +-(void)putBlobFromData:(NSData *)sourceData accessCondition:(AZSAccessCondition *)accessCondition requestOptions:(AZSBlobRequestOptions *)requestOptions operationContext:(AZSOperationContext *)operationContext completionHandler:(void (^)(NSError *))completionHandler { - AZSBlobRequestOptions *modifiedOptions = [[AZSBlobRequestOptions copyOptions:requestOptions] applyDefaultsFromOptions:self.client.defaultRequestOptions]; - AZSStorageCommand * command = [[AZSStorageCommand alloc] initWithStorageCredentials:self.client.credentials storageUri:self.storageUri]; + if (!operationContext) + { + operationContext = [[AZSOperationContext alloc] init]; + } + AZSStorageCommand * command = [[AZSStorageCommand alloc] initWithStorageCredentials:self.client.credentials storageUri:self.storageUri operationContext:operationContext]; NSString *contentMD5 = nil; - if (requestOptions.useTransactionalMD5 || requestOptions.storeBlobContentMD5) + if (requestOptions.storeBlobContentMD5) { - unsigned char md5Bytes[CC_MD5_DIGEST_LENGTH]; - CC_MD5(sourceData.bytes, sourceData.length, md5Bytes); - NSString *contentMD5String = [[[NSData alloc] initWithBytes:md5Bytes length:CC_MD5_DIGEST_LENGTH] base64EncodedStringWithOptions:0]; - - if (requestOptions.useTransactionalMD5) - { - contentMD5 = contentMD5String; - } - - if (requestOptions.storeBlobContentMD5) - { - self.properties.contentMD5 = contentMD5String; - } + contentMD5 = [AZSUtil calculateMD5FromData:sourceData]; } [command setBuildRequest:^ NSMutableURLRequest * (NSURLComponents *urlComponents, NSTimeInterval timeout, AZSOperationContext *operationContext) { - return [AZSBlobRequestFactory putBlockBlobWithLength:[sourceData length] blobProperties:self.properties contentMD5:contentMD5 cloudMetadata:self.metadata AccessCondition:accessCondition urlComponents:urlComponents timeout:timeout operationContext:operationContext]; + return [AZSBlobRequestFactory putBlockBlobWithLength:[sourceData length] blobProperties:self.properties contentMD5:contentMD5 cloudMetadata:self.metadata accessCondition:accessCondition urlComponents:urlComponents timeout:timeout operationContext:operationContext]; }]; [command setAuthenticationHandler:self.client.authenticationHandler]; @@ -239,18 +250,13 @@ -(void)uploadFromData:(NSData *)sourceData accessCondition:(AZSAccessCondition * [command setSource:sourceData]; - if (operationContext == nil) - { - operationContext = [[AZSOperationContext alloc] init]; - } - - [AZSExecutor ExecuteWithStorageCommand:command requestOptions:modifiedOptions operationContext:operationContext completionHandler:^(NSError *error, id result) + [AZSExecutor ExecuteWithStorageCommand:command requestOptions:requestOptions operationContext:operationContext completionHandler:^(NSError *error, id result) { completionHandler(error); }]; return; } - */ + -(void)uploadBlockFromData:(NSData *)sourceData blockID:(NSString *)blockID completionHandler:(void (^)(NSError*))completionHandler { @@ -329,7 +335,7 @@ -(void)uploadBlockListFromArray:(NSArray *)blockList accessCondition:(AZSAccessC [command setBuildRequest:^ NSMutableURLRequest * (NSURLComponents *urlComponents, NSTimeInterval timeout, AZSOperationContext *operationContext) { - return [AZSBlobRequestFactory putBlockListWithLength:[sourceData length] blobProperties:self.properties contentMD5:contentMD5 cloudMetadata:self.metadata AccessCondition:accessCondition urlComponents:urlComponents timeout:timeout operationContext:operationContext]; + return [AZSBlobRequestFactory putBlockListWithLength:[sourceData length] blobProperties:self.properties contentMD5:contentMD5 cloudMetadata:self.metadata accessCondition:accessCondition urlComponents:urlComponents timeout:timeout operationContext:operationContext]; }]; [command setAuthenticationHandler:self.client.authenticationHandler]; diff --git a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSCloudPageBlob.m b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSCloudPageBlob.m index 0c3b543..a583b6a 100644 --- a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSCloudPageBlob.m +++ b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSCloudPageBlob.m @@ -314,8 +314,8 @@ -(void)downloadPageRangesWithAZSULLRange:(AZSULLRange)range accessCondition:(AZS return nil; }]; - [command setPostProcessResponse:^id(NSHTTPURLResponse *urlResponse, AZSRequestResult *requestResult, NSOutputStream *outputStream, AZSOperationContext *operationContext, NSError **error) { - NSArray *pageRangesResponse = [AZSGetPageRangesResponse parseGetPageRangesResponseWithData:[outputStream propertyForKey:NSStreamDataWrittenToMemoryStreamKey] operationContext:operationContext error:error]; + [command setPostProcessResponse:^id(NSHTTPURLResponse *urlResponse, AZSRequestResult *requestResult, NSOutputStream *stream, AZSOperationContext *operationContext, NSError **error) { + NSArray *pageRangesResponse = [AZSGetPageRangesResponse parseGetPageRangesResponseWithData:[stream propertyForKey:NSStreamDataWrittenToMemoryStreamKey] operationContext:operationContext error:error]; if (*error) { @@ -485,16 +485,24 @@ -(void)runBlobUploadFromStreamWithContainer:(AZSPageBlobUploadFromStreamInputCon { @autoreleasepool { NSRunLoop *runLoopForUpload = [NSRunLoop currentRunLoop]; - BOOL __block blobFinished = NO; + __block BOOL blobFinished = NO; + __block NSError *error; // The blob will always already be created here. - AZSBlobUploadHelper *blobUploadHelper = [[AZSBlobUploadHelper alloc] initToPageBlob:inputContainer.targetBlob totalBlobSize:nil initialSequenceNumber:nil accessCondition:inputContainer.accessCondition requestOptions:inputContainer.blobRequestOptions operationContext:inputContainer.operationContext completionHandler:^(NSError * error) { + AZSBlobUploadHelper *blobUploadHelper = [[AZSBlobUploadHelper alloc] initToPageBlob:inputContainer.targetBlob totalBlobSize:nil initialSequenceNumber:nil accessCondition:inputContainer.accessCondition requestOptions:inputContainer.blobRequestOptions operationContext:inputContainer.operationContext completionHandler:^(NSError * err) { + error = err; + blobFinished = YES; [inputContainer.sourceStream close]; [inputContainer.sourceStream removeFromRunLoop:runLoopForUpload forMode:NSDefaultRunLoopMode]; - blobFinished = YES; if (inputContainer.completionHandler) { - inputContainer.completionHandler(error); + // We want to do this on the global async queue because the current thread was created to perform the 'upload from + // stream' call only, and is not being managed. This is needed to spin the runloop, but is not ideal for the resulting + // user callback. + dispatch_async(dispatch_get_global_queue( DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^(void) + { + inputContainer.completionHandler(error); + }); } }]; diff --git a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSConstants.h b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSConstants.h index 62d51b6..23525eb 100644 --- a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSConstants.h +++ b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSConstants.h @@ -44,6 +44,7 @@ FOUNDATION_EXPORT NSString *const AZSCBlobPageBlob; FOUNDATION_EXPORT NSInteger const AZSCKilobyte; FOUNDATION_EXPORT NSInteger const AZSCMaxBlockSize; +FOUNDATION_EXPORT NSInteger const AZSCPageSize; FOUNDATION_EXPORT NSInteger const AZSCSnapshotIndex; // Account Settings diff --git a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSConstants.m b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSConstants.m index bec767f..1b42d87 100644 --- a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSConstants.m +++ b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSConstants.m @@ -44,6 +44,7 @@ NSInteger const AZSCKilobyte = 1024; NSInteger const AZSCMaxBlockSize = 4 * AZSCKilobyte * AZSCKilobyte; +NSInteger const AZSCPageSize = 512; NSInteger const AZSCSnapshotIndex = 2; // Account Settings diff --git a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSErrors.h b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSErrors.h index 09d16c2..817d821 100644 --- a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSErrors.h +++ b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSErrors.h @@ -32,5 +32,7 @@ FOUNDATION_EXPORT NSString *const AZSInnerErrorString; #define AZSEXMLCreationError 7 #define AZSEOutputStreamError 8 #define AZSEOutputStreamFull 9 +#define AZSEInputStreamError 10 +#define AZSEInputStreamEmpty 11 #endif //__AZS_ERRORS_DEFINED__ \ No newline at end of file diff --git a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSExecutor.h b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSExecutor.h index bbd3f92..2df8f2b 100644 --- a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSExecutor.h +++ b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSExecutor.h @@ -18,6 +18,7 @@ #import @class AZSStorageCommand; +@class AZSStreamDownloadBuffer; @class AZSRequestOptions; @class AZSOperationContext; @@ -25,5 +26,8 @@ // The executor contains all the business logic of actually making/executing HTTP requests. // Funneling all requests through this one class allows us to implement retry policies, error handling, etc. @interface AZSExecutor : NSObject + +(void)ExecuteWithStorageCommand:(AZSStorageCommand *)storageCommand requestOptions:(AZSRequestOptions *)requestOptions operationContext:(AZSOperationContext *)operationContext completionHandler:(void (^)(NSError*, id))completionHandler; -@end ++(void)ExecuteWithStorageCommand:(AZSStorageCommand *)storageCommand requestOptions:(AZSRequestOptions *)requestOptions operationContext:(AZSOperationContext *)operationContext downloadBuffer:(AZSStreamDownloadBuffer *)downloadBuffer completionHandler:(void (^)(NSError*, id))completionHandler; + +@end \ No newline at end of file diff --git a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSExecutor.m b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSExecutor.m index a9a0216..5b93d42 100644 --- a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSExecutor.m +++ b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSExecutor.m @@ -16,6 +16,7 @@ // ----------------------------------------------------------------------------------------- #import +#import "AZSBlobInputStream.h" #import "AZSConstants.h" #import "AZSExecutor.h" #import "AZSOperationContext.h" @@ -23,6 +24,7 @@ #import "AZSEnums.h" #import "AZSStorageCommand.h" #import "AZSStorageUri.h" +#import "AZSStreamDownloadBuffer.h" #import "AZSRequestResult.h" #import "AZSErrors.h" #import "AZSRetryContext.h" @@ -30,276 +32,6 @@ #import "AZSUtil.h" #import "AZSStorageCredentials.h" -@interface AZSStreamDownloadBuffer : NSObject -{ - @public - CC_MD5_CTX _md5Context; -} - -@property (strong, readonly) NSOutputStream *stream; -@property (strong, readonly) NSMutableArray *queue; -@property NSUInteger currentLength; -@property BOOL streamWaiting; -@property NSUInteger maxSizeToBuffer; -@property (strong) NSData *currentDataToStream; -@property uint64_t totalSizeStreamed; -@property (strong, readonly) NSCondition *dataDownloadCondition; -@property BOOL calculateMD5; -@property (strong, readonly) AZSOperationContext *operationContext; -@property (strong) NSError *streamError; - --(instancetype)init AZS_DESIGNATED_INITIALIZER; --(instancetype)initWithStream:(NSOutputStream *)stream maxSizeToBuffer:(NSUInteger)maxSizeToBuffer calculateMD5:(BOOL)calculateMD5 operationContext:(AZSOperationContext *)operationContext AZS_DESIGNATED_INITIALIZER; --(void)stream:(NSStream *)stream handleEvent:(NSStreamEvent)eventCode; --(void)writeData:(NSData *)data; - -@end - -@implementation AZSStreamDownloadBuffer - --(instancetype)init -{ - return nil; -} - --(instancetype)initWithStream:(NSOutputStream *)stream maxSizeToBuffer:(NSUInteger)maxSizeToBuffer calculateMD5:(BOOL)calculateMD5 operationContext:(AZSOperationContext *)operationContext -{ - self = [super init]; - if (self) - { - _stream = stream; - _queue = [[NSMutableArray alloc] init]; - _currentLength = 0; - _streamWaiting = NO; - _operationContext = operationContext; - [_operationContext logAtLevel:AZSLogLevelDebug withMessage:@"Stream not waiting (init)"]; - _maxSizeToBuffer = maxSizeToBuffer; - _currentDataToStream = nil; - _totalSizeStreamed = 0; - _dataDownloadCondition = [[NSCondition alloc] init]; - _calculateMD5 = calculateMD5; - if (_calculateMD5) - { - CC_MD5_Init(&_md5Context); - } - } - - return self; -} - --(void)writeData:(NSData *)data -{ - if (self.calculateMD5) - { - CC_MD5_Update(&_md5Context, data.bytes, (unsigned int) data.length); - } - - [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"About to grab lock from data pushing"]; - - // TODO: Optimize to remove the need for the lock (or at least, for locking the whole thing.) - // To do this, we need to ensure that all data is written to the stream in the correct order (even if some writes are sync), - // and that the current length and total size are consistent. - - [self.dataDownloadCondition lock]; - - // TODO: self.streamWaiting should be set only if there is nothing in the buffer. Make sure this is true, then simplify this condition. - while (!(self.streamWaiting || (self.currentLength <= self.maxSizeToBuffer))) - { - [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"Waiting on datadownloadcondition in writeData."]; - [self.dataDownloadCondition wait]; - } - - if (self.streamError) - { - // If there's an error, return. - [self.dataDownloadCondition broadcast]; - [self.dataDownloadCondition unlock]; - return; - } - - if (self.streamWaiting) - { - [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"StreamWaiting = YES"]; - } - else - { - [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"StreamWaiting = NO"]; - } - - if (!self.streamWaiting) - { - [self.queue addObject:data]; - self.currentLength = self.currentLength + [data length]; - [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"Adding to queue. Current length = %ld, total amount streamed = %ld", (unsigned long)self.currentLength, (unsigned long)self.totalSizeStreamed]; - } - else - { - NSUInteger lengthWritten = [self.stream write:(const uint8_t *)[data bytes] maxLength:[data length]]; - [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"Wrote syncronously. LengthWritten = %ld, desired write size = %ld.", (unsigned long)lengthWritten, (unsigned long)[data length]]; - - if (lengthWritten == -1) - { - NSMutableDictionary *userInfo = [NSMutableDictionary dictionary]; - userInfo[AZSInnerErrorString] = self.stream.streamError; - NSError *streamError = [NSError errorWithDomain:AZSErrorDomain code:AZSEOutputStreamError userInfo:userInfo]; - self.streamError = streamError; - [self.operationContext logAtLevel:AZSLogLevelError withMessage:@"Error in writing to download stream, aborting download."]; - } - else if (lengthWritten == 0) - { - NSMutableDictionary *userInfo = [NSMutableDictionary dictionary]; - userInfo[AZSInnerErrorString] = self.stream.streamError; - NSError *streamError = [NSError errorWithDomain:AZSErrorDomain code:AZSEOutputStreamFull userInfo:userInfo]; - self.streamError = streamError; - [self.operationContext logAtLevel:AZSLogLevelError withMessage:@"DownloadStream is full but there is more pending data, aborting download."]; - } - else - { - self.totalSizeStreamed += lengthWritten; - self.streamWaiting = NO; - [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"Stream not waiting."]; - // TODO: Handle 0 and -1 case; - if (lengthWritten < [data length]) - { - NSUInteger lengthRemaining = [data length] - lengthWritten; - uint8_t buf[lengthRemaining]; - memcpy(buf, [data bytes] + lengthWritten, lengthRemaining); - [self.queue addObject:[NSData dataWithBytes:buf length:lengthRemaining]]; - self.currentLength = self.currentLength + [data length]; - } - - [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"Current length (wrote sync) = %ld, total amount streamed = %ld", (unsigned long)self.currentLength, (unsigned long)self.totalSizeStreamed]; - } - // The following broadcast should never actually wake up anything, because the condition is only waited on in two cases: - // - If the thread is done downloading and waiting for the buffer to clear (can't happen due to sync nature of didReceiveData and didCompleteWithError.) - // - If the buffer is full and didReceiveData is thus blocking (in which case this method shouldn't be called.) - // Leaving it in in case of any corner cases not yet thought of - this way, all writes from the buffer signals the condition. - [self.dataDownloadCondition broadcast]; - } - [self.dataDownloadCondition unlock]; -} - - --(void)stream:(NSStream *)stream handleEvent:(NSStreamEvent)eventCode -{ - if (![AZSUtil streamAvailable:stream]) - { - return; - } - - switch(eventCode) { - case NSStreamEventHasSpaceAvailable: - { - [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"NSStreamEventHasSpaceAvailable"]; - [self.dataDownloadCondition lock]; - if (self.streamError) - { - // If there's already an error, return. - [self.dataDownloadCondition broadcast]; - [self.dataDownloadCondition unlock]; - return; - } - - [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"Just grabbed lock from stream callback"]; - [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"Current thread name = %@",[NSThread currentThread]]; - - [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"HasSpaceAvailable"]; - if ((self.currentDataToStream != nil) || ([self.queue count] > 0)) - { - if (self.currentDataToStream == nil) - { - self.currentDataToStream = (NSData *) self.queue.firstObject; - [self.queue removeObjectAtIndex:0]; - } - NSUInteger lengthWritten = [self.stream write:(const uint8_t *)[self.currentDataToStream bytes] maxLength:[self.currentDataToStream length]]; - - if (lengthWritten == -1) - { - NSMutableDictionary *userInfo = [NSMutableDictionary dictionary]; - userInfo[AZSInnerErrorString] = self.stream.streamError; - NSError *streamError = [NSError errorWithDomain:AZSErrorDomain code:AZSEOutputStreamError userInfo:userInfo]; - self.streamError = streamError; - [self.operationContext logAtLevel:AZSLogLevelError withMessage:@"Error in writing to download stream, aborting download."]; - } - else if (lengthWritten == 0) - { - NSMutableDictionary *userInfo = [NSMutableDictionary dictionary]; - userInfo[AZSInnerErrorString] = self.stream.streamError; - NSError *streamError = [NSError errorWithDomain:AZSErrorDomain code:AZSEOutputStreamFull userInfo:userInfo]; - self.streamError = streamError; - [self.operationContext logAtLevel:AZSLogLevelError withMessage:@"DownloadStream is full but there is more pending data, aborting download."]; - } - else - { - self.currentLength = self.currentLength - lengthWritten; - self.totalSizeStreamed += lengthWritten; - [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"Wrote async. LengthWritten = %ld, desired write size = %ld.", (unsigned long)lengthWritten, (unsigned long)[self.currentDataToStream length]]; - [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"Current length (wrote async) = %ld, total amount streamed = %ld", (unsigned long)self.currentLength, (unsigned long)self.totalSizeStreamed]; - - if (lengthWritten < [self.currentDataToStream length]) - { - NSUInteger lengthRemaining = [self.currentDataToStream length] - lengthWritten; - uint8_t buf[lengthRemaining]; - memcpy(buf, [self.currentDataToStream bytes] + lengthWritten, lengthRemaining); - self.currentDataToStream = [NSData dataWithBytes:buf length:lengthRemaining]; - } - else - { - self.currentDataToStream = nil; - } - } - } - else - { - self.streamWaiting = YES; - [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"Stream waiting."]; - } - [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"About to signal and release lock from stream callback"]; - [self.dataDownloadCondition broadcast]; - [self.dataDownloadCondition unlock]; - - break; - } - case NSStreamEventEndEncountered: - { - [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"NSStreamEventEndEncountered"]; - break; - } - case NSStreamEventErrorOccurred: - { - [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"NSStreamEventErrorOccurred"]; - NSMutableDictionary *userInfo = [NSMutableDictionary dictionary]; - userInfo[AZSInnerErrorString] = self.stream.streamError; - NSError *streamError = [NSError errorWithDomain:AZSErrorDomain code:AZSEOutputStreamError userInfo:userInfo]; - self.streamError = streamError; - [self.operationContext logAtLevel:AZSLogLevelError withMessage:@"Error in writing to download stream, aborting download."]; - } - case NSStreamEventOpenCompleted: - { - [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"NSStreamEventOpenCompleted"]; - break; - } - case NSStreamEventNone: - { - [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"NSStreamEventNone"]; - break; - } - case NSStreamEventHasBytesAvailable: - { - [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"NSStreamEventHasBytesAvailable"]; - // Should never happen. - break; - } - default: - { - [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"NSStreamEventdefault"]; - NSLog(@"Default switch occurred."); - break; - } - } -} -@end - @interface AZSExecutor() @property (strong) AZSStorageCommand* storageCommand; @property (strong) AZSRequestOptions* requestOptions; @@ -308,35 +40,34 @@ @interface AZSExecutor() @property (strong) NSURLComponents *urlComponents; @property (strong) NSMutableURLRequest *request; @property (strong) AZSRequestResult *requestResult; -@property (strong) NSOutputStream *outputStream; @property (strong) NSHTTPURLResponse *httpResponse; @property (copy) void (^completionHandler)(NSError *, id); @property BOOL isSourceStreamSet; +@property (strong) AZSStreamDownloadBuffer *originalDownloadBuffer; @property (strong) AZSStreamDownloadBuffer *downloadBuffer; -@property (strong) NSRunLoop *runLoopForDownload; -@property dispatch_semaphore_t semaphoreForRunloopCreation; @property (strong) NSError *preProcessError; @property (strong) id retryPolicy; @property NSUInteger retryCount; @property AZSStorageLocation currentStorageLocation; @property AZSStorageLocationMode currentStorageLocationMode; +@property BOOL removeFromRunLoop; +@property (strong) NSOutputStream *outputStream; -(instancetype)init AZS_DESIGNATED_INITIALIZER; --(instancetype)initWithCommand:(AZSStorageCommand *)storageCommand requestOptions:(AZSRequestOptions *)requestOptions operationContext:(AZSOperationContext *) operationContext completionHandler:(void (^)(NSError *, id))completionHandler AZS_DESIGNATED_INITIALIZER; +-(instancetype)initWithCommand:(AZSStorageCommand *)storageCommand requestOptions:(AZSRequestOptions *)requestOptions operationContext:(AZSOperationContext *) operationContext downloadBuffer:(AZSStreamDownloadBuffer *)downloadBuffer completionHandler:(void (^)(NSError *, id))completionHandler AZS_DESIGNATED_INITIALIZER; + @end @implementation AZSExecutor +(void)ExecuteWithStorageCommand:(AZSStorageCommand *)storageCommand requestOptions:(AZSRequestOptions *)requestOptions operationContext:(AZSOperationContext *)operationContext completionHandler:(void (^)(NSError *, id))completionHandler { - AZSExecutor *executor = [[AZSExecutor alloc] initWithCommand:storageCommand requestOptions:requestOptions operationContext:operationContext completionHandler:completionHandler]; - [executor execute]; + [AZSExecutor ExecuteWithStorageCommand:storageCommand requestOptions:requestOptions operationContext:operationContext downloadBuffer:nil completionHandler:completionHandler]; } -+(void)ExecuteWithStorageCommand:(AZSStorageCommand *)storageCommand requestOptions:(AZSRequestOptions *)requestOptions operationContext:(AZSOperationContext *)operationContext retryCount:(NSUInteger)retryCount completionHandler:(void (^)(NSError *, id))completionHandler ++(void)ExecuteWithStorageCommand:(AZSStorageCommand *)storageCommand requestOptions:(AZSRequestOptions *)requestOptions operationContext:(AZSOperationContext *)operationContext downloadBuffer:(AZSStreamDownloadBuffer *)downloadBuffer completionHandler:(void (^)(NSError*, id))completionHandler { - AZSExecutor *executor = [[AZSExecutor alloc] initWithCommand:storageCommand requestOptions:requestOptions operationContext:operationContext completionHandler:completionHandler]; - executor.retryCount = retryCount; + AZSExecutor *executor = [[AZSExecutor alloc] initWithCommand:storageCommand requestOptions:requestOptions operationContext:operationContext downloadBuffer:downloadBuffer completionHandler:completionHandler]; [executor execute]; } @@ -523,37 +254,6 @@ -(void)URLSession:(NSURLSession *)session task:(NSURLSessionTask *)task didSendB } */ --(void)createAndSpinRunloopWithOutputStream:(id)outputStream -{ - @autoreleasepool { - self.runLoopForDownload = [NSRunLoop currentRunLoop]; - [outputStream scheduleInRunLoop:self.runLoopForDownload forMode:NSDefaultRunLoopMode]; - [self.outputStream open]; - dispatch_semaphore_signal(self.semaphoreForRunloopCreation); - - // TODO: Make the below timeout value for the runloop configurable. - BOOL runLoopSuccess = YES; - while (([AZSUtil streamAvailable:outputStream]) && runLoopSuccess) - { - @autoreleasepool { - NSDate *loopUntil = [NSDate dateWithTimeIntervalSinceNow:2.0];//[NSDate distantFuture]; - [self.runLoopForDownload runMode:NSDefaultRunLoopMode beforeDate:loopUntil]; - [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"(Waking up) Current thread name = %@",[NSThread currentThread]]; - if ([outputStream streamError]) - { - NSError *error = [outputStream streamError]; - [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"StreamError. Error code = %ld, error domain = %@, error userinfo = %@", (long)error.code, error.domain, error.userInfo]; - } - else - { - [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"No stream error while in runloop."]; - } - } - } - [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"Exiting spin."]; - } -} - -(void)URLSession:(NSURLSession *)session dataTask:(NSURLSessionDataTask *)dataTask didReceiveResponse:(NSURLResponse *)response completionHandler:(void (^)(NSURLSessionResponseDisposition))completionHandler { self.httpResponse = (NSHTTPURLResponse *) response; @@ -572,45 +272,27 @@ -(void)URLSession:(NSURLSession *)session dataTask:(NSURLSessionDataTask *)dataT self.requestResult = [[AZSRequestResult alloc] initWithStartTime:self.startTime location:self.currentStorageLocation response:self.httpResponse error:nil]; self.preProcessError = self.storageCommand.preProcessResponse(self.httpResponse, self.requestResult, self.operationContext); - if (self.preProcessError != nil) - { + // TODO: Don't bother with all the stream stuff (especially thread creation) if there is no body. + if (self.preProcessError) { + // In case of error, we want a memory stream and clean download buffer. self.outputStream = [NSOutputStream outputStreamToMemory]; + self.downloadBuffer = [[AZSStreamDownloadBuffer alloc] initWithOutputStream:self.outputStream maxSizeToBuffer:self.requestOptions.maximumDownloadBufferSize calculateMD5:(self.storageCommand.calculateResponseMD5 && (self.requestResult.contentReceivedMD5 != nil)) runLoopForDownload:self.requestOptions.runLoopForDownload operationContext:self.operationContext]; + + [self.downloadBuffer createAndSpinRunloop]; + self.removeFromRunLoop = YES; } - else - { - // TODO: Don't bother with all the stream stuff (especially thread creation) if there is no body. - if (self.storageCommand.destinationStream == nil) - { - self.outputStream = [NSOutputStream outputStreamToMemory]; + else { + // Otherwise we should use the same download buffer every time, and the user supplied stream (if any). + if (!self.originalDownloadBuffer) { + self.outputStream = self.storageCommand.destinationStream ?: [NSOutputStream outputStreamToMemory]; + self.originalDownloadBuffer = [[AZSStreamDownloadBuffer alloc] initWithOutputStream:self.outputStream maxSizeToBuffer:self.requestOptions.maximumDownloadBufferSize calculateMD5:(self.storageCommand.calculateResponseMD5 && (self.requestResult.contentReceivedMD5 != nil)) runLoopForDownload:self.requestOptions.runLoopForDownload operationContext:self.operationContext]; + [self.originalDownloadBuffer createAndSpinRunloop]; + self.removeFromRunLoop = YES; } - else - { - self.outputStream = self.storageCommand.destinationStream; - } - } - - self.downloadBuffer = [[AZSStreamDownloadBuffer alloc]initWithStream:self.outputStream maxSizeToBuffer:self.requestOptions.maximumDownloadBufferSize calculateMD5:(self.storageCommand.calculateResponseMD5 && (self.requestResult.contentReceivedMD5 != nil)) operationContext:self.operationContext]; - - [self.outputStream setDelegate:self.downloadBuffer]; - - self.runLoopForDownload = self.requestOptions.runLoopForDownload; - if (self.runLoopForDownload == nil) - { - // In this case, we will open the stream inside the createAndSpinRunloopWithOutputStream method. - self.semaphoreForRunloopCreation = dispatch_semaphore_create(0); - - [NSThread detachNewThreadSelector:@selector(createAndSpinRunloopWithOutputStream:) toTarget:self withObject:self.outputStream]; - dispatch_semaphore_wait(self.semaphoreForRunloopCreation, DISPATCH_TIME_FOREVER); + self.downloadBuffer = self.originalDownloadBuffer; } - else - { - [self.outputStream scheduleInRunLoop:self.runLoopForDownload forMode:NSDefaultRunLoopMode]; - [self.outputStream open]; - } - - completionHandler(NSURLSessionResponseAllow); } @@ -627,12 +309,7 @@ -(void)URLSession:(NSURLSession *)session task:(NSURLSessionTask *)task didCompl { // This is called upon task completion. If there were no error, *error will be nil. - if (self.downloadBuffer.calculateMD5) - { - unsigned char md5Bytes[CC_MD5_DIGEST_LENGTH]; - CC_MD5_Final(md5Bytes, &(self.downloadBuffer->_md5Context)); - self.requestResult.calculatedResponseMD5 = [[[NSData alloc] initWithBytes:md5Bytes length:CC_MD5_DIGEST_LENGTH] base64EncodedStringWithOptions:0]; - } + self.requestResult.calculatedResponseMD5 = [self.downloadBuffer checkMD5]; [self.downloadBuffer.dataDownloadCondition lock]; [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"Grabbed lock in didComplete."]; @@ -645,8 +322,10 @@ -(void)URLSession:(NSURLSession *)session task:(NSURLSessionTask *)task didCompl [self.downloadBuffer.dataDownloadCondition unlock]; [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"Released lock in didComplete."]; - [self.outputStream close]; - [self.outputStream removeFromRunLoop:self.runLoopForDownload forMode:NSDefaultRunLoopMode]; + self.downloadBuffer.downloadComplete = YES; + if (self.removeFromRunLoop) { + [self.downloadBuffer removeFromRunLoop]; + } if (error) // If DidCompleteWithError was passed an error { @@ -751,7 +430,7 @@ -(void)finishRequestWithSession:(NSURLSession *)session error:(NSError *)error r } // We cannot recover and retry the request if any data has been written to the caller's stream. - if (retry && (((self.storageCommand.destinationStream == self.outputStream) && self.downloadBuffer.totalSizeStreamed > 0))) + if (retry && (self.storageCommand.destinationStream == self.outputStream) && (self.downloadBuffer.totalSizeStreamed > 0)) { retry = NO; } @@ -790,12 +469,23 @@ -(void)finishRequestWithSession:(NSURLSession *)session error:(NSError *)error r } [self execute]; - -// [AZSExecutor ExecuteWithStorageCommand:self.storageCommand requestOptions:self.requestOptions operationContext:self.operationContext retryCount:self.retryCount completionHandler:self.completionHandler]; } else { + if (!self.downloadBuffer.streamError) { + self.downloadBuffer.streamError = error; + } + + if (self.originalDownloadBuffer && self.originalDownloadBuffer != self.downloadBuffer) + { + self.originalDownloadBuffer.streamError = self.downloadBuffer.streamError; + self.originalDownloadBuffer.downloadComplete = YES; + } self.operationContext.endTime = [NSDate date]; + if (self.removeFromRunLoop) { + [self.outputStream close]; + } + self.completionHandler(error, retval); } } @@ -822,7 +512,7 @@ +(AZSStorageLocation) getFirstLocationWithStorageLocationMode:(AZSStorageLocatio } } --(instancetype) initWithCommand:(AZSStorageCommand *)storageCommand requestOptions:(AZSRequestOptions *)requestOptions operationContext:(AZSOperationContext *) operationContext completionHandler:(void (^)(NSError *, id))completionHandler +-(instancetype) initWithCommand:(AZSStorageCommand *)storageCommand requestOptions:(AZSRequestOptions *)requestOptions operationContext:(AZSOperationContext *) operationContext downloadBuffer:(AZSStreamDownloadBuffer *)downloadBuffer completionHandler:(void (^)(NSError *, id))completionHandler { self = [super init]; if (self) @@ -831,6 +521,8 @@ -(instancetype) initWithCommand:(AZSStorageCommand *)storageCommand requestOptio _requestOptions = requestOptions; _operationContext = operationContext; _completionHandler = completionHandler; + _originalDownloadBuffer = downloadBuffer; + _downloadBuffer = downloadBuffer; _retryCount = 0; _retryPolicy = [operationContext.retryPolicy clone]; _currentStorageLocation = [AZSExecutor getFirstLocationWithStorageLocationMode:requestOptions.storageLocationMode]; @@ -851,4 +543,5 @@ -(NSTimeInterval) remainingTime return 60*60*24*7; // 7 days is the default timeout in iOS, although we don't have to stick to that. } } -@end + +@end \ No newline at end of file diff --git a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSStorageCommand.h b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSStorageCommand.h index 874f952..62d20eb 100644 --- a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSStorageCommand.h +++ b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSStorageCommand.h @@ -37,8 +37,8 @@ @property (copy) NSMutableURLRequest *(^buildRequest)(NSURLComponents *urlComponents, NSTimeInterval timeout, AZSOperationContext *operationContext); @property (copy) void(^signRequest)(NSMutableURLRequest *request, AZSOperationContext *operationContext); @property (copy) NSError *(^preProcessResponse)(NSHTTPURLResponse *response, AZSRequestResult *requestResult, AZSOperationContext *operationContext); -@property (copy) id(^postProcessResponse)(NSHTTPURLResponse *response, AZSRequestResult *requestResult, NSOutputStream *outputStream, AZSOperationContext *operationContext, NSError **error); -@property (copy) void(^processError)(NSOutputStream *outputStream, NSError **errorToPopulate, NSError **error); +@property (copy) id(^postProcessResponse)(NSHTTPURLResponse *response, AZSRequestResult *requestResult, NSOutputStream *stream, AZSOperationContext *operationContext, NSError **error); +@property (copy) void(^processError)(NSOutputStream *stream, NSError **errorToPopulate, NSError **error); @property (strong, nonatomic) NSData *source; @property (strong, nonatomic) NSOutputStream *destinationStream; @@ -48,5 +48,4 @@ -(void) setAllowedStorageLocation:(AZSAllowedStorageLocation)allowedStorageLocation; -(void) setAllowedStorageLocation:(AZSAllowedStorageLocation)allowedStorageLocation withLockLocation:(AZSStorageLocation)lockLocation error:(NSError **)error; - -@end +@end \ No newline at end of file diff --git a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSStorageCommand.m b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSStorageCommand.m index 168eaa0..b9f5636 100644 --- a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSStorageCommand.m +++ b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSStorageCommand.m @@ -53,8 +53,8 @@ -(instancetype) initWithStorageCredentials:(AZSStorageCredentials *)credentials // Give a default error-processing implementation. // TODO: This now couples the execution layer with the protocol layer. Decide if this is correct or not. - self.processError = ^void(NSOutputStream *outputStream, NSError **errorToPopulate, NSError **error) { - NSData *rawErrorData = [outputStream propertyForKey:NSStreamDataWrittenToMemoryStreamKey]; + self.processError = ^void(NSOutputStream *stream, NSError **errorToPopulate, NSError **error) { + NSData *rawErrorData = [stream propertyForKey:NSStreamDataWrittenToMemoryStreamKey]; if (rawErrorData.length > 0) { NSMutableDictionary *userInfo = [NSMutableDictionary dictionaryWithDictionary:(*errorToPopulate).userInfo]; diff --git a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSStreamDownloadBuffer.h b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSStreamDownloadBuffer.h new file mode 100644 index 0000000..45cc4cd --- /dev/null +++ b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSStreamDownloadBuffer.h @@ -0,0 +1,41 @@ +// ----------------------------------------------------------------------------------------- +// +// Copyright 2015 Microsoft Corporation +// +// Licensed under the MIT License; +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://spdx.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------------------- + +#import +@class AZSOperationContext; + +@interface AZSStreamDownloadBuffer : NSObject + +@property BOOL calculateMD5; +@property NSUInteger currentLength; +@property (strong, readonly) NSCondition *dataDownloadCondition; +@property (strong, readonly) AZSOperationContext *operationContext; +@property (strong) NSError *streamError; +@property BOOL streamClosed; +@property (readonly) uint64_t totalSizeStreamed; +@property BOOL downloadComplete; + +-(instancetype)initWithInputStream:(NSInputStream *)stream maxSizeToBuffer:(NSUInteger)maxSizeToBuffer calculateMD5:(BOOL)calculateMD5 runLoopForDownload:(NSRunLoop *)runLoop operationContext:(AZSOperationContext *)operationContext fireEventBlock:(void(^)())fireEventBlock setHasBytesAvailable:(void (^)())setHasBytesAvailable; +-(instancetype)initWithOutputStream:(NSOutputStream *)stream maxSizeToBuffer:(NSUInteger)maxSizeToBuffer calculateMD5:(BOOL)calculateMD5 runLoopForDownload:(NSRunLoop *)runLoop operationContext:(AZSOperationContext *)operationContext; +-(void)stream:(NSStream *)stream handleEvent:(NSStreamEvent)eventCode; +-(void)processDataWithProcess:(long(^)(uint8_t *, long))process; +-(void)writeData:(NSData *)data; +-(void)createAndSpinRunloop; +-(void)removeFromRunLoop; +-(NSString *)checkMD5; + +@end \ No newline at end of file diff --git a/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSStreamDownloadBuffer.m b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSStreamDownloadBuffer.m new file mode 100644 index 0000000..147f4c1 --- /dev/null +++ b/Lib/Azure Storage Client Library/Azure Storage Client Library/AZSStreamDownloadBuffer.m @@ -0,0 +1,387 @@ +// ----------------------------------------------------------------------------------------- +// +// Copyright 2015 Microsoft Corporation +// +// Licensed under the MIT License; +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://spdx.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------------------- + +#import +#import "AZSConstants.h" +#import "AZSErrors.h" +#import "AZSOperationContext.h" +#import "AZSStorageCredentials.h" +#import "AZSStreamDownloadBuffer.h" +#import "AZSUtil.h" + +@interface AZSStreamDownloadBuffer() + +@property CC_MD5_CTX md5Context; +@property (strong, readonly) NSMutableArray *queue; +@property (strong) NSRunLoop *runLoopForDownload; +@property BOOL streamWaiting; +@property NSUInteger maxSizeToBuffer; +@property NSInteger currentDataOffset; +@property (strong) NSData *currentDataToStream; +@property (strong, readonly) void(^fireEventBlock)(); +@property (strong, readonly) void(^setHasBytesAvailable)(); +@property (strong, readonly) NSStream *stream; + +-(instancetype)init AZS_DESIGNATED_INITIALIZER; +-(instancetype)initWithStream:(NSStream *)stream maxSizeToBuffer:(NSUInteger)maxSizeToBuffer calculateMD5:(BOOL)calculateMD5 runLoopForDownload:(NSRunLoop *)runLoop operationContext:(AZSOperationContext *)operationContext AZS_DESIGNATED_INITIALIZER; + +@end + +@implementation AZSStreamDownloadBuffer + +@synthesize streamError = _streamError; +@synthesize downloadComplete = _downloadComplete; + +-(instancetype)init +{ + return nil; +} + +-(instancetype)initWithStream:(NSStream *)stream maxSizeToBuffer:(NSUInteger)maxSizeToBuffer calculateMD5:(BOOL)calculateMD5 runLoopForDownload:(NSRunLoop *)runLoop operationContext:(AZSOperationContext *)operationContext +{ + self = [super init]; + if (self) { + _stream = stream; + _queue = [[NSMutableArray alloc] init]; + _currentLength = 0; + _streamWaiting = NO; + _operationContext = operationContext; + [_operationContext logAtLevel:AZSLogLevelDebug withMessage:@"Stream not waiting (init)"]; + _maxSizeToBuffer = maxSizeToBuffer; + _currentDataToStream = nil; + _totalSizeStreamed = 0; + _dataDownloadCondition = [[NSCondition alloc] init]; + _calculateMD5 = calculateMD5; + if (_calculateMD5) { + CC_MD5_Init(&_md5Context); + } + + _runLoopForDownload = runLoop; + } + + return self; +} + +-(instancetype)initWithInputStream:(NSInputStream *)stream maxSizeToBuffer:(NSUInteger)maxSizeToBuffer calculateMD5:(BOOL)calculateMD5 runLoopForDownload:(NSRunLoop *)runLoop operationContext:(AZSOperationContext *)operationContext fireEventBlock:(void (^)())fireEventBlock setHasBytesAvailable:(void (^)())setHasBytesAvailable +{ + self = [self initWithStream:stream maxSizeToBuffer:maxSizeToBuffer calculateMD5:calculateMD5 runLoopForDownload:runLoop operationContext:operationContext]; + if (self) { + _fireEventBlock = fireEventBlock; + _setHasBytesAvailable = setHasBytesAvailable; + } + + return self; +} + +-(instancetype)initWithOutputStream:(NSOutputStream *)stream maxSizeToBuffer:(NSUInteger)maxSizeToBuffer calculateMD5:(BOOL)calculateMD5 runLoopForDownload:(NSRunLoop *)runLoop operationContext:(AZSOperationContext *)operationContext +{ + self = [self initWithStream:stream maxSizeToBuffer:maxSizeToBuffer calculateMD5:calculateMD5 runLoopForDownload:runLoop operationContext:operationContext]; + if (self) { + [_stream setDelegate:self]; + } + + return self; +} + +-(BOOL) downloadComplete +{ + return _downloadComplete; +} + +-(void) setDownloadComplete:(BOOL)downloadComplete +{ + _downloadComplete = downloadComplete; + if (self.fireEventBlock) + { + self.fireEventBlock(); + } +} + +-(NSError *) streamError +{ + return _streamError; +} + +-(void) setStreamError:(NSError *)streamError +{ + _streamError = streamError; + if (self.fireEventBlock) + { + self.fireEventBlock(); + } +} + +-(void)processDataWithProcess:(long(^)(uint8_t *, long))process +{ + [self.dataDownloadCondition lock]; + if (self.streamError) { + // Return if there's already an error + [self.dataDownloadCondition broadcast]; + [self.dataDownloadCondition unlock]; + return; + } + + [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"Just grabbed lock from stream callback"]; + [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"Current thread name = %@",[NSThread currentThread]]; + + // If there is still data to process + if ((self.currentDataToStream && (self.currentDataOffset < self.currentDataToStream.length)) || ([self.queue count] > 0)) { + // If we need a new NSData object from the queue + if (!self.currentDataToStream || (self.currentDataOffset >= self.currentDataToStream.length)) { + self.currentDataToStream = (NSData *) self.queue.firstObject; + self.currentDataOffset = 0; + [self.queue removeObjectAtIndex:0]; + } + + long lengthProcessed = process((uint8_t *) ([self.currentDataToStream bytes] + self.currentDataOffset), [self.currentDataToStream length] - self.currentDataOffset); + [self updateOffsetWithData:self.currentDataToStream lengthProcessed:lengthProcessed]; + self.currentLength -= (lengthProcessed > 0) ? lengthProcessed : 0; + } + else { + self.streamWaiting = YES; + [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"Stream waiting."]; + } + + [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"About to signal and release lock from stream callback"]; + [self.dataDownloadCondition broadcast]; + [self.dataDownloadCondition unlock]; +} + +-(void)writeData:(NSData *)data +{ + _totalSizeStreamed += data.length; + + if (self.calculateMD5) { + CC_MD5_Update(&_md5Context, data.bytes, (unsigned int) data.length); + } + + [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"About to grab lock from data pushing"]; + + // TODO: Optimize to remove the need for the lock (or at least, for locking the whole thing.) + // To do this, we need to ensure that all data is written to the stream in the correct order (even if some writes are sync), + // and that the current length and total size are consistent. + + [self.dataDownloadCondition lock]; + + // TODO: self.streamWaiting should be set only if there is nothing in the buffer. Make sure this is true, then simplify this condition. + while (!(self.streamWaiting || (self.currentLength <= self.maxSizeToBuffer))) { + [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"Waiting on dataDownloadCondition in writeData."]; + [self.dataDownloadCondition wait]; + } + + if (self.streamError || self.streamClosed) { + // If there's an error or the stream is closed, return. + [self.dataDownloadCondition broadcast]; + [self.dataDownloadCondition unlock]; + return; + } + + if (self.streamWaiting) { + [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"StreamWaiting = YES"]; + } + else { + [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"StreamWaiting = NO"]; + } + + // self.stream must be either a NSOutputStream or an AZSInputStream, and only NSOutputStream has a write method. + if (self.streamWaiting && [self.stream respondsToSelector:@selector(write:maxLength:)]) { + // If the stream is waiting, write the data to it. + NSInteger lengthWritten = [((NSOutputStream *) self.stream) write:data.bytes maxLength:data.length]; + [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"Wrote syncronously. LengthWritten = %ld, desired write size = %ld.", lengthWritten, data.length]; + + // If any data is left unwritten, store it to be processed later. + self.currentDataOffset = 0; + [self updateOffsetWithData:data lengthProcessed:lengthWritten]; + if(self.currentDataToStream) { + self.currentLength += (lengthWritten > 0) ? (data.length - self.currentDataOffset) : 0; + } + + // The following broadcast should never actually wake up anything, because the condition is only waited on in two cases: + // - If the thread is done downloading and waiting for the buffer to clear (can't happen due to sync nature of didReceiveData and didCompleteWithError.) + // - If the buffer is full and didReceiveData is thus blocking (in which case this method shouldn't be called.) + // Leaving it in in case of any corner cases not yet thought of - this way, all writes from the buffer signals the condition. + [self.dataDownloadCondition broadcast]; + } + else { + // Otherwise, just add the data to the queue so it can be processed in order. + [self.queue addObject:data]; + self.currentLength += data.length; + [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"Adding to queue. Current length = %ld, total amount streamed = %ld", self.currentLength, self.totalSizeStreamed]; + } + + // Tells the input stream there is more data to process. + if (self.setHasBytesAvailable) { + self.setHasBytesAvailable(); + } + if (self.fireEventBlock) { + self.fireEventBlock(); + } + + [self.dataDownloadCondition unlock]; +} + +-(void)updateOffsetWithData:(NSData *)data lengthProcessed:(NSInteger)lengthProcessed +{ + if (lengthProcessed < 0) { + // Error Occurred + NSDictionary *userInfo = self.stream.streamError ? @{AZSInnerErrorString : self.stream.streamError} : nil; + self.streamError = [NSError errorWithDomain:AZSErrorDomain code:AZSEInputStreamError userInfo:userInfo]; + [self.operationContext logAtLevel:AZSLogLevelError withMessage:@"Error in processing download stream, aborting download."]; + } + else if (lengthProcessed == 0) { + // Capacity Reached + self.streamError = [NSError errorWithDomain:AZSErrorDomain code:AZSEInputStreamEmpty userInfo:@{AZSInnerErrorString : self.stream.streamError}]; + [self.operationContext logAtLevel:AZSLogLevelError withMessage:@"DownloadStream is empty, aborting download."]; + } + else { + // Increment self.currentDataOffset if doing so wouldn't go beyond the end of data and update self.currentDataToStream + [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"Process async. LengthRead = %ld, desired size = %ld.", (unsigned long) lengthProcessed, (unsigned long) [self.currentDataToStream length]]; + [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"Current length (process async) = %ld, total amount streamed = %ld", (unsigned long) self.currentLength, (unsigned long) self.totalSizeStreamed]; + + self.streamWaiting = NO; + [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"Stream not waiting."]; + + if (self.currentDataOffset + lengthProcessed <= [data length]) { + self.currentDataOffset += lengthProcessed; + self.currentDataToStream = data; + return; + } + } + + // If anything went wrong or no data was left over, reset these. + self.currentDataOffset = 0; + self.currentDataToStream = nil; +} + +-(void)stream:(NSStream *)stream handleEvent:(NSStreamEvent)eventCode +{ + if (![AZSUtil streamAvailable:stream]) { + return; + } + + switch(eventCode) { + case NSStreamEventHasSpaceAvailable: + { + [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"NSStreamEventHasSpaceAvailable"]; + [self processDataWithProcess:^long(uint8_t * buffer, long length) { + return [((NSOutputStream *) stream) write:buffer maxLength:length]; + }]; + + break; + } + case NSStreamEventEndEncountered: + { + [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"NSStreamEventEndEncountered"]; + break; + } + case NSStreamEventErrorOccurred: + { + [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"NSStreamEventErrorOccurred"]; + NSMutableDictionary *userInfo = [NSMutableDictionary dictionary]; + userInfo[AZSInnerErrorString] = stream.streamError; + NSError *streamError = [NSError errorWithDomain:AZSErrorDomain code:AZSEOutputStreamError userInfo:userInfo]; + self.streamError = streamError; + [self.operationContext logAtLevel:AZSLogLevelError withMessage:@"Error in writing to download stream, aborting download."]; + } + case NSStreamEventOpenCompleted: + { + [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"NSStreamEventOpenCompleted"]; + break; + } + case NSStreamEventNone: + { + [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"NSStreamEventNone"]; + break; + } + case NSStreamEventHasBytesAvailable: + { + // Should never happen. + [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"NSStreamEventHasBytesAvailable"]; + break; + } + default: + { + [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"NSStreamEventdefault"]; + break; + } + } +} + +-(void)createAndSpinRunloop +{ + if (!self.runLoopForDownload) { + // In this case, we will open the stream inside the createAndSpinRunloopWithStream method. + dispatch_semaphore_t semaphore = dispatch_semaphore_create(0); + [NSThread detachNewThreadSelector:@selector(createAndSpinRunloopWithSemaphore:) toTarget:self withObject:semaphore]; + dispatch_semaphore_wait(semaphore, DISPATCH_TIME_FOREVER); + } + else { + [_stream scheduleInRunLoop:self.runLoopForDownload forMode:NSDefaultRunLoopMode]; + [_stream open]; + } +} + +-(void)createAndSpinRunloopWithSemaphore:(dispatch_semaphore_t)semaphore +{ + @autoreleasepool { + self.runLoopForDownload = [NSRunLoop currentRunLoop]; + [self.stream scheduleInRunLoop:self.runLoopForDownload forMode:NSDefaultRunLoopMode]; + [self.stream open]; + dispatch_semaphore_signal(semaphore); + + // TODO: Make the below timeout value for the runloop configurable. + while ([AZSUtil streamAvailable:self.stream]) + { + @autoreleasepool { + NSDate *loopUntil = [NSDate dateWithTimeIntervalSinceNow:2]; + [self.runLoopForDownload runMode:NSDefaultRunLoopMode beforeDate:loopUntil]; + [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"(Waking up) Current thread name = %@",[NSThread currentThread]]; + + if ([self.stream streamError]) + { + NSError *error = [self.stream streamError]; + [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"StreamError. Error code = %ld, error domain = %@, error userinfo = %@", (long)error.code, error.domain, error.userInfo]; + } + else + { + [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"No stream error while in runloop."]; + } + } + } + [self.operationContext logAtLevel:AZSLogLevelDebug withMessage:@"Exiting spin."]; + } +} + +-(void)removeFromRunLoop +{ + [[self stream] close]; + [[self stream] removeFromRunLoop:self.runLoopForDownload forMode:NSDefaultRunLoopMode]; +} + +-(NSString *)checkMD5 +{ + if (self.calculateMD5) + { + unsigned char md5Bytes[CC_MD5_DIGEST_LENGTH]; + CC_MD5_Final(md5Bytes, &_md5Context); + return [[[NSData alloc] initWithBytes:md5Bytes length:CC_MD5_DIGEST_LENGTH] base64EncodedStringWithOptions:0]; + } + + return nil; +} + +@end \ No newline at end of file diff --git a/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSAccountSasTests.m b/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSAccountSasTests.m index 29767fc..1d6061b 100644 --- a/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSAccountSasTests.m +++ b/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSAccountSasTests.m @@ -262,8 +262,12 @@ - (void)testBlobAccountSasCombinationsWithResourceTypes:(AZSSharedAccessResource [self checkPassageOfError:error expectToPass:YES expectedHttpErrorCode:-1 message:@"Create SAS token"]; AZSCloudBlobClient *client = [self.account getBlobClient]; + AZSBlobRequestOptions *defaultOptions = [[AZSBlobRequestOptions alloc] init]; + defaultOptions.singleBlobUploadThreshold = 2; + client.defaultRequestOptions = defaultOptions; AZSStorageCredentials *credentials = [[AZSStorageCredentials alloc] initWithSASToken:accountSAS accountName:client.credentials.accountName]; AZSCloudBlobClient *sasClient = [[[AZSCloudStorageAccount alloc] initWithCredentials:credentials useHttps:NO error:&error] getBlobClient]; + sasClient.defaultRequestOptions = defaultOptions; [self checkPassageOfError:error expectToPass:YES expectedHttpErrorCode:-1 message:@"Access account with SAS token"]; AZSCloudBlobContainer *container = [client containerReferenceFromName:[NSString stringWithFormat:@"%@res%lu-perm%lu", [AZSTestHelpers uniqueName], resourceTypes, (unsigned long)permissions]]; @@ -302,7 +306,6 @@ - (void)testBlobAccountSasCombinationsWithResourceTypes:(AZSSharedAccessResource void (^createBlob)(id, void(^)(NSError *)) = ^void(id blob, void(^completionHandler)(NSError *)) { // TODO: Add check for create permissions once our library allows creating empty blobs - [blob uploadFromText:@"test data" completionHandler:^(NSError *err) { BOOL shouldPass = !((AZSCloudBlob *) blob).blobContainer.client.credentials.isSAS || [self isAccessAllowedWithParameters:sp acceptedPermissions:AZSSharedAccessPermissionsWrite diff --git a/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSBlobOutputStreamTests.m b/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSBlobOutputStreamTests.m deleted file mode 100644 index d492a4b..0000000 --- a/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSBlobOutputStreamTests.m +++ /dev/null @@ -1,443 +0,0 @@ -// ----------------------------------------------------------------------------------------- -// -// Copyright 2015 Microsoft Corporation -// -// Licensed under the MIT License; -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://spdx.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// ----------------------------------------------------------------------------------------- - -#import -#import -#import "AZSConstants.h" -#import "AZSBlobTestBase.h" -#import "AZSTestHelpers.h" -#import "AZSTestSemaphore.h" -#import "AZSClient.h" - -@interface AZSBlobUploadTestDelegate : NSObject - -@property (nonatomic, copy) void (^streamEventBlock)(NSStream *, NSStreamEvent); - --(instancetype)initWithBlock:(void(^)(NSStream *, NSStreamEvent))block; --(void)stream:(NSStream *)stream handleEvent:(NSStreamEvent)eventCode; - -@end - -@implementation AZSBlobUploadTestDelegate - --(void)stream:(NSStream *)stream handleEvent:(NSStreamEvent)eventCode -{ - self.streamEventBlock(stream, eventCode); -} - --(instancetype)initWithBlock:(void (^)(NSStream *, NSStreamEvent))block -{ - self = [super init]; - if (self) - { - self.streamEventBlock = block; - } - return self; -} - -@end - -@interface AZSBlobOutputStreamTests : AZSBlobTestBase -@property NSString *containerName; -@property AZSCloudBlobContainer *blobContainer; - -@end - - -@implementation AZSBlobOutputStreamTests - -- (void)setUp -{ - [super setUp]; - self.containerName = [NSString stringWithFormat:@"sampleioscontainer%@", [AZSTestHelpers uniqueName]]; - self.blobContainer = [self.blobClient containerReferenceFromName:self.containerName]; - AZSTestSemaphore *semaphore = [[AZSTestSemaphore alloc] init]; - - [self.blobContainer createContainerIfNotExistsWithCompletionHandler:^(NSError *error, BOOL exists) { - [semaphore signal]; - }]; - [semaphore wait]; - - // Put setup code here; it will be run once, before the first test case. -} - -- (void)tearDown -{ - // Put teardown code here; it will be run once, after the last test case. - AZSCloudBlobContainer *blobContainer = [self.blobClient containerReferenceFromName:self.containerName]; - AZSTestSemaphore *semaphore = [[AZSTestSemaphore alloc] init]; - - [blobContainer deleteContainerIfExistsWithCompletionHandler:^(NSError *error, BOOL exists) { - [semaphore signal]; - }]; - [semaphore wait]; - [super tearDown]; -} - -- (BOOL)runInternalTestOutputStreamWithBlobSize:(NSUInteger)blobSize blobToUpload:(AZSCloudBlob *)blob createOutputStreamCall:(AZSBlobOutputStream *(^)(AZSAccessCondition *, AZSBlobRequestOptions *, AZSOperationContext *))createOutputStreamCall -{ - AZSTestSemaphore *semaphore = [[AZSTestSemaphore alloc] init]; - unsigned int __block randSeed = (unsigned int)time(NULL); - - AZSUIntegerHolder *randSeedHolderCopy = [[AZSUIntegerHolder alloc]initWithNumber:randSeed]; - NSUInteger writeSize = 10000; - NSUInteger __block bytesWritten = 0; - BOOL __block uploadFailed = NO; - BOOL __block testFailedInDownload = NO; - - AZSBlobUploadTestDelegate *uploadDelegate = [[AZSBlobUploadTestDelegate alloc] initWithBlock:^(NSStream *stream, NSStreamEvent eventCode) { - NSOutputStream *outputStream = (NSOutputStream *)stream; - switch (eventCode) { - case NSStreamEventHasSpaceAvailable: - { - uint8_t buf[writeSize]; - int i = 0; - for (i = 0; i < writeSize && bytesWritten < blobSize; i++) - { - NSUInteger byteval = rand_r(&(randSeedHolderCopy->number)) % 256; - buf[i] = byteval; - bytesWritten++; - } - - NSInteger curBytesWritten = [outputStream write:(const uint8_t *)buf maxLength:i]; - if (curBytesWritten != writeSize) - { - XCTAssertTrue(NO, @"Incorrect number of bytes written to the stream."); - uploadFailed = YES; - } - break; - } - - default: - break; - } - }]; - - AZSBlobRequestOptions *options = [[AZSBlobRequestOptions alloc] init]; - options.absorbConditionalErrorsOnRetry = YES; - - AZSBlobOutputStream *blobOutputStream = createOutputStreamCall(nil, options, nil); - [blobOutputStream setDelegate:uploadDelegate]; - [blobOutputStream scheduleInRunLoop:[NSRunLoop currentRunLoop] forMode:NSDefaultRunLoopMode]; - [blobOutputStream open]; - - while ((!uploadFailed) && (bytesWritten < blobSize)) - { - BOOL loopSuccess = [[NSRunLoop currentRunLoop] runMode:NSDefaultRunLoopMode beforeDate:[NSDate distantFuture]]; - if (!loopSuccess) - { - break; - } - } - - [blobOutputStream close]; - [blobOutputStream removeFromRunLoop:[NSRunLoop currentRunLoop] forMode:NSDefaultRunLoopMode]; - - AZSByteValidationStream *targetStream = [[AZSByteValidationStream alloc]initWithRandomSeed:randSeed totalBlobSize:blobSize isUpload:NO]; - - [blob downloadToStream:((NSOutputStream *)targetStream) accessCondition:nil requestOptions:nil operationContext:nil completionHandler:^(NSError * error) { - XCTAssertNil(error, @"Error in downloading blob. Error code = %ld, error domain = %@, error userinfo = %@", (long)error.code, error.domain, error.userInfo); - XCTAssert(targetStream.totalBytes == blobSize, @"Downloaded blob is wrong size. Size = %ld, expected size = %ld", (unsigned long)targetStream.totalBytes, (unsigned long)(blobSize)); - XCTAssertFalse(targetStream.dataCorrupt, @"Downloaded blob is corrupt. Error count = %ld, first few errors = sample\nsample%@", (unsigned long)targetStream.errorCount, targetStream.errors); - - if ((error != nil) || (targetStream.totalBytes != blobSize) || (targetStream.dataCorrupt)) - { - testFailedInDownload = YES; - } - - [semaphore signal]; - }]; - [semaphore wait]; - return testFailedInDownload; -} - -// Note: This test works with ~200 MB blobs (you can watch memory consumption, it doesn't rise that far), -// but it takes a while. --(void)testBlockBlobOutputStreamIterate -{ - int failures = 0; - for (int i = 0; i < 2; i++) - { - @autoreleasepool { - NSString *blobName = @"blobName"; - AZSCloudBlockBlob *blob = [self.blobContainer blockBlobReferenceFromName:blobName]; - if ([self runInternalTestOutputStreamWithBlobSize:20000000 blobToUpload:blob createOutputStreamCall:^AZSBlobOutputStream *(AZSAccessCondition *accessCondition, AZSBlobRequestOptions *requestOptions, AZSOperationContext *operationContext) { - return [blob createOutputStreamWithAccessCondition:accessCondition requestOptions:requestOptions operationContext:operationContext]; - }]) - { - failures++; - } - } - } - XCTAssertEqual(0, failures, @"%d failure(s) detected.", failures); -} - -// Note: This test works with ~200 MB blobs (you can watch memory consumption, it doesn't rise that far), -// but it takes a while. --(void)testPageBlobOutputStreamCreateNewIterate -{ - int blobSize = 512*40000; - int failures = 0; - for (int i = 0; i < 2; i++) - { - @autoreleasepool { - NSString *blobName = @"blobName"; - AZSCloudPageBlob *blob = [self.blobContainer pageBlobReferenceFromName:blobName]; - if ([self runInternalTestOutputStreamWithBlobSize:blobSize blobToUpload:blob createOutputStreamCall:^AZSBlobOutputStream *(AZSAccessCondition *accessCondition, AZSBlobRequestOptions *requestOptions, AZSOperationContext *operationContext) { - return [blob createOutputStreamWithSize:[NSNumber numberWithInt:blobSize] sequenceNumber:[NSNumber numberWithInt:100] accessCondition:accessCondition requestOptions:requestOptions operationContext:operationContext]; - }]) - { - failures++; - } - } - } - XCTAssertEqual(0, failures, @"%d failure(s) detected.", failures); -} - -// Note: This test works with ~200 MB blobs (you can watch memory consumption, it doesn't rise that far), -// but it takes a while. --(void)testPageBlobOutputStreamExistingIterate -{ - int blobSize = 512*40000; - __block int failures = 0; - for (int i = 0; i < 2; i++) - { - @autoreleasepool { - AZSTestSemaphore *semaphore = [[AZSTestSemaphore alloc] init]; - NSString *blobName = @"blobName"; - AZSCloudPageBlob *blob = [self.blobContainer pageBlobReferenceFromName:blobName]; - [blob createWithSize:[NSNumber numberWithInt:blobSize] completionHandler:^(NSError * _Nullable error) { - XCTAssertNil(error, @"Error in creating blob. Error code = %ld, error domain = %@, error userinfo = %@", (long)error.code, error.domain, error.userInfo); - if ([self runInternalTestOutputStreamWithBlobSize:blobSize blobToUpload:blob createOutputStreamCall:^AZSBlobOutputStream *(AZSAccessCondition *accessCondition, AZSBlobRequestOptions *requestOptions, AZSOperationContext *operationContext) { - return [blob createOutputStreamWithAccessCondition:accessCondition requestOptions:requestOptions operationContext:operationContext]; - }]) - { - failures++; - } - [semaphore signal]; - }]; - [semaphore wait]; - } - } - XCTAssertEqual(0, failures, @"%d failure(s) detected.", failures); -} - - -// Note: This test works with ~200 MB blobs (you can watch memory consumption, it doesn't rise that far), -// but it takes a while. --(void)testAppendBlobOutputStreamCreateNewIterate -{ - int failures = 0; - for (int i = 0; i < 2; i++) - { - @autoreleasepool { - NSString *blobName = @"blobName"; - AZSCloudAppendBlob *blob = [self.blobContainer appendBlobReferenceFromName:blobName]; - if ([self runInternalTestOutputStreamWithBlobSize:20000000 blobToUpload:blob createOutputStreamCall:^AZSBlobOutputStream *(AZSAccessCondition *accessCondition, AZSBlobRequestOptions *requestOptions, AZSOperationContext *operationContext) { - return [blob createOutputStreamWithCreateNew:YES accessCondition:accessCondition requestOptions:requestOptions operationContext:operationContext]; - }]) - { - failures++; - } - } - } - XCTAssertEqual(0, failures, @"%d failure(s) detected.", failures); -} - -// Note: This test works with ~200 MB blobs (you can watch memory consumption, it doesn't rise that far), -// but it takes a while. --(void)testAppendBlobOutputStreamUseExistingIterate -{ - __block int failures = 0; - for (int i = 0; i < 2; i++) - { - @autoreleasepool { - AZSTestSemaphore *semaphore = [[AZSTestSemaphore alloc] init]; - NSString *blobName = @"blobName"; - AZSCloudAppendBlob *blob = [self.blobContainer appendBlobReferenceFromName:blobName]; - [blob createWithCompletionHandler:^(NSError * _Nullable error) { - XCTAssertNil(error, @"Error in creating blob. Error code = %ld, error domain = %@, error userinfo = %@", (long)error.code, error.domain, error.userInfo); - if ([self runInternalTestOutputStreamWithBlobSize:20000000 blobToUpload:blob createOutputStreamCall:^AZSBlobOutputStream *(AZSAccessCondition *accessCondition, AZSBlobRequestOptions *requestOptions, AZSOperationContext *operationContext) { - return [blob createOutputStreamWithCreateNew:NO accessCondition:accessCondition requestOptions:requestOptions operationContext:operationContext]; - }]) - { - failures++; - } - [semaphore signal]; - }]; - [semaphore wait]; - } - } - XCTAssertEqual(0, failures, @"%d failure(s) detected.", failures); -} - --(BOOL)runTestUploadFromStreamWithBlobSize:(NSUInteger)blobSize blobToUpload:(AZSCloudBlob *)blob uploadCall:(void (^)(NSInputStream * , AZSAccessCondition *, AZSBlobRequestOptions *, AZSOperationContext *, void(^)(NSError * error)))uploadCall -{ - AZSTestSemaphore *semaphore = [[AZSTestSemaphore alloc] init]; - unsigned int __block randSeed = (unsigned int)time(NULL); - BOOL __block testFailedInDownload = NO; - - AZSByteValidationStream *sourceStream = [[AZSByteValidationStream alloc]initWithRandomSeed:randSeed totalBlobSize:blobSize isUpload:YES]; - AZSBlobRequestOptions *options = [[AZSBlobRequestOptions alloc] init]; - options.maximumDownloadBufferSize = 20000000; - options.parallelismFactor = 2; - options.absorbConditionalErrorsOnRetry = YES; - - uploadCall((NSInputStream *)sourceStream, nil, options, nil, ^(NSError * error) { - XCTAssertNil(error, @"Error in uploading blob. Error code = %ld, error domain = %@, error userinfo = %@", (long)error.code, error.domain, error.userInfo); - - AZSByteValidationStream *targetStream =[[AZSByteValidationStream alloc]initWithRandomSeed:randSeed totalBlobSize:blobSize isUpload:NO]; - - [blob downloadToStream:((NSOutputStream *)targetStream) accessCondition:nil requestOptions:options operationContext:nil completionHandler:^(NSError * error) { - XCTAssertNil(error, @"Error in downloading blob. Error code = %ld, error domain = %@, error userinfo = %@", (long)error.code, error.domain, error.userInfo); - XCTAssert(targetStream.totalBytes == blobSize, @"Downloaded blob is wrong size. Size = %ld, expected size = %ld", (unsigned long)targetStream.totalBytes, (unsigned long)(blobSize)); - XCTAssertFalse(targetStream.dataCorrupt, @"Downloaded blob is corrupt. Error count = %ld, first few errors = sample\nsample%@", (unsigned long)targetStream.errorCount, targetStream.errors); - - if ((error != nil) || (targetStream.totalBytes != blobSize) || (targetStream.dataCorrupt)) - { - testFailedInDownload = YES; - } - - [semaphore signal]; - }]; - - }); - - [semaphore wait]; - return testFailedInDownload; -} - --(void)testBlockBlobFromToStreamIterate -{ - int failures = 0; - for (int i = 0; i < 2; i++) - { - @autoreleasepool { - - NSString *blobName = @"blobName"; - AZSCloudBlockBlob *blob = [self.blobContainer blockBlobReferenceFromName:blobName]; - - if ([self runTestUploadFromStreamWithBlobSize:20000000 blobToUpload:blob uploadCall:^(NSInputStream *sourceStream, AZSAccessCondition *accessCondition, AZSBlobRequestOptions *blobRequestOptions, AZSOperationContext *operationContext, void(^completionHandler)(NSError * error)) { - [blob uploadFromStream:sourceStream accessCondition:accessCondition requestOptions:blobRequestOptions operationContext:operationContext completionHandler:completionHandler]; - }]) - { - failures++; - } - } - } - XCTAssertEqual(0, failures, @"%d failure(s) detected.", failures); -} - --(void)testPageBlobFromToStreamUseExistingIterate -{ - int blobSize = 512*40000; - __block int failures = 0; - for (int i = 0; i < 2; i++) - { - @autoreleasepool { - AZSTestSemaphore *semaphore = [[AZSTestSemaphore alloc] init]; - NSString *blobName = @"blobName"; - AZSCloudPageBlob *blob = [self.blobContainer pageBlobReferenceFromName:blobName]; - - [blob createWithSize:[NSNumber numberWithInt:blobSize] completionHandler:^(NSError * _Nullable error) { - XCTAssertNil(error, @"Error in creating blob. Error code = %ld, error domain = %@, error userinfo = %@", (long)error.code, error.domain, error.userInfo); - if ([self runTestUploadFromStreamWithBlobSize:blobSize blobToUpload:blob uploadCall:^(NSInputStream *sourceStream, AZSAccessCondition *accessCondition, AZSBlobRequestOptions *blobRequestOptions, AZSOperationContext *operationContext, void(^completionHandler)(NSError * error)) { - [blob uploadFromStream:sourceStream accessCondition:accessCondition requestOptions:blobRequestOptions operationContext:operationContext completionHandler:completionHandler]; - }]) - { - failures++; - } - [semaphore signal]; - }]; - [semaphore wait]; - } - } - XCTAssertEqual(0, failures, @"%d failure(s) detected.", failures); -} - --(void)testPageBlobFromToStreamCreateNewIterate -{ - int blobSize = 512*40000; - int failures = 0; - for (int i = 0; i < 2; i++) - { - @autoreleasepool { - - NSString *blobName = @"blobName"; - AZSCloudPageBlob *blob = [self.blobContainer pageBlobReferenceFromName:blobName]; - - if ([self runTestUploadFromStreamWithBlobSize:blobSize blobToUpload:blob uploadCall:^(NSInputStream *sourceStream, AZSAccessCondition *accessCondition, AZSBlobRequestOptions *blobRequestOptions, AZSOperationContext *operationContext, void(^completionHandler)(NSError * error)) { - [blob uploadFromStream:sourceStream size:[NSNumber numberWithInt:blobSize] initialSequenceNumber:[NSNumber numberWithInt:100] accessCondition:accessCondition requestOptions:blobRequestOptions operationContext:operationContext completionHandler:completionHandler]; - }]) - { - failures++; - } - } - } - XCTAssertEqual(0, failures, @"%d failure(s) detected.", failures); -} - --(void)testAppendBlobFromToStreamUseExistingIterate -{ - int blobSize = 20000000; - __block int failures = 0; - for (int i = 0; i < 2; i++) - { - @autoreleasepool { - AZSTestSemaphore *semaphore = [[AZSTestSemaphore alloc] init]; - NSString *blobName = @"blobName"; - AZSCloudAppendBlob *blob = [self.blobContainer appendBlobReferenceFromName:blobName]; - - [blob createWithCompletionHandler:^(NSError * _Nullable error) { - XCTAssertNil(error, @"Error in creating blob. Error code = %ld, error domain = %@, error userinfo = %@", (long)error.code, error.domain, error.userInfo); - if ([self runTestUploadFromStreamWithBlobSize:blobSize blobToUpload:blob uploadCall:^(NSInputStream *sourceStream, AZSAccessCondition *accessCondition, AZSBlobRequestOptions *blobRequestOptions, AZSOperationContext *operationContext, void(^completionHandler)(NSError * error)) { - [blob uploadFromStream:sourceStream createNew:NO accessCondition:accessCondition requestOptions:blobRequestOptions operationContext:operationContext completionHandler:completionHandler]; - }]) - { - failures++; - } - [semaphore signal]; - }]; - [semaphore wait]; - } - } - XCTAssertEqual(0, failures, @"%d failure(s) detected.", failures); -} - --(void)testAppendBlobFromToStreamCreateNewIterate -{ - int blobSize = 20000000; - int failures = 0; - for (int i = 0; i < 2; i++) - { - @autoreleasepool { - - NSString *blobName = @"blobName"; - AZSCloudAppendBlob *blob = [self.blobContainer appendBlobReferenceFromName:blobName]; - - if ([self runTestUploadFromStreamWithBlobSize:blobSize blobToUpload:blob uploadCall:^(NSInputStream *sourceStream, AZSAccessCondition *accessCondition, AZSBlobRequestOptions *blobRequestOptions, AZSOperationContext *operationContext, void(^completionHandler)(NSError * error)) { - [blob uploadFromStream:sourceStream createNew:YES accessCondition:accessCondition requestOptions:blobRequestOptions operationContext:operationContext completionHandler:completionHandler]; - }]) - { - failures++; - } - } - } - XCTAssertEqual(0, failures, @"%d failure(s) detected.", failures); -} - -@end \ No newline at end of file diff --git a/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSBlobStreamTests.m b/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSBlobStreamTests.m new file mode 100644 index 0000000..32beb96 --- /dev/null +++ b/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSBlobStreamTests.m @@ -0,0 +1,508 @@ +// ----------------------------------------------------------------------------------------- +// +// Copyright 2015 Microsoft Corporation +// +// Licensed under the MIT License; +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://spdx.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------------------- + +#import +#import +#import "AZSConstants.h" +#import "AZSBlobInputStream.h" +#import "AZSBlobTestBase.h" +#import "AZSStreamDownloadBuffer.h" +#import "AZSTestHelpers.h" +#import "AZSTestSemaphore.h" +#import "AZSClient.h" + +@interface AZSBlobStreamTests : AZSBlobTestBase + +@property NSString *containerName; +@property AZSCloudBlobContainer *blobContainer; + +@end + + +@implementation AZSBlobStreamTests + +- (void)setUp +{ + [super setUp]; + self.containerName = [NSString stringWithFormat:@"sampleioscontainer%@", [AZSTestHelpers uniqueName]]; + self.blobContainer = [self.blobClient containerReferenceFromName:self.containerName]; + AZSTestSemaphore *semaphore = [[AZSTestSemaphore alloc] init]; + + [self.blobContainer createContainerIfNotExistsWithCompletionHandler:^(NSError *error, BOOL exists) { + [semaphore signal]; + }]; + [semaphore wait]; + + // Put setup code here; it will be run once, before the first test case. +} + +- (void)tearDown +{ + // Put teardown code here; it will be run once, after the last test case. + AZSCloudBlobContainer *blobContainer = [self.blobClient containerReferenceFromName:self.containerName]; + AZSTestSemaphore *semaphore = [[AZSTestSemaphore alloc] init]; + + [blobContainer deleteContainerIfExistsWithCompletionHandler:^(NSError *error, BOOL exists) { + [semaphore signal]; + }]; + [semaphore wait]; + [super tearDown]; +} + +- (int)runInternalTestStreamWithBlobSize:(const NSUInteger)blobSize blobToUpload:(AZSCloudBlob *)blob runloop:(NSRunLoop *)runloop createOutputStreamCall:(AZSBlobOutputStream *(^)(AZSAccessCondition *, AZSBlobRequestOptions *, AZSOperationContext *))createOutputStreamCall +{ + int __block testsFailedInDownload = 0; + unsigned int randSeed = (unsigned int)time(NULL); + BOOL useCurrentRunloop = !runloop; + runloop = runloop ?: [NSRunLoop currentRunLoop]; + + // step 1: upload the blob + AZSByteValidationStream *uploadDelegate = [[AZSByteValidationStream alloc] initWithRandomSeed:randSeed totalBlobSize:blobSize isUpload:NO failBlock:^(NSString *message) { + XCTFail(@"%@", message); + }]; + AZSBlobRequestOptions *options = [[AZSBlobRequestOptions alloc] init]; + options.absorbConditionalErrorsOnRetry = YES; + + AZSBlobOutputStream *blobOutputStream = createOutputStreamCall(nil, options, nil); + [blobOutputStream scheduleInRunLoop:runloop forMode:NSDefaultRunLoopMode]; + [blobOutputStream setDelegate:uploadDelegate]; + [blobOutputStream open]; + + // make the runloop iterate until either all the bytes are uploaded or an error is encountered + BOOL loopSuccess = YES; + while (loopSuccess && (!uploadDelegate.streamFailed) && (uploadDelegate.bytesWritten < blobSize)) { + if (useCurrentRunloop) { + // run the loop once + loopSuccess = [runloop runMode:NSDefaultRunLoopMode beforeDate:[NSDate dateWithTimeIntervalSinceNow:2]]; + } + else { + [NSThread sleepForTimeInterval:.25]; + } + } + + [blobOutputStream close]; + + // continue the runloop until the stream is properly closed + loopSuccess = YES; + while (loopSuccess && blobOutputStream.streamStatus != NSStreamStatusClosed) + { + if (useCurrentRunloop) { + loopSuccess = [runloop runMode:NSDefaultRunLoopMode beforeDate:[NSDate dateWithTimeIntervalSinceNow:2]]; + } + else { + [NSThread sleepForTimeInterval:.25]; + } + } + [blobOutputStream removeFromRunLoop:runloop forMode:NSDefaultRunLoopMode]; + + // Step 2: download the blob to an Input Stream + AZSByteValidationStream *downloadDelegate = [[AZSByteValidationStream alloc] initWithRandomSeed:randSeed totalBlobSize:blobSize isUpload:NO failBlock:^(NSString *message) { + XCTFail(@"%@", message); + }]; + + options.runLoopForDownload = runloop; + AZSBlobInputStream *readStream = [blob createInputStreamWithAccessCondition:nil requestOptions:options operationContext:nil]; + + [readStream scheduleInRunLoop:runloop forMode:NSDefaultRunLoopMode]; + [readStream setDelegate:downloadDelegate]; + [readStream open]; + + loopSuccess = YES; + while (loopSuccess && !downloadDelegate.streamFailed && !readStream.downloadBuffer.downloadComplete) { + if (useCurrentRunloop) { + loopSuccess = [runloop runMode:NSDefaultRunLoopMode beforeDate:[NSDate dateWithTimeIntervalSinceNow:2]]; + } + else { + [NSThread sleepForTimeInterval:.25]; + } + } + + [readStream close]; + [readStream removeFromRunLoop:runloop forMode:NSDefaultRunLoopMode]; + + // Step 3: download to Output Stream and validate the blob content + AZSByteValidationStream *downloadStream = [[AZSByteValidationStream alloc] initWithRandomSeed:randSeed totalBlobSize:blobSize isUpload:NO failBlock:^(NSString *message) { + XCTFail(@"%@", message); + }]; + + AZSTestSemaphore *semaphore = [[AZSTestSemaphore alloc] init]; + [blob downloadToStream:((NSOutputStream *) downloadStream) accessCondition:nil requestOptions:nil operationContext:nil completionHandler:^(NSError * error) { + XCTAssertNil(error, @"Error in downloading blob. Error code = %ld, error domain = %@, error userinfo = %@", (long)error.code, error.domain, error.userInfo); + XCTAssertEqual(downloadStream.totalBytes, blobSize, @"Downloaded blob is wrong size. Size = %ld, expected size = %ld", (unsigned long)downloadStream.totalBytes, (unsigned long)blobSize); + XCTAssertFalse(downloadStream.dataCorrupt, @"Downloaded blob is corrupt. Error count = %ld, first few errors = sample\nsample%@", (unsigned long)downloadStream.errorCount, downloadStream.errors); + + if ((error != nil) || (downloadStream.totalBytes != blobSize) || downloadStream.dataCorrupt) { + testsFailedInDownload++; + } + + [semaphore signal]; + }]; + [semaphore wait]; + + return testsFailedInDownload; +} + +-(void)testBlockBlobStreamError +{ + AZSCloudBlockBlob *blob = [[AZSCloudBlockBlob alloc] initWithContainer:self.blobContainer name:[AZSTestHelpers uniqueName]]; + + // Test InputStream + AZSBlobInputStream *inputStream = [blob createInputStream]; + __block BOOL downloadFailed = NO; + + AZSByteValidationStream *downloadDelegate = [[AZSByteValidationStream alloc] initWithRandomSeed:(unsigned int)time(NULL) totalBlobSize:0 isUpload:NO failBlock:^(NSString *message) { + [self checkPassageOfError:inputStream.streamError expectToPass:NO expectedHttpErrorCode:404 message:@"Blob unexpectedly found."]; + downloadFailed = YES; + }]; + + AZSTestSemaphore *semaphore = [[AZSTestSemaphore alloc] init]; + NSRunLoop *runloop = [self runloopWithSemaphore:semaphore]; + + [inputStream scheduleInRunLoop:runloop forMode:NSDefaultRunLoopMode]; + [inputStream setDelegate:downloadDelegate]; + [inputStream open]; + + while (!downloadDelegate.streamFailed && !downloadDelegate.downloadComplete) { + // TODO: Reduce when error handling is fixed. + [NSThread sleepForTimeInterval:2]; + } + + [semaphore signal]; + [inputStream close]; + + XCTAssertTrue(downloadFailed, @"Download unexpectedly succeeded."); + XCTAssertEqual(0, downloadDelegate.totalBytes); + + // Test OutputStream + semaphore = [[AZSTestSemaphore alloc] init]; + NSData *data = [NSData dataWithBytesNoCopy:calloc(8, sizeof(uint8_t)) length:8 freeWhenDone:YES]; + NSOutputStream *outputStream = [NSOutputStream outputStreamToBuffer:(uint8_t*)data.bytes capacity:8]; + + [blob downloadToStream:outputStream completionHandler:^(NSError * error) { + [self checkPassageOfError:error expectToPass:NO expectedHttpErrorCode:404 message:@"Blob unexpectedly found."]; + [semaphore signal]; + }]; + [semaphore wait]; + + for (int i = 0; i < data.length; i++) { + XCTAssertEqual(0, ((uint8_t*) data.bytes)[i]); + } +} + +// Note: This test works with ~20 MB blobs (you can watch memory consumption, it doesn't rise that far), +// but it takes a while. TOASK should they be 200MB instead? +-(void)testBlockBlobStreamIterate +{ + int failures = 0; + NSRunLoop *runloop = nil; + AZSTestSemaphore *semaphore = [[AZSTestSemaphore alloc] init]; + + for (int i = 0; i < 2; i++) + { + @autoreleasepool { + NSString *blobName = @"blobName"; + AZSCloudBlockBlob *blob = [self.blobContainer blockBlobReferenceFromName:blobName]; + failures += [self runInternalTestStreamWithBlobSize:20000000 blobToUpload:blob runloop:runloop createOutputStreamCall:^AZSBlobOutputStream *(AZSAccessCondition *accessCondition, AZSBlobRequestOptions *requestOptions, AZSOperationContext *operationContext) { + return [blob createOutputStreamWithAccessCondition:accessCondition requestOptions:requestOptions operationContext:operationContext]; + }]; + } + + runloop = runloop ?: [self runloopWithSemaphore:semaphore]; + } + + [semaphore signal]; + XCTAssertEqual(0, failures, @"%d failure(s) detected.", failures); +} + +// Note: This test works with ~20 MB blobs (you can watch memory consumption, it doesn't rise that far), +// but it takes a while. +-(void)testPageBlobStreamCreateNewIterate +{ + int blobSize = 512*40000; + int failures = 0; + NSRunLoop *runloop = nil; + AZSTestSemaphore *semaphore = [[AZSTestSemaphore alloc] init]; + + for (int i = 0; i < 2; i++) + { + @autoreleasepool { + NSString *blobName = @"blobName"; + AZSCloudPageBlob *blob = [self.blobContainer pageBlobReferenceFromName:blobName]; + failures += [self runInternalTestStreamWithBlobSize:blobSize blobToUpload:blob runloop:runloop createOutputStreamCall:^AZSBlobOutputStream *(AZSAccessCondition *accessCondition, AZSBlobRequestOptions *requestOptions, AZSOperationContext *operationContext) { + return [blob createOutputStreamWithSize:[NSNumber numberWithInt:blobSize] sequenceNumber:[NSNumber numberWithInt:100] accessCondition:accessCondition requestOptions:requestOptions operationContext:operationContext]; + }]; + } + + runloop = [self runloopWithSemaphore:semaphore]; + } + + [semaphore signal]; + XCTAssertEqual(0, failures, @"%d failure(s) detected.", failures); +} + +// Note: This test works with ~20 MB blobs (you can watch memory consumption, it doesn't rise that far), +// but it takes a while. +-(void)testPageBlobStreamExistingIterate +{ + int blobSize = 512*40000; + __block int failures = 0; + NSRunLoop *runloop = nil; + AZSTestSemaphore *runloopSemaphore = [[AZSTestSemaphore alloc] init]; + + for (int i = 0; i < 2; i++) + { + @autoreleasepool { + AZSTestSemaphore *semaphore = [[AZSTestSemaphore alloc] init]; + NSString *blobName = @"blobName"; + AZSCloudPageBlob *blob = [self.blobContainer pageBlobReferenceFromName:blobName]; + [blob createWithSize:[NSNumber numberWithInt:blobSize] completionHandler:^(NSError * __AZSNullable error) { + XCTAssertNil(error, @"Error in creating blob. Error code = %ld, error domain = %@, error userinfo = %@", (long)error.code, error.domain, error.userInfo); + failures += [self runInternalTestStreamWithBlobSize:blobSize blobToUpload:blob runloop:runloop createOutputStreamCall:^AZSBlobOutputStream *(AZSAccessCondition *accessCondition, AZSBlobRequestOptions *requestOptions, AZSOperationContext *operationContext) { + return [blob createOutputStreamWithAccessCondition:accessCondition requestOptions:requestOptions operationContext:operationContext]; + }]; + + [semaphore signal]; + }]; + + [semaphore wait]; + } + + runloop = [self runloopWithSemaphore:runloopSemaphore]; + } + + [runloopSemaphore signal]; + XCTAssertEqual(0, failures, @"%d failure(s) detected.", failures); +} + + +// Note: This test works with ~20 MB blobs (you can watch memory consumption, it doesn't rise that far), +// but it takes a while. +-(void)testAppendBlobStreamCreateNewIterate +{ + int failures = 0; + NSRunLoop *runloop = nil; + AZSTestSemaphore *semaphore = [[AZSTestSemaphore alloc] init]; + + for (int i = 0; i < 2; i++) + { + @autoreleasepool { + NSString *blobName = @"blobName"; + AZSCloudAppendBlob *blob = [self.blobContainer appendBlobReferenceFromName:blobName]; + failures += [self runInternalTestStreamWithBlobSize:20000000 blobToUpload:blob runloop:runloop createOutputStreamCall:^AZSBlobOutputStream *(AZSAccessCondition *accessCondition, AZSBlobRequestOptions *requestOptions, AZSOperationContext *operationContext) { + return [blob createOutputStreamWithCreateNew:YES accessCondition:accessCondition requestOptions:requestOptions operationContext:operationContext]; + }]; + } + + runloop = [self runloopWithSemaphore:semaphore]; + } + + [semaphore signal]; + XCTAssertEqual(0, failures, @"%d failure(s) detected.", failures); +} + +// Note: This test works with ~20 MB blobs (you can watch memory consumption, it doesn't rise that far), +// but it takes a while. +-(void)testAppendBlobStreamUseExistingIterate +{ + __block int failures = 0; + NSRunLoop *runloop = nil; + AZSTestSemaphore *runloopSemaphore = [[AZSTestSemaphore alloc] init]; + + for (int i = 0; i < 2; i++) + { + @autoreleasepool { + AZSTestSemaphore *semaphore = [[AZSTestSemaphore alloc] init]; + NSString *blobName = @"blobName"; + AZSCloudAppendBlob *blob = [self.blobContainer appendBlobReferenceFromName:blobName]; + + [blob createWithCompletionHandler:^(NSError * __AZSNullable error) { + XCTAssertNil(error, @"Error in creating blob. Error code = %ld, error domain = %@, error userinfo = %@", (long)error.code, error.domain, error.userInfo); + failures += [self runInternalTestStreamWithBlobSize:20000000 blobToUpload:blob runloop:runloop createOutputStreamCall:^AZSBlobOutputStream *(AZSAccessCondition *accessCondition, AZSBlobRequestOptions *requestOptions, AZSOperationContext *operationContext) { + return [blob createOutputStreamWithCreateNew:NO accessCondition:accessCondition requestOptions:requestOptions operationContext:operationContext]; + }]; + + [semaphore signal]; + }]; + [semaphore wait]; + } + + runloop = [self runloopWithSemaphore:runloopSemaphore]; + } + + [runloopSemaphore signal]; + XCTAssertEqual(0, failures, @"%d failure(s) detected.", failures); +} + +// TOASK this is doing the same thing as runInternalTestStreamWithBlobSize, right? +-(int)runTestUploadFromStreamWithBlobSize:(const NSUInteger)blobSize blobToUpload:(AZSCloudBlob *)blob uploadCall:(void (^)(NSInputStream * , AZSAccessCondition *, AZSBlobRequestOptions *, AZSOperationContext *, void(^)(NSError * error)))uploadCall +{ + AZSTestSemaphore *semaphore = [[AZSTestSemaphore alloc] init]; + unsigned int __block randSeed = (unsigned int)time(NULL); + int __block testsFailedInDownload = 0; + + AZSByteValidationStream *sourceStream = [[AZSByteValidationStream alloc]initWithRandomSeed:randSeed totalBlobSize:blobSize isUpload:YES failBlock:^(NSString *message) { + XCTFail(@"%@", message); + }]; + AZSBlobRequestOptions *options = [[AZSBlobRequestOptions alloc] init]; + options.parallelismFactor = 2; + options.absorbConditionalErrorsOnRetry = YES; + + uploadCall((NSInputStream *)sourceStream, nil, options, nil, ^(NSError * error) { + XCTAssertNil(error, @"Error in uploading blob. Error code = %ld, error domain = %@, error userinfo = %@", (long)error.code, error.domain, error.userInfo); + + AZSByteValidationStream *targetStream =[[AZSByteValidationStream alloc]initWithRandomSeed:randSeed totalBlobSize:blobSize isUpload:NO failBlock:^(NSString *message) { + XCTFail(@"%@", message); + }]; + + AZSBlobInputStream *readStream = [blob createInputStream]; + + AZSBlobTestSpinWrapper *wrapper = [[AZSBlobTestSpinWrapper alloc] init]; + wrapper.stream = readStream; + wrapper.delegate = targetStream; + + wrapper.completionHandler = ^() { + AZSByteValidationStream *newTargetStream = [[AZSByteValidationStream alloc] initWithRandomSeed:randSeed totalBlobSize:blobSize isUpload:NO failBlock:^(NSString *message) { + XCTFail(@"%@", message); + }]; + + [blob downloadToStream:((NSOutputStream *)newTargetStream) accessCondition:nil requestOptions:options operationContext:nil completionHandler:^(NSError * error) { + XCTAssertNil(error, @"Error in downloading blob. Error code = %ld, error domain = %@, error userinfo = %@", (long)error.code, error.domain, error.userInfo); + XCTAssert(newTargetStream.totalBytes == blobSize, @"Downloaded blob is wrong size. Size = %ld, expected size = %ld", (unsigned long)newTargetStream.totalBytes, (unsigned long)(blobSize)); + XCTAssertFalse(newTargetStream.dataCorrupt, @"Downloaded blob is corrupt. Error count = %ld, first few errors = sample\nsample%@", (unsigned long)newTargetStream.errorCount, newTargetStream.errors); + + if ((error != nil) || (newTargetStream.totalBytes != blobSize) || (newTargetStream.dataCorrupt)) + { + testsFailedInDownload++; + } + + [semaphore signal]; + }]; + }; + + [NSThread detachNewThreadSelector:@selector(scheduleStreamInNewThreaAndRunWithWrapper:) toTarget:self withObject:wrapper]; + }); + + [semaphore wait]; + return testsFailedInDownload; +} + +-(void)testBlockBlobFromToStreamIterate +{ + int failures = 0; + for (int i = 0; i < 2; i++) + { + @autoreleasepool { + + NSString *blobName = @"blobName"; + AZSCloudBlockBlob *blob = [self.blobContainer blockBlobReferenceFromName:blobName]; + + failures += [self runTestUploadFromStreamWithBlobSize:20000000 blobToUpload:blob uploadCall:^(NSInputStream *sourceStream, AZSAccessCondition *accessCondition, AZSBlobRequestOptions *blobRequestOptions, AZSOperationContext *operationContext, void(^completionHandler)(NSError * error)) { + [blob uploadFromStream:sourceStream accessCondition:accessCondition requestOptions:blobRequestOptions operationContext:operationContext completionHandler:completionHandler]; + }]; + } + } + XCTAssertEqual(0, failures, @"%d failure(s) detected.", failures); +} + +-(void)testPageBlobFromToStreamUseExistingIterate +{ + int blobSize = 512*40000; + __block int failures = 0; + for (int i = 0; i < 2; i++) + { + @autoreleasepool { + AZSTestSemaphore *semaphore = [[AZSTestSemaphore alloc] init]; + NSString *blobName = @"blobName"; + AZSCloudPageBlob *blob = [self.blobContainer pageBlobReferenceFromName:blobName]; + + [blob createWithSize:[NSNumber numberWithInt:blobSize] completionHandler:^(NSError * __AZSNullable error) { + XCTAssertNil(error, @"Error in creating blob. Error code = %ld, error domain = %@, error userinfo = %@", (long)error.code, error.domain, error.userInfo); + failures += [self runTestUploadFromStreamWithBlobSize:blobSize blobToUpload:blob uploadCall:^(NSInputStream *sourceStream, AZSAccessCondition *accessCondition, AZSBlobRequestOptions *blobRequestOptions, AZSOperationContext *operationContext, void(^completionHandler)(NSError * error)) { + [blob uploadFromStream:sourceStream accessCondition:accessCondition requestOptions:blobRequestOptions operationContext:operationContext completionHandler:completionHandler]; + }]; + + [semaphore signal]; + }]; + [semaphore wait]; + } + } + XCTAssertEqual(0, failures, @"%d failure(s) detected.", failures); +} + +-(void)testPageBlobFromToStreamCreateNewIterate +{ + int blobSize = 512*40000; + int failures = 0; + for (int i = 0; i < 2; i++) + { + @autoreleasepool { + + NSString *blobName = @"blobName"; + AZSCloudPageBlob *blob = [self.blobContainer pageBlobReferenceFromName:blobName]; + + failures += [self runTestUploadFromStreamWithBlobSize:blobSize blobToUpload:blob uploadCall:^(NSInputStream *sourceStream, AZSAccessCondition *accessCondition, AZSBlobRequestOptions *blobRequestOptions, AZSOperationContext *operationContext, void(^completionHandler)(NSError * error)) { + [blob uploadFromStream:sourceStream size:[NSNumber numberWithInt:blobSize] initialSequenceNumber:[NSNumber numberWithInt:100] accessCondition:accessCondition requestOptions:blobRequestOptions operationContext:operationContext completionHandler:completionHandler]; + }]; + } + } + XCTAssertEqual(0, failures, @"%d failure(s) detected.", failures); +} + +-(void)testAppendBlobFromToStreamUseExistingIterate +{ + int blobSize = 20000000; + __block int failures = 0; + for (int i = 0; i < 2; i++) + { + @autoreleasepool { + AZSTestSemaphore *semaphore = [[AZSTestSemaphore alloc] init]; + NSString *blobName = @"blobName"; + AZSCloudAppendBlob *blob = [self.blobContainer appendBlobReferenceFromName:blobName]; + + [blob createWithCompletionHandler:^(NSError * __AZSNullable error) { + XCTAssertNil(error, @"Error in creating blob. Error code = %ld, error domain = %@, error userinfo = %@", (long)error.code, error.domain, error.userInfo); + failures += [self runTestUploadFromStreamWithBlobSize:blobSize blobToUpload:blob uploadCall:^(NSInputStream *sourceStream, AZSAccessCondition *accessCondition, AZSBlobRequestOptions *blobRequestOptions, AZSOperationContext *operationContext, void(^completionHandler)(NSError * error)) { + [blob uploadFromStream:sourceStream createNew:NO accessCondition:accessCondition requestOptions:blobRequestOptions operationContext:operationContext completionHandler:completionHandler]; + }]; + + [semaphore signal]; + }]; + [semaphore wait]; + } + } + XCTAssertEqual(0, failures, @"%d failure(s) detected.", failures); +} + +-(void)testAppendBlobFromToStreamCreateNewIterate +{ + int blobSize = 20000000; + int failures = 0; + for (int i = 0; i < 2; i++) + { + @autoreleasepool { + NSString *blobName = @"blobName"; + AZSCloudAppendBlob *blob = [self.blobContainer appendBlobReferenceFromName:blobName]; + + failures += [self runTestUploadFromStreamWithBlobSize:blobSize blobToUpload:blob uploadCall:^(NSInputStream *sourceStream, AZSAccessCondition *accessCondition, AZSBlobRequestOptions *blobRequestOptions, AZSOperationContext *operationContext, void(^completionHandler)(NSError * error)) { + [blob uploadFromStream:sourceStream createNew:YES accessCondition:accessCondition requestOptions:blobRequestOptions operationContext:operationContext completionHandler:completionHandler]; + }]; + } + } + XCTAssertEqual(0, failures, @"%d failure(s) detected.", failures); +} + +@end diff --git a/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSBlobTestBase.h b/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSBlobTestBase.h index 06376a9..1b16890 100644 --- a/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSBlobTestBase.h +++ b/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSBlobTestBase.h @@ -18,10 +18,19 @@ #import "AZSTestBase.h" @class AZSCloudBlobClient; @class AZSCloudBlob; +@class AZSByteValidationStream; + +@interface AZSBlobTestSpinWrapper : NSObject +@property NSStream *stream; +@property AZSByteValidationStream *delegate; +@property (copy) void (^completionHandler)(); +@end @interface AZSBlobTestBase : AZSTestBase @property AZSCloudBlobClient *blobClient; -(void)waitForCopyToCompleteWithBlob:(AZSCloudBlob *)blobToMonitor completionHandler:(void (^)(NSError *, BOOL))completionHandler; +-(void)scheduleStreamInNewThreaAndRunWithWrapper:(AZSBlobTestSpinWrapper *)wrapper; @end + diff --git a/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSBlobTestBase.m b/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSBlobTestBase.m index 212bace..6537801 100644 --- a/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSBlobTestBase.m +++ b/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSBlobTestBase.m @@ -17,6 +17,11 @@ #import "AZSBlobTestBase.h" #import "AZSClient.h" +#import "AZSTestHelpers.h" + +@implementation AZSBlobTestSpinWrapper + +@end @implementation AZSBlobTestBase @@ -55,4 +60,29 @@ -(void)waitForCopyToCompleteWithBlob:(AZSCloudBlob *)blobToMonitor completionHan }]; } -@end \ No newline at end of file +// TOASK this doesn't actually create the new thread, rename?? +-(void)scheduleStreamInNewThreaAndRunWithWrapper:(AZSBlobTestSpinWrapper *)wrapper +{ + [wrapper.stream setDelegate:wrapper.delegate]; + [wrapper.stream scheduleInRunLoop:[NSRunLoop currentRunLoop] forMode:NSDefaultRunLoopMode]; + [wrapper.stream open]; + + while (!wrapper.delegate.streamFailed && (wrapper.stream.streamStatus != NSStreamStatusClosed)) + { + BOOL loopSuccess = [[NSRunLoop currentRunLoop] runMode:NSDefaultRunLoopMode beforeDate:[NSDate dateWithTimeIntervalSinceNow:2]]; + if (!loopSuccess) + { + break; + } + } + + [wrapper.stream close]; + [wrapper.stream removeFromRunLoop:[NSRunLoop currentRunLoop] forMode:NSDefaultRunLoopMode]; + + if (wrapper.completionHandler) + { + wrapper.completionHandler(); + } +} + +@end diff --git a/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSCloudBlockBlobTests.m b/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSCloudBlockBlobTests.m index 30c6d54..cfed0c8 100644 --- a/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSCloudBlockBlobTests.m +++ b/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSCloudBlockBlobTests.m @@ -708,7 +708,9 @@ -(void)testLargeBlobDownload [blockBlob uploadBlockListFromArray:blockArray accessCondition:nil requestOptions:nil operationContext:nil completionHandler:^(NSError * error) { XCTAssertNil(error, @"Error in uploading block list. Error code = %ld, error domain = %@, error userinfo = %@", (long)error.code, error.domain, error.userInfo); - AZSByteValidationStream *targetStream = [[AZSByteValidationStream alloc]initWithRandomSeed:randSeed totalBlobSize:blockSize*blockCount isUpload:NO]; + AZSByteValidationStream *targetStream = [[AZSByteValidationStream alloc]initWithRandomSeed:randSeed totalBlobSize:blockSize*blockCount isUpload:NO failBlock:^() { + XCTFail(@"Incorrect number of bytes written to the stream."); + }]; AZSBlobRequestOptions *requestOptions = [[AZSBlobRequestOptions alloc] init]; [blockBlob downloadToStream:(NSOutputStream *)targetStream accessCondition:nil requestOptions:requestOptions operationContext:nil completionHandler:^(NSError * error) { @@ -767,6 +769,7 @@ -(void)testUseTransactionalMD5 opContext.sendingRequest = validateSendingContentMD5; bro.disableContentMD5Validation = NO; bro.storeBlobContentMD5 = NO; + bro.singleBlobUploadThreshold = 2; [blockBlob uploadFromData:blockData accessCondition:nil requestOptions:bro operationContext:opContext completionHandler:^(NSError *error) { XCTAssertNil(error, @"Error in uploading blob. Error code = %ld, error domain = %@, error userinfo = %@", (long)error.code, error.domain, error.userInfo); @@ -1210,4 +1213,43 @@ -(void)testAccessConditions [semaphore wait]; } -@end \ No newline at end of file +-(void)testPutBlobvsPutBlockList +{ + AZSTestSemaphore *semaphore = [[AZSTestSemaphore alloc] init]; + + NSString *blob1Name = [NSString stringWithFormat:@"sampleblob%@", [AZSTestHelpers uniqueName]]; + AZSCloudBlockBlob *blob1 = [self.blobContainer blockBlobReferenceFromName:blob1Name]; + + NSString *blobText = @"sample blob text"; + AZSOperationContext *opContext = [[AZSOperationContext alloc] init]; + [blob1 uploadFromText:blobText accessCondition:nil requestOptions:nil operationContext:opContext completionHandler:^(NSError * _Nullable error) { + XCTAssertNil(error, @"Error in uploading blob. Error code = %ld, error domain = %@, error userinfo = %@", (long)error.code, error.domain, error.userInfo); + XCTAssertTrue(opContext.requestResults.count == 1, @"Incorrect number of request results - upload blob should only require one REST request."); + [blob1 downloadToTextWithCompletionHandler:^(NSError * _Nullable error, NSString * _Nullable downloadedBlobText) { + XCTAssertNil(error, @"Error in downloading blob. Error code = %ld, error domain = %@, error userinfo = %@", (long)error.code, error.domain, error.userInfo); + XCTAssertTrue([downloadedBlobText isEqualToString:blobText], @"Blob text incorrect."); + + AZSBlobRequestOptions *options = [[AZSBlobRequestOptions alloc] init]; + options.singleBlobUploadThreshold = 2; // Less than the blob text length + NSString *blob2Name = [NSString stringWithFormat:@"sampleblob%@", [AZSTestHelpers uniqueName]]; + AZSCloudBlockBlob *blob2 = [self.blobContainer blockBlobReferenceFromName:blob2Name]; + AZSOperationContext *opContext = [[AZSOperationContext alloc] init]; + + [blob2 uploadFromText:blobText accessCondition:nil requestOptions:options operationContext:opContext completionHandler:^(NSError * _Nullable error) { + XCTAssertNil(error, @"Error in uploading blob. Error code = %ld, error domain = %@, error userinfo = %@", (long)error.code, error.domain, error.userInfo); + XCTAssertTrue(opContext.requestResults.count == 2, @"Incorrect number of request results - upload blob should only require one REST request."); + + [blob2 downloadToTextWithCompletionHandler:^(NSError * _Nullable error, NSString * _Nullable downloadedBlobText) { + XCTAssertNil(error, @"Error in downloading blob. Error code = %ld, error domain = %@, error userinfo = %@", (long)error.code, error.domain, error.userInfo); + XCTAssertTrue([downloadedBlobText isEqualToString:blobText], @"Blob text incorrect."); + [semaphore signal]; + + }]; + }]; + }]; + }]; + + [semaphore wait]; +} + +@end diff --git a/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSCloudPageBlobTests.m b/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSCloudPageBlobTests.m index aee70c1..3e6af08 100644 --- a/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSCloudPageBlobTests.m +++ b/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSCloudPageBlobTests.m @@ -500,7 +500,9 @@ -(void)testSetSequenceNumber [self createSamplePageDataWithDataArrays:dataArrays offsets:offsets]; - AZSCloudPageBlob *pageBlob = [self.blobContainer pageBlobReferenceFromName:@"pageBlob"]; + AZSCloudPageBlob *_pageBlob = [self.blobContainer pageBlobReferenceFromName:@"pageBlob"]; + __weak AZSCloudPageBlob *pageBlob = _pageBlob; + [pageBlob createWithSize:totalBlobSize completionHandler:^(NSError *error) { XCTAssertNil(error, @"Error in blob creation. Error code = %ld, error domain = %@, error userinfo = %@", (long)error.code, error.domain, error.userInfo); diff --git a/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSTestBase.h b/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSTestBase.h index f44d8ff..6c54fe1 100644 --- a/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSTestBase.h +++ b/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSTestBase.h @@ -18,9 +18,13 @@ #import @class AZSStorageCredentials; @class AZSCloudStorageAccount; +@class AZSTestSemaphore; @interface AZSTestBase : XCTestCase @property AZSCloudStorageAccount *account; + -(void)checkPassageOfError:(NSError *)err expectToPass:(BOOL)expected expectedHttpErrorCode:(int)code message:(NSString *)message; +-(NSRunLoop *)runloopWithSemaphore:(AZSTestSemaphore *)semaphore; + @end \ No newline at end of file diff --git a/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSTestBase.m b/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSTestBase.m index af791a6..7bc3b59 100644 --- a/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSTestBase.m +++ b/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSTestBase.m @@ -17,7 +17,10 @@ #include +#import "AZSErrors.h" #import "AZSTestBase.h" +#import "AZSTestSemaphore.h" +#import "AZSUtil.h" #import "AZSConstants.h" #import "AZSCloudStorageAccount.h" #import "AZSOperationContext.h" @@ -88,5 +91,33 @@ - (void)checkPassageOfError:(NSError *)err expectToPass:(BOOL)expected expectedH } } +/** This method creates a new thread and spins its runloop untill the semaphore.done becomes YES */ +-(NSRunLoop *)runloopWithSemaphore:(AZSTestSemaphore *)semaphore +{ + __block volatile NSRunLoop *runloop = nil; + AZSTestSemaphore *runloopSemaphore = [[AZSTestSemaphore alloc] init]; + + [NSThread detachNewThreadSelector:@selector(executeBlock:) toTarget:self withObject:^() { + runloop = [NSRunLoop currentRunLoop]; + [runloopSemaphore signal]; + + while (!semaphore.done) { + BOOL loopSuccess = [runloop runMode:NSDefaultRunLoopMode beforeDate:[NSDate dateWithTimeIntervalSinceNow:2]]; + if (!loopSuccess) { + [[AZSUtil operationlessContext] logAtLevel:AZSLogLevelInfo withMessage:@"Runloop did not run."]; + [NSThread sleepForTimeInterval:.25]; + } + } + }]; + + [runloopSemaphore wait]; + + return (NSRunLoop *) runloop; +} + +-(void)executeBlock:(void(^)())block +{ + block(); +} @end diff --git a/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSTestHelpers.h b/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSTestHelpers.h index f5a8689..2a85993 100644 --- a/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSTestHelpers.h +++ b/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSTestHelpers.h @@ -41,17 +41,23 @@ @end -@interface AZSByteValidationStream : NSStream +@interface AZSByteValidationStream : NSStream @property BOOL dataCorrupt; +@property BOOL streamFailed; +@property BOOL downloadComplete; @property NSUInteger totalBytes; @property NSUInteger errorCount; +@property long bytesRead; +@property long bytesWritten; @property NSMutableString *errors; +@property(strong) void(^failBlock)(NSString *); // NSStream methods and properties: @property(assign) id delegate; @property(readonly) NSStreamStatus streamStatus; @property(readonly, copy) NSError *streamError; +@property(strong) void(^errorBlock)(); -(void)open; @@ -75,6 +81,7 @@ -(NSInteger)read:(uint8_t *)buffer maxLength:(NSUInteger)length; -(BOOL)getBuffer:(uint8_t **)buffer length:(NSUInteger *)len; --(instancetype)initWithRandomSeed:(unsigned int)seed totalBlobSize:(NSUInteger)totalBlobSize isUpload:(BOOL)isUpload; +-(instancetype)initWithRandomSeed:(unsigned int)seed totalBlobSize:(NSUInteger)totalBlobSize isUpload:(BOOL)isUpload failBlock:(void(^)())failBlock; +-(instancetype)init; @end \ No newline at end of file diff --git a/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSTestHelpers.m b/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSTestHelpers.m index 410f447..222e2f1 100644 --- a/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSTestHelpers.m +++ b/Lib/Azure Storage Client Library/Azure Storage Client LibraryTests/AZSTestHelpers.m @@ -19,8 +19,8 @@ #import "AZSConstants.h" #import "AZSTestHelpers.h" #import "AZSClient.h" - -@import ObjectiveC; +#import "AZSBlobInputStream.h" +#import "AZSStreamDownloadBuffer.h" @implementation AZSTestHelpers +(NSMutableData *)generateSampleDataWithSeed:(unsigned int *)seed length:(unsigned int)length @@ -92,6 +92,7 @@ -(instancetype)initWithNumber:(unsigned int)theNumber @end @interface AZSByteValidationStream() + @property AZSUIntegerHolder *currentSeed; @property BOOL isStreamOpen; @property BOOL isStreamClosed; @@ -101,7 +102,6 @@ @interface AZSByteValidationStream() @property NSMutableData *currentBuffer; @property BOOL isUpload; - @end void RunLoopSourceScheduleRoutine (void *info, CFRunLoopRef rl, CFStringRef mode) @@ -111,13 +111,19 @@ void RunLoopSourceScheduleRoutine (void *info, CFRunLoopRef rl, CFStringRef mode void RunLoopSourcePerformRoutine (void *info) { AZSByteValidationStream *stream = (__bridge AZSByteValidationStream *)info; + BOOL streamClosed = NO; + BOOL fireStreamErrorEvent = NO; @synchronized(stream) { if (stream.isStreamClosed) { streamClosed = YES; } + else if (stream.streamError) + { + fireStreamErrorEvent = YES; + } else if (!stream.isStreamOpen) { // TODO: Emit relevant events to the stream delegate here. @@ -130,6 +136,10 @@ void RunLoopSourcePerformRoutine (void *info) { [stream.delegate stream:stream handleEvent:NSStreamEventEndEncountered]; } + else if (fireStreamErrorEvent) + { + [stream.delegate stream:stream handleEvent:NSStreamEventErrorOccurred]; + } else { if (stream.isUpload) @@ -158,32 +168,50 @@ @interface AZSByteValidationStream() @implementation AZSByteValidationStream --(instancetype)initWithRandomSeed:(unsigned int)seed totalBlobSize:(NSUInteger)totalBlobSize isUpload:(BOOL)isUpload +-(instancetype)initWithRandomSeed:(unsigned int)seed totalBlobSize:(NSUInteger)totalBlobSize isUpload:(BOOL)isUpload failBlock:(void(^)())failBlock { self = [super init]; if (self) { - self.currentSeed = [[AZSUIntegerHolder alloc] initWithNumber:seed]; - self.dataCorrupt = NO; - self.isStreamOpen = NO; - self.isStreamClosed = NO; - CFRunLoopSourceContext context = {0, (__bridge void *)(self), NULL, NULL, NULL, NULL, NULL, + _currentSeed = [[AZSUIntegerHolder alloc] initWithNumber:seed]; + _dataCorrupt = NO; + _isStreamOpen = NO; + _isStreamClosed = NO; + CFRunLoopSourceContext context = + {0, (__bridge void *)(self), NULL, NULL, NULL, NULL, NULL, &RunLoopSourceScheduleRoutine, RunLoopSourceCancelRoutine, RunLoopSourcePerformRoutine}; - self.runLoopSource = CFRunLoopSourceCreate(NULL, 0, &context); - self.runLoopsRegistered = [NSMutableArray arrayWithCapacity:1]; - self.totalBytes = 0; - self.errorCount = 0; - self.errors = [NSMutableString stringWithString:AZSCEmptyString]; - self.totalBlobSize = totalBlobSize; - self.currentBuffer = [NSMutableData dataWithLength:AZSCKilobyte]; - self.isUpload = isUpload; - + _runLoopSource = CFRunLoopSourceCreate(NULL, 0, &context); + _runLoopsRegistered = [NSMutableArray arrayWithCapacity:1]; + _bytesRead = 0; + _totalBytes = 0; + _errorCount = 0; + _errors = [NSMutableString stringWithString:AZSCEmptyString]; + _totalBlobSize = totalBlobSize; + _currentBuffer = [NSMutableData dataWithLength:AZSCKilobyte]; + _isUpload = isUpload; + _currentSeed = [[AZSUIntegerHolder alloc] initWithNumber:seed]; + + __block int failCount = 0; + _failBlock = ^void(NSString *message) { + // TOASK Why 5? + if (failCount < 5) { + failBlock(message); + _streamFailed = YES; + failCount++; + } + }; } return self; } +// should not be used +-(instancetype) init +{ + return (self = nil); +} + -(void)fireStreamEvent { CFRunLoopSourceSignal(self.runLoopSource); @@ -249,6 +277,7 @@ -(BOOL) hasSpaceAvailable return YES; } +// This verifies that the buffer passed in contains exactly the sequence of bytes that is generated by rand_r with the self.currentSeed -(NSInteger)write:(const uint8_t *)buffer maxLength:(NSUInteger)length { for (int i = 0; i < length; i++) @@ -265,6 +294,7 @@ -(NSInteger)write:(const uint8_t *)buffer maxLength:(NSUInteger)length } } } + self.totalBytes += length; [self fireStreamEvent]; return length; @@ -276,9 +306,10 @@ -(BOOL) hasBytesAvailable return YES; } +// This generates a sequence of bytes with rand_r(seed) and put it in the given buffer -(NSInteger)read:(uint8_t *)buffer maxLength:(NSUInteger)length { - // Note: this method contains the extraneous local variables 'totalBlobSize' and 'totalBytes' because otherwise it contaminates profiling results. + // Note: this method contains the extraneous local variables 'totalBlobSize' and 'totalBytes' because otherwise it contaminates profiling results. TOASK why? NSUInteger bytesUploaded = 0; NSUInteger totalBlobSize = self.totalBlobSize; NSUInteger totalBytes = self.totalBytes; @@ -300,6 +331,109 @@ -(NSInteger)read:(uint8_t *)buffer maxLength:(NSUInteger)length return bytesUploaded; } +-(void)stream:(NSStream *)stream handleEvent:(NSStreamEvent)eventCode +{ + if (self.dataCorrupt) { + self.failBlock(@"Data has been corrupted."); + } + + switch (eventCode) { + case NSStreamEventHasBytesAvailable: + { + uint8_t temp[AZSCKilobyte]; + uint8_t *buffer = temp; + NSUInteger curBytesRead = 0; + + if (arc4random() % 2){ + // If unable to access internal buffer directly, make one ourselves. + curBytesRead = [(NSInputStream *) stream read:buffer maxLength:AZSCKilobyte]; + + if (curBytesRead > AZSCKilobyte) { + self.failBlock(@"Incorrect number of bytes written to the stream."); + self.streamFailed = YES; + } + } + else { + BOOL accessed = [(NSInputStream *) stream getBuffer:&buffer length:&curBytesRead]; + if (!accessed) { + self.failBlock(@"[stream getBuffer] failed!"); + self.streamFailed = YES; + } + } + + self.bytesRead += curBytesRead; + self.totalBytes += curBytesRead; + + for (int i = 0; i < curBytesRead; i++) { + NSUInteger byteVal = buffer[i]; + NSUInteger expectedByteVal = rand_r(&(self.currentSeed->number)) % 256; + if (expectedByteVal != byteVal) + { + self.dataCorrupt = YES; + self.errorCount++; + if (self.errorCount <= 5) + { + [self.errors appendFormat:@"Error discovered in stream validation. Expected value = %ld, actual value = %ld.\n", (unsigned long)byteVal, (unsigned long)expectedByteVal]; + } + } + } + + break; + } + + case NSStreamEventHasSpaceAvailable: + { + uint8_t buf[AZSCKilobyte]; + int i; + for (i = 0; i < AZSCKilobyte && self.bytesWritten < self.totalBlobSize; i++) + { + NSUInteger byteval = rand_r(&(self.currentSeed->number)) % 256; + buf[i] = byteval; + self.bytesWritten++; + } + + NSInteger curBytesWritten = [(NSOutputStream *) stream write:(const uint8_t *)buf maxLength:i]; + if (curBytesWritten != i) + { + // If fewer bytes are written then the data will be corrupted because the rest of the bytes in buf will be lost. + // If more bytes are written then the cap on maximum length was not adhered to. + self.failBlock(@"Incorrect number of bytes written to the stream."); + self.streamFailed = YES; + } + + break; + } + + case NSStreamEventEndEncountered: + if (self.bytesRead && self.bytesRead != self.totalBlobSize) { + self.failBlock(@"Incorrect number of bytes read from the stream."); + } + else if (self.bytesWritten && self.bytesWritten != self.totalBlobSize) { + self.failBlock(@"Incorrect number of bytes written to the stream."); + } + else if (!self.bytesRead && self.bytesWritten) { + self.failBlock(@"No bytes written to or read from the stream."); + } + + [stream close]; + [stream removeFromRunLoop:[NSRunLoop currentRunLoop] forMode:NSDefaultRunLoopMode]; + self.downloadComplete = YES; + + break; + + case NSStreamEventErrorOccurred: + self.failBlock(@"Error encountered."); + [stream close]; + [stream removeFromRunLoop:[NSRunLoop currentRunLoop] forMode:NSDefaultRunLoopMode]; + self.downloadComplete = YES; + + break; + + default: + break; + } +} + -(BOOL)getBuffer:(uint8_t **)buffer length:(NSUInteger *)len { return NO; diff --git a/Lib/Azure Storage Client Library/changelog.txt b/Lib/Azure Storage Client Library/changelog.txt index cb4e953..37acdb2 100644 --- a/Lib/Azure Storage Client Library/changelog.txt +++ b/Lib/Azure Storage Client Library/changelog.txt @@ -1,3 +1,6 @@ +2017.XX.YY Version 0.3.0 + * Added ability to generate an AZSBlobInputStream for downloading a blob. + 2017.05.07 Version 0.2.3 * Added support for Getting Service Properties for blob service. * Added support for Setting Service Properties for blob service. @@ -18,4 +21,4 @@ * Fixed a bug where the Cocoapod couldn't be used from a Swift library in some versions of XCode. 2015.09.22 Version 0.1.0 - * Initial Release + * Initial Release \ No newline at end of file diff --git a/README.md b/README.md index a1c3131..0b63a21 100644 --- a/README.md +++ b/README.md @@ -134,12 +134,6 @@ Also coming soon: API docs, a getting-started guide, and a downloadable framewor - Use Shared Key authentication or SAS authentication. - Access conditions, automatic retries and retry policies, and logging. -### Missing functionality - -The following functionality is all coming soon: -- If you want to download a blob's contents to a stream, this is possible using the DownloadToStream method on AZSCloudBlob. However, it is not yet possible to open an input stream that reads directly from the blob. -- There are a number of internal details that will change in the upcoming releases. If you look at the internals of the library (or fork it), be prepared. Much of this will be clean-up related. - ### Specific areas we would like feedback on: - NSOperation support. We have had requests to use NSOperation as the primary method of using the library - methods such as 'UploadFromText' would return an NSOperation, that could then be scheduled in an operation queue. However, this approach seems to have several drawbacks, notably along the lines of error handling and data return. (For example, imagine trying to implement a 'CreateIfNotExists' method, using 'Exists' and 'Create'. If they both returned an NSOperation, you could have the 'Create' operation depend on the 'Exists' operation, but the 'Create' operation would need to behave differently in the case that 'Exists' returns an error, or that the resource already exists.) If you have suggestions, please discuss in the wiki, or open an issue.