[JenkinsDownloadArtifacts] Supporting download to a relative path (#6… (

#6035)

* [JenkinsDownloadArtifacts] Supporting download to a relative path (#6020)

* Bumping version

* Supporting download to a relative path

* Supporting download of more than 5000 files

* throwing error for getArtifactItems

* resolving cr comments

* string fix

* Removing unused reference

* strings

Tasks/Common/azure-blobstorage-artifactProvider/Strings/resources.resjson/en-US/resources.resjson

@@ -1,5 +1,4 @@
 {
-  "loc.messages.ArtifactItemsTruncationWarning": "Downloaded items may be trucated at 5000 items, all artifact items might not have been downloaded.",
   "loc.messages.CreatedBlobForItem": "Created blob for item {0}. Blob uri: {1}",
   "loc.messages.CreatedContainer": "Created container {0}.",
   "loc.messages.ErrorInReadStream": "Error in Read stream: {0}",
@@ -9,5 +8,7 @@
   "loc.messages.FailedToListItemInsideContainer": "Failed to list items inside container: {0}. Error: {1}",
   "loc.messages.SuccessFullyFetchedItemList": "Successfully fetcted list of items",
   "loc.messages.UnableToFetchItem": "Unable to fetch item: {0}. Error: {1}",
-  "loc.messages.UploadingItem": "Uploading {0}."
+  "loc.messages.UploadingItem": "Uploading {0}.",
+  "loc.messages.ContinuationTokenExistsFetchingRemainingFiles": "Continuation token exists, trying to fetch the list of remaining files.",
+  "loc.messages.GetArtifactItemsNotSupported": "Get artifact items not supported, invalid code path"
 }

Tasks/Common/azure-blobstorage-artifactProvider/azureBlobStorageProvider.ts

@@ -5,12 +5,18 @@ import models = require('artifact-engine/Models');
 import tl = require('vsts-task-lib/task');
 
 export class AzureBlobProvider implements models.IArtifactProvider {
-    constructor(storageAccount: string, container: string, accessKey: string, prefixFolderPath?: string, host?: string) {
+
+    constructor(storageAccount: string, container: string, accessKey: string, prefixFolderPath?: string, host?: string, addPrefixToDownloadedItems?: boolean) {
         this._storageAccount = storageAccount;
         this._accessKey = accessKey;
         this._container = container;
-        this._prefixFolderPath = prefixFolderPath;
+        if (!!prefixFolderPath) {
+            this._prefixFolderPath = prefixFolderPath.endsWith("/") ? prefixFolderPath : prefixFolderPath + "/";
+        } else {
+            this._prefixFolderPath = "";
+        }
         this._blobSvc = azureStorage.createBlobService(this._storageAccount, this._accessKey, host);
+        this._addPrefixToDownloadedItems = !!addPrefixToDownloadedItems;
     }
 
     public putArtifactItem(item: models.ArtifactItem, readStream: NodeJS.ReadableStream): Promise<models.ArtifactItem> {
@@ -52,12 +58,18 @@ export class AzureBlobProvider implements models.IArtifactProvider {
     }
 
     public getArtifactItems(artifactItem: models.ArtifactItem): Promise<models.ArtifactItem[]> {
-        return this._getItems(this._container, artifactItem.path);
+        throw new Error(tl.loc("GetArtifactItemsNotSupported"));
     }
 
     public getArtifactItem(artifactItem: models.ArtifactItem): Promise<NodeJS.ReadableStream> {
         return new Promise((resolve, reject) => {
-            var readStream: NodeJS.ReadableStream = this._blobSvc.createReadStream(this._container, artifactItem.path, null);
+            var readStream: NodeJS.ReadableStream;
+            if (!this._addPrefixToDownloadedItems && !!this._prefixFolderPath) {
+                // Adding prefix path to get the absolute path
+                readStream = this._blobSvc.createReadStream(this._container, this._prefixFolderPath + artifactItem.path, null); 
+            } else {
+                readStream = this._blobSvc.createReadStream(this._container, artifactItem.path, null);
+            }
             resolve(readStream);
         });
     }
@@ -83,20 +95,34 @@ export class AzureBlobProvider implements models.IArtifactProvider {
     }
 
     private _getItems(container: string, parentRelativePath?: string): Promise<models.ArtifactItem[]> {
-        var promise = new Promise<models.ArtifactItem[]>((resolve, reject) => {
+        var promise = new Promise<models.ArtifactItem[]>(async (resolve, reject) => {
             var items: models.ArtifactItem[] = [];
+            var continuationToken = null;
+            var result;
+            do {
+                result = await this._getListOfItemsInsideContainer(container, parentRelativePath, continuationToken);
+                items = items.concat(this._convertBlobResultToArtifactItem(result.entries));
+                continuationToken = result.continuationToken;
+                if (!!continuationToken) {
+                    console.log(tl.loc("ContinuationTokenExistsFetchingRemainingFiles"));
+                }
+            } while (continuationToken);
+
+            console.log(tl.loc("SuccessFullyFetchedItemList"));
+            resolve(items);
+        });
 
-            this._blobSvc.listBlobsSegmentedWithPrefix(container, parentRelativePath, null, (error, result) => {
+        return promise;
+    }
+
+    private async _getListOfItemsInsideContainer(container, parentRelativePath, continuationToken): Promise<azureStorage.BlobService.ListBlobsResult> {
+        var promise = new Promise<azureStorage.BlobService.ListBlobsResult>((resolve, reject) => {
+            this._blobSvc.listBlobsSegmentedWithPrefix(container, parentRelativePath, continuationToken, async (error, result) => {
                 if (!!error) {
                     console.log(tl.loc("FailedToListItemInsideContainer", container, error.message));
                     reject(error);
                 } else {
-                    console.log(tl.loc("SuccessFullyFetchedItemList"));
-                    if (result.continuationToken) {
-                        tl.warning(tl.loc("ArtifactItemsTruncationWarning"));
-                    }
-                    items = this._convertBlobResultToArtifactItem(result.entries);
-                    resolve(items);
+                    resolve(result);
                 }
             });
         });
@@ -111,7 +137,12 @@ export class AzureBlobProvider implements models.IArtifactProvider {
             artifactitem.itemType = models.ItemType.File;
             artifactitem.fileLength = parseInt(element.contentLength);
             artifactitem.lastModified = new Date(element.lastModified + 'Z');
-            artifactitem.path = element.name;
+            if (!this._addPrefixToDownloadedItems && !!this._prefixFolderPath) {
+                 // Supplying relative path without prefix; removing the first occurence
+                artifactitem.path = element.name.replace(this._prefixFolderPath, "").trim();
+            } else {
+                artifactitem.path = element.name;
+            }
             artifactItems.push(artifactitem);
         });
 
@@ -124,4 +155,5 @@ export class AzureBlobProvider implements models.IArtifactProvider {
     private _prefixFolderPath: string;
     private _isContainerExists: boolean = false;
     private _blobSvc: azureStorage.BlobService;
+    private _addPrefixToDownloadedItems: boolean = false;
 }

Tasks/Common/azure-blobstorage-artifactProvider/blobservice.ts

@@ -20,7 +20,7 @@ export class BlobService {
         var fileProvider = new artifactProviders.FilesystemProvider(source);
         var azureProvider = new azureBlobProvider.AzureBlobProvider(this._storageAccountName, container, this._storageAccessKey, prefixFolderPath, this._host);
         var processor = new artifactProcessor.ArtifactEngine();
-        var processorOptions =  new artifactProcessor.ArtifactEngineOptions();
+        var processorOptions = new artifactProcessor.ArtifactEngineOptions();
         if (itemPattern) {
             processorOptions.itemPattern = itemPattern;
         }
@@ -37,11 +37,11 @@ export class BlobService {
         return uploadedUrls;
     }
 
-    public async downloadBlobs(destination: string, container: string, prefixFolderPath?: string, itemPattern?: string): Promise<void> {
+    public async downloadBlobs(destination: string, container: string, prefixFolderPath?: string, itemPattern?: string, addPrefixToDownloadedItems?: boolean): Promise<void> {
         var fileProvider = new artifactProviders.FilesystemProvider(destination);
-        var azureProvider = new azureBlobProvider.AzureBlobProvider(this._storageAccountName, container, this._storageAccessKey, prefixFolderPath, this._host);
+        var azureProvider = new azureBlobProvider.AzureBlobProvider(this._storageAccountName, container, this._storageAccessKey, prefixFolderPath, this._host, !!addPrefixToDownloadedItems);
         var processor = new artifactProcessor.ArtifactEngine();
-        var processorOptions =  new artifactProcessor.ArtifactEngineOptions();
+        var processorOptions = new artifactProcessor.ArtifactEngineOptions();
         if (itemPattern) {
             processorOptions.itemPattern = itemPattern;
         }

Tasks/Common/azure-blobstorage-artifactProvider/module.json

@@ -1,6 +1,5 @@
 {
   "messages": {
-    "ArtifactItemsTruncationWarning":"Downloaded items may be trucated at 5000 items, all artifact items might not have been downloaded.",
     "CreatedBlobForItem": "Created blob for item {0}. Blob uri: {1}",
     "CreatedContainer": "Created container {0}.",
     "ErrorInReadStream": "Error in Read stream: {0}",
@@ -10,6 +9,8 @@
     "FailedToListItemInsideContainer": "Failed to list items inside container: {0}. Error: {1}",
     "SuccessFullyFetchedItemList": "Successfully fetcted list of items",
     "UnableToFetchItem": "Unable to fetch item: {0}. Error: {1}",
-    "UploadingItem": "Uploading {0}."
+    "UploadingItem": "Uploading {0}.",
+    "ContinuationTokenExistsFetchingRemainingFiles": "Continuation token exists, trying to fetch the list of remaining files.",
+    "GetArtifactItemsNotSupported": "Get artifact items not supported, invalid code path"
   }
 }

Tasks/JenkinsDownloadArtifacts/AzureStorageArtifacts/AzureStorageArtifactDownloader.ts

@@ -26,7 +26,7 @@ export class AzureStorageArtifactDownloader {
 
     const blobService = new BlobService.BlobService(storageAccount.name, storageAccount.primaryAccessKey);
 
-    await blobService.downloadBlobs(downloadToPath, this.containerName, this.commonVirtualPath, fileType || "**");
+    await blobService.downloadBlobs(downloadToPath, this.containerName, this.commonVirtualPath, fileType || "**", false);
   }
 
   private async _getStorageAccountDetails(): Promise<StorageAccountInfo> {

Tasks/JenkinsDownloadArtifacts/Strings/resources.resjson/en-US/resources.resjson

@@ -109,7 +109,6 @@
   "loc.messages.ResourceNameCannotBeNull": "Resource name is required.",
   "loc.messages.activeDirectoryResourceIdUrlCannotBeEmpty": "Active directory resource url is required.",
   "loc.messages.StorageAccountCannotBeNull": "storage accountName is required.",
-  "loc.messages.ArtifactItemsTruncationWarning": "Downloaded items may be trucated at 5000 items, all artifact items might not have been downloaded.",
   "loc.messages.CreatedBlobForItem": "Created blob for item {0}. Blob uri: {1}",
   "loc.messages.CreatedContainer": "Created container {0}.",
   "loc.messages.ErrorInReadStream": "Error in Read stream: {0}",
@@ -122,5 +121,7 @@
   "loc.messages.UploadingItem": "Uploading {0}.",
   "loc.messages.FoundJobType": "Found job type %s",
   "loc.messages.CannotFindJobType": "Could not detect job type",
-  "loc.messages.InvalidJobName": "Invalid job name %s"
+  "loc.messages.InvalidJobName": "Invalid job name %s",
+  "loc.messages.ContinuationTokenExistsFetchingRemainingFiles": "Continuation token exists, trying to fetch the list of remaining files.",
+  "loc.messages.GetArtifactItemsNotSupported": "Get artifact items not supported, invalid code path"
 }

Tasks/JenkinsDownloadArtifacts/task.json

@@ -18,7 +18,7 @@
     "version": {
         "Major": 1,
         "Minor": 125,
-        "Patch": 1
+        "Patch": 2
     },
     "groups": [
         {
@@ -328,7 +328,6 @@
         "ResourceNameCannotBeNull": "Resource name is required.",
         "activeDirectoryResourceIdUrlCannotBeEmpty": "Active directory resource url is required.",
         "StorageAccountCannotBeNull": "storage accountName is required.",
-        "ArtifactItemsTruncationWarning": "Downloaded items may be trucated at 5000 items, all artifact items might not have been downloaded.",
         "CreatedBlobForItem": "Created blob for item {0}. Blob uri: {1}",
         "CreatedContainer": "Created container {0}.",
         "ErrorInReadStream": "Error in Read stream: {0}",
@@ -341,6 +340,8 @@
         "UploadingItem": "Uploading {0}.",
         "FoundJobType": "Found job type %s",
         "CannotFindJobType": "Could not detect job type",
-        "InvalidJobName": "Invalid job name %s"
+        "InvalidJobName": "Invalid job name %s",
+        "ContinuationTokenExistsFetchingRemainingFiles": "Continuation token exists, trying to fetch the list of remaining files.",
+        "GetArtifactItemsNotSupported": "Get artifact items not supported, invalid code path"
     }
 }

Tasks/JenkinsDownloadArtifacts/task.loc.json

@@ -18,7 +18,7 @@
   "version": {
     "Major": 1,
     "Minor": 125,
-    "Patch": 1
+    "Patch": 2
   },
   "groups": [
     {
@@ -328,7 +328,6 @@
     "ResourceNameCannotBeNull": "ms-resource:loc.messages.ResourceNameCannotBeNull",
     "activeDirectoryResourceIdUrlCannotBeEmpty": "ms-resource:loc.messages.activeDirectoryResourceIdUrlCannotBeEmpty",
     "StorageAccountCannotBeNull": "ms-resource:loc.messages.StorageAccountCannotBeNull",
-    "ArtifactItemsTruncationWarning": "ms-resource:loc.messages.ArtifactItemsTruncationWarning",
     "CreatedBlobForItem": "ms-resource:loc.messages.CreatedBlobForItem",
     "CreatedContainer": "ms-resource:loc.messages.CreatedContainer",
     "ErrorInReadStream": "ms-resource:loc.messages.ErrorInReadStream",
@@ -341,6 +340,8 @@
     "UploadingItem": "ms-resource:loc.messages.UploadingItem",
     "FoundJobType": "ms-resource:loc.messages.FoundJobType",
     "CannotFindJobType": "ms-resource:loc.messages.CannotFindJobType",
-    "InvalidJobName": "ms-resource:loc.messages.InvalidJobName"
+    "InvalidJobName": "ms-resource:loc.messages.InvalidJobName",
+    "ContinuationTokenExistsFetchingRemainingFiles": "ms-resource:loc.messages.ContinuationTokenExistsFetchingRemainingFiles",
+    "GetArtifactItemsNotSupported": "ms-resource:loc.messages.GetArtifactItemsNotSupported"
   }
 }