diff --git a/source/Calamari.Common/Calamari.Common.csproj b/source/Calamari.Common/Calamari.Common.csproj index 137715ff0..520c54d1a 100644 --- a/source/Calamari.Common/Calamari.Common.csproj +++ b/source/Calamari.Common/Calamari.Common.csproj @@ -10,7 +10,7 @@ $(DefineConstants);USE_ALPHAFS_FOR_LONG_FILE_PATH_SUPPORT;HAS_SSL3 - $(DefineConstants);USE_NUGET_V3_LIBS;WORKAROUND_FOR_EMPTY_STRING_BUG;HAS_NULLABLE_REF_TYPES + $(DefineConstants);WORKAROUND_FOR_EMPTY_STRING_BUG;HAS_NULLABLE_REF_TYPES @@ -18,7 +18,6 @@ - @@ -28,7 +27,6 @@ - @@ -36,6 +34,8 @@ + + diff --git a/source/Calamari.Common/Features/Packages/NuGet/LocalNuGetPackage.cs b/source/Calamari.Common/Features/Packages/NuGet/LocalNuGetPackage.cs index 52c9f4690..9967d4147 100644 --- a/source/Calamari.Common/Features/Packages/NuGet/LocalNuGetPackage.cs +++ b/source/Calamari.Common/Features/Packages/NuGet/LocalNuGetPackage.cs @@ -1,8 +1,4 @@ -#if USE_NUGET_V3_LIBS -using NuGet.Packaging; -#else -using NuGet; -#endif +using NuGet.Packaging; using System; using System.IO; using Calamari.Common.Plumbing; diff --git a/source/Calamari.Common/Plumbing/Extensions/VersionExtensions.cs b/source/Calamari.Common/Plumbing/Extensions/VersionExtensions.cs index 01a593ee7..b76bef1d9 100644 --- a/source/Calamari.Common/Plumbing/Extensions/VersionExtensions.cs +++ b/source/Calamari.Common/Plumbing/Extensions/VersionExtensions.cs @@ -1,5 +1,4 @@ -#if USE_NUGET_V3_LIBS -using System; +using System; using NuGet.Versioning; using Octopus.Versioning; @@ -23,5 +22,4 @@ public static NuGetVersion ToNuGetVersion(this IVersion version) version.Metadata); } } -} -#endif \ No newline at end of file +} \ No newline at end of file diff --git a/source/Calamari.Shared/Calamari.Shared.csproj b/source/Calamari.Shared/Calamari.Shared.csproj index 41d9bb93a..3058785c7 100644 --- a/source/Calamari.Shared/Calamari.Shared.csproj +++ b/source/Calamari.Shared/Calamari.Shared.csproj @@ -26,11 +26,11 @@ net462;netstandard2.1 - $(DefineConstants);USE_NUGET_V2_LIBS;SUPPORTS_POLLY;USE_OCTODIFF_EXE;WINDOWS_CERTIFICATE_STORE_SUPPORT + $(DefineConstants);SUPPORTS_POLLY;USE_OCTODIFF_EXE;WINDOWS_CERTIFICATE_STORE_SUPPORT anycpu - $(DefineConstants);USE_NUGET_V3_LIBS;SUPPORTS_POLLY + $(DefineConstants);SUPPORTS_POLLY CS8600;CS8601;CS8602;CS8603;CS8604;DE0003;DE0004 @@ -56,9 +56,9 @@ + - @@ -73,8 +73,6 @@ - - diff --git a/source/Calamari.Shared/Integration/Packages/Download/FeedCredentialsProvider.cs b/source/Calamari.Shared/Integration/Packages/Download/FeedCredentialsProvider.cs deleted file mode 100644 index f8e5dba04..000000000 --- a/source/Calamari.Shared/Integration/Packages/Download/FeedCredentialsProvider.cs +++ /dev/null @@ -1,95 +0,0 @@ -#if USE_NUGET_V2_LIBS -using System; -using System.Collections.Concurrent; -using System.Net; -using NuGet; - -namespace Calamari.Integration.Packages.Download -{ - public class FeedCredentialsProvider : ICredentialProvider - { - FeedCredentialsProvider() - { - } - - public static FeedCredentialsProvider Instance = new FeedCredentialsProvider(); - static readonly ConcurrentDictionary Credentials = new ConcurrentDictionary(); - static readonly ConcurrentDictionary Retries = new ConcurrentDictionary(); - - public void SetCredentials(Uri uri, ICredentials credential) - { - Credentials[Canonicalize(uri)] = credential; - } - - public ICredentials GetCredentials(Uri uri, IWebProxy proxy, CredentialType credentialType, bool retrying) - { - var url = Canonicalize(uri); - var retry = Retries.GetOrAdd(url, _ => new RetryTracker()); - - if (!retrying) - { - retry.Reset(); - } - else - { - var retryAllowed = retry.AttemptRetry(); - if (!retryAllowed) - return null; - } - - return new DynamicCachedCredential(url); - } - - ICredentials GetCurrentCredentials(string url) - { - ICredentials credential; - if (!Credentials.TryGetValue(url, out credential)) - { - credential = CredentialCache.DefaultNetworkCredentials; - } - - return credential; - } - - string Canonicalize(Uri uri) - { - return uri.Authority.ToLowerInvariant().Trim(); - } - - public class RetryTracker - { - const int MaxAttempts = 3; - int currentCount; - - public bool AttemptRetry() - { - if (currentCount > MaxAttempts) return false; - - currentCount++; - return true; - } - - public void Reset() - { - currentCount = 0; - } - } - - class DynamicCachedCredential : CredentialCache, ICredentials - { - readonly string url; - - public DynamicCachedCredential(string url) - { - this.url = url; - } - - NetworkCredential ICredentials.GetCredential(Uri uri, string authType) - { - var credential = Instance.GetCurrentCredentials(url); - return credential.GetCredential(uri, authType); - } - } - } -} -#endif \ No newline at end of file diff --git a/source/Calamari.Shared/Integration/Packages/Download/FixedFilePathResolver.cs b/source/Calamari.Shared/Integration/Packages/Download/FixedFilePathResolver.cs deleted file mode 100644 index 7ad8fbf50..000000000 --- a/source/Calamari.Shared/Integration/Packages/Download/FixedFilePathResolver.cs +++ /dev/null @@ -1,63 +0,0 @@ -#if USE_NUGET_V2_LIBS -using System; -using System.IO; -using NuGet; - -namespace Calamari.Integration.Packages.Download -{ - public class FixedFilePathResolver : IPackagePathResolver - { - readonly string packageName; - readonly string filePathNameToReturn; - - public FixedFilePathResolver(string packageName, string filePathNameToReturn) - { - if (packageName == null) - throw new ArgumentNullException("packageName"); - if (filePathNameToReturn == null) - throw new ArgumentNullException("filePathNameToReturn"); - - this.packageName = packageName; - this.filePathNameToReturn = filePathNameToReturn; - } - - public string GetInstallPath(IPackage package) - { - EnsureRightPackage(package.Id); - return Path.GetDirectoryName(filePathNameToReturn); - } - - public string GetPackageDirectory(IPackage package) - { - return GetPackageDirectory(package.Id, package.Version); - } - - public string GetPackageFileName(IPackage package) - { - return GetPackageFileName(package.Id, package.Version); - } - - public string GetPackageDirectory(string packageId, SemanticVersion version) - { - EnsureRightPackage(packageId); - return string.Empty; - } - - public string GetPackageFileName(string packageId, SemanticVersion version) - { - EnsureRightPackage(packageId); - return Path.GetFileName(filePathNameToReturn); - } - - void EnsureRightPackage(string packageId) - { - var samePackage = string.Equals(packageId, packageName, StringComparison.OrdinalIgnoreCase); - - if (!samePackage) - { - throw new ArgumentException(string.Format("Expected to be asked for the path for package {0}, but was instead asked for the path for package {1}", packageName, packageId)); - } - } - } -} -#endif \ No newline at end of file diff --git a/source/Calamari.Shared/Integration/Packages/Download/NuGetPackageDownloader.cs b/source/Calamari.Shared/Integration/Packages/Download/NuGetPackageDownloader.cs index 6abc0b65e..40e266d58 100644 --- a/source/Calamari.Shared/Integration/Packages/Download/NuGetPackageDownloader.cs +++ b/source/Calamari.Shared/Integration/Packages/Download/NuGetPackageDownloader.cs @@ -11,11 +11,6 @@ using Calamari.Integration.Packages.NuGet; using Octopus.Versioning; using PackageName = Calamari.Common.Features.Packages.PackageName; -#if USE_NUGET_V2_LIBS -using NuGet; -#else -using NuGet.Packaging; -#endif namespace Calamari.Integration.Packages.Download { @@ -124,11 +119,8 @@ PackagePhysicalFileMetadata DownloadPackage( void CheckWhetherThePackageHasDependencies(PackagePhysicalFileMetadata pkg) { var nuGetMetadata = new LocalNuGetPackage(pkg.FullFilePath).Metadata; -#if USE_NUGET_V3_LIBS var dependencies = nuGetMetadata.DependencyGroups.SelectMany(ds => ds.Packages).ToArray(); -#else - var dependencies = nuGetMetadata.DependencySets.SelectMany(ds => ds.Dependencies).ToArray(); -#endif + if (dependencies.Any()) Log.Info( "NuGet packages with dependencies are not currently supported, and dependencies won't be installed on the Tentacle. The package '{0} {1}' appears to have the following dependencies: {2}. For more information please see {3}", diff --git a/source/Calamari.Shared/Integration/Packages/NuGet/InternalNuGetPackageDownloader.cs b/source/Calamari.Shared/Integration/Packages/NuGet/InternalNuGetPackageDownloader.cs index a3c03fb47..a3130d8ff 100644 --- a/source/Calamari.Shared/Integration/Packages/NuGet/InternalNuGetPackageDownloader.cs +++ b/source/Calamari.Shared/Integration/Packages/NuGet/InternalNuGetPackageDownloader.cs @@ -98,61 +98,11 @@ private void DownloadPackageAction(string packageId, IVersion version, Uri feedU return; } -#if USE_NUGET_V2_LIBS - var timeout = GetHttpTimeout(); - if (IsHttp(feedUri.ToString())) - { - if (NuGetV3Downloader.CanHandle(feedUri, feedCredentials, timeout)) - { - NuGetV3Downloader.DownloadPackage(packageId, version, feedUri, feedCredentials, targetFilePath, timeout); - } - else - { - WarnIfHttpTimeoutHasBeenSet(); - NuGetV2Downloader.DownloadPackage(packageId, version.ToString(), feedUri, feedCredentials, targetFilePath); - } - } -#else - else - { - WarnIfHttpTimeoutHasBeenSet(); - NuGetV3LibDownloader.DownloadPackage(packageId, version, feedUri, feedCredentials, targetFilePath); - } -#endif - } - -#if USE_NUGET_V2_LIBS - TimeSpan GetHttpTimeout() - { - const string expectedTimespanFormat = "c"; - - // Equal to Timeout.InfiniteTimeSpan, which isn't available in net40 - var defaultTimeout = new TimeSpan(0, 0, 0, 0, -1); - - var rawTimeout = variables.Get(KnownVariables.NugetHttpTimeout); - if (string.IsNullOrWhiteSpace(rawTimeout)) - { - return defaultTimeout; - } - - if (TimeSpan.TryParseExact(rawTimeout, expectedTimespanFormat, null, out var parsedTimeout)) - { - return parsedTimeout; - } - - var exampleTimespan = new TimeSpan(0, 0, 1, 0).ToString(expectedTimespanFormat); - - var message = $"The variable {KnownVariables.NugetHttpTimeout} couldn't be parsed as a timespan. " + - $"Expected a value like '{exampleTimespan}' but received '{rawTimeout}'. " + - $"Defaulting to '{defaultTimeout.ToString(expectedTimespanFormat)}'."; - - Log.Warn(message); - return defaultTimeout; + WarnIfHttpTimeoutHasBeenSet(); + NuGetV3LibDownloader.DownloadPackage(packageId, version, feedUri, feedCredentials, targetFilePath); } -#endif - void WarnIfHttpTimeoutHasBeenSet() { if (variables.IsSet(KnownVariables.NugetHttpTimeout)) diff --git a/source/Calamari.Shared/Integration/Packages/NuGet/NuGetV2Downloader.cs b/source/Calamari.Shared/Integration/Packages/NuGet/NuGetV2Downloader.cs deleted file mode 100644 index e569e7b35..000000000 --- a/source/Calamari.Shared/Integration/Packages/NuGet/NuGetV2Downloader.cs +++ /dev/null @@ -1,68 +0,0 @@ -#if USE_NUGET_V2_LIBS -using System; -using System.IO; -using System.Net; -using Calamari.Common.Plumbing.Logging; -using Calamari.Integration.Packages.Download; -using NuGet; -using SemanticVersion = NuGet.SemanticVersion; - -namespace Calamari.Integration.Packages.NuGet -{ - public class NuGetV2Downloader - { - public static void DownloadPackage(string packageId, string packageVersion, Uri feedUri, - ICredentials feedCredentials, string targetFilePath) - { - SetFeedCredentials(feedUri, feedCredentials); - - var package = FindPackage(packageId, packageVersion, feedUri, out var downloader); - DownloadPackage(package, targetFilePath, downloader); - } - - private static IPackage FindPackage(string packageId, string packageVersion, Uri feed, out PackageDownloader downloader) - { - var remoteRepository = PackageRepositoryFactory.Default.CreateRepository(feed.AbsoluteUri); - - downloader = remoteRepository is DataServicePackageRepository dspr ? dspr.PackageDownloader : null; - - var requiredVersion = new SemanticVersion(packageVersion); - var package = remoteRepository.FindPackage(packageId, requiredVersion, true, true); - - if (package == null) - throw new Exception($"Could not find package {packageId} {packageVersion} in feed: '{feed}'"); - - if (!requiredVersion.Equals(package.Version)) - { - throw new Exception($"The package version '{package.Version}' returned from the package repository doesn't match the requested package version '{requiredVersion}'."); - } - - return package; - } - - private static void DownloadPackage(IPackage package, string fullPathToDownloadTo, PackageDownloader directDownloader) - { - Log.VerboseFormat("Found package {0} v{1}", package.Id, package.Version); - Log.Verbose("Downloading to: " + fullPathToDownloadTo); - - if (package is DataServicePackage dsp && directDownloader != null) - { - Log.Verbose("A direct download is possible; bypassing the NuGet machine cache"); - using (var targetFile = new FileStream(fullPathToDownloadTo, FileMode.CreateNew)) - directDownloader.DownloadPackage(dsp.DownloadUrl, dsp, targetFile); - return; - } - - var physical = new PhysicalFileSystem(Path.GetDirectoryName(fullPathToDownloadTo)); - var local = new LocalPackageRepository(new FixedFilePathResolver(package.Id, fullPathToDownloadTo), physical); - local.AddPackage(package); - } - - static void SetFeedCredentials(Uri feedUri, ICredentials feedCredentials) - { - FeedCredentialsProvider.Instance.SetCredentials(feedUri, feedCredentials); - HttpClient.DefaultCredentialProvider = FeedCredentialsProvider.Instance; - } - } -} -#endif \ No newline at end of file diff --git a/source/Calamari.Shared/Integration/Packages/NuGet/NuGetV3Downloader.cs b/source/Calamari.Shared/Integration/Packages/NuGet/NuGetV3Downloader.cs deleted file mode 100644 index 57f7e8f28..000000000 --- a/source/Calamari.Shared/Integration/Packages/NuGet/NuGetV3Downloader.cs +++ /dev/null @@ -1,359 +0,0 @@ -// Much of this class was based on code from https://github.com/NuGet/NuGet.Client. It was ported, as the NuGet libraries are .NET 4.5 and Calamari is .NET 4.0 -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.// -#if USE_NUGET_V2_LIBS -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Web; -using Calamari.Common.Commands; -using Calamari.Common.Plumbing.Extensions; -using Calamari.Common.Plumbing.Logging; -using Calamari.Deployment; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; -using Octopus.CoreUtilities.Extensions; -using Octopus.Versioning; -using Octopus.Versioning.Semver; - -namespace Calamari.Integration.Packages.NuGet -{ - internal static class NuGetV3Downloader - { - public static bool CanHandle(Uri feedUri, ICredentials feedCredentials, TimeSpan httpTimeout) - { - if (feedUri.ToString().EndsWith(".json", StringComparison.OrdinalIgnoreCase)) - { - return true; - } - - return IsJsonEndpoint(feedUri, feedCredentials, httpTimeout); - } - - static bool IsJsonEndpoint(Uri feedUri, ICredentials feedCredentials, TimeSpan httpTimeout) - { -#if NET40 - var request = WebRequest.Create(feedUri); - request.Credentials = feedCredentials; - request.Timeout = (int)httpTimeout.TotalMilliseconds; - using (var response = (HttpWebResponse)request.GetResponse()) - { - if (response.IsSuccessStatusCode()) - { - return response.ContentType == "application/json"; - } - - throw new HttpException((int)response.StatusCode, $"Received status code that indicate not successful response. Uri:{feedUri}"); - } -#else - var request = new HttpRequestMessage(HttpMethod.Get, feedUri); - - using (var httpClient = CreateHttpClient(feedCredentials, httpTimeout)) - { - var sending = httpClient.SendAsync(request); - sending.Wait(); - - using (var response = sending.Result) - { - response.EnsureSuccessStatusCode(); - - return response.Content.Headers.ContentType.MediaType == "application/json"; - } - } -#endif - } - - class PackageIdentifier - { - public PackageIdentifier(string packageId, IVersion version) - { - Id = packageId.ToLowerInvariant(); - Version = version.ToString().ToLowerInvariant(); - SemanticVersion = version; - SemanticVersionWithoutMetadata = new SemanticVersion(version.Major, version.Minor, version.Patch, version.Revision, version.Release, null); - } - - public string Id { get; } - public string Version { get; } - public IVersion SemanticVersion { get; } - public IVersion SemanticVersionWithoutMetadata { get; } - } - - public static void DownloadPackage(string packageId, IVersion version, Uri feedUri, ICredentials feedCredentials, string targetFilePath, TimeSpan httpTimeout) - { - var packageIdentifier = new PackageIdentifier(packageId, version); - - var downloadUri = GetDownloadUri(packageIdentifier, feedUri, feedCredentials, httpTimeout); - if (downloadUri == null) - { - throw new InvalidOperationException($"Unable to find url to download package: {version} with version: {version} from feed: {feedUri}"); - } - - Log.Verbose($"Downloading package from '{downloadUri}'"); - - using (var nupkgFile = new FileStream(targetFilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.None)) - { - GetHttp(downloadUri, feedCredentials, httpTimeout, pkgStream => - { - pkgStream.CopyTo(nupkgFile); - }); - } - } - - static Uri? GetDownloadUri(PackageIdentifier packageIdentifier, Uri feedUri, ICredentials feedCredentials, TimeSpan httpTimeout) - { - var json = GetServiceIndexJson(feedUri, feedCredentials, httpTimeout); - if (json == null) - { - throw new CommandException($"'{feedUri}' is not a valid NuGet v3 feed"); - } - - var resources = GetServiceResources(json); - - var packageBaseDownloadUri = GetPackageBaseDownloadUri(resources, packageIdentifier); - if (packageBaseDownloadUri != null) return packageBaseDownloadUri; - - return GetPackageRegistrationDownloadUri(feedCredentials, httpTimeout, resources, packageIdentifier); - } - - static Uri? GetPackageRegistrationDownloadUri(ICredentials feedCredentials, TimeSpan httpTimeout, IDictionary> resources, PackageIdentifier packageIdentifier) - { - var packageRegistrationUri = GetPackageRegistrationUri(resources, packageIdentifier.Id); - var packageRegistrationResponse = GetJsonResponse(packageRegistrationUri, feedCredentials, httpTimeout); - - // Package Registration Response structure - // https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource#response - var registrationPages = packageRegistrationResponse["items"]; - - // Registration Page structure - // https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource#registration-page-object - foreach (var registrationPage in registrationPages) - { - var registrationLeaves = registrationPage["items"]; - if (registrationLeaves == null) - { - // narrow version to specific page. - var versionedRegistrationPage = registrationPages.FirstOrDefault(x => VersionComparer.Default.Compare(new SemanticVersion(x["lower"].ToString()), packageIdentifier.SemanticVersionWithoutMetadata) <= 0 && VersionComparer.Default.Compare(new SemanticVersion(x["upper"].ToString()), packageIdentifier.SemanticVersionWithoutMetadata) >= 0); - - // If we can't find a page for the version we are looking for, return null. - if (versionedRegistrationPage == null) return null; - - var versionedRegistrationPageResponse = GetJsonResponse(new Uri(versionedRegistrationPage["@id"].ToString()), feedCredentials, httpTimeout); - registrationLeaves = versionedRegistrationPageResponse["items"]; - } - - // Leaf Structure - // https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource#registration-leaf-object-in-a-page - var leaf = registrationLeaves.FirstOrDefault(x => string.Equals(x["catalogEntry"]["version"].ToString(), packageIdentifier.Version, StringComparison.OrdinalIgnoreCase)); - // If we can't find the leaf registration for the version we are looking for, return null. - if (leaf == null) return null; - - var contentUri = leaf["packageContent"].ToString(); - - // Note: We reformat the packageContent Uri here as Artifactory (and possibly others) does not include +metadata suffixes on its packageContent Uri's - var downloadUri = new Uri($"{contentUri.Remove(contentUri.LastIndexOfAny("/".ToCharArray()) + 1)}{packageIdentifier.Version}"); - - return downloadUri; - } - - return null; - } - - static Uri? GetPackageBaseDownloadUri(IDictionary> resources, PackageIdentifier packageIdentifier) - { - var packageBaseUri = GetPackageBaseUri(resources); - - if (packageBaseUri?.AbsoluteUri.TrimEnd('/') != null) - { - return new Uri($"{packageBaseUri}/{packageIdentifier.Id}/{packageIdentifier.Version}/{packageIdentifier.Id}.{packageIdentifier.Version}.nupkg"); - } - - return null; - } - - static Uri GetPackageRegistrationUri(IDictionary> resources, string normalizedId) - { - var registrationUrl = NuGetServiceTypes.RegistrationsBaseUrl - .Where(serviceType => resources.ContainsKey(serviceType)) - .SelectMany(serviceType => resources[serviceType]) - .First() - .OriginalString.TrimEnd('/'); - - var packageRegistrationUri = new Uri($"{registrationUrl}/{normalizedId}/index.json"); - return packageRegistrationUri; - } - - static HttpClient CreateHttpClient(ICredentials credentials, TimeSpan httpTimeout) - { - var handler = new WebRequestHandler - { - Credentials = credentials, - AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate - }; - - var httpClient = new HttpClient(handler) - { - Timeout = httpTimeout - }; - - httpClient.DefaultRequestHeaders.Add("user-agent", "NuGet Client V3/3.4.3"); - - return httpClient; - } - - static void GetHttp(Uri uri, ICredentials credentials, TimeSpan httpTimeout, Action processContent) - { -#if NET40 - var request = WebRequest.Create(uri); - request.Credentials = credentials; - request.Timeout = (int)httpTimeout.TotalMilliseconds; - using (var response = (HttpWebResponse)request.GetResponse()) - { - if (response.IsSuccessStatusCode()) - { - using (var respStream = response.GetResponseStream()) - { - processContent(respStream); - } - } - else - { - throw new HttpException((int)response.StatusCode, $"Received status code that indicate not successful response. Uri:{uri}"); - } - } - -#else - var request = new HttpRequestMessage(HttpMethod.Get, uri); - - using (var httpClient = CreateHttpClient(credentials, httpTimeout)) - { - var sending = httpClient.SendAsync(request); - sending.Wait(); - using (var response = sending.Result) - { - response.EnsureSuccessStatusCode(); - var readingStream = response.Content.ReadAsStreamAsync(); - readingStream.Wait(); - processContent(readingStream.Result); - } - } -#endif - } - - static Uri? GetPackageBaseUri(IDictionary> resources) - { - // If index.json contains a flat container resource use that to directly - // construct package download urls. - if (resources.ContainsKey(NuGetServiceTypes.PackageBaseAddress)) - return resources[NuGetServiceTypes.PackageBaseAddress].FirstOrDefault(); - return null; - } - - static JObject? GetServiceIndexJson(Uri feedUri, ICredentials feedCredentials, TimeSpan httpTimeout) - { - var json = GetJsonResponse(feedUri, feedCredentials, httpTimeout); - - if (!IsValidV3Json(json)) return null; - - return json; - } - - static JObject GetJsonResponse(Uri feedUri, ICredentials feedCredentials, TimeSpan httpTimeout) - { - // Parse JSON for package base URL - JObject json = null; - GetHttp(feedUri, - feedCredentials, - httpTimeout, - stream => - { - using (var streamReader = new StreamReader(stream)) - using (var jsonReader = new JsonTextReader(streamReader)) - { - json = JObject.Load(jsonReader); - } - }); - return json; - } - - static bool IsValidV3Json(JObject json) - { - // Use SemVer instead of NuGetVersion, the service index should always be - // in strict SemVer format - JToken versionToken; - if (json.TryGetValue("version", out versionToken) && - versionToken.Type == JTokenType.String) - { - var version = VersionFactory.TryCreateSemanticVersion(((string)versionToken).SanitiseSemVerString()); - if (version != null && version.Major == 3) - { - return true; - } - } - return false; - } - - static IDictionary> GetServiceResources(JObject index) - { - var result = new Dictionary>(); - - JToken resources; - if (index.TryGetValue("resources", out resources)) - { - foreach (var resource in resources) - { - JToken type = resource["@type"]; - JToken id = resource["@id"]; - - if (type == null || id == null) - { - continue; - } - - if (type.Type == JTokenType.Array) - { - foreach (var nType in type) - { - AddEndpoint(result, nType, id); - } - } - else - { - AddEndpoint(result, type, id); - } - } - } - - return result; - } - - static void AddEndpoint(IDictionary> result, JToken typeToken, JToken idToken) - { - string type = (string)typeToken; - string id = (string)idToken; - - if (type == null || id == null) - { - return; - } - - List ids; - if (!result.TryGetValue(type, out ids)) - { - ids = new List(); - result.Add(type, ids); - } - - Uri uri; - if (Uri.TryCreate(id, UriKind.Absolute, out uri)) - { - ids.Add(new Uri(id)); - } - } - } -} -#endif \ No newline at end of file diff --git a/source/Calamari.Shared/Integration/Packages/NuGet/NuGetV3LibDownloader.cs b/source/Calamari.Shared/Integration/Packages/NuGet/NuGetV3LibDownloader.cs index 7d240dd42..ce27a83db 100644 --- a/source/Calamari.Shared/Integration/Packages/NuGet/NuGetV3LibDownloader.cs +++ b/source/Calamari.Shared/Integration/Packages/NuGet/NuGetV3LibDownloader.cs @@ -1,6 +1,4 @@ -#if USE_NUGET_V3_LIBS - -using System; +using System; using System.Net; using System.Threading; using NuGet.Configuration; @@ -8,17 +6,21 @@ using NuGet.Protocol; using NuGet.Protocol.Core.Types; using System.IO; -using Calamari.Common.Plumbing.Logging; +using System.Threading.Tasks; using Calamari.Common.Plumbing.Extensions; -using NuGet.DependencyResolver; -using NuGet.LibraryModel; +using NuGet.Commands; +using NuGet.Packaging.Core; using Octopus.Versioning; namespace Calamari.Integration.Packages.NuGet { public class NuGetV3LibDownloader { - public static void DownloadPackage(string packageId, IVersion version, Uri feedUri, ICredentials feedCredentials, string targetFilePath) + public static void DownloadPackage(string packageId, + IVersion version, + Uri feedUri, + ICredentials feedCredentials, + string targetFilePath) { ILogger logger = new NugetLogger(); var sourceRepository = Repository.Factory.GetCoreV3(feedUri.AbsoluteUri); @@ -28,47 +30,90 @@ public static void DownloadPackage(string packageId, IVersion version, Uri feedU sourceRepository.PackageSource.Credentials = new PackageSourceCredential("octopus", cred.UserName, cred.Password, true); } - using (var sourceCacheContext = new SourceCacheContext() { NoCache = true }) + var targetPath = Directory.GetParent(targetFilePath).FullName; + if (!Directory.Exists(targetPath)) { - var providers = new SourceRepositoryDependencyProvider(sourceRepository, logger, sourceCacheContext); - var libraryIdentity = new LibraryIdentity(packageId, version.ToNuGetVersion(), LibraryType.Package); + Directory.CreateDirectory(targetPath); + } - var targetPath = Directory.GetParent(targetFilePath).FullName; - if (!Directory.Exists(targetPath)) - { - Directory.CreateDirectory(targetPath); - } + var targetTempNupkg = Path.Combine(targetPath, Path.GetRandomFileName()); + var packageIdentity = new PackageIdentity(packageId, version.ToNuGetVersion()); + PerformNuGetDownload(packageIdentity, targetTempNupkg, sourceRepository, logger) + .GetAwaiter() + .GetResult(); - string targetTempNupkg = Path.Combine(targetPath, Path.GetRandomFileName()); - using (var nupkgStream = new FileStream(targetTempNupkg, - FileMode.Create, - FileAccess.ReadWrite, - FileShare.ReadWrite | FileShare.Delete, - 4096, - true)) - { - providers.CopyToAsync(libraryIdentity, nupkgStream, CancellationToken.None) - .GetAwaiter() - .GetResult(); - } + File.Move(targetTempNupkg, targetFilePath); - File.Move(targetTempNupkg, targetFilePath); - } } + static async Task PerformNuGetDownload(PackageIdentity packageIdentity, + string targetFilePath, + SourceRepository sourceRepository, + ILogger logger) + { + using var sourceCacheContext = new SourceCacheContext() { NoCache = true }; + var providers = new SourceRepositoryDependencyProvider(sourceRepository, + logger, + sourceCacheContext, + false, + true); + using var downloader = await providers.GetPackageDownloaderAsync(packageIdentity, sourceCacheContext, logger, CancellationToken.None); + await downloader.CopyNupkgFileToAsync(targetFilePath, default); + } + public class NugetLogger : ILogger { - public void LogDebug(string data) => Log.Verbose(data); - public void LogVerbose(string data) => Log.Verbose(data); - public void LogInformation(string data) => Log.Info(data); - public void LogMinimal(string data) => Log.Verbose(data); - public void LogWarning(string data) => Log.Warn(data); - public void LogError(string data) => Log.Error(data); - public void LogSummary(string data) => Log.Info(data); - public void LogInformationSummary(string data) => Log.Info(data); - public void LogErrorSummary(string data) => Log.Error(data); + public void LogDebug(string data) => Common.Plumbing.Logging.Log.Verbose(data); + public void LogVerbose(string data) => Common.Plumbing.Logging.Log.Verbose(data); + public void LogInformation(string data) => Common.Plumbing.Logging.Log.Info(data); + public void LogMinimal(string data) => Common.Plumbing.Logging.Log.Verbose(data); + public void LogWarning(string data) => Common.Plumbing.Logging.Log.Warn(data); + public void LogError(string data) => Common.Plumbing.Logging.Log.Error(data); + public void LogInformationSummary(string data) => Common.Plumbing.Logging.Log.Info(data); + + public void Log(LogLevel level, string data) + { + switch (level) + { + case LogLevel.Debug: + LogDebug(data); + break; + case LogLevel.Verbose: + LogVerbose(data); + break; + case LogLevel.Information: + LogInformation(data); + break; + case LogLevel.Minimal: + LogMinimal(data); + break; + case LogLevel.Warning: + LogWarning(data); + break; + case LogLevel.Error: + LogError(data); + break; + default: + throw new ArgumentOutOfRangeException(nameof(level), level, null); + } + } + + public Task LogAsync(LogLevel level, string data) + { + Log(level, data); + return Task.CompletedTask; + } + + public void Log(ILogMessage message) + { + Log(message.Level, message.Message); + } + + public Task LogAsync(ILogMessage message) + { + Log(message); + return Task.CompletedTask; + } } } -} - -#endif \ No newline at end of file +} \ No newline at end of file diff --git a/source/Calamari.Tests/Calamari.Tests.csproj b/source/Calamari.Tests/Calamari.Tests.csproj index 5c20d353a..a47473261 100644 --- a/source/Calamari.Tests/Calamari.Tests.csproj +++ b/source/Calamari.Tests/Calamari.Tests.csproj @@ -16,7 +16,7 @@ $(DefineConstants);NETCORE;AWS;AZURE_CORE;JAVA_SUPPORT - $(DefineConstants);NETFX;AWS;IIS_SUPPORT;USE_NUGET_V2_LIBS;USE_OCTODIFF_EXE;WINDOWS_CERTIFICATE_STORE_SUPPORT;WINDOWS_USER_ACCOUNT_SUPPORT;WINDOWS_REGISTRY_SUPPORT + $(DefineConstants);NETFX;AWS;IIS_SUPPORT;USE_OCTODIFF_EXE;WINDOWS_CERTIFICATE_STORE_SUPPORT;WINDOWS_USER_ACCOUNT_SUPPORT;WINDOWS_REGISTRY_SUPPORT diff --git a/source/Calamari.Tests/Fixtures/Integration/Packages/NuGetPackageDownloaderFixture.cs b/source/Calamari.Tests/Fixtures/Integration/Packages/NuGetPackageDownloaderFixture.cs index 92e21f37a..3111f92ca 100644 --- a/source/Calamari.Tests/Fixtures/Integration/Packages/NuGetPackageDownloaderFixture.cs +++ b/source/Calamari.Tests/Fixtures/Integration/Packages/NuGetPackageDownloaderFixture.cs @@ -64,92 +64,5 @@ public int AttemptsTheRightNumberOfTimesOnError(int maxDownloadAttempts) return calledCount; } - -#if USE_NUGET_V2_LIBS - const string SkipFreeBsdBecause = "performance on Mono+FreeBSD fluctuates significantly"; - - // We only support the specification of HTTP timeouts on V3 nuget endpoints in - // .NET framework. V2 nuget endpoints and .net core runtimes execute entirely - // different codepaths that don't give us an easy way to allow users to specify - // timeouts. - - [Test] - [NonParallelizable] - [RequiresNonFreeBSDPlatform(SkipFreeBsdBecause)] - [RequiresMinimumMonoVersion(5, 12, 0, Description = "HttpClient 4.3.2 broken on Mono - https://xamarin.github.io/bugzilla-archives/60/60315/bug.html#c7")] - public void TimesOutIfAValidTimeoutIsDefinedInVariables() - { - RunNugetV3TimeoutTest("00:00:01", TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(1)); - } - - [Test] - [NonParallelizable] - [RequiresNonFreeBSDPlatform(SkipFreeBsdBecause)] - [RequiresMinimumMonoVersion(5, 12, 0, Description = "HttpClient 4.3.2 broken on Mono - https://xamarin.github.io/bugzilla-archives/60/60315/bug.html#c7")] - public void IgnoresTheTimeoutIfAnInvalidTimeoutIsDefinedInVariables() - { - RunNugetV3TimeoutTest("this is not a valid timespan", TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(2)); - } - - [Test] - [NonParallelizable] - [RequiresNonFreeBSDPlatform(SkipFreeBsdBecause)] - [RequiresMinimumMonoVersion(5, 12, 0, Description = "HttpClient 4.3.2 broken on Mono - https://xamarin.github.io/bugzilla-archives/60/60315/bug.html#c7")] - public void DoesNotTimeOutIfTheServerRespondsBeforeTheTimeout() - { - RunNugetV3TimeoutTest("00:01:00", TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(1)); - } - - void RunNugetV3TimeoutTest(string timeoutInVariables, TimeSpan serverResponseTime, TimeSpan estimatedTimeout) - { - using (var server = new TestHttpServer(9001, serverResponseTime)) - { - var packageId = "FakePackageId"; - var version = VersionFactory.CreateSemanticVersion(1, 2, 3); - var feedCredentials = new CredentialCache(); - var targetFilePath = "FakeTargetFilePath"; - var filesystem = Substitute.For(); - var v3NugetUri = new Uri(server.BaseUrl + "/index.json"); - var variables = new CalamariVariables(); - - if (timeoutInVariables != null) - { - variables[KnownVariables.NugetHttpTimeout] = timeoutInVariables; - } - - var downloader = new InternalNuGetPackageDownloader(filesystem, variables); - - var stopwatch = new Stopwatch(); - - Action invocation = () => - { - stopwatch.Start(); - try - { - downloader.DownloadPackage( - packageId, - version, - v3NugetUri, - feedCredentials, - targetFilePath, - maxDownloadAttempts: 1, - downloadAttemptBackoff: TimeSpan.Zero - ); - } - finally - { - stopwatch.Stop(); - } - }; - - invocation.Should() - .ThrowExactly(); - - stopwatch.Elapsed - .Should() - .BeCloseTo(estimatedTimeout, TimeSpan.FromSeconds(0.5)); - } - } -#endif } } diff --git a/source/Calamari.Tests/Fixtures/Integration/Packages/TestHttpServer.cs b/source/Calamari.Tests/Fixtures/Integration/Packages/TestHttpServer.cs deleted file mode 100644 index 7ce63fde2..000000000 --- a/source/Calamari.Tests/Fixtures/Integration/Packages/TestHttpServer.cs +++ /dev/null @@ -1,45 +0,0 @@ -#if USE_NUGET_V2_LIBS -using System; -using System.Net; -using System.Threading; - -namespace Calamari.Tests.Fixtures.Integration.Packages -{ - public class TestHttpServer : IDisposable - { - readonly HttpListener listener; - - public int Port { get; } - public TimeSpan ResponseTime { get; } - - public string BaseUrl => $"http://localhost:{Port}"; - - public TestHttpServer(int port, TimeSpan responseTime) - { - Port = port; - ResponseTime = responseTime; - listener = new HttpListener - { - Prefixes = { BaseUrl + "/" } - }; - - listener.Start(); - listener.BeginGetContext(OnRequest, listener); - } - - void OnRequest(IAsyncResult result) - { - var context = listener.EndGetContext(result); - Thread.Sleep(ResponseTime); - var response = context.Response; - response.StatusCode = 200; - response.OutputStream.Close(); - } - - public void Dispose() - { - listener.Stop(); - } - } -} -#endif \ No newline at end of file diff --git a/source/Calamari.Tests/Fixtures/PackageDownload/NuGetFeedSupportFixture.cs b/source/Calamari.Tests/Fixtures/PackageDownload/NuGetFeedSupportFixture.cs index 3b4c87100..7c3d41f6e 100644 --- a/source/Calamari.Tests/Fixtures/PackageDownload/NuGetFeedSupportFixture.cs +++ b/source/Calamari.Tests/Fixtures/PackageDownload/NuGetFeedSupportFixture.cs @@ -72,7 +72,6 @@ public void ShouldSupportFeedzNuGetVersion3Feeds(string versionString) [Test] [TestCaseSource(nameof(ArtifactoryNuGet3SupportedVersionStrings))] [TestCaseSource(nameof(ArtifactoryNuGet2SupportedVersionStrings))] - [Platform("Net-4.5")] public void ArtifactoryShouldSupportNuGetVersion3Feeds(string versionString) { var calamariResult = DownloadPackage(TestNuGetPackageId, versionString, "nuget-local", ArtifactoryNuGetV3FeedUrl); diff --git a/source/Calamari.Tests/Fixtures/PackageDownload/PackageDownloadFixture.cs b/source/Calamari.Tests/Fixtures/PackageDownload/PackageDownloadFixture.cs index a494bd74a..6abc2fd95 100644 --- a/source/Calamari.Tests/Fixtures/PackageDownload/PackageDownloadFixture.cs +++ b/source/Calamari.Tests/Fixtures/PackageDownload/PackageDownloadFixture.cs @@ -354,9 +354,7 @@ public void PrivateNuGetFeedShouldDownloadPackage() result.AssertSuccess(); result.AssertOutput("Downloading NuGet package {0} v{1} from feed: '{2}'", FeedzPackage.PackageId, FeedzPackage.Version, AuthFeedUri); result.AssertOutput("Downloaded package will be stored in: '{0}'", FeedzPackage.DownloadFolder); -#if USE_NUGET_V2_LIBS - result.AssertOutput("Found package {0} v{1}", FeedzPackage.PackageId, FeedzPackage.Version); -#endif + AssertPackageHashMatchesExpected(result, ExpectedPackageHash); AssertPackageSizeMatchesExpected(result, ExpectedPackageSize); AssertStagePackageOutputVariableSet(result, FeedzPackage); @@ -393,9 +391,7 @@ public void PrivateNuGetFeedShouldByPassCacheAndDownloadPackage() result.AssertOutput("Downloading NuGet package {0} v{1} from feed: '{2}'", FeedzPackage.PackageId, FeedzPackage.Version, AuthFeedUri); result.AssertOutput("Downloaded package will be stored in: '{0}'", FeedzPackage.DownloadFolder); -#if USE_NUGET_V2_LIBS - result.AssertOutput("Found package {0} v{1}", FeedzPackage.PackageId, FeedzPackage.Version); -#endif + AssertPackageHashMatchesExpected(result, ExpectedPackageHash); AssertPackageSizeMatchesExpected(result, ExpectedPackageSize); AssertStagePackageOutputVariableSet(result, FeedzPackage);