From c57627ba36a59b3722ef5516ae213442b2b8a8c5 Mon Sep 17 00:00:00 2001 From: Steffen Forkmann Date: Thu, 5 Dec 2019 10:37:42 +0100 Subject: [PATCH 1/2] Cache package meta data during version check --- src/Paket.Core/Dependencies/NuGet.fs | 14 +- src/Paket.Core/Dependencies/NuGetCache.fs | 46 ++--- src/Paket.Core/Dependencies/NuGetV2.fs | 228 ++++++++++++---------- 3 files changed, 156 insertions(+), 132 deletions(-) diff --git a/src/Paket.Core/Dependencies/NuGet.fs b/src/Paket.Core/Dependencies/NuGet.fs index 0d51afca77..5e394fe30f 100644 --- a/src/Paket.Core/Dependencies/NuGet.fs +++ b/src/Paket.Core/Dependencies/NuGet.fs @@ -377,7 +377,7 @@ let GetTargetsFiles(targetFolder, (pkg : PackageName)) = [] let GetAnalyzerFiles(targetFolder) = getFilesMatching targetFolder "*.dll" "analyzers" "analyzer dlls" -let tryNuGetV3 (auth, nugetV3Url, package:PackageName) = +let tryNuGetV3 (force, auth, nugetV3Url, package:PackageName) = NuGetV3.findVersionsForPackage(nugetV3Url, auth, package) @@ -564,8 +564,8 @@ type GetVersionRequestResult = member x.Versions = x.Requests |> Array.collect (fun r -> r.Versions) -let getVersionsCached key f (source, auth, nugetURL, package) = - let request:NuGetCache.NuGetRequestGetVersions = f (auth, nugetURL, package) +let getVersionsCached key f (force, source, auth, nugetURL, package) = + let request:NuGetCache.NuGetRequestGetVersions = f (force, auth, nugetURL, package) NuGetCache.NuGetRequestGetVersions.ofFunc request.Url (fun _ -> async { match protocolCache.TryGetValue(source) with @@ -616,16 +616,16 @@ let GetVersions force alternativeProjectRoot root (parameters:GetPackageVersions | NuGetV2 source -> let auth = source.Authentication if String.containsIgnoreCase "artifactory" source.Url then - return [getVersionsCached "ODataNewestFirst" NuGetV2.tryGetAllVersionsFromNugetODataFindByIdNewestFirst (nugetSource, auth, source.Url, packageName) ] + return [ getVersionsCached "ODataNewestFirst" NuGetV2.tryGetAllVersionsFromNugetODataFindByIdNewestFirst (force, nugetSource, auth, source.Url, packageName) ] else let v2Feeds = - [ yield getVersionsCached "OData" NuGetV2.tryGetAllVersionsFromNugetODataFindById (nugetSource, auth, source.Url, packageName) - yield getVersionsCached "ODataWithFilter" NuGetV2.tryGetAllVersionsFromNugetODataWithFilter (nugetSource, auth, source.Url, packageName) ] + [ yield getVersionsCached "OData" NuGetV2.tryGetAllVersionsFromNugetODataFindById (force, nugetSource, auth, source.Url, packageName) + yield getVersionsCached "ODataWithFilter" NuGetV2.tryGetAllVersionsFromNugetODataWithFilter (force, nugetSource, auth, source.Url, packageName) ] return v2Feeds | NuGetV3 source -> let! versionsAPI = NuGetV3.getNuGetV3Resource source NuGetV3.AllVersionsAPI - return [ getVersionsCached "V3" tryNuGetV3 (nugetSource, source.Authentication, versionsAPI, packageName) ] + return [ getVersionsCached "V3" tryNuGetV3 (force, nugetSource, source.Authentication, versionsAPI, packageName) ] | LocalNuGet(path,Some _) -> return [ NuGetLocal.getAllVersionsFromLocalPath (true, path, packageName, alternativeProjectRoot, root) ] | LocalNuGet(path,None) -> diff --git a/src/Paket.Core/Dependencies/NuGetCache.fs b/src/Paket.Core/Dependencies/NuGetCache.fs index 19e9d2a7e3..5caa7664b2 100644 --- a/src/Paket.Core/Dependencies/NuGetCache.fs +++ b/src/Paket.Core/Dependencies/NuGetCache.fs @@ -432,44 +432,46 @@ let getCacheDataFromExtractedPackage (packageName:PackageName) (version:SemVerIn return None } -let getDetailsFromCacheOr force nugetURL (packageName:PackageName) (version:SemVerInfo) (getViaWebRequest : unit -> ODataSearchResult Async) : ODataSearchResult Async = - let writeCacheFile(result:NuGetPackageCache) = - let cacheFile = getCacheFiles force NuGetPackageCache.CurrentCacheVersion nugetURL packageName version - let serialized = JsonConvert.SerializeObject(result) - let cachedData = - try - if cacheFile.Exists then - use cacheReader = cacheFile.OpenText() - cacheReader.ReadToEnd() - else "" - with - | ex -> - traceWarnfn "Can't read cache file %O:%s Message: %O" cacheFile Environment.NewLine ex - "" - if String.CompareOrdinal(serialized, cachedData) <> 0 then - File.WriteAllText(cacheFile.FullName, serialized) +let writePackageDetailsCacheFile force nugetURL (packageName:PackageName) (version:SemVerInfo) (result:NuGetPackageCache) = + let cacheFile = getCacheFiles force NuGetPackageCache.CurrentCacheVersion nugetURL packageName version + let serialized = JsonConvert.SerializeObject(result) + let cachedData = + try + if cacheFile.Exists then + use cacheReader = cacheFile.OpenText() + cacheReader.ReadToEnd() + else "" + with + | ex -> + traceWarnfn "Can't read cache file %O:%s Message: %O" cacheFile Environment.NewLine ex + "" + if String.CompareOrdinal(serialized, cachedData) <> 0 then + File.WriteAllText(cacheFile.FullName, serialized) +let getDetailsFromCacheOr force nugetURL (packageName:PackageName) (version:SemVerInfo) (getViaWebRequest : unit -> ODataSearchResult Async) : ODataSearchResult Async = let getViaWebRequest () = async { let! result = getViaWebRequest() match result with | Match result -> - writeCacheFile result + writePackageDetailsCacheFile force nugetURL packageName version result | _ -> // TODO: Should we cache 404? Probably not. () return result } + async { match tryGetDetailsFromCache force nugetURL packageName version with - | None when force -> return! getViaWebRequest() + | None when force -> + return! getViaWebRequest() | None -> - let! result = getCacheDataFromExtractedPackage packageName version - match result with + match! getCacheDataFromExtractedPackage packageName version with | Some result -> - writeCacheFile result + writePackageDetailsCacheFile force nugetURL packageName version result return Match result - | _ -> return! getViaWebRequest() + | _ -> + return! getViaWebRequest() | Some res -> return res } diff --git a/src/Paket.Core/Dependencies/NuGetV2.fs b/src/Paket.Core/Dependencies/NuGetV2.fs index 0c3d0e3501..02ad3024e5 100644 --- a/src/Paket.Core/Dependencies/NuGetV2.fs +++ b/src/Paket.Core/Dependencies/NuGetV2.fs @@ -20,7 +20,92 @@ open Paket.Requirements open FSharp.Polyfill open System.Runtime.ExceptionServices -let private followODataLink auth url = +let private handleODataEntry nugetURL packageName version entry = + let officialName = + match (entry |> getNode "properties" |> optGetNode "Id") ++ (entry |> getNode "title") with + | Some node -> node.InnerText + | _ -> failwithf "Could not get official package name for package %O %O" packageName version + + let publishDate = + match entry |> getNode "properties" |> optGetNode "Published" with + | Some node -> + match DateTime.TryParse node.InnerText with + | true, date -> date + | _ -> DateTime.MinValue + | _ -> DateTime.MinValue + + let v = + match entry |> getNode "properties" |> optGetNode "Version" with + | Some node -> node.InnerText + | _ -> failwithf "Could not get official version no. for package %O %O" packageName version + + let downloadLink = + match entry |> getNode "content" |> optGetAttribute "type", + entry |> getNode "content" |> optGetAttribute "src" with + | Some "application/zip", Some link -> link + | Some "binary/octet-stream", Some link -> link + | _ -> failwithf "unable to find downloadLink for package %O %O" packageName version + + let licenseUrl = + match entry |> getNode "properties" |> optGetNode "LicenseUrl" with + | Some node -> node.InnerText + | _ -> "" + + let dependencies = + match entry |> getNode "properties" |> optGetNode "Dependencies" with + | Some node -> node.InnerText + | None -> failwithf "unable to find dependencies for package %O %O" packageName version + + let rawPackages = + let split (d : string) = + let a = d.Split ':' + let name = PackageName a.[0] + let version = VersionRequirement.Parse(if a.Length > 1 then a.[1] else "0") + (if a.Length > 2 && a.[2] <> "" then + let restriction = a.[2] + match PlatformMatching.extractPlatforms false restriction with + | Some p -> + Some p + | None -> + if not (restriction.StartsWith "_") then + Logging.traceWarnIfNotBefore ("Package", restriction, packageName, version) "Could not detect any platforms from '%s' in package %O %O, please tell the package authors" restriction packageName version + None + else Some PlatformMatching.ParsedPlatformPath.Empty) + |> Option.map (fun pp -> name, version, pp) + + dependencies + |> fun s -> s.Split([| '|' |], System.StringSplitOptions.RemoveEmptyEntries) + |> Array.choose split + + let frameworks = + rawPackages + |> Seq.map (fun (_,_,pp) -> pp) + |> Seq.distinctBy (fun pp -> pp.Platforms |> List.sort) + |> Seq.toList + + let cleanedPackages = + rawPackages + |> Seq.filter (fun (n,_,_) -> System.String.IsNullOrEmpty (n.ToString()) |> not) + |> Seq.toList + + let dependencies, warnings = addFrameworkRestrictionsToDependencies cleanedPackages frameworks + + for warning in warnings do + let message = warning.Format officialName version + Logging.traceWarnIfNotBefore message "%s" message + + { PackageName = officialName + DownloadUrl = downloadLink + SerializedDependencies = [] + SourceUrl = nugetURL + CacheVersion = NuGetPackageCache.CurrentCacheVersion + LicenseUrl = licenseUrl + Version = (SemVer.Parse v).Normalize() + Unlisted = publishDate = Constants.MagicUnlistingDate } + .WithDependencies dependencies + + +let private followODataLink force packageName auth nugetURL url = let rec followODataLinkSafe (knownVersions:Set<_>) (url:string) = async { let! raw = getFromUrl(auth, url, acceptXml) @@ -32,12 +117,32 @@ let private followODataLink auth url = | Some node -> node | None -> failwithf "unable to parse data from %s" url - let readEntryVersion = Some - >> optGetNode "properties" - >> optGetNode "Version" - >> Option.map (fun node -> node.InnerText) + let readEntryVersion node = + let v = + node + |> getNode "properties" + |> optGetNode "Version" + |> Option.map (fun node -> node.InnerText) + match v with + | Some v -> + let fullEntry = + try + Some(handleODataEntry nugetURL packageName v node) + with + | _ -> None + Some v, fullEntry + | None -> + None, None + + let entries = feed |> getNodes "entry" |> List.map readEntryVersion + let entriesVersions = entries |> List.choose fst + let fullMetaData = entries |> List.choose snd + for entry in fullMetaData do + try + NuGetCache.writePackageDetailsCacheFile force nugetURL packageName (SemVer.Parse entry.Version) entry + with + | _ -> () - let entriesVersions = feed |> getNodes "entry" |> List.choose readEntryVersion let newSet = entriesVersions |> Set.ofList let noNewFound = newSet.IsSubsetOf knownVersions if noNewFound then @@ -76,7 +181,7 @@ let private followODataLink auth url = let mutable uri = null // warn once per specific API endpoint, but try to cut the query let baseUrl = if Uri.TryCreate(url, UriKind.Absolute, &uri) then uri.AbsolutePath else url traceWarnIfNotBefore baseUrl - "At least one 'next' link (index %d) returned a empty result (noticed on '%O'): ['%s']" + "At least one 'next' link (index %d) returned an empty result (noticed on '%O'): ['%s']" i url (System.String.Join("' ; '", linksToFollow)) | None -> () return @@ -91,12 +196,12 @@ let private followODataLink auth url = let private tryGetAllVersionsFromNugetODataWithFilterWarnings = System.Collections.Concurrent.ConcurrentDictionary<_,_>() -let tryGetAllVersionsFromNugetODataWithFilter (auth, nugetURL, package:PackageName) = - let url = sprintf "%s/Packages?semVerLevel=2.0.0&$filter=Id eq '%O'" nugetURL package +let tryGetAllVersionsFromNugetODataWithFilter (force, auth, nugetURL, packageName:PackageName) = + let url = sprintf "%s/Packages?semVerLevel=2.0.0&$filter=Id eq '%O'" nugetURL packageName NuGetRequestGetVersions.ofSimpleFunc url (fun _ -> async { try - let! result = followODataLink auth url + let! result = followODataLink force packageName auth nugetURL url return SuccessResponse result with exn -> match tryGetAllVersionsFromNugetODataWithFilterWarnings.TryGetValue nugetURL with @@ -106,9 +211,9 @@ let tryGetAllVersionsFromNugetODataWithFilter (auth, nugetURL, package:PackageNa tryGetAllVersionsFromNugetODataWithFilterWarnings.TryAdd(nugetURL, true) |> ignore if verbose then printfn "Error while retrieving data from '%s': %O" url exn - let url = sprintf "%s/Packages?semVerLevel=2.0.0&$filter=tolower(Id) eq '%s'" nugetURL (package.CompareString) + let url = sprintf "%s/Packages?semVerLevel=2.0.0&$filter=tolower(Id) eq '%s'" nugetURL (packageName.CompareString) try - let! result = followODataLink auth url + let! result = followODataLink force packageName auth nugetURL url return SuccessResponse result with exn -> let cap = ExceptionDispatchInfo.Capture exn @@ -116,24 +221,24 @@ let tryGetAllVersionsFromNugetODataWithFilter (auth, nugetURL, package:PackageNa }) -let tryGetAllVersionsFromNugetODataFindById (auth, nugetURL, package:PackageName) = - let url = sprintf "%s/FindPackagesById()?semVerLevel=2.0.0&id='%O'" nugetURL package +let tryGetAllVersionsFromNugetODataFindById (force, auth, nugetURL, packageName:PackageName) = + let url = sprintf "%s/FindPackagesById()?semVerLevel=2.0.0&id='%O'" nugetURL packageName NuGetRequestGetVersions.ofSimpleFunc url (fun _ -> async { try - let! result = followODataLink auth url + let! result = followODataLink force packageName auth nugetURL url return SuccessResponse result with exn -> let cap = ExceptionDispatchInfo.Capture exn return UnknownError cap }) -let tryGetAllVersionsFromNugetODataFindByIdNewestFirst (auth, nugetURL, package:PackageName) = - let url = sprintf "%s/FindPackagesById()?semVerLevel=2.0.0&id='%O'&$orderby=Published desc" nugetURL package +let tryGetAllVersionsFromNugetODataFindByIdNewestFirst (force, auth, nugetURL, packageName:PackageName) = + let url = sprintf "%s/FindPackagesById()?semVerLevel=2.0.0&id='%O'&$orderby=Published desc" nugetURL packageName NuGetRequestGetVersions.ofSimpleFunc url (fun _ -> async { try - let! result = followODataLink auth url + let! result = followODataLink force packageName auth nugetURL url return SuccessResponse result with exn -> let cap = ExceptionDispatchInfo.Capture exn @@ -148,89 +253,6 @@ let private getXmlDoc url raw = | e -> raise (Exception(sprintf "Could not parse response from %s as OData.%sData:%s%s" url Environment.NewLine Environment.NewLine raw, e)) doc -let private handleODataEntry nugetURL packageName version entry = - let officialName = - match (entry |> getNode "properties" |> optGetNode "Id") ++ (entry |> getNode "title") with - | Some node -> node.InnerText - | _ -> failwithf "Could not get official package name for package %O %O" packageName version - - let publishDate = - match entry |> getNode "properties" |> optGetNode "Published" with - | Some node -> - match DateTime.TryParse node.InnerText with - | true, date -> date - | _ -> DateTime.MinValue - | _ -> DateTime.MinValue - - let v = - match entry |> getNode "properties" |> optGetNode "Version" with - | Some node -> node.InnerText - | _ -> failwithf "Could not get official version no. for package %O %O" packageName version - - let downloadLink = - match entry |> getNode "content" |> optGetAttribute "type", - entry |> getNode "content" |> optGetAttribute "src" with - | Some "application/zip", Some link -> link - | Some "binary/octet-stream", Some link -> link - | _ -> failwithf "unable to find downloadLink for package %O %O" packageName version - - let licenseUrl = - match entry |> getNode "properties" |> optGetNode "LicenseUrl" with - | Some node -> node.InnerText - | _ -> "" - - let dependencies = - match entry |> getNode "properties" |> optGetNode "Dependencies" with - | Some node -> node.InnerText - | None -> failwithf "unable to find dependencies for package %O %O" packageName version - - let rawPackages = - let split (d : string) = - let a = d.Split ':' - let name = PackageName a.[0] - let version = VersionRequirement.Parse(if a.Length > 1 then a.[1] else "0") - (if a.Length > 2 && a.[2] <> "" then - let restriction = a.[2] - match PlatformMatching.extractPlatforms false restriction with - | Some p -> - Some p - | None -> - if not (restriction.StartsWith "_") then - Logging.traceWarnIfNotBefore ("Package", restriction, packageName, version) "Could not detect any platforms from '%s' in package %O %O, please tell the package authors" restriction packageName version - None - else Some PlatformMatching.ParsedPlatformPath.Empty) - |> Option.map (fun pp -> name, version, pp) - - dependencies - |> fun s -> s.Split([| '|' |], System.StringSplitOptions.RemoveEmptyEntries) - |> Array.choose split - - let frameworks = - rawPackages - |> Seq.map (fun (_,_,pp) -> pp) - |> Seq.distinctBy (fun pp -> pp.Platforms |> List.sort) - |> Seq.toList - - let cleanedPackages = - rawPackages - |> Seq.filter (fun (n,_,_) -> System.String.IsNullOrEmpty (n.ToString()) |> not) - |> Seq.toList - - let dependencies, warnings = addFrameworkRestrictionsToDependencies cleanedPackages frameworks - - for warning in warnings do - let message = warning.Format officialName version - Logging.traceWarnIfNotBefore message "%s" message - - { PackageName = officialName - DownloadUrl = downloadLink - SerializedDependencies = [] - SourceUrl = nugetURL - CacheVersion = NuGetPackageCache.CurrentCacheVersion - LicenseUrl = licenseUrl - Version = (SemVer.Parse v).Normalize() - Unlisted = publishDate = Constants.MagicUnlistingDate } - .WithDependencies dependencies // parse search results. let parseODataListDetails (url,nugetURL,packageName:PackageName,version:SemVerInfo,doc) : ODataSearchResult = @@ -384,7 +406,7 @@ let getDetailsFromNuGetViaODataFast isVersionAssumed nugetSource (packageName:Pa } // parse search results. -let parseFindPackagesByIDODataListDetails (url,nugetURL,packageName:PackageName,version:SemVerInfo,doc) : ODataSearchResult = +let parseFindPackagesByIDODataListDetails (nugetURL,packageName:PackageName,version:SemVerInfo,doc) : ODataSearchResult = let feedNode = match doc |> getNode "feed" with | Some node -> node @@ -414,7 +436,7 @@ let rec parseFindPackagesByIDODataEntryDetails (url,nugetSource:NuGetSource,pack tracefn "%s" raw let doc = getXmlDoc url raw - match parseFindPackagesByIDODataListDetails (url,nugetSource.Url,packageName,version,doc) with + match parseFindPackagesByIDODataListDetails (nugetSource.Url,packageName,version,doc) with | EmptyResult -> let linksToFollow = doc From 8892711fd4de0a7a5ade955e4b675ba263626659 Mon Sep 17 00:00:00 2001 From: Steffen Forkmann Date: Thu, 5 Dec 2019 12:42:15 +0100 Subject: [PATCH 2/2] Cleanup --- src/Paket.Core/Dependencies/NuGetV2.fs | 14 +++++++------- src/Paket.Core/Dependencies/PackageResolver.fs | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Paket.Core/Dependencies/NuGetV2.fs b/src/Paket.Core/Dependencies/NuGetV2.fs index 02ad3024e5..47d196b81f 100644 --- a/src/Paket.Core/Dependencies/NuGetV2.fs +++ b/src/Paket.Core/Dependencies/NuGetV2.fs @@ -67,7 +67,7 @@ let private handleODataEntry nugetURL packageName version entry = | Some p -> Some p | None -> - if not (restriction.StartsWith "_") then + if not (restriction.StartsWith "_") then Logging.traceWarnIfNotBefore ("Package", restriction, packageName, version) "Could not detect any platforms from '%s' in package %O %O, please tell the package authors" restriction packageName version None else Some PlatformMatching.ParsedPlatformPath.Empty) @@ -117,18 +117,18 @@ let private followODataLink force packageName auth nugetURL url = | Some node -> node | None -> failwithf "unable to parse data from %s" url - let readEntryVersion node = - let v = + let readEntryVersion node = + let v = node |> getNode "properties" |> optGetNode "Version" |> Option.map (fun node -> node.InnerText) match v with - | Some v -> - let fullEntry = + | Some v -> + let fullEntry = try Some(handleODataEntry nugetURL packageName v node) - with + with | _ -> None Some v, fullEntry | None -> @@ -181,7 +181,7 @@ let private followODataLink force packageName auth nugetURL url = let mutable uri = null // warn once per specific API endpoint, but try to cut the query let baseUrl = if Uri.TryCreate(url, UriKind.Absolute, &uri) then uri.AbsolutePath else url traceWarnIfNotBefore baseUrl - "At least one 'next' link (index %d) returned an empty result (noticed on '%O'): ['%s']" + "At least one 'next' link (index %d) returned an empty result (noticed on '%O'): ['%s']" i url (System.String.Join("' ; '", linksToFollow)) | None -> () return diff --git a/src/Paket.Core/Dependencies/PackageResolver.fs b/src/Paket.Core/Dependencies/PackageResolver.fs index 46612d9b40..b2b2cd5071 100644 --- a/src/Paket.Core/Dependencies/PackageResolver.fs +++ b/src/Paket.Core/Dependencies/PackageResolver.fs @@ -1049,7 +1049,7 @@ let Resolve (getVersionsRaw : PackageVersionsFunc, getPreferredVersionsRaw : Pre try getAndReport details.Package.Sources Profile.BlockReason.PackageDetails workHandle with e -> - raise (Exception (sprintf "Unable to retrieve package details for '%O'-%s" details.Package.PackageName details.Version.AsString, e)) + raise (Exception (sprintf "Unable to retrieve package details for %O %O" details.Package.PackageName details.Version, e)) let startedGetVersionsRequests = System.Collections.Concurrent.ConcurrentDictionary<_,ResolverTaskMemory<_>>() let startRequestGetVersions (versions:GetPackageVersionsParameters) =