From e59c8629eb1a476eaacfaf53ce22afcae8010f8e Mon Sep 17 00:00:00 2001 From: Scott Arbeit Date: Thu, 2 Jan 2025 00:52:54 -0800 Subject: [PATCH] Refactor logging, caching, and CLI enhancements. Refactored table column definitions in Repository.CLI.fs to use AddColumns with an array of TableColumn objects. Added a new "When" column and moved "Updated at" to the end. Updated table row rendering to include the new ago property. In Services.CLI.fs, added a check for binary files before computing SHA-256 hash and reset the file stream. Updated comments for clarity and removed redundant binary file check code. Added Microsoft.Extensions.Caching.Memory namespace in Common.SDK.fs. Introduced checkMemoryCache function to initialize memory cache if null, called in getServer and postServer functions. Changed default BranchName in BranchDto to "root" in Dto.Shared.fs. --- src/Grace.Actors/Branch.Actor.fs | 2 +- .../MemoryCache.Extensions.Actor.fs | 2 +- src/Grace.CLI/Command/Branch.CLI.fs | 694 ++---------------- src/Grace.CLI/Command/Repository.CLI.fs | 20 +- src/Grace.CLI/Command/Services.CLI.fs | 7 +- src/Grace.SDK/Common.SDK.fs | 10 + src/Grace.Shared/Dto/Dto.Shared.fs | 2 +- src/Grace.Shared/Services.Shared.fs | 44 +- src/Grace.Shared/Utilities.Shared.fs | 37 +- 9 files changed, 143 insertions(+), 675 deletions(-) diff --git a/src/Grace.Actors/Branch.Actor.fs b/src/Grace.Actors/Branch.Actor.fs index 9e50632..ca88238 100644 --- a/src/Grace.Actors/Branch.Actor.fs +++ b/src/Grace.Actors/Branch.Actor.fs @@ -266,7 +266,7 @@ module Branch = && not <| (context.MethodName = "ReceiveReminderAsync") then log.LogInformation( - "{CurrentInstant}: Node: {HostName}; Duration: {duration_ms}ms; CorrelationId: {correlationId}; Finished {ActorName}.{MethodName}; RepositoryId: {RepositoryId}; BranchId: {Id}; BranchName: {BranchName}; ThreadCounts: {ThreadCounts}.", + "{CurrentInstant}: Node: {HostName}; Duration: {duration_ms}ms; CorrelationId: {correlationId}; Finished {ActorName}.{MethodName}; RepositoryId: {RepositoryId}; BranchId: {Id}; BranchName: {BranchName}; {ThreadCounts}.", getCurrentInstantExtended (), getMachineName, duration_ms, diff --git a/src/Grace.Actors/Extensions/MemoryCache.Extensions.Actor.fs b/src/Grace.Actors/Extensions/MemoryCache.Extensions.Actor.fs index 453d8bf..29de984 100644 --- a/src/Grace.Actors/Extensions/MemoryCache.Extensions.Actor.fs +++ b/src/Grace.Actors/Extensions/MemoryCache.Extensions.Actor.fs @@ -189,7 +189,7 @@ module MemoryCache = /// Create a new entry in MemoryCache to store the current thread count information. member this.CreateThreadCountEntry(threadInfo: string) = - use newCacheEntry = this.CreateEntry("ThreadCounts", Value = threadInfo, AbsoluteExpiration = DateTimeOffset.UtcNow.Add(TimeSpan.FromSeconds(1.0))) + use newCacheEntry = this.CreateEntry("ThreadCounts", Value = threadInfo, AbsoluteExpiration = DateTimeOffset.UtcNow.Add(TimeSpan.FromSeconds(6.0))) () /// Check if we have an entry in MemoryCache for the current ThreadCount. diff --git a/src/Grace.CLI/Command/Branch.CLI.fs b/src/Grace.CLI/Command/Branch.CLI.fs index 8b083cc..dd3635d 100644 --- a/src/Grace.CLI/Command/Branch.CLI.fs +++ b/src/Grace.CLI/Command/Branch.CLI.fs @@ -1834,7 +1834,7 @@ module Branch = member val public ReferenceId = String.Empty with get, set member val public Sha256Hash = Sha256Hash String.Empty with get, set - let private switchHandler2 parseResult (switchParameters: SwitchParameters) = + let private switchHandler parseResult (switchParameters: SwitchParameters) = task { try /// The GraceStatus at the beginning of running this command. @@ -2390,543 +2390,13 @@ module Branch = return -1 } - let private switchHandler parseResult (parameters: SwitchParameters) = - task { - try - if parseResult |> verbose then printParseResult parseResult - - let validateIncomingParameters = CommonValidations parseResult parameters - - match validateIncomingParameters with - | Ok _ -> - let getParameters = - Parameters.Branch.GetBranchParameters( - OwnerId = parameters.OwnerId, - OwnerName = parameters.OwnerName, - OrganizationId = parameters.OrganizationId, - OrganizationName = parameters.OrganizationName, - RepositoryId = parameters.RepositoryId, - RepositoryName = parameters.RepositoryName, - BranchId = $"{Current().BranchId}", - CorrelationId = parameters.CorrelationId - ) - - match! Branch.Get(getParameters) with - | Ok currentBranch -> - let mutable rootDirectoryId = DirectoryVersionId.Empty - let mutable rootDirectorySha256Hash = Sha256Hash String.Empty - let mutable previousDirectoryIds: HashSet = null - let repositoryId = RepositoryId.Parse(parameters.RepositoryId) - - if parseResult |> hasOutput then - return! - progress - .Columns(progressColumns) - .StartAsync(fun progressContext -> - task { - let t0 = progressContext.AddTask($"[{Color.DodgerBlue1}]Reading Grace index file.[/]") - - let t1 = - progressContext.AddTask($"[{Color.DodgerBlue1}]Scanning working directory for changes.[/]", autoStart = false) - - let t2 = progressContext.AddTask($"[{Color.DodgerBlue1}]Creating new directory verions.[/]", autoStart = false) - - let t3 = - progressContext.AddTask( - $"[{Color.DodgerBlue1}]Uploading changed files to object storage.[/]", - autoStart = false - ) - - let t4 = progressContext.AddTask($"[{Color.DodgerBlue1}]Uploading new directory versions.[/]", autoStart = false) - - let t5 = progressContext.AddTask($"[{Color.DodgerBlue1}]Create a save reference.[/]", autoStart = false) - - let t6 = - progressContext.AddTask( - $"[{Color.DodgerBlue1}]Getting new branch version from the server.[/]", - autoStart = false - ) - - let t7 = - progressContext.AddTask( - $"[{Color.DodgerBlue1}]Updating object cache and working directory.[/]", - autoStart = false - ) - - t0.Increment(0.0) - let! previousGraceStatus = readGraceStatusFile () - let mutable newGraceStatus = previousGraceStatus - t0.Value <- 100.0 - - if currentBranch.ReturnValue.SaveEnabled then - match! getGraceWatchStatus () with - | Some graceWatchStatus -> - t1.Value <- 100.0 - t2.Value <- 100.0 - t3.Value <- 100.0 - t4.Value <- 100.0 - t5.Value <- 100.0 - rootDirectoryId <- graceWatchStatus.RootDirectoryId - rootDirectorySha256Hash <- graceWatchStatus.RootDirectorySha256Hash - previousDirectoryIds <- graceWatchStatus.DirectoryIds - | None -> - t1.StartTask() - let! differences = scanForDifferences previousGraceStatus - t1.Value <- 100.0 - - t2.StartTask() - - let! (updatedGraceStatus, newDirectoryVersions) = - getNewGraceStatusAndDirectoryVersions previousGraceStatus differences - - newGraceStatus <- updatedGraceStatus - rootDirectoryId <- newGraceStatus.RootDirectoryId - rootDirectorySha256Hash <- newGraceStatus.RootDirectorySha256Hash - previousDirectoryIds <- newGraceStatus.Index.Keys.ToHashSet() - t2.Value <- 100.0 - - t3.StartTask() - - let updatedRelativePaths = - differences - .Select(fun difference -> - match difference.DifferenceType with - | Add -> - match difference.FileSystemEntryType with - | FileSystemEntryType.File -> Some difference.RelativePath - | FileSystemEntryType.Directory -> None - | Change -> - match difference.FileSystemEntryType with - | FileSystemEntryType.File -> Some difference.RelativePath - | FileSystemEntryType.Directory -> None - | Delete -> None) - .Where(fun relativePathOption -> relativePathOption.IsSome) - .Select(fun relativePath -> relativePath.Value) - - let newFileVersions = - updatedRelativePaths.Select(fun relativePath -> - newDirectoryVersions - .First(fun dv -> dv.Files.Exists(fun file -> file.RelativePath = relativePath)) - .Files.First(fun file -> file.RelativePath = relativePath)) - - let getUploadMetadataForFilesParameters = - Storage.GetUploadMetadataForFilesParameters( - OwnerId = parameters.OwnerId, - OwnerName = parameters.OwnerName, - OrganizationId = parameters.OrganizationId, - OrganizationName = parameters.OrganizationName, - RepositoryId = parameters.RepositoryId, - RepositoryName = parameters.RepositoryName, - CorrelationId = getCorrelationId parseResult, - FileVersions = - (newFileVersions - |> Seq.map (fun localFileVersion -> localFileVersion.ToFileVersion) - |> Seq.toArray) - ) - - let! uploadResult = uploadFilesToObjectStorage getUploadMetadataForFilesParameters - - t3.Value <- 100.0 - - t4.StartTask() - let saveParameters = SaveDirectoryVersionsParameters() - - saveParameters.DirectoryVersions <- newDirectoryVersions.Select(fun dv -> dv.ToDirectoryVersion).ToList() - - let! uploadDirectoryVersions = Directory.SaveDirectoryVersions saveParameters - - t4.Value <- 100.0 - - t5.StartTask() - - let! saveResult = - createSaveReference - newGraceStatus.Index[rootDirectoryId] - $"Save created during branch switch." - (getCorrelationId parseResult) - - match saveResult with - | Ok returnValue -> () - | Error error -> logToAnsiConsole Colors.Error $"{error}" - - t5.Value <- 100.0 - else - // Save is not enabled on this branch, so we can skip all of the above. - t1.Value <- 100.0 - t2.Value <- 100.0 - t3.Value <- 100.0 - t4.Value <- 100.0 - t5.Value <- 100.0 - // Get current values from the current branch based on finding no differences, because we're not creating a save or uploading anything. - let! (updatedGraceStatus, newDirectoryVersions) = - getNewGraceStatusAndDirectoryVersions previousGraceStatus (List()) - - newGraceStatus <- updatedGraceStatus - rootDirectoryId <- newGraceStatus.RootDirectoryId - rootDirectorySha256Hash <- newGraceStatus.RootDirectorySha256Hash - previousDirectoryIds <- newGraceStatus.Index.Keys.ToHashSet() - - t6.StartTask() - // If we have a branch name or ID to switch to, cool, let's do it. - //logToAnsiConsole Colors.Verbose $"parameters.BranchName: {parameters.BranchName}; parameters.BranchId: {parameters.BranchId}." - - if - (not <| String.IsNullOrEmpty(parameters.BranchId) - && parseResult.FindResultFor(Options.branchId).IsImplicit = false) - || (not <| String.IsNullOrEmpty(parameters.BranchName) - && parseResult.FindResultFor(Options.branchName).IsImplicit = false) - then - // First, let's make sure we're not switching to the current branch, which is a no-op. - //if not <| ((parseResult.FindResultFor(Options.branchId).IsImplicit = false && parameters.BranchId = Current().BranchId.ToString()) - // || (parseResult.FindResultFor(Options.branchName).IsImplicit = false && parameters.BranchName = Current().BranchName.ToString())) then - //if not <| parseResult.FindResultFor(Options.branchId).IsImplicit - // Get branch information based on Id and name. - //logToAnsiConsole Colors.Verbose "Get branch information based on Id and name." - let getParameters = - Parameters.Branch.GetBranchParameters( - OwnerId = parameters.OwnerId, - OwnerName = parameters.OwnerName, - OrganizationId = parameters.OrganizationId, - OrganizationName = parameters.OrganizationName, - RepositoryId = parameters.RepositoryId, - RepositoryName = parameters.RepositoryName, - CorrelationId = parameters.CorrelationId - ) - - if not <| parseResult.FindResultFor(Options.branchId).IsImplicit then - getParameters.BranchId <- parameters.BranchId - - if not <| parseResult.FindResultFor(Options.branchName).IsImplicit then - getParameters.BranchName <- parameters.BranchName - - let! branchGetResult = Branch.Get(getParameters) - - match branchGetResult with - | Ok returnValue -> - //logToAnsiConsole Colors.Verbose "Found branch information." - let branchDto = returnValue.ReturnValue - - let getVersionParameters = - GetBranchVersionParameters( - BranchId = $"{branchDto.BranchId}", - OwnerId = parameters.OwnerId, - OwnerName = parameters.OwnerName, - OrganizationId = parameters.OrganizationId, - OrganizationName = parameters.OrganizationName, - RepositoryId = $"{branchDto.RepositoryId}", - ReferenceId = parameters.ReferenceId, - Sha256Hash = parameters.Sha256Hash, - CorrelationId = parameters.CorrelationId - ) - //logToAnsiConsole Colors.Verbose "Calling GetVersion." - let! result = Branch.GetVersion getVersionParameters - t6.Value <- 100.0 - - t7.StartTask() - - match result with - | Ok returnValue -> - //logToAnsiConsole Colors.Verbose "Succeeded calling Branch.GetVersion." - let directoryIds = returnValue.ReturnValue - - let missingDirectoryIds = - directoryIds - .Where(fun directoryId -> not <| previousDirectoryIds.Contains(directoryId)) - .ToList() - - // Get missing directory versions from server. - let getByDirectoryIdParameters = - Parameters.Directory.GetByDirectoryIdsParameters( - RepositoryId = parameters.RepositoryId, - DirectoryId = $"{rootDirectoryId}", - DirectoryIds = missingDirectoryIds, - CorrelationId = parameters.CorrelationId - ) - - match! Directory.GetByDirectoryIds getByDirectoryIdParameters with - | Ok returnValue -> - //logToAnsiConsole Colors.Verbose $"Succeeded calling Directory.GetByDirectoryIds." - // Create a new version of GraceStatus that includes the new DirectoryVersions. - let newDirectoryVersions = returnValue.ReturnValue - - let newerGraceStatus = - updateGraceStatusWithNewDirectoryVersionsFromServer newGraceStatus newDirectoryVersions - //logToAnsiConsole Colors.Verbose $"Succeeded calling updateGraceStatusWithNewDirectoryVersions." - - // Identify files that we don't already have in object cache and download them. - for directoryVersion in newerGraceStatus.Index.Values do - match! - downloadFilesFromObjectStorage directoryVersion.Files (getCorrelationId parseResult) - with - | Ok _ -> - //logToAnsiConsole Colors.Verbose $"Succeeded downloading files from object storage for {directoryVersion.RelativePath}." - () - | Error error -> - logToAnsiConsole - Colors.Verbose - $"Failed downloading files from object storage for {directoryVersion.RelativePath}." - - logToAnsiConsole Colors.Error $"{error}" - - // Update working directory based on new GraceStatus.Index - do! - updateWorkingDirectory - newGraceStatus - newerGraceStatus - newDirectoryVersions - (getCorrelationId parseResult) - //logToAnsiConsole Colors.Verbose $"Succeeded calling updateWorkingDirectory." - - // Save the new Grace Status. - do! writeGraceStatusFile newerGraceStatus - - // Update graceconfig.json. - let configuration = Current() - configuration.BranchId <- branchDto.BranchId - configuration.BranchName <- branchDto.BranchName - updateConfiguration configuration - | Error error -> - logToAnsiConsole Colors.Verbose $"Failed calling Directory.GetByDirectoryIds." - - logToAnsiConsole Colors.Error $"{error}" - | Error error -> - logToAnsiConsole Colors.Verbose "Failed calling GetVersion." - logToAnsiConsole Colors.Error $"{error}" - - t7.Value <- 100 - | Error error -> logToAnsiConsole Colors.Error $"{error}" - - return 0 - //else - // logToAnsiConsole Colors.Highlighted $"The specified branch is already the current branch." - // t5.StopTask() - // t6.StopTask() - // return -1 - else - logToAnsiConsole Colors.Error (BranchError.getErrorMessage BranchError.EitherBranchIdOrBranchNameRequired) - - t6.StopTask() - t7.StopTask() - return -1 - }) - else - let! previousGraceStatus = readGraceStatusFile () - let mutable newGraceStatus = previousGraceStatus - - if currentBranch.ReturnValue.SaveEnabled then - match! getGraceWatchStatus () with - | Some graceWatchStatus -> - rootDirectoryId <- graceWatchStatus.RootDirectoryId - rootDirectorySha256Hash <- graceWatchStatus.RootDirectorySha256Hash - previousDirectoryIds <- graceWatchStatus.DirectoryIds - | None -> - let! differences = scanForDifferences previousGraceStatus - - let! (updatedGraceStatus, newDirectoryVersions) = getNewGraceStatusAndDirectoryVersions previousGraceStatus differences - - newGraceStatus <- updatedGraceStatus - rootDirectoryId <- newGraceStatus.RootDirectoryId - rootDirectorySha256Hash <- newGraceStatus.RootDirectorySha256Hash - previousDirectoryIds <- newGraceStatus.Index.Keys.ToHashSet() - - let updatedRelativePaths = - differences - .Select(fun difference -> - match difference.DifferenceType with - | Add -> - match difference.FileSystemEntryType with - | FileSystemEntryType.File -> Some difference.RelativePath - | FileSystemEntryType.Directory -> None - | Change -> - match difference.FileSystemEntryType with - | FileSystemEntryType.File -> Some difference.RelativePath - | FileSystemEntryType.Directory -> None - | Delete -> None) - .Where(fun relativePathOption -> relativePathOption.IsSome) - .Select(fun relativePath -> relativePath.Value) - - let newFileVersions = - updatedRelativePaths.Select(fun relativePath -> - newDirectoryVersions - .First(fun dv -> dv.Files.Exists(fun file -> file.RelativePath = relativePath)) - .Files.First(fun file -> file.RelativePath = relativePath)) - - let getUploadMetadataForFilesParameters = - Storage.GetUploadMetadataForFilesParameters( - OwnerId = parameters.OwnerId, - OwnerName = parameters.OwnerName, - OrganizationId = parameters.OrganizationId, - OrganizationName = parameters.OrganizationName, - RepositoryId = parameters.RepositoryId, - RepositoryName = parameters.RepositoryName, - CorrelationId = getCorrelationId parseResult, - FileVersions = - (newFileVersions - |> Seq.map (fun localFileVersion -> localFileVersion.ToFileVersion) - |> Seq.toArray) - ) - - let! uploadResult = uploadFilesToObjectStorage getUploadMetadataForFilesParameters - - let saveParameters = SaveDirectoryVersionsParameters() - - saveParameters.DirectoryVersions <- newDirectoryVersions.Select(fun dv -> dv.ToDirectoryVersion).ToList() - - let! uploadDirectoryVersions = Directory.SaveDirectoryVersions saveParameters - - let! saveResult = - createSaveReference - newGraceStatus.Index[rootDirectoryId] - $"Save created during branch switch." - (getCorrelationId parseResult) - - match saveResult with - | Ok returnValue -> () - | Error error -> logToAnsiConsole Colors.Error $"{error}" - else - // Save is not enabled on this branch, so we can skip all of the above. - // Get current values from the current branch based on finding no differences, because we're not creating a save or uploading anything. - let! (updatedGraceStatus, newDirectoryVersions) = - getNewGraceStatusAndDirectoryVersions previousGraceStatus (List()) - - newGraceStatus <- updatedGraceStatus - rootDirectoryId <- newGraceStatus.RootDirectoryId - rootDirectorySha256Hash <- newGraceStatus.RootDirectorySha256Hash - previousDirectoryIds <- newGraceStatus.Index.Keys.ToHashSet() - - // If we have a branch name or ID to switch to, cool, let's do it. - //logToAnsiConsole Colors.Verbose $"parameters.BranchName: {parameters.BranchName}; parameters.BranchId: {parameters.BranchId}." - - if - (not <| String.IsNullOrEmpty(parameters.BranchId) - && parseResult.FindResultFor(Options.branchId).IsImplicit = false) - || (not <| String.IsNullOrEmpty(parameters.BranchName) - && parseResult.FindResultFor(Options.branchName).IsImplicit = false) - then - // First, let's make sure we're not switching to the current branch, which is a no-op. - //if not <| ((parseResult.FindResultFor(Options.branchId).IsImplicit = false && parameters.BranchId = Current().BranchId.ToString()) - // || (parseResult.FindResultFor(Options.branchName).IsImplicit = false && parameters.BranchName = Current().BranchName.ToString())) then - //if not <| parseResult.FindResultFor(Options.branchId).IsImplicit - // Get branch information based on Id and name. - //logToAnsiConsole Colors.Verbose "Get branch information based on Id and name." - let getParameters = - GetBranchParameters( - OwnerId = parameters.OwnerId, - OwnerName = parameters.OwnerName, - OrganizationId = parameters.OrganizationId, - OrganizationName = parameters.OrganizationName, - RepositoryId = parameters.RepositoryId, - RepositoryName = parameters.RepositoryName, - CorrelationId = parameters.CorrelationId - ) - - if not <| parseResult.FindResultFor(Options.branchId).IsImplicit then - getParameters.BranchId <- parameters.BranchId - - if not <| parseResult.FindResultFor(Options.branchName).IsImplicit then - getParameters.BranchName <- parameters.BranchName - - let! branchGetResult = Branch.Get(getParameters) - - match branchGetResult with - | Ok returnValue -> - //logToAnsiConsole Colors.Verbose "Found branch information." - let branchDto = returnValue.ReturnValue - - let getVersionParameters = - GetBranchVersionParameters( - OwnerId = parameters.OwnerId, - OwnerName = parameters.OwnerName, - OrganizationId = parameters.OrganizationId, - OrganizationName = parameters.OrganizationName, - RepositoryId = $"{branchDto.RepositoryId}", - BranchId = $"{branchDto.BranchId}", - ReferenceId = parameters.ReferenceId, - Sha256Hash = parameters.Sha256Hash, - CorrelationId = parameters.CorrelationId - ) - //logToAnsiConsole Colors.Verbose "Calling GetVersion." - let! result = Branch.GetVersion getVersionParameters - - match result with - | Ok returnValue -> - //logToAnsiConsole Colors.Verbose "Succeeded calling Branch.GetVersion." - let directoryIds = returnValue.ReturnValue - - let missingDirectoryIds = - directoryIds - .Where(fun directoryId -> not <| previousDirectoryIds.Contains(directoryId)) - .ToList() - - // Get missing directory versions from server. - let getByDirectoryIdParameters = - Parameters.Directory.GetByDirectoryIdsParameters( - RepositoryId = parameters.RepositoryId, - DirectoryId = $"{rootDirectoryId}", - DirectoryIds = missingDirectoryIds, - CorrelationId = parameters.CorrelationId - ) - - match! Directory.GetByDirectoryIds getByDirectoryIdParameters with - | Ok returnValue -> - //logToAnsiConsole Colors.Verbose $"Succeeded calling Directory.GetByDirectoryIds." - // Create a new version of GraceStatus that includes the new DirectoryVersions. - let newDirectoryVersions = returnValue.ReturnValue - - let newerGraceStatus = updateGraceStatusWithNewDirectoryVersionsFromServer newGraceStatus newDirectoryVersions - //logToAnsiConsole Colors.Verbose $"Succeeded calling updateGraceStatusWithNewDirectoryVersions." - - // Identify files that we don't already have in object cache and download them. - for directoryVersion in newerGraceStatus.Index.Values do - match! downloadFilesFromObjectStorage directoryVersion.Files (getCorrelationId parseResult) with - | Ok _ -> - //logToAnsiConsole Colors.Verbose $"Succeeded downloading files from object storage for {directoryVersion.RelativePath}." - () - | Error error -> - logToAnsiConsole - Colors.Verbose - $"Failed downloading files from object storage for {directoryVersion.RelativePath}." - - logToAnsiConsole Colors.Error $"{error}" - - // Update working directory based on new GraceStatus.Index - do! updateWorkingDirectory newGraceStatus newerGraceStatus newDirectoryVersions (getCorrelationId parseResult) - //logToAnsiConsole Colors.Verbose $"Succeeded calling updateWorkingDirectory." - - // Save the new Grace Status. - do! writeGraceStatusFile newerGraceStatus - - // Update graceconfig.json. - let configuration = Current() - configuration.BranchId <- branchDto.BranchId - configuration.BranchName <- branchDto.BranchName - updateConfiguration configuration - | Error error -> - logToAnsiConsole Colors.Verbose $"Failed calling Directory.GetByDirectoryIds." - - logToAnsiConsole Colors.Error (Markup.Escape($"{error}")) - | Error error -> - logToAnsiConsole Colors.Verbose "Failed calling GetVersion." - logToAnsiConsole Colors.Error (Markup.Escape($"{error}")) - | Error error -> logToAnsiConsole Colors.Error (Markup.Escape($"{error}")) - - return 0 - else - logToAnsiConsole Colors.Highlighted $"Not implemented yet." - return -1 - | Error error -> return -1 - | Error error -> return -1 - with ex -> - return -1 - } - let private Switch = CommandHandler.Create(fun (parseResult: ParseResult) (switchParameters: SwitchParameters) -> task { try // Write the UpdatesInProgress file to let grace watch know to ignore these changes. do! File.WriteAllTextAsync(updateInProgressFileName, "`grace switch` is in progress.") - let! result = switchHandler2 parseResult switchParameters + let! result = switchHandler parseResult switchParameters return result finally File.Delete(updateInProgressFileName) @@ -3348,6 +2818,7 @@ module Branch = task { try if parseResult |> verbose then printParseResult parseResult + // Show repo and branch names. let getParameters = GetBranchParameters( @@ -3357,7 +2828,8 @@ module Branch = OrganizationName = parameters.OrganizationName, RepositoryId = parameters.RepositoryId, RepositoryName = parameters.RepositoryName, - BranchId = $"{Current().BranchId}", + BranchId = parameters.BranchId, + BranchName = parameters.BranchName, CorrelationId = parameters.CorrelationId ) @@ -3376,38 +2848,18 @@ module Branch = let latestParentBranchPromotion = parentBranchDto.LatestPromotion let basedOn = branchDto.BasedOn - let getReferencesParameters = - Parameters.Branch.GetReferencesParameters( - OwnerId = parameters.OwnerId, - OwnerName = parameters.OwnerName, - OrganizationId = parameters.OrganizationId, - OrganizationName = parameters.OrganizationName, - RepositoryId = parameters.RepositoryId, - RepositoryName = parameters.RepositoryName, - BranchId = $"{branchDto.BranchId}", - MaxCount = 1, - CorrelationId = parameters.CorrelationId - ) - - match! Branch.GetReferences(getReferencesParameters) with - | Ok returnValue -> - let latestReference = - if returnValue.ReturnValue.Count() > 0 then - returnValue.ReturnValue.First() - else - ReferenceDto.Default - - let getReferenceRowValue referenceDto = - if referenceDto.ReferenceId = ReferenceId.Empty then - $" None" - else if parseResult |> verbose then - $" {getShortSha256Hash referenceDto.Sha256Hash} - {ago referenceDto.CreatedAt} - {instantToLocalTime referenceDto.CreatedAt} [{Colors.Deemphasized}]- {referenceDto.ReferenceId} - {referenceDto.DirectoryId}[/]" - else - $" {getShortSha256Hash referenceDto.Sha256Hash} - {ago referenceDto.CreatedAt} - {instantToLocalTime referenceDto.CreatedAt} [{Colors.Deemphasized}]- {referenceDto.ReferenceId}[/]" + let getReferenceRowValue referenceDto = + if referenceDto.ReferenceId = ReferenceId.Empty then + $" None" + else if parseResult |> verbose then + $" {getShortSha256Hash referenceDto.Sha256Hash} - {ago referenceDto.CreatedAt} - {instantToLocalTime referenceDto.CreatedAt} [{Colors.Deemphasized}]- {referenceDto.ReferenceId} - {referenceDto.DirectoryId}[/]" + else + $" {getShortSha256Hash referenceDto.Sha256Hash} - {ago referenceDto.CreatedAt} - {instantToLocalTime referenceDto.CreatedAt} [{Colors.Deemphasized}]- {referenceDto.ReferenceId}[/]" - let permissions (branchDto: Dto.Branch.BranchDto) = - let sb = StringBuilder() + let permissions (branchDto: Dto.Branch.BranchDto) = + let sb = stringBuilderPool.Get() + try if branchDto.PromotionEnabled then sb.Append("Promotion/") |> ignore if branchDto.CommitEnabled then sb.Append("Commit/") |> ignore @@ -3416,73 +2868,75 @@ module Branch = if branchDto.SaveEnabled then sb.Append("Save/") |> ignore - if branchDto.TagEnabled then sb.Append("Tag") |> ignore + if branchDto.TagEnabled then sb.Append("Tag/") |> ignore + + if branchDto.ExternalEnabled then sb.Append("External/") |> ignore - if sb[sb.Length - 1] = '/' then sb.Remove(sb.Length - 1, 1) |> ignore + if sb.Length > 0 && sb[sb.Length - 1] = '/' then + sb.Remove(sb.Length - 1, 1) |> ignore sb.ToString() + finally + if not <| isNull sb then stringBuilderPool.Return(sb) + + let table = Table(Border = TableBorder.DoubleEdge) + table.ShowHeaders <- false + + table + .AddColumns(String.replicate (Current().OwnerName.Length) "_", String.replicate (Current().OwnerName.Length) "_") // Using Current().OwnerName.Length is aesthetically pleasing, there's no deeper reason for it. + .AddRow( + $"[{Colors.Important}]Owner[/]", + $"[{Colors.Important}]{Current().OwnerName}[/] [{Colors.Deemphasized}]- {Current().OwnerId}[/]" + ) + .AddRow( + $"[{Colors.Important}]Organization[/]", + $"[{Colors.Important}]{Current().OrganizationName}[/] [{Colors.Deemphasized}]- {Current().OrganizationId}[/]" + ) + .AddRow( + $"[{Colors.Important}]Repository[/]", + $"[{Colors.Important}]{Current().RepositoryName}[/] [{Colors.Deemphasized}]- {Current().RepositoryId}[/]" + ) + .AddRow(String.Empty, String.Empty) + .AddRow( + $"[{Colors.Important}]Branch[/]", + $"[{Colors.Important}]{branchDto.BranchName}[/] - Allows {permissions branchDto} [{Colors.Deemphasized}]- {branchDto.BranchId}[/]" + ) + //.AddRow(String.Empty, $"Permissions: {permissions}.") + .AddRow($" - latest save ", getReferenceRowValue latestSave) + .AddRow($" - latest checkpoint ", getReferenceRowValue latestCheckpoint) + .AddRow($" - latest commit", getReferenceRowValue latestCommit) + .AddRow($" - based on", getReferenceRowValue basedOn) + |> ignore + + if + branchDto.BasedOn.ReferenceId = parentBranchDto.LatestPromotion.ReferenceId + || branchDto.ParentBranchId = Constants.DefaultParentBranchId + then + table.AddRow($"", $"[{Colors.Added}] Based on latest promotion.[/]") |> ignore + else + table.AddRow($"", $"[{Colors.Important}] Not based on latest promotion.[/]") + |> ignore - let table = Table(Border = TableBorder.DoubleEdge) - table.ShowHeaders <- false + table.AddRow(String.Empty, String.Empty) |> ignore + if branchDto.ParentBranchId <> Constants.DefaultParentBranchId then table - .AddColumns(String.replicate (Current().OwnerName.Length) "_", String.replicate (Current().OwnerName.Length) "_") // Using Current().OwnerName.Length is aesthetically pleasing, there's no deeper reason for it. .AddRow( - $"[{Colors.Important}]Owner[/]", - $"[{Colors.Important}]{Current().OwnerName}[/] [{Colors.Deemphasized}]- {Current().OwnerId}[/]" + $"[{Colors.Important}]Parent branch[/]", + $"[{Colors.Important}]{parentBranchDto.BranchName}[/] - Allows {permissions parentBranchDto} [{Colors.Deemphasized}]- {parentBranchDto.BranchId}[/]" ) - .AddRow( - $"[{Colors.Important}]Organization[/]", - $"[{Colors.Important}]{Current().OrganizationName}[/] [{Colors.Deemphasized}]- {Current().OrganizationId}[/]" - ) - .AddRow( - $"[{Colors.Important}]Repository[/]", - $"[{Colors.Important}]{Current().RepositoryName}[/] [{Colors.Deemphasized}]- {Current().RepositoryId}[/]" - ) - .AddRow(String.Empty, String.Empty) - .AddRow( - $"[{Colors.Important}]Branch[/]", - $"[{Colors.Important}]{branchDto.BranchName}[/] - Allows {permissions branchDto} [{Colors.Deemphasized}]- {branchDto.BranchId}[/]" - ) - //.AddRow(String.Empty, $"Permissions: {permissions}.") - .AddRow($" - latest save ", getReferenceRowValue latestSave) - .AddRow($" - latest checkpoint ", getReferenceRowValue latestCheckpoint) - .AddRow($" - latest commit", getReferenceRowValue latestCommit) - .AddRow($" - based on", getReferenceRowValue basedOn) + .AddRow($" - latest promotion", getReferenceRowValue latestParentBranchPromotion) + .AddRow($"", $" {latestParentBranchPromotion.ReferenceText}") + |> ignore + else + table.AddRow($"[{Colors.Important}]Parent branch[/]", $"[{Colors.Important}] None[/]") |> ignore - if - branchDto.BasedOn.ReferenceId = parentBranchDto.LatestPromotion.ReferenceId - || branchDto.ParentBranchId = Constants.DefaultParentBranchId - then - table.AddRow($"", $"[{Colors.Added}] Based on latest promotion.[/]") |> ignore - else - table.AddRow($"", $"[{Colors.Important}] Not based on latest promotion.[/]") - |> ignore - - table.AddRow(String.Empty, String.Empty) |> ignore - - if branchDto.ParentBranchId <> Constants.DefaultParentBranchId then - table - .AddRow( - $"[{Colors.Important}]Parent branch[/]", - $"[{Colors.Important}]{parentBranchDto.BranchName}[/] - Allows {permissions parentBranchDto} [{Colors.Deemphasized}]- {parentBranchDto.BranchId}[/]" - ) - .AddRow($" - latest promotion", getReferenceRowValue latestParentBranchPromotion) - .AddRow($"", $" {latestParentBranchPromotion.ReferenceText}") - |> ignore - else - table.AddRow($"[{Colors.Important}]Parent branch[/]", $"[{Colors.Important}] None[/]") - |> ignore - - // Need to add this portion of the header after everything else is rendered so we know the width. - //let headerWidth = if table.Columns[0].Width.HasValue then table.Columns[0].Width.Value else 27 // 27 = longest current text - //table.Columns[0].Header <- Markup(String.replicate headerWidth "_") - AnsiConsole.Write(table) - return 0 - | Error error -> - logToAnsiConsole Colors.Error (Markup.Escape($"{error}")) - return -1 + // Need to add this portion of the header after everything else is rendered so we know the width. + //let headerWidth = if table.Columns[0].Width.HasValue then table.Columns[0].Width.Value else 27 // 27 = longest current text + //table.Columns[0].Header <- Markup(String.replicate headerWidth "_") + AnsiConsole.Write(table) + return 0 | Error error -> logToAnsiConsole Colors.Error (Markup.Escape($"{error}")) return -1 diff --git a/src/Grace.CLI/Command/Repository.CLI.fs b/src/Grace.CLI/Command/Repository.CLI.fs index 894b5c7..331438c 100644 --- a/src/Grace.CLI/Command/Repository.CLI.fs +++ b/src/Grace.CLI/Command/Repository.CLI.fs @@ -901,12 +901,14 @@ module Repository = if parseResult |> hasOutput then let table = Table(Border = TableBorder.DoubleEdge) - table - .AddColumn(TableColumn(Markup($"[{Colors.Important}]Branch[/]"))) - .AddColumn(TableColumn(Markup($"[{Colors.Important}]Updated at[/]"))) - .AddColumn(TableColumn(Markup($"[{Colors.Important}]Branch Id[/]"))) - .AddColumn(TableColumn(Markup($"[{Colors.Important}]Based on latest promotion[/]"))) - .AddColumn(TableColumn(Markup($"[{Colors.Important}]Parent branch[/]"))) + table.AddColumns( + [| TableColumn($"[{Colors.Important}]Branch[/]") + TableColumn($"[{Colors.Important}]Branch Id[/]") + TableColumn($"[{Colors.Important}]Based on latest promotion[/]") + TableColumn($"[{Colors.Important}]Parent branch[/]") + TableColumn($"[{Colors.Important}]When[/]", Alignment = Justify.Right) + TableColumn($"[{Colors.Important}]Updated at[/]") |] + ) |> ignore let allBranches = returnValue.ReturnValue @@ -941,6 +943,7 @@ module Repository = {| branchId = branch.BranchId branchName = branch.BranchName updatedAt = branch.UpdatedAt + ago = ago branch.CreatedAt parentBranchName = parent.branchName basedOnLatestPromotion = (branch.BasedOn.ReferenceId = parent.latestPromotion.ReferenceId) |}) ) @@ -954,13 +957,14 @@ module Repository = table.AddRow( br.branchName, - updatedAt, $"[{Colors.Deemphasized}]{br.branchId}[/]", (if br.basedOnLatestPromotion then $"[{Colors.Added}]Yes[/]" else $"[{Colors.Important}]No[/]"), - br.parentBranchName + br.parentBranchName, + br.ago, + $"[{Colors.Deemphasized}]{updatedAt}[/]" ) |> ignore diff --git a/src/Grace.CLI/Command/Services.CLI.fs b/src/Grace.CLI/Command/Services.CLI.fs index b88605f..ec44b70 100644 --- a/src/Grace.CLI/Command/Services.CLI.fs +++ b/src/Grace.CLI/Command/Services.CLI.fs @@ -1393,10 +1393,12 @@ module Services = use tempFileStream = File.Open(tempFilePath, fileStreamOptionsRead) + let! isBinary = isBinaryFile tempFileStream + tempFileStream.Position <- 0 let! sha256Hash = computeSha256ForFile tempFileStream relativeFilePath //logToConsole $"filePath: {filePath}; tempFilePath: {tempFilePath}; SHA256: {sha256Hash}" - // I'm going to rename this file below, using the SHA-256 hash, so I'll be polite and close the file stream here. + // I'm going to rename the temp file below, using the SHA-256 hash, so I'll close the file and dispose the stream first. do! tempFileStream.DisposeAsync() // Get the new name for this version of the file, including the SHA-256 hash. @@ -1418,9 +1420,6 @@ module Services = //logToConsole $"After moving temp file to object storage..." let objectFilePathInfo = FileInfo(objectFilePath) //logToConsole $"After creating FileInfo; Exists: {objectFilePathInfo.Exists}; FullName = {objectFilePathInfo.FullName}..." - use objectFileStream = objectFilePathInfo.Open(fileStreamOptionsRead) - //logToConsole $"After creating stream; .Length = {objectFileStream.Length}..." - let! isBinary = isBinaryFile objectFileStream //logToConsole $"Finished copyToObjectDirectory for {filePath}; isBinary: {isBinary}; moved temp file to object directory." let relativePath = Path.GetRelativePath(Current().RootDirectory, filePath) diff --git a/src/Grace.SDK/Common.SDK.fs b/src/Grace.SDK/Common.SDK.fs index 20d1ab8..4850119 100644 --- a/src/Grace.SDK/Common.SDK.fs +++ b/src/Grace.SDK/Common.SDK.fs @@ -18,6 +18,7 @@ open System.Net.Security open System.Text open System.Text.Json open System.Threading.Tasks +open Microsoft.Extensions.Caching.Memory // Supresses the warning for using AllowNoEncryption in Debug builds. #nowarn "0044" @@ -76,6 +77,13 @@ module Common = #endif httpClient + let checkMemoryCache () = + if isNull memoryCache then + let memoryCacheOptions = + MemoryCacheOptions(TrackStatistics = false, TrackLinkedCacheEntries = false, ExpirationScanFrequency = TimeSpan.FromSeconds(30.0)) + + memoryCache <- new MemoryCache(memoryCacheOptions) + /// /// Sends GET commands to Grace Server. /// @@ -87,6 +95,7 @@ module Common = let getServer<'T, 'U when 'T :> CommonParameters> (parameters: 'T, route: string) = task { try + checkMemoryCache () use httpClient = getHttpClient parameters.CorrelationId let startTime = getCurrentInstant () @@ -128,6 +137,7 @@ module Common = let postServer<'T, 'U when 'T :> CommonParameters> (parameters: 'T, route: string) = task { try + checkMemoryCache () use httpClient = getHttpClient parameters.CorrelationId let serverUriWithRoute = Uri($"{Current().ServerUri}/{route}") let startTime = getCurrentInstant () diff --git a/src/Grace.Shared/Dto/Dto.Shared.fs b/src/Grace.Shared/Dto/Dto.Shared.fs index 62d337d..d7094fb 100644 --- a/src/Grace.Shared/Dto/Dto.Shared.fs +++ b/src/Grace.Shared/Dto/Dto.Shared.fs @@ -222,7 +222,7 @@ module Dto = static member Default = { Class = nameof (BranchDto) BranchId = BranchId.Empty - BranchName = BranchName String.Empty + BranchName = BranchName "root" ParentBranchId = Constants.DefaultParentBranchId BasedOn = ReferenceDto.Default RepositoryId = RepositoryId.Empty diff --git a/src/Grace.Shared/Services.Shared.fs b/src/Grace.Shared/Services.Shared.fs index 2ad880b..09e9fc8 100644 --- a/src/Grace.Shared/Services.Shared.fs +++ b/src/Grace.Shared/Services.Shared.fs @@ -37,9 +37,7 @@ module Services = member this.Return(hashInstance: IncrementalHash) = // Reset the hash instance so it can be reused. - // We're calling GetHashAndReset() to reset - there's no Reset() - but because we're also calling GetHashAndReset() - // when we compute the hash values, calling it here on empty IncrementalHash instances will be as fast as possible. - // I'm betting that GetHashAndReset() on an empty IncrementalHash instance is faster than creating new instances. + // We're calling GetHashAndReset() to reset - there's no Reset() function. let throwaway = stackalloc SHA256.HashSizeInBytes hashInstance.GetHashAndReset(throwaway) |> ignore true // Indicates that the object is okay to be returned to the pool @@ -47,12 +45,50 @@ module Services = /// An ObjectPool for IncrementalHash instances. let incrementalHashPool = DefaultObjectPoolProvider().Create(IncrementalHashPolicy()) + /// The 0x00 character. + let nulChar = char (0) + + /// Checks if a file is a binary file by scanning the first 8K for a 0x00 character; if it finds one, we assume the file is binary. + /// + /// This is the same algorithm used by Git. + let isBinaryFile (stream: Stream) = + task { + // If the file is smaller than 8K, we'll check the whole file. + let defaultBytesToCheck = 8 * 1024 + + let bytesToCheck = + if stream.Length > defaultBytesToCheck then + defaultBytesToCheck + else + int (stream.Length) + + // Get a buffer to hold the part of the file we're going to check. + let startingBytes = ArrayPool.Shared.Rent(bytesToCheck) + + try + // Read the beginning of the file into the buffer. + let! bytesRead = stream.ReadAsync(startingBytes, 0, bytesToCheck) + + // Search for a 0x00 character. + match + startingBytes + |> Array.take bytesRead + |> Array.tryFind (fun b -> char (b) = nulChar) + with + | Some nul -> return true + | None -> return false + finally + // Return the rented buffer to the pool, even if an exception is thrown. + if not <| isNull startingBytes then ArrayPool.Shared.Return(startingBytes) + } + /// Computes the SHA-256 value for a given file, presented as a stream. /// /// Sha256Hash values for files are computed by hashing the file's contents, and then appending the relative path of the file, and the file length. let computeSha256ForFile (stream: Stream) (relativeFilePath: RelativePath) = task { - let bufferLength = 64 * 1024 // Did some informal perf testing on large files, this size was best, larger didn't help, and 64K is still on the small object heap. + // Did some informal perf testing on large files, this size was best, larger didn't help, and 64K is still on the small object heap. + let bufferLength = 64 * 1024 // Using object pooling for both of these. let buffer = ArrayPool.Shared.Rent(bufferLength) diff --git a/src/Grace.Shared/Utilities.Shared.fs b/src/Grace.Shared/Utilities.Shared.fs index 27043ab..a2d9a54 100644 --- a/src/Grace.Shared/Utilities.Shared.fs +++ b/src/Grace.Shared/Utilities.Shared.fs @@ -227,6 +227,7 @@ module Utilities = let normalizedTimeSpan = TimeSpan.FromMinutes(1.0) let normalizeFilePath (filePath: string) = + logToConsole $"In normalizeFilePath: filePath: {filePath}; isNull memoryCache: {isNull memoryCache}." let mutable result = String.Empty if not <| memoryCache.TryGetValue(filePath, &result) then @@ -253,42 +254,6 @@ module Utilities = let fileStreamOptionsWrite = FileStreamOptions(BufferSize = 8 * 1024, Mode = FileMode.Create, Access = FileAccess.Write, Share = FileShare.None, Options = FileOptions.Asynchronous) - let nulChar = char (0) - - /// Checks if a file is a binary file by scanning the first 8K for a 0x00 character; if it finds one, we assume the file is binary. - /// - /// This is the same algorithm used by Git. - let isBinaryFile (stream: Stream) = - task { - // If the file is smaller than 8K, we'll check the whole file. - let defaultBytesToCheck = 8 * 1024 - - let bytesToCheck = - if stream.Length > defaultBytesToCheck then - defaultBytesToCheck - else - int (stream.Length) - - // Get a buffer to hold the part of the file we're going to check. - let startingBytes = ArrayPool.Shared.Rent(bytesToCheck) - - try - // Read the beginning of the file into the buffer. - let! bytesRead = stream.ReadAsync(startingBytes, 0, bytesToCheck) - - // Search for a 0x00 character. - match - startingBytes - |> Array.take bytesRead - |> Array.tryFind (fun b -> char (b) = nulChar) - with - | Some nul -> return true - | None -> return false - finally - // Return the rented buffer to the pool, even if an exception is thrown. - if not <| isNull startingBytes then ArrayPool.Shared.Return(startingBytes) - } - /// Returns the directory of a file, relative to the root of the repository's working directory. let getRelativeDirectory (filePath: string) rootDirectory = let standardizedFilePath = normalizeFilePath filePath