Skip to content

Commit

Permalink
Refactor logging, caching, and CLI enhancements.
Browse files Browse the repository at this point in the history
Refactored table column definitions in Repository.CLI.fs to use AddColumns with an array of TableColumn objects. Added a new "When" column and moved "Updated at" to the end. Updated table row rendering to include the new ago property.

In Services.CLI.fs, added a check for binary files before computing SHA-256 hash and reset the file stream. Updated comments for clarity and removed redundant binary file check code.

Added Microsoft.Extensions.Caching.Memory namespace in Common.SDK.fs. Introduced checkMemoryCache function to initialize memory cache if null, called in getServer and postServer functions.

Changed default BranchName in BranchDto to "root" in Dto.Shared.fs.
  • Loading branch information
ScottArbeit committed Jan 2, 2025
1 parent f9f32cc commit e59c862
Show file tree
Hide file tree
Showing 9 changed files with 143 additions and 675 deletions.
2 changes: 1 addition & 1 deletion src/Grace.Actors/Branch.Actor.fs
Original file line number Diff line number Diff line change
Expand Up @@ -266,7 +266,7 @@ module Branch =
&& not <| (context.MethodName = "ReceiveReminderAsync")
then
log.LogInformation(
"{CurrentInstant}: Node: {HostName}; Duration: {duration_ms}ms; CorrelationId: {correlationId}; Finished {ActorName}.{MethodName}; RepositoryId: {RepositoryId}; BranchId: {Id}; BranchName: {BranchName}; ThreadCounts: {ThreadCounts}.",
"{CurrentInstant}: Node: {HostName}; Duration: {duration_ms}ms; CorrelationId: {correlationId}; Finished {ActorName}.{MethodName}; RepositoryId: {RepositoryId}; BranchId: {Id}; BranchName: {BranchName}; {ThreadCounts}.",
getCurrentInstantExtended (),
getMachineName,
duration_ms,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ module MemoryCache =

/// Create a new entry in MemoryCache to store the current thread count information.
member this.CreateThreadCountEntry(threadInfo: string) =
use newCacheEntry = this.CreateEntry("ThreadCounts", Value = threadInfo, AbsoluteExpiration = DateTimeOffset.UtcNow.Add(TimeSpan.FromSeconds(1.0)))
use newCacheEntry = this.CreateEntry("ThreadCounts", Value = threadInfo, AbsoluteExpiration = DateTimeOffset.UtcNow.Add(TimeSpan.FromSeconds(6.0)))
()

/// Check if we have an entry in MemoryCache for the current ThreadCount.
Expand Down
694 changes: 74 additions & 620 deletions src/Grace.CLI/Command/Branch.CLI.fs

Large diffs are not rendered by default.

20 changes: 12 additions & 8 deletions src/Grace.CLI/Command/Repository.CLI.fs
Original file line number Diff line number Diff line change
Expand Up @@ -901,12 +901,14 @@ module Repository =
if parseResult |> hasOutput then
let table = Table(Border = TableBorder.DoubleEdge)

table
.AddColumn(TableColumn(Markup($"[{Colors.Important}]Branch[/]")))
.AddColumn(TableColumn(Markup($"[{Colors.Important}]Updated at[/]")))
.AddColumn(TableColumn(Markup($"[{Colors.Important}]Branch Id[/]")))
.AddColumn(TableColumn(Markup($"[{Colors.Important}]Based on latest promotion[/]")))
.AddColumn(TableColumn(Markup($"[{Colors.Important}]Parent branch[/]")))
table.AddColumns(
[| TableColumn($"[{Colors.Important}]Branch[/]")
TableColumn($"[{Colors.Important}]Branch Id[/]")
TableColumn($"[{Colors.Important}]Based on latest promotion[/]")
TableColumn($"[{Colors.Important}]Parent branch[/]")
TableColumn($"[{Colors.Important}]When[/]", Alignment = Justify.Right)
TableColumn($"[{Colors.Important}]Updated at[/]") |]
)
|> ignore

let allBranches = returnValue.ReturnValue
Expand Down Expand Up @@ -941,6 +943,7 @@ module Repository =
{| branchId = branch.BranchId
branchName = branch.BranchName
updatedAt = branch.UpdatedAt
ago = ago branch.CreatedAt
parentBranchName = parent.branchName
basedOnLatestPromotion = (branch.BasedOn.ReferenceId = parent.latestPromotion.ReferenceId) |})
)
Expand All @@ -954,13 +957,14 @@ module Repository =

table.AddRow(
br.branchName,
updatedAt,
$"[{Colors.Deemphasized}]{br.branchId}[/]",
(if br.basedOnLatestPromotion then
$"[{Colors.Added}]Yes[/]"
else
$"[{Colors.Important}]No[/]"),
br.parentBranchName
br.parentBranchName,
br.ago,
$"[{Colors.Deemphasized}]{updatedAt}[/]"
)
|> ignore

Expand Down
7 changes: 3 additions & 4 deletions src/Grace.CLI/Command/Services.CLI.fs
Original file line number Diff line number Diff line change
Expand Up @@ -1393,10 +1393,12 @@ module Services =

use tempFileStream = File.Open(tempFilePath, fileStreamOptionsRead)

let! isBinary = isBinaryFile tempFileStream
tempFileStream.Position <- 0
let! sha256Hash = computeSha256ForFile tempFileStream relativeFilePath
//logToConsole $"filePath: {filePath}; tempFilePath: {tempFilePath}; SHA256: {sha256Hash}"

// I'm going to rename this file below, using the SHA-256 hash, so I'll be polite and close the file stream here.
// I'm going to rename the temp file below, using the SHA-256 hash, so I'll close the file and dispose the stream first.
do! tempFileStream.DisposeAsync()

// Get the new name for this version of the file, including the SHA-256 hash.
Expand All @@ -1418,9 +1420,6 @@ module Services =
//logToConsole $"After moving temp file to object storage..."
let objectFilePathInfo = FileInfo(objectFilePath)
//logToConsole $"After creating FileInfo; Exists: {objectFilePathInfo.Exists}; FullName = {objectFilePathInfo.FullName}..."
use objectFileStream = objectFilePathInfo.Open(fileStreamOptionsRead)
//logToConsole $"After creating stream; .Length = {objectFileStream.Length}..."
let! isBinary = isBinaryFile objectFileStream
//logToConsole $"Finished copyToObjectDirectory for {filePath}; isBinary: {isBinary}; moved temp file to object directory."
let relativePath = Path.GetRelativePath(Current().RootDirectory, filePath)

Expand Down
10 changes: 10 additions & 0 deletions src/Grace.SDK/Common.SDK.fs
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ open System.Net.Security
open System.Text
open System.Text.Json
open System.Threading.Tasks
open Microsoft.Extensions.Caching.Memory

// Supresses the warning for using AllowNoEncryption in Debug builds.
#nowarn "0044"
Expand Down Expand Up @@ -76,6 +77,13 @@ module Common =
#endif
httpClient

let checkMemoryCache () =
if isNull memoryCache then
let memoryCacheOptions =
MemoryCacheOptions(TrackStatistics = false, TrackLinkedCacheEntries = false, ExpirationScanFrequency = TimeSpan.FromSeconds(30.0))

memoryCache <- new MemoryCache(memoryCacheOptions)

/// <summary>
/// Sends GET commands to Grace Server.
/// </summary>
Expand All @@ -87,6 +95,7 @@ module Common =
let getServer<'T, 'U when 'T :> CommonParameters> (parameters: 'T, route: string) =
task {
try
checkMemoryCache ()
use httpClient = getHttpClient parameters.CorrelationId
let startTime = getCurrentInstant ()

Expand Down Expand Up @@ -128,6 +137,7 @@ module Common =
let postServer<'T, 'U when 'T :> CommonParameters> (parameters: 'T, route: string) =
task {
try
checkMemoryCache ()
use httpClient = getHttpClient parameters.CorrelationId
let serverUriWithRoute = Uri($"{Current().ServerUri}/{route}")
let startTime = getCurrentInstant ()
Expand Down
2 changes: 1 addition & 1 deletion src/Grace.Shared/Dto/Dto.Shared.fs
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,7 @@ module Dto =
static member Default =
{ Class = nameof (BranchDto)
BranchId = BranchId.Empty
BranchName = BranchName String.Empty
BranchName = BranchName "root"
ParentBranchId = Constants.DefaultParentBranchId
BasedOn = ReferenceDto.Default
RepositoryId = RepositoryId.Empty
Expand Down
44 changes: 40 additions & 4 deletions src/Grace.Shared/Services.Shared.fs
Original file line number Diff line number Diff line change
Expand Up @@ -37,22 +37,58 @@ module Services =

member this.Return(hashInstance: IncrementalHash) =
// Reset the hash instance so it can be reused.
// We're calling GetHashAndReset() to reset - there's no Reset() - but because we're also calling GetHashAndReset()
// when we compute the hash values, calling it here on empty IncrementalHash instances will be as fast as possible.
// I'm betting that GetHashAndReset() on an empty IncrementalHash instance is faster than creating new instances.
// We're calling GetHashAndReset() to reset - there's no Reset() function.
let throwaway = stackalloc<byte> SHA256.HashSizeInBytes
hashInstance.GetHashAndReset(throwaway) |> ignore
true // Indicates that the object is okay to be returned to the pool

/// An ObjectPool for IncrementalHash instances.
let incrementalHashPool = DefaultObjectPoolProvider().Create(IncrementalHashPolicy())

/// The 0x00 character.
let nulChar = char (0)

/// Checks if a file is a binary file by scanning the first 8K for a 0x00 character; if it finds one, we assume the file is binary.
///
/// This is the same algorithm used by Git.
let isBinaryFile (stream: Stream) =
task {
// If the file is smaller than 8K, we'll check the whole file.
let defaultBytesToCheck = 8 * 1024

let bytesToCheck =
if stream.Length > defaultBytesToCheck then
defaultBytesToCheck
else
int (stream.Length)

// Get a buffer to hold the part of the file we're going to check.
let startingBytes = ArrayPool<byte>.Shared.Rent(bytesToCheck)

try
// Read the beginning of the file into the buffer.
let! bytesRead = stream.ReadAsync(startingBytes, 0, bytesToCheck)

// Search for a 0x00 character.
match
startingBytes
|> Array.take bytesRead
|> Array.tryFind (fun b -> char (b) = nulChar)
with
| Some nul -> return true
| None -> return false
finally
// Return the rented buffer to the pool, even if an exception is thrown.
if not <| isNull startingBytes then ArrayPool<byte>.Shared.Return(startingBytes)
}

/// Computes the SHA-256 value for a given file, presented as a stream.
///
/// Sha256Hash values for files are computed by hashing the file's contents, and then appending the relative path of the file, and the file length.
let computeSha256ForFile (stream: Stream) (relativeFilePath: RelativePath) =
task {
let bufferLength = 64 * 1024 // Did some informal perf testing on large files, this size was best, larger didn't help, and 64K is still on the small object heap.
// Did some informal perf testing on large files, this size was best, larger didn't help, and 64K is still on the small object heap.
let bufferLength = 64 * 1024

// Using object pooling for both of these.
let buffer = ArrayPool<byte>.Shared.Rent(bufferLength)
Expand Down
37 changes: 1 addition & 36 deletions src/Grace.Shared/Utilities.Shared.fs
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,7 @@ module Utilities =
let normalizedTimeSpan = TimeSpan.FromMinutes(1.0)

let normalizeFilePath (filePath: string) =
logToConsole $"In normalizeFilePath: filePath: {filePath}; isNull memoryCache: {isNull memoryCache}."
let mutable result = String.Empty

if not <| memoryCache.TryGetValue(filePath, &result) then
Expand All @@ -253,42 +254,6 @@ module Utilities =
let fileStreamOptionsWrite =
FileStreamOptions(BufferSize = 8 * 1024, Mode = FileMode.Create, Access = FileAccess.Write, Share = FileShare.None, Options = FileOptions.Asynchronous)

let nulChar = char (0)

/// Checks if a file is a binary file by scanning the first 8K for a 0x00 character; if it finds one, we assume the file is binary.
///
/// This is the same algorithm used by Git.
let isBinaryFile (stream: Stream) =
task {
// If the file is smaller than 8K, we'll check the whole file.
let defaultBytesToCheck = 8 * 1024

let bytesToCheck =
if stream.Length > defaultBytesToCheck then
defaultBytesToCheck
else
int (stream.Length)

// Get a buffer to hold the part of the file we're going to check.
let startingBytes = ArrayPool<byte>.Shared.Rent(bytesToCheck)

try
// Read the beginning of the file into the buffer.
let! bytesRead = stream.ReadAsync(startingBytes, 0, bytesToCheck)

// Search for a 0x00 character.
match
startingBytes
|> Array.take bytesRead
|> Array.tryFind (fun b -> char (b) = nulChar)
with
| Some nul -> return true
| None -> return false
finally
// Return the rented buffer to the pool, even if an exception is thrown.
if not <| isNull startingBytes then ArrayPool<byte>.Shared.Return(startingBytes)
}

/// Returns the directory of a file, relative to the root of the repository's working directory.
let getRelativeDirectory (filePath: string) rootDirectory =
let standardizedFilePath = normalizeFilePath filePath
Expand Down

0 comments on commit e59c862

Please sign in to comment.