Add retries when running git fetch (#2405)

_**Issue**_:
Builds sometimes fail with:
```
Error: Unexpected error when running action. Error Message: Unable to find repoVersion in AL-Go-Settings.json, StackTrace: at Get-MaxAllowedObsoleteVersion, D:\a\BCApps\BCApps\build\scripts\GuardingV2ExtensionsHelper.psm1: line 264 <- at Update-AppSourceCopVersion, D:\a\BCApps\BCApps\build\scripts\GuardingV2ExtensionsHelper.psm1: line 208 <- at Enable-BreakingChangesCheck, D:\a\BCApps\BCApps\build\scripts\GuardingV2ExtensionsHelper.psm1: line 67 <- at <ScriptBlock>, D:\a\BCApps\BCApps\build\scripts\PreCompileApp.ps1: line 63 <- at <ScriptBlock>, D:\a\BCApps\BCApps\build\projects\System Application\.AL-Go\PreCompileApp.ps1: line 7 <- at <ScriptBlock>, C:\ProgramData\BcContainerHelper\6.0.29-preview1267\BcContainerHelper\AppHandling\Run-AlPipeline.ps1: line 2070 <- at <ScriptBlock>
```

The underlying error is
```
  fatal: unable to access 'https://github.com/microsoft/BCApps/': Recv failure: Connection was aborted
  fatal: invalid object name 'origin/main'.
```

or sometimes

```
  fatal: unable to access 'https://github.com/microsoft/BCApps/': OpenSSL SSL_read: SSL_ERROR_SYSCALL, errno 0
  fatal: invalid object name 'origin/main'.
```

_**Solution**_:
Add retries when running `git fetch` to catch the instability.

Fixes
[AB#557642](https://dynamicssmb2.visualstudio.com/1fcb79e7-ab07-432a-a3c6-6cf5a88ba4a5/_workitems/edit/557642)

---------

Co-authored-by: --unset <--unset>

build/scripts/EnlistmentHelperFunctions.psm1

@@ -448,5 +448,74 @@ function RunAndCheck {
     }
 }
 
+<#
+.SYNOPSIS
+    Invokes a command with retry logic.
+.DESCRIPTION
+    This function will invoke a command and retry it up to a specified number of times if it fails.
+    The function will sleep for an increasing amount of time between each retry.
+    The function will stop retrying if the maximum wait time is reached.
+.PARAMETER ScriptBlock
+    The script block to invoke.
+.PARAMETER RetryCount
+    The number of times to retry the command.
+.PARAMETER MaxWaitTimeBeforeLastAttempt
+    The maximum time in seconds to wait before
+.PARAMETER FirstDelay
+    The time in seconds to wait before the first retry.
+.PARAMETER MaxWaitBetweenRetries
+    The maximum time in seconds to wait between retries.
+#>
+function Invoke-CommandWithRetry {
+    [CmdletBinding()]
+    param (
+        [parameter(Mandatory = $true)]
+        [System.Management.Automation.ScriptBlock] $ScriptBlock,
+        [parameter(Mandatory = $false)]
+        [int] $RetryCount = 3,
+        [parameter(Mandatory = $false)]
+        [int] $MaxWaitTimeBeforeLastAttempt = 2 * 60 * 60,
+        [parameter(Mandatory = $false)]
+        [int] $FirstDelay = 60,
+        [parameter(Mandatory = $false)]
+        [ValidateRange(0, 60 * 60)]
+        [int] $MaxWaitBetweenRetries = 60 * 60
+    )
+    # Initialize the variables that will tell us when we should stop trying
+    $startTime = Get-Date
+    $retryNo = 0
+    # Start trying...
+    $nextSleepTime = $FirstDelay
+    while ($true) {
+        $retryNo++
+        try {
+            Invoke-Command -ScriptBlock $ScriptBlock -OutVariable output | Out-Null
+            return $output # Success!
+        }
+        catch [System.Exception] {
+            $exceptionMessage = $_.Exception.Message
+            $secondsSinceStart = ((Get-Date) - $startTime).TotalSeconds
+
+            # Determine if we should keep trying
+            $tryAgain = $retryNo -lt $RetryCount -and $secondsSinceStart -lt $MaxWaitTimeBeforeLastAttempt
+            # Try again, or stop?
+            if ($tryAgain) {
+                # Sleep
+                $sleepTime = [System.Math]::Min($nextSleepTime, $MaxWaitTimeBeforeLastAttempt - $secondsSinceStart) # don't sleep beyond the max time
+                $sleepTime = [System.Math]::Min($sleepTime, $MaxWaitBetweenRetries) # don't sleep for more than one hour (and don't go above what Start-Sleep can handle (2147483))
+                Write-Warning "Command failed with error '$exceptionMessage' in attempt no $retryNo after $secondsSinceStart seconds. Will retry up to $RetryCount times. Sleeping for $sleepTime seconds before trying again..."
+                Start-Sleep -Seconds $sleepTime
+                $nextSleepTime = 2 * $nextSleepTime # Next time sleep for longer
+                # Now try again
+            }
+            else {
+                # Failed!
+                $output | Write-Host
+                throw
+            }
+        }
+    }
+}
+
 Export-ModuleMember -Function *-*
 Export-ModuleMember -Function RunAndCheck

build/scripts/GuardingV2ExtensionsHelper.psm1

@@ -258,8 +258,10 @@ function Test-IsStrictModeEnabled
 }
 
 function Get-MaxAllowedObsoleteVersion() {
-    git fetch origin main
-    $alGoSettings = $(git show origin/main:.github/AL-Go-Settings.json) | ConvertFrom-Json
+    Import-Module -Name $PSScriptRoot\EnlistmentHelperFunctions.psm1
+
+    Invoke-CommandWithRetry -ScriptBlock { RunAndCheck git fetch origin main }
+    $alGoSettings = $(RunAndCheck git show origin/main:.github/AL-Go-Settings.json) | ConvertFrom-Json
     if (-not $alGoSettings.repoVersion) {
         throw "Unable to find repoVersion in AL-Go-Settings.json"
     }