Codeflow history API endpoint

src/Microsoft.DotNet.Darc/DarcLib/AzureDevOpsClient.cs

@@ -1871,4 +1871,14 @@ public async Task<IReadOnlyCollection<string>> GetGitTreeNames(string path, stri
             throw;
         }
     }
+
+    public Task<List<Commit>> FetchLatestRepoCommits(string repoUrl, string branch, int maxCount)
+        => throw new NotImplementedException();
+
+    public Task<List<Commit>> FetchLatestFetchNewerRepoCommitsAsyncRepoCommits(
+        string repoUrl,
+        string branch,
+        string commitSha,
+        int maxCount)
+        => throw new NotImplementedException();
 }

src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs

@@ -2,6 +2,7 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System;
+using System.Collections;
 using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.IO;
@@ -10,20 +11,26 @@
 using System.Net.Http;
 using System.Reflection;
 using System.Text;
+using System.Text.Json;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
+using LibGit2Sharp;
 using Maestro.Common;
 using Maestro.MergePolicyEvaluation;
 using Microsoft.DotNet.DarcLib.Helpers;
 using Microsoft.DotNet.DarcLib.Models;
 using Microsoft.DotNet.DarcLib.Models.GitHub;
+using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo;
+using Microsoft.DotNet.DarcLib.VirtualMonoRepo;
 using Microsoft.DotNet.Services.Utility;
 using Microsoft.Extensions.Caching.Memory;
 using Microsoft.Extensions.Logging;
+using Microsoft.TeamFoundation.TestManagement.WebApi;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using Newtonsoft.Json.Serialization;
 using Octokit;
+using static System.Net.WebRequestMethods;
 
 #nullable enable
 namespace Microsoft.DotNet.DarcLib;
@@ -1413,4 +1420,191 @@ public async Task<IReadOnlyCollection<string>> GetGitTreeNames(string path, stri
             throw;
         }
     }
+
+    public Task<List<Commit>> FetchLatestRepoCommitsAsync(string repoUrl, string branch, int maxCount)
+        => throw new NotImplementedException();
+
+    public async Task<List<Commit>> FetchNewerRepoCommitsAsync(
+        string repoUrl,
+        string branch,
+        string commitSha,
+        int maxCount)
+    {
+        if (maxCount <= 0)
+        {
+            maxCount = 100;
+        }
+
+        (string owner, string repo) = ParseRepoUri(repoUrl);
+
+        var request = new CommitRequest
+        {
+            Sha = branch ?? "main",
+        };
+
+        var options = new ApiOptions
+        {
+            PageSize = maxCount,
+            PageCount = 1,
+            StartPage = 1
+        };
+
+        var allCommits = new List<Commit>();
+
+        while (allCommits.Count < maxCount)
+        {
+            var commits = await GetClient(owner, repo)
+                .Repository
+                .Commit
+                .GetAll(owner, repo, request, options);
+
+            foreach (Octokit.GitHubCommit c in commits)
+            {
+                var convertedCommit = new Commit(
+                    c.Author?.Login,
+                    c.Commit.Sha,
+                    c.Commit.Message);
+
+                allCommits.Add(convertedCommit);
+                if (convertedCommit.Sha.Equals(commitSha))
+                {
+                    break;
+                }
+            }
+
+            if (commits.Count < options.PageSize)
+            {
+                break;
+            }
+
+            options.StartPage++;
+        }
+
+        return [.. allCommits.Take(maxCount)];
+    }
+
+    public async Task<ForwardFlow?> GetLastIncomingForwardFlow(string vmrUrl, string mappingName, string commit)
+    {
+        var content = await GetFileContentAtCommit(
+            vmrUrl,
+            commit,
+            VmrInfo.DefaultRelativeSourceManifestPath);
+
+        var lastForwardFlowRepoSha = SourceManifest
+            .FromJson(content)?
+            .GetRepoVersion(mappingName)
+            .CommitSha;
+
+        if (lastForwardFlowRepoSha == null)
+        {
+            return null;
+        }
+
+        int lineNumber = content.Split(Environment.NewLine)
+            .ToList()
+            .FindIndex(line => line.Contains(lastForwardFlowRepoSha));
+
+
+        string lastForwardFlowVmrSha = await BlameLineAsync(
+            vmrUrl,
+            commit,
+            VmrInfo.DefaultRelativeSourceManifestPath,
+            lineNumber);
+
+        return new ForwardFlow(lastForwardFlowRepoSha, lastForwardFlowVmrSha);
+    }
+
+    public async Task<Backflow?> GetLastIncomingBackflow(string repoUrl, string commit)
+    {
+        var content = await GetFileContentAtCommit(
+            repoUrl,
+            commit,
+            VersionFiles.VersionDetailsXml);
+
+        var lastBackflowVmrSha = VersionDetailsParser
+            .ParseVersionDetailsXml(content)?
+            .Source?
+            .Sha;
+
+        if (lastBackflowVmrSha == null)
+        {
+            return null;
+        }
+
+        int lineNumber = content
+            .Split(Environment.NewLine)
+            .ToList()
+            .FindIndex(line =>
+                line.Contains(VersionDetailsParser.SourceElementName) &&
+                line.Contains(lastBackflowVmrSha));
+
+        string lastBackflowRepoSha = await BlameLineAsync(
+            repoUrl,
+            commit,
+            VersionFiles.VersionDetailsXml,
+            lineNumber);
+
+        return new Backflow(lastBackflowVmrSha, lastBackflowRepoSha);
+    }
+
+    private async Task<string> BlameLineAsync(string repoUrl, string commitOrBranch, string filePath, int lineNumber)
+    {
+        (string owner, string repo) = ParseRepoUri(repoUrl);
+
+        string query = $@"""
+        {{
+          repository(owner: {owner}, name: {repo}) {{
+            object(expression: ""{commitOrBranch}:{filePath}"") {{
+              ... on Blob {{
+                blame {{
+                  ranges {{
+                    startingLine
+                    endingLine
+                    commit {{ oid }}
+                  }}
+                }}
+              }}
+            }}
+          }}
+        }}""";
+
+        var client = CreateHttpClient(repoUrl);
+
+        var requestBody = new { query };
+        var content = new StringContent(JsonConvert.SerializeObject(requestBody, _serializerSettings));
+
+        var response = await client.PostAsync("", content);
+        response.EnsureSuccessStatusCode();
+
+        using var stream = await response.Content.ReadAsStreamAsync();
+        using var doc = await JsonDocument.ParseAsync(stream);
+
+        var ranges = doc.RootElement
+            .GetProperty("data")
+            .GetProperty("repository")
+            .GetProperty("object")
+            .GetProperty("blame")
+            .GetProperty("ranges")
+            .EnumerateArray();
+
+        foreach (var range in ranges)
+        {
+            int start = range.GetProperty("startingLine").GetInt32();
+            int end = range.GetProperty("endingLine").GetInt32();
+
+            if (lineNumber >= start && lineNumber <= end)
+            {
+                return range.GetProperty("commit").GetProperty("oid").GetString()!;
+            }
+        }
+
+        throw new InvalidOperationException($"Line {lineNumber} not found in blame data.");
+    }
+
+    private async Task<string> GetFileContentAtCommit(string repoUrl, string commit, string filePath)
+    {
+        (string owner, string repo) = ParseRepoUri(repoUrl);
+        var file = await GetClient(repoUrl).Repository.Content.GetAllContentsByRef(owner, repo, filePath, commit);
+        return Encoding.UTF8.GetString(Convert.FromBase64String(file[0].Content));
+    }
 }

src/Microsoft.DotNet.Darc/DarcLib/Helpers/VersionDetailsParser.cs

@@ -5,9 +5,14 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Threading.Tasks;
 using System.Xml;
+using LibGit2Sharp;
 using Microsoft.DotNet.DarcLib.Models;
 using Microsoft.DotNet.DarcLib.Models.Darc;
+using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo;
+using Microsoft.TeamFoundation.Work.WebApi;
+using static System.Net.Mime.MediaTypeNames;
 
 #nullable enable
 namespace Microsoft.DotNet.DarcLib.Helpers;
@@ -56,13 +61,13 @@ public VersionDetails ParseVersionDetailsFile(string path, bool includePinned =
         return ParseVersionDetailsXml(content, includePinned: includePinned);
     }
 
-    public VersionDetails ParseVersionDetailsXml(string fileContents, bool includePinned = true)
+    public static VersionDetails ParseVersionDetailsXml(string fileContents, bool includePinned = true)
     {
         XmlDocument document = GetXmlDocument(fileContents);
         return ParseVersionDetailsXml(document, includePinned: includePinned);
     }
 
-    public VersionDetails ParseVersionDetailsXml(XmlDocument document, bool includePinned = true)
+    public static VersionDetails ParseVersionDetailsXml(XmlDocument document, bool includePinned = true)
     {
         XmlNodeList? dependencyNodes = document?.DocumentElement?.SelectNodes($"//{DependencyElementName}");
         if (dependencyNodes == null)

src/Microsoft.DotNet.Darc/DarcLib/IRemote.cs

@@ -1,6 +1,7 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System;
 using System.Collections.Generic;
 using System.Threading.Tasks;
 using Maestro.MergePolicyEvaluation;
@@ -254,5 +255,30 @@ Task CommitUpdatesAsync(
     /// <returns></returns>
     Task<IReadOnlyCollection<string>> GetGitTreeNames(string path, string repoUri, string branch);
 
-    #endregion
+    /// <summary>
+    /// Fetches the latest commits from a repository branch, up to maxCount
+    /// </summary>
+    /// <param name="repoUrl">Full url of the git repository</param>
+    /// <param name="branch">Name of the git branch in the repo</param>
+    /// <param name="maxCount">Maximum count of commits to fetch</param>
+    /// <returns>List of commits</returns>
+    Task<List<Commit>> FetchLatestRepoCommitsAsync(string repoUrl, string branch, int maxCount = 100);
+
+
+    /// <summary>
+    /// Fetches the latest commits from a repository branch that are newer than the specified
+    /// commit, fetching up to maxCount commits in total.
+    /// </summary>
+    /// <param name="repoUrl">Full url of the git repository</param>
+    /// <param name="branch">Name of the git branch in the repo</param>
+    /// <param name="commitSha">Sha of the commit to fetch newer commits than</param>
+    /// <param name="maxCount">Maximum count of commits to fetch</param>
+    /// <returns>List of commits</returns>
+    Task<List<Commit>> FetchNewerRepoCommitsAsync(string repoUrl, string branch, string commitSha, int maxCount = 100);
+
+
+    Task<Tuple<string, string>> GetLastIncomingForwardflow(string vmrUrl, string commit);
+    Task<Tuple<string, string>> GetLastIncomingBackflow(string repoUrl, string commit);
+
+        #endregion
 }

src/Microsoft.DotNet.Darc/DarcLib/IRemoteGitRepo.cs

@@ -181,6 +181,27 @@ Task<IEnumerable<int>> SearchPullRequestsAsync(
     /// Returns a list of tree names (directories) under a given path in a given branch
     /// </summary>
     Task<IReadOnlyCollection<string>> GetGitTreeNames(string path, string repoUri, string branch);
+
+    /// <summary>
+    /// Returns the latest commits for a given repository and branch, up to maxCount
+    /// </summary>
+    Task<List<Commit>> FetchLatestRepoCommitsAsync(string repoUrl, string branch, int maxCount);
+
+    /// <summary>
+    /// Returns the latest commits for a given repository and branch that are newer than
+    /// the given commit, fetching up to maxCount commits in total.
+    /// </summary>
+    Task<List<Commit>> FetchNewerRepoCommitsAsync(string repoUrl, string branch, string commitSha, int maxCount);
+
+    /// <summary>
+    /// Get the last forward flow that was merged onto the given VMR at the specified commit
+    /// </summary>
+    Task<Tuple<string, string>?> GetLastIncomingForwardFlow(string vmrUrl, string branch, string commit);
+
+    /// <summary>
+    /// Get the last back flow that was merged onto the given repo at the specified commit
+    /// </summary>
+    Task<Tuple<string, string>?> GetLastIncomingBackflow(string repoUrl, string commit);
 }
 
 #nullable disable

src/Microsoft.DotNet.Darc/DarcLib/Remote.cs

@@ -494,4 +494,28 @@ public async Task<IReadOnlyCollection<string>> GetGitTreeNames(string path, stri
     {
         return await _remoteGitClient.GetGitTreeNames(path, repoUri, branch);
     }
+
+    public Task<List<Commit>> FetchLatestRepoCommitsAsync(string repoUrl, string branch, int maxCount = 100)
+    {
+        return _remoteGitClient.FetchLatestRepoCommitsAsync(repoUrl, branch, maxCount);
+    }
+    public Task<List<Commit>> FetchNewerRepoCommitsAsync(
+        string repoUrl,
+        string branch,
+        string commitSha,
+        int maxCount = 100)
+    {
+        return _remoteGitClient.FetchNewerRepoCommitsAsync(repoUrl, branch, commitSha, maxCount);
+    }
+
+    public async Task<Tuple<string, string>> GetLastIncomingForwardflow(string vmrUrl, string commit)
+    {
+        await Task.CompletedTask;
+        return null;
+    }
+    public async Task<Tuple<string, string>> GetLastIncomingBackflow(string repoUrl, string commit)
+    {
+        await Task.CompletedTask;
+        return null;
+    }
 }

src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrCodeflower.cs

@@ -479,19 +479,16 @@ await HandleRevertedFiles(
     /// <summary>
     /// Finds the last backflow between a repo and a VMR.
     /// </summary>
-    private async Task<Backflow?> GetLastBackflow(NativePath repoPath)
+    private async Task<Backflow?> GetLastBackflow(string repoPath)
     {
-        // Last backflow SHA comes from Version.Details.xml in the repo
-        SourceDependency? source = _versionDetailsParser.ParseVersionDetailsFile(repoPath / VersionFiles.VersionDetailsXml).Source;
-        if (source is null)
+        var versionDetailsContent = await _localGitClient.GetFileFromGitAsync(repoPath, VersionFiles.VersionDetailsXml);
+        if (versionDetailsContent == null)
         {
             return null;
         }
 
-        string lastBackflowVmrSha = source.Sha;
-        string lastBackflowRepoSha = await _localGitClient.BlameLineAsync(
-            repoPath / VersionFiles.VersionDetailsXml,
-            line => line.Contains(VersionDetailsParser.SourceElementName) && line.Contains(lastBackflowVmrSha));
+        var lineNumber = VersionDetailsParser.SourceDependencyLineNumber(versionDetailsContent);
+        string lastBackflowRepoSha = await _localGitClient.BlameLineAsync(repoPath, VersionFiles.VersionDetailsXml, lineNumber);
 
         return new Backflow(lastBackflowVmrSha, lastBackflowRepoSha);
     }