diff --git a/src/Microsoft.DotNet.Darc/DarcLib/AzureDevOpsClient.cs b/src/Microsoft.DotNet.Darc/DarcLib/AzureDevOpsClient.cs index 61968429a2..66301f6269 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/AzureDevOpsClient.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/AzureDevOpsClient.cs @@ -1871,4 +1871,14 @@ public async Task> GetGitTreeNames(string path, stri throw; } } + + public Task> FetchLatestRepoCommits(string repoUrl, string branch, int maxCount) + => throw new NotImplementedException(); + + public Task> FetchLatestFetchNewerRepoCommitsAsyncRepoCommits( + string repoUrl, + string branch, + string commitSha, + int maxCount) + => throw new NotImplementedException(); } diff --git a/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs b/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs index e9c74750b9..3efcd1fede 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Collections; using System.Collections.Generic; using System.Collections.Immutable; using System.IO; @@ -10,20 +11,26 @@ using System.Net.Http; using System.Reflection; using System.Text; +using System.Text.Json; using System.Text.RegularExpressions; using System.Threading.Tasks; +using LibGit2Sharp; using Maestro.Common; using Maestro.MergePolicyEvaluation; using Microsoft.DotNet.DarcLib.Helpers; using Microsoft.DotNet.DarcLib.Models; using Microsoft.DotNet.DarcLib.Models.GitHub; +using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo; +using Microsoft.DotNet.DarcLib.VirtualMonoRepo; using Microsoft.DotNet.Services.Utility; using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; +using Microsoft.TeamFoundation.TestManagement.WebApi; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using Newtonsoft.Json.Serialization; using Octokit; +using static System.Net.WebRequestMethods; #nullable enable namespace Microsoft.DotNet.DarcLib; @@ -1413,4 +1420,191 @@ public async Task> GetGitTreeNames(string path, stri throw; } } + + public Task> FetchLatestRepoCommitsAsync(string repoUrl, string branch, int maxCount) + => throw new NotImplementedException(); + + public async Task> FetchNewerRepoCommitsAsync( + string repoUrl, + string branch, + string commitSha, + int maxCount) + { + if (maxCount <= 0) + { + maxCount = 100; + } + + (string owner, string repo) = ParseRepoUri(repoUrl); + + var request = new CommitRequest + { + Sha = branch ?? "main", + }; + + var options = new ApiOptions + { + PageSize = maxCount, + PageCount = 1, + StartPage = 1 + }; + + var allCommits = new List(); + + while (allCommits.Count < maxCount) + { + var commits = await GetClient(owner, repo) + .Repository + .Commit + .GetAll(owner, repo, request, options); + + foreach (Octokit.GitHubCommit c in commits) + { + var convertedCommit = new Commit( + c.Author?.Login, + c.Commit.Sha, + c.Commit.Message); + + allCommits.Add(convertedCommit); + if (convertedCommit.Sha.Equals(commitSha)) + { + break; + } + } + + if (commits.Count < options.PageSize) + { + break; + } + + options.StartPage++; + } + + return [.. allCommits.Take(maxCount)]; + } + + public async Task GetLastIncomingForwardFlow(string vmrUrl, string mappingName, string commit) + { + var content = await GetFileContentAtCommit( + vmrUrl, + commit, + VmrInfo.DefaultRelativeSourceManifestPath); + + var lastForwardFlowRepoSha = SourceManifest + .FromJson(content)? + .GetRepoVersion(mappingName) + .CommitSha; + + if (lastForwardFlowRepoSha == null) + { + return null; + } + + int lineNumber = content.Split(Environment.NewLine) + .ToList() + .FindIndex(line => line.Contains(lastForwardFlowRepoSha)); + + + string lastForwardFlowVmrSha = await BlameLineAsync( + vmrUrl, + commit, + VmrInfo.DefaultRelativeSourceManifestPath, + lineNumber); + + return new ForwardFlow(lastForwardFlowRepoSha, lastForwardFlowVmrSha); + } + + public async Task GetLastIncomingBackflow(string repoUrl, string commit) + { + var content = await GetFileContentAtCommit( + repoUrl, + commit, + VersionFiles.VersionDetailsXml); + + var lastBackflowVmrSha = VersionDetailsParser + .ParseVersionDetailsXml(content)? + .Source? + .Sha; + + if (lastBackflowVmrSha == null) + { + return null; + } + + int lineNumber = content + .Split(Environment.NewLine) + .ToList() + .FindIndex(line => + line.Contains(VersionDetailsParser.SourceElementName) && + line.Contains(lastBackflowVmrSha)); + + string lastBackflowRepoSha = await BlameLineAsync( + repoUrl, + commit, + VersionFiles.VersionDetailsXml, + lineNumber); + + return new Backflow(lastBackflowVmrSha, lastBackflowRepoSha); + } + + private async Task BlameLineAsync(string repoUrl, string commitOrBranch, string filePath, int lineNumber) + { + (string owner, string repo) = ParseRepoUri(repoUrl); + + string query = $@""" + {{ + repository(owner: {owner}, name: {repo}) {{ + object(expression: ""{commitOrBranch}:{filePath}"") {{ + ... on Blob {{ + blame {{ + ranges {{ + startingLine + endingLine + commit {{ oid }} + }} + }} + }} + }} + }} + }}"""; + + var client = CreateHttpClient(repoUrl); + + var requestBody = new { query }; + var content = new StringContent(JsonConvert.SerializeObject(requestBody, _serializerSettings)); + + var response = await client.PostAsync("", content); + response.EnsureSuccessStatusCode(); + + using var stream = await response.Content.ReadAsStreamAsync(); + using var doc = await JsonDocument.ParseAsync(stream); + + var ranges = doc.RootElement + .GetProperty("data") + .GetProperty("repository") + .GetProperty("object") + .GetProperty("blame") + .GetProperty("ranges") + .EnumerateArray(); + + foreach (var range in ranges) + { + int start = range.GetProperty("startingLine").GetInt32(); + int end = range.GetProperty("endingLine").GetInt32(); + + if (lineNumber >= start && lineNumber <= end) + { + return range.GetProperty("commit").GetProperty("oid").GetString()!; + } + } + + throw new InvalidOperationException($"Line {lineNumber} not found in blame data."); + } + + private async Task GetFileContentAtCommit(string repoUrl, string commit, string filePath) + { + (string owner, string repo) = ParseRepoUri(repoUrl); + var file = await GetClient(repoUrl).Repository.Content.GetAllContentsByRef(owner, repo, filePath, commit); + return Encoding.UTF8.GetString(Convert.FromBase64String(file[0].Content)); + } } diff --git a/src/Microsoft.DotNet.Darc/DarcLib/Helpers/VersionDetailsParser.cs b/src/Microsoft.DotNet.Darc/DarcLib/Helpers/VersionDetailsParser.cs index d044db8916..7b10e180be 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/Helpers/VersionDetailsParser.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/Helpers/VersionDetailsParser.cs @@ -5,9 +5,14 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Threading.Tasks; using System.Xml; +using LibGit2Sharp; using Microsoft.DotNet.DarcLib.Models; using Microsoft.DotNet.DarcLib.Models.Darc; +using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo; +using Microsoft.TeamFoundation.Work.WebApi; +using static System.Net.Mime.MediaTypeNames; #nullable enable namespace Microsoft.DotNet.DarcLib.Helpers; @@ -56,13 +61,13 @@ public VersionDetails ParseVersionDetailsFile(string path, bool includePinned = return ParseVersionDetailsXml(content, includePinned: includePinned); } - public VersionDetails ParseVersionDetailsXml(string fileContents, bool includePinned = true) + public static VersionDetails ParseVersionDetailsXml(string fileContents, bool includePinned = true) { XmlDocument document = GetXmlDocument(fileContents); return ParseVersionDetailsXml(document, includePinned: includePinned); } - public VersionDetails ParseVersionDetailsXml(XmlDocument document, bool includePinned = true) + public static VersionDetails ParseVersionDetailsXml(XmlDocument document, bool includePinned = true) { XmlNodeList? dependencyNodes = document?.DocumentElement?.SelectNodes($"//{DependencyElementName}"); if (dependencyNodes == null) diff --git a/src/Microsoft.DotNet.Darc/DarcLib/IRemote.cs b/src/Microsoft.DotNet.Darc/DarcLib/IRemote.cs index 38604ea021..0facb11325 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/IRemote.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/IRemote.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; using System.Collections.Generic; using System.Threading.Tasks; using Maestro.MergePolicyEvaluation; @@ -254,5 +255,30 @@ Task CommitUpdatesAsync( /// Task> GetGitTreeNames(string path, string repoUri, string branch); - #endregion + /// + /// Fetches the latest commits from a repository branch, up to maxCount + /// + /// Full url of the git repository + /// Name of the git branch in the repo + /// Maximum count of commits to fetch + /// List of commits + Task> FetchLatestRepoCommitsAsync(string repoUrl, string branch, int maxCount = 100); + + + /// + /// Fetches the latest commits from a repository branch that are newer than the specified + /// commit, fetching up to maxCount commits in total. + /// + /// Full url of the git repository + /// Name of the git branch in the repo + /// Sha of the commit to fetch newer commits than + /// Maximum count of commits to fetch + /// List of commits + Task> FetchNewerRepoCommitsAsync(string repoUrl, string branch, string commitSha, int maxCount = 100); + + + Task> GetLastIncomingForwardflow(string vmrUrl, string commit); + Task> GetLastIncomingBackflow(string repoUrl, string commit); + + #endregion } diff --git a/src/Microsoft.DotNet.Darc/DarcLib/IRemoteGitRepo.cs b/src/Microsoft.DotNet.Darc/DarcLib/IRemoteGitRepo.cs index f3967db0c7..b17f4660a8 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/IRemoteGitRepo.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/IRemoteGitRepo.cs @@ -181,6 +181,27 @@ Task> SearchPullRequestsAsync( /// Returns a list of tree names (directories) under a given path in a given branch /// Task> GetGitTreeNames(string path, string repoUri, string branch); + + /// + /// Returns the latest commits for a given repository and branch, up to maxCount + /// + Task> FetchLatestRepoCommitsAsync(string repoUrl, string branch, int maxCount); + + /// + /// Returns the latest commits for a given repository and branch that are newer than + /// the given commit, fetching up to maxCount commits in total. + /// + Task> FetchNewerRepoCommitsAsync(string repoUrl, string branch, string commitSha, int maxCount); + + /// + /// Get the last forward flow that was merged onto the given VMR at the specified commit + /// + Task?> GetLastIncomingForwardFlow(string vmrUrl, string branch, string commit); + + /// + /// Get the last back flow that was merged onto the given repo at the specified commit + /// + Task?> GetLastIncomingBackflow(string repoUrl, string commit); } #nullable disable diff --git a/src/Microsoft.DotNet.Darc/DarcLib/Remote.cs b/src/Microsoft.DotNet.Darc/DarcLib/Remote.cs index ce706da71a..1e61b2693c 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/Remote.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/Remote.cs @@ -494,4 +494,28 @@ public async Task> GetGitTreeNames(string path, stri { return await _remoteGitClient.GetGitTreeNames(path, repoUri, branch); } + + public Task> FetchLatestRepoCommitsAsync(string repoUrl, string branch, int maxCount = 100) + { + return _remoteGitClient.FetchLatestRepoCommitsAsync(repoUrl, branch, maxCount); + } + public Task> FetchNewerRepoCommitsAsync( + string repoUrl, + string branch, + string commitSha, + int maxCount = 100) + { + return _remoteGitClient.FetchNewerRepoCommitsAsync(repoUrl, branch, commitSha, maxCount); + } + + public async Task> GetLastIncomingForwardflow(string vmrUrl, string commit) + { + await Task.CompletedTask; + return null; + } + public async Task> GetLastIncomingBackflow(string repoUrl, string commit) + { + await Task.CompletedTask; + return null; + } } diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrCodeflower.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrCodeflower.cs index da4de78232..d3e30f5575 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrCodeflower.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrCodeflower.cs @@ -479,19 +479,16 @@ await HandleRevertedFiles( /// /// Finds the last backflow between a repo and a VMR. /// - private async Task GetLastBackflow(NativePath repoPath) + private async Task GetLastBackflow(string repoPath) { - // Last backflow SHA comes from Version.Details.xml in the repo - SourceDependency? source = _versionDetailsParser.ParseVersionDetailsFile(repoPath / VersionFiles.VersionDetailsXml).Source; - if (source is null) + var versionDetailsContent = await _localGitClient.GetFileFromGitAsync(repoPath, VersionFiles.VersionDetailsXml); + if (versionDetailsContent == null) { return null; } - string lastBackflowVmrSha = source.Sha; - string lastBackflowRepoSha = await _localGitClient.BlameLineAsync( - repoPath / VersionFiles.VersionDetailsXml, - line => line.Contains(VersionDetailsParser.SourceElementName) && line.Contains(lastBackflowVmrSha)); + var lineNumber = VersionDetailsParser.SourceDependencyLineNumber(versionDetailsContent); + string lastBackflowRepoSha = await _localGitClient.BlameLineAsync(repoPath, VersionFiles.VersionDetailsXml, lineNumber); return new Backflow(lastBackflowVmrSha, lastBackflowRepoSha); } diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs index f6326eeed8..074ca5a24c 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs @@ -3,17 +3,19 @@ using System.ComponentModel.DataAnnotations; using System.Net; -using ProductConstructionService.Api.v2018_07_16.Models; using Maestro.Data; using Microsoft.AspNetCore.ApiPagination; using Microsoft.AspNetCore.ApiVersioning; using Microsoft.AspNetCore.ApiVersioning.Swashbuckle; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; +using ProductConstructionService.Api.Controllers.Models; +using ProductConstructionService.Api.v2018_07_16.Models; +using ProductConstructionService.Common; using ProductConstructionService.DependencyFlow.WorkItems; using ProductConstructionService.WorkItems; - using Channel = Maestro.Data.Models.Channel; +using SubscriptionDAO = Maestro.Data.Models.Subscription; namespace ProductConstructionService.Api.Api.v2018_07_16.Controllers; @@ -27,17 +29,20 @@ public class SubscriptionsController : ControllerBase private readonly BuildAssetRegistryContext _context; private readonly IWorkItemProducerFactory _workItemProducerFactory; private readonly IGitHubInstallationIdResolver _installationIdResolver; + private readonly ICodeflowHistoryManager _codeflowHistoryManager; private readonly ILogger _logger; public SubscriptionsController( BuildAssetRegistryContext context, IWorkItemProducerFactory workItemProducerFactory, IGitHubInstallationIdResolver installationIdResolver, + ICodeflowHistoryManager codeflowHistoryManager, ILogger logger) { _context = context; _workItemProducerFactory = workItemProducerFactory; _installationIdResolver = installationIdResolver; + _codeflowHistoryManager = codeflowHistoryManager; _logger = logger; } @@ -53,7 +58,7 @@ public virtual IActionResult ListSubscriptions( int? channelId = null, bool? enabled = null) { - IQueryable query = _context.Subscriptions.Include(s => s.Channel); + IQueryable query = _context.Subscriptions.Include(s => s.Channel); if (!string.IsNullOrEmpty(sourceRepository)) { @@ -88,7 +93,7 @@ public virtual IActionResult ListSubscriptions( [ValidateModelState] public virtual async Task GetSubscription(Guid id) { - Maestro.Data.Models.Subscription? subscription = await _context.Subscriptions.Include(sub => sub.LastAppliedBuild) + SubscriptionDAO? subscription = await _context.Subscriptions.Include(sub => sub.LastAppliedBuild) .Include(sub => sub.Channel) .FirstOrDefaultAsync(sub => sub.Id == id); @@ -100,6 +105,14 @@ public virtual async Task GetSubscription(Guid id) return Ok(new Subscription(subscription)); } + [HttpPost("{id}/codeflowhistory")] + [SwaggerApiResponse(HttpStatusCode.Accepted, Type = typeof(Subscription), Description = "Subscription update has been triggered")] + [ValidateModelState] + public virtual async Task GetCodeflowHistory(Guid id) + { + return await GetCodeflowHistoryCore(id); + } + /// /// Trigger a manually by id /// @@ -116,7 +129,7 @@ public virtual async Task TriggerSubscription(Guid id, [FromQuery protected async Task TriggerSubscriptionCore(Guid id, int buildId, bool force = false) { - Maestro.Data.Models.Subscription? subscription = await _context.Subscriptions + SubscriptionDAO? subscription = await _context.Subscriptions .Include(sub => sub.LastAppliedBuild) .Include(sub => sub.Channel) .FirstOrDefaultAsync(sub => sub.Id == id); @@ -152,9 +165,72 @@ protected async Task TriggerSubscriptionCore(Guid id, int buildId return Accepted(new Subscription(subscription)); } + protected async Task GetCodeflowHistoryCore(Guid id, bool fetchNewChanges) + { + var subscription = await _context.Subscriptions + .Include(sub => sub.LastAppliedBuild) + .FirstOrDefaultAsync(sub => sub.Id == id); + + if (subscription == null || !subscription.SourceEnabled) + { + return NotFound(); + } + + var oppositeDirectionSubscription = await _context.Subscriptions + .Include(sub => sub.LastAppliedBuild) + .Include(sub => sub.Channel) + .Where(sub => + sub.SourceRepository == subscription.TargetRepository || + sub.TargetRepository == subscription.SourceRepository) + .FirstOrDefaultAsync(); + + if (oppositeDirectionSubscription?.SourceEnabled != true) + { + oppositeDirectionSubscription = null; + } + + bool isForwardFlow = !string.IsNullOrEmpty(subscription.TargetDirectory); + + CodeflowHistory? cachedFlows; + CodeflowHistory? oppositeCachedFlows; + + cachedFlows = await _codeflowHistoryManager.FetchLatestCodeflowHistoryAsync(id); + + oppositeCachedFlows = await _codeflowHistoryManager.FetchLatestCodeflowHistoryAsync( + oppositeDirectionSubscription?.Id); + + + var lastCommit = subscription.LastAppliedBuild.Commit; + + bool resultIsOutdated = IsCodeflowHistoryOutdated(subscription, cachedFlows) || + IsCodeflowHistoryOutdated(oppositeDirectionSubscription, oppositeCachedFlows); + + var result = new CodeflowHistoryResult + { + ResultIsOutdated = resultIsOutdated, + ForwardFlowHistory = isForwardFlow + ? cachedFlows + : oppositeCachedFlows, + BackflowHistory = isForwardFlow + ? oppositeCachedFlows + : cachedFlows, + }; + + return Ok(result); + } + + private static bool IsCodeflowHistoryOutdated( + SubscriptionDAO? subscription, + CodeflowHistory? cachedFlows) + { + string? lastCachedCodeflow = cachedFlows?.Codeflows.LastOrDefault()?.SourceCommitSha; + string? lastAppliedCommit = subscription?.LastAppliedBuild?.Commit; + return !string.Equals(lastCachedCodeflow, lastAppliedCommit, StringComparison.Ordinal); + } + private async Task EnqueueUpdateSubscriptionWorkItemAsync(Guid subscriptionId, int buildId, bool force = false) { - Maestro.Data.Models.Subscription? subscriptionToUpdate; + SubscriptionDAO? subscriptionToUpdate; if (buildId != 0) { // Update using a specific build @@ -229,7 +305,7 @@ public virtual async Task TriggerDailyUpdateAsync() foreach (var subscription in enabledSubscriptionsWithTargetFrequency) { - Maestro.Data.Models.Subscription? subscriptionWithBuilds = await _context.Subscriptions + SubscriptionDAO? subscriptionWithBuilds = await _context.Subscriptions .Where(s => s.Id == subscription.Id) .Include(s => s.Channel) .ThenInclude(c => c.BuildChannels) @@ -274,7 +350,7 @@ await workitemProducer.ProduceWorkItemAsync(new() [ValidateModelState] public virtual async Task UpdateSubscription(Guid id, [FromBody] SubscriptionUpdate update) { - Maestro.Data.Models.Subscription? subscription = await _context.Subscriptions.Where(sub => sub.Id == id) + SubscriptionDAO? subscription = await _context.Subscriptions.Where(sub => sub.Id == id) .FirstOrDefaultAsync(); if (subscription == null) @@ -320,7 +396,7 @@ public virtual async Task UpdateSubscription(Guid id, [FromBody] if (doUpdate) { - Maestro.Data.Models.Subscription? equivalentSubscription = await FindEquivalentSubscription(subscription); + SubscriptionDAO? equivalentSubscription = await FindEquivalentSubscription(subscription); if (equivalentSubscription != null) { return BadRequest( @@ -348,7 +424,7 @@ public virtual async Task UpdateSubscription(Guid id, [FromBody] [ValidateModelState] public virtual async Task DeleteSubscription(Guid id) { - Maestro.Data.Models.Subscription? subscription = + SubscriptionDAO? subscription = await _context.Subscriptions.FirstOrDefaultAsync(sub => sub.Id == id); if (subscription == null) @@ -379,7 +455,7 @@ public virtual async Task DeleteSubscription(Guid id) [Paginated(typeof(SubscriptionHistoryItem))] public virtual async Task GetSubscriptionHistory(Guid id) { - Maestro.Data.Models.Subscription? subscription = await _context.Subscriptions.Where(sub => sub.Id == id) + SubscriptionDAO? subscription = await _context.Subscriptions.Where(sub => sub.Id == id) .FirstOrDefaultAsync(); if (subscription == null) @@ -450,11 +526,11 @@ public virtual async Task Create([FromBody, Required] Subscriptio } } - Maestro.Data.Models.Subscription subscriptionModel = subscription.ToDb(); + SubscriptionDAO subscriptionModel = subscription.ToDb(); subscriptionModel.Channel = channel; subscriptionModel.Id = Guid.NewGuid(); - Maestro.Data.Models.Subscription? equivalentSubscription = await FindEquivalentSubscription(subscriptionModel); + SubscriptionDAO? equivalentSubscription = await FindEquivalentSubscription(subscriptionModel); if (equivalentSubscription != null) { return Conflict( @@ -541,7 +617,7 @@ protected async Task EnsureRepositoryRegistration(string repoUri) /// /// Subscription model with updated data. /// Subscription if it is found, null otherwise - private async Task FindEquivalentSubscription(Maestro.Data.Models.Subscription updatedOrNewSubscription) + private async Task FindEquivalentSubscription(SubscriptionDAO updatedOrNewSubscription) { // Compare subscriptions based on the 4 key elements: // - Channel diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2019_01_16/Controllers/SubscriptionsController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2019_01_16/Controllers/SubscriptionsController.cs index 44877ec45e..3fcc8c01ea 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2019_01_16/Controllers/SubscriptionsController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2019_01_16/Controllers/SubscriptionsController.cs @@ -7,8 +7,10 @@ using Microsoft.AspNetCore.ApiVersioning; using Microsoft.AspNetCore.ApiVersioning.Swashbuckle; using Microsoft.AspNetCore.Mvc; +using Microsoft.DotNet.DarcLib; using Microsoft.EntityFrameworkCore; using ProductConstructionService.Api.v2019_01_16.Models; +using ProductConstructionService.Common; using ProductConstructionService.WorkItems; namespace ProductConstructionService.Api.Api.v2019_01_16.Controllers; @@ -26,8 +28,9 @@ public SubscriptionsController( BuildAssetRegistryContext context, IWorkItemProducerFactory workItemProducerFactory, IGitHubInstallationIdResolver gitHubInstallationRetriever, + ICodeflowHistoryManager codeflowHistoryManager, ILogger logger) - : base(context, workItemProducerFactory, gitHubInstallationRetriever, logger) + : base(context, workItemProducerFactory, gitHubInstallationRetriever, codeflowHistoryManager, logger) { _context = context; } @@ -91,6 +94,14 @@ public override async Task GetSubscription(Guid id) return Ok(new Subscription(subscription)); } + [HttpPost("{id}/codeflowhistory")] + [SwaggerApiResponse(HttpStatusCode.Accepted, Type = typeof(Subscription), Description = "Subscription update has been triggered")] + [ValidateModelState] + public override async Task GetCodeflowHistory(Guid id) + { + return await GetCodeflowHistoryCore(id); + } + /// /// Trigger a manually by id /// diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/SubscriptionsController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/SubscriptionsController.cs index 1bf3fb5f2a..d55c556137 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/SubscriptionsController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/SubscriptionsController.cs @@ -7,9 +7,11 @@ using Microsoft.AspNetCore.ApiVersioning; using Microsoft.AspNetCore.ApiVersioning.Swashbuckle; using Microsoft.AspNetCore.Mvc; +using Microsoft.DotNet.DarcLib; using Microsoft.DotNet.GitHub.Authentication; using Microsoft.EntityFrameworkCore; using ProductConstructionService.Api.v2020_02_20.Models; +using ProductConstructionService.Common; using ProductConstructionService.WorkItems; namespace ProductConstructionService.Api.Api.v2020_02_20.Controllers; @@ -31,8 +33,9 @@ public SubscriptionsController( IGitHubClientFactory gitHubClientFactory, IGitHubInstallationIdResolver gitHubInstallationRetriever, IWorkItemProducerFactory workItemProducerFactory, + ICodeflowHistoryManager codeflowHistoryManager, ILogger logger) - : base(context, workItemProducerFactory, gitHubInstallationRetriever, logger) + : base(context, workItemProducerFactory, gitHubInstallationRetriever, codeflowHistoryManager, logger) { _context = context; _gitHubClientFactory = gitHubClientFactory; @@ -144,6 +147,14 @@ public override async Task TriggerSubscription(Guid id, [FromQuer return await TriggerSubscriptionCore(id, buildId, force); } + [HttpPost("{id}/codeflowhistory")] + [SwaggerApiResponse(HttpStatusCode.Accepted, Type = typeof(Subscription), Description = "Subscription update has been triggered")] + [ValidateModelState] + public override async Task GetCodeflowHistory(Guid id) + { + return await GetCodeflowHistoryCore(id); + } + [ApiRemoved] public sealed override Task UpdateSubscription(Guid id, [FromBody] ProductConstructionService.Api.v2018_07_16.Models.SubscriptionUpdate update) { diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Controllers/Models/CodeflowHistoryResult.cs b/src/ProductConstructionService/ProductConstructionService.Api/Controllers/Models/CodeflowHistoryResult.cs new file mode 100644 index 0000000000..91a08b8422 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Api/Controllers/Models/CodeflowHistoryResult.cs @@ -0,0 +1,13 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using ProductConstructionService.Common; + +namespace ProductConstructionService.Api.Controllers.Models; + +public class CodeflowHistoryResult +{ + public CodeflowHistory? ForwardFlowHistory { get; set; } + public CodeflowHistory? BackflowHistory { get; set; } + public bool ResultIsOutdated { get; set; } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistory.cs b/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistory.cs new file mode 100644 index 0000000000..50609f7cf5 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistory.cs @@ -0,0 +1,6 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.DotNet.DarcLib; + +namespace ProductConstructionService.Common; diff --git a/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs b/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs new file mode 100644 index 0000000000..0406edb5c3 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs @@ -0,0 +1,163 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.DotNet.DarcLib; +using Maestro.Data.Models; + +namespace ProductConstructionService.Common; + +public interface ICodeflowHistoryManager +{ + Task GetCodeflowHistory(Guid? subscriptionId); + Task FetchLatestCodeflowHistoryAsync(Guid? subscriptionId); +} + +public record CodeflowHistory( + List Commits, + List Codeflows); + +public record CodeflowHistoryResult( + CodeflowHistory? ForwardFlowHistory, + CodeflowHistory? BackflowHistory, + bool ResultIsOutdated); + +public record CodeflowRecord( + string SourceCommitSha, + string TargetCommitSha, + DateTimeOffset CodeflowMergeDate); + +public record CodeflowGraphCommit( + string CommitSha, + DateTimeOffset CommitDate, + string Author, + string Description, + CodeflowGraphCommit? IncomingCodeflow); + +public class CodeflowHistoryManager : ICodeflowHistoryManager +{ + private readonly IRedisCacheFactory _redisCacheFactory; + private readonly IRemoteFactory _remoteFactory; + + public CodeflowHistoryManager( + IRedisCacheFactory cacheFactory, + IRemoteFactory remoteFactory) + { + _redisCacheFactory = cacheFactory; + _remoteFactory = remoteFactory; + } + + public async Task GetCachedCodeflowHistoryAsync(Subscription subscription) + { + string id = subscription.Id.ToString()!; + var cache = _redisCacheFactory.Create(id); + return await cache.TryGetStateAsync(); + } + + public async Task GetCachedCodeflowHistoryAsync( + Subscription subscription, + string commitSha, + int commitFetchCount) + { + // todo this method returns the codeflow history starting from commitSha. + // It only reads from redis and never modifies the cache + } + public async Task FetchLatestCodeflowHistoryAsync( + Subscription subscription, + int commitFetchCount) + { + //todo acquire lock on the redis Zset here + var cachedCommits = await GetCachedCodeflowHistoryAsync(subscription.Id); + + var remote = await _remoteFactory.CreateRemoteAsync(subscription.TargetRepository); + + latestCachedCommitSha = cachedCommits?.Commits.FirstOrDefault()?.CommitSha; + + var latestCommits = await remote.FetchNewerRepoCommitsAsync( + subscription.TargetBranch, + subscription.TargetBranch, + latestCachedCommitSha, + commitFetchCount); + + if (latestCommits.Count == commitFetchCount && + latestCommits.LastOrDefault()?.CommitSha != latestCachedCommitSha) + { + // we have a gap in the history - throw away cache because we can't form a continuous history + cachedCommits = []; + } + else + { + latestCommits = latestCommits + .Where(commit => commit.CommitSha != latestCachedCommitSha) + .ToList(); + } + + var latestCachedCodeflow = cachedCommits?.Commits.FirstOrDefault( + commit => commit.IncomingCodeflows != null); + + var codeFlows = await FetchLatestIncomingCodeflows( + subscription.TargetRepository, + subscription.TargetBranch, + latestCommits, + remote); + + foreach (var commit in latestCommits) + { + string? sourceCommitSha = codeflows.GetCodeflowSourceCommit(commit.CommitSha); + commit.IncomingCodeflow = sourceCommitSha; + } + + // todo cache fresh commits and release lock on the Zset + await CacheCommits(latestCommits); + + return null; + } + + private async Task FetchLatestIncomingCodeflows( + string repo, + string branch, + List latestCommits, + IRemote? remote) + { + if (remote == null) + { + remote = await _remoteFactory.CreateRemoteAsync(repo); + } + + var lastFlow = remote.GetLastIncomingCodeflow(branch, latestCachedCommit?.CommitSha); + + //todo: implement this method + return null; + } + + private async Task CacheCommits(List commits) + { + // Cache the commits as part of the subscription's redis ZSet of CodeflowGraphCommit objects + if (commits.Count == 0) + { + return; + } + var cache = _redisCacheFactory.Create(subscription.Id.ToString()!); + await cache.SetStateAsync(new CodeflowHistory(commits, codeflows)); + + } +} + +class GraphCodeflows +{ + // keys: target repo commits that have incoing codeflows + // values: commit SHAs of those codeflows in the source repo + public Dictionary Codeflows { get; set; } = []; + + /// + /// Returns the source commit of the codeflow if targetCommitSha is a target commit of a codeflow. + /// Otherwise, return null + /// + public string? GetCodeflowSourceCommit(string targetCommitSha) + { + if (Codeflows.TryGetValue(targetCommitSha, out var sourceCommit)) + { + return sourceCommit; + } + return null; + } +}