diff --git a/implement/Pine.Core/TreeNodeWithStringPath.cs b/implement/Pine.Core/TreeNodeWithStringPath.cs index 600c59e7..2cdc8635 100644 --- a/implement/Pine.Core/TreeNodeWithStringPath.cs +++ b/implement/Pine.Core/TreeNodeWithStringPath.cs @@ -150,7 +150,7 @@ public static TreeNodeWithStringPath NonSortedTree(IReadOnlyList<(string name, T public TreeNodeWithStringPath SetNodeAtPathSorted(IReadOnlyList path, TreeNodeWithStringPath node) { - if (path.Count == 0) + if (path.Count is 0) return node; var pathFirstElement = path[0]; @@ -174,6 +174,13 @@ public TreeNodeWithStringPath SetNodeAtPathSorted(IReadOnlyList path, Tr return SortedTree(treeEntries); } + public static TreeNodeWithStringPath MergeBlobs( + TreeNodeWithStringPath left, + TreeNodeWithStringPath right) => + right.EnumerateBlobsTransitive() + .Aggregate(left, (acc, blob) => acc.SetNodeAtPathSorted(blob.path, Blob(blob.blobContent))); + + public static TreeNodeWithStringPath FilterNodesByPath( TreeNodeWithStringPath node, Func, bool> pathFilter, diff --git a/implement/pine/Elm/ElmAppDependencyResolution.cs b/implement/pine/Elm/ElmAppDependencyResolution.cs index 7ccabee8..12958b75 100644 --- a/implement/pine/Elm/ElmAppDependencyResolution.cs +++ b/implement/pine/Elm/ElmAppDependencyResolution.cs @@ -1,3 +1,4 @@ +using ElmTime.ElmSyntax; using Pine.Core; using Pine.Elm019; using System; @@ -7,9 +8,177 @@ namespace Pine.Elm; +public record AppCompilationUnits( + TreeNodeWithStringPath AppFiles, + IReadOnlyList<(TreeNodeWithStringPath files, ElmJsonStructure elmJson)> Packages) +{ + public static AppCompilationUnits WithoutPackages( + TreeNodeWithStringPath appCode) + { + return new AppCompilationUnits( + appCode, + Packages: []); + } +} + public class ElmAppDependencyResolution { - public static IReadOnlyDictionary, ReadOnlyMemory> MergePackagesElmModules( + public static (AppCompilationUnits files, IReadOnlyList entryModuleName) + AppCompilationUnitsForEntryPoint( + TreeNodeWithStringPath sourceFiles, + IReadOnlyList entryPointFilePath) + { + if (sourceFiles.GetNodeAtPath(entryPointFilePath) is not { } entryFileNode) + { + throw new Exception("Entry file not found: " + string.Join("/", entryPointFilePath)); + } + + if (entryFileNode is not TreeNodeWithStringPath.BlobNode entryFileBlob) + { + throw new Exception( + "Entry file is not a blob: " + string.Join("/", entryPointFilePath)); + } + + var entryFileText = + System.Text.Encoding.UTF8.GetString(entryFileBlob.Bytes.Span); + + if (ElmModule.ParseModuleName(entryFileText).IsOkOrNull() is not { } moduleName) + { + throw new Exception( + "Failed to parse module name from entry file: " + string.Join("/", entryPointFilePath)); + } + + var sourceFilesDict = + PineValueComposition.TreeToFlatDictionaryWithPathComparer(sourceFiles); + + var sourceFilesFiltered = + ElmCompiler.FilterTreeForCompilationRoots( + sourceFiles, + ImmutableHashSet.Create( + EnumerableExtension.EqualityComparer>(), + entryPointFilePath), + skipFilteringForSourceDirs: false); + + IReadOnlyList, IReadOnlyList>>> + remainingElmModulesNameAndImports = + sourceFilesFiltered + .EnumerateBlobsTransitive() + .Where(blob => blob.path.Last().EndsWith(".elm", StringComparison.OrdinalIgnoreCase)) + .Select(blob => + { + var moduleText = System.Text.Encoding.UTF8.GetString(blob.blobContent.Span); + + if (ElmModule.ParseModuleName(moduleText).IsOkOrNull() is not { } moduleName) + { + throw new Exception("Failed to parse module name from file: " + string.Join("/", blob.path)); + } + + return new KeyValuePair, IReadOnlyList>>( + moduleName, + [.. ElmModule.ParseModuleImportedModulesNames(moduleText)]); + }) + .ToImmutableList(); + + var remainingElmModulesImports = + remainingElmModulesNameAndImports + .SelectMany(kv => kv.Value) + .Where(importedModuleName => !remainingElmModulesNameAndImports.Any(kv => kv.Key.SequenceEqual(importedModuleName))) + .ToImmutableHashSet(EnumerableExtension.EqualityComparer>()); + + var packages = LoadPackagesForElmApp(sourceFilesDict); + + /* + * We filter packages to include only those needed for the current compilation entry point. + * + * Referencing two packages that expose modules with the same name is + * no problem as long as the app does not reference that module name. + * */ + + IReadOnlySet aggregateExposedModuleNames = + packages + .SelectMany(package => package.Value.elmJson.ExposedModules) + .ToImmutableHashSet(); + + IReadOnlyDictionary> + packagesFromExposedModuleName = + aggregateExposedModuleNames + .Select(moduleName => + { + return + new KeyValuePair>( + moduleName, + packages + .Where(package => package.Value.elmJson.ExposedModules.Contains(moduleName)) + .Select(package => package.Key) + .ToImmutableHashSet()); + }) + .ToImmutableDictionary(); + + var packagesToIncludeNames = + remainingElmModulesImports + .Select(importedModuleName => + { + var importedModuleNameFlat = string.Join(".", importedModuleName); + + if (!packagesFromExposedModuleName.TryGetValue(importedModuleNameFlat, out var packages)) + { + throw new Exception("Failed to find package for imported module: " + importedModuleNameFlat); + } + + if (packages.Count is not 1) + { + throw new Exception( + "Imported module " + importedModuleNameFlat + + " is exposed by multiple packages: " + string.Join(", ", packages)); + } + + return packages.First(); + }) + .ToImmutableHashSet(); + + IEnumerable enumeratePackageDependenciesTransitive(string packageName) + { + if (!packages.TryGetValue(packageName, out var package)) + { + yield break; + } + + var dependencies = + package.elmJson.Dependencies.Direct.EmptyIfNull() + .Concat(package.elmJson.Dependencies.Indirect.EmptyIfNull()) + .Concat(package.elmJson.Dependencies.Flat.EmptyIfNull()); + + foreach (var dependency in dependencies) + { + yield return dependency.Key; + + foreach (var transitiveDependency in enumeratePackageDependenciesTransitive(dependency.Key)) + { + yield return transitiveDependency; + } + } + } + + var packagesToInclude = + packages + .Where(kv => packagesToIncludeNames.Contains(kv.Key)) + .ToImmutableDictionary(); + + var packagesOrdered = + packagesToInclude + .OrderBy(kv => enumeratePackageDependenciesTransitive(kv.Key).Count()) + .ThenBy(kv => kv.Key) + .ToImmutableList(); + + return + (new AppCompilationUnits( + sourceFilesFiltered, + Packages: [.. packagesOrdered.Select(pkg => pkg.Value)]), + moduleName); + } + + public static IReadOnlyDictionary + LoadPackagesForElmApp( IReadOnlyDictionary, ReadOnlyMemory> appSourceFiles) { /* @@ -21,7 +190,7 @@ public static IReadOnlyDictionary, ReadOnlyMemory> M .Where(entry => entry.Key.Last() is "elm.json") .ToImmutableDictionary(); - var elmJsonAggregateDependencies = + var elmJsonAggregateDependenciesVersions = elmJsonFiles .SelectMany(elmJsonFile => { @@ -33,9 +202,9 @@ public static IReadOnlyDictionary, ReadOnlyMemory> M return new[] { - elmJsonParsed?.Dependencies.Direct, - elmJsonParsed?.Dependencies.Indirect, - elmJsonParsed?.Dependencies.Flat + elmJsonParsed?.Dependencies.Direct, + elmJsonParsed?.Dependencies.Indirect, + elmJsonParsed?.Dependencies.Flat } .WhereNotNull(); } @@ -47,6 +216,10 @@ public static IReadOnlyDictionary, ReadOnlyMemory> M } }) .SelectMany(dependency => dependency) + .ToImmutableDictionary(); + + var elmJsonAggregateDependencies = + elmJsonAggregateDependenciesVersions .ToImmutableDictionary( keySelector: dependency => dependency.Key, elementSelector: @@ -58,73 +231,33 @@ public static IReadOnlyDictionary, ReadOnlyMemory> M return packageFiles; }); - var sourceElmModulesNames = - appSourceFiles - .Where(entry => entry.Key.Last().EndsWith(".elm", StringComparison.OrdinalIgnoreCase)) - .Select(entry => ElmTime.ElmSyntax.ElmModule.ParseModuleName(entry.Value).WithDefault(null)) - .WhereNotNull() - .ToImmutableHashSet(EnumerableExtension.EqualityComparer>()); - - var packagesModulesNames = - new HashSet>(EnumerableExtension.EqualityComparer>()); - - var sourceFilesWithMergedPackages = + return elmJsonAggregateDependencies - .Aggregate( - seed: appSourceFiles, - func: (aggregate, package) => + .ToImmutableDictionary( + keySelector: + kv => kv.Key, + elementSelector: + kv => { - if (package.Key is "elm/core" || - package.Key is "elm/json" || - package.Key is "elm/bytes" || - package.Key is "elm/parser" || - package.Key is "elm/url") + try { - return aggregate; - } - - var packageExposedModuleFiles = - ElmPackage.ExposedModules(package.Value); - - return - packageExposedModuleFiles - .Aggregate( - seed: aggregate.ToImmutableDictionary(), - func: (innerAggregate, packageElmModuleFile) => + if (!kv.Value.TryGetValue(["elm.json"], out var elmJsonFile)) { - var relativePath = packageElmModuleFile.Key; + throw new Exception("Did not find elm.json file"); + } - var moduleName = packageElmModuleFile.Value.moduleName; + var elmJsonParsed = + System.Text.Json.JsonSerializer.Deserialize(elmJsonFile.Span) + ?? + throw new Exception("Parsing elm.json returned null"); - if (sourceElmModulesNames.Contains(moduleName)) - { - Console.WriteLine( - "Skipping Elm module file " + - string.Join("/", relativePath) + - " from package " + package.Key + " because it is already present in the source files."); - - return innerAggregate; - } - - if (packagesModulesNames.Contains(moduleName)) - { - Console.WriteLine( - "Skipping Elm module file " + - string.Join("/", relativePath) + - " from package " + package.Key + " because it is already present in the packages."); - - return innerAggregate; - } - - packagesModulesNames.Add(moduleName); - - return - innerAggregate.SetItem( - ["dependencies", .. package.Key.Split('/'), .. packageElmModuleFile.Key], - packageElmModuleFile.Value.fileContent); - }); + return (PineValueComposition.SortedTreeFromSetOfBlobsWithStringPath(kv.Value), elmJsonParsed); + } + catch (Exception e) + { + throw new Exception( + "Failed to load package: " + kv.Key + ": " + e.Message, e); + } }); - - return sourceFilesWithMergedPackages; } } diff --git a/implement/pine/Elm/ElmCompiler.cs b/implement/pine/Elm/ElmCompiler.cs index 0b587c9c..54aca0b3 100644 --- a/implement/pine/Elm/ElmCompiler.cs +++ b/implement/pine/Elm/ElmCompiler.cs @@ -2,6 +2,7 @@ using ElmTime.JavaScript; using Pine.Core; using Pine.Core.Elm; +using Pine.Elm019; using System; using System.Collections.Concurrent; using System.Collections.Generic; @@ -359,8 +360,10 @@ rootFilePaths.Count is 0 return InteractiveSessionPine.CompileInteractiveEnvironment( - appCodeTree: appCodeFilteredForRoots, + appCodeTree: + appCodeFilteredForRoots is null ? null : AppCompilationUnits.WithoutPackages(appCodeFilteredForRoots), overrideSkipLowering: true, + entryPointsFilePaths: rootFilePaths, elmCompiler: defaultCompiler); } @@ -641,4 +644,227 @@ public static bool CheckIfAppUsesLowering(TreeNodeWithStringPath appCode) return false; } + + public static TreeNodeWithStringPath FilterTreeForCompilationRoots( + TreeNodeWithStringPath tree, + IReadOnlySet> rootFilePaths, + bool skipFilteringForSourceDirs) + { + var trees = + rootFilePaths + .Select(rootFilePath => + FilterTreeForCompilationRoot( + tree, + rootFilePath, + skipFilteringForSourceDirs: skipFilteringForSourceDirs)) + .ToImmutableArray(); + + return + trees + .Aggregate( + seed: TreeNodeWithStringPath.EmptyTree, + TreeNodeWithStringPath.MergeBlobs); + } + + public static TreeNodeWithStringPath FilterTreeForCompilationRoot( + TreeNodeWithStringPath tree, + IReadOnlyList rootFilePath, + bool skipFilteringForSourceDirs) + { + Func, bool> keepElmModuleAtFilePath = + skipFilteringForSourceDirs + ? + _ => true + : + BuildPredicateFilePathIsInSourceDirectory(tree, rootFilePath); + + var allAvailableElmFiles = + tree + .EnumerateBlobsTransitive() + .Where(blobAtPath => blobAtPath.path.Last().EndsWith(".elm", StringComparison.OrdinalIgnoreCase)) + .ToImmutableArray(); + + var availableElmFiles = + allAvailableElmFiles + .Where(blobAtPath => keepElmModuleAtFilePath(blobAtPath.path)) + .ToImmutableArray(); + + var rootElmFiles = + availableElmFiles + .Where(c => c.path.SequenceEqual(rootFilePath)) + .ToImmutableArray(); + + var elmModulesIncluded = + ElmTime.ElmSyntax.ElmModule.ModulesTextOrderedForCompilationByDependencies( + rootModulesTexts: [.. rootElmFiles.Select(file => Encoding.UTF8.GetString(file.blobContent.Span))], + availableModulesTexts: [.. availableElmFiles.Select(file => Encoding.UTF8.GetString(file.blobContent.Span))]); + + var filePathsExcluded = + allAvailableElmFiles + .Where(elmFile => !elmModulesIncluded.Any(included => Encoding.UTF8.GetString(elmFile.blobContent.Span) == included)) + .Select(elmFile => elmFile.path) + .ToImmutableHashSet(EnumerableExtension.EqualityComparer>()); + + return + TreeNodeWithStringPath.FilterNodesByPath( + tree, + nodePath => + !filePathsExcluded.Contains(nodePath)); + } + + private static Func, bool> BuildPredicateFilePathIsInSourceDirectory( + TreeNodeWithStringPath tree, + IReadOnlyList rootFilePath) + { + if (FindElmJsonForEntryPoint(tree, rootFilePath) is not { } elmJsonForEntryPoint) + { + throw new Exception( + "Failed to find elm.json for entry point: " + string.Join("/", rootFilePath)); + } + + IReadOnlyList sourceDirectories = + [.. elmJsonForEntryPoint.elmJsonParsed.ParsedSourceDirectories]; + + bool filePathIsInSourceDirectory(IReadOnlyList filePath) + { + foreach (var sourceDirectory in sourceDirectories) + { + if (sourceDirectory.ParentLevel is not 0) + { + throw new NotImplementedException( + "ParentLevel in elm.json source-directories not implemented"); + } + + if (filePath.Count < sourceDirectory.Subdirectories.Count) + { + continue; + } + + if (filePath.Take(sourceDirectory.Subdirectories.Count).SequenceEqual(sourceDirectory.Subdirectories)) + { + return true; + } + } + + return false; + } + + return filePathIsInSourceDirectory; + } + + public static (IReadOnlyList filePath, ElmJsonStructure elmJsonParsed)? + FindElmJsonForEntryPoint( + TreeNodeWithStringPath sourceFiles, + IReadOnlyList entryPointFilePath) + { + // Collect all elm.json files from the tree, storing each parsed ElmJsonStructure along with its path: + var elmJsonFiles = + sourceFiles + .EnumerateBlobsTransitive() + .SelectMany(pathAndContent => + { + if (!pathAndContent.path.Last().EndsWith("elm.json", StringComparison.OrdinalIgnoreCase)) + { + return []; + } + + var elmJsonContent = pathAndContent.blobContent; + + try + { + var elmJsonParsed = + System.Text.Json.JsonSerializer.Deserialize(elmJsonContent.Span); + + return + new[] + { + (filePath: (IReadOnlyList)pathAndContent.path, elmJsonParsed) + }; + } + catch (Exception e) + { + return []; + } + }) + .ToImmutableDictionary( + keySelector: entry => entry.filePath, + elementSelector: entry => entry.elmJsonParsed, + keyComparer: EnumerableExtension.EqualityComparer>()); + + // Walk upwards from the directory of entryPointFilePath to find the "closest" elm.json + // that includes the entryPointFilePath in one of its source-directories: + IReadOnlyList currentDirectory = DirectoryOf(entryPointFilePath); + + while (true) + { + // See if there is an elm.json directly in this directory: + + IReadOnlyList elmJsonFilePath = [.. currentDirectory, "elm.json"]; + + if (elmJsonFiles.TryGetValue(elmJsonFilePath, out var elmJsonParsed) && elmJsonParsed is not null) + { + // We found an elm.json in the current directory; now check if it includes the entry point + // by verifying that entryPointFilePath is under one of its source-directories: + if (ElmJsonIncludesEntryPoint( + currentDirectory, elmJsonParsed, entryPointFilePath)) + { + return (elmJsonFilePath, elmJsonParsed); + } + } + + // If we are at the root (no parent to move up to), stop: + if (currentDirectory.Count is 0) + { + return null; + } + + // Move up one level: + currentDirectory = currentDirectory.Take(currentDirectory.Count - 1).ToArray(); + } + } + + /// + /// Returns all but the last segment of filePath (i.e. the directory path). + /// + private static IReadOnlyList DirectoryOf(IReadOnlyList filePath) + { + if (filePath.Count is 0) + return filePath; + + return [.. filePath.Take(filePath.Count - 1)]; + } + + /// + /// Checks whether the given elm.json file includes entryPointFilePath in one of its source-directories. + /// Since source-directories in elm.json are relative to the directory containing elm.json, + /// we build absolute paths and compare. + /// + private static bool ElmJsonIncludesEntryPoint( + IReadOnlyList elmJsonDirectory, + ElmJsonStructure elmJson, + IReadOnlyList entryPointFilePath) + { + // For each source directory in elm.json, build its absolute path (relative to elm.jsonDirectory), + // and check whether entryPointFilePath starts with that path. + + foreach (var sourceDir in elmJson.ParsedSourceDirectories) + { + // Combine the elmJsonDirectory with the subdirectories from sourceDir + // to get the absolute path to the "source directory": + IReadOnlyList absSourceDir = + [.. elmJsonDirectory, .. sourceDir.Subdirectories]; + + // Check if entryPointFilePath is "under" absSourceDir: + if (entryPointFilePath.Count >= absSourceDir.Count && + entryPointFilePath + .Take(absSourceDir.Count) + .SequenceEqual(absSourceDir)) + { + // The entry point sits in one of the source-directories recognized by this elm.json + return true; + } + } + + return false; + } } diff --git a/implement/pine/Elm/Platform/CommandLineApp.cs b/implement/pine/Elm/Platform/CommandLineApp.cs index 150a6383..65ee9828 100644 --- a/implement/pine/Elm/Platform/CommandLineApp.cs +++ b/implement/pine/Elm/Platform/CommandLineApp.cs @@ -154,6 +154,8 @@ public static CommandLineAppConfig ConfigFromSourceFilesAndModuleName( appCodeTree: sourceFiles, // TODO: Migrate lowering implementation for portability. overrideSkipLowering: true, + // TODO: Forward entry point. + entryPointsFilePaths: null, caching: true, autoPGO: null); diff --git a/implement/pine/Elm/Platform/WebServiceInterface.cs b/implement/pine/Elm/Platform/WebServiceInterface.cs index 10b5bc5c..ce754f31 100644 --- a/implement/pine/Elm/Platform/WebServiceInterface.cs +++ b/implement/pine/Elm/Platform/WebServiceInterface.cs @@ -763,7 +763,7 @@ public static Result ParseCreateVolatilePro ElmCompilerCache elmCompilerCache, PineVMParseCache parseCache) { - var asRecordResult = ElmValueEncoding.ParsePineValueAsRecord(pineValue); + var asRecordResult = ElmValueEncoding.ParsePineValueAsRecordTagged(pineValue); { if (asRecordResult.IsErrOrNull() is { } err) @@ -1153,39 +1153,17 @@ public static WebServiceConfig ConfigFromSourceFilesAndEntryFileName( TreeNodeWithStringPath sourceFiles, IReadOnlyList entryFileName) { - if (sourceFiles.GetNodeAtPath(entryFileName) is not { } entryFileNode) - { - throw new Exception("Entry file not found: " + string.Join(".", entryFileName)); - } - - if (entryFileNode is not TreeNodeWithStringPath.BlobNode entryFileBlob) - { - throw new Exception("Entry file is not a blob: " + string.Join(".", entryFileName)); - } - - var entryFileText = - System.Text.Encoding.UTF8.GetString(entryFileBlob.Bytes.Span); - - if (ElmModule.ParseModuleName(entryFileText).IsOkOrNull() is not { } moduleName) - { - throw new Exception("Failed to parse module name from entry file: " + string.Join(".", entryFileName)); - } - - var sourceFilesWithPackages = - PineValueComposition.SortedTreeFromSetOfBlobsWithStringPath( - ElmAppDependencyResolution.MergePackagesElmModules( - PineValueComposition.TreeToFlatDictionaryWithPathComparer(sourceFiles))); - - var sourceFilesFiltered = - ElmCompiler.FilterTreeForCompilationRoots( - sourceFilesWithPackages, - [entryFileName]); + var compilationUnitsPrepared = + ElmAppDependencyResolution.AppCompilationUnitsForEntryPoint( + sourceFiles, + entryFileName); using var interactiveSession = new InteractiveSessionPine( ElmCompiler.CompilerSourceContainerFilesDefault.Value, - appCodeTree: sourceFilesFiltered, + appCodeTree: compilationUnitsPrepared.files, overrideSkipLowering: true, + entryPointsFilePaths: null, caching: true, autoPGO: null); @@ -1194,11 +1172,11 @@ public static WebServiceConfig ConfigFromSourceFilesAndEntryFileName( var (declValue, _) = ElmInteractiveEnvironment.ParseFunctionFromElmModule( compiledModulesValue, - moduleName: string.Join(".", moduleName), + moduleName: string.Join(".", compilationUnitsPrepared.entryModuleName), declarationName: "webServiceMain", parseCache) .Extract(err => throw new Exception( - $"Failed parsing webServiceMain declaration from module {string.Join(".", moduleName)}: {err}")); + $"Failed parsing webServiceMain declaration from module {string.Join(".", compilationUnitsPrepared.entryModuleName)}: {err}")); return ConfigFromDeclarationValue(declValue); } diff --git a/implement/pine/ElmInteractive/ElmInteractive.cs b/implement/pine/ElmInteractive/ElmInteractive.cs index 93d29ac8..eb429278 100644 --- a/implement/pine/ElmInteractive/ElmInteractive.cs +++ b/implement/pine/ElmInteractive/ElmInteractive.cs @@ -2,6 +2,7 @@ using ElmTime.JavaScript; using Pine; using Pine.Core; +using Pine.Elm; using Pine.Elm019; using Pine.ElmInteractive; using System; @@ -156,8 +157,8 @@ closestBaseInMemory is null ? elmModulesTexts; using var evalElmPreparedJavaScriptEngine = - Pine.Elm.ElmCompiler.JavaScriptEngineFromElmCompilerSourceFiles( - Pine.Elm.ElmCompiler.CompilerSourceContainerFilesDefault.Value); + ElmCompiler.JavaScriptEngineFromElmCompilerSourceFiles( + ElmCompiler.CompilerSourceContainerFilesDefault.Value); Result chooseInitResult() { @@ -835,7 +836,12 @@ public static IReadOnlyList ModulesTextsFromAppCodeTree( IReadOnlyList> rootFilePaths) { var allModules = - ModulesFilePathsAndTextsFromAppCodeTree(appCodeTree, skipLowering) + ModulesFilePathsAndTextsFromAppCodeTree( + appCodeTree, + skipLowering, + entryPointsFilePaths: rootFilePaths.ToImmutableHashSet(EnumerableExtension.EqualityComparer>()), + skipFilteringForSourceDirs: false, + mergeKernelModules: true) .Where(file => !ShouldIgnoreSourceFile(file.filePath, file.fileContent)) .ToImmutableArray(); @@ -870,7 +876,10 @@ public static bool ShouldIgnoreSourceFile(IReadOnlyList filePath, ReadOn public static IReadOnlyList<(IReadOnlyList filePath, ReadOnlyMemory fileContent, string moduleText)> ModulesFilePathsAndTextsFromAppCodeTree( TreeNodeWithStringPath appCodeTree, - bool skipLowering) + bool skipLowering, + IReadOnlySet>? entryPointsFilePaths, + bool skipFilteringForSourceDirs, + bool mergeKernelModules) { var loweredTree = skipLowering @@ -879,8 +888,24 @@ public static bool ShouldIgnoreSourceFile(IReadOnlyList filePath, ReadOn : CompileTree(appCodeTree); + // After lowering we can remove all Elm modules not in the dependency tree of the entry points. + + var treeFiltered = + entryPointsFilePaths is null || entryPointsFilePaths.Count < 1 + ? + loweredTree + : + ElmCompiler.FilterTreeForCompilationRoots( + loweredTree, + entryPointsFilePaths, + skipFilteringForSourceDirs: skipFilteringForSourceDirs); + var treeWithKernelModules = - InteractiveSessionPine.MergeDefaultElmCoreAndKernelModules(loweredTree); + mergeKernelModules + ? + InteractiveSessionPine.MergeDefaultElmCoreAndKernelModules(treeFiltered) + : + treeFiltered; return [.. TreeToFlatDictionaryWithPathComparer(treeWithKernelModules) diff --git a/implement/pine/ElmInteractive/ElmInteractiveEnvironment.cs b/implement/pine/ElmInteractive/ElmInteractiveEnvironment.cs index a6739344..ef304e52 100644 --- a/implement/pine/ElmInteractive/ElmInteractiveEnvironment.cs +++ b/implement/pine/ElmInteractive/ElmInteractiveEnvironment.cs @@ -73,7 +73,7 @@ public static Result ApplyFunctionInElmModule( if (parseEnvResult.IsOkOrNull() is not { } parsedEnv) { - throw new System.NotImplementedException( + throw new NotImplementedException( "Unexpected result type: " + parseEnvResult.GetType()); } diff --git a/implement/pine/ElmInteractive/InteractiveSession.cs b/implement/pine/ElmInteractive/InteractiveSession.cs index ecfe7785..ee22496f 100644 --- a/implement/pine/ElmInteractive/InteractiveSession.cs +++ b/implement/pine/ElmInteractive/InteractiveSession.cs @@ -34,6 +34,7 @@ static IInteractiveSession Create( compilerSourceFiles: compilerSourceFiles, appCodeTree: appCodeTree, overrideSkipLowering: null, + entryPointsFilePaths: null, caching: pineConfig.Caching, autoPGO: pineConfig.DynamicPGOShare), diff --git a/implement/pine/ElmInteractive/InteractiveSessionPine.cs b/implement/pine/ElmInteractive/InteractiveSessionPine.cs index 0b5dfe3b..5ba3b4b8 100644 --- a/implement/pine/ElmInteractive/InteractiveSessionPine.cs +++ b/implement/pine/ElmInteractive/InteractiveSessionPine.cs @@ -2,6 +2,7 @@ using Pine.Core; using Pine.Core.PineVM; using Pine.Elm; +using Pine.Elm019; using Pine.ElmInteractive; using Pine.PineVM; using System; @@ -37,19 +38,32 @@ public class InteractiveSessionPine : IInteractiveSession static readonly ConcurrentDictionary, PineValue>>> TryParseModuleTextCache = new(); - private static JavaScript.IJavaScriptEngine ParseSubmissionOrCompileDefaultJavaScriptEngine { get; } = - BuildParseSubmissionOrCompileDefaultJavaScriptEngine(); - - private static JavaScript.IJavaScriptEngine BuildParseSubmissionOrCompileDefaultJavaScriptEngine() + public InteractiveSessionPine( + TreeNodeWithStringPath compilerSourceFiles, + TreeNodeWithStringPath? appCodeTree, + bool? overrideSkipLowering, + IReadOnlyList>? entryPointsFilePaths, + bool caching, + DynamicPGOShare? autoPGO) + : + this( + compilerSourceFiles: compilerSourceFiles, + appCodeTree: + appCodeTree is null ? null : AppCompilationUnits.WithoutPackages(appCodeTree), + overrideSkipLowering: overrideSkipLowering, + entryPointsFilePaths: entryPointsFilePaths, + BuildPineVM( + caching: caching, + autoPGO: autoPGO, + new PineVM.EvaluationConfig(ParseAndEvalCountLimit: 10_000_000))) { - return ElmCompiler.JavaScriptEngineFromElmCompilerSourceFiles( - ElmCompiler.CompilerSourceContainerFilesDefault.Value); } public InteractiveSessionPine( TreeNodeWithStringPath compilerSourceFiles, - TreeNodeWithStringPath? appCodeTree, + AppCompilationUnits? appCodeTree, bool? overrideSkipLowering, + IReadOnlyList>? entryPointsFilePaths, bool caching, DynamicPGOShare? autoPGO) : @@ -57,6 +71,7 @@ public InteractiveSessionPine( compilerSourceFiles: compilerSourceFiles, appCodeTree: appCodeTree, overrideSkipLowering: overrideSkipLowering, + entryPointsFilePaths: entryPointsFilePaths, BuildPineVM( caching: caching, autoPGO: autoPGO, @@ -68,20 +83,40 @@ public InteractiveSessionPine( TreeNodeWithStringPath compilerSourceFiles, TreeNodeWithStringPath? appCodeTree, bool? overrideSkipLowering, + IReadOnlyList>? entryPointsFilePaths, + IPineVM pineVM) + : + this( + compilerSourceFiles: compilerSourceFiles, + appCodeTree: + appCodeTree is null ? null : AppCompilationUnits.WithoutPackages(appCodeTree), + overrideSkipLowering: overrideSkipLowering, + entryPointsFilePaths: entryPointsFilePaths, + pineVM) + { + } + + public InteractiveSessionPine( + TreeNodeWithStringPath compilerSourceFiles, + AppCompilationUnits? appCodeTree, + bool? overrideSkipLowering, + IReadOnlyList>? entryPointsFilePaths, IPineVM pineVM) : this( compilerSourceFiles: compilerSourceFiles, appCodeTree: appCodeTree, overrideSkipLowering: overrideSkipLowering, + entryPointsFilePaths: entryPointsFilePaths, (pineVM, pineVMCache: null)) { } - private InteractiveSessionPine( + public InteractiveSessionPine( TreeNodeWithStringPath compilerSourceFiles, - TreeNodeWithStringPath? appCodeTree, + AppCompilationUnits? appCodeTree, bool? overrideSkipLowering, + IReadOnlyList>? entryPointsFilePaths, (IPineVM pineVM, PineVMCache? pineVMCache) pineVMAndCache) { pineVM = pineVMAndCache.pineVM; @@ -93,7 +128,8 @@ private InteractiveSessionPine( System.Threading.Tasks.Task.Run(() => CompileInteractiveEnvironment( appCodeTree: appCodeTree, - overrideSkipLowering: overrideSkipLowering)); + overrideSkipLowering: overrideSkipLowering, + entryPointsFilePaths: entryPointsFilePaths)); } public static (IPineVM, PineVMCache?) BuildPineVM( @@ -120,13 +156,15 @@ public static IPineVM BuildPineVM( return new PineVM( evalCache: cache?.EvalCache, - evaluationConfigDefault: evaluationConfig); + evaluationConfigDefault: evaluationConfig, + reportFunctionApplication: null); } private Result CompileInteractiveEnvironment( - TreeNodeWithStringPath? appCodeTree, - bool? overrideSkipLowering) + AppCompilationUnits? appCodeTree, + bool? overrideSkipLowering, + IReadOnlyList>? entryPointsFilePaths) { if (buildCompilerResult.IsOkOrNull() is not { } elmCompiler) { @@ -143,43 +181,38 @@ private Result CompileInteractiveEnvironment( CompileInteractiveEnvironment( appCodeTree, overrideSkipLowering: overrideSkipLowering, + entryPointsFilePaths: entryPointsFilePaths, elmCompiler, compileEnvPineVM); } public static Result CompileInteractiveEnvironment( - TreeNodeWithStringPath? appCodeTree, + AppCompilationUnits? appCodeTree, bool? overrideSkipLowering, + IReadOnlyList>? entryPointsFilePaths, ElmCompiler elmCompiler) => CompileInteractiveEnvironment( appCodeTree, overrideSkipLowering: overrideSkipLowering, + entryPointsFilePaths: entryPointsFilePaths, elmCompiler, compileEnvPineVM); public static Result CompileInteractiveEnvironment( - TreeNodeWithStringPath? appCodeTree, + AppCompilationUnits? appCodeTree, bool? overrideSkipLowering, + IReadOnlyList>? entryPointsFilePaths, ElmCompiler elmCompiler, IPineVM pineVM) { var skipLowering = overrideSkipLowering ?? - !ElmCompiler.CheckIfAppUsesLowering(appCodeTree ?? TreeNodeWithStringPath.EmptyTree); - - var appCodeTreeWithCoreModules = - /* - ElmCompiler.MergeElmCoreModules(appCodeTree ?? TreeNodeWithStringPath.EmptyTree); - */ - appCodeTree ?? TreeNodeWithStringPath.EmptyTree; + !ElmCompiler.CheckIfAppUsesLowering(appCodeTree?.AppFiles ?? TreeNodeWithStringPath.EmptyTree); - var orderedModules = - AppSourceFileTreesForIncrementalCompilation( - appCodeTreeWithCoreModules, - skipLowering: skipLowering) - .ToImmutableArray(); + var appSourceFiles = + appCodeTree ?? AppCompilationUnits.WithoutPackages(TreeNodeWithStringPath.EmptyTree); var initialStateElmValue = ElmValueInterop.PineValueEncodedAsInElmCompiler(PineValue.EmptyList); @@ -189,6 +222,72 @@ public static Result PineValue compiledNewEnvInCompiler = initialStateElmValueInCompiler; + var onlyKernelModulesTree = ElmCompiler.ElmCoreAndKernelModuleFilesDefault.Value; + + var compileKernelModulesResult = + CompileInteractiveEnvironmentUnitEncodedInCompiler( + compiledNewEnvInCompiler, + onlyKernelModulesTree, + entryPointsFilePaths: null, + elmCompiler, + pineVM: pineVM); + + if (compileKernelModulesResult.IsErrOrNull() is { } compileKernelModulesErr) + { + return "Failed to compile kernel modules: " + compileKernelModulesErr; + } + + if (compileKernelModulesResult.IsOkOrNull() is not { } compiledNewEnvInCompilerUnit) + { + throw new NotImplementedException( + "Unexpected result type: " + compileKernelModulesResult.GetType()); + } + + compiledNewEnvInCompiler = compiledNewEnvInCompilerUnit; + + foreach (var package in appSourceFiles.Packages) + { + var packageName = package.elmJson.Name; + + if (packageName is "elm/core" || + packageName is "elm/json" || + packageName is "elm/bytes" || + packageName is "elm/parser" || + packageName is "elm/url") + { + continue; + } + + var compilePackageResult = + CompileInteractiveEnvironmentPackageEncodedInCompiler( + compiledNewEnvInCompiler, + package.files, + package.elmJson, + elmCompiler, + pineVM: pineVM); + + if (compilePackageResult.IsErrOrNull() is { } compilePackageErr) + { + return "Compiling package " + package.elmJson.Name + " failed: " + compilePackageErr; + } + + if (compilePackageResult.IsOkOrNull() is not { } compilePackageOk) + { + throw new NotImplementedException( + "Unexpected result type: " + compilePackageResult.GetType()); + } + + compiledNewEnvInCompiler = compilePackageOk; + } + + var orderedModules = + AppSourceFileTreesForIncrementalCompilation( + appSourceFiles.AppFiles, + skipLowering: skipLowering, + entryPointsFilePaths: + entryPointsFilePaths?.ToImmutableHashSet(EnumerableExtension.EqualityComparer>())) + .ToImmutableArray(); + foreach (var compilationIncrement in orderedModules) { var parseResult = @@ -199,7 +298,9 @@ public static Result if (parseResult.IsErrOrNull() is { } parseErr) { - return "Failed parsing module " + string.Join(".", compilationIncrement.sourceModule.ModuleName) + ": " + parseErr; + return + "Failed parsing module " + + string.Join(".", compilationIncrement.sourceModule.ModuleName) + ": " + parseErr; } if (parseResult.IsOkOrNullable() is not { } parsedModule) @@ -259,6 +360,143 @@ public static Result return compiledNewEnvValue; } + + public static Result + CompileInteractiveEnvironmentPackageEncodedInCompiler( + PineValue initialStateElmValueInCompiler, + TreeNodeWithStringPath packageFiles, + ElmJsonStructure elmJson, + ElmCompiler elmCompiler, + IPineVM pineVM) + { + IReadOnlyDictionary> fileNameFromModuleName = + packageFiles.EnumerateBlobsTransitive() + .Where(blob => blob.path.Last().EndsWith(".elm", StringComparison.OrdinalIgnoreCase)) + .Select(blob => + { + var moduleText = System.Text.Encoding.UTF8.GetString(blob.blobContent.Span); + var moduleName = + ElmSyntax.ElmModule.ParseModuleName(moduleText) + .Extract(err => throw new Exception("Failed parsing module name: " + err)); + + return new KeyValuePair>( + string.Join(".", moduleName), + blob.path); + }) + .ToImmutableDictionary(); + + var entryPointsFilePaths = + elmJson.ExposedModules + .Select((moduleNameFlat) => fileNameFromModuleName[moduleNameFlat]) + .ToImmutableHashSet(EnumerableExtension.EqualityComparer>()); + + var compilationUnitResult = + CompileInteractiveEnvironmentUnitEncodedInCompiler( + initialStateElmValueInCompiler, + packageFiles, + entryPointsFilePaths: entryPointsFilePaths, + elmCompiler, + pineVM); + + if (compilationUnitResult.IsErrOrNull() is { } compilationUnitErr) + { + return "Failed to compile package: " + compilationUnitErr; + } + + if (compilationUnitResult.IsOkOrNull() is not { } compiledNewEnvInCompilerUnit) + { + throw new NotImplementedException( + "Unexpected result type: " + compilationUnitResult.GetType()); + } + + // TODO: Filter non-exposed modules. + + return compiledNewEnvInCompilerUnit; + } + + + public static Result + CompileInteractiveEnvironmentUnitEncodedInCompiler( + PineValue initialStateElmValueInCompiler, + TreeNodeWithStringPath sourceFiles, + IReadOnlySet>? entryPointsFilePaths, + ElmCompiler elmCompiler, + IPineVM pineVM) + { + PineValue compiledNewEnvInCompiler = initialStateElmValueInCompiler; + + var modulesToCompile = + ElmInteractive.ModulesFilePathsAndTextsFromAppCodeTree( + sourceFiles, + skipLowering: true, + entryPointsFilePaths: entryPointsFilePaths, + skipFilteringForSourceDirs: true, + mergeKernelModules: false); + + var modulesToCompileTexts = + modulesToCompile + .Where(sm => !ElmInteractive.ShouldIgnoreSourceFile(sm.filePath, sm.fileContent)) + .Select(sm => sm.moduleText) + .ToImmutableArray(); + + var modulesTextsOrdered = + ElmSyntax.ElmModule.ModulesTextOrderedForCompilationByDependencies( + rootModulesTexts: modulesToCompileTexts, + availableModulesTexts: []); + + var modulesToCompileOrdered = + modulesTextsOrdered + .Select(mt => modulesToCompile.First(c => c.moduleText == mt)) + .ToImmutableArray(); + + foreach (var compilationIncrement in modulesToCompileOrdered) + { + var moduleName = + ElmSyntax.ElmModule.ParseModuleName(compilationIncrement.moduleText) + .Extract(err => throw new Exception("Failed parsing module name: " + err)); + + var parseResult = + CachedTryParseModuleText( + compilationIncrement.moduleText, + elmCompiler, + pineVM); + + if (parseResult.IsErrOrNull() is { } parseErr) + { + return + "Failed parsing module " + + string.Join(".", moduleName) + ": " + parseErr; + } + + if (parseResult.IsOkOrNullable() is not { } parsedModule) + { + throw new NotImplementedException( + "Unexpected parse result type: " + parseResult.GetType()); + } + + var parsedModuleNameFlat = string.Join(".", parsedModule.Key); + + var compileModuleResult = + CompileOneElmModule( + compiledNewEnvInCompiler, + compilationIncrement.moduleText, + parsedModule.Value, + elmCompiler, + pineVM: pineVM); + + if (compileModuleResult.IsOkOrNull() is not { } compileModuleOk) + { + return + "Compiling module " + parsedModuleNameFlat + " failed: " + + compileModuleResult.Unpack(fromErr: err => err, fromOk: _ => "no err"); + } + + compiledNewEnvInCompiler = compileModuleOk; + } + + return compiledNewEnvInCompiler; + } + public static TreeNodeWithStringPath MergeDefaultElmCoreAndKernelModules( TreeNodeWithStringPath appCodeTree) => MergeDefaultElmCoreAndKernelModules( @@ -336,12 +574,16 @@ public record ParsedModule( public static IEnumerable<(TreeNodeWithStringPath tree, ParsedModule sourceModule)> AppSourceFileTreesForIncrementalCompilation( TreeNodeWithStringPath appSourceFiles, - bool skipLowering) + bool skipLowering, + IReadOnlySet>? entryPointsFilePaths) { var compileableSourceModules = ElmInteractive.ModulesFilePathsAndTextsFromAppCodeTree( appSourceFiles, - skipLowering: skipLowering) ?? []; + skipLowering: skipLowering, + entryPointsFilePaths: entryPointsFilePaths, + skipFilteringForSourceDirs: false, + mergeKernelModules: false) ?? []; var baseTree = compileableSourceModules @@ -720,8 +962,9 @@ public static IReadOnlyDictionary } */ - /* - * Begin a dedicated training phase, compiling a small Elm module and then doing code-analysis - * and PGO for the entirety based on that simple scenario. - * */ + /* + * Begin a dedicated training phase, compiling a small Elm module and then doing code-analysis + * and PGO for the entirety based on that simple scenario. + * */ /* @@ -1399,8 +1403,8 @@ public static IEnumerable CompareCompiledEnvironmentsAndAssertEqual( actualEnvParsed.Modules.Single(m => m.moduleName == moduleName); foreach (var comparisonReport in CompareCompiledModules(expectedModule, actualModule)) - { - yield return comparisonReport; + { + yield return comparisonReport; } } diff --git a/implement/test-elm-time/ElmInteractiveTests.cs b/implement/test-elm-time/ElmInteractiveTests.cs index f9eb21bc..6cef07dd 100644 --- a/implement/test-elm-time/ElmInteractiveTests.cs +++ b/implement/test-elm-time/ElmInteractiveTests.cs @@ -69,6 +69,7 @@ static ElmTime.ElmInteractive.IInteractiveSession newInteractiveSessionFromAppCo compilerSourceFiles: CompileElmProgramCodeFiles, appCodeTree: appCodeTree, overrideSkipLowering: null, + entryPointsFilePaths: null, caching: true, autoPGO: null); @@ -154,6 +155,7 @@ ElmTime.ElmInteractive.IInteractiveSession newInteractiveSessionFromAppCode(Tree compilerSourceFiles: CompileElmProgramCodeFiles, appCodeTree: appCodeTree, overrideSkipLowering: true, + entryPointsFilePaths: null, dynamicPGOShare.GetVMAutoUpdating()); { diff --git a/implement/test-elm-time/ExampleAppsTests.cs b/implement/test-elm-time/ExampleAppsTests.cs index 7d2a720e..25889835 100644 --- a/implement/test-elm-time/ExampleAppsTests.cs +++ b/implement/test-elm-time/ExampleAppsTests.cs @@ -111,6 +111,60 @@ public void Example_app_minimal_webservice_hello_world_sandbox() "response content as string"); } + + [TestMethod] + public void Example_app_Elm_editor_webservice_sandbox() + { + var webAppSource = + ExampleAppValueFromExampleName("elm-editor"); + + var webAppSourceTree = + PineValueComposition.ParseAsTreeWithStringPath(webAppSource) + .Extract(err => throw new Exception("Failed parsing app source files as tree: " + err)); + + var webServiceConfig = + WebServiceInterface.ConfigFromSourceFilesAndEntryFileName( + webAppSourceTree, + ["src", "Backend", "Main.elm"]); + + var webServiceApp = + new MutatingWebServiceApp(webServiceConfig); + + var eventResponse = + webServiceApp.EventHttpRequest( + new WebServiceInterface.HttpRequestEventStruct + ( + HttpRequestId: "1", + PosixTimeMilli: 0, + RequestContext: new WebServiceInterface.HttpRequestContext + ( + ClientAddress: null + ), + Request: new WebServiceInterface.HttpRequestProperties + ( + Method: "GET", + Uri: "/", + BodyAsBase64: null, + Headers: [] + ) + )); + + var allCommands = + webServiceApp.DequeueCommands(); + + var commandCreateVolatileProcess = + allCommands + .OfType() + .FirstOrDefault(); + + if (commandCreateVolatileProcess is null) + { + throw new Exception( + "Did not find expected command to create a volatile process among " + + allCommands.Count + " commands."); + } + } + [TestMethod] public void Example_app_demo_backend_state() { diff --git a/implement/test-elm-time/PGOTests.cs b/implement/test-elm-time/PGOTests.cs index db291306..af212080 100644 --- a/implement/test-elm-time/PGOTests.cs +++ b/implement/test-elm-time/PGOTests.cs @@ -45,6 +45,7 @@ module Test exposing (..) ElmCompiler.CompilerSourceFilesDefault.Value, appCodeTree: appCodeTree, overrideSkipLowering: true, + entryPointsFilePaths: null, caching: true, autoPGO: null); @@ -431,6 +432,7 @@ module Test exposing (..) ElmCompiler.CompilerSourceFilesDefault.Value, appCodeTree: appCodeTree, overrideSkipLowering: true, + entryPointsFilePaths: null, caching: true, autoPGO: null); @@ -1054,6 +1056,7 @@ listMapHelp f xs (Pine_kernel.concat [ [ f x ], acc ]) ElmCompiler.CompilerSourceFilesDefault.Value, appCodeTree: appCodeTree, overrideSkipLowering: true, + entryPointsFilePaths: null, caching: true, autoPGO: null); @@ -1471,6 +1474,7 @@ import Dict ElmCompiler.CompilerSourceFilesDefault.Value, appCodeTree: appCodeTree, overrideSkipLowering: true, + entryPointsFilePaths: null, caching: true, autoPGO: null);