Skip to content

Commit

Permalink
Expand Elm app build to model packages as separate compilation units
Browse files Browse the repository at this point in the history 
The previous approach of aggregating all modules from all packages and the app into a single set failed when multiple packages used unexposed modules with the same name.
  • Loading branch information
@Viir
Viir committed Jan 31, 2025
1 parent fa68370 commit 77778ed
Show file tree
Hide file tree
Showing 14 changed files with 829 additions and 147 deletions.
9 changes: 8 additions & 1 deletion implement/Pine.Core/TreeNodeWithStringPath.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ public static TreeNodeWithStringPath NonSortedTree(IReadOnlyList<(string name, T

public TreeNodeWithStringPath SetNodeAtPathSorted(IReadOnlyList<string> path, TreeNodeWithStringPath node)
{
if (path.Count == 0)
if (path.Count is 0)
return node;

var pathFirstElement = path[0];
Expand All @@ -174,6 +174,13 @@ public TreeNodeWithStringPath SetNodeAtPathSorted(IReadOnlyList<string> path, Tr
return SortedTree(treeEntries);
}

public static TreeNodeWithStringPath MergeBlobs(
TreeNodeWithStringPath left,
TreeNodeWithStringPath right) =>
right.EnumerateBlobsTransitive()
.Aggregate(left, (acc, blob) => acc.SetNodeAtPathSorted(blob.path, Blob(blob.blobContent)));


public static TreeNodeWithStringPath FilterNodesByPath(
TreeNodeWithStringPath node,
Func<IReadOnlyList<string>, bool> pathFilter,
Expand Down
265 changes: 199 additions & 66 deletions implement/pine/Elm/ElmAppDependencyResolution.cs
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
using ElmTime.ElmSyntax;
using Pine.Core;
using Pine.Elm019;
using System;
Expand All @@ -7,9 +8,177 @@

namespace Pine.Elm;

public record AppCompilationUnits(
TreeNodeWithStringPath AppFiles,
IReadOnlyList<(TreeNodeWithStringPath files, ElmJsonStructure elmJson)> Packages)
{
public static AppCompilationUnits WithoutPackages(
TreeNodeWithStringPath appCode)
{
return new AppCompilationUnits(
appCode,
Packages: []);
}
}

public class ElmAppDependencyResolution
{
public static IReadOnlyDictionary<IReadOnlyList<string>, ReadOnlyMemory<byte>> MergePackagesElmModules(
public static (AppCompilationUnits files, IReadOnlyList<string> entryModuleName)
AppCompilationUnitsForEntryPoint(
TreeNodeWithStringPath sourceFiles,
IReadOnlyList<string> entryPointFilePath)
{
if (sourceFiles.GetNodeAtPath(entryPointFilePath) is not { } entryFileNode)
{
throw new Exception("Entry file not found: " + string.Join("/", entryPointFilePath));
}

if (entryFileNode is not TreeNodeWithStringPath.BlobNode entryFileBlob)
{
throw new Exception(
"Entry file is not a blob: " + string.Join("/", entryPointFilePath));
}

var entryFileText =
System.Text.Encoding.UTF8.GetString(entryFileBlob.Bytes.Span);

if (ElmModule.ParseModuleName(entryFileText).IsOkOrNull() is not { } moduleName)
{
throw new Exception(
"Failed to parse module name from entry file: " + string.Join("/", entryPointFilePath));
}

var sourceFilesDict =
PineValueComposition.TreeToFlatDictionaryWithPathComparer(sourceFiles);

var sourceFilesFiltered =
ElmCompiler.FilterTreeForCompilationRoots(
sourceFiles,
ImmutableHashSet.Create(
EnumerableExtension.EqualityComparer<IReadOnlyList<string>>(),
entryPointFilePath),
skipFilteringForSourceDirs: false);

IReadOnlyList<KeyValuePair<IReadOnlyList<string>, IReadOnlyList<IReadOnlyList<string>>>>
remainingElmModulesNameAndImports =
sourceFilesFiltered
.EnumerateBlobsTransitive()
.Where(blob => blob.path.Last().EndsWith(".elm", StringComparison.OrdinalIgnoreCase))
.Select(blob =>
{
var moduleText = System.Text.Encoding.UTF8.GetString(blob.blobContent.Span);

if (ElmModule.ParseModuleName(moduleText).IsOkOrNull() is not { } moduleName)
{
throw new Exception("Failed to parse module name from file: " + string.Join("/", blob.path));
}

return new KeyValuePair<IReadOnlyList<string>, IReadOnlyList<IReadOnlyList<string>>>(
moduleName,
[.. ElmModule.ParseModuleImportedModulesNames(moduleText)]);
})
.ToImmutableList();

var remainingElmModulesImports =
remainingElmModulesNameAndImports
.SelectMany(kv => kv.Value)
.Where(importedModuleName => !remainingElmModulesNameAndImports.Any(kv => kv.Key.SequenceEqual(importedModuleName)))
.ToImmutableHashSet(EnumerableExtension.EqualityComparer<IReadOnlyList<string>>());

var packages = LoadPackagesForElmApp(sourceFilesDict);

/*
* We filter packages to include only those needed for the current compilation entry point.
*
* Referencing two packages that expose modules with the same name is
* no problem as long as the app does not reference that module name.
* */

IReadOnlySet<string> aggregateExposedModuleNames =
packages
.SelectMany(package => package.Value.elmJson.ExposedModules)
.ToImmutableHashSet();

IReadOnlyDictionary<string, IReadOnlySet<string>>
packagesFromExposedModuleName =
aggregateExposedModuleNames
.Select(moduleName =>
{
return
new KeyValuePair<string, IReadOnlySet<string>>(
moduleName,
packages
.Where(package => package.Value.elmJson.ExposedModules.Contains(moduleName))
.Select(package => package.Key)
.ToImmutableHashSet());
})
.ToImmutableDictionary();

var packagesToIncludeNames =
remainingElmModulesImports
.Select(importedModuleName =>
{
var importedModuleNameFlat = string.Join(".", importedModuleName);

if (!packagesFromExposedModuleName.TryGetValue(importedModuleNameFlat, out var packages))
{
throw new Exception("Failed to find package for imported module: " + importedModuleNameFlat);
}

if (packages.Count is not 1)
{
throw new Exception(
"Imported module " + importedModuleNameFlat +
" is exposed by multiple packages: " + string.Join(", ", packages));
}

return packages.First();
})
.ToImmutableHashSet();

IEnumerable<string> enumeratePackageDependenciesTransitive(string packageName)
{
if (!packages.TryGetValue(packageName, out var package))
{
yield break;
}

var dependencies =
package.elmJson.Dependencies.Direct.EmptyIfNull()
.Concat(package.elmJson.Dependencies.Indirect.EmptyIfNull())
.Concat(package.elmJson.Dependencies.Flat.EmptyIfNull());

foreach (var dependency in dependencies)
{
yield return dependency.Key;

foreach (var transitiveDependency in enumeratePackageDependenciesTransitive(dependency.Key))
{
yield return transitiveDependency;
}
}
}

var packagesToInclude =
packages
.Where(kv => packagesToIncludeNames.Contains(kv.Key))
.ToImmutableDictionary();

var packagesOrdered =
packagesToInclude
.OrderBy(kv => enumeratePackageDependenciesTransitive(kv.Key).Count())
.ThenBy(kv => kv.Key)
.ToImmutableList();

return
(new AppCompilationUnits(
sourceFilesFiltered,
Packages: [.. packagesOrdered.Select(pkg => pkg.Value)]),
moduleName);
}

public static IReadOnlyDictionary<string, (TreeNodeWithStringPath files, ElmJsonStructure elmJson)>
LoadPackagesForElmApp(
IReadOnlyDictionary<IReadOnlyList<string>, ReadOnlyMemory<byte>> appSourceFiles)
{
/*
Expand All @@ -21,7 +190,7 @@ public static IReadOnlyDictionary<IReadOnlyList<string>, ReadOnlyMemory<byte>> M
.Where(entry => entry.Key.Last() is "elm.json")
.ToImmutableDictionary();

var elmJsonAggregateDependencies =
var elmJsonAggregateDependenciesVersions =
elmJsonFiles
.SelectMany(elmJsonFile =>
{
Expand All @@ -33,9 +202,9 @@ public static IReadOnlyDictionary<IReadOnlyList<string>, ReadOnlyMemory<byte>> M
return
new[]
{
elmJsonParsed?.Dependencies.Direct,
elmJsonParsed?.Dependencies.Indirect,
elmJsonParsed?.Dependencies.Flat
elmJsonParsed?.Dependencies.Direct,
elmJsonParsed?.Dependencies.Indirect,
elmJsonParsed?.Dependencies.Flat
}
.WhereNotNull();
}
Expand All @@ -47,6 +216,10 @@ public static IReadOnlyDictionary<IReadOnlyList<string>, ReadOnlyMemory<byte>> M
}
})
.SelectMany(dependency => dependency)
.ToImmutableDictionary();

var elmJsonAggregateDependencies =
elmJsonAggregateDependenciesVersions
.ToImmutableDictionary(
keySelector: dependency => dependency.Key,
elementSelector:
Expand All @@ -58,73 +231,33 @@ public static IReadOnlyDictionary<IReadOnlyList<string>, ReadOnlyMemory<byte>> M
return packageFiles;
});

var sourceElmModulesNames =
appSourceFiles
.Where(entry => entry.Key.Last().EndsWith(".elm", StringComparison.OrdinalIgnoreCase))
.Select(entry => ElmTime.ElmSyntax.ElmModule.ParseModuleName(entry.Value).WithDefault(null))
.WhereNotNull()
.ToImmutableHashSet(EnumerableExtension.EqualityComparer<IReadOnlyList<string>>());

var packagesModulesNames =
new HashSet<IReadOnlyList<string>>(EnumerableExtension.EqualityComparer<IReadOnlyList<string>>());

var sourceFilesWithMergedPackages =
return
elmJsonAggregateDependencies
.Aggregate(
seed: appSourceFiles,
func: (aggregate, package) =>
.ToImmutableDictionary(
keySelector:
kv => kv.Key,
elementSelector:
kv =>
{
if (package.Key is "elm/core" ||
package.Key is "elm/json" ||
package.Key is "elm/bytes" ||
package.Key is "elm/parser" ||
package.Key is "elm/url")
try
{
return aggregate;
}

var packageExposedModuleFiles =
ElmPackage.ExposedModules(package.Value);

return
packageExposedModuleFiles
.Aggregate(
seed: aggregate.ToImmutableDictionary(),
func: (innerAggregate, packageElmModuleFile) =>
if (!kv.Value.TryGetValue(["elm.json"], out var elmJsonFile))
{
var relativePath = packageElmModuleFile.Key;
throw new Exception("Did not find elm.json file");
}

var moduleName = packageElmModuleFile.Value.moduleName;
var elmJsonParsed =
System.Text.Json.JsonSerializer.Deserialize<ElmJsonStructure>(elmJsonFile.Span)
??
throw new Exception("Parsing elm.json returned null");

if (sourceElmModulesNames.Contains(moduleName))
{
Console.WriteLine(
"Skipping Elm module file " +
string.Join("/", relativePath) +
" from package " + package.Key + " because it is already present in the source files.");

return innerAggregate;
}

if (packagesModulesNames.Contains(moduleName))
{
Console.WriteLine(
"Skipping Elm module file " +
string.Join("/", relativePath) +
" from package " + package.Key + " because it is already present in the packages.");

return innerAggregate;
}

packagesModulesNames.Add(moduleName);

return
innerAggregate.SetItem(
["dependencies", .. package.Key.Split('/'), .. packageElmModuleFile.Key],
packageElmModuleFile.Value.fileContent);
});
return (PineValueComposition.SortedTreeFromSetOfBlobsWithStringPath(kv.Value), elmJsonParsed);
}
catch (Exception e)
{
throw new Exception(
"Failed to load package: " + kv.Key + ": " + e.Message, e);
}
});

return sourceFilesWithMergedPackages;
}
}
Loading

0 comments on commit 77778ed

Please sign in to comment.