Migrate blobMain entry point to new make build infrastructure

+ Add the first version of loading Elm packages from GitHub and caching them locally.

implement/pine/Elm/ElmPackageSource.cs

@@ -0,0 +1,173 @@
+using Pine.Core;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.IO.Compression;
+using System.Net.Http;
+using System.Threading.Tasks;
+
+namespace Pine.Elm;
+
+/// <summary>
+/// Provides functionality to load Elm packages by either retrieving a locally cached ZIP file
+/// or downloading it from GitHub. The loaded package contents are returned as an in-memory
+/// dictionary mapping path segments to file contents. If no valid local cache is found,
+/// the package is fetched from GitHub and saved to the first cache directory for future use.
+///
+/// Usage:
+/// <code>
+/// var packageFiles = await ElmPackageSource.LoadElmPackageAsync("agu-z/elm-zip", "3.0.1");
+/// // Access file contents by path segments from the returned dictionary
+/// </code>
+/// </summary>
+public class ElmPackageSource
+{
+    /// <summary>
+    /// Default local cache directories (example: ~/.cache/elm-package).
+    /// You can add more directories if you want a search path.
+    /// </summary>
+    public static IReadOnlyList<string> LocalCacheDirectoriesDefault =>
+        [
+            Path.Combine(Filesystem.CacheDirectory, "elm-package"),
+        ];
+
+    /// <summary>
+    /// Public entry point that uses the default cache directories.
+    /// </summary>
+    public static async Task<IReadOnlyDictionary<IReadOnlyList<string>, ReadOnlyMemory<byte>>> LoadElmPackageAsync(
+        string packageName,
+        string versionId)
+    {
+        return await LoadElmPackageAsync(packageName, versionId, LocalCacheDirectoriesDefault);
+    }
+
+    /// <summary>
+    /// Tries to load from any of the given local cache directories; if not found, downloads from GitHub and saves to the first cache dir.
+    /// </summary>
+    public static async Task<IReadOnlyDictionary<IReadOnlyList<string>, ReadOnlyMemory<byte>>> LoadElmPackageAsync(
+        string packageName,
+        string versionId,
+        IReadOnlyList<string> localCacheDirectories)
+    {
+        // 1) Try to load from each cache directory, in order:
+        foreach (var cacheDirectory in localCacheDirectories)
+        {
+            // Construct the path for the cached file, for example "[email protected]".
+            var localZipPath = GetLocalZipPath(cacheDirectory, packageName, versionId);
+
+            if (File.Exists(localZipPath))
+            {
+                try
+                {
+                    // Attempt to load and parse the ZIP from disk.
+                    var data = await File.ReadAllBytesAsync(localZipPath);
+
+                    var fromCache = LoadElmPackageFromZipBytes(data);
+
+                    return fromCache;
+                }
+                catch
+                {
+                    // If anything failed (e.g. file is corrupt), ignore and keep going.
+                }
+            }
+        }
+
+        // 2) No valid cache found — download from GitHub:
+        var zipData = await DownloadPackageZipAsync(packageName, versionId);
+
+        var fromGitHub = LoadElmPackageFromZipBytes(zipData);
+
+        // 3) Write the ZIP to the *first* cache directory, if present (and possible).
+        if (localCacheDirectories.Count > 0)
+        {
+            var firstCache = localCacheDirectories[0];
+
+            try
+            {
+                Directory.CreateDirectory(firstCache); // Ensure the directory exists.
+
+                var localZipPath = GetLocalZipPath(firstCache, packageName, versionId);
+
+                await File.WriteAllBytesAsync(localZipPath, zipData);
+            }
+            catch
+            {
+                // Caching is non-critical. Swallow any exception here (e.g. no write permission).
+            }
+        }
+
+        return fromGitHub;
+    }
+
+    /// <summary>
+    /// Given the raw bytes of a ZIP file, extract all files and return them as a dictionary
+    /// from path segments to the file contents.
+    /// </summary>
+    private static IReadOnlyDictionary<IReadOnlyList<string>, ReadOnlyMemory<byte>> LoadElmPackageFromZipBytes(
+        byte[] zipBytes)
+    {
+        var result = new Dictionary<IReadOnlyList<string>, ReadOnlyMemory<byte>>(
+            EnumerableExtension.EqualityComparer<IReadOnlyList<string>>());
+
+        using var memoryStream = new MemoryStream(zipBytes);
+
+        using var archive = new System.IO.Compression.ZipArchive(memoryStream, ZipArchiveMode.Read);
+
+        foreach (var entry in archive.Entries)
+        {
+            // If it's a directory entry, skip
+            if (string.IsNullOrEmpty(entry.Name))
+                continue;
+
+            using var entryStream = entry.Open();
+            using var entryMemoryStream = new MemoryStream();
+            entryStream.CopyTo(entryMemoryStream);
+
+            var fileContents = entryMemoryStream.ToArray();
+
+            // Split the full name into segments
+            var pathSegments = entry.FullName.Split(new[] { '/' }, StringSplitOptions.RemoveEmptyEntries);
+
+            result[pathSegments] = fileContents;
+        }
+
+        return result;
+    }
+
+
+    /// <summary>
+    /// Downloads the ZIP from GitHub as a byte array.
+    /// </summary>
+    private static async Task<byte[]> DownloadPackageZipAsync(string packageName, string versionId)
+    {
+        // Example: "https://github.com/agu-z/elm-zip/archive/refs/tags/3.0.1.zip"
+
+        var downloadUrl = "https://github.com/" + packageName.Trim('/') + "/archive/refs/tags/" + versionId + ".zip";
+
+        using var httpClient = new HttpClient();
+
+        using var zipStream = await httpClient.GetStreamAsync(downloadUrl);
+
+        // Copy to memory
+        using var memoryStream = new MemoryStream();
+
+        await zipStream.CopyToAsync(memoryStream);
+
+        return memoryStream.ToArray();
+    }
+
+    /// <summary>
+    /// Constructs a local ZIP file path: e.g. "[email protected]" in the given cache directory.
+    /// Replaces slashes so the filename is filesystem-safe.
+    /// </summary>
+    private static string GetLocalZipPath(string cacheDirectory, string packageName, string versionId)
+    {
+        var safePkgName = packageName.Replace('/', '-');
+
+        // Example: "[email protected]"
+        var fileName = $"{safePkgName}@{versionId}.zip";
+
+        return Path.Combine(cacheDirectory, fileName);
+    }
+}

implement/pine/Elm/elm-compiler/src/CompileElmApp.elm

@@ -703,53 +703,18 @@ loweredForBlobEntryPoint :
     -> AppFiles
     -> Result (List (LocatedInSourceFiles CompilationError)) ( AppFiles, ElmMakeEntryPointStruct )
 loweredForBlobEntryPoint { compilationRootFilePath, compilationRootModule } sourceFiles =
-    ([ compilationRootModule.fileText
-     , String.trim """
-blob_main_as_base64 : String
-blob_main_as_base64 =
-    blobMain
-        |> Base64.fromBytes
-        |> Maybe.withDefault "Failed to encode as Base64"
-
-
-{-| Support function-level dead code elimination (<https://elm-lang.org/blog/small-assets-without-the-headache>) Elm code needed to inform the Elm compiler about our entry points.
--}
-main : Program Int {} String
-main =
-    Platform.worker
-        { init = always ( {}, Cmd.none )
-        , update = always (always ( blob_main_as_base64 |> always {}, Cmd.none ))
-        , subscriptions = always Sub.none
-        }
-"""
-     ]
-        |> String.join "\n\n"
-        |> addImportsInElmModuleText [ ( [ "Base64" ], Nothing ) ]
-        |> Result.mapError
-            (\err ->
-                [ LocatedInSourceFiles
-                    { filePath = compilationRootFilePath
-                    , locationInModuleText = Elm.Syntax.Range.emptyRange
-                    }
-                    (OtherCompilationError ("Failed to add import: " ++ err))
-                ]
-            )
-    )
-        |> Result.map
-            (\rootModuleText ->
-                ( sourceFiles
-                    |> updateFileContentAtPath (always (fileContentFromString rootModuleText)) compilationRootFilePath
-                , { elmMakeJavaScriptFunctionName =
-                        ((compilationRootModule.parsedSyntax.moduleDefinition
-                            |> Elm.Syntax.Node.value
-                            |> Elm.Syntax.Module.moduleName
-                         )
-                            ++ [ "blob_main_as_base64" ]
-                        )
-                            |> String.join "."
-                  }
+    Ok
+        ( sourceFiles
+        , { elmMakeJavaScriptFunctionName =
+                ((compilationRootModule.parsedSyntax.moduleDefinition
+                    |> Elm.Syntax.Node.value
+                    |> Elm.Syntax.Module.moduleName
+                 )
+                    ++ [ "blobMain" ]
                 )
-            )
+                    |> String.join "."
+          }
+        )
 
 
 sourceFileFunctionNameStart : String

implement/pine/Elm019/ElmPackage.cs

@@ -0,0 +1,110 @@
+using Pine.Core;
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Linq;
+using System.Text;
+using System.Text.Json;
+
+namespace Pine.Elm019;
+
+public class ElmPackage
+{
+    record ParsedModule(
+        IReadOnlyList<string> ModuleName,
+        IImmutableSet<IReadOnlyList<string>> ImportedModulesNames);
+
+    public static IReadOnlyDictionary<IReadOnlyList<string>, (ReadOnlyMemory<byte> fileContent, string moduleText, IReadOnlyList<string> moduleName)>
+        ExposedModules(
+        IReadOnlyDictionary<IReadOnlyList<string>, ReadOnlyMemory<byte>> packageSourceFiles)
+    {
+        Dictionary<IReadOnlyList<string>, (ReadOnlyMemory<byte> fileContent, string moduleText, IReadOnlyList<string> moduleName)> exposedModules =
+            new(EnumerableExtension.EqualityComparer<IReadOnlyList<string>>());
+
+        var elmJsonFile =
+            /*
+             * If package files come from GitHub zip archive, then elm.json file might not be at the root but in
+             * a directory named after the repository, e.g. "elm-flate-2.0.5".
+             * */
+            packageSourceFiles
+            .OrderBy(kv => kv.Key.Count)
+            .FirstOrDefault(kv => kv.Key.Last() is "elm.json");
+
+        if (elmJsonFile.Key is null)
+        {
+            return exposedModules;
+        }
+
+        var elmJson = JsonSerializer.Deserialize<ElmJsonStructure>(elmJsonFile.Value.Span);
+
+        Dictionary<IReadOnlyList<string>, (IReadOnlyList<string> filePath, ReadOnlyMemory<byte> fileContent, string moduleText, ParsedModule parsedModule)> parsedModulesByName =
+            new(EnumerableExtension.EqualityComparer<IReadOnlyList<string>>());
+
+        foreach (var (filePath, fileContent) in packageSourceFiles)
+        {
+            try
+            {
+                var moduleText = Encoding.UTF8.GetString(fileContent.Span);
+
+                if (ElmTime.ElmSyntax.ElmModule.ParseModuleName(moduleText).IsOkOrNull() is not { } moduleName)
+                {
+                    continue;
+                }
+
+                parsedModulesByName[moduleName] =
+                    (filePath,
+                    fileContent,
+                    moduleText,
+                    new ParsedModule(
+                        ModuleName: moduleName,
+                        ImportedModulesNames: [.. ElmTime.ElmSyntax.ElmModule.ParseModuleImportedModulesNames(moduleText)]));
+            }
+            catch (Exception)
+            {
+            }
+        }
+
+        IReadOnlySet<IReadOnlyList<string>> ListImportsOfModuleTransitive(IReadOnlyList<string> moduleName)
+        {
+            var queue =
+                new Queue<IReadOnlyList<string>>([moduleName]);
+
+            var set =
+                new HashSet<IReadOnlyList<string>>(EnumerableExtension.EqualityComparer<IReadOnlyList<string>>());
+
+            while (queue.TryDequeue(out var currentModuleName))
+            {
+                if (set.Add(currentModuleName) &&
+                    parsedModulesByName.TryGetValue(currentModuleName, out var currentModule))
+                {
+                    foreach (var importedModuleName in currentModule.parsedModule.ImportedModulesNames)
+                    {
+                        queue.Enqueue(importedModuleName);
+                    }
+                }
+            }
+
+            return set;
+        }
+
+        foreach (var exposedModuleNameFlat in elmJson.ExposedModules)
+        {
+            var exposedModuleName = exposedModuleNameFlat.Split('.').ToImmutableList();
+
+            var exposedNameIncludingDependencies =
+                ListImportsOfModuleTransitive(exposedModuleName)
+                .Prepend(exposedModuleName);
+
+            foreach (var moduleName in exposedNameIncludingDependencies)
+            {
+                if (parsedModulesByName.TryGetValue(moduleName, out var module))
+                {
+                    exposedModules[module.filePath] =
+                        (module.fileContent, module.moduleText, module.parsedModule.ModuleName);
+                }
+            }
+        }
+
+        return exposedModules;
+    }
+}

implement/pine/ElmSyntax/ElmModule.cs

@@ -146,6 +146,21 @@ public class DelegateComparer<T>(Func<T?, T?, int> func) : IComparer<T>
         public int Compare(T? x, T? y) => func(x, y);
     }
 
+    public static Result<string, IReadOnlyList<string>> ParseModuleName(ReadOnlyMemory<byte> moduleContent)
+    {
+        try
+        {
+            var moduleText =
+                System.Text.Encoding.UTF8.GetString(moduleContent.Span);
+
+            return ParseModuleName(moduleText);
+        }
+        catch (Exception exception)
+        {
+            return "Failed decoding text: " + exception.Message;
+        }
+    }
+
     public static Result<string, IReadOnlyList<string>> ParseModuleName(string moduleText)
     {
         {