v0.8.6+luau622

Author: khvzak

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "luau0-src"
-version = "0.8.5+luau617"
+version = "0.8.6+luau622"
 authors = ["Aleksandr Orlenko <[email protected]>"]
 edition = "2021"
 repository = "https://github.com/khvzak/luau-src-rs"

luau/CodeGen/include/Luau/CodeGen.h

@@ -2,6 +2,8 @@
 #pragma once
 
 #include <algorithm>
+#include <array>
+#include <memory>
 #include <string>
 #include <vector>
 
@@ -40,6 +42,26 @@ enum class CodeGenCompilationResult
     AllocationFailed = 9,                     // Native codegen failed due to an allocation error
 };
 
+struct ProtoCompilationFailure
+{
+    CodeGenCompilationResult result = CodeGenCompilationResult::Success;
+
+    std::string debugname;
+    int line = -1;
+};
+
+struct CompilationResult
+{
+    CodeGenCompilationResult result = CodeGenCompilationResult::Success;
+
+    std::vector<ProtoCompilationFailure> protoFailures;
+
+    [[nodiscard]] bool hasErrors() const
+    {
+        return result != CodeGenCompilationResult::Success || !protoFailures.empty();
+    }
+};
+
 struct CompilationStats
 {
     size_t bytecodeSizeBytes = 0;
@@ -49,23 +71,52 @@ struct CompilationStats
 
     uint32_t functionsTotal = 0;
     uint32_t functionsCompiled = 0;
+    uint32_t functionsBound = 0;
 };
 
 using AllocationCallback = void(void* context, void* oldPointer, size_t oldSize, void* newPointer, size_t newSize);
 
 bool isSupported();
 
+class SharedCodeGenContext;
+
+struct SharedCodeGenContextDeleter
+{
+    void operator()(const SharedCodeGenContext* context) const noexcept;
+};
+
+using UniqueSharedCodeGenContext = std::unique_ptr<SharedCodeGenContext, SharedCodeGenContextDeleter>;
+
+// Creates a new SharedCodeGenContext that can be used by multiple Luau VMs
+// concurrently, using either the default allocator parameters or custom
+// allocator parameters.
+[[nodiscard]] UniqueSharedCodeGenContext createSharedCodeGenContext();
+
+[[nodiscard]] UniqueSharedCodeGenContext createSharedCodeGenContext(AllocationCallback* allocationCallback, void* allocationCallbackContext);
+
+[[nodiscard]] UniqueSharedCodeGenContext createSharedCodeGenContext(
+    size_t blockSize, size_t maxTotalSize, AllocationCallback* allocationCallback, void* allocationCallbackContext);
+
+// Destroys the provided SharedCodeGenContext.  All Luau VMs using the
+// SharedCodeGenContext must be destroyed before this function is called.
+void destroySharedCodeGenContext(const SharedCodeGenContext* codeGenContext) noexcept;
+
 void create(lua_State* L, AllocationCallback* allocationCallback, void* allocationCallbackContext);
 void create(lua_State* L);
+void create(lua_State* L, SharedCodeGenContext* codeGenContext);
 
 // Check if native execution is enabled
 [[nodiscard]] bool isNativeExecutionEnabled(lua_State* L);
 
 // Enable or disable native execution according to `enabled` argument
 void setNativeExecutionEnabled(lua_State* L, bool enabled);
 
+using ModuleId = std::array<uint8_t, 16>;
+
 // Builds target function and all inner functions
-CodeGenCompilationResult compile(lua_State* L, int idx, unsigned int flags = 0, CompilationStats* stats = nullptr);
+CodeGenCompilationResult compile_DEPRECATED(lua_State* L, int idx, unsigned int flags = 0, CompilationStats* stats = nullptr);
+CompilationResult compile(lua_State* L, int idx, unsigned int flags = 0, CompilationStats* stats = nullptr);
+CompilationResult compile(const ModuleId& moduleId, lua_State* L, int idx, unsigned int flags = 0, CompilationStats* stats = nullptr);
 
 using AnnotatorFn = void (*)(void* context, std::string& result, int fid, int instpos);
 

luau/CodeGen/include/Luau/IrVisitUseDef.h

@@ -4,7 +4,7 @@
 #include "Luau/Common.h"
 #include "Luau/IrData.h"
 
-LUAU_FASTFLAG(LuauCodegenRemoveDeadStores3)
+LUAU_FASTFLAG(LuauCodegenRemoveDeadStores5)
 
 namespace Luau
 {
@@ -188,7 +188,7 @@ static void visitVmRegDefsUses(T& visitor, IrFunction& function, const IrInst& i
         visitor.def(inst.b);
         break;
     case IrCmd::FALLBACK_FORGPREP:
-        if (FFlag::LuauCodegenRemoveDeadStores3)
+        if (FFlag::LuauCodegenRemoveDeadStores5)
         {
             // This instruction doesn't always redefine Rn, Rn+1, Rn+2, so we have to mark it as implicit use
             visitor.useRange(vmRegOp(inst.b), 3);
@@ -216,7 +216,8 @@ static void visitVmRegDefsUses(T& visitor, IrFunction& function, const IrInst& i
 
         // After optimizations with DebugLuauAbortingChecks enabled, CHECK_TAG Rn, tag, block instructions are generated
     case IrCmd::CHECK_TAG:
-        visitor.maybeUse(inst.a);
+        if (!FFlag::LuauCodegenRemoveDeadStores5)
+            visitor.maybeUse(inst.a);
         break;
 
     default:

luau/CodeGen/include/Luau/NativeProtoExecData.h

@@ -0,0 +1,57 @@
+// This file is part of the Luau programming language and is licensed under MIT License; see LICENSE.txt for details
+#pragma once
+
+#include <memory>
+#include <stdint.h>
+
+namespace Luau
+{
+namespace CodeGen
+{
+
+// The NativeProtoExecData is constant metadata associated with a NativeProto.
+// We generally refer to the NativeProtoExecData via a pointer to the instruction
+// offsets array because this makes the logic in the entry gate simpler.
+
+class NativeModule;
+
+struct NativeProtoExecDataHeader
+{
+    // The NativeModule that owns this NativeProto.  This is initialized
+    // when the NativeProto is bound to the NativeModule via assignToModule().
+    NativeModule* nativeModule = nullptr;
+
+    // We store the native code offset until the code is allocated in executable
+    // pages, after which point we store the actual address.
+    const uint8_t* entryOffsetOrAddress = nullptr;
+
+    // The bytecode id of the proto
+    uint32_t bytecodeId = 0;
+
+    // The number of bytecode instructions in the proto.  This is the number of
+    // elements in the instruction offsets array following this header.
+    uint32_t bytecodeInstructionCount = 0;
+
+    // The size of the native code for this NativeProto, in bytes.
+    size_t nativeCodeSize = 0;
+};
+
+// Make sure that the instruction offsets array following the header will be
+// correctly aligned:
+static_assert(sizeof(NativeProtoExecDataHeader) % sizeof(uint32_t) == 0);
+
+struct NativeProtoExecDataDeleter
+{
+    void operator()(const uint32_t* instructionOffsets) const noexcept;
+};
+
+using NativeProtoExecDataPtr = std::unique_ptr<uint32_t[], NativeProtoExecDataDeleter>;
+
+[[nodiscard]] NativeProtoExecDataPtr createNativeProtoExecData(uint32_t bytecodeInstructionCount);
+void destroyNativeProtoExecData(const uint32_t* instructionOffsets) noexcept;
+
+[[nodiscard]] NativeProtoExecDataHeader& getNativeProtoExecDataHeader(uint32_t* instructionOffsets) noexcept;
+[[nodiscard]] const NativeProtoExecDataHeader& getNativeProtoExecDataHeader(const uint32_t* instructionOffsets) noexcept;
+
+} // namespace CodeGen
+} // namespace Luau

luau/CodeGen/include/Luau/SharedCodeAllocator.h

@@ -0,0 +1,165 @@
+// This file is part of the Luau programming language and is licensed under MIT License; see LICENSE.txt for details
+#pragma once
+
+#include "Luau/CodeGen.h"
+#include "Luau/Common.h"
+#include "Luau/NativeProtoExecData.h"
+
+#include <array>
+#include <atomic>
+#include <memory>
+#include <mutex>
+#include <optional>
+#include <stdint.h>
+#include <unordered_map>
+#include <vector>
+
+namespace Luau
+{
+namespace CodeGen
+{
+
+// SharedCodeAllocator is a native executable code allocator that provides
+// shared ownership of the native code.  Code is allocated on a per-module
+// basis.  Each module is uniquely identifiable via an id, which may be a hash
+// or other unique value.  Each module may contain multiple natively compiled
+// functions (protos).
+//
+// The module is the unit of shared ownership (i.e., it is where the reference
+// count is maintained).
+
+
+struct CodeAllocator;
+class NativeModule;
+class NativeModuleRef;
+class SharedCodeAllocator;
+
+
+// A NativeModule represents a single natively-compiled module (script).  It is
+// the unit of shared ownership and is thus where the reference count is
+// maintained.  It owns a set of NativeProtos, with associated native exec data,
+// and the allocated native data and code.
+class NativeModule
+{
+public:
+    NativeModule(SharedCodeAllocator* allocator, const std::optional<ModuleId>& moduleId, const uint8_t* moduleBaseAddress,
+        std::vector<NativeProtoExecDataPtr> nativeProtos) noexcept;
+
+    NativeModule(const NativeModule&) = delete;
+    NativeModule(NativeModule&&) = delete;
+    NativeModule& operator=(const NativeModule&) = delete;
+    NativeModule& operator=(NativeModule&&) = delete;
+
+    // The NativeModule must not be destroyed if there are any outstanding
+    // references.  It should thus only be destroyed by a call to release()
+    // that releases the last reference.
+    ~NativeModule() noexcept;
+
+    size_t addRef() const noexcept;
+    size_t addRefs(size_t count) const noexcept;
+    size_t release() const noexcept;
+    [[nodiscard]] size_t getRefcount() const noexcept;
+
+    [[nodiscard]] const std::optional<ModuleId>& getModuleId() const noexcept;
+
+    // Gets the base address of the executable native code for the module.
+    [[nodiscard]] const uint8_t* getModuleBaseAddress() const noexcept;
+
+    // Attempts to find the NativeProto with the given bytecode id.  If no
+    // NativeProto for that bytecode id exists, a null pointer is returned.
+    [[nodiscard]] const uint32_t* tryGetNativeProto(uint32_t bytecodeId) const noexcept;
+
+    [[nodiscard]] const std::vector<NativeProtoExecDataPtr>& getNativeProtos() const noexcept;
+
+private:
+    mutable std::atomic<size_t> refcount = 0;
+
+    SharedCodeAllocator* allocator = nullptr;
+    std::optional<ModuleId> moduleId = {};
+    const uint8_t* moduleBaseAddress = nullptr;
+
+    std::vector<NativeProtoExecDataPtr> nativeProtos = {};
+};
+
+// A NativeModuleRef is an owning reference to a NativeModule.  (Note:  We do
+// not use shared_ptr, to avoid complex state management in the Luau GC Proto
+// object.)
+class NativeModuleRef
+{
+public:
+    NativeModuleRef() noexcept = default;
+    NativeModuleRef(const NativeModule* nativeModule) noexcept;
+
+    NativeModuleRef(const NativeModuleRef& other) noexcept;
+    NativeModuleRef(NativeModuleRef&& other) noexcept;
+    NativeModuleRef& operator=(NativeModuleRef other) noexcept;
+
+    ~NativeModuleRef() noexcept;
+
+    void reset() noexcept;
+    void swap(NativeModuleRef& other) noexcept;
+
+    [[nodiscard]] bool empty() const noexcept;
+    explicit operator bool() const noexcept;
+
+    [[nodiscard]] const NativeModule* get() const noexcept;
+    [[nodiscard]] const NativeModule* operator->() const noexcept;
+    [[nodiscard]] const NativeModule& operator*() const noexcept;
+
+private:
+    const NativeModule* nativeModule = nullptr;
+};
+
+class SharedCodeAllocator
+{
+public:
+    SharedCodeAllocator(CodeAllocator* codeAllocator) noexcept;
+
+    SharedCodeAllocator(const SharedCodeAllocator&) = delete;
+    SharedCodeAllocator(SharedCodeAllocator&&) = delete;
+    SharedCodeAllocator& operator=(const SharedCodeAllocator&) = delete;
+    SharedCodeAllocator& operator=(SharedCodeAllocator&&) = delete;
+
+    ~SharedCodeAllocator() noexcept;
+
+    // If we have a NativeModule for the given ModuleId, an owning reference to
+    // it is returned.  Otherwise, an empty NativeModuleRef is returned.
+    [[nodiscard]] NativeModuleRef tryGetNativeModule(const ModuleId& moduleId) const noexcept;
+
+    // If we have a NativeModule for the given ModuleId, an owning reference to
+    // it is returned.  Otherwise, a new NativeModule is created for that ModuleId
+    // using the provided NativeProtos, data, and code (space is allocated for the
+    // data and code such that it can be executed).  Like std::map::insert, the
+    // bool result is true if a new module was created; false if an existing
+    // module is being returned.
+    std::pair<NativeModuleRef, bool> getOrInsertNativeModule(const ModuleId& moduleId, std::vector<NativeProtoExecDataPtr> nativeProtos,
+        const uint8_t* data, size_t dataSize, const uint8_t* code, size_t codeSize);
+
+    NativeModuleRef insertAnonymousNativeModule(
+        std::vector<NativeProtoExecDataPtr> nativeProtos, const uint8_t* data, size_t dataSize, const uint8_t* code, size_t codeSize);
+
+    // If a NativeModule exists for the given ModuleId and that NativeModule
+    // is no longer referenced, the NativeModule is destroyed.  This should
+    // usually only be called by NativeModule::release() when the reference
+    // count becomes zero
+    void eraseNativeModuleIfUnreferenced(const NativeModule& nativeModule);
+
+private:
+    struct ModuleIdHash
+    {
+        [[nodiscard]] size_t operator()(const ModuleId& moduleId) const noexcept;
+    };
+
+    [[nodiscard]] NativeModuleRef tryGetNativeModuleWithLockHeld(const ModuleId& moduleId) const noexcept;
+
+    mutable std::mutex mutex;
+
+    std::unordered_map<ModuleId, std::unique_ptr<NativeModule>, ModuleIdHash, std::equal_to<>> identifiedModules;
+
+    std::atomic<size_t> anonymousModuleCount = 0;
+
+    CodeAllocator* codeAllocator = nullptr;
+};
+
+} // namespace CodeGen
+} // namespace Luau