// This file is part of the Luau programming language and is licensed under MIT License; see LICENSE.txt for details #include "CodeGenContext.h" #include "CodeGenA64.h" #include "CodeGenLower.h" #include "CodeGenX64.h" #include "Luau/CodeBlockUnwind.h" #include "Luau/UnwindBuilder.h" #include "Luau/UnwindBuilderDwarf2.h" #include "Luau/UnwindBuilderWin.h" #include "lapi.h" LUAU_FASTFLAGVARIABLE(LuauCodegenContext, false) LUAU_FASTINT(LuauCodeGenBlockSize) LUAU_FASTINT(LuauCodeGenMaxTotalSize) namespace Luau { namespace CodeGen { static const Instruction kCodeEntryInsn = LOP_NATIVECALL; // From CodeGen.cpp extern void* gPerfLogContext; extern PerfLogFn gPerfLogFn; unsigned int getCpuFeaturesA64(); static void logPerfFunction(Proto* p, uintptr_t addr, unsigned size) { CODEGEN_ASSERT(FFlag::LuauCodegenContext); CODEGEN_ASSERT(p->source); const char* source = getstr(p->source); source = (source[0] == '=' || source[0] == '@') ? source + 1 : "[string]"; char name[256]; snprintf(name, sizeof(name), " %s:%d %s", source, p->linedefined, p->debugname ? getstr(p->debugname) : ""); if (gPerfLogFn) gPerfLogFn(gPerfLogContext, addr, size, name); } static void logPerfFunctions( const std::vector& moduleProtos, const uint8_t* nativeModuleBaseAddress, const std::vector& nativeProtos) { CODEGEN_ASSERT(FFlag::LuauCodegenContext); if (gPerfLogFn == nullptr) return; if (nativeProtos.size() > 0) gPerfLogFn(gPerfLogContext, uintptr_t(nativeModuleBaseAddress), unsigned(getNativeProtoExecDataHeader(nativeProtos[0].get()).entryOffsetOrAddress - nativeModuleBaseAddress), ""); auto protoIt = moduleProtos.begin(); for (const NativeProtoExecDataPtr& nativeProto : nativeProtos) { const NativeProtoExecDataHeader& header = getNativeProtoExecDataHeader(nativeProto.get()); while (protoIt != moduleProtos.end() && uint32_t((**protoIt).bytecodeid) != header.bytecodeId) { ++protoIt; } CODEGEN_ASSERT(protoIt != moduleProtos.end()); logPerfFunction(*protoIt, uintptr_t(header.entryOffsetOrAddress), uint32_t(header.nativeCodeSize)); } } // If Release is true, the native proto will be removed from the vector and // ownership will be assigned to the Proto object (for use with the // StandaloneCodeContext). If Release is false, the native proto will not be // removed from the vector (for use with the SharedCodeContext). template [[nodiscard]] static uint32_t bindNativeProtos(const std::vector& moduleProtos, NativeProtosVector& nativeProtos) { CODEGEN_ASSERT(FFlag::LuauCodegenContext); uint32_t protosBound = 0; auto protoIt = moduleProtos.begin(); for (auto& nativeProto : nativeProtos) { const NativeProtoExecDataHeader& header = getNativeProtoExecDataHeader(nativeProto.get()); while (protoIt != moduleProtos.end() && uint32_t((**protoIt).bytecodeid) != header.bytecodeId) { ++protoIt; } CODEGEN_ASSERT(protoIt != moduleProtos.end()); // The NativeProtoExecData is now owned by the VM and will be destroyed // via onDestroyFunction. Proto* proto = *protoIt; if constexpr (Release) { proto->execdata = nativeProto.release(); } else { proto->execdata = nativeProto.get(); } proto->exectarget = reinterpret_cast(header.entryOffsetOrAddress); proto->codeentry = &kCodeEntryInsn; ++protosBound; } return protosBound; } BaseCodeGenContext::BaseCodeGenContext(size_t blockSize, size_t maxTotalSize, AllocationCallback* allocationCallback, void* allocationCallbackContext) : codeAllocator{blockSize, maxTotalSize, allocationCallback, allocationCallbackContext} { CODEGEN_ASSERT(FFlag::LuauCodegenContext); CODEGEN_ASSERT(isSupported()); #if defined(_WIN32) unwindBuilder = std::make_unique(); #else unwindBuilder = std::make_unique(); #endif codeAllocator.context = unwindBuilder.get(); codeAllocator.createBlockUnwindInfo = createBlockUnwindInfo; codeAllocator.destroyBlockUnwindInfo = destroyBlockUnwindInfo; initFunctions(context); } [[nodiscard]] bool BaseCodeGenContext::initHeaderFunctions() { CODEGEN_ASSERT(FFlag::LuauCodegenContext); #if defined(__x86_64__) || defined(_M_X64) if (!X64::initHeaderFunctions(*this)) return false; #elif defined(__aarch64__) if (!A64::initHeaderFunctions(*this)) return false; #endif if (gPerfLogFn) gPerfLogFn(gPerfLogContext, uintptr_t(context.gateEntry), 4096, ""); return true; } StandaloneCodeGenContext::StandaloneCodeGenContext( size_t blockSize, size_t maxTotalSize, AllocationCallback* allocationCallback, void* allocationCallbackContext) : BaseCodeGenContext{blockSize, maxTotalSize, allocationCallback, allocationCallbackContext} { CODEGEN_ASSERT(FFlag::LuauCodegenContext); } [[nodiscard]] std::optional StandaloneCodeGenContext::tryBindExistingModule(const ModuleId&, const std::vector&) { CODEGEN_ASSERT(FFlag::LuauCodegenContext); // The StandaloneCodeGenContext does not support sharing of native code return {}; } [[nodiscard]] ModuleBindResult StandaloneCodeGenContext::bindModule(const std::optional&, const std::vector& moduleProtos, std::vector nativeProtos, const uint8_t* data, size_t dataSize, const uint8_t* code, size_t codeSize) { CODEGEN_ASSERT(FFlag::LuauCodegenContext); uint8_t* nativeData = nullptr; size_t sizeNativeData = 0; uint8_t* codeStart = nullptr; if (!codeAllocator.allocate(data, int(dataSize), code, int(codeSize), nativeData, sizeNativeData, codeStart)) { return {CodeGenCompilationResult::AllocationFailed}; } // Relocate the entry offsets to their final executable addresses: for (const NativeProtoExecDataPtr& nativeProto : nativeProtos) { NativeProtoExecDataHeader& header = getNativeProtoExecDataHeader(nativeProto.get()); header.entryOffsetOrAddress = codeStart + reinterpret_cast(header.entryOffsetOrAddress); } logPerfFunctions(moduleProtos, codeStart, nativeProtos); const uint32_t protosBound = bindNativeProtos(moduleProtos, nativeProtos); return {CodeGenCompilationResult::Success, protosBound}; } void StandaloneCodeGenContext::onCloseState() noexcept { CODEGEN_ASSERT(FFlag::LuauCodegenContext); // The StandaloneCodeGenContext is owned by the one VM that owns it, so when // that VM is destroyed, we destroy *this as well: delete this; } void StandaloneCodeGenContext::onDestroyFunction(void* execdata) noexcept { CODEGEN_ASSERT(FFlag::LuauCodegenContext); destroyNativeProtoExecData(static_cast(execdata)); } SharedCodeGenContext::SharedCodeGenContext( size_t blockSize, size_t maxTotalSize, AllocationCallback* allocationCallback, void* allocationCallbackContext) : BaseCodeGenContext{blockSize, maxTotalSize, allocationCallback, allocationCallbackContext} , sharedAllocator{&codeAllocator} { CODEGEN_ASSERT(FFlag::LuauCodegenContext); } [[nodiscard]] std::optional SharedCodeGenContext::tryBindExistingModule( const ModuleId& moduleId, const std::vector& moduleProtos) { CODEGEN_ASSERT(FFlag::LuauCodegenContext); NativeModuleRef nativeModule = sharedAllocator.tryGetNativeModule(moduleId); if (nativeModule.empty()) { return {}; } // Bind the native protos and acquire an owning reference for each: const uint32_t protosBound = bindNativeProtos(moduleProtos, nativeModule->getNativeProtos()); nativeModule->addRefs(protosBound); return {{CodeGenCompilationResult::Success, protosBound}}; } [[nodiscard]] ModuleBindResult SharedCodeGenContext::bindModule(const std::optional& moduleId, const std::vector& moduleProtos, std::vector nativeProtos, const uint8_t* data, size_t dataSize, const uint8_t* code, size_t codeSize) { CODEGEN_ASSERT(FFlag::LuauCodegenContext); const std::pair insertionResult = [&]() -> std::pair { if (moduleId.has_value()) { return sharedAllocator.getOrInsertNativeModule(*moduleId, std::move(nativeProtos), data, dataSize, code, codeSize); } else { return {sharedAllocator.insertAnonymousNativeModule(std::move(nativeProtos), data, dataSize, code, codeSize), true}; } }(); // If we did not get a NativeModule back, allocation failed: if (insertionResult.first.empty()) return {CodeGenCompilationResult::AllocationFailed}; // If we allocated a new module, log the function code ranges for perf: if (insertionResult.second) logPerfFunctions(moduleProtos, insertionResult.first->getModuleBaseAddress(), insertionResult.first->getNativeProtos()); // Bind the native protos and acquire an owning reference for each: const uint32_t protosBound = bindNativeProtos(moduleProtos, insertionResult.first->getNativeProtos()); insertionResult.first->addRefs(protosBound); return {CodeGenCompilationResult::Success, protosBound}; } void SharedCodeGenContext::onCloseState() noexcept { CODEGEN_ASSERT(FFlag::LuauCodegenContext); // The lifetime of the SharedCodeGenContext is managed separately from the // VMs that use it. When a VM is destroyed, we don't need to do anything // here. } void SharedCodeGenContext::onDestroyFunction(void* execdata) noexcept { CODEGEN_ASSERT(FFlag::LuauCodegenContext); getNativeProtoExecDataHeader(static_cast(execdata)).nativeModule->release(); } [[nodiscard]] UniqueSharedCodeGenContext createSharedCodeGenContext() { CODEGEN_ASSERT(FFlag::LuauCodegenContext); return createSharedCodeGenContext(size_t(FInt::LuauCodeGenBlockSize), size_t(FInt::LuauCodeGenMaxTotalSize), nullptr, nullptr); } [[nodiscard]] UniqueSharedCodeGenContext createSharedCodeGenContext(AllocationCallback* allocationCallback, void* allocationCallbackContext) { CODEGEN_ASSERT(FFlag::LuauCodegenContext); return createSharedCodeGenContext( size_t(FInt::LuauCodeGenBlockSize), size_t(FInt::LuauCodeGenMaxTotalSize), allocationCallback, allocationCallbackContext); } [[nodiscard]] UniqueSharedCodeGenContext createSharedCodeGenContext( size_t blockSize, size_t maxTotalSize, AllocationCallback* allocationCallback, void* allocationCallbackContext) { CODEGEN_ASSERT(FFlag::LuauCodegenContext); UniqueSharedCodeGenContext codeGenContext{new SharedCodeGenContext{blockSize, maxTotalSize, nullptr, nullptr}}; if (!codeGenContext->initHeaderFunctions()) return {}; return codeGenContext; } void destroySharedCodeGenContext(const SharedCodeGenContext* codeGenContext) noexcept { CODEGEN_ASSERT(FFlag::LuauCodegenContext); delete codeGenContext; } void SharedCodeGenContextDeleter::operator()(const SharedCodeGenContext* codeGenContext) const noexcept { CODEGEN_ASSERT(FFlag::LuauCodegenContext); destroySharedCodeGenContext(codeGenContext); } [[nodiscard]] static BaseCodeGenContext* getCodeGenContext(lua_State* L) noexcept { CODEGEN_ASSERT(FFlag::LuauCodegenContext); return static_cast(L->global->ecb.context); } static void onCloseState(lua_State* L) noexcept { CODEGEN_ASSERT(FFlag::LuauCodegenContext); getCodeGenContext(L)->onCloseState(); L->global->ecb = lua_ExecutionCallbacks{}; } static void onDestroyFunction(lua_State* L, Proto* proto) noexcept { CODEGEN_ASSERT(FFlag::LuauCodegenContext); getCodeGenContext(L)->onDestroyFunction(proto->execdata); proto->execdata = nullptr; proto->exectarget = 0; proto->codeentry = proto->code; } static int onEnter(lua_State* L, Proto* proto) { CODEGEN_ASSERT(FFlag::LuauCodegenContext); BaseCodeGenContext* codeGenContext = getCodeGenContext(L); CODEGEN_ASSERT(proto->execdata); CODEGEN_ASSERT(L->ci->savedpc >= proto->code && L->ci->savedpc < proto->code + proto->sizecode); uintptr_t target = proto->exectarget + static_cast(proto->execdata)[L->ci->savedpc - proto->code]; // Returns 1 to finish the function in the VM return GateFn(codeGenContext->context.gateEntry)(L, proto, target, &codeGenContext->context); } static int onEnterDisabled(lua_State* L, Proto* proto) { CODEGEN_ASSERT(FFlag::LuauCodegenContext); return 1; } // Defined in CodeGen.cpp void onDisable(lua_State* L, Proto* proto); static size_t getMemorySize(lua_State* L, Proto* proto) { CODEGEN_ASSERT(FFlag::LuauCodegenContext); const NativeProtoExecDataHeader& execDataHeader = getNativeProtoExecDataHeader(static_cast(proto->execdata)); const size_t execDataSize = sizeof(NativeProtoExecDataHeader) + execDataHeader.bytecodeInstructionCount * sizeof(Instruction); // While execDataSize is exactly the size of the allocation we made and hold for 'execdata' field, the code size is approximate // This is because code+data page is shared and owned by all Proto from a single module and each one can keep the whole region alive // So individual Proto being freed by GC will not reflect memory use by native code correctly return execDataSize + execDataHeader.nativeCodeSize; } static void initializeExecutionCallbacks(lua_State* L, BaseCodeGenContext* codeGenContext) noexcept { CODEGEN_ASSERT(FFlag::LuauCodegenContext); lua_ExecutionCallbacks* ecb = &L->global->ecb; ecb->context = codeGenContext; ecb->close = onCloseState; ecb->destroy = onDestroyFunction; ecb->enter = onEnter; ecb->disable = onDisable; ecb->getmemorysize = getMemorySize; } void create_NEW(lua_State* L) { CODEGEN_ASSERT(FFlag::LuauCodegenContext); return create_NEW(L, size_t(FInt::LuauCodeGenBlockSize), size_t(FInt::LuauCodeGenMaxTotalSize), nullptr, nullptr); } void create_NEW(lua_State* L, AllocationCallback* allocationCallback, void* allocationCallbackContext) { CODEGEN_ASSERT(FFlag::LuauCodegenContext); return create_NEW(L, size_t(FInt::LuauCodeGenBlockSize), size_t(FInt::LuauCodeGenMaxTotalSize), allocationCallback, allocationCallbackContext); } void create_NEW(lua_State* L, size_t blockSize, size_t maxTotalSize, AllocationCallback* allocationCallback, void* allocationCallbackContext) { CODEGEN_ASSERT(FFlag::LuauCodegenContext); std::unique_ptr codeGenContext = std::make_unique(blockSize, maxTotalSize, allocationCallback, allocationCallbackContext); if (!codeGenContext->initHeaderFunctions()) return; initializeExecutionCallbacks(L, codeGenContext.release()); } void create_NEW(lua_State* L, SharedCodeGenContext* codeGenContext) { CODEGEN_ASSERT(FFlag::LuauCodegenContext); initializeExecutionCallbacks(L, codeGenContext); } [[nodiscard]] static NativeProtoExecDataPtr createNativeProtoExecData(Proto* proto, const IrBuilder& ir) { CODEGEN_ASSERT(FFlag::LuauCodegenContext); NativeProtoExecDataPtr nativeExecData = createNativeProtoExecData(proto->sizecode); uint32_t instTarget = ir.function.entryLocation; for (int i = 0; i < proto->sizecode; ++i) { CODEGEN_ASSERT(ir.function.bcMapping[i].asmLocation >= instTarget); nativeExecData[i] = ir.function.bcMapping[i].asmLocation - instTarget; } // Set first instruction offset to 0 so that entering this function still // executes any generated entry code. nativeExecData[0] = 0; NativeProtoExecDataHeader& header = getNativeProtoExecDataHeader(nativeExecData.get()); header.entryOffsetOrAddress = reinterpret_cast(static_cast(instTarget)); header.bytecodeId = uint32_t(proto->bytecodeid); header.bytecodeInstructionCount = proto->sizecode; return nativeExecData; } template [[nodiscard]] static NativeProtoExecDataPtr createNativeFunction( AssemblyBuilder& build, ModuleHelpers& helpers, Proto* proto, uint32_t& totalIrInstCount, CodeGenCompilationResult& result) { CODEGEN_ASSERT(FFlag::LuauCodegenContext); IrBuilder ir; ir.buildFunctionIr(proto); unsigned instCount = unsigned(ir.function.instructions.size()); if (totalIrInstCount + instCount >= unsigned(FInt::CodegenHeuristicsInstructionLimit.value)) { result = CodeGenCompilationResult::CodeGenOverflowInstructionLimit; return {}; } totalIrInstCount += instCount; if (!lowerFunction(ir, build, helpers, proto, {}, /* stats */ nullptr, result)) { return {}; } return createNativeProtoExecData(proto, ir); } [[nodiscard]] static CompilationResult compileInternal( const std::optional& moduleId, lua_State* L, int idx, unsigned int flags, CompilationStats* stats) { CODEGEN_ASSERT(FFlag::LuauCodegenContext); CODEGEN_ASSERT(lua_isLfunction(L, idx)); const TValue* func = luaA_toobject(L, idx); Proto* root = clvalue(func)->l.p; if ((flags & CodeGen_OnlyNativeModules) != 0 && (root->flags & LPF_NATIVE_MODULE) == 0) return CompilationResult{CodeGenCompilationResult::NotNativeModule}; BaseCodeGenContext* codeGenContext = getCodeGenContext(L); if (codeGenContext == nullptr) return CompilationResult{CodeGenCompilationResult::CodeGenNotInitialized}; std::vector protos; gatherFunctions(protos, root, flags); // Skip protos that have been compiled during previous invocations of CodeGen::compile protos.erase(std::remove_if(protos.begin(), protos.end(), [](Proto* p) { return p == nullptr || p->execdata != nullptr; }), protos.end()); if (protos.empty()) return CompilationResult{CodeGenCompilationResult::NothingToCompile}; if (stats != nullptr) stats->functionsTotal = uint32_t(protos.size()); if (moduleId.has_value()) { if (std::optional existingModuleBindResult = codeGenContext->tryBindExistingModule(*moduleId, protos)) { if (stats != nullptr) stats->functionsBound = existingModuleBindResult->functionsBound; return CompilationResult{existingModuleBindResult->compilationResult}; } } #if defined(__aarch64__) static unsigned int cpuFeatures = getCpuFeaturesA64(); A64::AssemblyBuilderA64 build(/* logText= */ false, cpuFeatures); #else X64::AssemblyBuilderX64 build(/* logText= */ false); #endif ModuleHelpers helpers; #if defined(__aarch64__) A64::assembleHelpers(build, helpers); #else X64::assembleHelpers(build, helpers); #endif CompilationResult compilationResult; std::vector nativeProtos; nativeProtos.reserve(protos.size()); uint32_t totalIrInstCount = 0; for (size_t i = 0; i != protos.size(); ++i) { CodeGenCompilationResult protoResult = CodeGenCompilationResult::Success; NativeProtoExecDataPtr nativeExecData = createNativeFunction(build, helpers, protos[i], totalIrInstCount, protoResult); if (nativeExecData != nullptr) { nativeProtos.push_back(std::move(nativeExecData)); } else { compilationResult.protoFailures.push_back( {protoResult, protos[i]->debugname ? getstr(protos[i]->debugname) : "", protos[i]->linedefined}); } } // Very large modules might result in overflowing a jump offset; in this // case we currently abandon the entire module if (!build.finalize()) { compilationResult.result = CodeGenCompilationResult::CodeGenAssemblerFinalizationFailure; return compilationResult; } // If no functions were assembled, we don't need to allocate/copy executable pages for helpers if (nativeProtos.empty()) return compilationResult; if (stats != nullptr) { for (const NativeProtoExecDataPtr& nativeExecData : nativeProtos) { NativeProtoExecDataHeader& header = getNativeProtoExecDataHeader(nativeExecData.get()); stats->bytecodeSizeBytes += header.bytecodeInstructionCount * sizeof(Instruction); // Account for the native -> bytecode instruction offsets mapping: stats->nativeMetadataSizeBytes += header.bytecodeInstructionCount * sizeof(uint32_t); } stats->functionsCompiled += uint32_t(nativeProtos.size()); stats->nativeCodeSizeBytes += build.code.size() * sizeof(build.code[0]); stats->nativeDataSizeBytes += build.data.size(); } for (size_t i = 0; i < nativeProtos.size(); ++i) { NativeProtoExecDataHeader& header = getNativeProtoExecDataHeader(nativeProtos[i].get()); uint32_t begin = uint32_t(reinterpret_cast(header.entryOffsetOrAddress)); uint32_t end = i + 1 < nativeProtos.size() ? uint32_t(uintptr_t(getNativeProtoExecDataHeader(nativeProtos[i + 1].get()).entryOffsetOrAddress)) : uint32_t(build.code.size() * sizeof(build.code[0])); CODEGEN_ASSERT(begin < end); header.nativeCodeSize = end - begin; } const ModuleBindResult bindResult = codeGenContext->bindModule(moduleId, protos, std::move(nativeProtos), reinterpret_cast(build.data.data()), build.data.size(), reinterpret_cast(build.code.data()), build.code.size() * sizeof(build.code[0])); if (stats != nullptr) stats->functionsBound = bindResult.functionsBound; if (bindResult.compilationResult != CodeGenCompilationResult::Success) compilationResult.result = bindResult.compilationResult; return compilationResult; } CompilationResult compile_NEW(const ModuleId& moduleId, lua_State* L, int idx, unsigned int flags, CompilationStats* stats) { CODEGEN_ASSERT(FFlag::LuauCodegenContext); return compileInternal(moduleId, L, idx, flags, stats); } CompilationResult compile_NEW(lua_State* L, int idx, unsigned int flags, CompilationStats* stats) { CODEGEN_ASSERT(FFlag::LuauCodegenContext); return compileInternal({}, L, idx, flags, stats); } [[nodiscard]] bool isNativeExecutionEnabled_NEW(lua_State* L) { CODEGEN_ASSERT(FFlag::LuauCodegenContext); return getCodeGenContext(L) != nullptr && L->global->ecb.enter == onEnter; } void setNativeExecutionEnabled_NEW(lua_State* L, bool enabled) { CODEGEN_ASSERT(FFlag::LuauCodegenContext); if (getCodeGenContext(L) != nullptr) L->global->ecb.enter = enabled ? onEnter : onEnterDisabled; } } // namespace CodeGen } // namespace Luau