luau-lang/luau
1
0
Fork 0
mirror of https://github.com/luau-lang/luau.git synced 2024-12-14 06:00:39 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3
luau/CodeGen/src/IrLoweringX64.cpp

1243 lines
39 KiB
C++
Raw Normal View History

Sync to upstream/release/561 (#820) * Fix a potential debugger crash by adding checks for invalid stack index values --------- Co-authored-by: Arseny Kapoulkine <arseny.kapoulkine@gmail.com> Co-authored-by: Vyacheslav Egorov <vegorov@roblox.com>
2023-01-27 22:28:31 +00:00
// This file is part of the Luau programming language and is licensed under MIT License; see LICENSE.txt for details
#include "IrLoweringX64.h"
#include "Luau/CodeGen.h"
#include "Luau/DenseHash.h"
Sync to upstream/release/562 (#828) * Fixed rare use-after-free in analysis during table unification A lot of work these past months went into two new Luau components: * A near full rewrite of the typechecker using a new deferred constraint resolution system * Native code generation for AoT/JiT compilation of VM bytecode into x64 (avx)/arm64 instructions Both of these components are far from finished and we don't provide documentation on building and using them at this point. However, curious community members expressed interest in learning about changes that go into these components each week, so we are now listing them here in the 'sync' pull request descriptions. --- New typechecker can be enabled by setting DebugLuauDeferredConstraintResolution flag to 'true'. It is considered unstable right now, so try it at your own risk. Even though it already provides better type inference than the current one in some cases, our main goal right now is to reach feature parity with current typechecker. Features which improve over the capabilities of the current typechecker are marked as '(NEW)'. Changes to new typechecker: * Regular for loop index and parameters are now typechecked * Invalid type annotations on local variables are ignored to improve autocomplete * Fixed missing autocomplete type suggestions for function arguments * Type reduction is now performed to produce simpler types to be presented to the user (error messages, custom LSPs) * Internally, complex types like '((number | string) & ~(false?)) | string' can be produced, which is just 'string | number' when simplified * Fixed spots where support for unknown and never types was missing * (NEW) Length operator '#' is now valid to use on top table type, this type comes up when doing typeof(x) == "table" guards and isn't available in current typechecker --- Changes to native code generation: * Additional math library fast calls are now lowered to x64: math.ldexp, math.round, math.frexp, math.modf, math.sign and math.clamp
2023-02-03 19:26:13 +00:00
#include "Luau/IrDump.h"
#include "Luau/IrUtils.h"
Sync to upstream/release/561 (#820) * Fix a potential debugger crash by adding checks for invalid stack index values --------- Co-authored-by: Arseny Kapoulkine <arseny.kapoulkine@gmail.com> Co-authored-by: Vyacheslav Egorov <vegorov@roblox.com>
2023-01-27 22:28:31 +00:00
#include "EmitCommonX64.h"
#include "EmitInstructionX64.h"
#include "NativeState.h"
#include "lstate.h"
#include <algorithm>
namespace Luau
{
namespace CodeGen
{
static RegisterX64 gprAlocOrder[] = {rax, rdx, rcx, rbx, rsi, rdi, r8, r9, r10, r11};
IrLoweringX64::IrLoweringX64(AssemblyBuilderX64& build, ModuleHelpers& helpers, NativeState& data, Proto* proto, IrFunction& function)
: build(build)
, helpers(helpers)
, data(data)
, proto(proto)
, function(function)
{
freeGprMap.fill(true);
freeXmmMap.fill(true);
}
void IrLoweringX64::lower(AssemblyOptions options)
{
// While we will need a better block ordering in the future, right now we want to mostly preserve build order with fallbacks outlined
std::vector<uint32_t> sortedBlocks;
sortedBlocks.reserve(function.blocks.size());
for (uint32_t i = 0; i < function.blocks.size(); i++)
sortedBlocks.push_back(i);
std::sort(sortedBlocks.begin(), sortedBlocks.end(), [&](uint32_t idxA, uint32_t idxB) {
const IrBlock& a = function.blocks[idxA];
const IrBlock& b = function.blocks[idxB];
// Place fallback blocks at the end
if ((a.kind == IrBlockKind::Fallback) != (b.kind == IrBlockKind::Fallback))
return (a.kind == IrBlockKind::Fallback) < (b.kind == IrBlockKind::Fallback);
// Try to order by instruction order
return a.start < b.start;
});
DenseHashMap<uint32_t, uint32_t> bcLocations{~0u};
// Create keys for IR assembly locations that original bytecode instruction are interested in
for (const auto& [irLocation, asmLocation] : function.bcMapping)
{
if (irLocation != ~0u)
bcLocations[irLocation] = 0;
}
DenseHashMap<uint32_t, uint32_t> indexIrToBc{~0u};
bool outputEnabled = options.includeAssembly || options.includeIr;
if (outputEnabled && options.annotator)
{
// Create reverse mapping from IR location to bytecode location
for (size_t i = 0; i < function.bcMapping.size(); ++i)
{
uint32_t irLocation = function.bcMapping[i].irLocation;
if (irLocation != ~0u)
indexIrToBc[irLocation] = uint32_t(i);
}
}
IrToStringContext ctx{build.text, function.blocks, function.constants};
// We use this to skip outlined fallback blocks from IR/asm text output
size_t textSize = build.text.length();
uint32_t codeSize = build.getCodeSize();
bool seenFallback = false;
IrBlock dummy;
dummy.start = ~0u;
for (size_t i = 0; i < sortedBlocks.size(); ++i)
{
uint32_t blockIndex = sortedBlocks[i];
IrBlock& block = function.blocks[blockIndex];
LUAU_ASSERT(block.start != ~0u);
// If we want to skip fallback code IR/asm, we'll record when those blocks start once we see them
if (block.kind == IrBlockKind::Fallback && !seenFallback)
{
textSize = build.text.length();
codeSize = build.getCodeSize();
seenFallback = true;
}
if (options.includeIr)
build.logAppend("# %s_%u:\n", getBlockKindName(block.kind), blockIndex);
build.setLabel(block.label);
for (uint32_t index = block.start; true; index++)
{
LUAU_ASSERT(index < function.instructions.size());
// If IR instruction is the first one for the original bytecode, we can annotate it with source code text
if (outputEnabled && options.annotator)
{
if (uint32_t* bcIndex = indexIrToBc.find(index))
options.annotator(options.annotatorContext, build.text, proto->bytecodeid, *bcIndex);
}
// If bytecode needs the location of this instruction for jumps, record it
if (uint32_t* bcLocation = bcLocations.find(index))
*bcLocation = build.getCodeSize();
IrInst& inst = function.instructions[index];
// Nop is used to replace dead instructions in-place
// Because it doesn't have any effects aside from output (when enabled), we skip it completely
if (inst.cmd == IrCmd::NOP)
{
LUAU_ASSERT(inst.useCount == 0);
continue;
}
if (options.includeIr)
{
build.logAppend("# ");
toStringDetailed(ctx, inst, index);
}
IrBlock& next = i + 1 < sortedBlocks.size() ? function.blocks[sortedBlocks[i + 1]] : dummy;
lowerInst(inst, index, next);
freeLastUseRegs(inst, index);
if (isBlockTerminator(inst.cmd))
{
if (options.includeIr)
build.logAppend("#\n");
break;
}
}
}
if (outputEnabled && !options.includeOutlinedCode && seenFallback)
{
build.text.resize(textSize);
if (options.includeAssembly)
build.logAppend("; skipping %u bytes of outlined code\n", build.getCodeSize() - codeSize);
}
// Copy assembly locations of IR instructions that are mapped to bytecode instructions
for (auto& [irLocation, asmLocation] : function.bcMapping)
{
if (irLocation != ~0u)
asmLocation = bcLocations[irLocation];
}
}
void IrLoweringX64::lowerInst(IrInst& inst, uint32_t index, IrBlock& next)
{
switch (inst.cmd)
{
case IrCmd::LOAD_TAG:
inst.regX64 = allocGprReg(SizeX64::dword);
if (inst.a.kind == IrOpKind::VmReg)
build.mov(inst.regX64, luauRegTag(inst.a.index));
else if (inst.a.kind == IrOpKind::VmConst)
build.mov(inst.regX64, luauConstantTag(inst.a.index));
else
LUAU_ASSERT(!"Unsupported instruction form");
break;
case IrCmd::LOAD_POINTER:
inst.regX64 = allocGprReg(SizeX64::qword);
if (inst.a.kind == IrOpKind::VmReg)
build.mov(inst.regX64, luauRegValue(inst.a.index));
else if (inst.a.kind == IrOpKind::VmConst)
build.mov(inst.regX64, luauConstantValue(inst.a.index));
else
LUAU_ASSERT(!"Unsupported instruction form");
break;
case IrCmd::LOAD_DOUBLE:
inst.regX64 = allocXmmReg();
if (inst.a.kind == IrOpKind::VmReg)
build.vmovsd(inst.regX64, luauRegValue(inst.a.index));
else if (inst.a.kind == IrOpKind::VmConst)
build.vmovsd(inst.regX64, luauConstantValue(inst.a.index));
else
LUAU_ASSERT(!"Unsupported instruction form");
break;
case IrCmd::LOAD_INT:
LUAU_ASSERT(inst.a.kind == IrOpKind::VmReg);
inst.regX64 = allocGprReg(SizeX64::dword);
build.mov(inst.regX64, luauRegValueInt(inst.a.index));
break;
case IrCmd::LOAD_TVALUE:
inst.regX64 = allocXmmReg();
if (inst.a.kind == IrOpKind::VmReg)
build.vmovups(inst.regX64, luauReg(inst.a.index));
else if (inst.a.kind == IrOpKind::VmConst)
build.vmovups(inst.regX64, luauConstant(inst.a.index));
else if (inst.a.kind == IrOpKind::Inst)
build.vmovups(inst.regX64, xmmword[regOp(inst.a)]);
else
LUAU_ASSERT(!"Unsupported instruction form");
break;
case IrCmd::LOAD_NODE_VALUE_TV:
inst.regX64 = allocXmmReg();
build.vmovups(inst.regX64, luauNodeValue(regOp(inst.a)));
break;
case IrCmd::LOAD_ENV:
inst.regX64 = allocGprReg(SizeX64::qword);
build.mov(inst.regX64, sClosure);
build.mov(inst.regX64, qword[inst.regX64 + offsetof(Closure, env)]);
break;
case IrCmd::GET_ARR_ADDR:
if (inst.b.kind == IrOpKind::Inst)
{
inst.regX64 = allocGprRegOrReuse(SizeX64::qword, index, {inst.b});
build.mov(dwordReg(inst.regX64), regOp(inst.b));
build.shl(dwordReg(inst.regX64), kTValueSizeLog2);
build.add(inst.regX64, qword[regOp(inst.a) + offsetof(Table, array)]);
}
else if (inst.b.kind == IrOpKind::Constant)
{
inst.regX64 = allocGprRegOrReuse(SizeX64::qword, index, {inst.a});
build.mov(inst.regX64, qword[regOp(inst.a) + offsetof(Table, array)]);
if (uintOp(inst.b) != 0)
build.lea(inst.regX64, addr[inst.regX64 + uintOp(inst.b) * sizeof(TValue)]);
}
else
{
LUAU_ASSERT(!"Unsupported instruction form");
}
break;
case IrCmd::GET_SLOT_NODE_ADDR:
{
inst.regX64 = allocGprReg(SizeX64::qword);
ScopedReg tmp{*this, SizeX64::qword};
getTableNodeAtCachedSlot(build, tmp.reg, inst.regX64, regOp(inst.a), uintOp(inst.b));
break;
}
case IrCmd::STORE_TAG:
LUAU_ASSERT(inst.a.kind == IrOpKind::VmReg);
if (inst.b.kind == IrOpKind::Constant)
build.mov(luauRegTag(inst.a.index), tagOp(inst.b));
else
LUAU_ASSERT(!"Unsupported instruction form");
break;
case IrCmd::STORE_POINTER:
LUAU_ASSERT(inst.a.kind == IrOpKind::VmReg);
build.mov(luauRegValue(inst.a.index), regOp(inst.b));
break;
case IrCmd::STORE_DOUBLE:
LUAU_ASSERT(inst.a.kind == IrOpKind::VmReg);
if (inst.b.kind == IrOpKind::Constant)
{
ScopedReg tmp{*this, SizeX64::xmmword};
build.vmovsd(tmp.reg, build.f64(doubleOp(inst.b)));
build.vmovsd(luauRegValue(inst.a.index), tmp.reg);
}
else if (inst.b.kind == IrOpKind::Inst)
{
build.vmovsd(luauRegValue(inst.a.index), regOp(inst.b));
}
else
{
LUAU_ASSERT(!"Unsupported instruction form");
}
break;
case IrCmd::STORE_INT:
{
LUAU_ASSERT(inst.a.kind == IrOpKind::VmReg);
if (inst.b.kind == IrOpKind::Constant)
build.mov(luauRegValueInt(inst.a.index), intOp(inst.b));
else if (inst.b.kind == IrOpKind::Inst)
build.mov(luauRegValueInt(inst.a.index), regOp(inst.b));
else
LUAU_ASSERT(!"Unsupported instruction form");
break;
}
case IrCmd::STORE_TVALUE:
if (inst.a.kind == IrOpKind::VmReg)
build.vmovups(luauReg(inst.a.index), regOp(inst.b));
else if (inst.a.kind == IrOpKind::Inst)
build.vmovups(xmmword[regOp(inst.a)], regOp(inst.b));
else
LUAU_ASSERT(!"Unsupported instruction form");
break;
case IrCmd::STORE_NODE_VALUE_TV:
build.vmovups(luauNodeValue(regOp(inst.a)), regOp(inst.b));
break;
case IrCmd::ADD_INT:
inst.regX64 = allocGprRegOrReuse(SizeX64::dword, index, {inst.a});
if (inst.regX64 == regOp(inst.a) && intOp(inst.b) == 1)
build.inc(inst.regX64);
else if (inst.regX64 == regOp(inst.a))
build.add(inst.regX64, intOp(inst.b));
else
build.lea(inst.regX64, addr[regOp(inst.a) + intOp(inst.b)]);
break;
case IrCmd::SUB_INT:
inst.regX64 = allocGprRegOrReuse(SizeX64::dword, index, {inst.a});
if (inst.regX64 == regOp(inst.a) && intOp(inst.b) == 1)
build.dec(inst.regX64);
else if (inst.regX64 == regOp(inst.a))
build.sub(inst.regX64, intOp(inst.b));
else
build.lea(inst.regX64, addr[regOp(inst.a) - intOp(inst.b)]);
break;
case IrCmd::ADD_NUM:
inst.regX64 = allocXmmRegOrReuse(index, {inst.a, inst.b});
build.vaddsd(inst.regX64, regOp(inst.a), memRegDoubleOp(inst.b));
break;
case IrCmd::SUB_NUM:
inst.regX64 = allocXmmRegOrReuse(index, {inst.a, inst.b});
build.vsubsd(inst.regX64, regOp(inst.a), memRegDoubleOp(inst.b));
break;
case IrCmd::MUL_NUM:
inst.regX64 = allocXmmRegOrReuse(index, {inst.a, inst.b});
build.vmulsd(inst.regX64, regOp(inst.a), memRegDoubleOp(inst.b));
break;
case IrCmd::DIV_NUM:
inst.regX64 = allocXmmRegOrReuse(index, {inst.a, inst.b});
build.vdivsd(inst.regX64, regOp(inst.a), memRegDoubleOp(inst.b));
break;
case IrCmd::MOD_NUM:
{
inst.regX64 = allocXmmRegOrReuse(index, {inst.a, inst.b});
RegisterX64 lhs = regOp(inst.a);
if (inst.b.kind == IrOpKind::Inst)
{
ScopedReg tmp{*this, SizeX64::xmmword};
build.vdivsd(tmp.reg, lhs, memRegDoubleOp(inst.b));
build.vroundsd(tmp.reg, tmp.reg, tmp.reg, RoundingModeX64::RoundToNegativeInfinity);
build.vmulsd(tmp.reg, tmp.reg, memRegDoubleOp(inst.b));
build.vsubsd(inst.regX64, lhs, tmp.reg);
}
else
{
ScopedReg tmp1{*this, SizeX64::xmmword};
ScopedReg tmp2{*this, SizeX64::xmmword};
build.vmovsd(tmp1.reg, memRegDoubleOp(inst.b));
build.vdivsd(tmp2.reg, lhs, tmp1.reg);
build.vroundsd(tmp2.reg, tmp2.reg, tmp2.reg, RoundingModeX64::RoundToNegativeInfinity);
build.vmulsd(tmp1.reg, tmp2.reg, tmp1.reg);
build.vsubsd(inst.regX64, lhs, tmp1.reg);
}
break;
}
case IrCmd::POW_NUM:
{
inst.regX64 = allocXmmRegOrReuse(index, {inst.a, inst.b});
RegisterX64 lhs = regOp(inst.a);
if (inst.b.kind == IrOpKind::Inst)
{
// TODO: this doesn't happen with current local-only register allocation, but has to be handled in the future
LUAU_ASSERT(regOp(inst.b) != xmm0);
if (lhs != xmm0)
build.vmovsd(xmm0, lhs, lhs);
if (regOp(inst.b) != xmm1)
build.vmovsd(xmm1, regOp(inst.b), regOp(inst.b));
build.call(qword[rNativeContext + offsetof(NativeContext, libm_pow)]);
if (inst.regX64 != xmm0)
build.vmovsd(inst.regX64, xmm0, xmm0);
}
else if (inst.b.kind == IrOpKind::Constant)
{
double rhs = doubleOp(inst.b);
if (rhs == 2.0)
{
build.vmulsd(inst.regX64, lhs, lhs);
}
else if (rhs == 0.5)
{
build.vsqrtsd(inst.regX64, lhs, lhs);
}
else if (rhs == 3.0)
{
ScopedReg tmp{*this, SizeX64::xmmword};
build.vmulsd(tmp.reg, lhs, lhs);
build.vmulsd(inst.regX64, lhs, tmp.reg);
}
else
{
if (lhs != xmm0)
build.vmovsd(xmm0, xmm0, lhs);
build.vmovsd(xmm1, build.f64(rhs));
build.call(qword[rNativeContext + offsetof(NativeContext, libm_pow)]);
if (inst.regX64 != xmm0)
build.vmovsd(inst.regX64, xmm0, xmm0);
}
}
else
{
LUAU_ASSERT(!"Unsupported instruction form");
}
break;
}
case IrCmd::UNM_NUM:
{
inst.regX64 = allocXmmRegOrReuse(index, {inst.a});
RegisterX64 src = regOp(inst.a);
if (inst.regX64 == src)
{
build.vxorpd(inst.regX64, inst.regX64, build.f64(-0.0));
}
else
{
build.vmovsd(inst.regX64, src, src);
build.vxorpd(inst.regX64, inst.regX64, build.f64(-0.0));
}
break;
}
case IrCmd::NOT_ANY:
{
// TODO: if we have a single user which is a STORE_INT, we are missing the opportunity to write directly to target
inst.regX64 = allocGprRegOrReuse(SizeX64::dword, index, {inst.a, inst.b});
Label saveone, savezero, exit;
build.cmp(regOp(inst.a), LUA_TNIL);
build.jcc(ConditionX64::Equal, saveone);
build.cmp(regOp(inst.a), LUA_TBOOLEAN);
build.jcc(ConditionX64::NotEqual, savezero);
build.cmp(regOp(inst.b), 0);
build.jcc(ConditionX64::Equal, saveone);
build.setLabel(savezero);
build.mov(inst.regX64, 0);
build.jmp(exit);
build.setLabel(saveone);
build.mov(inst.regX64, 1);
build.setLabel(exit);
break;
}
case IrCmd::JUMP:
jumpOrFallthrough(blockOp(inst.a), next);
break;
case IrCmd::JUMP_IF_TRUTHY:
LUAU_ASSERT(inst.a.kind == IrOpKind::VmReg);
jumpIfTruthy(build, inst.a.index, labelOp(inst.b), labelOp(inst.c));
jumpOrFallthrough(blockOp(inst.c), next);
break;
case IrCmd::JUMP_IF_FALSY:
LUAU_ASSERT(inst.a.kind == IrOpKind::VmReg);
jumpIfFalsy(build, inst.a.index, labelOp(inst.b), labelOp(inst.c));
jumpOrFallthrough(blockOp(inst.c), next);
break;
case IrCmd::JUMP_EQ_TAG:
{
LUAU_ASSERT(inst.b.kind == IrOpKind::Inst || inst.b.kind == IrOpKind::Constant);
OperandX64 opb = inst.b.kind == IrOpKind::Inst ? regOp(inst.b) : OperandX64(tagOp(inst.b));
build.cmp(memRegTagOp(inst.a), opb);
if (isFallthroughBlock(blockOp(inst.d), next))
{
build.jcc(ConditionX64::Equal, labelOp(inst.c));
jumpOrFallthrough(blockOp(inst.d), next);
}
else
{
build.jcc(ConditionX64::NotEqual, labelOp(inst.d));
jumpOrFallthrough(blockOp(inst.c), next);
}
break;
}
case IrCmd::JUMP_EQ_BOOLEAN:
build.cmp(regOp(inst.a), boolOp(inst.b) ? 1 : 0);
build.jcc(ConditionX64::Equal, labelOp(inst.c));
jumpOrFallthrough(blockOp(inst.d), next);
break;
case IrCmd::JUMP_EQ_POINTER:
build.cmp(regOp(inst.a), regOp(inst.b));
build.jcc(ConditionX64::Equal, labelOp(inst.c));
jumpOrFallthrough(blockOp(inst.d), next);
break;
case IrCmd::JUMP_CMP_NUM:
{
LUAU_ASSERT(inst.c.kind == IrOpKind::Condition);
IrCondition cond = IrCondition(inst.c.index);
ScopedReg tmp{*this, SizeX64::xmmword};
// TODO: jumpOnNumberCmp should work on IrCondition directly
jumpOnNumberCmp(build, tmp.reg, memRegDoubleOp(inst.a), memRegDoubleOp(inst.b), getX64Condition(cond), labelOp(inst.d));
jumpOrFallthrough(blockOp(inst.e), next);
break;
}
case IrCmd::JUMP_CMP_ANY:
{
LUAU_ASSERT(inst.a.kind == IrOpKind::VmReg);
LUAU_ASSERT(inst.b.kind == IrOpKind::VmReg);
LUAU_ASSERT(inst.c.kind == IrOpKind::Condition);
IrCondition cond = IrCondition(inst.c.index);
jumpOnAnyCmpFallback(build, inst.a.index, inst.b.index, getX64Condition(cond), labelOp(inst.d));
jumpOrFallthrough(blockOp(inst.e), next);
break;
}
case IrCmd::TABLE_LEN:
inst.regX64 = allocXmmReg();
build.mov(rArg1, regOp(inst.a));
build.call(qword[rNativeContext + offsetof(NativeContext, luaH_getn)]);
build.vcvtsi2sd(inst.regX64, inst.regX64, eax);
break;
case IrCmd::NEW_TABLE:
inst.regX64 = allocGprReg(SizeX64::qword);
build.mov(rArg1, rState);
build.mov(dwordReg(rArg2), uintOp(inst.a));
build.mov(dwordReg(rArg3), uintOp(inst.b));
build.call(qword[rNativeContext + offsetof(NativeContext, luaH_new)]);
build.mov(inst.regX64, rax);
break;
case IrCmd::DUP_TABLE:
inst.regX64 = allocGprReg(SizeX64::qword);
// Re-ordered to avoid register conflict
build.mov(rArg2, regOp(inst.a));
build.mov(rArg1, rState);
build.call(qword[rNativeContext + offsetof(NativeContext, luaH_clone)]);
build.mov(inst.regX64, rax);
break;
case IrCmd::NUM_TO_INDEX:
{
inst.regX64 = allocGprReg(SizeX64::dword);
ScopedReg tmp{*this, SizeX64::xmmword};
convertNumberToIndexOrJump(build, tmp.reg, regOp(inst.a), inst.regX64, labelOp(inst.b));
break;
}
case IrCmd::DO_ARITH:
LUAU_ASSERT(inst.a.kind == IrOpKind::VmReg);
LUAU_ASSERT(inst.b.kind == IrOpKind::VmReg);
LUAU_ASSERT(inst.c.kind == IrOpKind::VmReg || inst.c.kind == IrOpKind::VmConst);
if (inst.c.kind == IrOpKind::VmReg)
callArithHelper(build, inst.a.index, inst.b.index, luauRegAddress(inst.c.index), TMS(intOp(inst.d)));
else
callArithHelper(build, inst.a.index, inst.b.index, luauConstantAddress(inst.c.index), TMS(intOp(inst.d)));
break;
case IrCmd::DO_LEN:
LUAU_ASSERT(inst.a.kind == IrOpKind::VmReg);
LUAU_ASSERT(inst.b.kind == IrOpKind::VmReg);
callLengthHelper(build, inst.a.index, inst.b.index);
break;
case IrCmd::GET_TABLE:
LUAU_ASSERT(inst.a.kind == IrOpKind::VmReg);
LUAU_ASSERT(inst.b.kind == IrOpKind::VmReg);
if (inst.c.kind == IrOpKind::VmReg)
{
callGetTable(build, inst.b.index, luauRegAddress(inst.c.index), inst.a.index);
}
else if (inst.c.kind == IrOpKind::Constant)
{
TValue n;
setnvalue(&n, uintOp(inst.c));
callGetTable(build, inst.b.index, build.bytes(&n, sizeof(n)), inst.a.index);
}
else
{
LUAU_ASSERT(!"Unsupported instruction form");
}
break;
case IrCmd::SET_TABLE:
LUAU_ASSERT(inst.a.kind == IrOpKind::VmReg);
LUAU_ASSERT(inst.b.kind == IrOpKind::VmReg);
if (inst.c.kind == IrOpKind::VmReg)
{
callSetTable(build, inst.b.index, luauRegAddress(inst.c.index), inst.a.index);
}
else if (inst.c.kind == IrOpKind::Constant)
{
TValue n;
setnvalue(&n, uintOp(inst.c));
callSetTable(build, inst.b.index, build.bytes(&n, sizeof(n)), inst.a.index);
}
else
{
LUAU_ASSERT(!"Unsupported instruction form");
}
break;
case IrCmd::GET_IMPORT:
LUAU_ASSERT(inst.a.kind == IrOpKind::VmReg);
emitInstGetImportFallback(build, inst.a.index, uintOp(inst.b));
break;
case IrCmd::CONCAT:
build.mov(rArg1, rState);
build.mov(dwordReg(rArg2), uintOp(inst.a));
build.mov(dwordReg(rArg3), uintOp(inst.b));
build.call(qword[rNativeContext + offsetof(NativeContext, luaV_concat)]);
emitUpdateBase(build);
break;
case IrCmd::GET_UPVALUE:
{
LUAU_ASSERT(inst.a.kind == IrOpKind::VmReg);
LUAU_ASSERT(inst.b.kind == IrOpKind::VmUpvalue);
ScopedReg tmp1{*this, SizeX64::qword};
ScopedReg tmp2{*this, SizeX64::xmmword};
build.mov(tmp1.reg, sClosure);
build.add(tmp1.reg, offsetof(Closure, l.uprefs) + sizeof(TValue) * inst.b.index);
// uprefs[] is either an actual value, or it points to UpVal object which has a pointer to value
Label skip;
// TODO: jumpIfTagIsNot can be generalized to take OperandX64 and then we can use it here; let's wait until we see this more though
build.cmp(dword[tmp1.reg + offsetof(TValue, tt)], LUA_TUPVAL);
build.jcc(ConditionX64::NotEqual, skip);
// UpVal.v points to the value (either on stack, or on heap inside each UpVal, but we can deref it unconditionally)
build.mov(tmp1.reg, qword[tmp1.reg + offsetof(TValue, value.gc)]);
build.mov(tmp1.reg, qword[tmp1.reg + offsetof(UpVal, v)]);
build.setLabel(skip);
build.vmovups(tmp2.reg, xmmword[tmp1.reg]);
build.vmovups(luauReg(inst.a.index), tmp2.reg);
break;
}
case IrCmd::SET_UPVALUE:
{
LUAU_ASSERT(inst.a.kind == IrOpKind::VmUpvalue);
LUAU_ASSERT(inst.b.kind == IrOpKind::VmReg);
Label next;
ScopedReg tmp1{*this, SizeX64::qword};
ScopedReg tmp2{*this, SizeX64::qword};
ScopedReg tmp3{*this, SizeX64::xmmword};
build.mov(tmp1.reg, sClosure);
build.mov(tmp2.reg, qword[tmp1.reg + offsetof(Closure, l.uprefs) + sizeof(TValue) * inst.a.index + offsetof(TValue, value.gc)]);
build.mov(tmp1.reg, qword[tmp2.reg + offsetof(UpVal, v)]);
build.vmovups(tmp3.reg, luauReg(inst.b.index));
build.vmovups(xmmword[tmp1.reg], tmp3.reg);
callBarrierObject(build, tmp1.reg, tmp2.reg, inst.b.index, next);
build.setLabel(next);
break;
}
case IrCmd::CHECK_TAG:
if (inst.a.kind == IrOpKind::Inst)
{
build.cmp(regOp(inst.a), tagOp(inst.b));
build.jcc(ConditionX64::NotEqual, labelOp(inst.c));
}
else if (inst.a.kind == IrOpKind::VmReg)
{
jumpIfTagIsNot(build, inst.a.index, lua_Type(tagOp(inst.b)), labelOp(inst.c));
}
else
{
LUAU_ASSERT(!"Unsupported instruction form");
}
break;
case IrCmd::CHECK_READONLY:
jumpIfTableIsReadOnly(build, regOp(inst.a), labelOp(inst.b));
break;
case IrCmd::CHECK_NO_METATABLE:
jumpIfMetatablePresent(build, regOp(inst.a), labelOp(inst.b));
break;
case IrCmd::CHECK_SAFE_ENV:
{
ScopedReg tmp{*this, SizeX64::qword};
jumpIfUnsafeEnv(build, tmp.reg, labelOp(inst.a));
break;
}
case IrCmd::CHECK_ARRAY_SIZE:
if (inst.b.kind == IrOpKind::Inst)
build.cmp(dword[regOp(inst.a) + offsetof(Table, sizearray)], regOp(inst.b));
else if (inst.b.kind == IrOpKind::Constant)
build.cmp(dword[regOp(inst.a) + offsetof(Table, sizearray)], uintOp(inst.b));
else
LUAU_ASSERT(!"Unsupported instruction form");
build.jcc(ConditionX64::BelowEqual, labelOp(inst.c));
break;
case IrCmd::CHECK_SLOT_MATCH:
{
LUAU_ASSERT(inst.b.kind == IrOpKind::VmConst);
ScopedReg tmp{*this, SizeX64::qword};
jumpIfNodeKeyNotInExpectedSlot(build, tmp.reg, regOp(inst.a), luauConstantValue(inst.b.index), labelOp(inst.c));
break;
}
case IrCmd::INTERRUPT:
emitInterrupt(build, uintOp(inst.a));
break;
case IrCmd::CHECK_GC:
{
Label skip;
callCheckGc(build, -1, false, skip);
build.setLabel(skip);
break;
}
case IrCmd::BARRIER_OBJ:
{
LUAU_ASSERT(inst.b.kind == IrOpKind::VmReg);
Label skip;
ScopedReg tmp{*this, SizeX64::qword};
callBarrierObject(build, tmp.reg, regOp(inst.a), inst.b.index, skip);
build.setLabel(skip);
break;
}
case IrCmd::BARRIER_TABLE_BACK:
{
Label skip;
callBarrierTableFast(build, regOp(inst.a), skip);
build.setLabel(skip);
break;
}
case IrCmd::BARRIER_TABLE_FORWARD:
{
LUAU_ASSERT(inst.b.kind == IrOpKind::VmReg);
Label skip;
ScopedReg tmp{*this, SizeX64::qword};
callBarrierTable(build, tmp.reg, regOp(inst.a), inst.b.index, skip);
build.setLabel(skip);
break;
}
case IrCmd::SET_SAVEDPC:
{
// This is like emitSetSavedPc, but using register allocation instead of relying on rax/rdx
ScopedReg tmp1{*this, SizeX64::qword};
ScopedReg tmp2{*this, SizeX64::qword};
build.mov(tmp2.reg, sCode);
build.add(tmp2.reg, uintOp(inst.a) * sizeof(Instruction));
build.mov(tmp1.reg, qword[rState + offsetof(lua_State, ci)]);
build.mov(qword[tmp1.reg + offsetof(CallInfo, savedpc)], tmp2.reg);
break;
}
case IrCmd::CLOSE_UPVALS:
{
LUAU_ASSERT(inst.a.kind == IrOpKind::VmReg);
Label next;
ScopedReg tmp1{*this, SizeX64::qword};
ScopedReg tmp2{*this, SizeX64::qword};
// L->openupval != 0
build.mov(tmp1.reg, qword[rState + offsetof(lua_State, openupval)]);
build.test(tmp1.reg, tmp1.reg);
build.jcc(ConditionX64::Zero, next);
// ra <= L->openuval->v
build.lea(tmp2.reg, addr[rBase + inst.a.index * sizeof(TValue)]);
build.cmp(tmp2.reg, qword[tmp1.reg + offsetof(UpVal, v)]);
build.jcc(ConditionX64::Above, next);
build.mov(rArg2, tmp2.reg);
build.mov(rArg1, rState);
build.call(qword[rNativeContext + offsetof(NativeContext, luaF_close)]);
build.setLabel(next);
break;
}
case IrCmd::CAPTURE:
// No-op right now
break;
// Fallbacks to non-IR instruction implementations
case IrCmd::LOP_SETLIST:
{
const Instruction* pc = proto->code + uintOp(inst.a);
Label next;
emitInstSetList(build, pc, next);
build.setLabel(next);
break;
}
case IrCmd::LOP_NAMECALL:
{
const Instruction* pc = proto->code + uintOp(inst.a);
emitInstNameCall(build, pc, uintOp(inst.a), proto->k, blockOp(inst.b).label, blockOp(inst.c).label);
break;
}
case IrCmd::LOP_CALL:
{
const Instruction* pc = proto->code + uintOp(inst.a);
emitInstCall(build, helpers, pc, uintOp(inst.a));
break;
}
case IrCmd::LOP_RETURN:
{
const Instruction* pc = proto->code + uintOp(inst.a);
emitInstReturn(build, helpers, pc, uintOp(inst.a));
break;
}
case IrCmd::LOP_FASTCALL:
emitInstFastCall(build, proto->code + uintOp(inst.a), uintOp(inst.a), labelOp(inst.b));
break;
case IrCmd::LOP_FASTCALL1:
emitInstFastCall1(build, proto->code + uintOp(inst.a), uintOp(inst.a), labelOp(inst.b));
break;
case IrCmd::LOP_FASTCALL2:
emitInstFastCall2(build, proto->code + uintOp(inst.a), uintOp(inst.a), labelOp(inst.b));
break;
case IrCmd::LOP_FASTCALL2K:
emitInstFastCall2K(build, proto->code + uintOp(inst.a), uintOp(inst.a), labelOp(inst.b));
break;
case IrCmd::LOP_FORNPREP:
emitInstForNPrep(build, proto->code + uintOp(inst.a), uintOp(inst.a), labelOp(inst.b), labelOp(inst.c));
break;
case IrCmd::LOP_FORNLOOP:
emitInstForNLoop(build, proto->code + uintOp(inst.a), uintOp(inst.a), labelOp(inst.b), labelOp(inst.c));
break;
case IrCmd::LOP_FORGLOOP:
emitinstForGLoop(build, proto->code + uintOp(inst.a), uintOp(inst.a), labelOp(inst.b), labelOp(inst.c), labelOp(inst.d));
break;
case IrCmd::LOP_FORGLOOP_FALLBACK:
emitinstForGLoopFallback(build, proto->code + uintOp(inst.a), uintOp(inst.a), labelOp(inst.b));
build.jmp(labelOp(inst.c));
break;
case IrCmd::LOP_FORGPREP_NEXT:
emitInstForGPrepNext(build, proto->code + uintOp(inst.a), labelOp(inst.b), labelOp(inst.c));
break;
case IrCmd::LOP_FORGPREP_INEXT:
emitInstForGPrepInext(build, proto->code + uintOp(inst.a), labelOp(inst.b), labelOp(inst.c));
break;
case IrCmd::LOP_FORGPREP_XNEXT_FALLBACK:
emitInstForGPrepXnextFallback(build, proto->code + uintOp(inst.a), uintOp(inst.a), labelOp(inst.b));
break;
case IrCmd::LOP_AND:
emitInstAnd(build, proto->code + uintOp(inst.a));
break;
case IrCmd::LOP_ANDK:
emitInstAndK(build, proto->code + uintOp(inst.a));
break;
case IrCmd::LOP_OR:
emitInstOr(build, proto->code + uintOp(inst.a));
break;
case IrCmd::LOP_ORK:
emitInstOrK(build, proto->code + uintOp(inst.a));
break;
case IrCmd::LOP_COVERAGE:
emitInstCoverage(build, uintOp(inst.a));
break;
// Full instruction fallbacks
case IrCmd::FALLBACK_GETGLOBAL:
emitFallback(build, data, LOP_GETGLOBAL, uintOp(inst.a));
break;
case IrCmd::FALLBACK_SETGLOBAL:
emitFallback(build, data, LOP_SETGLOBAL, uintOp(inst.a));
break;
case IrCmd::FALLBACK_GETTABLEKS:
emitFallback(build, data, LOP_GETTABLEKS, uintOp(inst.a));
break;
case IrCmd::FALLBACK_SETTABLEKS:
emitFallback(build, data, LOP_SETTABLEKS, uintOp(inst.a));
break;
case IrCmd::FALLBACK_NAMECALL:
emitFallback(build, data, LOP_NAMECALL, uintOp(inst.a));
break;
case IrCmd::FALLBACK_PREPVARARGS:
emitFallback(build, data, LOP_PREPVARARGS, uintOp(inst.a));
break;
case IrCmd::FALLBACK_GETVARARGS:
emitFallback(build, data, LOP_GETVARARGS, uintOp(inst.a));
break;
case IrCmd::FALLBACK_NEWCLOSURE:
emitFallback(build, data, LOP_NEWCLOSURE, uintOp(inst.a));
break;
case IrCmd::FALLBACK_DUPCLOSURE:
emitFallback(build, data, LOP_DUPCLOSURE, uintOp(inst.a));
break;
case IrCmd::FALLBACK_FORGPREP:
emitFallback(build, data, LOP_FORGPREP, uintOp(inst.a));
break;
default:
LUAU_ASSERT(!"Not supported yet");
break;
}
}
bool IrLoweringX64::isFallthroughBlock(IrBlock target, IrBlock next)
{
return target.start == next.start;
}
void IrLoweringX64::jumpOrFallthrough(IrBlock& target, IrBlock& next)
{
if (!isFallthroughBlock(target, next))
build.jmp(target.label);
}
OperandX64 IrLoweringX64::memRegDoubleOp(IrOp op) const
{
switch (op.kind)
{
case IrOpKind::Inst:
return regOp(op);
case IrOpKind::Constant:
return build.f64(doubleOp(op));
case IrOpKind::VmReg:
return luauRegValue(op.index);
case IrOpKind::VmConst:
return luauConstantValue(op.index);
default:
LUAU_ASSERT(!"Unsupported operand kind");
}
return noreg;
}
OperandX64 IrLoweringX64::memRegTagOp(IrOp op) const
{
switch (op.kind)
{
case IrOpKind::Inst:
return regOp(op);
case IrOpKind::VmReg:
return luauRegTag(op.index);
case IrOpKind::VmConst:
return luauConstantTag(op.index);
default:
LUAU_ASSERT(!"Unsupported operand kind");
}
return noreg;
}
RegisterX64 IrLoweringX64::regOp(IrOp op) const
{
LUAU_ASSERT(op.kind == IrOpKind::Inst);
return function.instructions[op.index].regX64;
}
IrConst IrLoweringX64::constOp(IrOp op) const
{
LUAU_ASSERT(op.kind == IrOpKind::Constant);
return function.constants[op.index];
}
uint8_t IrLoweringX64::tagOp(IrOp op) const
{
IrConst value = constOp(op);
LUAU_ASSERT(value.kind == IrConstKind::Tag);
return value.valueTag;
}
bool IrLoweringX64::boolOp(IrOp op) const
{
IrConst value = constOp(op);
LUAU_ASSERT(value.kind == IrConstKind::Bool);
return value.valueBool;
}
int IrLoweringX64::intOp(IrOp op) const
{
IrConst value = constOp(op);
LUAU_ASSERT(value.kind == IrConstKind::Int);
return value.valueInt;
}
unsigned IrLoweringX64::uintOp(IrOp op) const
{
IrConst value = constOp(op);
LUAU_ASSERT(value.kind == IrConstKind::Uint);
return value.valueUint;
}
double IrLoweringX64::doubleOp(IrOp op) const
{
IrConst value = constOp(op);
LUAU_ASSERT(value.kind == IrConstKind::Double);
return value.valueDouble;
}
IrBlock& IrLoweringX64::blockOp(IrOp op) const
{
LUAU_ASSERT(op.kind == IrOpKind::Block);
return function.blocks[op.index];
}
Label& IrLoweringX64::labelOp(IrOp op) const
{
return blockOp(op).label;
}
RegisterX64 IrLoweringX64::allocGprReg(SizeX64 preferredSize)
{
LUAU_ASSERT(
preferredSize == SizeX64::byte || preferredSize == SizeX64::word || preferredSize == SizeX64::dword || preferredSize == SizeX64::qword);
for (RegisterX64 reg : gprAlocOrder)
{
if (freeGprMap[reg.index])
{
freeGprMap[reg.index] = false;
return RegisterX64{preferredSize, reg.index};
}
}
LUAU_ASSERT(!"Out of GPR registers to allocate");
return noreg;
}
RegisterX64 IrLoweringX64::allocXmmReg()
{
for (size_t i = 0; i < freeXmmMap.size(); ++i)
{
if (freeXmmMap[i])
{
freeXmmMap[i] = false;
return RegisterX64{SizeX64::xmmword, uint8_t(i)};
}
}
LUAU_ASSERT(!"Out of XMM registers to allocate");
return noreg;
}
RegisterX64 IrLoweringX64::allocGprRegOrReuse(SizeX64 preferredSize, uint32_t index, std::initializer_list<IrOp> oprefs)
{
for (IrOp op : oprefs)
{
if (op.kind != IrOpKind::Inst)
continue;
IrInst& source = function.instructions[op.index];
if (source.lastUse == index && !source.reusedReg)
{
LUAU_ASSERT(source.regX64.size != SizeX64::xmmword);
LUAU_ASSERT(source.regX64 != noreg);
source.reusedReg = true;
return RegisterX64{preferredSize, source.regX64.index};
}
}
return allocGprReg(preferredSize);
}
RegisterX64 IrLoweringX64::allocXmmRegOrReuse(uint32_t index, std::initializer_list<IrOp> oprefs)
{
for (IrOp op : oprefs)
{
if (op.kind != IrOpKind::Inst)
continue;
IrInst& source = function.instructions[op.index];
if (source.lastUse == index && !source.reusedReg)
{
LUAU_ASSERT(source.regX64.size == SizeX64::xmmword);
LUAU_ASSERT(source.regX64 != noreg);
source.reusedReg = true;
return source.regX64;
}
}
return allocXmmReg();
}
void IrLoweringX64::freeReg(RegisterX64 reg)
{
if (reg.size == SizeX64::xmmword)
{
LUAU_ASSERT(!freeXmmMap[reg.index]);
freeXmmMap[reg.index] = true;
}
else
{
LUAU_ASSERT(!freeGprMap[reg.index]);
freeGprMap[reg.index] = true;
}
}
void IrLoweringX64::freeLastUseReg(IrInst& target, uint32_t index)
{
if (target.lastUse == index && !target.reusedReg)
{
LUAU_ASSERT(target.regX64 != noreg);
freeReg(target.regX64);
target.regX64 = noreg;
}
}
void IrLoweringX64::freeLastUseRegs(const IrInst& inst, uint32_t index)
{
auto checkOp = [this, index](IrOp op) {
if (op.kind == IrOpKind::Inst)
freeLastUseReg(function.instructions[op.index], index);
};
checkOp(inst.a);
checkOp(inst.b);
checkOp(inst.c);
checkOp(inst.d);
checkOp(inst.e);
}
ConditionX64 IrLoweringX64::getX64Condition(IrCondition cond) const
{
// TODO: this function will not be required when jumpOnNumberCmp starts accepting an IrCondition
switch (cond)
{
case IrCondition::Equal:
return ConditionX64::Equal;
case IrCondition::NotEqual:
return ConditionX64::NotEqual;
case IrCondition::Less:
return ConditionX64::Less;
case IrCondition::NotLess:
return ConditionX64::NotLess;
case IrCondition::LessEqual:
return ConditionX64::LessEqual;
case IrCondition::NotLessEqual:
return ConditionX64::NotLessEqual;
case IrCondition::Greater:
return ConditionX64::Greater;
case IrCondition::NotGreater:
return ConditionX64::NotGreater;
case IrCondition::GreaterEqual:
return ConditionX64::GreaterEqual;
case IrCondition::NotGreaterEqual:
return ConditionX64::NotGreaterEqual;
default:
LUAU_ASSERT(!"unsupported condition");
break;
}
return ConditionX64::Count;
}
IrLoweringX64::ScopedReg::ScopedReg(IrLoweringX64& owner, SizeX64 size)
: owner(owner)
{
if (size == SizeX64::xmmword)
reg = owner.allocXmmReg();
else
reg = owner.allocGprReg(size);
}
IrLoweringX64::ScopedReg::~ScopedReg()
{
if (reg != noreg)
owner.freeReg(reg);
}
void IrLoweringX64::ScopedReg::free()
{
LUAU_ASSERT(reg != noreg);
owner.freeReg(reg);
reg = noreg;
}
} // namespace CodeGen
} // namespace Luau
Reference in a new issue Copy permalink