From 77295c3610a0e471efb6389a9f31ad3c9fdf5dd6 Mon Sep 17 00:00:00 2001 From: Vighnesh-V Date: Fri, 11 Oct 2024 17:48:30 -0700 Subject: [PATCH] Sync to upstream/release/647 (#1469) # General Updates Fix an old solver crash that occurs in the presence of cyclic `requires()` ## New Solver - Improvements to Luau user-defined type function library - Avoid asserting on unexpected metatable types - Properties in user defined type functions should have a consistent iteration order - in this case it is insertion ordering # Runtime - Track VM allocations for telemetry --- Co-authored-by: Aaron Weiss Co-authored-by: Andy Friesen Co-authored-by: Hunter Goldstein Co-authored-by: James McNellis Co-authored-by: Varun Saini Co-authored-by: Vighnesh Vijay Co-authored-by: Vyacheslav Egorov --------- Co-authored-by: Aaron Weiss Co-authored-by: Alexander McCord Co-authored-by: Andy Friesen Co-authored-by: Aviral Goel Co-authored-by: David Cope Co-authored-by: Lily Brown Co-authored-by: Vyacheslav Egorov Co-authored-by: Junseo Yoo --- Analysis/include/Luau/TypeFunctionRuntime.h | 7 +- Analysis/src/BuiltinDefinitions.cpp | 4 +- Analysis/src/ConstraintSolver.cpp | 5 +- Analysis/src/Differ.cpp | 3 + Analysis/src/Frontend.cpp | 15 +- Analysis/src/Normalize.cpp | 60 ++--- Analysis/src/TypeChecker2.cpp | 7 +- Analysis/src/TypeFunction.cpp | 5 +- Analysis/src/TypeFunctionRuntime.cpp | 243 +++++++++++++++----- Ast/src/Ast.cpp | 5 + CLI/Flags.cpp | 10 + CMakeLists.txt | 2 + CodeGen/src/CodeGenContext.cpp | 4 +- Makefile | 2 + VM/include/lua.h | 2 + VM/src/lmem.cpp | 16 ++ bench/bench.py | 7 +- tests/Autocomplete.test.cpp | 2 +- tests/Fixture.cpp | 17 +- tests/Fixture.h | 42 +--- tests/SharedCodeAllocator.test.cpp | 4 + tests/TypeFunction.test.cpp | 2 +- tests/TypeFunction.user.test.cpp | 98 ++++++++ tests/TypeInfer.tables.test.cpp | 25 ++ tests/main.cpp | 10 + tools/flag-bisect.py | 2 +- 26 files changed, 425 insertions(+), 174 deletions(-) diff --git a/Analysis/include/Luau/TypeFunctionRuntime.h b/Analysis/include/Luau/TypeFunctionRuntime.h index eb5d19ee..44eef136 100644 --- a/Analysis/include/Luau/TypeFunctionRuntime.h +++ b/Analysis/include/Luau/TypeFunctionRuntime.h @@ -6,7 +6,7 @@ #include #include -#include +#include #include using lua_State = struct lua_State; @@ -182,7 +182,7 @@ struct TypeFunctionProperty struct TypeFunctionTableType { using Name = std::string; - using Props = std::unordered_map; + using Props = std::map; Props props; @@ -195,7 +195,7 @@ struct TypeFunctionTableType struct TypeFunctionClassType { using Name = std::string; - using Props = std::unordered_map; + using Props = std::map; Props props; @@ -260,6 +260,7 @@ bool isTypeUserData(lua_State* L, int idx); TypeFunctionTypeId getTypeUserData(lua_State* L, int idx); std::optional optionalTypeUserData(lua_State* L, int idx); +void registerTypesLibrary(lua_State* L); void registerTypeUserData(lua_State* L); void setTypeFunctionEnvironment(lua_State* L); diff --git a/Analysis/src/BuiltinDefinitions.cpp b/Analysis/src/BuiltinDefinitions.cpp index 21ae0f11..32692a6e 100644 --- a/Analysis/src/BuiltinDefinitions.cpp +++ b/Analysis/src/BuiltinDefinitions.cpp @@ -26,7 +26,6 @@ */ LUAU_FASTFLAG(LuauSolverV2); -LUAU_FASTFLAGVARIABLE(LuauDCRMagicFunctionTypeChecker, false); namespace Luau { @@ -931,8 +930,7 @@ TypeId makeStringMetatable(NotNull builtinTypes) formatFTV.isCheckedFunction = true; const TypeId formatFn = arena->addType(formatFTV); attachDcrMagicFunction(formatFn, dcrMagicFunctionFormat); - if (FFlag::LuauDCRMagicFunctionTypeChecker) - attachDcrMagicFunctionTypeCheck(formatFn, dcrMagicFunctionTypeCheckFormat); + attachDcrMagicFunctionTypeCheck(formatFn, dcrMagicFunctionTypeCheckFormat); const TypeId stringToStringType = makeFunction(*arena, std::nullopt, {}, {}, {stringType}, {}, {stringType}, /* checked */ true); diff --git a/Analysis/src/ConstraintSolver.cpp b/Analysis/src/ConstraintSolver.cpp index 6c4d78a0..7cb545f2 100644 --- a/Analysis/src/ConstraintSolver.cpp +++ b/Analysis/src/ConstraintSolver.cpp @@ -31,10 +31,7 @@ LUAU_FASTFLAGVARIABLE(DebugLuauLogSolver, false) LUAU_FASTFLAGVARIABLE(DebugLuauLogSolverIncludeDependencies, false) LUAU_FASTFLAGVARIABLE(DebugLuauLogBindings, false) LUAU_FASTINTVARIABLE(LuauSolverRecursionLimit, 500) - -// The default value here is 643 because the first release in which this was implemented is 644, -// and actively we want new changes to be off by default until they're enabled consciously. -LUAU_DYNAMIC_FASTINTVARIABLE(LuauTypeSolverRelease, 643) +LUAU_DYNAMIC_FASTINT(LuauTypeSolverRelease) namespace Luau { diff --git a/Analysis/src/Differ.cpp b/Analysis/src/Differ.cpp index 25687e11..b2cebc0b 100644 --- a/Analysis/src/Differ.cpp +++ b/Analysis/src/Differ.cpp @@ -13,6 +13,7 @@ namespace Luau { + std::string DiffPathNode::toString() const { switch (kind) @@ -944,12 +945,14 @@ std::vector>::const_reverse_iterator DifferEnvironment return visitingStack.crend(); } + DifferResult diff(TypeId ty1, TypeId ty2) { DifferEnvironment differEnv{ty1, ty2, std::nullopt, std::nullopt}; return diffUsingEnv(differEnv, ty1, ty2); } + DifferResult diffWithSymbols(TypeId ty1, TypeId ty2, std::optional symbol1, std::optional symbol2) { DifferEnvironment differEnv{ty1, ty2, symbol1, symbol2}; diff --git a/Analysis/src/Frontend.cpp b/Analysis/src/Frontend.cpp index edcabc4a..b2325d37 100644 --- a/Analysis/src/Frontend.cpp +++ b/Analysis/src/Frontend.cpp @@ -44,9 +44,9 @@ LUAU_FASTFLAGVARIABLE(DebugLuauLogSolverToJsonFile, false) LUAU_FASTFLAGVARIABLE(DebugLuauForbidInternalTypes, false) LUAU_FASTFLAGVARIABLE(DebugLuauForceStrictMode, false) LUAU_FASTFLAGVARIABLE(DebugLuauForceNonStrictMode, false) -LUAU_FASTFLAGVARIABLE(LuauSourceModuleUpdatedWithSelectedMode, false) LUAU_FASTFLAGVARIABLE(LuauUserDefinedTypeFunctionNoEvaluation, false) LUAU_DYNAMIC_FASTFLAGVARIABLE(LuauRunCustomModuleChecks, false) +LUAU_FASTFLAGVARIABLE(LuauMoreThoroughCycleDetection, false) LUAU_FASTFLAG(StudioReportLuauAny2) @@ -883,14 +883,18 @@ void Frontend::addBuildQueueItems( data.environmentScope = getModuleEnvironment(*sourceModule, data.config, frontendOptions.forAutocomplete); data.recordJsonLog = FFlag::DebugLuauLogSolverToJson; - Mode mode = sourceModule->mode.value_or(data.config.mode); + const Mode mode = sourceModule->mode.value_or(data.config.mode); - // in NoCheck mode we only need to compute the value of .cyclic for typeck // in the future we could replace toposort with an algorithm that can flag cyclic nodes by itself // however, for now getRequireCycles isn't expensive in practice on the cases we care about, and long term // all correct programs must be acyclic so this code triggers rarely if (cycleDetected) - data.requireCycles = getRequireCycles(fileResolver, sourceNodes, sourceNode.get(), mode == Mode::NoCheck); + { + if (FFlag::LuauMoreThoroughCycleDetection) + data.requireCycles = getRequireCycles(fileResolver, sourceNodes, sourceNode.get(), false); + else + data.requireCycles = getRequireCycles(fileResolver, sourceNodes, sourceNode.get(), mode == Mode::NoCheck); + } data.options = frontendOptions; @@ -922,8 +926,7 @@ void Frontend::checkBuildQueueItem(BuildQueueItem& item) else mode = sourceModule.mode.value_or(config.mode); - if (FFlag::LuauSourceModuleUpdatedWithSelectedMode) - item.sourceModule->mode = {mode}; + item.sourceModule->mode = {mode}; ScopePtr environmentScope = item.environmentScope; double timestamp = getTimestamp(); const std::vector& requireCycles = item.requireCycles; diff --git a/Analysis/src/Normalize.cpp b/Analysis/src/Normalize.cpp index 01f896d5..475f45c3 100644 --- a/Analysis/src/Normalize.cpp +++ b/Analysis/src/Normalize.cpp @@ -16,8 +16,6 @@ #include "Luau/Unifier.h" LUAU_FASTFLAGVARIABLE(DebugLuauCheckNormalizeInvariant, false) -LUAU_FASTFLAGVARIABLE(LuauFixReduceStackPressure, false); -LUAU_FASTFLAGVARIABLE(LuauFixCyclicTablesBlowingStack, false); LUAU_FASTINTVARIABLE(LuauNormalizeCacheLimit, 100000); LUAU_FASTFLAG(LuauSolverV2); @@ -25,16 +23,6 @@ LUAU_FASTFLAG(LuauSolverV2); LUAU_FASTFLAGVARIABLE(LuauUseNormalizeIntersectionLimit, false) LUAU_FASTINTVARIABLE(LuauNormalizeIntersectionLimit, 200) -static bool fixReduceStackPressure() -{ - return FFlag::LuauFixReduceStackPressure || FFlag::LuauSolverV2; -} - -static bool fixCyclicTablesBlowingStack() -{ - return FFlag::LuauFixCyclicTablesBlowingStack || FFlag::LuauSolverV2; -} - namespace Luau { @@ -2583,43 +2571,29 @@ std::optional Normalizer::intersectionOfTables(TypeId here, TypeId there if (tprop.readTy.has_value()) { // if the intersection of the read types of a property is uninhabited, the whole table is `never`. - if (fixReduceStackPressure()) + // We've seen these table prop elements before and we're about to ask if their intersection + // is inhabited + if (seenSet.contains(*hprop.readTy) && seenSet.contains(*tprop.readTy)) { - // We've seen these table prop elements before and we're about to ask if their intersection - // is inhabited - if (fixCyclicTablesBlowingStack()) - { - if (seenSet.contains(*hprop.readTy) && seenSet.contains(*tprop.readTy)) - { - seenSet.erase(*hprop.readTy); - seenSet.erase(*tprop.readTy); - return {builtinTypes->neverType}; - } - else - { - seenSet.insert(*hprop.readTy); - seenSet.insert(*tprop.readTy); - } - } - - NormalizationResult res = isIntersectionInhabited(*hprop.readTy, *tprop.readTy); - - // Cleanup - if (fixCyclicTablesBlowingStack()) - { - seenSet.erase(*hprop.readTy); - seenSet.erase(*tprop.readTy); - } - - if (NormalizationResult::True != res) - return {builtinTypes->neverType}; + seenSet.erase(*hprop.readTy); + seenSet.erase(*tprop.readTy); + return {builtinTypes->neverType}; } else { - if (NormalizationResult::False == isIntersectionInhabited(*hprop.readTy, *tprop.readTy)) - return {builtinTypes->neverType}; + seenSet.insert(*hprop.readTy); + seenSet.insert(*tprop.readTy); } + NormalizationResult res = isIntersectionInhabited(*hprop.readTy, *tprop.readTy); + + // Cleanup + seenSet.erase(*hprop.readTy); + seenSet.erase(*tprop.readTy); + + if (NormalizationResult::True != res) + return {builtinTypes->neverType}; + TypeId ty = simplifyIntersection(builtinTypes, NotNull{arena}, *hprop.readTy, *tprop.readTy).result; prop.readTy = ty; hereSubThere &= (ty == hprop.readTy); diff --git a/Analysis/src/TypeChecker2.cpp b/Analysis/src/TypeChecker2.cpp index db618815..0a47f0df 100644 --- a/Analysis/src/TypeChecker2.cpp +++ b/Analysis/src/TypeChecker2.cpp @@ -1625,8 +1625,11 @@ void TypeChecker2::indexExprMetatableHelper(AstExprIndexExpr* indexExpr, const M indexExprMetatableHelper(indexExpr, mtmt, exprType, indexType); else { - LUAU_ASSERT(tt || get(follow(metaTable->table))); - + if (!(DFInt::LuauTypeSolverRelease >= 647)) + { + LUAU_ASSERT(tt || get(follow(metaTable->table))); + } + // CLI-122161: We're not handling unions correctly (probably). reportError(CannotExtendTable{exprType, CannotExtendTable::Indexer, "indexer??"}, indexExpr->location); } } diff --git a/Analysis/src/TypeFunction.cpp b/Analysis/src/TypeFunction.cpp index f2a7fc8f..82bfeca9 100644 --- a/Analysis/src/TypeFunction.cpp +++ b/Analysis/src/TypeFunction.cpp @@ -48,6 +48,7 @@ LUAU_DYNAMIC_FASTINTVARIABLE(LuauTypeFamilyUseGuesserDepth, -1); LUAU_FASTFLAGVARIABLE(DebugLuauLogTypeFamilies, false) LUAU_FASTFLAGVARIABLE(LuauUserDefinedTypeFunctions2, false) LUAU_FASTFLAG(LuauUserDefinedTypeFunctionNoEvaluation) +LUAU_FASTFLAG(LuauUserTypeFunFixRegister) LUAU_DYNAMIC_FASTINT(LuauTypeSolverRelease) @@ -1018,9 +1019,11 @@ void TypeFunctionRuntime::prepareState() setTypeFunctionEnvironment(L); - // Register type userdata registerTypeUserData(L); + if (FFlag::LuauUserTypeFunFixRegister) + registerTypesLibrary(L); + luaL_sandbox(L); luaL_sandboxthread(L); } diff --git a/Analysis/src/TypeFunctionRuntime.cpp b/Analysis/src/TypeFunctionRuntime.cpp index b0d9285e..768288ad 100644 --- a/Analysis/src/TypeFunctionRuntime.cpp +++ b/Analysis/src/TypeFunctionRuntime.cpp @@ -15,6 +15,7 @@ LUAU_DYNAMIC_FASTINT(LuauTypeFunctionSerdeIterationLimit) LUAU_DYNAMIC_FASTINT(LuauTypeSolverRelease) +LUAU_FASTFLAGVARIABLE(LuauUserTypeFunFixRegister, false) namespace Luau { @@ -1381,11 +1382,11 @@ static int isEqualToType(lua_State* L) return 1; } -// Register the type userdata -void registerTypeUserData(lua_State* L) +void registerTypesLibrary(lua_State* L) { - // List of fields for type userdata - luaL_Reg typeUserdataFields[] = { + LUAU_ASSERT(FFlag::LuauUserTypeFunFixRegister); + + luaL_Reg fields[] = { {"unknown", createUnknown}, {"never", createNever}, {"any", createAny}, @@ -1395,8 +1396,7 @@ void registerTypeUserData(lua_State* L) {nullptr, nullptr} }; - // List of methods for type userdata - luaL_Reg typeUserdataMethods[] = { + luaL_Reg methods[] = { {"singleton", createSingleton}, {"negationof", createNegation}, {"unionof", createUnion}, @@ -1405,72 +1405,191 @@ void registerTypeUserData(lua_State* L) {"newfunction", createFunction}, {"copy", deepCopy}, - // Common methods - {"is", checkTag}, - - // Negation type methods - {"inner", getNegatedValue}, - - // Singleton type methods - {"value", getSingletonValue}, - - // Table type methods - {"setproperty", setTableProp}, - {"setreadproperty", setReadTableProp}, - {"setwriteproperty", setWriteTableProp}, - {"readproperty", readTableProp}, - {"writeproperty", writeTableProp}, - {"properties", getProps}, - {"setindexer", setTableIndexer}, - {"setreadindexer", setTableReadIndexer}, - {"setwriteindexer", setTableWriteIndexer}, - {"indexer", getIndexer}, - {"readindexer", getReadIndexer}, - {"writeindexer", getWriteIndexer}, - {"setmetatable", setTableMetatable}, - {"metatable", getMetatable}, - - // Function type methods - {"setparameters", setFunctionParameters}, - {"parameters", getFunctionParameters}, - {"setreturns", setFunctionReturns}, - {"returns", getFunctionReturns}, - - // Union and Intersection type methods - {"components", getComponents}, - - // Class type methods - {"parent", getClassParent}, - {"indexer", getIndexer}, {nullptr, nullptr} }; - // Create and register metatable for type userdata - luaL_newmetatable(L, "type"); - - // Protect metatable from being fetched. - lua_pushstring(L, "The metatable is locked"); - lua_setfield(L, -2, "__metatable"); - - // Set type userdata metatable's __eq to type_equals() - lua_pushcfunction(L, isEqualToType, "__eq"); - lua_setfield(L, -2, "__eq"); - - // Set type userdata metatable's __index to itself - lua_pushvalue(L, -1); // Push a copy of type userdata metatable - lua_setfield(L, -2, "__index"); - - luaL_register(L, nullptr, typeUserdataMethods); + luaL_register(L, "types", methods); // Set fields for type userdata - for (luaL_Reg* l = typeUserdataFields; l->name; l++) + for (luaL_Reg* l = fields; l->name; l++) { l->func(L); lua_setfield(L, -2, l->name); } - // Set types library as a global name "types" - lua_setglobal(L, "types"); + lua_pop(L, 1); +} + +static int typeUserdataIndex(lua_State* L) +{ + TypeFunctionTypeId self = getTypeUserData(L, 1); + const char* field = luaL_checkstring(L, 2); + + if (strcmp(field, "tag") == 0) + { + lua_pushstring(L, getTag(L, self).c_str()); + return 1; + } + + lua_pushvalue(L, lua_upvalueindex(1)); + lua_getfield(L, -1, field); + return 1; +} + +void registerTypeUserData(lua_State* L) +{ + if (FFlag::LuauUserTypeFunFixRegister) + { + luaL_Reg typeUserdataMethods[] = { + {"is", checkTag}, + + // Negation type methods + {"inner", getNegatedValue}, + + // Singleton type methods + {"value", getSingletonValue}, + + // Table type methods + {"setproperty", setTableProp}, + {"setreadproperty", setReadTableProp}, + {"setwriteproperty", setWriteTableProp}, + {"readproperty", readTableProp}, + {"writeproperty", writeTableProp}, + {"properties", getProps}, + {"setindexer", setTableIndexer}, + {"setreadindexer", setTableReadIndexer}, + {"setwriteindexer", setTableWriteIndexer}, + {"indexer", getIndexer}, + {"readindexer", getReadIndexer}, + {"writeindexer", getWriteIndexer}, + {"setmetatable", setTableMetatable}, + {"metatable", getMetatable}, + + // Function type methods + {"setparameters", setFunctionParameters}, + {"parameters", getFunctionParameters}, + {"setreturns", setFunctionReturns}, + {"returns", getFunctionReturns}, + + // Union and Intersection type methods + {"components", getComponents}, + + // Class type methods + {"parent", getClassParent}, + + {nullptr, nullptr} + }; + + // Create and register metatable for type userdata + luaL_newmetatable(L, "type"); + + // Protect metatable from being changed + lua_pushstring(L, "The metatable is locked"); + lua_setfield(L, -2, "__metatable"); + + lua_pushcfunction(L, isEqualToType, "__eq"); + lua_setfield(L, -2, "__eq"); + + // Indexing will be a dynamic function because some type fields are dynamic + lua_newtable(L); + luaL_register(L, nullptr, typeUserdataMethods); + lua_setreadonly(L, -1, true); + lua_pushcclosure(L, typeUserdataIndex, "__index", 1); + lua_setfield(L, -2, "__index"); + + lua_setreadonly(L, -1, true); + lua_pop(L, 1); + } + else + { + // List of fields for type userdata + luaL_Reg typeUserdataFields[] = { + {"unknown", createUnknown}, + {"never", createNever}, + {"any", createAny}, + {"boolean", createBoolean}, + {"number", createNumber}, + {"string", createString}, + {nullptr, nullptr} + }; + + // List of methods for type userdata + luaL_Reg typeUserdataMethods[] = { + {"singleton", createSingleton}, + {"negationof", createNegation}, + {"unionof", createUnion}, + {"intersectionof", createIntersection}, + {"newtable", createTable}, + {"newfunction", createFunction}, + {"copy", deepCopy}, + + // Common methods + {"is", checkTag}, + + // Negation type methods + {"inner", getNegatedValue}, + + // Singleton type methods + {"value", getSingletonValue}, + + // Table type methods + {"setproperty", setTableProp}, + {"setreadproperty", setReadTableProp}, + {"setwriteproperty", setWriteTableProp}, + {"readproperty", readTableProp}, + {"writeproperty", writeTableProp}, + {"properties", getProps}, + {"setindexer", setTableIndexer}, + {"setreadindexer", setTableReadIndexer}, + {"setwriteindexer", setTableWriteIndexer}, + {"indexer", getIndexer}, + {"readindexer", getReadIndexer}, + {"writeindexer", getWriteIndexer}, + {"setmetatable", setTableMetatable}, + {"metatable", getMetatable}, + + // Function type methods + {"setparameters", setFunctionParameters}, + {"parameters", getFunctionParameters}, + {"setreturns", setFunctionReturns}, + {"returns", getFunctionReturns}, + + // Union and Intersection type methods + {"components", getComponents}, + + // Class type methods + {"parent", getClassParent}, + {"indexer", getIndexer}, + {nullptr, nullptr} + }; + + // Create and register metatable for type userdata + luaL_newmetatable(L, "type"); + + // Protect metatable from being fetched. + lua_pushstring(L, "The metatable is locked"); + lua_setfield(L, -2, "__metatable"); + + // Set type userdata metatable's __eq to type_equals() + lua_pushcfunction(L, isEqualToType, "__eq"); + lua_setfield(L, -2, "__eq"); + + // Set type userdata metatable's __index to itself + lua_pushvalue(L, -1); // Push a copy of type userdata metatable + lua_setfield(L, -2, "__index"); + + luaL_register(L, nullptr, typeUserdataMethods); + + // Set fields for type userdata + for (luaL_Reg* l = typeUserdataFields; l->name; l++) + { + l->func(L); + lua_setfield(L, -2, l->name); + } + + // Set types library as a global name "types" + lua_setglobal(L, "types"); + } // Sets up a destructor for the type userdata. lua_setuserdatadtor(L, kTypeUserdataTag, deallocTypeUserData); diff --git a/Ast/src/Ast.cpp b/Ast/src/Ast.cpp index ff7c7cc6..f5deac2f 100644 --- a/Ast/src/Ast.cpp +++ b/Ast/src/Ast.cpp @@ -5,6 +5,11 @@ LUAU_FASTFLAG(LuauNativeAttribute); +// The default value here is 643 because the first release in which this was implemented is 644, +// and actively we want new changes to be off by default until they're enabled consciously. +// The flag is placed in AST project here to be common in all Luau libraries +LUAU_DYNAMIC_FASTINTVARIABLE(LuauTypeSolverRelease, 643) + namespace Luau { diff --git a/CLI/Flags.cpp b/CLI/Flags.cpp index ee5918c9..c0bb485f 100644 --- a/CLI/Flags.cpp +++ b/CLI/Flags.cpp @@ -2,11 +2,14 @@ #include "Luau/Common.h" #include "Luau/ExperimentalFlags.h" +#include // TODO: remove with LuauTypeSolverRelease #include #include #include +LUAU_DYNAMIC_FASTINT(LuauTypeSolverRelease) + static void setLuauFlag(std::string_view name, bool state) { for (Luau::FValue* flag = Luau::FValue::list; flag; flag = flag->next) @@ -23,6 +26,13 @@ static void setLuauFlag(std::string_view name, bool state) static void setLuauFlags(bool state) { + if (state) + { + // Setting flags to 'true' means enabling all Luau flags including new type solver + // In that case, it is provided with all fixes enabled (as if each fix had its own boolean flag) + DFInt::LuauTypeSolverRelease.value = std::numeric_limits::max(); + } + for (Luau::FValue* flag = Luau::FValue::list; flag; flag = flag->next) if (strncmp(flag->name, "Luau", 4) == 0) flag->value = state; diff --git a/CMakeLists.txt b/CMakeLists.txt index c8053cc1..51fa919e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -124,6 +124,8 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") # Some gcc versions treat var in `if (type var = val)` as unused # Some gcc versions treat variables used in constexpr if blocks as unused list(APPEND LUAU_OPTIONS -Wno-unused) + # some gcc versions warn maybe uninitialized on optional members on structs + list(APPEND LUAU_OPTIONS -Wno-maybe-uninitialized) endif() # Enabled in CI; we should be warning free on our main compiler versions but don't guarantee being warning free everywhere diff --git a/CodeGen/src/CodeGenContext.cpp b/CodeGen/src/CodeGenContext.cpp index a463091d..c3c58f78 100644 --- a/CodeGen/src/CodeGenContext.cpp +++ b/CodeGen/src/CodeGenContext.cpp @@ -346,13 +346,13 @@ void SharedCodeGenContextDeleter::operator()(const SharedCodeGenContext* codeGen return static_cast(L->global->ecb.context); } -static void onCloseState(lua_State* L) noexcept +static void onCloseState(lua_State* L) { getCodeGenContext(L)->onCloseState(); L->global->ecb = lua_ExecutionCallbacks{}; } -static void onDestroyFunction(lua_State* L, Proto* proto) noexcept +static void onDestroyFunction(lua_State* L, Proto* proto) { getCodeGenContext(L)->onDestroyFunction(proto->execdata); proto->execdata = nullptr; diff --git a/Makefile b/Makefile index cb199de8..1f082f38 100644 --- a/Makefile +++ b/Makefile @@ -82,8 +82,10 @@ LDFLAGS= # some gcc versions treat var in `if (type var = val)` as unused # some gcc versions treat variables used in constexpr if blocks as unused +# some gcc versions warn maybe uninitalized on optional members on structs ifeq ($(findstring g++,$(shell $(CXX) --version)),g++) CXXFLAGS+=-Wno-unused + CXXFLAGS+=-Wno-maybe-uninitialized endif # enabled in CI; we should be warning free on our main compiler versions but don't guarantee being warning free everywhere diff --git a/VM/include/lua.h b/VM/include/lua.h index 4ee9306e..dbb313c2 100644 --- a/VM/include/lua.h +++ b/VM/include/lua.h @@ -453,6 +453,8 @@ struct lua_Callbacks void (*debugstep)(lua_State* L, lua_Debug* ar); // gets called after each instruction in single step mode void (*debuginterrupt)(lua_State* L, lua_Debug* ar); // gets called when thread execution is interrupted by break in another thread void (*debugprotectederror)(lua_State* L); // gets called when protected call results in an error + + void (*onallocate)(lua_State* L, size_t osize, size_t nsize); // gets called when memory is allocated }; typedef struct lua_Callbacks lua_Callbacks; diff --git a/VM/src/lmem.cpp b/VM/src/lmem.cpp index 5ff5de72..f65d79dc 100644 --- a/VM/src/lmem.cpp +++ b/VM/src/lmem.cpp @@ -504,6 +504,11 @@ void* luaM_new_(lua_State* L, size_t nsize, uint8_t memcat) g->totalbytes += nsize; g->memcatbytes[memcat] += nsize; + if (LUAU_UNLIKELY(!!g->cb.onallocate)) + { + g->cb.onallocate(L, 0, nsize); + } + return block; } @@ -539,6 +544,11 @@ GCObject* luaM_newgco_(lua_State* L, size_t nsize, uint8_t memcat) g->totalbytes += nsize; g->memcatbytes[memcat] += nsize; + if (LUAU_UNLIKELY(!!g->cb.onallocate)) + { + g->cb.onallocate(L, 0, nsize); + } + return (GCObject*)block; } @@ -618,6 +628,12 @@ void* luaM_realloc_(lua_State* L, void* block, size_t osize, size_t nsize, uint8 LUAU_ASSERT((nsize == 0) == (result == NULL)); g->totalbytes = (g->totalbytes - osize) + nsize; g->memcatbytes[memcat] += nsize - osize; + + if (LUAU_UNLIKELY(!!g->cb.onallocate)) + { + g->cb.onallocate(L, osize, nsize); + } + return result; } diff --git a/bench/bench.py b/bench/bench.py index 002dfadb..bc06b663 100644 --- a/bench/bench.py +++ b/bench/bench.py @@ -508,6 +508,9 @@ def runTest(subdir, filename, filepath): filepath = os.path.abspath(filepath) mainVm = os.path.abspath(arguments.vm) + if not os.path.isfile(mainVm): + print(f"{colored(Color.RED, 'ERROR')}: VM executable '{mainVm}' does not exist.") + sys.exit(1) # Process output will contain the test name and execution times mainOutput = getVmOutput(substituteArguments(mainVm, getExtraArguments(filepath)) + " " + filepath) @@ -887,9 +890,11 @@ def run(args, argsubcb): analyzeResult('', mainResult, compareResults) else: all_files = [subdir + os.sep + filename for subdir, dirs, files in os.walk(arguments.folder) for filename in files] + if len(all_files) == 0: + print(f"{colored(Color.YELLOW, 'WARNING')}: No test files found in '{arguments.folder}'.") for filepath in sorted(all_files): subdir, filename = os.path.split(filepath) - if filename.endswith(".lua"): + if filename.endswith(".lua") or filename.endswith(".luau"): if arguments.run_test == None or re.match(arguments.run_test, filename[:-4]): runTest(subdir, filename, filepath) diff --git a/tests/Autocomplete.test.cpp b/tests/Autocomplete.test.cpp index 82e5c252..53c418cd 100644 --- a/tests/Autocomplete.test.cpp +++ b/tests/Autocomplete.test.cpp @@ -30,7 +30,7 @@ template struct ACFixtureImpl : BaseType { ACFixtureImpl() - : BaseType(true, true) + : BaseType(true) { } diff --git a/tests/Fixture.cpp b/tests/Fixture.cpp index 66e3ac30..bf254f80 100644 --- a/tests/Fixture.cpp +++ b/tests/Fixture.cpp @@ -25,9 +25,7 @@ static const char* mainModuleName = "MainModule"; LUAU_FASTFLAG(LuauSolverV2); -LUAU_FASTFLAG(DebugLuauFreezeArena); LUAU_FASTFLAG(DebugLuauLogSolverToJsonFile) -LUAU_FASTFLAG(LuauDCRMagicFunctionTypeChecker); LUAU_DYNAMIC_FASTINT(LuauTypeSolverRelease) extern std::optional randomSeed; // tests/main.cpp @@ -152,15 +150,8 @@ const Config& TestConfigResolver::getConfig(const ModuleName& name) const return defaultConfig; } -Fixture::Fixture(bool freeze, bool prepareAutocomplete) - : sff_DebugLuauFreezeArena(FFlag::DebugLuauFreezeArena, freeze) - , sff_LuauDCRMagicFunctionTypeChecker(FFlag::LuauDCRMagicFunctionTypeChecker, true) - // The first value of LuauTypeSolverRelease was 643, so as long as this is - // some number greater than 900 (5 years worth of releases), all tests that - // run under the new solver will run against all of the changes guarded by - // this flag. - , sff_LuauTypeSolverRelease(DFInt::LuauTypeSolverRelease, std::numeric_limits::max()) - , frontend( +Fixture::Fixture(bool prepareAutocomplete) + : frontend( &fileResolver, &configResolver, {/* retainFullTypeGraphs= */ true, /* forAutocomplete */ false, /* runLintChecks */ false, /* randomConstraintResolutionSeed */ randomSeed} @@ -583,8 +574,8 @@ LoadDefinitionFileResult Fixture::loadDefinition(const std::string& source) return result; } -BuiltinsFixture::BuiltinsFixture(bool freeze, bool prepareAutocomplete) - : Fixture(freeze, prepareAutocomplete) +BuiltinsFixture::BuiltinsFixture(bool prepareAutocomplete) + : Fixture(prepareAutocomplete) { Luau::unfreeze(frontend.globals.globalTypes); Luau::unfreeze(frontend.globalsForAutocomplete.globalTypes); diff --git a/tests/Fixture.h b/tests/Fixture.h index bd0db4ef..4f2050c1 100644 --- a/tests/Fixture.h +++ b/tests/Fixture.h @@ -25,6 +25,8 @@ #include #include +LUAU_FASTFLAG(DebugLuauFreezeArena) + namespace Luau { @@ -63,7 +65,7 @@ struct TestConfigResolver : ConfigResolver struct Fixture { - explicit Fixture(bool freeze = true, bool prepareAutocomplete = false); + explicit Fixture(bool prepareAutocomplete = false); ~Fixture(); // Throws Luau::ParseErrors if the parse fails. @@ -100,36 +102,14 @@ struct Fixture TypeId requireTypeAlias(const std::string& name); TypeId requireExportedType(const ModuleName& moduleName, const std::string& name); - // TODO: Should this be in a container of some kind? Seems a little silly - // to have a bunch of flags sitting on the text fixture. + // While most flags can be flipped inside the unit test, some code changes affect the state that is part of Fixture initialization + // Most often those are changes related to builtin type definitions. + // In that case, flag can be forced to 'true' using the example below: + // ScopedFastFlag sff_LuauExampleFlagDefinition{FFlag::LuauExampleFlagDefinition, true}; - // We have a couple flags that are OK to set for all tests and, in some - // cases, cannot easily be flipped on or off on a per-test basis. For these - // we set them as part of constructing the test fixture. - - /* From the original commit: - * - * > This enables arena freezing for all but two unit tests. Arena - * > freezing marks the `TypeArena`'s underlying memory as read-only, - * > raising an access violation whenever you mutate it. This is useful - * > for tracking down violations of Luau's memory model. - */ - ScopedFastFlag sff_DebugLuauFreezeArena; - - /* Magic typechecker functions for the new solver are initialized when the - * typechecker frontend is initialized, which is done at the beginning of - * the test: we set this flag as part of the fixture as we always want to - * enable the magic functions for, say, `string.format`. - */ - ScopedFastFlag sff_LuauDCRMagicFunctionTypeChecker; - - /* While the new solver is being rolled out we are using a monotonically - * increasing version number to track new changes, we just set it to a - * sufficiently high number in tests to ensure that any guards in prod - * code pass in tests (so we don't accidentally reintroduce a bug before - * it's unflagged). - */ - ScopedFastInt sff_LuauTypeSolverRelease; + // Arena freezing marks the `TypeArena`'s underlying memory as read-only, raising an access violation whenever you mutate it. + // This is useful for tracking down violations of Luau's memory model. + ScopedFastFlag sff_DebugLuauFreezeArena{FFlag::DebugLuauFreezeArena, true}; TestFileResolver fileResolver; TestConfigResolver configResolver; @@ -158,7 +138,7 @@ struct Fixture struct BuiltinsFixture : Fixture { - BuiltinsFixture(bool freeze = true, bool prepareAutocomplete = false); + BuiltinsFixture(bool prepareAutocomplete = false); }; std::optional pathExprToModuleName(const ModuleName& currentModuleName, const std::vector& segments); diff --git a/tests/SharedCodeAllocator.test.cpp b/tests/SharedCodeAllocator.test.cpp index bba8daad..13bf9f98 100644 --- a/tests/SharedCodeAllocator.test.cpp +++ b/tests/SharedCodeAllocator.test.cpp @@ -175,6 +175,8 @@ TEST_CASE("NativeModuleRefRefcounting") REQUIRE(modRefA->getRefcount() == 1); REQUIRE(modRefB->getRefcount() == 1); +#if defined(__linux__) && defined(__GNUC__) +#else // NativeModuleRef self move assignment: { NativeModuleRef modRef1{modRefA}; @@ -183,6 +185,8 @@ TEST_CASE("NativeModuleRefRefcounting") REQUIRE(modRefA->getRefcount() == 2); } +#endif + REQUIRE(modRefA->getRefcount() == 1); REQUIRE(modRefB->getRefcount() == 1); diff --git a/tests/TypeFunction.test.cpp b/tests/TypeFunction.test.cpp index fb53ee87..68bfff57 100644 --- a/tests/TypeFunction.test.cpp +++ b/tests/TypeFunction.test.cpp @@ -21,7 +21,7 @@ struct TypeFunctionFixture : Fixture TypeFunction swapFunction; TypeFunctionFixture() - : Fixture(true, false) + : Fixture(false) { swapFunction = TypeFunction{ /* name */ "Swap", diff --git a/tests/TypeFunction.user.test.cpp b/tests/TypeFunction.user.test.cpp index 72ba0fad..5a3dca10 100644 --- a/tests/TypeFunction.user.test.cpp +++ b/tests/TypeFunction.user.test.cpp @@ -11,6 +11,7 @@ LUAU_FASTFLAG(LuauSolverV2) LUAU_FASTFLAG(LuauUserDefinedTypeFunctionsSyntax2) LUAU_FASTFLAG(LuauUserDefinedTypeFunctions2) LUAU_FASTFLAG(LuauUserDefinedTypeFunctionNoEvaluation) +LUAU_FASTFLAG(LuauUserTypeFunFixRegister) TEST_SUITE_BEGIN("UserDefinedTypeFunctionTests"); @@ -1101,4 +1102,101 @@ TEST_CASE_FIXTURE(BuiltinsFixture, "udtf_strip_indexer") CHECK(toString(tpm->givenTp) == "{ foo: string }"); } +TEST_CASE_FIXTURE(BuiltinsFixture, "no_type_methods_on_types") +{ + ScopedFastFlag newSolver{FFlag::LuauSolverV2, true}; + ScopedFastFlag udtfSyntax{FFlag::LuauUserDefinedTypeFunctionsSyntax2, true}; + ScopedFastFlag udtf{FFlag::LuauUserDefinedTypeFunctions2, true}; + ScopedFastFlag luauUserTypeFunFixRegister{FFlag::LuauUserTypeFunFixRegister, true}; + + CheckResult result = check(R"( + type function test(x) + return if types.is(x, "number") then types.string else types.boolean + end + local function ok(tbl: test): never return tbl end + )"); + + LUAU_REQUIRE_ERROR_COUNT(4, result); + CHECK(toString(result.errors[0]) == R"('test' type function errored at runtime: [string "test"]:3: attempt to call a nil value)"); +} + +TEST_CASE_FIXTURE(BuiltinsFixture, "no_types_functions_on_type") +{ + ScopedFastFlag newSolver{FFlag::LuauSolverV2, true}; + ScopedFastFlag udtfSyntax{FFlag::LuauUserDefinedTypeFunctionsSyntax2, true}; + ScopedFastFlag udtf{FFlag::LuauUserDefinedTypeFunctions2, true}; + ScopedFastFlag luauUserTypeFunFixRegister{FFlag::LuauUserTypeFunFixRegister, true}; + + CheckResult result = check(R"( + type function test(x) + return x.singleton("a") + end + local function ok(tbl: test): never return tbl end + )"); + + LUAU_REQUIRE_ERROR_COUNT(4, result); + CHECK(toString(result.errors[0]) == R"('test' type function errored at runtime: [string "test"]:3: attempt to call a nil value)"); +} + +TEST_CASE_FIXTURE(BuiltinsFixture, "no_metatable_writes") +{ + ScopedFastFlag newSolver{FFlag::LuauSolverV2, true}; + ScopedFastFlag udtfSyntax{FFlag::LuauUserDefinedTypeFunctionsSyntax2, true}; + ScopedFastFlag udtf{FFlag::LuauUserDefinedTypeFunctions2, true}; + ScopedFastFlag luauUserTypeFunFixRegister{FFlag::LuauUserTypeFunFixRegister, true}; + + CheckResult result = check(R"( + type function test(x) + local a = x.__index + a.is = function() return false end + return types.singleton(x.is("number")) + end + local function ok(tbl: test): never return tbl end + )"); + + LUAU_REQUIRE_ERROR_COUNT(4, result); + CHECK(toString(result.errors[0]) == R"('test' type function errored at runtime: [string "test"]:4: attempt to index nil with 'is')"); +} + +TEST_CASE_FIXTURE(BuiltinsFixture, "no_eq_field") +{ + ScopedFastFlag newSolver{FFlag::LuauSolverV2, true}; + ScopedFastFlag udtfSyntax{FFlag::LuauUserDefinedTypeFunctionsSyntax2, true}; + ScopedFastFlag udtf{FFlag::LuauUserDefinedTypeFunctions2, true}; + ScopedFastFlag luauUserTypeFunFixRegister{FFlag::LuauUserTypeFunFixRegister, true}; + + CheckResult result = check(R"( + type function test(x) + return types.singleton(x.__eq(x, types.number)) + end + local function ok(tbl: test): never return tbl end + )"); + + LUAU_REQUIRE_ERROR_COUNT(4, result); + CHECK(toString(result.errors[0]) == R"('test' type function errored at runtime: [string "test"]:3: attempt to call a nil value)"); +} + +TEST_CASE_FIXTURE(BuiltinsFixture, "tag_field") +{ + ScopedFastFlag newSolver{FFlag::LuauSolverV2, true}; + ScopedFastFlag udtfSyntax{FFlag::LuauUserDefinedTypeFunctionsSyntax2, true}; + ScopedFastFlag udtf{FFlag::LuauUserDefinedTypeFunctions2, true}; + ScopedFastFlag luauUserTypeFunFixRegister{FFlag::LuauUserTypeFunFixRegister, true}; + + CheckResult result = check(R"( + type function test(x) + return types.singleton(x.tag) + end + + local function ok1(tbl: test): never return tbl end + local function ok2(tbl: test): never return tbl end + local function ok3(tbl: test<{}>): never return tbl end + )"); + + LUAU_REQUIRE_ERROR_COUNT(3, result); + CHECK(toString(result.errors[0]) == R"(Type pack '"number"' could not be converted into 'never'; at [0], "number" is not a subtype of never)"); + CHECK(toString(result.errors[1]) == R"(Type pack '"string"' could not be converted into 'never'; at [0], "string" is not a subtype of never)"); + CHECK(toString(result.errors[2]) == R"(Type pack '"table"' could not be converted into 'never'; at [0], "table" is not a subtype of never)"); +} + TEST_SUITE_END(); diff --git a/tests/TypeInfer.tables.test.cpp b/tests/TypeInfer.tables.test.cpp index eef76cab..c324901d 100644 --- a/tests/TypeInfer.tables.test.cpp +++ b/tests/TypeInfer.tables.test.cpp @@ -4866,4 +4866,29 @@ TEST_CASE_FIXTURE(BuiltinsFixture, "subtyping_with_a_metatable_table_path") ); } +TEST_CASE_FIXTURE(BuiltinsFixture, "metatable_union_type") +{ + + ScopedFastFlag _{FFlag::LuauSolverV2, true}; + + // This will have one (legitimate) error but previously would crash. + auto result = check(R"( + local function set(key, value) + local Message = {} + function Message.new(message) + local self = message or {} + setmetatable(self, Message) + return self + end + local self = Message.new(nil) + self[key] = value + end + )"); + LUAU_REQUIRE_ERROR_COUNT(1, result); + CHECK_EQ( + "Cannot add indexer to table '{ @metatable t1, (nil & ~(false?)) | { } } where t1 = { new: (a) -> { @metatable t1, (a & ~(false?)) | { } } }'", + toString(result.errors[0]) + ); +} + TEST_SUITE_END(); diff --git a/tests/main.cpp b/tests/main.cpp index 4612829b..d80589c1 100644 --- a/tests/main.cpp +++ b/tests/main.cpp @@ -27,9 +27,13 @@ #include #endif +#include // TODO: remove with LuauTypeSolverRelease #include + #include +LUAU_DYNAMIC_FASTINT(LuauTypeSolverRelease) + // Indicates if verbose output is enabled; can be overridden via --verbose // Currently, this enables output from 'print', but other verbose output could be enabled eventually. bool verbose = false; @@ -411,6 +415,12 @@ int main(int argc, char** argv) printf("Using RNG seed %u\n", *randomSeed); } + // New Luau type solver uses a temporary scheme where fixes are made under a single version flag + // When flags are enabled, new solver is enabled with all new features and fixes + // When it's disabled, this value should have no effect (all uses under a new solver) + // Flag setup argument can still be used to override this to a specific value if desired + DFInt::LuauTypeSolverRelease.value = std::numeric_limits::max(); + if (std::vector flags; doctest::parseCommaSepArgs(argc, argv, "--fflags=", flags)) setFastFlags(flags); diff --git a/tools/flag-bisect.py b/tools/flag-bisect.py index 01f3ef7c..55663a78 100644 --- a/tools/flag-bisect.py +++ b/tools/flag-bisect.py @@ -135,7 +135,7 @@ def add_argument_parsers(parser): interestness_parser.add_argument('--auto', dest='mode', action='store_const', const=InterestnessMode.AUTO, default=InterestnessMode.AUTO, help='Automatically figure out which one of --pass or --fail should be used') interestness_parser.add_argument('--fail', dest='mode', action='store_const', const=InterestnessMode.FAIL, - help='You want this if omitting --fflags=true causes tests to fail') + help='You want this if passing --fflags=true causes tests to fail') interestness_parser.add_argument('--pass', dest='mode', action='store_const', const=InterestnessMode.PASS, help='You want this if passing --fflags=true causes tests to pass') interestness_parser.add_argument('--timeout', dest='timeout', type=int, default=0, metavar='SECONDS',