luau/Analysis/include/Luau/Subtyping.h

// This file is part of the Luau programming language and is licensed under MIT License; see LICENSE.txt for details
#pragma once

#include "Luau/DenseHash.h"
#include "Luau/EqSatSimplification.h"
#include "Luau/Set.h"
#include "Luau/SubtypingVariance.h"
#include "Luau/TypeCheckLimits.h"
#include "Luau/TypeFunction.h"
#include "Luau/TypeFwd.h"
#include "Luau/TypePairHash.h"
#include "Luau/TypePath.h"

#include <vector>
#include <optional>

namespace Luau
{

template<typename A, typename B>
struct TryPair;
struct InternalErrorReporter;

class TypeIds;
class Normalizer;
struct NormalizedExternType;
struct NormalizedFunctionType;
struct NormalizedStringType;
struct NormalizedType;
struct Property;
struct Scope;
struct TableIndexer;
struct TypeArena;
struct TypeCheckLimits;

struct SubtypingReasoning
{
    // The path, relative to the _root subtype_, where subtyping failed.
    Path subPath;
    // The path, relative to the _root supertype_, where subtyping failed.
    Path superPath;
    SubtypingVariance variance = SubtypingVariance::Covariant;

    bool operator==(const SubtypingReasoning& other) const;
};

struct SubtypingReasoningHash
{
    size_t operator()(const SubtypingReasoning& r) const;
};

using SubtypingReasonings = DenseHashSet<SubtypingReasoning, SubtypingReasoningHash>;
inline const SubtypingReasoning kEmptyReasoning = SubtypingReasoning{TypePath::kEmpty, TypePath::kEmpty, SubtypingVariance::Invalid};

struct SubtypingResult
{
    bool isSubtype = false;
    bool normalizationTooComplex = false;
    bool isCacheable = true;
    ErrorVec errors;
    /// The reason for isSubtype to be false. May not be present even if
    /// isSubtype is false, depending on the input types.
    SubtypingReasonings reasoning{kEmptyReasoning};
    DenseHashMap<TypePackId, TypePackId> mappedGenericPacks{nullptr};

    // If this subtype result required testing free types, we might be making
    // assumptions about what the free type eventually resolves to.  If so,
    // those assumptions are recorded here.
    std::vector<SubtypeConstraint> assumedConstraints;

    /// If any generic bounds were invalid, report them here
    std::vector<GenericBoundsMismatch> genericBoundsMismatches;

    SubtypingResult& andAlso(const SubtypingResult& other);
    SubtypingResult& orElse(const SubtypingResult& other);
    SubtypingResult& withBothComponent(TypePath::Component component);
    SubtypingResult& withSuperComponent(TypePath::Component component);
    SubtypingResult& withSubComponent(TypePath::Component component);
    SubtypingResult& withBothPath(TypePath::Path path);
    SubtypingResult& withSubPath(TypePath::Path path);
    SubtypingResult& withSuperPath(TypePath::Path path);
    SubtypingResult& withErrors(ErrorVec& err);
    SubtypingResult& withError(TypeError err);

    // Only negates the `isSubtype`.
    static SubtypingResult negate(const SubtypingResult& result);
    static SubtypingResult all(const std::vector<SubtypingResult>& results);
    static SubtypingResult any(const std::vector<SubtypingResult>& results);
};

struct SubtypingEnvironment
{
    struct GenericBounds
    {
        TypeIds lowerBound;
        TypeIds upperBound;
    };

    // TODO: Clip with LuauSubtypingGenericsDoesntUseVariance
    struct GenericBounds_DEPRECATED
    {
        DenseHashSet<TypeId> lowerBound{nullptr};
        DenseHashSet<TypeId> upperBound{nullptr};
    };

    /* For nested subtyping relationship tests of mapped generic bounds, we keep the outer environment immutable */
    SubtypingEnvironment* parent = nullptr;

    /// Applies `mappedGenerics` to the given type.
    /// This is used specifically to substitute for generics in type function instances.
    std::optional<TypeId> applyMappedGenerics(
        NotNull<BuiltinTypes> builtinTypes,
        NotNull<TypeArena> arena,
        TypeId ty,
        NotNull<InternalErrorReporter> iceReporter
    );
    // TODO: Clip with LuauSubtypingGenericsDoesntUseVariance
    std::optional<TypeId> applyMappedGenerics_DEPRECATED(NotNull<BuiltinTypes> builtinTypes, NotNull<TypeArena> arena, TypeId ty);

    const TypeId* tryFindSubstitution(TypeId ty) const;
    // TODO: Clip with LuauSubtypingGenericsDoesntUseVariance
    const SubtypingResult* tryFindSubtypingResult(std::pair<TypeId, TypeId> subAndSuper) const;

    bool containsMappedType(TypeId ty) const;
    bool containsMappedPack(TypePackId tp) const;

    GenericBounds& getMappedTypeBounds(TypeId ty, NotNull<InternalErrorReporter> iceReporter);
    // TODO: Clip with LuauSubtypingGenericsDoesntUseVariance
    GenericBounds_DEPRECATED& getMappedTypeBounds_DEPRECATED(TypeId ty);
    TypePackId* getMappedPackBounds(TypePackId tp);

    /*
     * When we encounter a generic over the course of a subtyping test, we need
     * to tentatively map that generic onto a type on the other side. We map to a
     * vector of bounds, since generics may be shadowed by nested types. The back
     * of each vector represents the current scope.
     */
    DenseHashMap<TypeId, std::vector<GenericBounds>> mappedGenerics{nullptr};
    // TODO: Clip with LuauSubtypingGenericsDoesntUseVariance
    DenseHashMap<TypeId, GenericBounds_DEPRECATED> mappedGenerics_DEPRECATED{nullptr};
    DenseHashMap<TypePackId, TypePackId> mappedGenericPacks{nullptr};

    /*
     * See the test cyclic_tables_are_assumed_to_be_compatible_with_extern_types for
     * details.
     *
     * An empty value is equivalent to a nonexistent key.
     */
    DenseHashMap<TypeId, TypeId> substitutions{nullptr};

    // TODO: Clip with LuauSubtypingGenericsDoesntUseVariance
    DenseHashMap<std::pair<TypeId, TypeId>, SubtypingResult, TypePairHash> ephemeralCache{{}};

    // We use this cache to track pairs of subtypes that we tried to subtype, and found them to be in the seen set at the time.
    // In those situations, we return True, but mark the result as not cacheable, because we don't want to cache broader results which
    // led to the seen pair. However, those results were previously being cache in the ephemeralCache, and we still want to cache them somewhere
    // for performance reasons.
    DenseHashMap<std::pair<TypeId, TypeId>, SubtypingResult, TypePairHash> seenSetCache{{}};
};

struct Subtyping
{
    NotNull<BuiltinTypes> builtinTypes;
    NotNull<TypeArena> arena;
    NotNull<Simplifier> simplifier;
    NotNull<Normalizer> normalizer;
    NotNull<TypeFunctionRuntime> typeFunctionRuntime;
    NotNull<InternalErrorReporter> iceReporter;

    TypeCheckLimits limits;

    enum class Variance
    {
        Covariant,
        Contravariant
    };

    // TODO: Clip this along with LuauSubtypingGenericsDoesntUseVariance?
    Variance variance = Variance::Covariant;

    using SeenSet = Set<std::pair<TypeId, TypeId>, TypePairHash>;
    using SeenTypePackSet = Set<std::pair<TypePackId, TypePackId>, TypePairHash>;

    SeenSet seenTypes{{}};
    SeenTypePackSet seenPacks{{}};

    Subtyping(
        NotNull<BuiltinTypes> builtinTypes,
        NotNull<TypeArena> typeArena,
        NotNull<Simplifier> simplifier,
        NotNull<Normalizer> normalizer,
        NotNull<TypeFunctionRuntime> typeFunctionRuntime,
        NotNull<InternalErrorReporter> iceReporter
    );

    Subtyping(const Subtyping&) = delete;
    Subtyping& operator=(const Subtyping&) = delete;

    Subtyping(Subtyping&&) = default;
    Subtyping& operator=(Subtyping&&) = default;

    // Only used by unit tests to test that the cache works.
    const DenseHashMap<std::pair<TypeId, TypeId>, SubtypingResult, TypePairHash>& peekCache() const
    {
        return resultCache;
    }

    // TODO cache
    // TODO cyclic types
    // TODO recursion limits

    SubtypingResult isSubtype(TypeId subTy, TypeId superTy, NotNull<Scope> scope);
    SubtypingResult isSubtype(
        TypePackId subTp,
        TypePackId superTp,
        NotNull<Scope> scope,
        std::optional<std::vector<TypeId>> bindableGenerics = std::nullopt
    );

private:
    DenseHashMap<std::pair<TypeId, TypeId>, SubtypingResult, TypePairHash> resultCache{{}};

    SubtypingResult cache(SubtypingEnvironment& env, SubtypingResult res, TypeId subTy, TypeId superTy);

    SubtypingResult isCovariantWith(SubtypingEnvironment& env, TypeId subTy, TypeId superTy, NotNull<Scope> scope);
    SubtypingResult isCovariantWith(SubtypingEnvironment& env, TypePackId subTp, TypePackId superTp, NotNull<Scope> scope);

    template<typename SubTy, typename SuperTy>
    SubtypingResult isContravariantWith(SubtypingEnvironment& env, SubTy&& subTy, SuperTy&& superTy, NotNull<Scope> scope);

    template<typename SubTy, typename SuperTy>
    SubtypingResult isInvariantWith(SubtypingEnvironment& env, SubTy&& subTy, SuperTy&& superTy, NotNull<Scope> scope);

    template<typename SubTy, typename SuperTy>
    SubtypingResult isCovariantWith(SubtypingEnvironment& env, const TryPair<const SubTy*, const SuperTy*>& pair, NotNull<Scope> scope);

    template<typename SubTy, typename SuperTy>
    SubtypingResult isContravariantWith(SubtypingEnvironment& env, const TryPair<const SubTy*, const SuperTy*>& pair, NotNull<Scope>);

    template<typename SubTy, typename SuperTy>
    SubtypingResult isInvariantWith(SubtypingEnvironment& env, const TryPair<const SubTy*, const SuperTy*>& pair, NotNull<Scope>);

    SubtypingResult isCovariantWith(SubtypingEnvironment& env, TypeId subTy, const UnionType* superUnion, NotNull<Scope> scope);
    SubtypingResult isCovariantWith(SubtypingEnvironment& env, const UnionType* subUnion, TypeId superTy, NotNull<Scope> scope);
    SubtypingResult isCovariantWith(SubtypingEnvironment& env, TypeId subTy, const IntersectionType* superIntersection, NotNull<Scope> scope);
    SubtypingResult isCovariantWith(SubtypingEnvironment& env, const IntersectionType* subIntersection, TypeId superTy, NotNull<Scope> scope);

    SubtypingResult isCovariantWith(SubtypingEnvironment& env, const NegationType* subNegation, TypeId superTy, NotNull<Scope> scope);
    SubtypingResult isCovariantWith(SubtypingEnvironment& env, const TypeId subTy, const NegationType* superNegation, NotNull<Scope> scope);

    SubtypingResult isCovariantWith(SubtypingEnvironment& env, const PrimitiveType* subPrim, const PrimitiveType* superPrim, NotNull<Scope> scope);
    SubtypingResult isCovariantWith(
        SubtypingEnvironment& env,
        const SingletonType* subSingleton,
        const PrimitiveType* superPrim,
        NotNull<Scope> scope
    );
    SubtypingResult isCovariantWith(
        SubtypingEnvironment& env,
        const SingletonType* subSingleton,
        const SingletonType* superSingleton,
        NotNull<Scope> scope
    );
    SubtypingResult isCovariantWith(SubtypingEnvironment& env, const TableType* subTable, const TableType* superTable, NotNull<Scope> scope);
    SubtypingResult isCovariantWith(SubtypingEnvironment& env, const MetatableType* subMt, const MetatableType* superMt, NotNull<Scope> scope);
    SubtypingResult isCovariantWith(SubtypingEnvironment& env, const MetatableType* subMt, const TableType* superTable, NotNull<Scope> scope);
    SubtypingResult isCovariantWith(
        SubtypingEnvironment& env,
        const ExternType* subExternType,
        const ExternType* superExternType,
        NotNull<Scope> scope
    );
    SubtypingResult
    isCovariantWith(SubtypingEnvironment& env, TypeId subTy, const ExternType* subExternType, TypeId superTy, const TableType* superTable, NotNull<Scope>);
    SubtypingResult isCovariantWith(
        SubtypingEnvironment& env,
        const FunctionType* subFunction,
        const FunctionType* superFunction,
        NotNull<Scope> scope
    );
    SubtypingResult isCovariantWith(SubtypingEnvironment& env, const TableType* subTable, const PrimitiveType* superPrim, NotNull<Scope> scope);
    SubtypingResult isCovariantWith(SubtypingEnvironment& env, const PrimitiveType* subPrim, const TableType* superTable, NotNull<Scope> scope);
    SubtypingResult isCovariantWith(SubtypingEnvironment& env, const SingletonType* subSingleton, const TableType* superTable, NotNull<Scope> scope);

    SubtypingResult isCovariantWith(
        SubtypingEnvironment& env,
        const TableIndexer& subIndexer,
        const TableIndexer& superIndexer,
        NotNull<Scope> scope
    );
    SubtypingResult
    isCovariantWith(SubtypingEnvironment& env, const Property& subProperty, const Property& superProperty, const std::string& name, NotNull<Scope>);

    SubtypingResult isCovariantWith(
        SubtypingEnvironment& env,
        const std::shared_ptr<const NormalizedType>& subNorm,
        const std::shared_ptr<const NormalizedType>& superNorm,
        NotNull<Scope> scope
    );
    SubtypingResult isCovariantWith(
        SubtypingEnvironment& env,
        const NormalizedExternType& subExternType,
        const NormalizedExternType& superExternType,
        NotNull<Scope> scope
    );
    SubtypingResult isCovariantWith(
        SubtypingEnvironment& env,
        const NormalizedExternType& subExternType,
        const TypeIds& superTables,
        NotNull<Scope> scope
    );
    SubtypingResult isCovariantWith(
        SubtypingEnvironment& env,
        const NormalizedStringType& subString,
        const NormalizedStringType& superString,
        NotNull<Scope> scope
    );
    SubtypingResult isCovariantWith(
        SubtypingEnvironment& env,
        const NormalizedStringType& subString,
        const TypeIds& superTables,
        NotNull<Scope> scope
    );
    SubtypingResult
    isCovariantWith(SubtypingEnvironment& env, const NormalizedFunctionType& subFunction, const NormalizedFunctionType& superFunction, NotNull<Scope>);
    SubtypingResult isCovariantWith(SubtypingEnvironment& env, const TypeIds& subTypes, const TypeIds& superTypes, NotNull<Scope> scope);

    SubtypingResult isCovariantWith(
        SubtypingEnvironment& env,
        const VariadicTypePack* subVariadic,
        const VariadicTypePack* superVariadic,
        NotNull<Scope> scope
    );
    SubtypingResult isCovariantWith(
        SubtypingEnvironment& env,
        const TypeFunctionInstanceType* subFunctionInstance,
        const TypeId superTy,
        NotNull<Scope> scope
    );
    SubtypingResult isCovariantWith(
        SubtypingEnvironment& env,
        const TypeId subTy,
        const TypeFunctionInstanceType* superFunctionInstance,
        NotNull<Scope> scope
    );

    bool bindGeneric(SubtypingEnvironment& env, TypeId subTp, TypeId superTp);
    bool bindGeneric(SubtypingEnvironment& env, TypePackId subTp, TypePackId superTp);

    template<typename T, typename Container>
    TypeId makeAggregateType(const Container& container, TypeId orElse);

    std::pair<TypeId, ErrorVec> handleTypeFunctionReductionResult(const TypeFunctionInstanceType* functionInstance, NotNull<Scope> scope);

    [[noreturn]] void unexpected(TypeId ty);
    [[noreturn]] void unexpected(TypePackId tp);

    SubtypingResult trySemanticSubtyping(SubtypingEnvironment& env,
                                         TypeId subTy,
                                         TypeId superTy,
                                         NotNull<Scope> scope,
                                         SubtypingResult& original);

    SubtypingResult checkGenericBounds(
        const SubtypingEnvironment::GenericBounds& bounds,
        SubtypingEnvironment& env,
        NotNull<Scope> scope,
        std::string_view genericName
    );

    // TODO: Clip with LuauSubtypingReportGenericBoundMismatches
    SubtypingResult checkGenericBounds_DEPRECATED(const SubtypingEnvironment::GenericBounds& bounds, SubtypingEnvironment& env, NotNull<Scope> scope);

    static void maybeUpdateBounds(
        TypeId here,
        TypeId there,
        TypeIds& boundsToUpdate,
        const TypeIds& firstBoundsToCheck,
        const TypeIds& secondBoundsToCheck
    );
};

} // namespace Luau