query_common.h

#pragma once
#include "gaia/config/config.h"
 
#include <type_traits>
 
#include "gaia/cnt/darray.h"
#include "gaia/cnt/map.h"
#include "gaia/cnt/sarray.h"
#include "gaia/cnt/sarray_ext.h"
#include "gaia/core/bit_utils.h"
#include "gaia/core/hashing_policy.h"
#include "gaia/core/span.h"
#include "gaia/core/utility.h"
#include "gaia/ecs/api.h"
#include "gaia/ecs/component.h"
#include "gaia/ecs/id.h"
#include "gaia/ecs/query_fwd.h"
#include "gaia/ecs/query_mask.h"
 
namespace gaia {
  namespace ecs {
    class World;
    class Archetype;
    using InheritedTermDataView = std::span<const void* const>;
    GAIA_NODISCARD uint32_t world_rel_version(const World& world, Entity relation);
    GAIA_NODISCARD bool world_has_entity_term(const World& world, Entity entity, Entity term);
    GAIA_NODISCARD bool world_has_entity_term_in(const World& world, Entity entity, Entity term);
    GAIA_NODISCARD bool world_has_entity_term_direct(const World& world, Entity entity, Entity term);
    GAIA_NODISCARD bool world_term_uses_inherit_policy(const World& world, Entity term);
    GAIA_NODISCARD bool world_relation_uses_non_fragmenting_storage(const World& world, Entity relation);
    GAIA_NODISCARD bool world_component_uses_sparse_storage(const World& world, Entity component);
    GAIA_NODISCARD bool world_component_is_non_fragmenting(const World& world, Entity component);
    GAIA_NODISCARD uint32_t world_count_direct_term_entities(const World& world, Entity term);
    GAIA_NODISCARD uint32_t world_count_in_term_entities(const World& world, Entity term);
    GAIA_NODISCARD uint32_t world_count_direct_term_entities_direct(const World& world, Entity term);
    void world_collect_direct_term_entities(const World& world, Entity term, cnt::darray<Entity>& out);
    void world_collect_in_term_entities(const World& world, Entity term, cnt::darray<Entity>& out);
    void world_collect_direct_term_entities_direct(const World& world, Entity term, cnt::darray<Entity>& out);
    GAIA_NODISCARD bool
    world_each_direct_term_entity(const World& world, Entity term, void* ctx, bool (*func)(void*, Entity));
    GAIA_NODISCARD bool
    world_each_in_term_entity(const World& world, Entity term, void* ctx, bool (*func)(void*, Entity));
    GAIA_NODISCARD bool
    world_each_direct_term_entity_direct(const World& world, Entity term, void* ctx, bool (*func)(void*, Entity));
    GAIA_NODISCARD bool world_entity_enabled(const World& world, Entity entity);
    GAIA_NODISCARD bool world_entity_prefab(const World& world, Entity entity);
    GAIA_NODISCARD const Archetype* world_entity_archetype(const World& world, Entity entity);
    void world_finish_write(World& world, Entity term, Entity entity);
    GAIA_NODISCARD uint32_t world_component_index_bucket_size(const World& world, Entity term);
    GAIA_NODISCARD uint32_t world_component_index_comp_idx(const World& world, const Archetype& archetype, Entity term);
    GAIA_NODISCARD uint32_t
    world_component_index_match_count(const World& world, const Archetype& archetype, Entity term);
    GAIA_NODISCARD GroupId group_by_func_depth_order(const World& world, const Archetype& archetype, Entity relation);
    template <typename T>
    GAIA_NODISCARD decltype(auto) world_direct_entity_arg(World& world, Entity entity);
    template <typename T>
    GAIA_NODISCARD decltype(auto) world_query_entity_arg(World& world, Entity entity);
    template <typename T>
    GAIA_NODISCARD Entity world_query_arg_id(World& world);
    template <typename T>
    GAIA_NODISCARD decltype(auto) world_query_entity_arg_by_id(World& world, Entity entity, Entity id);
    template <typename T>
    GAIA_NODISCARD decltype(auto) world_query_entity_arg_by_id_raw(World& world, Entity entity, Entity id);
    GAIA_NODISCARD uint32_t world_entity_archetype_version(const World& world, Entity entity);
 
    static constexpr uint32_t MAX_ITEMS_IN_QUERY = 12U;
    static constexpr uint32_t MAX_TRAV_DEPTH = 128U;
 
    GAIA_GCC_WARNING_PUSH()
    // GCC is unnecessarily too strict about shadowing.
    // We have a globally defined entity All and thinks QueryOpKind::All shadows it.
    GAIA_GCC_WARNING_DISABLE("-Wshadow")
 
    enum class QueryOpKind : uint8_t {
      All,
      Or,
      Not,
      Any,
      Count
    };
    enum class QueryAccess : uint8_t {
      None,
      Read,
      Write,
      Match
    };
    enum class QueryInputFlags : uint8_t {
      None,
      Variable
    };
    enum class QueryTravKind : uint8_t {
      None = 0x00,
      Self = 0x01,
      Up = 0x02,
      Down = 0x04
    };
 
    GAIA_GCC_WARNING_POP()
 
    
    GAIA_NODISCARD constexpr QueryTravKind operator|(QueryTravKind lhs, QueryTravKind rhs) {
      return (QueryTravKind)((uint8_t)lhs | (uint8_t)rhs);
    }
 
    GAIA_NODISCARD constexpr bool query_trav_has(QueryTravKind value, QueryTravKind bit) {
      return (((uint8_t)value) & ((uint8_t)bit)) != 0;
    }
 
    using QueryLookupHash = core::direct_hash_key<uint64_t>;
    using QueryEntityArray = cnt::sarray<Entity, MAX_ITEMS_IN_QUERY>;
    using QuerySerMap = cnt::map<QueryId, QuerySerBuffer>;
 
    struct QueryArchetypeCacheCursor {
      uint32_t index = 0;
      uint32_t revision = 0;
    };
 
    using QueryArchetypeCacheIndexMap = cnt::map<EntityLookupKey, QueryArchetypeCacheCursor>;
    using EntityToArchetypeVersionMap = cnt::map<EntityLookupKey, uint32_t>;
 
    static constexpr uint16_t ComponentIndexBad = (uint16_t)-1;
 
    struct ComponentIndexEntry {
      Archetype* pArchetype = nullptr;
      uint16_t compIdx = ComponentIndexBad;
      uint16_t matchCount = 0;
 
      GAIA_NODISCARD bool matches(const Archetype* pOther) const {
        return pArchetype == pOther;
      }
    };
 
    class ComponentIndexEntryArray {
    public:
      using value_type = ComponentIndexEntry;
      using size_type = uint32_t;
 
    private:
      ComponentIndexEntry m_inline{};
      cnt::darray<ComponentIndexEntry> m_items;
      size_type m_size = 0;
 
      void collapse_to_inline_if_needed() {
        if (m_size != 1 || m_items.empty())
          return;
 
        m_inline = m_items[0];
        m_items.clear();
      }
 
    public:
      ComponentIndexEntryArray() = default;
      ComponentIndexEntryArray(ComponentIndexEntryArray&&) noexcept = default;
      ComponentIndexEntryArray(const ComponentIndexEntryArray&) = default;
      ComponentIndexEntryArray& operator=(ComponentIndexEntryArray&&) noexcept = default;
      ComponentIndexEntryArray& operator=(const ComponentIndexEntryArray&) = default;
 
      GAIA_NODISCARD bool empty() const noexcept {
        return m_size == 0;
      }
 
      GAIA_NODISCARD size_type size() const noexcept {
        return m_size;
      }
 
      GAIA_NODISCARD ComponentIndexEntry* data() noexcept {
        return m_size <= 1 ? &m_inline : m_items.data();
      }
 
      GAIA_NODISCARD const ComponentIndexEntry* data() const noexcept {
        return m_size <= 1 ? &m_inline : m_items.data();
      }
 
      GAIA_NODISCARD ComponentIndexEntry* begin() noexcept {
        return data();
      }
 
      GAIA_NODISCARD const ComponentIndexEntry* begin() const noexcept {
        return data();
      }
 
      GAIA_NODISCARD ComponentIndexEntry* end() noexcept {
        return data() + m_size;
      }
 
      GAIA_NODISCARD const ComponentIndexEntry* end() const noexcept {
        return data() + m_size;
      }
 
      GAIA_NODISCARD ComponentIndexEntry& operator[](size_type idx) noexcept {
        GAIA_ASSERT(idx < m_size);
        return data()[idx];
      }
 
      GAIA_NODISCARD const ComponentIndexEntry& operator[](size_type idx) const noexcept {
        GAIA_ASSERT(idx < m_size);
        return data()[idx];
      }
 
      GAIA_NODISCARD ComponentIndexEntry& back() noexcept {
        GAIA_ASSERT(m_size > 0);
        return (*this)[m_size - 1];
      }
 
      GAIA_NODISCARD const ComponentIndexEntry& back() const noexcept {
        GAIA_ASSERT(m_size > 0);
        return (*this)[m_size - 1];
      }
 
      void push_back(ComponentIndexEntry entry) {
        if (m_size == 0) {
          m_inline = entry;
          m_size = 1;
          return;
        }
 
        if (m_size == 1) {
          m_items.push_back(m_inline);
          m_items.push_back(entry);
          m_size = 2;
          return;
        }
 
        m_items.push_back(entry);
        ++m_size;
      }
 
      void pop_back() {
        GAIA_ASSERT(m_size > 0);
        if (m_size == 1) {
          m_size = 0;
          return;
        }
 
        m_items.pop_back();
        --m_size;
        collapse_to_inline_if_needed();
      }
    };
 
    struct SingleArchetypeLookupItem {
      EntityLookupKey key = EntityBadLookupKey;
      ComponentIndexEntry entry{};
 
      GAIA_NODISCARD bool matches(EntityLookupKey other) const {
        return key == other;
      }
    };
 
    using SingleArchetypeLookup = cnt::sarray_ext<SingleArchetypeLookupItem, ChunkHeader::MAX_COMPONENTS * 4>;
 
    static constexpr QueryId QueryIdBad = (QueryId)-1;
    static constexpr GroupId GroupIdMax = ((GroupId)-1) - 1;
 
    struct QueryHandle {
      static constexpr uint32_t IdMask = QueryIdBad;
 
    private:
      struct HandleData {
        QueryId id;
        uint32_t gen;
      };
 
      union {
        HandleData data;
        uint64_t val;
      };
 
    public:
      constexpr QueryHandle() noexcept: val((uint64_t)-1) {}
 
      QueryHandle(QueryId id, uint32_t gen) {
        data.id = id;
        data.gen = gen;
      }
      ~QueryHandle() = default;
 
      QueryHandle(QueryHandle&&) noexcept = default;
      QueryHandle(const QueryHandle&) = default;
      QueryHandle& operator=(QueryHandle&&) noexcept = default;
      QueryHandle& operator=(const QueryHandle&) = default;
 
      GAIA_NODISCARD constexpr bool operator==(const QueryHandle& other) const noexcept {
        return val == other.val;
      }
      GAIA_NODISCARD constexpr bool operator!=(const QueryHandle& other) const noexcept {
        return val != other.val;
      }
 
      GAIA_NODISCARD auto id() const {
        return data.id;
      }
      GAIA_NODISCARD auto gen() const {
        return data.gen;
      }
      GAIA_NODISCARD auto value() const {
        return val;
      }
    };
 
    inline static const QueryHandle QueryHandleBad = QueryHandle();
 
    struct QueryHandleLookupKey {
      using LookupHash = core::direct_hash_key<uint64_t>;
 
    private:
      QueryHandle m_handle;
      LookupHash m_hash;
 
      static LookupHash calc(QueryHandle handle) {
        return {core::calculate_hash64(handle.value())};
      }
 
    public:
      static constexpr bool IsDirectHashKey = true;
 
      QueryHandleLookupKey() = default;
      explicit QueryHandleLookupKey(QueryHandle handle): m_handle(handle), m_hash(calc(handle)) {}
      ~QueryHandleLookupKey() = default;
 
      QueryHandleLookupKey(const QueryHandleLookupKey&) = default;
      QueryHandleLookupKey(QueryHandleLookupKey&&) = default;
      QueryHandleLookupKey& operator=(const QueryHandleLookupKey&) = default;
      QueryHandleLookupKey& operator=(QueryHandleLookupKey&&) = default;
 
      QueryHandle handle() const {
        return m_handle;
      }
 
      size_t hash() const {
        return (size_t)m_hash.hash;
      }
 
      bool operator==(const QueryHandleLookupKey& other) const {
        if GAIA_LIKELY (m_hash != other.m_hash)
          return false;
 
        return m_handle == other.m_handle;
      }
 
      bool operator!=(const QueryHandleLookupKey& other) const {
        return !operator==(other);
      }
    };
 
    inline static const QueryHandleLookupKey QueryHandleBadLookupKey = QueryHandleLookupKey(QueryHandleBad);
 
    struct QueryInput {
      static constexpr uint8_t TravDepthUnlimited = 0;
 
      QueryOpKind op = QueryOpKind::All;
      QueryAccess access = QueryAccess::Read;
      Entity id;
      Entity entSrc = EntityBad;
      Entity entTrav = EntityBad;
      QueryTravKind travKind = QueryTravKind::Self | QueryTravKind::Up;
      uint8_t travDepth = TravDepthUnlimited;
      QueryMatchKind matchKind = QueryMatchKind::Semantic;
    };
 
    struct QueryTermOptions {
      static constexpr uint8_t TravDepthUnlimited = QueryInput::TravDepthUnlimited;
 
      Entity entSrc = EntityBad;
      Entity entTrav = EntityBad;
      QueryTravKind travKind = QueryTravKind::Self | QueryTravKind::Up;
      uint8_t travDepth = TravDepthUnlimited;
      QueryAccess access = QueryAccess::None;
      QueryMatchKind matchKind = QueryMatchKind::Semantic;
 
      QueryTermOptions& src(Entity source) {
        entSrc = source;
        return *this;
      }
 
      QueryTermOptions& trav(Entity relation = ChildOf) {
        entTrav = relation;
        travKind = QueryTravKind::Self | QueryTravKind::Up;
        travDepth = TravDepthUnlimited;
        return *this;
      }
 
      QueryTermOptions& trav_up(Entity relation = ChildOf) {
        entTrav = relation;
        travKind = QueryTravKind::Up;
        travDepth = TravDepthUnlimited;
        return *this;
      }
 
      QueryTermOptions& trav_parent(Entity relation = ChildOf) {
        entTrav = relation;
        travKind = QueryTravKind::Up;
        travDepth = 1;
        return *this;
      }
 
      QueryTermOptions& trav_self_parent(Entity relation = ChildOf) {
        entTrav = relation;
        travKind = QueryTravKind::Self | QueryTravKind::Up;
        travDepth = 1;
        return *this;
      }
 
      QueryTermOptions& trav_down(Entity relation = ChildOf) {
        entTrav = relation;
        travKind = QueryTravKind::Down;
        travDepth = TravDepthUnlimited;
        return *this;
      }
 
      QueryTermOptions& trav_self_down(Entity relation = ChildOf) {
        entTrav = relation;
        travKind = QueryTravKind::Self | QueryTravKind::Down;
        travDepth = TravDepthUnlimited;
        return *this;
      }
 
      QueryTermOptions& trav_child(Entity relation = ChildOf) {
        entTrav = relation;
        travKind = QueryTravKind::Down;
        travDepth = 1;
        return *this;
      }
 
      QueryTermOptions& trav_self_child(Entity relation = ChildOf) {
        entTrav = relation;
        travKind = QueryTravKind::Self | QueryTravKind::Down;
        travDepth = 1;
        return *this;
      }
 
      QueryTermOptions& trav_kind(QueryTravKind kind) {
        travKind = kind;
        return *this;
      }
 
      QueryTermOptions& trav_depth(uint8_t maxDepth) {
        travDepth = maxDepth;
        return *this;
      }
 
      QueryTermOptions& no_access() {
        access = QueryAccess::Match;
        return *this;
      }
 
      QueryTermOptions& read() {
        access = QueryAccess::Read;
        return *this;
      }
 
      QueryTermOptions& write() {
        access = QueryAccess::Write;
        return *this;
      }
 
      QueryTermOptions& direct() {
        matchKind = QueryMatchKind::Direct;
        return *this;
      }
 
      QueryTermOptions& in() {
        matchKind = QueryMatchKind::In;
        return *this;
      }
    };
 
    struct QueryAccessSet {
      cnt::sarray<Entity, MAX_ITEMS_IN_QUERY> reads;
      cnt::sarray<Entity, MAX_ITEMS_IN_QUERY> writes;
      uint8_t readCnt = 0;
      uint8_t writeCnt = 0;
      bool hasPairPatterns = false;
 
      GAIA_NODISCARD std::span<const Entity> reads_view() const {
        return {reads.data(), readCnt};
      }
 
      GAIA_NODISCARD std::span<const Entity> writes_view() const {
        return {writes.data(), writeCnt};
      }
 
      void add_read(Entity entity) {
        if (entity == EntityBad || core::has(reads_view(), entity))
          return;
 
        GAIA_ASSERT(readCnt < MAX_ITEMS_IN_QUERY);
        if (readCnt < MAX_ITEMS_IN_QUERY) {
          reads[readCnt++] = entity;
          hasPairPatterns |= entity.pair() && (is_wildcard(entity) || is_variable((EntityId)entity.id()) ||
                                               is_variable((EntityId)entity.gen()));
        }
      }
 
      void add_write(Entity entity) {
        if (entity == EntityBad || core::has(writes_view(), entity))
          return;
 
        GAIA_ASSERT(writeCnt < MAX_ITEMS_IN_QUERY);
        if (writeCnt < MAX_ITEMS_IN_QUERY) {
          writes[writeCnt++] = entity;
          hasPairPatterns |= entity.pair() && (is_wildcard(entity) || is_variable((EntityId)entity.id()) ||
                                               is_variable((EntityId)entity.gen()));
        }
      }
 
      GAIA_NODISCARD QueryAccess access(Entity entity) const {
        if (core::has(writes_view(), entity))
          return QueryAccess::Write;
        if (core::has(reads_view(), entity))
          return QueryAccess::Read;
        return QueryAccess::None;
      }
    };
 
    struct QueryTerm {
      Entity id;
      Entity src;
      Entity entTrav;
      QueryTravKind travKind;
      uint8_t travDepth;
      QueryMatchKind matchKind;
      Archetype* srcArchetype;
      QueryOpKind op;
      uint8_t fieldIndex = 0;
      QueryAccess access = QueryAccess::None;
 
      bool operator==(const QueryTerm& other) const {
        return id == other.id && src == other.src && entTrav == other.entTrav && travKind == other.travKind &&
               travDepth == other.travDepth && matchKind == other.matchKind && op == other.op;
      }
      bool operator!=(const QueryTerm& other) const {
        return !operator==(other);
      }
    };
 
    constexpr bool query_term_less_for_lookup(const QueryTerm& lhs, const QueryTerm& rhs) {
      if (lhs.op != rhs.op)
        return lhs.op < rhs.op;
 
      if (lhs.id != rhs.id)
        return SortComponentCond()(lhs.id, rhs.id);
 
      if (lhs.src != rhs.src)
        return SortComponentCond()(lhs.src, rhs.src);
 
      if (lhs.entTrav != rhs.entTrav)
        return SortComponentCond()(lhs.entTrav, rhs.entTrav);
 
      if (lhs.travKind != rhs.travKind)
        return (uint8_t)lhs.travKind < (uint8_t)rhs.travKind;
 
      if (lhs.travDepth != rhs.travDepth)
        return lhs.travDepth < rhs.travDepth;
 
      return (uint8_t)lhs.matchKind < (uint8_t)rhs.matchKind;
    }
 
    inline void canonicalize_lookup_terms(std::span<QueryTerm> terms) {
      const auto idsCnt = (uint32_t)terms.size();
      if (idsCnt > 0) {
        uint32_t orCnt = 0;
        uint32_t orIdx = BadIndex;
        GAIA_FOR(idsCnt) {
          if (terms[i].op != QueryOpKind::Or)
            continue;
          ++orCnt;
          orIdx = i;
          if (orCnt > 1)
            break;
        }
 
        if (orCnt == 1)
          terms[orIdx].op = QueryOpKind::All;
      }
 
      core::sort(terms.data(), terms.data() + idsCnt, [](const QueryTerm& left, const QueryTerm& right) {
        return query_term_less_for_lookup(left, right);
      });
    }
 
    inline void canonicalize_lookup_changed(std::span<Entity> changed) {
      const auto changedCnt = (uint32_t)changed.size();
      if (changedCnt > 1)
        core::sort(changed.data(), changed.data() + changedCnt, SortComponentCond{});
    }
 
    GAIA_NODISCARD inline bool term_has_variables(const QueryTerm& term) {
      if (is_variable(term.src))
        return true;
 
      if (term.id.pair())
        return is_variable(EntityId(term.id.id())) || is_variable(EntityId(term.id.gen()));
 
      return is_variable(EntityId(term.id.id()));
    }
 
    GAIA_NODISCARD inline bool query_term_maps_to_current_archetype(const QueryTerm& term) {
      return term.src == EntityBad && term.entTrav == EntityBad && !term_has_variables(term);
    }
 
    GAIA_NODISCARD inline bool query_term_uses_potential_inherited_id_matching(const QueryTerm& term) {
      const auto id = term.id;
      return term.matchKind == QueryMatchKind::Semantic && term.src == EntityBad && term.entTrav == EntityBad &&
             !term_has_variables(term) && !is_wildcard(id) && !is_variable((EntityId)id.id()) &&
             (!id.pair() || !is_variable((EntityId)id.gen()));
    }
 
    using QueryTermArray = cnt::sarray_ext<QueryTerm, MAX_ITEMS_IN_QUERY>;
    using QueryTermSpan = std::span<QueryTerm>;
    using QueryRemappingArray = cnt::sarray_ext<uint8_t, MAX_ITEMS_IN_QUERY>;
 
    struct QueryIdentity {
      QueryHandle handle = {};
      QueryId serId = QueryIdBad;
 
      GAIA_NODISCARD QuerySerBuffer& ser_buffer(World* world) {
        return query_buffer(*world, serId);
      }
      void ser_buffer_reset(World* world) {
        query_buffer_reset(*world, serId);
      }
    };
 
    struct QueryCtx {
      const World* w{};
      ComponentCache* cc{};
      QueryLookupHash hashLookup{};
      QueryIdentity q{};
 
      enum QueryFlags : uint16_t {
        Empty = 0x00,
        SortEntities = 0x01,
        SortGroups = 0x02,
        Complex = 0x04,
        Recompile = 0x08,
        HasSourceTerms = 0x10,
        HasVariableTerms = 0x20,
        MatchPrefab = 0x40,
        HasPrefabTerms = 0x80,
        OrderGroups = 0x100,
        HasMatchTerms = 0x200,
        HasPairTerms = 0x400,
      };
 
      enum class CachePolicy : uint8_t {
        Immediate,
        Lazy,
        Dynamic,
      };
 
      enum class CreateArchetypeMatchKind : uint8_t {
        Vm,
        DirectStructuralTerms,
      };
 
      enum class DynamicCacheKind : uint8_t {
        None,
        RelationOnly,
        DirectSource,
        TraversedSource,
        Variable,
        Mixed
      };
 
      enum class DirectTargetEvalKind : uint8_t {
        Generic,
        SingleAllDirect,
        SingleAllSemanticIs,
        SingleAllInIs,
        SingleAllInherited,
        SingleAllDirectMixed,
        SingleAllDirectMixedDirect,
        SingleAllSemanticIsMixed,
        SingleAllInIsMixed,
        SingleAllInheritedMixed,
      };
 
      enum DependencyFlags : uint16_t {
        DependencyNone = 0x00,
        DependencyHasSourceTerms = 0x01,
        DependencyHasVariableTerms = 0x02,
        DependencyHasPositiveTerms = 0x04,
        DependencyHasNegativeTerms = 0x08,
        DependencyHasAnyTerms = 0x10,
        DependencyHasWildcardTerms = 0x20,
        DependencyHasSort = 0x40,
        DependencyHasGroup = 0x80,
        DependencyHasTraversalTerms = 0x100,
        DependencyHasEntityFilterTerms = 0x200,
        DependencyHasInheritedDataTerms = 0x400,
        DependencyHasPotentialInheritedIdTerms = 0x800,
      };
 
      struct Data {
        struct Dependencies {
          QueryEntityArray createSelectors;
          QueryEntityArray exclusions;
          QueryEntityArray relations;
          QueryEntityArray sourceEntities;
          uint8_t createSelectorCnt = 0;
          uint8_t exclusionCnt = 0;
          uint8_t relationCnt = 0;
          uint8_t sourceEntityCnt = 0;
          uint8_t sourceTermCnt = 0;
          DependencyFlags flags = DependencyNone;
 
          void clear() {
            createSelectorCnt = 0;
            exclusionCnt = 0;
            relationCnt = 0;
            sourceEntityCnt = 0;
            sourceTermCnt = 0;
            flags = DependencyNone;
          }
 
          GAIA_NODISCARD std::span<const Entity> create_selectors_view() const {
            return {createSelectors.data(), createSelectorCnt};
          }
 
          GAIA_NODISCARD std::span<const Entity> exclusions_view() const {
            return {exclusions.data(), exclusionCnt};
          }
 
          GAIA_NODISCARD std::span<const Entity> relations_view() const {
            return {relations.data(), relationCnt};
          }
 
          GAIA_NODISCARD std::span<const Entity> src_entities_view() const {
            return {sourceEntities.data(), sourceEntityCnt};
          }
 
          void set_dep_flag(DependencyFlags dependency) {
            flags = (DependencyFlags)(flags | dependency);
          }
 
          GAIA_NODISCARD bool has_dep_flag(DependencyFlags dependency) const {
            return (flags & dependency) != 0;
          }
 
          void add_rel(Entity relation) {
            if (relation == EntityBad || core::has(relations_view(), relation))
              return;
 
            GAIA_ASSERT(relationCnt < MAX_ITEMS_IN_QUERY);
            relations[relationCnt++] = relation;
          }
 
          void add_src_entity(Entity entity) {
            if (entity == EntityBad || core::has(src_entities_view(), entity))
              return;
 
            GAIA_ASSERT(sourceEntityCnt < MAX_ITEMS_IN_QUERY);
            sourceEntities[sourceEntityCnt++] = entity;
          }
 
          GAIA_NODISCARD bool can_reuse_src_cache() const {
            return sourceTermCnt > 0 && sourceTermCnt == sourceEntityCnt;
          }
        };
 
        struct LookupIdentity {
          cnt::sarray<QueryTerm, MAX_ITEMS_IN_QUERY> lookupTerms;
          QueryEntityArray changed;
          QueryEntityArray groupDeps;
        };
 
        QueryEntityArray ids;
        cnt::sarray<QueryTerm, MAX_ITEMS_IN_QUERY> terms;
        QueryArchetypeCacheIndexMap lastMatchedArchetypeIdx_All;
        QueryArchetypeCacheIndexMap lastMatchedArchetypeIdx_Or;
        QueryArchetypeCacheIndexMap lastMatchedArchetypeIdx_Not;
        uint8_t idsCnt = 0;
        uint8_t changedCnt = 0;
        QueryEntityArray changed;
        cnt::sarray<uint8_t, MAX_ITEMS_IN_QUERY> changedFields;
        QueryEntityArray groupDeps;
        Entity sortBy;
        TSortByFunc sortByFunc;
        Entity groupBy;
        TGroupByFunc groupByFunc;
        QueryMask queryMask;
        uint32_t as_mask_0;
        uint32_t as_mask_1;
        uint8_t firstNot;
        uint8_t firstAny;
        uint8_t firstOr;
        uint8_t groupDepCnt = 0;
        uint16_t readWriteMask;
        uint16_t flags;
        uint16_t cacheSrcTrav = 0;
        DirectTargetEvalKind directTargetEvalKind = DirectTargetEvalKind::Generic;
        Entity directTargetEvalId = EntityBad;
        bool canDirectTargetEval = false;
        bool canDirectEntitySeedEvalShape = false;
        bool hasOnlyDirectOrTerms = false;
        bool canReuseDynamicCache = false;
        Dependencies deps;
        CachePolicy cachePolicy = CachePolicy::Lazy;
        CreateArchetypeMatchKind createArchetypeMatchKind = CreateArchetypeMatchKind::Vm;
        DynamicCacheKind dynamicCacheKind = DynamicCacheKind::None;
        LookupIdentity lookupIdentity;
 
        GAIA_NODISCARD bool has_match_terms() const {
          return (flags & QueryFlags::HasMatchTerms) != 0;
        }
 
        GAIA_NODISCARD std::span<const Entity> ids_view() const {
          return {ids.data(), idsCnt};
        }
 
        GAIA_NODISCARD std::span<const Entity> changed_view() const {
          return {changed.data(), changedCnt};
        }
 
        GAIA_NODISCARD std::span<const uint8_t> changed_fields_view() const {
          return {changedFields.data(), changedCnt};
        }
 
        GAIA_NODISCARD std::span<const Entity> group_deps_view() const {
          return {groupDeps.data(), groupDepCnt};
        }
 
        GAIA_NODISCARD std::span<const QueryTerm> lookup_terms_view() const {
          return {lookupIdentity.lookupTerms.data(), idsCnt};
        }
 
        GAIA_NODISCARD bool uses_term_access_identity() const {
          return (flags & (QueryFlags::HasMatchTerms | QueryFlags::HasPairTerms)) != 0 ||
                 (idsCnt != 0 && lookup_terms_view().back().op == QueryOpKind::Any);
        }
 
        GAIA_NODISCARD std::span<QueryTerm> lookup_terms_view_mut() {
          return {lookupIdentity.lookupTerms.data(), idsCnt};
        }
 
        GAIA_NODISCARD std::span<const Entity> changed_lookup_view() const {
          return {lookupIdentity.changed.data(), changedCnt};
        }
 
        GAIA_NODISCARD std::span<Entity> changed_lookup_view_mut() {
          return {lookupIdentity.changed.data(), changedCnt};
        }
 
        GAIA_NODISCARD std::span<const Entity> group_deps_lookup_view() const {
          return {lookupIdentity.groupDeps.data(), groupDepCnt};
        }
 
        GAIA_NODISCARD std::span<Entity> group_deps_lookup_view_mut() {
          return {lookupIdentity.groupDeps.data(), groupDepCnt};
        }
 
        void add_group_dep(Entity relation) {
          if (relation == EntityBad || core::has(group_deps_view(), relation))
            return;
 
          GAIA_ASSERT(groupDepCnt < MAX_ITEMS_IN_QUERY);
          if (groupDepCnt < MAX_ITEMS_IN_QUERY)
            groupDeps[groupDepCnt++] = relation;
        }
 
        void add_group_deps() {
          deps.set_dep_flag(DependencyHasGroup);
 
          const bool hasBuiltInGroupDep = groupBy != EntityBad && (groupByFunc == group_by_func_default ||
                                                                   groupByFunc == group_by_func_depth_order);
          if (hasBuiltInGroupDep)
            deps.add_rel(groupBy);
          for (auto relation: group_deps_view())
            deps.add_rel(relation);
        }
 
        GAIA_NODISCARD std::span<QueryTerm> terms_view_mut() {
          return {terms.data(), idsCnt};
        }
        GAIA_NODISCARD std::span<const QueryTerm> terms_view() const {
          return {terms.data(), idsCnt};
        }
 
        GAIA_NODISCARD DynamicCacheKind calc_dynamic_cache_kind() const {
          if (cachePolicy != CachePolicy::Dynamic)
            return DynamicCacheKind::None;
 
          const bool hasVariables = deps.has_dep_flag(DependencyHasVariableTerms);
          const bool hasSources = deps.has_dep_flag(DependencyHasSourceTerms);
          const bool hasTraversal = deps.has_dep_flag(DependencyHasTraversalTerms);
          const bool hasRelations = deps.relationCnt != 0;
          if (hasSources && hasTraversal && !hasVariables)
            return DynamicCacheKind::TraversedSource;
 
          const uint8_t depCnt = (uint8_t)hasVariables + (uint8_t)hasSources + (uint8_t)hasRelations;
          if (depCnt > 1)
            return DynamicCacheKind::Mixed;
          if (hasVariables)
            return DynamicCacheKind::Variable;
          if (hasSources)
            return hasTraversal ? DynamicCacheKind::TraversedSource : DynamicCacheKind::DirectSource;
          if (hasRelations)
            return DynamicCacheKind::RelationOnly;
          return DynamicCacheKind::None;
        }
 
        GAIA_NODISCARD bool uses_direct_src_version_tracking() const {
          return canReuseDynamicCache && dynamicCacheKind == DynamicCacheKind::DirectSource;
        }
 
        GAIA_NODISCARD bool uses_src_trav_snapshot() const {
          return canReuseDynamicCache && dynamicCacheKind == DynamicCacheKind::TraversedSource;
        }
 
        GAIA_NODISCARD bool calc_can_reuse_dynamic_cache() const {
          if (cachePolicy != CachePolicy::Dynamic)
            return false;
 
          switch (dynamicCacheKind) {
            case DynamicCacheKind::None:
              return false;
            case DynamicCacheKind::RelationOnly:
            case DynamicCacheKind::Variable:
              return true;
            case DynamicCacheKind::DirectSource:
              return deps.can_reuse_src_cache();
            case DynamicCacheKind::TraversedSource:
              return deps.can_reuse_src_cache() && cacheSrcTrav != 0;
            case DynamicCacheKind::Mixed:
              if (!deps.has_dep_flag(DependencyHasSourceTerms))
                return true;
              if (!deps.can_reuse_src_cache())
                return false;
              return !deps.has_dep_flag(DependencyHasTraversalTerms) || cacheSrcTrav != 0;
          }
 
          GAIA_ASSERT(false);
          return false;
        }
 
        void refresh_lookup_keys() {
          GAIA_FOR(idsCnt) lookupIdentity.lookupTerms[i] = terms[i];
          canonicalize_lookup_terms(lookup_terms_view_mut());
          GAIA_FOR(changedCnt) lookupIdentity.changed[i] = changed[i];
          canonicalize_lookup_changed(changed_lookup_view_mut());
          GAIA_FOR(groupDepCnt) lookupIdentity.groupDeps[i] = groupDeps[i];
          canonicalize_lookup_changed(group_deps_lookup_view_mut());
        }
 
        GAIA_NODISCARD bool lookup_keys_equal(const Data& other) const {
          {
            const auto left = lookup_terms_view();
            const auto right = other.lookup_terms_view();
            GAIA_FOR((uint32_t)left.size()) {
              if (left[i] != right[i])
                return false;
            }
          }
 
          if (uses_term_access_identity() || other.uses_term_access_identity()) {
            const auto left = lookup_terms_view();
            const auto right = other.lookup_terms_view();
            GAIA_FOR((uint32_t)left.size()) {
              if (left[i].access != right[i].access || left[i].fieldIndex != right[i].fieldIndex)
                return false;
            }
          }
 
          {
            const auto left = changed_lookup_view();
            const auto right = other.changed_lookup_view();
            GAIA_FOR((uint32_t)left.size()) {
              if (left[i] != right[i])
                return false;
            }
          }
 
          {
            const auto left = group_deps_lookup_view();
            const auto right = other.group_deps_lookup_view();
            GAIA_FOR((uint32_t)left.size()) {
              if (left[i] != right[i])
                return false;
            }
          }
 
          return true;
        }
 
        GAIA_NODISCARD QueryLookupHash::Type hash_lookup_key_payload() const {
          QueryLookupHash::Type payloadHash = 0;
          const bool useTermAccess = uses_term_access_identity();
 
          // Ids & ops
          {
            QueryLookupHash::Type hash = 0;
 
            for (const auto& pair: lookup_terms_view()) {
              const auto op = (QueryLookupHash::Type)pair.op;
              const auto access = !useTermAccess ? op
                                                 : (op | ((QueryLookupHash::Type)pair.access << 8) |
                                                    ((QueryLookupHash::Type)pair.fieldIndex << 16));
              hash = core::hash_combine(hash, access);
              hash = core::hash_combine(hash, (QueryLookupHash::Type)pair.id.value());
              hash = core::hash_combine(hash, (QueryLookupHash::Type)pair.src.value());
              hash = core::hash_combine(hash, (QueryLookupHash::Type)pair.entTrav.value());
              hash = core::hash_combine(hash, (QueryLookupHash::Type)(uint8_t)pair.travKind);
              hash = core::hash_combine(hash, (QueryLookupHash::Type)pair.travDepth);
              hash = core::hash_combine(hash, (QueryLookupHash::Type)(uint8_t)pair.matchKind);
            }
            hash = core::hash_combine(hash, (QueryLookupHash::Type)idsCnt);
            hash = core::hash_combine(hash, (QueryLookupHash::Type)(!useTermAccess ? readWriteMask : 0));
            hash = core::hash_combine(hash, (QueryLookupHash::Type)cacheSrcTrav);
 
            const bool matchPrefab = (flags & QueryFlags::MatchPrefab) != 0;
            hash = core::hash_combine(hash, (QueryLookupHash::Type)matchPrefab);
 
            payloadHash = hash;
          }
 
          // Filters
          {
            QueryLookupHash::Type hash = 0;
 
            for (const auto entity: changed_lookup_view())
              hash = core::hash_combine(hash, (QueryLookupHash::Type)entity.value());
            hash = core::hash_combine(hash, (QueryLookupHash::Type)changedCnt);
 
            payloadHash = core::hash_combine(payloadHash, hash);
          }
 
          // Explicit grouping dependencies
          {
            QueryLookupHash::Type hash = 0;
 
            for (const auto entity: group_deps_lookup_view())
              hash = core::hash_combine(hash, (QueryLookupHash::Type)entity.value());
            hash = core::hash_combine(hash, (QueryLookupHash::Type)groupDepCnt);
 
            payloadHash = core::hash_combine(payloadHash, hash);
          }
 
          return payloadHash;
        }
 
        GAIA_NODISCARD bool grouping_payload_equal(const Data& other) const {
          if (groupBy != other.groupBy)
            return false;
          if (groupByFunc != other.groupByFunc)
            return false;
          return (flags & QueryFlags::OrderGroups) == (other.flags & QueryFlags::OrderGroups);
        }
 
        GAIA_NODISCARD bool sort_payload_equal(const Data& other) const {
          return sortBy == other.sortBy && sortByFunc == other.sortByFunc;
        }
 
        GAIA_NODISCARD bool has_sort_payload() const {
          return sortBy != EntityBad || sortByFunc != nullptr;
        }
 
        GAIA_NODISCARD QueryLookupHash::Type hash_sort_payload() const {
          QueryLookupHash::Type hash = 0;
          hash = core::hash_combine(hash, (QueryLookupHash::Type)sortBy.value());
          hash = core::hash_combine(hash, (QueryLookupHash::Type)sortByFunc);
          return hash;
        }
 
        GAIA_NODISCARD bool identity_payload_equal(const Data& other) const {
          if (idsCnt != other.idsCnt)
            return false;
          if (changedCnt != other.changedCnt)
            return false;
          if (groupDepCnt != other.groupDepCnt)
            return false;
          if (has_match_terms() != other.has_match_terms())
            return false;
          if (!uses_term_access_identity() && readWriteMask != other.readWriteMask)
            return false;
          if (cacheSrcTrav != other.cacheSrcTrav)
            return false;
          if (!lookup_keys_equal(other))
            return false;
          if (!sort_payload_equal(other))
            return false;
          return grouping_payload_equal(other);
        }
 
        GAIA_NODISCARD QueryLookupHash::Type hash_grouping_payload() const {
          QueryLookupHash::Type hash = 0;
          hash = core::hash_combine(hash, (QueryLookupHash::Type)groupBy.value());
          hash = core::hash_combine(hash, (QueryLookupHash::Type)groupByFunc);
          hash = core::hash_combine(hash, (QueryLookupHash::Type)((flags & QueryFlags::OrderGroups) != 0));
          return hash;
        }
 
        GAIA_NODISCARD QueryLookupHash::Type hash_identity_payload() const {
          QueryLookupHash::Type hash = hash_lookup_key_payload();
          if (has_sort_payload())
            hash = core::hash_combine(hash, hash_sort_payload());
          return core::hash_combine(hash, hash_grouping_payload());
        }
 
        GAIA_NODISCARD QueryLookupHash calc_lookup_hash() const {
          return {core::calculate_hash64(hash_identity_payload())};
        }
      } data{}; 
      // Make sure that MAX_ITEMS_IN_QUERY can fit into data.readWriteMask
      static_assert(MAX_ITEMS_IN_QUERY < 16);
 
      void init(World* pWorld) {
        w = pWorld;
        cc = &comp_cache_mut(*pWorld);
      }
 
      void refresh() {
#if GAIA_ASSERT_ENABLED
        // Sorting reads component values even when the callback does not.
        if (data.has_match_terms() && data.sortByFunc != nullptr) {
          GAIA_FOR(data.idsCnt) {
            GAIA_ASSERT(data.terms[i].access != QueryAccess::Match || data.terms[i].id != data.sortBy);
          }
        }
#endif
        const auto mask0_old = data.as_mask_0;
        const auto mask1_old = data.as_mask_1;
        const auto isComplex_old = data.flags & QueryFlags::Complex;
        const auto hasSourceTerms_old = data.flags & QueryFlags::HasSourceTerms;
        const auto hasVariableTerms_old = data.flags & QueryFlags::HasVariableTerms;
        const auto cachePolicy_old = data.cachePolicy;
        const auto createArchetypeMatchKind_old = data.createArchetypeMatchKind;
        const auto dynamicCacheKind_old = data.dynamicCacheKind;
        const auto canDirectTargetEval_old = data.canDirectTargetEval;
        const auto canDirectEntitySeedEvalShape_old = data.canDirectEntitySeedEvalShape;
        const auto hasOnlyDirectOrTerms_old = data.hasOnlyDirectOrTerms;
        const auto canReuseDynamicCache_old = data.canReuseDynamicCache;
        const auto dependencyFlags_old = data.deps.flags;
        const auto createSelectorCnt_old = data.deps.createSelectorCnt;
        const auto exclusionCnt_old = data.deps.exclusionCnt;
        const auto relationCnt_old = data.deps.relationCnt;
        const auto sourceEntityCnt_old = data.deps.sourceEntityCnt;
        const auto sourceTermCnt_old = data.deps.sourceTermCnt;
        auto createSelectors_old = data.deps.createSelectors;
        auto exclusions_old = data.deps.exclusions;
        auto relations_old = data.deps.relations;
        auto sourceEntities_old = data.deps.sourceEntities;
 
        // Update masks
        {
          uint32_t as_mask_0 = 0;
          uint32_t as_mask_1 = 0;
          bool isComplex = false;
          bool hasSourceTerms = false;
          bool hasVariableTerms = false;
          bool hasPrefabTerms = false;
          bool hasCreateSelector = false;
          bool canDirectCreateArchetypeMatch = true;
          bool hasEntityFilterTerms = false;
          bool canDirectTargetEval = true;
          bool hasOnlyDirectOrTerms = true;
          bool hasOrTerms = false;
          bool hasDirectTargetEvalPositiveTerms = false;
          const QueryTerm* pSingleDirectTargetAllTerm = nullptr;
          bool singleDirectTargetEvalPossible = true;
          bool directTargetEvalHasRemainingTerms = false;
          bool directTargetEvalAllTermsDirect = true;
          QueryEntityArray idsNoSrc;
          QueryEntityArray createSelectorsAll;
          QueryEntityArray createSelectorsOr;
          uint32_t idsNoSrcCnt = 0;
          uint8_t createSelectorAllCnt = 0;
          uint8_t createSelectorOrCnt = 0;
          data.deps.clear();
          data.directTargetEvalKind = DirectTargetEvalKind::Generic;
          data.directTargetEvalId = EntityBad;
          if (data.sortByFunc != nullptr)
            data.deps.set_dep_flag(DependencyHasSort);
          if (data.groupBy != EntityBad)
            data.add_group_deps();
 
          auto terms = data.terms_view();
          const auto cnt = (uint32_t)terms.size();
          GAIA_FOR(cnt) {
            const auto& term = terms[i];
            const auto id = term.id;
            hasPrefabTerms |= id == Prefab;
            const bool isDirectIsTerm = term.src == EntityBad && term.entTrav == EntityBad &&
                                        !term_has_variables(term) && term.matchKind != QueryMatchKind::Direct &&
                                        id.pair() && id.id() == Is.id() && !is_wildcard(id.gen()) &&
                                        !is_variable((EntityId)id.gen());
            const bool isPotentialInheritedTerm = query_term_uses_potential_inherited_id_matching(term);
            const bool isInheritedTerm = isPotentialInheritedTerm && world_term_uses_inherit_policy(*w, id);
            directTargetEvalAllTermsDirect &= !isDirectIsTerm && !isInheritedTerm;
            const bool isNonFragmentingTerm =
                term.src == EntityBad && term.entTrav == EntityBad && !term_has_variables(term) &&
                ((id.pair() && world_relation_uses_non_fragmenting_storage(*w, pair_rel(*w, id))) ||
                 (!id.pair() && world_component_is_non_fragmenting(*w, id)));
            hasEntityFilterTerms |= isNonFragmentingTerm || isDirectIsTerm || isInheritedTerm;
          }
 
          GAIA_FOR(cnt) {
            const auto& term = terms[i];
            const auto id = term.id;
            if (term.src != EntityBad || term.entTrav != EntityBad || term_has_variables(term)) {
              singleDirectTargetEvalPossible = false;
              canDirectTargetEval = false;
              hasOnlyDirectOrTerms = false;
            }
            switch (term.op) {
              case QueryOpKind::All:
                hasDirectTargetEvalPositiveTerms = true;
                if (pSingleDirectTargetAllTerm == nullptr)
                  pSingleDirectTargetAllTerm = &term;
                else
                  singleDirectTargetEvalPossible = false;
                hasOnlyDirectOrTerms = false;
                break;
              case QueryOpKind::Or:
                hasDirectTargetEvalPositiveTerms = true;
                hasOrTerms = true;
                directTargetEvalHasRemainingTerms = true;
                break;
              case QueryOpKind::Not:
                directTargetEvalHasRemainingTerms = true;
                break;
              case QueryOpKind::Any:
              case QueryOpKind::Count:
                singleDirectTargetEvalPossible = false;
                canDirectTargetEval = false;
                hasOnlyDirectOrTerms = false;
                break;
            }
            const bool isDirectIsTerm = term.src == EntityBad && term.entTrav == EntityBad &&
                                        !term_has_variables(term) && term.matchKind != QueryMatchKind::Direct &&
                                        id.pair() && id.id() == Is.id() && !is_wildcard(id.gen()) &&
                                        !is_variable((EntityId)id.gen());
            const bool isPotentialInheritedTerm = query_term_uses_potential_inherited_id_matching(term);
            const bool isInheritedTerm = isPotentialInheritedTerm && world_term_uses_inherit_policy(*w, id);
            const bool isCachedInheritedDataTerm = isInheritedTerm && !world_component_uses_sparse_storage(*w, id);
            const bool isNonFragmentingTerm =
                term.src == EntityBad && term.entTrav == EntityBad && !term_has_variables(term) &&
                ((id.pair() && world_relation_uses_non_fragmenting_storage(*w, pair_rel(*w, id))) ||
                 (!id.pair() && world_component_is_non_fragmenting(*w, id)));
            canDirectCreateArchetypeMatch &= term.src == EntityBad;
            if (id.pair() && (is_wildcard(id.id()) || is_wildcard(id.gen())))
              data.deps.set_dep_flag(DependencyHasWildcardTerms);
            const bool hasDynamicRelationUsage =
                term.entTrav != EntityBad || term.src != EntityBad || term_has_variables(term);
            if (id.pair() && hasDynamicRelationUsage && !is_wildcard(id.id()) && !is_variable((EntityId)id.id()))
              data.deps.add_rel(pair_rel(*w, id));
            if (term.entTrav != EntityBad) {
              data.deps.add_rel(term.entTrav);
              data.deps.set_dep_flag(DependencyHasTraversalTerms);
            }
            if (term.src != EntityBad) {
              hasSourceTerms = true;
              data.deps.set_dep_flag(DependencyHasSourceTerms);
              ++data.deps.sourceTermCnt;
              if (!is_variable(term.src))
                data.deps.add_src_entity(term.src);
            }
 
            if (term_has_variables(term)) {
              hasVariableTerms = true;
              data.deps.set_dep_flag(DependencyHasVariableTerms);
              isComplex = true;
              continue;
            }
 
            if (isPotentialInheritedTerm)
              data.deps.set_dep_flag(DependencyHasPotentialInheritedIdTerms);
 
            if (isNonFragmentingTerm || isDirectIsTerm || isInheritedTerm) {
              data.deps.set_dep_flag(DependencyHasEntityFilterTerms);
              if (isCachedInheritedDataTerm)
                data.deps.set_dep_flag(DependencyHasInheritedDataTerms);
              if (id.pair() && !is_wildcard(id.id()) && !is_variable((EntityId)id.id()))
                data.deps.add_rel(pair_rel(*w, id));
              continue;
            }
 
            if (hasEntityFilterTerms && term.op == QueryOpKind::Or) {
              isComplex = true;
              continue;
            }
 
            // Source terms are evaluated separately by the VM.
            // They should not affect archetype-level query masks.
            if (term.src != EntityBad) {
              continue;
            }
 
            // ANY terms are not hard requirements and must not affect archetype prefilter masks.
            if (term.op != QueryOpKind::Any)
              idsNoSrc[idsNoSrcCnt++] = id;
 
            if (term.op == QueryOpKind::All || term.op == QueryOpKind::Or) {
              hasCreateSelector = true;
              data.deps.set_dep_flag(DependencyHasPositiveTerms);
              if (term.op == QueryOpKind::All)
                createSelectorsAll[createSelectorAllCnt++] = id;
              else
                createSelectorsOr[createSelectorOrCnt++] = id;
            } else if (term.op == QueryOpKind::Not) {
              data.deps.set_dep_flag(DependencyHasNegativeTerms);
              data.deps.exclusions[data.deps.exclusionCnt++] = id;
            } else if (term.op == QueryOpKind::Any) {
              data.deps.set_dep_flag(DependencyHasAnyTerms);
            }
 
            // Build the Is mask.
            // We will use it to identify entities with an Is relationship quickly.
            const bool allowSemanticIs = !(
                term.matchKind == QueryMatchKind::Direct && id.pair() && id.id() == Is.id() && !is_wildcard(id.gen()));
            if (!id.pair()) {
              const auto j = (uint32_t)i;
              const auto has_as = allowSemanticIs ? (uint32_t)is_base(*w, id) : 0U;
              as_mask_0 |= (has_as << j);
            } else {
              const bool idIsWildcard = is_wildcard(id.id());
              const bool isGenWildcard = is_wildcard(id.gen());
              isComplex |= (idIsWildcard || isGenWildcard);
 
              if (!idIsWildcard) {
                const auto j = (uint32_t)i;
                const auto e = pair_rel(*w, id);
                const auto has_as = allowSemanticIs ? (uint32_t)is_base(*w, e) : 0U;
                as_mask_0 |= (has_as << j);
              }
 
              if (!isGenWildcard) {
                const auto j = (uint32_t)i;
                const auto e = pair_tgt(*w, id);
                const auto has_as = allowSemanticIs ? (uint32_t)is_base(*w, e) : 0U;
                as_mask_1 |= (has_as << j);
              }
            }
          }
 
          if (singleDirectTargetEvalPossible && pSingleDirectTargetAllTerm != nullptr) {
            const auto& term = *pSingleDirectTargetAllTerm;
            const auto id = term.id;
            if (term.matchKind == QueryMatchKind::In && id.pair() && id.id() == Is.id() && !is_wildcard(id.gen()) &&
                !is_variable((EntityId)id.gen())) {
              data.directTargetEvalKind = directTargetEvalHasRemainingTerms ? DirectTargetEvalKind::SingleAllInIsMixed
                                                                            : DirectTargetEvalKind::SingleAllInIs;
            } else if (
                term.matchKind == QueryMatchKind::Semantic && id.pair() && id.id() == Is.id() &&
                !is_wildcard(id.gen()) && !is_variable((EntityId)id.gen())) {
              data.directTargetEvalKind = directTargetEvalHasRemainingTerms
                                              ? DirectTargetEvalKind::SingleAllSemanticIsMixed
                                              : DirectTargetEvalKind::SingleAllSemanticIs;
            } else if (
                term.matchKind == QueryMatchKind::Semantic && !is_wildcard(id) && !is_variable((EntityId)id.id()) &&
                (!id.pair() || !is_variable((EntityId)id.gen())) && world_term_uses_inherit_policy(*w, id)) {
              data.directTargetEvalKind = directTargetEvalHasRemainingTerms
                                              ? DirectTargetEvalKind::SingleAllInheritedMixed
                                              : DirectTargetEvalKind::SingleAllInherited;
            } else {
              data.directTargetEvalKind = !directTargetEvalHasRemainingTerms ? DirectTargetEvalKind::SingleAllDirect
                                          : directTargetEvalAllTermsDirect
                                              ? DirectTargetEvalKind::SingleAllDirectMixedDirect
                                              : DirectTargetEvalKind::SingleAllDirectMixed;
            }
            data.directTargetEvalId = id;
          }
          data.canDirectTargetEval = canDirectTargetEval && hasDirectTargetEvalPositiveTerms;
          data.canDirectEntitySeedEvalShape =
              data.canDirectTargetEval && data.sortByFunc == nullptr && data.groupBy == EntityBad;
          data.hasOnlyDirectOrTerms = hasOnlyDirectOrTerms && hasOrTerms;
 
          // Update the mask
          data.as_mask_0 = as_mask_0;
          data.as_mask_1 = as_mask_1;
          data.deps.createSelectorCnt = 0;
          if (createSelectorAllCnt != 0) {
            auto selector_rank = [](Entity term) {
              if (!term.pair())
                return 2;
              if (!is_wildcard(term.id()) && !is_wildcard(term.gen()))
                return 0;
              if (is_wildcard(term.id()) && is_wildcard(term.gen()))
                return 3;
              return 1;
            };
 
            // For immediate structural queries, we choose one required ALL selector as the create-time wake-up key.
            // This choice is ordered by:
            //   1) smaller component index bucket size first
            //   2) if equal, more specific selector first
            uint8_t bestIdx = 0;
            auto bestBucketSize = world_component_index_bucket_size(*w, createSelectorsAll[0]);
            auto bestRank = selector_rank(createSelectorsAll[0]);
            GAIA_FOR2_(1, createSelectorAllCnt, i) {
              const auto bucketSize = world_component_index_bucket_size(*w, createSelectorsAll[i]);
              const auto rank = selector_rank(createSelectorsAll[i]);
              if (bucketSize < bestBucketSize || (bucketSize == bestBucketSize && rank < bestRank)) {
                bestBucketSize = bucketSize;
                bestRank = rank;
                bestIdx = (uint8_t)i;
              }
            }
            data.deps.createSelectors[data.deps.createSelectorCnt++] = createSelectorsAll[bestIdx];
          } else {
            GAIA_FOR(createSelectorOrCnt) {
              data.deps.createSelectors[data.deps.createSelectorCnt++] = createSelectorsOr[i];
            }
          }
          if (hasPrefabTerms)
            data.flags |= QueryCtx::QueryFlags::HasPrefabTerms;
          else
            data.flags &= (QueryCtx::QueryFlags)(~(uint16_t)QueryCtx::QueryFlags::HasPrefabTerms);
 
          if (hasSourceTerms)
            data.flags |= QueryCtx::QueryFlags::HasSourceTerms;
          else
            data.flags &= (QueryCtx::QueryFlags)(~(uint16_t)QueryCtx::QueryFlags::HasSourceTerms);
 
          if (hasVariableTerms)
            data.flags |= QueryCtx::QueryFlags::HasVariableTerms;
          else
            data.flags &= (QueryCtx::QueryFlags)(~(uint16_t)QueryCtx::QueryFlags::HasVariableTerms);
 
          if (hasSourceTerms || hasVariableTerms)
            data.cachePolicy = CachePolicy::Dynamic;
          else if (
              !hasEntityFilterTerms && data.sortByFunc == nullptr && data.groupBy == EntityBad && hasCreateSelector)
            data.cachePolicy = CachePolicy::Immediate;
          else
            data.cachePolicy = CachePolicy::Lazy;
 
          data.createArchetypeMatchKind = data.cachePolicy == CachePolicy::Immediate && canDirectCreateArchetypeMatch
                                              ? CreateArchetypeMatchKind::DirectStructuralTerms
                                              : CreateArchetypeMatchKind::Vm;
          data.dynamicCacheKind = data.calc_dynamic_cache_kind();
 
          // Traversed-source snapshot caching is only effective for traversed source terms.
          if (!data.deps.has_dep_flag(DependencyHasSourceTerms) || !data.deps.has_dep_flag(DependencyHasTraversalTerms))
            data.cacheSrcTrav = 0;
 
          data.canReuseDynamicCache = data.calc_can_reuse_dynamic_cache();
 
          // Calculate the component mask for simple queries
          isComplex |= ((data.as_mask_0 + data.as_mask_1) != 0);
          if (isComplex) {
            data.queryMask = {};
            data.flags |= QueryCtx::QueryFlags::Complex;
          } else {
            data.queryMask = build_entity_mask(EntitySpan{idsNoSrc.data(), idsNoSrcCnt});
            data.flags &= (QueryCtx::QueryFlags)(~(uint16_t)QueryCtx::QueryFlags::Complex);
          }
        }
 
        // Request recompilation of the query if the mask has changed
        data.refresh_lookup_keys();
 
        // Request recompilation of the query if the mask has changed
        if (mask0_old != data.as_mask_0 || mask1_old != data.as_mask_1 ||
            isComplex_old != (data.flags & QueryFlags::Complex) ||
            hasSourceTerms_old != (data.flags & QueryFlags::HasSourceTerms) ||
            hasVariableTerms_old != (data.flags & QueryFlags::HasVariableTerms) ||
            canDirectTargetEval_old != data.canDirectTargetEval ||
            canDirectEntitySeedEvalShape_old != data.canDirectEntitySeedEvalShape ||
            hasOnlyDirectOrTerms_old != data.hasOnlyDirectOrTerms ||
            canReuseDynamicCache_old != data.canReuseDynamicCache || cachePolicy_old != data.cachePolicy ||
            createArchetypeMatchKind_old != data.createArchetypeMatchKind ||
            dynamicCacheKind_old != data.dynamicCacheKind || dependencyFlags_old != data.deps.flags ||
            createSelectorCnt_old != data.deps.createSelectorCnt || exclusionCnt_old != data.deps.exclusionCnt ||
            relationCnt_old != data.deps.relationCnt || sourceEntityCnt_old != data.deps.sourceEntityCnt ||
            sourceTermCnt_old != data.deps.sourceTermCnt || createSelectors_old != data.deps.createSelectors ||
            exclusions_old != data.deps.exclusions || relations_old != data.deps.relations ||
            sourceEntities_old != data.deps.sourceEntities)
          data.flags |= QueryCtx::QueryFlags::Recompile;
      }
 
      GAIA_NODISCARD static bool
      equals_no_handle_assumption(const QueryCtx& leftCtx, const QueryCtx& rightCtx) noexcept {
        // Lookup hash must match
        if (leftCtx.hashLookup != rightCtx.hashLookup)
          return false;
 
        const auto& left = leftCtx.data;
        const auto& right = rightCtx.data;
        return left.identity_payload_equal(right);
      }
 
      GAIA_NODISCARD bool operator==(const QueryCtx& other) const noexcept {
        // Comparison expected to be done only the first time the query is set up
        GAIA_ASSERT(q.handle.id() == QueryIdBad);
        // Fast path when cache ids are set
        // if (queryId != QueryIdBad && queryId == other.queryId)
        //  return true;
 
        return equals_no_handle_assumption(*this, other);
      }
 
      GAIA_NODISCARD bool operator!=(const QueryCtx& other) const noexcept {
        return !operator==(other);
      }
    };
 
    struct query_sort_cond {
      constexpr bool operator()(const QueryTerm& lhs, const QueryTerm& rhs) const {
        return query_term_less_for_lookup(lhs, rhs);
      }
    };
 
    inline void sort(QueryCtx& ctx) {
      const uint32_t idsCnt = ctx.data.idsCnt;
      const uint32_t changedCnt = ctx.data.changedCnt;
 
      auto& ctxData = ctx.data;
      // Canonicalize degenerate OR queries: a single OR term has AND semantics.
      // Rewriting it here keeps ordering/hash behavior identical to an explicit ALL term.
      if (idsCnt > 0) {
        uint32_t orCnt = 0;
        uint32_t orIdx = BadIndex;
        GAIA_FOR(idsCnt) {
          if (ctxData.terms[i].op != QueryOpKind::Or)
            continue;
          ++orCnt;
          orIdx = i;
          if (orCnt > 1)
            break;
        }
 
        if (orCnt == 1)
          ctxData.terms[orIdx].op = QueryOpKind::All;
      }
 
      // Sort data. Necessary for correct hash calculation.
      // Without sorting query.all<XXX, YYY> would be different than query.all<YYY, XXX>.
      // Also makes sure data is in optimal order for query processing.
      core::sort(
          ctxData.terms.data(), ctxData.terms.data() + ctxData.idsCnt, query_sort_cond{}, //
          [&](uint32_t left, uint32_t right) {
            core::swap(ctxData.ids[left], ctxData.ids[right]);
            core::swap(ctxData.terms[left], ctxData.terms[right]);
 
            // Make sure masks remains correct after sorting
            core::swap_bits(ctxData.readWriteMask, left, right);
            core::swap_bits(ctxData.as_mask_0, left, right);
            core::swap_bits(ctxData.as_mask_1, left, right);
          });
 
      if (idsCnt > 0) {
        uint32_t i = 0;
        while (i < idsCnt && ctxData.terms[i].op == QueryOpKind::All)
          ++i;
        ctxData.firstOr = (uint8_t)i;
        while (i < idsCnt && ctxData.terms[i].op == QueryOpKind::Or)
          ++i;
        ctxData.firstNot = (uint8_t)i;
        while (i < idsCnt && ctxData.terms[i].op == QueryOpKind::Not)
          ++i;
        ctxData.firstAny = (uint8_t)i;
      } else
        ctxData.firstOr = ctxData.firstNot = ctxData.firstAny = 0;
 
      // Canonicalize filter order. This enables monotonic component lookup in filter matching
      // and keeps cache keys stable regardless of changed() call order.
      if (changedCnt > 1) {
        core::sort(ctxData.changed.data(), ctxData.changed.data() + changedCnt, SortComponentCond{});
      }
 
      GAIA_FOR(changedCnt) {
        const auto comp = ctxData.changed[i];
        uint32_t compIdx = 0;
        while (compIdx < idsCnt && ctxData.ids[compIdx] != comp)
          ++compIdx;
 
        GAIA_ASSERT(compIdx < idsCnt);
        ctxData.changedFields[i] = compIdx < idsCnt ? (uint8_t)compIdx : (uint8_t)0xFF;
      }
    }
 
    inline void normalize_cache_src_trav(QueryCtx& ctx) {
      auto& ctxData = ctx.data;
      if (ctxData.cacheSrcTrav == 0)
        return;
 
      bool hasTraversedSourceTerm = false;
      for (const auto& term: ctxData.terms_view()) {
        if (term.src == EntityBad || term.entTrav == EntityBad)
          continue;
 
        hasTraversedSourceTerm = true;
        break;
      }
 
      if (!hasTraversedSourceTerm)
        ctxData.cacheSrcTrav = 0;
    }
 
    inline void calc_lookup_hash(QueryCtx& ctx) {
      GAIA_ASSERT(ctx.cc != nullptr);
      // Make sure we don't calculate the hash twice
      GAIA_ASSERT(ctx.hashLookup.hash == 0);
 
      ctx.hashLookup = ctx.data.calc_lookup_hash();
    }
 
    template <uint32_t MAX_COMPONENTS>
    GAIA_NODISCARD inline uint32_t comp_idx(const QueryTerm* pTerms, Entity entity, Entity src) {
      // We let the compiler know the upper iteration bound at compile-time.
      // This way it can optimize better (e.g. loop unrolling, vectorization).
      GAIA_FOR(MAX_COMPONENTS) {
        if (pTerms[i].id == entity && pTerms[i].src == src)
          return i;
      }
 
      GAIA_ASSERT(false);
      return BadIndex;
    }
  } // namespace ecs
} // namespace gaia