query_info.h

#pragma once
#include "gaia/config/config.h"
 
#include "gaia/cnt/darray.h"
#include "gaia/cnt/darray_ext.h"
#include "gaia/cnt/ilist.h"
#include "gaia/config/profiler.h"
#include "gaia/core/hashing_policy.h"
#include "gaia/core/utility.h"
#include "gaia/ecs/api.h"
#include "gaia/ecs/archetype.h"
#include "gaia/ecs/archetype_common.h"
#include "gaia/ecs/chunk_iterator.h"
#include "gaia/ecs/component.h"
#include "gaia/ecs/component_cache.h"
#include "gaia/ecs/id.h"
#include "gaia/ecs/query_common.h"
#include "gaia/ecs/query_match_stamps.h"
#include "gaia/ecs/vm.h"
#include "gaia/mem/mem_utils.h"
#include "gaia/mem/smallblock_allocator.h"
 
namespace gaia {
  namespace ecs {
    struct Entity;
    class World;
 
    uint32_t world_version(const World& world);
    uint32_t world_archetype_delete_version(const World& world);
    Entity world_query_first_inherited_owner(const World& world, const Archetype& archetype, Entity term);
    const void* world_query_inherited_arg_data_const_ptr(const World& world, Entity owner, Entity id);
 
    using EntityToArchetypeMap = cnt::map<EntityLookupKey, ComponentIndexEntryArray>;
    struct ArchetypeCompIndices {
      uint8_t indices[ChunkHeader::MAX_COMPONENTS];
    };
 
    struct ArchetypeInheritedData {
      const void* data[ChunkHeader::MAX_COMPONENTS];
    };
 
    struct DirectChunkEntry {
      const Archetype* pArchetype = nullptr;
      Chunk* pChunk = nullptr;
      const uint8_t* pCompIndices = nullptr;
      const void* pData = nullptr;
      uint32_t dataOffset = UINT32_MAX;
      uint16_t rowFrom = 0;
      uint16_t rowTo = 0;
    };
 
    struct QueryMatchScratch {
      GAIA_USE_SMALLBLOCK(QueryMatchScratch)
 
      
      cnt::darr<const Archetype*> matchesArr;
      ArchetypeMatchStamps matchStamps;
      cnt::darray<const Archetype*> matchesArrDynPrev;
      uint32_t matchVersion = 0;
 
      void clear_temporary_matches() {
        matchesArr.clear();
        matchesArrDynPrev.clear();
        matchStamps.clear();
        matchVersion = 0;
      }
 
      void clear_temporary_matches_keep_stamps() {
        matchesArr.clear();
        matchesArrDynPrev.clear();
      }
 
      void reset_stamps() {
        matchStamps.clear();
        matchVersion = 0;
      }
 
      GAIA_NODISCARD uint32_t next_match_version() {
        ++matchVersion;
        if (matchVersion != 0)
          return matchVersion;
 
        reset_stamps();
        matchVersion = 1;
        return matchVersion;
      }
    };
 
    GAIA_NODISCARD QueryMatchScratch& query_match_scratch_acquire(World& world);
    void query_match_scratch_release(World& world, bool keepStamps);
 
    struct QueryInfoCreationCtx {
      QueryCtx* pQueryCtx;
      const EntityToArchetypeMap* pEntityToArchetypeMap;
      std::span<const Archetype*> allArchetypes;
    };
 
    class QueryInfo {
    public:
      uint32_t idx = 0;
      uint32_t gen = 0;
 
      enum class MatchArchetypeQueryRet : uint8_t {
        Fail,
        Ok,
        Skip
      };
 
    private:
      struct Instruction {
        Entity id;
        QueryOpKind op;
      };
 
      struct SortData {
        Chunk* pChunk;
        uint32_t archetypeIdx;
        uint16_t startRow;
        uint16_t count;
      };
 
      struct GroupData {
        GroupId groupId;
        uint32_t idxFirst;
        uint32_t idxLast;
        bool needsSorting;
      };
 
      struct SrcTravSnapshotItem {
        Entity entity = EntityBad;
        uint32_t sourceVersion = 0;
 
        GAIA_NODISCARD bool operator==(const SrcTravSnapshotItem& other) const {
          return entity == other.entity && sourceVersion == other.sourceVersion;
        }
 
        GAIA_NODISCARD bool operator!=(const SrcTravSnapshotItem& other) const {
          return !operator==(other);
        }
      };
 
    public:
      enum class InvalidationKind : uint8_t {
        Result,
        Seed,
        All
      };
 
    private:
      struct QueryPlan {
        QueryCtx ctx;
        vm::VirtualMachine vm;
      };
 
      struct QueryState {
        enum DirtyFlags : uint8_t { Clean = 0x00, Seed = 0x01, Result = 0x02, All = Seed | Result };
 
        cnt::set<const Archetype*> seedArchetypeSet;
        CArchetypeDArray seedArchetypeCache;
 
        cnt::set<const Archetype*> archetypeSet;
        CArchetypeDArray archetypeCache;
 
        struct GroupedPayload {
          cnt::darray<GroupId> archetypeGroupIds;
          cnt::darray<GroupData> archetypeGroupData;
          mutable GroupData selectedGroupData{};
          mutable bool selectedGroupDataValid = false;
          bool dataPending = false;
 
          void clear() {
            archetypeGroupIds = {};
            archetypeGroupData = {};
            selectedGroupData = {};
            selectedGroupDataValid = false;
            dataPending = false;
          }
 
          void clear_transient() {
            archetypeGroupIds.clear();
            archetypeGroupData.clear();
            selectedGroupData = {};
            selectedGroupDataValid = false;
            dataPending = false;
          }
        };
 
        struct ExecPayload {
          cnt::darray<ArchetypeCompIndices> archetypeCompIndices;
          cnt::darray<ArchetypeInheritedData> archetypeInheritedData;
          cnt::darray<DirectChunkEntry> directChunks;
          cnt::darray<const void*> directChunkData;
          uint32_t directChunksWorldVersion = UINT32_MAX;
          uint32_t directChunksEnabledVersion = UINT32_MAX;
          uint32_t directChunksDeleteVersion = UINT32_MAX;
          uint32_t directChunksResultRevision = 0;
          uint32_t directChunksIdxFrom = UINT32_MAX;
          uint32_t directChunksIdxTo = UINT32_MAX;
          uint32_t directChunksDataFields[MAX_ITEMS_IN_QUERY]{};
          uint32_t directChunksDataFieldCount = 0;
          const void* directChunksCompIndicesData = nullptr;
          bool compIndicesPending = false;
          bool inheritedDataPending = false;
 
          void clear() {
            archetypeCompIndices = {};
            archetypeInheritedData = {};
            directChunks = {};
            directChunkData = {};
            directChunksWorldVersion = UINT32_MAX;
            directChunksEnabledVersion = UINT32_MAX;
            directChunksDeleteVersion = UINT32_MAX;
            directChunksResultRevision = 0;
            directChunksIdxFrom = UINT32_MAX;
            directChunksIdxTo = UINT32_MAX;
            directChunksDataFieldCount = 0;
            directChunksCompIndicesData = nullptr;
            compIndicesPending = false;
            inheritedDataPending = false;
          }
 
          void clear_transient() {
            archetypeCompIndices.clear();
            archetypeInheritedData.clear();
            directChunks.clear();
            directChunkData.clear();
            directChunksWorldVersion = UINT32_MAX;
            directChunksEnabledVersion = UINT32_MAX;
            directChunksDeleteVersion = UINT32_MAX;
            directChunksResultRevision = 0;
            directChunksIdxFrom = UINT32_MAX;
            directChunksIdxTo = UINT32_MAX;
            directChunksDataFieldCount = 0;
            directChunksCompIndicesData = nullptr;
            compIndicesPending = false;
            inheritedDataPending = false;
          }
        };
 
        struct NonTrivialPayload {
          cnt::darray<uint8_t> archetypeBarrierPasses;
          cnt::darray<SortData> archetypeSortData;
          uint32_t sortVersion{};
          uint32_t barrierRelVersion = UINT32_MAX;
          uint32_t barrierEnabledVersion = UINT32_MAX;
          uint8_t barrierMayPrune = 0;
 
          void clear() {
            archetypeSortData = {};
            archetypeBarrierPasses = {};
            sortVersion = 0;
            barrierRelVersion = UINT32_MAX;
            barrierEnabledVersion = UINT32_MAX;
            barrierMayPrune = 0;
          }
 
          void clear_transient() {
            archetypeSortData.clear();
            archetypeBarrierPasses.clear();
            sortVersion = 0;
            barrierRelVersion = UINT32_MAX;
            barrierEnabledVersion = UINT32_MAX;
            barrierMayPrune = 0;
          }
        };
 
        ExecPayload exec;
        GroupedPayload grouped;
        NonTrivialPayload nonTrivial;
 
        struct DynamicCacheState {
          struct RelationPayload {
            cnt::sarray<uint32_t, MAX_ITEMS_IN_QUERY> versions;
 
            GAIA_NODISCARD bool changed(const World& world, std::span<const Entity> relations) const {
              const auto cnt = (uint32_t)relations.size();
              GAIA_FOR(cnt) {
                if (versions[i] != world_rel_version(world, relations[i]))
                  return true;
              }
 
              return false;
            }
 
            void snapshot(const World& world, std::span<const Entity> relations) {
              const auto cnt = (uint32_t)relations.size();
              GAIA_FOR(cnt)
              versions[i] = world_rel_version(world, relations[i]);
            }
          };
 
          struct DirectSourcePayload {
            cnt::sarray<uint32_t, MAX_ITEMS_IN_QUERY> entityVersions;
 
            GAIA_NODISCARD bool changed(const World& world, std::span<const Entity> sourceEntities) const {
              const auto cnt = (uint32_t)sourceEntities.size();
              GAIA_FOR(cnt) {
                if (entityVersions[i] != world_entity_archetype_version(world, sourceEntities[i]))
                  return true;
              }
 
              return false;
            }
 
            void snapshot(const World& world, std::span<const Entity> sourceEntities) {
              const auto cnt = (uint32_t)sourceEntities.size();
              GAIA_FOR(cnt)
              entityVersions[i] = world_entity_archetype_version(world, sourceEntities[i]);
            }
          };
 
          struct TraversedSourcePayload {
            cnt::darray<SrcTravSnapshotItem> snapshot;
            uint32_t enabledVersion = UINT32_MAX;
            bool overflowed = false;
 
            void clear() {
              snapshot.clear();
              enabledVersion = UINT32_MAX;
              overflowed = false;
            }
 
            GAIA_NODISCARD bool is_overflowed() const {
              return overflowed;
            }
 
            GAIA_NODISCARD bool versions_changed(const World& world) const {
              if (enabledVersion != world_enabled_hierarchy_version(world))
                return true;
 
              const auto cnt = (uint32_t)snapshot.size();
              GAIA_FOR(cnt) {
                const auto& item = snapshot[i];
                if (item.sourceVersion != world_entity_archetype_version(world, item.entity))
                  return true;
              }
 
              return false;
            }
 
            GAIA_NODISCARD bool changed(const World& world, const cnt::darray<SrcTravSnapshotItem>& items) const {
              if (enabledVersion != world_enabled_hierarchy_version(world))
                return true;
 
              if (items.size() != snapshot.size())
                return true;
 
              const auto cnt = (uint32_t)snapshot.size();
              GAIA_FOR(cnt) {
                if (items[i] != snapshot[i])
                  return true;
              }
 
              return false;
            }
 
            void capture(const World& world, const cnt::darray<SrcTravSnapshotItem>& items) {
              snapshot = items;
              enabledVersion = world_enabled_hierarchy_version(world);
              overflowed = false;
            }
 
            void mark_overflowed() {
              snapshot.clear();
              enabledVersion = UINT32_MAX;
              overflowed = true;
            }
          };
 
          struct VariablePayload {
            cnt::sarray<Entity, MaxVarCnt> bindings;
            uint8_t bindingMask = 0;
 
            void clear() {
              bindingMask = 0;
            }
 
            GAIA_NODISCARD bool
            changed(const cnt::sarray<Entity, MaxVarCnt>& runtimeBindings, uint8_t runtimeBindingMask) const {
              if (bindingMask != runtimeBindingMask)
                return true;
              if (runtimeBindingMask == 0)
                return false;
 
              GAIA_FOR(MaxVarCnt) {
                const auto bit = (uint8_t(1) << i);
                if ((runtimeBindingMask & bit) == 0)
                  continue;
 
                if (bindings[i] != runtimeBindings[i])
                  return true;
              }
 
              return false;
            }
 
            void snapshot(const cnt::sarray<Entity, MaxVarCnt>& runtimeBindings, uint8_t runtimeBindingMask) {
              bindings = runtimeBindings;
              bindingMask = runtimeBindingMask;
            }
          };
 
          RelationPayload relation;
          DirectSourcePayload directSource;
          TraversedSourcePayload traversedSource;
          VariablePayload variable;
 
          void clear_input_snapshots() {
            traversedSource.clear();
            variable.clear();
          }
        };
 
        DynamicCacheState dynamic;
 
        ArchetypeId lastArchetypeId{};
        uint32_t resultCacheRevision = 1;
        uint8_t resultCacheMayNeedPrefabFilter = 0;
        uint8_t dirtyFlags = DirtyFlags::All;
 
        void clear_seed_cache() {
          seedArchetypeSet = {};
          seedArchetypeCache = {};
        }
 
        void clear_result_cache() {
          archetypeSet = {};
          archetypeCache = {};
          resultCacheMayNeedPrefabFilter = 0;
          exec.clear();
          grouped.clear();
          nonTrivial.clear();
        }
 
        void clear_transient_result_cache() {
          archetypeCache.clear();
          resultCacheMayNeedPrefabFilter = 0;
          exec.clear_transient();
          grouped.clear_transient();
          nonTrivial.clear_transient();
        }
 
        void clear_cache() {
          clear_seed_cache();
          clear_result_cache();
        }
 
        void reset() {
          clear_cache();
          dynamic.clear_input_snapshots();
          lastArchetypeId = 0;
          dirtyFlags = DirtyFlags::All;
        }
 
        void invalidate_seed() {
          dirtyFlags = (uint8_t)(dirtyFlags | DirtyFlags::Seed | DirtyFlags::Result);
        }
 
        void invalidate_result() {
          dirtyFlags = (uint8_t)(dirtyFlags | DirtyFlags::Result);
        }
 
        void invalidate_all() {
          dirtyFlags = DirtyFlags::All;
        }
 
        GAIA_NODISCARD bool seed_dirty() const {
          return (dirtyFlags & DirtyFlags::Seed) != 0;
        }
 
        GAIA_NODISCARD bool result_dirty() const {
          return (dirtyFlags & DirtyFlags::Result) != 0;
        }
 
        GAIA_NODISCARD bool needs_refresh() const {
          return seed_dirty() || result_dirty();
        }
 
        void clear_dirty() {
          dirtyFlags = DirtyFlags::Clean;
        }
      };
 
      uint32_t m_refs = 0;
#if GAIA_ECS_TEST_HOOKS
      uint32_t m_testMatchPassCount = 0;
#endif
 
      QueryPlan m_plan;
      QueryState m_state;
 
      enum QueryCmdType : uint8_t { ALL, OR, NOT };
 
      void reset_matching_cache(bool trackMembershipChange) {
        if (trackMembershipChange && !m_state.archetypeCache.empty())
          mark_result_cache_membership_changed();
 
        m_state.reset();
 
        auto& ctxData = m_plan.ctx.data;
        ctxData.lastMatchedArchetypeIdx_All = {};
        ctxData.lastMatchedArchetypeIdx_Or = {};
        ctxData.lastMatchedArchetypeIdx_Not = {};
      }
 
      void clear_result_cache() {
        m_state.clear_result_cache();
      }
 
      void invalidate_result_barriers() {
        m_state.nonTrivial.barrierRelVersion = UINT32_MAX;
        m_state.nonTrivial.barrierEnabledVersion = UINT32_MAX;
      }
 
      void mark_result_cache_membership_changed() {
        invalidate_result_barriers();
        ++m_state.resultCacheRevision;
        if (m_state.resultCacheRevision != 0)
          return;
 
        // Reserve 0 as the "never synced" value in QueryCache.
        m_state.resultCacheRevision = 1;
      }
 
      GAIA_NODISCARD bool has_dyn_terms() const {
        return m_plan.ctx.data.cachePolicy == QueryCtx::CachePolicy::Dynamic;
      }
 
      GAIA_NODISCARD bool can_reuse_dyn_cache() const {
        return m_plan.ctx.data.canReuseDynamicCache;
      }
 
      GAIA_NODISCARD bool uses_direct_src_version_tracking() const {
        return m_plan.ctx.data.uses_direct_src_version_tracking();
      }
 
      GAIA_NODISCARD bool uses_src_trav_snapshot() const {
        return m_plan.ctx.data.uses_src_trav_snapshot();
      }
 
      GAIA_NODISCARD bool dyn_rel_versions_changed() const {
        return m_state.dynamic.relation.changed(*world(), m_plan.ctx.data.deps.relations_view());
      }
 
      GAIA_NODISCARD bool dyn_var_bindings_changed(
          const cnt::sarray<Entity, MaxVarCnt>& runtimeVarBindings, uint8_t runtimeVarBindingMask) const {
        return m_state.dynamic.variable.changed(runtimeVarBindings, runtimeVarBindingMask);
      }
 
      GAIA_NODISCARD bool direct_src_versions_changed() const {
        return m_state.dynamic.directSource.changed(*world(), m_plan.ctx.data.deps.src_entities_view());
      }
 
      template <typename Func>
      void each_reusable_src_entity(Func&& func) const {
        const auto terms = m_plan.ctx.data.terms_view();
        const auto cnt = (uint32_t)terms.size();
        GAIA_FOR(cnt) {
          const auto& term = terms[i];
          if (term.src == EntityBad || is_variable(term.src))
            continue;
 
          (void)vm::detail::each_lookup_src(*world(), term, term.src, [&](Entity source) {
            return func(source);
          });
        }
      }
 
      GAIA_NODISCARD bool build_src_trav_snapshot(cnt::darray<SrcTravSnapshotItem>& items) const {
        const auto maxItems = (uint32_t)m_plan.ctx.data.cacheSrcTrav;
        items.clear();
        if (maxItems == 0)
          return false;
 
        bool overflowed = false;
        each_reusable_src_entity([&](Entity source) {
          if (items.size() >= maxItems) {
            overflowed = true;
            return true;
          }
          items.push_back({source, world_entity_archetype_version(*world(), source)});
          return false;
        });
 
        return !overflowed;
      }
 
      GAIA_NODISCARD bool traversed_src_versions_changed() const {
        return m_state.dynamic.traversedSource.versions_changed(*world());
      }
 
      GAIA_NODISCARD bool traversed_src_inputs_changed(bool relationVersionsChanged) {
        if (m_state.dynamic.traversedSource.is_overflowed())
          return true;
 
        if (!relationVersionsChanged)
          return traversed_src_versions_changed();
 
        cnt::darray<SrcTravSnapshotItem> scratch;
        if (!build_src_trav_snapshot(scratch))
          return true;
 
        return m_state.dynamic.traversedSource.changed(*world(), scratch);
      }
 
      GAIA_NODISCARD bool mixed_dyn_inputs_changed(
          const cnt::sarray<Entity, MaxVarCnt>& runtimeVarBindings, uint8_t runtimeVarBindingMask) {
        const auto& deps = m_plan.ctx.data.deps;
        if (deps.has_dep_flag(QueryCtx::DependencyHasVariableTerms) &&
            dyn_var_bindings_changed(runtimeVarBindings, runtimeVarBindingMask))
          return true;
 
        const bool relationVersionsChanged = dyn_rel_versions_changed();
        const bool hasSourceTerms = deps.has_dep_flag(QueryCtx::DependencyHasSourceTerms);
        const bool hasTraversalTerms = deps.has_dep_flag(QueryCtx::DependencyHasTraversalTerms);
        if (relationVersionsChanged && !(hasSourceTerms && hasTraversalTerms))
          return true;
 
        if (!hasSourceTerms)
          return relationVersionsChanged;
 
        if (m_state.dynamic.traversedSource.is_overflowed())
          return true;
 
        if (!hasTraversalTerms)
          return direct_src_versions_changed();
 
        return traversed_src_inputs_changed(relationVersionsChanged);
      }
 
      GAIA_NODISCARD bool
      dyn_inputs_changed(const cnt::sarray<Entity, MaxVarCnt>& runtimeVarBindings, uint8_t runtimeVarBindingMask) {
        if (!can_reuse_dyn_cache())
          return false;
 
        switch (m_plan.ctx.data.dynamicCacheKind) {
          case QueryCtx::DynamicCacheKind::None:
            return false;
          case QueryCtx::DynamicCacheKind::RelationOnly:
            return dyn_rel_versions_changed();
          case QueryCtx::DynamicCacheKind::DirectSource:
            return direct_src_versions_changed();
          case QueryCtx::DynamicCacheKind::TraversedSource:
            return traversed_src_inputs_changed(dyn_rel_versions_changed());
          case QueryCtx::DynamicCacheKind::Variable:
            return dyn_var_bindings_changed(runtimeVarBindings, runtimeVarBindingMask);
          case QueryCtx::DynamicCacheKind::Mixed:
            return mixed_dyn_inputs_changed(runtimeVarBindings, runtimeVarBindingMask);
        }
 
        GAIA_ASSERT(false);
        return true;
      }
 
      void snapshot_dyn_rel_inputs() {
        m_state.dynamic.relation.snapshot(*world(), m_plan.ctx.data.deps.relations_view());
      }
 
      void snapshot_dyn_direct_src_inputs() {
        m_state.dynamic.directSource.snapshot(*world(), m_plan.ctx.data.deps.src_entities_view());
      }
 
      void snapshot_dyn_traversed_src_inputs() {
        cnt::darray<SrcTravSnapshotItem> scratch;
        if (build_src_trav_snapshot(scratch))
          m_state.dynamic.traversedSource.capture(*world(), scratch);
        else
          m_state.dynamic.traversedSource.mark_overflowed();
      }
 
      void
      snapshot_dyn_var_inputs(const cnt::sarray<Entity, MaxVarCnt>& runtimeVarBindings, uint8_t runtimeVarBindingMask) {
        m_state.dynamic.variable.snapshot(runtimeVarBindings, runtimeVarBindingMask);
      }
 
      void
      snapshot_dyn_inputs(const cnt::sarray<Entity, MaxVarCnt>& runtimeVarBindings, uint8_t runtimeVarBindingMask) {
        if (!can_reuse_dyn_cache())
          return;
 
        m_state.dynamic.clear_input_snapshots();
 
        switch (m_plan.ctx.data.dynamicCacheKind) {
          case QueryCtx::DynamicCacheKind::None:
            return;
          case QueryCtx::DynamicCacheKind::RelationOnly:
            snapshot_dyn_rel_inputs();
            return;
          case QueryCtx::DynamicCacheKind::DirectSource:
            snapshot_dyn_direct_src_inputs();
            return;
          case QueryCtx::DynamicCacheKind::TraversedSource:
            snapshot_dyn_rel_inputs();
            snapshot_dyn_traversed_src_inputs();
            return;
          case QueryCtx::DynamicCacheKind::Variable:
            snapshot_dyn_var_inputs(runtimeVarBindings, runtimeVarBindingMask);
            return;
          case QueryCtx::DynamicCacheKind::Mixed: {
            const auto& deps = m_plan.ctx.data.deps;
            snapshot_dyn_rel_inputs();
            if (deps.has_dep_flag(QueryCtx::DependencyHasSourceTerms)) {
              if (deps.has_dep_flag(QueryCtx::DependencyHasTraversalTerms))
                snapshot_dyn_traversed_src_inputs();
              else
                snapshot_dyn_direct_src_inputs();
            }
            if (deps.has_dep_flag(QueryCtx::DependencyHasVariableTerms))
              snapshot_dyn_var_inputs(runtimeVarBindings, runtimeVarBindingMask);
            return;
          }
        }
 
        GAIA_ASSERT(false);
      }
      template <typename TType>
      GAIA_NODISCARD bool has_inter([[maybe_unused]] QueryOpKind op, bool isReadWrite) const {
        using T = core::raw_t<TType>;
 
        if constexpr (std::is_same_v<T, Entity>) {
          // Entities are read-only.
          GAIA_ASSERT(!isReadWrite);
          // Skip Entity input args. Entities are always there.
          return true;
        } else {
          Entity id;
 
          if constexpr (is_pair<T>::value) {
            const auto rel = m_plan.ctx.cc->get<typename T::rel>().entity;
            const auto tgt = m_plan.ctx.cc->get<typename T::tgt>().entity;
            id = (Entity)Pair(rel, tgt);
          } else {
            id = m_plan.ctx.cc->get<T>().entity;
          }
 
          const auto& ctxData = m_plan.ctx.data;
          const auto compIdx = comp_idx<MAX_ITEMS_IN_QUERY>(ctxData.terms.data(), id, EntityBad);
 
          if (op != ctxData.terms[compIdx].op)
            return false;
 
          // Read-write mask must match
          const uint32_t maskRW = (uint32_t)ctxData.readWriteMask & (1U << compIdx);
          const uint32_t maskXX = (uint32_t)isReadWrite << compIdx;
          return maskRW == maskXX;
        }
      }
 
      template <typename T>
      GAIA_NODISCARD bool has_inter(QueryOpKind op) const {
        // static_assert(is_raw_v<<T>, "has() must be used with raw types");
        constexpr bool isReadWrite = core::is_mut_v<T>;
        return has_inter<T>(op, isReadWrite);
      }
 
    public:
      void add_ref() {
        ++m_refs;
        GAIA_ASSERT(m_refs != 0);
      }
 
      void dec_ref() {
        GAIA_ASSERT(m_refs > 0);
        --m_refs;
      }
 
      uint32_t refs() const {
        return m_refs;
      }
 
      void init(World* world) {
        m_plan.ctx.w = world;
      }
 
      void reset() {
        reset_matching_cache(true);
      }
 
      void invalidate(InvalidationKind kind = InvalidationKind::All) {
        switch (kind) {
          case InvalidationKind::Result:
            m_state.invalidate_result();
            invalidate_result_barriers();
            break;
          case InvalidationKind::Seed:
            m_state.invalidate_seed();
            break;
          case InvalidationKind::All:
            m_state.invalidate_all();
            break;
        }
      }
 
      void invalidate_seed() {
        invalidate(InvalidationKind::Seed);
      }
 
      void invalidate_result() {
        invalidate(InvalidationKind::Result);
      }
 
      void invalidate_sort() {
        if (m_plan.ctx.data.sortByFunc != nullptr)
          m_plan.ctx.data.flags |= QueryCtx::QueryFlags::SortEntities;
      }
 
      GAIA_NODISCARD static QueryInfo create(
          QueryId id, QueryCtx&& ctx, const EntityToArchetypeMap& entityToArchetypeMap,
          std::span<const Archetype*> allArchetypes) {
        // Make sure query items are sorted
        sort(ctx);
 
        QueryInfo info;
        info.idx = id;
        info.gen = 0;
 
        info.m_plan.ctx = GAIA_MOV(ctx);
        info.m_plan.ctx.q.handle = {id, 0};
 
        // Compile the query
        info.compile(entityToArchetypeMap, allArchetypes);
 
        return info;
      }
 
      GAIA_NODISCARD static QueryInfo create(uint32_t idx, uint32_t gen, void* pCtx) {
        auto* pCreationCtx = (QueryInfoCreationCtx*)pCtx;
        auto& queryCtx = *pCreationCtx->pQueryCtx;
        auto& entityToArchetypeMap = (EntityToArchetypeMap&)*pCreationCtx->pEntityToArchetypeMap;
 
        // Make sure query items are sorted
        sort(queryCtx);
 
        QueryInfo info;
        info.idx = idx;
        info.gen = gen;
 
        info.m_plan.ctx = GAIA_MOV(queryCtx);
        info.m_plan.ctx.q.handle = {idx, gen};
 
        // Compile the query
        info.compile(entityToArchetypeMap, pCreationCtx->allArchetypes);
 
        return info;
      }
 
      GAIA_NODISCARD static QueryHandle handle(const QueryInfo& info) {
        return QueryHandle(info.idx, info.gen);
      }
 
      void compile(const EntityToArchetypeMap& entityToArchetypeMap, std::span<const Archetype*> allArchetypes) {
        GAIA_PROF_SCOPE(queryinfo::compile);
 
        // Compile the opcodes
        m_plan.vm.compile(entityToArchetypeMap, allArchetypes, m_plan.ctx);
      }
 
      void recompile() {
        GAIA_PROF_SCOPE(queryinfo::recompile);
 
        // Compile the opcodes
        m_plan.vm.create_opcodes(m_plan.ctx);
      }
 
      GAIA_NODISCARD QueryCtx::CachePolicy cache_policy() const {
        return m_plan.ctx.data.cachePolicy;
      }
 
      GAIA_NODISCARD bool has_grouped_payload() const {
        return m_plan.ctx.data.groupBy != EntityBad;
      }
 
      template <typename Container>
      void group_ids(Container& out, bool sortGroups) {
        out.clear();
        ensure_group_data(sortGroups);
 
        const auto cnt = (uint32_t)m_state.grouped.archetypeGroupData.size();
        out.reserve(cnt);
        GAIA_FOR(cnt) {
          const auto groupId = m_state.grouped.archetypeGroupData[i].groupId;
          if (groupId != 0)
            out.push_back(groupId);
        }
      }
 
      GAIA_NODISCARD bool has_sorted_payload() const {
        return m_plan.ctx.data.sortByFunc != nullptr;
      }
 
      GAIA_NODISCARD uint32_t result_cache_rev() const {
        return m_state.resultCacheRevision;
      }
 
      GAIA_NODISCARD bool result_cache_may_need_prefab_filter() const {
        return m_state.resultCacheMayNeedPrefabFilter != 0;
      }
 
      GAIA_NODISCARD bool can_update_with_new_archetype() const {
        // Only immediate structural queries participate in archetype-create propagation.
        return m_plan.vm.is_compiled() && cache_policy() == QueryCtx::CachePolicy::Immediate &&
               !m_state.needs_refresh();
      }
 
      GAIA_NODISCARD bool can_use_direct_create_archetype_match() const {
        return m_plan.ctx.data.createArchetypeMatchKind == QueryCtx::CreateArchetypeMatchKind::DirectStructuralTerms;
      }
 
      GAIA_NODISCARD bool direct_create_archetype_match_uses_is() const {
        const auto& ctxData = m_plan.ctx.data;
        return (ctxData.as_mask_0 + ctxData.as_mask_1) != 0;
      }
 
      GAIA_NODISCARD bool operator==(const QueryCtx& other) const {
        return m_plan.ctx == other;
      }
 
      GAIA_NODISCARD bool operator!=(const QueryCtx& other) const {
        return m_plan.ctx != other;
      }
 
      struct CleanUpTmpArchetypeMatches {
        World& world;
        bool keepStamps;
 
        explicit CleanUpTmpArchetypeMatches(World& world, bool keepStamps): world(world), keepStamps(keepStamps) {}
        CleanUpTmpArchetypeMatches(const CleanUpTmpArchetypeMatches&) = delete;
        CleanUpTmpArchetypeMatches(CleanUpTmpArchetypeMatches&&) = delete;
        CleanUpTmpArchetypeMatches& operator=(const CleanUpTmpArchetypeMatches&) = delete;
        CleanUpTmpArchetypeMatches& operator=(CleanUpTmpArchetypeMatches&&) = delete;
 
        ~CleanUpTmpArchetypeMatches() {
          query_match_scratch_release(world, keepStamps);
        }
      };
 
      template <typename ArchetypeLookup>
      void match(
          const ArchetypeLookup& entityToArchetypeMap, std::span<const Archetype*> allArchetypes,
          const EntityToArchetypeVersionMap* pEntityToArchetypeMapVersions, ArchetypeId archetypeLastId,
          const cnt::sarray<Entity, MaxVarCnt>& runtimeVarBindings, uint8_t runtimeVarBindingMask) {
        auto& ctxData = m_plan.ctx.data;
 
        // Recompile if necessary
        if ((ctxData.flags & QueryCtx::QueryFlags::Recompile) != 0)
          recompile();
 
        // Skip if nothing has been compiled.
        if (!m_plan.vm.is_compiled())
          return;
 
        const bool hasDynamicTerms = has_dyn_terms();
        const bool canReuseDynamicCache = can_reuse_dyn_cache();
        const bool refreshDynamicCache =
            hasDynamicTerms && (!canReuseDynamicCache || m_state.needs_refresh() ||
                                dyn_inputs_changed(runtimeVarBindings, runtimeVarBindingMask));
        bool compareDynamicMembership = false;
        QueryMatchScratch* pMatchScratch = nullptr;
 
        if (refreshDynamicCache) {
          compareDynamicMembership = !m_state.archetypeCache.empty();
          if (compareDynamicMembership) {
            auto& w = *world();
            pMatchScratch = &query_match_scratch_acquire(w);
            pMatchScratch->matchesArrDynPrev = m_state.archetypeCache;
          }
          // Dynamic queries keep their cached result as long as tracked runtime inputs stay stable.
          reset_matching_cache(!compareDynamicMembership);
        } else if (m_state.seed_dirty()) {
          reset_matching_cache(true);
        } else if (m_state.result_dirty()) {
          sync_result_cache_from_seed_cache();
          if (m_state.lastArchetypeId == archetypeLastId) {
            sort_entities();
            sort_cache_groups();
            m_state.clear_dirty();
            return;
          }
        }
 
        // Skip if no new archetype appeared and the cached match set is still valid.
        GAIA_ASSERT(archetypeLastId >= m_state.lastArchetypeId);
        if (!m_state.needs_refresh() && m_state.lastArchetypeId == archetypeLastId &&
            (!hasDynamicTerms || canReuseDynamicCache)) {
          // Sort entities if necessary
          sort_entities();
          return;
        }
 
        m_state.lastArchetypeId = archetypeLastId;
 
        GAIA_PROF_SCOPE(queryinfo::match);
 
        auto& w = *world();
        auto& matchScratch = pMatchScratch != nullptr ? *pMatchScratch : query_match_scratch_acquire(w);
        CleanUpTmpArchetypeMatches autoCleanup(w, true);
 
        // Prepare the context
        vm::MatchingCtx ctx{};
        ctx.pWorld = &w;
        // ctx.targetEntities = {};
        ctx.allArchetypes = allArchetypes;
        if constexpr (std::is_same_v<ArchetypeLookup, EntityToArchetypeMap>) {
          GAIA_ASSERT(pEntityToArchetypeMapVersions != nullptr);
          ctx.archetypeLookup = vm::make_archetype_lookup_view(entityToArchetypeMap, *pEntityToArchetypeMapVersions);
        } else {
          (void)pEntityToArchetypeMapVersions;
          ctx.archetypeLookup = vm::make_archetype_lookup_view(entityToArchetypeMap);
        }
 
        ctx.pMatchesArr = &matchScratch.matchesArr;
        ctx.pMatchesStampByArchetypeId = &matchScratch.matchStamps;
        ctx.matchesVersion = matchScratch.next_match_version();
        ctx.pLastMatchedArchetypeIdx_All = &ctxData.lastMatchedArchetypeIdx_All;
        ctx.pLastMatchedArchetypeIdx_Or = &ctxData.lastMatchedArchetypeIdx_Or;
        ctx.pLastMatchedArchetypeIdx_Not = &ctxData.lastMatchedArchetypeIdx_Not;
        ctx.queryMask = ctxData.queryMask;
        ctx.as_mask_0 = ctxData.as_mask_0;
        ctx.as_mask_1 = ctxData.as_mask_1;
        ctx.flags = ctxData.flags;
        ctx.varBindings = runtimeVarBindings;
        ctx.varBindingMask = runtimeVarBindingMask;
 
        // Run the virtual machine
#if GAIA_ECS_TEST_HOOKS
        ++m_testMatchPassCount;
#endif
        m_plan.vm.exec(ctx);
 
        // Write found matches to cache
        const bool trackMembershipChangeOnAdd = !compareDynamicMembership;
        for (const auto* pArchetype: *ctx.pMatchesArr) {
          if (hasDynamicTerms) {
            add_archetype_to_cache(pArchetype, trackMembershipChangeOnAdd, false);
          } else {
            add_archetype_to_seed_cache(pArchetype);
            add_archetype_to_cache(pArchetype, true, false);
          }
        }
 
        if (compareDynamicMembership) {
          if (!has_same_result_membership_as(matchScratch.matchesArrDynPrev))
            mark_result_cache_membership_changed();
        }
 
        // Sort entities if necessary
        sort_entities();
        // Sort cache groups if necessary
        sort_cache_groups();
        snapshot_dyn_inputs(runtimeVarBindings, runtimeVarBindingMask);
        m_state.clear_dirty();
      }
 
      bool match_one(
          const Archetype& archetype, EntitySpan targetEntities,
          const cnt::sarray<Entity, MaxVarCnt>& runtimeVarBindings, uint8_t runtimeVarBindingMask) {
        auto& ctxData = m_plan.ctx.data;
 
        // Recompile if necessary
        if ((ctxData.flags & QueryCtx::QueryFlags::Recompile) != 0)
          recompile();
 
        // Skip if nothing has been compiled.
        if (!m_plan.vm.is_compiled())
          return false;
 
        const bool hasDynamicTerms = has_dyn_terms();
        const bool canReuseDynamicCache = can_reuse_dyn_cache();
        if ((hasDynamicTerms && (!canReuseDynamicCache || m_state.needs_refresh() ||
                                 dyn_inputs_changed(runtimeVarBindings, runtimeVarBindingMask))) ||
            m_state.seed_dirty()) {
          // Dynamic queries keep their cached result as long as tracked runtime inputs stay stable.
          reset_matching_cache(true);
        } else if (m_state.result_dirty()) {
          sync_result_cache_from_seed_cache();
        }
 
        GAIA_PROF_SCOPE(queryinfo::match1);
 
        auto& w = *world();
        auto& matchScratch = query_match_scratch_acquire(w);
        CleanUpTmpArchetypeMatches autoCleanup(w, true);
 
        // Prepare the context
        vm::MatchingCtx ctx{};
        ctx.pWorld = &w;
        ctx.targetEntities = targetEntities;
        const auto* pArchetype = &archetype;
        ctx.allArchetypes = std::span((const Archetype**)&pArchetype, 1);
        ctx.archetypeLookup = {};
        ctx.pMatchesArr = &matchScratch.matchesArr;
        ctx.pMatchesStampByArchetypeId = &matchScratch.matchStamps;
        ctx.matchesVersion = matchScratch.next_match_version();
        ctx.pLastMatchedArchetypeIdx_All = nullptr;
        ctx.pLastMatchedArchetypeIdx_Or = nullptr;
        ctx.pLastMatchedArchetypeIdx_Not = nullptr;
        ctx.queryMask = ctxData.queryMask;
        ctx.as_mask_0 = ctxData.as_mask_0;
        ctx.as_mask_1 = ctxData.as_mask_1;
        ctx.flags = ctxData.flags;
        ctx.varBindings = runtimeVarBindings;
        ctx.varBindingMask = runtimeVarBindingMask;
 
        // Run the virtual machine
#if GAIA_ECS_TEST_HOOKS
        ++m_testMatchPassCount;
#endif
        m_plan.vm.exec(ctx);
        const bool matched = !ctx.pMatchesArr->empty();
 
        // Write found matches to cache
        for (const auto* pArch: *ctx.pMatchesArr) {
          if (hasDynamicTerms) {
            add_archetype_to_cache(pArch, true, false);
          } else {
            add_archetype_to_seed_cache(pArch);
            add_archetype_to_cache(pArch, true, false);
          }
        }
        snapshot_dyn_inputs(runtimeVarBindings, runtimeVarBindingMask);
        m_state.clear_dirty();
        return matched;
      }
 
      void ensure_matches(
          const EntityToArchetypeMap& entityToArchetypeMap, std::span<const Archetype*> allArchetypes,
          const EntityToArchetypeVersionMap& entityToArchetypeMapVersions, ArchetypeId archetypeLastId,
          const cnt::sarray<Entity, MaxVarCnt>& runtimeVarBindings, uint8_t runtimeVarBindingMask) {
        match(
            entityToArchetypeMap, allArchetypes, &entityToArchetypeMapVersions, archetypeLastId, runtimeVarBindings,
            runtimeVarBindingMask);
      }
 
      void ensure_matches_transient(
          const EntityToArchetypeMap& entityToArchetypeMap, std::span<const Archetype*> allArchetypes,
          const EntityToArchetypeVersionMap& entityToArchetypeMapVersions,
          const cnt::sarray<Entity, MaxVarCnt>& runtimeVarBindings, uint8_t runtimeVarBindingMask) {
        auto& ctxData = m_plan.ctx.data;
 
        if ((ctxData.flags & QueryCtx::QueryFlags::Recompile) != 0)
          recompile();
 
        if (!m_plan.vm.is_compiled())
          return;
 
        m_state.clear_transient_result_cache();
 
        auto& w = *world();
        auto& matchScratch = query_match_scratch_acquire(w);
        CleanUpTmpArchetypeMatches autoCleanup(w, true);
 
        vm::MatchingCtx ctx{};
        ctx.pWorld = &w;
        ctx.allArchetypes = allArchetypes;
        ctx.archetypeLookup = vm::make_archetype_lookup_view(entityToArchetypeMap, entityToArchetypeMapVersions);
        ctx.pMatchesArr = &matchScratch.matchesArr;
        ctx.pMatchesStampByArchetypeId = &matchScratch.matchStamps;
        ctx.matchesVersion = matchScratch.next_match_version();
        ctx.pLastMatchedArchetypeIdx_All = nullptr;
        ctx.pLastMatchedArchetypeIdx_Or = nullptr;
        ctx.pLastMatchedArchetypeIdx_Not = nullptr;
        ctx.queryMask = ctxData.queryMask;
        ctx.as_mask_0 = ctxData.as_mask_0;
        ctx.as_mask_1 = ctxData.as_mask_1;
        ctx.flags = ctxData.flags;
        ctx.varBindings = runtimeVarBindings;
        ctx.varBindingMask = runtimeVarBindingMask;
 
        m_plan.vm.exec(ctx);
 
        m_state.archetypeCache.reserve(ctx.pMatchesArr->size());
        if (ctxData.groupBy != EntityBad)
          m_state.grouped.archetypeGroupIds.reserve(ctx.pMatchesArr->size());
        for (const auto* pArchetype: *ctx.pMatchesArr)
          add_archetype_to_transient_cache(pArchetype);
 
        sort_entities();
        ensure_group_data(false);
      }
 
      bool ensure_matches_one(
          const Archetype& archetype, EntitySpan targetEntities,
          const cnt::sarray<Entity, MaxVarCnt>& runtimeVarBindings, uint8_t runtimeVarBindingMask) {
        return match_one(archetype, targetEntities, runtimeVarBindings, runtimeVarBindingMask);
      }
 
      bool ensure_matches_one_transient(
          const Archetype& archetype, EntitySpan targetEntities,
          const cnt::sarray<Entity, MaxVarCnt>& runtimeVarBindings, uint8_t runtimeVarBindingMask) {
        auto& ctxData = m_plan.ctx.data;
 
        if ((ctxData.flags & QueryCtx::QueryFlags::Recompile) != 0)
          recompile();
 
        if (!m_plan.vm.is_compiled())
          return false;
 
        m_state.clear_transient_result_cache();
 
        auto& w = *world();
        auto& matchScratch = query_match_scratch_acquire(w);
        CleanUpTmpArchetypeMatches autoCleanup(w, true);
 
        vm::MatchingCtx ctx{};
        ctx.pWorld = &w;
        ctx.targetEntities = targetEntities;
        const auto* pArchetype = &archetype;
        ctx.allArchetypes = std::span((const Archetype**)&pArchetype, 1);
        ctx.archetypeLookup = {};
        ctx.pMatchesArr = &matchScratch.matchesArr;
        ctx.pMatchesStampByArchetypeId = &matchScratch.matchStamps;
        ctx.matchesVersion = matchScratch.next_match_version();
        ctx.pLastMatchedArchetypeIdx_All = nullptr;
        ctx.pLastMatchedArchetypeIdx_Or = nullptr;
        ctx.pLastMatchedArchetypeIdx_Not = nullptr;
        ctx.queryMask = ctxData.queryMask;
        ctx.as_mask_0 = ctxData.as_mask_0;
        ctx.as_mask_1 = ctxData.as_mask_1;
        ctx.flags = ctxData.flags;
        ctx.varBindings = runtimeVarBindings;
        ctx.varBindingMask = runtimeVarBindingMask;
 
        m_plan.vm.exec(ctx);
        const bool matched = !ctx.pMatchesArr->empty();
 
        m_state.archetypeCache.reserve(ctx.pMatchesArr->size());
        if (ctxData.groupBy != EntityBad)
          m_state.grouped.archetypeGroupIds.reserve(ctx.pMatchesArr->size());
        for (const auto* pArch: *ctx.pMatchesArr)
          add_archetype_to_transient_cache(pArch);
 
        ensure_group_data(false);
        return matched;
      }
 
      bool register_archetype(const Archetype& archetype, Entity matchedSelector = EntityBad, bool assumeNew = false) {
        auto& ctxData = m_plan.ctx.data;
 
        // Recompile if necessary.
        if ((ctxData.flags & QueryCtx::QueryFlags::Recompile) != 0)
          recompile();
 
        if (!can_update_with_new_archetype())
          return false;
 
        const bool hadMatchBefore = !assumeNew && m_state.archetypeSet.contains(&archetype);
        if (can_use_direct_create_archetype_match()) {
          const bool usesIs = direct_create_archetype_match_uses_is();
          bool hasOrTerms = false;
          bool matchedOrTerm = false;
          for (const auto& term: ctxData.terms_view()) {
            if (term.id == matchedSelector) {
              if (term.op == QueryOpKind::Or)
                matchedOrTerm = true;
              continue;
            }
 
            const bool present = usesIs ? vm::detail::match_single_id_on_archetype(*world(), archetype, term.id)
                                        : world_component_index_match_count(*world(), archetype, term.id) != 0;
            if (term.op == QueryOpKind::Or) {
              hasOrTerms = true;
              matchedOrTerm |= present;
              continue;
            }
            if (term.op == QueryOpKind::Any)
              continue;
 
            const bool matched = term.op == QueryOpKind::Not ? !present : present;
            if (!matched)
              return false;
          }
          if (hasOrTerms && !matchedOrTerm)
            return false;
          if (hadMatchBefore)
            return false;
 
          if (assumeNew)
            add_new_archetype_to_immediate_caches(&archetype, true);
          else {
            add_archetype_to_seed_cache(&archetype, false);
            add_archetype_to_cache(&archetype, true, false);
          }
          return true;
        }
 
        SingleArchetypeLookup singleArchetypeLookup;
        auto addLookupUnique = [&](Entity key, uint16_t compIdx, uint16_t matchCount) {
          const auto keyLookup = EntityLookupKey(key);
          const auto itLookup =
              core::find_if(singleArchetypeLookup.begin(), singleArchetypeLookup.end(), [&](const auto& item) {
                return item.matches(keyLookup);
              });
          if (itLookup != singleArchetypeLookup.end()) {
            auto& entry = itLookup->entry;
            entry.matchCount = (uint16_t)(entry.matchCount + matchCount);
            if (compIdx != ComponentIndexBad)
              entry.compIdx = compIdx;
            return;
          }
          singleArchetypeLookup.push_back(
              SingleArchetypeLookupItem{
                  keyLookup, ComponentIndexEntry{const_cast<Archetype*>(&archetype), compIdx, matchCount}});
        };
        auto archetypeIds = archetype.ids_view();
        const auto cntIds = (uint32_t)archetypeIds.size();
        GAIA_FOR(cntIds) {
          const auto entity = archetypeIds[i];
          singleArchetypeLookup.push_back(
              SingleArchetypeLookupItem{
                  EntityLookupKey(entity), ComponentIndexEntry{const_cast<Archetype*>(&archetype), (uint16_t)i, 1}});
 
          if (!entity.pair())
            continue;
 
          // Wildcard pair lookups use the same special records as the world-level
          // entity-to-archetype map, so incremental matching can reuse the normal VM path.
          const auto relKind = entity.entity() ? EntityKind::EK_Uni : EntityKind::EK_Gen;
          const auto rel = Entity((EntityId)entity.id(), 0, false, false, relKind);
          const auto tgt = Entity((EntityId)entity.gen(), 0, false, false, entity.kind());
          addLookupUnique(Pair(All, tgt), ComponentIndexBad, 1);
          addLookupUnique(Pair(rel, All), ComponentIndexBad, 1);
          addLookupUnique(Pair(All, All), ComponentIndexBad, 1);
        }
 
        auto lastMatchedArchetypeIdx_All = GAIA_MOV(ctxData.lastMatchedArchetypeIdx_All);
        auto lastMatchedArchetypeIdx_Or = GAIA_MOV(ctxData.lastMatchedArchetypeIdx_Or);
        auto lastMatchedArchetypeIdx_Not = GAIA_MOV(ctxData.lastMatchedArchetypeIdx_Not);
        GAIA_ASSERT(ctxData.lastMatchedArchetypeIdx_All.empty());
        GAIA_ASSERT(ctxData.lastMatchedArchetypeIdx_Or.empty());
        GAIA_ASSERT(ctxData.lastMatchedArchetypeIdx_Not.empty());
 
        const auto* pArchetype = &archetype;
        const cnt::sarray<Entity, MaxVarCnt> noRuntimeVarBindings{};
        match(
            singleArchetypeLookup, std::span((const Archetype**)&pArchetype, 1), nullptr, archetype.id(),
            noRuntimeVarBindings, 0);
        ctxData.lastMatchedArchetypeIdx_All = GAIA_MOV(lastMatchedArchetypeIdx_All);
        ctxData.lastMatchedArchetypeIdx_Or = GAIA_MOV(lastMatchedArchetypeIdx_Or);
        ctxData.lastMatchedArchetypeIdx_Not = GAIA_MOV(lastMatchedArchetypeIdx_Not);
        const bool matched = assumeNew ? m_state.archetypeSet.contains(&archetype)
                                       : !hadMatchBefore && m_state.archetypeSet.contains(&archetype);
 
        if (!matched)
          return false;
 
        sort_entities();
        sort_cache_groups();
        return true;
      }
 
      void calculate_sort_data() {
        GAIA_PROF_SCOPE(queryinfo::calc_sort_data);
 
        m_state.nonTrivial.archetypeSortData.clear();
 
        // The function doesn't do any moves and expects that all chunks have their data sorted already.
        // We use a min-heap / priority queue - like structure during query iteration to merge sorted tables:
        // - we hold a cursor into each sorted chunk
        // - we compare the next entity from each chunk using your sorting function
        // - we then pick the smallest one (like k-way merge sort), and advance that cursor
        // This is esentially what this function does:
        // while (any_chunk_has_entities) {
        //  find_chunk_with_smallest_next_element();
        //  yield(entity_from_that_chunk);
        //  adv_cursor_for_that_chunk();
        // }
        // This produces a globally sorted view without modifying actual data. It's a balance between
        // performance and memory usage. We could also sort the data in-place across all chunks, but that
        // would generated too many data moves (entities + all of their components).
 
        struct Cursor {
          uint32_t chunkIdx = 0;
          uint16_t row = 0;
        };
 
        auto& archetypes = m_state.archetypeCache;
 
        // Initialize cursors. We will need as many as there are archetypes.
        cnt::sarray_ext<Cursor, 128> cursors(archetypes.size());
 
        uint32_t currArchetypeIdx = (uint32_t)-1;
        Chunk* pCurrentChunk = nullptr;
        uint16_t currentStartRow = 0;
        uint16_t currentRow = 0;
 
        const void* pDataMin = nullptr;
        const void* pDataCurr = nullptr;
 
        while (true) {
          uint32_t minArchetypeIdx = (uint32_t)-1;
          Entity minEntity = EntityBad;
 
          // Find the next entity across all tables/chunks
          for (uint32_t t = 0; t < archetypes.size(); ++t) {
            const auto* pArchetype = archetypes[t];
            const auto& chunks = pArchetype->chunks();
            auto& cur = cursors[t];
 
            while (cur.chunkIdx < chunks.size() && cur.row >= chunks[cur.chunkIdx]->size()) {
              ++cur.chunkIdx;
              cur.row = 0;
            }
 
            if (cur.chunkIdx >= chunks.size())
              continue;
 
            const auto* pChunk = pArchetype->chunks()[cur.chunkIdx];
            auto entity = pChunk->entity_view()[cur.row];
 
            if (m_plan.ctx.data.sortBy != ecs::EntityBad) {
              auto compIdx = world_component_index_comp_idx(*m_plan.ctx.w, *pArchetype, m_plan.ctx.data.sortBy);
              if (compIdx == BadIndex)
                compIdx = pChunk->comp_idx(m_plan.ctx.data.sortBy);
              pDataCurr = pChunk->comp_ptr(compIdx, cur.row);
            } else
              pDataCurr = &pChunk->entity_view()[cur.row];
 
            if (minEntity == EntityBad) {
              minEntity = entity;
              minArchetypeIdx = t;
              pDataMin = pDataCurr;
              continue;
            }
 
            if (m_plan.ctx.data.sortByFunc(*m_plan.ctx.w, pDataCurr, pDataMin) < 0) {
              minEntity = entity;
              minArchetypeIdx = t;
            }
          }
 
          // No more results found, we can stop
          if (minArchetypeIdx == (uint32_t)-1)
            break;
 
          auto& cur = cursors[minArchetypeIdx];
          const auto& chunks = archetypes[minArchetypeIdx]->chunks();
          Chunk* pChunk = chunks[cur.chunkIdx];
 
          if (minArchetypeIdx == currArchetypeIdx && pChunk == pCurrentChunk) {
            // Current slice
          } else {
            // End previous slice
            if (pCurrentChunk != nullptr) {
              m_state.nonTrivial.archetypeSortData.push_back(
                  {pCurrentChunk, currArchetypeIdx, currentStartRow, (uint16_t)(currentRow - currentStartRow)});
            }
 
            // Start a new slice
            currArchetypeIdx = minArchetypeIdx;
            pCurrentChunk = pChunk;
            currentStartRow = cur.row;
          }
 
          ++cur.row;
          currentRow = cur.row;
        }
 
        if (pCurrentChunk != nullptr) {
          m_state.nonTrivial.archetypeSortData.push_back(
              {pCurrentChunk, currArchetypeIdx, currentStartRow, (uint16_t)(currentRow - currentStartRow)});
        }
      }
 
      void sort_entities() {
        if (m_plan.ctx.data.sortByFunc == nullptr)
          return;
 
        if ((m_plan.ctx.data.flags & QueryCtx::QueryFlags::SortEntities) == 0 && m_state.nonTrivial.sortVersion != 0)
          return;
        m_plan.ctx.data.flags &= ~QueryCtx::QueryFlags::SortEntities;
 
        // First, sort entities in archetypes
        for (const auto* pArchetype: m_state.archetypeCache)
          const_cast<Archetype*>(pArchetype)->sort_entities(m_plan.ctx.data.sortBy, m_plan.ctx.data.sortByFunc);
 
        // Now that entites are sorted, we can start creating slices
        calculate_sort_data();
        m_state.nonTrivial.sortVersion = ::gaia::ecs::world_version(*world());
      }
 
      void sort_cache_groups() {
        if ((m_plan.ctx.data.flags & QueryCtx::QueryFlags::SortGroups) == 0)
          return;
        m_plan.ctx.data.flags &= ~QueryCtx::QueryFlags::SortGroups;
 
        if ((m_plan.ctx.data.flags & QueryCtx::QueryFlags::OrderGroups) != 0)
          ensure_group_data(true);
      }
 
      void swap_archetype_cache_entry(uint32_t left, uint32_t right) {
        auto* pTmpArchetype = m_state.archetypeCache[left];
        m_state.archetypeCache[left] = m_state.archetypeCache[right];
        m_state.archetypeCache[right] = pTmpArchetype;
 
        if (left < m_state.grouped.archetypeGroupIds.size() && right < m_state.grouped.archetypeGroupIds.size()) {
          const auto tmp = m_state.grouped.archetypeGroupIds[left];
          m_state.grouped.archetypeGroupIds[left] = m_state.grouped.archetypeGroupIds[right];
          m_state.grouped.archetypeGroupIds[right] = tmp;
        }
 
        if (left < m_state.exec.archetypeCompIndices.size() && right < m_state.exec.archetypeCompIndices.size()) {
          auto tmp = m_state.exec.archetypeCompIndices[left];
          m_state.exec.archetypeCompIndices[left] = m_state.exec.archetypeCompIndices[right];
          m_state.exec.archetypeCompIndices[right] = tmp;
        }
 
        if (left < m_state.exec.archetypeInheritedData.size() && right < m_state.exec.archetypeInheritedData.size()) {
          auto tmp = m_state.exec.archetypeInheritedData[left];
          m_state.exec.archetypeInheritedData[left] = m_state.exec.archetypeInheritedData[right];
          m_state.exec.archetypeInheritedData[right] = tmp;
        }
 
        if (left < m_state.nonTrivial.archetypeBarrierPasses.size() &&
            right < m_state.nonTrivial.archetypeBarrierPasses.size()) {
          const auto tmp = m_state.nonTrivial.archetypeBarrierPasses[left];
          m_state.nonTrivial.archetypeBarrierPasses[left] = m_state.nonTrivial.archetypeBarrierPasses[right];
          m_state.nonTrivial.archetypeBarrierPasses[right] = tmp;
        }
      }
 
      void ensure_comp_indices() {
        if (!m_state.exec.compIndicesPending)
          return;
 
        m_state.exec.archetypeCompIndices.clear();
        m_state.exec.archetypeCompIndices.reserve(m_state.archetypeCache.size());
        for (const auto* pArchetype: m_state.archetypeCache)
          m_state.exec.archetypeCompIndices.push_back(create_comp_indices(pArchetype));
 
        m_state.exec.compIndicesPending = false;
      }
 
      void
      ensure_direct_chunks(uint32_t idxFrom, uint32_t idxTo, const uint32_t* pDataFields, uint32_t dataFieldCount) {
        ensure_comp_indices();
        const auto* pCompIndicesData = m_state.exec.archetypeCompIndices.data();
        const auto dataFieldsMatch = [&]() {
          if (m_state.exec.directChunksDataFieldCount != dataFieldCount)
            return false;
          GAIA_FOR(dataFieldCount) {
            if (m_state.exec.directChunksDataFields[i] != pDataFields[i])
              return false;
          }
          return true;
        };
 
        const auto currWorldVersion = ::gaia::ecs::world_version(*world());
        const auto currEnabledVersion = world_enabled_hierarchy_version(*world());
        const auto currDeleteVersion = world_archetype_delete_version(*world());
        if (m_state.exec.directChunksWorldVersion == currWorldVersion &&
            m_state.exec.directChunksEnabledVersion == currEnabledVersion &&
            m_state.exec.directChunksDeleteVersion == currDeleteVersion &&
            m_state.exec.directChunksResultRevision == m_state.resultCacheRevision &&
            m_state.exec.directChunksIdxFrom == idxFrom && m_state.exec.directChunksIdxTo == idxTo &&
            dataFieldsMatch() && m_state.exec.directChunksCompIndicesData == pCompIndicesData)
          return;
 
        m_state.exec.directChunks.clear();
        m_state.exec.directChunkData.clear();
        m_state.exec.directChunksIdxFrom = idxFrom;
        m_state.exec.directChunksIdxTo = idxTo;
        m_state.exec.directChunksDataFieldCount = dataFieldCount;
        GAIA_FOR(dataFieldCount) {
          m_state.exec.directChunksDataFields[i] = pDataFields[i];
        }
        m_state.exec.directChunksResultRevision = m_state.resultCacheRevision;
        m_state.exec.directChunksWorldVersion = currWorldVersion;
        m_state.exec.directChunksEnabledVersion = currEnabledVersion;
        m_state.exec.directChunksDeleteVersion = currDeleteVersion;
        m_state.exec.directChunksCompIndicesData = pCompIndicesData;
 
        uint32_t chunkCnt = 0;
        for (uint32_t i = idxFrom; i < idxTo; ++i)
          chunkCnt += (uint32_t)m_state.archetypeCache[i]->chunks().size();
        m_state.exec.directChunks.reserve(chunkCnt);
        m_state.exec.directChunkData.reserve(chunkCnt * dataFieldCount);
 
        for (uint32_t i = idxFrom; i < idxTo; ++i) {
          const auto* pArchetype = m_state.archetypeCache[i];
          if (pArchetype->is_req_del())
            continue;
          const auto* pCompIndices = m_state.exec.archetypeCompIndices[i].indices;
          for (auto* pChunk: pArchetype->chunks()) {
            const auto from = detail::ChunkIterImpl::start_index(pChunk, Constraints::EnabledOnly);
            const auto to = detail::ChunkIterImpl::end_index(pChunk, Constraints::EnabledOnly);
            if (from != to) {
              const void* pData = nullptr;
              uint32_t dataOffset = UINT32_MAX;
              if (dataFieldCount == 1) {
                const auto dataFieldIdx = pDataFields[0];
                const auto compIdx =
                    dataFieldIdx < ChunkHeader::MAX_COMPONENTS ? pCompIndices[dataFieldIdx] : uint8_t(0xFF);
                if (compIdx != 0xFF)
                  pData = pChunk->comp_rec_view()[compIdx].pData;
              } else if (dataFieldCount > 1) {
                dataOffset = (uint32_t)m_state.exec.directChunkData.size();
                const auto recs = pChunk->comp_rec_view();
                GAIA_FOR(dataFieldCount) {
                  const auto dataFieldIdx = pDataFields[i];
                  const auto compIdx =
                      dataFieldIdx < ChunkHeader::MAX_COMPONENTS ? pCompIndices[dataFieldIdx] : uint8_t(0xFF);
                  m_state.exec.directChunkData.push_back(compIdx != 0xFF ? recs[compIdx].pData : nullptr);
                }
              }
              m_state.exec.directChunks.push_back({pArchetype, pChunk, pCompIndices, pData, dataOffset, from, to});
            }
          }
        }
      }
 
      GAIA_NODISCARD bool has_inherited_data_payload() const {
        return ctx().data.deps.has_dep_flag(QueryCtx::DependencyHasInheritedDataTerms);
      }
 
      void ensure_inherited_data() {
        if (!m_state.exec.inheritedDataPending)
          return;
 
        if (!has_inherited_data_payload()) {
          m_state.exec.archetypeInheritedData.clear();
          m_state.exec.inheritedDataPending = false;
          return;
        }
 
        m_state.exec.archetypeInheritedData.clear();
        m_state.exec.archetypeInheritedData.reserve(m_state.archetypeCache.size());
        for (const auto* pArchetype: m_state.archetypeCache)
          create_inherited_data(pArchetype);
 
        m_state.exec.inheritedDataPending = false;
      }
 
      void ensure_group_data(bool orderGroups) {
        if (m_plan.ctx.data.groupBy == EntityBad || !m_state.grouped.dataPending)
          return;
 
        if (!orderGroups && (m_plan.ctx.data.flags & QueryCtx::QueryFlags::OrderGroups) == 0)
          return;
 
        const auto groupCnt = (uint32_t)m_state.grouped.archetypeGroupIds.size();
        // Include the initial cache index so equal groups keep deterministic order under a strict comparator.
        cnt::darray_ext<uint64_t, 32> sortKeys;
        sortKeys.resize(groupCnt);
        GAIA_FOR(groupCnt) {
          sortKeys[i] = ((uint64_t)m_state.grouped.archetypeGroupIds[i] << 32U) | i;
        }
 
        struct sort_cond {
          bool operator()(uint64_t a, uint64_t b) const {
            return a < b;
          }
        };
 
        core::sort(sortKeys, sort_cond{}, [&](uint32_t left, uint32_t right) {
          const auto tmp = sortKeys[left];
          sortKeys[left] = sortKeys[right];
          sortKeys[right] = tmp;
          swap_archetype_cache_entry(left, right);
        });
 
        m_state.grouped.archetypeGroupData.clear();
        m_state.grouped.selectedGroupDataValid = false;
 
        if (m_state.grouped.archetypeGroupIds.empty()) {
          m_state.grouped.dataPending = false;
          return;
        }
 
        GroupId groupId = m_state.grouped.archetypeGroupIds[0];
        uint32_t idxFirst = 0;
        const auto cnt = (uint32_t)m_state.grouped.archetypeGroupIds.size();
        for (uint32_t i = 1; i < cnt; ++i) {
          if (m_state.grouped.archetypeGroupIds[i] == groupId)
            continue;
 
          m_state.grouped.archetypeGroupData.push_back({groupId, idxFirst, i - 1, false});
          groupId = m_state.grouped.archetypeGroupIds[i];
          idxFirst = i;
        }
 
        m_state.grouped.archetypeGroupData.push_back({groupId, idxFirst, cnt - 1, false});
        m_state.grouped.dataPending = false;
      }
 
      void ensure_depth_order_hierarchy_barrier_cache_inter() {
        if (!world_relation_depth_order_prunes_disabled_subtrees(*world(), m_plan.ctx.data.groupBy))
          return;
 
        ensure_group_data(true);
 
        const auto currRelationVersion = world_rel_version(*world(), m_plan.ctx.data.groupBy);
        const auto currEnabledVersion = world_enabled_hierarchy_version(*world());
        if (m_state.nonTrivial.barrierRelVersion == currRelationVersion &&
            m_state.nonTrivial.barrierEnabledVersion == currEnabledVersion)
          return;
 
        m_state.nonTrivial.archetypeBarrierPasses.resize(m_state.archetypeCache.size(), 1);
        m_state.nonTrivial.barrierMayPrune = 0;
 
        const auto relation = m_plan.ctx.data.groupBy;
        for (uint32_t i = 0; i < m_state.archetypeCache.size(); ++i) {
          const auto* pArchetype = m_state.archetypeCache[i];
          auto& barrierPasses = m_state.nonTrivial.archetypeBarrierPasses[i];
          barrierPasses = 1;
 
          auto ids = pArchetype->ids_view();
          for (auto idsIdx: pArchetype->pair_rel_indices(relation)) {
            const auto pair = ids[idsIdx];
            const auto parent = world_pair_target_if_alive(*world(), pair);
            if (parent == EntityBad || !world_entity_enabled_hierarchy(*world(), parent, relation)) {
              barrierPasses = 0;
              m_state.nonTrivial.barrierMayPrune = 1;
              break;
            }
          }
        }
 
        m_state.nonTrivial.barrierRelVersion = currRelationVersion;
        m_state.nonTrivial.barrierEnabledVersion = currEnabledVersion;
      }
 
      ArchetypeCompIndices create_comp_indices(const Archetype* pArchetype) {
        ArchetypeCompIndices cacheData{};
        core::fill(cacheData.indices, cacheData.indices + ChunkHeader::MAX_COMPONENTS, (uint8_t)0xFF);
        const auto terms = ctx().data.terms_view();
        const auto cnt = (uint32_t)terms.size();
        GAIA_FOR(cnt) {
          const auto& term = terms[i];
          const auto fieldIdx = term.fieldIndex;
          const auto queryId = term.id;
          if (!query_term_maps_to_current_archetype(term))
            continue;
 
          if (!queryId.pair() && world_component_uses_sparse_storage(*world(), queryId)) {
#if GAIA_ASSERT_ENABLED
            // Verify that the component is indeed not present on the archetype, otherwise our matching logic is flawed.
            const auto compIdx = core::get_index_unsafe(pArchetype->ids_view(), queryId);
            GAIA_ASSERT(compIdx != BadIndex);
#endif
            cacheData.indices[fieldIdx] = 0xFF;
            continue;
          }
 
          auto compIdx = world_component_index_comp_idx(*world(), *pArchetype, queryId);
          if (compIdx == BadIndex) {
            // Wildcard/semantic terms are not represented by an exact component index entry.
            // OR terms may also be absent from this matching archetype. Use a safe lookup so
            // absent terms stay unmapped instead of becoming a past-the-end chunk column.
            compIdx = core::get_index(pArchetype->ids_view(), queryId);
          }
 
          cacheData.indices[fieldIdx] = compIdx != BadIndex ? (uint8_t)compIdx : (uint8_t)0xFF;
        }
        return cacheData;
      }
 
      void create_inherited_data(const Archetype* pArchetype) {
        ArchetypeInheritedData inheritedData{};
        core::fill(inheritedData.data, inheritedData.data + ChunkHeader::MAX_COMPONENTS, nullptr);
 
        const auto terms = ctx().data.terms_view();
        const auto cnt = (uint32_t)terms.size();
        GAIA_FOR(cnt) {
          const auto& term = terms[i];
          if (term.src != EntityBad || term.entTrav != EntityBad || term_has_variables(term))
            continue;
          if (term.matchKind != QueryMatchKind::Semantic)
            continue;
          const auto queryId = term.id;
          if (queryId == EntityBad || is_wildcard(queryId) || is_variable((EntityId)queryId.id()))
            continue;
          if (world_component_uses_sparse_storage(*world(), queryId))
            continue;
          if (!world_term_uses_inherit_policy(*world(), queryId))
            continue;
          if (pArchetype->has(queryId))
            continue;
 
          const auto owner = world_query_first_inherited_owner(*world(), *pArchetype, queryId);
          GAIA_ASSERT(owner != EntityBad);
          inheritedData.data[term.fieldIndex] = world_query_inherited_arg_data_const_ptr(*world(), owner, queryId);
        }
 
        m_state.exec.archetypeInheritedData.push_back(inheritedData);
      }
 
      void update_result_cache_prefab_filter_state(const Archetype* pArchetype) {
        if (!matches_prefab_entities() && pArchetype->has(Prefab))
          m_state.resultCacheMayNeedPrefabFilter = 1;
      }
 
      void add_archetype_to_cache_no_grouping(
          const Archetype* pArchetype, bool trackMembershipChange, bool assumeAbsent = false) {
        GAIA_PROF_SCOPE(queryinfo::add_cache_ng);
 
        if (!assumeAbsent && m_state.archetypeSet.contains(pArchetype))
          return;
        GAIA_ASSERT(assumeAbsent || !m_state.archetypeSet.contains(pArchetype));
 
        m_state.archetypeSet.emplace(pArchetype);
        m_state.archetypeCache.push_back(pArchetype);
        update_result_cache_prefab_filter_state(pArchetype);
        m_state.exec.compIndicesPending = true;
        m_state.exec.inheritedDataPending = true;
        m_state.nonTrivial.barrierRelVersion = UINT32_MAX;
        m_state.nonTrivial.barrierEnabledVersion = UINT32_MAX;
        if (trackMembershipChange)
          mark_result_cache_membership_changed();
      }
 
      void add_archetype_to_seed_cache(const Archetype* pArchetype, bool assumeAbsent = false) {
        if (!assumeAbsent && m_state.seedArchetypeSet.contains(pArchetype))
          return;
        GAIA_ASSERT(assumeAbsent || !m_state.seedArchetypeSet.contains(pArchetype));
 
        m_state.seedArchetypeSet.emplace(pArchetype);
        m_state.seedArchetypeCache.push_back(pArchetype);
      }
 
      void add_new_archetype_to_immediate_caches(const Archetype* pArchetype, bool trackMembershipChange) {
        GAIA_ASSERT(m_plan.ctx.data.groupBy == EntityBad);
        GAIA_ASSERT(m_plan.ctx.data.sortByFunc == nullptr);
        GAIA_ASSERT(!m_state.seedArchetypeSet.contains(pArchetype));
        GAIA_ASSERT(!m_state.archetypeSet.contains(pArchetype));
 
        m_state.seedArchetypeSet.emplace(pArchetype);
        m_state.seedArchetypeCache.push_back(pArchetype);
 
        m_state.archetypeSet.emplace(pArchetype);
        m_state.archetypeCache.push_back(pArchetype);
        update_result_cache_prefab_filter_state(pArchetype);
        m_state.exec.compIndicesPending = true;
        m_state.exec.inheritedDataPending = true;
        if (trackMembershipChange)
          mark_result_cache_membership_changed();
      }
 
      void add_archetype_to_cache_w_grouping(
          const Archetype* pArchetype, bool trackMembershipChange, bool assumeAbsent = false) {
        GAIA_PROF_SCOPE(queryinfo::add_cache_wg);
 
        if (!assumeAbsent && m_state.archetypeSet.contains(pArchetype))
          return;
        GAIA_ASSERT(assumeAbsent || !m_state.archetypeSet.contains(pArchetype));
 
        m_state.grouped.selectedGroupDataValid = false;
 
        const GroupId groupId = m_plan.ctx.data.groupByFunc(*m_plan.ctx.w, *pArchetype, m_plan.ctx.data.groupBy);
 
        m_state.archetypeSet.emplace(pArchetype);
        m_state.archetypeCache.push_back(pArchetype);
        update_result_cache_prefab_filter_state(pArchetype);
        m_state.grouped.archetypeGroupIds.push_back(groupId);
        m_state.grouped.dataPending = true;
        m_state.exec.compIndicesPending = true;
        m_state.exec.inheritedDataPending = true;
        m_state.nonTrivial.barrierRelVersion = UINT32_MAX;
        m_state.nonTrivial.barrierEnabledVersion = UINT32_MAX;
        m_plan.ctx.data.flags |= QueryCtx::QueryFlags::SortGroups;
        if (trackMembershipChange)
          mark_result_cache_membership_changed();
      }
 
      void add_archetype_to_cache(const Archetype* pArchetype, bool trackMembershipChange, bool assumeAbsent) {
        if (m_plan.ctx.data.sortByFunc != nullptr)
          m_plan.ctx.data.flags |= QueryCtx::QueryFlags::SortEntities;
 
        if (m_plan.ctx.data.groupBy != EntityBad)
          add_archetype_to_cache_w_grouping(pArchetype, trackMembershipChange, assumeAbsent);
        else
          add_archetype_to_cache_no_grouping(pArchetype, trackMembershipChange, assumeAbsent);
      }
 
      void add_archetype_to_transient_cache(const Archetype* pArchetype) {
        m_state.archetypeCache.push_back(pArchetype);
        update_result_cache_prefab_filter_state(pArchetype);
        m_state.exec.compIndicesPending = true;
        m_state.exec.inheritedDataPending = true;
        if (m_plan.ctx.data.groupBy != EntityBad) {
          const auto groupId = m_plan.ctx.data.groupByFunc(*m_plan.ctx.w, *pArchetype, m_plan.ctx.data.groupBy);
          m_state.grouped.archetypeGroupIds.push_back(groupId);
          m_state.grouped.dataPending = true;
        }
      }
 
      GAIA_NODISCARD const GroupData* selected_group_data(GroupId runtimeGroupId) const {
        const_cast<QueryInfo*>(this)->ensure_group_data(true);
        if (m_plan.ctx.data.groupBy == EntityBad || runtimeGroupId == 0)
          return nullptr;
 
        if (!m_state.grouped.selectedGroupDataValid || m_state.grouped.selectedGroupData.groupId != runtimeGroupId) {
          uint32_t left = 0;
          uint32_t right = (uint32_t)m_state.grouped.archetypeGroupData.size();
          while (left < right) {
            const uint32_t mid = left + ((right - left) >> 1);
            const auto midGroupId = m_state.grouped.archetypeGroupData[mid].groupId;
            if (midGroupId < runtimeGroupId)
              left = mid + 1;
            else
              right = mid;
          }
 
          if (left < m_state.grouped.archetypeGroupData.size() &&
              m_state.grouped.archetypeGroupData[left].groupId == runtimeGroupId) {
            m_state.grouped.selectedGroupData = m_state.grouped.archetypeGroupData[left];
            m_state.grouped.selectedGroupDataValid = true;
            return &m_state.grouped.selectedGroupData;
          }
 
          m_state.grouped.selectedGroupData = {};
          m_state.grouped.selectedGroupDataValid = false;
          return nullptr;
        }
 
        return &m_state.grouped.selectedGroupData;
      }
 
      GAIA_NODISCARD bool has_same_result_membership_as_seed_cache() const {
        if (m_state.archetypeSet.size() != m_state.seedArchetypeSet.size())
          return false;
 
        for (const auto* pArchetype: m_state.seedArchetypeCache) {
          if (!m_state.archetypeSet.contains(pArchetype))
            return false;
        }
 
        return true;
      }
 
      GAIA_NODISCARD bool has_same_result_membership_as(const CArchetypeDArray& archetypeCache) const {
        if (m_state.archetypeSet.size() != archetypeCache.size())
          return false;
 
        for (const auto* pArchetype: archetypeCache) {
          if (!m_state.archetypeSet.contains(pArchetype))
            return false;
        }
 
        return true;
      }
 
      void sync_result_cache_from_seed_cache() {
        const bool membershipChanged = !has_same_result_membership_as_seed_cache();
        clear_result_cache();
        const auto cnt = (uint32_t)m_state.seedArchetypeCache.size();
        GAIA_FOR(cnt) {
          add_archetype_to_cache(m_state.seedArchetypeCache[i], false, false);
        }
        if (membershipChanged)
          mark_result_cache_membership_changed();
      }
 
      bool del_archetype_from_cache(const Archetype* pArchetype) {
        const auto it = m_state.archetypeSet.find(pArchetype);
        if (it == m_state.archetypeSet.end())
          return false;
 
        m_state.archetypeSet.erase(it);
 
        const auto archetypeIdx = core::get_index(m_state.archetypeCache, pArchetype);
        GAIA_ASSERT(archetypeIdx != BadIndex);
        if (archetypeIdx == BadIndex)
          return true;
 
        if (m_plan.ctx.data.sortByFunc != nullptr)
          m_plan.ctx.data.flags |= QueryCtx::QueryFlags::SortEntities;
 
        core::swap_erase(m_state.archetypeCache, archetypeIdx);
        if (archetypeIdx < m_state.exec.archetypeCompIndices.size())
          core::swap_erase(m_state.exec.archetypeCompIndices, archetypeIdx);
        if (archetypeIdx < m_state.exec.archetypeInheritedData.size())
          core::swap_erase(m_state.exec.archetypeInheritedData, archetypeIdx);
        if (archetypeIdx < m_state.grouped.archetypeGroupIds.size())
          core::swap_erase(m_state.grouped.archetypeGroupIds, archetypeIdx);
        if (archetypeIdx < m_state.nonTrivial.archetypeBarrierPasses.size())
          core::swap_erase(m_state.nonTrivial.archetypeBarrierPasses, archetypeIdx);
 
        if (m_plan.ctx.data.groupBy != EntityBad) {
          m_state.grouped.selectedGroupDataValid = false;
          m_state.grouped.archetypeGroupData.clear();
          m_state.grouped.dataPending = true;
          m_plan.ctx.data.flags |= QueryCtx::QueryFlags::SortGroups;
        }
        m_state.nonTrivial.barrierRelVersion = UINT32_MAX;
        m_state.nonTrivial.barrierEnabledVersion = UINT32_MAX;
 
        mark_result_cache_membership_changed();
        return true;
      }
 
      bool del_archetype_from_seed_cache(const Archetype* pArchetype) {
        const auto it = m_state.seedArchetypeSet.find(pArchetype);
        if (it == m_state.seedArchetypeSet.end())
          return false;
 
        m_state.seedArchetypeSet.erase(it);
 
        const auto archetypeIdx = core::get_index(m_state.seedArchetypeCache, pArchetype);
        GAIA_ASSERT(archetypeIdx != BadIndex);
        if (archetypeIdx == BadIndex)
          return true;
 
        core::swap_erase(m_state.seedArchetypeCache, archetypeIdx);
        return true;
      }
 
      GAIA_NODISCARD World* world() {
        GAIA_ASSERT(m_plan.ctx.w != nullptr);
        return const_cast<World*>(m_plan.ctx.w);
      }
      GAIA_NODISCARD const World* world() const {
        GAIA_ASSERT(m_plan.ctx.w != nullptr);
        return m_plan.ctx.w;
      }
 
      GAIA_NODISCARD QuerySerBuffer& ser_buffer() {
        return m_plan.ctx.q.ser_buffer(world());
      }
      void ser_buffer_reset() {
        m_plan.ctx.q.ser_buffer_reset(world());
      }
 
      GAIA_NODISCARD QueryCtx& ctx() {
        return m_plan.ctx;
      }
      GAIA_NODISCARD const QueryCtx& ctx() const {
        return m_plan.ctx;
      }
 
      GAIA_NODISCARD util::str bytecode() const {
        return m_plan.vm.bytecode(*world());
      }
 
      GAIA_NODISCARD uint32_t op_count() const {
        return m_plan.vm.op_count();
      }
 
      GAIA_NODISCARD uint64_t op_signature() const {
        return m_plan.vm.op_signature();
      }
 
      GAIA_NODISCARD bool has_filters() const {
        const auto& ctxData = m_plan.ctx.data;
        return ctxData.changedCnt > 0;
      }
 
      GAIA_NODISCARD bool has_entity_filter_terms() const {
        const auto& ctxData = m_plan.ctx.data;
        return ctxData.deps.has_dep_flag(QueryCtx::DependencyHasEntityFilterTerms);
      }
 
      GAIA_NODISCARD bool has_potential_inherited_id_terms() const {
        const auto& ctxData = m_plan.ctx.data;
        return ctxData.deps.has_dep_flag(QueryCtx::DependencyHasPotentialInheritedIdTerms);
      }
 
      GAIA_NODISCARD QueryCtx::DirectTargetEvalKind direct_target_eval_kind() const {
        return m_plan.ctx.data.directTargetEvalKind;
      }
 
      GAIA_NODISCARD Entity direct_target_eval_id() const {
        return m_plan.ctx.data.directTargetEvalId;
      }
 
      GAIA_NODISCARD bool can_direct_target_eval() const {
        return m_plan.ctx.data.canDirectTargetEval;
      }
 
      GAIA_NODISCARD bool can_direct_entity_seed_eval_shape() const {
        return m_plan.ctx.data.canDirectEntitySeedEvalShape;
      }
 
      GAIA_NODISCARD bool has_only_direct_or_terms() const {
        return m_plan.ctx.data.hasOnlyDirectOrTerms;
      }
 
      GAIA_NODISCARD bool matches_prefab_entities() const {
        const auto& ctxData = m_plan.ctx.data;
        return (ctxData.flags & QueryCtx::QueryFlags::MatchPrefab) != 0 ||
               (ctxData.flags & QueryCtx::QueryFlags::HasPrefabTerms) != 0;
      }
 
      template <typename... T>
      GAIA_NODISCARD bool has_any() const {
        return (has_inter<T>(QueryOpKind::Any) || ...);
      }
 
      template <typename... T>
      GAIA_NODISCARD bool has_or() const {
        return (has_inter<T>(QueryOpKind::Or) || ...);
      }
 
      template <typename... T>
      GAIA_NODISCARD bool has_all() const {
        return (has_inter<T>(QueryOpKind::All) && ...);
      }
 
      template <typename... T>
      GAIA_NODISCARD bool has_no() const {
        return (!has_inter<T>(QueryOpKind::Not) && ...);
      }
 
      void remove(Archetype* pArchetype) {
        GAIA_PROF_SCOPE(queryinfo::remove);
 
        (void)del_archetype_from_seed_cache(pArchetype);
        (void)del_archetype_from_cache(pArchetype);
      }
 
      std::span<const uint8_t> indices_mapping_view(uint32_t archetypeIdx) const {
        const_cast<QueryInfo*>(this)->ensure_comp_indices();
        const auto& ctxData = m_state.exec.archetypeCompIndices[archetypeIdx];
        return {ctxData.indices, ChunkHeader::MAX_COMPONENTS};
      }
 
      std::span<const DirectChunkEntry>
      direct_chunk_view(uint32_t idxFrom, uint32_t idxTo, const uint32_t* pDataFields, uint32_t dataFieldCount) const {
        const_cast<QueryInfo*>(this)->ensure_direct_chunks(idxFrom, idxTo, pDataFields, dataFieldCount);
        return {m_state.exec.directChunks.data(), m_state.exec.directChunks.size()};
      }
 
      std::span<const void* const> direct_chunk_data_view() const {
        return {m_state.exec.directChunkData.data(), m_state.exec.directChunkData.size()};
      }
 
      InheritedTermDataView inherited_data_view(uint32_t archetypeIdx) const {
        const_cast<QueryInfo*>(this)->ensure_inherited_data();
        if (archetypeIdx >= m_state.exec.archetypeInheritedData.size())
          return {};
        const auto& ctxData = m_state.exec.archetypeInheritedData[archetypeIdx];
        return {ctxData.data, ChunkHeader::MAX_COMPONENTS};
      }
 
      InheritedTermDataView inherited_data_view(const Archetype* pArchetype) const {
        if (!has_inherited_data_payload())
          return {};
        const auto archetypeIdx = core::get_index(m_state.archetypeCache, pArchetype);
        if (archetypeIdx == BadIndex)
          return {};
        return inherited_data_view((uint32_t)archetypeIdx);
      }
 
      void ensure_depth_order_hierarchy_barrier_cache() {
        ensure_depth_order_hierarchy_barrier_cache_inter();
      }
 
      std::span<const uint8_t> try_indices_mapping_view(const Archetype* pArchetype) const {
        if (m_state.exec.compIndicesPending)
          return {};
        const auto archetypeIdx = core::get_index(m_state.archetypeCache, pArchetype);
        if (archetypeIdx == BadIndex)
          return {};
        return indices_mapping_view(archetypeIdx);
      }
 
      InheritedTermDataView try_inherited_data_view(const Archetype* pArchetype) const {
        if (!has_inherited_data_payload() || m_state.exec.inheritedDataPending)
          return {};
        const auto archetypeIdx = core::get_index(m_state.archetypeCache, pArchetype);
        if (archetypeIdx == BadIndex)
          return {};
        return inherited_data_view((uint32_t)archetypeIdx);
      }
 
      GAIA_NODISCARD GroupId group_id(uint32_t archetypeIdx) const {
        const_cast<QueryInfo*>(this)->ensure_group_data(true);
        GAIA_ASSERT(archetypeIdx < m_state.grouped.archetypeGroupIds.size());
        return m_state.grouped.archetypeGroupIds[archetypeIdx];
      }
 
      GAIA_NODISCARD bool barrier_passes(uint32_t archetypeIdx) const {
        const_cast<QueryInfo*>(this)->ensure_depth_order_hierarchy_barrier_cache();
        if (m_state.nonTrivial.archetypeBarrierPasses.empty())
          return true;
        GAIA_ASSERT(archetypeIdx < m_state.nonTrivial.archetypeBarrierPasses.size());
        return m_state.nonTrivial.archetypeBarrierPasses[archetypeIdx] != 0;
      }
 
      GAIA_NODISCARD bool barrier_may_prune() const {
        const_cast<QueryInfo*>(this)->ensure_depth_order_hierarchy_barrier_cache();
        return m_state.nonTrivial.barrierMayPrune != 0;
      }
 
      GAIA_NODISCARD CArchetypeDArray::iterator begin() {
        return m_state.archetypeCache.begin();
      }
 
      GAIA_NODISCARD CArchetypeDArray::const_iterator begin() const {
        return m_state.archetypeCache.begin();
      }
 
      GAIA_NODISCARD CArchetypeDArray::const_iterator cbegin() const {
        return m_state.archetypeCache.begin();
      }
 
      GAIA_NODISCARD CArchetypeDArray::iterator end() {
        return m_state.archetypeCache.end();
      }
 
      GAIA_NODISCARD CArchetypeDArray::const_iterator end() const {
        return m_state.archetypeCache.end();
      }
 
      GAIA_NODISCARD CArchetypeDArray::const_iterator cend() const {
        return m_state.archetypeCache.end();
      }
 
      GAIA_NODISCARD std::span<const Archetype*> cache_archetype_view() const {
        return std::span{(const Archetype**)m_state.archetypeCache.data(), m_state.archetypeCache.size()};
      }
 
      GAIA_NODISCARD std::span<const SortData> cache_sort_view() const {
        return std::span{m_state.nonTrivial.archetypeSortData.data(), m_state.nonTrivial.archetypeSortData.size()};
      }
 
      GAIA_NODISCARD std::span<const GroupData> group_data_view() const {
        const_cast<QueryInfo*>(this)->ensure_group_data(true);
        return std::span{m_state.grouped.archetypeGroupData.data(), m_state.grouped.archetypeGroupData.size()};
      }
 
#if GAIA_ECS_TEST_HOOKS
      using TestDynamicCacheKind = QueryCtx::DynamicCacheKind;
 
      GAIA_NODISCARD TestDynamicCacheKind test_dynamic_cache_kind() const {
        return m_plan.ctx.data.dynamicCacheKind;
      }
 
      GAIA_NODISCARD bool test_uses_direct_src_version_tracking() const {
        return m_plan.ctx.data.uses_direct_src_version_tracking();
      }
 
      GAIA_NODISCARD bool test_uses_src_trav_snapshot() const {
        return m_plan.ctx.data.uses_src_trav_snapshot();
      }
 
      GAIA_NODISCARD uint32_t test_match_pass_count() const {
        return m_testMatchPassCount;
      }
 
      GAIA_NODISCARD static constexpr uint32_t test_query_info_size() {
        return (uint32_t)sizeof(QueryInfo);
      }
 
      GAIA_NODISCARD static constexpr uint32_t test_query_plan_size() {
        return (uint32_t)sizeof(QueryPlan);
      }
 
      GAIA_NODISCARD static constexpr uint32_t test_query_state_size() {
        return (uint32_t)sizeof(QueryState);
      }
 
      GAIA_NODISCARD static constexpr uint32_t test_dynamic_cache_state_size() {
        return (uint32_t)sizeof(QueryState::DynamicCacheState);
      }
 
      GAIA_NODISCARD static constexpr uint32_t test_query_state_dynamic_offset() {
        return (uint32_t)offsetof(QueryState, dynamic);
      }
 
      void test_add_transient_archetype(const Archetype* pArchetype) {
        GAIA_ASSERT(pArchetype != nullptr);
        if (pArchetype == nullptr)
          return;
 
        m_state.clear_transient_result_cache();
        add_archetype_to_transient_cache(pArchetype);
      }
#endif
    };
  } // namespace ecs
} // namespace gaia