Gaia-ECS v1.0.0
A simple and powerful entity component system
Loading...
Searching...
No Matches
query_typed.inl
1#pragma once
2
3#include "gaia/ecs/query_adapter_typed.inl"
4
5namespace gaia {
6 namespace ecs {
7 namespace detail {
9 inline TypedQueryExecState build_typed_query_exec_state(
10 World& world, const QueryInfo& queryInfo, const TypedQueryArgMeta* pMetas, uint32_t argCount) {
11 TypedQueryExecState state{};
12 QueryImpl::DirectChunkArgEvalDesc directChunkDescs[MAX_ITEMS_IN_QUERY]{};
13 bool hasSparseArgs = false;
14 state.argCount = argCount;
15 GAIA_FOR(argCount) {
16 state.argIds[i] = pMetas[i].termId;
17 state.writeFlags[i] = pMetas[i].isWrite;
18 if (pMetas[i].isWrite) {
19 state.hasWriteArgs = true;
20 if (state.firstWriteArg == MAX_ITEMS_IN_QUERY)
21 state.firstWriteArg = (uint32_t)i;
22 }
23 state.needsInheritedArgIds = state.needsInheritedArgIds || !pMetas[i].isEntity;
24 hasSparseArgs = hasSparseArgs || pMetas[i].usesSparseStorage;
25 directChunkDescs[i] = {pMetas[i].termId, pMetas[i].isEntity, pMetas[i].isPair, pMetas[i].usesSparseStorage};
26 }
27
28 state.canUseDirectChunkEval =
29 QueryImpl::can_use_direct_chunk_term_eval_descs(world, queryInfo, directChunkDescs, argCount);
30 state.canUseSparseChunkEval = hasSparseArgs && QueryImpl::can_use_sparse_chunk_term_eval_descs(
31 world, queryInfo, directChunkDescs, argCount);
32 if (state.needsInheritedArgIds)
33 state.hasInheritedTerms = queryInfo.has_potential_inherited_id_terms();
34 return state;
35 }
36
37#if GAIA_ECS_TEST_HOOKS
38 template <typename Func>
39 inline QueryImpl::QueryPlan QueryImpl::test_typed_plan(Func func) {
40 auto& queryInfo = fetch();
41 match_all(queryInfo);
42
43 using InputArgs = decltype(core::func_args(&Func::operator()));
44 #if GAIA_ASSERT_ENABLED
45 GAIA_ASSERT(typed_query_args_match_query(queryInfo, InputArgs{}));
46 #endif
47 auto& world = *const_cast<World*>(queryInfo.world());
48 TypedQueryArgMeta metas[MAX_ITEMS_IN_QUERY]{};
49 const auto argCount = init_typed_query_arg_metas(metas, world, InputArgs{});
50 const auto state = build_typed_query_exec_state(world, queryInfo, metas, argCount);
51 (void)func;
52 return prepare_query_plan(queryInfo, state);
53 }
54
55 inline QueryImpl::QueryPlan QueryImpl::test_iter_plan(Constraints constraints) {
56 auto& queryInfo = fetch();
57 match_all(queryInfo);
58 return prepare_query_plan(queryInfo, constraints);
59 }
60#endif
61
62 inline void finish_typed_chunk_state(
63 World& world, Chunk* pChunk, uint16_t from, uint16_t to, const TypedQueryExecState& state);
64
65 inline void finish_typed_iter_state(QueryImpl& query, Iter& it, const TypedQueryExecState& state);
66
67 inline void noop_row_done(uint32_t) {}
68
69 template <typename Func, typename ViewsTuple, typename... T>
70 inline void invoke_typed_query_row_erased(
71 void* pFunc, Iter& it, ViewsTuple& dataPointerTuple, uint32_t row, bool directLocalIndex);
72
73 template <typename OnRowDone, typename... T>
74 inline void run_typed_chunk_views(
75 const QueryInfo* pQueryInfo, Iter& it, void* pFunc, bool directViews, OnRowDone&& onRowDone,
76 void (*invokeDirectRow)(
77 void*, Iter&, std::tuple<decltype(std::declval<Iter&>().template sview_auto<T>())...>&, uint32_t, bool),
78 void (*invokeMappedRow)(
79 void*, Iter&, std::tuple<decltype(std::declval<Iter&>().template view_auto_any<T>())...>&, uint32_t,
80 bool),
81 core::func_type_list<T...>);
82
83 template <typename Func, typename... T>
84 inline void run_typed_chunk_direct_iter_fast_cb(QueryImpl&, Iter& it, void* pFunc, const TypedQueryExecState&);
85
86 template <typename Func, typename... T>
87 inline void
88 run_typed_chunk_direct_iter_cb(QueryImpl& query, Iter& it, void* pFunc, const TypedQueryExecState& state);
89
90 template <typename Func, typename... T>
91 inline void run_typed_chunk_mapped_iter_cb(
92 QueryImpl& query, const QueryInfo& queryInfo, Iter& it, void* pFunc, const TypedQueryExecState& state);
93
94 template <typename Func, typename... T>
95 inline void run_typed_sparse_chunk_rows(
96 Chunk* pChunk, uint16_t from, uint16_t to, Func& func, const TypedQueryExecState& state,
97 core::func_type_list<T...> types);
98
99 template <typename Func, typename... T>
100 GAIA_NODISCARD inline auto typed_run_direct_fast_chunk_ptr(core::func_type_list<T...>) {
101 return &run_typed_chunk_direct_iter_fast_cb<Func, T...>;
102 }
103
104 template <typename Func, typename... T>
105 GAIA_NODISCARD inline auto typed_run_direct_chunk_ptr(core::func_type_list<T...>) {
106 return &run_typed_chunk_direct_iter_cb<Func, T...>;
107 }
108
109 template <typename Func, typename... T>
110 GAIA_NODISCARD inline auto typed_run_mapped_chunk_ptr(core::func_type_list<T...>) {
111 return &run_typed_chunk_mapped_iter_cb<Func, T...>;
112 }
113
114 template <typename Func, typename... T>
115 inline void run_typed_sparse_plan_cb(
116 QueryImpl& query, QueryInfo& queryInfo, const QueryImpl::QueryPlan& plan, void* pFunc,
117 const TypedQueryExecState& state) {
118 auto& func = *static_cast<Func*>(pFunc);
119 if (plan.mode == QueryImpl::QueryPlanMode::EntitySeed) {
120 query.run_query_on_sparse_entities_typed(queryInfo, state, func, core::func_type_list<T...>{});
121 return;
122 }
123
124 GAIA_ASSERT(plan.mode == QueryImpl::QueryPlanMode::SparseDense);
125 if ((plan.flags & QueryImpl::QueryPlanFlag_Filtered) != 0)
126 query.run_query_on_chunks_sparse_typed<true>(queryInfo, plan, state, func, core::func_type_list<T...>{});
127 else
128 query.run_query_on_chunks_sparse_typed<false>(queryInfo, plan, state, func, core::func_type_list<T...>{});
129 }
130
131 template <typename Func, typename... T>
132 GAIA_NODISCARD inline auto typed_run_sparse_plan_ptr(core::func_type_list<T...>) {
133 using RunSparsePlan =
134 void (*)(QueryImpl&, QueryInfo&, const QueryImpl::QueryPlan&, void*, const TypedQueryExecState&);
135 if constexpr (typed_query_args_use_sparse_storage_v<T...>)
136 return RunSparsePlan(&run_typed_sparse_plan_cb<Func, T...>);
137 else
138 return RunSparsePlan(nullptr);
139 }
140
141 template <typename Func, typename... T>
142 inline void run_typed_chunk_direct_iter_fast_cb(QueryImpl&, Iter& it, void* pFunc, const TypedQueryExecState&) {
143 run_typed_chunk_views(
144 nullptr, it, pFunc, true, noop_row_done,
145 &invoke_typed_query_row_erased<
146 Func, std::tuple<decltype(std::declval<Iter&>().template sview_auto<T>())...>, T...>,
147 &invoke_typed_query_row_erased<
148 Func, std::tuple<decltype(std::declval<Iter&>().template view_auto_any<T>())...>, T...>,
149 core::func_type_list<T...>{});
150 }
151
152 template <typename Func, typename... T>
153 inline void
154 run_typed_chunk_direct_iter_cb(QueryImpl& query, Iter& it, void* pFunc, const TypedQueryExecState& state) {
155 auto& func = *static_cast<Func*>(pFunc);
156 run_typed_chunk_direct_finish(query, it, func, state, core::func_type_list<T...>{});
157 }
158
159 template <typename Func, typename... T>
160 inline void run_typed_chunk_mapped_iter_cb(
161 QueryImpl& query, const QueryInfo& queryInfo, Iter& it, void* pFunc, const TypedQueryExecState& state) {
162 auto& func = *static_cast<Func*>(pFunc);
163 if constexpr (typed_query_args_use_sparse_storage_v<T...>) {
164 if (state.canUseSparseChunkEval) {
165 auto boundState = state;
166 auto& world = *const_cast<World*>(queryInfo.world());
167 bind_typed_sparse_stores(boundState, world, core::func_type_list<T...>{});
168 auto* pChunk = const_cast<Chunk*>(it.chunk());
169 run_typed_sparse_chunk_rows(
170 pChunk, it.row_begin(), it.row_end(), func, boundState, core::func_type_list<T...>{});
171 if (boundState.hasWriteArgs)
172 finish_typed_chunk_state(world, pChunk, it.row_begin(), it.row_end(), boundState);
173 return;
174 }
175 }
176 run_typed_chunk_mapped_finish(query, queryInfo, it, func, state, core::func_type_list<T...>{});
177 }
178
179 template <typename Func, typename... T>
180 inline void each_iter_dispatch(
181 QueryImpl& query, const QueryInfo& queryInfo, Iter& it, Func& func, const TypedQueryExecState& state,
182 core::func_type_list<T...>) {
183 if (state.canUseDirectChunkEval) {
184 run_typed_chunk_views(
185 nullptr, it, &func, true, noop_row_done,
186 &invoke_typed_query_row_erased<
187 Func, std::tuple<decltype(std::declval<Iter&>().template sview_auto<T>())...>, T...>,
188 &invoke_typed_query_row_erased<
189 Func, std::tuple<decltype(std::declval<Iter&>().template view_auto_any<T>())...>, T...>,
190 core::func_type_list<T...>{});
191 finish_typed_chunk_state(*it.world(), const_cast<Chunk*>(it.chunk()), it.row_begin(), it.row_end(), state);
192 } else
193 run_typed_chunk_unmapped(query, queryInfo, it, func, state, core::func_type_list<T...>{});
194 }
195
196 template <typename Func, typename... T>
197 inline void run_typed_chunk_direct_finish(
198 QueryImpl& query, Iter& it, Func& func, const TypedQueryExecState& state, core::func_type_list<T...>) {
199 run_typed_chunk_views(
200 nullptr, it, &func, true, noop_row_done,
201 &invoke_typed_query_row_erased<
202 Func, std::tuple<decltype(std::declval<Iter&>().template sview_auto<T>())...>, T...>,
203 &invoke_typed_query_row_erased<
204 Func, std::tuple<decltype(std::declval<Iter&>().template view_auto_any<T>())...>, T...>,
205 core::func_type_list<T...>{});
206 finish_typed_iter_state(query, it, state);
207 }
208
209 template <typename Func, typename... T>
210 inline void run_typed_chunk_mapped_finish(
211 QueryImpl& query, const QueryInfo& queryInfo, Iter& it, Func& func, const TypedQueryExecState& state,
212 core::func_type_list<T...>) {
213 run_typed_chunk_views(
214 &queryInfo, it, &func, false, noop_row_done,
215 &invoke_typed_query_row_erased<
216 Func, std::tuple<decltype(std::declval<Iter&>().template sview_auto<T>())...>, T...>,
217 &invoke_typed_query_row_erased<
218 Func, std::tuple<decltype(std::declval<Iter&>().template view_auto_any<T>())...>, T...>,
219 core::func_type_list<T...>{});
220 finish_typed_iter_state(query, it, state);
221 }
222
223 template <typename Func, typename... T>
224 inline void run_typed_chunk_direct_walk_cb(
225 QueryImpl& query, const QueryInfo&, Iter& it, void* pFunc, const TypedQueryExecState& state) {
226 run_typed_chunk_direct_finish(query, it, *static_cast<Func*>(pFunc), state, core::func_type_list<T...>{});
227 }
228
229 template <typename Func, typename... T>
230 inline void run_typed_chunk_mapped_walk_cb(
231 QueryImpl& query, const QueryInfo& queryInfo, Iter& it, void* pFunc, const TypedQueryExecState& state) {
232 run_typed_chunk_mapped_finish(
233 query, queryInfo, it, *static_cast<Func*>(pFunc), state, core::func_type_list<T...>{});
234 }
235
236 template <typename Func, typename... T>
237 inline void each_walk_dispatch_direct(
238 QueryImpl& query, QueryInfo& queryInfo, std::span<const Entity> entities, Constraints constraints, Func& func,
239 const TypedQueryExecState& state, core::func_type_list<T...>) {
240 query.each_walk_inter(
241 queryInfo, entities, constraints, &func, state, &run_typed_chunk_direct_walk_cb<Func, T...>);
242 }
243
244 template <typename Func, typename... T>
245 inline void each_walk_dispatch_mapped(
246 QueryImpl& query, QueryInfo& queryInfo, std::span<const Entity> entities, Constraints constraints, Func& func,
247 const TypedQueryExecState& state, core::func_type_list<T...>) {
248 query.each_walk_inter(
249 queryInfo, entities, constraints, &func, state, &run_typed_chunk_mapped_walk_cb<Func, T...>);
250 }
251
252 template <typename Func, typename... T>
253 inline void run_typed_chunk_unmapped(
254 QueryImpl& query, const QueryInfo& queryInfo, Iter& it, Func& func, const TypedQueryExecState& state,
255 core::func_type_list<T...> types) {
256 auto& world = *const_cast<World*>(queryInfo.world());
257 auto* pChunk = const_cast<Chunk*>(it.chunk());
258 const bool hasEntityFilters = queryInfo.has_entity_filter_terms();
259
260 if (!hasEntityFilters) {
261 run_typed_chunk_views(
262 &queryInfo, it, &func, false, noop_row_done,
263 &invoke_typed_query_row_erased<
264 Func, std::tuple<decltype(std::declval<Iter&>().template sview_auto<T>())...>, T...>,
265 &invoke_typed_query_row_erased<
266 Func, std::tuple<decltype(std::declval<Iter&>().template view_auto_any<T>())...>, T...>,
267 types);
268 finish_typed_chunk_state(world, pChunk, it.row_begin(), it.row_end(), state);
269 } else {
270 run_typed_chunk_views(
271 &queryInfo, it, &func, false,
272 [&](uint32_t row) {
273 finish_typed_chunk_state(
274 world, pChunk, (uint16_t)(it.row_begin() + row), (uint16_t)(it.row_begin() + row + 1), state);
275 },
276 &invoke_typed_query_row_erased<
277 Func, std::tuple<decltype(std::declval<Iter&>().template sview_auto<T>())...>, T...>,
278 &invoke_typed_query_row_erased<
279 Func, std::tuple<decltype(std::declval<Iter&>().template view_auto_any<T>())...>, T...>,
280 types);
281 }
282
283 it.clear_touched_writes();
284 }
285
286 template <typename Func, typename... T>
287 inline void run_typed_cached_seed_runs_basic(
288 QueryImpl& query, World& world, Constraints constraints, Func& func, std::span<const BfsChunkRun> runs,
289 const TypedQueryExecState& state) {
290 Iter it;
291 it.init_query_state(&world, constraints, false);
292 const Archetype* pLastArchetype = nullptr;
293 for (const auto& run: runs) {
294 if (run.pArchetype != pLastArchetype) {
295 it.set_archetype(run.pArchetype);
296 pLastArchetype = run.pArchetype;
297 }
298
299 it.set_chunk(run.pChunk, run.from, run.to);
300 it.set_group_id(0);
301 run_typed_chunk_direct_finish(query, it, func, state, core::func_type_list<T...>{});
302 }
303 }
304
305 template <typename Func, typename... T>
306 inline void run_typed_cached_seed_runs_entity_init(
307 QueryImpl& query, QueryInfo& queryInfo, World& world, Constraints constraints, Func& func,
308 std::span<const BfsChunkRun> runs, const TypedQueryExecState& state) {
309 Iter it;
310 it.init_query_state(&world, constraints, false);
311 const Archetype* pLastArchetype = nullptr;
312 uint8_t indices[ChunkHeader::MAX_COMPONENTS];
313 Entity termIds[ChunkHeader::MAX_COMPONENTS];
314 for (const auto& run: runs) {
315 const auto& ec = ::gaia::ecs::fetch(world, run.pChunk->entity_view()[run.from]);
316 query.init_direct_entity_iter(queryInfo, world, ec, it, indices, termIds, pLastArchetype);
317 it.set_chunk(run.pChunk, run.from, run.to);
318 it.set_group_id(0);
319 run_typed_chunk_direct_finish(query, it, func, state, core::func_type_list<T...>{});
320 }
321 }
322
323 template <typename Func, typename... T>
324 inline void run_typed_cached_seed_runs(
325 QueryImpl& query, QueryInfo& queryInfo, World& world, Constraints constraints, Func& func,
326 std::span<const BfsChunkRun> runs, bool canUseBasicInit, const TypedQueryExecState& state) {
327 if (canUseBasicInit)
328 run_typed_cached_seed_runs_basic<Func, T...>(query, world, constraints, func, runs, state);
329 else
330 run_typed_cached_seed_runs_entity_init<Func, T...>(query, queryInfo, world, constraints, func, runs, state);
331 }
332
333 inline void QueryImpl::each_walk_inter(
334 QueryInfo& queryInfo, std::span<const Entity> entities, Constraints constraints, void* pFunc,
335 const TypedQueryExecState& state,
336 void (*runChunk)(QueryImpl&, const QueryInfo&, Iter&, void*, const TypedQueryExecState&)) {
337 auto& world = *queryInfo.world();
338 auto& walkData = ensure_each_walk_data();
339 Iter it;
340 it.init_query_state(&world, constraints, false);
341 if (!walkData.cachedRuns.empty()) {
342 const auto& runs = walkData.cachedRuns;
343 const Archetype* pLastArchetype = nullptr;
344 for (const auto& run: runs) {
345 if (run.pArchetype != pLastArchetype) {
346 it.set_archetype(run.pArchetype);
347 pLastArchetype = run.pArchetype;
348 }
349
350 it.set_chunk(run.pChunk, run.from, run.to);
351 it.set_group_id(0);
352 runChunk(*this, queryInfo, it, pFunc, state);
353 }
354 return;
355 }
356
357 const Archetype* pLastArchetype = nullptr;
358 uint8_t indices[ChunkHeader::MAX_COMPONENTS];
359 Entity termIds[ChunkHeader::MAX_COMPONENTS];
360 for (const auto entity: entities) {
361 const auto& ec = ::gaia::ecs::fetch(world, entity);
362 init_direct_entity_iter(queryInfo, world, ec, it, indices, termIds, pLastArchetype);
363 runChunk(*this, queryInfo, it, pFunc, state);
364 }
365 }
366
367 inline void finish_typed_iter_state(QueryImpl& query, Iter& it, const TypedQueryExecState& state) {
368 query.finish_typed_iter_writes_runtime(it, state.argIds, state.writeFlags, state.argCount, state.firstWriteArg);
369 it.clear_touched_writes();
370 }
371
372 inline void finish_typed_chunk_state(
373 World& world, Chunk* pChunk, uint16_t from, uint16_t to, const TypedQueryExecState& state) {
374 QueryImpl::finish_typed_chunk_writes_runtime(
375 world, pChunk, from, to, state.argIds, state.writeFlags, state.argCount, state.firstWriteArg,
376 state.sparseStores);
377 }
378
379 template <typename InvokeRow, typename OnRowDone>
380 inline void run_typed_query_rows_runtime(
381 const QueryInfo* pQueryInfo, Iter& it, InvokeRow&& invokeRow, OnRowDone&& onRowDone) {
382 const auto cnt = it.size();
383 const bool hasEntityFilters = pQueryInfo != nullptr && pQueryInfo->has_entity_filter_terms();
384
385 if (!hasEntityFilters) {
386 GAIA_FOR(cnt) {
387 invokeRow((uint32_t)i);
388 onRowDone((uint32_t)i);
389 }
390 return;
391 }
392
393 const auto entities = it.template view<Entity>();
394 GAIA_FOR(cnt) {
395 if (!QueryImpl::match_entity_filters(*pQueryInfo->world(), entities[i], *pQueryInfo))
396 continue;
397 invokeRow((uint32_t)i);
398 onRowDone((uint32_t)i);
399 }
400 }
401
402 template <typename Func, typename ViewsTuple, typename... T>
403 inline void invoke_typed_query_row(
404 Iter& it, Func& func, ViewsTuple& dataPointerTuple, uint32_t row, bool directLocalIndex,
405 core::func_type_list<T...>) {
406 if constexpr (sizeof...(T) > 0) {
407 if (directLocalIndex) {
408 std::apply(
409 [&](auto&... views) {
410 func(views[row]...);
411 },
412 dataPointerTuple);
413 } else {
414 std::apply(
415 [&](auto&... views) {
416 func(views[it.template acc_index<T>(row)]...);
417 },
418 dataPointerTuple);
419 }
420 } else
421 func();
422 }
423
424 template <typename Func, typename ViewsTuple, typename... T>
425 inline void invoke_typed_query_row_erased(
426 void* pFunc, Iter& it, ViewsTuple& dataPointerTuple, uint32_t row, bool directLocalIndex) {
427 auto& func = *static_cast<Func*>(pFunc);
428 invoke_typed_query_row(it, func, dataPointerTuple, row, directLocalIndex, core::func_type_list<T...>{});
429 }
430
431 template <typename OnRowDone, typename ViewsTuple>
432 inline void run_typed_tuple_rows(
433 const QueryInfo* pQueryInfo, Iter& it, void* pFunc, ViewsTuple& dataPointerTuple, bool directLocalIndex,
434 void (*invokeRow)(void*, Iter&, ViewsTuple&, uint32_t, bool), OnRowDone&& onRowDone) {
435 run_typed_query_rows_runtime(
436 pQueryInfo, it,
437 [&](uint32_t row) {
438 invokeRow(pFunc, it, dataPointerTuple, row, directLocalIndex);
439 },
440 GAIA_FWD(onRowDone));
441 }
442
443 template <typename OnRowDone, typename... T>
444 inline void run_typed_chunk_views(
445 const QueryInfo* pQueryInfo, Iter& it, void* pFunc, bool directViews, OnRowDone&& onRowDone,
446 void (*invokeDirectRow)(
447 void*, Iter&, std::tuple<decltype(std::declval<Iter&>().template sview_auto<T>())...>&, uint32_t, bool),
448 void (*invokeMappedRow)(
449 void*, Iter&, std::tuple<decltype(std::declval<Iter&>().template view_auto_any<T>())...>&, uint32_t,
450 bool),
451 core::func_type_list<T...>) {
452 if constexpr (sizeof...(T) > 0) {
453 if (directViews) {
454 auto dataPointerTuple = std::make_tuple(it.template sview_auto<T>()...);
455 run_typed_tuple_rows(pQueryInfo, it, pFunc, dataPointerTuple, true, invokeDirectRow, GAIA_FWD(onRowDone));
456 } else {
457 auto dataPointerTuple = std::make_tuple(it.template view_auto_any<T>()...);
458 run_typed_tuple_rows(pQueryInfo, it, pFunc, dataPointerTuple, false, invokeMappedRow, GAIA_FWD(onRowDone));
459 }
460 } else {
461 auto dataPointerTuple = std::tuple<>{};
462 run_typed_tuple_rows(
463 pQueryInfo, it, pFunc, dataPointerTuple, directViews, directViews ? invokeDirectRow : invokeMappedRow,
464 GAIA_FWD(onRowDone));
465 }
466 }
467
468 template <typename ContainerOut>
469 inline void typed_arr_push(World& world, Entity entity, ContainerOut& outArray) {
470 using ContainerItemType = typename ContainerOut::value_type;
471 if constexpr (std::is_same_v<ContainerItemType, Entity>)
472 outArray.push_back(entity);
473 else {
474 auto tmp = world_direct_entity_arg<ContainerItemType>(world, entity);
475 outArray.push_back(tmp);
476 }
477 }
478
479 template <bool UseFilters, typename ContainerOut>
480 inline void run_typed_arr_rows(
481 QueryImpl& query, const QueryInfo& queryInfo, Iter& it, ContainerOut& outArray, uint32_t changedWorldVersion,
482 uint32_t archetypeIdx, const Archetype* pArchetype, Chunk* pChunk, uint16_t from, uint16_t to,
483 bool needsBarrierCache, bool canUseDirectChunkEval) {
484 using ContainerItemType = typename ContainerOut::value_type;
485 const bool barrierPasses = !needsBarrierCache || queryInfo.barrier_passes(archetypeIdx);
486 if GAIA_UNLIKELY (!query.can_process_archetype_inter(queryInfo, *pArchetype, it.constraints(), barrierPasses))
487 return;
488 if GAIA_UNLIKELY (from == to)
489 return;
490
491 GAIA_PROF_SCOPE(query::arr);
492
493 it.set_archetype(pArchetype);
494 it.set_chunk(pChunk, from, to);
495 it.set_group_id(0);
496
497 const auto cnt = it.size();
498 if (cnt == 0)
499 return;
500
501 if constexpr (UseFilters) {
502 if (!QueryImpl::match_filters(*pChunk, queryInfo, changedWorldVersion))
503 return;
504 }
505
506 const bool hasEntityFilters = queryInfo.has_entity_filter_terms();
507 if (canUseDirectChunkEval) {
508 const auto dataView = it.template sview_auto<ContainerItemType>();
509 GAIA_FOR(cnt) {
510 auto tmp = dataView[i];
511 outArray.push_back(tmp);
512 }
513 return;
514 }
515
516 const auto dataView = it.template view<ContainerItemType>();
517 if (!hasEntityFilters) {
518 GAIA_FOR(cnt) {
519 const auto idx = it.template acc_index<ContainerItemType>(i);
520 auto tmp = dataView[idx];
521 outArray.push_back(tmp);
522 }
523 return;
524 }
525
526 const auto entities = it.template view<Entity>();
527 GAIA_FOR(cnt) {
528 if (!QueryImpl::match_entity_filters(*queryInfo.world(), entities[i], queryInfo))
529 continue;
530 const auto idx = it.template acc_index<ContainerItemType>(i);
531 auto tmp = dataView[idx];
532 outArray.push_back(tmp);
533 }
534 }
535
536 inline void QueryImpl::finish_typed_chunk_writes_runtime(
537 World& world, Chunk* pChunk, uint16_t from, uint16_t to, const Entity* pArgIds, const bool* pWriteFlags,
538 uint32_t argCnt, uint32_t firstWriteArg, void* const* pSparseStores) {
539 if (firstWriteArg >= argCnt || from >= to)
540 return;
541
542 Entity seenTerms[ChunkHeader::MAX_COMPONENTS]{};
543 uint32_t seenCnt = 0;
544 const auto entities = pChunk->entity_view();
545 const auto finish_term = [&](Entity term, bool sparseStoreBound) {
546 GAIA_FOR(seenCnt) {
547 if (seenTerms[i] == term)
548 return;
549 }
550
551 seenTerms[seenCnt++] = term;
552 if (sparseStoreBound) {
553 for (uint16_t row = from; row < to; ++row)
554 world_notify_on_set_entity(world, term, entities[row]);
555 return;
556 }
557 if (!world_component_uses_sparse_storage(world, term)) {
558 const auto compIdx = core::get_index(pChunk->ids_view(), term);
559 if (compIdx != BadIndex) {
560 pChunk->finish_write(compIdx, from, to);
561 return;
562 }
563 }
564
565 for (uint16_t row = from; row < to; ++row)
566 world_finish_write(world, term, entities[row]);
567 };
568
569 for (uint32_t i = firstWriteArg; i < argCnt; ++i) {
570 if (!pWriteFlags[i])
571 continue;
572 const auto term = pArgIds[i];
573 if (term != EntityBad)
574 finish_term(term, pSparseStores != nullptr && pSparseStores[i] != nullptr);
575 }
576 }
577
578 template <typename... T>
579 inline void QueryImpl::finish_typed_chunk_writes(World& world, Chunk* pChunk, uint16_t from, uint16_t to) {
580 TypedQueryArgMeta metas[MAX_ITEMS_IN_QUERY]{};
581 const auto argCount = init_typed_query_arg_metas(metas, world, core::func_type_list<T...>{});
582 Entity argIds[MAX_ITEMS_IN_QUERY]{};
583 bool writeFlags[MAX_ITEMS_IN_QUERY]{};
584 uint32_t firstWriteArg = argCount;
585 GAIA_FOR(argCount) {
586 argIds[i] = metas[i].termId;
587 writeFlags[i] = metas[i].isWrite;
588 if (metas[i].isWrite && firstWriteArg == argCount)
589 firstWriteArg = (uint32_t)i;
590 }
591 finish_typed_chunk_writes_runtime(world, pChunk, from, to, argIds, writeFlags, argCount, firstWriteArg);
592 }
593
594 inline void QueryImpl::finish_typed_iter_writes_runtime(
595 Iter& it, const Entity* pArgIds, const bool* pWriteFlags, uint32_t argCnt, uint32_t firstWriteArg) {
596 if (firstWriteArg >= argCnt)
597 return;
598
599 auto* pChunk = const_cast<Chunk*>(it.chunk());
600 if (pChunk == nullptr || it.row_begin() >= it.row_end())
601 return;
602
603 auto& world = *it.world();
604 auto entities = it.entity_rows();
605 Entity seenTerms[ChunkHeader::MAX_COMPONENTS]{};
606 uint32_t seenCnt = 0;
607 const auto finish_term = [&](Entity term, uint32_t fieldIdx) {
608 GAIA_FOR(seenCnt) {
609 if (seenTerms[i] == term)
610 return;
611 }
612
613 seenTerms[seenCnt++] = term;
614 const bool usesSparseStorage = world_component_uses_sparse_storage(world, term);
615 auto compIdx = uint8_t(0xFF);
616 if (it.comp_indices() != nullptr && fieldIdx < ChunkHeader::MAX_COMPONENTS)
617 compIdx = it.comp_indices()[fieldIdx];
618 else if (!usesSparseStorage)
619 compIdx = (uint8_t)pChunk->comp_idx(term);
620
621 if (compIdx != 0xFF && !usesSparseStorage) {
622 pChunk->finish_write(compIdx, it.row_begin(), it.row_end());
623 return;
624 }
625
626 GAIA_FOR(entities.size()) {
627 world_finish_write(world, term, entities[i]);
628 }
629 };
630
631 for (uint32_t i = firstWriteArg; i < argCnt; ++i) {
632 if (!pWriteFlags[i])
633 continue;
634
635 Entity term = pArgIds[i];
636 if (it.term_ids() != nullptr && i < ChunkHeader::MAX_COMPONENTS && it.term_ids()[i] != EntityBad)
637 term = it.term_ids()[i];
638 if (term == EntityBad)
639 continue;
640
641 finish_term(term, i);
642 }
643 }
644
655 template <typename T, typename View>
656 GAIA_NODISCARD inline decltype(auto) typed_direct_chunk_arg_at(View& view, uint32_t row, uint16_t from) {
657 using U = typename actual_type_t<T>::Type;
658 if constexpr (mem::is_soa_layout_v<U>)
659 return view[from + row];
660 else
661 return view[row];
662 }
663
674 template <typename Func, typename ViewsTuple, typename... T, size_t... I>
675 inline void invoke_typed_direct_chunk_row(
676 Func& func, ViewsTuple& views, uint32_t row, uint16_t from, core::func_type_list<T...>,
677 std::index_sequence<I...>) {
678 func(typed_direct_chunk_arg_at<T>(std::get<I>(views), row, from)...);
679 }
680
683 struct TypedDirectChunkRun {
685 Chunk* pChunk = nullptr;
687 const uint8_t* pCompIndices = nullptr;
689 const void* const* pData = nullptr;
691 uint16_t from = 0;
693 uint16_t to = 0;
694 };
695
704 template <typename Func, typename T>
705 inline void run_typed_direct_chunk_row_arg(
706 const TypedDirectChunkRun& chunkRun, Func& func, const TypedQueryExecState& state, core::func_type_list<T>) {
707 Chunk* pChunk = chunkRun.pChunk;
708 const auto from = chunkRun.from;
709 const auto cnt = (uint32_t)(chunkRun.to - from);
710 using U = typename actual_type_t<T>::Type;
711 if constexpr (std::is_same_v<U, Entity>) {
712 const auto data = pChunk->entity_view();
713 GAIA_FOR(cnt)
714 func(data[from + i]);
715 } else if constexpr (!mem::is_soa_layout_v<U>) {
716 const auto* pData = chunkRun.pData != nullptr ? chunkRun.pData[0] : nullptr;
717 if (pData == nullptr)
718 pData = pChunk->comp_rec_view()[pChunk->comp_idx(state.argIds[0])].pData;
719 auto* data = (U*)pData + from;
720 GAIA_FOR(cnt)
721 func(data[i]);
722 } else {
723 auto view = pChunk->template sview_auto<T>(from, chunkRun.to);
724 GAIA_FOR(cnt)
725 func(view[from + i]);
726 }
727 }
728
729 template <typename T>
730 GAIA_NODISCARD constexpr bool typed_direct_chunk_data_cacheable_arg() {
731 using U = typename actual_type_t<T>::Type;
732 return !std::is_same_v<U, Entity> && !mem::is_soa_layout_v<U>;
733 }
734
735 template <typename... T>
736 GAIA_NODISCARD constexpr bool typed_direct_chunk_data_cacheable(core::func_type_list<T...>) {
737 if constexpr (sizeof...(T) == 0)
738 return false;
739 else
740 return (typed_direct_chunk_data_cacheable_arg<T>() && ...);
741 }
742
743 template <typename T>
744 GAIA_NODISCARD inline decltype(auto)
745 typed_direct_chunk_cached_arg(const void* const* pData, uint32_t argIdx, uint32_t row) {
746 using U = typename actual_type_t<T>::Type;
747 auto* data = (U*)pData[argIdx];
748 return data[row];
749 }
750
751 template <typename Func, typename... T, size_t... I>
752 inline void invoke_typed_direct_chunk_cached_row(
753 Func& func, const void* const* pData, uint32_t row, core::func_type_list<T...>, std::index_sequence<I...>) {
754 func(typed_direct_chunk_cached_arg<T>(pData, (uint32_t)I, row)...);
755 }
756
757 template <uint32_t ArgCount>
758 GAIA_NODISCARD inline bool typed_direct_chunk_cached_data_valid(const void* const* pData) {
759 if (pData == nullptr)
760 return false;
761 GAIA_FOR(ArgCount) {
762 if (pData[i] == nullptr)
763 return false;
764 }
765 return true;
766 }
767
778 template <typename Func, typename... T>
779 inline void run_typed_direct_chunk_rows(
780 const TypedDirectChunkRun& chunkRun, Func& func, const TypedQueryExecState& state,
781 core::func_type_list<T...>) {
782 Chunk* pChunk = chunkRun.pChunk;
783 const auto from = chunkRun.from;
784 const auto to = chunkRun.to;
785 if constexpr (sizeof...(T) == 1) {
786 run_typed_direct_chunk_row_arg(chunkRun, func, state, core::func_type_list<T...>{});
787 } else if constexpr (sizeof...(T) > 0) {
788 const auto cnt = (uint32_t)(to - from);
789 if constexpr (typed_direct_chunk_data_cacheable(core::func_type_list<T...>{})) {
790 if (typed_direct_chunk_cached_data_valid<(uint32_t)sizeof...(T)>(chunkRun.pData)) {
791 GAIA_FOR(cnt)
792 invoke_typed_direct_chunk_cached_row(
793 func, chunkRun.pData, (uint32_t)i, core::func_type_list<T...>{}, std::index_sequence_for<T...>{});
794 return;
795 }
796 }
797
798 auto views = std::make_tuple(pChunk->template sview_auto<T>(from, to)...);
799 GAIA_FOR(cnt)
800 invoke_typed_direct_chunk_row(
801 func, views, (uint32_t)i, from, core::func_type_list<T...>{}, std::index_sequence_for<T...>{});
802 } else {
803 const auto cnt = (uint32_t)(to - from);
804 GAIA_FOR(cnt)
805 func();
806 }
807 }
808
809 template <typename T>
810 struct TypedSparseQueryView {
811 using U = typename actual_type_t<T>::Type;
812 const Entity* pEntities = nullptr;
813 void* pStore = nullptr;
814
815 GAIA_NODISCARD decltype(auto) operator[](size_t idx) const {
816 if constexpr (core::is_mut_v<typename actual_type_t<T>::TypeOriginal>)
817 return world_typed_sparse_store_mut<U>(pStore, pEntities[idx]);
818 else
819 return world_typed_sparse_store_get<U>(pStore, pEntities[idx]);
820 }
821 };
822
823 template <typename T>
824 GAIA_NODISCARD inline auto typed_sparse_chunk_view(
825 Chunk* pChunk, uint16_t from, uint16_t to, const TypedQueryExecState& state, uint32_t argIdx) {
826 using U = typename actual_type_t<T>::Type;
827 if constexpr (auto_storage_policy_v<U> == DataStorageType::Sparse)
828 return TypedSparseQueryView<T>{pChunk->entity_view().data() + from, state.sparseStores[argIdx]};
829 else
830 return pChunk->template sview_auto<T>(from, to);
831 }
832
833 template <typename T, typename View>
834 GAIA_NODISCARD inline decltype(auto) typed_sparse_chunk_arg_at(View& view, uint32_t row, uint16_t from) {
835 using U = typename actual_type_t<T>::Type;
836 if constexpr (auto_storage_policy_v<U> == DataStorageType::Sparse)
837 return view[row];
838 else
839 return typed_direct_chunk_arg_at<T>(view, row, from);
840 }
841
842 template <typename Func, typename... T, size_t... I>
843 inline void run_typed_sparse_chunk_rows_impl(
844 Chunk* pChunk, uint16_t from, uint16_t to, Func& func, const TypedQueryExecState& state,
845 core::func_type_list<T...>, std::index_sequence<I...>) {
846 auto views = std::make_tuple(typed_sparse_chunk_view<T>(pChunk, from, to, state, (uint32_t)I)...);
847 const auto cnt = (uint32_t)(to - from);
848 GAIA_FOR(cnt)
849 func(typed_sparse_chunk_arg_at<T>(std::get<I>(views), (uint32_t)i, from)...);
850 }
851
852 template <typename Func, typename... T>
853 inline void run_typed_sparse_chunk_rows(
854 Chunk* pChunk, uint16_t from, uint16_t to, Func& func, const TypedQueryExecState& state,
855 core::func_type_list<T...> types) {
856 run_typed_sparse_chunk_rows_impl(pChunk, from, to, func, state, types, std::index_sequence_for<T...>{});
857 }
858
859 template <typename T>
860 GAIA_NODISCARD inline decltype(auto)
861 typed_sparse_entity_arg(World& world, Entity entity, const TypedQueryExecState& state, uint32_t argIdx) {
862 using U = typename actual_type_t<T>::Type;
863 if constexpr (std::is_same_v<U, Entity>)
864 return entity;
865 else if constexpr (auto_storage_policy_v<U> == DataStorageType::Sparse) {
866 if (world_typed_sparse_store_has<U>(state.sparseStores[argIdx], entity)) {
867 if constexpr (core::is_mut_v<typename actual_type_t<T>::TypeOriginal>)
868 return world_typed_sparse_store_mut<U>(state.sparseStores[argIdx], entity);
869 else
870 return world_typed_sparse_store_get<U>(state.sparseStores[argIdx], entity);
871 }
872 }
873 if constexpr (core::is_mut_v<typename actual_type_t<T>::TypeOriginal>)
874 return world_query_entity_arg_by_id_raw<T>(world, entity, state.argIds[argIdx]);
875 else
876 return world_query_entity_arg_by_id<T>(world, entity, state.argIds[argIdx]);
877 }
878
879 template <typename Func, typename... T, size_t... I>
880 inline void invoke_typed_sparse_entity(
881 World& world, Entity entity, const TypedQueryExecState& state, Func& func, core::func_type_list<T...>,
882 std::index_sequence<I...>) {
883 func(typed_sparse_entity_arg<T>(world, entity, state, (uint32_t)I)...);
884 }
885
890 inline QueryImpl::QueryPlan
891 QueryImpl::prepare_query_plan(const QueryInfo& queryInfo, const TypedQueryExecState& state) const {
892 QueryPlan plan{};
893 plan.payloadKind = exec_payload_kind(queryInfo, Constraints::EnabledOnly);
894
895 const bool hasFilters = queryInfo.has_filters();
896 const bool hasSortedPayload = queryInfo.has_sorted_payload();
897 const bool hasDepthOrderBarrier = has_depth_order_hierarchy_enabled_barrier(queryInfo);
898 const bool depthOrderBarrierPrunes = hasDepthOrderBarrier && depth_order_hierarchy_barrier_prunes(queryInfo);
899 if (hasFilters)
900 plan.flags |= QueryPlanFlag_Filtered;
901 if (queryInfo.has_entity_filter_terms())
902 plan.flags |= QueryPlanFlag_EntityFilter;
903 if (queryInfo.has_inherited_data_payload())
904 plan.flags |= QueryPlanFlag_InheritedPayload;
905 if (queryInfo.has_grouped_payload())
906 plan.flags |= QueryPlanFlag_Grouped;
907 if (hasSortedPayload || hasDepthOrderBarrier)
908 plan.flags |= QueryPlanFlag_Sorted;
909 if (hasDepthOrderBarrier && !depthOrderBarrierPrunes)
910 plan.payloadKind = ExecPayloadKind::Grouped;
911 if (depthOrderBarrierPrunes)
912 plan.flags |= QueryPlanFlag_BarrierCache;
913
914 const bool canDirectEntitySeed = !hasFilters && can_use_direct_entity_seed_eval(queryInfo);
915 const bool canDirectChunks = !hasSortedPayload && (!hasDepthOrderBarrier || !depthOrderBarrierPrunes);
916
917 auto setDenseRange = [&]() -> bool {
918 const auto cacheRange = selected_query_cache_range(queryInfo);
919 plan.idxFrom = cacheRange.idxFrom;
920 plan.idxTo = cacheRange.idxTo;
921 if (cacheRange.hasSelectedGroup) {
922 plan.flags |= QueryPlanFlag_Grouped;
923 plan.payloadKind = ExecPayloadKind::Grouped;
924 if (!cacheRange.valid)
925 return false;
926 }
927 return plan.idxFrom < plan.idxTo;
928 };
929
930 if ((state.canUseDirectChunkEval || state.canUseSparseChunkEval) && !canDirectEntitySeed && canDirectChunks) {
931 if (!setDenseRange()) {
932 plan.mode = QueryPlanMode::Empty;
933 plan.idxFrom = 0;
934 plan.idxTo = 0;
935 return plan;
936 }
937
938 plan.mode = state.canUseSparseChunkEval ? QueryPlanMode::SparseDense : QueryPlanMode::DirectDense;
939 return plan;
940 }
941
942 if (canDirectEntitySeed) {
943 plan.mode = QueryPlanMode::EntitySeed;
944 return plan;
945 }
946
947 if (!setDenseRange()) {
948 plan.mode = QueryPlanMode::Empty;
949 plan.idxFrom = 0;
950 plan.idxTo = 0;
951 return plan;
952 }
953
954 if (hasSortedPayload) {
955 plan.mode = QueryPlanMode::Sorted;
956 plan.payloadKind = ExecPayloadKind::NonTrivial;
957 return plan;
958 }
959
960 if (hasDepthOrderBarrier && depthOrderBarrierPrunes) {
961 plan.mode = QueryPlanMode::Traversal;
962 plan.payloadKind = ExecPayloadKind::NonTrivial;
963 return plan;
964 }
965
966 if ((plan.flags & QueryPlanFlag_InheritedPayload) != 0) {
967 plan.mode = QueryPlanMode::Traversal;
968 plan.payloadKind = ExecPayloadKind::NonTrivial;
969 return plan;
970 }
971
972 if (hasDepthOrderBarrier) {
973 plan.payloadKind = ExecPayloadKind::Grouped;
974 return plan;
975 }
976
977 if (!canDirectChunks) {
978 plan.mode = QueryPlanMode::Sorted;
979 return plan;
980 }
981
982 if (!state.canUseDirectChunkEval) {
983 plan.mode = QueryPlanMode::MappedDense;
984 return plan;
985 }
986
987 return plan;
988 }
989
990 template <bool HasFilters, typename Func, typename... T>
991 inline void QueryImpl::run_query_on_chunks_sparse_typed(
992 QueryInfo& queryInfo, const QueryPlan& plan, const TypedQueryExecState& state, Func& func,
993 core::func_type_list<T...> types) {
994 GAIA_ASSERT(plan.mode == QueryPlanMode::SparseDense);
995 auto& world = *queryInfo.world();
996 auto boundState = state;
997 bind_typed_sparse_stores(boundState, world, types);
998 const auto cacheView = queryInfo.cache_archetype_view();
999 if (plan.idxFrom >= plan.idxTo)
1000 return;
1001
1002 if (boundState.hasWriteArgs)
1003 ::gaia::ecs::update_version(*m_worldVersion);
1004
1005 const bool canSkipProcessCheck =
1006 !queryInfo.result_cache_may_need_prefab_filter() && (plan.flags & QueryPlanFlag_BarrierCache) == 0;
1007 lock(*m_storage.world());
1008 for (uint32_t i = plan.idxFrom; i < plan.idxTo; ++i) {
1009 const auto* pArchetype = cacheView[i];
1010 if (canSkipProcessCheck) {
1011 if GAIA_UNLIKELY (pArchetype->is_req_del())
1012 continue;
1013 } else if GAIA_UNLIKELY (!can_process_archetype_inter(queryInfo, *pArchetype, Constraints::EnabledOnly))
1014 continue;
1015
1016 std::span<const uint8_t> indicesView;
1017 if constexpr (HasFilters)
1018 indicesView = queryInfo.indices_mapping_view(i);
1019
1020 for (auto* pChunk: pArchetype->chunks()) {
1021 const auto from = Iter::start_index(pChunk);
1022 const auto to = Iter::end_index(pChunk);
1023 if GAIA_UNLIKELY (from == to)
1024 continue;
1025 if constexpr (HasFilters) {
1026 if GAIA_UNLIKELY (!match_filters(*pChunk, queryInfo, m_changedWorldVersion, indicesView))
1027 continue;
1028 }
1029
1030 GAIA_PROF_SCOPE(query_func);
1031 run_typed_sparse_chunk_rows(pChunk, from, to, func, boundState, types);
1032 if (boundState.hasWriteArgs)
1033 finish_typed_chunk_state(world, pChunk, from, to, boundState);
1034 }
1035 }
1036
1037 unlock(*m_storage.world());
1038 commit_cmd_buffer_st(*m_storage.world());
1039 commit_cmd_buffer_mt(*m_storage.world());
1040 m_changedWorldVersion = *m_worldVersion;
1041 }
1042
1043 template <typename Func, typename... T>
1044 inline void QueryImpl::run_query_on_sparse_entities_typed(
1045 QueryInfo& queryInfo, const TypedQueryExecState& state, Func& func, core::func_type_list<T...> types) {
1046 auto& world = *queryInfo.world();
1047 auto boundState = state;
1048 bind_typed_sparse_stores(boundState, world, types);
1049 if (boundState.hasWriteArgs)
1050 ::gaia::ecs::update_version(*m_worldVersion);
1051
1052 auto execEntity = [&](Entity entity) {
1053 invoke_typed_sparse_entity(world, entity, boundState, func, types, std::index_sequence_for<T...>{});
1054 if (boundState.hasWriteArgs)
1055 finish_typed_query_args_by_id(world, entity, boundState);
1056 };
1057
1058 if (boundState.hasWriteArgs) {
1059 auto& scratch = direct_query_scratch();
1060 const auto seedInfo = build_direct_entity_seed(world, queryInfo, scratch.entities);
1061 for (const auto entity: scratch.entities) {
1062 if (!match_direct_entity_constraints(world, queryInfo, entity, Constraints::EnabledOnly) ||
1063 !match_direct_entity_terms(world, entity, queryInfo, seedInfo))
1064 continue;
1065 execEntity(entity);
1066 }
1067 } else {
1068 const auto seedPlan = direct_entity_seed_plan(world, queryInfo);
1069 if (seedPlan.preferOrSeed) {
1070 for_each_direct_or_union(world, queryInfo, Constraints::EnabledOnly, [&](Entity entity) {
1071 execEntity(entity);
1072 return true;
1073 });
1074 } else {
1075 (void)for_each_direct_all_seed(world, queryInfo, seedPlan, Constraints::EnabledOnly, [&](Entity entity) {
1076 execEntity(entity);
1077 return true;
1078 });
1079 }
1080 }
1081
1082 m_changedWorldVersion = *m_worldVersion;
1083 }
1084
1095 template <bool HasFilters, typename Func, typename... T>
1096 inline void QueryImpl::run_query_on_chunks_direct_typed(
1097 QueryInfo& queryInfo, const QueryPlan& plan, const TypedQueryExecState& state, Func& func,
1098 core::func_type_list<T...> types) {
1099 auto& world = *queryInfo.world();
1100 if constexpr (HasFilters) {
1101 GAIA_ASSERT(plan.mode == QueryPlanMode::DirectDense);
1102 GAIA_ASSERT((plan.flags & QueryPlanFlag_Filtered) != 0);
1103 } else {
1104 GAIA_ASSERT(plan.mode == QueryPlanMode::DirectDense);
1105 }
1106
1107 auto cacheView = queryInfo.cache_archetype_view();
1108 if (plan.idxFrom >= plan.idxTo)
1109 return;
1110
1111 if (state.hasWriteArgs)
1112 ::gaia::ecs::update_version(*m_worldVersion);
1113
1114 const bool canSkipProcessCheck =
1115 !queryInfo.result_cache_may_need_prefab_filter() && (plan.flags & QueryPlanFlag_BarrierCache) == 0;
1116 lock(*m_storage.world());
1117 if constexpr (!HasFilters) {
1118 if (!state.hasWriteArgs && canSkipProcessCheck && plan.idxTo - plan.idxFrom >= 1024) {
1119 const auto terms = queryInfo.ctx().data.terms_view();
1120 uint32_t dataFields[MAX_ITEMS_IN_QUERY]{};
1121 uint32_t dataFieldCount = 0;
1122 if constexpr (typed_direct_chunk_data_cacheable(types)) {
1123 dataFieldCount = state.argCount;
1124 GAIA_FOR(state.argCount) {
1125 const auto argId = state.argIds[i];
1126 uint32_t termIdx = UINT32_MAX;
1127 for (uint32_t j = 0; j < terms.size(); ++j) {
1128 if (terms[j].id == argId) {
1129 termIdx = (uint32_t)j;
1130 break;
1131 }
1132 }
1133 if (termIdx == UINT32_MAX) {
1134 dataFieldCount = 0;
1135 break;
1136 }
1137 dataFields[i] = termIdx;
1138 }
1139 }
1140 const auto chunkView = queryInfo.direct_chunk_view(plan.idxFrom, plan.idxTo, dataFields, dataFieldCount);
1141 const auto dataView = queryInfo.direct_chunk_data_view();
1142 uint32_t chunkIdx = 0;
1143 for (const auto& entry: chunkView) {
1144 if constexpr (sizeof...(T) > 1 && typed_direct_chunk_data_cacheable(types)) {
1147 constexpr uint32_t PrefetchDistance = 8;
1148 const auto prefetchIdx = chunkIdx + PrefetchDistance;
1149 if (prefetchIdx < chunkView.size()) {
1150 const auto& prefetchEntry = chunkView[prefetchIdx];
1151 if (prefetchEntry.dataOffset != UINT32_MAX) {
1152 const auto* pPrefetchData = &dataView[prefetchEntry.dataOffset];
1153 GAIA_FOR_(sizeof...(T), dataIdx) {
1154 if (pPrefetchData[dataIdx] != nullptr)
1155 gaia::prefetch(pPrefetchData[dataIdx], PrefetchHint::PREFETCH_HINT_T1);
1156 }
1157 }
1158 }
1159 ++chunkIdx;
1160 }
1161 const void* pSingleData[1]{};
1162 const void* const* pData = nullptr;
1163 if (entry.pData != nullptr) {
1167 pSingleData[0] = entry.pData;
1168 pData = pSingleData;
1169 } else if (entry.dataOffset != UINT32_MAX)
1170 pData = &dataView[entry.dataOffset];
1171 const TypedDirectChunkRun chunkRun{entry.pChunk, entry.pCompIndices, pData, entry.rowFrom, entry.rowTo};
1172
1173 GAIA_PROF_SCOPE(query_func);
1174 run_typed_direct_chunk_rows(chunkRun, func, state, types);
1175 }
1176
1177 unlock(*m_storage.world());
1178 commit_cmd_buffer_st(*m_storage.world());
1179 commit_cmd_buffer_mt(*m_storage.world());
1180 m_changedWorldVersion = *m_worldVersion;
1181 return;
1182 }
1183 }
1184
1185 for (uint32_t i = plan.idxFrom; i < plan.idxTo; ++i) {
1186 const auto* pArchetype = cacheView[i];
1187 if (canSkipProcessCheck) {
1188 if GAIA_UNLIKELY (pArchetype->is_req_del())
1189 continue;
1190 } else if GAIA_UNLIKELY (!can_process_archetype_inter(queryInfo, *pArchetype, Constraints::EnabledOnly))
1191 continue;
1192
1193 std::span<const uint8_t> indicesView;
1194 if constexpr (HasFilters)
1195 indicesView = queryInfo.indices_mapping_view(i);
1196
1197 const auto& chunks = pArchetype->chunks();
1198 for (auto* pChunk: chunks) {
1199 const auto from = Iter::start_index(pChunk);
1200 const auto to = Iter::end_index(pChunk);
1201 if GAIA_UNLIKELY (from == to)
1202 continue;
1203
1204 if constexpr (HasFilters) {
1205 if GAIA_UNLIKELY (!match_filters(*pChunk, queryInfo, m_changedWorldVersion, indicesView))
1206 continue;
1207 }
1208
1209 GAIA_PROF_SCOPE(query_func);
1210 run_typed_direct_chunk_rows({pChunk, nullptr, nullptr, from, to}, func, state, types);
1211 if (state.hasWriteArgs)
1212 finish_typed_chunk_state(world, pChunk, from, to, state);
1213 }
1214 }
1215
1216 unlock(*m_storage.world());
1217 commit_cmd_buffer_st(*m_storage.world());
1218 commit_cmd_buffer_mt(*m_storage.world());
1219 m_changedWorldVersion = *m_worldVersion;
1220 }
1221
1222 inline void QueryImpl::run_query_on_chunks_direct_iter(
1223 QueryInfo& queryInfo, const QueryPlan& plan, const TypedQueryExecState& state, void* pFunc,
1224 void (*runChunk)(QueryImpl&, Iter& it, void*, const TypedQueryExecState&)) {
1225 auto& world = *queryInfo.world();
1226 GAIA_ASSERT(plan.mode == QueryPlanMode::DirectDense);
1227 GAIA_ASSERT(!queryInfo.has_filters());
1228 auto cacheView = queryInfo.cache_archetype_view();
1229 if (plan.idxFrom >= plan.idxTo)
1230 return;
1231
1232 if (state.hasWriteArgs)
1233 ::gaia::ecs::update_version(*m_worldVersion);
1234
1235 lock(*m_storage.world());
1236 Iter it;
1237 it.init_query_state(queryInfo.world(), Constraints::EnabledOnly, false);
1238 const Archetype* pLastArchetype = nullptr;
1239 for (uint32_t i = plan.idxFrom; i < plan.idxTo; ++i) {
1240 const auto* pArchetype = cacheView[i];
1241 if GAIA_UNLIKELY (!can_process_archetype_inter(queryInfo, *pArchetype, Constraints::EnabledOnly))
1242 continue;
1243
1244 const auto& chunks = pArchetype->chunks();
1245 for (auto* pChunk: chunks) {
1246 const auto from = Iter::start_index(pChunk);
1247 const auto to = Iter::end_index(pChunk);
1248 if GAIA_UNLIKELY (from == to)
1249 continue;
1250
1251 GAIA_PROF_SCOPE(query_func);
1252 if (pArchetype != pLastArchetype) {
1253 it.set_archetype(pArchetype);
1254 pLastArchetype = pArchetype;
1255 }
1256 it.set_chunk(pChunk, from, to);
1257 it.set_group_id(0);
1258 it.ctx(m_ctx);
1259 runChunk(*this, it, pFunc, state);
1260 finish_typed_chunk_state(world, pChunk, from, to, state);
1261 }
1262 }
1263
1264 unlock(*m_storage.world());
1265 commit_cmd_buffer_st(*m_storage.world());
1266 commit_cmd_buffer_mt(*m_storage.world());
1267 m_changedWorldVersion = *m_worldVersion;
1268 }
1269
1270 inline void QueryImpl::run_query_on_chunks_direct(
1271 QueryInfo& queryInfo, const QueryPlan& plan, const TypedQueryExecState& state, void* pFunc,
1272 void (*runChunk)(QueryImpl&, Iter& it, void*, const TypedQueryExecState&)) {
1273 auto& world = *queryInfo.world();
1274 auto cacheView = queryInfo.cache_archetype_view();
1275 if (plan.idxFrom >= plan.idxTo)
1276 return;
1277
1278 if (state.hasWriteArgs)
1279 ::gaia::ecs::update_version(*m_worldVersion);
1280
1281 GAIA_ASSERT(plan.mode == QueryPlanMode::DirectDense);
1282 const bool hasFilters = (plan.flags & QueryPlanFlag_Filtered) != 0;
1283
1284 lock(*m_storage.world());
1285 Iter it;
1286 it.init_query_state(queryInfo.world(), Constraints::EnabledOnly, false);
1287 const Archetype* pLastArchetype = nullptr;
1288
1289 for (uint32_t i = plan.idxFrom; i < plan.idxTo; ++i) {
1290 const auto* pArchetype = cacheView[i];
1291 if GAIA_UNLIKELY (!can_process_archetype_inter(queryInfo, *pArchetype, Constraints::EnabledOnly))
1292 continue;
1293
1294 std::span<const uint8_t> indicesView;
1295 if (hasFilters)
1296 indicesView = queryInfo.indices_mapping_view(i);
1297
1298 const auto& chunks = pArchetype->chunks();
1299 for (auto* pChunk: chunks) {
1300 const auto from = Iter::start_index(pChunk);
1301 const auto to = Iter::end_index(pChunk);
1302 if GAIA_UNLIKELY (from == to)
1303 continue;
1304
1305 if (hasFilters) {
1306 if GAIA_UNLIKELY (!match_filters(*pChunk, queryInfo, m_changedWorldVersion, indicesView))
1307 continue;
1308 }
1309
1310 GAIA_PROF_SCOPE(query_func);
1311 if (pArchetype != pLastArchetype) {
1312 it.set_archetype(pArchetype);
1313 pLastArchetype = pArchetype;
1314 }
1315 it.set_chunk(pChunk, from, to);
1316 it.set_group_id(0);
1317 it.ctx(m_ctx);
1318 runChunk(*this, it, pFunc, state);
1319 finish_typed_chunk_state(world, pChunk, from, to, state);
1320 }
1321 }
1322
1323 unlock(*m_storage.world());
1324 commit_cmd_buffer_st(*m_storage.world());
1325 commit_cmd_buffer_mt(*m_storage.world());
1326 m_changedWorldVersion = *m_worldVersion;
1327 }
1328
1329 template <QueryExecType ExecType>
1330 inline void QueryImpl::each_inter(
1331 QueryInfo& queryInfo, const QueryPlan& plan, void* pFunc, const TypedQueryExecState& state,
1332 const TypedQueryErasedOps& ops) {
1333 if (plan.mode == QueryPlanMode::Empty)
1334 return;
1335
1336 if (plan.mode == QueryPlanMode::EntitySeed) {
1337 GAIA_PROF_SCOPE(query_func);
1338 each_direct_inter(
1339 queryInfo, Constraints::EnabledOnly, pFunc, state, ops.runDirectChunk, ops.needsInheritedArgIds,
1340 ops.invokeInherited);
1341 return;
1342 }
1343
1344 if (state.canUseDirectChunkEval) {
1345 if constexpr (ExecType == QueryExecType::Default) {
1346 if (plan.mode == QueryPlanMode::DirectDense) {
1347 run_query_on_chunks_direct(queryInfo, plan, state, pFunc, ops.runDirectFastChunk);
1348 return;
1349 }
1350 }
1351 TypedDirectChunkCallback cb{this, pFunc, &state, ops.runDirectChunk};
1352 run_query_on_chunks<ExecType, IterModeEnabled>(queryInfo, cb);
1353 } else {
1354 TypedMappedChunkCallback cb{this, &queryInfo, pFunc, &state, ops.runMappedChunk};
1355 run_query_on_chunks<ExecType, IterModeEnabled>(queryInfo, cb);
1356 }
1357 }
1358
1359 template <QueryExecType ExecType, typename Func>
1360 inline void QueryImpl::each_typed_inter(QueryInfo& queryInfo, Func func) {
1361 using InputArgs = decltype(core::func_args(&Func::operator()));
1362
1363#if GAIA_ASSERT_ENABLED
1364 GAIA_ASSERT(typed_query_args_match_query(queryInfo, InputArgs{}));
1365#endif
1366 auto& world = *const_cast<World*>(queryInfo.world());
1367 TypedQueryArgMeta metas[MAX_ITEMS_IN_QUERY]{};
1368 const auto argCount = init_typed_query_arg_metas(metas, world, InputArgs{});
1369 const auto state = build_typed_query_exec_state(world, queryInfo, metas, argCount);
1370 const auto plan = prepare_query_plan(queryInfo, state);
1371 if constexpr (ExecType == QueryExecType::Default) {
1372 if constexpr (typed_query_arg_list_uses_sparse_storage_v<InputArgs>) {
1373 if (plan.mode == QueryPlanMode::EntitySeed) {
1374 run_query_on_sparse_entities_typed(queryInfo, state, func, InputArgs{});
1375 return;
1376 }
1377 if (plan.mode == QueryPlanMode::SparseDense) {
1378 if ((plan.flags & QueryPlanFlag_Filtered) != 0)
1379 run_query_on_chunks_sparse_typed<true>(queryInfo, plan, state, func, InputArgs{});
1380 else
1381 run_query_on_chunks_sparse_typed<false>(queryInfo, plan, state, func, InputArgs{});
1382 return;
1383 }
1384 }
1385 if (plan.mode == QueryPlanMode::DirectDense) {
1386 if ((plan.flags & QueryPlanFlag_Filtered) != 0)
1387 run_query_on_chunks_direct_typed<true>(queryInfo, plan, state, func, InputArgs{});
1388 else
1389 run_query_on_chunks_direct_typed<false>(queryInfo, plan, state, func, InputArgs{});
1390 return;
1391 }
1392 }
1393
1394 TypedQueryErasedOps ops;
1395 ops.runDirectFastChunk = typed_run_direct_fast_chunk_ptr<Func>(InputArgs{});
1396 ops.runDirectChunk = typed_run_direct_chunk_ptr<Func>(InputArgs{});
1397 ops.runMappedChunk = typed_run_mapped_chunk_ptr<Func>(InputArgs{});
1398 ops.invokeInherited = typed_invoke_inherited_ptr<Func>(InputArgs{});
1399 ops.needsInheritedArgIds = state.needsInheritedArgIds;
1400 each_inter<ExecType>(queryInfo, plan, &func, state, ops);
1401 }
1402
1403 inline void QueryImpl::each_typed_erased(
1404 QueryExecType execType, void* pFunc, const TypedQueryExecState& state, const TypedQueryErasedOps& ops) {
1405 auto& queryInfo = fetch();
1406 match_all(queryInfo);
1407 const auto plan = prepare_query_plan(queryInfo, state);
1408 if (execType == QueryExecType::Default && ops.runSparsePlan != nullptr &&
1409 (plan.mode == QueryPlanMode::SparseDense || plan.mode == QueryPlanMode::EntitySeed)) {
1410 ops.runSparsePlan(*this, queryInfo, plan, pFunc, state);
1411 return;
1412 }
1413
1414 switch (execType) {
1415 case QueryExecType::Parallel:
1416 each_inter<QueryExecType::Parallel>(queryInfo, plan, pFunc, state, ops);
1417 break;
1418 case QueryExecType::ParallelPerf:
1419 each_inter<QueryExecType::ParallelPerf>(queryInfo, plan, pFunc, state, ops);
1420 break;
1421 case QueryExecType::ParallelEff:
1422 each_inter<QueryExecType::ParallelEff>(queryInfo, plan, pFunc, state, ops);
1423 break;
1424 default:
1425 each_inter<QueryExecType::Default>(queryInfo, plan, pFunc, state, ops);
1426 break;
1427 }
1428 }
1429
1430 template <typename Func, std::enable_if_t<!detail::is_query_iter_callback_v<Func>, int>>
1431 inline void QueryImpl::each(Func func) {
1432 each(func, QueryExecType::Default);
1433 }
1434
1435 template <typename Func, std::enable_if_t<!detail::is_query_iter_callback_v<Func>, int>>
1436 inline void QueryImpl::each(Func func, QueryExecType execType) {
1437 auto& queryInfo = fetch();
1438 match_all(queryInfo);
1439
1440 switch (execType) {
1441 case QueryExecType::Parallel:
1442 each_typed_inter<QueryExecType::Parallel>(queryInfo, func);
1443 break;
1444 case QueryExecType::ParallelPerf:
1445 each_typed_inter<QueryExecType::ParallelPerf>(queryInfo, func);
1446 break;
1447 case QueryExecType::ParallelEff:
1448 each_typed_inter<QueryExecType::ParallelEff>(queryInfo, func);
1449 break;
1450 default:
1451 each_typed_inter<QueryExecType::Default>(queryInfo, func);
1452 break;
1453 }
1454 }
1455
1456 template <QueryExecType ExecType>
1457 inline void QueryImpl::each_iter_inter_erased(
1458 QueryInfo& queryInfo, const QueryPlan& plan, void* pFunc, const TypedQueryExecState& state,
1459 void (*runDirectFastChunk)(QueryImpl&, Iter&, void*, const TypedQueryExecState&),
1460 void (*runMappedChunk)(QueryImpl&, const QueryInfo&, Iter&, void*, const TypedQueryExecState&)) {
1461 TypedIterErasedCallback cb{this, pFunc, &state, runDirectFastChunk, runMappedChunk};
1462
1463 if (plan.mode == QueryPlanMode::Empty)
1464 return;
1465
1466 if (plan.mode == QueryPlanMode::EntitySeed) {
1467 each_direct_iter_inter(queryInfo, Constraints::EnabledOnly, cb);
1468 return;
1469 }
1470
1471 if constexpr (ExecType == QueryExecType::Default) {
1472 if (plan.mode == QueryPlanMode::DirectDense) {
1473 run_query_on_chunks_direct_iter(queryInfo, plan, state, pFunc, runDirectFastChunk);
1474 return;
1475 }
1476 }
1477
1478 run_query_on_chunks<ExecType, IterModeEnabled>(queryInfo, cb);
1479 }
1480
1481 inline void QueryImpl::each_iter_erased(
1482 QueryExecType execType, void* pFunc, const TypedQueryExecState& state,
1483 void (*runDirectFastChunk)(QueryImpl&, Iter&, void*, const TypedQueryExecState&),
1484 void (*runMappedChunk)(QueryImpl&, const QueryInfo&, Iter&, void*, const TypedQueryExecState&)) {
1485 auto& queryInfo = fetch();
1486 match_all(queryInfo);
1487 const auto plan = prepare_query_plan(queryInfo, state);
1488
1489 switch (execType) {
1490 case QueryExecType::Parallel:
1491 each_iter_inter_erased<QueryExecType::Parallel>(
1492 queryInfo, plan, pFunc, state, runDirectFastChunk, runMappedChunk);
1493 break;
1494 case QueryExecType::ParallelPerf:
1495 each_iter_inter_erased<QueryExecType::ParallelPerf>(
1496 queryInfo, plan, pFunc, state, runDirectFastChunk, runMappedChunk);
1497 break;
1498 case QueryExecType::ParallelEff:
1499 each_iter_inter_erased<QueryExecType::ParallelEff>(
1500 queryInfo, plan, pFunc, state, runDirectFastChunk, runMappedChunk);
1501 break;
1502 default:
1503 each_iter_inter_erased<QueryExecType::Default>(
1504 queryInfo, plan, pFunc, state, runDirectFastChunk, runMappedChunk);
1505 break;
1506 }
1507 }
1508
1509 template <typename Func>
1510 inline void QueryImpl::each_iter(Iter& it, Func func) {
1511 using InputArgs = decltype(core::func_args(&Func::operator()));
1512 auto& queryInfo = fetch();
1513
1514#if GAIA_ASSERT_ENABLED
1515 GAIA_ASSERT(typed_query_args_match_query(queryInfo, InputArgs{}));
1516#endif
1517 TypedQueryArgMeta metas[MAX_ITEMS_IN_QUERY]{};
1518 const auto argCount = init_typed_query_arg_metas(metas, *it.world(), InputArgs{});
1519 const auto state = build_typed_query_exec_state(*it.world(), queryInfo, metas, argCount);
1520 it.ctx(m_ctx);
1521 each_iter_dispatch(*this, queryInfo, it, func, state, InputArgs{});
1522 }
1523
1524 inline void QueryImpl::each_iter_erased(
1525 Iter& it, void* pFunc, const TypedQueryExecState& state,
1526 void (*runDirectFastChunk)(QueryImpl&, Iter&, void*, const TypedQueryExecState&),
1527 void (*runMappedChunk)(QueryImpl&, const QueryInfo&, Iter&, void*, const TypedQueryExecState&)) {
1528 auto& queryInfo = fetch();
1529 it.ctx(m_ctx);
1530 if (state.canUseDirectChunkEval) {
1531 runDirectFastChunk(*this, it, pFunc, state);
1532 finish_typed_chunk_state(*it.world(), const_cast<Chunk*>(it.chunk()), it.row_begin(), it.row_end(), state);
1533 } else
1534 runMappedChunk(*this, queryInfo, it, pFunc, state);
1535 }
1536
1537 inline void QueryImpl::each_direct_inter(
1538 QueryInfo& queryInfo, Constraints constraints, void* pFunc, const TypedQueryExecState& state,
1539 void (*runDirectChunk)(QueryImpl&, Iter&, void*, const TypedQueryExecState&), bool needsInheritedArgIds,
1540 void (*invokeInherited)(World&, Entity, const Entity*, void*)) {
1541 auto& world = *queryInfo.world();
1542 const auto plan = direct_entity_seed_plan(world, queryInfo);
1543 const bool hasWriteTerms = queryInfo.ctx().data.readWriteMask != 0;
1544
1545 auto exec_direct_entity = [&](Entity entity) {
1546 uint8_t indices[ChunkHeader::MAX_COMPONENTS];
1547 Entity termIds[ChunkHeader::MAX_COMPONENTS];
1548 Iter it;
1549 it.set_constraints(constraints);
1550 init_direct_entity_iter(queryInfo, world, entity, it, indices, termIds);
1551 it.ctx(m_ctx);
1552 runDirectChunk(*this, it, pFunc, state);
1553 };
1554
1555 auto exec_entity = [&](Entity entity) {
1556 if (needsInheritedArgIds && state.hasInheritedTerms) {
1557 invokeInherited(world, entity, state.argIds, pFunc);
1558 finish_typed_query_args_by_id(world, entity, state);
1559 return;
1560 }
1561
1562 exec_direct_entity(entity);
1563 };
1564
1565 if (!hasWriteTerms && !plan.preferOrSeed) {
1566 const auto* pSeedTerm = find_direct_all_seed_term(queryInfo, plan);
1567 if (pSeedTerm != nullptr && can_use_direct_seed_run_cache(world, queryInfo, *pSeedTerm)) {
1568 DirectEntitySeedInfo seedInfo{};
1569 seedInfo.seededAllTerm = pSeedTerm->id;
1570 seedInfo.seededAllMatchKind = pSeedTerm->matchKind;
1571 seedInfo.seededFromAll = true;
1572 if (!state.hasInheritedTerms) {
1573 const auto runs = cached_direct_seed_runs(queryInfo, *pSeedTerm, seedInfo, constraints);
1574 if (state.canUseDirectChunkEval) {
1575 Iter it;
1576 it.init_query_state(&world, constraints, false);
1577 const Archetype* pLastArchetype = nullptr;
1578 for (const auto& run: runs) {
1579 if (run.pArchetype != pLastArchetype) {
1580 it.set_archetype(run.pArchetype);
1581 pLastArchetype = run.pArchetype;
1582 }
1583 it.set_chunk(run.pChunk, run.from, run.to);
1584 it.set_group_id(0);
1585 it.ctx(m_ctx);
1586 runDirectChunk(*this, it, pFunc, state);
1587 }
1588 } else {
1589 Iter it;
1590 it.init_query_state(&world, constraints, false);
1591 const Archetype* pLastArchetype = nullptr;
1592 uint8_t indices[ChunkHeader::MAX_COMPONENTS];
1593 Entity termIds[ChunkHeader::MAX_COMPONENTS];
1594 for (const auto& run: runs) {
1595 const auto& ec = ::gaia::ecs::fetch(world, run.pChunk->entity_view()[run.from]);
1596 init_direct_entity_iter(queryInfo, world, ec, it, indices, termIds, pLastArchetype);
1597 it.set_chunk(run.pChunk, run.from, run.to);
1598 it.set_group_id(0);
1599 it.ctx(m_ctx);
1600 runDirectChunk(*this, it, pFunc, state);
1601 }
1602 }
1603 } else {
1604 const auto entities = cached_direct_seed_chunk_entities(queryInfo, *pSeedTerm, seedInfo, constraints);
1605 for (const auto entity: entities)
1606 exec_entity(entity);
1607 }
1608 return;
1609 }
1610 }
1611
1612 auto walk_entities = [&](auto&& execEntity) {
1613 if (hasWriteTerms) {
1614 auto& scratch = direct_query_scratch();
1615 const auto seedInfo = build_direct_entity_seed(world, queryInfo, scratch.entities);
1616 for (const auto entity: scratch.entities) {
1617 if (!match_direct_entity_constraints(world, queryInfo, entity, constraints))
1618 continue;
1619 if (!match_direct_entity_terms(world, entity, queryInfo, seedInfo))
1620 continue;
1621 execEntity(entity);
1622 }
1623 return;
1624 }
1625
1626 if (plan.preferOrSeed) {
1627 for_each_direct_or_union(world, queryInfo, constraints, [&](Entity entity) {
1628 execEntity(entity);
1629 return true;
1630 });
1631 return;
1632 }
1633
1634 (void)for_each_direct_all_seed(world, queryInfo, plan, constraints, [&](Entity entity) {
1635 execEntity(entity);
1636 return true;
1637 });
1638 };
1639
1640 walk_entities(exec_entity);
1641 }
1642
1643 template <typename Func, std::enable_if_t<!detail::is_query_walk_core_callback_v<Func>, int>>
1644 inline void QueryImpl::each_walk(Func func, Entity relation, TravOrder order, Constraints constraints) {
1645 auto& queryInfo = fetch();
1646 match_all(queryInfo);
1647 const auto ordered = ordered_entities_walk(queryInfo, relation, order, constraints);
1648
1649 using InputArgs = decltype(core::func_args(&Func::operator()));
1650 GAIA_ASSERT(typed_query_args_match_query(queryInfo, InputArgs{}));
1651 GAIA_ASSERT(can_use_direct_target_eval(queryInfo));
1652 if (!can_use_direct_target_eval(queryInfo))
1653 return;
1654
1655 auto& world = *queryInfo.world();
1656 TypedQueryArgMeta metas[MAX_ITEMS_IN_QUERY]{};
1657 const auto argCount = init_typed_query_arg_metas(metas, world, InputArgs{});
1658 const auto state = build_typed_query_exec_state(world, queryInfo, metas, argCount);
1659 if (state.canUseDirectChunkEval) {
1660 each_walk_dispatch_direct(*this, queryInfo, ordered, constraints, func, state, InputArgs{});
1661 } else {
1662 each_walk_dispatch_mapped(*this, queryInfo, ordered, constraints, func, state, InputArgs{});
1663 }
1664 }
1665
1666 template <bool UseFilters, typename ContainerOut>
1667 inline void QueryImpl::arr_inter(QueryInfo& queryInfo, ContainerOut& outArray, Constraints constraints) {
1668 using ContainerItemType = typename ContainerOut::value_type;
1669 if constexpr (!UseFilters) {
1670 if (can_use_direct_entity_seed_eval(queryInfo)) {
1671 auto& world = *queryInfo.world();
1672 const auto plan = direct_entity_seed_plan(world, queryInfo);
1673 if (plan.preferOrSeed) {
1674 for_each_direct_or_union(world, queryInfo, constraints, [&](Entity entity) {
1675 typed_arr_push(world, entity, outArray);
1676 });
1677 return;
1678 }
1679
1680 (void)for_each_direct_all_seed(world, queryInfo, plan, constraints, [&](Entity entity) {
1681 typed_arr_push(world, entity, outArray);
1682 return true;
1683 });
1684
1685 return;
1686 }
1687 }
1688
1689 auto& world = *queryInfo.world();
1690 const auto meta = typed_query_arg_meta<ContainerItemType>(world);
1691 const DirectChunkArgEvalDesc desc{meta.termId, meta.isEntity, meta.isPair, meta.usesSparseStorage};
1692 Iter it;
1693 it.init_query_state(queryInfo.world(), constraints, false);
1694 const bool canUseDirectChunkEval = !UseFilters && !queryInfo.has_entity_filter_terms() &&
1695 can_use_direct_chunk_term_eval_descs(world, queryInfo, &desc, 1) &&
1696 can_use_direct_chunk_iteration_fastpath(queryInfo);
1697 const auto cacheView = queryInfo.cache_archetype_view();
1698 const bool needsBarrierCache = needs_depth_order_hierarchy_barrier_cache(queryInfo, constraints);
1699 const bool hasSortedArrayPayload = queryInfo.has_sorted_payload() || needsBarrierCache;
1700 const auto sortView =
1701 hasSortedArrayPayload ? queryInfo.cache_sort_view() : decltype(queryInfo.cache_sort_view()){};
1702 if (needsBarrierCache)
1703 queryInfo.ensure_depth_order_hierarchy_barrier_cache();
1704 const auto cacheRange = selected_query_cache_range(queryInfo);
1705 if (!cacheRange.valid)
1706 return;
1707 const auto idxFrom = cacheRange.idxFrom;
1708 const auto idxTo = cacheRange.idxTo;
1709
1710 if (!sortView.empty()) {
1711 for (const auto& view: sortView) {
1712 if (view.archetypeIdx < idxFrom || view.archetypeIdx >= idxTo)
1713 continue;
1714
1715 const bool barrierPasses = !needsBarrierCache || queryInfo.barrier_passes(view.archetypeIdx);
1716 if GAIA_UNLIKELY (!can_process_archetype_inter(
1717 queryInfo, *cacheView[view.archetypeIdx], constraints, barrierPasses))
1718 continue;
1719
1720 const auto viewFrom = view.startRow;
1721 const auto viewTo = (uint16_t)(view.startRow + view.count);
1722 uint16_t minStartRow = 0;
1723 uint16_t minEndRow = 0;
1724 chunk_effective_range(view.pChunk, constraints, needsBarrierCache, barrierPasses, minStartRow, minEndRow);
1725 const auto startRow = core::get_max(minStartRow, viewFrom);
1726 const auto endRow = core::get_min(minEndRow, viewTo);
1727 if (startRow == endRow)
1728 continue;
1729
1730 run_typed_arr_rows<UseFilters>(
1731 *this, queryInfo, it, outArray, m_changedWorldVersion, view.archetypeIdx, cacheView[view.archetypeIdx],
1732 view.pChunk, startRow, endRow, needsBarrierCache, canUseDirectChunkEval);
1733 }
1734 return;
1735 }
1736
1737 for (uint32_t i = idxFrom; i < idxTo; ++i) {
1738 auto* pArchetype = cacheView[i];
1739 const bool barrierPasses = !needsBarrierCache || queryInfo.barrier_passes(i);
1740 if GAIA_UNLIKELY (!can_process_archetype_inter(queryInfo, *pArchetype, constraints, barrierPasses))
1741 continue;
1742
1743 const auto& chunks = pArchetype->chunks();
1744 for (auto* pChunk: chunks) {
1745 uint16_t from = 0;
1746 uint16_t to = 0;
1747 chunk_effective_range(pChunk, constraints, needsBarrierCache, barrierPasses, from, to);
1748 run_typed_arr_rows<UseFilters>(
1749 *this, queryInfo, it, outArray, m_changedWorldVersion, i, pArchetype, pChunk, from, to,
1750 needsBarrierCache, canUseDirectChunkEval);
1751 }
1752 }
1753 }
1754
1755 template <typename Container>
1756 inline void QueryImpl::arr(Container& outArray, Constraints constraints) {
1757 const auto entCnt = count(constraints);
1758 if (entCnt == 0)
1759 return;
1760
1761 outArray.reserve(entCnt);
1762 auto& queryInfo = fetch();
1763 match_all(queryInfo);
1764
1765 const bool hasFilters = queryInfo.has_filters();
1766 if (hasFilters) {
1767 arr_inter<true>(queryInfo, outArray, constraints);
1768 } else {
1769 arr_inter<false>(queryInfo, outArray, constraints);
1770 }
1771 }
1773 } // namespace detail
1774 } // namespace ecs
1775} // namespace gaia
gaia::ecs::Iter::end_index
static GAIA_NODISCARD uint16_t end_index(Chunk *pChunk) noexcept
Returns the end of the enabled row range in a chunk.
Definition chunk_iterator.h:1989
gaia::ecs::Iter::start_index
static GAIA_NODISCARD uint16_t start_index(Chunk *pChunk) noexcept
Returns the first enabled row in a chunk.
Definition chunk_iterator.h:1982
gaia::ecs::detail::QueryImpl::chunk_effective_range
static void chunk_effective_range(Chunk *pChunk, Constraints constraints, bool needsBarrierCache, bool barrierPasses, uint16_t &from, uint16_t &to) noexcept
Calculates the row range of a chunk after applying row constraints and depth-order barrier state.
Definition query.h:1937
gaia::ecs::detail::QueryImpl::arr
void arr(Container &outArray, Constraints constraints=Constraints::EnabledOnly)
Appends all components or entities matching the query to the output array.
gaia::ecs::detail::QueryImpl::prepare_query_plan
GAIA_NODISCARD QueryPlan prepare_query_plan(const QueryInfo &queryInfo, Constraints constraints) const
Selects the prepared execution plan for public iterator callbacks.
Definition query.h:3649
gaia::ecs::detail::QueryImpl::needs_depth_order_hierarchy_barrier_cache
static GAIA_NODISCARD bool needs_depth_order_hierarchy_barrier_cache(const QueryInfo &queryInfo, Constraints constraints)
Checks whether the current row constraints require the depth-order hierarchy barrier cache.
Definition query.h:1926
gaia::ecs::detail::QueryImpl::ExecPayloadKind::Grouped
@ Grouped
Batches carry group ids but do not require sorted slices or inherited/barrier metadata.
gaia::ecs::detail::QueryImpl::ExecPayloadKind::NonTrivial
@ NonTrivial
Batches require non-trivial side payload such as sorted slices, inherited data, or barriers.
gaia::ecs::detail::QueryImpl::QueryPlanFlag_Sorted
@ QueryPlanFlag_Sorted
The plan may need sorted cache slices. Runners use them only with non-trivial payload.
Definition query.h:2158
gaia::ecs::detail::QueryImpl::QueryPlanFlag_Filtered
@ QueryPlanFlag_Filtered
The query has per-chunk filters such as changed terms.
Definition query.h:2150
gaia::ecs::detail::QueryImpl::QueryPlanFlag_InheritedPayload
@ QueryPlanFlag_InheritedPayload
The query carries inherited component data into iterator payloads.
Definition query.h:2154
gaia::ecs::detail::QueryImpl::QueryPlanFlag_BarrierCache
@ QueryPlanFlag_BarrierCache
The plan must use the depth-order hierarchy barrier cache when checking archetype/row ranges.
Definition query.h:2160
gaia::ecs::detail::QueryImpl::QueryPlanFlag_Grouped
@ QueryPlanFlag_Grouped
The query uses grouped payload/ranges or grouped cache ordering.
Definition query.h:2156
gaia::ecs::detail::QueryImpl::QueryPlanFlag_EntityFilter
@ QueryPlanFlag_EntityFilter
The query has entity-filter terms that require per-entity rechecks.
Definition query.h:2152
gaia::ecs::detail::QueryImpl::each_iter
void each_iter(Iter &it, Func func)
Runs a typed callback against an already prepared iterator. This is used by higher-level adapters tha...
gaia::ecs::detail::QueryImpl::exec_payload_kind
static GAIA_NODISCARD ExecPayloadKind exec_payload_kind(const QueryInfo &queryInfo, Constraints constraints)
Classifies the generic batch payload needed for a matched query under row constraints.
Definition query.h:2107
gaia::ecs::detail::QueryImpl::each
void each(Func func)
Iterates query matches using the default execution mode.
Definition query.h:6151
gaia::ecs::detail::QueryImpl::depth_order_hierarchy_barrier_prunes
static GAIA_NODISCARD bool depth_order_hierarchy_barrier_prunes(const QueryInfo &queryInfo)
Checks whether cached depth-order barrier results can prune any matched archetype.
Definition query.h:1953
gaia::ecs::detail::QueryImpl::has_depth_order_hierarchy_enabled_barrier
static GAIA_NODISCARD bool has_depth_order_hierarchy_enabled_barrier(const QueryInfo &queryInfo)
Checks whether depth-order grouping can prune disabled hierarchy subtrees.
Definition query.h:1915
gaia::ecs::detail::QueryImpl::match_all
void match_all(QueryInfo &queryInfo)
Matches the query against all relevant archetypes.
Definition query.h:953
gaia::ecs::detail::QueryImpl::ordered_entities_walk
GAIA_NODISCARD std::span< const Entity > ordered_entities_walk(QueryInfo &queryInfo, Entity relation, TravOrder order, Constraints constraints=Constraints::EnabledOnly)
Builds and caches relation traversal order for the current query result.
Definition query.h:6493
gaia::ecs::detail::QueryImpl::count
uint32_t count(Constraints constraints=Constraints::EnabledOnly)
Calculates the number of entities matching the query.
Definition query.h:6287
gaia::ecs::detail::QueryImpl::match_filters
static GAIA_NODISCARD bool match_filters(const Chunk &chunk, const QueryInfo &queryInfo, uint32_t changedWorldVersion, std::span< const uint8_t > compIndices)
Returns whether a chunk passes the query's changed filters.
Definition query.h:1799
gaia::ecs::detail::QueryImpl::each_walk
void each_walk(Func func, Entity relation, TravOrder order=TravOrder::Down, Constraints constraints=Constraints::EnabledOnly)
Iterates entities matching the query in a requested relation traversal order. For relation R this tre...
Definition query.h:6838
gaia::ecs::detail::QueryImpl::QueryPlanMode::Sorted
@ Sorted
Sorted payload execution that must preserve cache-provided chunk order.
gaia::ecs::detail::QueryImpl::QueryPlanMode::SparseDense
@ SparseDense
Typed cached chunk iteration with compile-time sparse payload access.
gaia::ecs::detail::QueryImpl::QueryPlanMode::DirectDense
@ DirectDense
Direct cached archetype/chunk iteration with query-term indices matching storage layout.
gaia::ecs::detail::QueryImpl::QueryPlanMode::EntitySeed
@ EntitySeed
Direct entity-seed evaluation over explicitly selected entities.
gaia::ecs::detail::QueryImpl::QueryPlanMode::MappedDense
@ MappedDense
Typed dense cached archetype/chunk iteration using mapped component access. Public Iter callbacks use...
gaia::ecs::detail::QueryImpl::QueryPlanMode::Empty
@ Empty
The selected group/range has no matching archetypes, so execution can return immediately.
gaia::ecs::detail::QueryImpl::QueryPlanMode::Traversal
@ Traversal
Traversal or inherited payload execution that requires the mapped generic path.
gaia::ecs::detail::QueryImpl::fetch
QueryInfo & fetch()
Fetches the QueryInfo object. Creates or refreshes the backing QueryInfo if needed.
Definition query.h:899
gaia::ecs::detail::QueryImpl::can_process_archetype_inter
GAIA_NODISCARD bool can_process_archetype_inter(const QueryInfo &queryInfo, const Archetype &archetype, Constraints constraints, int8_t barrierPasses=-1) const
Checks whether a matched archetype can be processed for the current row constraints.
Definition query.h:1993
gaia::ecs::detail::QueryImpl::each_direct_inter
void each_direct_inter(QueryInfo &queryInfo, Constraints constraints, void *pFunc, const TypedQueryExecState &state, void(*runDirectChunk)(QueryImpl &, Iter &, void *, const TypedQueryExecState &), bool needsInheritedArgIds, void(*invokeInherited)(World &, Entity, const Entity *, void *))
Runs an erased typed callback over entities selected by a direct sparse or target-term seed.
gaia::ecs::detail::QueryImpl::QueryPlan::payloadKind
ExecPayloadKind payloadKind
Payload layout required by generic chunk-batch runners independent of sorted-cache availability.
Definition query.h:2170