Gaia-ECS v1.0.0
A simple and powerful entity component system
Loading...
Searching...
No Matches
query_typed.inl
1#pragma once
2
3#include "gaia/ecs/query_adapter_typed.inl"
4
5namespace gaia {
6 namespace ecs {
7 namespace detail {
9 inline TypedQueryExecState build_typed_query_exec_state(
10 World& world, const QueryInfo& queryInfo, const TypedQueryArgMeta* pMetas, uint32_t argCount) {
11 TypedQueryExecState state{};
12 QueryImpl::DirectChunkArgEvalDesc directChunkDescs[MAX_ITEMS_IN_QUERY]{};
13 bool hasSparseArgs = false;
14 state.argCount = argCount;
15 GAIA_FOR(argCount) {
16#if GAIA_ASSERT_ENABLED
17 GAIA_ASSERT(ChunkIterImpl::allows_term_access(queryInfo.ctx().data, pMetas[i].termId, false));
18#endif
19 state.argIds[i] = pMetas[i].termId;
20 state.writeFlags[i] = pMetas[i].isWrite;
21 if (pMetas[i].isWrite) {
22 state.hasWriteArgs = true;
23 if (state.firstWriteArg == MAX_ITEMS_IN_QUERY)
24 state.firstWriteArg = (uint32_t)i;
25 }
26 state.needsInheritedArgIds = state.needsInheritedArgIds || !pMetas[i].isEntity;
27 hasSparseArgs = hasSparseArgs || pMetas[i].usesSparseStorage;
28 directChunkDescs[i] = {pMetas[i].termId, pMetas[i].isEntity, pMetas[i].isPair, pMetas[i].usesSparseStorage};
29 }
30
31 state.canUseDirectChunkEval =
32 QueryImpl::can_use_direct_chunk_term_eval_descs(world, queryInfo, directChunkDescs, argCount);
33 state.canUseSparseChunkEval = hasSparseArgs && QueryImpl::can_use_sparse_chunk_term_eval_descs(
34 world, queryInfo, directChunkDescs, argCount);
35 if (state.needsInheritedArgIds)
36 state.hasInheritedTerms = queryInfo.has_potential_inherited_id_terms();
37 return state;
38 }
39
40#if GAIA_ECS_TEST_HOOKS
41 template <typename Func>
42 inline QueryImpl::QueryPlan QueryImpl::test_typed_plan(Func func) {
43 auto& queryInfo = fetch();
44 match_all(queryInfo);
45
46 using InputArgs = decltype(core::func_args(&Func::operator()));
47 #if GAIA_ASSERT_ENABLED
48 GAIA_ASSERT(typed_query_args_match_query(queryInfo, InputArgs{}));
49 #endif
50 auto& world = *const_cast<World*>(queryInfo.world());
51 TypedQueryArgMeta metas[MAX_ITEMS_IN_QUERY]{};
52 const auto argCount = init_typed_query_arg_metas(metas, world, InputArgs{});
53 const auto state = build_typed_query_exec_state(world, queryInfo, metas, argCount);
54 (void)func;
55 return prepare_query_plan(queryInfo, state);
56 }
57
58 inline QueryImpl::QueryPlan QueryImpl::test_iter_plan(Constraints constraints) {
59 auto& queryInfo = fetch();
60 match_all(queryInfo);
61 return prepare_query_plan(queryInfo, constraints);
62 }
63#endif
64
65 inline void finish_typed_chunk_state(
66 World& world, Chunk* pChunk, uint16_t from, uint16_t to, const TypedQueryExecState& state);
67
68 inline void finish_typed_iter_state(QueryImpl& query, Iter& it, const TypedQueryExecState& state);
69
70 inline void noop_row_done(uint32_t) {}
71
72 template <typename Func, typename ViewsTuple, typename... T>
73 inline void invoke_typed_query_row_erased(
74 void* pFunc, Iter& it, ViewsTuple& dataPointerTuple, uint32_t row, bool directLocalIndex);
75
76 template <typename OnRowDone, typename... T>
77 inline void run_typed_chunk_views(
78 const QueryInfo* pQueryInfo, Iter& it, void* pFunc, bool directViews, OnRowDone&& onRowDone,
79 void (*invokeDirectRow)(
80 void*, Iter&, std::tuple<decltype(std::declval<Iter&>().template sview_auto<T>())...>&, uint32_t, bool),
81 void (*invokeMappedRow)(
82 void*, Iter&, std::tuple<decltype(std::declval<Iter&>().template view_auto_any<T>())...>&, uint32_t,
83 bool),
84 core::func_type_list<T...>);
85
86 template <typename Func, typename... T>
87 inline void run_typed_chunk_direct_iter_fast_cb(QueryImpl&, Iter& it, void* pFunc, const TypedQueryExecState&);
88
89 template <typename Func, typename... T>
90 inline void
91 run_typed_chunk_direct_iter_cb(QueryImpl& query, Iter& it, void* pFunc, const TypedQueryExecState& state);
92
93 template <typename Func, typename... T>
94 inline void run_typed_chunk_mapped_iter_cb(
95 QueryImpl& query, const QueryInfo& queryInfo, Iter& it, void* pFunc, const TypedQueryExecState& state);
96
97 template <typename Func, typename... T>
98 inline void run_typed_sparse_chunk_rows(
99 Chunk* pChunk, uint16_t from, uint16_t to, Func& func, const TypedQueryExecState& state,
100 core::func_type_list<T...> types);
101
102 template <typename Func, typename... T>
103 GAIA_NODISCARD inline auto typed_run_direct_fast_chunk_ptr(core::func_type_list<T...>) {
104 return &run_typed_chunk_direct_iter_fast_cb<Func, T...>;
105 }
106
107 template <typename Func, typename... T>
108 GAIA_NODISCARD inline auto typed_run_direct_chunk_ptr(core::func_type_list<T...>) {
109 return &run_typed_chunk_direct_iter_cb<Func, T...>;
110 }
111
112 template <typename Func, typename... T>
113 GAIA_NODISCARD inline auto typed_run_mapped_chunk_ptr(core::func_type_list<T...>) {
114 return &run_typed_chunk_mapped_iter_cb<Func, T...>;
115 }
116
117 template <typename Func, typename... T>
118 inline void run_typed_sparse_plan_cb(
119 QueryImpl& query, QueryInfo& queryInfo, const QueryImpl::QueryPlan& plan, void* pFunc,
120 const TypedQueryExecState& state) {
121 auto& func = *static_cast<Func*>(pFunc);
122 if (plan.mode == QueryImpl::QueryPlanMode::EntitySeed) {
123 query.run_query_on_sparse_entities_typed(queryInfo, state, func, core::func_type_list<T...>{});
124 return;
125 }
126
127 GAIA_ASSERT(plan.mode == QueryImpl::QueryPlanMode::SparseDense);
128 if ((plan.flags & QueryImpl::QueryPlanFlag_Filtered) != 0)
129 query.run_query_on_chunks_sparse_typed<true>(queryInfo, plan, state, func, core::func_type_list<T...>{});
130 else
131 query.run_query_on_chunks_sparse_typed<false>(queryInfo, plan, state, func, core::func_type_list<T...>{});
132 }
133
134 template <typename Func, typename... T>
135 GAIA_NODISCARD inline auto typed_run_sparse_plan_ptr(core::func_type_list<T...>) {
136 using RunSparsePlan =
137 void (*)(QueryImpl&, QueryInfo&, const QueryImpl::QueryPlan&, void*, const TypedQueryExecState&);
138 if constexpr (typed_query_args_use_sparse_storage_v<T...>)
139 return RunSparsePlan(&run_typed_sparse_plan_cb<Func, T...>);
140 else
141 return RunSparsePlan(nullptr);
142 }
143
144 template <typename Func, typename... T>
145 inline void run_typed_chunk_direct_iter_fast_cb(QueryImpl&, Iter& it, void* pFunc, const TypedQueryExecState&) {
146 run_typed_chunk_views(
147 nullptr, it, pFunc, true, noop_row_done,
148 &invoke_typed_query_row_erased<
149 Func, std::tuple<decltype(std::declval<Iter&>().template sview_auto<T>())...>, T...>,
150 &invoke_typed_query_row_erased<
151 Func, std::tuple<decltype(std::declval<Iter&>().template view_auto_any<T>())...>, T...>,
152 core::func_type_list<T...>{});
153 }
154
155 template <typename Func, typename... T>
156 inline void
157 run_typed_chunk_direct_iter_cb(QueryImpl& query, Iter& it, void* pFunc, const TypedQueryExecState& state) {
158 auto& func = *static_cast<Func*>(pFunc);
159 run_typed_chunk_direct_finish(query, it, func, state, core::func_type_list<T...>{});
160 }
161
162 template <typename Func, typename... T>
163 inline void run_typed_chunk_mapped_iter_cb(
164 QueryImpl& query, const QueryInfo& queryInfo, Iter& it, void* pFunc, const TypedQueryExecState& state) {
165 auto& func = *static_cast<Func*>(pFunc);
166 if constexpr (typed_query_args_use_sparse_storage_v<T...>) {
167 if (state.canUseSparseChunkEval) {
168 auto boundState = state;
169 auto& world = *const_cast<World*>(queryInfo.world());
170 bind_typed_sparse_stores(boundState, world, core::func_type_list<T...>{});
171 auto* pChunk = const_cast<Chunk*>(it.chunk());
172 run_typed_sparse_chunk_rows(
173 pChunk, it.row_begin(), it.row_end(), func, boundState, core::func_type_list<T...>{});
174 if (boundState.hasWriteArgs)
175 finish_typed_chunk_state(world, pChunk, it.row_begin(), it.row_end(), boundState);
176 return;
177 }
178 }
179 run_typed_chunk_mapped_finish(query, queryInfo, it, func, state, core::func_type_list<T...>{});
180 }
181
182 template <typename Func, typename... T>
183 inline void each_iter_dispatch(
184 const QueryInfo& queryInfo, Iter& it, Func& func, const TypedQueryExecState& state,
185 core::func_type_list<T...>) {
186 if (state.canUseDirectChunkEval) {
187 run_typed_chunk_views(
188 nullptr, it, &func, true, noop_row_done,
189 &invoke_typed_query_row_erased<
190 Func, std::tuple<decltype(std::declval<Iter&>().template sview_auto<T>())...>, T...>,
191 &invoke_typed_query_row_erased<
192 Func, std::tuple<decltype(std::declval<Iter&>().template view_auto_any<T>())...>, T...>,
193 core::func_type_list<T...>{});
194 finish_typed_chunk_state(*it.world(), const_cast<Chunk*>(it.chunk()), it.row_begin(), it.row_end(), state);
195 } else
196 run_typed_chunk_unmapped(queryInfo, it, func, state, core::func_type_list<T...>{});
197 }
198
199 template <typename Func, typename... T>
200 inline void run_typed_chunk_direct_finish(
201 QueryImpl& query, Iter& it, Func& func, const TypedQueryExecState& state, core::func_type_list<T...>) {
202 run_typed_chunk_views(
203 nullptr, it, &func, true, noop_row_done,
204 &invoke_typed_query_row_erased<
205 Func, std::tuple<decltype(std::declval<Iter&>().template sview_auto<T>())...>, T...>,
206 &invoke_typed_query_row_erased<
207 Func, std::tuple<decltype(std::declval<Iter&>().template view_auto_any<T>())...>, T...>,
208 core::func_type_list<T...>{});
209 finish_typed_iter_state(query, it, state);
210 }
211
212 template <typename Func, typename... T>
213 inline void run_typed_chunk_mapped_finish(
214 QueryImpl& query, const QueryInfo& queryInfo, Iter& it, Func& func, const TypedQueryExecState& state,
215 core::func_type_list<T...>) {
216 run_typed_chunk_views(
217 &queryInfo, it, &func, false, noop_row_done,
218 &invoke_typed_query_row_erased<
219 Func, std::tuple<decltype(std::declval<Iter&>().template sview_auto<T>())...>, T...>,
220 &invoke_typed_query_row_erased<
221 Func, std::tuple<decltype(std::declval<Iter&>().template view_auto_any<T>())...>, T...>,
222 core::func_type_list<T...>{});
223 finish_typed_iter_state(query, it, state);
224 }
225
226 template <typename Func, typename... T>
227 inline void run_typed_chunk_direct_walk_cb(
228 QueryImpl& query, const QueryInfo&, Iter& it, void* pFunc, const TypedQueryExecState& state) {
229 run_typed_chunk_direct_finish(query, it, *static_cast<Func*>(pFunc), state, core::func_type_list<T...>{});
230 }
231
232 template <typename Func, typename... T>
233 inline void run_typed_chunk_mapped_walk_cb(
234 QueryImpl& query, const QueryInfo& queryInfo, Iter& it, void* pFunc, const TypedQueryExecState& state) {
235 run_typed_chunk_mapped_finish(
236 query, queryInfo, it, *static_cast<Func*>(pFunc), state, core::func_type_list<T...>{});
237 }
238
239 template <typename Func, typename... T>
240 inline void each_walk_dispatch_direct(
241 QueryImpl& query, QueryInfo& queryInfo, std::span<const Entity> entities, Constraints constraints, Func& func,
242 const TypedQueryExecState& state, core::func_type_list<T...>) {
243 query.each_walk_inter(
244 queryInfo, entities, constraints, &func, state, &run_typed_chunk_direct_walk_cb<Func, T...>);
245 }
246
247 template <typename Func, typename... T>
248 inline void each_walk_dispatch_mapped(
249 QueryImpl& query, QueryInfo& queryInfo, std::span<const Entity> entities, Constraints constraints, Func& func,
250 const TypedQueryExecState& state, core::func_type_list<T...>) {
251 query.each_walk_inter(
252 queryInfo, entities, constraints, &func, state, &run_typed_chunk_mapped_walk_cb<Func, T...>);
253 }
254
255 template <typename Func, typename... T>
256 inline void run_typed_chunk_unmapped(
257 const QueryInfo& queryInfo, Iter& it, Func& func, const TypedQueryExecState& state,
258 core::func_type_list<T...> types) {
259 auto& world = *const_cast<World*>(queryInfo.world());
260 auto* pChunk = const_cast<Chunk*>(it.chunk());
261 const bool hasEntityFilters = queryInfo.has_entity_filter_terms();
262
263 if (!hasEntityFilters) {
264 run_typed_chunk_views(
265 &queryInfo, it, &func, false, noop_row_done,
266 &invoke_typed_query_row_erased<
267 Func, std::tuple<decltype(std::declval<Iter&>().template sview_auto<T>())...>, T...>,
268 &invoke_typed_query_row_erased<
269 Func, std::tuple<decltype(std::declval<Iter&>().template view_auto_any<T>())...>, T...>,
270 types);
271 finish_typed_chunk_state(world, pChunk, it.row_begin(), it.row_end(), state);
272 } else {
273 run_typed_chunk_views(
274 &queryInfo, it, &func, false,
275 [&](uint32_t row) {
276 finish_typed_chunk_state(
277 world, pChunk, (uint16_t)(it.row_begin() + row), (uint16_t)(it.row_begin() + row + 1), state);
278 },
279 &invoke_typed_query_row_erased<
280 Func, std::tuple<decltype(std::declval<Iter&>().template sview_auto<T>())...>, T...>,
281 &invoke_typed_query_row_erased<
282 Func, std::tuple<decltype(std::declval<Iter&>().template view_auto_any<T>())...>, T...>,
283 types);
284 }
285
286 it.clear_touched_writes();
287 }
288
289 template <typename Func, typename... T>
290 inline void run_typed_cached_seed_runs_basic(
291 QueryImpl& query, World& world, Constraints constraints, Func& func, std::span<const BfsChunkRun> runs,
292 const TypedQueryExecState& state) {
293 Iter it;
294 it.init_query_state(&world, constraints, false);
295 const Archetype* pLastArchetype = nullptr;
296 for (const auto& run: runs) {
297 if (run.pArchetype != pLastArchetype) {
298 it.set_archetype(run.pArchetype);
299 pLastArchetype = run.pArchetype;
300 }
301
302 it.set_chunk(run.pChunk, run.from, run.to);
303 it.set_group_id(0);
304 run_typed_chunk_direct_finish(query, it, func, state, core::func_type_list<T...>{});
305 }
306 }
307
308 template <typename Func, typename... T>
309 inline void run_typed_cached_seed_runs_entity_init(
310 QueryImpl& query, QueryInfo& queryInfo, World& world, Constraints constraints, Func& func,
311 std::span<const BfsChunkRun> runs, const TypedQueryExecState& state) {
312 Iter it;
313 it.init_query_state(&world, constraints, false);
314#if GAIA_ASSERT_ENABLED
315 it.set_query_access(&queryInfo.ctx().data);
316#endif
317 const Archetype* pLastArchetype = nullptr;
318 uint8_t indices[ChunkHeader::MAX_COMPONENTS];
319 Entity termIds[ChunkHeader::MAX_COMPONENTS];
320 for (const auto& run: runs) {
321 const auto& ec = ::gaia::ecs::fetch(world, run.pChunk->entity_view()[run.from]);
322 query.init_direct_entity_iter(queryInfo, world, ec, it, indices, termIds, pLastArchetype);
323 it.set_chunk(run.pChunk, run.from, run.to);
324 it.set_group_id(0);
325 run_typed_chunk_direct_finish(query, it, func, state, core::func_type_list<T...>{});
326 }
327 }
328
329 template <typename Func, typename... T>
330 inline void run_typed_cached_seed_runs(
331 QueryImpl& query, QueryInfo& queryInfo, World& world, Constraints constraints, Func& func,
332 std::span<const BfsChunkRun> runs, bool canUseBasicInit, const TypedQueryExecState& state) {
333 if (canUseBasicInit)
334 run_typed_cached_seed_runs_basic<Func, T...>(query, world, constraints, func, runs, state);
335 else
336 run_typed_cached_seed_runs_entity_init<Func, T...>(query, queryInfo, world, constraints, func, runs, state);
337 }
338
339 inline void QueryImpl::each_walk_inter(
340 QueryInfo& queryInfo, std::span<const Entity> entities, Constraints constraints, void* pFunc,
341 const TypedQueryExecState& state,
342 void (*runChunk)(QueryImpl&, const QueryInfo&, Iter&, void*, const TypedQueryExecState&)) {
343 auto& world = *queryInfo.world();
344 auto& walkData = ensure_each_walk_data();
345 Iter it;
346 it.init_query_state(&world, constraints, false);
347#if GAIA_ASSERT_ENABLED
348 it.set_query_access(&queryInfo.ctx().data);
349#endif
350 if (!walkData.cachedRuns.empty()) {
351 const auto& runs = walkData.cachedRuns;
352 const Archetype* pLastArchetype = nullptr;
353 for (const auto& run: runs) {
354 if (run.pArchetype != pLastArchetype) {
355 it.set_archetype(run.pArchetype);
356 pLastArchetype = run.pArchetype;
357 }
358
359 it.set_chunk(run.pChunk, run.from, run.to);
360 it.set_group_id(0);
361 runChunk(*this, queryInfo, it, pFunc, state);
362 }
363 return;
364 }
365
366 const Archetype* pLastArchetype = nullptr;
367 uint8_t indices[ChunkHeader::MAX_COMPONENTS];
368 Entity termIds[ChunkHeader::MAX_COMPONENTS];
369 for (const auto entity: entities) {
370 const auto& ec = ::gaia::ecs::fetch(world, entity);
371 init_direct_entity_iter(queryInfo, world, ec, it, indices, termIds, pLastArchetype);
372 runChunk(*this, queryInfo, it, pFunc, state);
373 }
374 }
375
376 inline void finish_typed_iter_state(QueryImpl& query, Iter& it, const TypedQueryExecState& state) {
377 query.finish_typed_iter_writes_runtime(it, state.argIds, state.writeFlags, state.argCount, state.firstWriteArg);
378 it.clear_touched_writes();
379 }
380
381 inline void finish_typed_chunk_state(
382 World& world, Chunk* pChunk, uint16_t from, uint16_t to, const TypedQueryExecState& state) {
383 QueryImpl::finish_typed_chunk_writes_runtime(
384 world, pChunk, from, to, state.argIds, state.writeFlags, state.argCount, state.firstWriteArg,
385 state.sparseStores);
386 }
387
388 template <typename InvokeRow, typename OnRowDone>
389 inline void run_typed_query_rows_runtime(
390 const QueryInfo* pQueryInfo, Iter& it, InvokeRow&& invokeRow, OnRowDone&& onRowDone) {
391 const auto cnt = it.size();
392 const bool hasEntityFilters = pQueryInfo != nullptr && pQueryInfo->has_entity_filter_terms();
393
394 if (!hasEntityFilters) {
395 GAIA_FOR(cnt) {
396 invokeRow((uint32_t)i);
397 onRowDone((uint32_t)i);
398 }
399 return;
400 }
401
402 const auto entities = it.template view<Entity>();
403 GAIA_FOR(cnt) {
404 if (!QueryImpl::match_entity_filters(*pQueryInfo->world(), entities[i], *pQueryInfo))
405 continue;
406 invokeRow((uint32_t)i);
407 onRowDone((uint32_t)i);
408 }
409 }
410
411 template <typename Func, typename ViewsTuple, typename... T>
412 inline void invoke_typed_query_row(
413 Iter& it, Func& func, ViewsTuple& dataPointerTuple, uint32_t row, bool directLocalIndex,
414 core::func_type_list<T...>) {
415 if constexpr (sizeof...(T) > 0) {
416 if (directLocalIndex) {
417 std::apply(
418 [&](auto&... views) {
419 func(views[row]...);
420 },
421 dataPointerTuple);
422 } else {
423 std::apply(
424 [&](auto&... views) {
425 func(views[it.template acc_index<T>(row)]...);
426 },
427 dataPointerTuple);
428 }
429 } else
430 func();
431 }
432
433 template <typename Func, typename ViewsTuple, typename... T>
434 inline void invoke_typed_query_row_erased(
435 void* pFunc, Iter& it, ViewsTuple& dataPointerTuple, uint32_t row, bool directLocalIndex) {
436 auto& func = *static_cast<Func*>(pFunc);
437 invoke_typed_query_row(it, func, dataPointerTuple, row, directLocalIndex, core::func_type_list<T...>{});
438 }
439
440 template <typename OnRowDone, typename ViewsTuple>
441 inline void run_typed_tuple_rows(
442 const QueryInfo* pQueryInfo, Iter& it, void* pFunc, ViewsTuple& dataPointerTuple, bool directLocalIndex,
443 void (*invokeRow)(void*, Iter&, ViewsTuple&, uint32_t, bool), OnRowDone&& onRowDone) {
444 run_typed_query_rows_runtime(
445 pQueryInfo, it,
446 [&](uint32_t row) {
447 invokeRow(pFunc, it, dataPointerTuple, row, directLocalIndex);
448 },
449 GAIA_FWD(onRowDone));
450 }
451
452 template <typename OnRowDone, typename... T>
453 inline void run_typed_chunk_views(
454 const QueryInfo* pQueryInfo, Iter& it, void* pFunc, bool directViews, OnRowDone&& onRowDone,
455 void (*invokeDirectRow)(
456 void*, Iter&, std::tuple<decltype(std::declval<Iter&>().template sview_auto<T>())...>&, uint32_t, bool),
457 void (*invokeMappedRow)(
458 void*, Iter&, std::tuple<decltype(std::declval<Iter&>().template view_auto_any<T>())...>&, uint32_t,
459 bool),
460 core::func_type_list<T...>) {
461 if constexpr (sizeof...(T) > 0) {
462 if (directViews) {
463 auto dataPointerTuple = std::make_tuple(it.template sview_auto<T>()...);
464 run_typed_tuple_rows(pQueryInfo, it, pFunc, dataPointerTuple, true, invokeDirectRow, GAIA_FWD(onRowDone));
465 } else {
466 auto dataPointerTuple = std::make_tuple(it.template view_auto_any<T>()...);
467 run_typed_tuple_rows(pQueryInfo, it, pFunc, dataPointerTuple, false, invokeMappedRow, GAIA_FWD(onRowDone));
468 }
469 } else {
470 auto dataPointerTuple = std::tuple<>{};
471 run_typed_tuple_rows(
472 pQueryInfo, it, pFunc, dataPointerTuple, directViews, directViews ? invokeDirectRow : invokeMappedRow,
473 GAIA_FWD(onRowDone));
474 }
475 }
476
477 template <typename ContainerOut>
478 inline void typed_arr_push(World& world, Entity entity, ContainerOut& outArray) {
479 using ContainerItemType = typename ContainerOut::value_type;
480 if constexpr (std::is_same_v<ContainerItemType, Entity>)
481 outArray.push_back(entity);
482 else {
483 auto tmp = world_direct_entity_arg<ContainerItemType>(world, entity);
484 outArray.push_back(tmp);
485 }
486 }
487
488 template <bool UseFilters, typename ContainerOut>
489 inline void run_typed_arr_rows(
490 QueryImpl& query, const QueryInfo& queryInfo, Iter& it, ContainerOut& outArray, uint32_t changedWorldVersion,
491 uint32_t archetypeIdx, const Archetype* pArchetype, Chunk* pChunk, uint16_t from, uint16_t to,
492 bool needsBarrierCache, bool canUseDirectChunkEval) {
493 using ContainerItemType = typename ContainerOut::value_type;
494 const bool barrierPasses = !needsBarrierCache || queryInfo.barrier_passes(archetypeIdx);
495 if GAIA_UNLIKELY (!query.can_process_archetype_inter(queryInfo, *pArchetype, it.constraints(), barrierPasses))
496 return;
497 if GAIA_UNLIKELY (from == to)
498 return;
499
500 GAIA_PROF_SCOPE(query::arr);
501
502 it.set_archetype(pArchetype);
503 it.set_chunk(pChunk, from, to);
504 it.set_group_id(0);
505
506 const auto cnt = it.size();
507 if (cnt == 0)
508 return;
509
510 if constexpr (UseFilters) {
511 if (!QueryImpl::match_filters(*pChunk, queryInfo, changedWorldVersion))
512 return;
513 }
514
515 const bool hasEntityFilters = queryInfo.has_entity_filter_terms();
516 if (canUseDirectChunkEval) {
517 const auto dataView = it.template sview_auto<ContainerItemType>();
518 GAIA_FOR(cnt) {
519 auto tmp = dataView[i];
520 outArray.push_back(tmp);
521 }
522 return;
523 }
524
525 const auto dataView = it.template view<ContainerItemType>();
526 if (!hasEntityFilters) {
527 GAIA_FOR(cnt) {
528 const auto idx = it.template acc_index<ContainerItemType>(i);
529 auto tmp = dataView[idx];
530 outArray.push_back(tmp);
531 }
532 return;
533 }
534
535 const auto entities = it.template view<Entity>();
536 GAIA_FOR(cnt) {
537 if (!QueryImpl::match_entity_filters(*queryInfo.world(), entities[i], queryInfo))
538 continue;
539 const auto idx = it.template acc_index<ContainerItemType>(i);
540 auto tmp = dataView[idx];
541 outArray.push_back(tmp);
542 }
543 }
544
545 inline void QueryImpl::finish_typed_chunk_writes_runtime(
546 World& world, Chunk* pChunk, uint16_t from, uint16_t to, const Entity* pArgIds, const bool* pWriteFlags,
547 uint32_t argCnt, uint32_t firstWriteArg, void* const* pSparseStores) {
548 if (firstWriteArg >= argCnt || from >= to)
549 return;
550
551 Entity seenTerms[ChunkHeader::MAX_COMPONENTS]{};
552 uint32_t seenCnt = 0;
553 const auto entities = pChunk->entity_view();
554 const auto finish_term = [&](Entity term, bool sparseStoreBound) {
555 GAIA_FOR(seenCnt) {
556 if (seenTerms[i] == term)
557 return;
558 }
559
560 seenTerms[seenCnt++] = term;
561 if (sparseStoreBound) {
562 for (uint16_t row = from; row < to; ++row)
563 world_notify_on_set_entity(world, term, entities[row]);
564 return;
565 }
566 if (!world_component_uses_sparse_storage(world, term)) {
567 const auto compIdx = core::get_index(pChunk->ids_view(), term);
568 if (compIdx != BadIndex) {
569 pChunk->finish_write(compIdx, from, to);
570 return;
571 }
572 }
573
574 for (uint16_t row = from; row < to; ++row)
575 world_finish_write(world, term, entities[row]);
576 };
577
578 for (uint32_t i = firstWriteArg; i < argCnt; ++i) {
579 if (!pWriteFlags[i])
580 continue;
581 const auto term = pArgIds[i];
582 if (term != EntityBad)
583 finish_term(term, pSparseStores != nullptr && pSparseStores[i] != nullptr);
584 }
585 }
586
587 template <typename... T>
588 inline void QueryImpl::finish_typed_chunk_writes(World& world, Chunk* pChunk, uint16_t from, uint16_t to) {
589 TypedQueryArgMeta metas[MAX_ITEMS_IN_QUERY]{};
590 const auto argCount = init_typed_query_arg_metas(metas, world, core::func_type_list<T...>{});
591 Entity argIds[MAX_ITEMS_IN_QUERY]{};
592 bool writeFlags[MAX_ITEMS_IN_QUERY]{};
593 uint32_t firstWriteArg = argCount;
594 GAIA_FOR(argCount) {
595 argIds[i] = metas[i].termId;
596 writeFlags[i] = metas[i].isWrite;
597 if (metas[i].isWrite && firstWriteArg == argCount)
598 firstWriteArg = (uint32_t)i;
599 }
600 finish_typed_chunk_writes_runtime(world, pChunk, from, to, argIds, writeFlags, argCount, firstWriteArg);
601 }
602
603 inline void QueryImpl::finish_typed_iter_writes_runtime(
604 Iter& it, const Entity* pArgIds, const bool* pWriteFlags, uint32_t argCnt, uint32_t firstWriteArg) {
605 if (firstWriteArg >= argCnt)
606 return;
607
608 auto* pChunk = const_cast<Chunk*>(it.chunk());
609 if (pChunk == nullptr || it.row_begin() >= it.row_end())
610 return;
611
612 auto& world = *it.world();
613 auto entities = it.entity_rows();
614 Entity seenTerms[ChunkHeader::MAX_COMPONENTS]{};
615 uint32_t seenCnt = 0;
616 const auto finish_term = [&](Entity term, uint32_t fieldIdx) {
617 GAIA_FOR(seenCnt) {
618 if (seenTerms[i] == term)
619 return;
620 }
621
622 seenTerms[seenCnt++] = term;
623 const bool usesSparseStorage = world_component_uses_sparse_storage(world, term);
624 auto compIdx = uint8_t(0xFF);
625 if (it.comp_indices() != nullptr && fieldIdx < ChunkHeader::MAX_COMPONENTS)
626 compIdx = it.comp_indices()[fieldIdx];
627 else if (!usesSparseStorage)
628 compIdx = (uint8_t)pChunk->comp_idx(term);
629
630 if (compIdx != 0xFF && !usesSparseStorage) {
631 pChunk->finish_write(compIdx, it.row_begin(), it.row_end());
632 return;
633 }
634
635 GAIA_FOR(entities.size()) {
636 world_finish_write(world, term, entities[i]);
637 }
638 };
639
640 for (uint32_t i = firstWriteArg; i < argCnt; ++i) {
641 if (!pWriteFlags[i])
642 continue;
643
644 Entity term = pArgIds[i];
645 if (it.term_ids() != nullptr && i < ChunkHeader::MAX_COMPONENTS && it.term_ids()[i] != EntityBad)
646 term = it.term_ids()[i];
647 if (term == EntityBad)
648 continue;
649
650 finish_term(term, i);
651 }
652 }
653
664 template <typename T, typename View>
665 GAIA_NODISCARD inline decltype(auto) typed_direct_chunk_arg_at(View& view, uint32_t row, uint16_t from) {
666 using U = typename actual_type_t<T>::Type;
667 if constexpr (mem::is_soa_layout_v<U>)
668 return view[from + row];
669 else
670 return view[row];
671 }
672
683 template <typename Func, typename ViewsTuple, typename... T, size_t... I>
684 inline void invoke_typed_direct_chunk_row(
685 Func& func, ViewsTuple& views, uint32_t row, uint16_t from, core::func_type_list<T...>,
686 std::index_sequence<I...>) {
687 func(typed_direct_chunk_arg_at<T>(std::get<I>(views), row, from)...);
688 }
689
692 struct TypedDirectChunkRun {
694 Chunk* pChunk = nullptr;
696 const uint8_t* pCompIndices = nullptr;
698 const void* const* pData = nullptr;
700 uint16_t from = 0;
702 uint16_t to = 0;
703 };
704
713 template <typename Func, typename T>
714 inline void run_typed_direct_chunk_row_arg(
715 const TypedDirectChunkRun& chunkRun, Func& func, const TypedQueryExecState& state, core::func_type_list<T>) {
716 Chunk* pChunk = chunkRun.pChunk;
717 const auto from = chunkRun.from;
718 const auto cnt = (uint32_t)(chunkRun.to - from);
719 using U = typename actual_type_t<T>::Type;
720 if constexpr (std::is_same_v<U, Entity>) {
721 const auto data = pChunk->entity_view();
722 GAIA_FOR(cnt)
723 func(data[from + i]);
724 } else if constexpr (!mem::is_soa_layout_v<U>) {
725 const auto* pData = chunkRun.pData != nullptr ? chunkRun.pData[0] : nullptr;
726 if (pData == nullptr)
727 pData = pChunk->comp_rec_view()[pChunk->comp_idx(state.argIds[0])].pData;
728 auto* data = (U*)pData + from;
729 GAIA_FOR(cnt)
730 func(data[i]);
731 } else {
732 auto view = pChunk->template sview_auto<T>(from, chunkRun.to);
733 GAIA_FOR(cnt)
734 func(view[from + i]);
735 }
736 }
737
738 template <typename T>
739 GAIA_NODISCARD constexpr bool typed_direct_chunk_data_cacheable_arg() {
740 using U = typename actual_type_t<T>::Type;
741 return !std::is_same_v<U, Entity> && !mem::is_soa_layout_v<U>;
742 }
743
744 template <typename... T>
745 GAIA_NODISCARD constexpr bool typed_direct_chunk_data_cacheable(core::func_type_list<T...>) {
746 if constexpr (sizeof...(T) == 0)
747 return false;
748 else
749 return (typed_direct_chunk_data_cacheable_arg<T>() && ...);
750 }
751
752 template <typename T>
753 GAIA_NODISCARD inline decltype(auto)
754 typed_direct_chunk_cached_arg(const void* const* pData, uint32_t argIdx, uint32_t row) {
755 using U = typename actual_type_t<T>::Type;
756 auto* data = (U*)pData[argIdx];
757 return data[row];
758 }
759
760 template <typename Func, typename... T, size_t... I>
761 inline void invoke_typed_direct_chunk_cached_row(
762 Func& func, const void* const* pData, uint32_t row, core::func_type_list<T...>, std::index_sequence<I...>) {
763 func(typed_direct_chunk_cached_arg<T>(pData, (uint32_t)I, row)...);
764 }
765
766 template <uint32_t ArgCount>
767 GAIA_NODISCARD inline bool typed_direct_chunk_cached_data_valid(const void* const* pData) {
768 if (pData == nullptr)
769 return false;
770 GAIA_FOR(ArgCount) {
771 if (pData[i] == nullptr)
772 return false;
773 }
774 return true;
775 }
776
787 template <typename Func, typename... T>
788 inline void run_typed_direct_chunk_rows(
789 const TypedDirectChunkRun& chunkRun, Func& func, const TypedQueryExecState& state,
790 core::func_type_list<T...>) {
791 Chunk* pChunk = chunkRun.pChunk;
792 const auto from = chunkRun.from;
793 const auto to = chunkRun.to;
794 if constexpr (sizeof...(T) == 1) {
795 run_typed_direct_chunk_row_arg(chunkRun, func, state, core::func_type_list<T...>{});
796 } else if constexpr (sizeof...(T) > 0) {
797 const auto cnt = (uint32_t)(to - from);
798 if constexpr (typed_direct_chunk_data_cacheable(core::func_type_list<T...>{})) {
799 if (typed_direct_chunk_cached_data_valid<(uint32_t)sizeof...(T)>(chunkRun.pData)) {
800 GAIA_FOR(cnt)
801 invoke_typed_direct_chunk_cached_row(
802 func, chunkRun.pData, (uint32_t)i, core::func_type_list<T...>{}, std::index_sequence_for<T...>{});
803 return;
804 }
805 }
806
807 auto views = std::make_tuple(pChunk->template sview_auto<T>(from, to)...);
808 GAIA_FOR(cnt)
809 invoke_typed_direct_chunk_row(
810 func, views, (uint32_t)i, from, core::func_type_list<T...>{}, std::index_sequence_for<T...>{});
811 } else {
812 const auto cnt = (uint32_t)(to - from);
813 GAIA_FOR(cnt)
814 func();
815 }
816 }
817
818 template <typename T>
819 struct TypedSparseQueryView {
820 using U = typename actual_type_t<T>::Type;
821 using UOriginal = typename actual_type_t<T>::TypeOriginal;
822 const Entity* pEntities = nullptr;
823 void* pStore = nullptr;
824
825 GAIA_NODISCARD decltype(auto) operator[](size_t idx) const {
826 if constexpr (core::is_mut_v<UOriginal>)
827 return world_typed_sparse_store_mut<U>(pStore, pEntities[idx]);
828 else
829 return world_typed_sparse_store_get<U>(pStore, pEntities[idx]);
830 }
831 };
832
833 template <typename T>
834 GAIA_NODISCARD inline auto typed_sparse_chunk_view(
835 Chunk* pChunk, uint16_t from, uint16_t to, const TypedQueryExecState& state, uint32_t argIdx) {
836 if constexpr (uses_ct_sparse_storage_v<T>)
837 return TypedSparseQueryView<T>{pChunk->entity_view().data() + from, state.sparseStores[argIdx]};
838 else
839 return pChunk->template sview_auto<T>(from, to);
840 }
841
842 template <typename T, typename View>
843 GAIA_NODISCARD inline decltype(auto) typed_sparse_chunk_arg_at(View& view, uint32_t row, uint16_t from) {
844 if constexpr (uses_ct_sparse_storage_v<T>)
845 return view[row];
846 else
847 return typed_direct_chunk_arg_at<T>(view, row, from);
848 }
849
850 template <typename Func, typename... T, size_t... I>
851 inline void run_typed_sparse_chunk_rows_impl(
852 Chunk* pChunk, uint16_t from, uint16_t to, Func& func, const TypedQueryExecState& state,
853 core::func_type_list<T...>, std::index_sequence<I...>) {
854 auto views = std::make_tuple(typed_sparse_chunk_view<T>(pChunk, from, to, state, (uint32_t)I)...);
855 const auto cnt = (uint32_t)(to - from);
856 GAIA_FOR(cnt)
857 func(typed_sparse_chunk_arg_at<T>(std::get<I>(views), (uint32_t)i, from)...);
858 }
859
860 template <typename Func, typename... T>
861 inline void run_typed_sparse_chunk_rows(
862 Chunk* pChunk, uint16_t from, uint16_t to, Func& func, const TypedQueryExecState& state,
863 core::func_type_list<T...> types) {
864 run_typed_sparse_chunk_rows_impl(pChunk, from, to, func, state, types, std::index_sequence_for<T...>{});
865 }
866
867 template <typename T>
868 GAIA_NODISCARD inline decltype(auto)
869 typed_sparse_entity_arg(World& world, Entity entity, const TypedQueryExecState& state, uint32_t argIdx) {
870 using U = typename actual_type_t<T>::Type;
871 using UOriginal = typename actual_type_t<T>::TypeOriginal;
872 if constexpr (std::is_same_v<U, Entity>)
873 return entity;
874 else if constexpr (uses_ct_sparse_storage_v<T>) {
875 if constexpr (core::is_mut_v<UOriginal>) {
876 if (auto* pValue = world_typed_sparse_store_try_mut<U>(state.sparseStores[argIdx], entity);
877 pValue != nullptr)
878 return *pValue;
879 } else {
880 if (const auto* pValue = world_typed_sparse_store_try_get<U>(state.sparseStores[argIdx], entity);
881 pValue != nullptr)
882 return *pValue;
883 }
884 }
885 if constexpr (core::is_mut_v<UOriginal>)
886 return world_query_entity_arg_by_id_raw<T>(world, entity, state.argIds[argIdx]);
887 else
888 return world_query_entity_arg_by_id<T>(world, entity, state.argIds[argIdx]);
889 }
890
891 template <typename Func, typename... T, size_t... I>
892 inline void invoke_typed_sparse_entity(
893 World& world, Entity entity, const TypedQueryExecState& state, Func& func, core::func_type_list<T...>,
894 std::index_sequence<I...>) {
895 func(typed_sparse_entity_arg<T>(world, entity, state, (uint32_t)I)...);
896 }
897
902 inline QueryImpl::QueryPlan
903 QueryImpl::prepare_query_plan(const QueryInfo& queryInfo, const TypedQueryExecState& state) const {
904 QueryPlan plan{};
905 plan.payloadKind = exec_payload_kind(queryInfo, Constraints::EnabledOnly);
906
907 const bool hasFilters = queryInfo.has_filters();
908 const bool hasSortedPayload = queryInfo.has_sorted_payload();
909 const bool hasDepthOrderBarrier = has_depth_order_hierarchy_enabled_barrier(queryInfo);
910 const bool depthOrderBarrierPrunes = hasDepthOrderBarrier && depth_order_hierarchy_barrier_prunes(queryInfo);
911 if (hasFilters)
912 plan.flags |= QueryPlanFlag_Filtered;
913 if (queryInfo.has_entity_filter_terms())
914 plan.flags |= QueryPlanFlag_EntityFilter;
915 if (queryInfo.has_inherited_data_payload())
916 plan.flags |= QueryPlanFlag_InheritedPayload;
917 if (queryInfo.has_grouped_payload())
918 plan.flags |= QueryPlanFlag_Grouped;
919 if (hasSortedPayload || hasDepthOrderBarrier)
920 plan.flags |= QueryPlanFlag_Sorted;
921 if (hasDepthOrderBarrier && !depthOrderBarrierPrunes)
922 plan.payloadKind = ExecPayloadKind::Grouped;
923 if (depthOrderBarrierPrunes)
924 plan.flags |= QueryPlanFlag_BarrierCache;
925
926 const bool canDirectEntitySeed = !hasFilters && can_use_direct_entity_seed_eval(queryInfo);
927 const bool canDirectChunks = !hasSortedPayload && (!hasDepthOrderBarrier || !depthOrderBarrierPrunes);
928
929 auto setDenseRange = [&]() -> bool {
930 const auto cacheRange = selected_query_cache_range(queryInfo);
931 plan.idxFrom = cacheRange.idxFrom;
932 plan.idxTo = cacheRange.idxTo;
933 if (cacheRange.hasSelectedGroup) {
934 plan.flags |= QueryPlanFlag_Grouped;
935 plan.payloadKind = ExecPayloadKind::Grouped;
936 if (!cacheRange.valid)
937 return false;
938 }
939 return plan.idxFrom < plan.idxTo;
940 };
941
942 if ((state.canUseDirectChunkEval || state.canUseSparseChunkEval) && !canDirectEntitySeed && canDirectChunks) {
943 if (!setDenseRange()) {
944 plan.mode = QueryPlanMode::Empty;
945 plan.idxFrom = 0;
946 plan.idxTo = 0;
947 return plan;
948 }
949
950 plan.mode = state.canUseSparseChunkEval ? QueryPlanMode::SparseDense : QueryPlanMode::DirectDense;
951 return plan;
952 }
953
954 if (canDirectEntitySeed) {
955 plan.mode = QueryPlanMode::EntitySeed;
956 return plan;
957 }
958
959 if (!setDenseRange()) {
960 plan.mode = QueryPlanMode::Empty;
961 plan.idxFrom = 0;
962 plan.idxTo = 0;
963 return plan;
964 }
965
966 if (hasSortedPayload) {
967 plan.mode = QueryPlanMode::Sorted;
968 plan.payloadKind = ExecPayloadKind::NonTrivial;
969 return plan;
970 }
971
972 if (hasDepthOrderBarrier && depthOrderBarrierPrunes) {
973 plan.mode = QueryPlanMode::Traversal;
974 plan.payloadKind = ExecPayloadKind::NonTrivial;
975 return plan;
976 }
977
978 if ((plan.flags & QueryPlanFlag_InheritedPayload) != 0) {
979 plan.mode = QueryPlanMode::Traversal;
980 plan.payloadKind = ExecPayloadKind::NonTrivial;
981 return plan;
982 }
983
984 if (hasDepthOrderBarrier) {
985 plan.payloadKind = ExecPayloadKind::Grouped;
986 return plan;
987 }
988
989 if (!canDirectChunks) {
990 plan.mode = QueryPlanMode::Sorted;
991 return plan;
992 }
993
994 if (!state.canUseDirectChunkEval) {
995 plan.mode = QueryPlanMode::MappedDense;
996 return plan;
997 }
998
999 return plan;
1000 }
1001
1002 template <bool HasFilters, typename Func, typename... T>
1003 inline void QueryImpl::run_query_on_chunks_sparse_typed(
1004 QueryInfo& queryInfo, const QueryPlan& plan, const TypedQueryExecState& state, Func& func,
1005 core::func_type_list<T...> types) {
1006 GAIA_ASSERT(plan.mode == QueryPlanMode::SparseDense);
1007 auto& world = *queryInfo.world();
1008 auto boundState = state;
1009 bind_typed_sparse_stores(boundState, world, types);
1010 const auto cacheView = queryInfo.cache_archetype_view();
1011 if (plan.idxFrom >= plan.idxTo)
1012 return;
1013
1014 if (boundState.hasWriteArgs)
1015 ::gaia::ecs::update_version(*m_worldVersion);
1016
1017 const bool canSkipProcessCheck =
1018 !queryInfo.result_cache_may_need_prefab_filter() && (plan.flags & QueryPlanFlag_BarrierCache) == 0;
1019 lock(*m_storage.world());
1020 for (uint32_t i = plan.idxFrom; i < plan.idxTo; ++i) {
1021 const auto* pArchetype = cacheView[i];
1022 if (canSkipProcessCheck) {
1023 if GAIA_UNLIKELY (pArchetype->is_req_del())
1024 continue;
1025 } else if GAIA_UNLIKELY (!can_process_archetype_inter(queryInfo, *pArchetype, Constraints::EnabledOnly))
1026 continue;
1027
1028 std::span<const uint8_t> indicesView;
1029 if constexpr (HasFilters)
1030 indicesView = queryInfo.indices_mapping_view(i);
1031
1032 for (auto* pChunk: pArchetype->chunks()) {
1033 const auto from = Iter::start_index(pChunk);
1034 const auto to = Iter::end_index(pChunk);
1035 if GAIA_UNLIKELY (from == to)
1036 continue;
1037 if constexpr (HasFilters) {
1038 if GAIA_UNLIKELY (!match_filters(*pChunk, queryInfo, m_changedWorldVersion, indicesView))
1039 continue;
1040 }
1041
1042 GAIA_PROF_SCOPE(query_func);
1043 run_typed_sparse_chunk_rows(pChunk, from, to, func, boundState, types);
1044 if (boundState.hasWriteArgs)
1045 finish_typed_chunk_state(world, pChunk, from, to, boundState);
1046 }
1047 }
1048
1049 unlock(*m_storage.world());
1050 commit_cmd_buffer_st(*m_storage.world());
1051 commit_cmd_buffer_mt(*m_storage.world());
1052 m_changedWorldVersion = *m_worldVersion;
1053 }
1054
1055 template <typename Func, typename... T>
1056 inline void QueryImpl::run_query_on_sparse_entities_typed(
1057 QueryInfo& queryInfo, const TypedQueryExecState& state, Func& func, core::func_type_list<T...> types) {
1058 auto& world = *queryInfo.world();
1059 auto boundState = state;
1060 bind_typed_sparse_stores(boundState, world, types);
1061 if (boundState.hasWriteArgs)
1062 ::gaia::ecs::update_version(*m_worldVersion);
1063
1064 auto execEntity = [&](Entity entity) {
1065 invoke_typed_sparse_entity(world, entity, boundState, func, types, std::index_sequence_for<T...>{});
1066 if (boundState.hasWriteArgs)
1067 finish_typed_query_args_by_id(world, entity, boundState);
1068 };
1069
1070 if (boundState.hasWriteArgs) {
1071 DirectQueryScratchScope scratchScope;
1072 auto& scratch = scratchScope.scratch;
1073 const auto seedInfo = build_direct_entity_seed(world, queryInfo, scratch);
1074 for (const auto entity: scratch.entities) {
1075 if (!match_direct_entity_constraints(world, queryInfo, entity, Constraints::EnabledOnly) ||
1076 !match_direct_entity_terms(world, entity, queryInfo, seedInfo))
1077 continue;
1078 execEntity(entity);
1079 }
1080 } else {
1081 const auto seedPlan = direct_entity_seed_plan(world, queryInfo);
1082 if (seedPlan.preferOrSeed) {
1083 each_direct_or_union(world, queryInfo, Constraints::EnabledOnly, [&](Entity entity) {
1084 execEntity(entity);
1085 return true;
1086 });
1087 } else {
1088 (void)each_direct_all_seed(world, queryInfo, seedPlan, Constraints::EnabledOnly, [&](Entity entity) {
1089 execEntity(entity);
1090 return true;
1091 });
1092 }
1093 }
1094
1095 finish_unlocked_each();
1096 }
1097
1108 template <bool HasFilters, typename Func, typename... T>
1109 inline void QueryImpl::run_query_on_chunks_direct_typed(
1110 QueryInfo& queryInfo, const QueryPlan& plan, const TypedQueryExecState& state, Func& func,
1111 core::func_type_list<T...> types) {
1112 auto& world = *queryInfo.world();
1113 if constexpr (HasFilters) {
1114 GAIA_ASSERT(plan.mode == QueryPlanMode::DirectDense);
1115 GAIA_ASSERT((plan.flags & QueryPlanFlag_Filtered) != 0);
1116 } else {
1117 GAIA_ASSERT(plan.mode == QueryPlanMode::DirectDense);
1118 }
1119
1120 auto cacheView = queryInfo.cache_archetype_view();
1121 if (plan.idxFrom >= plan.idxTo)
1122 return;
1123
1124 if (state.hasWriteArgs)
1125 ::gaia::ecs::update_version(*m_worldVersion);
1126
1127 const bool canSkipProcessCheck =
1128 !queryInfo.result_cache_may_need_prefab_filter() && (plan.flags & QueryPlanFlag_BarrierCache) == 0;
1129 lock(*m_storage.world());
1130 if constexpr (!HasFilters) {
1131 if (!state.hasWriteArgs && canSkipProcessCheck && plan.idxTo - plan.idxFrom >= 1024) {
1132 const auto terms = queryInfo.ctx().data.terms_view();
1133 uint32_t dataFields[MAX_ITEMS_IN_QUERY]{};
1134 uint32_t dataFieldCount = 0;
1135 if constexpr (typed_direct_chunk_data_cacheable(types)) {
1136 dataFieldCount = state.argCount;
1137 GAIA_FOR(state.argCount) {
1138 const auto argId = state.argIds[i];
1139 uint32_t termIdx = UINT32_MAX;
1140 for (uint32_t j = 0; j < terms.size(); ++j) {
1141 if (terms[j].id == argId) {
1142 termIdx = (uint32_t)j;
1143 break;
1144 }
1145 }
1146 if (termIdx == UINT32_MAX) {
1147 dataFieldCount = 0;
1148 break;
1149 }
1150 dataFields[i] = termIdx;
1151 }
1152 }
1153 const auto chunkView = queryInfo.direct_chunk_view(plan.idxFrom, plan.idxTo, dataFields, dataFieldCount);
1154 const auto dataView = queryInfo.direct_chunk_data_view();
1155 uint32_t chunkIdx = 0;
1156 for (const auto& entry: chunkView) {
1157 if constexpr (sizeof...(T) > 1 && typed_direct_chunk_data_cacheable(types)) {
1160 constexpr uint32_t PrefetchDistance = 8;
1161 const auto prefetchIdx = chunkIdx + PrefetchDistance;
1162 if (prefetchIdx < chunkView.size()) {
1163 const auto& prefetchEntry = chunkView[prefetchIdx];
1164 if (prefetchEntry.dataOffset != UINT32_MAX) {
1165 const auto* pPrefetchData = &dataView[prefetchEntry.dataOffset];
1166 GAIA_FOR_(sizeof...(T), dataIdx) {
1167 if (pPrefetchData[dataIdx] != nullptr)
1168 gaia::prefetch(pPrefetchData[dataIdx], PrefetchHint::PREFETCH_HINT_T1);
1169 }
1170 }
1171 }
1172 ++chunkIdx;
1173 }
1174 const void* pSingleData[1]{};
1175 const void* const* pData = nullptr;
1176 if (entry.pData != nullptr) {
1180 pSingleData[0] = entry.pData;
1181 pData = pSingleData;
1182 } else if (entry.dataOffset != UINT32_MAX)
1183 pData = &dataView[entry.dataOffset];
1184 const TypedDirectChunkRun chunkRun{entry.pChunk, entry.pCompIndices, pData, entry.rowFrom, entry.rowTo};
1185
1186 GAIA_PROF_SCOPE(query_func);
1187 run_typed_direct_chunk_rows(chunkRun, func, state, types);
1188 }
1189
1190 unlock(*m_storage.world());
1191 commit_cmd_buffer_st(*m_storage.world());
1192 commit_cmd_buffer_mt(*m_storage.world());
1193 m_changedWorldVersion = *m_worldVersion;
1194 return;
1195 }
1196 }
1197
1198 for (uint32_t i = plan.idxFrom; i < plan.idxTo; ++i) {
1199 const auto* pArchetype = cacheView[i];
1200 if (canSkipProcessCheck) {
1201 if GAIA_UNLIKELY (pArchetype->is_req_del())
1202 continue;
1203 } else if GAIA_UNLIKELY (!can_process_archetype_inter(queryInfo, *pArchetype, Constraints::EnabledOnly))
1204 continue;
1205
1206 std::span<const uint8_t> indicesView;
1207 if constexpr (HasFilters)
1208 indicesView = queryInfo.indices_mapping_view(i);
1209
1210 const auto& chunks = pArchetype->chunks();
1211 for (auto* pChunk: chunks) {
1212 const auto from = Iter::start_index(pChunk);
1213 const auto to = Iter::end_index(pChunk);
1214 if GAIA_UNLIKELY (from == to)
1215 continue;
1216
1217 if constexpr (HasFilters) {
1218 if GAIA_UNLIKELY (!match_filters(*pChunk, queryInfo, m_changedWorldVersion, indicesView))
1219 continue;
1220 }
1221
1222 GAIA_PROF_SCOPE(query_func);
1223 run_typed_direct_chunk_rows({pChunk, nullptr, nullptr, from, to}, func, state, types);
1224 if (state.hasWriteArgs)
1225 finish_typed_chunk_state(world, pChunk, from, to, state);
1226 }
1227 }
1228
1229 unlock(*m_storage.world());
1230 commit_cmd_buffer_st(*m_storage.world());
1231 commit_cmd_buffer_mt(*m_storage.world());
1232 m_changedWorldVersion = *m_worldVersion;
1233 }
1234
1235 inline void QueryImpl::run_query_on_chunks_direct_iter(
1236 QueryInfo& queryInfo, const QueryPlan& plan, const TypedQueryExecState& state, void* pFunc,
1237 void (*runChunk)(QueryImpl&, Iter& it, void*, const TypedQueryExecState&)) {
1238 auto& world = *queryInfo.world();
1239 GAIA_ASSERT(plan.mode == QueryPlanMode::DirectDense);
1240 GAIA_ASSERT(!queryInfo.has_filters());
1241 auto cacheView = queryInfo.cache_archetype_view();
1242 if (plan.idxFrom >= plan.idxTo)
1243 return;
1244
1245 if (state.hasWriteArgs)
1246 ::gaia::ecs::update_version(*m_worldVersion);
1247
1248 lock(*m_storage.world());
1249 Iter it;
1250 it.init_query_state(queryInfo.world(), Constraints::EnabledOnly, false);
1251#if GAIA_ASSERT_ENABLED
1252 it.set_query_access(&queryInfo.ctx().data);
1253#endif
1254 const Archetype* pLastArchetype = nullptr;
1255 for (uint32_t i = plan.idxFrom; i < plan.idxTo; ++i) {
1256 const auto* pArchetype = cacheView[i];
1257 if GAIA_UNLIKELY (!can_process_archetype_inter(queryInfo, *pArchetype, Constraints::EnabledOnly))
1258 continue;
1259
1260 const auto& chunks = pArchetype->chunks();
1261 for (auto* pChunk: chunks) {
1262 const auto from = Iter::start_index(pChunk);
1263 const auto to = Iter::end_index(pChunk);
1264 if GAIA_UNLIKELY (from == to)
1265 continue;
1266
1267 GAIA_PROF_SCOPE(query_func);
1268 if (pArchetype != pLastArchetype) {
1269 it.set_archetype(pArchetype);
1270 pLastArchetype = pArchetype;
1271 }
1272 it.set_chunk(pChunk, from, to);
1273 it.set_group_id(0);
1274 it.ctx(m_ctx);
1275 runChunk(*this, it, pFunc, state);
1276 finish_typed_chunk_state(world, pChunk, from, to, state);
1277 }
1278 }
1279
1280 unlock(*m_storage.world());
1281 commit_cmd_buffer_st(*m_storage.world());
1282 commit_cmd_buffer_mt(*m_storage.world());
1283 m_changedWorldVersion = *m_worldVersion;
1284 }
1285
1286 inline void QueryImpl::run_query_on_chunks_direct(
1287 QueryInfo& queryInfo, const QueryPlan& plan, const TypedQueryExecState& state, void* pFunc,
1288 void (*runChunk)(QueryImpl&, Iter& it, void*, const TypedQueryExecState&)) {
1289 auto& world = *queryInfo.world();
1290 auto cacheView = queryInfo.cache_archetype_view();
1291 if (plan.idxFrom >= plan.idxTo)
1292 return;
1293
1294 if (state.hasWriteArgs)
1295 ::gaia::ecs::update_version(*m_worldVersion);
1296
1297 GAIA_ASSERT(plan.mode == QueryPlanMode::DirectDense);
1298 const bool hasFilters = (plan.flags & QueryPlanFlag_Filtered) != 0;
1299
1300 lock(*m_storage.world());
1301 Iter it;
1302 it.init_query_state(queryInfo.world(), Constraints::EnabledOnly, false);
1303#if GAIA_ASSERT_ENABLED
1304 it.set_query_access(&queryInfo.ctx().data);
1305#endif
1306 const Archetype* pLastArchetype = nullptr;
1307
1308 for (uint32_t i = plan.idxFrom; i < plan.idxTo; ++i) {
1309 const auto* pArchetype = cacheView[i];
1310 if GAIA_UNLIKELY (!can_process_archetype_inter(queryInfo, *pArchetype, Constraints::EnabledOnly))
1311 continue;
1312
1313 std::span<const uint8_t> indicesView;
1314 if (hasFilters)
1315 indicesView = queryInfo.indices_mapping_view(i);
1316
1317 const auto& chunks = pArchetype->chunks();
1318 for (auto* pChunk: chunks) {
1319 const auto from = Iter::start_index(pChunk);
1320 const auto to = Iter::end_index(pChunk);
1321 if GAIA_UNLIKELY (from == to)
1322 continue;
1323
1324 if (hasFilters) {
1325 if GAIA_UNLIKELY (!match_filters(*pChunk, queryInfo, m_changedWorldVersion, indicesView))
1326 continue;
1327 }
1328
1329 GAIA_PROF_SCOPE(query_func);
1330 if (pArchetype != pLastArchetype) {
1331 it.set_archetype(pArchetype);
1332 pLastArchetype = pArchetype;
1333 }
1334 it.set_chunk(pChunk, from, to);
1335 it.set_group_id(0);
1336 it.ctx(m_ctx);
1337 runChunk(*this, it, pFunc, state);
1338 finish_typed_chunk_state(world, pChunk, from, to, state);
1339 }
1340 }
1341
1342 unlock(*m_storage.world());
1343 commit_cmd_buffer_st(*m_storage.world());
1344 commit_cmd_buffer_mt(*m_storage.world());
1345 m_changedWorldVersion = *m_worldVersion;
1346 }
1347
1348 template <QueryExecType ExecType>
1349 inline void QueryImpl::each_inter(
1350 QueryInfo& queryInfo, const QueryPlan& plan, void* pFunc, const TypedQueryExecState& state,
1351 const TypedQueryErasedOps& ops) {
1352 if (plan.mode == QueryPlanMode::Empty)
1353 return;
1354
1355 if (plan.mode == QueryPlanMode::EntitySeed) {
1356 GAIA_PROF_SCOPE(query_func);
1357 each_direct_inter(
1358 queryInfo, Constraints::EnabledOnly, pFunc, state, ops.runDirectChunk, ops.needsInheritedArgIds,
1359 ops.invokeInherited);
1360 return;
1361 }
1362
1363 if (state.canUseDirectChunkEval) {
1364 if constexpr (ExecType == QueryExecType::Default) {
1365 if (plan.mode == QueryPlanMode::DirectDense) {
1366 run_query_on_chunks_direct(queryInfo, plan, state, pFunc, ops.runDirectFastChunk);
1367 return;
1368 }
1369 }
1370 TypedDirectChunkCallback cb{this, pFunc, &state, ops.runDirectChunk};
1371 run_query_on_chunks<ExecType, IterModeEnabled>(queryInfo, cb);
1372 } else {
1373 TypedMappedChunkCallback cb{this, &queryInfo, pFunc, &state, ops.runMappedChunk};
1374 run_query_on_chunks<ExecType, IterModeEnabled>(queryInfo, cb);
1375 }
1376 }
1377
1378#if GAIA_ASSERT_ENABLED
1379 template <typename Func>
1380 inline void QueryImpl::validate_typed_access() {
1381 using InputArgs = decltype(core::func_args(&Func::operator()));
1382
1383 auto& queryInfo = fetch();
1384 TypedQueryArgMeta metas[MAX_ITEMS_IN_QUERY]{};
1385 const auto argCount = init_typed_query_arg_metas(metas, *queryInfo.world(), InputArgs{});
1386 GAIA_FOR(argCount) {
1387 GAIA_ASSERT(ChunkIterImpl::allows_term_access(queryInfo.ctx().data, metas[i].termId, false));
1388 }
1389 }
1390#endif
1391
1392 template <QueryExecType ExecType, typename Func>
1393 inline void QueryImpl::each_typed_inter(QueryInfo& queryInfo, Func func) {
1394 using InputArgs = decltype(core::func_args(&Func::operator()));
1395
1396#if GAIA_ASSERT_ENABLED
1397 GAIA_ASSERT(typed_query_args_match_query(queryInfo, InputArgs{}));
1398#endif
1399 auto& world = *const_cast<World*>(queryInfo.world());
1400 TypedQueryArgMeta metas[MAX_ITEMS_IN_QUERY]{};
1401 const auto argCount = init_typed_query_arg_metas(metas, world, InputArgs{});
1402 const auto state = build_typed_query_exec_state(world, queryInfo, metas, argCount);
1403 const auto plan = prepare_query_plan(queryInfo, state);
1404 if constexpr (ExecType == QueryExecType::Default) {
1405 if constexpr (typed_query_arg_list_uses_sparse_storage_v<InputArgs>) {
1406 if (plan.mode == QueryPlanMode::EntitySeed) {
1407 run_query_on_sparse_entities_typed(queryInfo, state, func, InputArgs{});
1408 return;
1409 }
1410 if (plan.mode == QueryPlanMode::SparseDense) {
1411 if ((plan.flags & QueryPlanFlag_Filtered) != 0)
1412 run_query_on_chunks_sparse_typed<true>(queryInfo, plan, state, func, InputArgs{});
1413 else
1414 run_query_on_chunks_sparse_typed<false>(queryInfo, plan, state, func, InputArgs{});
1415 return;
1416 }
1417 }
1418 if (plan.mode == QueryPlanMode::DirectDense) {
1419 if ((plan.flags & QueryPlanFlag_Filtered) != 0)
1420 run_query_on_chunks_direct_typed<true>(queryInfo, plan, state, func, InputArgs{});
1421 else
1422 run_query_on_chunks_direct_typed<false>(queryInfo, plan, state, func, InputArgs{});
1423 return;
1424 }
1425 }
1426
1427 TypedQueryErasedOps ops;
1428 ops.runDirectFastChunk = typed_run_direct_fast_chunk_ptr<Func>(InputArgs{});
1429 ops.runDirectChunk = typed_run_direct_chunk_ptr<Func>(InputArgs{});
1430 ops.runMappedChunk = typed_run_mapped_chunk_ptr<Func>(InputArgs{});
1431 ops.invokeInherited = typed_invoke_inherited_ptr<Func>(InputArgs{});
1432 ops.needsInheritedArgIds = state.needsInheritedArgIds;
1433 each_inter<ExecType>(queryInfo, plan, &func, state, ops);
1434 }
1435
1436 inline void QueryImpl::each_typed_erased(
1437 QueryExecType execType, void* pFunc, const TypedQueryExecState& state, const TypedQueryErasedOps& ops) {
1438 auto& queryInfo = fetch();
1439 match_all(queryInfo);
1440 const auto plan = prepare_query_plan(queryInfo, state);
1441 if (execType == QueryExecType::Default && ops.runSparsePlan != nullptr &&
1442 (plan.mode == QueryPlanMode::SparseDense || plan.mode == QueryPlanMode::EntitySeed)) {
1443 ops.runSparsePlan(*this, queryInfo, plan, pFunc, state);
1444 return;
1445 }
1446
1447 switch (execType) {
1448 case QueryExecType::Parallel:
1449 each_inter<QueryExecType::Parallel>(queryInfo, plan, pFunc, state, ops);
1450 break;
1451 case QueryExecType::ParallelPerf:
1452 each_inter<QueryExecType::ParallelPerf>(queryInfo, plan, pFunc, state, ops);
1453 break;
1454 case QueryExecType::ParallelEff:
1455 each_inter<QueryExecType::ParallelEff>(queryInfo, plan, pFunc, state, ops);
1456 break;
1457 default:
1458 each_inter<QueryExecType::Default>(queryInfo, plan, pFunc, state, ops);
1459 break;
1460 }
1461 }
1462
1463 template <typename Func, std::enable_if_t<!detail::is_query_iter_callback_v<Func>, int>>
1464 inline void QueryImpl::each(Func func) {
1465 each(func, QueryExecType::Default);
1466 }
1467
1468 template <typename Func, std::enable_if_t<!detail::is_query_iter_callback_v<Func>, int>>
1469 inline void QueryImpl::each(Func func, QueryExecType execType) {
1470 auto& queryInfo = fetch();
1471 match_all(queryInfo);
1472
1473 switch (execType) {
1474 case QueryExecType::Parallel:
1475 each_typed_inter<QueryExecType::Parallel>(queryInfo, func);
1476 break;
1477 case QueryExecType::ParallelPerf:
1478 each_typed_inter<QueryExecType::ParallelPerf>(queryInfo, func);
1479 break;
1480 case QueryExecType::ParallelEff:
1481 each_typed_inter<QueryExecType::ParallelEff>(queryInfo, func);
1482 break;
1483 default:
1484 each_typed_inter<QueryExecType::Default>(queryInfo, func);
1485 break;
1486 }
1487 }
1488
1489 template <QueryExecType ExecType>
1490 inline void QueryImpl::each_iter_inter_erased(
1491 QueryInfo& queryInfo, const QueryPlan& plan, void* pFunc, const TypedQueryExecState& state,
1492 void (*runDirectFastChunk)(QueryImpl&, Iter&, void*, const TypedQueryExecState&),
1493 void (*runMappedChunk)(QueryImpl&, const QueryInfo&, Iter&, void*, const TypedQueryExecState&)) {
1494 TypedIterErasedCallback cb{this, pFunc, &state, runDirectFastChunk, runMappedChunk};
1495
1496 if (plan.mode == QueryPlanMode::Empty)
1497 return;
1498
1499 if (plan.mode == QueryPlanMode::EntitySeed) {
1500 each_direct_iter_inter(queryInfo, Constraints::EnabledOnly, cb);
1501 return;
1502 }
1503
1504 if constexpr (ExecType == QueryExecType::Default) {
1505 if (plan.mode == QueryPlanMode::DirectDense) {
1506 run_query_on_chunks_direct_iter(queryInfo, plan, state, pFunc, runDirectFastChunk);
1507 return;
1508 }
1509 }
1510
1511 run_query_on_chunks<ExecType, IterModeEnabled>(queryInfo, cb);
1512 }
1513
1514 inline void QueryImpl::each_iter_erased(
1515 QueryExecType execType, void* pFunc, const TypedQueryExecState& state,
1516 void (*runDirectFastChunk)(QueryImpl&, Iter&, void*, const TypedQueryExecState&),
1517 void (*runMappedChunk)(QueryImpl&, const QueryInfo&, Iter&, void*, const TypedQueryExecState&)) {
1518 auto& queryInfo = fetch();
1519 match_all(queryInfo);
1520 const auto plan = prepare_query_plan(queryInfo, state);
1521
1522 switch (execType) {
1523 case QueryExecType::Parallel:
1524 each_iter_inter_erased<QueryExecType::Parallel>(
1525 queryInfo, plan, pFunc, state, runDirectFastChunk, runMappedChunk);
1526 break;
1527 case QueryExecType::ParallelPerf:
1528 each_iter_inter_erased<QueryExecType::ParallelPerf>(
1529 queryInfo, plan, pFunc, state, runDirectFastChunk, runMappedChunk);
1530 break;
1531 case QueryExecType::ParallelEff:
1532 each_iter_inter_erased<QueryExecType::ParallelEff>(
1533 queryInfo, plan, pFunc, state, runDirectFastChunk, runMappedChunk);
1534 break;
1535 default:
1536 each_iter_inter_erased<QueryExecType::Default>(
1537 queryInfo, plan, pFunc, state, runDirectFastChunk, runMappedChunk);
1538 break;
1539 }
1540 }
1541
1542 template <typename Func>
1543 inline void QueryImpl::each_iter(Iter& it, Func func) {
1544 using InputArgs = decltype(core::func_args(&Func::operator()));
1545 auto& queryInfo = fetch();
1546
1547#if GAIA_ASSERT_ENABLED
1548 GAIA_ASSERT(typed_query_args_match_query(queryInfo, InputArgs{}));
1549#endif
1550 TypedQueryArgMeta metas[MAX_ITEMS_IN_QUERY]{};
1551 const auto argCount = init_typed_query_arg_metas(metas, *it.world(), InputArgs{});
1552 const auto state = build_typed_query_exec_state(*it.world(), queryInfo, metas, argCount);
1553 it.ctx(m_ctx);
1554 each_iter_dispatch(queryInfo, it, func, state, InputArgs{});
1555 }
1556
1557 inline void QueryImpl::each_iter_erased(
1558 Iter& it, void* pFunc, const TypedQueryExecState& state,
1559 void (*runDirectFastChunk)(QueryImpl&, Iter&, void*, const TypedQueryExecState&),
1560 void (*runMappedChunk)(QueryImpl&, const QueryInfo&, Iter&, void*, const TypedQueryExecState&)) {
1561 auto& queryInfo = fetch();
1562 it.ctx(m_ctx);
1563 if (state.canUseDirectChunkEval) {
1564 runDirectFastChunk(*this, it, pFunc, state);
1565 finish_typed_chunk_state(*it.world(), const_cast<Chunk*>(it.chunk()), it.row_begin(), it.row_end(), state);
1566 } else
1567 runMappedChunk(*this, queryInfo, it, pFunc, state);
1568 }
1569
1570 inline void QueryImpl::each_direct_inter(
1571 QueryInfo& queryInfo, Constraints constraints, void* pFunc, const TypedQueryExecState& state,
1572 void (*runDirectChunk)(QueryImpl&, Iter&, void*, const TypedQueryExecState&), bool needsInheritedArgIds,
1573 void (*invokeInherited)(World&, Entity, const Entity*, void*)) {
1574 auto& world = *queryInfo.world();
1575 const auto plan = direct_entity_seed_plan(world, queryInfo);
1576 const bool hasWriteTerms = queryInfo.ctx().data.readWriteMask != 0;
1577
1578 auto exec_direct_entity = [&](Entity entity) {
1579 uint8_t indices[ChunkHeader::MAX_COMPONENTS];
1580 Entity termIds[ChunkHeader::MAX_COMPONENTS];
1581 Iter it;
1582 it.set_constraints(constraints);
1583 init_direct_entity_iter(queryInfo, world, entity, it, indices, termIds);
1584 it.ctx(m_ctx);
1585 runDirectChunk(*this, it, pFunc, state);
1586 };
1587
1588 auto exec_entity = [&](Entity entity) {
1589 if (needsInheritedArgIds && state.hasInheritedTerms) {
1590 invokeInherited(world, entity, state.argIds, pFunc);
1591 finish_typed_query_args_by_id(world, entity, state);
1592 return;
1593 }
1594
1595 exec_direct_entity(entity);
1596 };
1597
1598 if (!hasWriteTerms && !plan.preferOrSeed) {
1599 const auto* pSeedTerm = find_direct_all_seed_term(queryInfo, plan);
1600 if (pSeedTerm != nullptr && can_use_direct_seed_run_cache(world, queryInfo, *pSeedTerm)) {
1601 DirectEntitySeedInfo seedInfo{};
1602 seedInfo.seededAllTerm = pSeedTerm->id;
1603 seedInfo.seededAllMatchKind = pSeedTerm->matchKind;
1604 seedInfo.seededFromAll = true;
1605 if (!state.hasInheritedTerms) {
1606 const auto runs = cached_direct_seed_runs(queryInfo, *pSeedTerm, seedInfo, constraints);
1607 if (state.canUseDirectChunkEval) {
1608 Iter it;
1609 it.init_query_state(&world, constraints, false);
1610#if GAIA_ASSERT_ENABLED
1611 it.set_query_access(&queryInfo.ctx().data);
1612#endif
1613 const Archetype* pLastArchetype = nullptr;
1614 for (const auto& run: runs) {
1615 if (run.pArchetype != pLastArchetype) {
1616 it.set_archetype(run.pArchetype);
1617 pLastArchetype = run.pArchetype;
1618 }
1619 it.set_chunk(run.pChunk, run.from, run.to);
1620 it.set_group_id(0);
1621 it.ctx(m_ctx);
1622 runDirectChunk(*this, it, pFunc, state);
1623 }
1624 } else {
1625 Iter it;
1626 it.init_query_state(&world, constraints, false);
1627#if GAIA_ASSERT_ENABLED
1628 it.set_query_access(&queryInfo.ctx().data);
1629#endif
1630 const Archetype* pLastArchetype = nullptr;
1631 uint8_t indices[ChunkHeader::MAX_COMPONENTS];
1632 Entity termIds[ChunkHeader::MAX_COMPONENTS];
1633 for (const auto& run: runs) {
1634 const auto& ec = ::gaia::ecs::fetch(world, run.pChunk->entity_view()[run.from]);
1635 init_direct_entity_iter(queryInfo, world, ec, it, indices, termIds, pLastArchetype);
1636 it.set_chunk(run.pChunk, run.from, run.to);
1637 it.set_group_id(0);
1638 it.ctx(m_ctx);
1639 runDirectChunk(*this, it, pFunc, state);
1640 }
1641 }
1642 } else {
1643 const auto entities = cached_direct_seed_chunk_entities(queryInfo, *pSeedTerm, seedInfo, constraints);
1644 for (const auto entity: entities)
1645 exec_entity(entity);
1646 }
1647 finish_unlocked_each();
1648 return;
1649 }
1650 }
1651
1652 auto walk_entities = [&](auto&& execEntity) {
1653 if (hasWriteTerms) {
1654 DirectQueryScratchScope scratchScope;
1655 auto& scratch = scratchScope.scratch;
1656 const auto seedInfo = build_direct_entity_seed(world, queryInfo, scratch);
1657 for (const auto entity: scratch.entities) {
1658 if (!match_direct_entity_constraints(world, queryInfo, entity, constraints))
1659 continue;
1660 if (!match_direct_entity_terms(world, entity, queryInfo, seedInfo))
1661 continue;
1662 execEntity(entity);
1663 }
1664 return;
1665 }
1666
1667 if (plan.preferOrSeed) {
1668 each_direct_or_union(world, queryInfo, constraints, [&](Entity entity) {
1669 execEntity(entity);
1670 return true;
1671 });
1672 return;
1673 }
1674
1675 (void)each_direct_all_seed(world, queryInfo, plan, constraints, [&](Entity entity) {
1676 execEntity(entity);
1677 return true;
1678 });
1679 };
1680
1681 walk_entities(exec_entity);
1682 finish_unlocked_each();
1683 }
1684
1685 template <typename Func, std::enable_if_t<!detail::is_query_walk_core_callback_v<Func>, int>>
1686 inline void QueryImpl::each_walk(Func func, Entity relation, TravOrder order, Constraints constraints) {
1687 auto& queryInfo = fetch();
1688 match_all(queryInfo);
1689 const auto ordered = ordered_entities_walk(queryInfo, relation, order, constraints);
1690
1691 using InputArgs = decltype(core::func_args(&Func::operator()));
1692 GAIA_ASSERT(typed_query_args_match_query(queryInfo, InputArgs{}));
1693 GAIA_ASSERT(can_use_direct_target_eval(queryInfo));
1694 if (!can_use_direct_target_eval(queryInfo))
1695 return;
1696
1697 auto& world = *queryInfo.world();
1698 TypedQueryArgMeta metas[MAX_ITEMS_IN_QUERY]{};
1699 const auto argCount = init_typed_query_arg_metas(metas, world, InputArgs{});
1700 const auto state = build_typed_query_exec_state(world, queryInfo, metas, argCount);
1701 if (state.canUseDirectChunkEval) {
1702 each_walk_dispatch_direct(*this, queryInfo, ordered, constraints, func, state, InputArgs{});
1703 } else {
1704 each_walk_dispatch_mapped(*this, queryInfo, ordered, constraints, func, state, InputArgs{});
1705 }
1706 }
1707
1708 template <bool UseFilters, typename ContainerOut>
1709 inline void QueryImpl::arr_inter(QueryInfo& queryInfo, ContainerOut& outArray, Constraints constraints) {
1710 using ContainerItemType = typename ContainerOut::value_type;
1711 if constexpr (!UseFilters) {
1712 if (can_use_direct_entity_seed_eval(queryInfo)) {
1713 auto& world = *queryInfo.world();
1714 const auto plan = direct_entity_seed_plan(world, queryInfo);
1715 if (plan.preferOrSeed) {
1716 each_direct_or_union(world, queryInfo, constraints, [&](Entity entity) {
1717 typed_arr_push(world, entity, outArray);
1718 });
1719 return;
1720 }
1721
1722 (void)each_direct_all_seed(world, queryInfo, plan, constraints, [&](Entity entity) {
1723 typed_arr_push(world, entity, outArray);
1724 return true;
1725 });
1726
1727 return;
1728 }
1729 }
1730
1731 auto& world = *queryInfo.world();
1732 const auto meta = typed_query_arg_meta<ContainerItemType>(world);
1733 const DirectChunkArgEvalDesc desc{meta.termId, meta.isEntity, meta.isPair, meta.usesSparseStorage};
1734 Iter it;
1735 it.init_query_state(queryInfo.world(), constraints, false);
1736#if GAIA_ASSERT_ENABLED
1737 it.set_query_access(&queryInfo.ctx().data);
1738#endif
1739 const bool canUseDirectChunkEval = !UseFilters && !queryInfo.has_entity_filter_terms() &&
1740 can_use_direct_chunk_term_eval_descs(world, queryInfo, &desc, 1) &&
1741 can_use_direct_chunk_iteration_fastpath(queryInfo);
1742 const auto cacheView = queryInfo.cache_archetype_view();
1743 const bool needsBarrierCache = needs_depth_order_hierarchy_barrier_cache(queryInfo, constraints);
1744 const bool hasSortedArrayPayload = queryInfo.has_sorted_payload() || needsBarrierCache;
1745 const auto sortView =
1746 hasSortedArrayPayload ? queryInfo.cache_sort_view() : decltype(queryInfo.cache_sort_view()){};
1747 if (needsBarrierCache)
1748 queryInfo.ensure_depth_order_hierarchy_barrier_cache();
1749 const auto cacheRange = selected_query_cache_range(queryInfo);
1750 if (!cacheRange.valid)
1751 return;
1752 const auto idxFrom = cacheRange.idxFrom;
1753 const auto idxTo = cacheRange.idxTo;
1754
1755 if (!sortView.empty()) {
1756 for (const auto& view: sortView) {
1757 if (view.archetypeIdx < idxFrom || view.archetypeIdx >= idxTo)
1758 continue;
1759
1760 const bool barrierPasses = !needsBarrierCache || queryInfo.barrier_passes(view.archetypeIdx);
1761 if GAIA_UNLIKELY (!can_process_archetype_inter(
1762 queryInfo, *cacheView[view.archetypeIdx], constraints, barrierPasses))
1763 continue;
1764
1765 const auto viewFrom = view.startRow;
1766 const auto viewTo = (uint16_t)(view.startRow + view.count);
1767 uint16_t minStartRow = 0;
1768 uint16_t minEndRow = 0;
1769 chunk_effective_range(view.pChunk, constraints, needsBarrierCache, barrierPasses, minStartRow, minEndRow);
1770 const auto startRow = core::get_max(minStartRow, viewFrom);
1771 const auto endRow = core::get_min(minEndRow, viewTo);
1772 if (startRow == endRow)
1773 continue;
1774
1775 run_typed_arr_rows<UseFilters>(
1776 *this, queryInfo, it, outArray, m_changedWorldVersion, view.archetypeIdx, cacheView[view.archetypeIdx],
1777 view.pChunk, startRow, endRow, needsBarrierCache, canUseDirectChunkEval);
1778 }
1779 return;
1780 }
1781
1782 for (uint32_t i = idxFrom; i < idxTo; ++i) {
1783 auto* pArchetype = cacheView[i];
1784 const bool barrierPasses = !needsBarrierCache || queryInfo.barrier_passes(i);
1785 if GAIA_UNLIKELY (!can_process_archetype_inter(queryInfo, *pArchetype, constraints, barrierPasses))
1786 continue;
1787
1788 const auto& chunks = pArchetype->chunks();
1789 for (auto* pChunk: chunks) {
1790 uint16_t from = 0;
1791 uint16_t to = 0;
1792 chunk_effective_range(pChunk, constraints, needsBarrierCache, barrierPasses, from, to);
1793 run_typed_arr_rows<UseFilters>(
1794 *this, queryInfo, it, outArray, m_changedWorldVersion, i, pArchetype, pChunk, from, to,
1795 needsBarrierCache, canUseDirectChunkEval);
1796 }
1797 }
1798 }
1799
1800 template <typename Container>
1801 inline void QueryImpl::arr(Container& outArray, Constraints constraints) {
1802#if GAIA_ASSERT_ENABLED
1803 {
1804 auto& queryInfo = fetch();
1805 GAIA_ASSERT(
1806 ChunkIterImpl::allows_term_access(
1807 queryInfo.ctx().data, typed_query_arg_meta<typename Container::value_type>(*queryInfo.world()).termId,
1808 false));
1809 }
1810#endif
1811 const auto entCnt = count(constraints);
1812 if (entCnt == 0)
1813 return;
1814
1815 outArray.reserve(entCnt);
1816 auto& queryInfo = fetch();
1817 match_all(queryInfo);
1818
1819 const bool hasFilters = queryInfo.has_filters();
1820 if (hasFilters) {
1821 arr_inter<true>(queryInfo, outArray, constraints);
1822 } else {
1823 arr_inter<false>(queryInfo, outArray, constraints);
1824 }
1825 }
1827 } // namespace detail
1828 } // namespace ecs
1829} // namespace gaia
gaia::ecs::Iter::end_index
static GAIA_NODISCARD uint16_t end_index(Chunk *pChunk) noexcept
Returns the end of the enabled row range in a chunk.
Definition chunk_iterator.h:2116
gaia::ecs::Iter::start_index
static GAIA_NODISCARD uint16_t start_index(Chunk *pChunk) noexcept
Returns the first enabled row in a chunk.
Definition chunk_iterator.h:2109
gaia::ecs::detail::QueryImpl::chunk_effective_range
static void chunk_effective_range(Chunk *pChunk, Constraints constraints, bool needsBarrierCache, bool barrierPasses, uint16_t &from, uint16_t &to) noexcept
Calculates the row range of a chunk after applying row constraints and depth-order barrier state.
Definition query.h:2099
gaia::ecs::detail::QueryImpl::arr
void arr(Container &outArray, Constraints constraints=Constraints::EnabledOnly)
Appends all components or entities matching the query to the output array.
gaia::ecs::detail::QueryImpl::prepare_query_plan
GAIA_NODISCARD QueryPlan prepare_query_plan(const QueryInfo &queryInfo, Constraints constraints) const
Selects the prepared execution plan for public iterator callbacks.
Definition query.h:3836
gaia::ecs::detail::QueryImpl::needs_depth_order_hierarchy_barrier_cache
static GAIA_NODISCARD bool needs_depth_order_hierarchy_barrier_cache(const QueryInfo &queryInfo, Constraints constraints)
Checks whether the current row constraints require the depth-order hierarchy barrier cache.
Definition query.h:2088
gaia::ecs::detail::QueryImpl::ExecPayloadKind::Grouped
@ Grouped
Batches carry group ids but do not require sorted slices or inherited/barrier metadata.
gaia::ecs::detail::QueryImpl::ExecPayloadKind::NonTrivial
@ NonTrivial
Batches require non-trivial side payload such as sorted slices, inherited data, or barriers.
gaia::ecs::detail::QueryImpl::QueryPlanFlag_Sorted
@ QueryPlanFlag_Sorted
The plan may need sorted cache slices. Runners use them only with non-trivial payload.
Definition query.h:2320
gaia::ecs::detail::QueryImpl::QueryPlanFlag_Filtered
@ QueryPlanFlag_Filtered
The query has per-chunk filters such as changed terms.
Definition query.h:2312
gaia::ecs::detail::QueryImpl::QueryPlanFlag_InheritedPayload
@ QueryPlanFlag_InheritedPayload
The query carries inherited component data into iterator payloads.
Definition query.h:2316
gaia::ecs::detail::QueryImpl::QueryPlanFlag_BarrierCache
@ QueryPlanFlag_BarrierCache
The plan must use the depth-order hierarchy barrier cache when checking archetype/row ranges.
Definition query.h:2322
gaia::ecs::detail::QueryImpl::QueryPlanFlag_Grouped
@ QueryPlanFlag_Grouped
The query uses grouped payload/ranges or grouped cache ordering.
Definition query.h:2318
gaia::ecs::detail::QueryImpl::QueryPlanFlag_EntityFilter
@ QueryPlanFlag_EntityFilter
The query has entity-filter terms that require per-entity rechecks.
Definition query.h:2314
gaia::ecs::detail::QueryImpl::each_iter
void each_iter(Iter &it, Func func)
Runs a typed callback against an already prepared iterator. This is used by higher-level adapters tha...
gaia::ecs::detail::QueryImpl::exec_payload_kind
static GAIA_NODISCARD ExecPayloadKind exec_payload_kind(const QueryInfo &queryInfo, Constraints constraints)
Classifies the generic batch payload needed for a matched query under row constraints.
Definition query.h:2269
gaia::ecs::detail::QueryImpl::each
void each(Func func)
Iterates query matches using the default execution mode. Each callback receives a contiguous chunk wi...
Definition query.h:6581
gaia::ecs::detail::QueryImpl::depth_order_hierarchy_barrier_prunes
static GAIA_NODISCARD bool depth_order_hierarchy_barrier_prunes(const QueryInfo &queryInfo)
Checks whether cached depth-order barrier results can prune any matched archetype.
Definition query.h:2115
gaia::ecs::detail::QueryImpl::has_depth_order_hierarchy_enabled_barrier
static GAIA_NODISCARD bool has_depth_order_hierarchy_enabled_barrier(const QueryInfo &queryInfo)
Checks whether depth-order grouping can prune disabled hierarchy subtrees.
Definition query.h:2077
gaia::ecs::detail::QueryImpl::match_all
void match_all(QueryInfo &queryInfo)
Matches the query against all relevant archetypes.
Definition query.h:1094
gaia::ecs::detail::QueryImpl::ordered_entities_walk
GAIA_NODISCARD std::span< const Entity > ordered_entities_walk(QueryInfo &queryInfo, Entity relation, TravOrder order, Constraints constraints=Constraints::EnabledOnly)
Builds and caches relation traversal order for the current query result.
Definition query.h:6926
gaia::ecs::detail::QueryImpl::count
uint32_t count(Constraints constraints=Constraints::EnabledOnly)
Calculates the number of entities matching the query.
Definition query.h:6720
gaia::ecs::detail::QueryImpl::match_filters
static GAIA_NODISCARD bool match_filters(const Chunk &chunk, const QueryInfo &queryInfo, uint32_t changedWorldVersion, std::span< const uint8_t > compIndices)
Returns whether a chunk passes the query's changed filters.
Definition query.h:1961
gaia::ecs::detail::QueryImpl::each_walk
void each_walk(Func func, Entity relation, TravOrder order=TravOrder::Down, Constraints constraints=Constraints::EnabledOnly)
Iterates entities matching the query in a requested relation traversal order. For relation R this tre...
Definition query.h:7267
gaia::ecs::detail::QueryImpl::QueryPlanMode::Sorted
@ Sorted
Sorted payload execution that must preserve cache-provided chunk order.
gaia::ecs::detail::QueryImpl::QueryPlanMode::SparseDense
@ SparseDense
Typed cached chunk iteration with compile-time sparse payload access.
gaia::ecs::detail::QueryImpl::QueryPlanMode::DirectDense
@ DirectDense
Direct cached archetype/chunk iteration with query-term indices matching storage layout.
gaia::ecs::detail::QueryImpl::QueryPlanMode::EntitySeed
@ EntitySeed
Direct entity-seed evaluation over explicitly selected entities.
gaia::ecs::detail::QueryImpl::QueryPlanMode::MappedDense
@ MappedDense
Typed dense cached archetype/chunk iteration using mapped component access. Public Iter callbacks use...
gaia::ecs::detail::QueryImpl::QueryPlanMode::Empty
@ Empty
The selected group/range has no matching archetypes, so execution can return immediately.
gaia::ecs::detail::QueryImpl::QueryPlanMode::Traversal
@ Traversal
Traversal or inherited payload execution that requires the mapped generic path.
gaia::ecs::detail::QueryImpl::fetch
QueryInfo & fetch()
Fetches the QueryInfo object. Creates or refreshes the backing QueryInfo if needed.
Definition query.h:1039
gaia::ecs::detail::QueryImpl::can_process_archetype_inter
GAIA_NODISCARD bool can_process_archetype_inter(const QueryInfo &queryInfo, const Archetype &archetype, Constraints constraints, int8_t barrierPasses=-1) const
Checks whether a matched archetype can be processed for the current row constraints.
Definition query.h:2155
gaia::ecs::detail::QueryImpl::each_direct_inter
void each_direct_inter(QueryInfo &queryInfo, Constraints constraints, void *pFunc, const TypedQueryExecState &state, void(*runDirectChunk)(QueryImpl &, Iter &, void *, const TypedQueryExecState &), bool needsInheritedArgIds, void(*invokeInherited)(World &, Entity, const Entity *, void *))
Runs an erased typed callback over entities selected by a direct sparse or target-term seed.
gaia::ecs::detail::QueryImpl::QueryPlan::payloadKind
ExecPayloadKind payloadKind
Payload layout required by generic chunk-batch runners independent of sorted-cache availability.
Definition query.h:2332