MayaFlux 1.0.0
Digital-First Multimedia Processing Framework
Loading...
Searching...
No Matches
VisionAnalyzer.cpp
Go to the documentation of this file.
1#include "VisionAnalyzer.hpp"
2
8
9namespace MayaFlux::Yantra {
10
11namespace {
12
13 struct TrackReducePC {
14 uint32_t count;
15 };
16
17
18 struct HistogramPC {
19 uint32_t width;
20 uint32_t height;
21 uint32_t channels;
22 };
23
24 struct RegionSamplePC {
25 uint32_t px, py, pw, ph;
26 };
27
28 [[nodiscard]] bool inside(const Kinesis::Vision::BoundingBox& box, const glm::vec2& p)
29 {
30 return p.x >= box.x && p.x <= box.x + box.w
31 && p.y >= box.y && p.y <= box.y + box.h;
32 }
33
34 /** @brief Route one VisionResult into its matching VisionAnalysis field. */
35 void collect_into(
36 std::optional<std::vector<Kinesis::Vision::TrackResult>>& raw_tracks,
37 std::optional<std::vector<Kinesis::Vision::Keypoint>>& raw_keypoints,
38 Kinesis::Vision::VisionAnalysis& analysis,
39 const Kinesis::Vision::VisionResult& result)
40 {
41 if (const auto* tracks = std::get_if<std::vector<Kinesis::Vision::TrackResult>>(&result.structured))
42 raw_tracks = *tracks;
43
44 if (const auto* keypoints = std::get_if<std::vector<Kinesis::Vision::Keypoint>>(&result.structured))
45 raw_keypoints = *keypoints;
46
47 if (const auto* contours = std::get_if<std::vector<Kinesis::Vision::Contour>>(&result.structured)) {
48 analysis.find_elements = Kinesis::Vision::FindElementsAnalysis {
49 .contours = *contours,
50 .boxes = result.component_boxes,
51 };
52 }
53
54 if (result.flow) {
55 analysis.estimate_motion = result.flow;
56 analysis.motion_activity = result.motion_energy;
57 }
58
59 if (result.images.canny)
60 analysis.detect_edges = result.images.canny;
61 }
62
63} // namespace
64
69
71{
72 for (const auto& executor : m_executors) {
73 if (executor)
74 executor->reset();
75 }
76 m_run = {};
77 m_results = std::make_shared<const std::vector<Kinesis::Vision::VisionResult>>();
78}
79
81{
83 return m_executors[m_label_lane].get();
84 return m_executors.empty() ? nullptr : m_executors.front().get();
85}
86
88 const std::vector<Kinesis::Vision::TrackResult>& tracks, uint32_t w, uint32_t h)
89{
91 result.tracks = tracks;
92
93 if (tracks.empty())
94 return result;
95
96 std::vector<float> track_data;
97 track_data.reserve(tracks.size() * 4);
98 std::vector<float> extra_data;
99 extra_data.reserve(tracks.size() * 2);
100 for (const auto& t : tracks) {
101 track_data.push_back(t.position.x);
102 track_data.push_back(t.position.y);
103 track_data.push_back(t.previous.x);
104 track_data.push_back(t.previous.y);
105 extra_data.push_back(static_cast<float>(t.age));
106 extra_data.push_back(t.tracked ? 1.0F : 0.0F);
107 }
108
109 if (!m_track_reducer) {
110 m_track_reducer = std::make_shared<ShaderExecutionContext<>>(
112 .shader_path = "track_reduce.comp.spv",
113 .workgroup_size = { 256, 1, 1 },
114 .push_constant_size = sizeof(TrackReducePC) });
115 }
116
117 m_track_reducer->input(0, track_data)
118 .output(1, 3 * sizeof(glm::vec4))
119 .input(2, extra_data)
120 .push(TrackReducePC { .count = static_cast<uint32_t>(tracks.size()) });
121
122 const auto output = m_track_reducer->execute(Datum<std::vector<Kakshya::DataVariant>> {}, ExecutionContext {});
123 const auto raw = ShaderExecutionContext<>::read_output<float>(output, 1);
124
125 if (raw.size() >= 12) {
126 const auto n = static_cast<float>(tracks.size());
127 result.centroid = { raw[0] / n, raw[1] / n };
128 result.velocity = { raw[2] / n, raw[3] / n };
129 result.bounds = {
130 .x = raw[4],
131 .y = raw[5],
132 .w = raw[6] - raw[4],
133 .h = raw[7] - raw[5],
134 };
135 result.mean_speed_px = glm::length(glm::vec2 {
136 result.velocity.x * static_cast<float>(w),
137 result.velocity.y * static_cast<float>(h) });
138 result.mean_age = raw[8] / n;
139 result.max_age = static_cast<uint32_t>(std::lround(raw[9]));
140 result.tracked_count = static_cast<uint32_t>(std::lround(raw[10]));
141 result.lost_count = static_cast<uint32_t>(tracks.size()) - result.tracked_count;
142 }
143
144 return result;
145}
146
148 const std::vector<Kinesis::Vision::Keypoint>& keypoints)
149{
151 result.keypoints = keypoints;
152
153 if (keypoints.empty())
154 return result;
155
156 std::vector<float> point_data;
157 point_data.reserve(keypoints.size() * 4);
158 std::vector<float> extra_data;
159 extra_data.reserve(keypoints.size() * 2);
160 for (const auto& k : keypoints) {
161 point_data.push_back(k.position.x);
162 point_data.push_back(k.position.y);
163 point_data.push_back(k.position.x);
164 point_data.push_back(k.position.y);
165 extra_data.push_back(k.response);
166 extra_data.push_back(0.0F);
167 }
168
169 if (!m_track_reducer) {
170 m_track_reducer = std::make_shared<ShaderExecutionContext<>>(
172 .shader_path = "track_reduce.comp.spv",
173 .workgroup_size = { 256, 1, 1 },
174 .push_constant_size = sizeof(TrackReducePC) });
175 }
176
177 m_track_reducer->input(0, point_data)
178 .output(1, 3 * sizeof(glm::vec4))
179 .input(2, extra_data)
180 .push(TrackReducePC { .count = static_cast<uint32_t>(keypoints.size()) });
181
182 const auto output = m_track_reducer->execute(Datum<std::vector<Kakshya::DataVariant>> {}, ExecutionContext {});
183 const auto raw = ShaderExecutionContext<>::read_output<float>(output, 1);
184
185 if (raw.size() >= 12) {
186 const auto n = static_cast<float>(keypoints.size());
187 result.centroid = { raw[0] / n, raw[1] / n };
188 result.mean_response = raw[8] / n;
189 result.bounds = {
190 .x = raw[4],
191 .y = raw[5],
192 .w = raw[6] - raw[4],
193 .h = raw[7] - raw[5],
194 };
195 }
196
197 return result;
198}
199
201{
202 const auto n = fe.boxes.size();
203 fe.shapes.assign(n, {});
204 fe.nearest_neighbor_distance_px.assign(n, 0.0F);
205 if (n == 0)
206 return;
207
208 const auto fw = static_cast<float>(w);
209 const auto fh = static_cast<float>(h);
210
211 std::vector<float> element_data;
212 element_data.reserve(n * 4);
213 std::vector<float> centroid_data;
214 centroid_data.reserve(n * 2);
215
216 for (const auto& b : fe.boxes) {
217 const auto it = std::ranges::find_if(fe.contours,
218 [&](const auto& c) { return c.label_id == b.label_id; });
219 const float area_px = (it != fe.contours.end()) ? it->area * fw * fh : 0.0F;
220 const float perimeter_px = (it != fe.contours.end()) ? it->perimeter : 0.0F;
221 element_data.push_back(area_px);
222 element_data.push_back(perimeter_px);
223 element_data.push_back(b.w * fw);
224 element_data.push_back(b.h * fh);
225
226 centroid_data.push_back((b.x + b.w * 0.5F) * fw);
227 centroid_data.push_back((b.y + b.h * 0.5F) * fh);
228 }
229
230 if (!m_shape_ctx) {
231 m_shape_ctx = std::make_shared<ShaderExecutionContext<>>(
233 .shader_path = "element_shape.comp.spv",
234 .workgroup_size = { 256, 1, 1 },
235 .push_constant_size = sizeof(TrackReducePC) });
236 }
237 m_shape_ctx->input(0, element_data)
238 .output(1, n * sizeof(glm::vec2))
239 .push(TrackReducePC { .count = static_cast<uint32_t>(n) });
240
241 const auto shape_output = m_shape_ctx->execute(Datum<std::vector<Kakshya::DataVariant>> {}, ExecutionContext {});
242 const auto shape_raw = ShaderExecutionContext<>::read_output<float>(shape_output, 1);
243 for (size_t i = 0; i < n && (i + 1) * 2 <= shape_raw.size(); ++i) {
244 fe.shapes[i].compactness = shape_raw[i * 2];
245 fe.shapes[i].aspect_ratio = shape_raw[i * 2 + 1];
246 }
247
248 if (n < 2)
249 return;
250
251 if (!m_neighbor_ctx) {
252 m_neighbor_ctx = std::make_shared<ShaderExecutionContext<>>(
254 .shader_path = "nearest_neighbor.comp.spv",
255 .workgroup_size = { 256, 1, 1 },
256 .push_constant_size = sizeof(TrackReducePC) });
257 }
258 m_neighbor_ctx->input(0, centroid_data)
259 .output(1, n * sizeof(float))
260 .push(TrackReducePC { .count = static_cast<uint32_t>(n) });
261
262 const auto dist_output = m_neighbor_ctx->execute(Datum<std::vector<Kakshya::DataVariant>> {}, ExecutionContext {});
263 const auto dist_raw = ShaderExecutionContext<>::read_output<float>(dist_output, 1);
264 for (size_t i = 0; i < n && i < dist_raw.size(); ++i)
265 fe.nearest_neighbor_distance_px[i] = dist_raw[i];
266}
267
269 const Kinesis::Vision::VisionResult& result,
270 const std::shared_ptr<Core::VKImage>& gray,
271 uint32_t w, uint32_t h)
272{
274
275 if (result.images.sobel)
277
278 if (gray) {
279 out.histogram = compute_histogram(gray, w, h);
280 out.mean_brightness = compute_mean_brightness(out.histogram);
281 }
282
283 return out;
284}
285
287 const std::shared_ptr<Core::VKImage>& image, uint32_t w, uint32_t h)
288{
290 m_gradient_sample_ctx = std::make_shared<TextureExecutionContext>(
292 .shader_path = "region_sample.comp.spv",
293 .workgroup_size = { 256, 1, 1 },
294 .push_constant_size = sizeof(RegionSamplePC) },
295 Portal::Graphics::ImageFormat::RGBA8,
297 1,
298 std::vector<GpuBufferBinding> {
299 { .set = 0, .binding = 2, .direction = GpuBufferBinding::Direction::OUTPUT, .element_type = GpuBufferBinding::ElementType::FLOAT32 } });
300 }
301
302 auto& foundry = Portal::Graphics::get_shader_foundry();
303
304 m_gradient_sample_ctx->stage_image(image);
305 m_gradient_sample_ctx->set_push_constants(RegionSamplePC { .px = 0, .py = 0, .pw = w, .ph = h });
306 m_gradient_sample_ctx->set_output_size(2, 3 * sizeof(glm::vec4));
307 m_gradient_sample_ctx->set_output_dimensions(1, 1);
308
309 const auto fence = m_gradient_sample_ctx->dispatch_async({});
310 m_gradient_sample_ctx->clear_output_dimensions();
311 foundry.wait_for_fence(fence);
312 foundry.release_fence(fence);
313
314 const auto gpu_result = m_gradient_sample_ctx->collect_result();
315
317 if (auto it = gpu_result.aux.find(2); it != gpu_result.aux.end() && it->second.size() >= 3 * sizeof(glm::vec4)) {
318 std::array<glm::vec4, 3> raw {};
319 std::memcpy(raw.data(), it->second.data(), sizeof(raw));
320 out.mean = raw[0];
321 out.min_val = raw[1];
322 out.max_val = raw[2];
323 }
324 return out;
325}
326
327std::array<uint32_t, 256> VisionAnalyzer::compute_histogram(
328 const std::shared_ptr<Core::VKImage>& image, uint32_t w, uint32_t h)
329{
330 if (!m_histogram_ctx) {
331 m_histogram_ctx = std::make_shared<TextureExecutionContext>(
333 .shader_path = "otsu_histogram.comp.spv",
334 .workgroup_size = { 8, 8, 1 },
335 .push_constant_size = sizeof(HistogramPC) },
336 Portal::Graphics::ImageFormat::RGBA8,
338 1,
339 std::vector<GpuBufferBinding> {
340 { .set = 0, .binding = 3, .direction = GpuBufferBinding::Direction::INPUT_OUTPUT, .element_type = GpuBufferBinding::ElementType::UINT32 } },
342 }
343
344 m_histogram_ctx->ensure_shared_buffer(0, 3, 768, GpuBufferBinding::ElementType::UINT32,
346
347 const std::array<uint32_t, 768> zero {};
348 m_histogram_ctx->upload_shared_raw(0, 3, reinterpret_cast<const uint8_t*>(zero.data()), sizeof(zero));
349
350 m_histogram_ctx->stage_image(image);
351 m_histogram_ctx->set_push_constants(HistogramPC { .width = w, .height = h, .channels = 0 });
352 m_histogram_ctx->set_output_dimensions(w, h);
353
354 auto& foundry = Portal::Graphics::get_shader_foundry();
355 const auto fence = m_histogram_ctx->dispatch_async({});
356 m_histogram_ctx->clear_output_dimensions();
357 foundry.wait_for_fence(fence);
358 foundry.release_fence(fence);
359
360 std::array<uint32_t, 256> histogram {};
361 m_histogram_ctx->download_shared(0, 3, histogram.data(), sizeof(histogram));
362 return histogram;
363}
364
365float VisionAnalyzer::compute_mean_brightness(const std::array<uint32_t, 256>& histogram)
366{
368 m_brightness_reducer = std::make_shared<ShaderExecutionContext<>>(
370 .shader_path = "histogram_reduce.comp.spv",
371 .workgroup_size = { 256, 1, 1 },
372 .push_constant_size = 0 });
373 }
374
375 const std::vector<uint32_t> hist_vec(histogram.begin(), histogram.end());
377 .output(1, sizeof(float));
378
379 const auto output = m_brightness_reducer->execute(Datum<std::vector<Kakshya::DataVariant>> {}, ExecutionContext {});
380 const auto raw = ShaderExecutionContext<>::read_output<float>(output, 1);
381 return raw.empty() ? 0.0F : raw[0];
382}
383
384std::shared_ptr<Core::VKImage> VisionAnalyzer::resolve_image(
385 const std::shared_ptr<Kakshya::SignalSourceContainer>& source,
386 size_t index)
387{
388 if (!source)
389 return nullptr;
390
391 if (auto tc = std::dynamic_pointer_cast<Kakshya::TextureContainer>(source))
392 return tc->to_image(static_cast<uint32_t>(index));
393
394 if (auto wc = std::dynamic_pointer_cast<Kakshya::WindowContainer>(source))
395 return wc->to_image();
396
397 auto vc = std::dynamic_pointer_cast<Kakshya::VideoStreamContainer>(source);
398 if (!vc)
399 return nullptr;
400
401 const uint32_t w = vc->get_width();
402 const uint32_t h = vc->get_height();
403
404 const void* raw = source->get_raw_data();
405 if (!raw)
406 return nullptr;
407
408 if (!m_upload_image || m_upload_w != w || m_upload_h != h) {
410 w, h, Portal::Graphics::ImageFormat::RGBA8, nullptr);
411 m_upload_w = w;
412 m_upload_h = h;
413 }
414 if (!m_upload_image)
415 return nullptr;
416
418 return m_upload_image;
419}
420
422 const std::shared_ptr<Kakshya::SignalSourceContainer>& source,
423 size_t index)
424{
425 const auto image = resolve_image(source, index);
426 if (!image)
427 return {};
428
429 auto analysis = analyze_resolved(image);
430 if (std::dynamic_pointer_cast<Kakshya::TextureContainer>(source))
431 analysis.layer = index;
432 return analysis;
433}
434
436 const std::shared_ptr<Core::VKImage>& image)
437{
438 return analyze_resolved(image);
439}
440
441std::optional<Kinesis::Vision::VisionAnalysis> VisionAnalyzer::advance(
442 const std::shared_ptr<Core::VKImage>& image)
443{
444 if (!m_run.active) {
445 if (!image)
446 return std::nullopt;
447
449 if (!m_frame)
450 return std::nullopt;
451
452 m_run.resolved = Kinesis::Vision::resolve(config<VisionAnalyzerConfig>().effective_query());
453 for (auto& entry : m_run.resolved) {
454 for (auto& step : entry.sequence.steps) {
457 step.deferred = true;
458 }
459 }
460
461 m_run.results.clear();
462 m_run.results.reserve(m_run.resolved.size());
463 m_run.w = m_frame->get_width();
464 m_run.h = m_frame->get_height();
465 m_run.active = true;
466 m_label_lane = 0;
467 }
468
469 const size_t lane = m_run.results.size();
470 if (lane < m_run.resolved.size() && !run_lane(lane, m_run.resolved, m_run.results, m_frame))
471 return std::nullopt;
472
473 if (m_run.results.size() < m_run.resolved.size())
474 return std::nullopt;
475
476 m_run.active = false;
477 auto analysis = assemble(m_run.resolved, m_run.results, m_run.w, m_run.h);
478 m_results = std::make_shared<const std::vector<Kinesis::Vision::VisionResult>>(std::move(m_run.results));
479 return analysis;
480}
481
482std::shared_ptr<Core::VKImage> VisionAnalyzer::hold_frame(const std::shared_ptr<Core::VKImage>& image)
483{
485 auto held = loom.acquire_cached_image(m_frame_cache, {
486 .width = image->get_width(),
487 .height = image->get_height(),
488 .format = Portal::Graphics::ImageFormat::RGBA32F,
490 });
491
492 constexpr auto filter = Portal::Graphics::FilterMode::NEAREST;
493 if (!held || !loom.can_blit(image, held, filter) || !loom.blit_layer(image, held, { .filter = filter }))
494 return nullptr;
495 return held;
496}
497
499 const std::shared_ptr<Core::VKImage>& image)
500{
501 const auto resolved = Kinesis::Vision::resolve(config<VisionAnalyzerConfig>().effective_query());
502
503 std::vector<Kinesis::Vision::VisionResult> results;
504 results.reserve(resolved.size());
505 m_label_lane = 0;
506
507 for (size_t lane = 0; lane < resolved.size(); ++lane) {
508 if (!run_lane(lane, resolved, results, image))
509 results.emplace_back();
510 }
511
512 auto analysis = assemble(resolved, results, image->get_width(), image->get_height());
513 m_results = std::make_shared<const std::vector<Kinesis::Vision::VisionResult>>(std::move(results));
514 return analysis;
515}
516
518 size_t lane,
519 const std::vector<Kinesis::Vision::ResolvedSequence>& resolved,
520 std::vector<Kinesis::Vision::VisionResult>& results,
521 const std::shared_ptr<Core::VKImage>& image)
522{
523 if (m_executors.size() < resolved.size())
524 m_executors.resize(resolved.size());
525
526 const auto& entry = resolved[lane];
527 auto& executor = m_executors[lane];
528 if (!executor)
529 executor = std::make_unique<VisionGpuExecutor>();
530
531 if (std::ranges::any_of(entry.sequence.steps,
532 [](const auto& step) { return step.op == Kinesis::Vision::VisionOp::ConnectedComponents; }))
533 m_label_lane = lane;
534
535 const auto seeded = entry.seed_field ? entry.seed_field(results[entry.seed_lane]) : nullptr;
536 const auto& source = seeded ? seeded : image;
537 auto result = executor->run(entry.sequence, source, source->get_width(), source->get_height());
538
539 if (result.status != Kinesis::Vision::VisionStatus::COMPLETE)
540 return false;
541
542 results.push_back(std::move(result));
543 return true;
544}
545
547 const std::vector<Kinesis::Vision::ResolvedSequence>& resolved,
548 const std::vector<Kinesis::Vision::VisionResult>& results,
549 uint32_t w, uint32_t h)
550{
552 std::optional<std::vector<Kinesis::Vision::TrackResult>> raw_tracks;
553 std::optional<std::vector<Kinesis::Vision::Keypoint>> raw_keypoints;
554
555 for (size_t lane = 0; lane < results.size(); ++lane) {
556 const auto& result = results[lane];
557 const auto& entry = resolved[lane];
558
559 collect_into(raw_tracks, raw_keypoints, analysis, result);
560
561 if (result.images.sobel) {
562 const auto& gray = result.gray || !entry.seed_field ? result.gray : results[entry.seed_lane].gray;
563 analysis.measure_appearance = measure_appearance(result, gray, w, h);
564 }
565 }
566
567 const auto& filtering = config<VisionAnalyzerConfig>().context;
568
569 if (raw_tracks && filtering.track_objects) {
570 const auto& ctx = *filtering.track_objects;
571 if (ctx.subject_track_id) {
572 const auto id = *ctx.subject_track_id;
573 std::erase_if(*raw_tracks, [&](const auto& t) { return t.id != id; });
574 } else if (ctx.subject_bounds) {
575 const auto& bounds = *ctx.subject_bounds;
576 std::erase_if(*raw_tracks, [&](const auto& t) { return !inside(bounds, t.position); });
577 }
578 }
579
580 if (raw_tracks)
581 analysis.track_objects = reduce_tracks(*raw_tracks, w, h);
582
583 if (raw_keypoints && filtering.detect_features_bounds) {
584 const auto& bounds = *filtering.detect_features_bounds;
585 std::erase_if(*raw_keypoints, [&](const auto& k) { return !inside(bounds, k.position); });
586 }
587
588 if (raw_keypoints)
589 analysis.detect_features = reduce_keypoints(*raw_keypoints);
590
591 if (analysis.find_elements)
592 compute_shapes(*analysis.find_elements, w, h);
593
594 return analysis;
595}
596
598{
599 output_type output;
600 output.metadata = input.metadata;
601 output.facets = input.facets;
602
603 if (const auto image = input.facets.get(Kinesis::Vision::vision_frame())) {
605 return output;
606 }
607
608 if (!input.data) {
609 MF_ERROR(Journal::Component::Yantra, Journal::Context::ComputeMatrix, "VisionAnalyzer: missing container");
610 return output;
611 }
612
613 output.facets.set(analyze_vision(input.data, config<VisionAnalyzerConfig>().frame_index));
614 return output;
615}
616
617} // namespace MayaFlux::Yantra
#define MF_ERROR(comp, ctx,...)
Core::GlobalInputConfig input
Definition Config.cpp:42
IO::ImageData image
Definition Decoder.cpp:47
uint32_t h
Definition InkPress.cpp:29
size_t b
std::vector< float > * out
uint32_t index
Definition VKDevice.cpp:122
size_t count
uint32_t channels
uint32_t ph
uint32_t py
uint32_t pw
uint32_t px
Analyzer<shared_ptr<SignalSourceContainer>, vector<DataVariant>> whose query and context are its conf...
uint32_t width
uint32_t height
void set(T value)
Stores a described struct under its type identity.
Definition Facets.hpp:107
void upload_data(const std::shared_ptr< Core::VKImage > &image, const void *data, size_t size, const std::shared_ptr< Buffers::VKBuffer > &staging=nullptr, bool deferred=false)
Upload pixels to an image.
std::shared_ptr< Core::VKImage > create_2d(uint32_t width, uint32_t height, ImageFormat format=ImageFormat::RGBA8, const void *data=nullptr, uint32_t mip_levels=1)
Create a 2D texture.
const Config & config() const
The stored config struct of type Config, or its defaults when none was set.
void set_config(Config config)
Stores a described config struct, whole, under its own type.
Concrete GpuExecutionContext for a single fixed shader with fixed bindings.
@ SCALAR
Return SSBO readback via collect_result(); no image download.
Kinesis::Vision::MeasureAppearanceAnalysis measure_appearance(const Kinesis::Vision::VisionResult &result, const std::shared_ptr< Core::VKImage > &gray, uint32_t w, uint32_t h)
MeasureAppearance's whole-frame gradient sample (region_sample.comp over result.images....
float compute_mean_brightness(const std::array< uint32_t, 256 > &histogram)
Dispatch histogram_reduce.comp over an already-computed histogram.
std::shared_ptr< Core::VKImage > m_upload_image
std::shared_ptr< ShaderExecutionContext<> > m_shape_ctx
Kinesis::Vision::FieldSample sample_full_frame(const std::shared_ptr< Core::VKImage > &image, uint32_t w, uint32_t h)
Dispatch region_sample.comp over the whole image (no region concept for MeasureAppearance).
VisionAnalyzer(VisionAnalyzerConfig config={})
Construct from the settings, as EnergyAnalyzer does.
void compute_shapes(Kinesis::Vision::FindElementsAnalysis &fe, uint32_t w, uint32_t h)
Shape descriptors (element_shape.comp) and nearest-neighbour distances (nearest_neighbor....
std::shared_ptr< Core::VKImage > m_frame
Kinesis::Vision::VisionAnalysis analyze_resolved(const std::shared_ptr< Core::VKImage > &image)
Shared body of both analyze_vision() overloads, once a GPU image is in hand: resolve the configured q...
bool run_lane(size_t lane, const std::vector< Kinesis::Vision::ResolvedSequence > &resolved, std::vector< Kinesis::Vision::VisionResult > &results, const std::shared_ptr< Core::VKImage > &image)
Run or poll sequence lane of resolved on its executor.
std::shared_ptr< Core::VKImage > hold_frame(const std::shared_ptr< Core::VKImage > &image)
Copy image into this analyzer's own frame image.
Portal::Graphics::ImageCacheEntry m_frame_cache
Kinesis::Vision::DetectFeaturesAnalysis reduce_keypoints(const std::vector< Kinesis::Vision::Keypoint > &keypoints)
Dispatch track_reduce.comp over keypoint positions (fed as their own previous, so velocity reduces to...
std::vector< std::unique_ptr< VisionGpuExecutor > > m_executors
std::shared_ptr< TextureExecutionContext > m_histogram_ctx
std::shared_ptr< ShaderExecutionContext<> > m_neighbor_ctx
std::shared_ptr< const std::vector< Kinesis::Vision::VisionResult > > m_results
void reset()
Abandon outstanding work and clear retained executor state.
VisionGpuExecutor * get_executor() const
The executor that ran the FindElements sequence, or the first executor when the query has none.
std::optional< Kinesis::Vision::VisionAnalysis > advance(const std::shared_ptr< Core::VKImage > &image)
Advance an analysis of image by one sequence per call, without waiting on deferred GPU work.
std::shared_ptr< ShaderExecutionContext<> > m_brightness_reducer
output_type analyze_implementation(const input_type &input) override
Uses input.data as the source container, or a supplied frame facet, and calls analyze_vision().
Kinesis::Vision::VisionAnalysis assemble(const std::vector< Kinesis::Vision::ResolvedSequence > &resolved, const std::vector< Kinesis::Vision::VisionResult > &results, uint32_t w, uint32_t h)
Route every completed sequence result into one VisionAnalysis, apply context filtering and run the GP...
Kinesis::Vision::TrackObjectsAnalysis reduce_tracks(const std::vector< Kinesis::Vision::TrackResult > &tracks, uint32_t w, uint32_t h)
Dispatch track_reduce.comp over tracks and fill centroid/velocity/bounds/mean_speed_px/age stats on t...
std::shared_ptr< Core::VKImage > resolve_image(const std::shared_ptr< Kakshya::SignalSourceContainer > &source, size_t index)
Resolve source/index to a GPU image, the same way VisionProcessor does: TextureContainer's own to_ima...
Kinesis::Vision::VisionAnalysis analyze_vision(const std::shared_ptr< Kakshya::SignalSourceContainer > &source, size_t index=0)
Real entry point: resolve the configured query against source, run it, run the GPU analysis.
std::shared_ptr< TextureExecutionContext > m_gradient_sample_ctx
std::shared_ptr< ShaderExecutionContext<> > m_track_reducer
std::array< uint32_t, 256 > compute_histogram(const std::shared_ptr< Core::VKImage > &image, uint32_t w, uint32_t h)
Dispatch otsu_histogram.comp over image, zeroing the buffer first.
Stateful GPU dispatch engine for VisionSequence execution.
@ ComputeMatrix
Compute operations (Yantra - algorithms, matrices, DSP)
@ Yantra
DSP algorithms, computational units, matrix operations, Grammar.
const Facet< std::shared_ptr< Core::VKImage > > & vision_frame()
Facet key for the frame a VisionAnalysis applies to, carried in a Datum's facets beside the analysis.
std::vector< ResolvedSequence > resolve(const VisionQuery &query)
Resolve a VisionQuery into the ResolvedSequences needed to satisfy every active intent.
MAYAFLUX_API ShaderFoundry & get_shader_foundry()
Get the global shader compiler instance.
@ NEAREST
Nearest neighbor (pixelated)
@ COMPUTE
Storage buffer for compute shaders (device-local)
Keypoint reduction, computed on GPU by the same reduction shader as TrackObjectsAnalysis (track_reduc...
Per-channel mean/min/max over a sampled image region, computed on GPU (Yantra's region_sample....
FindElements output: contours plus the per-component BoundingBoxes ConnectedComponents already comput...
std::shared_ptr< Core::VKImage > sobel
MeasureAppearance output: whole-frame photometric characterisation, computed on GPU (region_sample....
Track reduction, computed on GPU (Yantra's track_reduce.comp) from the (context-filtered) subject tra...
std::optional< FindElementsAnalysis > find_elements
std::optional< MeasureAppearanceAnalysis > measure_appearance
std::optional< DetectFeaturesAnalysis > detect_features
std::optional< TrackObjectsAnalysis > track_objects
Composed analysis output: one populated field per active intent that produced a result this call.
ImageOutputs images
Image output of whichever image-producing ops ran in this sequence, one named slot per op.
Result of executing a VisionSequence on one frame.
enum MayaFlux::Portal::Graphics::GpuBufferBinding::ElementType FLOAT32
Plain-data description of the compute shader to dispatch.
Facets facets
Typed results that travel with the data, read by type.
Definition DataIO.hpp:32
std::unordered_map< std::string, std::any > metadata
Associated metadata.
Definition DataIO.hpp:31
Input/Output container for computation pipeline data flow with structure preservation.
Definition DataIO.hpp:27
Context information controlling how a compute operation executes.
Settings of VisionAnalyzer, stored whole in the operation's facets.
std::vector< Kinesis::Vision::ResolvedSequence > resolved
std::vector< Kinesis::Vision::VisionResult > results