MayaFlux 0.5.0
Digital-First Multimedia Processing Framework
Loading...
Searching...
No Matches
VisionGpuDispatch.cpp
Go to the documentation of this file.
2
6
7namespace MayaFlux::Yantra {
8
9using namespace Portal::Graphics;
10using namespace Kinesis::Vision;
11
12// ============================================================================
13// Internal push constant layouts
14// ============================================================================
15
16namespace {
17 constexpr uint32_t CC_BACKGROUND_HOST = 0xFFFFFFFFU;
18 constexpr uint32_t CC_UNCLAIMED_HOST = 0U;
19
20 /** Standard 2D workgroup used by all pixel-to-pixel vision shaders */
21 constexpr std::array<uint32_t, 3> k_wg2d { 8, 8, 1 };
22 /** Maximum number of connected components that can be labeled in a single pass */
23 constexpr uint32_t k_max_components = 4096;
24 /** Maximum number of points that can be stored in a single contour */
25 constexpr uint32_t k_max_points_per_contour = 4096;
26 /** Maximum number of contours that can be stored in a single pass */
27 constexpr uint32_t k_max_holes_per_label = 4;
28 /** Maximum number of trace slots that can be stored in a single pass (for contour tracing) */
29 constexpr uint32_t k_max_trace_slots = k_max_components * (1U + k_max_holes_per_label);
30
31 struct ThresholdPC {
32 float value;
33 };
34 struct ThresholdAdaptivePC {
35 uint32_t block_size;
36 float offset;
37 };
38 struct OtsuHistPC {
39 uint32_t width;
40 uint32_t height;
41 };
42 struct NormalizePC {
43 float scale;
44 float offset;
45 };
46 struct MorphPC {
47 uint32_t radius;
48 };
49 struct HarrisPC {
50 float k;
51 uint32_t pass;
52 uint32_t width;
53 uint32_t height;
54 };
55 struct ExtractPC {
56 float threshold;
57 uint32_t nms_radius;
58 };
59 struct CannyPC {
60 float sigma;
61 float lo;
62 float hi;
63 };
64 struct RgbaToGrayPC {
65 float wr;
66 float wg;
67 float wb;
68 float wa;
69 };
70 struct GaussianPC {
71 uint32_t radius;
72 uint32_t width;
73 uint32_t height;
74 };
75 struct CompletedOp {
76 std::shared_ptr<Core::VKImage> output;
77 std::shared_ptr<Core::VKImage> input;
78 };
79 struct ClassifyPC {
80 float threshold;
81 float value;
82 };
83 struct HysteresisPC {
84 uint32_t width;
85 uint32_t height;
86 };
87 struct FinalizePC {
88 float threshold;
89 };
90 struct ExtractPeaksPC {
91 float threshold;
92 uint32_t nms_radius;
93 uint32_t width;
94 uint32_t height;
95 uint32_t max_keypoints;
96 };
97 struct CCBlockInitPC {
98 uint32_t width;
99 uint32_t height;
100 uint32_t block_width;
101 uint32_t block_height;
102 };
103 struct CCMergePC {
104 uint32_t width;
105 uint32_t height;
106 uint32_t block_width;
107 uint32_t block_height;
108 };
109 struct CCCompressPC {
110 uint32_t block_width;
111 uint32_t block_height;
112 };
113 struct CCFinalLabelPC {
114 uint32_t width;
115 uint32_t height;
116 uint32_t block_width;
117 uint32_t block_height;
120 };
121 struct CCResetPC {
122 uint32_t lut_size;
123 uint32_t max_components;
124 };
125 struct ContourSegmentsPC {
126 uint32_t width;
127 uint32_t height;
128 uint32_t max_segments;
129 };
130 struct ContourLinkPC {
131 uint32_t width;
132 uint32_t height;
133 uint32_t phase;
134 };
135 struct ContourClearPC {
136 uint32_t width;
137 uint32_t height;
138 };
139 struct ContourRenderPC {
140 uint32_t width;
141 uint32_t height;
142 uint32_t max_components;
144 uint32_t max_contours;
145 };
146 struct ContourMarchPC {
147 uint32_t width;
148 uint32_t height;
149 uint32_t max_components;
150 uint32_t max_points_per_contour;
152 uint32_t phase;
153 float min_area;
155 };
156 struct ContourCompactPC {
157 uint32_t max_components;
158 uint32_t max_holes_per_label;
159 };
160
161 /**
162 * @brief 2D Gaussian kernel for convolution, cached by (radius, sigma
163 * bit pattern).
164 *
165 * Sigma is a tuning parameter that rarely changes frame to frame;
166 * recomputing exp() over (2*radius+1)^2 taps and reallocating the
167 * kernel every call is pure repeated work for an identical result.
168 *
169 * @param radius Radius of the kernel in pixels. Kernel size is (2*radius + 1)^2.
170 * @param sigma Standard deviation of the Gaussian.
171 * @return Normalized kernel weights as a flat vector in row-major order.
172 */
173 const std::vector<float>& gaussian_kernel_2d(uint32_t radius, float sigma)
174 {
175 static std::unordered_map<uint64_t, std::vector<float>> cache;
176 const uint64_t key = (static_cast<uint64_t>(std::bit_cast<uint32_t>(sigma)) << 32)
177 | radius;
178 auto it = cache.find(key);
179 if (it != cache.end())
180 return it->second;
181
182 const uint32_t diam = 2 * radius + 1;
183 std::vector<float> k(static_cast<size_t>(diam) * diam);
184 float sum = 0.0F;
185 for (uint32_t y = 0; y < diam; ++y) {
186 for (uint32_t x = 0; x < diam; ++x) {
187 const float fx = static_cast<float>(x) - static_cast<float>(radius);
188 const float fy = static_cast<float>(y) - static_cast<float>(radius);
189 const float v = std::exp(-(fx * fx + fy * fy) / (2.0F * sigma * sigma));
190 k[y * diam + x] = v;
191 sum += v;
192 }
193 }
194 for (auto& v : k)
195 v /= sum;
196
197 return cache.emplace(key, std::move(k)).first->second;
198 }
199
200 /**
201 * @brief 1D Gaussian kernel for separable convolution, cached by
202 * (radius, sigma bit pattern).
203 *
204 * Sigma is a tuning parameter that rarely changes frame to frame;
205 * recomputing exp() per tap and reallocating the kernel every call
206 * is pure repeated work for an identical result. Mirrors
207 * VisionExecutor::gaussian_kernel's caching rationale for the CPU path.
208 *
209 * @param radius Radius of the kernel in pixels. Kernel size is (2*radius + 1).
210 * @param sigma Standard deviation of the Gaussian.
211 * @return Normalized kernel weights.
212 */
213 const std::vector<float>& gaussian_kernel_1d(uint32_t radius, float sigma)
214 {
215 static std::unordered_map<uint64_t, std::vector<float>> cache;
216 const uint64_t key = (static_cast<uint64_t>(std::bit_cast<uint32_t>(sigma)) << 32)
217 | radius;
218 auto it = cache.find(key);
219 if (it != cache.end())
220 return it->second;
221
222 const uint32_t size = 2 * radius + 1;
223 std::vector<float> k(size);
224 float sum = 0.0F;
225 for (uint32_t i = 0; i < size; ++i) {
226 const float x = static_cast<float>(i) - static_cast<float>(radius);
227 k[i] = std::exp(-(x * x) / (2.0F * sigma * sigma));
228 sum += k[i];
229 }
230 for (auto& v : k)
231 v /= sum;
232
233 return cache.emplace(key, std::move(k)).first->second;
234 }
235
236} // namespace
237
239 : pixel {
241 Portal::Graphics::ImageFormat::RGBA32F,
242 TextureExecutionContext::OutputMode::IMAGE,
243 1,
244 std::vector<GpuBufferBinding> {
245 { .set = 0, .binding = 2, .direction = GpuBufferBinding::Direction::INPUT, .element_type = GpuBufferBinding::ElementType::FLOAT32 },
246 },
248 }
249 , structured {
250 GpuComputeConfig {},
252 TextureExecutionContext::OutputMode::SCALAR,
253 0,
254 std::vector<GpuBufferBinding> {
255 { .set = 0, .binding = 1, .direction = GpuBufferBinding::Direction::OUTPUT, .element_type = GpuBufferBinding::ElementType::UINT32 },
256 { .set = 0, .binding = 2, .direction = GpuBufferBinding::Direction::OUTPUT, .element_type = GpuBufferBinding::ElementType::FLOAT32 },
257 { .set = 0, .binding = 3, .direction = GpuBufferBinding::Direction::OUTPUT, .element_type = GpuBufferBinding::ElementType::UINT32 },
258 { .set = 0, .binding = 4, .direction = GpuBufferBinding::Direction::OUTPUT, .element_type = GpuBufferBinding::ElementType::UINT32 },
259 { .set = 1, .binding = 0, .direction = GpuBufferBinding::Direction::INPUT_OUTPUT, .element_type = GpuBufferBinding::ElementType::FLOAT32 },
260 { .set = 1, .binding = 1, .direction = GpuBufferBinding::Direction::INPUT_OUTPUT, .element_type = GpuBufferBinding::ElementType::FLOAT32 },
261 },
262 GpuBufferBinding::ElementType::IMAGE_STORAGE,
263 0,
264 }
265 , labels {
268 TextureExecutionContext::OutputMode::IMAGE,
269 1,
270 std::vector<GpuBufferBinding> {
271 { .set = 0, .binding = 2, .direction = GpuBufferBinding::Direction::OUTPUT, .element_type = GpuBufferBinding::ElementType::UINT32 },
272 { .set = 0, .binding = 4, .direction = GpuBufferBinding::Direction::OUTPUT, .element_type = GpuBufferBinding::ElementType::IMAGE_STORAGE },
273 },
274 GpuBufferBinding::ElementType::IMAGE_STORAGE,
275 }
276 , cc_pipeline {
279 TextureExecutionContext::OutputMode::IMAGE,
280 1,
281 std::vector<GpuBufferBinding> {
282 { .set = 0, .binding = 2, .direction = GpuBufferBinding::Direction::INPUT_OUTPUT, .element_type = GpuBufferBinding::ElementType::UINT32 },
283 { .set = 0, .binding = 3, .direction = GpuBufferBinding::Direction::INPUT_OUTPUT, .element_type = GpuBufferBinding::ElementType::UINT32 },
284 { .set = 0, .binding = 4, .direction = GpuBufferBinding::Direction::INPUT_OUTPUT, .element_type = GpuBufferBinding::ElementType::UINT32 },
285 { .set = 0, .binding = 5, .direction = GpuBufferBinding::Direction::INPUT_OUTPUT, .element_type = GpuBufferBinding::ElementType::UINT32 },
286 { .set = 0, .binding = 6, .direction = GpuBufferBinding::Direction::INPUT_OUTPUT, .element_type = GpuBufferBinding::ElementType::UINT32 },
287 { .set = 0, .binding = 7, .direction = GpuBufferBinding::Direction::INPUT_OUTPUT, .element_type = GpuBufferBinding::ElementType::UINT32 },
288 { .set = 0, .binding = 8, .direction = GpuBufferBinding::Direction::INPUT_OUTPUT, .element_type = GpuBufferBinding::ElementType::UINT32 },
289 { .set = 0, .binding = 9, .direction = GpuBufferBinding::Direction::INPUT_OUTPUT, .element_type = GpuBufferBinding::ElementType::UINT32 },
290 { .set = 0, .binding = 10, .direction = GpuBufferBinding::Direction::INPUT_OUTPUT, .element_type = GpuBufferBinding::ElementType::UINT32 },
291 { .set = 1, .binding = 0, .direction = GpuBufferBinding::Direction::INPUT_OUTPUT, .element_type = GpuBufferBinding::ElementType::UINT32 },
292 { .set = 1, .binding = 1, .direction = GpuBufferBinding::Direction::INPUT_OUTPUT, .element_type = GpuBufferBinding::ElementType::UINT32 },
293 { .set = 1, .binding = 2, .direction = GpuBufferBinding::Direction::INPUT_OUTPUT, .element_type = GpuBufferBinding::ElementType::UINT32 },
294 { .set = 1, .binding = 3, .direction = GpuBufferBinding::Direction::INPUT_OUTPUT, .element_type = GpuBufferBinding::ElementType::UINT32 },
295 { .set = 1, .binding = 4, .direction = GpuBufferBinding::Direction::INPUT_OUTPUT, .element_type = GpuBufferBinding::ElementType::UINT32 },
296 { .set = 1, .binding = 5, .direction = GpuBufferBinding::Direction::INPUT_OUTPUT, .element_type = GpuBufferBinding::ElementType::UINT32 },
297 { .set = 1, .binding = 6, .direction = GpuBufferBinding::Direction::INPUT_OUTPUT, .element_type = GpuBufferBinding::ElementType::UINT32 },
298 { .set = 1, .binding = 7, .direction = GpuBufferBinding::Direction::INPUT_OUTPUT, .element_type = GpuBufferBinding::ElementType::UINT32 },
299 { .set = 1, .binding = 8, .direction = GpuBufferBinding::Direction::INPUT_OUTPUT, .element_type = GpuBufferBinding::ElementType::FLOAT32 },
300 { .set = 1, .binding = 9, .direction = GpuBufferBinding::Direction::INPUT_OUTPUT, .element_type = GpuBufferBinding::ElementType::UINT32 },
301 { .set = 1, .binding = 10, .direction = GpuBufferBinding::Direction::INPUT_OUTPUT, .element_type = GpuBufferBinding::ElementType::UINT32 },
302 { .set = 1, .binding = 11, .direction = GpuBufferBinding::Direction::INPUT_OUTPUT, .element_type = GpuBufferBinding::ElementType::FLOAT32 },
303 { .set = 2, .binding = 0, .direction = GpuBufferBinding::Direction::INPUT_OUTPUT, .element_type = GpuBufferBinding::ElementType::FLOAT32 },
304 { .set = 2, .binding = 1, .direction = GpuBufferBinding::Direction::INPUT_OUTPUT, .element_type = GpuBufferBinding::ElementType::FLOAT32 },
305 },
306 GpuBufferBinding::ElementType::IMAGE_STORAGE,
307 0,
308 }
309{
310 structured.set_output_size(1, sizeof(uint32_t));
311 structured.set_output_size(2, static_cast<size_t>(4096) * 4 * sizeof(float));
312 structured.set_output_size(3, static_cast<size_t>(256) * sizeof(uint32_t));
313 structured.set_output_size(4, sizeof(uint32_t));
314}
315
316// ============================================================================
317// vision_gpu_config
318// ============================================================================
319
320GpuComputeConfig VisionGpuExecutor::config(VisionOp op, const VisionParams& /*params*/)
321{
322 switch (op) {
323 case VisionOp::Threshold: {
324 const auto spec = ShaderSpec::Assemble {}
325 .storage_image("out", BindingDirection::Output)
326 .storage_image("src", BindingDirection::Input)
327 .pc("threshold")
328 .op(KernelOp::CompareGE)
329 .workgroup(k_wg2d[0], k_wg2d[1])
330 .build();
331 return config_from_spec(spec);
332 }
333 case VisionOp::RgbaToGray: {
334 const auto spec = ShaderSpec::Assemble {}
335 .storage_image("out", BindingDirection::Output)
336 .storage_image("src", BindingDirection::Input)
337 .pc("wr")
338 .pc("wg")
339 .pc("wb")
340 .pc("wa")
341 .op(KernelOp::ChannelDot)
342 .workgroup(k_wg2d[0], k_wg2d[1])
343 .build();
344 return config_from_spec(spec);
345 }
346 case VisionOp::GrayToRgba: {
347 const auto spec = ShaderSpec::Assemble {}
348 .storage_image("out", BindingDirection::Output)
349 .storage_image("src", BindingDirection::Input)
350 .op(KernelOp::ChannelReplicate)
351 .workgroup(k_wg2d[0], k_wg2d[1])
352 .build();
353 return config_from_spec(spec);
354 }
355 case VisionOp::GaussianBlur: {
356 const auto spec = ShaderSpec::Assemble {}
357 .tmpl(KernelTemplate::Convolve2D)
358 .storage_image("out", BindingDirection::Output)
359 .storage_image("src", BindingDirection::Input)
360 .ssbo("kernel", BindingDirection::Input, Kakshya::GpuDataFormat::FLOAT32)
364 .workgroup(k_wg2d[0], k_wg2d[1])
365 .build();
366 return config_from_spec(spec);
367 }
368 case VisionOp::NormalizeRange: {
369 const auto spec = ShaderSpec::Assemble {}
370 .storage_image("out", BindingDirection::Output)
371 .storage_image("src", BindingDirection::Input)
372 .pc("scale")
373 .pc("offset")
374 .op(KernelOp::ScaleOffset)
375 .workgroup(k_wg2d[0], k_wg2d[1])
376 .build();
377 return config_from_spec(spec);
378 }
379 case VisionOp::NormalizeInplace: {
380 const auto spec = ShaderSpec::Assemble {}
381 .storage_image("out", BindingDirection::Output)
382 .storage_image("src", BindingDirection::Input)
383 .op(KernelOp::Scale)
384 .workgroup(k_wg2d[0], k_wg2d[1])
385 .build();
386 return config_from_spec(spec);
387 }
388 case VisionOp::Canny: {
389 const auto spec = ShaderSpec::Assemble {}
390 .storage_image("out", BindingDirection::Output)
391 .storage_image("src", BindingDirection::Input)
392 .pc("threshold")
393 .pc("value")
394 .op(KernelOp::CompareGEPreserve)
395 .workgroup(k_wg2d[0], k_wg2d[1])
396 .build();
397 return config_from_spec(spec);
398 }
399 case VisionOp::RgbaToHsv:
400 return { .shader_path = "rgba_to_hsv.comp.spv", .workgroup_size = k_wg2d };
401 case VisionOp::Downsample2x:
402 return { .shader_path = "downsample_2x.comp.spv", .workgroup_size = k_wg2d };
403 case VisionOp::FilterSeparable:
404 return { .shader_path = "filter_separable.comp.spv", .workgroup_size = k_wg2d };
405 case VisionOp::Sobel:
406 return { .shader_path = "sobel.comp.spv", .workgroup_size = k_wg2d };
407 case VisionOp::Scharr:
408 return { .shader_path = "scharr.comp.spv", .workgroup_size = k_wg2d };
409 case VisionOp::Erode:
410 return { .shader_path = "erode.comp.spv", .workgroup_size = k_wg2d, .push_constant_size = sizeof(MorphPC) };
411 case VisionOp::Dilate:
412 return { .shader_path = "dilate.comp.spv", .workgroup_size = k_wg2d, .push_constant_size = sizeof(MorphPC) };
413 case VisionOp::Open:
414 return { .shader_path = "open.comp.spv", .workgroup_size = k_wg2d, .push_constant_size = sizeof(MorphPC) };
415 case VisionOp::Close:
416 return { .shader_path = "close.comp.spv", .workgroup_size = k_wg2d, .push_constant_size = sizeof(MorphPC) };
417 case VisionOp::MorphGradient:
418 return { .shader_path = "morph_gradient.comp.spv", .workgroup_size = k_wg2d, .push_constant_size = sizeof(MorphPC) };
419 case VisionOp::HarrisResponse:
420 return { .shader_path = "harris_response.comp.spv", .workgroup_size = k_wg2d, .push_constant_size = sizeof(HarrisPC) };
421 case VisionOp::ExtractPeaks:
422 return { .shader_path = "extract_peaks.comp.spv", .workgroup_size = { 256, 1, 1 }, .push_constant_size = sizeof(ExtractPC) };
423 case VisionOp::ConnectedComponents:
424 return { .shader_path = "cc_colorize.comp.spv", .workgroup_size = k_wg2d, .push_constant_size = sizeof(uint32_t) * 2 };
425 case VisionOp::FindContours:
426 return { .shader_path = "contour_segments.comp.spv", .workgroup_size = k_wg2d, .push_constant_size = sizeof(ContourSegmentsPC) };
427 case VisionOp::ThresholdAdaptive:
428 return { .shader_path = "threshold_adaptive.comp.spv", .workgroup_size = k_wg2d, .push_constant_size = sizeof(ThresholdAdaptivePC) };
429 case VisionOp::ThresholdOtsu:
430 return { .shader_path = "threshold_otsu.comp.spv", .workgroup_size = { 256, 1, 1 } };
431 default:
433 }
434}
435
436// ============================================================================
437// run_gpu
438// ============================================================================
439
440VisionResult VisionGpuExecutor::run(
441 VisionGpuContexts& contexts,
442 const VisionSequence& sequence,
443 const std::shared_ptr<Core::VKImage>& image,
444 uint32_t w, uint32_t h)
445{
446 auto& pixel_ctx = contexts.pixel;
447 auto& structured_ctx = contexts.structured;
448 auto& label_ctx = contexts.labels;
449 auto& cc_pipeline = contexts.cc_pipeline;
450
451 VisionResult result;
452 result.w = w;
453 result.h = h;
454
455 auto& foundry = Portal::Graphics::get_shader_foundry();
456 std::shared_ptr<Core::VKImage> current = image;
457
459 std::string last_shader_path;
460 std::shared_ptr<Core::VKImage> last_staged;
461 uint32_t last_w = 0, last_h = 0;
462 std::unordered_map<size_t, CompletedOp> completed_ops;
463
464 pixel_ctx.prepare_output_image(w, h);
465 last_w = w;
466 last_h = h;
467
468 for (const auto& step : sequence.steps) {
469 const auto cfg = config(step.op, step.params);
470
471 if (cfg.shader_id == Portal::Graphics::INVALID_SHADER && cfg.shader_path.empty()) {
473 "run_gpu: no GPU implementation for VisionOp {}",
474 static_cast<int>(step.op));
475 return VisionResult {};
476 }
477
478 if (cfg.shader_id != last_shader_id || cfg.shader_path != last_shader_path) {
479 pixel_ctx.swap_shader(cfg);
480 last_shader_id = cfg.shader_id;
481 last_shader_path = cfg.shader_path;
482 }
483 if (current != last_staged) {
484 pixel_ctx.stage_image(current);
485 last_staged = current;
486 }
487 if (w != last_w || h != last_h) {
488 pixel_ctx.prepare_output_image(w, h);
489 last_w = w;
490 last_h = h;
491 }
492 pixel_ctx.set_output_dimensions(w, h);
493
494 switch (step.op) {
495 case VisionOp::Downsample2x: {
496 const uint32_t new_w = std::max(1U, w / 2);
497 const uint32_t new_h = std::max(1U, h / 2);
498
499 pixel_ctx.prepare_output_image(new_w, new_h);
500 pixel_ctx.set_output_dimensions(new_w, new_h);
501 {
502 const auto f = pixel_ctx.dispatch_async({});
503 foundry.wait_for_fence(f);
504 foundry.release_fence(f);
505 }
506 pixel_ctx.clear_output_dimensions();
507
508 auto downsampled = pixel_ctx.get_output_image(0);
509 completed_ops[Kinesis::Vision::hash_vision_step(step.op, step.params)] = { .output = downsampled, .input = current };
510 current = downsampled;
511 last_staged = current;
512
513 w = new_w;
514 h = new_h;
515 result.w = w;
516 result.h = h;
517 last_w = w;
518 last_h = h;
519
520 continue;
521 }
522 case VisionOp::Threshold:
523 pixel_ctx.set_push_constants(ThresholdPC {
524 .value = std::get<ThresholdParams>(step.params).value });
525 break;
526 case VisionOp::NormalizeRange: {
527 const auto& p = std::get<NormalizeRangeParams>(step.params);
528 const float scale = (p.hi > p.lo) ? 1.0F / (p.hi - p.lo) : 1.0F;
529 const float off = (p.hi > p.lo) ? -p.lo / (p.hi - p.lo) : 0.0F;
530 pixel_ctx.set_push_constants(NormalizePC { .scale = scale, .offset = off });
531 break;
532 }
533 case VisionOp::RgbaToGray:
534 pixel_ctx.set_push_constants(RgbaToGrayPC {
535 .wr = 0.299F, .wg = 0.587F, .wb = 0.114F, .wa = 0.0F });
536 break;
537 case VisionOp::GaussianBlur: {
538 const auto& p = std::get<GaussianBlurParams>(step.params);
539 const auto radius = static_cast<uint32_t>(std::ceil(p.sigma * 3.0F));
540 const auto& weights = gaussian_kernel_1d(radius, p.sigma);
541 pixel_ctx.set_binding_data(2, std::span<const float>(weights));
542 pixel_ctx.set_push_constants(GaussianPC { .radius = radius, .width = w, .height = h });
543 break;
544 }
545 case VisionOp::Erode:
546 case VisionOp::Dilate:
547 case VisionOp::MorphGradient:
548 pixel_ctx.set_push_constants(MorphPC {
549 .radius = std::get<MorphParams>(step.params).radius });
550 break;
551 case VisionOp::ThresholdAdaptive: {
552 const auto& p = std::get<ThresholdAdaptiveParams>(step.params);
553 pixel_ctx.set_push_constants(ThresholdAdaptivePC { .block_size = p.block_size, .offset = p.offset });
554 break;
555 }
556 case VisionOp::ThresholdOtsu: {
557 structured_ctx.swap_shader({
558 .shader_path = "otsu_histogram.comp.spv",
559 .workgroup_size = k_wg2d,
560 .push_constant_size = sizeof(OtsuHistPC),
561 });
562 std::vector<uint32_t> zeros(256, 0);
563 structured_ctx.set_binding_data(3, std::span<const uint32_t>(zeros));
564 structured_ctx.stage_image(current);
565 structured_ctx.set_push_constants(OtsuHistPC { .width = w, .height = h });
566 structured_ctx.set_output_dimensions(w, h);
567 {
568 const auto f = structured_ctx.dispatch_async({});
569 structured_ctx.clear_output_dimensions();
570 foundry.wait_for_fence(f);
571 foundry.release_fence(f);
572 }
573
574 const auto hist_check = structured_ctx.collect_result();
575 std::vector<uint32_t> hist_readback(256, 0);
576 if (auto it = hist_check.aux.find(3); it != hist_check.aux.end())
577 std::memcpy(hist_readback.data(), it->second.data(), 256 * sizeof(uint32_t));
578
579 structured_ctx.swap_shader({
580 .shader_path = "otsu_select.comp.spv",
581 .workgroup_size = { 256, 1, 1 },
582 });
583 structured_ctx.set_binding_data(3, std::span<const uint32_t>(hist_readback));
584 structured_ctx.set_output_dimensions(256, 1);
585
586 {
587 const auto f = structured_ctx.dispatch_async({});
588 structured_ctx.clear_output_dimensions();
589 foundry.wait_for_fence(f);
590 foundry.release_fence(f);
591 }
592
593 const auto sel_result = structured_ctx.collect_result();
594 uint32_t best_bin = 0;
595 if (auto it = sel_result.aux.find(4); it != sel_result.aux.end())
596 std::memcpy(&best_bin, it->second.data(), sizeof(uint32_t));
597 const float t_norm = static_cast<float>(best_bin) / 255.0F;
598
599 const auto apply_cfg = config_from_spec(
601 .storage_image("out", BindingDirection::Output)
602 .storage_image("src", BindingDirection::Input)
603 .pc("threshold")
604 .op(KernelOp::CompareGE)
605 .workgroup(k_wg2d[0], k_wg2d[1])
606 .build());
607 pixel_ctx.swap_shader(apply_cfg);
608 pixel_ctx.stage_image(current);
609 pixel_ctx.set_push_constants(ThresholdPC { .value = t_norm });
610 pixel_ctx.prepare_output_image(w, h);
611 {
612 const auto f = pixel_ctx.dispatch_async({});
613 foundry.wait_for_fence(f);
614 foundry.release_fence(f);
615 }
616 auto thresholded = pixel_ctx.get_output_image(0);
617
618 completed_ops[Kinesis::Vision::hash_vision_step(step.op, step.params)] = { .output = thresholded, .input = current };
619 result.debug_labels = thresholded;
620 current = thresholded;
621 last_staged = current;
622 result.structured = std::monostate {};
623 continue;
624 }
625 case VisionOp::Open:
626 case VisionOp::Close: {
627 const auto radius = std::get<MorphParams>(step.params).radius;
628 const bool is_open = (step.op == VisionOp::Open);
629
630 const GpuComputeConfig first_cfg {
631 .shader_path = is_open ? "erode.comp.spv" : "dilate.comp.spv",
632 .workgroup_size = k_wg2d,
633 .push_constant_size = sizeof(MorphPC),
634 };
635 pixel_ctx.swap_shader(first_cfg);
636 pixel_ctx.stage_image(current);
637 pixel_ctx.set_push_constants(MorphPC { .radius = radius });
638 pixel_ctx.prepare_output_image(w, h);
639 pixel_ctx.set_output_dimensions(w, h);
640 {
641 const auto f = pixel_ctx.dispatch_async({});
642 foundry.wait_for_fence(f);
643 foundry.release_fence(f);
644 }
645 auto intermediate = pixel_ctx.get_output_image(0);
646
647 const GpuComputeConfig second_cfg {
648 .shader_path = is_open ? "dilate.comp.spv" : "erode.comp.spv",
649 .workgroup_size = k_wg2d,
650 .push_constant_size = sizeof(MorphPC),
651 };
652 pixel_ctx.swap_shader(second_cfg);
653 pixel_ctx.stage_image(intermediate);
654 pixel_ctx.set_push_constants(MorphPC { .radius = radius });
655 pixel_ctx.prepare_output_image(w, h);
656 pixel_ctx.set_output_dimensions(w, h);
657 {
658 const auto f = pixel_ctx.dispatch_async({});
659 foundry.wait_for_fence(f);
660 foundry.release_fence(f);
661 }
662
663 auto opened_closed = pixel_ctx.get_output_image(0);
664 completed_ops[Kinesis::Vision::hash_vision_step(step.op, step.params)] = { .output = opened_closed, .input = current };
665 current = opened_closed;
666 last_staged = current;
667 result.structured = std::monostate {};
668 continue;
669 }
670 case VisionOp::Canny: {
671 const auto& p = std::get<CannyParams>(step.params);
672 auto canny_input = current;
673
674 const auto blur_key = Kinesis::Vision::hash_vision_step(
675 VisionOp::GaussianBlur, GaussianBlurParams { .sigma = p.sigma });
676 std::shared_ptr<Core::VKImage> blurred;
677 if (auto it = completed_ops.find(blur_key);
678 it != completed_ops.end() && it->second.input == canny_input) {
679 blurred = it->second.output;
680 } else {
681 const auto radius = static_cast<uint32_t>(std::ceil(p.sigma * 3.0F));
682 const auto& weights = gaussian_kernel_1d(radius, p.sigma);
683 const auto blur_cfg = config(VisionOp::GaussianBlur, GaussianBlurParams { .sigma = p.sigma });
684 pixel_ctx.swap_shader(blur_cfg);
685 pixel_ctx.stage_image(canny_input);
686 pixel_ctx.set_binding_data(2, std::span<const float>(weights));
687 pixel_ctx.set_push_constants(GaussianPC { .radius = radius, .width = w, .height = h });
688 pixel_ctx.prepare_output_image(w, h);
689 {
690 const auto f = pixel_ctx.dispatch_async({});
691 foundry.wait_for_fence(f);
692 foundry.release_fence(f);
693 }
694 blurred = pixel_ctx.get_output_image(0);
695 completed_ops[blur_key] = { .output = blurred, .input = canny_input };
696 }
697
698 const auto sobel_key = Kinesis::Vision::hash_vision_step(VisionOp::Sobel, std::monostate {});
699 std::shared_ptr<Core::VKImage> grad;
700 if (auto it = completed_ops.find(sobel_key);
701 it != completed_ops.end() && it->second.input == blurred) {
702 grad = it->second.output;
703 } else {
704 const auto sobel_cfg = config(VisionOp::Sobel, std::monostate {});
705 pixel_ctx.swap_shader(sobel_cfg);
706 pixel_ctx.stage_image(blurred);
707 pixel_ctx.prepare_output_image(w, h);
708 {
709 const auto f = pixel_ctx.dispatch_async({});
710 foundry.wait_for_fence(f);
711 foundry.release_fence(f);
712 }
713 grad = pixel_ctx.get_output_image(0);
714 completed_ops[sobel_key] = { .output = grad, .input = blurred };
715 }
716
717 const GpuComputeConfig nms_cfg {
718 .shader_path = "canny_nms.comp.spv",
719 .workgroup_size = k_wg2d,
720 };
721 pixel_ctx.swap_shader(nms_cfg);
722 pixel_ctx.stage_image(grad);
723 pixel_ctx.prepare_output_image(w, h);
724 {
725 const auto f = pixel_ctx.dispatch_async({});
726 foundry.wait_for_fence(f);
727 foundry.release_fence(f);
728 }
729 auto suppressed = pixel_ctx.get_output_image(0);
730
731 const auto classify_cfg = config(VisionOp::Canny, step.params);
732 pixel_ctx.swap_shader(classify_cfg);
733 pixel_ctx.stage_image(suppressed);
734 pixel_ctx.set_push_constants(ClassifyPC { .threshold = p.low_threshold, .value = 0.5F });
735 pixel_ctx.prepare_output_image(w, h);
736 {
737 const auto f = pixel_ctx.dispatch_async({});
738 foundry.wait_for_fence(f);
739 foundry.release_fence(f);
740 }
741 auto classified_weak = pixel_ctx.get_output_image(0);
742
743 pixel_ctx.stage_image(classified_weak);
744 pixel_ctx.set_push_constants(ClassifyPC { .threshold = p.high_threshold, .value = 1.0F });
745 pixel_ctx.prepare_output_image(w, h);
746 {
747 const auto f = pixel_ctx.dispatch_async({});
748 foundry.wait_for_fence(f);
749 foundry.release_fence(f);
750 }
751 auto classified = pixel_ctx.get_output_image(0);
752
753 constexpr uint32_t k_max_hysteresis_rounds = 64;
754 label_ctx.set_output_size(2, sizeof(uint32_t));
755 label_ctx.set_output_dimensions(w, h);
756 label_ctx.swap_shader({
757 .shader_path = "canny_hysteresis.comp.spv",
758 .workgroup_size = k_wg2d,
759 .push_constant_size = sizeof(HysteresisPC),
760 });
761 label_ctx.stage_image_at(0, classified, GpuBufferBinding::ElementType::IMAGE_STORAGE);
762 label_ctx.slot_binding(0).direction = GpuBufferBinding::Direction::INPUT_OUTPUT;
763 {
764 uint32_t zero = 0;
765 label_ctx.set_binding_data(2, std::span<const uint32_t>(&zero, 1));
766 const HysteresisPC hpc { .width = w, .height = h };
767 ExecutionContext chained_ctx;
768 chained_ctx.mode = ExecutionMode::CHAINED;
769 chained_ctx.parameters = ChainedParams {
770 .pass_count = k_max_hysteresis_rounds,
771 .pc_updater = [hpc](uint32_t, void* dst) { std::memcpy(dst, &hpc, sizeof(HysteresisPC)); },
772 .passes_per_batch = k_max_hysteresis_rounds,
773 };
774 label_ctx.execute(Datum<> {}, chained_ctx);
775 }
776 label_ctx.slot_binding(0).direction = GpuBufferBinding::Direction::OUTPUT;
777 auto hysteresis_result = classified;
778
779 const auto finalize_cfg = config_from_spec(
781 .storage_image("out", BindingDirection::Output)
782 .storage_image("src", BindingDirection::Input)
783 .pc("threshold")
784 .op(KernelOp::CompareGE)
785 .workgroup(k_wg2d[0], k_wg2d[1])
786 .build());
787 pixel_ctx.swap_shader(finalize_cfg);
788 pixel_ctx.stage_image(hysteresis_result);
789 pixel_ctx.set_push_constants(FinalizePC { .threshold = 1.0F });
790 pixel_ctx.prepare_output_image(w, h);
791 {
792 const auto f = pixel_ctx.dispatch_async({});
793 foundry.wait_for_fence(f);
794 foundry.release_fence(f);
795 }
796 auto finalized = pixel_ctx.get_output_image(0);
797
798 completed_ops[Kinesis::Vision::hash_vision_step(step.op, step.params)] = { .output = finalized, .input = canny_input };
799
800 result.debug_labels = finalized;
801 current = finalized;
802 last_staged = current;
803 result.structured = std::monostate {};
804 continue;
805 }
806 case VisionOp::HarrisResponse: {
807 const auto& p = std::get<HarrisParams>(step.params);
808 const auto radius = static_cast<uint32_t>(std::ceil(p.sigma * 3.0F));
809 const auto& weights = gaussian_kernel_1d(radius, p.sigma);
810
811 auto harris_input = current;
812 pixel_ctx.swap_shader({ .shader_path = "harris_grad_pack.comp.spv", .workgroup_size = k_wg2d });
813 pixel_ctx.stage_image(current);
814 pixel_ctx.prepare_output_image(w, h);
815 {
816 const auto f = pixel_ctx.dispatch_async({});
817 foundry.wait_for_fence(f);
818 foundry.release_fence(f);
819 }
820 auto packed = pixel_ctx.get_output_image(0);
821
822 const auto blur_cfg = config(VisionOp::GaussianBlur, GaussianBlurParams { .sigma = p.sigma });
823 pixel_ctx.swap_shader(blur_cfg);
824 pixel_ctx.stage_image(packed);
825 pixel_ctx.set_binding_data(2, std::span<const float>(weights));
826 pixel_ctx.set_push_constants(GaussianPC { .radius = radius, .width = w, .height = h });
827 pixel_ctx.prepare_output_image(w, h);
828 {
829 const auto f = pixel_ctx.dispatch_async({});
830 foundry.wait_for_fence(f);
831 foundry.release_fence(f);
832 }
833 auto smoothed = pixel_ctx.get_output_image(0);
834
835 const GpuComputeConfig harris_resp_cfg {
836 .shader_path = "harris_response.comp.spv",
837 .workgroup_size = k_wg2d,
838 .push_constant_size = sizeof(HarrisPC),
839 };
840 pixel_ctx.swap_shader(harris_resp_cfg);
841 pixel_ctx.stage_image(smoothed);
842 pixel_ctx.set_push_constants(HarrisPC { .k = p.k, .pass = 0U, .width = w, .height = h });
843 pixel_ctx.prepare_output_image(w, h);
844 {
845 const auto f = pixel_ctx.dispatch_async({});
846 foundry.wait_for_fence(f);
847 foundry.release_fence(f);
848 }
849
850 pixel_ctx.set_push_constants(HarrisPC { .k = p.k, .pass = 1U, .width = w, .height = h });
851 {
852 const auto f = pixel_ctx.dispatch_async({});
853 foundry.wait_for_fence(f);
854 foundry.release_fence(f);
855 }
856 current = pixel_ctx.get_output_image(0);
857 completed_ops[Kinesis::Vision::hash_vision_step(step.op, step.params)] = { .output = current, .input = harris_input };
858
859 result.structured = std::monostate {};
860 continue;
861 }
862 case VisionOp::ExtractPeaks: {
863 const auto& p = std::get<ExtractPeaksParams>(step.params);
864 constexpr uint32_t k_max_kp = 4096;
865
866 structured_ctx.swap_shader({
867 .shader_path = "extract_peaks.comp.spv",
868 .workgroup_size = { 8, 8, 1 },
869 .push_constant_size = sizeof(ExtractPeaksPC),
870 });
871
872 structured_ctx.set_output_size(1, sizeof(uint32_t));
873 structured_ctx.set_output_size(2, static_cast<size_t>(k_max_kp) * 4 * sizeof(float));
874
875 structured_ctx.stage_image(current);
876 structured_ctx.set_push_constants(ExtractPeaksPC {
877 .threshold = p.threshold,
878 .nms_radius = p.nms_radius,
879 .width = w,
880 .height = h,
881 .max_keypoints = k_max_kp,
882 });
883
884 structured_ctx.set_output_dimensions(w, h);
885 const auto fence = structured_ctx.dispatch_async({});
886 structured_ctx.clear_output_dimensions();
887 foundry.wait_for_fence(fence);
888 foundry.release_fence(fence);
889
890 if (sequence.track_follows_peaks) {
891 result.structured = std::monostate {};
892 result.w = w;
893 result.h = h;
894 continue;
895 }
896
897 const auto gpu_result = structured_ctx.collect_result();
898
899 uint32_t count = 0;
900 if (auto it = gpu_result.aux.find(1); it != gpu_result.aux.end())
901 std::memcpy(&count, it->second.data(), sizeof(uint32_t));
902 count = std::min(count, k_max_kp);
903
904 struct GpuKp {
905 float x, y, response, pad;
906 };
907 std::vector<GpuKp> raw(count);
908 if (count > 0) {
909 if (auto it = gpu_result.aux.find(2); it != gpu_result.aux.end())
910 std::memcpy(raw.data(), it->second.data(), count * sizeof(GpuKp));
911 }
912
913 std::vector<Kinesis::Vision::Keypoint> kpts;
914 kpts.reserve(count);
915 for (const auto& kp : raw) {
916 kpts.push_back({ .position = { kp.x, kp.y },
917 .response = kp.response,
918 .scale = 1.0F,
919 .angle = 0.0F });
920 }
921 std::ranges::sort(kpts, [](const auto& a, const auto& b) { return a.response > b.response; });
922
923 result.structured = std::move(kpts);
924 result.w = 0;
925 result.h = 0;
926 continue;
927 }
928 case VisionOp::ConnectedComponents: {
929 const auto& p = std::get<Kinesis::Vision::ConnectedComponentsParams>(step.params);
930
931 if (!current) {
932 continue;
933 }
934
935 const auto seed_input = current;
936
937 const uint32_t block_width = (w + 1U) / 2U;
938 const uint32_t block_height = (h + 1U) / 2U;
939 const double block_diagonal = std::sqrt(
940 static_cast<double>(block_width) * block_width + static_cast<double>(block_height) * block_height);
941 const auto k_compress_passes = static_cast<uint32_t>(std::ceil(std::log2(std::max(2.0, block_diagonal))));
942
943 cc_pipeline.ensure_shared_buffer(0, 2, static_cast<size_t>(block_width) * block_height, GpuBufferBinding::ElementType::UINT32);
944 cc_pipeline.ensure_shared_buffer(0, 3, 1, GpuBufferBinding::ElementType::UINT32);
945 cc_pipeline.ensure_shared_buffer(0, 4, static_cast<size_t>(block_width) * block_height, GpuBufferBinding::ElementType::UINT32);
946 cc_pipeline.ensure_shared_buffer(0, 5, 1, GpuBufferBinding::ElementType::UINT32);
947 cc_pipeline.ensure_shared_buffer(0, 6, static_cast<size_t>(w) * h, GpuBufferBinding::ElementType::UINT32);
948 cc_pipeline.ensure_shared_buffer(0, 7, static_cast<size_t>(k_max_components) * 2, GpuBufferBinding::ElementType::UINT32);
949 cc_pipeline.ensure_shared_buffer(0, 8, static_cast<size_t>(k_max_components) * 2, GpuBufferBinding::ElementType::UINT32);
950 cc_pipeline.ensure_shared_buffer(0, 9, k_max_components, GpuBufferBinding::ElementType::UINT32);
951
952 const CCBlockInitPC init_pc { .width = w, .height = h, .block_width = block_width, .block_height = block_height };
953 const CCMergePC merge_pc { .width = w, .height = h, .block_width = block_width, .block_height = block_height };
954 const CCFinalLabelPC final_pc { .width = w, .height = h, .block_width = block_width, .block_height = block_height, .max_components = k_max_components, .export_labels = 1U };
955
956 cc_pipeline.swap_shader({ .shader_path = "cc_reset.comp.spv", .workgroup_size = { 256, 1, 1 }, .push_constant_size = sizeof(CCResetPC) });
957 cc_pipeline.set_push_constants(CCResetPC {
958 .lut_size = block_width * block_height,
959 .max_components = k_max_components,
960 });
961 cc_pipeline.set_output_dimensions(std::max(block_width * block_height, k_max_components), 1);
962 {
963 const auto reset_fence = cc_pipeline.dispatch_async({});
964 foundry.wait_for_fence(reset_fence);
965 foundry.release_fence(reset_fence);
966 }
967 cc_pipeline.clear_output_dimensions();
968
969 const std::array<uint32_t, 3> block_groups {
970 (block_width + k_wg2d[0] - 1U) / k_wg2d[0],
971 (block_height + k_wg2d[1] - 1U) / k_wg2d[1],
972 1U
973 };
974
975 std::vector<DependencyStage> cc_stages;
976
977 cc_stages.push_back({
978 .config = { .shader_path = "cc_block_init.comp.spv", .workgroup_size = k_wg2d, .push_constant_size = sizeof(CCBlockInitPC) },
979 .stage_fn = [&](GpuDispatchCore& ctx) {
980 cc_pipeline.stage_image_at(1, seed_input, GpuBufferBinding::ElementType::IMAGE_STORAGE);
981 ctx.set_push_constants(init_pc);
982 cc_pipeline.set_output_dimensions(block_width, block_height); },
983 .hazard_fn = [&](GpuDispatchCore& ctx) -> std::vector<Portal::Graphics::HazardResource> {
984 return {
985 ctx.shared_buffer_hazard(
986 { .set = 0, .binding = 2, .direction = GpuBufferBinding::Direction::INPUT_OUTPUT, .element_type = GpuBufferBinding::ElementType::UINT32 }),
987 };
988 },
989 .explicit_groups = block_groups,
990 });
991
992 cc_stages.push_back({
993 .config = { .shader_path = "cc_merge.comp.spv", .workgroup_size = k_wg2d, .push_constant_size = sizeof(CCMergePC) },
994 .stage_fn = [&](GpuDispatchCore& ctx) {
995 cc_pipeline.stage_image_at(1, seed_input, GpuBufferBinding::ElementType::IMAGE_STORAGE);
996 ctx.set_push_constants(merge_pc);
997 cc_pipeline.set_output_dimensions(block_width, block_height); },
998 .hazard_fn = [&](GpuDispatchCore& ctx) -> std::vector<Portal::Graphics::HazardResource> {
999 return {
1000 ctx.shared_buffer_hazard(
1001 { .set = 0, .binding = 2, .direction = GpuBufferBinding::Direction::INPUT_OUTPUT, .element_type = GpuBufferBinding::ElementType::UINT32 }),
1002 };
1003 },
1004 .explicit_groups = block_groups,
1005 });
1006
1007 ExecutionContext cc_ctx;
1008 cc_ctx.mode = ExecutionMode::DEPENDENCY;
1009 DependencyParams params;
1010 params.stages = cc_stages;
1011 cc_ctx.parameters = params;
1012 cc_pipeline.execute(Datum<> {}, cc_ctx);
1013
1014 cc_pipeline.ensure_shared_buffer(0, 10, 3, GpuBufferBinding::ElementType::UINT32,
1016 const std::array<uint32_t, 3> full_grid_indirect {
1017 (block_width + 7U) / 8U,
1018 (block_height + 7U) / 8U,
1019 1U
1020 };
1021 cc_pipeline.upload_shared_raw(0, 10, reinterpret_cast<const uint8_t*>(full_grid_indirect.data()), full_grid_indirect.size() * sizeof(uint32_t));
1022
1023 cc_pipeline.swap_shader({ .shader_path = "cc_compress.comp.spv", .workgroup_size = k_wg2d, .push_constant_size = sizeof(CCCompressPC) });
1024 cc_pipeline.set_push_constants(CCCompressPC { .block_width = block_width, .block_height = block_height });
1025 cc_pipeline.set_output_dimensions(block_width, block_height);
1026 {
1027 const auto fence = cc_pipeline.dispatch_async({});
1028 foundry.wait_for_fence(fence);
1029 foundry.release_fence(fence);
1030 }
1031 cc_pipeline.clear_output_dimensions();
1032
1033 cc_pipeline.swap_shader({ .shader_path = "cc_final_label.comp.spv", .workgroup_size = k_wg2d, .push_constant_size = sizeof(CCFinalLabelPC) });
1034 cc_pipeline.stage_image_at(1, seed_input, GpuBufferBinding::ElementType::IMAGE_STORAGE);
1035 cc_pipeline.set_push_constants(final_pc);
1036 cc_pipeline.set_output_dimensions(w, h);
1037 cc_pipeline.prepare_output_image(w, h);
1038 {
1039 const auto fence = cc_pipeline.dispatch_async({});
1040 foundry.wait_for_fence(fence);
1041 foundry.release_fence(fence);
1042 }
1043 cc_pipeline.clear_output_dimensions();
1044
1045 result.debug_labels = p.with_colors ? cc_pipeline.get_output_image(0) : nullptr;
1046
1047 uint32_t compact_count = 0;
1048 cc_pipeline.download_shared(0, 5, &compact_count, sizeof(uint32_t));
1049 compact_count = std::min(compact_count, k_max_components);
1050
1052 cc_result.count = compact_count;
1053 cc_result.boxes.reserve(compact_count);
1054
1055 if (compact_count > 0) {
1056 std::vector<glm::uvec2> bmin(compact_count);
1057 std::vector<glm::uvec2> bmax(compact_count);
1058 std::vector<uint32_t> bcount(compact_count);
1059 cc_pipeline.download_shared(0, 7, bmin.data(), bmin.size() * sizeof(glm::uvec2));
1060 cc_pipeline.download_shared(0, 8, bmax.data(), bmax.size() * sizeof(glm::uvec2));
1061 cc_pipeline.download_shared(0, 9, bcount.data(), bcount.size() * sizeof(uint32_t));
1062
1063 const float inv_w = 1.0F / static_cast<float>(w);
1064 const float inv_h = 1.0F / static_cast<float>(h);
1065
1066 for (uint32_t i = 0; i < compact_count; ++i) {
1067 if (bcount[i] == 0)
1068 continue;
1069 const float x = static_cast<float>(bmin[i].x) * inv_w;
1070 const float y = static_cast<float>(bmin[i].y) * inv_h;
1071 const float bw = static_cast<float>(bmax[i].x - bmin[i].x + 1) * inv_w;
1072 const float bh = static_cast<float>(bmax[i].y - bmin[i].y + 1) * inv_h;
1073 cc_result.boxes.push_back({ .x = x, .y = y, .w = bw, .h = bh, .confidence = 1.0F, .label_id = i + 1 });
1074 }
1075 }
1076
1077 result.structured = std::move(cc_result);
1078 result.w = 0;
1079 result.h = 0;
1080 continue;
1081 }
1082 case VisionOp::FindContours: {
1083 if (!sequence.contours_follow_cc) {
1085 "run_gpu: FindContours requires ConnectedComponents(export_labels=true) as the immediately preceding step");
1086 return VisionResult {};
1087 }
1088
1089 const auto& p = std::get<Kinesis::Vision::FindContoursParams>(step.params);
1090
1091 cc_pipeline.ensure_shared_buffer(1, 4, static_cast<size_t>(k_max_components) + 1U, GpuBufferBinding::ElementType::UINT32);
1092 cc_pipeline.ensure_shared_buffer(1, 5, static_cast<size_t>(k_max_components) * k_max_holes_per_label, GpuBufferBinding::ElementType::UINT32);
1093 cc_pipeline.ensure_shared_buffer(1, 6, static_cast<size_t>(k_max_trace_slots) * 2U, GpuBufferBinding::ElementType::UINT32);
1094 cc_pipeline.ensure_shared_buffer(1, 7, 1U, GpuBufferBinding::ElementType::UINT32);
1095 cc_pipeline.ensure_shared_buffer(1, 8, static_cast<size_t>(k_max_trace_slots) * k_max_points_per_contour * 2U, GpuBufferBinding::ElementType::FLOAT32);
1096 cc_pipeline.ensure_shared_buffer(1, 9, 1U, GpuBufferBinding::ElementType::UINT32);
1097 cc_pipeline.ensure_shared_buffer(1, 10, static_cast<size_t>(k_max_trace_slots) * 4U, GpuBufferBinding::ElementType::UINT32);
1098 cc_pipeline.ensure_shared_buffer(1, 11, static_cast<size_t>(k_max_trace_slots) * 2U, GpuBufferBinding::ElementType::FLOAT32);
1099 cc_pipeline.ensure_shared_buffer(2, 0, k_max_components, GpuBufferBinding::ElementType::FLOAT32);
1100 cc_pipeline.ensure_shared_buffer(2, 1, k_max_components, GpuBufferBinding::ElementType::FLOAT32);
1101
1102 {
1103 std::vector<uint32_t> owner_reset(static_cast<size_t>(k_max_components) + 1U, CC_UNCLAIMED_HOST);
1104 cc_pipeline.upload_shared_raw(1, 4, reinterpret_cast<const uint8_t*>(owner_reset.data()), owner_reset.size() * sizeof(uint32_t));
1105
1106 std::vector<uint32_t> hole_owner_reset(static_cast<size_t>(k_max_components) * k_max_holes_per_label, CC_UNCLAIMED_HOST);
1107 cc_pipeline.upload_shared_raw(1, 5, reinterpret_cast<const uint8_t*>(hole_owner_reset.data()), hole_owner_reset.size() * sizeof(uint32_t));
1108
1109 const uint32_t zero = 0;
1110 cc_pipeline.upload_shared_raw(1, 7, reinterpret_cast<const uint8_t*>(&zero), sizeof(uint32_t));
1111 cc_pipeline.upload_shared_raw(1, 9, reinterpret_cast<const uint8_t*>(&zero), sizeof(uint32_t));
1112 }
1113
1114 auto max_points = p.max_points_per_contour > 0 ? std::min<uint32_t>(p.max_points_per_contour, k_max_points_per_contour) : k_max_points_per_contour;
1115 cc_pipeline.swap_shader({ .shader_path = "contour_march.comp.spv", .workgroup_size = k_wg2d, .push_constant_size = sizeof(ContourMarchPC) });
1116 cc_pipeline.stage_image_at(1, image, GpuBufferBinding::ElementType::IMAGE_SAMPLED);
1117 cc_pipeline.prepare_output_image(w, h);
1118 cc_pipeline.set_push_constants(ContourMarchPC {
1119 .width = w,
1120 .height = h,
1121 .max_components = k_max_components,
1122 .max_points_per_contour = max_points,
1123 .max_holes_per_label = k_max_holes_per_label,
1124 .phase = 0U,
1125 .min_area = p.min_area,
1126 .compacted_count = 0U });
1127
1128 cc_pipeline.set_output_dimensions(w, h);
1129 {
1130 const auto fence = cc_pipeline.dispatch_async({});
1131 foundry.wait_for_fence(fence);
1132 foundry.release_fence(fence);
1133 }
1134
1135 cc_pipeline.set_push_constants(ContourMarchPC {
1136 .width = w,
1137 .height = h,
1138 .max_components = k_max_components,
1139 .max_points_per_contour = max_points,
1140 .max_holes_per_label = k_max_holes_per_label,
1141 .phase = 1U,
1142 .min_area = p.min_area,
1143 .compacted_count = 0U });
1144 {
1145 const auto fence = cc_pipeline.dispatch_async({});
1146 foundry.wait_for_fence(fence);
1147 foundry.release_fence(fence);
1148 }
1149
1150 cc_pipeline.clear_output_dimensions();
1151
1152 cc_pipeline.swap_shader({ .shader_path = "contour_compact.comp.spv", .workgroup_size = { 256, 1, 1 }, .push_constant_size = sizeof(ContourCompactPC) });
1153 cc_pipeline.set_push_constants(ContourCompactPC { .max_components = k_max_components, .max_holes_per_label = k_max_holes_per_label });
1154 const uint32_t total_owner_slots = k_max_components * (1U + k_max_holes_per_label);
1155 cc_pipeline.set_output_dimensions(total_owner_slots, 1);
1156 {
1157 const auto fence = cc_pipeline.dispatch_async({});
1158 foundry.wait_for_fence(fence);
1159 foundry.release_fence(fence);
1160 }
1161 cc_pipeline.clear_output_dimensions();
1162
1163 uint32_t compacted_count = 0;
1164 cc_pipeline.download_shared(1, 7, &compacted_count, sizeof(uint32_t));
1165
1166 cc_pipeline.swap_shader({ .shader_path = "contour_march.comp.spv", .workgroup_size = { 256, 1, 1 }, .push_constant_size = sizeof(ContourMarchPC) });
1167 cc_pipeline.set_push_constants(ContourMarchPC {
1168 .width = w,
1169 .height = h,
1170 .max_components = k_max_components,
1171 .max_points_per_contour = max_points,
1172 .max_holes_per_label = k_max_holes_per_label,
1173 .phase = 2U,
1174 .min_area = p.min_area,
1175 .compacted_count = compacted_count });
1176 cc_pipeline.set_output_dimensions(std::max(compacted_count, 1U), 1);
1177 {
1178 const auto fence = cc_pipeline.dispatch_async({});
1179 foundry.wait_for_fence(fence);
1180 foundry.release_fence(fence);
1181 }
1182 cc_pipeline.clear_output_dimensions();
1183
1184 if (p.max_contours > 0U) {
1185 constexpr uint32_t k = 12U;
1186 constexpr uint32_t total_passes = k * (k + 1U) / 2U;
1187
1188 cc_pipeline.swap_shader(config_from_spec(
1190 .tmpl(KernelTemplate::BitonicSort)
1191 .start_set(2)
1192 .ssbo("keys", BindingDirection::InOut, Kakshya::GpuDataFormat::FLOAT32)
1193 .ssbo("indices", BindingDirection::InOut, Kakshya::GpuDataFormat::FLOAT32)
1197 .pc("descending", Kakshya::GpuDataFormat::UINT32)
1198 .workgroup(256)
1199 .build()));
1200
1201 cc_pipeline.set_output_dimensions(k_max_components, 1U);
1202
1203 ExecutionContext bitonic_ctx;
1204 bitonic_ctx.mode = ExecutionMode::CHAINED;
1205 bitonic_ctx.parameters = ChainedParams {
1206 .pass_count = total_passes,
1207 .pc_updater = [k](uint32_t p_idx, void* pc_ptr) {
1208 uint32_t stage = 0, pass = 0, remaining = p_idx;
1209 for (uint32_t s = 0; s < k; ++s) {
1210 if (remaining <= s) {
1211 stage = s;
1212 pass = remaining;
1213 break;
1214 }
1215 remaining -= (s + 1);
1216 }
1217 struct PC {
1218 uint32_t stage, pass, count, descending;
1219 };
1220 *static_cast<PC*>(pc_ptr) = { .stage = stage, .pass = pass, .count = k_max_components, .descending = 1U };
1221 },
1222 };
1223
1224 cc_pipeline.execute(Datum<std::vector<Kakshya::DataVariant>> {}, bitonic_ctx);
1225 cc_pipeline.clear_output_dimensions();
1226 }
1227
1228 if (p.as_image) {
1229 cc_pipeline.swap_shader({ .shader_path = "contour_render_clear.comp.spv", .workgroup_size = k_wg2d, .push_constant_size = sizeof(ContourClearPC) });
1230 cc_pipeline.prepare_output_image(w, h);
1231 cc_pipeline.set_push_constants(ContourClearPC { .width = w, .height = h });
1232 cc_pipeline.set_output_dimensions(w, h);
1233 {
1234 const auto fence = cc_pipeline.dispatch_async({});
1235 foundry.wait_for_fence(fence);
1236 foundry.release_fence(fence);
1237 }
1238 cc_pipeline.clear_output_dimensions();
1239
1240 cc_pipeline.swap_shader({ .shader_path = "contour_render.comp.spv", .workgroup_size = { 256, 1, 1 }, .push_constant_size = sizeof(ContourRenderPC) });
1241 cc_pipeline.set_push_constants(ContourRenderPC { .width = w, .height = h, .max_components = k_max_components, .max_points_per_contour = k_max_points_per_contour, .max_contours = p.max_contours });
1242 const uint32_t render_slots = p.max_contours > 0U ? std::min(p.max_contours, k_max_components) : k_max_trace_slots;
1243 cc_pipeline.set_output_dimensions(render_slots * k_max_points_per_contour, 1U);
1244 {
1245 const auto fence = cc_pipeline.dispatch_async({});
1246 foundry.wait_for_fence(fence);
1247 foundry.release_fence(fence);
1248 }
1249 cc_pipeline.clear_output_dimensions();
1250
1251 result.debug_contours = cc_pipeline.get_output_image(0);
1252 result.structured = std::monostate {};
1253 result.w = 0;
1254 result.h = 0;
1255 continue;
1256 }
1257
1258 std::vector<glm::uvec4> meta(compacted_count);
1259 cc_pipeline.download_shared(1, 10, meta.data(), compacted_count * sizeof(glm::uvec4));
1260 std::vector<glm::vec2> area_perim(compacted_count);
1261 cc_pipeline.download_shared(1, 11, area_perim.data(), compacted_count * sizeof(glm::vec2));
1262 uint32_t points_written = 0;
1263 cc_pipeline.download_shared(1, 9, &points_written, sizeof(uint32_t));
1264 std::vector<glm::vec2> flat_points_full(points_written);
1265
1266 if (points_written > 0)
1267 cc_pipeline.download_shared(1, 8, flat_points_full.data(), static_cast<size_t>(points_written) * sizeof(glm::vec2));
1268
1269 std::vector<uint32_t> order;
1270 if (p.max_contours > 0U) {
1271 const uint32_t take = std::min(p.max_contours, compacted_count);
1272 std::vector<float> sorted_indices(take);
1273 cc_pipeline.download_shared(2, 1, sorted_indices.data(), take * sizeof(float));
1274 order.reserve(take);
1275 for (float f : sorted_indices)
1276 order.push_back(static_cast<uint32_t>(f));
1277 } else {
1278 order.resize(compacted_count);
1279 for (uint32_t i = 0; i < compacted_count; ++i)
1280 order[i] = i;
1281 }
1282
1283 std::vector<Kinesis::Vision::Contour> out_contours;
1284 out_contours.reserve(order.size());
1285
1286 for (uint32_t idx : order) {
1287 if (idx >= compacted_count)
1288 continue;
1289 const auto& m = meta[idx];
1290 if (m.y < 3)
1291 continue;
1292
1293 std::vector<glm::vec2> pts(
1294 flat_points_full.begin() + m.x,
1295 flat_points_full.begin() + m.x + m.y);
1296 const glm::vec2 ap = area_perim[idx];
1297 out_contours.push_back({ .points = std::move(pts), .area = ap.x, .perimeter = ap.y, .parent_label = m.z });
1298 }
1299
1300 result.structured = std::move(out_contours);
1301 result.w = 0;
1302 result.h = 0;
1303 continue;
1304 }
1305 default:
1306 break;
1307 }
1308
1309 auto dispatch_input = current;
1310 const auto fence = pixel_ctx.dispatch_async({});
1311 foundry.wait_for_fence(fence);
1312 foundry.release_fence(fence);
1313
1314 pixel_ctx.clear_output_dimensions();
1315 current = pixel_ctx.get_output_image(0);
1316 last_staged = current;
1317 completed_ops[Kinesis::Vision::hash_vision_step(step.op, step.params)] = { .output = current, .input = dispatch_input };
1318 }
1319
1320 return result;
1321}
1322
1323VisionResult VisionGpuExecutor::run(
1324 const VisionSequence& sequence,
1325 const std::shared_ptr<Core::VKImage>& image,
1326 uint32_t w, uint32_t h)
1327{
1328 if (!m_contexts)
1329 m_contexts = std::make_unique<VisionGpuContexts>();
1330
1331 return run(*m_contexts, sequence, image, w, h);
1332}
1333
1334} // namespace MayaFlux::Yantra
#define MF_ERROR(comp, ctx,...)
vk::Fence fence
Core::GlobalInputConfig input
Definition Config.cpp:38
IO::ImageData image
Definition Decoder.cpp:64
uint32_t width
Definition Decoder.cpp:66
glm::vec2 current
uint32_t h
Definition InkPress.cpp:28
size_t a
size_t b
size_t count
uint32_t block_width
float value
float scale
float lo
uint32_t max_holes_per_label
std::shared_ptr< Core::VKImage > output
float threshold
float offset
uint32_t max_keypoints
uint32_t block_height
uint32_t phase
uint32_t nms_radius
uint32_t height
float wb
uint32_t radius
uint32_t max_points_per_contour
uint32_t pass
float wg
uint32_t max_components
uint32_t block_size
float k
float min_area
uint32_t export_labels
float wa
uint32_t compacted_count
float sigma
uint32_t max_contours
float wr
float hi
uint32_t max_segments
uint32_t lut_size
GPU execution layer for Kinesis::Vision::VisionSequence.
Assemble & op(KernelOp o)
Set the named operation the emitter will lower to SPIR-V.
ShaderSpec build()
Finalise and return the ShaderSpec.
Assemble & pc(std::string name, Kakshya::GpuDataFormat format)
Declare a push constant field with explicit format.
Assemble & ssbo(std::string name, BindingDirection direction, Kakshya::GpuDataFormat format, Kakshya::DataModality modality=Kakshya::DataModality::SCALAR_F32)
Declare an SSBO binding.
Assemble & workgroup(uint32_t x, uint32_t y=1, uint32_t z=1)
Override workgroup size.
Assemble & tmpl(KernelTemplate t)
Set the kernel template.
Assemble & storage_image(std::string name, BindingDirection direction=BindingDirection::Output)
Declare a storage image binding (image2D).
Fluent assembler producing a ShaderSpec.
Non-template base that owns all type-independent GPU dispatch logic.
GpuExecutionContext specialisation for image compute shaders.
void run()
Definition main.cpp:22
@ ComputeMatrix
Compute operations (Yantra - algorithms, matrices, DSP)
@ Yantra
DSP algorithms, computational units, matrix operations, Grammar.
std::vector< double > sum(std::span< const double > data, size_t n_windows, uint32_t hop_size, uint32_t window_size)
Sum per window.
Definition Analysis.cpp:469
size_t hash_vision_step(VisionOp op, const VisionParams &params)
Hash a VisionStep's op and parameters together.
Definition VisionOp.hpp:369
std::variant< std::monostate, ThresholdParams, ThresholdAdaptiveParams, NormalizeRangeParams, GaussianBlurParams, FilterSeparableParams, CannyParams, MorphParams, HarrisParams, ExtractPeaksParams, TrackKeypointsParams, ConnectedComponentsParams, FindContoursParams > VisionParams
Parameter variant covering all ops that carry parameters.
Definition VisionOp.hpp:148
VisionOp
Named operations available in a VisionSequence.
Definition VisionOp.hpp:29
constexpr ShaderID INVALID_SHADER
MAYAFLUX_API ShaderFoundry & get_shader_foundry()
Get the global shader compiler instance.
ImageFormat
User-friendly image format enum.
@ RGBA32F
Four channel 32-bit float.
@ INDIRECT
Indirect draw/dispatch buffer (device-local)
Portal::Graphics::GpuComputeConfig GpuComputeConfig
GpuComputeConfig config_from_spec(const Portal::Graphics::ShaderSpec &spec)
Derive a GpuComputeConfig from a ShaderSpec.
static constexpr DomainSpec Graphics
Domain constant for Graphics domain.
Definition Creator.hpp:308
std::vector< BoundingBox > boxes
One BoundingBox per component, label 1..count.
uint32_t count
Number of components found.
Result of connected component labelling.
std::shared_ptr< Core::VKImage > debug_labels
std::shared_ptr< Core::VKImage > debug_contours
Result of executing a VisionSequence on one frame.
Ordered sequence of VisionSteps describing a complete vision pipeline.
Definition VisionOp.hpp:163
enum MayaFlux::Portal::Graphics::GpuBufferBinding::Direction INPUT
enum MayaFlux::Portal::Graphics::GpuBufferBinding::ElementType FLOAT32
Declares a single storage buffer or image binding a compute shader expects.
Plain-data description of the compute shader to dispatch.
Parameters for ExecutionMode::CHAINED.
Input/Output container for computation pipeline data flow with structure preservation.
Definition DataIO.hpp:24
std::vector< DependencyStage > stages
Parameters for ExecutionMode::DEPENDENCY.
ExecutionMode mode
Execution mode controlling scheduling behavior.
ExecutionParams parameters
Optional parameters specific to the execution mode.
Context information controlling how a compute operation executes.
TextureExecutionContext labels
Image + aux SSBO.
VisionGpuContexts()
Construct all three contexts in place with the one correct binding layout for every currently GPU-imp...
TextureExecutionContext pixel
Image pipeline.
TextureExecutionContext structured
Buffer-only readback.
Fixed set of TextureExecutionContexts covering every GPU-implemented VisionOp shape.