13 struct TrackReducePC {
24 struct RegionSamplePC {
28 [[nodiscard]]
bool inside(
const Kinesis::Vision::BoundingBox& box,
const glm::vec2& p)
30 return p.x >= box.x && p.x <= box.x + box.w
31 && p.y >= box.y && p.y <= box.y + box.h;
36 std::optional<std::vector<Kinesis::Vision::TrackResult>>& raw_tracks,
37 std::optional<std::vector<Kinesis::Vision::Keypoint>>& raw_keypoints,
38 Kinesis::Vision::VisionAnalysis& analysis,
39 const Kinesis::Vision::VisionResult& result)
41 if (
const auto* tracks = std::get_if<std::vector<Kinesis::Vision::TrackResult>>(&result.structured))
44 if (
const auto* keypoints = std::get_if<std::vector<Kinesis::Vision::Keypoint>>(&result.structured))
45 raw_keypoints = *keypoints;
47 if (
const auto* contours = std::get_if<std::vector<Kinesis::Vision::Contour>>(&result.structured)) {
48 analysis.find_elements = Kinesis::Vision::FindElementsAnalysis {
49 .contours = *contours,
50 .boxes = result.component_boxes,
55 analysis.estimate_motion = result.flow;
56 analysis.motion_activity = result.motion_energy;
59 if (result.images.canny)
60 analysis.detect_edges = result.images.canny;
77 m_results = std::make_shared<const std::vector<Kinesis::Vision::VisionResult>>();
88 const std::vector<Kinesis::Vision::TrackResult>& tracks, uint32_t w, uint32_t
h)
96 std::vector<float> track_data;
97 track_data.reserve(tracks.size() * 4);
98 std::vector<float> extra_data;
99 extra_data.reserve(tracks.size() * 2);
100 for (
const auto& t : tracks) {
101 track_data.push_back(t.position.x);
102 track_data.push_back(t.position.y);
103 track_data.push_back(t.previous.x);
104 track_data.push_back(t.previous.y);
105 extra_data.push_back(
static_cast<float>(t.age));
106 extra_data.push_back(t.tracked ? 1.0F : 0.0F);
113 .workgroup_size = { 256, 1, 1 },
114 .push_constant_size =
sizeof(TrackReducePC) });
118 .output(1, 3 *
sizeof(glm::vec4))
119 .
input(2, extra_data)
120 .push(TrackReducePC { .count =
static_cast<uint32_t
>(tracks.size()) });
125 if (raw.size() >= 12) {
126 const auto n =
static_cast<float>(tracks.size());
127 result.
centroid = { raw[0] / n, raw[1] / n };
128 result.
velocity = { raw[2] / n, raw[3] / n };
132 .w = raw[6] - raw[4],
133 .h = raw[7] - raw[5],
136 result.
velocity.x *
static_cast<float>(w),
137 result.
velocity.y *
static_cast<float>(
h) });
139 result.
max_age =
static_cast<uint32_t
>(std::lround(raw[9]));
140 result.
tracked_count =
static_cast<uint32_t
>(std::lround(raw[10]));
148 const std::vector<Kinesis::Vision::Keypoint>& keypoints)
153 if (keypoints.empty())
156 std::vector<float> point_data;
157 point_data.reserve(keypoints.size() * 4);
158 std::vector<float> extra_data;
159 extra_data.reserve(keypoints.size() * 2);
160 for (
const auto& k : keypoints) {
161 point_data.push_back(k.position.x);
162 point_data.push_back(k.position.y);
163 point_data.push_back(k.position.x);
164 point_data.push_back(k.position.y);
165 extra_data.push_back(k.response);
166 extra_data.push_back(0.0F);
173 .workgroup_size = { 256, 1, 1 },
174 .push_constant_size =
sizeof(TrackReducePC) });
178 .output(1, 3 *
sizeof(glm::vec4))
179 .
input(2, extra_data)
180 .push(TrackReducePC { .count =
static_cast<uint32_t
>(keypoints.size()) });
185 if (raw.size() >= 12) {
186 const auto n =
static_cast<float>(keypoints.size());
187 result.
centroid = { raw[0] / n, raw[1] / n };
192 .w = raw[6] - raw[4],
193 .h = raw[7] - raw[5],
202 const auto n = fe.
boxes.size();
208 const auto fw =
static_cast<float>(w);
209 const auto fh =
static_cast<float>(
h);
211 std::vector<float> element_data;
212 element_data.reserve(n * 4);
213 std::vector<float> centroid_data;
214 centroid_data.reserve(n * 2);
216 for (
const auto&
b : fe.
boxes) {
217 const auto it = std::ranges::find_if(fe.
contours,
218 [&](
const auto& c) { return c.label_id == b.label_id; });
219 const float area_px = (it != fe.
contours.end()) ? it->area * fw * fh : 0.0F;
220 const float perimeter_px = (it != fe.
contours.end()) ? it->perimeter : 0.0F;
221 element_data.push_back(area_px);
222 element_data.push_back(perimeter_px);
223 element_data.push_back(
b.w * fw);
224 element_data.push_back(
b.h * fh);
226 centroid_data.push_back((
b.x +
b.w * 0.5F) * fw);
227 centroid_data.push_back((
b.y +
b.h * 0.5F) * fh);
231 m_shape_ctx = std::make_shared<ShaderExecutionContext<>>(
234 .workgroup_size = { 256, 1, 1 },
235 .push_constant_size =
sizeof(TrackReducePC) });
238 .output(1, n *
sizeof(glm::vec2))
239 .push(TrackReducePC { .count =
static_cast<uint32_t
>(n) });
243 for (
size_t i = 0; i < n && (i + 1) * 2 <= shape_raw.size(); ++i) {
244 fe.
shapes[i].compactness = shape_raw[i * 2];
245 fe.
shapes[i].aspect_ratio = shape_raw[i * 2 + 1];
255 .workgroup_size = { 256, 1, 1 },
256 .push_constant_size =
sizeof(TrackReducePC) });
259 .output(1, n *
sizeof(
float))
260 .push(TrackReducePC { .count =
static_cast<uint32_t
>(n) });
264 for (
size_t i = 0; i < n && i < dist_raw.size(); ++i)
270 const std::shared_ptr<Core::VKImage>& gray,
271 uint32_t w, uint32_t
h)
287 const std::shared_ptr<Core::VKImage>&
image, uint32_t w, uint32_t
h)
293 .workgroup_size = { 256, 1, 1 },
294 .push_constant_size =
sizeof(RegionSamplePC) },
295 Portal::Graphics::ImageFormat::RGBA8,
298 std::vector<GpuBufferBinding> {
311 foundry.wait_for_fence(fence);
312 foundry.release_fence(fence);
317 if (
auto it = gpu_result.aux.find(2); it != gpu_result.aux.end() && it->second.size() >= 3 *
sizeof(glm::vec4)) {
318 std::array<glm::vec4, 3> raw {};
319 std::memcpy(raw.data(), it->second.data(),
sizeof(raw));
321 out.min_val = raw[1];
322 out.max_val = raw[2];
328 const std::shared_ptr<Core::VKImage>&
image, uint32_t w, uint32_t
h)
334 .workgroup_size = { 8, 8, 1 },
335 .push_constant_size =
sizeof(HistogramPC) },
336 Portal::Graphics::ImageFormat::RGBA8,
339 std::vector<GpuBufferBinding> {
347 const std::array<uint32_t, 768> zero {};
348 m_histogram_ctx->upload_shared_raw(0, 3,
reinterpret_cast<const uint8_t*
>(zero.data()),
sizeof(zero));
351 m_histogram_ctx->set_push_constants(HistogramPC { .width = w, .height =
h, .channels = 0 });
357 foundry.wait_for_fence(fence);
358 foundry.release_fence(fence);
360 std::array<uint32_t, 256> histogram {};
361 m_histogram_ctx->download_shared(0, 3, histogram.data(),
sizeof(histogram));
371 .workgroup_size = { 256, 1, 1 },
372 .push_constant_size = 0 });
375 const std::vector<uint32_t> hist_vec(histogram.begin(), histogram.end());
377 .output(1,
sizeof(
float));
381 return raw.empty() ? 0.0F : raw[0];
385 const std::shared_ptr<Kakshya::SignalSourceContainer>& source,
391 if (
auto tc = std::dynamic_pointer_cast<Kakshya::TextureContainer>(source))
392 return tc->to_image(
static_cast<uint32_t
>(
index));
394 if (
auto wc = std::dynamic_pointer_cast<Kakshya::WindowContainer>(source))
395 return wc->to_image();
397 auto vc = std::dynamic_pointer_cast<Kakshya::VideoStreamContainer>(source);
401 const uint32_t w = vc->get_width();
402 const uint32_t
h = vc->get_height();
404 const void* raw = source->get_raw_data();
410 w,
h, Portal::Graphics::ImageFormat::RGBA8,
nullptr);
422 const std::shared_ptr<Kakshya::SignalSourceContainer>& source,
430 if (std::dynamic_pointer_cast<Kakshya::TextureContainer>(source))
431 analysis.layer =
index;
436 const std::shared_ptr<Core::VKImage>&
image)
442 const std::shared_ptr<Core::VKImage>&
image)
454 for (
auto& step : entry.sequence.steps) {
457 step.deferred =
true;
478 m_results = std::make_shared<const std::vector<Kinesis::Vision::VisionResult>>(std::move(
m_run.
results));
486 .width =
image->get_width(),
488 .format = Portal::Graphics::ImageFormat::RGBA32F,
493 if (!held || !loom.can_blit(
image, held, filter) || !loom.blit_layer(
image, held, { .filter = filter }))
499 const std::shared_ptr<Core::VKImage>&
image)
503 std::vector<Kinesis::Vision::VisionResult> results;
504 results.reserve(resolved.size());
507 for (
size_t lane = 0; lane < resolved.size(); ++lane) {
509 results.emplace_back();
513 m_results = std::make_shared<const std::vector<Kinesis::Vision::VisionResult>>(std::move(results));
519 const std::vector<Kinesis::Vision::ResolvedSequence>& resolved,
520 std::vector<Kinesis::Vision::VisionResult>& results,
521 const std::shared_ptr<Core::VKImage>&
image)
526 const auto& entry = resolved[lane];
529 executor = std::make_unique<VisionGpuExecutor>();
531 if (std::ranges::any_of(entry.sequence.steps,
532 [](
const auto& step) { return step.op == Kinesis::Vision::VisionOp::ConnectedComponents; }))
535 const auto seeded = entry.seed_field ? entry.seed_field(results[entry.seed_lane]) :
nullptr;
536 const auto& source = seeded ? seeded :
image;
537 auto result = executor->run(entry.sequence, source, source->get_width(), source->get_height());
542 results.push_back(std::move(result));
547 const std::vector<Kinesis::Vision::ResolvedSequence>& resolved,
548 const std::vector<Kinesis::Vision::VisionResult>& results,
549 uint32_t w, uint32_t
h)
552 std::optional<std::vector<Kinesis::Vision::TrackResult>> raw_tracks;
553 std::optional<std::vector<Kinesis::Vision::Keypoint>> raw_keypoints;
555 for (
size_t lane = 0; lane < results.size(); ++lane) {
556 const auto& result = results[lane];
557 const auto& entry = resolved[lane];
559 collect_into(raw_tracks, raw_keypoints, analysis, result);
561 if (result.images.sobel) {
562 const auto& gray = result.gray || !entry.seed_field ? result.gray : results[entry.seed_lane].gray;
567 const auto& filtering = config<VisionAnalyzerConfig>().context;
569 if (raw_tracks && filtering.track_objects) {
570 const auto& ctx = *filtering.track_objects;
571 if (ctx.subject_track_id) {
572 const auto id = *ctx.subject_track_id;
573 std::erase_if(*raw_tracks, [&](
const auto& t) {
return t.id != id; });
574 }
else if (ctx.subject_bounds) {
575 const auto& bounds = *ctx.subject_bounds;
576 std::erase_if(*raw_tracks, [&](
const auto& t) {
return !inside(bounds, t.position); });
583 if (raw_keypoints && filtering.detect_features_bounds) {
584 const auto& bounds = *filtering.detect_features_bounds;
585 std::erase_if(*raw_keypoints, [&](
const auto& k) {
return !inside(bounds, k.position); });
#define MF_ERROR(comp, ctx,...)
Core::GlobalInputConfig input
std::vector< float > * out
Analyzer<shared_ptr<SignalSourceContainer>, vector<DataVariant>> whose query and context are its conf...
void set(T value)
Stores a described struct under its type identity.
void upload_data(const std::shared_ptr< Core::VKImage > &image, const void *data, size_t size, const std::shared_ptr< Buffers::VKBuffer > &staging=nullptr, bool deferred=false)
Upload pixels to an image.
static TextureLoom & instance()
std::shared_ptr< Core::VKImage > create_2d(uint32_t width, uint32_t height, ImageFormat format=ImageFormat::RGBA8, const void *data=nullptr, uint32_t mip_levels=1)
Create a 2D texture.
const Config & config() const
The stored config struct of type Config, or its defaults when none was set.
void set_config(Config config)
Stores a described config struct, whole, under its own type.
Concrete GpuExecutionContext for a single fixed shader with fixed bindings.
@ SCALAR
Return SSBO readback via collect_result(); no image download.
Kinesis::Vision::MeasureAppearanceAnalysis measure_appearance(const Kinesis::Vision::VisionResult &result, const std::shared_ptr< Core::VKImage > &gray, uint32_t w, uint32_t h)
MeasureAppearance's whole-frame gradient sample (region_sample.comp over result.images....
float compute_mean_brightness(const std::array< uint32_t, 256 > &histogram)
Dispatch histogram_reduce.comp over an already-computed histogram.
std::shared_ptr< Core::VKImage > m_upload_image
std::shared_ptr< ShaderExecutionContext<> > m_shape_ctx
Kinesis::Vision::FieldSample sample_full_frame(const std::shared_ptr< Core::VKImage > &image, uint32_t w, uint32_t h)
Dispatch region_sample.comp over the whole image (no region concept for MeasureAppearance).
VisionAnalyzer(VisionAnalyzerConfig config={})
Construct from the settings, as EnergyAnalyzer does.
void compute_shapes(Kinesis::Vision::FindElementsAnalysis &fe, uint32_t w, uint32_t h)
Shape descriptors (element_shape.comp) and nearest-neighbour distances (nearest_neighbor....
std::shared_ptr< Core::VKImage > m_frame
Kinesis::Vision::VisionAnalysis analyze_resolved(const std::shared_ptr< Core::VKImage > &image)
Shared body of both analyze_vision() overloads, once a GPU image is in hand: resolve the configured q...
bool run_lane(size_t lane, const std::vector< Kinesis::Vision::ResolvedSequence > &resolved, std::vector< Kinesis::Vision::VisionResult > &results, const std::shared_ptr< Core::VKImage > &image)
Run or poll sequence lane of resolved on its executor.
std::shared_ptr< Core::VKImage > hold_frame(const std::shared_ptr< Core::VKImage > &image)
Copy image into this analyzer's own frame image.
Portal::Graphics::ImageCacheEntry m_frame_cache
Kinesis::Vision::DetectFeaturesAnalysis reduce_keypoints(const std::vector< Kinesis::Vision::Keypoint > &keypoints)
Dispatch track_reduce.comp over keypoint positions (fed as their own previous, so velocity reduces to...
std::vector< std::unique_ptr< VisionGpuExecutor > > m_executors
std::shared_ptr< TextureExecutionContext > m_histogram_ctx
std::shared_ptr< ShaderExecutionContext<> > m_neighbor_ctx
std::shared_ptr< const std::vector< Kinesis::Vision::VisionResult > > m_results
void reset()
Abandon outstanding work and clear retained executor state.
VisionGpuExecutor * get_executor() const
The executor that ran the FindElements sequence, or the first executor when the query has none.
std::optional< Kinesis::Vision::VisionAnalysis > advance(const std::shared_ptr< Core::VKImage > &image)
Advance an analysis of image by one sequence per call, without waiting on deferred GPU work.
std::shared_ptr< ShaderExecutionContext<> > m_brightness_reducer
output_type analyze_implementation(const input_type &input) override
Uses input.data as the source container, or a supplied frame facet, and calls analyze_vision().
Kinesis::Vision::VisionAnalysis assemble(const std::vector< Kinesis::Vision::ResolvedSequence > &resolved, const std::vector< Kinesis::Vision::VisionResult > &results, uint32_t w, uint32_t h)
Route every completed sequence result into one VisionAnalysis, apply context filtering and run the GP...
Kinesis::Vision::TrackObjectsAnalysis reduce_tracks(const std::vector< Kinesis::Vision::TrackResult > &tracks, uint32_t w, uint32_t h)
Dispatch track_reduce.comp over tracks and fill centroid/velocity/bounds/mean_speed_px/age stats on t...
std::shared_ptr< Core::VKImage > resolve_image(const std::shared_ptr< Kakshya::SignalSourceContainer > &source, size_t index)
Resolve source/index to a GPU image, the same way VisionProcessor does: TextureContainer's own to_ima...
Kinesis::Vision::VisionAnalysis analyze_vision(const std::shared_ptr< Kakshya::SignalSourceContainer > &source, size_t index=0)
Real entry point: resolve the configured query against source, run it, run the GPU analysis.
std::shared_ptr< TextureExecutionContext > m_gradient_sample_ctx
std::shared_ptr< ShaderExecutionContext<> > m_track_reducer
std::array< uint32_t, 256 > compute_histogram(const std::shared_ptr< Core::VKImage > &image, uint32_t w, uint32_t h)
Dispatch otsu_histogram.comp over image, zeroing the buffer first.
Stateful GPU dispatch engine for VisionSequence execution.
@ ComputeMatrix
Compute operations (Yantra - algorithms, matrices, DSP)
@ Yantra
DSP algorithms, computational units, matrix operations, Grammar.
const Facet< std::shared_ptr< Core::VKImage > > & vision_frame()
Facet key for the frame a VisionAnalysis applies to, carried in a Datum's facets beside the analysis.
std::vector< ResolvedSequence > resolve(const VisionQuery &query)
Resolve a VisionQuery into the ResolvedSequences needed to satisfy every active intent.
MAYAFLUX_API ShaderFoundry & get_shader_foundry()
Get the global shader compiler instance.
@ NEAREST
Nearest neighbor (pixelated)
@ COMPUTE
Storage buffer for compute shaders (device-local)
std::vector< Keypoint > keypoints
Keypoint reduction, computed on GPU by the same reduction shader as TrackObjectsAnalysis (track_reduc...
Per-channel mean/min/max over a sampled image region, computed on GPU (Yantra's region_sample....
std::vector< Contour > contours
std::vector< float > nearest_neighbor_distance_px
std::vector< BoundingBox > boxes
std::vector< ElementShape > shapes
FindElements output: contours plus the per-component BoundingBoxes ConnectedComponents already comput...
std::shared_ptr< Core::VKImage > sobel
MeasureAppearance output: whole-frame photometric characterisation, computed on GPU (region_sample....
std::vector< TrackResult > tracks
Track reduction, computed on GPU (Yantra's track_reduce.comp) from the (context-filtered) subject tra...
std::optional< FindElementsAnalysis > find_elements
std::optional< MeasureAppearanceAnalysis > measure_appearance
std::optional< DetectFeaturesAnalysis > detect_features
std::optional< TrackObjectsAnalysis > track_objects
Composed analysis output: one populated field per active intent that produced a result this call.
ImageOutputs images
Image output of whichever image-producing ops ran in this sequence, one named slot per op.
Result of executing a VisionSequence on one frame.
enum MayaFlux::Portal::Graphics::GpuBufferBinding::ElementType FLOAT32
Plain-data description of the compute shader to dispatch.
Facets facets
Typed results that travel with the data, read by type.
std::unordered_map< std::string, std::any > metadata
Associated metadata.
Input/Output container for computation pipeline data flow with structure preservation.
Context information controlling how a compute operation executes.
Settings of VisionAnalyzer, stored whole in the operation's facets.
std::vector< Kinesis::Vision::ResolvedSequence > resolved
std::vector< Kinesis::Vision::VisionResult > results