MayaFlux 0.5.0
Digital-First Multimedia Processing Framework
Loading...
Searching...
No Matches

◆ run()

VisionResult MayaFlux::Kinesis::Vision::VisionExecutor::run ( const VisionSequence sequence,
std::span< const float >  frame,
uint32_t  w,
uint32_t  h 
)

Execute a VisionSequence on one frame.

Parameters
sequenceOrdered steps to execute.
frameNormalised float input. RGBA (4 floats/pixel) for RgbaToGray/RgbaToHsv; single-channel otherwise.
wFrame width in pixels.
hFrame height in pixels.
Returns
VisionResult with pixel_image (DataVariant) and/or structured output.

Definition at line 97 of file VisionExecutor.cpp.

101{
102 ensure_slots(w, h);
103 const size_t slot_min = static_cast<size_t>(m_slot_w) * m_slot_h * 4;
104 if (slot_vec(k_slot_nxt).size() < slot_min)
105 slot_vec(k_slot_nxt).resize(slot_min);
106 if (slot_vec(k_slot_cur).capacity() < slot_min)
107 slot_vec(k_slot_cur).reserve(slot_min);
108
109 auto& cur_vec = slot_vec(k_slot_cur);
110 cur_vec.assign(frame.begin(), frame.end());
111
112 m_pass.begin(sequence, w, h);
114
115 size_t nxt = k_slot_nxt;
116 auto en = static_cast<Eigen::Index>(m_pass.plane_size());
117 const bool wants_tracking = tracks_keypoints(sequence);
118 const bool peaks_feed_track = track_follows_peaks(sequence);
119
120 for (m_pass.index = 0; m_pass.index < sequence.steps.size(); ++m_pass.index) {
121 const auto& step = m_pass.step();
122
123 switch (step.op) {
125 uint32_t new_w = 0, new_h = 0;
126 downsample_2x(slot_vec(m_pass.current), slot_vec(nxt), w, h, m_pass.channels, new_w, new_h);
127 w = new_w;
128 h = new_h;
130 en = static_cast<Eigen::Index>(m_pass.plane_size());
131 std::swap(m_pass.current, nxt);
132 m_pass.result.structured = std::monostate {};
133 break;
134 }
135
138 m_pass.channels = 1;
139 std::swap(m_pass.current, nxt);
140
141 if (wants_tracking && !peaks_feed_track) {
144 }
145
146 m_pass.result.structured = std::monostate {};
147 break;
148 }
149
150 case VisionOp::RgbaToHsv: {
152 m_pass.channels = 3;
153 std::swap(m_pass.current, nxt);
154 m_pass.result.structured = std::monostate {};
155 break;
156 }
157
160 m_pass.channels = 4;
161 std::swap(m_pass.current, nxt);
162 m_pass.result.structured = std::monostate {};
163 break;
164 }
165
166 case VisionOp::Threshold: {
167 const auto& p = get_params<ThresholdParams>(step.params, step.op);
168 slot_map_mut(m_pass.current, en) = (slot_map(m_pass.current, en) >= p.value).cast<float>();
169 m_pass.result.structured = std::monostate {};
170 break;
171 }
172
174 const auto& p = get_params<ThresholdAdaptiveParams>(step.params, step.op);
175 threshold_adaptive(slot_vec(m_pass.current), slot_vec(nxt), w, h, p.block_size, p.offset);
176 std::swap(m_pass.current, nxt);
177 m_pass.result.structured = std::monostate {};
178 break;
179 }
180
183 std::swap(m_pass.current, nxt);
184 m_pass.result.structured = std::monostate {};
185 break;
186 }
187
189 auto m = slot_map_mut(m_pass.current, en);
190 const float mn = m.minCoeff();
191 const float mx = m.maxCoeff();
192 if (mx > mn)
193 m = (m - mn) / (mx - mn);
194 m_pass.result.structured = std::monostate {};
195 break;
196 }
197
199 const auto& p = get_params<NormalizeRangeParams>(step.params, step.op);
200 if (p.hi > p.lo) {
201 auto m = slot_map_mut(m_pass.current, en);
202 m = ((m - p.lo) / (p.hi - p.lo)).max(0.0F).min(1.0F);
203 }
204 m_pass.result.structured = std::monostate {};
205 break;
206 }
207
209 const auto& p = get_params<GaussianBlurParams>(step.params, step.op);
210 const auto& kern = gaussian_kernel(p.sigma);
212 std::swap(m_pass.current, nxt);
213 m_pass.result.structured = std::monostate {};
214 break;
215 }
216
218 const auto& p = get_params<FilterSeparableParams>(step.params, step.op);
219 filter_separable(slot_vec(m_pass.current), slot_vec(k_slot_tmp), slot_vec(nxt), w, h, p.kernel_x, p.kernel_y);
220 std::swap(m_pass.current, nxt);
221 m_pass.result.structured = std::monostate {};
222 break;
223 }
224
225 case VisionOp::Sobel: {
226 GradientResult grad;
228 slot_vec(k_slot_tmp), w, h);
229
230 auto dx = slot_map(k_slot_dx, en);
231 auto dy = slot_map(k_slot_dy, en);
232 slot_map_mut(nxt, en) = (dx.square() + dy.square()).sqrt();
233 const float peak = slot_map(nxt, en).maxCoeff();
234
235 if (peak > 0.0F)
236 slot_map_mut(nxt, en) /= peak;
237
238 grad.dx.assign(slot_vec(k_slot_dx).begin(), slot_vec(k_slot_dx).begin() + en);
239 grad.dy.assign(slot_vec(k_slot_dy).begin(), slot_vec(k_slot_dy).begin() + en);
240 grad.magnitude.assign(slot_vec(nxt).begin(), slot_vec(nxt).begin() + en);
241 grad.angle.resize(static_cast<size_t>(en));
242
243 P::transform(P::par_unseq,
244 slot_vec(k_slot_dx).begin(), slot_vec(k_slot_dx).begin() + en,
245 slot_vec(k_slot_dy).begin(), grad.angle.begin(),
246 [](float gx, float gy) { return std::atan2(gy, gx); });
247
248 m_pass.result.structured = std::move(grad);
249 std::swap(m_pass.current, nxt);
250 break;
251 }
252
253 case VisionOp::Scharr: {
254 GradientResult grad;
256 slot_vec(k_slot_tmp), w, h);
257
258 auto dx = slot_map(k_slot_dx, en);
259 auto dy = slot_map(k_slot_dy, en);
260 slot_map_mut(nxt, en) = (dx.square() + dy.square()).sqrt();
261
262 const float peak = slot_map(nxt, en).maxCoeff();
263 if (peak > 0.0F)
264 slot_map_mut(nxt, en) /= peak;
265
266 grad.dx.assign(slot_vec(k_slot_dx).begin(), slot_vec(k_slot_dx).begin() + en);
267 grad.dy.assign(slot_vec(k_slot_dy).begin(), slot_vec(k_slot_dy).begin() + en);
268 grad.magnitude.assign(slot_vec(nxt).begin(), slot_vec(nxt).begin() + en);
269 grad.angle.resize(static_cast<size_t>(en));
270
271 P::transform(P::par_unseq,
272 slot_vec(k_slot_dx).begin(), slot_vec(k_slot_dx).begin() + en,
273 slot_vec(k_slot_dy).begin(), grad.angle.begin(),
274 [](float gx, float gy) { return std::atan2(gy, gx); });
275
276 m_pass.result.structured = std::move(grad);
277 std::swap(m_pass.current, nxt);
278 break;
279 }
280
281 case VisionOp::Canny: {
282 const auto& p = get_params<CannyParams>(step.params, step.op);
283 canny(slot_vec(m_pass.current), slot_vec(nxt), w, h, p.sigma, p.low_threshold, p.high_threshold);
284 std::swap(m_pass.current, nxt);
285 m_pass.result.structured = std::monostate {};
286 break;
287 }
288
289 case VisionOp::Erode: {
290 const auto& p = get_params<MorphParams>(step.params, step.op);
291 erode(slot_vec(m_pass.current), slot_vec(nxt), w, h, p.radius);
292 std::swap(m_pass.current, nxt);
293 m_pass.result.structured = std::monostate {};
294 break;
295 }
296
297 case VisionOp::Dilate: {
298 const auto& p = get_params<MorphParams>(step.params, step.op);
299 dilate(slot_vec(m_pass.current), slot_vec(nxt), w, h, p.radius);
300 std::swap(m_pass.current, nxt);
301 m_pass.result.structured = std::monostate {};
302 break;
303 }
304
305 case VisionOp::Open: {
306 const auto& p = get_params<MorphParams>(step.params, step.op);
307 open(slot_vec(m_pass.current), slot_vec(k_slot_tmp), slot_vec(nxt), w, h, p.radius);
308 std::swap(m_pass.current, nxt);
309 m_pass.result.structured = std::monostate {};
310 break;
311 }
312
313 case VisionOp::Close: {
314 const auto& p = get_params<MorphParams>(step.params, step.op);
315 close(slot_vec(m_pass.current), slot_vec(k_slot_tmp), slot_vec(nxt), w, h, p.radius);
316 std::swap(m_pass.current, nxt);
317 m_pass.result.structured = std::monostate {};
318 break;
319 }
320
322 const auto& p = get_params<MorphParams>(step.params, step.op);
324 std::swap(m_pass.current, nxt);
325 m_pass.result.structured = std::monostate {};
326 break;
327 }
328
330 const size_t n = m_pass.plane_size();
332 std::span<const float>(slot_vec(m_pass.current)).subspan(0, n),
333 w, h);
334 slot_vec(m_pass.current).clear();
335 m_pass.result.w = 0;
336 m_pass.result.h = 0;
337 break;
338 }
339
341 const auto& p = get_params<FindContoursParams>(step.params, step.op);
342 const size_t n = m_pass.plane_size();
344 std::span<const float>(slot_vec(m_pass.current)).subspan(0, n),
345 w, h, p.min_area, p.max_contours);
346 slot_vec(m_pass.current).clear();
347 m_pass.result.w = 0;
348 m_pass.result.h = 0;
349 break;
350 }
351
353 const auto& p = get_params<HarrisParams>(step.params, step.op);
359 slot_vec(nxt),
360 w, h, p.k, gaussian_kernel(p.sigma));
361 std::swap(m_pass.current, nxt);
362 m_pass.result.structured = std::monostate {};
363 break;
364 }
365
367 const auto& p = get_params<ExtractPeaksParams>(step.params, step.op);
368 auto kpts = extract_peaks(slot_vec(m_pass.current), w, h, p.threshold, p.nms_radius);
369 m_prev_keypoints = kpts;
370
371 if (wants_tracking && !peaks_feed_track)
372 std::swap(std::get<std::vector<float>>(m_prev_gray), m_curr_gray_cache);
373
374 m_pass.result.structured = std::move(kpts);
375 if (!peaks_feed_track) {
376 slot_vec(m_pass.current).clear();
377 m_pass.result.w = 0;
378 m_pass.result.h = 0;
379 }
380 break;
381 }
382
384 const auto& p = get_params<TrackKeypointsParams>(step.params, step.op);
385
386 auto& prev_vec = std::get<std::vector<float>>(m_prev_gray);
387 if (prev_vec.empty() || m_prev_keypoints.empty()) {
388 std::swap(prev_vec, slot_vec(m_pass.current));
389 m_pass.result.structured = std::vector<TrackResult> {};
390 slot_vec(m_pass.current).clear();
391 m_pass.result.w = 0;
392 m_pass.result.h = 0;
393 break;
394 }
395
396 std::vector<glm::vec2> prev_pos;
397 prev_pos.reserve(m_prev_keypoints.size());
398 for (const auto& kp : m_prev_keypoints)
399 prev_pos.push_back(kp.position);
400
401 auto tracked = track_keypoints(
402 prev_vec, slot_vec(m_pass.current),
403 w, h, prev_pos,
404 p.window_radius, p.max_iterations,
405 p.eigen_threshold, p.error_threshold);
406
407 std::swap(prev_vec, slot_vec(m_pass.current));
408 m_pass.result.structured = std::move(tracked);
409 slot_vec(m_pass.current).clear();
410 m_pass.result.w = 0;
411 m_pass.result.h = 0;
412 break;
413 }
414
415 case VisionOp::Snapshot: {
416 if (slot_vec(m_pass.current).empty())
417 break;
418 const size_t n = m_pass.plane_size() * m_pass.channels;
419 SnapshotEntry entry;
420 entry.pixels.assign(slot_vec(m_pass.current).begin(), slot_vec(m_pass.current).begin() + n);
421 entry.w = w;
422 entry.h = h;
423 entry.channels = m_pass.channels;
424 m_pass.result.snapshots.push_back(std::move(entry));
425 break;
426 }
427 }
428 }
429
431 slot_vec(m_pass.current).reserve(static_cast<size_t>(m_slot_w) * m_slot_h * 4);
432
433 VisionResult out = std::move(m_pass.result);
434 m_pass.result = VisionResult {};
435 return out;
436}
uint32_t h
Definition InkPress.cpp:28
std::array< Kakshya::DataVariant, k_slot_count > m_slots
Eigen::Map< Eigen::ArrayXf > slot_map_mut(size_t i, Eigen::Index n) noexcept
Zero-copy mutable Eigen::Map<ArrayXf> over slot i.
std::vector< float > & slot_vec(size_t i) noexcept
Mutable reference to the vector<float> inside slot i.
CpuVisionPass m_pass
Walk state for the current run: sequence position, geometry, working slot index, and the result under...
void ensure_slots(uint32_t w, uint32_t h)
Ensure all slots are sized to n_pixels floats.
const std::vector< float > & gaussian_kernel(float sigma)
Return a reference to the precomputed 1D Gaussian kernel for sigma.
Eigen::Map< const Eigen::ArrayXf > slot_map(size_t i, Eigen::Index n) const noexcept
Zero-copy read-only Eigen::Map<const ArrayXf> over slot i.
std::vector< double > max(std::span< const double > data, size_t n_windows, uint32_t hop_size, uint32_t window_size)
Maximum value per window.
Definition Analysis.cpp:436
void rgba_to_gray(std::span< const float > rgba, std::span< float > dst, uint32_t w, uint32_t h)
Convert RGBA to luminance gray using BT.601 coefficients.
Definition PixelOps.cpp:18
bool track_follows_peaks(const VisionSequence &seq)
True when an ExtractPeaks step is immediately followed by TrackKeypoints.
Definition VisionOp.hpp:441
std::vector< float > filter_separable(std::span< const float > src, uint32_t w, uint32_t h, std::span< const float > kernel_x, std::span< const float > kernel_y)
Apply a separable 2D filter via two 1D passes.
std::vector< float > canny(std::span< const float > gray, uint32_t w, uint32_t h, float sigma, float low_threshold, float high_threshold)
Canny edge detector.
Definition Gradient.cpp:132
void gray_to_rgba(std::span< const float > gray, std::span< float > dst, uint32_t w, uint32_t h)
Global threshold writing into caller-supplied buffer.
Definition PixelOps.cpp:138
std::vector< Keypoint > extract_peaks(std::span< const float > response, uint32_t w, uint32_t h, float threshold, uint32_t nms_radius)
Extract peaks from a response map via non-maximum suppression.
Definition Harris.cpp:184
void erode(std::span< const float > mask, std::span< float > dst, uint32_t w, uint32_t h, uint32_t radius)
Erosion writing into dst.
std::vector< float > harris_response(std::span< const float > gray, uint32_t w, uint32_t h, float k, float sigma)
Compute the Harris corner response map.
Definition Harris.cpp:19
ComponentResult connected_components(std::span< const float > mask, uint32_t w, uint32_t h)
Label connected foreground components in a binary mask.
void rgba_to_hsv(std::span< const float > rgba, std::span< float > dst, uint32_t w, uint32_t h)
Convert RGBA to HSV writing into caller-supplied buffer.
Definition PixelOps.cpp:94
std::vector< Contour > find_contours(std::span< const float > mask, uint32_t w, uint32_t h, float min_area, uint32_t max_contours)
Extract outer contours from a binary mask.
Definition Contours.cpp:70
void close(std::span< const float > mask, std::span< float > tmp, std::span< float > dst, uint32_t w, uint32_t h, uint32_t radius)
Morphological closing writing into dst.
void sobel(std::span< const float > gray, std::span< float > dx, std::span< float > dy, std::span< float > tmp, uint32_t w, uint32_t h)
Sobel gradient writing dx and dy into caller-supplied buffers.
Definition Gradient.cpp:66
void dilate(std::span< const float > mask, std::span< float > dst, uint32_t w, uint32_t h, uint32_t radius)
Dilation writing into dst.
bool tracks_keypoints(const VisionSequence &seq)
True when any step tracks keypoints.
Definition VisionOp.hpp:428
std::vector< TrackResult > track_keypoints(std::span< const float > prev_gray, std::span< const float > curr_gray, uint32_t w, uint32_t h, std::span< const glm::vec2 > prev_points, uint32_t window_radius, uint32_t max_iterations, float eigen_threshold, float error_threshold)
Track keypoints from prev_gray to curr_gray via Lucas-Kanade.
void scharr(std::span< const float > gray, std::span< float > dx, std::span< float > dy, std::span< float > tmp, uint32_t w, uint32_t h)
Scharr gradient writing dx and dy into caller-supplied buffers.
Definition Gradient.cpp:99
void threshold_otsu(std::span< const float > gray, std::span< float > dst)
Otsu threshold writing into caller-supplied buffer.
Definition PixelOps.cpp:299
void threshold_adaptive(std::span< const float > gray, std::span< float > dst, uint32_t w, uint32_t h, uint32_t block_size, float offset)
Adaptive threshold writing into caller-supplied buffer.
Definition PixelOps.cpp:258
void downsample_2x(std::span< const float > src, std::span< float > dst, uint32_t w, uint32_t h, uint32_t &new_w, uint32_t &new_h)
2x box-filter downsample writing into a caller-supplied buffer.
Definition PixelOps.cpp:443
void morph_gradient(std::span< const float > mask, std::span< float > tmp, std::span< float > dst, uint32_t w, uint32_t h, uint32_t radius)
Morphological gradient (dilate - erode) writing into dst.
void open(std::span< const float > mask, std::span< float > tmp, std::span< float > dst, uint32_t w, uint32_t h, uint32_t radius)
Morphological opening writing into dst.
std::shared_ptr< Vruta::Routine > sequence(std::vector< std::pair< double, std::function< void()> > > sequence, Vruta::ProcessingToken token)
Creates a temporal sequence that executes callbacks at specified time offsets.
Definition Tasks.cpp:36
double peak(const std::vector< double > &data)
Find peak amplitude in single-channel data.
Definition Yantra.cpp:268
void begin(const VisionSequence &seq, uint32_t width, uint32_t height)
Reset the walk band for a fresh run.
const VisionStep & step() const noexcept
void set_geometry(uint32_t width, uint32_t height) noexcept
std::vector< SnapshotEntry > snapshots

References MayaFlux::Kinesis::Vision::GradientResult::angle, MayaFlux::Kinesis::Vision::VisionPass< Handle >::begin(), MayaFlux::Kinesis::Vision::canny(), MayaFlux::Kinesis::Vision::Canny, MayaFlux::Kinesis::Vision::SnapshotEntry::channels, MayaFlux::Kinesis::Vision::VisionPass< Handle >::channels, MayaFlux::Kinesis::Vision::close(), MayaFlux::Kinesis::Vision::Close, MayaFlux::Kinesis::Vision::connected_components(), MayaFlux::Kinesis::Vision::ConnectedComponents, MayaFlux::Kinesis::Vision::VisionPass< Handle >::current, MayaFlux::Kinesis::Vision::dilate(), MayaFlux::Kinesis::Vision::Dilate, MayaFlux::Kinesis::Vision::Downsample2x, MayaFlux::Kinesis::Vision::downsample_2x(), MayaFlux::Kinesis::Vision::GradientResult::dx, MayaFlux::Kinesis::Vision::GradientResult::dy, ensure_slots(), MayaFlux::Kinesis::Vision::erode(), MayaFlux::Kinesis::Vision::Erode, MayaFlux::Kinesis::Vision::extract_peaks(), MayaFlux::Kinesis::Vision::ExtractPeaks, MayaFlux::Kinesis::Vision::filter_separable(), MayaFlux::Kinesis::Vision::FilterSeparable, MayaFlux::Kinesis::Vision::find_contours(), MayaFlux::Kinesis::Vision::FindContours, gaussian_kernel(), MayaFlux::Kinesis::Vision::GaussianBlur, MayaFlux::Kinesis::Vision::gray_to_rgba(), MayaFlux::Kinesis::Vision::GrayToRgba, MayaFlux::Kinesis::Vision::SnapshotEntry::h, MayaFlux::Kinesis::Vision::VisionResult::h, h, MayaFlux::Kinesis::Vision::harris_response(), MayaFlux::Kinesis::Vision::HarrisResponse, MayaFlux::Kinesis::Vision::VisionPass< Handle >::index, k_slot_cur, k_slot_dx, k_slot_dy, k_slot_ixx, k_slot_ixy, k_slot_iyy, k_slot_nxt, k_slot_sxx, k_slot_sxy, k_slot_syy, k_slot_tmp, m_curr_gray_cache, m_pass, m_prev_gray, m_prev_keypoints, m_slot_h, m_slot_w, m_slots, MayaFlux::Kinesis::Vision::GradientResult::magnitude, MayaFlux::Kinesis::Vision::morph_gradient(), MayaFlux::Kinesis::Vision::MorphGradient, MayaFlux::Kinesis::Vision::NormalizeInplace, MayaFlux::Kinesis::Vision::NormalizeRange, MayaFlux::Kinesis::Vision::open(), MayaFlux::Kinesis::Vision::Open, MayaFlux::peak(), MayaFlux::Kinesis::Vision::VisionResult::pixel_image, MayaFlux::Kinesis::Vision::SnapshotEntry::pixels, MayaFlux::Kinesis::Vision::VisionPass< Handle >::plane_size(), MayaFlux::Kinesis::Vision::VisionPass< Handle >::result, MayaFlux::Kinesis::Vision::rgba_to_gray(), MayaFlux::Kinesis::Vision::rgba_to_hsv(), MayaFlux::Kinesis::Vision::RgbaToGray, MayaFlux::Kinesis::Vision::RgbaToHsv, MayaFlux::Kinesis::Vision::scharr(), MayaFlux::Kinesis::Vision::Scharr, MayaFlux::Kinesis::Vision::VisionPass< Handle >::set_geometry(), slot_map(), slot_map_mut(), slot_vec(), MayaFlux::Kinesis::Vision::Snapshot, MayaFlux::Kinesis::Vision::VisionResult::snapshots, MayaFlux::Kinesis::Vision::sobel(), MayaFlux::Kinesis::Vision::Sobel, MayaFlux::Kinesis::Vision::VisionPass< Handle >::step(), MayaFlux::Kinesis::Vision::VisionResult::structured, MayaFlux::Kinesis::Vision::Threshold, MayaFlux::Kinesis::Vision::threshold_adaptive(), MayaFlux::Kinesis::Vision::threshold_otsu(), MayaFlux::Kinesis::Vision::ThresholdAdaptive, MayaFlux::Kinesis::Vision::ThresholdOtsu, MayaFlux::Kinesis::Vision::track_follows_peaks(), MayaFlux::Kinesis::Vision::track_keypoints(), MayaFlux::Kinesis::Vision::TrackKeypoints, MayaFlux::Kinesis::Vision::tracks_keypoints(), MayaFlux::Kinesis::Vision::SnapshotEntry::w, and MayaFlux::Kinesis::Vision::VisionResult::w.

Referenced by MayaFlux::Yantra::VisionAnalyzer< InputType, OutputType >::analyze_implementation(), MayaFlux::Kakshya::VisionProcessor::process(), and MayaFlux::Buffers::ImageCVProcessor< T >::processing_function().

+ Here is the call graph for this function:
+ Here is the caller graph for this function: