MayaFlux 1.0.0
Digital-First Multimedia Processing Framework
Loading...
Searching...
No Matches
PhonemeAnalyzer.hpp
Go to the documentation of this file.
1#pragma once
2
6
7#include "Analyzer.hpp"
8#include <Eigen/Dense>
9
12
14
15/**
16 * @file PhonemeAnalyzer.hpp
17 * @brief Which phoneme is spoken near each 20 ms frame of an utterance, from a learned network run on the GPU.
18 */
19
20namespace MayaFlux::Yantra {
21
22/**
23 * @enum PhonemeMethod
24 * @brief The per-frame values a PhonemeAnalyzer reports as its output data.
25 */
26enum class PhonemeMethod : uint8_t {
27 POSTERIORS, ///< One column per class: the probability of each of the network's classes
28 BROAD_CLASSES, ///< Three columns: the summed probability of the vowels, of the consonants, and of everything else
29 HELD_CLASSES ///< Three columns of 0 or 1: whether the frame's centre lies in the estimated range of a vowel, of a consonant, or of neither
30};
31
32/**
33 * @struct PhonemeAnalyzerConfig
34 * @brief Settings of a PhonemeAnalyzer.
35 *
36 * The network fixes a frame at 400 samples of 16 kHz audio every 320 samples,
37 * so the window and hop of the base config are not settings. sample_rate
38 * states the rate of the input and exists so that a wrong one is refused with
39 * its reason: the analyzer does not convert.
40 */
47
48 /** Values reported as the output data, as a PhonemeMethod value. */
50 /** Sample rate in Hz of the input. The network runs at 16000 only, so any other value is refused. */
51 double sample_rate { static_cast<double>(HubertPhonemeModel::sample_rate) };
52 /** Also read back the network's state of every frame. They are large: 768 values per frame. */
53 bool keep_embeddings { false };
54};
55
56MF_DESCRIBE(PhonemeAnalyzerConfig)
57
58/**
59 * @struct PhonemeEvent
60 * @brief One phoneme the network recognised.
61 *
62 * start and end are estimates. The network marks a phoneme on one frame, and
63 * the boundary between two neighbours is put where the weak evidence on the
64 * frames between them changes sides. Measured on 60 LibriSpeech utterances
65 * against a forced alignment, the boundary between two correctly recognised
66 * neighbours fell a median of 12.5 ms off, with 94% within 40 ms. Pauses are
67 * not detected: a phoneme before a pause runs through it, the first starts at
68 * sample 0 and the last ends with the utterance.
69 */
70struct MAYAFLUX_API PhonemeEvent {
71 /** Class of the phoneme, 0 to HubertPhonemeModel::phonemes - 1. */
72 uint32_t phoneme { 0 };
73 /** Frame it was recognised on. */
74 size_t frame { 0 };
75 /** First sample of that frame. The frame spans 400 samples from there. */
76 size_t sample { 0 };
77 /** Probability the network gave the phoneme on that frame. */
78 float probability { 0.0F };
79 /** Estimated first sample of the phoneme. */
80 size_t start { 0 };
81 /** Estimated sample one past the phoneme's last. */
82 size_t end { 0 };
83};
84
85/**
86 * @struct PhonemeAnalysis
87 * @brief Analysis result of a PhonemeAnalyzer.
88 */
90 /** Settings the analysis was produced under. */
92 /** Frames of the utterance. */
93 size_t frames { 0 };
94 /** Probability of every class on every frame, frame after frame, HubertPhonemeModel::classes values each. */
95 std::vector<float> posteriors;
96 /** Most probable class of each frame. Mostly the blank class: a phoneme is emitted on one frame. */
97 std::vector<uint32_t> best;
98 /** The phonemes in order: every frame whose most probable class is a phoneme and differs from the frame before. */
99 std::vector<PhonemeEvent> events;
100 /** First and one past the last sample of each frame. */
101 std::vector<std::pair<size_t, size_t>> frame_positions;
102 /** State of every frame before the final layer, HubertPhonemeModel::hidden_width values each. Kept on request. */
103 std::vector<float> embeddings;
104
105 /**
106 * @brief The probabilities as a frames by classes float32 tensor.
107 * @return The container, or nullptr when there are no frames.
108 */
109 [[nodiscard]] std::shared_ptr<Kakshya::TensorContainer> posterior_tensor(std::string name) const
110 {
111 return as_tensor(posteriors, HubertPhonemeModel::classes, std::move(name));
112 }
113
114 /**
115 * @brief The states as a frames by hidden_width float32 tensor.
116 * @return The container, or nullptr when the states were not kept.
117 */
118 [[nodiscard]] std::shared_ptr<Kakshya::TensorContainer> embedding_tensor(std::string name) const
119 {
120 return as_tensor(embeddings, HubertPhonemeModel::hidden_width, std::move(name));
121 }
122
123private:
124 [[nodiscard]] static std::shared_ptr<Kakshya::TensorContainer> as_tensor(const std::vector<float>& values, uint64_t width, std::string name)
125 {
126 if (values.empty()) {
127 return nullptr;
128 }
129 auto data = Kakshya::TensorData::from_values<float>(values, { values.size() / width, width }, std::move(name));
130 return data ? std::make_shared<Kakshya::TensorContainer>(std::move(*data)) : nullptr;
131 }
132};
133
134MF_DESCRIBE(PhonemeAnalysis)
135
136/**
137 * @brief Field layout of the records a PhonemeAnalyzer with Composite output gives, one record per recognised phoneme.
138 *
139 * phoneme is the ARPABET symbol, class its class, frame the frame it was
140 * recognised on, sample the first sample of that frame, start and end the
141 * estimated sample range as in PhonemeEvent, probability what the network gave
142 * it on its frame, and vowel is 1 for a vowel and 0 otherwise.
143 */
145{
147 layout.add_field<std::string>("phoneme");
148 layout.add_field<uint32_t>("class");
149 layout.add_field<uint64_t>("frame");
150 layout.add_field<uint64_t>("sample");
151 layout.add_field<uint64_t>("start");
152 layout.add_field<uint64_t>("end");
153 layout.add_field<double>("probability");
154 layout.add_field<uint8_t>("vowel");
155 return layout;
156}
157
158/** @brief Region attribute: the probability the network gave the region's phoneme. */
160
161/** @brief Region attribute: the class of the region's phoneme. */
163
164/**
165 * @brief One region per recognised phoneme, labelled with its ARPABET symbol.
166 *
167 * A region covers the estimated sample range of its phoneme, see PhonemeEvent.
168 * Each carries phoneme_probability and phoneme_class.
169 *
170 * @param analysis The analysis whose events become regions.
171 * @param first_channel First channel the regions cover.
172 * @param last_channel Last channel the regions cover, inclusive.
173 * @param name Name of the group.
174 */
176 const PhonemeAnalysis& analysis, uint32_t first_channel, uint32_t last_channel, std::string name = "phonemes")
177{
178 Kakshya::RegionGroup group(std::move(name));
179 for (const auto& event : analysis.events) {
180 if (event.end <= event.start) {
181 continue;
182 }
183 auto region = Kakshya::Region::audio_span(event.start, event.end - 1,
184 first_channel, last_channel, std::string(HubertPhonemeModel::label(event.phoneme)));
185 region.set(phoneme_probability, static_cast<double>(event.probability));
186 region.set(phoneme_class, event.phoneme);
187 group.add_region(region);
188 }
189 return group;
190}
191
192/**
193 * @class PhonemeAnalyzer
194 * @brief Which phoneme is spoken near each frame of a mono 16 kHz utterance, from a learned network.
195 *
196 * Where WaveformAnalyzer measures how a waveform repeats and VadAnalyzer says
197 * whether a sound is speech, this says what is being said, as phonemes: every
198 * frame is judged with the whole utterance as its context, which no measure of
199 * a single window can do.
200 *
201 * The analyzer is named for the task. Today it runs one network, a HuBERT base
202 * fine tuned on read English, recognised by its tensors; weights for any other
203 * network are rejected with the reason. Its classes are the 39 ARPABET
204 * phonemes, an unknown class, the blank class and two unused markers, named by
205 * HubertPhonemeModel::label.
206 *
207 * The network emits each phoneme on one frame and the blank class between, so
208 * the result tells which phonemes occur and on which frames. Where each starts
209 * and ends is an estimate, described at PhonemeEvent. One call is one utterance
210 * of at most 30 s: nothing carries
211 * over between calls, and a longer input is refused with the reason. Cut a
212 * long recording at its pauses first, for instance with VadAnalyzer and
213 * IntervalExtractor, which also keeps the network off everything that is not
214 * speech. One channel only.
215 *
216 * The weights are an explicit dependency: the caller loads the network's
217 * tensors and passes them. The network runs in native float on the vulkan
218 * target and has no CPU twin. It is not in the default registry, because it
219 * cannot be built without its weights; catalog it with a factory to use it in
220 * a grammar. It must not run on the audio thread.
221 *
222 * With a matrix as OutputType the output data is the per frame values the
223 * method selects. HELD_CLASSES is the one made for IntervalExtractor: with
224 * samples_per_score 320 and min_run, min_gap and padding set for single frames
225 * it gives the vowel and consonant intervals of the utterance. With Kakshya::CompositeArray it is one record per recognised
226 * phoneme in the layout of phoneme_record_layout, whatever the method. The
227 * analysis is the same either way.
228 */
229template <ComputeData InputType = std::vector<Kakshya::DataVariant>, ComputeData OutputType = Eigen::MatrixXd>
230class PhonemeAnalyzer : public Analyzer<InputType, OutputType> {
231public:
236
237 static constexpr Target target = targets::vulkan;
238
239 /**
240 * @brief Construct from the network weights.
241 * @param weights The tensors of the network, as in hubert_base_phoneme_en.safetensors.
242 * @param config Analysis settings.
243 * @throws std::invalid_argument When diagnose refuses the config, or the weights are missing a tensor or one has the wrong shape.
244 */
246 {
247 if (const std::string reason = diagnose(config); !reason.empty()) {
249 std::source_location::current(), "PhonemeAnalyzer: {}", reason);
250 }
251 if (const std::string reason = HubertPhonemeModel::diagnose(weights); !reason.empty()) {
253 std::source_location::current(), "PhonemeAnalyzer: {}", reason);
254 }
255 m_weights = std::make_shared<const Kakshya::TensorSet>(weights);
256 this->set_config(std::move(config));
257 }
258
259 [[nodiscard]] ConfigRefs configs() const override
260 {
261 return this->stored_configs(supports {});
262 }
263
264 /**
265 * @brief Why a config cannot be analysed.
266 * @return The reason, empty when the config is usable: its sample_rate is the 16000 Hz the network runs at.
267 */
268 [[nodiscard]] static std::string diagnose(const PhonemeAnalyzerConfig& config)
269 {
270 if (config.sample_rate != static_cast<double>(HubertPhonemeModel::sample_rate)) {
271 return std::format("sample_rate {} is not the {} Hz the network runs at; convert the input first, for example with Kinesis::Discrete::resample",
273 }
274 return {};
275 }
276
277 /**
278 * @brief Type-safe phoneme analysis
279 */
281 {
282 auto result = this->analyze_data(data);
283 return safe_any_cast_or_throw<PhonemeAnalysis>(result);
284 }
285
286 PhonemeAnalysis analyze_phonemes(const InputType& data)
287 {
288 return this->analyze_phonemes(input_type { data });
289 }
290
292 {
293 return safe_any_cast_or_throw<PhonemeAnalysis>(this->get_current_analysis());
294 }
295
296 /**
297 * @brief Append to a recording the caller shares with other work.
298 *
299 * Call it before the first analysis. The recording must be compute
300 * isolated.
301 *
302 * @throws std::logic_error when the network has already been created.
303 */
304 void share_recording(std::shared_ptr<GpuDispatchRecorder> recording)
305 {
306 if (m_model) {
308 std::source_location::current(), "The recording can only be shared before the network starts");
309 }
310 m_recording = std::move(recording);
311 }
312
313 /**
314 * @brief Where the states of the latest utterance are on the device, for a consumer on the same recording.
315 */
317 {
318 return network().hidden_buffer();
319 }
320
321 /**
322 * @brief Where the probabilities of the latest utterance are on the device, for a consumer on the same recording.
323 */
328
329 /** @brief Frames the latest utterance left on the device. */
330 [[nodiscard]] size_t resident_frames() const noexcept
331 {
332 return m_model ? m_model->resident_frames() : 0;
333 }
334
335 [[nodiscard]] AnalysisType get_analysis_type() const override
336 {
338 }
339
340 [[nodiscard]] std::string get_name() const override
341 {
342 return "PhonemeAnalyzer";
343 }
344
345 /**
346 * @brief One while a device is up, zero otherwise.
347 */
348 static float fitness(const input_type& /*input*/)
349 {
351 }
352
353protected:
355 {
356 try {
357 const auto& cfg = this->template config<PhonemeAnalyzerConfig>();
358 if (const std::string reason = diagnose(cfg); !reason.empty()) {
360 std::source_location::current(), "PhonemeAnalyzer: {}", reason);
361 }
362
363 auto [data_span, structure_info] = OperationHelper::extract_structured_double(
364 const_cast<input_type&>(input));
365
366 if (data_span.size() != 1) {
368 std::source_location::current(), "PhonemeAnalyzer analyses one channel and was given {}", data_span.size());
369 }
370
371 const std::vector<float> samples(data_span[0].begin(), data_span[0].end());
372
373 constexpr size_t classes = HubertPhonemeModel::classes;
374 PhonemeAnalysis analysis;
375 analysis.config = cfg;
376 analysis.posteriors = network().process(samples);
377 analysis.frames = analysis.posteriors.size() / classes;
378 if (cfg.keep_embeddings) {
379 analysis.embeddings = network().hidden_states();
380 }
381 describe_frames(analysis);
382 bound_events(analysis, samples.size());
383
384 this->store_current_analysis(analysis);
385
386 output_type output = make_output(analysis, static_cast<PhonemeMethod>(cfg.method), structure_info);
387 output.metadata = input.metadata;
388 output.facets = input.facets;
389 output.facets.set(analysis);
390 return output;
391 } catch (const std::exception& e) {
392 MF_ERROR(Journal::Component::Yantra, Journal::Context::ComputeMatrix, "Phoneme analysis failed: {}", e.what());
393 output_type error_result = empty_output();
394 error_result.metadata = input.metadata;
395 error_result.facets = input.facets;
396 return error_result;
397 }
398 }
399
400private:
402 {
403 if (!m_model) {
404 m_model = std::make_unique<HubertPhonemeModel>(*m_weights, m_recording);
405 }
406 return *m_model;
407 }
408
410 {
411 if constexpr (CompositeArrayData<OutputType>) {
413 } else {
414 return output_type {};
415 }
416 }
417
419 {
420 if constexpr (CompositeArrayData<OutputType>) {
421 structure_info.composite_layout = phoneme_record_layout();
422 std::vector<std::vector<double>> columns(7);
423 for (const auto& event : analysis.events) {
424 columns[0].push_back(static_cast<double>(event.phoneme));
425 columns[1].push_back(static_cast<double>(event.frame));
426 columns[2].push_back(static_cast<double>(event.sample));
427 columns[3].push_back(static_cast<double>(event.start));
428 columns[4].push_back(static_cast<double>(event.end));
429 columns[5].push_back(static_cast<double>(event.probability));
430 columns[6].push_back(HubertPhonemeModel::is_vowel(event.phoneme) ? 1.0 : 0.0);
431 }
432 output_type output = this->convert_result(columns, structure_info);
433 for (size_t row = 0; row < analysis.events.size(); ++row) {
434 output.data.set_text(row, "phoneme", HubertPhonemeModel::label(analysis.events[row].phoneme));
435 }
436 return output;
437 } else {
438 auto channel_values = select_values(analysis, method);
439 return this->convert_result(channel_values, structure_info);
440 }
441 }
442
443 static void describe_frames(PhonemeAnalysis& analysis)
444 {
445 constexpr size_t classes = HubertPhonemeModel::classes;
446 analysis.best.reserve(analysis.frames);
447 analysis.frame_positions.reserve(analysis.frames);
448 uint32_t previous = HubertPhonemeModel::blank_class;
449 for (size_t frame = 0; frame < analysis.frames; ++frame) {
450 const float* row = analysis.posteriors.data() + frame * classes;
451 const auto top = static_cast<uint32_t>(std::max_element(row, row + classes) - row);
452 const size_t start = frame * HubertPhonemeModel::frame_hop;
453 analysis.best.push_back(top);
454 analysis.frame_positions.emplace_back(start, start + HubertPhonemeModel::frame_span);
455 if (top != previous && top < HubertPhonemeModel::phonemes) {
456 analysis.events.push_back({ .phoneme = top, .frame = frame, .sample = start, .probability = row[top] });
457 }
458 previous = top;
459 }
460 }
461
462 /**
463 * Gives every event its sample range. The first frame of the later of two neighbours is the one, after the
464 * earlier spike and up to the later one, that maximises the log probability of the earlier phoneme on the
465 * frames before it plus that of the later phoneme from it on, probabilities taken among the phonemes only.
466 */
467 static void bound_events(PhonemeAnalysis& analysis, size_t samples)
468 {
469 constexpr size_t classes = HubertPhonemeModel::classes;
470 constexpr size_t phonemes = HubertPhonemeModel::phonemes;
472 auto& events = analysis.events;
473 if (events.empty()) {
474 return;
475 }
476 const auto evidence = [&](size_t frame, uint32_t phoneme) {
477 const float* row = analysis.posteriors.data() + frame * classes;
478 double mass = 0.0;
479 for (size_t index = 0; index < phonemes; ++index) {
480 mass += row[index];
481 }
482 return mass > 0.0 ? std::log(row[phoneme] / mass + 1.0e-30) : 0.0;
483 };
484
485 events.front().start = 0;
486 for (size_t k = 0; k + 1 < events.size(); ++k) {
487 const size_t first = events[k].frame;
488 const size_t last = events[k + 1].frame;
489 const uint32_t earlier = events[k].phoneme;
490 const uint32_t later = events[k + 1].phoneme;
491
492 double score = 0.0;
493 for (size_t frame = first + 1; frame <= last; ++frame) {
494 score += evidence(frame, later);
495 }
496 double best = score;
497 size_t split = first + 1;
498 for (size_t candidate = first + 2; candidate <= last; ++candidate) {
499 score += evidence(candidate - 1, earlier) - evidence(candidate - 1, later);
500 if (score > best) {
501 best = score;
502 split = candidate;
503 }
504 }
505 const size_t boundary = std::min(split * HubertPhonemeModel::frame_hop + lead, samples);
506 events[k].end = boundary;
507 events[k + 1].start = boundary;
508 }
509 events.back().end = samples;
510 }
511
512 static std::vector<std::vector<double>> select_values(const PhonemeAnalysis& analysis, PhonemeMethod method)
513 {
514 constexpr size_t classes = HubertPhonemeModel::classes;
515 if (method == PhonemeMethod::HELD_CLASSES) {
516 std::vector<std::vector<double>> held(3, std::vector<double>(analysis.frames, 0.0));
517 size_t current = 0;
518 for (size_t frame = 0; frame < analysis.frames; ++frame) {
519 const size_t centre = frame * HubertPhonemeModel::frame_hop + HubertPhonemeModel::frame_span / 2;
520 while (current < analysis.events.size() && analysis.events[current].end <= centre) {
521 ++current;
522 }
523 const bool inside = current < analysis.events.size() && analysis.events[current].start <= centre;
524 held[!inside ? 2 : HubertPhonemeModel::is_vowel(analysis.events[current].phoneme) ? 0 : 1][frame] = 1.0;
525 }
526 return held;
527 }
528 const bool broad = method == PhonemeMethod::BROAD_CLASSES;
529 std::vector<std::vector<double>> columns(broad ? 3 : classes, std::vector<double>(analysis.frames, 0.0));
530 for (size_t frame = 0; frame < analysis.frames; ++frame) {
531 const float* row = analysis.posteriors.data() + frame * classes;
532 for (uint32_t index = 0; index < classes; ++index) {
533 if (!broad) {
534 columns[index][frame] = row[index];
535 } else if (index >= HubertPhonemeModel::phonemes) {
536 columns[2][frame] += row[index];
537 } else {
538 columns[HubertPhonemeModel::is_vowel(index) ? 0 : 1][frame] += row[index];
539 }
540 }
541 }
542 return columns;
543 }
544
545 std::shared_ptr<const Kakshya::TensorSet> m_weights;
546 std::shared_ptr<GpuDispatchRecorder> m_recording;
547 mutable std::unique_ptr<HubertPhonemeModel> m_model;
548};
549
550/// Standard phoneme analyzer: DataVariant -> MatrixXd
552
553/// Record phoneme analyzer: DataVariant -> one Composite record per recognised phoneme
555
556/// Container phoneme analyzer: SignalContainer -> MatrixXd
558
559} // namespace MayaFlux::Yantra
Modern, digital-first analyzer framework for Maya Flux.
#define MF_ERROR(comp, ctx,...)
#define MF_DESCRIBE(...)
Requests a generated member description for a struct at namespace scope.
Definition Describe.hpp:14
A HuBERT base network with a phoneme head run on the GPU, one phoneme distribution per 20 ms frame.
Typed key for an arbitrary attribute stored in a Facets bag.
Definition Facets.hpp:26
void set(T value)
Stores a struct under its type identity.
Definition Facets.hpp:107
Owns a sequence of values that share a CompositeLayout.
bool add_field(std::string name)
Declare a named arithmetic, GLM, text, or binary field.
Definition Composite.hpp:74
Shared field description and byte layout for a CompositeArray.
Definition Composite.hpp:60
static std::optional< TensorData > from_values(std::span< const T > values, std::vector< uint64_t > shape={}, std::string name={})
Contiguous tensor holding a copy of typed values.
Definition Tensor.hpp:75
bool is_initialized() const
Check if compiler is initialized.
void store_current_analysis(AnalysisResultType &&result) const
Definition Analyzer.hpp:234
std::any get_current_analysis() const
Access cached analysis from last operation.
Definition Analyzer.hpp:175
std::any analyze_data(const input_type &data)
User-facing analysis method - returns analysis results directly.
Definition Analyzer.hpp:138
Template-flexible analyzer base with instance-defined I/O types.
Definition Analyzer.hpp:101
void set_config(Config config)
Stores a config struct, whole, under its own type.
ConfigRefs stored_configs(Supports< Configs... >) const
The stored configs among those listed, in listed order.
output_type convert_result(std::vector< std::vector< double > > &result_data, DataStructureInfo &metadata)
Convert processed double data back to OutputType using metadata and optional callback.
DeviceBuffer posterior_buffer()
The device buffer of the distributions, classes rows of frames.
static constexpr uint32_t frame_hop
Samples between the starts of successive frames.
static constexpr uint32_t phonemes
Classes 0 to phonemes - 1 are phonemes.
DeviceBuffer hidden_buffer()
The device buffer of the states, hidden_width rows of frames.
static constexpr uint32_t classes
Scores per frame.
static bool is_vowel(uint32_t index) noexcept
Whether a class is a vowel phoneme.
static std::string diagnose(const Kakshya::TensorSet &weights)
Check that a tensor set holds the weights the network needs.
static constexpr uint32_t blank_class
Class the network emits between phonemes.
static constexpr uint32_t sample_rate
Sample rate in Hz the network was trained and runs at.
static std::string_view label(uint32_t index) noexcept
Name of a class: the ARPABET symbol of a phoneme, or a bracketed name for the others.
std::vector< float > process(std::span< const float > samples)
Run one utterance.
static constexpr uint32_t frame_span
Samples one frame's features are computed from.
static constexpr uint32_t hidden_width
Width of the state before the final layer.
std::vector< float > hidden_states()
The state before the final layer for every frame of the latest utterance.
HuBERT base fine tuned for English phonemes at 16 kHz: an utterance in, a distribution over phonemes ...
static std::tuple< NumericChannels, DataStructureInfo > extract_structured_double(T &compute_data)
Extract structured double data from Datum container or direct ComputeData with automatic container ha...
size_t resident_frames() const noexcept
Frames the latest utterance left on the device.
HubertPhonemeModel::DeviceBuffer posterior_buffer()
Where the probabilities of the latest utterance are on the device, for a consumer on the same recordi...
Supports< PhonemeAnalyzerConfig > supports
PhonemeAnalyzer(const Kakshya::TensorSet &weights, PhonemeAnalyzerConfig config={})
Construct from the network weights.
void share_recording(std::shared_ptr< GpuDispatchRecorder > recording)
Append to a recording the caller shares with other work.
AnalysisType get_analysis_type() const override
Gets the analysis type category for this analyzer.
Analyzer< InputType, OutputType > base_type
PhonemeAnalysis analyze_phonemes(const InputType &data)
std::shared_ptr< const Kakshya::TensorSet > m_weights
static void describe_frames(PhonemeAnalysis &analysis)
static std::vector< std::vector< double > > select_values(const PhonemeAnalysis &analysis, PhonemeMethod method)
std::string get_name() const override
Get operation name for debugging/introspection.
ConfigRefs configs() const override
The base view of the config structs this operation holds.
static std::string diagnose(const PhonemeAnalyzerConfig &config)
Why a config cannot be analysed.
static float fitness(const input_type &)
One while a device is up, zero otherwise.
HubertPhonemeModel::DeviceBuffer hidden_buffer()
Where the states of the latest utterance are on the device, for a consumer on the same recording.
PhonemeAnalysis analyze_phonemes(const input_type &data)
Type-safe phoneme analysis.
HubertPhonemeModel & network() const
output_type analyze_implementation(const input_type &input) override
Pure virtual analysis implementation - derived classes implement this.
output_type make_output(const PhonemeAnalysis &analysis, PhonemeMethod method, DataStructureInfo &structure_info)
PhonemeAnalysis get_phoneme_analysis() const
static void bound_events(PhonemeAnalysis &analysis, size_t samples)
Gives every event its sample range.
Which phoneme is spoken near each frame of a mono 16 kHz utterance, from a learned network.
An owning array of schema-bearing Composite elements.
Definition DataSpec.hpp:43
@ ComputeMatrix
Compute operations (Yantra - algorithms, matrices, DSP).
@ Yantra
DSP algorithms, computational units, matrix operations, Grammar.
MAYAFLUX_API ShaderFoundry & get_shader_foundry()
Get the global shader compiler instance.
PhonemeAnalyzer< std::shared_ptr< Kakshya::SignalSourceContainer >, Eigen::MatrixXd > ContainerPhonemeAnalyzer
Container phoneme analyzer: SignalContainer -> MatrixXd.
AnalysisType
Categories of analysis operations for discovery and organization.
Definition Analyzer.hpp:33
@ PATTERN
Pattern recognition and matching.
Definition Analyzer.hpp:39
const Facet< uint32_t > phoneme_class
Region attribute: the class of the region's phoneme.
PhonemeAnalyzer< std::vector< Kakshya::DataVariant >, Kakshya::CompositeArray > CompositePhonemeAnalyzer
Record phoneme analyzer: DataVariant -> one Composite record per recognised phoneme.
PhonemeMethod
The per-frame values a PhonemeAnalyzer reports as its output data.
@ HELD_CLASSES
Three columns of 0 or 1: whether the frame's centre lies in the estimated range of a vowel,...
@ POSTERIORS
One column per class: the probability of each of the network's classes.
@ BROAD_CLASSES
Three columns: the summed probability of the vowels, of the consonants, and of everything else.
PhonemeAnalyzer< std::vector< Kakshya::DataVariant >, Eigen::MatrixXd > StandardPhonemeAnalyzer
Standard phoneme analyzer: DataVariant -> MatrixXd.
Kakshya::CompositeLayout phoneme_record_layout()
Field layout of the records a PhonemeAnalyzer with Composite output gives, one record per recognised ...
const Facet< double > phoneme_probability
Region attribute: the probability the network gave the region's phoneme.
std::vector< std::reference_wrapper< const OperationConfig > > ConfigRefs
Borrowed views of the configs an operation holds.
Kakshya::RegionGroup phoneme_regions(const PhonemeAnalysis &analysis, uint32_t first_channel, uint32_t last_channel, std::string name="phonemes")
One region per recognised phoneme, labelled with its ARPABET symbol.
void add_region(const Region &region)
Organizes related signal regions into a categorized collection.
static Region audio_span(uint64_t start_frame, uint64_t end_frame, uint32_t start_channel, uint32_t end_channel, const std::string &label="")
Create a Region representing a span in audio (frames and channels).
Definition Region.hpp:196
Named tensors that belong together, such as a model's weights or I/O.
Definition Tensor.hpp:167
Stores a scoped enum in its underlying integer type.
uint32_t window_size
Analysis window length in samples.
Definition Analyzer.hpp:64
uint32_t hop_size
Distance between successive windows in samples.
Definition Analyzer.hpp:66
Settings every analyzer family accepts.
Definition Analyzer.hpp:62
std::optional< Kakshya::CompositeLayout > composite_layout
Dimensions, modality, source type, and optional Composite field schema.
T data
The actual computation data.
Definition DataIO.hpp:64
Facets facets
Typed results that travel with the data, read by type.
Definition DataIO.hpp:68
std::unordered_map< std::string, std::any > metadata
Associated metadata.
Definition DataIO.hpp:67
Input/Output container for computation pipeline data flow with structure preservation.
Definition DataIO.hpp:63
Where a result of the latest utterance is on the device.
PhonemeAnalyzerConfig config
Settings the analysis was produced under.
std::shared_ptr< Kakshya::TensorContainer > embedding_tensor(std::string name) const
The states as a frames by hidden_width float32 tensor.
std::vector< PhonemeEvent > events
The phonemes in order: every frame whose most probable class is a phoneme and differs from the frame ...
std::vector< std::pair< size_t, size_t > > frame_positions
First and one past the last sample of each frame.
size_t frames
Frames of the utterance.
static std::shared_ptr< Kakshya::TensorContainer > as_tensor(const std::vector< float > &values, uint64_t width, std::string name)
std::shared_ptr< Kakshya::TensorContainer > posterior_tensor(std::string name) const
The probabilities as a frames by classes float32 tensor.
std::vector< uint32_t > best
Most probable class of each frame.
std::vector< float > posteriors
Probability of every class on every frame, frame after frame, HubertPhonemeModel::classes values each...
std::vector< float > embeddings
State of every frame before the final layer, HubertPhonemeModel::hidden_width values each.
Analysis result of a PhonemeAnalyzer.
Reflect::EnumByte< PhonemeMethod > method
Values reported as the output data, as a PhonemeMethod value.
double sample_rate
Sample rate in Hz of the input.
bool keep_embeddings
Also read back the network's state of every frame.
Settings of a PhonemeAnalyzer.
size_t sample
First sample of that frame.
float probability
Probability the network gave the phoneme on that frame.
size_t start
Estimated first sample of the phoneme.
size_t end
Estimated sample one past the phoneme's last.
size_t frame
Frame it was recognised on.
uint32_t phoneme
Class of the phoneme, 0 to HubertPhonemeModel::phonemes - 1.
One phoneme the network recognised.
The config structs an operation class accepts, declared as using supports.
A set of (device, runtime) pairs.