97 std::vector<uint32_t>
best;
109 [[nodiscard]] std::shared_ptr<Kakshya::TensorContainer>
posterior_tensor(std::string name)
const
118 [[nodiscard]] std::shared_ptr<Kakshya::TensorContainer>
embedding_tensor(std::string name)
const
124 [[nodiscard]]
static std::shared_ptr<Kakshya::TensorContainer>
as_tensor(
const std::vector<float>& values, uint64_t width, std::string name)
126 if (values.empty()) {
130 return data ? std::make_shared<Kakshya::TensorContainer>(std::move(*data)) :
nullptr;
147 layout.
add_field<std::string>(
"phoneme");
176 const PhonemeAnalysis& analysis, uint32_t first_channel, uint32_t last_channel, std::string name =
"phonemes")
179 for (
const auto& event : analysis.
events) {
180 if (event.end <= event.start) {
229template <ComputeData InputType = std::vector<Kakshya::DataVariant>, ComputeData OutputType = Eigen::MatrixXd>
247 if (
const std::string reason =
diagnose(
config); !reason.empty()) {
249 std::source_location::current(),
"PhonemeAnalyzer: {}", reason);
253 std::source_location::current(),
"PhonemeAnalyzer: {}", reason);
255 m_weights = std::make_shared<const Kakshya::TensorSet>(weights);
271 return std::format(
"sample_rate {} is not the {} Hz the network runs at; convert the input first, for example with Kinesis::Discrete::resample",
283 return safe_any_cast_or_throw<PhonemeAnalysis>(result);
308 std::source_location::current(),
"The recording can only be shared before the network starts");
340 [[nodiscard]] std::string
get_name()
const override
342 return "PhonemeAnalyzer";
358 if (
const std::string reason =
diagnose(cfg); !reason.empty()) {
360 std::source_location::current(),
"PhonemeAnalyzer: {}", reason);
366 if (data_span.size() != 1) {
368 std::source_location::current(),
"PhonemeAnalyzer analyses one channel and was given {}", data_span.size());
371 const std::vector<float> samples(data_span[0].begin(), data_span[0].end());
378 if (cfg.keep_embeddings) {
391 }
catch (
const std::exception& e) {
422 std::vector<std::vector<double>> columns(7);
423 for (
const auto& event : analysis.
events) {
424 columns[0].push_back(
static_cast<double>(event.phoneme));
425 columns[1].push_back(
static_cast<double>(event.frame));
426 columns[2].push_back(
static_cast<double>(event.sample));
427 columns[3].push_back(
static_cast<double>(event.start));
428 columns[4].push_back(
static_cast<double>(event.end));
429 columns[5].push_back(
static_cast<double>(event.probability));
433 for (
size_t row = 0; row < analysis.
events.size(); ++row) {
449 for (
size_t frame = 0; frame < analysis.
frames; ++frame) {
450 const float* row = analysis.
posteriors.data() + frame * classes;
451 const auto top =
static_cast<uint32_t
>(std::max_element(row, row + classes) - row);
453 analysis.
best.push_back(top);
456 analysis.
events.push_back({ .phoneme = top, .frame = frame, .sample = start, .probability = row[top] });
472 auto& events = analysis.
events;
473 if (events.empty()) {
476 const auto evidence = [&](
size_t frame, uint32_t phoneme) {
477 const float* row = analysis.
posteriors.data() + frame * classes;
479 for (
size_t index = 0; index < phonemes; ++index) {
482 return mass > 0.0 ? std::log(row[phoneme] / mass + 1.0e-30) : 0.0;
485 events.front().start = 0;
486 for (
size_t k = 0; k + 1 < events.size(); ++k) {
487 const size_t first = events[k].frame;
488 const size_t last = events[k + 1].frame;
489 const uint32_t earlier = events[k].phoneme;
490 const uint32_t later = events[k + 1].phoneme;
493 for (
size_t frame = first + 1; frame <= last; ++frame) {
494 score += evidence(frame, later);
497 size_t split = first + 1;
498 for (
size_t candidate = first + 2; candidate <= last; ++candidate) {
499 score += evidence(candidate - 1, earlier) - evidence(candidate - 1, later);
506 events[k].end = boundary;
507 events[k + 1].start = boundary;
509 events.back().end = samples;
516 std::vector<std::vector<double>> held(3, std::vector<double>(analysis.
frames, 0.0));
518 for (
size_t frame = 0; frame < analysis.
frames; ++frame) {
520 while (current < analysis.
events.size() && analysis.
events[current].end <= centre) {
523 const bool inside = current < analysis.
events.size() && analysis.
events[current].start <= centre;
529 std::vector<std::vector<double>> columns(broad ? 3 : classes, std::vector<double>(analysis.
frames, 0.0));
530 for (
size_t frame = 0; frame < analysis.
frames; ++frame) {
531 const float* row = analysis.
posteriors.data() + frame * classes;
532 for (uint32_t index = 0; index < classes; ++index) {
534 columns[index][frame] = row[index];
536 columns[2][frame] += row[index];
547 mutable std::unique_ptr<HubertPhonemeModel>
m_model;
Modern, digital-first analyzer framework for Maya Flux.
#define MF_ERROR(comp, ctx,...)
#define MF_DESCRIBE(...)
Requests a generated member description for a struct at namespace scope.
A HuBERT base network with a phoneme head run on the GPU, one phoneme distribution per 20 ms frame.
Typed key for an arbitrary attribute stored in a Facets bag.
void set(T value)
Stores a struct under its type identity.
Owns a sequence of values that share a CompositeLayout.
bool add_field(std::string name)
Declare a named arithmetic, GLM, text, or binary field.
Shared field description and byte layout for a CompositeArray.
static std::optional< TensorData > from_values(std::span< const T > values, std::vector< uint64_t > shape={}, std::string name={})
Contiguous tensor holding a copy of typed values.
bool is_initialized() const
Check if compiler is initialized.
void store_current_analysis(AnalysisResultType &&result) const
std::any get_current_analysis() const
Access cached analysis from last operation.
std::any analyze_data(const input_type &data)
User-facing analysis method - returns analysis results directly.
Template-flexible analyzer base with instance-defined I/O types.
const Config & config() const
void set_config(Config config)
Stores a config struct, whole, under its own type.
ConfigRefs stored_configs(Supports< Configs... >) const
The stored configs among those listed, in listed order.
output_type convert_result(std::vector< std::vector< double > > &result_data, DataStructureInfo &metadata)
Convert processed double data back to OutputType using metadata and optional callback.
DeviceBuffer posterior_buffer()
The device buffer of the distributions, classes rows of frames.
static constexpr uint32_t frame_hop
Samples between the starts of successive frames.
static constexpr uint32_t phonemes
Classes 0 to phonemes - 1 are phonemes.
DeviceBuffer hidden_buffer()
The device buffer of the states, hidden_width rows of frames.
static constexpr uint32_t classes
Scores per frame.
static bool is_vowel(uint32_t index) noexcept
Whether a class is a vowel phoneme.
static std::string diagnose(const Kakshya::TensorSet &weights)
Check that a tensor set holds the weights the network needs.
static constexpr uint32_t blank_class
Class the network emits between phonemes.
static constexpr uint32_t sample_rate
Sample rate in Hz the network was trained and runs at.
static std::string_view label(uint32_t index) noexcept
Name of a class: the ARPABET symbol of a phoneme, or a bracketed name for the others.
std::vector< float > process(std::span< const float > samples)
Run one utterance.
static constexpr uint32_t frame_span
Samples one frame's features are computed from.
static constexpr uint32_t hidden_width
Width of the state before the final layer.
std::vector< float > hidden_states()
The state before the final layer for every frame of the latest utterance.
HuBERT base fine tuned for English phonemes at 16 kHz: an utterance in, a distribution over phonemes ...
static std::tuple< NumericChannels, DataStructureInfo > extract_structured_double(T &compute_data)
Extract structured double data from Datum container or direct ComputeData with automatic container ha...
size_t resident_frames() const noexcept
Frames the latest utterance left on the device.
HubertPhonemeModel::DeviceBuffer posterior_buffer()
Where the probabilities of the latest utterance are on the device, for a consumer on the same recordi...
Supports< PhonemeAnalyzerConfig > supports
Datum< OutputType > output_type
PhonemeAnalyzer(const Kakshya::TensorSet &weights, PhonemeAnalyzerConfig config={})
Construct from the network weights.
void share_recording(std::shared_ptr< GpuDispatchRecorder > recording)
Append to a recording the caller shares with other work.
AnalysisType get_analysis_type() const override
Gets the analysis type category for this analyzer.
Analyzer< InputType, OutputType > base_type
PhonemeAnalysis analyze_phonemes(const InputType &data)
std::shared_ptr< const Kakshya::TensorSet > m_weights
static void describe_frames(PhonemeAnalysis &analysis)
static std::vector< std::vector< double > > select_values(const PhonemeAnalysis &analysis, PhonemeMethod method)
static constexpr Target target
std::string get_name() const override
Get operation name for debugging/introspection.
ConfigRefs configs() const override
The base view of the config structs this operation holds.
static std::string diagnose(const PhonemeAnalyzerConfig &config)
Why a config cannot be analysed.
static float fitness(const input_type &)
One while a device is up, zero otherwise.
HubertPhonemeModel::DeviceBuffer hidden_buffer()
Where the states of the latest utterance are on the device, for a consumer on the same recording.
PhonemeAnalysis analyze_phonemes(const input_type &data)
Type-safe phoneme analysis.
HubertPhonemeModel & network() const
std::unique_ptr< HubertPhonemeModel > m_model
static output_type empty_output()
Datum< InputType > input_type
output_type analyze_implementation(const input_type &input) override
Pure virtual analysis implementation - derived classes implement this.
output_type make_output(const PhonemeAnalysis &analysis, PhonemeMethod method, DataStructureInfo &structure_info)
PhonemeAnalysis get_phoneme_analysis() const
static void bound_events(PhonemeAnalysis &analysis, size_t samples)
Gives every event its sample range.
std::shared_ptr< GpuDispatchRecorder > m_recording
Which phoneme is spoken near each frame of a mono 16 kHz utterance, from a learned network.
An owning array of schema-bearing Composite elements.
@ ComputeMatrix
Compute operations (Yantra - algorithms, matrices, DSP).
@ Yantra
DSP algorithms, computational units, matrix operations, Grammar.
MAYAFLUX_API ShaderFoundry & get_shader_foundry()
Get the global shader compiler instance.
PhonemeAnalyzer< std::shared_ptr< Kakshya::SignalSourceContainer >, Eigen::MatrixXd > ContainerPhonemeAnalyzer
Container phoneme analyzer: SignalContainer -> MatrixXd.
AnalysisType
Categories of analysis operations for discovery and organization.
@ PATTERN
Pattern recognition and matching.
const Facet< uint32_t > phoneme_class
Region attribute: the class of the region's phoneme.
PhonemeAnalyzer< std::vector< Kakshya::DataVariant >, Kakshya::CompositeArray > CompositePhonemeAnalyzer
Record phoneme analyzer: DataVariant -> one Composite record per recognised phoneme.
PhonemeMethod
The per-frame values a PhonemeAnalyzer reports as its output data.
@ HELD_CLASSES
Three columns of 0 or 1: whether the frame's centre lies in the estimated range of a vowel,...
@ POSTERIORS
One column per class: the probability of each of the network's classes.
@ BROAD_CLASSES
Three columns: the summed probability of the vowels, of the consonants, and of everything else.
PhonemeAnalyzer< std::vector< Kakshya::DataVariant >, Eigen::MatrixXd > StandardPhonemeAnalyzer
Standard phoneme analyzer: DataVariant -> MatrixXd.
Kakshya::CompositeLayout phoneme_record_layout()
Field layout of the records a PhonemeAnalyzer with Composite output gives, one record per recognised ...
const Facet< double > phoneme_probability
Region attribute: the probability the network gave the region's phoneme.
std::vector< std::reference_wrapper< const OperationConfig > > ConfigRefs
Borrowed views of the configs an operation holds.
Kakshya::RegionGroup phoneme_regions(const PhonemeAnalysis &analysis, uint32_t first_channel, uint32_t last_channel, std::string name="phonemes")
One region per recognised phoneme, labelled with its ARPABET symbol.
void add_region(const Region ®ion)
Organizes related signal regions into a categorized collection.
static Region audio_span(uint64_t start_frame, uint64_t end_frame, uint32_t start_channel, uint32_t end_channel, const std::string &label="")
Create a Region representing a span in audio (frames and channels).
Named tensors that belong together, such as a model's weights or I/O.
Stores a scoped enum in its underlying integer type.
uint32_t window_size
Analysis window length in samples.
uint32_t hop_size
Distance between successive windows in samples.
Settings every analyzer family accepts.
std::optional< Kakshya::CompositeLayout > composite_layout
Dimensions, modality, source type, and optional Composite field schema.
T data
The actual computation data.
Facets facets
Typed results that travel with the data, read by type.
std::unordered_map< std::string, std::any > metadata
Associated metadata.
Input/Output container for computation pipeline data flow with structure preservation.
Where a result of the latest utterance is on the device.
PhonemeAnalyzerConfig config
Settings the analysis was produced under.
std::shared_ptr< Kakshya::TensorContainer > embedding_tensor(std::string name) const
The states as a frames by hidden_width float32 tensor.
std::vector< PhonemeEvent > events
The phonemes in order: every frame whose most probable class is a phoneme and differs from the frame ...
std::vector< std::pair< size_t, size_t > > frame_positions
First and one past the last sample of each frame.
size_t frames
Frames of the utterance.
static std::shared_ptr< Kakshya::TensorContainer > as_tensor(const std::vector< float > &values, uint64_t width, std::string name)
std::shared_ptr< Kakshya::TensorContainer > posterior_tensor(std::string name) const
The probabilities as a frames by classes float32 tensor.
std::vector< uint32_t > best
Most probable class of each frame.
std::vector< float > posteriors
Probability of every class on every frame, frame after frame, HubertPhonemeModel::classes values each...
std::vector< float > embeddings
State of every frame before the final layer, HubertPhonemeModel::hidden_width values each.
Analysis result of a PhonemeAnalyzer.
Reflect::EnumByte< PhonemeMethod > method
Values reported as the output data, as a PhonemeMethod value.
double sample_rate
Sample rate in Hz of the input.
bool keep_embeddings
Also read back the network's state of every frame.
Settings of a PhonemeAnalyzer.
size_t sample
First sample of that frame.
float probability
Probability the network gave the phoneme on that frame.
size_t start
Estimated first sample of the phoneme.
size_t end
Estimated sample one past the phoneme's last.
size_t frame
Frame it was recognised on.
uint32_t phoneme
Class of the phoneme, 0 to HubertPhonemeModel::phonemes - 1.
One phoneme the network recognised.
The config structs an operation class accepts, declared as using supports.
A set of (device, runtime) pairs.