113 std::vector<std::pair<size_t, size_t>>
speech;
119 std::vector<std::pair<size_t, size_t>>
voiced;
133 layout.add_field<uint32_t>(
"segment");
151template <ComputeData InputType = std::vector<Kakshya::DataVariant>, ComputeData OutputType = Kakshya::CompositeArray>
172 if (
const std::string reason =
diagnose(
config); !reason.empty()) {
174 std::source_location::current(),
"PhonemeExtractor: {}", reason);
195 if (
config.gate.enabled && (!std::isfinite(
config.gate.threshold) ||
config.gate.threshold <= 0.0 ||
config.gate.hop == 0)) {
196 return "gate threshold must be finite and greater than 0, and its hop greater than 0";
208 const auto*
record = result.facets.template get<PhonemeExtraction>();
211 std::source_location::current(),
"Phoneme extraction produced no result");
232 return "PhonemeExtractor";
239 if (
const std::string reason =
diagnose(cfg); !reason.empty()) {
241 std::source_location::current(),
"PhonemeExtractor: {}", reason);
245 if (channels.size() != 1) {
247 std::source_location::current(),
"PhonemeExtractor analyses one channel and was given {}", channels.size());
249 const std::span<const double> samples { channels[0].data(), channels[0].size() };
253 if (!samples.empty()) {
267 }
catch (
const std::exception& e) {
308 std::vector<Piece> pieces;
309 for (
const size_t row : rows) {
310 const auto&
event =
record.events[row];
311 if (event.end <= event.start) {
315 pieces.push_back({
event.start,
event.end, row });
319 pieces.push_back({ start, end, row });
327 for (uint32_t index = 0; index <
record.speech.size(); ++index) {
328 const auto [first, last] =
record.speech[index];
332 const auto part = samples.subspan(first, last - first);
333 const auto analysis =
m_phonemes.analyze_phonemes(
334 std::vector<Kakshya::DataVariant> { std::vector<double>(part.begin(), part.end()) });
335 for (
auto event : analysis.events) {
336 event.sample += first;
337 event.start += first;
339 record.events.push_back(event);
340 record.segments.push_back(index);
361 std::vector<size_t> rows;
362 for (
size_t row = 0; row <
record.events.size(); ++row) {
384 std::vector<std::vector<double>> columns(8);
385 for (
const size_t row : rows) {
386 const auto&
event =
record.events[row];
387 columns[0].push_back(
static_cast<double>(event.phoneme));
388 columns[1].push_back(
static_cast<double>(event.frame));
389 columns[2].push_back(
static_cast<double>(event.sample));
390 columns[3].push_back(
static_cast<double>(event.start));
391 columns[4].push_back(
static_cast<double>(event.end));
392 columns[5].push_back(
static_cast<double>(event.probability));
394 columns[7].push_back(
static_cast<double>(
record.segments[row]));
397 for (
size_t k = 0; k < rows.size(); ++k) {
401 }
else if constexpr (std::is_same_v<OutputType, Kakshya::RegionGroup>) {
404 const auto&
event =
record.events[piece.row];
412 output.
data = std::move(group);
415 std::vector<std::pair<size_t, size_t>> ranges;
417 ranges.emplace_back(piece.start, piece.end);
419 if (
record.config.merge_adjacent) {
#define MF_ERROR(comp, ctx,...)
#define MF_DESCRIBE(...)
Requests a generated member description for a struct at namespace scope.
Which phoneme is spoken near each 20 ms frame of an utterance, from a learned network run on the GPU.
Where speech is in a recording, with optionally how voiced each stretch is and its pitch.
void set(T value)
Stores a struct under its type identity.
Owns a sequence of values that share a CompositeLayout.
Shared field description and byte layout for a CompositeArray.
bool is_initialized() const
Check if compiler is initialized.
const Config & config() const
void set_config(Config config)
Stores a config struct, whole, under its own type.
output_type apply_operation(const input_type &input)
Public synchronous execution interface.
ConfigRefs stored_configs(Supports< Configs... >) const
The stored configs among those listed, in listed order.
output_type convert_result(std::vector< std::vector< double > > &result_data, DataStructureInfo &metadata)
Convert processed double data back to OutputType using metadata and optional callback.
static bool is_vowel(uint32_t index) noexcept
Whether a class is a vowel phoneme.
static std::string_view label(uint32_t index) noexcept
Name of a class: the ARPABET symbol of a phoneme, or a bracketed name for the others.
static constexpr uint32_t frame_span
Samples one frame's features are computed from.
static constexpr size_t max_samples
Longest utterance one call accepts, 30 s.
static std::tuple< NumericChannels, DataStructureInfo > extract_structured_double(T &compute_data)
Extract structured double data from Datum container or direct ComputeData with automatic container ha...
static constexpr uint32_t sample_rate
Sample rate in Hz the network was trained and runs at.
static std::string diagnose(const SpeechSegmenterConfig &config)
Why a config cannot be segmented with.
An owning array of schema-bearing Composite elements.
@ ComputeMatrix
Compute operations (Yantra - algorithms, matrices, DSP).
@ Yantra
DSP algorithms, computational units, matrix operations, Grammar.
std::vector< double > slice_intervals(std::span< const double > data, const std::vector< std::pair< size_t, size_t > > &intervals)
Copy data from a set of half-open intervals into a flat vector.
std::vector< std::pair< size_t, size_t > > merge_intervals(const std::vector< std::pair< size_t, size_t > > &intervals)
Merge overlapping or adjacent half-open intervals.
std::vector< std::pair< size_t, size_t > > intersect_intervals(const std::vector< std::pair< size_t, size_t > > &first, const std::vector< std::pair< size_t, size_t > > &second)
Parts covered by both of two sets of half-open intervals.
MAYAFLUX_API ShaderFoundry & get_shader_foundry()
Get the global shader compiler instance.
ExtractionType
Categories of extraction operations for discovery and organization.
@ PATTERN_BASED
Extract based on pattern recognition.
SpeechSegmenterConfig speech_segmenter_config(const PhonemeExtractorConfig &config)
The segmenter settings a PhonemeExtractorConfig implies.
PhonemeExtractor< std::vector< Kakshya::DataVariant >, Kakshya::RegionGroup > RegionPhonemeExtractor
Regions: one region per selected phoneme.
std::vector< std::pair< size_t, size_t > > voiced_intervals(StandardWaveformAnalyzer &analyzer, std::span< const double > samples, double threshold, uint32_t hop)
The stretches of a recording where the waveform is voiced.
PhonemeExtractor< std::vector< Kakshya::DataVariant >, Kakshya::CompositeArray > StandardPhonemeExtractor
Records: one Composite record per selected phoneme.
SpeechSegmenter< std::vector< Kakshya::DataVariant >, Kakshya::RegionGroup > RegionSpeechSegmenter
Regions: one region per speech interval.
const Facet< uint32_t > phoneme_class
Region attribute: the class of the region's phoneme.
Kakshya::CompositeLayout phoneme_extraction_layout()
Field layout of the records a PhonemeExtractor gives: the phoneme record layout with the speech inter...
PhonemeSelection
Which recognised phonemes the output holds.
@ CONSONANTS
Phonemes that are not vowels.
@ ALL
Every recognised phoneme.
@ VOWELS
Vowel phonemes only.
@ HELD_CLASSES
Three columns of 0 or 1: whether the frame's centre lies in the estimated range of a vowel,...
PhonemeAnalyzer< std::vector< Kakshya::DataVariant >, Eigen::MatrixXd > StandardPhonemeAnalyzer
Standard phoneme analyzer: DataVariant -> MatrixXd.
Kakshya::CompositeLayout phoneme_record_layout()
Field layout of the records a PhonemeAnalyzer with Composite output gives, one record per recognised ...
PhonemeExtractor< std::vector< Kakshya::DataVariant >, std::vector< Kakshya::DataVariant > > AudioPhonemeExtractor
Audio: the audio of the selected phonemes, concatenated.
const Facet< double > phoneme_probability
Region attribute: the probability the network gave the region's phoneme.
std::vector< std::reference_wrapper< const OperationConfig > > ConfigRefs
Borrowed views of the configs an operation holds.
WaveformAnalyzerConfig voicing_analyzer_config(double sample_rate, uint32_t hop=256)
The waveform analyzer settings the voicing measure uses at the given rate and hop.
std::pair< std::shared_ptr< Kriya::BufferPipeline >, std::shared_ptr< Buffers::AudioBuffer > > record(const std::shared_ptr< Kakshya::DynamicSoundStream > &stream, uint32_t input_channel, double seconds, std::optional< uint32_t > monitor, uint32_t channel)
Record a live input channel into a stream for a number of seconds.
void add_region(const Region ®ion)
Organizes related signal regions into a categorized collection.
static Region audio_span(uint64_t start_frame, uint64_t end_frame, uint32_t start_channel, uint32_t end_channel, const std::string &label="")
Create a Region representing a span in audio (frames and channels).
Named tensors that belong together, such as a model's weights or I/O.
Stores a scoped enum in its underlying integer type.
std::optional< Kakshya::CompositeLayout > composite_layout
Dimensions, modality, source type, and optional Composite field schema.
T data
The actual computation data.
Facets facets
Typed results that travel with the data, read by type.
std::unordered_map< std::string, std::any > metadata
Associated metadata.
Input/Output container for computation pipeline data flow with structure preservation.
Reflect::EnumByte< PhonemeMethod > method
Values reported as the output data, as a PhonemeMethod value.
double sample_rate
Sample rate in Hz of the input.
Settings of a PhonemeAnalyzer.
uint32_t phoneme
Class of the phoneme, 0 to HubertPhonemeModel::phonemes - 1.
One phoneme the network recognised.
double sample_rate
Sample rate in Hz of the input.
SpeechSettings speech
How speech is found and measured.
Settings of a SpeechSegmenter.
bool measure
Also measure the voiced fraction and pitch of each speech interval.
uint32_t max_run
Longest speech interval in samples, 0 for no cap.
How speech runs are found and measured.
The config structs an operation class accepts, declared as using supports.
A set of (device, runtime) pairs.
bool enabled
Trim vowel ranges to voiced stretches.
uint32_t hop
Samples between waveform windows.
double threshold
Periodicity at or above which a window is voiced.
Trims vowel ranges to the stretches where the waveform is periodic.