MayaFlux 1.0.0
Digital-First Multimedia Processing Framework
Loading...
Searching...
No Matches

◆ run()

std::vector< float > MayaFlux::Yantra::HubertPhonemeModel::Impl::run ( std::span< const float > samples)
inlinenodiscard

Definition at line 307 of file HubertPhonemeModel.cpp.

308 {
309 const auto lengths = feature_lengths(samples.size());
310 const uint32_t frames = lengths.back();
311 ensure_capacity(samples.size(), lengths);
312 upload_normalised(samples);
313
314 const auto front_at = [&](size_t slot) { return front.address(address(WeightsFront), slot); };
315 const auto block_at = [&](uint32_t block, size_t slot) { return blocks[block].address(address(WeightsBlock0 + block), slot); };
316
317 std::vector<DependencyStage> stages;
318 const auto dense = [&](uint64_t src, uint32_t in, uint64_t weight, uint64_t bias, uint64_t dst, uint32_t out, Activation activation = Activation::None) {
319 stages.push_back(TensorKernels::conv1d_stage({ .c_in = in, .c_out = out, .kernel = 1, .activation = activation }, src, weight, bias, dst, 1, frames));
320 };
321 const auto layer_norm = [&](uint64_t data, uint32_t channels, uint64_t weight, uint64_t bias) {
322 stages.push_back(TensorKernels::normalize_stats_stage(data, address(Stats), 1, channels, frames, k_norm_epsilon));
323 stages.push_back(TensorKernels::normalize_apply_stage(data, address(Stats), weight, bias, data, 1, channels, frames, NormAffine::Along));
324 };
325
326 uint64_t features = address(Audio);
327 auto length = static_cast<uint32_t>(samples.size());
328 for (uint32_t i = 0; i < k_feature_layers; ++i) {
329 const uint64_t dst = address(i % 2 == 0 ? WideA : WideB);
330 stages.push_back(TensorKernels::conv1d_stage(feature_layer(i), features, front_at(FeatureWeight0 + i), 0, dst, 1, length));
331 if (i == 0) {
332 stages.push_back(TensorKernels::normalize_stats_stage(dst, address(Stats), k_features, lengths[0], 1, k_norm_epsilon));
333 stages.push_back(TensorKernels::normalize_apply_stage(dst, address(Stats), front_at(FeatureNormWeight), front_at(FeatureNormBias),
334 dst, k_features, lengths[0], 1, NormAffine::Outer, Activation::Gelu));
335 }
336 features = dst;
337 length = lengths[i];
338 }
339
340 const uint64_t hidden = address(Hidden);
341 layer_norm(features, k_features, front_at(ProjectionNormWeight), front_at(ProjectionNormBias));
342 dense(features, k_features, front_at(ProjectionWeight), front_at(ProjectionBias), hidden, k_width);
343
345 { .c_in = k_width, .c_out = k_width, .kernel = k_position_kernel, .groups = k_position_groups, .pad = k_position_kernel / 2, .trim = 1, .activation = Activation::Gelu },
346 hidden, address(WeightsPosition), front_at(PositionBias), address(Positional), 1, frames));
347 stages.push_back(TensorKernels::add_stage(hidden, address(Positional), hidden, k_width * frames));
348 layer_norm(hidden, k_width, front_at(EncoderNormWeight), front_at(EncoderNormBias));
349
350 const float score_scale = 1.0F / std::sqrt(static_cast<float>(k_head_width));
351 for (uint32_t block = 0; block < k_blocks; ++block) {
352 dense(hidden, k_width, block_at(block, QueryWeight), block_at(block, QueryBias), address(Query), k_width);
353 dense(hidden, k_width, block_at(block, KeyWeight), block_at(block, KeyBias), address(Key), k_width);
354 dense(hidden, k_width, block_at(block, ValueWeight), block_at(block, ValueBias), address(Value), k_width);
355 stages.push_back(TensorKernels::attention_scores_stage(address(Query), address(Key), address(Scores), k_heads, k_head_width, frames, score_scale));
356 stages.push_back(TensorKernels::softmax_stage(address(Scores), k_heads * frames, frames, 1));
357 stages.push_back(TensorKernels::attention_mix_stage(address(Scores), address(Value), address(Query), k_heads, k_head_width, frames));
358 dense(address(Query), k_width, block_at(block, OutputWeight), block_at(block, OutputBias), address(Key), k_width);
359 stages.push_back(TensorKernels::add_stage(hidden, address(Key), hidden, k_width * frames));
360 layer_norm(hidden, k_width, block_at(block, NormWeight), block_at(block, NormBias));
361
362 dense(hidden, k_width, block_at(block, ExpandWeight), block_at(block, ExpandBias), address(Expanded), k_expanded, Activation::Gelu);
363 dense(address(Expanded), k_expanded, block_at(block, ReduceWeight), block_at(block, ReduceBias), address(Key), k_width);
364 stages.push_back(TensorKernels::add_stage(hidden, address(Key), hidden, k_width * frames));
365 layer_norm(hidden, k_width, block_at(block, FinalNormWeight), block_at(block, FinalNormBias));
366 }
367
368 dense(hidden, k_width, front_at(HeadWeight), front_at(HeadBias), address(Posterior), k_classes);
369 stages.push_back(TensorKernels::softmax_stage(address(Posterior), 1, k_classes, frames));
370
371 context->record_stages(stages);
372 resident = frames;
373
374 std::vector<float> rows(static_cast<size_t>(k_classes) * frames);
375 context->download_shared(0, Posterior, rows.data(), rows.size() * sizeof(float));
376 return frame_major(rows, k_classes, frames);
377 }
DependencyStage softmax_stage(uint64_t data, uint32_t outer, uint32_t extent, uint32_t inner)
One nn_softmax dispatch: softmax of a tensor [outer, extent, inner] along extent, in place.
Definition Stages.hpp:228
DependencyStage grouped_conv1d_stage(const GroupedConv1d &layer, uint64_t src, uint64_t weights, uint64_t bias, uint64_t dst, uint32_t batch, uint32_t t_in)
One nn_conv1d_grouped dispatch: [batch, c_in, t_in] to [batch, c_out, grouped_conv1d_length].
Definition Stages.hpp:110
DependencyStage add_stage(uint64_t a, uint64_t b, uint64_t dst, uint32_t count)
One nn_add dispatch: the element-wise sum of two sequences of count floats.
Definition Stages.hpp:197
DependencyStage normalize_apply_stage(uint64_t src, uint64_t stats, uint64_t weight, uint64_t bias, uint64_t dst, uint32_t outer, uint32_t extent, uint32_t inner, NormAffine affine, Activation activation=Activation::None)
The nn_normalize dispatch that writes (x - mean) * scale from the measures of normalize_stats_stage.
Definition Stages.hpp:175
DependencyStage normalize_stats_stage(uint64_t src, uint64_t stats, uint32_t outer, uint32_t extent, uint32_t inner, float epsilon)
The nn_normalize dispatch that measures every row of a tensor [outer, extent, inner] along extent.
Definition Stages.hpp:152
DependencyStage attention_scores_stage(uint64_t q, uint64_t k, uint64_t scores, uint32_t heads, uint32_t dim, uint32_t time, float scale)
One nn_attention_scores dispatch: scale times the dot product of every query with every key,...
Definition Stages.hpp:214
DependencyStage attention_mix_stage(uint64_t weights, uint64_t v, uint64_t dst, uint32_t heads, uint32_t dim, uint32_t time)
One nn_attention_mix dispatch: the values of every position mixed by each query's attention weights.
Definition Stages.hpp:245
DependencyStage conv1d_stage(const Conv1d &layer, uint64_t src, uint64_t weights, uint64_t bias, uint64_t dst, uint32_t batch, uint32_t t_in)
One nn_conv1d dispatch: [batch, c_in, t_in] to [batch, c_out, conv1d_length].
Definition Stages.hpp:70
Activation
Output activation of a layer.
Definition Layers.hpp:16
static constexpr DomainSpec Audio
Domain constant for Audio domain.
Definition Creator.hpp:497
std::unique_ptr< ShaderExecutionContext<> > context
void upload_normalised(std::span< const float > samples)
void ensure_capacity(size_t samples, const std::array< uint32_t, k_feature_layers > &lengths)
std::array< TensorKernels::PackedWeights, k_blocks > blocks

References MayaFlux::Yantra::TensorKernels::add_stage(), address(), MayaFlux::Yantra::TensorKernels::attention_mix_stage(), MayaFlux::Yantra::TensorKernels::attention_scores_stage(), MayaFlux::Audio, blocks, context, MayaFlux::Yantra::TensorKernels::conv1d_stage(), ensure_capacity(), front, MayaFlux::Yantra::TensorKernels::grouped_conv1d_stage(), MayaFlux::Yantra::TensorKernels::normalize_apply_stage(), MayaFlux::Yantra::TensorKernels::normalize_stats_stage(), resident, MayaFlux::Yantra::TensorKernels::softmax_stage(), and upload_normalised().

Here is the call graph for this function: