17 constexpr uint32_t k_feature_layers = 7;
18 constexpr uint32_t k_features = 512;
20 constexpr uint32_t k_expanded = 3072;
21 constexpr uint32_t k_blocks = 12;
22 constexpr uint32_t k_heads = 12;
23 constexpr uint32_t k_head_width = k_width / k_heads;
24 constexpr uint32_t k_position_kernel = 128;
25 constexpr uint32_t k_position_groups = 16;
27 constexpr float k_norm_epsilon = 1.0e-5F;
28 constexpr double k_input_epsilon = 1.0e-7;
30 constexpr std::array<uint32_t, k_feature_layers> k_feature_kernels { 10, 3, 3, 3, 3, 2, 2 };
31 constexpr std::array<uint32_t, k_feature_layers> k_feature_strides { 5, 2, 2, 2, 2, 2, 2 };
33 constexpr std::array<std::string_view, k_classes> k_labels {
34 "AA",
"AE",
"AH",
"AO",
"AW",
"AY",
"B",
"CH",
"D",
"DH",
"EH",
"ER",
"EY",
"F",
"G",
"HH",
"IH",
"IY",
"JH",
"K",
35 "L",
"M",
"N",
"NG",
"OW",
"OY",
"P",
"R",
"S",
"SH",
"T",
"TH",
"UH",
"UW",
"V",
"W",
"Y",
"Z",
"ZH",
36 "[UNK]",
"[PAD]",
"<s>",
"</s>"
39 constexpr std::array<uint32_t, 15> k_vowels { 0, 1, 2, 3, 4, 5, 10, 11, 12, 16, 17, 24, 25, 32, 33 };
43 FeatureNormWeight = FeatureWeight0 + k_feature_layers,
77 enum Position :
size_t {
83 enum Buffer : uint32_t {
104 .c_in = index == 0 ? 1U : k_features,
106 .kernel = k_feature_kernels[index],
107 .stride = k_feature_strides[index],
108 .activation = index == 0 ? Activation::None : Activation::Gelu,
112 std::vector<TensorRequirement> front_specs()
114 std::vector<TensorRequirement> specs;
115 specs.reserve(FrontCount);
116 for (uint32_t i = 0; i < k_feature_layers; ++i) {
117 specs.push_back({ std::format(
"hubert.feature_extractor.conv_layers.{}.conv.weight", i),
118 { k_features, i == 0 ? 1U : k_features, k_feature_kernels[i] } });
120 specs.push_back({
"hubert.feature_extractor.conv_layers.0.layer_norm.weight", { k_features } });
121 specs.push_back({
"hubert.feature_extractor.conv_layers.0.layer_norm.bias", { k_features } });
122 specs.push_back({
"hubert.feature_projection.layer_norm.weight", { k_features } });
123 specs.push_back({
"hubert.feature_projection.layer_norm.bias", { k_features } });
124 specs.push_back({
"hubert.feature_projection.projection.weight", { k_width, k_features } });
125 specs.push_back({
"hubert.feature_projection.projection.bias", { k_width } });
126 specs.push_back({
"hubert.encoder.pos_conv_embed.conv.bias", { k_width } });
127 specs.push_back({
"hubert.encoder.layer_norm.weight", { k_width } });
128 specs.push_back({
"hubert.encoder.layer_norm.bias", { k_width } });
129 specs.push_back({
"lm_head.weight", { k_classes, k_width } });
130 specs.push_back({
"lm_head.bias", { k_classes } });
134 std::vector<TensorRequirement> block_specs(uint32_t block)
136 const std::string prefix = std::format(
"hubert.encoder.layers.{}.", block);
137 const auto dense = [&](std::vector<TensorRequirement>& specs, std::string_view name, uint32_t out, uint32_t in) {
138 specs.push_back({ std::format(
"{}{}.weight", prefix, name), { out, in } });
139 specs.push_back({ std::format(
"{}{}.bias", prefix, name), { out } });
141 const auto norm = [&](std::vector<TensorRequirement>& specs, std::string_view name) {
142 specs.push_back({ std::format(
"{}{}.weight", prefix, name), { k_width } });
143 specs.push_back({ std::format(
"{}{}.bias", prefix, name), { k_width } });
145 std::vector<TensorRequirement> specs;
146 specs.reserve(BlockCount);
147 dense(specs,
"attention.q_proj", k_width, k_width);
148 dense(specs,
"attention.k_proj", k_width, k_width);
149 dense(specs,
"attention.v_proj", k_width, k_width);
150 dense(specs,
"attention.out_proj", k_width, k_width);
151 norm(specs,
"layer_norm");
152 dense(specs,
"feed_forward.intermediate_dense", k_expanded, k_width);
153 dense(specs,
"feed_forward.output_dense", k_width, k_expanded);
154 norm(specs,
"final_layer_norm");
158 std::vector<TensorRequirement> position_specs()
161 {
"hubert.encoder.pos_conv_embed.conv.weight_g", { 1, 1, k_position_kernel } },
162 {
"hubert.encoder.pos_conv_embed.conv.weight_v", { k_width, k_width / k_position_groups, k_position_kernel } },
171 std::source_location::current(),
"HubertPhonemeModel: {}", packed.error());
173 return std::move(*packed);
182 const size_t rows = packed.offsets.size() > PositionDirection
183 ? (packed.values.size() - packed.offsets[PositionDirection]) / k_position_kernel
185 const float* gain = packed.values.data() + packed.offsets[PositionGain];
186 const float* direction = packed.values.data() + packed.offsets[PositionDirection];
188 std::array<double, k_position_kernel> norms {};
189 for (
size_t row = 0; row < rows; ++row) {
190 for (
size_t tap = 0; tap < k_position_kernel; ++tap) {
191 const double value = direction[row * k_position_kernel + tap];
192 norms[tap] += value * value;
195 std::vector<float> folded(rows * k_position_kernel);
196 for (
size_t row = 0; row < rows; ++row) {
197 for (
size_t tap = 0; tap < k_position_kernel; ++tap) {
198 const double norm = std::sqrt(norms[tap]);
199 folded[row * k_position_kernel + tap] = norm > 0.0
200 ?
static_cast<float>(
static_cast<double>(gain[tap]) * direction[row * k_position_kernel + tap] / norm)
207 std::array<uint32_t, k_feature_layers> feature_lengths(
size_t samples)
209 std::array<uint32_t, k_feature_layers> lengths {};
210 auto length =
static_cast<uint32_t
>(samples);
211 for (uint32_t i = 0; i < k_feature_layers; ++i) {
218 std::vector<float> frame_major(
const std::vector<float>& rows,
size_t channels,
size_t frames)
220 std::vector<float> out(rows.size());
221 for (
size_t channel = 0; channel < channels; ++channel) {
222 for (
size_t frame = 0; frame < frames; ++frame) {
223 out[frame * channels + channel] = rows[channel * frames + frame];
233 std::unique_ptr<ShaderExecutionContext<>>
context;
235 std::array<TensorKernels::PackedWeights, k_blocks>
blocks;
243 context->set_recording(recording);
244 if (!context->ensure_gpu_ready()) {
246 std::source_location::current(),
"HubertPhonemeModel needs a graphics device and none is available");
249 front = pack(weights, front_specs());
251 front.values = std::vector<float> {};
253 for (uint32_t block = 0; block < k_blocks; ++block) {
254 blocks[block] = pack(weights, block_specs(block));
256 blocks[block].values = std::vector<float> {};
259 const auto folded = fold_position_weights(pack(weights, position_specs()));
262 context->upload_shared_raw(0, WeightsPosition,
reinterpret_cast<const uint8_t*
>(folded.data()), folded.size() *
sizeof(
float));
265 void ensure_capacity(
size_t samples,
const std::array<uint32_t, k_feature_layers>& lengths)
269 const size_t frames = lengths.back();
270 context->ensure_shared_buffer(0,
Audio, samples, type, BufferUsageHint::COMPUTE);
271 context->ensure_shared_buffer(0, WideA,
static_cast<size_t>(k_features) * lengths[0], type, BufferUsageHint::COMPUTE);
272 context->ensure_shared_buffer(0, WideB,
static_cast<size_t>(k_features) * lengths[1], type, BufferUsageHint::COMPUTE);
273 context->ensure_shared_buffer(0, Stats, 2 * std::max<size_t>(k_features, frames), type, BufferUsageHint::COMPUTE);
274 for (
const uint32_t buffer : { Hidden, Positional, Query, Key, Value }) {
275 context->ensure_shared_buffer(0, buffer, k_width * frames, type, BufferUsageHint::COMPUTE);
277 context->ensure_shared_buffer(0, Scores, k_heads * frames * frames, type, BufferUsageHint::COMPUTE);
278 context->ensure_shared_buffer(0, Expanded, k_expanded * frames, type, BufferUsageHint::COMPUTE);
279 context->ensure_shared_buffer(0, Posterior, k_classes * frames, type);
282 [[nodiscard]] uint64_t
address(uint32_t buffer)
const
284 return context->shared_buffer_address(0, buffer);
290 for (
const float sample : samples) {
293 const double mean = sum /
static_cast<double>(samples.size());
295 for (
const float sample : samples) {
296 square += (sample - mean) * (sample - mean);
298 const double scale = 1.0 / std::sqrt(square /
static_cast<double>(samples.size()) + k_input_epsilon);
300 std::vector<float> normalised(samples.size());
301 for (
size_t i = 0; i < samples.size(); ++i) {
302 normalised[i] =
static_cast<float>((samples[i] - mean) * scale);
304 context->upload_shared_raw(0,
Audio,
reinterpret_cast<const uint8_t*
>(normalised.data()), normalised.size() *
sizeof(
float));
307 [[nodiscard]] std::vector<float>
run(std::span<const float> samples)
309 const auto lengths = feature_lengths(samples.size());
310 const uint32_t frames = lengths.back();
314 const auto front_at = [&](
size_t slot) {
return front.address(
address(WeightsFront), slot); };
315 const auto block_at = [&](uint32_t block,
size_t slot) {
return blocks[block].address(
address(WeightsBlock0 + block), slot); };
317 std::vector<DependencyStage> stages;
318 const auto dense = [&](uint64_t src, uint32_t in, uint64_t weight, uint64_t bias, uint64_t dst, uint32_t out,
Activation activation = Activation::None) {
319 stages.push_back(
TensorKernels::conv1d_stage({ .c_in = in, .c_out = out, .kernel = 1, .activation = activation }, src, weight, bias, dst, 1, frames));
321 const auto layer_norm = [&](uint64_t data, uint32_t channels, uint64_t weight, uint64_t bias) {
327 auto length =
static_cast<uint32_t
>(samples.size());
328 for (uint32_t i = 0; i < k_feature_layers; ++i) {
329 const uint64_t dst =
address(i % 2 == 0 ? WideA : WideB);
334 dst, k_features, lengths[0], 1, NormAffine::Outer, Activation::Gelu));
340 const uint64_t hidden =
address(Hidden);
341 layer_norm(features, k_features, front_at(ProjectionNormWeight), front_at(ProjectionNormBias));
342 dense(features, k_features, front_at(ProjectionWeight), front_at(ProjectionBias), hidden, k_width);
345 { .c_in = k_width, .c_out = k_width, .kernel = k_position_kernel, .groups = k_position_groups, .pad = k_position_kernel / 2, .trim = 1, .activation = Activation::Gelu },
346 hidden,
address(WeightsPosition), front_at(PositionBias),
address(Positional), 1, frames));
348 layer_norm(hidden, k_width, front_at(EncoderNormWeight), front_at(EncoderNormBias));
350 const float score_scale = 1.0F / std::sqrt(
static_cast<float>(k_head_width));
351 for (uint32_t block = 0; block < k_blocks; ++block) {
352 dense(hidden, k_width, block_at(block, QueryWeight), block_at(block, QueryBias),
address(Query), k_width);
353 dense(hidden, k_width, block_at(block, KeyWeight), block_at(block, KeyBias),
address(Key), k_width);
354 dense(hidden, k_width, block_at(block, ValueWeight), block_at(block, ValueBias),
address(Value), k_width);
358 dense(
address(Query), k_width, block_at(block, OutputWeight), block_at(block, OutputBias),
address(Key), k_width);
360 layer_norm(hidden, k_width, block_at(block, NormWeight), block_at(block, NormBias));
362 dense(hidden, k_width, block_at(block, ExpandWeight), block_at(block, ExpandBias),
address(Expanded), k_expanded, Activation::Gelu);
363 dense(
address(Expanded), k_expanded, block_at(block, ReduceWeight), block_at(block, ReduceBias),
address(Key), k_width);
365 layer_norm(hidden, k_width, block_at(block, FinalNormWeight), block_at(block, FinalNormBias));
368 dense(hidden, k_width, front_at(HeadWeight), front_at(HeadBias),
address(Posterior), k_classes);
371 context->record_stages(stages);
374 std::vector<float> rows(
static_cast<size_t>(k_classes) * frames);
375 context->download_shared(0, Posterior, rows.data(), rows.size() *
sizeof(
float));
376 return frame_major(rows, k_classes, frames);
388 for (uint32_t block = 0; block < k_blocks; ++block) {
398 return index < k_labels.size() ? k_labels[index] : std::string_view {};
403 return std::ranges::find(k_vowels, index) != k_vowels.end();
413 if (samples < frame_span || samples > std::numeric_limits<uint32_t>::max()) {
416 return feature_lengths(samples).back();
426 if (
const std::string reason =
diagnose(weights); !reason.empty()) {
428 std::source_location::current(),
"HubertPhonemeModel: {}", reason);
430 m_impl = std::make_unique<Impl>(weights, std::move(recording));
439 std::source_location::current(),
440 "HubertPhonemeModel: an utterance of {} samples is longer than the {} one call accepts; run it in pieces, for instance the intervals a voice activity analysis gives",
447 return m_impl->run(samples);
452 const size_t frames =
m_impl->resident;
457 m_impl->context->download_shared(0, Hidden, rows.data(), rows.size() *
sizeof(
float));
469 return { .context =
m_impl->context.get(), .set = 0, .binding = Posterior };
476 return { .context =
m_impl->context.get(), .set = 0, .binding = Hidden };
A HuBERT base network with a phoneme head run on the GPU, one phoneme distribution per 20 ms frame.
The stages that run the tensor kernels.
Checking a set of tensors against what a network needs, and packing it into one float32 buffer.
Shared command buffer for GPU dispatch work.
DeviceBuffer posterior_buffer()
The device buffer of the distributions, classes rows of frames.
static constexpr uint32_t phonemes
Classes 0 to phonemes - 1 are phonemes.
HubertPhonemeModel(const Kakshya::TensorSet &weights)
Upload the weights and prepare the GPU resources.
DeviceBuffer hidden_buffer()
The device buffer of the states, hidden_width rows of frames.
static constexpr uint32_t classes
Scores per frame.
static bool is_vowel(uint32_t index) noexcept
Whether a class is a vowel phoneme.
std::unique_ptr< Impl > m_impl
static std::string diagnose(const Kakshya::TensorSet &weights)
Check that a tensor set holds the weights the network needs.
static constexpr uint32_t blank_class
Class the network emits between phonemes.
static PhonemeVocabulary vocabulary() noexcept
The vocabulary of this network: the names, the phoneme count, the blank class and the vowels.
static std::string_view label(uint32_t index) noexcept
Name of a class: the ARPABET symbol of a phoneme, or a bracketed name for the others.
std::vector< float > process(std::span< const float > samples)
Run one utterance.
static constexpr uint32_t frame_span
Samples one frame's features are computed from.
size_t resident_frames() const noexcept
Frames the latest utterance left on the device.
static constexpr uint32_t hidden_width
Width of the state before the final layer.
static constexpr size_t max_samples
Longest utterance one call accepts, 30 s.
std::vector< float > hidden_states()
The state before the final layer for every frame of the latest utterance.
static uint32_t frame_count(size_t samples) noexcept
Frames an utterance of this many samples gives, 0 below frame_span samples.
Concrete GpuExecutionContext for a single fixed shader with fixed bindings.
@ ComputeMatrix
Compute operations (Yantra - algorithms, matrices, DSP).
@ Yantra
DSP algorithms, computational units, matrix operations, Grammar.
BufferUsageHint
Semantic usage hint for buffer allocation and memory properties.
@ COMPUTE
Storage buffer for compute shaders (device-local).
NormAffine
Which index selects the weight and bias applied after a normalisation.
DependencyStage softmax_stage(uint64_t data, uint32_t outer, uint32_t extent, uint32_t inner)
One nn_softmax dispatch: softmax of a tensor [outer, extent, inner] along extent, in place.
DependencyStage grouped_conv1d_stage(const GroupedConv1d &layer, uint64_t src, uint64_t weights, uint64_t bias, uint64_t dst, uint32_t batch, uint32_t t_in)
One nn_conv1d_grouped dispatch: [batch, c_in, t_in] to [batch, c_out, grouped_conv1d_length].
Activation
Output activation of a layer.
DependencyStage add_stage(uint64_t a, uint64_t b, uint64_t dst, uint32_t count)
One nn_add dispatch: the element-wise sum of two sequences of count floats.
DependencyStage normalize_apply_stage(uint64_t src, uint64_t stats, uint64_t weight, uint64_t bias, uint64_t dst, uint32_t outer, uint32_t extent, uint32_t inner, NormAffine affine, Activation activation=Activation::None)
The nn_normalize dispatch that writes (x - mean) * scale from the measures of normalize_stats_stage.
DependencyStage normalize_stats_stage(uint64_t src, uint64_t stats, uint32_t outer, uint32_t extent, uint32_t inner, float epsilon)
The nn_normalize dispatch that measures every row of a tensor [outer, extent, inner] along extent.
DependencyStage attention_scores_stage(uint64_t q, uint64_t k, uint64_t scores, uint32_t heads, uint32_t dim, uint32_t time, float scale)
One nn_attention_scores dispatch: scale times the dot product of every query with every key,...
std::string check_tensors(const Kakshya::TensorSet &weights, std::span< const TensorRequirement > required)
Whether a set holds every required tensor with the required shape.
constexpr uint32_t element_workgroup
Invocations per workgroup of the element-wise kernels.
uint32_t conv1d_length(const Conv1d &layer, uint32_t t_in)
Output length of a convolution, 0 when the window does not fit the padded input.
std::expected< PackedWeights, std::string > pack_weights(const Kakshya::TensorSet &weights, std::span< const TensorRequirement > required)
Convert the required tensors to float32 and lay them out in the order of required.
DependencyStage attention_mix_stage(uint64_t weights, uint64_t v, uint64_t dst, uint32_t heads, uint32_t dim, uint32_t time)
One nn_attention_mix dispatch: the values of every position mixed by each query's attention weights.
DependencyStage conv1d_stage(const Conv1d &layer, uint64_t src, uint64_t weights, uint64_t bias, uint64_t dst, uint32_t batch, uint32_t t_in)
One nn_conv1d dispatch: [batch, c_in, t_in] to [batch, c_out, conv1d_length].
void upload_packed_weights(GpuDispatchCore &context, uint32_t set, size_t binding, const PackedWeights &packed)
Allocate a shared buffer on a context and fill it with packed weights.
Activation
Output activation of a layer.
static constexpr DomainSpec Audio
Domain constant for Audio domain.
Named tensors that belong together, such as a model's weights or I/O.
enum MayaFlux::Portal::Graphics::GpuBufferBinding::ElementType FLOAT32
Plain-data description of the compute shader to dispatch.
Where a result of the latest utterance is on the device.
std::unique_ptr< ShaderExecutionContext<> > context
Impl(const Kakshya::TensorSet &weights, std::shared_ptr< GpuDispatchRecorder > shared)
void upload_normalised(std::span< const float > samples)
uint64_t address(uint32_t buffer) const
std::vector< float > run(std::span< const float > samples)
std::shared_ptr< GpuDispatchRecorder > recording
void ensure_capacity(size_t samples, const std::array< uint32_t, k_feature_layers > &lengths)
TensorKernels::PackedWeights front
std::array< TensorKernels::PackedWeights, k_blocks > blocks
The classes of a phoneme network: their names, how many are phonemes, the blank class,...
Push constants of nn_conv1d.
One convolution layer over [batch, c_in, t_in], with weights [c_out, c_in, kernel].
The required tensors as float32 in one run, in the order they were required.
One tensor a network needs.