21 [[nodiscard]]
inline uint32_t
groups_for(
size_t invocations, uint32_t local)
23 return static_cast<uint32_t
>((invocations + local - 1) / local);
30 [[nodiscard]]
inline std::array<uint32_t, 3>
group_grid(
size_t invocations, uint32_t local)
32 const size_t groups = std::max<size_t>((invocations + local - 1) / local, 1);
34 return {
static_cast<uint32_t
>(columns),
static_cast<uint32_t
>((groups + columns - 1) / columns), 1U };
37 template <
typename PC>
42 .stage_fn = [pc](
GpuDispatchCore& core) { core.set_push_constants(pc); },
47 template <
typename PC>
51 .config =
GpuComputeConfig { .shader_path = shader, .workgroup_size = { local, 1, 1 }, .push_constant_size =
sizeof(PC) },
52 .stage_fn = [pc](
GpuDispatchCore& core) { core.set_push_constants(pc); },
53 .explicit_groups = std::array<uint32_t, 3> { groups, 1U, 1U },
71 uint64_t bias, uint64_t dst, uint32_t batch, uint32_t t_in)
74 if (layer.
c_in == 0 || layer.
c_out == 0 || batch == 0 || t_out == 0) {
75 detail::refuse(
"Conv1d has no input channels, output channels, batch or output length");
90 .kernel_size = layer.
kernel,
111 uint64_t bias, uint64_t dst, uint32_t batch, uint32_t t_in)
114 if (layer.
c_in == 0 || layer.
c_out == 0 || batch == 0 || t_out == 0) {
115 detail::refuse(
"Grouped Conv1d has no input channels, output channels, batch or output length");
118 detail::refuse(
"Grouped Conv1d needs channel counts that are multiples of its groups");
130 .c_out = layer.
c_out,
132 .kernel_size = layer.
kernel,
153 uint32_t outer, uint32_t extent, uint32_t inner,
float epsilon)
155 if (outer == 0 || extent == 0 || inner == 0) {
156 detail::refuse(
"Normalize needs an outer, a normalised and an inner extent of at least 1");
158 const NormalizePC pc { .src = src, .stats = stats, .weight = 0, .bias = 0, .dst = 0,
159 .outer = outer, .extent = extent, .inner = inner, .epsilon = epsilon, .flags = 0, .reserved = 0 };
178 if (outer == 0 || extent == 0 || inner == 0) {
179 detail::refuse(
"Normalize needs an outer, a normalised and an inner extent of at least 1");
188 const NormalizePC pc { .src = src, .stats = stats, .weight = weight, .bias = bias, .dst = dst,
189 .outer = outer, .extent = extent, .inner = inner, .epsilon = 0.0F, .flags = flags, .reserved = 0 };
202 const AddPC pc { .a = a, .b = b, .dst = dst, .count = count, .reserved = 0 };
215 uint32_t heads, uint32_t dim, uint32_t time,
float scale)
217 if (heads == 0 || dim == 0 || time == 0) {
218 detail::refuse(
"Attention needs heads, a head width and positions");
220 const AttentionScoresPC pc { .q = q, .k = k, .scores = scores, .heads = heads, .dim = dim, .time = time, .scale = scale };
230 if (outer == 0 || extent == 0 || inner == 0) {
231 detail::refuse(
"Softmax needs an outer, a normalised and an inner extent of at least 1");
233 const SoftmaxPC pc { .data = data, .outer = outer, .extent = extent, .inner = inner, .reserved = 0 };
246 uint32_t heads, uint32_t dim, uint32_t time)
248 if (heads == 0 || dim == 0 || time == 0) {
249 detail::refuse(
"Attention needs heads, a head width and positions");
251 const AttentionMixPC pc { .weights = weights, .v = v, .dst = dst, .heads = heads, .dim = dim, .time = time, .reserved = 0 };
268 uint32_t chunk_size, uint32_t context, uint32_t pad)
270 if (chunks == 0 || chunk_size == 0 || pad >= context + chunk_size) {
271 detail::refuse(
"Frame audio needs chunks, a chunk size and a pad smaller than one frame");
273 const FrameAudioPC pc { .samples = samples, .frames = frames, .chunks = chunks, .chunk_size = chunk_size, .context = context, .pad = pad };
287 uint32_t channels, uint32_t time)
289 if (batch == 0 || channels == 0 || time == 0) {
292 const MagnitudePC pc { .src = src, .dst = dst, .batch = batch, .channels = channels, .time = time, .reserved = 0 };
312 uint64_t b_hh, uint64_t state, uint64_t hs, uint32_t steps, uint32_t hidden)
315 detail::refuse(
"LSTM needs at least one step and a hidden width from 1 to 256");
317 const LstmPC pc { .x = x, .w_ih = w_ih, .w_hh = w_hh, .b_ih = b_ih, .b_hh = b_hh, .state = state, .hs = hs, .steps = steps, .hidden = hidden };
Descriptions of the layers the tensor kernels run, and their output sizes.
The push constant layouts of the tensor kernels, their flag bits and limits.
Non-template base that owns all type-independent GPU dispatch logic.
DependencyStage make_stage(const char *shader, uint32_t local, const PC &pc, uint32_t groups)
void refuse(std::string_view message)
DependencyStage make_grid_stage(const char *shader, const PC &pc, size_t invocations)
uint32_t groups_for(size_t invocations, uint32_t local)
std::array< uint32_t, 3 > group_grid(size_t invocations, uint32_t local)
Workgroup counts covering the invocations, along x up to max_groups_per_axis and then along y.
constexpr uint32_t normalize_flag_affine_outer
nn_normalize flag bit: weight and bias are indexed by the outer index instead of the position along e...
constexpr uint32_t normalize_flag_affine
nn_normalize flag bit: multiply by weight and add bias.
NormAffine
Which index selects the weight and bias applied after a normalisation.
@ Outer
The outer index, as a per channel normalisation over time does.
constexpr uint32_t conv1d_activation_shift
nn_conv1d and nn_conv1d_grouped flag bits 2 and 3: the output activation, 0 none, 1 ReLU,...
DependencyStage frame_audio_stage(uint64_t samples, uint64_t frames, uint32_t chunks, uint32_t chunk_size, uint32_t context, uint32_t pad)
One nn_frame_audio dispatch: cut a sample stream into overlapping frames, reflect padded on the right...
DependencyStage softmax_stage(uint64_t data, uint32_t outer, uint32_t extent, uint32_t inner)
One nn_softmax dispatch: softmax of a tensor [outer, extent, inner] along extent, in place.
constexpr uint32_t max_groups_per_axis
Most workgroups the stages put on one axis; a larger dispatch continues along y.
DependencyStage grouped_conv1d_stage(const GroupedConv1d &layer, uint64_t src, uint64_t weights, uint64_t bias, uint64_t dst, uint32_t batch, uint32_t t_in)
One nn_conv1d_grouped dispatch: [batch, c_in, t_in] to [batch, c_out, grouped_conv1d_length].
constexpr const char * attention_mix_shader
constexpr const char * conv1d_shader
constexpr const char * softmax_shader
Activation
Output activation of a layer.
constexpr uint32_t frame_audio_length(uint32_t chunk_size, uint32_t context, uint32_t pad)
Length of one frame nn_frame_audio writes.
constexpr const char * frame_audio_shader
DependencyStage add_stage(uint64_t a, uint64_t b, uint64_t dst, uint32_t count)
One nn_add dispatch: the element-wise sum of two sequences of count floats.
DependencyStage normalize_apply_stage(uint64_t src, uint64_t stats, uint64_t weight, uint64_t bias, uint64_t dst, uint32_t outer, uint32_t extent, uint32_t inner, NormAffine affine, Activation activation=Activation::None)
The nn_normalize dispatch that writes (x - mean) * scale from the measures of normalize_stats_stage.
uint32_t grouped_conv1d_length(const GroupedConv1d &layer, uint32_t t_in)
Output length of a grouped convolution after its trim, 0 when nothing is left.
DependencyStage normalize_stats_stage(uint64_t src, uint64_t stats, uint32_t outer, uint32_t extent, uint32_t inner, float epsilon)
The nn_normalize dispatch that measures every row of a tensor [outer, extent, inner] along extent.
constexpr const char * normalize_shader
DependencyStage attention_scores_stage(uint64_t q, uint64_t k, uint64_t scores, uint32_t heads, uint32_t dim, uint32_t time, float scale)
One nn_attention_scores dispatch: scale times the dot product of every query with every key,...
DependencyStage lstm_sequence_stage(uint64_t x, uint64_t w_ih, uint64_t w_hh, uint64_t b_ih, uint64_t b_hh, uint64_t state, uint64_t hs, uint32_t steps, uint32_t hidden)
One nn_lstm_sequence dispatch: an LSTM cell over a whole sequence in one workgroup.
constexpr const char * grouped_conv1d_shader
constexpr uint32_t element_workgroup
Invocations per workgroup of the element-wise kernels.
uint32_t conv1d_length(const Conv1d &layer, uint32_t t_in)
Output length of a convolution, 0 when the window does not fit the padded input.
constexpr uint32_t normalize_flag_apply
nn_normalize flag bit: write the normalised values instead of measuring the rows.
constexpr uint32_t conv1d_flag_bias
nn_conv1d flag bit: add the bias.
constexpr uint32_t normalize_activation_shift
nn_normalize flag bits 3 and 4: the output activation, 0 none, 3 GELU.
constexpr const char * magnitude_shader
DependencyStage magnitude_stage(uint64_t src, uint64_t dst, uint32_t batch, uint32_t channels, uint32_t time)
One nn_magnitude dispatch: sqrt(re^2 + im^2) of a complex signal stored as two channel halves.
DependencyStage attention_mix_stage(uint64_t weights, uint64_t v, uint64_t dst, uint32_t heads, uint32_t dim, uint32_t time)
One nn_attention_mix dispatch: the values of every position mixed by each query's attention weights.
DependencyStage conv1d_stage(const Conv1d &layer, uint64_t src, uint64_t weights, uint64_t bias, uint64_t dst, uint32_t batch, uint32_t t_in)
One nn_conv1d dispatch: [batch, c_in, t_in] to [batch, c_out, conv1d_length].
constexpr const char * add_shader
constexpr const char * attention_scores_shader
constexpr uint32_t lstm_max_hidden
Widest hidden state nn_lstm_sequence supports.
constexpr uint32_t lstm_workgroup
Invocations of the one workgroup nn_lstm_sequence runs in.
constexpr const char * lstm_sequence_shader
constexpr uint32_t conv1d_flag_relu_input
nn_conv1d flag bit: apply ReLU to the input.
Plain-data description of the compute shader to dispatch.
One stage in a record_stages call.
Push constants of nn_add.
Push constants of nn_attention_mix.
Push constants of nn_attention_scores.
Push constants of nn_conv1d.
One convolution layer over [batch, c_in, t_in], with weights [c_out, c_in, kernel].
Push constants of nn_frame_audio.
Push constants of nn_conv1d_grouped.
One grouped convolution layer over [batch, c_in, t_in], with weights [c_out, c_in / groups,...
Push constants of nn_lstm_sequence.
Push constants of nn_magnitude.
Push constants of nn_normalize.
Push constants of nn_softmax.