MayaFlux 1.0.0
Digital-First Multimedia Processing Framework
Loading...
Searching...
No Matches
Stages.hpp
Go to the documentation of this file.
1#pragma once
2
4
5#include "Layers.hpp"
6#include "Layouts.hpp"
7
8/**
9 * @file Stages.hpp
10 * @brief The stages that run the tensor kernels.
11 *
12 * Each function returns a DependencyStage for record_stages, so a network is a
13 * list of stages recorded into one submission. Addresses are device addresses
14 * of float32 buffers, from a context's shared buffers (shared_buffer_address).
15 */
16
18
19namespace detail {
20
21 [[nodiscard]] inline uint32_t groups_for(size_t invocations, uint32_t local)
22 {
23 return static_cast<uint32_t>((invocations + local - 1) / local);
24 }
25
26 /**
27 * Workgroup counts covering the invocations, along x up to max_groups_per_axis and then along y. The kernels
28 * that number their invocations along both axes take any such grid.
29 */
30 [[nodiscard]] inline std::array<uint32_t, 3> group_grid(size_t invocations, uint32_t local)
31 {
32 const size_t groups = std::max<size_t>((invocations + local - 1) / local, 1);
33 const size_t columns = std::min<size_t>(groups, max_groups_per_axis);
34 return { static_cast<uint32_t>(columns), static_cast<uint32_t>((groups + columns - 1) / columns), 1U };
35 }
36
37 template <typename PC>
38 [[nodiscard]] DependencyStage make_grid_stage(const char* shader, const PC& pc, size_t invocations)
39 {
40 return DependencyStage {
41 .config = GpuComputeConfig { .shader_path = shader, .workgroup_size = { element_workgroup, 1, 1 }, .push_constant_size = sizeof(PC) },
42 .stage_fn = [pc](GpuDispatchCore& core) { core.set_push_constants(pc); },
43 .explicit_groups = group_grid(invocations, element_workgroup),
44 };
45 }
46
47 template <typename PC>
48 [[nodiscard]] DependencyStage make_stage(const char* shader, uint32_t local, const PC& pc, uint32_t groups)
49 {
50 return DependencyStage {
51 .config = GpuComputeConfig { .shader_path = shader, .workgroup_size = { local, 1, 1 }, .push_constant_size = sizeof(PC) },
52 .stage_fn = [pc](GpuDispatchCore& core) { core.set_push_constants(pc); },
53 .explicit_groups = std::array<uint32_t, 3> { groups, 1U, 1U },
54 };
55 }
56
57}
58
59/**
60 * @brief One nn_conv1d dispatch: [batch, c_in, t_in] to [batch, c_out, conv1d_length].
61 * @param layer The layer.
62 * @param src Device address of the input.
63 * @param weights Device address of the weights, [c_out, c_in, kernel].
64 * @param bias Device address of the bias, [c_out], or 0 for none.
65 * @param dst Device address of the output.
66 * @param batch Independent repeats of the input.
67 * @param t_in Input length.
68 * @throws std::invalid_argument When the layer is empty or the output would be empty.
69 */
70[[nodiscard]] inline DependencyStage conv1d_stage(const Conv1d& layer, uint64_t src, uint64_t weights,
71 uint64_t bias, uint64_t dst, uint32_t batch, uint32_t t_in)
72{
73 const uint32_t t_out = conv1d_length(layer, t_in);
74 if (layer.c_in == 0 || layer.c_out == 0 || batch == 0 || t_out == 0) {
75 detail::refuse("Conv1d has no input channels, output channels, batch or output length");
76 }
77 const uint32_t flags = (bias != 0 ? conv1d_flag_bias : 0U)
78 | (layer.relu_input ? conv1d_flag_relu_input : 0U)
79 | (static_cast<uint32_t>(layer.activation) << conv1d_activation_shift);
80 const Conv1dPC pc {
81 .src = src,
82 .weights = weights,
83 .bias = bias,
84 .dst = dst,
85 .batch = batch,
86 .c_in = layer.c_in,
87 .t_in = t_in,
88 .c_out = layer.c_out,
89 .t_out = t_out,
90 .kernel_size = layer.kernel,
91 .stride = layer.stride,
92 .pad = layer.pad,
93 .flags = flags,
94 .dilation = layer.dilation,
95 };
96 return detail::make_grid_stage(conv1d_shader, pc, static_cast<size_t>(batch) * layer.c_out * t_out);
97}
98
99/**
100 * @brief One nn_conv1d_grouped dispatch: [batch, c_in, t_in] to [batch, c_out, grouped_conv1d_length].
101 * @param layer The layer. Both channel counts must be multiples of groups.
102 * @param src Device address of the input.
103 * @param weights Device address of the weights, [c_out, c_in / groups, kernel].
104 * @param bias Device address of the bias, [c_out], or 0 for none.
105 * @param dst Device address of the output.
106 * @param batch Independent repeats of the input.
107 * @param t_in Input length.
108 * @throws std::invalid_argument When the layer is empty, the groups do not divide the channels or the output would be empty.
109 */
110[[nodiscard]] inline DependencyStage grouped_conv1d_stage(const GroupedConv1d& layer, uint64_t src, uint64_t weights,
111 uint64_t bias, uint64_t dst, uint32_t batch, uint32_t t_in)
112{
113 const uint32_t t_out = grouped_conv1d_length(layer, t_in);
114 if (layer.c_in == 0 || layer.c_out == 0 || batch == 0 || t_out == 0) {
115 detail::refuse("Grouped Conv1d has no input channels, output channels, batch or output length");
116 }
117 if (layer.c_in % layer.groups != 0 || layer.c_out % layer.groups != 0) {
118 detail::refuse("Grouped Conv1d needs channel counts that are multiples of its groups");
119 }
120 const uint32_t flags = (bias != 0 ? conv1d_flag_bias : 0U)
121 | (static_cast<uint32_t>(layer.activation) << conv1d_activation_shift);
122 const GroupedConv1dPC pc {
123 .src = src,
124 .weights = weights,
125 .bias = bias,
126 .dst = dst,
127 .batch = batch,
128 .c_in = layer.c_in,
129 .t_in = t_in,
130 .c_out = layer.c_out,
131 .t_out = t_out,
132 .kernel_size = layer.kernel,
133 .stride = layer.stride,
134 .pad = layer.pad,
135 .flags = flags,
136 .groups = layer.groups,
137 };
138 return detail::make_grid_stage(grouped_conv1d_shader, pc, static_cast<size_t>(batch) * layer.c_out * t_out);
139}
140
141/**
142 * @brief The nn_normalize dispatch that measures every row of a tensor [outer, extent, inner] along extent.
143 *
144 * Writes the mean and 1 / sqrt(variance + epsilon) of each (outer, inner) row
145 * to stats, two floats per row. The variance is the mean squared deviation.
146 * Follow it with normalize_apply_stage.
147 *
148 * @param src Device address of the tensor.
149 * @param stats Device address of the row measures, 2 * outer * inner floats.
150 * @throws std::invalid_argument When an extent is 0.
151 */
152[[nodiscard]] inline DependencyStage normalize_stats_stage(uint64_t src, uint64_t stats,
153 uint32_t outer, uint32_t extent, uint32_t inner, float epsilon)
154{
155 if (outer == 0 || extent == 0 || inner == 0) {
156 detail::refuse("Normalize needs an outer, a normalised and an inner extent of at least 1");
157 }
158 const NormalizePC pc { .src = src, .stats = stats, .weight = 0, .bias = 0, .dst = 0,
159 .outer = outer, .extent = extent, .inner = inner, .epsilon = epsilon, .flags = 0, .reserved = 0 };
160 return detail::make_grid_stage(normalize_shader, pc, static_cast<size_t>(outer) * inner);
161}
162
163/**
164 * @brief The nn_normalize dispatch that writes (x - mean) * scale from the measures of normalize_stats_stage.
165 *
166 * @param src Device address of the tensor.
167 * @param stats Device address of the row measures.
168 * @param weight Device address of the weight, ignored when affine is None.
169 * @param bias Device address of the bias, ignored when affine is None.
170 * @param dst Device address of the output, which may be src.
171 * @param affine Which index selects the weight and bias.
172 * @param activation Applied last. None and Gelu are supported.
173 * @throws std::invalid_argument When an extent is 0 or the activation is not supported.
174 */
175[[nodiscard]] inline DependencyStage normalize_apply_stage(uint64_t src, uint64_t stats, uint64_t weight, uint64_t bias,
176 uint64_t dst, uint32_t outer, uint32_t extent, uint32_t inner, NormAffine affine, Activation activation = Activation::None)
177{
178 if (outer == 0 || extent == 0 || inner == 0) {
179 detail::refuse("Normalize needs an outer, a normalised and an inner extent of at least 1");
180 }
181 if (activation != Activation::None && activation != Activation::Gelu) {
182 detail::refuse("Normalize applies no activation or GELU");
183 }
184 const uint32_t flags = normalize_flag_apply
185 | (affine != NormAffine::None ? normalize_flag_affine : 0U)
187 | (static_cast<uint32_t>(activation) << normalize_activation_shift);
188 const NormalizePC pc { .src = src, .stats = stats, .weight = weight, .bias = bias, .dst = dst,
189 .outer = outer, .extent = extent, .inner = inner, .epsilon = 0.0F, .flags = flags, .reserved = 0 };
190 return detail::make_grid_stage(normalize_shader, pc, static_cast<size_t>(outer) * extent * inner);
191}
192
193/**
194 * @brief One nn_add dispatch: the element-wise sum of two sequences of count floats. dst may be a or b.
195 * @throws std::invalid_argument When count is 0.
196 */
197[[nodiscard]] inline DependencyStage add_stage(uint64_t a, uint64_t b, uint64_t dst, uint32_t count)
198{
199 if (count == 0) {
200 detail::refuse("Add needs at least one element");
201 }
202 const AddPC pc { .a = a, .b = b, .dst = dst, .count = count, .reserved = 0 };
203 return detail::make_grid_stage(add_shader, pc, count);
204}
205
206/**
207 * @brief One nn_attention_scores dispatch: scale times the dot product of every query with every key, per head.
208 *
209 * q and k are [heads * dim, time], head h owning channels h * dim onwards.
210 * scores is [heads, time, time], indexed head, query, key.
211 *
212 * @throws std::invalid_argument When heads, dim or time is 0.
213 */
214[[nodiscard]] inline DependencyStage attention_scores_stage(uint64_t q, uint64_t k, uint64_t scores,
215 uint32_t heads, uint32_t dim, uint32_t time, float scale)
216{
217 if (heads == 0 || dim == 0 || time == 0) {
218 detail::refuse("Attention needs heads, a head width and positions");
219 }
220 const AttentionScoresPC pc { .q = q, .k = k, .scores = scores, .heads = heads, .dim = dim, .time = time, .scale = scale };
221 return detail::make_grid_stage(attention_scores_shader, pc, static_cast<size_t>(heads) * time * time);
222}
223
224/**
225 * @brief One nn_softmax dispatch: softmax of a tensor [outer, extent, inner] along extent, in place.
226 * @throws std::invalid_argument When an extent is 0.
227 */
228[[nodiscard]] inline DependencyStage softmax_stage(uint64_t data, uint32_t outer, uint32_t extent, uint32_t inner)
229{
230 if (outer == 0 || extent == 0 || inner == 0) {
231 detail::refuse("Softmax needs an outer, a normalised and an inner extent of at least 1");
232 }
233 const SoftmaxPC pc { .data = data, .outer = outer, .extent = extent, .inner = inner, .reserved = 0 };
234 return detail::make_grid_stage(softmax_shader, pc, static_cast<size_t>(outer) * inner);
235}
236
237/**
238 * @brief One nn_attention_mix dispatch: the values of every position mixed by each query's attention weights.
239 *
240 * weights is [heads, time, time] as attention_scores_stage and softmax_stage
241 * leave it. v and dst are [heads * dim, time].
242 *
243 * @throws std::invalid_argument When heads, dim or time is 0.
244 */
245[[nodiscard]] inline DependencyStage attention_mix_stage(uint64_t weights, uint64_t v, uint64_t dst,
246 uint32_t heads, uint32_t dim, uint32_t time)
247{
248 if (heads == 0 || dim == 0 || time == 0) {
249 detail::refuse("Attention needs heads, a head width and positions");
250 }
251 const AttentionMixPC pc { .weights = weights, .v = v, .dst = dst, .heads = heads, .dim = dim, .time = time, .reserved = 0 };
252 return detail::make_grid_stage(attention_mix_shader, pc, static_cast<size_t>(heads) * dim * time);
253}
254
255/**
256 * @brief One nn_frame_audio dispatch: cut a sample stream into overlapping frames, reflect padded on the right.
257 *
258 * samples holds context + chunks * chunk_size values. Frame n is the context +
259 * chunk_size values starting at n * chunk_size, followed by pad values that
260 * mirror the frame about its last value without repeating it. pad must be
261 * smaller than context + chunk_size.
262 *
263 * @param samples Device address of the samples.
264 * @param frames Device address of the frames, [chunks, frame_audio_length].
265 * @throws std::invalid_argument When chunks or chunk_size is 0 or pad is a frame or more.
266 */
267[[nodiscard]] inline DependencyStage frame_audio_stage(uint64_t samples, uint64_t frames, uint32_t chunks,
268 uint32_t chunk_size, uint32_t context, uint32_t pad)
269{
270 if (chunks == 0 || chunk_size == 0 || pad >= context + chunk_size) {
271 detail::refuse("Frame audio needs chunks, a chunk size and a pad smaller than one frame");
272 }
273 const FrameAudioPC pc { .samples = samples, .frames = frames, .chunks = chunks, .chunk_size = chunk_size, .context = context, .pad = pad };
275 detail::groups_for(static_cast<size_t>(chunks) * frame_audio_length(chunk_size, context, pad), element_workgroup));
276}
277
278/**
279 * @brief One nn_magnitude dispatch: sqrt(re^2 + im^2) of a complex signal stored as two channel halves.
280 *
281 * src is [batch, 2 * channels, time] with the real parts in the first half of
282 * the channels and the imaginary parts in the second. dst is [batch, channels, time].
283 *
284 * @throws std::invalid_argument When batch, channels or time is 0.
285 */
286[[nodiscard]] inline DependencyStage magnitude_stage(uint64_t src, uint64_t dst, uint32_t batch,
287 uint32_t channels, uint32_t time)
288{
289 if (batch == 0 || channels == 0 || time == 0) {
290 detail::refuse("Magnitude needs a batch, channels and time");
291 }
292 const MagnitudePC pc { .src = src, .dst = dst, .batch = batch, .channels = channels, .time = time, .reserved = 0 };
294 detail::groups_for(static_cast<size_t>(batch) * channels * time, element_workgroup));
295}
296
297/**
298 * @brief One nn_lstm_sequence dispatch: an LSTM cell over a whole sequence in one workgroup.
299 *
300 * The input width equals the hidden width. Weights are in the order of PyTorch:
301 * gate rows input, forget, cell, output. w_ih and w_hh are [4 * hidden, hidden],
302 * b_ih and b_hh are [4 * hidden]. state holds h then c, 2 * hidden floats, and is
303 * read at the start and written at the end so the recurrence continues across
304 * dispatches. hs receives the hidden vector of every step.
305 *
306 * @param x Device address of the input, [steps, hidden].
307 * @param hs Device address of the output, [steps, hidden].
308 * @param hidden Hidden width, 1 to lstm_max_hidden.
309 * @throws std::invalid_argument When steps or hidden is out of range.
310 */
311[[nodiscard]] inline DependencyStage lstm_sequence_stage(uint64_t x, uint64_t w_ih, uint64_t w_hh, uint64_t b_ih,
312 uint64_t b_hh, uint64_t state, uint64_t hs, uint32_t steps, uint32_t hidden)
313{
314 if (steps == 0 || hidden == 0 || hidden > lstm_max_hidden) {
315 detail::refuse("LSTM needs at least one step and a hidden width from 1 to 256");
316 }
317 const LstmPC pc { .x = x, .w_ih = w_ih, .w_hh = w_hh, .b_ih = b_ih, .b_hh = b_hh, .state = state, .hs = hs, .steps = steps, .hidden = hidden };
319}
320
321} // namespace MayaFlux::Yantra::TensorKernels
Descriptions of the layers the tensor kernels run, and their output sizes.
The push constant layouts of the tensor kernels, their flag bits and limits.
Non-template base that owns all type-independent GPU dispatch logic.
DependencyStage make_stage(const char *shader, uint32_t local, const PC &pc, uint32_t groups)
Definition Stages.hpp:48
void refuse(std::string_view message)
Definition Layers.hpp:74
DependencyStage make_grid_stage(const char *shader, const PC &pc, size_t invocations)
Definition Stages.hpp:38
uint32_t groups_for(size_t invocations, uint32_t local)
Definition Stages.hpp:21
std::array< uint32_t, 3 > group_grid(size_t invocations, uint32_t local)
Workgroup counts covering the invocations, along x up to max_groups_per_axis and then along y.
Definition Stages.hpp:30
constexpr uint32_t normalize_flag_affine_outer
nn_normalize flag bit: weight and bias are indexed by the outer index instead of the position along e...
Definition Layouts.hpp:46
constexpr uint32_t normalize_flag_affine
nn_normalize flag bit: multiply by weight and add bias.
Definition Layouts.hpp:44
NormAffine
Which index selects the weight and bias applied after a normalisation.
Definition Layers.hpp:66
@ Outer
The outer index, as a per channel normalisation over time does.
Definition Layers.hpp:69
constexpr uint32_t conv1d_activation_shift
nn_conv1d and nn_conv1d_grouped flag bits 2 and 3: the output activation, 0 none, 1 ReLU,...
Definition Layouts.hpp:39
DependencyStage frame_audio_stage(uint64_t samples, uint64_t frames, uint32_t chunks, uint32_t chunk_size, uint32_t context, uint32_t pad)
One nn_frame_audio dispatch: cut a sample stream into overlapping frames, reflect padded on the right...
Definition Stages.hpp:267
DependencyStage softmax_stage(uint64_t data, uint32_t outer, uint32_t extent, uint32_t inner)
One nn_softmax dispatch: softmax of a tensor [outer, extent, inner] along extent, in place.
Definition Stages.hpp:228
constexpr uint32_t max_groups_per_axis
Most workgroups the stages put on one axis; a larger dispatch continues along y.
Definition Layouts.hpp:21
DependencyStage grouped_conv1d_stage(const GroupedConv1d &layer, uint64_t src, uint64_t weights, uint64_t bias, uint64_t dst, uint32_t batch, uint32_t t_in)
One nn_conv1d_grouped dispatch: [batch, c_in, t_in] to [batch, c_out, grouped_conv1d_length].
Definition Stages.hpp:110
constexpr const char * attention_mix_shader
Definition Layouts.hpp:32
constexpr const char * conv1d_shader
Definition Layouts.hpp:24
constexpr const char * softmax_shader
Definition Layouts.hpp:31
Activation
Output activation of a layer.
Definition Layers.hpp:16
constexpr uint32_t frame_audio_length(uint32_t chunk_size, uint32_t context, uint32_t pad)
Length of one frame nn_frame_audio writes.
Definition Layers.hpp:120
constexpr const char * frame_audio_shader
Definition Layouts.hpp:23
DependencyStage add_stage(uint64_t a, uint64_t b, uint64_t dst, uint32_t count)
One nn_add dispatch: the element-wise sum of two sequences of count floats.
Definition Stages.hpp:197
DependencyStage normalize_apply_stage(uint64_t src, uint64_t stats, uint64_t weight, uint64_t bias, uint64_t dst, uint32_t outer, uint32_t extent, uint32_t inner, NormAffine affine, Activation activation=Activation::None)
The nn_normalize dispatch that writes (x - mean) * scale from the measures of normalize_stats_stage.
Definition Stages.hpp:175
uint32_t grouped_conv1d_length(const GroupedConv1d &layer, uint32_t t_in)
Output length of a grouped convolution after its trim, 0 when nothing is left.
Definition Layers.hpp:104
DependencyStage normalize_stats_stage(uint64_t src, uint64_t stats, uint32_t outer, uint32_t extent, uint32_t inner, float epsilon)
The nn_normalize dispatch that measures every row of a tensor [outer, extent, inner] along extent.
Definition Stages.hpp:152
constexpr const char * normalize_shader
Definition Layouts.hpp:28
DependencyStage attention_scores_stage(uint64_t q, uint64_t k, uint64_t scores, uint32_t heads, uint32_t dim, uint32_t time, float scale)
One nn_attention_scores dispatch: scale times the dot product of every query with every key,...
Definition Stages.hpp:214
DependencyStage lstm_sequence_stage(uint64_t x, uint64_t w_ih, uint64_t w_hh, uint64_t b_ih, uint64_t b_hh, uint64_t state, uint64_t hs, uint32_t steps, uint32_t hidden)
One nn_lstm_sequence dispatch: an LSTM cell over a whole sequence in one workgroup.
Definition Stages.hpp:311
constexpr const char * grouped_conv1d_shader
Definition Layouts.hpp:27
constexpr uint32_t element_workgroup
Invocations per workgroup of the element-wise kernels.
Definition Layouts.hpp:15
uint32_t conv1d_length(const Conv1d &layer, uint32_t t_in)
Output length of a convolution, 0 when the window does not fit the padded input.
Definition Layers.hpp:88
constexpr uint32_t normalize_flag_apply
nn_normalize flag bit: write the normalised values instead of measuring the rows.
Definition Layouts.hpp:42
constexpr uint32_t conv1d_flag_bias
nn_conv1d flag bit: add the bias.
Definition Layouts.hpp:35
constexpr uint32_t normalize_activation_shift
nn_normalize flag bits 3 and 4: the output activation, 0 none, 3 GELU.
Definition Layouts.hpp:48
constexpr const char * magnitude_shader
Definition Layouts.hpp:25
DependencyStage magnitude_stage(uint64_t src, uint64_t dst, uint32_t batch, uint32_t channels, uint32_t time)
One nn_magnitude dispatch: sqrt(re^2 + im^2) of a complex signal stored as two channel halves.
Definition Stages.hpp:286
DependencyStage attention_mix_stage(uint64_t weights, uint64_t v, uint64_t dst, uint32_t heads, uint32_t dim, uint32_t time)
One nn_attention_mix dispatch: the values of every position mixed by each query's attention weights.
Definition Stages.hpp:245
DependencyStage conv1d_stage(const Conv1d &layer, uint64_t src, uint64_t weights, uint64_t bias, uint64_t dst, uint32_t batch, uint32_t t_in)
One nn_conv1d dispatch: [batch, c_in, t_in] to [batch, c_out, conv1d_length].
Definition Stages.hpp:70
constexpr const char * add_shader
Definition Layouts.hpp:29
constexpr const char * attention_scores_shader
Definition Layouts.hpp:30
constexpr uint32_t lstm_max_hidden
Widest hidden state nn_lstm_sequence supports.
Definition Layouts.hpp:19
constexpr uint32_t lstm_workgroup
Invocations of the one workgroup nn_lstm_sequence runs in.
Definition Layouts.hpp:17
constexpr const char * lstm_sequence_shader
Definition Layouts.hpp:26
constexpr uint32_t conv1d_flag_relu_input
nn_conv1d flag bit: apply ReLU to the input.
Definition Layouts.hpp:37
Plain-data description of the compute shader to dispatch.
One stage in a record_stages call.
Push constants of nn_add.
Definition Layouts.hpp:135
Push constants of nn_attention_mix.
Definition Layouts.hpp:164
Push constants of nn_attention_scores.
Definition Layouts.hpp:144
Push constants of nn_conv1d.
Definition Layouts.hpp:51
One convolution layer over [batch, c_in, t_in], with weights [c_out, c_in, kernel].
Definition Layers.hpp:31
Push constants of nn_frame_audio.
Definition Layouts.hpp:69
Push constants of nn_conv1d_grouped.
Definition Layouts.hpp:102
One grouped convolution layer over [batch, c_in, t_in], with weights [c_out, c_in / groups,...
Definition Layers.hpp:51
Push constants of nn_lstm_sequence.
Definition Layouts.hpp:89
Push constants of nn_magnitude.
Definition Layouts.hpp:79
Push constants of nn_normalize.
Definition Layouts.hpp:120
Push constants of nn_softmax.
Definition Layouts.hpp:155