MayaFlux 1.0.0
Digital-First Multimedia Processing Framework
Loading...
Searching...
No Matches
TensorConvert.cpp
Go to the documentation of this file.
1#include "TensorConvert.hpp"
2
3#include "SmallFloat.hpp"
4
5#include <bit>
6
7namespace MayaFlux::Kakshya {
8
9namespace {
10
11 enum class Family : uint8_t {
12 SIGNED,
13 UNSIGNED,
14 BOOL,
15 FLOAT,
17 };
18
19 struct Layout {
20 Family family {};
21 uint8_t bits {};
22 bool brain {};
23 };
24
25 struct Scalar {
26 double real {};
27 uint64_t magnitude {};
28 bool negative {};
29 bool exact {};
30 };
31
32 using Decoder = Scalar (*)(const std::byte*);
33 using Encoder = void (*)(std::byte*, const Scalar&);
34
35 bool one_of(uint8_t bits, std::initializer_list<uint8_t> widths)
36 {
37 return std::ranges::find(widths, bits) != widths.end();
38 }
39
40 std::optional<Layout> layout_of(TensorDType dtype, std::string& why)
41 {
42 if (dtype.lanes != 1) {
43 why = dtype.name() + " holds several lanes per element";
44 return std::nullopt;
45 }
46
47 switch (dtype.kind) {
49 if (one_of(dtype.bits, { 8, 16, 32, 64 }))
50 return Layout { .family = Family::SIGNED, .bits = dtype.bits, .brain = false };
51 break;
53 if (one_of(dtype.bits, { 8, 16, 32, 64 }))
54 return Layout { .family = Family::UNSIGNED, .bits = dtype.bits, .brain = false };
55 break;
57 if (dtype.bits == 8)
58 return Layout { .family = Family::BOOL, .bits = 8, .brain = false };
59 break;
61 if (one_of(dtype.bits, { 16, 32, 64 }))
62 return Layout { .family = Family::FLOAT, .bits = dtype.bits, .brain = false };
63 break;
65 if (dtype.bits == 16)
66 return Layout { .family = Family::FLOAT, .bits = 16, .brain = true };
67 break;
69 if (one_of(dtype.bits, { 64, 128 }))
70 return Layout { .family = Family::COMPLEX, .bits = dtype.bits, .brain = false };
71 break;
72 default:
73 break;
74 }
75
76 why = dtype.name() + " is not supported by conversion";
77 return std::nullopt;
78 }
79
80 bool holds_quantization(Layout layout)
81 {
82 return (layout.family == Family::SIGNED || layout.family == Family::UNSIGNED) && layout.bits <= 32;
83 }
84
85 bool same_quantization(const std::optional<TensorQuantization>& a, const std::optional<TensorQuantization>& b)
86 {
87 if (!a || !b)
88 return !a && !b;
89 return a->scale == b->scale && a->zero_point == b->zero_point && a->axis == b->axis;
90 }
91
92 double wide_real(const Scalar& value)
93 {
94 constexpr uint64_t exact_limit = uint64_t { 1 } << 53;
95 if (!value.exact || value.magnitude <= exact_limit)
96 return value.real;
97
98 const int shift = std::bit_width(value.magnitude) - 53;
99 uint64_t mantissa = value.magnitude >> shift;
100 if ((value.magnitude & ((uint64_t { 1 } << shift) - 1U)) != 0U)
101 mantissa |= 1U;
102 const double magnitude = std::ldexp(static_cast<double>(mantissa), shift);
103 return value.negative ? -magnitude : magnitude;
104 }
105
106 template <typename T>
107 T read(const std::byte* source)
108 {
109 T value;
110 std::memcpy(&value, source, sizeof(T));
111 return value;
112 }
113
114 template <typename T>
115 void write(std::byte* destination, T value)
116 {
117 std::memcpy(destination, &value, sizeof(T));
118 }
119
120 template <typename T>
121 Scalar decode_signed(const std::byte* source)
122 {
123 const auto value = read<T>(source);
124 Scalar result;
125 result.exact = true;
126 result.real = static_cast<double>(value);
127 result.negative = value < 0;
128 result.magnitude = value < 0
129 ? static_cast<uint64_t>(-(static_cast<int64_t>(value) + 1)) + 1U
130 : static_cast<uint64_t>(value);
131 return result;
132 }
133
134 template <typename T>
135 Scalar decode_unsigned(const std::byte* source)
136 {
137 const auto value = read<T>(source);
138 Scalar result;
139 result.exact = true;
140 result.real = static_cast<double>(value);
141 result.magnitude = value;
142 return result;
143 }
144
145 Scalar decode_bool(const std::byte* source)
146 {
147 Scalar result;
148 result.exact = true;
149 result.magnitude = *source != std::byte { 0 } ? 1U : 0U;
150 result.real = static_cast<double>(result.magnitude);
151 return result;
152 }
153
154 Scalar decode_f32(const std::byte* source)
155 {
156 return Scalar { .real = static_cast<double>(read<float>(source)) };
157 }
158
159 Scalar decode_f64(const std::byte* source)
160 {
161 return Scalar { .real = read<double>(source) };
162 }
163
164 Scalar decode_f16(const std::byte* source)
165 {
166 return Scalar { .real = decode_small_float(read<uint16_t>(source), 5, 10) };
167 }
168
169 Scalar decode_bf16(const std::byte* source)
170 {
171 return Scalar { .real = decode_small_float(read<uint16_t>(source), 8, 7) };
172 }
173
174 template <typename T>
175 T integer_from_real(double value)
176 {
177 if (std::isnan(value))
178 return T {};
179 const double rounded = std::nearbyint(value);
180 if (rounded <= static_cast<double>(std::numeric_limits<T>::min()))
181 return std::numeric_limits<T>::min();
182 if (rounded >= static_cast<double>(std::numeric_limits<T>::max()))
183 return std::numeric_limits<T>::max();
184 return static_cast<T>(rounded);
185 }
186
187 template <typename T>
188 void encode_signed(std::byte* destination, const Scalar& value)
189 {
190 T result {};
191 if (!value.exact) {
192 result = integer_from_real<T>(value.real);
193 } else if (value.negative) {
194 const uint64_t limit = static_cast<uint64_t>(std::numeric_limits<T>::max()) + 1U;
195 result = value.magnitude >= limit
196 ? std::numeric_limits<T>::min()
197 : static_cast<T>(-static_cast<int64_t>(value.magnitude));
198 } else {
199 result = value.magnitude > static_cast<uint64_t>(std::numeric_limits<T>::max())
200 ? std::numeric_limits<T>::max()
201 : static_cast<T>(value.magnitude);
202 }
203 write(destination, result);
204 }
205
206 template <typename T>
207 void encode_unsigned(std::byte* destination, const Scalar& value)
208 {
209 T result {};
210 if (!value.exact) {
211 result = integer_from_real<T>(value.real);
212 } else if (value.negative) {
213 result = 0;
214 } else {
215 result = value.magnitude > static_cast<uint64_t>(std::numeric_limits<T>::max())
216 ? std::numeric_limits<T>::max()
217 : static_cast<T>(value.magnitude);
218 }
219 write(destination, result);
220 }
221
222 void encode_bool(std::byte* destination, const Scalar& value)
223 {
224 const bool truth = value.exact ? value.magnitude != 0U : value.real != 0.0;
225 *destination = truth ? std::byte { 1 } : std::byte { 0 };
226 }
227
228 void encode_f32(std::byte* destination, const Scalar& value)
229 {
230 write(destination, static_cast<float>(wide_real(value)));
231 }
232
233 void encode_f64(std::byte* destination, const Scalar& value)
234 {
235 write(destination, value.real);
236 }
237
238 void encode_f16(std::byte* destination, const Scalar& value)
239 {
240 write(destination, static_cast<uint16_t>(encode_small_float(wide_real(value), 5, 10)));
241 }
242
243 void encode_bf16(std::byte* destination, const Scalar& value)
244 {
245 write(destination, static_cast<uint16_t>(encode_small_float(wide_real(value), 8, 7)));
246 }
247
248 Decoder decoder_for(Layout layout)
249 {
250 switch (layout.family) {
251 case Family::SIGNED:
252 switch (layout.bits) {
253 case 8:
254 return decode_signed<int8_t>;
255 case 16:
256 return decode_signed<int16_t>;
257 case 32:
258 return decode_signed<int32_t>;
259 default:
260 return decode_signed<int64_t>;
261 }
262 case Family::UNSIGNED:
263 switch (layout.bits) {
264 case 8:
265 return decode_unsigned<uint8_t>;
266 case 16:
267 return decode_unsigned<uint16_t>;
268 case 32:
269 return decode_unsigned<uint32_t>;
270 default:
271 return decode_unsigned<uint64_t>;
272 }
273 case Family::BOOL:
274 return decode_bool;
275 default:
276 switch (layout.bits) {
277 case 16:
278 return layout.brain ? decode_bf16 : decode_f16;
279 case 32:
280 return decode_f32;
281 default:
282 return decode_f64;
283 }
284 }
285 }
286
287 Encoder encoder_for(Layout layout)
288 {
289 switch (layout.family) {
290 case Family::SIGNED:
291 switch (layout.bits) {
292 case 8:
293 return encode_signed<int8_t>;
294 case 16:
295 return encode_signed<int16_t>;
296 case 32:
297 return encode_signed<int32_t>;
298 default:
299 return encode_signed<int64_t>;
300 }
301 case Family::UNSIGNED:
302 switch (layout.bits) {
303 case 8:
304 return encode_unsigned<uint8_t>;
305 case 16:
306 return encode_unsigned<uint16_t>;
307 case 32:
308 return encode_unsigned<uint32_t>;
309 default:
310 return encode_unsigned<uint64_t>;
311 }
312 case Family::BOOL:
313 return encode_bool;
314 default:
315 switch (layout.bits) {
316 case 16:
317 return layout.brain ? encode_bf16 : encode_f16;
318 case 32:
319 return encode_f32;
320 default:
321 return encode_f64;
322 }
323 }
324 }
325
326 template <typename Visit>
327 void walk(const TensorAccess& source, Visit visit)
328 {
329 const auto& spec = source.spec();
330 const uint64_t count = source.element_count();
331 const auto element = static_cast<int64_t>(spec.dtype.element_bytes());
332 const std::vector<int64_t> strides = spec.resolved_strides();
333 const std::byte* base = source.element_data();
334 std::vector<uint64_t> index(spec.shape.size(), 0);
335
336 for (uint64_t n = 0; n < count; ++n) {
337 int64_t offset = 0;
338 for (size_t axis = 0; axis < index.size(); ++axis)
339 offset += strides[axis] * static_cast<int64_t>(index[axis]);
340 visit(n, base + offset * element, index);
341 for (size_t axis = index.size(); axis-- > 0;) {
342 if (++index[axis] < spec.shape[axis])
343 break;
344 index[axis] = 0;
345 }
346 }
347 }
348
349 double zero_point_at(const TensorQuantization& quantization, size_t slice)
350 {
351 return quantization.zero_point.empty() ? 0.0 : static_cast<double>(quantization.zero_point[slice]);
352 }
353
354 void convert_real(const TensorAccess& source, Layout from, Layout to, const std::optional<TensorQuantization>& requested, std::byte* out)
355 {
356 const Decoder decode = decoder_for(from);
357 const Encoder encode = encoder_for(to);
358 const size_t out_bytes = to.family == Family::BOOL ? 1U : to.bits / 8U;
359 const TensorQuantization* in_q = source.spec().quantization ? &*source.spec().quantization : nullptr;
360 const TensorQuantization* out_q = requested ? &*requested : nullptr;
361
362 walk(source, [&](uint64_t n, const std::byte* element, const std::vector<uint64_t>& index) {
363 Scalar value = decode(element);
364 if (in_q != nullptr) {
365 const size_t slice = in_q->axis ? index[*in_q->axis] : 0U;
366 value.real = in_q->scale[slice] * (value.real - zero_point_at(*in_q, slice));
367 value.exact = false;
368 }
369 if (out_q != nullptr) {
370 const size_t slice = out_q->axis ? index[*out_q->axis] : 0U;
371 const double zero = zero_point_at(*out_q, slice);
372 const double scaled = value.real / out_q->scale[slice];
373 value.real = std::isnan(scaled) ? zero : std::nearbyint(scaled) + zero;
374 value.exact = false;
375 }
376 encode(out + n * out_bytes, value);
377 });
378 }
379
380 void convert_complex(const TensorAccess& source, Layout from, Layout to, std::byte* out)
381 {
382 const size_t out_bytes = to.bits / 8U;
383 walk(source, [&](uint64_t n, const std::byte* element, const std::vector<uint64_t>&) {
384 const std::complex<double> value = from.bits == 64
385 ? std::complex<double>(read<std::complex<float>>(element))
386 : read<std::complex<double>>(element);
387 std::byte* destination = out + n * out_bytes;
388 if (to.bits == 64) {
389 write(destination, std::complex<float>(static_cast<float>(value.real()), static_cast<float>(value.imag())));
390 } else {
391 write(destination, value);
392 }
393 });
394 }
395
396}
397
399 const TensorSpec& source, TensorDType target, const std::optional<TensorQuantization>& quantization)
400{
401 if (auto problem = source.diagnose(); !problem.empty())
402 return problem;
403
404 const std::string label = source.name.empty() ? std::string("tensor") : "tensor '" + source.name + "'";
405 if (!source.device.is_host_accessible())
406 return label + ": memory is not readable by the host";
407
408 std::string why;
409 const auto from = layout_of(source.dtype, why);
410 if (!from)
411 return label + ": source " + why;
412 const auto to = layout_of(target, why);
413 if (!to)
414 return label + ": target " + why;
415
416 if ((from->family == Family::COMPLEX) != (to->family == Family::COMPLEX))
417 return label + ": complex and real element types do not convert to each other";
418
419 if (source.quantization && !holds_quantization(*from))
420 return label + ": quantization on " + source.dtype.name() + " storage is not supported";
421
422 if (quantization) {
423 if (!holds_quantization(*to))
424 return label + ": " + target.name() + " cannot hold quantization, use an integer type of at most 32 bits";
425
426 TensorSpec probe = source;
427 probe.dtype = target;
428 probe.strides.clear();
429 probe.byte_offset = 0;
430 probe.quantization = quantization;
431 if (auto problem = probe.diagnose(); !problem.empty())
432 return problem;
433
434 const bool usable = std::ranges::all_of(quantization->scale,
435 [](double scale) { return std::isfinite(scale) && scale != 0.0; });
436 if (!usable)
437 return label + ": quantization scales must be finite and non-zero";
438 }
439
440 return {};
441}
442
443std::expected<TensorData, std::string> convert_tensor(
444 const TensorAccess& source, TensorDType target, const std::optional<TensorQuantization>& quantization)
445{
446 const TensorSpec& from_spec = source.spec();
447 if (auto problem = diagnose_conversion(from_spec, target, quantization); !problem.empty())
448 return std::unexpected(std::move(problem));
449
450 if (from_spec.dtype == target && same_quantization(from_spec.quantization, quantization)) {
451 auto copy = TensorData::copy_of(source);
452 if (!copy)
453 return std::unexpected("tensor '" + from_spec.name + "': the copy could not be made");
454 return std::move(*copy);
455 }
456
457 TensorSpec spec = from_spec;
458 spec.dtype = target;
459 spec.strides.clear();
460 spec.byte_offset = 0;
461 spec.device = {};
462 spec.quantization = quantization;
463
464 auto result = TensorData::allocate(std::move(spec));
465 if (!result)
466 return std::unexpected("tensor '" + from_spec.name + "': the result could not be allocated");
467 if (source.element_count() == 0)
468 return std::move(*result);
469
470 std::string unused;
471 const Layout from = *layout_of(from_spec.dtype, unused);
472 const Layout to = *layout_of(target, unused);
473 std::byte* out = result->storage().data();
474
475 if (from.family == Family::COMPLEX) {
476 convert_complex(source, from, to, out);
477 } else {
478 convert_real(source, from, to, quantization, out);
479 }
480 return std::move(*result);
481}
482
483}
Validated, non-owning view of tensor memory and its layout.
static std::optional< TensorData > copy_of(const TensorAccess &view)
Dense C-ordered copy of any tensor view.
Definition Tensor.cpp:126
static std::optional< TensorData > allocate(TensorSpec spec)
Zero-filled tensor for a spec.
Definition Tensor.cpp:72
std::string diagnose_conversion(const TensorSpec &source, TensorDType target, const std::optional< TensorQuantization > &quantization)
Empty when a tensor can be converted to a type, otherwise the reason it cannot.
std::expected< TensorData, std::string > convert_tensor(const TensorAccess &source, TensorDType target, const std::optional< TensorQuantization > &quantization)
Convert the elements of a tensor to another element type.
uint32_t encode_small_float(double value, int exponent_bits, int mantissa_bits) noexcept
Encode a double as a binary floating point pattern with the given field widths.
double decode_small_float(uint32_t bits, int exponent_bits, int mantissa_bits) noexcept
Decode a binary floating point pattern with the given field widths.
void convert_complex(std::span< From > source, std::span< To > destination, ComplexConversionStrategy strategy)
Legacy interface - redirects to convert_data.
@ COMPLEX
2 rows → complex (row 0 = real, row 1 = imag)
std::vector< double > slice(std::span< const double > data, double start_ratio, double end_ratio)
Extract a contiguous slice by ratio, returning a new vector.
std::vector< double > count(std::span< const double > data, size_t n_windows, uint32_t hop_size, uint32_t window_size)
Sample count per window (as double for pipeline uniformity).
Definition Analysis.cpp:506
std::function< float(glm::vec2)> scaled(std::function< float(glm::vec2)> norm, float lo, float hi)
Rescale a normalized projection onto an arbitrary range.
Source source()
Begin a Source chain.
Definition Plot.hpp:127
std::string name() const
Short readable name such as f32, bf16, i64, u8.
Element type as kind, bit width and lane count.
Affine quantization: real = scale * (stored - zero_point).
std::string diagnose() const
Empty when the spec is self-consistent, otherwise the reason it is not.
std::optional< TensorQuantization > quantization
std::vector< int64_t > strides
Complete description of a tensor: type, shape, layout, meaning.