11 enum class Family : uint8_t {
27 uint64_t magnitude {};
32 using Decoder = Scalar (*)(
const std::byte*);
33 using Encoder = void (*)(std::byte*,
const Scalar&);
35 bool one_of(uint8_t bits, std::initializer_list<uint8_t> widths)
37 return std::ranges::find(widths, bits) != widths.end();
40 std::optional<Layout> layout_of(
TensorDType dtype, std::string& why)
42 if (dtype.lanes != 1) {
43 why = dtype.name() +
" holds several lanes per element";
49 if (one_of(dtype.bits, { 8, 16, 32, 64 }))
50 return Layout { .family = Family::SIGNED, .bits = dtype.bits, .brain =
false };
53 if (one_of(dtype.bits, { 8, 16, 32, 64 }))
54 return Layout { .family = Family::UNSIGNED, .bits = dtype.bits, .brain =
false };
58 return Layout { .family = Family::BOOL, .bits = 8, .brain =
false };
61 if (one_of(dtype.bits, { 16, 32, 64 }))
62 return Layout { .family = Family::FLOAT, .bits = dtype.bits, .brain =
false };
66 return Layout { .family = Family::FLOAT, .bits = 16, .brain =
true };
69 if (one_of(dtype.bits, { 64, 128 }))
70 return Layout { .family = Family::COMPLEX, .bits = dtype.bits, .brain =
false };
76 why = dtype.name() +
" is not supported by conversion";
80 bool holds_quantization(Layout layout)
82 return (layout.family == Family::SIGNED || layout.family == Family::UNSIGNED) && layout.bits <= 32;
85 bool same_quantization(
const std::optional<TensorQuantization>& a,
const std::optional<TensorQuantization>& b)
89 return a->scale == b->scale && a->zero_point == b->zero_point && a->axis == b->axis;
92 double wide_real(
const Scalar& value)
94 constexpr uint64_t exact_limit = uint64_t { 1 } << 53;
95 if (!value.exact || value.magnitude <= exact_limit)
98 const int shift = std::bit_width(value.magnitude) - 53;
99 uint64_t mantissa = value.magnitude >> shift;
100 if ((value.magnitude & ((uint64_t { 1 } << shift) - 1U)) != 0U)
102 const double magnitude = std::ldexp(
static_cast<double>(mantissa), shift);
103 return value.negative ? -magnitude : magnitude;
106 template <
typename T>
107 T read(
const std::byte* source)
110 std::memcpy(&value, source,
sizeof(T));
114 template <
typename T>
115 void write(std::byte* destination, T value)
117 std::memcpy(destination, &value,
sizeof(T));
120 template <
typename T>
121 Scalar decode_signed(
const std::byte* source)
123 const auto value = read<T>(source);
126 result.real =
static_cast<double>(value);
127 result.negative = value < 0;
128 result.magnitude = value < 0
129 ?
static_cast<uint64_t
>(-(
static_cast<int64_t
>(value) + 1)) + 1U
130 : static_cast<uint64_t>(value);
134 template <
typename T>
135 Scalar decode_unsigned(
const std::byte* source)
137 const auto value = read<T>(source);
140 result.real =
static_cast<double>(value);
141 result.magnitude = value;
145 Scalar decode_bool(
const std::byte* source)
149 result.magnitude = *
source != std::byte { 0 } ? 1U : 0
U;
150 result.real =
static_cast<double>(result.magnitude);
154 Scalar decode_f32(
const std::byte* source)
156 return Scalar { .real =
static_cast<double>(read<float>(source)) };
159 Scalar decode_f64(
const std::byte* source)
161 return Scalar { .real = read<double>(source) };
164 Scalar decode_f16(
const std::byte* source)
169 Scalar decode_bf16(
const std::byte* source)
174 template <
typename T>
175 T integer_from_real(
double value)
177 if (std::isnan(value))
179 const double rounded = std::nearbyint(value);
180 if (rounded <=
static_cast<double>(std::numeric_limits<T>::min()))
181 return std::numeric_limits<T>::min();
182 if (rounded >=
static_cast<double>(std::numeric_limits<T>::max()))
183 return std::numeric_limits<T>::max();
184 return static_cast<T>(rounded);
187 template <
typename T>
188 void encode_signed(std::byte* destination,
const Scalar& value)
192 result = integer_from_real<T>(value.real);
193 }
else if (value.negative) {
194 const uint64_t limit =
static_cast<uint64_t
>(std::numeric_limits<T>::max()) + 1U;
195 result = value.magnitude >= limit
196 ? std::numeric_limits<T>::min()
197 : static_cast<
T>(-static_cast<int64_t>(value.magnitude));
199 result = value.magnitude >
static_cast<uint64_t
>(std::numeric_limits<T>::max())
200 ? std::numeric_limits<T>::max()
201 :
static_cast<T>(value.magnitude);
203 write(destination, result);
206 template <
typename T>
207 void encode_unsigned(std::byte* destination,
const Scalar& value)
211 result = integer_from_real<T>(value.real);
212 }
else if (value.negative) {
215 result = value.magnitude >
static_cast<uint64_t
>(std::numeric_limits<T>::max())
216 ? std::numeric_limits<T>::max()
217 :
static_cast<T>(value.magnitude);
219 write(destination, result);
222 void encode_bool(std::byte* destination,
const Scalar& value)
224 const bool truth = value.exact ? value.magnitude != 0
U : value.real != 0.0;
225 *destination = truth ? std::byte { 1 } : std::byte { 0 };
228 void encode_f32(std::byte* destination,
const Scalar& value)
230 write(destination,
static_cast<float>(wide_real(value)));
233 void encode_f64(std::byte* destination,
const Scalar& value)
235 write(destination, value.real);
238 void encode_f16(std::byte* destination,
const Scalar& value)
240 write(destination,
static_cast<uint16_t
>(
encode_small_float(wide_real(value), 5, 10)));
243 void encode_bf16(std::byte* destination,
const Scalar& value)
245 write(destination,
static_cast<uint16_t
>(
encode_small_float(wide_real(value), 8, 7)));
248 Decoder decoder_for(Layout layout)
250 switch (layout.family) {
252 switch (layout.bits) {
254 return decode_signed<int8_t>;
256 return decode_signed<int16_t>;
258 return decode_signed<int32_t>;
260 return decode_signed<int64_t>;
262 case Family::UNSIGNED:
263 switch (layout.bits) {
265 return decode_unsigned<uint8_t>;
267 return decode_unsigned<uint16_t>;
269 return decode_unsigned<uint32_t>;
271 return decode_unsigned<uint64_t>;
276 switch (layout.bits) {
278 return layout.brain ? decode_bf16 : decode_f16;
287 Encoder encoder_for(Layout layout)
289 switch (layout.family) {
291 switch (layout.bits) {
293 return encode_signed<int8_t>;
295 return encode_signed<int16_t>;
297 return encode_signed<int32_t>;
299 return encode_signed<int64_t>;
301 case Family::UNSIGNED:
302 switch (layout.bits) {
304 return encode_unsigned<uint8_t>;
306 return encode_unsigned<uint16_t>;
308 return encode_unsigned<uint32_t>;
310 return encode_unsigned<uint64_t>;
315 switch (layout.bits) {
317 return layout.brain ? encode_bf16 : encode_f16;
326 template <
typename Visit>
329 const auto& spec =
source.spec();
331 const auto element =
static_cast<int64_t
>(spec.dtype.element_bytes());
332 const std::vector<int64_t> strides = spec.resolved_strides();
333 const std::byte* base =
source.element_data();
334 std::vector<uint64_t> index(spec.shape.size(), 0);
336 for (uint64_t n = 0; n <
count; ++n) {
338 for (
size_t axis = 0; axis < index.size(); ++axis)
339 offset += strides[axis] *
static_cast<int64_t
>(index[axis]);
340 visit(n, base + offset * element, index);
341 for (
size_t axis = index.size(); axis-- > 0;) {
342 if (++index[axis] < spec.shape[axis])
351 return quantization.zero_point.empty() ? 0.0 :
static_cast<double>(quantization.zero_point[
slice]);
354 void convert_real(
const TensorAccess& source, Layout from, Layout to,
const std::optional<TensorQuantization>& requested, std::byte* out)
356 const Decoder decode = decoder_for(from);
357 const Encoder encode = encoder_for(to);
358 const size_t out_bytes = to.family == Family::BOOL ? 1U : to.bits / 8U;
362 walk(source, [&](uint64_t n,
const std::byte* element,
const std::vector<uint64_t>& index) {
363 Scalar value = decode(element);
364 if (in_q !=
nullptr) {
365 const size_t slice = in_q->axis ? index[*in_q->axis] : 0
U;
366 value.real = in_q->scale[
slice] * (value.real - zero_point_at(*in_q, slice));
369 if (out_q !=
nullptr) {
370 const size_t slice = out_q->axis ? index[*out_q->axis] : 0
U;
371 const double zero = zero_point_at(*out_q, slice);
372 const double scaled = value.real / out_q->scale[
slice];
373 value.real = std::isnan(scaled) ? zero : std::nearbyint(scaled) + zero;
376 encode(out + n * out_bytes, value);
382 const size_t out_bytes = to.bits / 8U;
383 walk(source, [&](uint64_t n,
const std::byte* element,
const std::vector<uint64_t>&) {
384 const std::complex<double> value = from.bits == 64
385 ? std::complex<double>(read<std::complex<float>>(element))
386 : read<std::complex<double>>(element);
387 std::byte* destination = out + n * out_bytes;
389 write(destination, std::complex<float>(static_cast<float>(value.real()), static_cast<float>(value.imag())));
391 write(destination, value);
401 if (
auto problem = source.diagnose(); !problem.empty())
404 const std::string label = source.name.empty() ? std::string(
"tensor") :
"tensor '" + source.name +
"'";
405 if (!source.device.is_host_accessible())
406 return label +
": memory is not readable by the host";
409 const auto from = layout_of(source.dtype, why);
411 return label +
": source " + why;
412 const auto to = layout_of(target, why);
414 return label +
": target " + why;
416 if ((from->family == Family::COMPLEX) != (to->family == Family::COMPLEX))
417 return label +
": complex and real element types do not convert to each other";
419 if (source.quantization && !holds_quantization(*from))
420 return label +
": quantization on " + source.dtype.name() +
" storage is not supported";
423 if (!holds_quantization(*to))
424 return label +
": " + target.
name() +
" cannot hold quantization, use an integer type of at most 32 bits";
427 probe.
dtype = target;
431 if (
auto problem = probe.
diagnose(); !problem.empty())
434 const bool usable = std::ranges::all_of(quantization->scale,
435 [](
double scale) { return std::isfinite(scale) && scale != 0.0; });
437 return label +
": quantization scales must be finite and non-zero";
448 return std::unexpected(std::move(problem));
450 if (from_spec.
dtype == target && same_quantization(from_spec.
quantization, quantization)) {
453 return std::unexpected(
"tensor '" + from_spec.
name +
"': the copy could not be made");
454 return std::move(*copy);
466 return std::unexpected(
"tensor '" + from_spec.
name +
"': the result could not be allocated");
467 if (source.element_count() == 0)
468 return std::move(*result);
471 const Layout from = *layout_of(from_spec.
dtype, unused);
472 const Layout to = *layout_of(target, unused);
473 std::byte* out = result->storage().data();
475 if (from.family == Family::COMPLEX) {
478 convert_real(source, from, to, quantization, out);
480 return std::move(*result);
Validated, non-owning view of tensor memory and its layout.
static std::optional< TensorData > copy_of(const TensorAccess &view)
Dense C-ordered copy of any tensor view.
static std::optional< TensorData > allocate(TensorSpec spec)
Zero-filled tensor for a spec.
std::string diagnose_conversion(const TensorSpec &source, TensorDType target, const std::optional< TensorQuantization > &quantization)
Empty when a tensor can be converted to a type, otherwise the reason it cannot.
std::expected< TensorData, std::string > convert_tensor(const TensorAccess &source, TensorDType target, const std::optional< TensorQuantization > &quantization)
Convert the elements of a tensor to another element type.
uint32_t encode_small_float(double value, int exponent_bits, int mantissa_bits) noexcept
Encode a double as a binary floating point pattern with the given field widths.
double decode_small_float(uint32_t bits, int exponent_bits, int mantissa_bits) noexcept
Decode a binary floating point pattern with the given field widths.
void convert_complex(std::span< From > source, std::span< To > destination, ComplexConversionStrategy strategy)
Legacy interface - redirects to convert_data.
@ COMPLEX
2 rows → complex (row 0 = real, row 1 = imag)
std::vector< double > slice(std::span< const double > data, double start_ratio, double end_ratio)
Extract a contiguous slice by ratio, returning a new vector.
std::vector< double > count(std::span< const double > data, size_t n_windows, uint32_t hop_size, uint32_t window_size)
Sample count per window (as double for pipeline uniformity).
std::function< float(glm::vec2)> scaled(std::function< float(glm::vec2)> norm, float lo, float hi)
Rescale a normalized projection onto an arbitrary range.
std::string name() const
Short readable name such as f32, bf16, i64, u8.
Element type as kind, bit width and lane count.
Affine quantization: real = scale * (stored - zero_point).
std::string diagnose() const
Empty when the spec is self-consistent, otherwise the reason it is not.
std::optional< TensorQuantization > quantization
std::vector< int64_t > strides
Complete description of a tensor: type, shape, layout, meaning.