MayaFlux 1.0.0
Digital-First Multimedia Processing Framework
Loading...
Searching...
No Matches
TensorCodec.cpp
Go to the documentation of this file.
1#include "TensorCodec.hpp"
2
3#include <bit>
4#include <charconv>
5#include <fstream>
6
7namespace MayaFlux::IO::Detail {
8
9namespace {
10
11 using Kind = Kakshya::TensorDTypeKind;
12
13 struct SafetensorsEntry {
14 std::string_view name;
15 Kind kind;
16 uint8_t bits;
17 };
18
19 constexpr std::array<SafetensorsEntry, 22> k_safetensors { {
20 { "BOOL", Kind::BOOL, 8 },
21 { "F4", Kind::FLOAT4_E2M1FN, 4 },
22 { "F6_E2M3", Kind::FLOAT6_E2M3FN, 6 },
23 { "F6_E3M2", Kind::FLOAT6_E3M2FN, 6 },
24 { "U8", Kind::UINT, 8 },
25 { "I8", Kind::INT, 8 },
26 { "F8_E5M2", Kind::FLOAT8_E5M2, 8 },
27 { "F8_E4M3", Kind::FLOAT8_E4M3FN, 8 },
28 { "F8_E8M0", Kind::FLOAT8_E8M0FNU, 8 },
29 { "F8_E4M3FNUZ", Kind::FLOAT8_E4M3FNUZ, 8 },
30 { "F8_E5M2FNUZ", Kind::FLOAT8_E5M2FNUZ, 8 },
31 { "I16", Kind::INT, 16 },
32 { "U16", Kind::UINT, 16 },
33 { "F16", Kind::FLOAT, 16 },
34 { "BF16", Kind::BFLOAT, 16 },
35 { "I32", Kind::INT, 32 },
36 { "U32", Kind::UINT, 32 },
37 { "F32", Kind::FLOAT, 32 },
38 { "C64", Kind::COMPLEX, 64 },
39 { "F64", Kind::FLOAT, 64 },
40 { "I64", Kind::INT, 64 },
41 { "U64", Kind::UINT, 64 },
42 } };
43
44}
45
46bool host_is_little_endian() noexcept
47{
48 return std::endian::native == std::endian::little;
49}
50
52{
53 const size_t bits = (dtype.kind == Kind::COMPLEX || dtype.kind == Kind::BCOMPLEX)
54 ? dtype.bits / 2U
55 : dtype.bits;
56 return bits % 8 == 0 ? bits / 8 : 0;
57}
58
59void swap_components(std::span<std::byte> data, size_t component_bytes) noexcept
60{
61 if (component_bytes <= 1)
62 return;
63 for (size_t offset = 0; offset + component_bytes <= data.size(); offset += component_bytes)
64 std::reverse(data.begin() + static_cast<std::ptrdiff_t>(offset),
65 data.begin() + static_cast<std::ptrdiff_t>(offset + component_bytes));
66}
67
68std::optional<Kakshya::TensorDType> safetensors_dtype(std::string_view name) noexcept
69{
70 for (const auto& entry : k_safetensors) {
71 if (entry.name == name)
72 return Kakshya::TensorDType { .kind = entry.kind, .bits = entry.bits, .lanes = 1 };
73 }
74 return std::nullopt;
75}
76
77std::optional<std::string_view> safetensors_name(Kakshya::TensorDType dtype) noexcept
78{
79 if (dtype.lanes != 1)
80 return std::nullopt;
81 for (const auto& entry : k_safetensors) {
82 if (entry.kind == dtype.kind && entry.bits == dtype.bits)
83 return entry.name;
84 }
85 return std::nullopt;
86}
87
88std::optional<NpyTypestr> npy_dtype(std::string_view typestr, std::string& why)
89{
90 size_t pos = 0;
91 bool big_endian = false;
92 if (!typestr.empty()) {
93 switch (typestr[0]) {
94 case '>':
95 big_endian = true;
96 pos = 1;
97 break;
98 case '=':
99 big_endian = !host_is_little_endian();
100 pos = 1;
101 break;
102 case '<':
103 case '|':
104 pos = 1;
105 break;
106 default:
107 break;
108 }
109 }
110 if (pos >= typestr.size()) {
111 why = "malformed numpy type string '" + std::string(typestr) + "'";
112 return std::nullopt;
113 }
114
115 const char code = typestr[pos++];
116 unsigned size = 0;
117 const char* end = typestr.data() + typestr.size();
118 const auto [parsed, error] = std::from_chars(typestr.data() + pos, end, size);
119 if (error != std::errc {} || parsed != end || size == 0 || size > 16) {
120 why = "malformed numpy type string '" + std::string(typestr) + "'";
121 return std::nullopt;
122 }
123
124 const auto bits = static_cast<uint8_t>(size * 8);
125 const auto make = [&](Kind kind) -> std::optional<NpyTypestr> {
126 return NpyTypestr { .dtype = { .kind = kind, .bits = bits, .lanes = 1 }, .big_endian = big_endian && size > 1 };
127 };
128
129 switch (code) {
130 case 'f':
131 if (size == 2 || size == 4 || size == 8)
132 return make(Kind::FLOAT);
133 break;
134 case 'i':
135 if (size == 1 || size == 2 || size == 4 || size == 8)
136 return make(Kind::INT);
137 break;
138 case 'u':
139 if (size == 1 || size == 2 || size == 4 || size == 8)
140 return make(Kind::UINT);
141 break;
142 case 'b':
143 if (size == 1)
144 return make(Kind::BOOL);
145 break;
146 case 'c':
147 if (size == 8 || size == 16)
148 return make(Kind::COMPLEX);
149 break;
150 default:
151 break;
152 }
153
154 why = "numpy type '" + std::string(typestr) + "' is not supported";
155 return std::nullopt;
156}
157
158std::optional<std::string> npy_typestr(Kakshya::TensorDType dtype)
159{
160 if (dtype.lanes != 1)
161 return std::nullopt;
162
163 const auto sized = [&](char code) -> std::optional<std::string> {
164 const unsigned size = dtype.bits / 8U;
165 return std::string(size == 1 ? "|" : "<") + code + std::to_string(size);
166 };
167
168 switch (dtype.kind) {
169 case Kind::FLOAT:
170 return (dtype.bits == 16 || dtype.bits == 32 || dtype.bits == 64) ? sized('f') : std::nullopt;
171 case Kind::INT:
172 return (dtype.bits == 8 || dtype.bits == 16 || dtype.bits == 32 || dtype.bits == 64) ? sized('i') : std::nullopt;
173 case Kind::UINT:
174 return (dtype.bits == 8 || dtype.bits == 16 || dtype.bits == 32 || dtype.bits == 64) ? sized('u') : std::nullopt;
175 case Kind::BOOL:
176 return dtype.bits == 8 ? std::optional<std::string> { "|b1" } : std::nullopt;
177 case Kind::COMPLEX:
178 return (dtype.bits == 64 || dtype.bits == 128) ? sized('c') : std::nullopt;
179 default:
180 return std::nullopt;
181 }
182}
183
184std::optional<FileHead> read_file_head(const std::string& path, size_t limit)
185{
186 std::ifstream file(path, std::ios::binary);
187 if (!file)
188 return std::nullopt;
189
190 file.seekg(0, std::ios::end);
191 FileHead head;
192 head.file_size = static_cast<uint64_t>(file.tellg());
193 file.seekg(0);
194
195 head.bytes.resize(static_cast<size_t>(std::min<uint64_t>(limit, head.file_size)));
196 file.read(reinterpret_cast<char*>(head.bytes.data()), static_cast<std::streamsize>(head.bytes.size()));
197 head.bytes.resize(static_cast<size_t>(file.gcount()));
198 return head;
199}
200
201std::string lower_extension(const std::string& path)
202{
203 auto ext = std::filesystem::path(path).extension().string();
204 if (!ext.empty() && ext[0] == '.')
205 ext.erase(0, 1);
206 std::ranges::transform(ext, ext.begin(), [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
207 return ext;
208}
209
210std::optional<std::span<const std::byte>> dense_bytes(
211 const Kakshya::TensorData& tensor, std::optional<Kakshya::TensorData>& dense)
212{
213 const auto view = tensor.access();
214 if (!view)
215 return std::nullopt;
216
217 const auto raw = view->bytes();
218 if (!raw.empty() || view->element_count() == 0)
219 return raw;
220
221 dense = Kakshya::TensorData::copy_of(*view);
222 if (!dense)
223 return std::nullopt;
224 const auto gathered = dense->access();
225 if (!gathered)
226 return std::nullopt;
227 return gathered->bytes();
228}
229
230}
static std::optional< TensorData > copy_of(const TensorAccess &view)
Dense C-ordered copy of any tensor view.
Definition Tensor.cpp:126
std::optional< TensorAccess > access() const
Borrow the tensor, or std::nullopt when it holds nothing valid.
Definition Tensor.cpp:150
Owning host tensor: a TensorSpec and the bytes it describes.
Definition Tensor.hpp:42
std::string lower_extension(const std::string &path)
Lowercase extension of a path without its dot.
size_t component_bytes(Kakshya::TensorDType dtype) noexcept
Bytes of one independently ordered component of an element.
std::optional< std::string > npy_typestr(Kakshya::TensorDType dtype)
Little-endian numpy type string for an element type, when numpy has one.
std::optional< Kakshya::TensorDType > safetensors_dtype(std::string_view name) noexcept
Element type for a safetensors dtype name such as F32 or BF16.
std::optional< std::span< const std::byte > > dense_bytes(const Kakshya::TensorData &tensor, std::optional< Kakshya::TensorData > &dense)
Elements of a tensor as dense C-order bytes.
void swap_components(std::span< std::byte > data, size_t component_bytes) noexcept
Reverse the byte order of every component in place.
std::optional< std::string_view > safetensors_name(Kakshya::TensorDType dtype) noexcept
Safetensors dtype name for an element type, when the format has one.
bool host_is_little_endian() noexcept
Whether this machine stores multi-byte values least significant byte first.
std::optional< NpyTypestr > npy_dtype(std::string_view typestr, std::string &why)
Parse a numpy type string such as <f4, >i8 or |b1.
std::optional< FileHead > read_file_head(const std::string &path, size_t limit)
Read up to limit leading bytes of a file, or std::nullopt if it cannot be opened.
TensorDTypeKind
Element family of a tensor.
std::vector< std::byte > bytes
First bytes of a file together with its total size.
Element type and byte order read from a numpy type string.
Element type as kind, bit width and lane count.