MayaFlux 1.0.0
Digital-First Multimedia Processing Framework
Loading...
Searching...
No Matches
NpyCodec.cpp
Go to the documentation of this file.
1#include "NpyCodec.hpp"
2
3#include "TensorCodec.hpp"
4
5#include <charconv>
6
7namespace MayaFlux::IO::Detail {
8
9namespace {
10
11 constexpr std::array<uint8_t, 6> npy_magic { 0x93, 'N', 'U', 'M', 'P', 'Y' };
12
13 std::string_view value_after_key(std::string_view text, std::string_view key)
14 {
15 for (const char quote : { '\'', '"' }) {
16 std::string token;
17 token += quote;
18 token += key;
19 token += quote;
20 auto pos = text.find(token);
21 if (pos == std::string_view::npos) {
22 continue;
23 }
24 pos += token.size();
25 while (pos < text.size() && (text[pos] == ' ' || text[pos] == ':')) {
26 ++pos;
27 }
28 return text.substr(pos);
29 }
30 return {};
31 }
32
33 std::optional<std::vector<uint64_t>> parse_shape(std::string_view value)
34 {
35 if (value.empty() || value[0] != '(') {
36 return std::nullopt;
37 }
38 const auto close = value.find(')');
39 if (close == std::string_view::npos) {
40 return std::nullopt;
41 }
42
43 std::vector<uint64_t> shape;
44 std::string_view body = value.substr(1, close - 1);
45 size_t pos = 0;
46 while (pos < body.size()) {
47 if (body[pos] == ' ' || body[pos] == ',') {
48 ++pos;
49 continue;
50 }
51 uint64_t extent = 0;
52 const auto [end, error] = std::from_chars(body.data() + pos, body.data() + body.size(), extent);
53 if (error != std::errc {}) {
54 return std::nullopt;
55 }
56 shape.push_back(extent);
57 pos = static_cast<size_t>(end - body.data());
58 if (pos < body.size() && body[pos] == 'L') {
59 ++pos;
60 }
61 }
62 return shape;
63 }
64
65 std::string shape_text(const std::vector<uint64_t>& shape)
66 {
67 std::string text = "(";
68 for (size_t i = 0; i < shape.size(); ++i) {
69 if (i > 0) {
70 text += ", ";
71 }
72 text += std::to_string(shape[i]);
73 }
74 if (shape.size() == 1) {
75 text += ",";
76 }
77 return text + ")";
78 }
79
80 std::vector<std::byte> build_header(std::string_view body, uint8_t major)
81 {
82 const size_t prefix = major == 1 ? 10 : 12;
83 std::string header(body);
84 const size_t padding = (64 - (prefix + header.size() + 1) % 64) % 64;
85 header.append(padding, ' ');
86 header.push_back('\n');
87
88 std::vector<std::byte> bytes;
89 bytes.reserve(prefix + header.size());
90 for (const auto byte : npy_magic) {
91 bytes.push_back(static_cast<std::byte>(byte));
92 }
93 bytes.push_back(static_cast<std::byte>(major));
94 bytes.push_back(std::byte { 0 });
95 const size_t length_bytes = major == 1 ? 2 : 4;
96 for (size_t i = 0; i < length_bytes; ++i) {
97 bytes.push_back(static_cast<std::byte>((header.size() >> (8 * i)) & 0xFF));
98 }
99 for (const char c : header) {
100 bytes.push_back(static_cast<std::byte>(c));
101 }
102 return bytes;
103 }
104
105}
106
107std::optional<size_t> npy_prefix_size(std::span<const std::byte> leading, std::string& why)
108{
109 if (leading.size() < 10 || !std::equal(npy_magic.begin(), npy_magic.end(), leading.begin(),
110 [](uint8_t expected, std::byte actual) { return expected == std::to_integer<uint8_t>(actual); })) {
111 why = "missing numpy magic string";
112 return std::nullopt;
113 }
114
115 const auto major = std::to_integer<uint8_t>(leading[6]);
116 size_t length = 0;
117 size_t prefix = 0;
118 if (major == 1) {
119 length = std::to_integer<size_t>(leading[8]) | (std::to_integer<size_t>(leading[9]) << 8);
120 prefix = 10;
121 } else if (major == 2 || major == 3) {
122 if (leading.size() < 12) {
123 why = "numpy header is truncated";
124 return std::nullopt;
125 }
126 for (size_t i = 0; i < 4; ++i) {
127 length |= std::to_integer<size_t>(leading[8 + i]) << (8 * i);
128 }
129 prefix = 12;
130 } else {
131 why = "numpy format version " + std::to_string(major) + " is not supported";
132 return std::nullopt;
133 }
134 return prefix + length;
135}
136
137std::optional<NpyHeader> parse_npy_header(
138 std::span<const std::byte> prefix, std::string_view name, std::string& why)
139{
140 const auto total = npy_prefix_size(prefix, why);
141 if (!total || *total != prefix.size()) {
142 if (total) {
143 why = "numpy header is truncated";
144 }
145 return std::nullopt;
146 }
147
148 const size_t header_start = std::to_integer<uint8_t>(prefix[6]) == 1 ? 10 : 12;
149 const std::string_view text(reinterpret_cast<const char*>(prefix.data()) + header_start,
150 prefix.size() - header_start);
151
152 const auto descr = value_after_key(text, "descr");
153 if (descr.empty() || (descr[0] != '\'' && descr[0] != '"')) {
154 why = descr.empty() ? "numpy header has no descr" : "structured numpy dtypes are not supported";
155 return std::nullopt;
156 }
157 const auto descr_end = descr.find(descr[0], 1);
158 if (descr_end == std::string_view::npos) {
159 why = "numpy descr is malformed";
160 return std::nullopt;
161 }
162 const auto typestr = npy_dtype(descr.substr(1, descr_end - 1), why);
163 if (!typestr) {
164 return std::nullopt;
165 }
166
167 const auto order = value_after_key(text, "fortran_order");
168 bool fortran = false;
169 if (order.starts_with("True")) {
170 fortran = true;
171 } else if (!order.starts_with("False")) {
172 why = "numpy header has no fortran_order";
173 return std::nullopt;
174 }
175
176 const auto shape = parse_shape(value_after_key(text, "shape"));
177 if (!shape) {
178 why = "numpy header has no valid shape";
179 return std::nullopt;
180 }
181
182 NpyHeader header;
183 header.big_endian = typestr->big_endian;
184 header.data_offset = *total;
185 header.spec = Kakshya::TensorSpec::contiguous(typestr->dtype, *shape, std::string(name));
186
187 if (fortran && shape->size() > 1) {
188 header.spec.strides.assign(shape->size(), 1);
189 for (size_t i = 1; i < shape->size(); ++i) {
190 header.spec.strides[i] = header.spec.strides[i - 1] * static_cast<int64_t>(std::max<uint64_t>((*shape)[i - 1], 1));
191 }
192 }
193
194 if (const auto reason = header.spec.diagnose(); !reason.empty()) {
195 why = reason;
196 return std::nullopt;
197 }
198 return header;
199}
200
201std::optional<std::vector<std::byte>> encode_npy(const Kakshya::TensorData& tensor, std::string& why)
202{
203 const auto& spec = tensor.spec();
204 const auto typestr = npy_typestr(spec.dtype);
205 if (!typestr) {
206 why = "numpy has no type for element type " + spec.dtype.name();
207 return std::nullopt;
208 }
209
210 std::optional<Kakshya::TensorData> gathered;
211 const auto payload = dense_bytes(tensor, gathered);
212 if (!payload) {
213 why = "tensor '" + spec.name + "' could not be read";
214 return std::nullopt;
215 }
216
217 const std::string body = "{'descr': '" + *typestr + "', 'fortran_order': False, 'shape': "
218 + shape_text(spec.shape) + ", }";
219 auto bytes = build_header(body, 1);
220 if (bytes.size() - 10 > 0xFFFF) {
221 bytes = build_header(body, 2);
222 }
223
224 const size_t data_start = bytes.size();
225 bytes.insert(bytes.end(), payload->begin(), payload->end());
226 if (!host_is_little_endian()) {
227 swap_components(std::span<std::byte>(bytes).subspan(data_start), component_bytes(spec.dtype));
228 }
229 return bytes;
230}
231
232}
const TensorSpec & spec() const noexcept
Layout of the stored bytes.
Definition Tensor.hpp:105
Owning host tensor: a TensorSpec and the bytes it describes.
Definition Tensor.hpp:42
std::optional< NpyHeader > parse_npy_header(std::span< const std::byte > prefix, std::string_view name, std::string &why)
Decode a .npy prefix into a tensor spec.
Definition NpyCodec.cpp:137
size_t component_bytes(Kakshya::TensorDType dtype) noexcept
Bytes of one independently ordered component of an element.
std::optional< std::vector< std::byte > > encode_npy(const Kakshya::TensorData &tensor, std::string &why)
Encode a tensor as a complete .npy array in dense C order.
Definition NpyCodec.cpp:201
std::optional< std::string > npy_typestr(Kakshya::TensorDType dtype)
Little-endian numpy type string for an element type, when numpy has one.
std::optional< std::span< const std::byte > > dense_bytes(const Kakshya::TensorData &tensor, std::optional< Kakshya::TensorData > &dense)
Elements of a tensor as dense C-order bytes.
std::optional< size_t > npy_prefix_size(std::span< const std::byte > leading, std::string &why)
Total prefix size (magic, version, length field and header) of a .npy array.
Definition NpyCodec.cpp:107
void swap_components(std::span< std::byte > data, size_t component_bytes) noexcept
Reverse the byte order of every component in place.
bool host_is_little_endian() noexcept
Whether this machine stores multi-byte values least significant byte first.
std::optional< NpyTypestr > npy_dtype(std::string_view typestr, std::string &why)
Parse a numpy type string such as <f4, >i8 or |b1.
void error(Component component, Context context, std::source_location location, std::string_view message)
Log an error message and optionally throw an exception.
void close(std::span< const float > mask, std::span< float > tmp, std::span< float > dst, uint32_t w, uint32_t h, uint32_t radius)
Morphological closing writing into dst.
std::string shape_text(const std::vector< uint64_t > &shape)
Kakshya::TensorSpec spec
Definition NpyCodec.hpp:9
Decoded header of one .npy array.
Definition NpyCodec.hpp:8
std::string diagnose() const
Empty when the spec is self-consistent, otherwise the reason it is not.
std::vector< int64_t > strides
static TensorSpec contiguous(TensorDType dtype, std::vector< uint64_t > shape, std::string name={})
C-ordered spec with no offset, axes or quantization.