11 constexpr std::array<uint8_t, 6> npy_magic { 0x93,
'N',
'U',
'M',
'P',
'Y' };
13 std::string_view value_after_key(std::string_view text, std::string_view key)
15 for (
const char quote : {
'\'',
'"' }) {
20 auto pos = text.find(token);
21 if (pos == std::string_view::npos) {
25 while (pos < text.size() && (text[pos] ==
' ' || text[pos] ==
':')) {
28 return text.substr(pos);
33 std::optional<std::vector<uint64_t>> parse_shape(std::string_view value)
35 if (value.empty() || value[0] !=
'(') {
38 const auto close = value.find(
')');
39 if (close == std::string_view::npos) {
43 std::vector<uint64_t> shape;
44 std::string_view body = value.substr(1, close - 1);
46 while (pos < body.size()) {
47 if (body[pos] ==
' ' || body[pos] ==
',') {
52 const auto [end,
error] = std::from_chars(body.data() + pos, body.data() + body.size(), extent);
53 if (error != std::errc {}) {
56 shape.push_back(extent);
57 pos =
static_cast<size_t>(end - body.data());
58 if (pos < body.size() && body[pos] ==
'L') {
65 std::string
shape_text(
const std::vector<uint64_t>& shape)
67 std::string text =
"(";
68 for (
size_t i = 0; i < shape.size(); ++i) {
72 text += std::to_string(shape[i]);
74 if (shape.size() == 1) {
80 std::vector<std::byte> build_header(std::string_view body, uint8_t major)
82 const size_t prefix = major == 1 ? 10 : 12;
83 std::string header(body);
84 const size_t padding = (64 - (prefix + header.size() + 1) % 64) % 64;
85 header.append(padding,
' ');
86 header.push_back(
'\n');
88 std::vector<std::byte> bytes;
89 bytes.reserve(prefix + header.size());
90 for (
const auto byte : npy_magic) {
91 bytes.push_back(
static_cast<std::byte
>(
byte));
93 bytes.push_back(
static_cast<std::byte
>(major));
94 bytes.push_back(std::byte { 0 });
95 const size_t length_bytes = major == 1 ? 2 : 4;
96 for (
size_t i = 0; i < length_bytes; ++i) {
97 bytes.push_back(
static_cast<std::byte
>((header.size() >> (8 * i)) & 0xFF));
99 for (
const char c : header) {
100 bytes.push_back(
static_cast<std::byte
>(c));
107std::optional<size_t>
npy_prefix_size(std::span<const std::byte> leading, std::string& why)
109 if (leading.size() < 10 || !std::equal(npy_magic.begin(), npy_magic.end(), leading.begin(),
110 [](uint8_t expected, std::byte actual) { return expected == std::to_integer<uint8_t>(actual); })) {
111 why =
"missing numpy magic string";
115 const auto major = std::to_integer<uint8_t>(leading[6]);
119 length = std::to_integer<size_t>(leading[8]) | (std::to_integer<size_t>(leading[9]) << 8);
121 }
else if (major == 2 || major == 3) {
122 if (leading.size() < 12) {
123 why =
"numpy header is truncated";
126 for (
size_t i = 0; i < 4; ++i) {
127 length |= std::to_integer<size_t>(leading[8 + i]) << (8 * i);
131 why =
"numpy format version " + std::to_string(major) +
" is not supported";
134 return prefix + length;
138 std::span<const std::byte> prefix, std::string_view name, std::string& why)
141 if (!total || *total != prefix.size()) {
143 why =
"numpy header is truncated";
148 const size_t header_start = std::to_integer<uint8_t>(prefix[6]) == 1 ? 10 : 12;
149 const std::string_view text(
reinterpret_cast<const char*
>(prefix.data()) + header_start,
150 prefix.size() - header_start);
152 const auto descr = value_after_key(text,
"descr");
153 if (descr.empty() || (descr[0] !=
'\'' && descr[0] !=
'"')) {
154 why = descr.empty() ?
"numpy header has no descr" :
"structured numpy dtypes are not supported";
157 const auto descr_end = descr.find(descr[0], 1);
158 if (descr_end == std::string_view::npos) {
159 why =
"numpy descr is malformed";
162 const auto typestr =
npy_dtype(descr.substr(1, descr_end - 1), why);
167 const auto order = value_after_key(text,
"fortran_order");
168 bool fortran =
false;
169 if (order.starts_with(
"True")) {
171 }
else if (!order.starts_with(
"False")) {
172 why =
"numpy header has no fortran_order";
176 const auto shape = parse_shape(value_after_key(text,
"shape"));
178 why =
"numpy header has no valid shape";
187 if (fortran && shape->size() > 1) {
189 for (
size_t i = 1; i < shape->size(); ++i) {
190 header.
spec.
strides[i] = header.
spec.
strides[i - 1] *
static_cast<int64_t
>(std::max<uint64_t>((*shape)[i - 1], 1));
194 if (
const auto reason = header.
spec.
diagnose(); !reason.empty()) {
203 const auto& spec = tensor.
spec();
206 why =
"numpy has no type for element type " + spec.dtype.name();
210 std::optional<Kakshya::TensorData> gathered;
211 const auto payload =
dense_bytes(tensor, gathered);
213 why =
"tensor '" + spec.name +
"' could not be read";
217 const std::string body =
"{'descr': '" + *typestr +
"', 'fortran_order': False, 'shape': "
218 + shape_text(spec.shape) +
", }";
219 auto bytes = build_header(body, 1);
220 if (bytes.size() - 10 > 0xFFFF) {
221 bytes = build_header(body, 2);
224 const size_t data_start = bytes.size();
225 bytes.insert(bytes.end(), payload->begin(), payload->end());
const TensorSpec & spec() const noexcept
Layout of the stored bytes.
Owning host tensor: a TensorSpec and the bytes it describes.
std::optional< NpyHeader > parse_npy_header(std::span< const std::byte > prefix, std::string_view name, std::string &why)
Decode a .npy prefix into a tensor spec.
size_t component_bytes(Kakshya::TensorDType dtype) noexcept
Bytes of one independently ordered component of an element.
std::optional< std::vector< std::byte > > encode_npy(const Kakshya::TensorData &tensor, std::string &why)
Encode a tensor as a complete .npy array in dense C order.
std::optional< std::string > npy_typestr(Kakshya::TensorDType dtype)
Little-endian numpy type string for an element type, when numpy has one.
std::optional< std::span< const std::byte > > dense_bytes(const Kakshya::TensorData &tensor, std::optional< Kakshya::TensorData > &dense)
Elements of a tensor as dense C-order bytes.
std::optional< size_t > npy_prefix_size(std::span< const std::byte > leading, std::string &why)
Total prefix size (magic, version, length field and header) of a .npy array.
void swap_components(std::span< std::byte > data, size_t component_bytes) noexcept
Reverse the byte order of every component in place.
bool host_is_little_endian() noexcept
Whether this machine stores multi-byte values least significant byte first.
std::optional< NpyTypestr > npy_dtype(std::string_view typestr, std::string &why)
Parse a numpy type string such as <f4, >i8 or |b1.
void error(Component component, Context context, std::source_location location, std::string_view message)
Log an error message and optionally throw an exception.
void close(std::span< const float > mask, std::span< float > tmp, std::span< float > dst, uint32_t w, uint32_t h, uint32_t radius)
Morphological closing writing into dst.
std::string shape_text(const std::vector< uint64_t > &shape)
std::string diagnose() const
Empty when the spec is self-consistent, otherwise the reason it is not.
std::vector< int64_t > strides
static TensorSpec contiguous(TensorDType dtype, std::vector< uint64_t > shape, std::string name={})
C-ordered spec with no offset, axes or quantization.