MayaFlux 1.0.0
Digital-First Multimedia Processing Framework
Loading...
Searching...
No Matches
NpyFormat.cpp
Go to the documentation of this file.
1#include "NpyFormat.hpp"
2
3#include "Detail/NpyCodec.hpp"
5#include "TensorFormat.hpp"
6
8
9namespace MayaFlux::IO {
10
12{
14 .name = "npy",
15 .extensions = { "npy" },
16 .probe = &NpyReader::probe,
17 .make_reader = []() -> std::unique_ptr<TensorReader> { return std::make_unique<NpyReader>(); },
18 .make_writer = []() -> std::unique_ptr<TensorWriter> { return std::make_unique<NpyWriter>(); } });
19
20 MF_INFO(Journal::Component::IO, Journal::Context::Init, "NpyReader registered for: npy");
21}
22
23bool NpyReader::probe(std::span<const std::byte> head, uint64_t)
24{
25 std::string why;
26 return head.size() >= 10 && Detail::npy_prefix_size(head.first(std::min<size_t>(head.size(), 12)), why).has_value();
27}
28
29bool NpyReader::can_read(const std::string& filepath) const
30{
31 const auto path = resolve_path(filepath);
32 if (Detail::lower_extension(path) == "npy") {
33 return true;
34 }
35 const auto head = Detail::read_file_head(path, 12);
36 return head && probe(head->bytes, head->file_size);
37}
38
40{
41 if (m_file.is_open()) {
42 m_file.close();
43 }
44 m_spec = {};
45 m_data_offset = 0;
46 m_open = false;
47}
48
49bool NpyReader::open(const std::string& filepath, FileReadOptions)
50{
51 close();
52 m_path = resolve_path(filepath);
53 m_file.open(m_path, std::ios::binary);
54 if (!m_file) {
55 return fail("cannot open '" + m_path + "'");
56 }
57
58 m_file.seekg(0, std::ios::end);
59 const auto file_size = static_cast<uint64_t>(m_file.tellg());
60 m_file.seekg(0);
61
62 std::array<std::byte, 12> leading {};
63 m_file.read(reinterpret_cast<char*>(leading.data()), static_cast<std::streamsize>(leading.size()));
64 const auto got = static_cast<size_t>(m_file.gcount());
65 m_file.clear();
66
67 std::string why;
68 const auto prefix_size = Detail::npy_prefix_size(std::span<const std::byte>(leading).first(got), why);
69 if (!prefix_size || *prefix_size > file_size) {
70 close();
71 return fail(prefix_size ? "numpy header is truncated" : why);
72 }
73
74 std::vector<std::byte> prefix(*prefix_size);
75 m_file.seekg(0);
76 if (!m_file.read(reinterpret_cast<char*>(prefix.data()), static_cast<std::streamsize>(prefix.size()))) {
77 close();
78 return fail("numpy header is truncated");
79 }
80
81 const auto stem = std::filesystem::path(m_path).stem().string();
82 const auto header = Detail::parse_npy_header(prefix, stem, why);
83 if (!header) {
84 close();
85 return fail(why);
86 }
87
88 const auto bytes = header->spec.dtype.storage_bytes(header->spec.element_count().value_or(0));
89 if (!bytes || header->data_offset + *bytes > file_size) {
90 close();
91 return fail("numpy data is shorter than its header declares");
92 }
93
94 m_spec = header->spec;
95 m_data_offset = header->data_offset;
96 m_big_endian = header->big_endian;
97 m_open = true;
98 m_last_error.clear();
99 return true;
100}
101
102std::vector<Kakshya::TensorSpec> NpyReader::describe() const
103{
104 return m_open ? std::vector<Kakshya::TensorSpec> { m_spec } : std::vector<Kakshya::TensorSpec> {};
105}
106
107std::optional<Kakshya::TensorData> NpyReader::read(std::string_view name)
108{
109 if (!m_open) {
110 fail("no file is open");
111 return std::nullopt;
112 }
113 if (!name.empty() && name != m_spec.name) {
114 fail("no tensor named '" + std::string(name) + "'");
115 return std::nullopt;
116 }
117
119 if (!tensor) {
120 fail("cannot allocate tensor '" + m_spec.name + "'");
121 return std::nullopt;
122 }
123
124 const auto storage = tensor->storage();
125 if (!storage.empty()) {
126 m_file.clear();
127 m_file.seekg(static_cast<std::streamoff>(m_data_offset));
128 if (!m_file.read(reinterpret_cast<char*>(storage.data()), static_cast<std::streamsize>(storage.size()))) {
129 fail("numpy data is truncated");
130 return std::nullopt;
131 }
134 }
135 }
136 return tensor;
137}
138
139bool NpyWriter::write(const std::string& filepath, const Kakshya::TensorSet& tensors)
140{
141 m_last_error.clear();
142 if (tensors.tensors.size() != 1) {
143 m_last_error = "a .npy file holds exactly one tensor, received " + std::to_string(tensors.tensors.size());
144 return false;
145 }
146
147 const auto bytes = Detail::encode_npy(tensors.tensors.front(), m_last_error);
148 if (!bytes) {
149 return false;
150 }
151
152 std::ofstream file(filepath, std::ios::binary | std::ios::trunc);
153 if (!file) {
154 m_last_error = "cannot create '" + filepath + "'";
155 return false;
156 }
157 file.write(reinterpret_cast<const char*>(bytes->data()), static_cast<std::streamsize>(bytes->size()));
158 file.close();
159 if (!file) {
160 m_last_error = "failed while writing '" + filepath + "'";
161 return false;
162 }
163 return true;
164}
165
166}
#define MF_INFO(comp, ctx,...)
static std::string resolve_path(const std::string &filepath)
Resolve a filepath against the project source root if not found as-is.
Kakshya::TensorSpec m_spec
Definition NpyFormat.hpp:51
bool open(const std::string &filepath, FileReadOptions options=FileReadOptions::ALL) override
Open a file for reading.
Definition NpyFormat.cpp:49
std::vector< Kakshya::TensorSpec > describe() const override
Specs of every tensor in the open file, in file order, without reading data.
void close() override
Close the currently open file.
Definition NpyFormat.cpp:39
bool can_read(const std::string &filepath) const override
Check if a file can be read by this reader.
Definition NpyFormat.cpp:29
static void register_with_registry()
Register the format, with its writer, in the TensorFormatRegistry.
Definition NpyFormat.cpp:11
static bool probe(std::span< const std::byte > head, uint64_t file_size)
Whether the leading bytes are the numpy magic string.
Definition NpyFormat.cpp:23
std::optional< Kakshya::TensorData > read(std::string_view name) override
Read one tensor by name.
bool write(const std::string &filepath, const Kakshya::TensorSet &tensors) override
Write every tensor in the set to a file.
static TensorFormatRegistry & instance()
The process-wide registry.
void register_format(TensorFormat format)
Add a format, replacing any earlier one with the same name.
bool fail(std::string reason)
Record a failure reason and return false.
static std::optional< TensorData > allocate(TensorSpec spec)
Zero-filled tensor for a spec.
Definition Tensor.cpp:72
std::string lower_extension(const std::string &path)
Lowercase extension of a path without its dot.
std::optional< NpyHeader > parse_npy_header(std::span< const std::byte > prefix, std::string_view name, std::string &why)
Decode a .npy prefix into a tensor spec.
Definition NpyCodec.cpp:137
size_t component_bytes(Kakshya::TensorDType dtype) noexcept
Bytes of one independently ordered component of an element.
std::optional< std::vector< std::byte > > encode_npy(const Kakshya::TensorData &tensor, std::string &why)
Encode a tensor as a complete .npy array in dense C order.
Definition NpyCodec.cpp:201
std::optional< size_t > npy_prefix_size(std::span< const std::byte > leading, std::string &why)
Total prefix size (magic, version, length field and header) of a .npy array.
Definition NpyCodec.cpp:107
void swap_components(std::span< std::byte > data, size_t component_bytes) noexcept
Reverse the byte order of every component in place.
bool host_is_little_endian() noexcept
Whether this machine stores multi-byte values least significant byte first.
std::optional< FileHead > read_file_head(const std::string &path, size_t limit)
Read up to limit leading bytes of a file, or std::nullopt if it cannot be opened.
FileReadOptions
Generic options for file reading behavior.
@ Init
Engine/subsystem initialization.
@ IO
Networking, file handling, streaming.
Everything the registry needs to know about one tensor file format.
std::vector< TensorData > tensors
Definition Tensor.hpp:168
Named tensors that belong together, such as a model's weights or I/O.
Definition Tensor.hpp:167