MayaFlux 1.0.0
Digital-First Multimedia Processing Framework
Loading...
Searching...
No Matches
NpzFormat.cpp
Go to the documentation of this file.
1#include "NpzFormat.hpp"
2
3#include "Detail/NpyCodec.hpp"
5#include "TensorFormat.hpp"
6
8
9#include "miniz.h"
10
11namespace MayaFlux::IO {
12
13namespace Detail {
14
15 struct NpzArchive {
16 mz_zip_archive zip {};
17 bool active {};
18
20 {
21 if (active) {
22 mz_zip_reader_end(&zip);
23 }
24 }
25 };
26
27}
28
29namespace {
30
31 constexpr std::string_view npy_suffix = ".npy";
32
33 std::vector<std::byte> read_entry_prefix(mz_zip_archive& zip, uint32_t index, std::string& why)
34 {
35 auto* iterator = mz_zip_reader_extract_iter_new(&zip, index, 0);
36 if (!iterator) {
37 why = "cannot open archive entry";
38 return {};
39 }
40
41 std::array<std::byte, 12> leading {};
42 const size_t got = mz_zip_reader_extract_iter_read(iterator, leading.data(), leading.size());
43
44 std::vector<std::byte> prefix;
45 if (const auto total = Detail::npy_prefix_size(std::span<const std::byte>(leading).first(got), why)) {
46 prefix.resize(*total);
47 const size_t have = std::min(got, *total);
48 std::memcpy(prefix.data(), leading.data(), have);
49 const size_t more = *total - have;
50 if (more > 0 && mz_zip_reader_extract_iter_read(iterator, prefix.data() + have, more) != more) {
51 why = "numpy header is truncated";
52 prefix.clear();
53 }
54 }
55 mz_zip_reader_extract_iter_free(iterator);
56 return prefix;
57 }
58
59}
60
61NpzReader::NpzReader() = default;
62
67
69{
71 .name = "npz",
72 .extensions = { "npz" },
73 .probe = {},
74 .make_reader = []() -> std::unique_ptr<TensorReader> { return std::make_unique<NpzReader>(); },
75 .make_writer = []() -> std::unique_ptr<TensorWriter> { return std::make_unique<NpzWriter>(); } });
76
77 MF_INFO(Journal::Component::IO, Journal::Context::Init, "NpzReader registered for: npz");
78}
79
80bool NpzReader::can_read(const std::string& filepath) const
81{
82 return Detail::lower_extension(resolve_path(filepath)) == "npz";
83}
84
86{
87 m_archive.reset();
88 m_entries.clear();
89 m_open = false;
90}
91
92bool NpzReader::open(const std::string& filepath, FileReadOptions)
93{
94 close();
95 m_path = resolve_path(filepath);
96
97 m_archive = std::make_unique<Detail::NpzArchive>();
98 if (!mz_zip_reader_init_file(&m_archive->zip, m_path.c_str(), 0)) {
99 const std::string reason = mz_zip_get_error_string(mz_zip_get_last_error(&m_archive->zip));
100 close();
101 return fail("cannot open archive '" + m_path + "': " + reason);
102 }
103 m_archive->active = true;
104
105 const mz_uint count = mz_zip_reader_get_num_files(&m_archive->zip);
106 for (mz_uint i = 0; i < count; ++i) {
107 mz_zip_archive_file_stat stat {};
108 if (!mz_zip_reader_file_stat(&m_archive->zip, i, &stat) || stat.m_is_directory) {
109 continue;
110 }
111
112 const std::string entry_name = stat.m_filename;
113 if (!entry_name.ends_with(npy_suffix)) {
114 continue;
115 }
116 const std::string name = entry_name.substr(0, entry_name.size() - npy_suffix.size());
117
118 std::string why;
119 const auto prefix = read_entry_prefix(m_archive->zip, i, why);
120 const auto header = prefix.empty() ? std::nullopt : Detail::parse_npy_header(prefix, name, why);
121 if (!header) {
123 "NpzReader: skipping '{}': {}", entry_name, why);
124 continue;
125 }
126
127 const auto bytes = header->spec.dtype.storage_bytes(header->spec.element_count().value_or(0));
128 if (!bytes || header->data_offset + *bytes > stat.m_uncomp_size) {
130 "NpzReader: skipping '{}': data is shorter than its header declares", entry_name);
131 continue;
132 }
133
134 m_entries.push_back({ .index = i, .spec = header->spec, .data_offset = header->data_offset, .big_endian = header->big_endian });
135 }
136
137 m_open = true;
138 m_last_error.clear();
139 return true;
140}
141
142std::vector<Kakshya::TensorSpec> NpzReader::describe() const
143{
144 std::vector<Kakshya::TensorSpec> specs;
145 specs.reserve(m_entries.size());
146 for (const auto& entry : m_entries) {
147 specs.push_back(entry.spec);
148 }
149 return specs;
150}
151
152std::optional<Kakshya::TensorData> NpzReader::read(std::string_view name)
153{
154 if (!m_open) {
155 fail("no file is open");
156 return std::nullopt;
157 }
158
159 const auto entry = std::ranges::find_if(m_entries,
160 [name](const Entry& candidate) { return candidate.spec.name == name; });
161 if (entry == m_entries.end()) {
162 fail("no tensor named '" + std::string(name) + "'");
163 return std::nullopt;
164 }
165
166 size_t size = 0;
167 void* heap = mz_zip_reader_extract_to_heap(&m_archive->zip, entry->index, &size, 0);
168 if (!heap) {
169 fail("cannot extract '" + std::string(name) + "'");
170 return std::nullopt;
171 }
172
173 auto tensor = Kakshya::TensorData::allocate(entry->spec);
174 const auto storage = tensor ? tensor->storage() : std::span<std::byte> {};
175 if (!tensor || entry->data_offset + storage.size() > size) {
176 mz_free(heap);
177 fail("tensor '" + std::string(name) + "' is truncated");
178 return std::nullopt;
179 }
180
181 if (!storage.empty()) {
182 std::memcpy(storage.data(), static_cast<const std::byte*>(heap) + entry->data_offset, storage.size());
183 if (entry->big_endian == Detail::host_is_little_endian()) {
184 Detail::swap_components(storage, Detail::component_bytes(entry->spec.dtype));
185 }
186 }
187 mz_free(heap);
188 return tensor;
189}
190
191bool NpzWriter::write(const std::string& filepath, const Kakshya::TensorSet& tensors)
192{
193 m_last_error.clear();
194 if (!tensors.is_consistent()) {
195 m_last_error = "tensors must be valid with unique, non-empty names";
196 return false;
197 }
198
199 std::vector<std::vector<std::byte>> encoded;
200 encoded.reserve(tensors.tensors.size());
201 for (const auto& tensor : tensors.tensors) {
202 auto bytes = Detail::encode_npy(tensor, m_last_error);
203 if (!bytes) {
204 return false;
205 }
206 encoded.push_back(std::move(*bytes));
207 }
208
209 mz_zip_archive zip {};
210 if (!mz_zip_writer_init_file(&zip, filepath.c_str(), 0)) {
211 m_last_error = "cannot create '" + filepath + "'";
212 return false;
213 }
214
215 for (size_t i = 0; i < encoded.size(); ++i) {
216 const std::string entry = tensors.tensors[i].spec().name + std::string(npy_suffix);
217 if (!mz_zip_writer_add_mem(&zip, entry.c_str(), encoded[i].data(), encoded[i].size(), MZ_NO_COMPRESSION)) {
218 m_last_error = std::string("cannot add '") + entry + "': " + mz_zip_get_error_string(mz_zip_get_last_error(&zip));
219 mz_zip_writer_end(&zip);
220 return false;
221 }
222 }
223
224 const bool finalized = mz_zip_writer_finalize_archive(&zip) != 0;
225 mz_zip_writer_end(&zip);
226 if (!finalized) {
227 m_last_error = "cannot finalize '" + filepath + "'";
228 return false;
229 }
230 return true;
231}
232
233}
#define MF_INFO(comp, ctx,...)
#define MF_WARN(comp, ctx,...)
static std::string resolve_path(const std::string &filepath)
Resolve a filepath against the project source root if not found as-is.
bool open(const std::string &filepath, FileReadOptions options=FileReadOptions::ALL) override
Open a file for reading.
Definition NpzFormat.cpp:92
static void register_with_registry()
Register the format, with its writer, in the TensorFormatRegistry.
Definition NpzFormat.cpp:68
std::vector< Kakshya::TensorSpec > describe() const override
Specs of every tensor in the open file, in file order, without reading data.
void close() override
Close the currently open file.
Definition NpzFormat.cpp:85
std::optional< Kakshya::TensorData > read(std::string_view name) override
Read one tensor by name.
std::vector< Entry > m_entries
Definition NpzFormat.hpp:57
std::unique_ptr< Detail::NpzArchive > m_archive
Definition NpzFormat.hpp:56
bool can_read(const std::string &filepath) const override
Check if a file can be read by this reader.
Definition NpzFormat.cpp:80
bool write(const std::string &filepath, const Kakshya::TensorSet &tensors) override
Write every tensor in the set to a file.
static TensorFormatRegistry & instance()
The process-wide registry.
void register_format(TensorFormat format)
Add a format, replacing any earlier one with the same name.
bool fail(std::string reason)
Record a failure reason and return false.
static std::optional< TensorData > allocate(TensorSpec spec)
Zero-filled tensor for a spec.
Definition Tensor.cpp:72
std::string lower_extension(const std::string &path)
Lowercase extension of a path without its dot.
std::optional< NpyHeader > parse_npy_header(std::span< const std::byte > prefix, std::string_view name, std::string &why)
Decode a .npy prefix into a tensor spec.
Definition NpyCodec.cpp:137
size_t component_bytes(Kakshya::TensorDType dtype) noexcept
Bytes of one independently ordered component of an element.
std::optional< std::vector< std::byte > > encode_npy(const Kakshya::TensorData &tensor, std::string &why)
Encode a tensor as a complete .npy array in dense C order.
Definition NpyCodec.cpp:201
std::optional< size_t > npy_prefix_size(std::span< const std::byte > leading, std::string &why)
Total prefix size (magic, version, length field and header) of a .npy array.
Definition NpyCodec.cpp:107
void swap_components(std::span< std::byte > data, size_t component_bytes) noexcept
Reverse the byte order of every component in place.
bool host_is_little_endian() noexcept
Whether this machine stores multi-byte values least significant byte first.
FileReadOptions
Generic options for file reading behavior.
@ FileIO
Filesystem I/O operations.
@ Init
Engine/subsystem initialization.
@ IO
Networking, file handling, streaming.
Kakshya::TensorSpec spec
Definition NpzFormat.hpp:51
Everything the registry needs to know about one tensor file format.
std::vector< TensorData > tensors
Definition Tensor.hpp:168
bool is_consistent() const
Whether every tensor is valid and named, with no duplicate names.
Definition Tensor.cpp:194
Named tensors that belong together, such as a model's weights or I/O.
Definition Tensor.hpp:167