16 mz_zip_archive
zip {};
22 mz_zip_reader_end(&
zip);
31 constexpr std::string_view npy_suffix =
".npy";
33 std::vector<std::byte> read_entry_prefix(mz_zip_archive& zip, uint32_t index, std::string& why)
35 auto* iterator = mz_zip_reader_extract_iter_new(&zip, index, 0);
37 why =
"cannot open archive entry";
41 std::array<std::byte, 12> leading {};
42 const size_t got = mz_zip_reader_extract_iter_read(iterator, leading.data(), leading.size());
44 std::vector<std::byte> prefix;
46 prefix.resize(*total);
47 const size_t have = std::min(got, *total);
48 std::memcpy(prefix.data(), leading.data(), have);
49 const size_t more = *total - have;
50 if (more > 0 && mz_zip_reader_extract_iter_read(iterator, prefix.data() + have, more) != more) {
51 why =
"numpy header is truncated";
55 mz_zip_reader_extract_iter_free(iterator);
72 .extensions = {
"npz" },
74 .make_reader = []() -> std::unique_ptr<TensorReader> {
return std::make_unique<NpzReader>(); },
75 .make_writer = []() -> std::unique_ptr<TensorWriter> {
return std::make_unique<NpzWriter>(); } });
97 m_archive = std::make_unique<Detail::NpzArchive>();
99 const std::string reason = mz_zip_get_error_string(mz_zip_get_last_error(&
m_archive->zip));
101 return fail(
"cannot open archive '" +
m_path +
"': " + reason);
105 const mz_uint count = mz_zip_reader_get_num_files(&
m_archive->zip);
106 for (mz_uint i = 0; i < count; ++i) {
107 mz_zip_archive_file_stat stat {};
108 if (!mz_zip_reader_file_stat(&
m_archive->zip, i, &stat) || stat.m_is_directory) {
112 const std::string entry_name = stat.m_filename;
113 if (!entry_name.ends_with(npy_suffix)) {
116 const std::string name = entry_name.substr(0, entry_name.size() - npy_suffix.size());
119 const auto prefix = read_entry_prefix(
m_archive->zip, i, why);
123 "NpzReader: skipping '{}': {}", entry_name, why);
127 const auto bytes = header->spec.dtype.storage_bytes(header->spec.element_count().value_or(0));
128 if (!bytes || header->data_offset + *bytes > stat.m_uncomp_size) {
130 "NpzReader: skipping '{}': data is shorter than its header declares", entry_name);
134 m_entries.push_back({ .index = i, .spec = header->spec, .data_offset = header->data_offset, .big_endian = header->big_endian });
144 std::vector<Kakshya::TensorSpec> specs;
147 specs.push_back(entry.spec);
155 fail(
"no file is open");
159 const auto entry = std::ranges::find_if(
m_entries,
160 [name](
const Entry& candidate) {
return candidate.
spec.
name == name; });
162 fail(
"no tensor named '" + std::string(name) +
"'");
167 void* heap = mz_zip_reader_extract_to_heap(&
m_archive->zip, entry->index, &size, 0);
169 fail(
"cannot extract '" + std::string(name) +
"'");
174 const auto storage = tensor ? tensor->storage() : std::span<std::byte> {};
175 if (!tensor || entry->data_offset + storage.size() > size) {
177 fail(
"tensor '" + std::string(name) +
"' is truncated");
181 if (!storage.empty()) {
182 std::memcpy(storage.data(),
static_cast<const std::byte*
>(heap) + entry->data_offset, storage.size());
195 m_last_error =
"tensors must be valid with unique, non-empty names";
199 std::vector<std::vector<std::byte>> encoded;
200 encoded.reserve(tensors.
tensors.size());
201 for (
const auto& tensor : tensors.
tensors) {
206 encoded.push_back(std::move(*bytes));
209 mz_zip_archive zip {};
210 if (!mz_zip_writer_init_file(&zip, filepath.c_str(), 0)) {
215 for (
size_t i = 0; i < encoded.size(); ++i) {
216 const std::string entry = tensors.
tensors[i].spec().name + std::string(npy_suffix);
217 if (!mz_zip_writer_add_mem(&zip, entry.c_str(), encoded[i].data(), encoded[i].size(), MZ_NO_COMPRESSION)) {
218 m_last_error = std::string(
"cannot add '") + entry +
"': " + mz_zip_get_error_string(mz_zip_get_last_error(&zip));
219 mz_zip_writer_end(&zip);
224 const bool finalized = mz_zip_writer_finalize_archive(&zip) != 0;
225 mz_zip_writer_end(&zip);
#define MF_INFO(comp, ctx,...)
#define MF_WARN(comp, ctx,...)
static std::string resolve_path(const std::string &filepath)
Resolve a filepath against the project source root if not found as-is.
bool open(const std::string &filepath, FileReadOptions options=FileReadOptions::ALL) override
Open a file for reading.
static void register_with_registry()
Register the format, with its writer, in the TensorFormatRegistry.
std::vector< Kakshya::TensorSpec > describe() const override
Specs of every tensor in the open file, in file order, without reading data.
void close() override
Close the currently open file.
std::optional< Kakshya::TensorData > read(std::string_view name) override
Read one tensor by name.
std::vector< Entry > m_entries
std::unique_ptr< Detail::NpzArchive > m_archive
bool can_read(const std::string &filepath) const override
Check if a file can be read by this reader.
bool write(const std::string &filepath, const Kakshya::TensorSet &tensors) override
Write every tensor in the set to a file.
bool fail(std::string reason)
Record a failure reason and return false.
static std::optional< TensorData > allocate(TensorSpec spec)
Zero-filled tensor for a spec.
std::string lower_extension(const std::string &path)
Lowercase extension of a path without its dot.
std::optional< NpyHeader > parse_npy_header(std::span< const std::byte > prefix, std::string_view name, std::string &why)
Decode a .npy prefix into a tensor spec.
size_t component_bytes(Kakshya::TensorDType dtype) noexcept
Bytes of one independently ordered component of an element.
std::optional< std::vector< std::byte > > encode_npy(const Kakshya::TensorData &tensor, std::string &why)
Encode a tensor as a complete .npy array in dense C order.
std::optional< size_t > npy_prefix_size(std::span< const std::byte > leading, std::string &why)
Total prefix size (magic, version, length field and header) of a .npy array.
void swap_components(std::span< std::byte > data, size_t component_bytes) noexcept
Reverse the byte order of every component in place.
bool host_is_little_endian() noexcept
Whether this machine stores multi-byte values least significant byte first.
FileReadOptions
Generic options for file reading behavior.
@ FileIO
Filesystem I/O operations.
@ Init
Engine/subsystem initialization.
@ IO
Networking, file handling, streaming.
std::vector< TensorData > tensors
bool is_consistent() const
Whether every tensor is valid and named, with no duplicate names.
Named tensors that belong together, such as a model's weights or I/O.