MayaFlux 1.0.0
Digital-First Multimedia Processing Framework
Loading...
Searching...
No Matches
NpzFormat.hpp
Go to the documentation of this file.
1#pragma once
2
3#include "TensorReader.hpp"
4#include "TensorWriter.hpp"
5
6namespace MayaFlux::IO {
7
8namespace Detail {
9 struct NpzArchive;
10}
11
12/**
13 * @class NpzReader
14 * @brief Reader for NumPy .npz archives: a zip file of .npy arrays.
15 *
16 * Each archive entry ending in .npy is one tensor named after the entry.
17 * open() reads only each entry's header, so describe() is cheap even for a
18 * compressed archive; read() decompresses a single entry. Entries numpy
19 * cannot describe to this reader are skipped with a warning.
20 *
21 * Usage:
22 * @code
23 * NpzReader reader;
24 * reader.open("dataset.npz");
25 * auto labels = reader.read("labels");
26 * @endcode
27 */
28class MAYAFLUX_API NpzReader : public TensorReader {
29public:
31 NpzReader(const NpzReader&) = delete;
32 NpzReader& operator=(const NpzReader&) = delete;
33 ~NpzReader() override;
34
35 /** @brief Register the format, with its writer, in the TensorFormatRegistry. */
36 static void register_with_registry();
37
38 [[nodiscard]] bool can_read(const std::string& filepath) const override;
39 bool open(const std::string& filepath, FileReadOptions options = FileReadOptions::ALL) override;
40 void close() override;
41 [[nodiscard]] bool is_open() const override { return m_open; }
42 [[nodiscard]] std::vector<std::string> get_supported_extensions() const override { return { "npz" }; }
43
44 [[nodiscard]] std::string format_name() const override { return "npz"; }
45 [[nodiscard]] std::vector<Kakshya::TensorSpec> describe() const override;
46 [[nodiscard]] std::optional<Kakshya::TensorData> read(std::string_view name) override;
47
48private:
49 struct Entry {
50 uint32_t index {};
52 size_t data_offset {};
53 bool big_endian {};
54 };
55
56 std::unique_ptr<Detail::NpzArchive> m_archive;
57 std::vector<Entry> m_entries;
58 bool m_open {};
59};
60
61/**
62 * @class NpzWriter
63 * @brief Writer for NumPy .npz archives, one stored (uncompressed) .npy entry per tensor.
64 *
65 * Entry names are the tensor names. A tensor numpy has no element type for is
66 * refused, and so is the whole set.
67 */
68class MAYAFLUX_API NpzWriter : public TensorWriter {
69public:
70 bool write(const std::string& filepath, const Kakshya::TensorSet& tensors) override;
71 [[nodiscard]] std::string get_last_error() const override { return m_last_error; }
72 [[nodiscard]] std::vector<std::string> get_supported_extensions() const override { return { "npz" }; }
73
74private:
75 std::string m_last_error;
76};
77
78}
bool open(const std::string &filepath, FileReadOptions options=FileReadOptions::ALL) override
Open a file for reading.
Definition NpzFormat.cpp:92
static void register_with_registry()
Register the format, with its writer, in the TensorFormatRegistry.
Definition NpzFormat.cpp:68
void close() override
Close the currently open file.
Definition NpzFormat.cpp:85
std::vector< Entry > m_entries
Definition NpzFormat.hpp:57
bool is_open() const override
Check if a file is currently open.
Definition NpzFormat.hpp:41
NpzReader(const NpzReader &)=delete
std::unique_ptr< Detail::NpzArchive > m_archive
Definition NpzFormat.hpp:56
std::string format_name() const override
Format identifier such as safetensors or npy.
Definition NpzFormat.hpp:44
bool can_read(const std::string &filepath) const override
Check if a file can be read by this reader.
Definition NpzFormat.cpp:80
NpzReader & operator=(const NpzReader &)=delete
std::vector< std::string > get_supported_extensions() const override
Get supported file extensions for this reader.
Definition NpzFormat.hpp:42
bool write(const std::string &filepath, const Kakshya::TensorSet &tensors) override
Write every tensor in the set to a file.
std::vector< std::string > get_supported_extensions() const override
Extensions this format writes, without dots.
Definition NpzFormat.hpp:72
std::string get_last_error() const override
Reason the last write failed, or empty.
Definition NpzFormat.hpp:71
Writer for NumPy .npz archives, one stored (uncompressed) .npy entry per tensor.
Definition NpzFormat.hpp:68
Typed FileReader boundary for formats that store named tensors.
Single-shot writer of a TensorSet to one file format.
FileReadOptions
Generic options for file reading behavior.
@ ALL
All options enabled.
Kakshya::TensorSpec spec
Definition NpzFormat.hpp:51
Named tensors that belong together, such as a model's weights or I/O.
Definition Tensor.hpp:167
Complete description of a tensor: type, shape, layout, meaning.