MayaFlux 1.0.0
Digital-First Multimedia Processing Framework
Loading...
Searching...
No Matches
TensorReader.hpp
Go to the documentation of this file.
1#pragma once
2
5
6namespace MayaFlux::IO {
7
8/**
9 * @struct TensorReadOptions
10 * @brief Selection for TensorReader::read_tensors().
11 */
13 /** @brief Tensors to read, in this order. Empty reads every tensor in file order. */
14 std::vector<std::string> names;
15};
16
17/**
18 * @class TensorReader
19 * @brief Typed FileReader boundary for formats that store named tensors.
20 *
21 * A concrete reader parses only the file's index on open(), so describe() and
22 * tensor_metadata() answer without reading tensor data. read() then loads one
23 * tensor at its native element type, with byte order converted to the host's,
24 * so a large file can be inspected and read selectively. The inherited
25 * DataVariant and container methods are unsupported.
26 *
27 * A new format derives from this class, registers a TensorFormat with the
28 * TensorFormatRegistry, and nothing else changes.
29 *
30 * Usage:
31 * @code
32 * auto reader = io.open_tensor_reader("model.safetensors");
33 * for (const auto& spec : reader->describe()) { }
34 * auto weight = reader->read("encoder.0.weight");
35 * auto everything = reader->read_tensors();
36 * @endcode
37 */
38class MAYAFLUX_API TensorReader : public FileReader {
39public:
40 ~TensorReader() override = default;
41
42 /** @brief Format identifier such as safetensors or npy. */
43 [[nodiscard]] virtual std::string format_name() const = 0;
44
45 /** @brief Specs of every tensor in the open file, in file order, without reading data. */
46 [[nodiscard]] virtual std::vector<Kakshya::TensorSpec> describe() const = 0;
47
48 /** @brief Free-form string metadata stored in the file, empty if the format has none. */
49 [[nodiscard]] virtual std::map<std::string, std::string> tensor_metadata() const { return {}; }
50
51 /**
52 * @brief Read one tensor by name.
53 * @return The tensor, or std::nullopt with get_last_error() set.
54 */
55 [[nodiscard]] virtual std::optional<Kakshya::TensorData> read(std::string_view name) = 0;
56
57 /**
58 * @brief Read a selection of tensors together with the file's metadata.
59 * @return The set, or std::nullopt if any requested tensor is missing or unreadable.
60 */
61 [[nodiscard]] std::optional<Kakshya::TensorSet> read_tensors(const TensorReadOptions& options = {});
62
63 /** @brief Spec of one tensor by name, without reading data. */
64 [[nodiscard]] std::optional<Kakshya::TensorSpec> find_spec(std::string_view name) const;
65
66 [[nodiscard]] std::optional<FileMetadata> get_metadata() const override;
67 [[nodiscard]] std::vector<FileRegion> get_regions() const override { return {}; }
68 std::vector<Kakshya::DataVariant> read_all() override;
69 std::vector<Kakshya::DataVariant> read_region(const FileRegion&) override;
70 std::shared_ptr<Kakshya::SignalSourceContainer> create_container() override;
71 bool load_into_container(std::shared_ptr<Kakshya::SignalSourceContainer>) override;
72 [[nodiscard]] std::vector<uint64_t> get_read_position() const override { return {}; }
73 bool seek(const std::vector<uint64_t>&) override { return false; }
74 [[nodiscard]] std::type_index get_data_type() const override;
75 [[nodiscard]] std::type_index get_container_type() const override;
76 [[nodiscard]] std::string get_last_error() const override { return m_last_error; }
77 [[nodiscard]] bool supports_streaming() const override { return false; }
78 [[nodiscard]] uint64_t get_preferred_chunk_size() const override { return 0; }
79 [[nodiscard]] size_t get_num_dimensions() const override { return 0; }
80 [[nodiscard]] std::vector<uint64_t> get_dimension_sizes() const override { return {}; }
81
82protected:
83 /** @brief Record a failure reason and return false. */
84 bool fail(std::string reason)
85 {
86 m_last_error = std::move(reason);
87 return false;
88 }
89
90 std::string m_last_error;
91 std::string m_path;
92};
93
94}
Abstract interface for reading various file formats into containers.
std::string get_last_error() const override
Get the last error message.
std::vector< FileRegion > get_regions() const override
Get semantic regions from the file.
uint64_t get_preferred_chunk_size() const override
Get the preferred chunk size for streaming.
std::vector< uint64_t > get_read_position() const override
Get current read position in primary dimension.
virtual std::map< std::string, std::string > tensor_metadata() const
Free-form string metadata stored in the file, empty if the format has none.
virtual std::vector< Kakshya::TensorSpec > describe() const =0
Specs of every tensor in the open file, in file order, without reading data.
bool fail(std::string reason)
Record a failure reason and return false.
virtual std::optional< Kakshya::TensorData > read(std::string_view name)=0
Read one tensor by name.
bool supports_streaming() const override
Check if streaming is supported for the current file.
~TensorReader() override=default
virtual std::string format_name() const =0
Format identifier such as safetensors or npy.
std::optional< Kakshya::TensorSet > read_tensors(const TensorReadOptions &options={})
Read a selection of tensors together with the file's metadata.
std::vector< uint64_t > get_dimension_sizes() const override
Get size of each dimension in the file data.
bool seek(const std::vector< uint64_t > &) override
Seek to a specific position in the file.
size_t get_num_dimensions() const override
Get the dimensionality of the file data.
Typed FileReader boundary for formats that store named tensors.
auto create_container(Args &&... args) -> std::shared_ptr< ContainerType >
creates a new container of the specified type
Definition Depot.hpp:67
Generic region descriptor for any file type.
std::vector< std::string > names
Tensors to read, in this order.
Selection for TensorReader::read_tensors().