MayaFlux 1.0.0
Digital-First Multimedia Processing Framework
Loading...
Searching...
No Matches
SafetensorsFormat.hpp
Go to the documentation of this file.
1#pragma once
2
3#include "TensorReader.hpp"
4#include "TensorWriter.hpp"
5
6#include <fstream>
7
8namespace MayaFlux::IO {
9
10/**
11 * @class SafetensorsReader
12 * @brief Reader for the safetensors format: a JSON index followed by packed tensor bytes.
13 *
14 * open() parses the index only. Each tensor is then read with one seek, at its
15 * native element type, including bfloat16, float8 and sub-byte types. An index
16 * entry with an element type this build does not know is skipped with a
17 * warning; every other tensor stays readable.
18 *
19 * Usage:
20 * @code
21 * SafetensorsReader reader;
22 * reader.open("model.safetensors");
23 * auto bias = reader.read("layer.bias");
24 * @endcode
25 */
26class MAYAFLUX_API SafetensorsReader : public TensorReader {
27public:
28 SafetensorsReader() = default;
29 ~SafetensorsReader() override { close(); }
30
31 /** @brief Register the format, with its writer, in the TensorFormatRegistry. */
32 static void register_with_registry();
33
34 /** @brief Whether leading bytes and file size look like a safetensors header. */
35 [[nodiscard]] static bool probe(std::span<const std::byte> head, uint64_t file_size);
36
37 [[nodiscard]] bool can_read(const std::string& filepath) const override;
38 bool open(const std::string& filepath, FileReadOptions options = FileReadOptions::ALL) override;
39 void close() override;
40 [[nodiscard]] bool is_open() const override { return m_open; }
41 [[nodiscard]] std::vector<std::string> get_supported_extensions() const override { return { "safetensors" }; }
42
43 [[nodiscard]] std::string format_name() const override { return "safetensors"; }
44 [[nodiscard]] std::vector<Kakshya::TensorSpec> describe() const override;
45 [[nodiscard]] std::map<std::string, std::string> tensor_metadata() const override { return m_metadata; }
46 [[nodiscard]] std::optional<Kakshya::TensorData> read(std::string_view name) override;
47
48private:
49 struct Entry {
51 uint64_t begin {};
52 uint64_t end {};
53 };
54
55 bool reject(std::string reason);
56
57 std::ifstream m_file;
58 std::vector<Entry> m_entries;
59 std::map<std::string, std::string> m_metadata;
60 uint64_t m_data_start {};
61 bool m_open {};
62};
63
64/**
65 * @class SafetensorsWriter
66 * @brief Writer for the safetensors format.
67 *
68 * Orders tensors wider elements first, then by name, so every tensor stays
69 * aligned to its element size. Pads the index to eight bytes and stores the
70 * set's string metadata. Element types safetensors has no name for are refused.
71 */
72class MAYAFLUX_API SafetensorsWriter : public TensorWriter {
73public:
74 bool write(const std::string& filepath, const Kakshya::TensorSet& tensors) override;
75 [[nodiscard]] std::string get_last_error() const override { return m_last_error; }
76 [[nodiscard]] std::vector<std::string> get_supported_extensions() const override { return { "safetensors" }; }
77
78private:
79 std::string m_last_error;
80};
81
82}
void close() override
Close the currently open file.
std::map< std::string, std::string > tensor_metadata() const override
Free-form string metadata stored in the file, empty if the format has none.
bool is_open() const override
Check if a file is currently open.
std::string format_name() const override
Format identifier such as safetensors or npy.
std::vector< std::string > get_supported_extensions() const override
Get supported file extensions for this reader.
std::map< std::string, std::string > m_metadata
std::vector< std::string > get_supported_extensions() const override
Extensions this format writes, without dots.
std::string get_last_error() const override
Reason the last write failed, or empty.
bool write(const std::string &filepath, const Kakshya::TensorSet &tensors) override
Write every tensor in the set to a file.
Writer for the safetensors format.
Typed FileReader boundary for formats that store named tensors.
Single-shot writer of a TensorSet to one file format.
FileReadOptions
Generic options for file reading behavior.
@ ALL
All options enabled.
Named tensors that belong together, such as a model's weights or I/O.
Definition Tensor.hpp:167
Complete description of a tensor: type, shape, layout, meaning.