MayaFlux 1.0.0
Digital-First Multimedia Processing Framework
Loading...
Searching...
No Matches
SafetensorsFormat.cpp
Go to the documentation of this file.
2
4#include "TensorFormat.hpp"
5
7
8#include <nlohmann/json.hpp>
9
10namespace MayaFlux::IO {
11
12namespace {
13
14 constexpr uint64_t max_header_bytes = 100'000'000;
15
16 uint64_t read_le64(const std::byte* bytes)
17 {
18 uint64_t value = 0;
19 for (size_t i = 0; i < 8; ++i)
20 value |= static_cast<uint64_t>(std::to_integer<uint8_t>(bytes[i])) << (8 * i);
21 return value;
22 }
23
24 void write_le64(std::string& out, uint64_t value)
25 {
26 for (size_t i = 0; i < 8; ++i)
27 out.push_back(static_cast<char>((value >> (8 * i)) & 0xFF));
28 }
29
30}
31
33{
35 .name = "safetensors",
36 .extensions = { "safetensors" },
38 .make_reader = []() -> std::unique_ptr<TensorReader> { return std::make_unique<SafetensorsReader>(); },
39 .make_writer = []() -> std::unique_ptr<TensorWriter> { return std::make_unique<SafetensorsWriter>(); } });
40
42 "SafetensorsReader registered for: safetensors");
43}
44
45bool SafetensorsReader::probe(std::span<const std::byte> head, uint64_t file_size)
46{
47 if (head.size() < 9 || file_size < 9)
48 return false;
49 const uint64_t header_size = read_le64(head.data());
50 return header_size > 0 && header_size <= max_header_bytes && header_size <= file_size - 8
51 && head[8] == std::byte { '{' };
52}
53
54bool SafetensorsReader::can_read(const std::string& filepath) const
55{
56 const auto path = resolve_path(filepath);
57 if (Detail::lower_extension(path) == "safetensors")
58 return true;
59 const auto head = Detail::read_file_head(path, 16);
60 return head && probe(head->bytes, head->file_size);
61}
62
63bool SafetensorsReader::reject(std::string reason)
64{
65 close();
66 return fail(std::move(reason));
67}
68
70{
71 if (m_file.is_open())
72 m_file.close();
73 m_entries.clear();
74 m_metadata.clear();
75 m_data_start = 0;
76 m_open = false;
77}
78
79bool SafetensorsReader::open(const std::string& filepath, FileReadOptions)
80{
81 close();
82 m_path = resolve_path(filepath);
83 m_file.open(m_path, std::ios::binary);
84 if (!m_file)
85 return reject("cannot open '" + m_path + "'");
86
87 m_file.seekg(0, std::ios::end);
88 const auto file_size = static_cast<uint64_t>(m_file.tellg());
89 m_file.seekg(0);
90
91 std::array<std::byte, 8> prefix {};
92 if (!m_file.read(reinterpret_cast<char*>(prefix.data()), 8))
93 return reject("file is shorter than a safetensors prefix");
94
95 const uint64_t header_size = read_le64(prefix.data());
96 if (header_size == 0 || header_size > max_header_bytes || header_size > file_size - 8)
97 return reject("safetensors header size " + std::to_string(header_size) + " is invalid");
98
99 std::string text(static_cast<size_t>(header_size), '\0');
100 if (!m_file.read(text.data(), static_cast<std::streamsize>(header_size)))
101 return reject("safetensors header is truncated");
102
103 const auto index = nlohmann::ordered_json::parse(text, nullptr, false);
104 if (index.is_discarded() || !index.is_object())
105 return reject("safetensors header is not a JSON object");
106
107 const uint64_t data_size = file_size - 8 - header_size;
108
109 for (const auto& [key, value] : index.items()) {
110 if (key == "__metadata__") {
111 if (!value.is_object())
112 return reject("__metadata__ must be an object");
113 for (const auto& [meta_key, meta_value] : value.items()) {
114 if (!meta_value.is_string())
115 return reject("__metadata__ value for '" + meta_key + "' is not a string");
116 m_metadata[meta_key] = meta_value.get<std::string>();
117 }
118 continue;
119 }
120
121 if (!value.is_object())
122 return reject("tensor '" + key + "' is not an object");
123
124 const auto dtype_it = value.find("dtype");
125 const auto shape_it = value.find("shape");
126 const auto offsets_it = value.find("data_offsets");
127 if (dtype_it == value.end() || !dtype_it->is_string() || shape_it == value.end() || !shape_it->is_array()
128 || offsets_it == value.end() || !offsets_it->is_array() || offsets_it->size() != 2)
129 return reject("tensor '" + key + "' lacks dtype, shape or data_offsets");
130
131 const auto dtype = Detail::safetensors_dtype(dtype_it->get<std::string>());
132 if (!dtype) {
134 "SafetensorsReader: skipping '{}' with unsupported dtype '{}'",
135 key, dtype_it->get<std::string>());
136 continue;
137 }
138
139 std::vector<uint64_t> shape;
140 for (const auto& extent : *shape_it) {
141 if (!extent.is_number_unsigned())
142 return reject("tensor '" + key + "' has a non-integer extent");
143 shape.push_back(extent.get<uint64_t>());
144 }
145 if (!(*offsets_it)[0].is_number_unsigned() || !(*offsets_it)[1].is_number_unsigned())
146 return reject("tensor '" + key + "' has non-integer data_offsets");
147
148 Entry entry;
149 entry.begin = (*offsets_it)[0].get<uint64_t>();
150 entry.end = (*offsets_it)[1].get<uint64_t>();
151 entry.spec = Kakshya::TensorSpec::contiguous(*dtype, std::move(shape), key);
152
153 if (const auto why = entry.spec.diagnose(); !why.empty())
154 return reject(why);
155
156 const auto bytes = dtype->storage_bytes(entry.spec.element_count().value_or(0));
157 if (!bytes || entry.end < entry.begin || entry.end > data_size || entry.end - entry.begin != *bytes)
158 return reject("tensor '" + key + "' has inconsistent data_offsets");
159
160 m_entries.push_back(std::move(entry));
161 }
162
163 m_data_start = 8 + header_size;
164 m_open = true;
165 m_last_error.clear();
166 return true;
167}
168
169std::vector<Kakshya::TensorSpec> SafetensorsReader::describe() const
170{
171 std::vector<Kakshya::TensorSpec> specs;
172 specs.reserve(m_entries.size());
173 for (const auto& entry : m_entries)
174 specs.push_back(entry.spec);
175 return specs;
176}
177
178std::optional<Kakshya::TensorData> SafetensorsReader::read(std::string_view name)
179{
180 if (!m_open) {
181 fail("no file is open");
182 return std::nullopt;
183 }
184
185 const auto entry = std::ranges::find_if(m_entries,
186 [name](const Entry& candidate) { return candidate.spec.name == name; });
187 if (entry == m_entries.end()) {
188 fail("no tensor named '" + std::string(name) + "'");
189 return std::nullopt;
190 }
191
192 auto tensor = Kakshya::TensorData::allocate(entry->spec);
193 if (!tensor) {
194 fail("cannot allocate tensor '" + std::string(name) + "'");
195 return std::nullopt;
196 }
197
198 const auto storage = tensor->storage();
199 if (!storage.empty()) {
200 m_file.clear();
201 m_file.seekg(static_cast<std::streamoff>(m_data_start + entry->begin));
202 if (!m_file.read(reinterpret_cast<char*>(storage.data()), static_cast<std::streamsize>(storage.size()))) {
203 fail("tensor '" + std::string(name) + "' is truncated");
204 return std::nullopt;
205 }
207 Detail::swap_components(storage, Detail::component_bytes(entry->spec.dtype));
208 }
209 return tensor;
210}
211
212bool SafetensorsWriter::write(const std::string& filepath, const Kakshya::TensorSet& tensors)
213{
214 m_last_error.clear();
215 if (!tensors.is_consistent()) {
216 m_last_error = "tensors must be valid with unique, non-empty names";
217 return false;
218 }
219
220 nlohmann::ordered_json index = nlohmann::ordered_json::object();
221 if (!tensors.metadata.empty()) {
222 auto meta = nlohmann::ordered_json::object();
223 for (const auto& [key, value] : tensors.metadata)
224 meta[key] = value;
225 index["__metadata__"] = std::move(meta);
226 }
227
228 std::vector<size_t> order(tensors.tensors.size());
229 std::iota(order.begin(), order.end(), size_t { 0 });
230 std::ranges::sort(order, [&tensors](size_t a, size_t b) {
231 const auto& left = tensors.tensors[a].spec();
232 const auto& right = tensors.tensors[b].spec();
233 const auto left_width = std::max<size_t>(1, left.dtype.element_bytes());
234 const auto right_width = std::max<size_t>(1, right.dtype.element_bytes());
235 return left_width != right_width ? left_width > right_width : left.name < right.name;
236 });
237
238 std::vector<std::optional<Kakshya::TensorData>> gathered(tensors.tensors.size());
239 std::vector<std::span<const std::byte>> payloads;
240 payloads.reserve(tensors.tensors.size());
241
242 uint64_t offset = 0;
243 for (size_t i = 0; i < order.size(); ++i) {
244 const auto& tensor = tensors.tensors[order[i]];
245 const auto& spec = tensor.spec();
246
247 const auto name = Detail::safetensors_name(spec.dtype);
248 if (!name) {
249 m_last_error = "tensor '" + spec.name + "' has element type " + spec.dtype.name()
250 + " which safetensors cannot store";
251 return false;
252 }
253 const auto bytes = Detail::dense_bytes(tensor, gathered[i]);
254 if (!bytes) {
255 m_last_error = "tensor '" + spec.name + "' could not be read";
256 return false;
257 }
258
259 index[spec.name] = {
260 { "dtype", std::string(*name) },
261 { "shape", spec.shape },
262 { "data_offsets", std::vector<uint64_t> { offset, offset + bytes->size() } }
263 };
264 offset += bytes->size();
265 payloads.push_back(*bytes);
266 }
267
268 std::string header = index.dump();
269 header.append((8 - header.size() % 8) % 8, ' ');
270
271 std::string prefix;
272 write_le64(prefix, header.size());
273
274 std::ofstream file(filepath, std::ios::binary | std::ios::trunc);
275 if (!file) {
276 m_last_error = "cannot create '" + filepath + "'";
277 return false;
278 }
279 file.write(prefix.data(), static_cast<std::streamsize>(prefix.size()));
280 file.write(header.data(), static_cast<std::streamsize>(header.size()));
281
282 std::vector<std::byte> swapped;
283 for (size_t i = 0; i < payloads.size(); ++i) {
284 std::span<const std::byte> payload = payloads[i];
286 swapped.assign(payload.begin(), payload.end());
287 Detail::swap_components(swapped, Detail::component_bytes(tensors.tensors[order[i]].spec().dtype));
288 payload = swapped;
289 }
290 file.write(reinterpret_cast<const char*>(payload.data()), static_cast<std::streamsize>(payload.size()));
291 }
292
293 file.close();
294 if (!file) {
295 m_last_error = "failed while writing '" + filepath + "'";
296 return false;
297 }
298 return true;
299}
300
301}
#define MF_INFO(comp, ctx,...)
#define MF_WARN(comp, ctx,...)
static std::string resolve_path(const std::string &filepath)
Resolve a filepath against the project source root if not found as-is.
bool open(const std::string &filepath, FileReadOptions options=FileReadOptions::ALL) override
Open a file for reading.
bool reject(std::string reason)
void close() override
Close the currently open file.
static void register_with_registry()
Register the format, with its writer, in the TensorFormatRegistry.
std::vector< Kakshya::TensorSpec > describe() const override
Specs of every tensor in the open file, in file order, without reading data.
static bool probe(std::span< const std::byte > head, uint64_t file_size)
Whether leading bytes and file size look like a safetensors header.
std::optional< Kakshya::TensorData > read(std::string_view name) override
Read one tensor by name.
bool can_read(const std::string &filepath) const override
Check if a file can be read by this reader.
std::map< std::string, std::string > m_metadata
bool write(const std::string &filepath, const Kakshya::TensorSet &tensors) override
Write every tensor in the set to a file.
static TensorFormatRegistry & instance()
The process-wide registry.
void register_format(TensorFormat format)
Add a format, replacing any earlier one with the same name.
bool fail(std::string reason)
Record a failure reason and return false.
static std::optional< TensorData > allocate(TensorSpec spec)
Zero-filled tensor for a spec.
Definition Tensor.cpp:72
std::string lower_extension(const std::string &path)
Lowercase extension of a path without its dot.
size_t component_bytes(Kakshya::TensorDType dtype) noexcept
Bytes of one independently ordered component of an element.
std::optional< Kakshya::TensorDType > safetensors_dtype(std::string_view name) noexcept
Element type for a safetensors dtype name such as F32 or BF16.
std::optional< std::span< const std::byte > > dense_bytes(const Kakshya::TensorData &tensor, std::optional< Kakshya::TensorData > &dense)
Elements of a tensor as dense C-order bytes.
void swap_components(std::span< std::byte > data, size_t component_bytes) noexcept
Reverse the byte order of every component in place.
std::optional< std::string_view > safetensors_name(Kakshya::TensorDType dtype) noexcept
Safetensors dtype name for an element type, when the format has one.
bool host_is_little_endian() noexcept
Whether this machine stores multi-byte values least significant byte first.
std::optional< FileHead > read_file_head(const std::string &path, size_t limit)
Read up to limit leading bytes of a file, or std::nullopt if it cannot be opened.
FileReadOptions
Generic options for file reading behavior.
@ FileIO
Filesystem I/O operations.
@ Init
Engine/subsystem initialization.
@ IO
Networking, file handling, streaming.
Everything the registry needs to know about one tensor file format.
std::vector< TensorData > tensors
Definition Tensor.hpp:168
std::map< std::string, std::string > metadata
Definition Tensor.hpp:169
bool is_consistent() const
Whether every tensor is valid and named, with no duplicate names.
Definition Tensor.cpp:194
Named tensors that belong together, such as a model's weights or I/O.
Definition Tensor.hpp:167
std::optional< uint64_t > element_count() const noexcept
Product of the shape, or std::nullopt on overflow.
std::string diagnose() const
Empty when the spec is self-consistent, otherwise the reason it is not.
static TensorSpec contiguous(TensorDType dtype, std::vector< uint64_t > shape, std::string name={})
C-ordered spec with no offset, axes or quantization.