MayaFlux 1.0.0
Digital-First Multimedia Processing Framework
Loading...
Searching...
No Matches

◆ write()

bool MayaFlux::IO::SafetensorsWriter::write ( const std::string & filepath,
const Kakshya::TensorSet & tensors )
overridevirtual

Write every tensor in the set to a file.

Parameters
filepathDestination path.
tensorsTensors with unique, non-empty names.
Returns
True only if the complete file was written.

Implements MayaFlux::IO::TensorWriter.

Definition at line 212 of file SafetensorsFormat.cpp.

213{
214 m_last_error.clear();
215 if (!tensors.is_consistent()) {
216 m_last_error = "tensors must be valid with unique, non-empty names";
217 return false;
218 }
219
220 nlohmann::ordered_json index = nlohmann::ordered_json::object();
221 if (!tensors.metadata.empty()) {
222 auto meta = nlohmann::ordered_json::object();
223 for (const auto& [key, value] : tensors.metadata)
224 meta[key] = value;
225 index["__metadata__"] = std::move(meta);
226 }
227
228 std::vector<size_t> order(tensors.tensors.size());
229 std::iota(order.begin(), order.end(), size_t { 0 });
230 std::ranges::sort(order, [&tensors](size_t a, size_t b) {
231 const auto& left = tensors.tensors[a].spec();
232 const auto& right = tensors.tensors[b].spec();
233 const auto left_width = std::max<size_t>(1, left.dtype.element_bytes());
234 const auto right_width = std::max<size_t>(1, right.dtype.element_bytes());
235 return left_width != right_width ? left_width > right_width : left.name < right.name;
236 });
237
238 std::vector<std::optional<Kakshya::TensorData>> gathered(tensors.tensors.size());
239 std::vector<std::span<const std::byte>> payloads;
240 payloads.reserve(tensors.tensors.size());
241
242 uint64_t offset = 0;
243 for (size_t i = 0; i < order.size(); ++i) {
244 const auto& tensor = tensors.tensors[order[i]];
245 const auto& spec = tensor.spec();
246
247 const auto name = Detail::safetensors_name(spec.dtype);
248 if (!name) {
249 m_last_error = "tensor '" + spec.name + "' has element type " + spec.dtype.name()
250 + " which safetensors cannot store";
251 return false;
252 }
253 const auto bytes = Detail::dense_bytes(tensor, gathered[i]);
254 if (!bytes) {
255 m_last_error = "tensor '" + spec.name + "' could not be read";
256 return false;
257 }
258
259 index[spec.name] = {
260 { "dtype", std::string(*name) },
261 { "shape", spec.shape },
262 { "data_offsets", std::vector<uint64_t> { offset, offset + bytes->size() } }
263 };
264 offset += bytes->size();
265 payloads.push_back(*bytes);
266 }
267
268 std::string header = index.dump();
269 header.append((8 - header.size() % 8) % 8, ' ');
270
271 std::string prefix;
272 write_le64(prefix, header.size());
273
274 std::ofstream file(filepath, std::ios::binary | std::ios::trunc);
275 if (!file) {
276 m_last_error = "cannot create '" + filepath + "'";
277 return false;
278 }
279 file.write(prefix.data(), static_cast<std::streamsize>(prefix.size()));
280 file.write(header.data(), static_cast<std::streamsize>(header.size()));
281
282 std::vector<std::byte> swapped;
283 for (size_t i = 0; i < payloads.size(); ++i) {
284 std::span<const std::byte> payload = payloads[i];
286 swapped.assign(payload.begin(), payload.end());
287 Detail::swap_components(swapped, Detail::component_bytes(tensors.tensors[order[i]].spec().dtype));
288 payload = swapped;
289 }
290 file.write(reinterpret_cast<const char*>(payload.data()), static_cast<std::streamsize>(payload.size()));
291 }
292
293 file.close();
294 if (!file) {
295 m_last_error = "failed while writing '" + filepath + "'";
296 return false;
297 }
298 return true;
299}
size_t component_bytes(Kakshya::TensorDType dtype) noexcept
Bytes of one independently ordered component of an element.
std::optional< std::span< const std::byte > > dense_bytes(const Kakshya::TensorData &tensor, std::optional< Kakshya::TensorData > &dense)
Elements of a tensor as dense C-order bytes.
void swap_components(std::span< std::byte > data, size_t component_bytes) noexcept
Reverse the byte order of every component in place.
std::optional< std::string_view > safetensors_name(Kakshya::TensorDType dtype) noexcept
Safetensors dtype name for an element type, when the format has one.
bool host_is_little_endian() noexcept
Whether this machine stores multi-byte values least significant byte first.

References MayaFlux::IO::Detail::component_bytes(), MayaFlux::IO::Detail::dense_bytes(), MayaFlux::IO::Detail::host_is_little_endian(), MayaFlux::Kakshya::TensorSet::is_consistent(), m_last_error, MayaFlux::Kakshya::TensorSet::metadata, MayaFlux::IO::Detail::safetensors_name(), MayaFlux::IO::Detail::swap_components(), and MayaFlux::Kakshya::TensorSet::tensors.

Here is the call graph for this function: