MayaFlux 1.0.0
Digital-First Multimedia Processing Framework
Loading...
Searching...
No Matches

◆ open()

bool MayaFlux::IO::SafetensorsReader::open ( const std::string & filepath,
FileReadOptions options = FileReadOptions::ALL )
overridevirtual

Open a file for reading.

Parameters
filepathPath to the file.
optionsReading options (see FileReadOptions).
Returns
true if file was successfully opened.

Implements MayaFlux::IO::FileReader.

Definition at line 79 of file SafetensorsFormat.cpp.

80{
81 close();
82 m_path = resolve_path(filepath);
83 m_file.open(m_path, std::ios::binary);
84 if (!m_file)
85 return reject("cannot open '" + m_path + "'");
86
87 m_file.seekg(0, std::ios::end);
88 const auto file_size = static_cast<uint64_t>(m_file.tellg());
89 m_file.seekg(0);
90
91 std::array<std::byte, 8> prefix {};
92 if (!m_file.read(reinterpret_cast<char*>(prefix.data()), 8))
93 return reject("file is shorter than a safetensors prefix");
94
95 const uint64_t header_size = read_le64(prefix.data());
96 if (header_size == 0 || header_size > max_header_bytes || header_size > file_size - 8)
97 return reject("safetensors header size " + std::to_string(header_size) + " is invalid");
98
99 std::string text(static_cast<size_t>(header_size), '\0');
100 if (!m_file.read(text.data(), static_cast<std::streamsize>(header_size)))
101 return reject("safetensors header is truncated");
102
103 const auto index = nlohmann::ordered_json::parse(text, nullptr, false);
104 if (index.is_discarded() || !index.is_object())
105 return reject("safetensors header is not a JSON object");
106
107 const uint64_t data_size = file_size - 8 - header_size;
108
109 for (const auto& [key, value] : index.items()) {
110 if (key == "__metadata__") {
111 if (!value.is_object())
112 return reject("__metadata__ must be an object");
113 for (const auto& [meta_key, meta_value] : value.items()) {
114 if (!meta_value.is_string())
115 return reject("__metadata__ value for '" + meta_key + "' is not a string");
116 m_metadata[meta_key] = meta_value.get<std::string>();
117 }
118 continue;
119 }
120
121 if (!value.is_object())
122 return reject("tensor '" + key + "' is not an object");
123
124 const auto dtype_it = value.find("dtype");
125 const auto shape_it = value.find("shape");
126 const auto offsets_it = value.find("data_offsets");
127 if (dtype_it == value.end() || !dtype_it->is_string() || shape_it == value.end() || !shape_it->is_array()
128 || offsets_it == value.end() || !offsets_it->is_array() || offsets_it->size() != 2)
129 return reject("tensor '" + key + "' lacks dtype, shape or data_offsets");
130
131 const auto dtype = Detail::safetensors_dtype(dtype_it->get<std::string>());
132 if (!dtype) {
134 "SafetensorsReader: skipping '{}' with unsupported dtype '{}'",
135 key, dtype_it->get<std::string>());
136 continue;
137 }
138
139 std::vector<uint64_t> shape;
140 for (const auto& extent : *shape_it) {
141 if (!extent.is_number_unsigned())
142 return reject("tensor '" + key + "' has a non-integer extent");
143 shape.push_back(extent.get<uint64_t>());
144 }
145 if (!(*offsets_it)[0].is_number_unsigned() || !(*offsets_it)[1].is_number_unsigned())
146 return reject("tensor '" + key + "' has non-integer data_offsets");
147
148 Entry entry;
149 entry.begin = (*offsets_it)[0].get<uint64_t>();
150 entry.end = (*offsets_it)[1].get<uint64_t>();
151 entry.spec = Kakshya::TensorSpec::contiguous(*dtype, std::move(shape), key);
152
153 if (const auto why = entry.spec.diagnose(); !why.empty())
154 return reject(why);
155
156 const auto bytes = dtype->storage_bytes(entry.spec.element_count().value_or(0));
157 if (!bytes || entry.end < entry.begin || entry.end > data_size || entry.end - entry.begin != *bytes)
158 return reject("tensor '" + key + "' has inconsistent data_offsets");
159
160 m_entries.push_back(std::move(entry));
161 }
162
163 m_data_start = 8 + header_size;
164 m_open = true;
165 m_last_error.clear();
166 return true;
167}
#define MF_WARN(comp, ctx,...)
static std::string resolve_path(const std::string &filepath)
Resolve a filepath against the project source root if not found as-is.
bool reject(std::string reason)
void close() override
Close the currently open file.
std::map< std::string, std::string > m_metadata
std::optional< Kakshya::TensorDType > safetensors_dtype(std::string_view name) noexcept
Element type for a safetensors dtype name such as F32 or BF16.
@ FileIO
Filesystem I/O operations.
@ IO
Networking, file handling, streaming.
static TensorSpec contiguous(TensorDType dtype, std::vector< uint64_t > shape, std::string name={})
C-ordered spec with no offset, axes or quantization.

References MayaFlux::IO::SafetensorsReader::Entry::begin, close(), MayaFlux::Kakshya::TensorSpec::contiguous(), MayaFlux::Kakshya::TensorSpec::diagnose(), MayaFlux::Kakshya::TensorSpec::element_count(), MayaFlux::IO::SafetensorsReader::Entry::end, MayaFlux::Journal::FileIO, MayaFlux::Journal::IO, m_data_start, m_entries, m_file, MayaFlux::IO::TensorReader::m_last_error, m_metadata, m_open, MayaFlux::IO::TensorReader::m_path, MF_WARN, reject(), MayaFlux::IO::FileReader::resolve_path(), MayaFlux::IO::Detail::safetensors_dtype(), and MayaFlux::IO::SafetensorsReader::Entry::spec.

Here is the call graph for this function: