Open a file for reading.
80{
86
87 m_file.seekg(0, std::ios::end);
88 const auto file_size =
static_cast<uint64_t
>(
m_file.tellg());
90
91 std::array<std::byte, 8> prefix {};
92 if (!
m_file.read(
reinterpret_cast<char*
>(prefix.data()), 8))
93 return reject(
"file is shorter than a safetensors prefix");
94
95 const uint64_t header_size = read_le64(prefix.data());
96 if (header_size == 0 || header_size > max_header_bytes || header_size > file_size - 8)
97 return reject(
"safetensors header size " + std::to_string(header_size) +
" is invalid");
98
99 std::string text(static_cast<size_t>(header_size), '\0');
100 if (!
m_file.read(text.data(),
static_cast<std::streamsize
>(header_size)))
101 return reject(
"safetensors header is truncated");
102
103 const auto index = nlohmann::ordered_json::parse(text, nullptr, false);
104 if (index.is_discarded() || !index.is_object())
105 return reject(
"safetensors header is not a JSON object");
106
107 const uint64_t data_size = file_size - 8 - header_size;
108
109 for (const auto& [key, value] : index.items()) {
110 if (key == "__metadata__") {
111 if (!value.is_object())
112 return reject(
"__metadata__ must be an object");
113 for (const auto& [meta_key, meta_value] : value.items()) {
114 if (!meta_value.is_string())
115 return reject(
"__metadata__ value for '" + meta_key +
"' is not a string");
116 m_metadata[meta_key] = meta_value.get<std::string>();
117 }
118 continue;
119 }
120
121 if (!value.is_object())
122 return reject(
"tensor '" + key +
"' is not an object");
123
124 const auto dtype_it = value.find("dtype");
125 const auto shape_it = value.find("shape");
126 const auto offsets_it = value.find("data_offsets");
127 if (dtype_it == value.end() || !dtype_it->is_string() || shape_it == value.end() || !shape_it->is_array()
128 || offsets_it == value.end() || !offsets_it->is_array() || offsets_it->size() != 2)
129 return reject(
"tensor '" + key +
"' lacks dtype, shape or data_offsets");
130
132 if (!dtype) {
134 "SafetensorsReader: skipping '{}' with unsupported dtype '{}'",
135 key, dtype_it->get<std::string>());
136 continue;
137 }
138
139 std::vector<uint64_t> shape;
140 for (const auto& extent : *shape_it) {
141 if (!extent.is_number_unsigned())
142 return reject(
"tensor '" + key +
"' has a non-integer extent");
143 shape.push_back(extent.get<uint64_t>());
144 }
145 if (!(*offsets_it)[0].is_number_unsigned() || !(*offsets_it)[1].is_number_unsigned())
146 return reject(
"tensor '" + key +
"' has non-integer data_offsets");
147
149 entry.
begin = (*offsets_it)[0].get<uint64_t>();
150 entry.end = (*offsets_it)[1].get<uint64_t>();
152
153 if (const auto why = entry.spec.diagnose(); !why.empty())
155
156 const auto bytes = dtype->storage_bytes(entry.spec.element_count().value_or(0));
157 if (!bytes || entry.end < entry.begin || entry.end > data_size || entry.end - entry.begin != *bytes)
158 return reject(
"tensor '" + key +
"' has inconsistent data_offsets");
159
161 }
162
166 return true;
167}
#define MF_WARN(comp, ctx,...)
static std::string resolve_path(const std::string &filepath)
Resolve a filepath against the project source root if not found as-is.
bool reject(std::string reason)
void close() override
Close the currently open file.
std::vector< Entry > m_entries
std::map< std::string, std::string > m_metadata
std::optional< Kakshya::TensorDType > safetensors_dtype(std::string_view name) noexcept
Element type for a safetensors dtype name such as F32 or BF16.
@ FileIO
Filesystem I/O operations.
@ IO
Networking, file handling, streaming.
static TensorSpec contiguous(TensorDType dtype, std::vector< uint64_t > shape, std::string name={})
C-ordered spec with no offset, axes or quantization.