15 .extensions = {
"npy" },
17 .make_reader = []() -> std::unique_ptr<TensorReader> {
return std::make_unique<NpyReader>(); },
18 .make_writer = []() -> std::unique_ptr<TensorWriter> {
return std::make_unique<NpyWriter>(); } });
26 return head.size() >= 10 &&
Detail::npy_prefix_size(head.first(std::min<size_t>(head.size(), 12)), why).has_value();
36 return head &&
probe(head->bytes, head->file_size);
58 m_file.seekg(0, std::ios::end);
59 const auto file_size =
static_cast<uint64_t
>(
m_file.tellg());
62 std::array<std::byte, 12> leading {};
63 m_file.read(
reinterpret_cast<char*
>(leading.data()),
static_cast<std::streamsize
>(leading.size()));
64 const auto got =
static_cast<size_t>(
m_file.gcount());
69 if (!prefix_size || *prefix_size > file_size) {
71 return fail(prefix_size ?
"numpy header is truncated" : why);
74 std::vector<std::byte> prefix(*prefix_size);
76 if (!
m_file.read(
reinterpret_cast<char*
>(prefix.data()),
static_cast<std::streamsize
>(prefix.size()))) {
78 return fail(
"numpy header is truncated");
81 const auto stem = std::filesystem::path(
m_path).stem().string();
88 const auto bytes = header->spec.dtype.storage_bytes(header->spec.element_count().value_or(0));
89 if (!bytes || header->data_offset + *bytes > file_size) {
91 return fail(
"numpy data is shorter than its header declares");
104 return m_open ? std::vector<Kakshya::TensorSpec> {
m_spec } : std::vector<Kakshya::TensorSpec> {};
110 fail(
"no file is open");
113 if (!name.empty() && name !=
m_spec.name) {
114 fail(
"no tensor named '" + std::string(name) +
"'");
120 fail(
"cannot allocate tensor '" +
m_spec.name +
"'");
124 const auto storage = tensor->storage();
125 if (!storage.empty()) {
128 if (!
m_file.read(
reinterpret_cast<char*
>(storage.data()),
static_cast<std::streamsize
>(storage.size()))) {
129 fail(
"numpy data is truncated");
142 if (tensors.
tensors.size() != 1) {
143 m_last_error =
"a .npy file holds exactly one tensor, received " + std::to_string(tensors.
tensors.size());
152 std::ofstream file(filepath, std::ios::binary | std::ios::trunc);
157 file.write(
reinterpret_cast<const char*
>(bytes->data()),
static_cast<std::streamsize
>(bytes->size()));
160 m_last_error =
"failed while writing '" + filepath +
"'";
#define MF_INFO(comp, ctx,...)
static std::string resolve_path(const std::string &filepath)
Resolve a filepath against the project source root if not found as-is.
Kakshya::TensorSpec m_spec
bool open(const std::string &filepath, FileReadOptions options=FileReadOptions::ALL) override
Open a file for reading.
std::vector< Kakshya::TensorSpec > describe() const override
Specs of every tensor in the open file, in file order, without reading data.
void close() override
Close the currently open file.
bool can_read(const std::string &filepath) const override
Check if a file can be read by this reader.
static void register_with_registry()
Register the format, with its writer, in the TensorFormatRegistry.
static bool probe(std::span< const std::byte > head, uint64_t file_size)
Whether the leading bytes are the numpy magic string.
std::optional< Kakshya::TensorData > read(std::string_view name) override
Read one tensor by name.
bool write(const std::string &filepath, const Kakshya::TensorSet &tensors) override
Write every tensor in the set to a file.
bool fail(std::string reason)
Record a failure reason and return false.
static std::optional< TensorData > allocate(TensorSpec spec)
Zero-filled tensor for a spec.
std::string lower_extension(const std::string &path)
Lowercase extension of a path without its dot.
std::optional< NpyHeader > parse_npy_header(std::span< const std::byte > prefix, std::string_view name, std::string &why)
Decode a .npy prefix into a tensor spec.
size_t component_bytes(Kakshya::TensorDType dtype) noexcept
Bytes of one independently ordered component of an element.
std::optional< std::vector< std::byte > > encode_npy(const Kakshya::TensorData &tensor, std::string &why)
Encode a tensor as a complete .npy array in dense C order.
std::optional< size_t > npy_prefix_size(std::span< const std::byte > leading, std::string &why)
Total prefix size (magic, version, length field and header) of a .npy array.
void swap_components(std::span< std::byte > data, size_t component_bytes) noexcept
Reverse the byte order of every component in place.
bool host_is_little_endian() noexcept
Whether this machine stores multi-byte values least significant byte first.
std::optional< FileHead > read_file_head(const std::string &path, size_t limit)
Read up to limit leading bytes of a file, or std::nullopt if it cannot be opened.
FileReadOptions
Generic options for file reading behavior.
@ Init
Engine/subsystem initialization.
@ IO
Networking, file handling, streaming.
std::vector< TensorData > tensors
Named tensors that belong together, such as a model's weights or I/O.