MayaFlux 1.0.0
Digital-First Multimedia Processing Framework
Loading...
Searching...
No Matches
RawTensorReader.cpp
Go to the documentation of this file.
1#include "RawTensorReader.hpp"
2
4
5namespace MayaFlux::IO {
6
7bool RawTensorReader::set_layout(std::vector<RawTensorEntry> entries)
8{
9 for (size_t i = 0; i < entries.size(); ++i) {
10 const auto& spec = entries[i].spec;
11 if (spec.name.empty()) {
12 return fail("every raw tensor needs a name");
13 }
14 if (const auto why = spec.diagnose(); !why.empty()) {
15 return fail(why);
16 }
17 if (!spec.device.is_host_accessible()) {
18 return fail("tensor '" + spec.name + "' is not host memory");
19 }
20 for (size_t j = i + 1; j < entries.size(); ++j) {
21 if (entries[j].spec.name == spec.name) {
22 return fail("duplicate tensor name '" + spec.name + "'");
23 }
24 }
25 }
26 m_layout = std::move(entries);
27 return true;
28}
29
30bool RawTensorReader::can_read(const std::string& filepath) const
31{
32 std::error_code error;
33 return std::filesystem::is_regular_file(resolve_path(filepath), error);
34}
35
37{
38 if (m_file.is_open()) {
39 m_file.close();
40 }
41 m_offsets.clear();
42 m_open = false;
43}
44
45bool RawTensorReader::open(const std::string& filepath, FileReadOptions)
46{
47 close();
48 if (m_layout.empty()) {
49 return fail("set_layout() must describe the tensors before open()");
50 }
51
52 m_path = resolve_path(filepath);
53 m_file.open(m_path, std::ios::binary);
54 if (!m_file) {
55 return fail("cannot open '" + m_path + "'");
56 }
57 m_file.seekg(0, std::ios::end);
58 const auto file_size = static_cast<uint64_t>(m_file.tellg());
59
60 uint64_t cursor = 0;
61 for (const auto& entry : m_layout) {
62 const uint64_t begin = entry.file_offset.value_or(cursor);
63 const auto range = entry.spec.byte_range();
64 if (!range || range->second > file_size || begin > file_size - range->second) {
65 close();
66 return fail("tensor '" + entry.spec.name + "' extends past the end of the file");
67 }
68 m_offsets.push_back(begin);
69 cursor = begin + range->second;
70 }
71
72 m_open = true;
73 m_last_error.clear();
74 return true;
75}
76
77std::vector<Kakshya::TensorSpec> RawTensorReader::describe() const
78{
79 std::vector<Kakshya::TensorSpec> specs;
80 specs.reserve(m_layout.size());
81 for (const auto& entry : m_layout) {
82 specs.push_back(entry.spec);
83 }
84 return specs;
85}
86
87std::optional<Kakshya::TensorData> RawTensorReader::read(std::string_view name)
88{
89 if (!m_open) {
90 fail("no file is open");
91 return std::nullopt;
92 }
93
94 const auto entry = std::ranges::find_if(m_layout,
95 [name](const RawTensorEntry& candidate) { return candidate.spec.name == name; });
96 if (entry == m_layout.end()) {
97 fail("no tensor named '" + std::string(name) + "'");
98 return std::nullopt;
99 }
100
101 auto tensor = Kakshya::TensorData::allocate(entry->spec);
102 if (!tensor) {
103 fail("cannot allocate tensor '" + std::string(name) + "'");
104 return std::nullopt;
105 }
106
107 const auto storage = tensor->storage();
108 if (!storage.empty()) {
109 m_file.clear();
110 m_file.seekg(static_cast<std::streamoff>(m_offsets[static_cast<size_t>(entry - m_layout.begin())]));
111 if (!m_file.read(reinterpret_cast<char*>(storage.data()), static_cast<std::streamsize>(storage.size()))) {
112 fail("tensor '" + std::string(name) + "' is truncated");
113 return std::nullopt;
114 }
115 if (entry->big_endian == Detail::host_is_little_endian()) {
116 Detail::swap_components(storage, Detail::component_bytes(entry->spec.dtype));
117 }
118 }
119 return tensor;
120}
121
122}
static std::string resolve_path(const std::string &filepath)
Resolve a filepath against the project source root if not found as-is.
std::vector< Kakshya::TensorSpec > describe() const override
Specs of every tensor in the open file, in file order, without reading data.
std::vector< RawTensorEntry > m_layout
std::vector< uint64_t > m_offsets
bool can_read(const std::string &filepath) const override
Check if a file can be read by this reader.
void close() override
Close the currently open file.
bool open(const std::string &filepath, FileReadOptions options=FileReadOptions::ALL) override
Open a file for reading.
std::optional< Kakshya::TensorData > read(std::string_view name) override
Read one tensor by name.
bool set_layout(std::vector< RawTensorEntry > entries)
Declare the tensors the file holds.
bool fail(std::string reason)
Record a failure reason and return false.
static std::optional< TensorData > allocate(TensorSpec spec)
Zero-filled tensor for a spec.
Definition Tensor.cpp:72
size_t component_bytes(Kakshya::TensorDType dtype) noexcept
Bytes of one independently ordered component of an element.
void swap_components(std::span< std::byte > data, size_t component_bytes) noexcept
Reverse the byte order of every component in place.
bool host_is_little_endian() noexcept
Whether this machine stores multi-byte values least significant byte first.
FileReadOptions
Generic options for file reading behavior.
Kakshya::TensorSpec spec
Named layout of the tensor: dtype, shape, strides and offset.
Where one tensor sits in a headerless file, and what it is.