MayaFlux 1.0.0
Digital-First Multimedia Processing Framework
Loading...
Searching...
No Matches
TensorCodec.hpp
Go to the documentation of this file.
1#pragma once
2
4
5namespace MayaFlux::IO::Detail {
6
7/** @brief Whether this machine stores multi-byte values least significant byte first. */
8[[nodiscard]] bool host_is_little_endian() noexcept;
9
10/**
11 * @brief Bytes of one independently ordered component of an element.
12 *
13 * A complex element is two components; every other element is one. Zero for
14 * sub-byte types, which have no byte order.
15 */
16[[nodiscard]] size_t component_bytes(Kakshya::TensorDType dtype) noexcept;
17
18/** @brief Reverse the byte order of every component in place. */
19void swap_components(std::span<std::byte> data, size_t component_bytes) noexcept;
20
21/** @brief Element type for a safetensors dtype name such as F32 or BF16. */
22[[nodiscard]] std::optional<Kakshya::TensorDType> safetensors_dtype(std::string_view name) noexcept;
23
24/** @brief Safetensors dtype name for an element type, when the format has one. */
25[[nodiscard]] std::optional<std::string_view> safetensors_name(Kakshya::TensorDType dtype) noexcept;
26
27/** @brief Element type and byte order read from a numpy type string. */
32
33/**
34 * @brief Parse a numpy type string such as <f4, >i8 or |b1.
35 * @param why Receives the reason when the type is not supported.
36 */
37[[nodiscard]] std::optional<NpyTypestr> npy_dtype(std::string_view typestr, std::string& why);
38
39/** @brief Little-endian numpy type string for an element type, when numpy has one. */
40[[nodiscard]] std::optional<std::string> npy_typestr(Kakshya::TensorDType dtype);
41
42/** @brief First bytes of a file together with its total size. */
43struct FileHead {
44 std::vector<std::byte> bytes;
45 uint64_t file_size {};
46};
47
48/** @brief Read up to @p limit leading bytes of a file, or std::nullopt if it cannot be opened. */
49[[nodiscard]] std::optional<FileHead> read_file_head(const std::string& path, size_t limit);
50
51/** @brief Lowercase extension of a path without its dot. */
52[[nodiscard]] std::string lower_extension(const std::string& path);
53
54/**
55 * @brief Elements of a tensor as dense C-order bytes.
56 * @param tensor Source tensor.
57 * @param dense Receives a gathered copy only when the tensor is strided; the
58 * returned span then points into it, so keep it alive while the span is used.
59 * @return The bytes, or std::nullopt when the tensor is invalid.
60 */
61[[nodiscard]] std::optional<std::span<const std::byte>> dense_bytes(
62 const Kakshya::TensorData& tensor, std::optional<Kakshya::TensorData>& dense);
63
64}
Owning host tensor: a TensorSpec and the bytes it describes.
Definition Tensor.hpp:42
std::string lower_extension(const std::string &path)
Lowercase extension of a path without its dot.
size_t component_bytes(Kakshya::TensorDType dtype) noexcept
Bytes of one independently ordered component of an element.
std::optional< std::string > npy_typestr(Kakshya::TensorDType dtype)
Little-endian numpy type string for an element type, when numpy has one.
std::optional< Kakshya::TensorDType > safetensors_dtype(std::string_view name) noexcept
Element type for a safetensors dtype name such as F32 or BF16.
std::optional< std::span< const std::byte > > dense_bytes(const Kakshya::TensorData &tensor, std::optional< Kakshya::TensorData > &dense)
Elements of a tensor as dense C-order bytes.
void swap_components(std::span< std::byte > data, size_t component_bytes) noexcept
Reverse the byte order of every component in place.
std::optional< std::string_view > safetensors_name(Kakshya::TensorDType dtype) noexcept
Safetensors dtype name for an element type, when the format has one.
bool host_is_little_endian() noexcept
Whether this machine stores multi-byte values least significant byte first.
std::optional< NpyTypestr > npy_dtype(std::string_view typestr, std::string &why)
Parse a numpy type string such as <f4, >i8 or |b1.
std::optional< FileHead > read_file_head(const std::string &path, size_t limit)
Read up to limit leading bytes of a file, or std::nullopt if it cannot be opened.
std::vector< std::byte > bytes
First bytes of a file together with its total size.
Element type and byte order read from a numpy type string.
Element type as kind, bit width and lane count.