Skip to content

Blosc

blosc

Partial decoding of Blosc-1 frames: read only the blocks covering a byte range.

A Blosc-1 frame is header(16 B) | bstarts(int32 * nblocks) | block payloads where every block except the last holds blocksize uncompressed bytes and is compressed independently (shuffle is applied per block). Given the header and the offsets table, a contiguous run of blocks can be fetched with one HTTP range request and re-framed into a smaller valid Blosc frame that the regular decoder (numcodecs) decompresses. Format reference: c-blosc README_HEADER.

BloscHeader dataclass

Blosc-1 frame header with the start and end offset of every block.

Source code in packages/barograph-met/src/baro/met/blosc.py 
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
@dataclass(frozen=True)
class BloscHeader:
    """Blosc-1 frame header with the start and end offset of every block."""

    version: int
    versionlz: int
    flags: int
    typesize: int
    nbytes: int
    blocksize: int
    cbytes: int
    starts: NDArray[np.int64]
    ends: NDArray[np.int64]

    @property
    def nblocks(self) -> int:
        """Number of compressed blocks."""
        return len(self.starts)

    @property
    def header_size(self) -> int:
        """Bytes of header plus block-offset table."""
        return _HEADER + 4 * self.nblocks

    def block_span(self, i: int) -> tuple[int, int]:
        """Uncompressed byte span of block ``i``."""
        return i * self.blocksize, min((i + 1) * self.blocksize, self.nbytes)

    def compressed_span(self, first: int, last: int) -> tuple[int, int]:
        """Smallest compressed byte span ``[start, stop)`` holding blocks ``first..last``.

        Multi-threaded compressors may store blocks out of order; the span is
        then a covering range that can include unrelated blocks (still valid).
        """
        run = slice(first, last + 1)
        return int(self.starts[run].min()), int(self.ends[run].max())

    def blocks_for(self, start: int, stop: int) -> tuple[int, int]:
        """Decodable inclusive block range covering uncompressed bytes ``[start, stop)``.

        A trailing partial ("leftover") block is decoded with different split
        rules than a full block, and a frame whose ``nbytes`` is below its
        ``blocksize`` is rejected by the decoder. A run made of the leftover
        block alone is therefore widened to include the preceding block.
        """
        if not 0 <= start < stop <= self.nbytes:
            raise ValueError(f"byte range [{start}, {stop}) outside frame of {self.nbytes} B")
        first, last = start // self.blocksize, (stop - 1) // self.blocksize
        leftover = self.nbytes % self.blocksize != 0
        if first == last == self.nblocks - 1 and leftover and first > 0:
            first -= 1
        return first, last
header_size property

Bytes of header plus block-offset table.

nblocks property

Number of compressed blocks.

block_span(i)

Uncompressed byte span of block i.

Source code in packages/barograph-met/src/baro/met/blosc.py 
55
56
57
def block_span(self, i: int) -> tuple[int, int]:
    """Uncompressed byte span of block ``i``."""
    return i * self.blocksize, min((i + 1) * self.blocksize, self.nbytes)
blocks_for(start, stop)

Decodable inclusive block range covering uncompressed bytes [start, stop).

A trailing partial ("leftover") block is decoded with different split rules than a full block, and a frame whose nbytes is below its blocksize is rejected by the decoder. A run made of the leftover block alone is therefore widened to include the preceding block.

Source code in packages/barograph-met/src/baro/met/blosc.py 
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
def blocks_for(self, start: int, stop: int) -> tuple[int, int]:
    """Decodable inclusive block range covering uncompressed bytes ``[start, stop)``.

    A trailing partial ("leftover") block is decoded with different split
    rules than a full block, and a frame whose ``nbytes`` is below its
    ``blocksize`` is rejected by the decoder. A run made of the leftover
    block alone is therefore widened to include the preceding block.
    """
    if not 0 <= start < stop <= self.nbytes:
        raise ValueError(f"byte range [{start}, {stop}) outside frame of {self.nbytes} B")
    first, last = start // self.blocksize, (stop - 1) // self.blocksize
    leftover = self.nbytes % self.blocksize != 0
    if first == last == self.nblocks - 1 and leftover and first > 0:
        first -= 1
    return first, last
compressed_span(first, last)

Smallest compressed byte span [start, stop) holding blocks first..last.

Multi-threaded compressors may store blocks out of order; the span is then a covering range that can include unrelated blocks (still valid).

Source code in packages/barograph-met/src/baro/met/blosc.py 
59
60
61
62
63
64
65
66
def compressed_span(self, first: int, last: int) -> tuple[int, int]:
    """Smallest compressed byte span ``[start, stop)`` holding blocks ``first..last``.

    Multi-threaded compressors may store blocks out of order; the span is
    then a covering range that can include unrelated blocks (still valid).
    """
    run = slice(first, last + 1)
    return int(self.starts[run].min()), int(self.ends[run].max())

UnsupportedFrameError

Bases: ValueError

The frame cannot be partially decoded; callers must fall back to a full read.

Source code in packages/barograph-met/src/baro/met/blosc.py 
27
28
class UnsupportedFrameError(ValueError):
    """The frame cannot be partially decoded; callers must fall back to a full read."""

decode_run(header, first, last, payload)

Decode blocks first..last from their compressed payload.

payload must be exactly frame[compressed_span(first, last)]. Returns the uncompressed bytes frame_uncompressed[block_span(first)[0]:block_span(last)[1]].

Source code in packages/barograph-met/src/baro/met/blosc.py 
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
def decode_run(header: BloscHeader, first: int, last: int, payload: bytes) -> bytes:
    """Decode blocks ``first..last`` from their compressed ``payload``.

    ``payload`` must be exactly ``frame[compressed_span(first, last)]``. Returns
    the uncompressed bytes ``frame_uncompressed[block_span(first)[0]:block_span(last)[1]]``.
    """
    c0, c1 = header.compressed_span(first, last)
    if len(payload) != c1 - c0:
        raise ValueError(f"payload has {len(payload)} bytes, expected {c1 - c0}")
    n = last - first + 1
    u0, _ = header.block_span(first)
    _, u1 = header.block_span(last)
    new_header_size = _HEADER + 4 * n
    starts = header.starts[first : last + 1] - c0 + new_header_size
    head = struct.pack(
        "<BBBBiii",
        header.version,
        header.versionlz,
        header.flags,
        header.typesize,
        u1 - u0,
        header.blocksize,
        new_header_size + len(payload),
    )
    frame = head + starts.astype("<i4").tobytes() + payload
    out = numcodecs.Blosc().decode(frame)
    data = bytes(out)
    if len(data) != u1 - u0:
        raise UnsupportedFrameError("partial decode returned an unexpected size")
    return data

header_prefix_size(first16)

Total header size (header + offsets) announced by the first 16 bytes.

Source code in packages/barograph-met/src/baro/met/blosc.py 
85
86
87
88
89
90
91
92
def header_prefix_size(first16: bytes) -> int:
    """Total header size (header + offsets) announced by the first 16 bytes."""
    if len(first16) < _HEADER:
        raise UnsupportedFrameError("truncated blosc header")
    nbytes, blocksize = (int(x) for x in struct.unpack("<ii", first16[4:12]))
    if blocksize <= 0:
        raise UnsupportedFrameError("invalid blocksize")
    return _HEADER + 4 * (-(-nbytes // blocksize))

parse_header(prefix)

Parse a Blosc-1 header and its block-offset table from a frame prefix.

Source code in packages/barograph-met/src/baro/met/blosc.py 
123
124
125
126
127
128
129
130
131
132
133
134
135
def parse_header(prefix: bytes) -> BloscHeader:
    """Parse a Blosc-1 header and its block-offset table from a frame prefix."""
    if len(prefix) < _HEADER:
        raise UnsupportedFrameError("truncated blosc header")
    version, versionlz, flags, typesize = prefix[0], prefix[1], prefix[2], prefix[3]
    nbytes, blocksize, cbytes = struct.unpack("<iii", prefix[4:16])
    _check_frame(version, flags, nbytes, blocksize)
    nblocks = -(-nbytes // blocksize)
    need = _HEADER + 4 * nblocks
    if len(prefix) < need:
        raise UnsupportedFrameError(f"prefix too short: need {need} bytes")
    starts, ends = _block_spans(prefix[_HEADER:need], need, cbytes)
    return BloscHeader(version, versionlz, flags, typesize, nbytes, blocksize, cbytes, starts, ends)