2 * Copyright (C) 2015 The Android Open Source Project
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
25 #include "fec_private.h"
27 using rs_unique_ptr = std::unique_ptr<void, decltype(&free_rs_char)>;
29 /* prints a hexdump of `data' using warn(...) */
30 static void dump(const char *name, uint64_t value, const uint8_t *data,
33 const int bytes_per_line = 16;
34 char hex[bytes_per_line * 3 + 1];
35 char prn[bytes_per_line + 1];
37 warn("%s (%" PRIu64 ") (%zu bytes):", name ? name : "", value, size);
44 for (size_t n = 0; n < size; n += bytes_per_line) {
45 memset(hex, 0, sizeof(hex));
46 memset(prn, 0, sizeof(prn));
48 for (size_t m = 0; m < bytes_per_line; ++m) {
50 sprintf(&hex[m * 3], "%02x ", data[n + m]);
52 if (isprint(data[n + m])) {
58 strcpy(&hex[m * 3], " ");
62 warn(" %04zu %s %s", n, hex, prn);
66 /* checks if `offset' is within a corrupted block */
67 static inline bool is_erasure(fec_handle *f, uint64_t offset,
70 if (unlikely(offset >= f->data_size)) {
74 /* ideally, we would like to know if a specific byte on this block has
75 been corrupted, but knowing whether any of them is can be useful as
76 well, because often the entire block is corrupted */
78 uint64_t n = offset / FEC_BLOCKSIZE;
80 return !verity_check_block(f, &f->verity.hash[n * SHA256_DIGEST_LENGTH],
84 /* check if `offset' is within a block expected to contain zeros */
85 static inline bool is_zero(fec_handle *f, uint64_t offset)
87 verity_info *v = &f->verity;
89 if (!v->hash || unlikely(offset >= f->data_size)) {
93 uint64_t hash_offset = (offset / FEC_BLOCKSIZE) * SHA256_DIGEST_LENGTH;
95 if (unlikely(hash_offset >
96 v->hash_data_blocks * FEC_BLOCKSIZE - SHA256_DIGEST_LENGTH)) {
100 return !memcmp(v->zero_hash, &v->hash[hash_offset], SHA256_DIGEST_LENGTH);
103 /* reads and decodes a single block starting from `offset', returns the number
104 of bytes corrected in `errors' */
105 static int __ecc_read(fec_handle *f, void *rs, uint8_t *dest, uint64_t offset,
106 bool use_erasures, uint8_t *ecc_data, size_t *errors)
108 check(offset % FEC_BLOCKSIZE == 0);
109 ecc_info *e = &f->ecc;
111 /* reverse interleaving: calculate the RS block that includes the requested
113 uint64_t rsb = offset - (offset / (e->rounds * FEC_BLOCKSIZE)) *
114 e->rounds * FEC_BLOCKSIZE;
116 int erasures[e->rsn];
119 /* verity is required to check for erasures */
120 check(!use_erasures || f->verity.hash);
122 for (int i = 0; i < e->rsn; ++i) {
123 uint64_t interleaved = fec_ecc_interleave(rsb * e->rsn + i, e->rsn,
126 if (interleaved == offset) {
130 /* copy raw data to reconstruct the RS block */
131 uint8_t bbuf[FEC_BLOCKSIZE];
133 if (unlikely(interleaved >= e->start) ||
134 is_zero(f, interleaved)) {
135 memset(bbuf, 0, FEC_BLOCKSIZE);
137 if (!raw_pread(f, bbuf, FEC_BLOCKSIZE, interleaved)) {
138 error("failed to read: %s", strerror(errno));
142 if (use_erasures && neras <= e->roots &&
143 is_erasure(f, interleaved, bbuf)) {
144 erasures[neras++] = i;
148 for (int j = 0; j < FEC_BLOCKSIZE; ++j) {
149 ecc_data[j * FEC_RSM + i] = bbuf[j];
153 check(data_index >= 0);
156 uint8_t copy[FEC_RSM];
158 for (int i = 0; i < FEC_BLOCKSIZE; ++i) {
159 /* copy parity data */
160 if (!raw_pread(f, &ecc_data[i * FEC_RSM + e->rsn], e->roots,
161 e->start + (i + rsb) * e->roots)) {
162 error("failed to read ecc data: %s", strerror(errno));
166 /* for debugging decoding failures, because decode_rs_char can mangle
168 if (unlikely(use_erasures)) {
169 memcpy(copy, &ecc_data[i * FEC_RSM], FEC_RSM);
173 int rc = decode_rs_char(rs, &ecc_data[i * FEC_RSM], erasures, neras);
175 if (unlikely(rc < 0)) {
177 error("RS block %" PRIu64 ": decoding failed (%d erasures)",
179 dump("raw RS block", rsb, copy, FEC_RSM);
180 } else if (!f->verity.hash) {
181 warn("RS block %" PRIu64 ": decoding failed", rsb);
183 debug("RS block %" PRIu64 ": decoding failed", rsb);
188 } else if (unlikely(rc > 0)) {
189 check(rc <= (use_erasures ? e->roots : e->roots / 2));
193 dest[i] = ecc_data[i * FEC_RSM + data_index];
197 warn("RS block %" PRIu64 ": corrected %zu errors", rsb, nerrs);
201 return FEC_BLOCKSIZE;
204 /* initializes RS decoder and allocates memory for interleaving */
205 static int ecc_init(fec_handle *f, rs_unique_ptr& rs,
206 std::unique_ptr<uint8_t[]>& ecc_data)
210 rs.reset(init_rs_char(FEC_PARAMS(f->ecc.roots)));
213 error("failed to initialize RS");
218 ecc_data.reset(new (std::nothrow) uint8_t[FEC_RSM * FEC_BLOCKSIZE]);
220 if (unlikely(!ecc_data)) {
221 error("failed to allocate ecc buffer");
229 /* reads `count' bytes from `offset' and corrects possible errors without
230 erasure detection, returning the number of corrected bytes in `errors' */
231 static ssize_t ecc_read(fec_handle *f, uint8_t *dest, size_t count,
232 uint64_t offset, size_t *errors)
236 check(offset < f->data_size);
237 check(offset + count <= f->data_size);
240 debug("[%" PRIu64 ", %" PRIu64 ")", offset, offset + count);
242 rs_unique_ptr rs(NULL, free_rs_char);
243 std::unique_ptr<uint8_t[]> ecc_data;
245 if (ecc_init(f, rs, ecc_data) == -1) {
249 uint64_t curr = offset / FEC_BLOCKSIZE;
250 size_t coff = (size_t)(offset - curr * FEC_BLOCKSIZE);
253 uint8_t data[FEC_BLOCKSIZE];
256 /* there's no erasure detection without verity metadata */
257 if (__ecc_read(f, rs.get(), data, curr * FEC_BLOCKSIZE, false,
258 ecc_data.get(), errors) == -1) {
262 size_t copy = FEC_BLOCKSIZE - coff;
268 memcpy(dest, &data[coff], copy);
279 /* reads `count' bytes from `offset', corrects possible errors with
280 erasure detection, and verifies the integrity of read data using
281 verity hash tree; returns the number of corrections in `errors' */
282 static ssize_t verity_read(fec_handle *f, uint8_t *dest, size_t count,
283 uint64_t offset, size_t *errors)
287 check(offset < f->data_size);
288 check(offset + count <= f->data_size);
289 check(f->verity.hash);
292 debug("[%" PRIu64 ", %" PRIu64 ")", offset, offset + count);
294 rs_unique_ptr rs(NULL, free_rs_char);
295 std::unique_ptr<uint8_t[]> ecc_data;
297 if (f->ecc.start && ecc_init(f, rs, ecc_data) == -1) {
301 uint64_t curr = offset / FEC_BLOCKSIZE;
302 size_t coff = (size_t)(offset - curr * FEC_BLOCKSIZE);
304 uint8_t data[FEC_BLOCKSIZE];
306 uint64_t max_hash_block = (f->verity.hash_data_blocks * FEC_BLOCKSIZE -
307 SHA256_DIGEST_LENGTH) / SHA256_DIGEST_LENGTH;
310 check(curr <= max_hash_block);
312 uint8_t *hash = &f->verity.hash[curr * SHA256_DIGEST_LENGTH];
313 uint64_t curr_offset = curr * FEC_BLOCKSIZE;
315 bool expect_zeros = is_zero(f, curr_offset);
317 /* if we are in read-only mode and expect to read a zero block,
318 skip reading and just return zeros */
319 if (f->mode & O_RDONLY && expect_zeros) {
320 memset(data, 0, FEC_BLOCKSIZE);
324 /* copy raw data without error correction */
325 if (!raw_pread(f, data, FEC_BLOCKSIZE, curr_offset)) {
326 error("failed to read: %s", strerror(errno));
330 if (likely(verity_check_block(f, hash, data))) {
334 /* we know the block is supposed to contain zeros, so return zeros
335 instead of trying to correct it */
337 memset(data, 0, FEC_BLOCKSIZE);
342 /* fatal error without ecc */
343 error("[%" PRIu64 ", %" PRIu64 "): corrupted block %" PRIu64,
344 offset, offset + count, curr);
347 debug("[%" PRIu64 ", %" PRIu64 "): corrupted block %" PRIu64,
348 offset, offset + count, curr);
351 /* try to correct without erasures first, because checking for
352 erasure locations is slower */
353 if (__ecc_read(f, rs.get(), data, curr_offset, false, ecc_data.get(),
354 errors) == FEC_BLOCKSIZE &&
355 verity_check_block(f, hash, data)) {
359 /* try to correct with erasures */
360 if (__ecc_read(f, rs.get(), data, curr_offset, true, ecc_data.get(),
361 errors) == FEC_BLOCKSIZE &&
362 verity_check_block(f, hash, data)) {
366 error("[%" PRIu64 ", %" PRIu64 "): corrupted block %" PRIu64
367 " (offset %" PRIu64 ") cannot be recovered",
368 offset, offset + count, curr, curr_offset);
369 dump("decoded block", curr, data, FEC_BLOCKSIZE);
375 /* update the corrected block to the file if we are in r/w mode */
376 if (f->mode & O_RDWR &&
377 !raw_pwrite(f, data, FEC_BLOCKSIZE, curr_offset)) {
378 error("failed to write: %s", strerror(errno));
383 size_t copy = FEC_BLOCKSIZE - coff;
389 memcpy(dest, &data[coff], copy);
400 /* sets the internal file position to `offset' relative to `whence' */
401 int fec_seek(struct fec_handle *f, int64_t offset, int whence)
405 if (whence == SEEK_SET) {
412 } else if (whence == SEEK_CUR) {
413 if (offset < 0 && f->pos < (uint64_t)-offset) {
416 } else if (offset > 0 && (uint64_t)offset > UINT64_MAX - f->pos) {
422 } else if (whence == SEEK_END) {
426 } else if ((uint64_t)-offset > f->size) {
431 f->pos = f->size + offset;
440 /* reads up to `count' bytes starting from the internal file position using
441 error correction and integrity validation, if available */
442 ssize_t fec_read(struct fec_handle *f, void *buf, size_t count)
444 ssize_t rc = fec_pread(f, buf, count, f->pos);
447 check(f->pos < UINT64_MAX - rc);
454 /* for a file with size `max', returns the number of bytes we can read starting
455 from `offset', up to `count' bytes */
456 static inline size_t get_max_count(uint64_t offset, size_t count, uint64_t max)
460 } else if (offset > max - count) {
461 return (size_t)(max - offset);
467 /* reads `count' bytes from `f->fd' starting from `offset', and copies the
469 bool raw_pread(fec_handle *f, void *buf, size_t count, uint64_t offset)
474 uint8_t *p = (uint8_t *)buf;
475 size_t remaining = count;
477 while (remaining > 0) {
478 ssize_t n = TEMP_FAILURE_RETRY(pread64(f->fd, p, remaining, offset));
492 /* writes `count' bytes from `buf' to `f->fd' to a file position `offset' */
493 bool raw_pwrite(fec_handle *f, const void *buf, size_t count, uint64_t offset)
498 const uint8_t *p = (const uint8_t *)buf;
499 size_t remaining = count;
501 while (remaining > 0) {
502 ssize_t n = TEMP_FAILURE_RETRY(pwrite64(f->fd, p, remaining, offset));
516 /* reads up to `count' bytes starting from `offset' using error correction and
517 integrity validation, if available */
518 ssize_t fec_pread(struct fec_handle *f, void *buf, size_t count,
524 if (unlikely(offset > UINT64_MAX - count)) {
529 if (f->verity.hash) {
530 return process(f, (uint8_t *)buf,
531 get_max_count(offset, count, f->data_size), offset,
533 } else if (f->ecc.start) {
534 check(f->ecc.start < f->size);
536 count = get_max_count(offset, count, f->data_size);
537 ssize_t rc = process(f, (uint8_t *)buf, count, offset, ecc_read);
543 /* return raw data if pure ecc read fails; due to interleaving
544 the specific blocks the caller wants may still be fine */
546 count = get_max_count(offset, count, f->size);
549 if (raw_pread(f, buf, count, offset)) {