}
-static int rice_count_exact(int32_t *res, int n, int k)
+static uint64_t rice_count_exact(int32_t *res, int n, int k)
{
int i;
- int count = 0;
+ uint64_t count = 0;
for (i = 0; i < n; i++) {
int32_t v = -2 * res[i] - 1;
}
-static int subframe_count_exact(FlacEncodeContext *s, FlacSubframe *sub,
- int pred_order)
+static uint64_t subframe_count_exact(FlacEncodeContext *s, FlacSubframe *sub,
+ int pred_order)
{
int p, porder, psize;
int i, part_end;
- int count = 0;
+ uint64_t count = 0;
/* subframe header */
count += 8;
/**
* Solve for d/dk(rice_encode_count) = n-((sum-(n>>1))>>(k+1)) = 0.
*/
-static int find_optimal_param(uint32_t sum, int n)
+static int find_optimal_param(uint64_t sum, int n)
{
int k;
- uint32_t sum2;
+ uint64_t sum2;
if (sum <= n >> 1)
return 0;
sum2 = sum - (n >> 1);
- k = av_log2(n < 256 ? FASTDIV(sum2, n) : sum2 / n);
+ k = av_log2(av_clipl_int32(sum2 / n));
return FFMIN(k, MAX_RICE_PARAM);
}
-static uint32_t calc_optimal_rice_params(RiceContext *rc, int porder,
- uint32_t *sums, int n, int pred_order)
+static uint64_t calc_optimal_rice_params(RiceContext *rc, int porder,
+ uint64_t *sums, int n, int pred_order)
{
int i;
int k, cnt, part;
- uint32_t all_bits;
+ uint64_t all_bits;
part = (1 << porder);
all_bits = 4 * part;
static void calc_sums(int pmin, int pmax, uint32_t *data, int n, int pred_order,
- uint32_t sums[][MAX_PARTITIONS])
+ uint64_t sums[][MAX_PARTITIONS])
{
int i, j;
int parts;
res = &data[pred_order];
res_end = &data[n >> pmax];
for (i = 0; i < parts; i++) {
- uint32_t sum = 0;
+ uint64_t sum = 0;
while (res < res_end)
sum += *(res++);
sums[pmax][i] = sum;
}
-static uint32_t calc_rice_params(RiceContext *rc, int pmin, int pmax,
+static uint64_t calc_rice_params(RiceContext *rc, int pmin, int pmax,
int32_t *data, int n, int pred_order)
{
int i;
- uint32_t bits[MAX_PARTITION_ORDER+1];
+ uint64_t bits[MAX_PARTITION_ORDER+1];
int opt_porder;
RiceContext tmp_rc;
uint32_t *udata;
- uint32_t sums[MAX_PARTITION_ORDER+1][MAX_PARTITIONS];
+ uint64_t sums[MAX_PARTITION_ORDER+1][MAX_PARTITIONS];
assert(pmin >= 0 && pmin <= MAX_PARTITION_ORDER);
assert(pmax >= 0 && pmax <= MAX_PARTITION_ORDER);
}
-static uint32_t find_subframe_rice_params(FlacEncodeContext *s,
+static uint64_t find_subframe_rice_params(FlacEncodeContext *s,
FlacSubframe *sub, int pred_order)
{
int pmin = get_max_p_order(s->options.min_partition_order,
int pmax = get_max_p_order(s->options.max_partition_order,
s->frame.blocksize, pred_order);
- uint32_t bits = 8 + pred_order * sub->obits + 2 + 4;
+ uint64_t bits = 8 + pred_order * sub->obits + 2 + 4;
if (sub->type == FLAC_SUBFRAME_LPC)
bits += 4 + 5 + pred_order * s->options.lpc_coeff_precision;
bits += calc_rice_params(&sub->rc, pmin, pmax, sub->residual,
sub->type = FLAC_SUBFRAME_FIXED;
if (s->options.lpc_type == FF_LPC_TYPE_NONE ||
s->options.lpc_type == FF_LPC_TYPE_FIXED || n <= max_order) {
- uint32_t bits[MAX_FIXED_ORDER+1];
+ uint64_t bits[MAX_FIXED_ORDER+1];
if (max_order > MAX_FIXED_ORDER)
max_order = MAX_FIXED_ORDER;
opt_order = 0;
omethod == ORDER_METHOD_4LEVEL ||
omethod == ORDER_METHOD_8LEVEL) {
int levels = 1 << omethod;
- uint32_t bits[1 << ORDER_METHOD_8LEVEL];
+ uint64_t bits[1 << ORDER_METHOD_8LEVEL];
int order;
int opt_index = levels-1;
opt_order = max_order-1;
opt_order++;
} else if (omethod == ORDER_METHOD_SEARCH) {
// brute-force optimal order search
- uint32_t bits[MAX_LPC_ORDER];
+ uint64_t bits[MAX_LPC_ORDER];
opt_order = 0;
bits[0] = UINT32_MAX;
for (i = min_order-1; i < max_order; i++) {
}
opt_order++;
} else if (omethod == ORDER_METHOD_LOG) {
- uint32_t bits[MAX_LPC_ORDER];
+ uint64_t bits[MAX_LPC_ORDER];
int step;
opt_order = min_order - 1 + (max_order-min_order)/3;
static int encode_frame(FlacEncodeContext *s)
{
- int ch, count;
+ int ch;
+ uint64_t count;
count = count_frame_header(s);
count += (8 - (count & 7)) & 7; // byte alignment
count += 16; // CRC-16
- return count >> 3;
+ count >>= 3;
+ if (count > INT_MAX)
+ return AVERROR_BUG;
+ return count;
}
/* fallback to verbatim mode if the compressed frame is larger than it
would be if encoded uncompressed. */
- if (frame_bytes > s->max_framesize) {
+ if (frame_bytes < 0 || frame_bytes > s->max_framesize) {
s->frame.verbatim_only = 1;
frame_bytes = encode_frame(s);
+ if (frame_bytes < 0) {
+ av_log(avctx, AV_LOG_ERROR, "Bad frame count\n");
+ return frame_bytes;
+ }
}
if ((ret = ff_alloc_packet(avpkt, frame_bytes))) {