smacker: add sanity check for length in smacker_decode_tree()

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

Bug-Id: 1098
Cc: libav-stable@libav.org
Signed-off-by: Sean McGovern <gseanmcg@gmail.com>
static int smacker_decode_header_tree(SmackVContext *smk, BitstreamContext *bc,
                                      int **recodes, int *last, int size)
{
    int res;
    HuffContext huff;
    HuffContext tmp1, tmp2;
    VLC vlc[2] = { { 0 } };
    int escapes[3];
    DBCtx ctx;
    int err = 0;

    if(size >= UINT_MAX>>4){ // (((size + 3) >> 2) + 3) << 2 must not overflow
        av_log(smk->avctx, AV_LOG_ERROR, "size too large\n");
        return AVERROR_INVALIDDATA;
    }

    tmp1.length = 256;
    tmp1.maxlength = 0;
    tmp1.current = 0;
    tmp1.bits = av_mallocz(256 * 4);
    tmp1.lengths = av_mallocz(256 * sizeof(int));
    tmp1.values = av_mallocz(256 * sizeof(int));

    tmp2.length = 256;
    tmp2.maxlength = 0;
    tmp2.current = 0;
    tmp2.bits = av_mallocz(256 * 4);
    tmp2.lengths = av_mallocz(256 * sizeof(int));
    tmp2.values = av_mallocz(256 * sizeof(int));
    if (!tmp1.bits || !tmp1.lengths || !tmp1.values ||
        !tmp2.bits || !tmp2.lengths || !tmp2.values) {
        err = AVERROR(ENOMEM);
        goto error;
    }

    if (bitstream_read_bit(bc)) {
        smacker_decode_tree(bc, &tmp1, 0, 0);
        bitstream_skip(bc, 1);
        res = init_vlc(&vlc[0], SMKTREE_BITS, tmp1.length,
                    tmp1.lengths, sizeof(int), sizeof(int),
                    tmp1.bits, sizeof(uint32_t), sizeof(uint32_t), INIT_VLC_LE);
        if(res < 0) {
            av_log(smk->avctx, AV_LOG_ERROR, "Cannot build VLC table\n");
            err = res;
            goto error;
        }
    } else {
        av_log(smk->avctx, AV_LOG_ERROR, "Skipping low bytes tree\n");
    }
    if (bitstream_read_bit(bc)) {
        smacker_decode_tree(bc, &tmp2, 0, 0);
        bitstream_skip(bc, 1);
        res = init_vlc(&vlc[1], SMKTREE_BITS, tmp2.length,
                    tmp2.lengths, sizeof(int), sizeof(int),
                    tmp2.bits, sizeof(uint32_t), sizeof(uint32_t), INIT_VLC_LE);
        if(res < 0) {
            av_log(smk->avctx, AV_LOG_ERROR, "Cannot build VLC table\n");
            err = res;
            goto error;
        }
    } else {
        av_log(smk->avctx, AV_LOG_ERROR, "Skipping high bytes tree\n");
    }

    escapes[0]  = bitstream_read(bc, 8);
    escapes[0] |= bitstream_read(bc, 8) << 8;
    escapes[1]  = bitstream_read(bc, 8);
    escapes[1] |= bitstream_read(bc, 8) << 8;
    escapes[2]  = bitstream_read(bc, 8);
    escapes[2] |= bitstream_read(bc, 8) << 8;

    last[0] = last[1] = last[2] = -1;

    ctx.escapes[0] = escapes[0];
    ctx.escapes[1] = escapes[1];
    ctx.escapes[2] = escapes[2];
    ctx.v1 = &vlc[0];
    ctx.v2 = &vlc[1];
    ctx.recode1 = tmp1.values;
    ctx.recode2 = tmp2.values;
    ctx.last = last;

    huff.length = ((size + 3) >> 2) + 4;
    huff.maxlength = 0;
    huff.current = 0;
    huff.values = av_mallocz(huff.length * sizeof(int));
    if (!huff.values) {
        err = AVERROR(ENOMEM);
        goto error;
    }

    if ((res = smacker_decode_bigtree(bc, &huff, &ctx)) < 0)
        err = res;
    bitstream_skip(bc, 1);
    if(ctx.last[0] == -1) ctx.last[0] = huff.current++;
    if(ctx.last[1] == -1) ctx.last[1] = huff.current++;
    if(ctx.last[2] == -1) ctx.last[2] = huff.current++;
    if (ctx.last[0] >= huff.length ||
        ctx.last[1] >= huff.length ||
        ctx.last[2] >= huff.length) {
        av_log(smk->avctx, AV_LOG_ERROR, "Huffman codes out of range\n");
        err = AVERROR_INVALIDDATA;
    }

    *recodes = huff.values;

error:
    if(vlc[0].table)
        ff_free_vlc(&vlc[0]);
    if(vlc[1].table)
        ff_free_vlc(&vlc[1]);
    av_free(tmp1.bits);
    av_free(tmp1.lengths);
    av_free(tmp1.values);
    av_free(tmp2.bits);
    av_free(tmp2.lengths);
    av_free(tmp2.values);

    return err;
}

static int smacker_decode_header_tree(SmackVContext *smk, BitstreamContext *bc,
                                      int **recodes, int *last, int size)
{
    int res;
    HuffContext huff;
    HuffContext tmp1, tmp2;
    VLC vlc[2] = { { 0 } };
    int escapes[3];
    DBCtx ctx;
    int err = 0;

    if(size >= UINT_MAX>>4){ // (((size + 3) >> 2) + 3) << 2 must not overflow
        av_log(smk->avctx, AV_LOG_ERROR, "size too large\n");
        return AVERROR_INVALIDDATA;
    }

    tmp1.length = 256;
    tmp1.maxlength = 0;
    tmp1.current = 0;
    tmp1.bits = av_mallocz(256 * 4);
    tmp1.lengths = av_mallocz(256 * sizeof(int));
    tmp1.values = av_mallocz(256 * sizeof(int));

    tmp2.length = 256;
    tmp2.maxlength = 0;
    tmp2.current = 0;
    tmp2.bits = av_mallocz(256 * 4);
    tmp2.lengths = av_mallocz(256 * sizeof(int));
    tmp2.values = av_mallocz(256 * sizeof(int));
    if (!tmp1.bits || !tmp1.lengths || !tmp1.values ||
        !tmp2.bits || !tmp2.lengths || !tmp2.values) {
        err = AVERROR(ENOMEM);
        goto error;
    }

    if (bitstream_read_bit(bc)) {
        smacker_decode_tree(bc, &tmp1, 0, 0);
        bitstream_skip(bc, 1);
        res = init_vlc(&vlc[0], SMKTREE_BITS, tmp1.length,
                    tmp1.lengths, sizeof(int), sizeof(int),
                    tmp1.bits, sizeof(uint32_t), sizeof(uint32_t), INIT_VLC_LE);
        if(res < 0) {
            av_log(smk->avctx, AV_LOG_ERROR, "Cannot build VLC table\n");
            err = res;
            goto error;
        }
    } else {
        av_log(smk->avctx, AV_LOG_ERROR, "Skipping low bytes tree\n");
    }
    if (bitstream_read_bit(bc)) {
        smacker_decode_tree(bc, &tmp2, 0, 0);
        bitstream_skip(bc, 1);
        res = init_vlc(&vlc[1], SMKTREE_BITS, tmp2.length,
                    tmp2.lengths, sizeof(int), sizeof(int),
                    tmp2.bits, sizeof(uint32_t), sizeof(uint32_t), INIT_VLC_LE);
        if(res < 0) {
            av_log(smk->avctx, AV_LOG_ERROR, "Cannot build VLC table\n");
            err = res;
            goto error;
        }
    } else {
        av_log(smk->avctx, AV_LOG_ERROR, "Skipping high bytes tree\n");
    }

    escapes[0]  = bitstream_read(bc, 8);
    escapes[0] |= bitstream_read(bc, 8) << 8;
    escapes[1]  = bitstream_read(bc, 8);
    escapes[1] |= bitstream_read(bc, 8) << 8;
    escapes[2]  = bitstream_read(bc, 8);
    escapes[2] |= bitstream_read(bc, 8) << 8;

    last[0] = last[1] = last[2] = -1;

    ctx.escapes[0] = escapes[0];
    ctx.escapes[1] = escapes[1];
    ctx.escapes[2] = escapes[2];
    ctx.v1 = &vlc[0];
    ctx.v2 = &vlc[1];
    ctx.recode1 = tmp1.values;
    ctx.recode2 = tmp2.values;
    ctx.last = last;

    huff.length = ((size + 3) >> 2) + 4;
    huff.maxlength = 0;
    huff.current = 0;
    huff.values = av_mallocz(huff.length * sizeof(int));
    if (!huff.values) {
        err = AVERROR(ENOMEM);
        goto error;
    }

    if ((res = smacker_decode_bigtree(bc, &huff, &ctx)) < 0)
        err = res;
    bitstream_skip(bc, 1);
    if(ctx.last[0] == -1) ctx.last[0] = huff.current++;
    if(ctx.last[1] == -1) ctx.last[1] = huff.current++;
    if(ctx.last[2] == -1) ctx.last[2] = huff.current++;
    if (ctx.last[0] >= huff.length ||
        ctx.last[1] >= huff.length ||
        ctx.last[2] >= huff.length) {
        av_log(smk->avctx, AV_LOG_ERROR, "Huffman codes out of range\n");
        err = AVERROR_INVALIDDATA;
    }

    *recodes = huff.values;

error:
    if(vlc[0].table)
        ff_free_vlc(&vlc[0]);
    if(vlc[1].table)
        ff_free_vlc(&vlc[1]);
    av_free(tmp1.bits);
    av_free(tmp1.lengths);
    av_free(tmp1.values);
    av_free(tmp2.bits);
    av_free(tmp2.lengths);
    av_free(tmp2.values);

    return err;
}
