qdm2: check output buffer size before decoding
(cherry picked from commit 7d49f79f1cd47783a963a757a6563b9cac29db62)
Signed-off-by: Reinhard Tartler <siretart@tauware.de>
(cherry picked from commit 73472053516f82b7d273a3d42c583f894077a191)
Conflicts:
libavcodec/qdm2.c
(cherry picked from commit cfb9b47a1ecdc9e88e6561aa213d98245ee70267)
Signed-off-by: Reinhard Tartler <siretart@tauware.de>
diff --git a/libavcodec/qdm2.c b/libavcodec/qdm2.c
index 95c8c97..023fe99 100644
--- a/libavcodec/qdm2.c
+++ b/libavcodec/qdm2.c
@@ -1974,13 +1974,20 @@
{
QDM2Context *s = avctx->priv_data;
int16_t *out = data;
- int i;
+ int i, out_size;
if(!buf)
return 0;
if(buf_size < s->checksum_size)
return -1;
+ out_size = 16 * s->channels * s->frame_size *
+ av_get_bits_per_sample_format(avctx->sample_fmt)/8;
+ if (*data_size < out_size) {
+ av_log(avctx, AV_LOG_ERROR, "Output buffer is too small\n");
+ return AVERROR(EINVAL);
+ }
+
av_log(avctx, AV_LOG_DEBUG, "decode(%d): %p[%d] -> %p[%d]\n",
buf_size, buf, s->checksum_size, data, *data_size);
@@ -1990,7 +1997,7 @@
out += s->channels * s->frame_size;
}
- *data_size = (uint8_t*)out - (uint8_t*)data;
+ *data_size = out_size;
return buf_size;
}