commit 468cc41d6d7bba97f67e9a6c8386ca98cde41190
author Mans Rullgard <mans@mansr.com> Mon Apr 23 13:16:33 2012 +0100
committer Reinhard Tartler <siretart@tauware.de> Wed May 09 22:34:07 2012 +0200
tree 5359a505dd0ca7d2c55ab5513e51fcd833c14f69
parent 6c9b404dba6f29e44ff9ce385c84759e73082cd2

vqavideo: return error if image size is not a multiple of block size

The decoder assumes in various places that the image size
is a multiple of the block size, and there is no obvious
way to support odd sizes.  Bailing out early if the header
specifies a bad size avoids various errors later on.

Fixes CVE-2012-0947.

Signed-off-by: Mans Rullgard <mans@mansr.com>
(cherry picked from commit 58b2e0f0f2fc96c1158e04f8aba95cbe6157a1a3)

Signed-off-by: Reinhard Tartler <siretart@tauware.de>
(cherry picked from commit d5207e2af81580dd5e6277b354c8b459c3624f26)

Signed-off-by: Reinhard Tartler <siretart@tauware.de>
(cherry picked from commit c71c77e56fcc6d469d45e1c8ce04aa053124d3f8)

Signed-off-by: Reinhard Tartler <siretart@tauware.de>
(cherry picked from commit c90da45d5a7a4045dbf22fba52c63ef55d207269)

Signed-off-by: Reinhard Tartler <siretart@tauware.de>

libavcodec/vqavideo.c

diff --git a/libavcodec/vqavideo.c b/libavcodec/vqavideo.c
index 00df736..f34a631 100644
--- a/libavcodec/vqavideo.c
+++ b/libavcodec/vqavideo.c
@@ -163,6 +163,12 @@
         return -1;
     }
 
+    if (s->width  & (s->vector_width  - 1) ||
+        s->height & (s->vector_height - 1)) {
+        av_log(avctx, AV_LOG_ERROR, "Image size not multiple of block size\n");
+        return AVERROR_INVALIDDATA;
+    }
+
     /* allocate codebooks */
     s->codebook_size = MAX_CODEBOOK_SIZE;
     s->codebook = av_malloc(s->codebook_size);