commit 5629c3910188182a23ca0d46abd5a2350f502c58
author Alex Converse <alex.converse@gmail.com> Thu Jan 26 17:30:49 2012 +0100
committer Reinhard Tartler <siretart@tauware.de> Mon May 28 23:46:08 2012 +0200
tree b79ea139abc6f0412c178a6f3fe577c4ae428614
parent 7867cbaf6c2583642c8f3a909a9c6f00ec01b162

kmvc: Check palsize.

Fixes: CVE-2011-3952

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Based on fix by Michael Niedermayer
(cherry picked from commit 386741f887714d3e46c9e8fe577e326a7964037b)
(cherry picked from commit 416849f2e06227b1b4a451c392f100db1d709a0c)

Signed-off-by: Reinhard Tartler <siretart@tauware.de>
(cherry picked from commit e7392dc349291eb94379d8cfb7ef73d32a768858)

Signed-off-by: Reinhard Tartler <siretart@tauware.de>

libavcodec/kmvc.c

diff --git a/libavcodec/kmvc.c b/libavcodec/kmvc.c
index 30939ab..69b5937 100644
--- a/libavcodec/kmvc.c
+++ b/libavcodec/kmvc.c
@@ -33,6 +33,7 @@
 #define KMVC_KEYFRAME 0x80
 #define KMVC_PALETTE  0x40
 #define KMVC_METHOD   0x0F
+#define MAX_PALSIZE   256
 
 /*
  * Decoder context
@@ -43,7 +44,7 @@
 
     int setpal;
     int palsize;
-    uint32_t pal[256];
+    uint32_t pal[MAX_PALSIZE];
     uint8_t *cur, *prev;
     uint8_t *frm0, *frm1;
 } KmvcContext;
@@ -366,6 +367,10 @@
         c->palsize = 127;
     } else {
         c->palsize = AV_RL16(avctx->extradata + 10);
+        if (c->palsize >= MAX_PALSIZE) {
+            av_log(avctx, AV_LOG_ERROR, "KMVC palette too large\n");
+            return AVERROR_INVALIDDATA;
+        }
     }
 
     if (avctx->extradata_size == 1036) {        // palette in extradata