commit 4f07a3aa2c6b7356c28646692261aa9080605fcc
author Michael Niedermayer <michaelni@gmx.at> Thu Jul 28 14:59:54 2011 +0200
committer Reinhard Tartler <siretart@tauware.de> Wed Nov 02 21:49:55 2011 +0100
tree 656e8e76ecf1e72765415f7c6d564f06262364d3
parent 04888edef344b09daaabbc30b3fa5ab32d3bd866

Fix memory (re)allocation in matroskadec.c, related to MSVR-11-0080.

Whitespace of the patch cleaned up by Aurel
Some of the issues have been reported by Steve Manzuik / Microsoft Vulnerability Research (MSVR)
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

(cherry picked from commit 956c901c68eff78288f40e3c8f41ee2fa081d4a8)

Further suggestions from Kostya <kostya.shishkov@gmail.com> have been
implemented by Reinhard Tartler <siretart@tauware.de>

Signed-off-by: Reinhard Tartler <siretart@tauware.de>
(cherry picked from commit 77d2ef13a8fa630e5081f14bde3fd20f84c90aec)

NB: MSVR-11-0080 doesn't seem to exist. This issue seems to be known
as MSVR11-011 instead.

Fixes: CVE-2011-3504

Signed-off-by: Reinhard Tartler <siretart@tauware.de>

libavformat/matroskadec.c

diff --git a/libavformat/matroskadec.c b/libavformat/matroskadec.c
index ac12f1e..d9ffec3 100644
--- a/libavformat/matroskadec.c
+++ b/libavformat/matroskadec.c
@@ -752,11 +752,15 @@
     uint32_t id = syntax->id;
     uint64_t length;
     int res;
+    void *newelem;
 
     data = (char *)data + syntax->data_offset;
     if (syntax->list_elem_size) {
         EbmlList *list = data;
-        list->elem = av_realloc(list->elem, (list->nb_elem+1)*syntax->list_elem_size);
+        newelem = av_realloc(list->elem, (list->nb_elem+1)*syntax->list_elem_size);
+        if (!newelem)
+            return AVERROR(ENOMEM);
+        list->elem = newelem;
         data = (char*)list->elem + list->nb_elem*syntax->list_elem_size;
         memset(data, 0, syntax->list_elem_size);
         list->nb_elem++;
@@ -873,6 +877,7 @@
     uint8_t* data = *buf;
     int isize = *buf_size;
     uint8_t* pkt_data = NULL;
+    uint8_t* newpktdata;
     int pkt_size = isize;
     int result = 0;
     int olen;
@@ -899,7 +904,12 @@
         zstream.avail_in = isize;
         do {
             pkt_size *= 3;
-            pkt_data = av_realloc(pkt_data, pkt_size);
+            newpktdata = av_realloc(pkt_data, pkt_size);
+            if (!newpktdata) {
+                inflateEnd(&zstream);
+                goto failed;
+            }
+            pkt_data = newpktdata;
             zstream.avail_out = pkt_size - zstream.total_out;
             zstream.next_out = pkt_data + zstream.total_out;
             result = inflate(&zstream, Z_NO_FLUSH);
@@ -920,7 +930,12 @@
         bzstream.avail_in = isize;
         do {
             pkt_size *= 3;
-            pkt_data = av_realloc(pkt_data, pkt_size);
+            newpktdata = av_realloc(pkt_data, pkt_size);
+            if (!newpktdata) {
+                BZ2_bzDecompressEnd(&bzstream);
+                goto failed;
+            }
+            pkt_data = newpktdata;
             bzstream.avail_out = pkt_size - bzstream.total_out_lo32;
             bzstream.next_out = pkt_data + bzstream.total_out_lo32;
             result = BZ2_bzDecompress(&bzstream);
@@ -975,13 +990,17 @@
     }
 }
 
-static void matroska_merge_packets(AVPacket *out, AVPacket *in)
+static int matroska_merge_packets(AVPacket *out, AVPacket *in)
 {
-    out->data = av_realloc(out->data, out->size+in->size);
+    void *newdata = av_realloc(out->data, out->size+in->size);
+    if (!newdata)
+        return AVERROR(ENOMEM);
+    out->data = newdata;
     memcpy(out->data+out->size, in->data, in->size);
     out->size += in->size;
     av_destruct_packet(in);
     av_free(in);
+    return 0;
 }
 
 static void matroska_convert_tag(AVFormatContext *s, EbmlList *list,
@@ -1462,11 +1481,13 @@
         memcpy(pkt, matroska->packets[0], sizeof(AVPacket));
         av_free(matroska->packets[0]);
         if (matroska->num_packets > 1) {
+            void *newpackets;
             memmove(&matroska->packets[0], &matroska->packets[1],
                     (matroska->num_packets - 1) * sizeof(AVPacket *));
-            matroska->packets =
-                av_realloc(matroska->packets, (matroska->num_packets - 1) *
-                           sizeof(AVPacket *));
+            newpackets = av_realloc(matroska->packets,
+                            (matroska->num_packets - 1) * sizeof(AVPacket *));
+            if (newpackets)
+                matroska->packets = newpackets;
         } else {
             av_freep(&matroska->packets);
         }