From: Monty <xiphmont@xiph.org>
Date: Tue, 18 Mar 2008 15:39:43 +0000 (+0000)
Subject: Add code to prevent heap attacks by exploiting dim=bignum and
X-Git-Tag: v1.3.3~247
X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=7fae73780c2a6eaa4846bf11c38576109170ad54;p=platform%2Fupstream%2Flibvorbis.git

Add code to prevent heap attacks by exploiting dim=bignum and
partition_codewords = partion_values^dim.  partition_codewords is
actually overdetermined; in the case of inconsistency, mark stream
undecodable.


svn path=/trunk/vorbis/; revision=14598
---

diff --git a/doc/Vorbis_I_spec.pdf b/doc/Vorbis_I_spec.pdf
index c16352c..f9a4332 100644
Binary files a/doc/Vorbis_I_spec.pdf and b/doc/Vorbis_I_spec.pdf differ
diff --git a/doc/xml/08-residue.xml b/doc/xml/08-residue.xml
index c5df3be..2141be0 100644
--- a/doc/xml/08-residue.xml
+++ b/doc/xml/08-residue.xml
@@ -199,21 +199,29 @@ Header decode for all three residue types is identical.</para>
 </programlisting>
 
 <para>
-<varname>[residue_begin]</varname> and <varname>[residue_end]</varname> select the specific
-sub-portion of each vector that is actually coded; it implements akin
-to a bandpass where, for coding purposes, the vector effectively
-begins at element <varname>[residue_begin]</varname> and ends at
-<varname>[residue_end]</varname>.  Preceding and following values in the unpacked
-vectors are zeroed.  Note that for residue type 2, these values as
-well as <varname>[residue_partition_size]</varname>apply to the interleaved
-vector, not the individual vectors before interleave.
+<varname>[residue_begin]</varname> and
+<varname>[residue_end]</varname> select the specific sub-portion of
+each vector that is actually coded; it implements akin to a bandpass
+where, for coding purposes, the vector effectively begins at element
+<varname>[residue_begin]</varname> and ends at
+<varname>[residue_end]</varname>.  Preceding and following values in
+the unpacked vectors are zeroed.  Note that for residue type 2, these
+values as well as <varname>[residue_partition_size]</varname>apply to
+the interleaved vector, not the individual vectors before interleave.
 <varname>[residue_partition_size]</varname> is as explained above,
 <varname>[residue_classifications]</varname> is the number of possible
 classification to which a partition can belong and
-<varname>[residue_classbook]</varname> is the codebook number used to code
-classification codewords.  The number of dimensions in book
-<varname>[residue_classbook]</varname> determines how many classification values
-are grouped into a single classification codeword.</para>
+<varname>[residue_classbook]</varname> is the codebook number used to
+code classification codewords.  The number of dimensions in book
+<varname>[residue_classbook]</varname> determines how many
+classification values are grouped into a single classification
+codeword.  Note that the number of entries and dimensions in book
+<varname>[residue_classbook]</varname>, along with
+<varname>[residue_classifications]</varname>, overdetermines to
+possible number of classification codewords.  If
+<varname>[residue_classifications]</varname>^<varname>[residue_classbook]</varname>.dimensions
+does not equal <varname>[residue_classbook]</varname>.entries, the
+bitstream should be regarded to be undecodable. </para>
 
 <para>
 Next we read a bitmap pattern that specifies which partition classes
diff --git a/lib/misc.c b/lib/misc.c
index c2a66a9..90c1eb8 100644
--- a/lib/misc.c
+++ b/lib/misc.c
@@ -190,7 +190,10 @@ void _VDBG_dump(void){
   pthread_mutex_unlock(&memlock);
 }
 
-extern void *_VDBG_malloc(void *ptr,long bytes,char *file,long line){
+void *_VDBG_malloc(void *ptr,long bytes,char *file,long line){
+  if(bytes<=0)
+    fprintf(stderr,"bad malloc request (%ld bytes) from %s:%ld\n",bytes,file,line);
+
   bytes+=HEAD_ALIGN;
   if(ptr){
     ptr-=HEAD_ALIGN;
@@ -203,7 +206,7 @@ extern void *_VDBG_malloc(void *ptr,long bytes,char *file,long line){
   return _insert(ptr,bytes,file,line);
 }
 
-extern void _VDBG_free(void *ptr,char *file,long line){
+void _VDBG_free(void *ptr,char *file,long line){
   if(ptr){
     ptr-=HEAD_ALIGN;
     _ripremove(ptr);
diff --git a/lib/misc.h b/lib/misc.h
index de06c38..c50fcdc 100644
--- a/lib/misc.h
+++ b/lib/misc.h
@@ -19,6 +19,8 @@
 #define _V_RANDOM_H_
 #include "vorbis/codec.h"
 
+#define DEBUG_MALLOC
+
 extern int analysis_noisy;
 
 extern void *_vorbis_block_alloc(vorbis_block *vb,long bytes);
@@ -29,6 +31,7 @@ extern void _analysis_output(char *base,int i,float *v,int n,int bark,int dB,
 #ifdef DEBUG_MALLOC
 
 #define _VDBG_GRAPHFILE "malloc.m"
+#undef _VDBG_GRAPHFILE
 extern void *_VDBG_malloc(void *ptr,long bytes,char *file,long line); 
 extern void _VDBG_free(void *ptr,char *file,long line); 
 
diff --git a/lib/res0.c b/lib/res0.c
index 7b73121..1e78014 100644
--- a/lib/res0.c
+++ b/lib/res0.c
@@ -223,6 +223,20 @@ vorbis_info_residue *res0_unpack(vorbis_info *vi,oggpack_buffer *opb){
   for(j=0;j<acc;j++)
     if(info->booklist[j]>=ci->books)goto errout;
 
+  /* verify the phrasebook is not specifying an impossible or
+     inconsistent partitioning scheme. */
+  {
+    int entries = ci->book_param[info->groupbook]->entries;
+    int dim = ci->book_param[info->groupbook]->dim;
+    int partvals = 1;
+    while(dim>0){
+      partvals *= info->partitions;
+      if(partvals > entries) goto errout;
+      dim--;
+    }
+    if(partvals != entries) goto errout;
+  }
+
   return(info);
  errout:
   res0_free_info(info);
@@ -263,7 +277,7 @@ vorbis_look_residue *res0_look(vorbis_dsp_state *vd,
     }
   }
 
-  look->partvals=rint(pow((float)look->parts,(float)dim));
+  look->partvals=look->phrasebook->entries;
   look->stages=maxstage;
   look->decodemap=_ogg_malloc(look->partvals*sizeof(*look->decodemap));
   for(j=0;j<look->partvals;j++){