From: Monty <xiphmont@xiph.org>
Date: Wed, 5 Jan 2000 10:15:00 +0000 (+0000)
Subject: More VQ utility work.  Utils now include:
X-Git-Tag: v1.3.3~1298
X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=fabd9fca0c0543344a67455c5c285772a968d2c6;p=platform%2Fupstream%2Flibvorbis.git

More VQ utility work.  Utils now include:

lspvqtrain
genericvqtrain
vqbuild
vqcascade

Monty

svn path=/trunk/vorbis/; revision=224
---

diff --git a/vq/Makefile.in b/vq/Makefile.in
index 49e6cc1..5e4ba0e 100644
--- a/vq/Makefile.in
+++ b/vq/Makefile.in
@@ -1,4 +1,4 @@
-# $Id: Makefile.in,v 1.5 2000/01/05 03:11:10 xiphmont Exp $
+# $Id: Makefile.in,v 1.6 2000/01/05 10:14:53 xiphmont Exp $
 
 ###############################################################################
 #                                                                             #
@@ -25,10 +25,11 @@ AR=@AR@
 RANLIB=@RANLIB@
 LIBS=@LIBS@ -lm
 
-HFILES =	../include/vorbis/codebook.h vqgen.h vqext.h
+HFILES =	../include/vorbis/codebook.h vqgen.h vqext.h bookutil.h
 
-OFILES = 	vqgen.o vqsplit.o 
-ALLOFILES = 	$(OFILES) lspdata.o train.o build.o
+OFILES = 	vqgen.o vqsplit.o bookutil.o
+ALLOFILES = 	$(OFILES) lspdata.o genericdata.o train.o build.o run.o\
+		cascade.o
 
 all:
 	$(MAKE) target CFLAGS="$(OPT)"
@@ -39,14 +40,20 @@ debug:
 profile: 
 	$(MAKE) target CFLAGS="$(PROFILE)"
 
-target:	lspvqtrain vqbuild
+target:	lspvqtrain genericvqtrain vqbuild vqcascade
 
 lspvqtrain:	$(OFILES) lspdata.o train.o
 	$(CC) $(CFLAGS) $(LDFLAGS) $^ -o $@ $(LIBS)
 
+genericvqtrain:	$(OFILES) genericdata.o train.o
+	$(CC) $(CFLAGS) $(LDFLAGS) $^ -o $@ $(LIBS)
+
 vqbuild:	$(OFILES) build.o
 	$(CC) $(CFLAGS) $(LDFLAGS) $^ -o $@ $(LIBS)
 
+vqcascade:	$(OFILES) run.o cascade.o
+	$(CC) $(CFLAGS) $(LDFLAGS) $^ -o $@ $(LIBS)
+
 $(ALLOFILES): 	$(HFILES)
 
 .c.o:
diff --git a/vq/bookutil.c b/vq/bookutil.c
new file mode 100644
index 0000000..9639ade
--- /dev/null
+++ b/vq/bookutil.c
@@ -0,0 +1,347 @@
+/********************************************************************
+ *                                                                  *
+ * THIS FILE IS PART OF THE Ogg Vorbis SOFTWARE CODEC SOURCE CODE.  *
+ * USE, DISTRIBUTION AND REPRODUCTION OF THIS SOURCE IS GOVERNED BY *
+ * THE GNU PUBLIC LICENSE 2, WHICH IS INCLUDED WITH THIS SOURCE.    *
+ * PLEASE READ THESE TERMS DISTRIBUTING.                            *
+ *                                                                  *
+ * THE OggSQUISH SOURCE CODE IS (C) COPYRIGHT 1994-2000             *
+ * by Monty <monty@xiph.org> and The XIPHOPHORUS Company            *
+ * http://www.xiph.org/                                             *
+ *                                                                  *
+ ********************************************************************
+
+ function: utility functions for loading .vqh and .vqd files
+ last mod: $Id: bookutil.c,v 1.1 2000/01/05 10:14:54 xiphmont Exp $
+
+ ********************************************************************/
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <math.h>
+#include <string.h>
+#include <errno.h>
+#include "vorbis/codebook.h"
+#include "bookutil.h"
+
+void codebook_unquantize(codebook *b){
+  long j,k;
+  double mindel=float24_unpack(b->q_min);
+  double delta=float24_unpack(b->q_delta);
+  if(!b->valuelist)b->valuelist=malloc(sizeof(double)*b->entries*b->dim);
+  
+  for(j=0;j<b->entries;j++){
+    double last=0.;
+    for(k=0;k<b->dim;k++){
+      double val=b->quantlist[j*b->dim+k]*delta+last+mindel;
+      b->valuelist[j*b->dim+k]=val;
+      if(b->q_sequencep)last=val;
+
+    }
+  }
+}
+
+/* A few little utils for reading files */
+/* read a line.  Use global, persistent buffering */
+static char *linebuffer=NULL;
+static int  lbufsize=0;
+char *get_line(FILE *in){
+  long sofar=0;
+  if(feof(in))return NULL;
+
+  while(1){
+    int gotline=0;
+
+    while(!gotline){
+      if(sofar>=lbufsize){
+        if(!lbufsize){  
+          lbufsize=1024;
+          linebuffer=malloc(lbufsize);
+        }else{
+          lbufsize*=2;
+          linebuffer=realloc(linebuffer,lbufsize);
+        }
+      }
+      {
+        long c=fgetc(in);
+        switch(c){
+        case '\n':
+        case EOF:
+          gotline=1;
+          break;
+        default:
+          linebuffer[sofar++]=c;
+          linebuffer[sofar]='\0';
+          break;
+        }
+      }
+    }
+    
+    if(linebuffer[0]=='#'){
+      sofar=0;
+    }else{
+      return(linebuffer);
+    }
+  }
+}
+
+/* read the next numerical value from the given file */
+static char *value_line_buff=NULL;
+
+int get_line_value(FILE *in,double *value){
+  char *next;
+
+  if(!value_line_buff)return(-1);
+
+  *value=strtod(value_line_buff, &next);
+  if(next==value_line_buff){
+    value_line_buff=NULL;
+    return(-1);
+  }else{
+    value_line_buff=next+1;
+    return(0);
+  }
+}
+
+int get_next_value(FILE *in,double *value){
+  while(1){
+    if(get_line_value(in,value)){
+      value_line_buff=get_line(in);
+      if(!value_line_buff)return(-1);
+    }else{
+      return(0);
+    }
+  }
+}
+
+int get_next_ivalue(FILE *in,long *ivalue){
+  double value;
+  int ret=get_next_value(in,&value);
+  *ivalue=value;
+  return(ret);
+}
+
+static double sequence_base=0.;
+void reset_next_value(void){
+  value_line_buff=NULL;
+  sequence_base=0.;
+}
+
+int get_vector(codebook *b,FILE *in,double *a){
+  int i;
+
+  while(1){
+
+    if(get_next_value(in,a))
+      break;
+
+    for(i=1;i<b->dim;i++)
+      if(get_line_value(in,a+i))
+	break;
+    
+    if(i==b->dim){
+      for(i=0;i<b->dim;i++)a[i]-=sequence_base;
+      if(b->q_sequencep)sequence_base=a[b->dim-1];
+      return(0);
+    }
+    sequence_base=0.;
+  }
+
+  return(-1);
+}
+
+/* read lines fromt he beginning until we find one containing the
+   specified string */
+char *find_seek_to(FILE *in,char *s){
+  rewind(in);
+  while(1){
+    char *line=get_line(in);
+    if(line){
+      if(strstr(line,s))
+	return(line);
+    }else
+      return(NULL);
+  }
+}
+
+
+/* this reads the format as written by vqbuild; innocent (legal)
+   tweaking of the file that would not affect its valid header-ness
+   will break this routine */
+
+codebook *codebook_load(char *filename){
+  codebook *b=calloc(1,sizeof(codebook));
+  encode_aux *a=calloc(1,sizeof(encode_aux));
+  FILE *in=fopen(filename,"r");
+  char *line;
+  long i;
+
+  b->encode_tree=a;
+
+  if(in==NULL){
+    fprintf(stderr,"Couldn't open codebook %s\n",filename);
+    exit(1);
+  }
+
+  /* find the codebook struct */
+  find_seek_to(in,"static codebook _vq_book_");
+
+  /* get the major important values */
+  line=get_line(in);
+  if(sscanf(line,"%ld, %ld, %ld, %ld, %d, %d",
+	    &(b->dim),&(b->entries),&(b->q_min),&(b->q_delta),&(b->q_quant),
+	    &(b->q_sequencep))!=6){
+    fprintf(stderr,"syntax in %s in line:\t %s",filename,line);
+    exit(1);
+  }
+
+  /* find the auxiliary encode struct (if any) */
+  find_seek_to(in,"static encode_aux _vq_aux_");
+  /* how big? */
+  line=get_line(in);
+  line=get_line(in);
+  line=get_line(in);
+  line=get_line(in);
+  line=get_line(in);
+  line=get_line(in);
+  line=get_line(in);
+  if(sscanf(line,"%ld, %ld",&(a->aux),&(a->alloc))!=2){
+    fprintf(stderr,"syntax in %s in line:\t %s",filename,line);
+    exit(1);
+  }
+    
+  /* load the quantized entries */
+  find_seek_to(in,"static long _vq_quantlist_");
+  reset_next_value();
+  b->quantlist=malloc(sizeof(long)*b->entries*b->dim);
+  for(i=0;i<b->entries*b->dim;i++)
+    if(get_next_ivalue(in,b->quantlist+i)){
+      fprintf(stderr,"out of data while reading codebook %s\n",filename);
+      exit(1);
+    }
+
+  /* load the codewords */
+  find_seek_to(in,"static long _vq_codelist");
+  reset_next_value();
+  b->codelist=malloc(sizeof(long)*b->entries);
+  for(i=0;i<b->entries;i++)
+    if(get_next_ivalue(in,b->codelist+i)){
+      fprintf(stderr,"out of data while reading codebook %s\n",filename);
+      exit(1);
+    }
+
+  /* load the lengthlist */
+  find_seek_to(in,"static long _vq_lengthlist");
+  reset_next_value();
+  b->lengthlist=malloc(sizeof(long)*b->entries);
+  for(i=0;i<b->entries;i++)
+    if(get_next_ivalue(in,b->lengthlist+i)){
+      fprintf(stderr,"out of data while reading codebook %s\n",filename);
+      exit(1);
+    }
+
+  /* load ptr0 */
+  find_seek_to(in,"static long _vq_ptr0");
+  reset_next_value();
+  a->ptr0=malloc(sizeof(long)*a->aux);
+  for(i=0;i<a->aux;i++)
+    if(get_next_ivalue(in,a->ptr0+i)){
+      fprintf(stderr,"out of data while reading codebook %s\n",filename);
+      exit(1);
+    }
+
+  /* load ptr1 */
+  find_seek_to(in,"static long _vq_ptr1");
+  reset_next_value();
+  a->ptr1=malloc(sizeof(long)*a->aux);
+  for(i=0;i<a->aux;i++)
+    if(get_next_ivalue(in,a->ptr1+i)){
+      fprintf(stderr,"out of data while reading codebook %s\n",filename);
+      exit(1);
+    }
+
+
+  /* load p */
+  find_seek_to(in,"static long _vq_p_");
+  reset_next_value();
+  a->p=malloc(sizeof(long)*a->aux);
+  for(i=0;i<a->aux;i++)
+    if(get_next_ivalue(in,a->p+i)){
+      fprintf(stderr,"out of data while reading codebook %s\n",filename);
+      exit(1);
+    }
+
+  /* load q */
+  find_seek_to(in,"static long _vq_q_");
+  reset_next_value();
+  a->q=malloc(sizeof(long)*a->aux);
+  for(i=0;i<a->aux;i++)
+    if(get_next_ivalue(in,a->q+i)){
+      fprintf(stderr,"out of data while reading codebook %s\n",filename);
+      exit(1);
+    }
+
+  /* got it all */
+  fclose(in);
+
+  /* might as well unquantize the entries while we're at it */
+  codebook_unquantize(b);
+
+  /* don't need n and c */
+  return(b);
+}
+
+int codebook_entry(codebook *b,double *val){
+  encode_aux *t=b->encode_tree;
+  int ptr=0,k;
+  double *n=alloca(b->dim*sizeof(double));
+
+  while(1){
+    double c=0.;
+    double *p=b->valuelist+t->p[ptr]*b->dim;
+    double *q=b->valuelist+t->q[ptr]*b->dim;
+    
+    for(k=0;k<b->dim;k++){
+      n[k]=p[k]-q[k];
+      c-=(p[k]+q[k])*n[k];
+    }
+    c/=2.;
+
+    for(k=0;k<b->dim;k++)
+      c+=n[k]*val[k];
+    if(c>0.) /* in A */
+      ptr= -t->ptr0[ptr];
+    else     /* in B */
+      ptr= -t->ptr1[ptr];
+    if(ptr<=0)break;
+  }
+  return(-ptr);
+}
+
+/* 24 bit float (not IEEE; nonnormalized mantissa +
+   biased exponent ): neeeeemm mmmmmmmm mmmmmmmm */
+
+#define VQ_FEXP_BIAS 20 /* bias toward values smaller than 1. */
+long float24_pack(double val){
+  int sign=0;
+  long exp;
+  long mant;
+  if(val<0){
+    sign=0x800000;
+    val= -val;
+  }
+  exp= floor(log(val)/log(2));
+  mant=rint(ldexp(val,17-exp));
+  exp=(exp+VQ_FEXP_BIAS)<<18;
+
+  return(sign|exp|mant);
+}
+
+double float24_unpack(long val){
+  double mant=val&0x3ffff;
+  double sign=val&0x800000;
+  double exp =(val&0x7c0000)>>18;
+  if(sign)mant= -mant;
+  return(ldexp(mant,exp-17-VQ_FEXP_BIAS));
+}
+
diff --git a/vq/bookutil.h b/vq/bookutil.h
new file mode 100644
index 0000000..174fa48
--- /dev/null
+++ b/vq/bookutil.h
@@ -0,0 +1,42 @@
+/********************************************************************
+ *                                                                  *
+ * THIS FILE IS PART OF THE Ogg Vorbis SOFTWARE CODEC SOURCE CODE.  *
+ * USE, DISTRIBUTION AND REPRODUCTION OF THIS SOURCE IS GOVERNED BY *
+ * THE GNU PUBLIC LICENSE 2, WHICH IS INCLUDED WITH THIS SOURCE.    *
+ * PLEASE READ THESE TERMS DISTRIBUTING.                            *
+ *                                                                  *
+ * THE OggSQUISH SOURCE CODE IS (C) COPYRIGHT 1994-2000             *
+ * by Monty <monty@xiph.org> and The XIPHOPHORUS Company            *
+ * http://www.xiph.org/                                             *
+ *                                                                  *
+ ********************************************************************
+
+ function: utility functions for loading .vqh and .vqd files
+ last mod: $Id: bookutil.h,v 1.1 2000/01/05 10:14:54 xiphmont Exp $
+
+ ********************************************************************/
+
+#ifndef _V_BOOKUTIL_H_
+#define _V_BOOKUTIL_H_
+
+#include <stdio.h>
+#include "vorbis/codebook.h"
+
+extern void      codebook_unquantize(codebook *b);
+extern char     *get_line(FILE *in);
+extern int       get_line_value(FILE *in,double *value);
+extern int       get_next_value(FILE *in,double *value);
+extern int       get_next_ivalue(FILE *in,long *ivalue);
+extern void      reset_next_value(void);
+extern int       get_vector(codebook *b,FILE *in,double *a);
+extern char     *find_seek_to(FILE *in,char *s);
+
+extern codebook *codebook_load(char *filename);
+extern int       codebook_entry(codebook *b,double *val);
+
+
+extern long float24_pack(double val);
+extern double float24_unpack(long val);
+
+#endif
+
diff --git a/vq/build.c b/vq/build.c
index a7f5e93..470db7d 100644
--- a/vq/build.c
+++ b/vq/build.c
@@ -12,7 +12,7 @@
  ********************************************************************
 
  function: utility main for building codebooks from training sets
- last mod: $Id: build.c,v 1.7 1999/12/30 07:26:59 xiphmont Exp $
+ last mod: $Id: build.c,v 1.8 2000/01/05 10:14:54 xiphmont Exp $
 
  ********************************************************************/
 
@@ -21,7 +21,10 @@
 #include <math.h>
 #include <string.h>
 #include <errno.h>
+#include "vorbis/codebook.h"
+
 #include "vqgen.h"
+#include "vqsplit.h"
 
 static char *linebuffer=NULL;
 static int  lbufsize=0;
@@ -71,10 +74,10 @@ static char *rline(FILE *in,FILE *out){
 
 int main(int argc,char *argv[]){
   vqgen v;
-  vqbook b;
+  codebook b;
   quant_meta q;
-  long *quantlist=NULL;
 
+  long *quantlist=NULL;
   int entries=-1,dim=-1,aux=-1;
   FILE *out=NULL;
   FILE *in=NULL;
@@ -198,7 +201,7 @@ int main(int argc,char *argv[]){
  " ********************************************************************/\n\n");
 
   fprintf(out,"#ifndef _V_%s_VQH_\n#define _V_%s_VQH_\n",name,name);
-  fprintf(out,"#include \"vqgen.h\"\n\n");
+  fprintf(out,"#include \"vorbis/codebook.h\"\n\n");
 
   /* first, the static vectors, then the book structure to tie it together. */
   /* quantlist */
@@ -234,59 +237,65 @@ int main(int argc,char *argv[]){
 
   /* ptr0 */
   fprintf(out,"static long _vq_ptr0_%s[] = {\n",name);
-  for(j=0;j<b.aux;){
+  for(j=0;j<b.encode_tree->aux;){
     fprintf(out,"\t");
-    for(k=0;k<8 && j<b.aux;k++,j++)
-      fprintf(out,"%5ld,",b.ptr0[j]);
+    for(k=0;k<8 && j<b.encode_tree->aux;k++,j++)
+      fprintf(out,"%6ld,",b.encode_tree->ptr0[j]);
     fprintf(out,"\n");
   }
   fprintf(out,"};\n\n");
 
   /* ptr1 */
   fprintf(out,"static long _vq_ptr1_%s[] = {\n",name);
-  for(j=0;j<b.aux;){
+  for(j=0;j<b.encode_tree->aux;){
     fprintf(out,"\t");
-    for(k=0;k<8 && j<b.aux;k++,j++)
-      fprintf(out,"%6ld,",b.ptr1[j]);
+    for(k=0;k<8 && j<b.encode_tree->aux;k++,j++)
+      fprintf(out,"%6ld,",b.encode_tree->ptr1[j]);
     fprintf(out,"\n");
   }
   fprintf(out,"};\n\n");
 
   /* p */
   fprintf(out,"static long _vq_p_%s[] = {\n",name);
-  for(j=0;j<b.aux;){
+  for(j=0;j<b.encode_tree->aux;){
     fprintf(out,"\t");
-    for(k=0;k<8 && j<b.aux;k++,j++)
-      fprintf(out,"%6ld,",b.p[j]);
+    for(k=0;k<8 && j<b.encode_tree->aux;k++,j++)
+      fprintf(out,"%6ld,",b.encode_tree->p[j]);
     fprintf(out,"\n");
   }
   fprintf(out,"};\n\n");
 
   /* q */
   fprintf(out,"static long _vq_q_%s[] = {\n",name);
-  for(j=0;j<b.aux;){
+  for(j=0;j<b.encode_tree->aux;){
     fprintf(out,"\t");
-    for(k=0;k<8 && j<b.aux;k++,j++)
-      fprintf(out,"%6ld,",b.q[j]);
+    for(k=0;k<8 && j<b.encode_tree->aux;k++,j++)
+      fprintf(out,"%6ld,",b.encode_tree->q[j]);
     fprintf(out,"\n");
   }
   fprintf(out,"};\n\n");
 
   /* tie it all together */
-  fprintf(out,"static vqbook _vq_book_%s = {\n",name);
+
+  fprintf(out,"static encode_aux _vq_aux_%s = {\n",name);
+  fprintf(out,"\t_vq_ptr0_%s,\n",name);
+  fprintf(out,"\t_vq_ptr1_%s,\n",name);
+  fprintf(out,"\t0,\n");
+  fprintf(out,"\t0,\n");
+  fprintf(out,"\t_vq_p_%s,\n",name);
+  fprintf(out,"\t_vq_q_%s,\n",name);
+  fprintf(out,"\t%ld, %ld\n};\n\n",b.encode_tree->aux,b.encode_tree->aux);
+
+  fprintf(out,"static codebook _vq_book_%s = {\n",name);
   fprintf(out,"\t%ld, %ld, %ld, %ld, %d, %d,\n",
 	  b.dim,b.entries,q.min,q.delta,q.quant,q.sequencep);
   fprintf(out,"\t0,\n"); /* valuelist */
   fprintf(out,"\t_vq_quantlist_%s,\n",name);
   fprintf(out,"\t_vq_codelist_%s,\n",name);
   fprintf(out,"\t_vq_lengthlist_%s,\n",name);
-  fprintf(out,"\t_vq_ptr0_%s,\n",name);
-  fprintf(out,"\t_vq_ptr1_%s,\n",name);
-  fprintf(out,"\t0,\n",name);
-  fprintf(out,"\t0,\n",name);
-  fprintf(out,"\t_vq_p_%s,\n",name);
-  fprintf(out,"\t_vq_q_%s,\n",name);
-  fprintf(out,"\t%ld, %ld };\n\n",b.aux,b.aux);
+  fprintf(out,"\t&_vq_aux_%s,\n",name);
+  fprintf(out,"\t0\n");
+  fprintf(out,"};\n\n");
 
   fprintf(out,"\n#endif\n");
   fclose(out);
diff --git a/vq/cascade.c b/vq/cascade.c
new file mode 100644
index 0000000..a198722
--- /dev/null
+++ b/vq/cascade.c
@@ -0,0 +1,39 @@
+/********************************************************************
+ *                                                                  *
+ * THIS FILE IS PART OF THE Ogg Vorbis SOFTWARE CODEC SOURCE CODE.  *
+ * USE, DISTRIBUTION AND REPRODUCTION OF THIS SOURCE IS GOVERNED BY *
+ * THE GNU PUBLIC LICENSE 2, WHICH IS INCLUDED WITH THIS SOURCE.    *
+ * PLEASE READ THESE TERMS DISTRIBUTING.                            *
+ *                                                                  *
+ * THE OggSQUISH SOURCE CODE IS (C) COPYRIGHT 1994-2000             *
+ * by Monty <monty@xiph.org> and The XIPHOPHORUS Company            *
+ * http://www.xiph.org/                                             *
+ *                                                                  *
+ ********************************************************************
+
+ function: function call to do simple data cascading
+ last mod: $Id: cascade.c,v 1.1 2000/01/05 10:14:55 xiphmont Exp $
+
+ ********************************************************************/
+
+/* this one just outputs to stdout */
+
+#include "bookutil.h"
+
+void process_vector(codebook *b,double *a){
+  int entry=codebook_entry(b,a);
+  double *e=b->valuelist+b->dim*entry;
+  int i;
+
+  for(i=0;i<b->dim;i++)
+    fprintf(stdout,"%f, ",a[i]-e[i]);
+  fprintf(stdout,"\n");
+}
+
+void process_usage(void){
+  fprintf(stderr,
+	  "usage: vqcascade <codebook>.vqh datafile.vqd [datafile.vqd]...\n\n"
+	  "       data can be taken on stdin.  residual error data sent to\n"
+	  "       stdout.\n\n");
+
+}
diff --git a/vq/genericdata.c b/vq/genericdata.c
new file mode 100644
index 0000000..5279a2a
--- /dev/null
+++ b/vq/genericdata.c
@@ -0,0 +1,54 @@
+/********************************************************************
+ *                                                                  *
+ * THIS FILE IS PART OF THE Ogg Vorbis SOFTWARE CODEC SOURCE CODE.  *
+ * USE, DISTRIBUTION AND REPRODUCTION OF THIS SOURCE IS GOVERNED BY *
+ * THE GNU PUBLIC LICENSE 2, WHICH IS INCLUDED WITH THIS SOURCE.    *
+ * PLEASE READ THESE TERMS DISTRIBUTING.                            *
+ *                                                                  *
+ * THE OggSQUISH SOURCE CODE IS (C) COPYRIGHT 1994-2000             *
+ * by Monty <monty@xiph.org> and The XIPHOPHORUS Company            *
+ * http://www.xiph.org/                                             *
+ *                                                                  *
+ ********************************************************************
+
+ function: generic euclidian distance metric for VQ codebooks
+ last mod: $Id: genericdata.c,v 1.1 2000/01/05 10:14:55 xiphmont Exp $
+
+ ********************************************************************/
+
+#include <stdlib.h>
+#include <math.h>
+#include <stdio.h>
+#include "vqgen.h"
+#include "vqext.h"
+
+char *vqext_booktype="GENERICdata";  
+int vqext_aux=0;                
+quant_meta q={0,0,0,0};          /* non sequence data; each scalar 
+				    independent */
+
+double *vqext_weight(vqgen *v,double *p){
+  /*noop*/
+  return(p);
+}
+
+                            /* candidate,actual */
+double vqext_metric(vqgen *v,double *e, double *p){
+  int i;
+  double acc=0.;
+  for(i=0;i<v->elements;i++){
+    double val=p[i]-e[i];
+    acc+=val*val;
+  }
+  return acc;
+}
+
+void vqext_addpoint_adj(vqgen *v,double *b,int start,int dim,int cols){
+  vqgen_addpoint(v,b+start,NULL);
+}
+
+void vqext_preprocess(vqgen *v){
+  /* noop */
+}
+
+
diff --git a/vq/run.c b/vq/run.c
index 80a385c..587231a 100644
--- a/vq/run.c
+++ b/vq/run.c
@@ -1,5 +1,3 @@
-#define CODEBOOK _vq_book_lsp_256_32_44100_8
-#include "lsp_256_32_44100_8.vqh"
 /********************************************************************
  *                                                                  *
  * THIS FILE IS PART OF THE Ogg Vorbis SOFTWARE CODEC SOURCE CODE.  *
@@ -13,272 +11,86 @@
  *                                                                  *
  ********************************************************************
 
- function: utility main for loading/testing/running finished codebooks
- last mod: $Id: run.c,v 1.3 2000/01/05 03:11:11 xiphmont Exp $
+ function: utility main for loading and operating on codebooks
+ last mod: $Id: run.c,v 1.4 2000/01/05 10:14:55 xiphmont Exp $
 
  ********************************************************************/
 
+#include <unistd.h>
 #include <stdlib.h>
 #include <stdio.h>
 #include <math.h>
 #include <string.h>
 #include <errno.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <fcntl.h>
 
-/* this is a bit silly; it's a C stub used by a Perl script to build a
-   quick executable against the chosen codebook and run tests */
-
-static char *linebuffer=NULL;
-static int  lbufsize=0;
-static char *rline(FILE *in){
-  long sofar=0;
-  if(feof(in))return NULL;
-
-  while(1){
-    int gotline=0;
-
-    while(!gotline){
-      if(sofar>=lbufsize){
-	if(!lbufsize){	
-	  lbufsize=1024;
-	  linebuffer=malloc(lbufsize);
-	}else{
-	  lbufsize*=2;
-	  linebuffer=realloc(linebuffer,lbufsize);
-	}
-      }
-      {
-	long c=fgetc(in);
-	switch(c){
-	case '\n':
-	case EOF:
-	  gotline=1;
-	  break;
-	default:
-	  linebuffer[sofar++]=c;
-	  linebuffer[sofar]='\0';
-	  break;
-	}
-      }
-    }
-    
-    if(linebuffer[0]=='#'){
-      sofar=0;
-    }else{
-      return(linebuffer);
-    }
-  }
-}
-
-void vqbook_unquantize(vqbook *b){
-  long j,k;
-  double mindel=float24_unpack(b->min);
-  double delta=float24_unpack(b->delta);
-  if(!b->valuelist)b->valuelist=malloc(sizeof(double)*b->entries*b->dim);
-  
-  for(j=0;j<b->entries;j++){
-    double last=0.;
-    for(k=0;k<b->dim;k++){
-      double val=b->quantlist[j*b->dim+k]*delta+last+mindel;
-      b->valuelist[j*b->dim+k]=val;
-      if(b->sequencep)last=val;
-
-    }
-  }
-}
-
-double _ssqe(int el,double *a, double *b){
-  int i;
-  double acc=0.;
-  for(i=0;i<el;i++){
-    double val=(a[i]-b[i]);
-    acc+=val*val;
-  }
-  return acc;
-}
-
+#include "bookutil.h"
 
 /* command line:
-   run outbase [-m] [-s <start>,<n>] datafile [-s <start>,<n>] [datafile...]
+   utilname input_book.vqh input_data.vqd [input_data.vqd]
 
-   produces: outbase-residue.m (error between input data and chosen codewords;
-                                can be used to cascade)
-             outbase-cells.m   (2d gnuplot file of cells if -m)
+   produces output data on stdout
+   (may also take input data from stdin)
 
-   currently assumes a 'sequenced' codebook wants its test data to be
-   normalized to begin at zero... */
+ */
 
-int main(int argc,char *argv[]){
-  vqbook *b=&CODEBOOK;
-  FILE *cells=NULL;
-  FILE *residue=NULL;
-  FILE *in=NULL;
-  char *line,*name;
-  long i,j,k;
-  int start=0,num=-1;
+extern void process_vector(codebook *b,double *a,char *basename);
+extern void process_usage(void);
 
-  double mean=0.,br_mean=0.,mean_count=0.;
-  
+int main(int argc,char *argv[]){
+  char *name;
+  char *basename;
+  double *a=NULL;
+  codebook *b=NULL;
   argv++;
 
   if(*argv==NULL){
-    fprintf(stderr,"Need a basename.\n");
+    process_usage();
     exit(1);
   }
 
   name=strdup(*argv);
-  vqbook_unquantize(b);
+  b=codebook_load(name);
+  a=alloca(sizeof(double)*b->dim);
+  argv=argv++;
 
   {
-    char buf[80];
-    snprintf(buf,80,"%s-residue.vqd",name);
-    residue=fopen(buf,"w");
-    if(!residue){
-      fprintf(stderr,"Unable to open output file %s\n",buf);
-      exit(1);
-    }
-  }   
-
-  /* parse the command line; handle things as the come */
-  argv=argv++;
+    char *dot;
+    basename=strrchr(name,'/');
+    if(basename)
+      basename=strdup(basename);
+    else
+      basename=strdup(name);
+    dot=strchr(basename,'.');
+    if(dot)*dot='\0';
+  }
+  
   while(*argv){
-    if(argv[0][0]=='-'){
-      /* it's an option */
-      switch(argv[0][1]){
-      case 'm':
-	{
-	  char buf[80];
-	  snprintf(buf,80,"%s-cells.m",name);
-	  cells=fopen(buf,"w");
-	  if(!cells){
-	    fprintf(stderr,"Unable to open output file %s\n",buf);
-	    exit(1);
-	  }
-	}   
-	argv++;
-	break;
-      case 's':
-	if(!argv[1]){
-	  fprintf(stderr,"Option %s missing argument.\n",argv[0]);
-	  exit(1);
-	}      
-	if(sscanf(argv[1],"%d,%d",&start,&num)!=2){
-          num= -1;
-          if(sscanf(argv[1],"%d",&start)!=1){
-	    fprintf(stderr,"Option %s syntax error.\n",argv[0]);
-            exit(1);
-	  }
-        }
-	argv+=2;
-        break;
-      }
-    }else{
-      /* it's an input file */
-      char *file=strdup(*argv++);
-      FILE *in;
-      int cols=-1;
-
-      in=fopen(file,"r");
-      if(in==NULL){
-        fprintf(stderr,"Could not open input file %s\n",file);
-        exit(1);
-      }
-
-      while((line=rline(in))){
-        if(cols==-1){
-          char *temp=line;
-          while(*temp==' ')temp++;
-          for(cols=0;*temp;cols++){
-            while(*temp>32)temp++;
-            while(*temp==' ')temp++;
-          }
-        }
-        {
-          int i;
-          double *p=malloc(cols*sizeof(double));
-          if(start*num+b->dim>cols){
-            fprintf(stderr,"ran out of columns reading %s\n",file);
-            exit(1);
-          }
-          while(*line==' ')line++;
-          for(i=0;i<cols;i++){
-
-            /* static length buffer bug workaround */
-            char *temp=line;
-            char old;
-            while(*temp>32)temp++;
-
-            old=temp[0];
-            temp[0]='\0';
-            p[i]=atof(line);
-            temp[0]=old;
-            
-            while(*line>32)line++;
-            while(*line==' ')line++;
-          }
-          if(num<=0)num=(cols-start)/b->dim;
-          for(i=num-1;i>=0;i--){
-	    int entry;
-	    double *base=p+start+i*b->dim;
-
-	    /* if this is a sequenced book and i||start,
-	       normalize the beginning to zero */
-	    if(b->sequencep && (i>0 || start>0)){
-	      for(k=0;k<b->dim;k++)
-		base[k]-= *(base-1);
-	    }
-
-	    /* assign the point */
-	    entry=vqenc_entry(b,base);
-
-	    /* accumulate metrics */
-	    mean+=_ssqe(b->dim,base,b->valuelist+entry*b->dim);
-
-	    /* brute force it as a sanity check of the decision
-               table... did that work better? */
-	    {
-	      double best=_ssqe(b->dim,base,b->valuelist);
-	      int bestj=0;
-	      for(j=0;j<b->entries;j++){
-		double this=_ssqe(b->dim,base,b->valuelist+j*b->dim);
-		if(this<best){
-		  best=this;
-		  bestj=j;
-		}
-	      }
-
-	      br_mean+=best;
-	    }
-
-	    mean_count+=b->dim;
-
-	    /* paint the cell if -m */
-	    if(cells){
-	      fprintf(cells,"%g %g\n%g %g\n\n",
-		      base[0],base[1],
-		      b->valuelist[0+entry*b->dim],
-		      b->valuelist[1+entry*b->dim]);
-	    }
-
-	    /* write cascading data */
+    /* only input files */
+    char *file=strdup(*argv++);
+    FILE *in=fopen(file,"r");
+    reset_next_value();
 
+    while(get_vector(b,in,a)!=-1)
+      process_vector(b,a,basename);
 
+    fclose(in);
+  }
 
-	  }
-          free(p);
-        }
-      }
-      fclose(in);
+  /* take any data from stdin */
+  {
+    struct stat st;
+    if(fstat(STDIN_FILENO,&st)==-1){
+      fprintf(stderr,"Could not stat STDIN\n");
+      exit(1);
+    }
+    if((S_IFIFO|S_IFREG|S_IFSOCK)&st.st_mode){
+      reset_next_value();
+      while(get_vector(b,stdin,a)!=-1)
+	process_vector(b,a,basename);
     }
   }
-  
-  
-  fclose(residue);
-  if(cells)fclose(cells);
-
-  /* print accumulated error statistics */
-  fprintf(stderr,"results:\n\tsqrt(mean squared error) from decision tree:%g\n"
-	  "\tsqrt(mean squared error) by brute force:%g\n\n",
-	  sqrt(mean/mean_count),sqrt(br_mean/mean_count));
-
   return 0;
 }
diff --git a/vq/train.c b/vq/train.c
index bf63f13..b8d6845 100644
--- a/vq/train.c
+++ b/vq/train.c
@@ -12,7 +12,7 @@
  ********************************************************************
 
  function: utility main for training codebooks
- last mod: $Id: train.c,v 1.11 1999/12/30 07:27:02 xiphmont Exp $
+ last mod: $Id: train.c,v 1.12 2000/01/05 10:14:56 xiphmont Exp $
 
  ********************************************************************/
 
@@ -24,46 +24,15 @@
 #include <signal.h>
 #include "vqgen.h"
 #include "vqext.h"
+#include "bookutil.h"
 
-static char *linebuffer=NULL;
-static int  lbufsize=0;
 static char *rline(FILE *in,FILE *out,int pass){
-  long sofar=0;
-  if(feof(in))return NULL;
-
   while(1){
-    int gotline=0;
-
-    while(!gotline){
-      if(sofar+1>=lbufsize){
-	if(!lbufsize){	
-	  lbufsize=16;
-	  linebuffer=malloc(lbufsize);
-	}else{
-	  lbufsize*=2;
-	  linebuffer=realloc(linebuffer,lbufsize);
-	}
-      }
-      {
-	long c=fgetc(in);
-	switch(c){
-	case '\n':
-	case EOF:
-	  gotline=1;
-	  break;
-	default:
-	  linebuffer[sofar++]=c;
-	  linebuffer[sofar]='\0';
-	  break;
-	}
-      }
-    }
-    
-    if(linebuffer[0]=='#'){
-      if(pass)fprintf(out,"%s\n",linebuffer);
-      sofar=0;
+    char *line=get_line(in);
+    if(line && line[0]=='#'){
+      if(pass)fprintf(out,"%s\n",line);
     }else{
-      return(linebuffer);
+      return(line);
     }
   }
 }
diff --git a/vq/vqgen.c b/vq/vqgen.c
index 9425441..024d589 100644
--- a/vq/vqgen.c
+++ b/vq/vqgen.c
@@ -12,7 +12,7 @@
  ********************************************************************
 
  function: train a VQ codebook 
- last mod: $Id: vqgen.c,v 1.25 1999/12/30 07:27:04 xiphmont Exp $
+ last mod: $Id: vqgen.c,v 1.26 2000/01/05 10:14:57 xiphmont Exp $
 
  ********************************************************************/
 
@@ -29,7 +29,9 @@
 #include <stdio.h>
 #include <math.h>
 #include <string.h>
+
 #include "vqgen.h"
+#include "bookutil.h"
 
 /* Codebook generation happens in two steps: 
 
@@ -80,6 +82,38 @@ void _vqgen_seed(vqgen *v){
 
 /* External calls *******************************************************/
 
+void spinnit(char *s,int n){
+  static int p=0;
+  static long lasttime=0;
+  long test;
+  struct timeval thistime;
+
+  gettimeofday(&thistime,NULL);
+  test=thistime.tv_sec*10+thistime.tv_usec/100000;
+  if(lasttime!=test){
+    lasttime=test;
+
+    fprintf(stderr,"%s%d ",s,n);
+
+    p++;if(p>3)p=0;
+    switch(p){
+    case 0:
+      fprintf(stderr,"|    \r");
+      break;
+    case 1:
+      fprintf(stderr,"/    \r");
+      break;
+    case 2:
+      fprintf(stderr,"-    \r");
+      break;
+    case 3:
+      fprintf(stderr,"\\    \r");
+      break;
+    }
+    fflush(stderr);
+  }
+}
+
 /* We have two forms of quantization; in the first, each vector
    element in the codebook entry is orthogonal.  Residues would use this
    quantization for example.
@@ -93,33 +127,6 @@ void _vqgen_seed(vqgen *v){
    the encoded (delta) values, not abs positions. This requires minor
    additional encode-side trickery. */
 
-/* 24 bit float (not IEEE; nonnormalized mantissa +
-   biased exponent ): neeeeemm mmmmmmmm mmmmmmmm */
-
-#define VQ_FEXP_BIAS 20 /* bias toward values smaller than 1. */
-long float24_pack(double val){
-  int sign=0;
-  long exp;
-  long mant;
-  if(val<0){
-    sign=0x800000;
-    val= -val;
-  }
-  exp= floor(log(val)/log(2));
-  mant=rint(ldexp(val,17-exp));
-  exp=(exp+VQ_FEXP_BIAS)<<18;
-
-  return(sign|exp|mant);
-}
-
-double float24_unpack(long val){
-  double mant=val&0x3ffff;
-  double sign=val&0x800000;
-  double exp =(val&0x7c0000)>>18;
-  if(sign)mant= -mant;
-  return(ldexp(mant,exp-17-VQ_FEXP_BIAS));
-}
-
 void vqgen_quantize(vqgen *v,quant_meta *q){
 
   double maxdel;
@@ -247,21 +254,6 @@ double vqgen_iterate(vqgen *v){
   long   *nearcount=malloc(v->entries*sizeof(long));
   double *nearbias=malloc(v->entries*desired2*sizeof(double));
 
-#ifdef NOISY
-  char buff[80];
-  FILE *assig;
-  FILE *bias;
-  FILE *cells;
-  sprintf(buff,"cells%d.m",v->it);
-  cells=fopen(buff,"w");
-  sprintf(buff,"assig%d.m",v->it);
-  assig=fopen(buff,"w");
-  sprintf(buff,"bias%d.m",v->it);
-  bias=fopen(buff,"w");
-#endif
-
-  fprintf(stderr,"Pass #%d... ",v->it);
-
   if(v->entries<2){
     fprintf(stderr,"generation requires at least two entries\n");
     exit(1);
@@ -277,6 +269,9 @@ double vqgen_iterate(vqgen *v){
     double secondmetric=v->metric_func(v,_now(v,1),ppt)+v->bias[1];
     long   firstentry=0;
     long   secondentry=1;
+
+    spinnit("working... ",v->points-i);
+
     if(firstmetric>secondmetric){
       double temp=firstmetric;
       firstmetric=secondmetric;
@@ -301,7 +296,7 @@ double vqgen_iterate(vqgen *v){
     }
       
     j=firstentry;
-    meterror+=sqrt(_dist_sq(v,_now(v,j),ppt));
+    meterror+=sqrt(_dist_sq(v,_now(v,j),ppt)/v->elements);
     /* set up midpoints for next iter */
     if(v->assigned[j]++)
       for(k=0;k<v->elements;k++)
@@ -310,13 +305,6 @@ double vqgen_iterate(vqgen *v){
       for(k=0;k<v->elements;k++)
 	vN(new,j)[k]=ppt[k];
 
-   
-#ifdef NOISY
-    fprintf(cells,"%g %g\n%g %g\n\n",
-	    _now(v,j)[0],_now(v,j)[1],
-	    ppt[0],ppt[1]);
-#endif
-
     for(j=0;j<v->entries;j++){
       
       double thismetric;
@@ -356,6 +344,8 @@ double vqgen_iterate(vqgen *v){
   for(i=0;i<v->entries;i++){
     double *nearbiasptr=nearbias+desired2*i;
 
+    spinnit("working... ",v->entries-i);
+
     /* due to the delayed sorting, we likely need to finish it off....*/
     if(nearcount[i]>desired)
       qsort(nearbiasptr,nearcount[i],sizeof(double),directdsort);
@@ -370,15 +360,13 @@ double vqgen_iterate(vqgen *v){
     if(v->assigned[j])
       for(k=0;k<v->elements;k++)
 	_now(v,j)[k]=vN(new,j)[k]/v->assigned[j];
-#ifdef NOISY
-    fprintf(assig,"%ld\n",v->assigned[j]);
-    fprintf(bias,"%g\n",v->bias[j]);
-#endif
   }
 
   asserror/=(v->entries*fdesired);
+
+  fprintf(stderr,"Pass #%d... ",v->it);
   fprintf(stderr,": dist %g(%g) metric error=%g \n",
-	  asserror,fdesired,meterror/v->points/v->elements);
+	  asserror,fdesired,meterror/v->points);
   v->it++;
   
   free(new);
diff --git a/vq/vqgen.h b/vq/vqgen.h
index 7f2a478..7cbd86d 100644
--- a/vq/vqgen.h
+++ b/vq/vqgen.h
@@ -12,13 +12,15 @@
  ********************************************************************
 
  function: build a VQ codebook 
- last mod: $Id: vqgen.h,v 1.8 1999/12/30 07:27:05 xiphmont Exp $
+ last mod: $Id: vqgen.h,v 1.9 2000/01/05 10:14:58 xiphmont Exp $
 
  ********************************************************************/
 
 #ifndef _VQGEN_H_
 #define _VQGEN_H_
 
+#include <sys/time.h>
+
 typedef struct vqgen{
   int it;
   int elements;
@@ -39,36 +41,6 @@ typedef struct vqgen{
   double *(*weight_func) (struct vqgen *v,double *point);
 } vqgen;
 
-typedef struct vqbook{
-  long dim;           /* codebook dimensions (elements per vector) */
-  long entries;       /* codebook entries */
-
-  long   min;         /* packed 24 bit float; quant value 0 maps to minval */
-  long   delta;       /* packed 24 bit float; val 1 - val 0 == delta */       
-  int    quant;       /* 0 < quant <= 16 */
-  int    sequencep;   /* bitflag */
-
-  double *valuelist;  /* list of dim*entries actual entry values */
-  long   *quantlist;  /* list of dim*entries quantized entry values */
-  long   *codelist;   /* list of bitstream codewords for each entry */
-  long   *lengthlist; /* codeword lengths in bits */
-
-  /* auxiliary encoding/decoding information */
-  /* encode: provided pre-calculated partitioning tree */
-  /* decode: hufftree */
-  long   *ptr0;
-  long   *ptr1;
-
-  /* auxiliary encoding information. Not used in decode */
-  double *n;         /* decision hyperplanes: sum(x_i*n_i)[0<=i<dim]=c */ 
-  double *c;
-  long   *p;
-  long   *q;
-  long   aux;
-  long   alloc;
-
-} vqbook;
-
 typedef struct {
   long   min;       /* packed 24 bit float */       
   long   delta;     /* packed 24 bit float */       
@@ -96,12 +68,12 @@ extern void vqgen_addpoint(vqgen *v, double *p,double *aux);
 extern double vqgen_iterate(vqgen *v);
 extern void vqgen_unquantize(vqgen *v,quant_meta *q);
 extern void vqgen_quantize(vqgen *v,quant_meta *q);
+
+extern void spinnit(char *s,int n);
+
 extern long float24_pack(double val);
 extern double float24_unpack(long val);
 
-extern void vqsp_book(vqgen *v,vqbook *b,long *quantlist);
-extern int vqenc_entry(vqbook *b,double *val);
-
 #endif
 
 
diff --git a/vq/vqsplit.c b/vq/vqsplit.c
index e08dbc9..aaf6525 100644
--- a/vq/vqsplit.c
+++ b/vq/vqsplit.c
@@ -12,7 +12,7 @@
  ********************************************************************
 
  function: build a VQ codebook and the encoding decision 'tree'
- last mod: $Id: vqsplit.c,v 1.8 2000/01/05 03:11:12 xiphmont Exp $
+ last mod: $Id: vqsplit.c,v 1.9 2000/01/05 10:14:59 xiphmont Exp $
 
  ********************************************************************/
 
@@ -29,9 +29,12 @@
 #include <stdio.h>
 #include <math.h>
 #include <string.h>
-#include "vqgen.h"
 #include <sys/time.h>
 
+#include "vqgen.h"
+#include "vqsplit.h"
+#include "bookutil.h"
+
 /* Codebook generation happens in two steps: 
 
    1) Train the codebook with data collected from the encoder: We use
@@ -64,6 +67,7 @@ int iascsort(const void *a,const void *b){
   return(av-bv);
 }
 
+/* grab it from vqgen.c */
 extern double _dist_sq(vqgen *v,double *a, double *b);
 
 /* goes through the split, but just counts it and returns a metric*/
@@ -121,42 +125,12 @@ void pq_in_out(vqgen *v,double *n,double *c,double *p,double *q){
   }
 }
 
-static void spinnit(int n){
-  static int p=0;
-  static long lasttime=0;
-  long test;
-  struct timeval thistime;
-
-  gettimeofday(&thistime,NULL);
-  test=thistime.tv_sec*10+thistime.tv_usec/100000;
-  if(lasttime!=test){
-    lasttime=test;
-
-    fprintf(stderr," %d ",n);
-
-    p++;if(p>3)p=0;
-    switch(p){
-    case 0:
-      fprintf(stderr,"|    \r");
-      break;
-    case 1:
-      fprintf(stderr,"/    \r");
-      break;
-    case 2:
-      fprintf(stderr,"-    \r");
-      break;
-    case 3:
-      fprintf(stderr,"\\    \r");
-      break;
-    }
-    fflush(stderr);
-  }
-}
-
-int lp_split(vqgen *v,vqbook *b,
+int lp_split(vqgen *v,codebook *b,
 	     long *entryindex,long entries, 
 	     long *pointindex,long points,
-	     long depth){
+	     long depth, long *pointsofar){
+
+  encode_aux *t=b->encode_tree;
 
   /* The encoder, regardless of book, will be using a straight
      euclidian distance-to-point metric to determine closest point.
@@ -179,6 +153,9 @@ int lp_split(vqgen *v,vqbook *b,
 
   long   besti=-1;
   long   bestj=-1;
+
+  char spinbuf[80];
+  sprintf(spinbuf,"splitting [%ld left]... ",v->points-*pointsofar);
   
   /* which cells do points belong to?  Do this before n^2 best pair chooser. */
 
@@ -187,7 +164,7 @@ int lp_split(vqgen *v,vqbook *b,
     long   firstentry=0;
     double firstmetric=_dist_sq(v,_now(v,entryindex[0]),ppt);
     
-    if(points*entries>64*1024)spinnit(entries);
+    if(points*entries>64*1024)spinnit(spinbuf,entries);
 
     for(j=1;j<entries;j++){
       double thismetric=_dist_sq(v,_now(v,entryindex[j]),ppt);
@@ -212,7 +189,7 @@ int lp_split(vqgen *v,vqbook *b,
     double this;
     for(i=0;i<entries-1;i++){
       for(j=i+1;j<entries;j++){
-	spinnit(entries-i);
+	spinnit(spinbuf,entries-i);
 	pq_in_out(v,n,&c,_now(v,entryindex[i]),_now(v,entryindex[j]));
 	vqsp_count(v,membership,
 		   entryindex,entries, 
@@ -247,7 +224,7 @@ int lp_split(vqgen *v,vqbook *b,
     /* eventually, we want to select the closest entry and figure n/c
        from p/q (because storing n/c is too large */
     for(k=0;k<v->elements;k++){
-      spinnit(entries);
+      spinnit(spinbuf,entries);
       
       p[k]=0.;
       for(j=0;j<entries;j++)
@@ -265,7 +242,7 @@ int lp_split(vqgen *v,vqbook *b,
       double ref_best=0.;
       double ref_j=-1;
       double this;
-      spinnit(entries-i);
+      spinnit(spinbuf,entries-i);
       
       for(k=0;k<v->elements;k++)
 	q[k]=2*p[k]-ppi[k];
@@ -353,35 +330,37 @@ int lp_split(vqgen *v,vqbook *b,
     }
   }
 
-  fprintf(stderr,"split: total=%ld depth=%ld set A=%ld:%ld:%ld=B\n",
-	  entries,depth,entriesA-entriesC,entriesC,entriesB-entriesC);
+  /*  fprintf(stderr,"split: total=%ld depth=%ld set A=%ld:%ld:%ld=B\n",
+      entries,depth,entriesA-entriesC,entriesC,entriesB-entriesC);*/
   {
-    long thisaux=b->aux++;
-    if(b->aux>=b->alloc){
-      b->alloc*=2;
-      b->ptr0=realloc(b->ptr0,sizeof(long)*b->alloc);
-      b->ptr1=realloc(b->ptr1,sizeof(long)*b->alloc);
-      b->p=realloc(b->p,sizeof(long)*b->alloc);
-      b->q=realloc(b->q,sizeof(long)*b->alloc);
+    long thisaux=t->aux++;
+    if(t->aux>=t->alloc){
+      t->alloc*=2;
+      t->ptr0=realloc(t->ptr0,sizeof(long)*t->alloc);
+      t->ptr1=realloc(t->ptr1,sizeof(long)*t->alloc);
+      t->p=realloc(t->p,sizeof(long)*t->alloc);
+      t->q=realloc(t->q,sizeof(long)*t->alloc);
     }
     
-    b->p[thisaux]=besti;
-    b->q[thisaux]=bestj;
+    t->p[thisaux]=besti;
+    t->q[thisaux]=bestj;
     
     if(entriesA==1){
       ret=1;
-      b->ptr0[thisaux]=entryA[0];
+      t->ptr0[thisaux]=entryA[0];
+      *pointsofar+=pointsA;
     }else{
-      b->ptr0[thisaux]= -b->aux;
-      ret=lp_split(v,b,entryA,entriesA,pointindex,pointsA,depth+1); 
+      t->ptr0[thisaux]= -t->aux;
+      ret=lp_split(v,b,entryA,entriesA,pointindex,pointsA,depth+1,pointsofar); 
     }
     if(entriesB==1){
       ret++;
-      b->ptr1[thisaux]=entryB[0];
+      t->ptr1[thisaux]=entryB[0];
+      *pointsofar+=points-pointsA;
     }else{
-      b->ptr1[thisaux]= -b->aux;
+      t->ptr1[thisaux]= -t->aux;
       ret+=lp_split(v,b,entryB,entriesB,pointindex+pointsA,points-pointsA,
-		    depth+1); 
+		    depth+1,pointsofar); 
     }
   }
   free(entryA);
@@ -389,7 +368,7 @@ int lp_split(vqgen *v,vqbook *b,
   return(ret);
 }
 
-static int _node_eq(vqbook *v, long a, long b){
+static int _node_eq(encode_aux *v, long a, long b){
   long    Aptr0=v->ptr0[a];
   long    Aptr1=v->ptr1[a];
   long    Ap   =v->p[a];
@@ -398,7 +377,6 @@ static int _node_eq(vqbook *v, long a, long b){
   long    Bptr1=v->ptr1[b];
   long    Bp   =v->p[b];
   long    Bq   =v->q[b];
-  int i;
 
   /* the possibility of choosing the same p and q, but switched, can;t
      happen because we always look for the best p/q in the same search
@@ -410,25 +388,28 @@ static int _node_eq(vqbook *v, long a, long b){
   return(0);
 }
 
-void vqsp_book(vqgen *v, vqbook *b, long *quantlist){
+void vqsp_book(vqgen *v, codebook *b, long *quantlist){
   long *entryindex=malloc(sizeof(long)*v->entries);
   long *pointindex=malloc(sizeof(long)*v->points);
   long *membership=malloc(sizeof(long)*v->points);
   long i,j;
+  encode_aux *t;
+
+  memset(b,0,sizeof(codebook));
+  t=b->encode_tree=calloc(1,sizeof(encode_aux));
 
-  memset(b,0,sizeof(vqbook));
   for(i=0;i<v->entries;i++)entryindex[i]=i;
   for(i=0;i<v->points;i++)pointindex[i]=i;
   
   b->dim=v->elements;
   b->entries=v->entries;
-  b->alloc=4096;
-
-  b->ptr0=malloc(sizeof(long)*b->alloc);
-  b->ptr1=malloc(sizeof(long)*b->alloc);
-  b->p=malloc(sizeof(long)*b->alloc);
-  b->q=malloc(sizeof(long)*b->alloc);
   b->lengthlist=calloc(b->entries,sizeof(long));
+
+  t->alloc=4096;
+  t->ptr0=malloc(sizeof(long)*t->alloc);
+  t->ptr1=malloc(sizeof(long)*t->alloc);
+  t->p=malloc(sizeof(long)*t->alloc);
+  t->q=malloc(sizeof(long)*t->alloc);
   
   /* which cells do points belong to?  Only do this once. */
 
@@ -450,10 +431,13 @@ void vqsp_book(vqgen *v, vqbook *b, long *quantlist){
 
 
   /* first, generate the encoding decision heirarchy */
-  fprintf(stderr,"Total leaves: %d\n",
-	  lp_split(v,b,entryindex,v->entries, 
-		   pointindex,v->points,0));
-  
+  {
+    long pointsofar=0;
+    fprintf(stderr,"Total leaves: %d            \n",
+	    lp_split(v,b,entryindex,v->entries, 
+		     pointindex,v->points,0,&pointsofar));
+  }
+
   free(entryindex);
   free(pointindex);
 
@@ -461,7 +445,7 @@ void vqsp_book(vqgen *v, vqbook *b, long *quantlist){
   /* The tree is likely big and redundant.  Pare and reroute branches */
   {
     int changedflag=1;
-    long *unique_entries=alloca(b->aux*sizeof(long));
+    long *unique_entries=alloca(t->aux*sizeof(long));
 
     while(changedflag){
       int nodes=0;
@@ -471,12 +455,12 @@ void vqsp_book(vqgen *v, vqbook *b, long *quantlist){
       /* span the tree node by node; list unique decision nodes and
 	 short circuit redundant branches */
 
-      for(i=0;i<b->aux;){
+      for(i=0;i<t->aux;){
 	int k;
 
 	/* check list of unique decisions */
 	for(j=0;j<nodes;j++)
-	  if(_node_eq(b,i,unique_entries[j]))break;
+	  if(_node_eq(t,i,unique_entries[j]))break;
 
 	if(j==nodes){
 	  /* a new entry */
@@ -485,29 +469,29 @@ void vqsp_book(vqgen *v, vqbook *b, long *quantlist){
 	  /* a redundant entry; find all higher nodes referencing it and
              short circuit them to the previously noted unique entry */
 	  changedflag=1;
-	  for(k=0;k<b->aux;k++){
-	    if(b->ptr0[k]==-i)b->ptr0[k]=-unique_entries[j];
-	    if(b->ptr1[k]==-i)b->ptr1[k]=-unique_entries[j];
+	  for(k=0;k<t->aux;k++){
+	    if(t->ptr0[k]==-i)t->ptr0[k]=-unique_entries[j];
+	    if(t->ptr1[k]==-i)t->ptr1[k]=-unique_entries[j];
 	  }
 
 	  /* Now, we need to fill in the hole from this redundant
              entry in the listing.  Insert the last entry in the list.
              Fix the forward pointers to that last entry */
-	  b->aux--;
-	  b->ptr0[i]=b->ptr0[b->aux];
-	  b->ptr1[i]=b->ptr1[b->aux];
-	  b->p[i]=b->p[b->aux];
-	  b->q[i]=b->q[b->aux];
-	  for(k=0;k<b->aux;k++){
-	    if(b->ptr0[k]==-b->aux)b->ptr0[k]=-i;
-	    if(b->ptr1[k]==-b->aux)b->ptr1[k]=-i;
+	  t->aux--;
+	  t->ptr0[i]=t->ptr0[t->aux];
+	  t->ptr1[i]=t->ptr1[t->aux];
+	  t->p[i]=t->p[t->aux];
+	  t->q[i]=t->q[t->aux];
+	  for(k=0;k<t->aux;k++){
+	    if(t->ptr0[k]==-t->aux)t->ptr0[k]=-i;
+	    if(t->ptr1[k]==-t->aux)t->ptr1[k]=-i;
 	  }
 	  /* hole plugged */
 
 	}
       }
 
-      fprintf(stderr," %ld remaining\n",b->aux);
+      fprintf(stderr," %ld remaining\n",t->aux);
     }
   }
 
@@ -521,7 +505,7 @@ void vqsp_book(vqgen *v, vqbook *b, long *quantlist){
 
     b->valuelist=v->entrylist; /* temporary for vqenc_entry; replaced later */
     for(i=0;i<v->points;i++){
-      int ret=vqenc_entry(b,v->pointlist+i*v->elements);
+      int ret=codebook_entry(b,v->pointlist+i*v->elements);
       probability[ret]++;
     }
     for(i=0;i<v->entries;i++)membership[i]=i;
@@ -567,7 +551,7 @@ void vqsp_book(vqgen *v, vqbook *b, long *quantlist){
     /* unneccessary metric */
     memset(probability,0,sizeof(long)*v->entries);
     for(i=0;i<v->points;i++){
-      int ret=vqenc_entry(b,v->pointlist+i*v->elements);
+      int ret=codebook_entry(b,v->pointlist+i*v->elements);
       probability[ret]++;
     }
     
@@ -589,11 +573,11 @@ void vqsp_book(vqgen *v, vqbook *b, long *quantlist){
     /* rearrange storage; ptr0/1 first as it needs a reverse index */
     /* n and c stay unchanged */
     for(i=0;i<b->entries;i++)revindex[index[i]]=i;
-    for(i=0;i<b->aux;i++){
-      if(b->ptr0[i]>=0)b->ptr0[i]=revindex[b->ptr0[i]];
-      if(b->ptr1[i]>=0)b->ptr1[i]=revindex[b->ptr1[i]];
-      b->p[i]=revindex[b->p[i]];
-      b->q[i]=revindex[b->q[i]];
+    for(i=0;i<t->aux;i++){
+      if(t->ptr0[i]>=0)t->ptr0[i]=revindex[t->ptr0[i]];
+      if(t->ptr1[i]>=0)t->ptr1[i]=revindex[t->ptr1[i]];
+      t->p[i]=revindex[t->p[i]];
+      t->q[i]=revindex[t->q[i]];
     }
     free(revindex);
 
@@ -641,33 +625,3 @@ void vqsp_book(vqgen *v, vqbook *b, long *quantlist){
   fprintf(stderr,"Done.\n\n");
 }
 
-/* slow version for use here that does not use a preexpanded n/c. */
-int vqenc_entry(vqbook *b,double *val){
-  int ptr=0,k;
-  double *n=alloca(b->dim*sizeof(double));
-
-  while(1){
-    double c=0.;
-    double *p=b->valuelist+b->p[ptr]*b->dim;
-    double *q=b->valuelist+b->q[ptr]*b->dim;
-    
-    for(k=0;k<b->dim;k++){
-      n[k]=p[k]-q[k];
-      c-=(p[k]+q[k])*n[k];
-    }
-    c/=2.;
-
-    for(k=0;k<b->dim;k++)
-      c+=n[k]*val[k];
-    if(c>0.) /* in A */
-      ptr= -b->ptr0[ptr];
-    else     /* in B */
-      ptr= -b->ptr1[ptr];
-    if(ptr<=0)break;
-  }
-  return(-ptr);
-}
-
-
-
-
diff --git a/vq/vqsplit.h b/vq/vqsplit.h
new file mode 100644
index 0000000..6c91046
--- /dev/null
+++ b/vq/vqsplit.h
@@ -0,0 +1,32 @@
+/********************************************************************
+ *                                                                  *
+ * THIS FILE IS PART OF THE Ogg Vorbis SOFTWARE CODEC SOURCE CODE.  *
+ * USE, DISTRIBUTION AND REPRODUCTION OF THIS SOURCE IS GOVERNED BY *
+ * THE GNU PUBLIC LICENSE 2, WHICH IS INCLUDED WITH THIS SOURCE.    *
+ * PLEASE READ THESE TERMS DISTRIBUTING.                            *
+ *                                                                  *
+ * THE OggSQUISH SOURCE CODE IS (C) COPYRIGHT 1994-2000             *
+ * by Monty <monty@xiph.org> and The XIPHOPHORUS Company            *
+ * http://www.xiph.org/                                             *
+ *                                                                  *
+ ********************************************************************
+
+ function: build a VQ codebook decision tree 
+ last mod: $Id: vqsplit.h,v 1.1 2000/01/05 10:15:00 xiphmont Exp $
+
+ ********************************************************************/
+
+#ifndef _VQSPL_H_
+#define _VQSPL_H_
+
+#include "vorbis/codebook.h"
+
+extern void vqsp_book(vqgen *v,codebook *b,long *quantlist);
+extern int vqenc_entry(codebook *b,double *val);
+
+#endif
+
+
+
+
+