vq/latticebuild.c

   1 /********************************************************************
   2  *                                                                  *
   3  * THIS FILE IS PART OF THE Ogg Vorbis SOFTWARE CODEC SOURCE CODE.  *
   4  * USE, DISTRIBUTION AND REPRODUCTION OF THIS SOURCE IS GOVERNED BY *
   5  * THE GNU PUBLIC LICENSE 2, WHICH IS INCLUDED WITH THIS SOURCE.    *
   6  * PLEASE READ THESE TERMS DISTRIBUTING.                            *
   7  *                                                                  *
   8  * THE OggSQUISH SOURCE CODE IS (C) COPYRIGHT 1994-2000             *
   9  * by Monty <monty@xiph.org> and The XIPHOPHORUS Company            *
  10  * http://www.xiph.org/                                             *
  11  *                                                                  *
  12  ********************************************************************
  13
  14  function: utility main for building codebooks from lattice descriptions
  15  last mod: $Id: latticebuild.c,v 1.3 2000/07/12 09:36:18 xiphmont Exp $
  16
  17  ********************************************************************/
  18
  19 #include <stdlib.h>
  20 #include <stdio.h>
  21 #include <math.h>
  22 #include <string.h>
  23 #include <errno.h>
  24 #include "vorbis/codebook.h"
  25 #include "../lib/sharedbook.h"
  26 #include "bookutil.h"
  27
  28 /* The purpose of this util is actually just to count hits and finish
  29    packaging the description into a static codebook.
  30
  31    command line:
  32    vqlattice description.vql datafile.vqd
  33
  34    the lattice description file contains five lines:
  35
  36    <n> <dim> <multiplicitavep> <sequentialp>
  37    <value_0> <value_1> <value_2> ... <value_n-1>
  38    <m>
  39    <thresh_0> <thresh_1> <thresh_2> ... <thresh_m-2>
  40    <map_0> <map_1> <map_2> ... <map_m-1>
  41
  42    vqlattice sends residual data (for the next stage) to stdout, and
  43    produces description.vqh */
  44
  45 static int ilog(unsigned int v){
  46   int ret=0;
  47   while(v){
  48     ret++;
  49     v>>=1;
  50   }
  51   return(ret);
  52 }
  53
  54 int main(int argc,char *argv[]){
  55   codebook b;
  56   static_codebook c;
  57   encode_aux_threshmatch t;
  58   double *quantlist;
  59   long *hits;
  60
  61   int entries=-1,dim=-1,quantvals=-1,addmul=-1,threshvals=-1,sequencep=0;
  62   FILE *out=NULL;
  63   FILE *in=NULL;
  64   char *line,*name;
  65   long j,k;
  66
  67   memset(&b,0,sizeof(b));
  68   memset(&c,0,sizeof(c));
  69   memset(&t,0,sizeof(t));
  70
  71   if(argv[1]==NULL){
  72     fprintf(stderr,"Need a lattice description file on the command line.\n");
  73     exit(1);
  74   }
  75
  76   {
  77     char *ptr;
  78     char *filename=calloc(strlen(argv[1])+4,1);
  79
  80     strcpy(filename,argv[1]);
  81     in=fopen(filename,"r");
  82     if(!in){
  83       fprintf(stderr,"Could not open input file %s\n",filename);
  84       exit(1);
  85     }
  86
  87     ptr=strrchr(filename,'.');
  88     if(ptr){
  89       *ptr='\0';
  90       name=strdup(filename);
  91       sprintf(ptr,".vqh");
  92     }else{
  93       name=strdup(filename);
  94       strcat(filename,".vqh");
  95     }
  96
  97     out=fopen(filename,"w");
  98     if(out==NULL){
  99       fprintf(stderr,"Unable to open %s for writing\n",filename);
 100       exit(1);
 101     }
 102   }
 103
 104   /* read the description */
 105   line=get_line(in);
 106   if(sscanf(line,"%d %d %d %d",&quantvals,&dim,&addmul,&sequencep)!=4){
 107     if(sscanf(line,"%d %d %d",&quantvals,&dim,&addmul)!=3){
 108       fprintf(stderr,"Syntax error reading book file (line 1)\n");
 109       exit(1);
 110     }
 111   }
 112   entries=pow(quantvals,dim);
 113   c.thresh_tree=&t;
 114   c.dim=dim;
 115   c.entries=entries;
 116   c.lengthlist=malloc(entries*sizeof(long));
 117   c.maptype=1;
 118   c.q_sequencep=sequencep;
 119   c.quantlist=calloc(quantvals,sizeof(long));
 120
 121   quantlist=malloc(sizeof(long)*c.dim*c.entries);
 122   hits=malloc(c.entries*sizeof(long));
 123   for(j=0;j<entries;j++)hits[j]=1;
 124   for(j=0;j<entries;j++)c.lengthlist[j]=1;
 125
 126   reset_next_value();
 127   setup_line(in);
 128   for(j=0;j<quantvals;j++){
 129     if(get_line_value(in,quantlist+j)==-1){
 130       fprintf(stderr,"Ran out of data on line 2 of description file\n");
 131       exit(1);
 132     }
 133   }
 134
 135   line=setup_line(in);
 136   if(sscanf(line,"%d",&threshvals)!=1){
 137     fprintf(stderr,"Syntax error reading book file (line 3)\n");
 138     exit(1);
 139   }
 140
 141   t.quantthresh=calloc(threshvals-1,sizeof(double));
 142   t.quantmap=calloc(threshvals,sizeof(int));
 143   t.threshvals=threshvals;
 144   t.quantvals=quantvals;
 145
 146   setup_line(in);
 147   for(j=0;j<threshvals-1;j++){
 148     if(get_line_value(in,t.quantthresh+j)==-1){
 149       fprintf(stderr,"Ran out of data on line 4 of description file\n");
 150       exit(1);
 151     }
 152   }
 153
 154   setup_line(in);
 155   for(j=0;j<threshvals;j++){
 156     if(get_next_ivalue(in,t.quantmap+j)==-1){
 157       fprintf(stderr,"Ran out of data on line 5 of description file\n");
 158       exit(1);
 159     }
 160   }
 161
 162   /* gen a real quant list from the more easily human-grokked input */
 163   {
 164     double min=quantlist[0];
 165     double mindel=-1;
 166     int fac=1;
 167     for(j=1;j<quantvals;j++)if(quantlist[j]<min)min=quantlist[j];
 168     for(j=0;j<quantvals;j++)
 169       if(min!=quantlist[j] && (mindel==-1 || quantlist[j]-min<mindel))
 170         mindel=quantlist[j]-min;
 171
 172     fprintf(stderr,"min=%g mindel=%g\n",min,mindel);
 173     j=0;
 174     while(j<quantvals){
 175       for(j=0;j<quantvals;j++){
 176         double test=(quantlist[j]-min)/(mindel/fac);
 177         if( fabs(rint(test)-test)>.000001) break;
 178       }
 179       if(j<quantvals)fac++;
 180     }
 181
 182     mindel/=fac;
 183
 184     c.q_min=_float32_pack(min);
 185     c.q_delta=_float32_pack(mindel);
 186     c.q_quant=0;
 187
 188     min=_float32_unpack(c.q_min);
 189     mindel=_float32_unpack(c.q_delta);
 190     for(j=0;j<quantvals;j++){
 191       c.quantlist[j]=rint((quantlist[j]-min)/mindel);
 192       if(ilog(c.quantlist[j])>c.q_quant)c.q_quant=ilog(c.quantlist[j]);
 193     }
 194   }
 195
 196   vorbis_book_init_encode(&b,&c);
 197   fclose(in);
 198
 199   /* at this point, we have enough to do best entry matching.  We
 200      don't have counts yet, but that's OK; match, write residual data,
 201      and track hits.  The build word lengths when we're done */
 202
 203   /* we have to do our own de-interleave */
 204   if(argv[2]==NULL){
 205     fprintf(stderr,"Need a data file on the command line.\n");
 206     exit(1);
 207   }
 208   in=fopen(argv[2],"r");
 209   if(!in){
 210     fprintf(stderr,"Could not open input file %s\n",argv[2]);
 211     exit(1);
 212   }
 213
 214   {
 215     int cols=0;
 216     long lines=0;
 217     double *vec;
 218     int interleaved;
 219     line=setup_line(in);
 220     /* count cols before we start reading */
 221     {
 222       char *temp=line;
 223       while(*temp==' ')temp++;
 224       for(cols=0;*temp;cols++){
 225         while(*temp>32)temp++;
 226         while(*temp==' ')temp++;
 227       }
 228     }
 229     vec=alloca(cols*sizeof(double));
 230     interleaved=cols/dim;
 231
 232     while(line){
 233       lines++;
 234
 235       if(!(lines&0xff))spinnit("lines so far...",lines);
 236
 237       /* don't need to deinterleave; besterror does that for us */
 238       for(j=0;j<cols;j++)
 239         if(get_line_value(in,vec+j)){
 240           fprintf(stderr,"Too few columns on line %ld in data file\n",lines);
 241           exit(1);
 242         }
 243
 244       /* process */
 245       for(j=0;j<interleaved;j++)
 246         hits[vorbis_book_besterror(&b,vec+j,interleaved,addmul)]++;
 247
 248       /* write */
 249       for(j=0;j<cols;j++)
 250         fprintf(stdout,"%g, ",vec[j]);
 251       fprintf(stdout,"\n");
 252
 253       line=setup_line(in);
 254     }
 255   }
 256   fclose(in);
 257
 258   /* build the codeword lengths */
 259   memset(c.lengthlist,0,sizeof(long)*entries);
 260   build_tree_from_lengths(entries,hits,c.lengthlist);
 261
 262   /* save the book in C header form */
 263   fprintf(out,
 264  "/********************************************************************\n"
 265  " *                                                                  *\n"
 266  " * THIS FILE IS PART OF THE Ogg Vorbis SOFTWARE CODEC SOURCE CODE.  *\n"
 267  " * USE, DISTRIBUTION AND REPRODUCTION OF THIS SOURCE IS GOVERNED BY *\n"
 268  " * THE GNU PUBLIC LICENSE 2, WHICH IS INCLUDED WITH THIS SOURCE.    *\n"
 269  " * PLEASE READ THESE TERMS DISTRIBUTING.                            *\n"
 270  " *                                                                  *\n"
 271  " * THE OggSQUISH SOURCE CODE IS (C) COPYRIGHT 1994-1999             *\n"
 272  " * by 1999 Monty <monty@xiph.org> and The XIPHOPHORUS Company       *\n"
 273  " * http://www.xiph.org/                                             *\n"
 274  " *                                                                  *\n"
 275  " ********************************************************************\n"
 276  "\n"
 277  " function: static codebook autogenerated by vq/latticebuild\n"
 278  "\n"
 279  " ********************************************************************/\n\n");
 280
 281   fprintf(out,"#ifndef _V_%s_VQH_\n#define _V_%s_VQH_\n",name,name);
 282   fprintf(out,"#include \"vorbis/codebook.h\"\n\n");
 283
 284   /* first, the static vectors, then the book structure to tie it together. */
 285   /* quantlist */
 286   fprintf(out,"static long _vq_quantlist_%s[] = {\n",name);
 287   for(j=0;j<_book_maptype1_quantvals(&c);j++){
 288     fprintf(out,"\t%ld,\n",c.quantlist[j]);
 289   }
 290   fprintf(out,"};\n\n");
 291
 292   /* lengthlist */
 293   fprintf(out,"static long _vq_lengthlist_%s[] = {\n",name);
 294   for(j=0;j<c.entries;){
 295     fprintf(out,"\t");
 296     for(k=0;k<16 && j<c.entries;k++,j++)
 297       fprintf(out,"%2ld,",c.lengthlist[j]);
 298     fprintf(out,"\n");
 299   }
 300   fprintf(out,"};\n\n");
 301
 302   /* quantthresh */
 303   fprintf(out,"static double _vq_quantthresh_%s[] = {\n",name);
 304   for(j=0;j<t.threshvals-1;){
 305     fprintf(out,"\t");
 306     for(k=0;k<8 && j<t.threshvals-1;k++,j++)
 307       fprintf(out,"%.5g, ",t.quantthresh[j]);
 308     fprintf(out,"\n");
 309   }
 310   fprintf(out,"};\n\n");
 311
 312   /* quantmap */
 313   fprintf(out,"static long _vq_quantmap_%s[] = {\n",name);
 314   for(j=0;j<t.threshvals;){
 315     fprintf(out,"\t");
 316     for(k=0;k<8 && j<t.threshvals;k++,j++)
 317       fprintf(out,"%5ld,",t.quantmap[j]);
 318     fprintf(out,"\n");
 319   }
 320   fprintf(out,"};\n\n");
 321
 322   /* tie it all together */
 323
 324   fprintf(out,"static encode_aux_threshmatch _vq_aux_%s = {\n",name);
 325   fprintf(out,"\t_vq_quantthresh_%s,\n",name);
 326   fprintf(out,"\t_vq_quantmap_%s,\n",name);
 327   fprintf(out,"\t%d,\n",t.quantvals);
 328   fprintf(out,"\t%d\n};\n\n",t.threshvals);
 329
 330   fprintf(out,"static static_codebook _vq_book_%s = {\n",name);
 331
 332   fprintf(out,"\t%ld, %ld,\n",c.dim,c.entries);
 333   fprintf(out,"\t_vq_lengthlist_%s,\n",name);
 334   fprintf(out,"\t1, %ld, %ld, %d, %d,\n",
 335           c.q_min,c.q_delta,c.q_quant,c.q_sequencep);
 336   fprintf(out,"\t_vq_quantlist_%s,\n",name);
 337   fprintf(out,"\tNULL,\n");
 338   fprintf(out,"\t&_vq_aux_%s\n",name);
 339   fprintf(out,"};\n\n");
 340
 341   fprintf(out,"\n#endif\n");
 342   fclose(out);
 343
 344   fprintf(stderr,"\r                                                     "
 345           "\nDone.\n");
 346   exit(0);
 347 }