vq/train.c

   1 /********************************************************************
   2  *                                                                  *
   3  * THIS FILE IS PART OF THE Ogg Vorbis SOFTWARE CODEC SOURCE CODE.  *
   4  * USE, DISTRIBUTION AND REPRODUCTION OF THIS SOURCE IS GOVERNED BY *
   5  * THE GNU PUBLIC LICENSE 2, WHICH IS INCLUDED WITH THIS SOURCE.    *
   6  * PLEASE READ THESE TERMS DISTRIBUTING.                            *
   7  *                                                                  *
   8  * THE OggSQUISH SOURCE CODE IS (C) COPYRIGHT 1994-1999             *
   9  * by 1999 Monty <monty@xiph.org> and The XIPHOPHORUS Company       *
  10  * http://www.xiph.org/                                             *
  11  *                                                                  *
  12  ********************************************************************
  13
  14  function: utility main for training codebooks
  15  author: Monty <xiphmont@mit.edu>
  16  modifications by: Monty
  17  last modification date: Dec 15 1999
  18
  19  ********************************************************************/
  20
  21 #include <stdlib.h>
  22 #include <stdio.h>
  23 #include <math.h>
  24 #include <string.h>
  25 #include <errno.h>
  26 #include <signal.h>
  27 #include "vqgen.h"
  28 #include "vqext.h"
  29
  30 static char *linebuffer=NULL;
  31 static int  lbufsize=0;
  32 static char *rline(FILE *in,FILE *out,int pass){
  33   long sofar=0;
  34   if(feof(in))return NULL;
  35
  36   while(1){
  37     int gotline=0;
  38
  39     while(!gotline){
  40       if(sofar+1>=lbufsize){
  41         if(!lbufsize){
  42           lbufsize=16;
  43           linebuffer=malloc(lbufsize);
  44         }else{
  45           lbufsize*=2;
  46           linebuffer=realloc(linebuffer,lbufsize);
  47         }
  48       }
  49       {
  50         long c=fgetc(in);
  51         switch(c){
  52         case '\n':
  53         case EOF:
  54           gotline=1;
  55           break;
  56         default:
  57           linebuffer[sofar++]=c;
  58           linebuffer[sofar]='\0';
  59           break;
  60         }
  61       }
  62     }
  63
  64     if(linebuffer[0]=='#'){
  65       if(pass)fprintf(out,"%s",linebuffer);
  66       sofar=0;
  67     }else{
  68       return(linebuffer);
  69     }
  70   }
  71 }
  72
  73 /* command line:
  74    trainvq  vqfile [options] trainfile [trainfile]
  75
  76    options: -params     entries,dim,quant
  77             -subvector  start[,num]
  78             -error      desired_error
  79             -iterations iterations
  80 */
  81
  82 static void usage(void){
  83   fprintf(stderr, "\nOggVorbis %s VQ codebook trainer\n\n"
  84           "<foo>vqtrain vqfile [options] [datasetfile] [datasetfile]\n"
  85           "options: -p[arams]     <entries,dim,quant>\n"
  86           "         -s[ubvector]  <start[,num]>\n"
  87           "         -e[rror]      <desired_error>\n"
  88           "         -i[terations] <maxiterations>\n\n"
  89           "examples:\n"
  90           "   train a new codebook to 1%% tolerance on datafile 'foo':\n"
  91           "      xxxvqtrain book -p 256,6,8 -e .01 foo\n"
  92           "      (produces a trained set in book-0.vqi)\n\n"
  93           "   continue training 'book-0.vqi' (produces book-1.vqi):\n"
  94           "      xxxvqtrain book-0.vqi\n\n"
  95           "   add subvector from element 1 to <dimension> from files\n"
  96           "      data*.m to the training in progress, prodicing book-1.vqi:\n"
  97           "      xxxvqtrain book-0.vqi -s 1,1 data*.m\n\n",vqext_booktype);
  98 }
  99
 100 int exiting=0;
 101 void setexit(int dummy){
 102   fprintf(stderr,"\nexiting... please wait to finish this iteration\n");
 103   exiting=1;
 104 }
 105
 106 int main(int argc,char *argv[]){
 107   vqgen v;
 108   quant_return q;
 109
 110   int entries=-1,dim=-1,quant=-1;
 111   int start=0,num=-1;
 112   double desired=.05;
 113   int iter=1000;
 114
 115   FILE *out=NULL;
 116   char *line;
 117   long i,j,k;
 118   int init=0;
 119
 120   argv++;
 121   if(!*argv){
 122     usage();
 123     exit(0);
 124   }
 125
 126   /* get the book name, a preexisting book to continue training */
 127   {
 128     FILE *in=NULL;
 129     char *filename=alloca(strlen(*argv)+30),*ptr;
 130
 131     strcpy(filename,*argv);
 132     in=fopen(filename,"r");
 133     ptr=strrchr(filename,'-');
 134     if(ptr){
 135       int num;
 136       ptr++;
 137       num=atoi(ptr);
 138       sprintf(ptr,"%d.vqi",num+1);
 139     }else
 140       strcat(filename,"-0.vqi");
 141
 142     out=fopen(filename,"w");
 143     if(out==NULL){
 144       fprintf(stderr,"Unable to open %s for writing\n",filename);
 145       exit(1);
 146     }
 147
 148     if(in){
 149       /* we wish to suck in a preexisting book and continue to train it */
 150       double a;
 151
 152       line=rline(in,out,1);
 153       if(strcmp(line,vqext_booktype)){
 154         fprintf(stderr,"wrong book type; %s!=%s\n",line,vqext_booktype);
 155         exit(1);
 156       }
 157
 158       line=rline(in,out,1);
 159       if(sscanf(line,"%d %d",&entries,&dim)!=2){
 160         fprintf(stderr,"Syntax error reading book file\n");
 161         exit(1);
 162       }
 163
 164       vqgen_init(&v,dim,entries,vqext_metric);
 165       init=1;
 166
 167       /* quant setup */
 168       line=rline(in,out,1);
 169       if(sscanf(line,"%lf %lf %d %d",&q.minval,&q.delt,
 170                 &q.addtoquant,&quant)!=4){
 171         fprintf(stderr,"Syntax error reading book file\n");
 172         exit(1);
 173       }
 174
 175       /* quantized entries */
 176       for(j=0;j<entries;j++)
 177         for(k=0;k<dim;k++)
 178           line=rline(in,out,0);
 179
 180       /* unquantized entries */
 181       i=0;
 182       for(j=0;j<entries;j++){
 183         for(k=0;k<dim;k++){
 184           line=rline(in,out,0);
 185           sscanf(line,"%lf",&a);
 186           v.entrylist[i++]=a;
 187         }
 188       }
 189
 190         /* bias, points */
 191       i=0;
 192       for(j=0;j<entries;j++){
 193         line=rline(in,out,0);
 194         sscanf(line,"%lf",&a);
 195         v.bias[i++]=a;
 196       }
 197
 198       {
 199         double b[80];
 200         i=0;
 201         v.entries=0; /* hack to avoid reseeding */
 202         while(1){
 203           for(k=0;k<dim && k<80;k++){
 204             line=rline(in,out,0);
 205             if(!line)break;
 206             sscanf(line,"%lf",b+k);
 207           }
 208           if(feof(in))break;
 209           vqgen_addpoint(&v,b);
 210         }
 211         v.entries=entries;
 212       }
 213
 214       fclose(in);
 215     }
 216   }
 217
 218   /* get the rest... */
 219   argv=argv++;
 220   while(*argv){
 221     if(argv[0][0]=='-'){
 222       /* it's an option */
 223       if(!argv[1]){
 224         fprintf(stderr,"Option %s missing argument.\n",argv[0]);
 225         exit(1);
 226       }
 227       switch(argv[0][1]){
 228       case 'p':
 229         if(sscanf(argv[1],"%d,%d,%d",&entries,&dim,&quant)!=3)
 230           goto syner;
 231         break;
 232       case 's':
 233         if(sscanf(argv[1],"%d,%d",&start,&num)!=2){
 234           num= -1;
 235           if(sscanf(argv[1],"%d",&start)!=1)
 236             goto syner;
 237         }
 238         break;
 239       case 'e':
 240         if(sscanf(argv[1],"%lf",&desired)!=1)
 241           goto syner;
 242         break;
 243       case 'i':
 244         if(sscanf(argv[1],"%d",&iter)!=1)
 245           goto syner;
 246         break;
 247       default:
 248         fprintf(stderr,"Unknown option %s\n",argv[0]);
 249         exit(1);
 250       }
 251       argv+=2;
 252     }else{
 253       /* it's an input file */
 254       char *file=strdup(*argv++);
 255       FILE *in;
 256       int cols=-1;
 257
 258       if(!init){
 259         if(dim==-1 || entries==-1 || quant==-1){
 260           fprintf(stderr,"-p required when training a new set\n");
 261           exit(1);
 262         }
 263         vqgen_init(&v,dim,entries,vqext_metric);
 264         init=1;
 265       }
 266
 267       in=fopen(file,"r");
 268       if(in==NULL){
 269         fprintf(stderr,"Could not open input file %s\n",file);
 270         exit(1);
 271       }
 272       fprintf(out,"# training file entry: %s\n",file);
 273
 274       while((line=rline(in,out,0))){
 275         if(cols==-1){
 276           char *temp=line;
 277           while(*temp==' ')temp++;
 278           for(cols=0;*temp;cols++){
 279             while(*temp>32)temp++;
 280             while(*temp==' ')temp++;
 281           }
 282         }
 283         {
 284           int i;
 285           double *b=malloc(cols*sizeof(double));
 286           if(start*num+dim>cols){
 287             fprintf(stderr,"ran out of columns reading %s\n",file);
 288             exit(1);
 289           }
 290           while(*line==' ')line++;
 291           for(i=0;i<cols;i++){
 292
 293             /* static length buffer bug workaround */
 294             char *temp=line;
 295             char old;
 296             while(*temp>32)temp++;
 297
 298             old=temp[0];
 299             temp[0]='\0';
 300             b[i]=atof(line);
 301             temp[0]=old;
 302
 303             while(*line>32)line++;
 304             while(*line==' ')line++;
 305           }
 306           if(num<=0)num=(cols-start)/dim;
 307           for(i=0;i<num;i++){
 308             vqext_adjdata(b,start+i*dim,dim);
 309             vqgen_addpoint(&v,b+start+i*dim);
 310           }
 311           free(b);
 312         }
 313       }
 314       fclose(in);
 315     }
 316   }
 317
 318   if(!init){
 319     fprintf(stderr,"No input files!\n");
 320     exit(1);
 321   }
 322
 323   /* train the book */
 324   signal(SIGTERM,setexit);
 325   signal(SIGINT,setexit);
 326
 327   for(i=0;i<iter && !exiting;i++){
 328     double result;
 329     if(i!=0)vqext_unquantize(&v,&q);
 330     result=vqgen_iterate(&v);
 331     q=vqext_quantize(&v,quant);
 332     if(result<desired)break;
 333   }
 334
 335   /* save the book */
 336
 337   fprintf(out,"# OggVorbis VQ codebook trainer, intermediate file\n");
 338   fprintf(out,"%s\n",vqext_booktype);
 339   fprintf(out,"%d %d\n",entries,dim);
 340   fprintf(out,"%g %g %d %d\n",q.minval,q.delt,q.addtoquant,quant);
 341
 342   /* quantized entries */
 343   fprintf(out,"# quantized entries---\n");
 344   i=0;
 345   for(j=0;j<entries;j++)
 346     for(k=0;k<dim;k++)
 347       fprintf(out,"%d\n",(int)(rint(v.entrylist[i++])));
 348
 349   /* dequantize */
 350   vqext_unquantize(&v,&q);
 351
 352   fprintf(out,"# unquantized entries---\n");
 353   i=0;
 354   for(j=0;j<entries;j++)
 355     for(k=0;k<dim;k++)
 356       fprintf(out,"%f\n",v.entrylist[i++]);
 357
 358   /* unquantized entries */
 359
 360   fprintf(out,"# biases---\n");
 361   i=0;
 362   for(j=0;j<entries;j++)
 363     fprintf(out,"%f\n",v.bias[i++]);
 364
 365   fprintf(out,"# points---\n");
 366   i=0;
 367   for(j=0;j<v.points;j++)
 368     for(k=0;k<dim && k<80;k++)
 369       fprintf(out,"%f\n",v.pointlist[i++]);
 370
 371   fclose(out);
 372   exit(0);
 373
 374   syner:
 375     fprintf(stderr,"Syntax error in argument '%s'\n",*argv);
 376     exit(1);
 377 }