vq/train.c

   1 /********************************************************************
   2  *                                                                  *
   3  * THIS FILE IS PART OF THE Ogg Vorbis SOFTWARE CODEC SOURCE CODE.  *
   4  * USE, DISTRIBUTION AND REPRODUCTION OF THIS SOURCE IS GOVERNED BY *
   5  * THE GNU PUBLIC LICENSE 2, WHICH IS INCLUDED WITH THIS SOURCE.    *
   6  * PLEASE READ THESE TERMS DISTRIBUTING.                            *
   7  *                                                                  *
   8  * THE OggSQUISH SOURCE CODE IS (C) COPYRIGHT 1994-2000             *
   9  * by Monty <monty@xiph.org> and The XIPHOPHORUS Company            *
  10  * http://www.xiph.org/                                             *
  11  *                                                                  *
  12  ********************************************************************
  13
  14  function: utility main for training codebooks
  15  last mod: $Id: train.c,v 1.11 1999/12/30 07:27:02 xiphmont Exp $
  16
  17  ********************************************************************/
  18
  19 #include <stdlib.h>
  20 #include <stdio.h>
  21 #include <math.h>
  22 #include <string.h>
  23 #include <errno.h>
  24 #include <signal.h>
  25 #include "vqgen.h"
  26 #include "vqext.h"
  27
  28 static char *linebuffer=NULL;
  29 static int  lbufsize=0;
  30 static char *rline(FILE *in,FILE *out,int pass){
  31   long sofar=0;
  32   if(feof(in))return NULL;
  33
  34   while(1){
  35     int gotline=0;
  36
  37     while(!gotline){
  38       if(sofar+1>=lbufsize){
  39         if(!lbufsize){
  40           lbufsize=16;
  41           linebuffer=malloc(lbufsize);
  42         }else{
  43           lbufsize*=2;
  44           linebuffer=realloc(linebuffer,lbufsize);
  45         }
  46       }
  47       {
  48         long c=fgetc(in);
  49         switch(c){
  50         case '\n':
  51         case EOF:
  52           gotline=1;
  53           break;
  54         default:
  55           linebuffer[sofar++]=c;
  56           linebuffer[sofar]='\0';
  57           break;
  58         }
  59       }
  60     }
  61
  62     if(linebuffer[0]=='#'){
  63       if(pass)fprintf(out,"%s\n",linebuffer);
  64       sofar=0;
  65     }else{
  66       return(linebuffer);
  67     }
  68   }
  69 }
  70
  71 /* command line:
  72    trainvq  vqfile [options] trainfile [trainfile]
  73
  74    options: -params     entries,dim,quant
  75             -subvector  start[,num]
  76             -error      desired_error
  77             -iterations iterations
  78 */
  79
  80 static void usage(void){
  81   fprintf(stderr, "\nOggVorbis %s VQ codebook trainer\n\n"
  82           "<foo>vqtrain vqfile [options] [datasetfile] [datasetfile]\n"
  83           "options: -p[arams]     <entries,dim,quant>\n"
  84           "         -s[ubvector]  <start[,num]>\n"
  85           "         -e[rror]      <desired_error>\n"
  86           "         -i[terations] <maxiterations>\n\n"
  87           "examples:\n"
  88           "   train a new codebook to 1%% tolerance on datafile 'foo':\n"
  89           "      xxxvqtrain book -p 256,6,8 -e .01 foo\n"
  90           "      (produces a trained set in book-0.vqi)\n\n"
  91           "   continue training 'book-0.vqi' (produces book-1.vqi):\n"
  92           "      xxxvqtrain book-0.vqi\n\n"
  93           "   add subvector from element 1 to <dimension> from files\n"
  94           "      data*.m to the training in progress, prodicing book-1.vqi:\n"
  95           "      xxxvqtrain book-0.vqi -s 1,1 data*.m\n\n",vqext_booktype);
  96 }
  97
  98 int exiting=0;
  99 void setexit(int dummy){
 100   fprintf(stderr,"\nexiting... please wait to finish this iteration\n");
 101   exiting=1;
 102 }
 103
 104 int main(int argc,char *argv[]){
 105   vqgen v;
 106
 107   int entries=-1,dim=-1;
 108   int start=0,num=-1;
 109   double desired=.05;
 110   int iter=1000;
 111
 112   FILE *out=NULL;
 113   char *line;
 114   long i,j,k;
 115   int init=0;
 116   q.quant=-1;
 117
 118   argv++;
 119   if(!*argv){
 120     usage();
 121     exit(0);
 122   }
 123
 124   /* get the book name, a preexisting book to continue training */
 125   {
 126     FILE *in=NULL;
 127     char *filename=alloca(strlen(*argv)+30),*ptr;
 128
 129     strcpy(filename,*argv);
 130     in=fopen(filename,"r");
 131     ptr=strrchr(filename,'-');
 132     if(ptr){
 133       int num;
 134       ptr++;
 135       num=atoi(ptr);
 136       sprintf(ptr,"%d.vqi",num+1);
 137     }else
 138       strcat(filename,"-0.vqi");
 139
 140     out=fopen(filename,"w");
 141     if(out==NULL){
 142       fprintf(stderr,"Unable to open %s for writing\n",filename);
 143       exit(1);
 144     }
 145
 146     if(in){
 147       /* we wish to suck in a preexisting book and continue to train it */
 148       double a;
 149
 150       line=rline(in,out,1);
 151       if(strcmp(line,vqext_booktype)){
 152         fprintf(stderr,"wrong book type; %s!=%s\n",line,vqext_booktype);
 153         exit(1);
 154       }
 155
 156       line=rline(in,out,1);
 157       if(sscanf(line,"%d %d %d",&entries,&dim,&vqext_aux)!=3){
 158         fprintf(stderr,"Syntax error reading book file\n");
 159         exit(1);
 160       }
 161
 162       vqgen_init(&v,dim,vqext_aux,entries,vqext_metric,vqext_weight);
 163       init=1;
 164
 165       /* quant setup */
 166       line=rline(in,out,1);
 167       if(sscanf(line,"%ld %ld %d %d",&q.min,&q.delta,
 168                 &q.quant,&q.sequencep)!=4){
 169         fprintf(stderr,"Syntax error reading book file\n");
 170         exit(1);
 171       }
 172
 173       /* quantized entries */
 174       i=0;
 175       for(j=0;j<entries;j++){
 176         for(k=0;k<dim;k++){
 177           line=rline(in,out,0);
 178           sscanf(line,"%lf",&a);
 179           v.entrylist[i++]=a;
 180         }
 181       }
 182       vqgen_unquantize(&v,&q);
 183
 184       /* bias, points */
 185       i=0;
 186       for(j=0;j<entries;j++){
 187         line=rline(in,out,0);
 188         sscanf(line,"%lf",&a);
 189         v.bias[i++]=a;
 190       }
 191
 192       {
 193         double *b=alloca((dim+vqext_aux)*sizeof(double));
 194         i=0;
 195         v.entries=0; /* hack to avoid reseeding */
 196         while(1){
 197           for(k=0;k<dim+vqext_aux;k++){
 198             line=rline(in,out,0);
 199             if(!line)break;
 200             sscanf(line,"%lf",b+k);
 201           }
 202           if(feof(in))break;
 203           vqgen_addpoint(&v,b,b+dim);
 204         }
 205         v.entries=entries;
 206       }
 207
 208       fclose(in);
 209     }
 210   }
 211
 212   /* get the rest... */
 213   argv=argv++;
 214   while(*argv){
 215     if(argv[0][0]=='-'){
 216       /* it's an option */
 217       if(!argv[1]){
 218         fprintf(stderr,"Option %s missing argument.\n",argv[0]);
 219         exit(1);
 220       }
 221       switch(argv[0][1]){
 222       case 'p':
 223         if(sscanf(argv[1],"%d,%d,%d",&entries,&dim,&q.quant)!=3)
 224           goto syner;
 225         break;
 226       case 's':
 227         if(sscanf(argv[1],"%d,%d",&start,&num)!=2){
 228           num= -1;
 229           if(sscanf(argv[1],"%d",&start)!=1)
 230             goto syner;
 231         }
 232         break;
 233       case 'e':
 234         if(sscanf(argv[1],"%lf",&desired)!=1)
 235           goto syner;
 236         break;
 237       case 'i':
 238         if(sscanf(argv[1],"%d",&iter)!=1)
 239           goto syner;
 240         break;
 241       default:
 242         fprintf(stderr,"Unknown option %s\n",argv[0]);
 243         exit(1);
 244       }
 245       argv+=2;
 246     }else{
 247       /* it's an input file */
 248       char *file=strdup(*argv++);
 249       FILE *in;
 250       int cols=-1;
 251
 252       if(!init){
 253         if(dim==-1 || entries==-1 || q.quant==-1){
 254           fprintf(stderr,"-p required when training a new set\n");
 255           exit(1);
 256         }
 257         vqgen_init(&v,dim,vqext_aux,entries,vqext_metric,vqext_weight);
 258         init=1;
 259       }
 260
 261       in=fopen(file,"r");
 262       if(in==NULL){
 263         fprintf(stderr,"Could not open input file %s\n",file);
 264         exit(1);
 265       }
 266       fprintf(out,"# training file entry: %s\n",file);
 267
 268       while((line=rline(in,out,0))){
 269         if(cols==-1){
 270           char *temp=line;
 271           while(*temp==' ')temp++;
 272           for(cols=0;*temp;cols++){
 273             while(*temp>32)temp++;
 274             while(*temp==' ')temp++;
 275           }
 276         }
 277         {
 278           int i;
 279           double *b=malloc(cols*sizeof(double));
 280           if(start*num+dim>cols){
 281             fprintf(stderr,"ran out of columns reading %s\n",file);
 282             exit(1);
 283           }
 284           while(*line==' ')line++;
 285           for(i=0;i<cols;i++){
 286
 287             /* static length buffer bug workaround */
 288             char *temp=line;
 289             char old;
 290             while(*temp>32)temp++;
 291
 292             old=temp[0];
 293             temp[0]='\0';
 294             b[i]=atof(line);
 295             temp[0]=old;
 296
 297             while(*line>32)line++;
 298             while(*line==' ')line++;
 299           }
 300           if(num<=0)num=(cols-start)/dim;
 301           for(i=0;i<num;i++)
 302             vqext_addpoint_adj(&v,b,start+i*dim,dim,cols);
 303
 304           free(b);
 305         }
 306       }
 307       fclose(in);
 308     }
 309   }
 310
 311   if(!init){
 312     fprintf(stderr,"No input files!\n");
 313     exit(1);
 314   }
 315
 316   vqext_preprocess(&v);
 317
 318   /* train the book */
 319   signal(SIGTERM,setexit);
 320   signal(SIGINT,setexit);
 321
 322   for(i=0;i<iter && !exiting;i++){
 323     double result;
 324     if(i!=0)vqgen_unquantize(&v,&q);
 325     result=vqgen_iterate(&v);
 326     vqgen_quantize(&v,&q);
 327     if(result<desired)break;
 328   }
 329
 330   /* save the book */
 331
 332   fprintf(out,"# OggVorbis VQ codebook trainer, intermediate file\n");
 333   fprintf(out,"%s\n",vqext_booktype);
 334   fprintf(out,"%d %d %d\n",entries,dim,vqext_aux);
 335   fprintf(out,"%ld %ld %d %d\n",q.min,q.delta,q.quant,q.sequencep);
 336
 337   /* quantized entries */
 338   fprintf(out,"# quantized entries---\n");
 339   i=0;
 340   for(j=0;j<entries;j++)
 341     for(k=0;k<dim;k++)
 342       fprintf(out,"%d\n",(int)(rint(v.entrylist[i++])));
 343
 344   fprintf(out,"# biases---\n");
 345   i=0;
 346   for(j=0;j<entries;j++)
 347     fprintf(out,"%f\n",v.bias[i++]);
 348
 349   fprintf(out,"# points---\n");
 350   i=0;
 351   for(j=0;j<v.points;j++)
 352     for(k=0;k<dim+vqext_aux;k++)
 353       fprintf(out,"%f\n",v.pointlist[i++]);
 354
 355   fclose(out);
 356   exit(0);
 357
 358   syner:
 359     fprintf(stderr,"Syntax error in argument '%s'\n",*argv);
 360     exit(1);
 361 }