scripts/s3file.py

   1 # Copyright (c) 2006 Carnegie Mellon University
   2 #
   3 # You may copy and modify this freely under the same terms as
   4 # Sphinx-III
   5
   6 """Read/write Sphinx-III binary parameter files.
   7
   8 All the various binary parameter files created by SphinxTrain and used
   9 by Sphinx-III and PocketSphinx share a common file format.  This
  10 module contains some base classes for reading and writing these files.
  11 """
  12
  13 __author__ = "David Huggins-Daines <dhuggins@cs.cmu.edu>"
  14 __version__ = "$Revision: 8994 $"
  15
  16 from struct import unpack, pack
  17 from numpy import array,reshape,shape,fromstring
  18
  19 class S3File(object):
  20     "Read Sphinx-III binary files"
  21     def __init__(self, filename=None, mode="rb"):
  22         if filename != None:
  23             self.open(filename, mode)
  24
  25     def getall(self):
  26         return self._params
  27
  28     def __getitem__(self, key):
  29         return self._params[key]
  30
  31     def __setitem__(self, key, value):
  32         self._params[key] = value
  33
  34     def __delitem__(self, key):
  35         del self._params[key]
  36
  37     def __iter__(self):
  38         return iter(self._params)
  39
  40     def __len__(self):
  41         return len(self._params)
  42
  43     def open(self, filename, mode="rb"):
  44         self.filename = filename
  45         self.fh = file(filename, mode)
  46         self.readheader()
  47
  48     def readheader(self):
  49         """
  50         Read binary header.  Sets the following attributes:
  51
  52           - fileattr (a dictionary of attribute-value pairs)
  53           - swap (a byteswap string as used by the struct module)
  54           - otherend (a flag indicating if the file is wrong-endian
  55                   for the current platform)
  56           - data_start (offset of the start of data in the file)
  57         """
  58         spam = self.fh.readline()
  59         self.fileattr = {}
  60         if spam != "s3\n":
  61             raise Exception("File ID not found or invalid: " + spam)
  62         while True:
  63             spam = self.fh.readline()
  64             if spam == "":
  65                 raise Exception("EOF while reading headers")
  66             if spam.endswith("endhdr\n"):
  67                 break
  68             sp = spam.find(' ')
  69             k = spam[0:sp].strip()
  70             v = spam[sp:].strip()
  71             self.fileattr[k] = v
  72         # This is 0x11223344 in the file's byte order
  73         spam = unpack("<i", self.fh.read(4))[0]
  74         if spam == 0x11223344:
  75             self.swap = "<" # little endian
  76         elif spam == 0x44332211:
  77             self.swap = ">" # big endian
  78         else:
  79             raise Exception("Invalid byte-order mark %08x" % spam)
  80         # Now determine whether we need to swap to get to native
  81         # byteorder (shouldn't this be easier???)
  82         self.otherend = (unpack('=i', pack(self.swap + 'i', spam))[0] != spam)
  83         self.data_start = self.fh.tell()
  84
  85     def read3d(self):
  86         self.d1 = unpack(self.swap + "I", self.fh.read(4))[0]
  87         self.d2 = unpack(self.swap + "I", self.fh.read(4))[0]
  88         self.d3 = unpack(self.swap + "I", self.fh.read(4))[0]
  89         self._nfloats = unpack(self.swap + "I", self.fh.read(4))[0]
  90         if self._nfloats != self.d1 * self.d2 * self.d3:
  91             raise Exception(("Number of data points %d doesn't match "
  92                              + "total %d = %d*%d*%d")
  93                             %
  94                             (self._nfloats,
  95                              self.d1 * self.d2 * self.d3,
  96                              self.d1, self.d2, self.d3))
  97         spam = self.fh.read(self._nfloats * 4)
  98         params = fromstring(spam, 'f')
  99         if self.otherend:
 100             params = params.byteswap()
 101         return reshape(params, (self.d1, self.d2, self.d3)).astype('d')
 102
 103     def read2d(self):
 104         self.d1 = unpack(self.swap + "I", self.fh.read(4))[0]
 105         self.d2 = unpack(self.swap + "I", self.fh.read(4))[0]
 106         self._nfloats = unpack(self.swap + "I", self.fh.read(4))[0]
 107         if self._nfloats != self.d1 * self.d2:
 108             raise Exception(("Number of data points %d doesn't match "
 109                              + "total %d = %d*%d")
 110                             %
 111                             (self._nfloats,
 112                              self.d1 * self.d2,
 113                              self.d1, self.d2))
 114         spam = self.fh.read(self._nfloats * 4)
 115         params = fromstring(spam, 'f')
 116         if self.otherend:
 117             params = params.byteswap()
 118         return reshape(params, (self.d1, self.d2)).astype('d')
 119
 120     def read1d(self):
 121         self.d1 = unpack(self.swap + "I", self.fh.read(4))[0]
 122         self._nfloats = unpack(self.swap + "I", self.fh.read(4))[0]
 123         if self._nfloats != self.d1:
 124             raise Exception(("Number of data points %d doesn't match "
 125                              + "total %d")
 126                             %
 127                             (self._nfloats, self.d1))
 128         spam = self.fh.read(self._nfloats * 4)
 129         params = fromstring(spam, 'f')
 130         if self.otherend:
 131             params = params.byteswap()
 132         return params.astype('d')
 133
 134 class S3File_write:
 135     "Write Sphinx-III binary files"
 136     def __init__(self, filename=None, mode="wb", attr={"version":1.0}):
 137         self.fileattr = attr
 138         if filename != None:
 139             self.open(filename)
 140
 141     def open(self, filename):
 142         self.filename = filename
 143         self.fh = file(filename, "wb")
 144         self.writeheader()
 145
 146     def writeheader(self):
 147         self.fh.write("s3\n")
 148         for k,v in self.fileattr.iteritems():
 149             self.fh.write("%s %s\n" % (k,v))
 150         # Make sure the binary data lives on a 4-byte boundary
 151         lsb = (self.fh.tell() + len("endhdr\n")) & 3
 152         if lsb != 0:
 153             align = 4-lsb
 154             self.fh.write("%sendhdr\n" % (" " * align))
 155         else:
 156             self.fh.write("endhdr\n")
 157         self.fh.write(pack("=i", 0x11223344))
 158         self.data_start = self.fh.tell()
 159
 160     def write3d(self, stuff):
 161         d1, d2, d3 = shape(stuff)
 162         self.fh.write(pack("=IIII",
 163                            d1, d2, d3,
 164                            d1 * d2 * d3))
 165         stuff.ravel().astype('f').tofile(self.fh)
 166
 167     def write2d(self, stuff):
 168         d1, d2 = shape(stuff)
 169         self.fh.write(pack("=III",
 170                            d1, d2,
 171                            d1 * d2))
 172         stuff.ravel().astype('f').tofile(self.fh)
 173
 174     def write1d(self, stuff):
 175         d1 = len(stuff)
 176         self.fh.write(pack("=II", d1, d1))
 177         stuff.ravel().astype('f').tofile(self.fh)
 178
 179     def __del__(self):
 180         self.close()
 181
 182     def close(self):
 183         self.fh.close()