From 447576da195e098d54c8e1279f7bb3ff1b5f2d49 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Jos=C3=A9=20Fonseca?= Date: Fri, 27 Jan 2012 14:27:13 +0000 Subject: [PATCH] Pickle more data types in binary. --- common/pickle.hpp | 147 +++++++++++++++++++++++++++++++++++++++++----------- scripts/unpickle.py | 25 +++++++-- 2 files changed, 140 insertions(+), 32 deletions(-) diff --git a/common/pickle.hpp b/common/pickle.hpp index c8f90ae..f333960 100644 --- a/common/pickle.hpp +++ b/common/pickle.hpp @@ -32,6 +32,7 @@ #include #include +#include #include #include @@ -89,22 +90,22 @@ private: SETITEMS = 'u', BINFLOAT = 'G', - PROTO = 0x80, - NEWOBJ = 0x81, - EXT1 = 0x82, - EXT2 = 0x83, - EXT4 = 0x84, - TUPLE1 = 0x85, - TUPLE2 = 0x86, - TUPLE3 = 0x87, - NEWTRUE = 0x88, - NEWFALSE = 0x89, - LONG1 = 0x8a, - LONG4 = 0x8b, + PROTO = '\x80', + NEWOBJ = '\x81', + EXT1 = '\x82', + EXT2 = '\x83', + EXT4 = '\x84', + TUPLE1 = '\x85', + TUPLE2 = '\x86', + TUPLE3 = '\x87', + NEWTRUE = '\x88', + NEWFALSE = '\x89', + LONG1 = '\x8a', + LONG4 = '\x8b', }; public: - PickleWriter(std::ostream &_os) : + PickleWriter(std::ostream &_os) : os(_os) { os.put(PROTO); @@ -158,13 +159,6 @@ public: os.put(TUPLE); } - inline void putInt(int i) { - os.put( i & 0xff); - os.put((i >> 8) & 0xff); - os.put((i >> 16) & 0xff); - os.put((i >> 24) & 0xff); - } - inline void writeString(const char *s, size_t length) { if (!s) { writeNone(); @@ -176,7 +170,7 @@ public: os.put(length); } else { os.put(BINSTRING); - putInt(length); + putInt32(length); } os.write(s, length); @@ -205,16 +199,111 @@ public: os.put(b ? NEWTRUE : NEWFALSE); } - inline void writeInt(long i) { - // TODO: binary - os.put(INT); - os << i << '\n'; + inline void writeInt(uint8_t i) { + os.put(BININT1); + os.put(i); + } + + inline void writeInt(uint16_t i) { + if (i < 0x100) { + writeInt((uint8_t)i); + } else { + os.put(BININT2); + putInt16(i); + } + } + + inline void writeInt(int32_t i) { + if (0 <= i && i < 0x10000) { + writeInt((uint16_t)i); + } else { + os.put(BININT); + putInt32(i); + } + } + + inline void writeInt(uint32_t i) { + if (i < 0x8000000) { + writeInt((int32_t)i); + } else { + writeLong(i); + } } - inline void writeFloat(float f) { - // TODO: binary - os.put(FLOAT); - os << f << '\n'; + inline void writeInt(long long i) { + if (-0x8000000 <= i && i < 0x8000000) { + writeInt((int32_t)i); + } else { + writeLong(i); + } + } + + inline void writeInt(unsigned long long i) { + if (i < 0x8000000) { + writeInt((int32_t)i); + } else { + writeLong(i); + } + } + + inline void writeFloat(double f) { + union { + double f; + char c[8]; + } u; + + assert(sizeof u.f == sizeof u.c); + u.f = f; + + os.put(BINFLOAT); + os.put(u.c[7]); + os.put(u.c[6]); + os.put(u.c[5]); + os.put(u.c[4]); + os.put(u.c[3]); + os.put(u.c[2]); + os.put(u.c[1]); + os.put(u.c[0]); + } + +protected: + inline void putInt16(uint16_t i) { + os.put( i & 0xff); + os.put( i >> 8 ); + } + + inline void putInt32(uint32_t i) { + os.put( i & 0xff); + os.put((i >> 8) & 0xff); + os.put((i >> 16) & 0xff); + os.put( i >> 24 ); + } + + template< class T > + inline void writeLong(T l) { + os.put(LONG1); + + if (l == 0) { + os.put(0); + return; + } + + unsigned c = 1; + // Same as l >> (8 * sizeof l), but without the warnings + T sign = l < 0 ? ~0 : 0; + while ((l >> (8 * c)) != sign) { + ++c; + } + // Add an extra byte if sign bit doesn't match + if (((l >> (8 * c - 1)) & 1) != ((l >> (8 * sizeof l - 1)) & 1)) { + ++c; + } + os.put(c); + + for (unsigned i = 0; i < c; ++ i) { + os.put(l & 0xff); + l >>= 8; + } } }; diff --git a/scripts/unpickle.py b/scripts/unpickle.py index efda4a6..efb8608 100755 --- a/scripts/unpickle.py +++ b/scripts/unpickle.py @@ -33,12 +33,25 @@ Run as: ''' +import optparse import cPickle as pickle import sys import time def main(): + optparser = optparse.OptionParser( + usage="\n\tapitrace pickle trace. %prog [options]") + optparser.add_option( + '--quiet', + action="store_true", dest="quiet", default=False, + help="don't dump calls to stdout") + + (options, args) = optparser.parse_args(sys.argv[1:]) + + if args: + optparser.error('unexpected arguments') + calls = 0 startTime = time.time() while True: @@ -48,12 +61,18 @@ def main(): break else: callNo, functionName, args, ret = call - if False: - sys.stdout.write('%u %s%r = %r\n' % (callNo, functionName, tuple(args), ret)) + if not options.quiet: + sys.stdout.write('%u ' % callNo) + sys.stdout.write(functionName) + sys.stdout.write('(' + ', '.join(map(repr, args)) + ')') + if ret is not None: + sys.stdout.write(' = ') + sys.stdout.write(repr(ret)) + sys.stdout.write('\n') calls += 1 stopTime = time.time() duration = stopTime - startTime - sys.stderr.write('%u calls, %.03f secs, %u class/sec\n' % (calls, duration, calls/duration)) + sys.stderr.write('%u calls, %.03f secs, %u calls/sec\n' % (calls, duration, calls/duration)) if __name__ == '__main__': -- 2.7.4