From: Sergey Karayev Date: Sat, 12 Jul 2014 02:23:47 +0000 (-0700) Subject: [example] image classification web demo X-Git-Tag: submit/tizen/20180823.020014~662^2 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=dd546171bae1c3608ce70111c8ffb46c766ea9df;p=platform%2Fupstream%2Fcaffeonacl.git [example] image classification web demo --- diff --git a/data/ilsvrc12/get_ilsvrc_aux.sh b/data/ilsvrc12/get_ilsvrc_aux.sh index 3fa58dc..b9b85d2 100755 --- a/data/ilsvrc12/get_ilsvrc_aux.sh +++ b/data/ilsvrc12/get_ilsvrc_aux.sh @@ -4,6 +4,7 @@ # This script downloads the imagenet example auxiliary files including: # - the ilsvrc12 image mean, binaryproto # - synset ids and words +# - Python pickle-format data of ImageNet graph structure and relative infogain # - the training splits with labels DIR="$( cd "$(dirname "$0")" ; pwd -P )" diff --git a/docs/getting_pretrained_models.md b/docs/getting_pretrained_models.md index bbac5ac..14e6ee9 100644 --- a/docs/getting_pretrained_models.md +++ b/docs/getting_pretrained_models.md @@ -8,7 +8,8 @@ layout: default Note that unlike Caffe itself, these models are licensed for **academic research / non-commercial use only**. If you have any questions, please get in touch with us. -This page will be updated as more models become available. +*UPDATE* July 2014: we are actively working on a service for hosting user-uploaded model definition and trained weight files. +Soon, the community will be able to easily contribute different architectures! ### ImageNet @@ -26,4 +27,6 @@ This page will be updated as more models become available. validation accuracy 57.258% and loss 1.83948. - This model obtains a top-1 accuracy 57.1% and a top-5 accuracy 80.2% on the validation set, using just the center crop. (Using the average of 10 crops, (4 + 1 center) * 2 mirror, should obtain a bit higher accuracy) +### Auxiliary Data + Additionally, you will probably eventually need some auxiliary data (mean image, synset list, etc.): run `data/ilsvrc12/get_ilsvrc_aux.sh` from the root directory to obtain it. diff --git a/examples/web_demo/app.py b/examples/web_demo/app.py new file mode 100644 index 0000000..9bc4ed5 --- /dev/null +++ b/examples/web_demo/app.py @@ -0,0 +1,215 @@ +import os +import time +import cPickle +import datetime +import logging +import flask +import werkzeug +import optparse +import tornado.wsgi +import tornado.httpserver +import numpy as np +import pandas as pd +from PIL import Image as PILImage +import cStringIO as StringIO +import urllib +import caffe +import exifutil + +REPO_DIRNAME = os.path.abspath(os.path.dirname(__file__) + '/../..') +UPLOAD_FOLDER = '/tmp/caffe_demos_uploads' +ALLOWED_IMAGE_EXTENSIONS = set(['png', 'bmp', 'jpg', 'jpe', 'jpeg', 'gif']) + +# Obtain the flask app object +app = flask.Flask(__name__) + + +@app.route('/') +def index(): + return flask.render_template('index.html', has_result=False) + + +@app.route('/classify_url', methods=['GET']) +def classify_url(): + imageurl = flask.request.args.get('imageurl', '') + try: + string_buffer = StringIO.StringIO( + urllib.urlopen(imageurl).read()) + image = caffe.io.load_image(string_buffer) + + except Exception as err: + # For any exception we encounter in reading the image, we will just + # not continue. + logging.info('URL Image open error: %s', err) + return flask.render_template( + 'index.html', has_result=True, + result=(False, 'Cannot open image from URL.') + ) + + logging.info('Image: %s', imageurl) + result = app.clf.classify_image(image) + return flask.render_template( + 'index.html', has_result=True, result=result, imagesrc=imageurl) + + +@app.route('/classify_upload', methods=['POST']) +def classify_upload(): + try: + # We will save the file to disk for possible data collection. + imagefile = flask.request.files['imagefile'] + filename_ = str(datetime.datetime.now()).replace(' ', '_') + \ + werkzeug.secure_filename(imagefile.filename) + filename = os.path.join(UPLOAD_FOLDER, filename_) + imagefile.save(filename) + logging.info('Saving to %s.', filename) + image = exifutil.open_oriented_im(filename) + + except Exception as err: + logging.info('Uploaded image open error: %s', err) + return flask.render_template( + 'index.html', has_result=True, + result=(False, 'Cannot open uploaded image.') + ) + + result = app.clf.classify_image(image) + return flask.render_template( + 'index.html', has_result=True, result=result, + imagesrc=embed_image_html(image) + ) + + +def embed_image_html(image): + """Creates an image embedded in HTML base64 format.""" + image_pil = PILImage.fromarray((255 * image).astype('uint8')) + image_pil = image_pil.resize((256, 256)) + string_buf = StringIO.StringIO() + image_pil.save(string_buf, format='png') + data = string_buf.getvalue().encode('base64').replace('\n', '') + return 'data:image/png;base64,' + data + + +def allowed_file(filename): + return ( + '.' in filename and + filename.rsplit('.', 1)[1] in ALLOWED_IMAGE_EXTENSIONS + ) + + +class ImagenetClassifier(object): + default_args = { + 'model_def_file': ( + '{}/examples/imagenet/imagenet_deploy.prototxt'.format(REPO_DIRNAME)), + 'pretrained_model_file': ( + '{}/examples/imagenet/caffe_reference_imagenet_model'.format(REPO_DIRNAME)), + 'mean_file': ( + '{}/python/caffe/imagenet/ilsvrc_2012_mean.npy'.format(REPO_DIRNAME)), + 'class_labels_file': ( + '{}/data/ilsvrc12/synset_words.txt'.format(REPO_DIRNAME)), + 'bet_file': ( + '{}/data/ilsvrc12/imagenet.bet.pickle'.format(REPO_DIRNAME)), + } + for key, val in default_args.iteritems(): + if not os.path.exists(val): + raise Exception( + "File for {} is missing. Should be at: {}".format(key, val)) + default_args['image_dim'] = 227 + default_args['gpu_mode'] = True + + def __init__(self, model_def_file, pretrained_model_file, mean_file, + class_labels_file, bet_file, image_dim, gpu_mode=False): + logging.info('Loading net and associated files...') + self.net = caffe.Classifier( + model_def_file, pretrained_model_file, input_scale=255, + image_dims=(image_dim, image_dim), gpu=gpu_mode, + mean_file=mean_file, channel_swap=(2, 1, 0) + ) + + with open(class_labels_file) as f: + labels_df = pd.DataFrame([ + { + 'synset_id': l.strip().split(' ')[0], + 'name': ' '.join(l.strip().split(' ')[1:]).split(',')[0] + } + for l in f.readlines() + ]) + self.labels = labels_df.sort('synset_id')['name'].values + + self.bet = cPickle.load(open(bet_file)) + # A bias to prefer children nodes in single-chain paths + # I am setting the value to 0.1 as a quick, simple model. + # We could use better psychological models here... + self.bet['infogain'] -= np.array(self.bet['preferences']) * 0.1 + + def classify_image(self, image): + try: + starttime = time.time() + scores = self.net.predict([image], oversample=True).flatten() + endtime = time.time() + + indices = (-scores).argsort()[:5] + predictions = self.labels[indices] + + # In addition to the prediction text, we will also produce + # the length for the progress bar visualization. + meta = [ + (p, '%.5f' % scores[i]) + for i, p in zip(indices, predictions) + ] + logging.info('result: %s', str(meta)) + + # Compute expected information gain + expected_infogain = np.dot( + self.bet['probmat'], scores[self.bet['idmapping']]) + expected_infogain *= self.bet['infogain'] + + # sort the scores + infogain_sort = expected_infogain.argsort()[::-1] + bet_result = [(self.bet['words'][v], '%.5f' % expected_infogain[v]) + for v in infogain_sort[:5]] + logging.info('bet result: %s', str(bet_result)) + + return (True, meta, bet_result, '%.3f' % (endtime - starttime)) + + except Exception as err: + logging.info('Classification error: %s', err) + return (False, 'Something went wrong when classifying the ' + 'image. Maybe try another one?') + + +def start_tornado(app, port=5000): + http_server = tornado.httpserver.HTTPServer( + tornado.wsgi.WSGIContainer(app)) + http_server.listen(port) + print("Tornado server starting on port {}".format(port)) + tornado.ioloop.IOLoop.instance().start() + + +def start_from_terminal(app): + """ + Parse command line options and start the server. + """ + parser = optparse.OptionParser() + parser.add_option( + '-d', '--debug', + help="enable debug mode", + action="store_true", default=False) + parser.add_option( + '-p', '--port', + help="which port to serve content on", + type='int', default=5000) + opts, args = parser.parse_args() + + # Initialize classifier + app.clf = ImagenetClassifier(**ImagenetClassifier.default_args) + + if opts.debug: + app.run(debug=True, host='0.0.0.0', port=opts.port) + else: + start_tornado(app, opts.port) + + +if __name__ == '__main__': + logging.getLogger().setLevel(logging.INFO) + if not os.path.exists(UPLOAD_FOLDER): + os.makedirs(UPLOAD_FOLDER) + start_from_terminal(app) diff --git a/examples/web_demo/exifutil.py b/examples/web_demo/exifutil.py new file mode 100644 index 0000000..8c07aa8 --- /dev/null +++ b/examples/web_demo/exifutil.py @@ -0,0 +1,33 @@ +""" +This script handles the skimage exif problem. +""" + +from PIL import Image +import numpy as np + +ORIENTATIONS = { # used in apply_orientation + 2: (Image.FLIP_LEFT_RIGHT,), + 3: (Image.ROTATE_180,), + 4: (Image.FLIP_TOP_BOTTOM,), + 5: (Image.FLIP_LEFT_RIGHT, Image.ROTATE_90), + 6: (Image.ROTATE_270,), + 7: (Image.FLIP_LEFT_RIGHT, Image.ROTATE_270), + 8: (Image.ROTATE_90,) +} + + +def open_oriented_im(im_path): + im = Image.open(im_path) + if hasattr(im, '_getexif'): + exif = im._getexif() + if exif is not None and 274 in exif: + orientation = exif[274] + im = apply_orientation(im, orientation) + return np.asarray(im).astype(np.float32) / 255. + + +def apply_orientation(im, orientation): + if orientation in ORIENTATIONS: + for method in ORIENTATIONS[orientation]: + im = im.transpose(method) + return im diff --git a/examples/web_demo/readme.md b/examples/web_demo/readme.md new file mode 100644 index 0000000..559c41e --- /dev/null +++ b/examples/web_demo/readme.md @@ -0,0 +1,30 @@ +--- +title: Web demo +description: Image classification demo running as a Flask web server. +category: example +layout: default +include_in_docs: true +--- + +# Web Demo + +## Requirements + +The demo server requires Python with some dependencies. +To make sure you have the dependencies, please run `pip install -r examples/web_demo/requirements.txt`, and also make sure that you've compiled the Python Caffe interface and that it is on your `PYTHONPATH` (see [installation instructions](/installation.html)). + +Make sure that you have obtained the Caffe Reference ImageNet Model and the ImageNet Auxiliary Data ([instructions](/getting_pretrained_models.html)). +NOTE: if you run into trouble, try re-downloading the auxiliary files. + +## Run + +Running `python examples/web_demo/app.py` will bring up the demo server, accessible at `http://0.0.0.0:5000`. +You can enable debug mode of the web server, or switch to a different port: + + % python examples/web_demo/app.py -h + Usage: app.py [options] + + Options: + -h, --help show this help message and exit + -d, --debug enable debug mode + -p PORT, --port=PORT which port to serve content on diff --git a/examples/web_demo/templates/index.html b/examples/web_demo/templates/index.html new file mode 100644 index 0000000..8789334 --- /dev/null +++ b/examples/web_demo/templates/index.html @@ -0,0 +1,138 @@ + + + + + + + + + Caffe Demos + + + + + + + + + + + + + + +
+
+

Caffe Demos

+

+ The Caffe neural network library makes implementing state-of-the-art computer vision systems easy. +

+
+ +
+

Classification

+ Click for a Quick Example +
+ + {% if has_result %} + {% if not result[0] %} + +
{{ result[1] }} Did you provide a valid URL or a valid image file?
+ {% else %} +
+ +
+
+ +
+
+
    + {% for single_pred in result[2] %} +
  • + {{ single_pred[1] }} +

    + {{ single_pred[0] }} +

    +
    • + {% endfor %} +
+
+
+
    + {% for single_pred in result[1] %} +
  • + {{ single_pred[1] }} +

    + {{ single_pred[0] }} +

    +
    • + {% endfor %} +
+
+
+
+ +
+
+

CNN took {{ result[3] }} seconds.

+ {% endif %} +
+ {% endif %} + +
+
+
+ + + + +
+
+
+ +
+
+ + +
+ +
+
+ +
+ + +