From: A. Unique TensorFlower Date: Fri, 23 Mar 2018 21:41:02 +0000 (-0700) Subject: Benchmarker for LPIRC CVPR2018. X-Git-Tag: tflite-v0.1.7~106^2^2~33 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=18832ec8497a6acc6f828808e5ea3a2859548efa;p=platform%2Fupstream%2Ftensorflow.git Benchmarker for LPIRC CVPR2018. PiperOrigin-RevId: 190276899 --- diff --git a/tensorflow/contrib/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicBenchmarker.java b/tensorflow/contrib/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicBenchmarker.java new file mode 100644 index 0000000..d010288 --- /dev/null +++ b/tensorflow/contrib/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicBenchmarker.java @@ -0,0 +1,197 @@ +/*Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +package org.tensorflow.ovic; + +import android.graphics.Bitmap; +import android.os.SystemClock; +import android.util.Log; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.MappedByteBuffer; + +/** + * Class that benchmarks image classifier models. + * + *

===================== General workflow ======================= + * + * 

{@code
+ * benchmarker = new OvicBenchmarker();
+ * benchmarker.getReadyToTest(labelInputStream, model);
+ * while (!benchmarker.shouldStop()) {
+ *   Bitmap bitmap = ...
+ *   benchmarker.doTestIteration(bitmap);
+ * }
+ * }
+ */ +public class OvicBenchmarker { + /** Tag for the {@link Log}. */ + private static final String TAG = "OvicBenchmarker"; + + /** Evaluation transformation parameters. */ + private static final float CENTRAL_FRACTION = 0.875f; + + /** Dimensions of inputs. */ + private static final int DIM_BATCH_SIZE = 1; + private static final int DIM_PIXEL_SIZE = 3; + private int imgHeight = 224; + private int imgWidth = 224; + + /* Preallocated buffers for storing image data in. */ + private int[] intValues = null; + + /** A ByteBuffer to hold image data, to be feed into classifier as inputs. */ + private ByteBuffer imgData = null; + + private OvicClassifier classifier; + + /** Total runtime in ms. */ + private double totalRuntime = 0.0; + /** Total allowed runtime in ms. */ + private double wallTime = 20000 * 30.0; + + private Boolean benchmarkStarted = null; + + /** + * Initializes an {@link OvicBenchmarker} + * + * @param wallTime: a double number specifying the total amount of time to benchmark. + */ + public OvicBenchmarker(double wallTime) { + benchmarkStarted = false; + totalRuntime = 0.0; + this.wallTime = wallTime; + } + + /** Check whether the benchmarker should stop. */ + public Boolean shouldStop() { + if (totalRuntime >= wallTime) { + Log.e( + TAG, + "Total runtime " + + Double.toString(totalRuntime) + + " exceeded walltime " + + Double.toString(wallTime)); + return true; + } + return false; + } + + /** Check whether the benchmarker is ready to start classifying images. */ + public Boolean readyToTest() { + return (classifier != null); + } + + /** + * Getting the benchmarker ready for classifying images. + * + * @param labelInputStream: an {@link InputStream} specifying where the list of labels should be + * read from. + * @param model: a {@link MappedByteBuffer} model to benchmark. + */ + public void getReadyToTest(InputStream labelInputStream, MappedByteBuffer model) { + try { + Log.i(TAG, "Creating classifier."); + classifier = new OvicClassifier(labelInputStream, model); + int [] inputDims = classifier.getInputDims(); + imgHeight = inputDims[1]; + imgWidth = inputDims[2]; + // Only accept QUANTIZED_UINT8 input. + imgData = ByteBuffer.allocateDirect(DIM_BATCH_SIZE * imgHeight * imgWidth * DIM_PIXEL_SIZE); + imgData.order(ByteOrder.nativeOrder()); + intValues = new int[imgHeight * imgWidth]; + } catch (Exception e) { + Log.e(TAG, e.getMessage()); + Log.e(TAG, "Failed to initialize ImageNet classifier for the benchmarker."); + } + } + + /** Return how many classes are predicted per image. */ + public int getNumPredictions() { + return classifier.getNumPredictions(); + } + + /** + * Perform test on a single bitmap image. + * + * @param bitmap: a {@link Bitmap} image to classify. + */ + public OvicSingleImageResult doTestIteration(Bitmap bitmap) + throws IOException, InterruptedException { + if (shouldStop() || !readyToTest()) { + return null; + } + OvicSingleImageResult iterResult = null; + try { + Log.i(TAG, "Converting bitmap."); + convertBitmapToInput(bitmap); + Log.i(TAG, "Classifying image."); + iterResult = classifier.classifyByteBuffer(imgData); + } catch (RuntimeException e) { + Log.e(TAG, e.getMessage()); + Log.e(TAG, "Failed to classify image."); + } + if (iterResult == null || iterResult.latency == null) { + throw new RuntimeException("Classification result or timing is invalid."); + } + Log.d(TAG, "Native inference latency: " + iterResult.latency); + Log.i(TAG, iterResult.toString()); + + if (!benchmarkStarted) { // Skip the first image to discount warming-up time. + benchmarkStarted = true; + } else { + totalRuntime += (double) iterResult.latency; + } + return iterResult; + } + + /** + * Writes Image data into a {@link ByteBuffer}. + * + * @param bitmap: a {@link Bitmap} source image. + */ + private void convertBitmapToInput(Bitmap bitmap) throws RuntimeException { + if (imgData == null) { + throw new RuntimeException("Benchmarker is not yet ready to test."); + } + imgData.rewind(); + // Perform transformations corresponding to evaluation mode. + float width = (float) bitmap.getWidth(); + float height = (float) bitmap.getHeight(); + int stWidth = Math.round((width - width * CENTRAL_FRACTION) / 2); + int stHeight = Math.round((height - height * CENTRAL_FRACTION) / 2); + int newWidth = Math.round(width - stWidth * 2); + int newHeight = Math.round(height - stHeight * 2); + bitmap = Bitmap.createBitmap(bitmap, stWidth, stHeight, newWidth, newHeight); + bitmap = Bitmap.createScaledBitmap(bitmap, imgWidth, imgHeight, true); + bitmap.getPixels(intValues, 0, bitmap.getWidth(), 0, 0, bitmap.getWidth(), bitmap.getHeight()); + + // Convert the image to ByteBuffer. + int pixel = 0; + long startTime = SystemClock.uptimeMillis(); + + for (int i = 0; i < imgHeight; ++i) { + for (int j = 0; j < imgWidth; ++j) { + final int val = intValues[pixel++]; + imgData.put((byte) ((val >> 16) & 0xFF)); + imgData.put((byte) ((val >> 8) & 0xFF)); + imgData.put((byte) (val & 0xFF)); + } + } + long endTime = SystemClock.uptimeMillis(); + Log.d(TAG, "Timecost to put values into ByteBuffer: " + Long.toString(endTime - startTime)); + } +} diff --git a/tensorflow/contrib/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicClassifier.java b/tensorflow/contrib/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicClassifier.java new file mode 100644 index 0000000..b2dfd8f --- /dev/null +++ b/tensorflow/contrib/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicClassifier.java @@ -0,0 +1,209 @@ +/*Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +package org.tensorflow.ovic; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.ByteBuffer; +import java.nio.MappedByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.PriorityQueue; +import org.tensorflow.lite.Interpreter; +import org.tensorflow.lite.TestHelper; + +/** Benchmark ImageNet Classifier with Tensorflow Lite. */ +public class OvicClassifier { + + /** Tag for the {@link Log}. */ + private static final String TAG = "OvicClassifier"; + + /** Number of results to show (i.e. the "K" in top-K predictions). */ + private static final int RESULTS_TO_SHOW = 5; + + /** An instance of the driver class to run model inference with Tensorflow Lite. */ + private Interpreter tflite; + + /** Labels corresponding to the output of the vision model. */ + private List labelList; + + /** An array to hold inference results, to be feed into Tensorflow Lite as outputs. */ + private byte[][] inferenceOutputArray = null; + /** An array to hold final prediction probabilities. */ + private float[][] labelProbArray = null; + + /** Input resultion. */ + private int[] inputDims = null; + /** Whether the model runs as float or quantized. */ + private Boolean outputIsFloat = null; + + private PriorityQueue> sortedLabels = + new PriorityQueue<>( + RESULTS_TO_SHOW, + new Comparator>() { + @Override + public int compare(Map.Entry o1, Map.Entry o2) { + return (o1.getValue()).compareTo(o2.getValue()); + } + }); + + /** Initializes an {@code OvicClassifier}. */ + OvicClassifier(InputStream labelInputStream, MappedByteBuffer model) + throws IOException, RuntimeException { + if (model == null) { + throw new RuntimeException("Input model is empty."); + } + labelList = loadLabelList(labelInputStream); + // OVIC uses one thread for CPU inference. + tflite = new Interpreter(model, 1); + inputDims = TestHelper.getInputDims(tflite, 0); + if (inputDims.length != 4) { + throw new RuntimeException("The model's input dimensions must be 4 (BWHC)."); + } + if (inputDims[0] != 1) { + throw new RuntimeException("The model must have a batch size of 1, got " + + inputDims[0] + " instead."); + } + if (inputDims[3] != 3) { + throw new RuntimeException("The model must have three color channels, got " + + inputDims[3] + " instead."); + } + int minSide = Math.min(inputDims[1], inputDims[2]); + int maxSide = Math.max(inputDims[1], inputDims[2]); + if (minSide <= 0 || maxSide > 1000) { + throw new RuntimeException("The model's resolution must be between (0, 1000]."); + } + String outputDataType = TestHelper.getOutputDataType(tflite, 0); + if (outputDataType.equals("float")) { + outputIsFloat = true; + } else if (outputDataType.equals("byte")) { + outputIsFloat = false; + } else { + throw new RuntimeException("Cannot process output type: " + outputDataType); + } + inferenceOutputArray = new byte[1][labelList.size()]; + labelProbArray = new float[1][labelList.size()]; + } + + /** Classifies a {@link ByteBuffer} image. */ + // @throws RuntimeException if model is uninitialized. + OvicSingleImageResult classifyByteBuffer(ByteBuffer imgData) throws RuntimeException { + if (tflite == null) { + throw new RuntimeException(TAG + ": ImageNet classifier has not been initialized; Failed."); + } + if (outputIsFloat == null) { + throw new RuntimeException(TAG + ": Classifier output type has not been resolved."); + } + if (outputIsFloat) { + tflite.run(imgData, labelProbArray); + } else { + tflite.run(imgData, inferenceOutputArray); + /** Convert results to float */ + for (int i = 0; i < inferenceOutputArray[0].length; i++) { + labelProbArray[0][i] = (inferenceOutputArray[0][i] & 0xff) / 255.0f; + } + } + OvicSingleImageResult iterResult = computeTopKLabels(); + iterResult.latency = getLastNativeInferenceLatencyMilliseconds(); + return iterResult; + } + + /** Return the probability array of all classes. */ + public float[][] getlabelProbArray() { + return labelProbArray; + } + + /** Return the number of top labels predicted by the classifier. */ + public int getNumPredictions() { + return RESULTS_TO_SHOW; + } + + /** Return the four dimensions of the input image. */ + public int[] getInputDims() { + return inputDims; + } + + /* + * Get native inference latency of last image classification run. + * @throws RuntimeException if model is uninitialized. + */ + public Long getLastNativeInferenceLatencyMilliseconds() { + if (tflite == null) { + throw new RuntimeException(TAG + ": ImageNet classifier has not been initialized; Failed."); + } + Long latency = tflite.getLastNativeInferenceDurationNanoseconds(); + return (latency == null) ? null : (Long) (latency / 1000000); + } + + /** Closes tflite to release resources. */ + public void close() { + tflite.close(); + tflite = null; + } + + /** Reads label list from Assets. */ + private static List loadLabelList(InputStream labelInputStream) throws IOException { + List labelList = new ArrayList(); + try (BufferedReader reader = + new BufferedReader(new InputStreamReader(labelInputStream, StandardCharsets.UTF_8))) { + String line; + while ((line = reader.readLine()) != null) { + labelList.add(line); + } + } + return labelList; + } + + /** Computes top-K labels. */ + private OvicSingleImageResult computeTopKLabels() { + if (labelList == null) { + throw new RuntimeException("Label file has not been loaded."); + } + for (int i = 0; i < labelList.size(); ++i) { + sortedLabels.add(new AbstractMap.SimpleEntry<>(i, labelProbArray[0][i])); + if (sortedLabels.size() > RESULTS_TO_SHOW) { + sortedLabels.poll(); + } + } + OvicSingleImageResult singleImageResult = new OvicSingleImageResult(); + if (sortedLabels.size() != RESULTS_TO_SHOW) { + throw new RuntimeException( + "Number of returned labels does not match requirement: " + + sortedLabels.size() + + " returned, but " + + RESULTS_TO_SHOW + + " required."); + } + for (int i = 0; i < RESULTS_TO_SHOW; ++i) { + Map.Entry label = sortedLabels.poll(); + // ImageNet model prediction indices are 0-based. + singleImageResult.topKIndices.add(label.getKey()); + singleImageResult.topKClasses.add(labelList.get(label.getKey())); + singleImageResult.topKProbs.add(label.getValue()); + } + // Labels with lowest probability are returned first, hence need to reverse them. + Collections.reverse(singleImageResult.topKIndices); + Collections.reverse(singleImageResult.topKClasses); + Collections.reverse(singleImageResult.topKProbs); + return singleImageResult; + } +} diff --git a/tensorflow/contrib/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicSingleImageResult.java b/tensorflow/contrib/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicSingleImageResult.java new file mode 100644 index 0000000..4af9a65 --- /dev/null +++ b/tensorflow/contrib/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicSingleImageResult.java @@ -0,0 +1,54 @@ +/*Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +package org.tensorflow.ovic; + +import java.util.ArrayList; + +/** Result class for inference run on a single image. */ +public class OvicSingleImageResult { + + /** Top K classes and probabilities. */ + public ArrayList topKClasses; + public ArrayList topKProbs; + public ArrayList topKIndices; + + /** Latency (ms). */ + public Long latency; + + OvicSingleImageResult() { + topKClasses = new ArrayList<>(); + topKProbs = new ArrayList<>(); + topKIndices = new ArrayList<>(); + latency = -1L; + } + + @Override + public String toString() { + String textToShow = latency + "ms"; + for (int k = 0; k < topKProbs.size(); ++k) { + textToShow += + "\nPrediction [" + + k + + "] = Class " + + Integer.toString(topKIndices.get(k)) + + " (" + + topKClasses.get(k) + + ") : " + + Float.toString(topKProbs.get(k)); + } + return textToShow; + } + +} diff --git a/tensorflow/contrib/lite/java/ovic/src/test/java/org/tensorflow/ovic/OvicClassifierTest.java b/tensorflow/contrib/lite/java/ovic/src/test/java/org/tensorflow/ovic/OvicClassifierTest.java new file mode 100644 index 0000000..4fd23a9 --- /dev/null +++ b/tensorflow/contrib/lite/java/ovic/src/test/java/org/tensorflow/ovic/OvicClassifierTest.java @@ -0,0 +1,176 @@ +/*Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +package org.tensorflow.ovic; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.fail; + +import java.awt.image.BufferedImage; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.Paths; +import javax.imageio.ImageIO; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link org.tensorflow.ovic.OvicClassifier}. */ +@RunWith(JUnit4.class) +public final class OvicClassifierTest { + + private OvicClassifier classifier; + private InputStream labelsInputStream = null; + private MappedByteBuffer quantizedModel = null; + private MappedByteBuffer floatModel = null; + private MappedByteBuffer lowResModel = null; + private ByteBuffer testImage = null; + private ByteBuffer lowResTestImage = null; + private OvicSingleImageResult testResult = null; + private static final String LABELS_PATH = "testdata/labels.txt"; + private static final String QUANTIZED_MODEL_PATH = "testdata/quantized_model.lite"; + private static final String LOW_RES_MODEL_PATH = "testdata/low_res_model.lite"; + private static final String FLOAT_MODEL_PATH = "testdata/float_model.lite"; + private static final String TEST_IMAGE_PATH = "testdata/test_image_224.jpg"; + private static final String TEST_LOW_RES_IMAGE_PATH = "testdata/test_image_128.jpg"; + private static final int TEST_IMAGE_GROUNDTRUTH = 653; // "military uniform" + + @Before + public void setUp() { + try { + File labelsfile = new File(getTestDir(LABELS_PATH)); + labelsInputStream = new FileInputStream(labelsfile); + quantizedModel = loadModelFile(getTestDir(QUANTIZED_MODEL_PATH)); + floatModel = loadModelFile(getTestDir(FLOAT_MODEL_PATH)); + lowResModel = loadModelFile(getTestDir(LOW_RES_MODEL_PATH)); + File imageFile = new File(getTestDir(TEST_IMAGE_PATH)); + BufferedImage img = ImageIO.read(imageFile); + testImage = toByteBuffer(img); + // Low res image and models. + imageFile = new File(getTestDir(TEST_LOW_RES_IMAGE_PATH)); + img = ImageIO.read(imageFile); + lowResTestImage = toByteBuffer(img); + } catch (IOException e) { + System.out.print(e.getMessage()); + } + System.out.println("Successful setup"); + } + + private static String getTestDir(String testfile) throws IOException { + return Paths.get("third_party/tensorflow/contrib/lite/java/ovic/src/", testfile).toString(); + } + + @Test + public void ovicClassifier_quantizedModelCreateSuccess() throws Exception { + classifier = new OvicClassifier(labelsInputStream, quantizedModel); + assertThat(classifier != null).isTrue(); + } + + @Test + public void ovicClassifier_floatModelCreateSuccess() throws Exception { + classifier = new OvicClassifier(labelsInputStream, floatModel); + assertThat(classifier != null).isTrue(); + } + + @Test + public void ovicClassifier_quantizedModelClassifySuccess() throws Exception { + classifier = new OvicClassifier(labelsInputStream, quantizedModel); + testResult = classifier.classifyByteBuffer(testImage); + assertCorrectTopK(testResult); + } + + @Test + public void ovicClassifier_floatModelClassifySuccess() throws Exception { + classifier = new OvicClassifier(labelsInputStream, floatModel); + testResult = classifier.classifyByteBuffer(testImage); + assertCorrectTopK(testResult); + } + + @Test + public void ovicClassifier_lowResModelClassifySuccess() throws Exception { + classifier = new OvicClassifier(labelsInputStream, lowResModel); + testResult = classifier.classifyByteBuffer(lowResTestImage); + assertCorrectTopK(testResult); + } + + @Test + public void ovicClassifier_latencyNotNull() throws Exception { + classifier = new OvicClassifier(labelsInputStream, floatModel); + testResult = classifier.classifyByteBuffer(testImage); + assertThat(testResult.latency != null).isTrue(); + } + + @Test + public void ovicClassifier_mismatchedInputResolutionFails() throws Exception { + classifier = new OvicClassifier(labelsInputStream, lowResModel); + int[] inputDims = classifier.getInputDims(); + assertThat((inputDims[1] == 128) && (inputDims[2] == 128)).isTrue(); + try { + testResult = classifier.classifyByteBuffer(testImage); + fail(); + } catch (RuntimeException e) { + assertThat(e) + .hasMessageThat() + .contains( + "Failed to get input dimensions. 0-th input should have 49152 bytes, " + + "but found 150528 bytes."); + } + } + + private static ByteBuffer toByteBuffer(BufferedImage image) { + ByteBuffer imgData = ByteBuffer.allocateDirect( + image.getHeight() * image.getWidth() * 3); + imgData.order(ByteOrder.nativeOrder()); + for (int y = 0; y < image.getHeight(); y++) { + for (int x = 0; x < image.getWidth(); x++) { + int val = image.getRGB(x, y); + imgData.put((byte) ((val >> 16) & 0xFF)); + imgData.put((byte) ((val >> 8) & 0xFF)); + imgData.put((byte) (val & 0xFF)); + } + } + return imgData; + } + + private static void assertCorrectTopK(OvicSingleImageResult testResult) { + assertThat(testResult.topKClasses.size() > 0).isTrue(); + Boolean topKAccurate = false; + // Assert that the correct class is in the top K. + for (int i = 0; i < testResult.topKIndices.size(); i++) { + if (testResult.topKIndices.get(i) == TEST_IMAGE_GROUNDTRUTH) { + topKAccurate = true; + break; + } + } + System.out.println(testResult.toString()); + System.out.flush(); + assertThat(topKAccurate).isTrue(); + } + + private static MappedByteBuffer loadModelFile(String modelFilePath) throws IOException { + File modelfile = new File(modelFilePath); + FileInputStream inputStream = new FileInputStream(modelfile); + FileChannel fileChannel = inputStream.getChannel(); + long startOffset = 0L; + long declaredLength = fileChannel.size(); + return fileChannel.map(FileChannel.MapMode.READ_ONLY, startOffset, declaredLength); + } +} diff --git a/tensorflow/contrib/lite/java/ovic/src/testdata/labels.txt b/tensorflow/contrib/lite/java/ovic/src/testdata/labels.txt new file mode 100644 index 0000000..fe81123 --- /dev/null +++ b/tensorflow/contrib/lite/java/ovic/src/testdata/labels.txt @@ -0,0 +1,1001 @@ +background +tench +goldfish +great white shark +tiger shark +hammerhead +electric ray +stingray +cock +hen +ostrich +brambling +goldfinch +house finch +junco +indigo bunting +robin +bulbul +jay +magpie +chickadee +water ouzel +kite +bald eagle +vulture +great grey owl +European fire salamander +common newt +eft +spotted salamander +axolotl +bullfrog +tree frog +tailed frog +loggerhead +leatherback turtle +mud turtle +terrapin +box turtle +banded gecko +common iguana +American chameleon +whiptail +agama +frilled lizard +alligator lizard +Gila monster +green lizard +African chameleon +Komodo dragon +African crocodile +American alligator +triceratops +thunder snake +ringneck snake +hognose snake +green snake +king snake +garter snake +water snake +vine snake +night snake +boa constrictor +rock python +Indian cobra +green mamba +sea snake +horned viper +diamondback +sidewinder +trilobite +harvestman +scorpion +black and gold garden spider +barn spider +garden spider +black widow +tarantula +wolf spider +tick +centipede +black grouse +ptarmigan +ruffed grouse +prairie chicken +peacock +quail +partridge +African grey +macaw +sulphur-crested cockatoo +lorikeet +coucal +bee eater +hornbill +hummingbird +jacamar +toucan +drake +red-breasted merganser +goose +black swan +tusker +echidna +platypus +wallaby +koala +wombat +jellyfish +sea anemone +brain coral +flatworm +nematode +conch +snail +slug +sea slug +chiton +chambered nautilus +Dungeness crab +rock crab +fiddler crab +king crab +American lobster +spiny lobster +crayfish +hermit crab +isopod +white stork +black stork +spoonbill +flamingo +little blue heron +American egret +bittern +crane +limpkin +European gallinule +American coot +bustard +ruddy turnstone +red-backed sandpiper +redshank +dowitcher +oystercatcher +pelican +king penguin +albatross +grey whale +killer whale +dugong +sea lion +Chihuahua +Japanese spaniel +Maltese dog +Pekinese +Shih-Tzu +Blenheim spaniel +papillon +toy terrier +Rhodesian ridgeback +Afghan hound +basset +beagle +bloodhound +bluetick +black-and-tan coonhound +Walker hound +English foxhound +redbone +borzoi +Irish wolfhound +Italian greyhound +whippet +Ibizan hound +Norwegian elkhound +otterhound +Saluki +Scottish deerhound +Weimaraner +Staffordshire bullterrier +American Staffordshire terrier +Bedlington terrier +Border terrier +Kerry blue terrier +Irish terrier +Norfolk terrier +Norwich terrier +Yorkshire terrier +wire-haired fox terrier +Lakeland terrier +Sealyham terrier +Airedale +cairn +Australian terrier +Dandie Dinmont +Boston bull +miniature schnauzer +giant schnauzer +standard schnauzer +Scotch terrier +Tibetan terrier +silky terrier +soft-coated wheaten terrier +West Highland white terrier +Lhasa +flat-coated retriever +curly-coated retriever +golden retriever +Labrador retriever +Chesapeake Bay retriever +German short-haired pointer +vizsla +English setter +Irish setter +Gordon setter +Brittany spaniel +clumber +English springer +Welsh springer spaniel +cocker spaniel +Sussex spaniel +Irish water spaniel +kuvasz +schipperke +groenendael +malinois +briard +kelpie +komondor +Old English sheepdog +Shetland sheepdog +collie +Border collie +Bouvier des Flandres +Rottweiler +German shepherd +Doberman +miniature pinscher +Greater Swiss Mountain dog +Bernese mountain dog +Appenzeller +EntleBucher +boxer +bull mastiff +Tibetan mastiff +French bulldog +Great Dane +Saint Bernard +Eskimo dog +malamute +Siberian husky +dalmatian +affenpinscher +basenji +pug +Leonberg +Newfoundland +Great Pyrenees +Samoyed +Pomeranian +chow +keeshond +Brabancon griffon +Pembroke +Cardigan +toy poodle +miniature poodle +standard poodle +Mexican hairless +timber wolf +white wolf +red wolf +coyote +dingo +dhole +African hunting dog +hyena +red fox +kit fox +Arctic fox +grey fox +tabby +tiger cat +Persian cat +Siamese cat +Egyptian cat +cougar +lynx +leopard +snow leopard +jaguar +lion +tiger +cheetah +brown bear +American black bear +ice bear +sloth bear +mongoose +meerkat +tiger beetle +ladybug +ground beetle +long-horned beetle +leaf beetle +dung beetle +rhinoceros beetle +weevil +fly +bee +ant +grasshopper +cricket +walking stick +cockroach +mantis +cicada +leafhopper +lacewing +dragonfly +damselfly +admiral +ringlet +monarch +cabbage butterfly +sulphur butterfly +lycaenid +starfish +sea urchin +sea cucumber +wood rabbit +hare +Angora +hamster +porcupine +fox squirrel +marmot +beaver +guinea pig +sorrel +zebra +hog +wild boar +warthog +hippopotamus +ox +water buffalo +bison +ram +bighorn +ibex +hartebeest +impala +gazelle +Arabian camel +llama +weasel +mink +polecat +black-footed ferret +otter +skunk +badger +armadillo +three-toed sloth +orangutan +gorilla +chimpanzee +gibbon +siamang +guenon +patas +baboon +macaque +langur +colobus +proboscis monkey +marmoset +capuchin +howler monkey +titi +spider monkey +squirrel monkey +Madagascar cat +indri +Indian elephant +African elephant +lesser panda +giant panda +barracouta +eel +coho +rock beauty +anemone fish +sturgeon +gar +lionfish +puffer +abacus +abaya +academic gown +accordion +acoustic guitar +aircraft carrier +airliner +airship +altar +ambulance +amphibian +analog clock +apiary +apron +ashcan +assault rifle +backpack +bakery +balance beam +balloon +ballpoint +Band Aid +banjo +bannister +barbell +barber chair +barbershop +barn +barometer +barrel +barrow +baseball +basketball +bassinet +bassoon +bathing cap +bath towel +bathtub +beach wagon +beacon +beaker +bearskin +beer bottle +beer glass +bell cote +bib +bicycle-built-for-two +bikini +binder +binoculars +birdhouse +boathouse +bobsled +bolo tie +bonnet +bookcase +bookshop +bottlecap +bow +bow tie +brass +brassiere +breakwater +breastplate +broom +bucket +buckle +bulletproof vest +bullet train +butcher shop +cab +caldron +candle +cannon +canoe +can opener +cardigan +car mirror +carousel +carpenter's kit +carton +car wheel +cash machine +cassette +cassette player +castle +catamaran +CD player +cello +cellular telephone +chain +chainlink fence +chain mail +chain saw +chest +chiffonier +chime +china cabinet +Christmas stocking +church +cinema +cleaver +cliff dwelling +cloak +clog +cocktail shaker +coffee mug +coffeepot +coil +combination lock +computer keyboard +confectionery +container ship +convertible +corkscrew +cornet +cowboy boot +cowboy hat +cradle +crane +crash helmet +crate +crib +Crock Pot +croquet ball +crutch +cuirass +dam +desk +desktop computer +dial telephone +diaper +digital clock +digital watch +dining table +dishrag +dishwasher +disk brake +dock +dogsled +dome +doormat +drilling platform +drum +drumstick +dumbbell +Dutch oven +electric fan +electric guitar +electric locomotive +entertainment center +envelope +espresso maker +face powder +feather boa +file +fireboat +fire engine +fire screen +flagpole +flute +folding chair +football helmet +forklift +fountain +fountain pen +four-poster +freight car +French horn +frying pan +fur coat +garbage truck +gasmask +gas pump +goblet +go-kart +golf ball +golfcart +gondola +gong +gown +grand piano +greenhouse +grille +grocery store +guillotine +hair slide +hair spray +half track +hammer +hamper +hand blower +hand-held computer +handkerchief +hard disc +harmonica +harp +harvester +hatchet +holster +home theater +honeycomb +hook +hoopskirt +horizontal bar +horse cart +hourglass +iPod +iron +jack-o'-lantern +jean +jeep +jersey +jigsaw puzzle +jinrikisha +joystick +kimono +knee pad +knot +lab coat +ladle +lampshade +laptop +lawn mower +lens cap +letter opener +library +lifeboat +lighter +limousine +liner +lipstick +Loafer +lotion +loudspeaker +loupe +lumbermill +magnetic compass +mailbag +mailbox +maillot +maillot +manhole cover +maraca +marimba +mask +matchstick +maypole +maze +measuring cup +medicine chest +megalith +microphone +microwave +military uniform +milk can +minibus +miniskirt +minivan +missile +mitten +mixing bowl +mobile home +Model T +modem +monastery +monitor +moped +mortar +mortarboard +mosque +mosquito net +motor scooter +mountain bike +mountain tent +mouse +mousetrap +moving van +muzzle +nail +neck brace +necklace +nipple +notebook +obelisk +oboe +ocarina +odometer +oil filter +organ +oscilloscope +overskirt +oxcart +oxygen mask +packet +paddle +paddlewheel +padlock +paintbrush +pajama +palace +panpipe +paper towel +parachute +parallel bars +park bench +parking meter +passenger car +patio +pay-phone +pedestal +pencil box +pencil sharpener +perfume +Petri dish +photocopier +pick +pickelhaube +picket fence +pickup +pier +piggy bank +pill bottle +pillow +ping-pong ball +pinwheel +pirate +pitcher +plane +planetarium +plastic bag +plate rack +plow +plunger +Polaroid camera +pole +police van +poncho +pool table +pop bottle +pot +potter's wheel +power drill +prayer rug +printer +prison +projectile +projector +puck +punching bag +purse +quill +quilt +racer +racket +radiator +radio +radio telescope +rain barrel +recreational vehicle +reel +reflex camera +refrigerator +remote control +restaurant +revolver +rifle +rocking chair +rotisserie +rubber eraser +rugby ball +rule +running shoe +safe +safety pin +saltshaker +sandal +sarong +sax +scabbard +scale +school bus +schooner +scoreboard +screen +screw +screwdriver +seat belt +sewing machine +shield +shoe shop +shoji +shopping basket +shopping cart +shovel +shower cap +shower curtain +ski +ski mask +sleeping bag +slide rule +sliding door +slot +snorkel +snowmobile +snowplow +soap dispenser +soccer ball +sock +solar dish +sombrero +soup bowl +space bar +space heater +space shuttle +spatula +speedboat +spider web +spindle +sports car +spotlight +stage +steam locomotive +steel arch bridge +steel drum +stethoscope +stole +stone wall +stopwatch +stove +strainer +streetcar +stretcher +studio couch +stupa +submarine +suit +sundial +sunglass +sunglasses +sunscreen +suspension bridge +swab +sweatshirt +swimming trunks +swing +switch +syringe +table lamp +tank +tape player +teapot +teddy +television +tennis ball +thatch +theater curtain +thimble +thresher +throne +tile roof +toaster +tobacco shop +toilet seat +torch +totem pole +tow truck +toyshop +tractor +trailer truck +tray +trench coat +tricycle +trimaran +tripod +triumphal arch +trolleybus +trombone +tub +turnstile +typewriter keyboard +umbrella +unicycle +upright +vacuum +vase +vault +velvet +vending machine +vestment +viaduct +violin +volleyball +waffle iron +wall clock +wallet +wardrobe +warplane +washbasin +washer +water bottle +water jug +water tower +whiskey jug +whistle +wig +window screen +window shade +Windsor tie +wine bottle +wing +wok +wooden spoon +wool +worm fence +wreck +yawl +yurt +web site +comic book +crossword puzzle +street sign +traffic light +book jacket +menu +plate +guacamole +consomme +hot pot +trifle +ice cream +ice lolly +French loaf +bagel +pretzel +cheeseburger +hotdog +mashed potato +head cabbage +broccoli +cauliflower +zucchini +spaghetti squash +acorn squash +butternut squash +cucumber +artichoke +bell pepper +cardoon +mushroom +Granny Smith +strawberry +orange +lemon +fig +pineapple +banana +jackfruit +custard apple +pomegranate +hay +carbonara +chocolate sauce +dough +meat loaf +pizza +potpie +burrito +red wine +espresso +cup +eggnog +alp +bubble +cliff +coral reef +geyser +lakeside +promontory +sandbar +seashore +valley +volcano +ballplayer +groom +scuba diver +rapeseed +daisy +yellow lady's slipper +corn +acorn +hip +buckeye +coral fungus +agaric +gyromitra +stinkhorn +earthstar +hen-of-the-woods +bolete +ear +toilet tissue