[Android/Test] add tc to check classification result
authorJaeyun <jy1210.jung@samsung.com>
Thu, 2 Apr 2020 10:53:06 +0000 (19:53 +0900)
committerMyungJoo Ham <myungjoo.ham@samsung.com>
Tue, 7 Apr 2020 02:34:16 +0000 (11:34 +0900)
Using image classification model (tf-lite), check label with max score.
Also add minor tc for coverage.

Signed-off-by: Jaeyun <jy1210.jung@samsung.com>
api/android/api/src/androidTest/java/org/nnsuite/nnstreamer/APITestCommon.java
api/android/api/src/androidTest/java/org/nnsuite/nnstreamer/APITestPipeline.java
api/android/api/src/androidTest/java/org/nnsuite/nnstreamer/APITestSingleShot.java
api/android/api/src/androidTest/java/org/nnsuite/nnstreamer/APITestTensorsData.java

index ccdd5bc..9f4fd54 100644 (file)
@@ -14,6 +14,7 @@ import org.junit.runner.RunWith;
 import java.io.File;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
+import java.nio.file.Files;
 
 import static org.junit.Assert.*;
 
@@ -66,6 +67,62 @@ public class APITestCommon {
     }
 
     /**
+     * Reads raw image file (orange) and returns TensorsData instance.
+     */
+    public static TensorsData readRawImageData() {
+        String root = Environment.getExternalStorageDirectory().getAbsolutePath();
+        File raw = new File(root + "/nnstreamer/test/orange.raw");
+
+        if (!raw.exists()) {
+            fail();
+        }
+
+        TensorsInfo info = new TensorsInfo();
+        info.addTensorInfo(NNStreamer.TensorType.UINT8, new int[]{3,224,224,1});
+
+        int size = info.getTensorSize(0);
+        TensorsData data = TensorsData.allocate(info);
+
+        try {
+            byte[] content = Files.readAllBytes(raw.toPath());
+            if (content.length != size) {
+                fail();
+            }
+
+            ByteBuffer buffer = TensorsData.allocateByteBuffer(size);
+            buffer.put(content);
+
+            data.setTensorData(0, buffer);
+        } catch (Exception e) {
+            fail();
+        }
+
+        return data;
+    }
+
+    /**
+     * Gets the label index with max score, for tensorflow-lite image classification.
+     */
+    public static int getMaxScore(ByteBuffer buffer) {
+        int index = -1;
+        int maxScore = 0;
+
+        if (isValidBuffer(buffer, 1001)) {
+            for (int i = 0; i < 1001; i++) {
+                /* convert unsigned byte */
+                int score = (buffer.get(i) & 0xFF);
+
+                if (score > maxScore) {
+                    maxScore = score;
+                    index = i;
+                }
+            }
+        }
+
+        return index;
+    }
+
+    /**
      * Gets the File object of tensorflow-lite model.
      * Note that, to invoke model in the storage, the permission READ_EXTERNAL_STORAGE is required.
      */
index 470727f..be0532c 100644 (file)
@@ -177,8 +177,9 @@ public class APITestPipeline {
 
     @Test
     public void testRegisterNullDataCb_n() {
-        String desc = "videotestsrc ! videoconvert ! video/x-raw,format=RGB ! " +
-                "tensor_converter ! tensor_sink name=sinkx";
+        String desc = "appsrc name=srcx ! " +
+                "other/tensor,dimension=(string)2:10:10:1,type=(string)uint8,framerate=(fraction)0/1 ! " +
+                "tensor_sink name=sinkx";
 
         try (Pipeline pipe = new Pipeline(desc)) {
             pipe.registerSinkCallback("sinkx", null);
@@ -190,8 +191,9 @@ public class APITestPipeline {
 
     @Test
     public void testRegisterDataCbInvalidName_n() {
-        String desc = "videotestsrc ! videoconvert ! video/x-raw,format=RGB ! " +
-                "tensor_converter ! tensor_sink name=sinkx";
+        String desc = "appsrc name=srcx ! " +
+                "other/tensor,dimension=(string)2:10:10:1,type=(string)uint8,framerate=(fraction)0/1 ! " +
+                "tensor_sink name=sinkx";
 
         try (Pipeline pipe = new Pipeline(desc)) {
             pipe.registerSinkCallback("invalid_sink", mSinkCb);
@@ -203,8 +205,9 @@ public class APITestPipeline {
 
     @Test
     public void testRegisterDataCbNullName_n() {
-        String desc = "videotestsrc ! videoconvert ! video/x-raw,format=RGB ! " +
-                "tensor_converter ! tensor_sink name=sinkx";
+        String desc = "appsrc name=srcx ! " +
+                "other/tensor,dimension=(string)2:10:10:1,type=(string)uint8,framerate=(fraction)0/1 ! " +
+                "tensor_sink name=sinkx";
 
         try (Pipeline pipe = new Pipeline(desc)) {
             pipe.registerSinkCallback(null, mSinkCb);
@@ -215,9 +218,24 @@ public class APITestPipeline {
     }
 
     @Test
+    public void testRegisterDataCbEmptyName_n() {
+        String desc = "appsrc name=srcx ! " +
+                "other/tensor,dimension=(string)2:10:10:1,type=(string)uint8,framerate=(fraction)0/1 ! " +
+                "tensor_sink name=sinkx";
+
+        try (Pipeline pipe = new Pipeline(desc)) {
+            pipe.registerSinkCallback("", mSinkCb);
+            fail();
+        } catch (Exception e) {
+            /* expected */
+        }
+    }
+
+    @Test
     public void testUnregisterNullDataCb_n() {
-        String desc = "videotestsrc ! videoconvert ! video/x-raw,format=RGB ! " +
-                "tensor_converter ! tensor_sink name=sinkx";
+        String desc = "appsrc name=srcx ! " +
+                "other/tensor,dimension=(string)2:10:10:1,type=(string)uint8,framerate=(fraction)0/1 ! " +
+                "tensor_sink name=sinkx";
 
         try (Pipeline pipe = new Pipeline(desc)) {
             pipe.unregisterSinkCallback("sinkx", null);
@@ -229,8 +247,9 @@ public class APITestPipeline {
 
     @Test
     public void testUnregisterDataCbNullName_n() {
-        String desc = "videotestsrc ! videoconvert ! video/x-raw,format=RGB ! " +
-                "tensor_converter ! tensor_sink name=sinkx";
+        String desc = "appsrc name=srcx ! " +
+                "other/tensor,dimension=(string)2:10:10:1,type=(string)uint8,framerate=(fraction)0/1 ! " +
+                "tensor_sink name=sinkx";
 
         try (Pipeline pipe = new Pipeline(desc)) {
             pipe.unregisterSinkCallback(null, mSinkCb);
@@ -241,9 +260,24 @@ public class APITestPipeline {
     }
 
     @Test
+    public void testUnregisterDataCbEmptyName_n() {
+        String desc = "appsrc name=srcx ! " +
+                "other/tensor,dimension=(string)2:10:10:1,type=(string)uint8,framerate=(fraction)0/1 ! " +
+                "tensor_sink name=sinkx";
+
+        try (Pipeline pipe = new Pipeline(desc)) {
+            pipe.unregisterSinkCallback("", mSinkCb);
+            fail();
+        } catch (Exception e) {
+            /* expected */
+        }
+    }
+
+    @Test
     public void testUnregisteredDataCb_n() {
-        String desc = "videotestsrc ! videoconvert ! video/x-raw,format=RGB ! " +
-                "tensor_converter ! tensor_sink name=sinkx";
+        String desc = "appsrc name=srcx ! " +
+                "other/tensor,dimension=(string)2:10:10:1,type=(string)uint8,framerate=(fraction)0/1 ! " +
+                "tensor_sink name=sinkx";
 
         try (Pipeline pipe = new Pipeline(desc)) {
             pipe.unregisterSinkCallback("sinkx", mSinkCb);
@@ -254,6 +288,31 @@ public class APITestPipeline {
     }
 
     @Test
+    public void testUnregisterInvalidCb_n() {
+        String desc = "appsrc name=srcx ! " +
+                "other/tensor,dimension=(string)2:10:10:1,type=(string)uint8,framerate=(fraction)0/1 ! " +
+                "tensor_sink name=sinkx";
+
+        try (Pipeline pipe = new Pipeline(desc)) {
+            /* register callback */
+            Pipeline.NewDataCallback cb1 = new Pipeline.NewDataCallback() {
+                @Override
+                public void onNewDataReceived(TensorsData data) {
+                    mReceived++;
+                }
+            };
+
+            pipe.registerSinkCallback("sinkx", cb1);
+
+            /* unregistered callback */
+            pipe.unregisterSinkCallback("sinkx", mSinkCb);
+            fail();
+        } catch (Exception e) {
+            /* expected */
+        }
+    }
+
+    @Test
     public void testRemoveDataCb() {
         String desc = "appsrc name=srcx ! " +
                 "other/tensor,dimension=(string)2:10:10:1,type=(string)uint8,framerate=(fraction)0/1 ! " +
@@ -311,6 +370,56 @@ public class APITestPipeline {
             TensorsInfo info = new TensorsInfo();
             info.addTensorInfo(NNStreamer.TensorType.UINT8, new int[]{2,10,10,1});
 
+            pipe.registerSinkCallback("sinkx", mSinkCb);
+
+            /* try to register same cb */
+            pipe.registerSinkCallback("sinkx", mSinkCb);
+
+            /* start pipeline */
+            pipe.start();
+
+            /* push input buffer */
+            for (int i = 0; i < 10; i++) {
+                /* dummy input */
+                pipe.inputData("srcx", info.allocate());
+                Thread.sleep(50);
+            }
+
+            /* pause pipeline and unregister sink callback */
+            Thread.sleep(100);
+            pipe.stop();
+
+            pipe.unregisterSinkCallback("sinkx", mSinkCb);
+            Thread.sleep(100);
+
+            /* start pipeline again */
+            pipe.start();
+
+            /* push input buffer again */
+            for (int i = 0; i < 10; i++) {
+                /* dummy input */
+                pipe.inputData("srcx", info.allocate());
+                Thread.sleep(50);
+            }
+
+            /* check received data from sink */
+            assertFalse(mInvalidState);
+            assertEquals(10, mReceived);
+        } catch (Exception e) {
+            fail();
+        }
+    }
+
+    @Test
+    public void testMultipleDataCb() {
+        String desc = "appsrc name=srcx ! " +
+                "other/tensor,dimension=(string)2:10:10:1,type=(string)uint8,framerate=(fraction)0/1 ! " +
+                "tensor_sink name=sinkx";
+
+        try (Pipeline pipe = new Pipeline(desc)) {
+            TensorsInfo info = new TensorsInfo();
+            info.addTensorInfo(NNStreamer.TensorType.UINT8, new int[]{2,10,10,1});
+
             /* register three callbacks */
             Pipeline.NewDataCallback cb1 = new Pipeline.NewDataCallback() {
                 @Override
@@ -430,6 +539,62 @@ public class APITestPipeline {
     }
 
     @Test
+    public void testClassificationResult() {
+        if (!NNStreamer.isAvailable(NNStreamer.NNFWType.TENSORFLOW_LITE)) {
+            /* cannot run the test */
+            return;
+        }
+
+        File model = APITestCommon.getTFLiteImgModel();
+        String desc = "appsrc name=srcx ! " +
+                "other/tensor,dimension=(string)3:224:224:1,type=(string)uint8,framerate=(fraction)0/1 ! " +
+                "tensor_filter framework=tensorflow-lite model=" + model.getAbsolutePath() + " ! " +
+                "tensor_sink name=sinkx";
+
+        try (Pipeline pipe = new Pipeline(desc)) {
+            TensorsInfo info = new TensorsInfo();
+            info.addTensorInfo(NNStreamer.TensorType.UINT8, new int[]{3,224,224,1});
+
+            /* register sink callback */
+            pipe.registerSinkCallback("sinkx", new Pipeline.NewDataCallback() {
+                @Override
+                public void onNewDataReceived(TensorsData data) {
+                    if (data == null || data.getTensorsCount() != 1) {
+                        mInvalidState = true;
+                        return;
+                    }
+
+                    ByteBuffer buffer = data.getTensorData(0);
+                    int labelIndex = APITestCommon.getMaxScore(buffer);
+
+                    /* check label index (orange) */
+                    if (labelIndex != 951) {
+                        mInvalidState = true;
+                    }
+
+                    mReceived++;
+                }
+            });
+
+            /* start pipeline */
+            pipe.start();
+
+            /* push input buffer */
+            TensorsData in = APITestCommon.readRawImageData();
+            pipe.inputData("srcx", in);
+
+            /* sleep 1000 to invoke */
+            Thread.sleep(1000);
+
+            /* check received data from sink */
+            assertFalse(mInvalidState);
+            assertTrue(mReceived > 0);
+        } catch (Exception e) {
+            fail();
+        }
+    }
+
+    @Test
     public void testInputBuffer() {
         String desc = "appsrc name=srcx ! " +
                 "other/tensor,dimension=(string)2:10:10:1,type=(string)uint8,framerate=(fraction)0/1 ! " +
@@ -507,6 +672,26 @@ public class APITestPipeline {
     }
 
     @Test
+    public void testInputEmptyName_n() {
+        String desc = "appsrc name=srcx ! " +
+                "other/tensor,dimension=(string)2:10:10:1,type=(string)uint8,framerate=(fraction)0/1 ! " +
+                "tensor_sink name=sinkx";
+
+        try (Pipeline pipe = new Pipeline(desc)) {
+            TensorsInfo info = new TensorsInfo();
+            info.addTensorInfo(NNStreamer.TensorType.UINT8, new int[]{2,10,10,1});
+
+            /* start pipeline */
+            pipe.start();
+
+            pipe.inputData("", TensorsData.allocate(info));
+            fail();
+        } catch (Exception e) {
+            /* expected */
+        }
+    }
+
+    @Test
     public void testInputNullData_n() {
         String desc = "appsrc name=srcx ! " +
                 "other/tensor,dimension=(string)2:10:10:1,type=(string)uint8,framerate=(fraction)0/1 ! " +
@@ -524,6 +709,29 @@ public class APITestPipeline {
     }
 
     @Test
+    public void testInputInvalidData_n() {
+        String desc = "appsrc name=srcx ! " +
+                "other/tensor,dimension=(string)2:10:10:1,type=(string)uint8,framerate=(fraction)0/1 ! " +
+                "tensor_sink name=sinkx";
+
+        try (Pipeline pipe = new Pipeline(desc)) {
+            /* start pipeline */
+            pipe.start();
+
+            TensorsInfo info = new TensorsInfo();
+            info.addTensorInfo(NNStreamer.TensorType.UINT8, new int[]{4,10,10,2});
+
+            TensorsData in = TensorsData.allocate(info);
+
+            /* push data with invalid size */
+            pipe.inputData("srcx", in);
+            fail();
+        } catch (Exception e) {
+            /* expected */
+        }
+    }
+
+    @Test
     public void testSelectSwitch() {
         String desc = "appsrc name=srcx ! " +
                 "other/tensor,dimension=(string)2:10:10:1,type=(string)uint8,framerate=(fraction)0/1 ! " +
@@ -628,6 +836,26 @@ public class APITestPipeline {
     }
 
     @Test
+    public void testGetSwitchEmptyName_n() {
+        String desc = "appsrc name=srcx ! " +
+                "other/tensor,dimension=(string)2:10:10:1,type=(string)uint8,framerate=(fraction)0/1 ! " +
+                "output-selector name=outs " +
+                "outs.src_0 ! tensor_sink name=sinkx async=false " +
+                "outs.src_1 ! tensor_sink async=false";
+
+        try (Pipeline pipe = new Pipeline(desc)) {
+            /* start pipeline */
+            pipe.start();
+
+            /* get pad list with empty name */
+            pipe.getSwitchPads("");
+            fail();
+        } catch (Exception e) {
+            /* expected */
+        }
+    }
+
+    @Test
     public void testSelectInvalidPad_n() {
         String desc = "appsrc name=srcx ! " +
                 "other/tensor,dimension=(string)2:10:10:1,type=(string)uint8,framerate=(fraction)0/1 ! " +
@@ -668,6 +896,26 @@ public class APITestPipeline {
     }
 
     @Test
+    public void testSelectEmptyPad_n() {
+        String desc = "appsrc name=srcx ! " +
+                "other/tensor,dimension=(string)2:10:10:1,type=(string)uint8,framerate=(fraction)0/1 ! " +
+                "output-selector name=outs " +
+                "outs.src_0 ! tensor_sink name=sinkx async=false " +
+                "outs.src_1 ! tensor_sink async=false";
+
+        try (Pipeline pipe = new Pipeline(desc)) {
+            /* start pipeline */
+            pipe.start();
+
+            /* empty pad name */
+            pipe.selectSwitchPad("outs", "");
+            fail();
+        } catch (Exception e) {
+            /* expected */
+        }
+    }
+
+    @Test
     public void testSelectNullSwitchName_n() {
         String desc = "appsrc name=srcx ! " +
                 "other/tensor,dimension=(string)2:10:10:1,type=(string)uint8,framerate=(fraction)0/1 ! " +
@@ -688,6 +936,26 @@ public class APITestPipeline {
     }
 
     @Test
+    public void testSelectEmptySwitchName_n() {
+        String desc = "appsrc name=srcx ! " +
+                "other/tensor,dimension=(string)2:10:10:1,type=(string)uint8,framerate=(fraction)0/1 ! " +
+                "output-selector name=outs " +
+                "outs.src_0 ! tensor_sink name=sinkx async=false " +
+                "outs.src_1 ! tensor_sink async=false";
+
+        try (Pipeline pipe = new Pipeline(desc)) {
+            /* start pipeline */
+            pipe.start();
+
+            /* empty switch name */
+            pipe.selectSwitchPad("", "src_1");
+            fail();
+        } catch (Exception e) {
+            /* expected */
+        }
+    }
+
+    @Test
     public void testControlValve() {
         String desc = "appsrc name=srcx ! " +
                 "other/tensor,dimension=(string)2:10:10:1,type=(string)uint8,framerate=(fraction)0/1 ! " +
@@ -760,7 +1028,7 @@ public class APITestPipeline {
             /* start pipeline */
             pipe.start();
 
-            /* control valve with invalid name */
+            /* control valve with null name */
             pipe.controlValve(null, false);
             fail();
         } catch (Exception e) {
@@ -769,6 +1037,26 @@ public class APITestPipeline {
     }
 
     @Test
+    public void testControlEmptyValveName_n() {
+        String desc = "appsrc name=srcx ! " +
+                "other/tensor,dimension=(string)2:10:10:1,type=(string)uint8,framerate=(fraction)0/1 ! " +
+                "tee name=t " +
+                "t. ! queue ! tensor_sink " +
+                "t. ! queue ! valve name=valvex ! tensor_sink name=sinkx";
+
+        try (Pipeline pipe = new Pipeline(desc)) {
+            /* start pipeline */
+            pipe.start();
+
+            /* control valve with empty name */
+            pipe.controlValve("", false);
+            fail();
+        } catch (Exception e) {
+            /* expected */
+        }
+    }
+
+    @Test
     public void testAMCsrc() {
         String root = Environment.getExternalStorageDirectory().getAbsolutePath();
         String media = root + "/nnstreamer/test/test_video.mp4";
index bae0b0a..101a1e7 100644 (file)
@@ -4,7 +4,6 @@ import android.os.Environment;
 import android.support.test.rule.GrantPermissionRule;
 import android.support.test.runner.AndroidJUnit4;
 
-import org.junit.After;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -88,6 +87,26 @@ public class APITestSingleShot {
     }
 
     @Test
+    public void testSetInvalidInputInfo_n() {
+        if (!NNStreamer.isAvailable(NNStreamer.NNFWType.TENSORFLOW_LITE)) {
+            /* cannot run the test */
+            return;
+        }
+
+        try {
+            SingleShot single = new SingleShot(APITestCommon.getTFLiteImgModel());
+
+            TensorsInfo newInfo = new TensorsInfo();
+            newInfo.addTensorInfo(NNStreamer.TensorType.UINT8, new int[]{2,2,2,2});
+
+            single.setInputInfo(newInfo);
+            fail();
+        } catch (Exception e) {
+            /* expected */
+        }
+    }
+
+    @Test
     public void testSetInputInfo() {
         if (!NNStreamer.isAvailable(NNStreamer.NNFWType.TENSORFLOW_LITE)) {
             /* cannot run the test */
@@ -159,6 +178,35 @@ public class APITestSingleShot {
     }
 
     @Test
+    public void testClassificationResult() {
+        if (!NNStreamer.isAvailable(NNStreamer.NNFWType.TENSORFLOW_LITE)) {
+            /* cannot run the test */
+            return;
+        }
+
+        try {
+            SingleShot single = new SingleShot(APITestCommon.getTFLiteImgModel());
+
+            /* let's ignore timeout (set 10 sec) */
+            single.setTimeout(10000);
+
+            /* single-shot invoke */
+            TensorsData in = APITestCommon.readRawImageData();
+            TensorsData out = single.invoke(in);
+            int labelIndex = APITestCommon.getMaxScore(out.getTensorData(0));
+
+            /* check label index (orange) */
+            if (labelIndex != 951) {
+                fail();
+            }
+
+            single.close();
+        } catch (Exception e) {
+            fail();
+        }
+    }
+
+    @Test
     public void testInvokeDynamicVary() {
         if (!NNStreamer.isAvailable(NNStreamer.NNFWType.TENSORFLOW_LITE)) {
             /* cannot run the test */
@@ -444,7 +492,7 @@ public class APITestSingleShot {
     }
 
     @Test
-    public void testUnknownPropertyName_n() {
+    public void testSetUnknownPropertyName_n() {
         if (!NNStreamer.isAvailable(NNStreamer.NNFWType.TENSORFLOW_LITE)) {
             /* cannot run the test */
             return;
@@ -478,6 +526,23 @@ public class APITestSingleShot {
     }
 
     @Test
+    public void testSetEmptyPropertyName_n() {
+        if (!NNStreamer.isAvailable(NNStreamer.NNFWType.TENSORFLOW_LITE)) {
+            /* cannot run the test */
+            return;
+        }
+
+        try {
+            SingleShot single = new SingleShot(APITestCommon.getTFLiteImgModel());
+
+            single.setValue("", "ANY");
+            fail();
+        } catch (Exception e) {
+            /* expected */
+        }
+    }
+
+    @Test
     public void testSetNullPropertyValue_n() {
         if (!NNStreamer.isAvailable(NNStreamer.NNFWType.TENSORFLOW_LITE)) {
             /* cannot run the test */
@@ -495,6 +560,23 @@ public class APITestSingleShot {
     }
 
     @Test
+    public void testSetPropertyDimension() {
+        if (!NNStreamer.isAvailable(NNStreamer.NNFWType.TENSORFLOW_LITE)) {
+            /* cannot run the test */
+            return;
+        }
+
+        try {
+            SingleShot single = new SingleShot(APITestCommon.getTFLiteAddModel());
+
+            single.setValue("input", "5:1:1:1");
+            single.close();
+        } catch (Exception e) {
+            fail();
+        }
+    }
+
+    @Test
     public void testGetPropertyDimension() {
         if (!NNStreamer.isAvailable(NNStreamer.NNFWType.TENSORFLOW_LITE)) {
             /* cannot run the test */
index 012f77a..f9932b3 100644 (file)
@@ -73,6 +73,18 @@ public class APITestTensorsData {
     }
 
     @Test
+    public void testAllocateEmptyInfo_n() {
+        try {
+            TensorsInfo info = new TensorsInfo();
+
+            TensorsData.allocate(info);
+            fail();
+        } catch (Exception e) {
+            /* expected */
+        }
+    }
+
+    @Test
     public void testAllocateNullInfo_n() {
         try {
             TensorsData.allocate(null);