Add Pose detector services

 - 3fps detection on RPI CPU mode
 - tflite model file

Signed-off-by: Kwang Son <k.son@samsung.com>

diff --git a/Fitness/Services/PoseDetector.cs b/Fitness/Services/PoseDetector.cs
new file mode 100644 (file)
index 0000000..22d7fcf
--- /dev/null
+++ b/Fitness/Services/PoseDetector.cs
@@ -0,0 +1,97 @@
+using System.Threading;
+using System.Threading.Tasks;
+using Tizen.Applications;
+using Tizen.Multimedia.Vision;
+
+namespace Fitness.Services
+{
+    public class PoseDetector
+    {
+        private InferenceModelConfiguration pdConfig;
+
+        public PoseDetector()
+        {
+            pdConfig = new InferenceModelConfiguration();
+            pdConfig.WeightFilePath = Application.Current.DirectoryInfo.Resource + "tflite/pld_tflite_model.tflite";
+            pdConfig.MeanValue = 0.0;
+            pdConfig.StdValue = 1.0;
+            pdConfig.Backend = InferenceBackendType.TFLite;
+            pdConfig.Device = InferenceTargetDevice.CPU;
+            pdConfig.DataType = InferenceDataType.Float32;
+            pdConfig.TensorSize = new Tizen.Multimedia.Size(192, 192);
+            pdConfig.TensorChannels = 3;
+            pdConfig.InputNodeName = "image";
+            pdConfig.OutputNodeName = new string[]
+            {
+                    "Convolution_Pose_Machine/stage_5_out",
+            };
+            pdConfig.LoadInferenceModel();
+        }
+
+        public enum BodyPart
+        {
+            Head,
+            Neck,
+            RightUpArm,
+            RightLowArm,
+            RightHand,
+            LeftUpArm,
+            LeftLowArm,
+            LeftHand,
+            RightUpLeg,
+            RightLowLeg,
+            RightFoot,
+            LeftUpLeg,
+            LeftLowLeg,
+            LeftFoot,
+        }
+
+        public Task<Landmark[,]> Detect(Tizen.Multimedia.TriplePlane plane, uint height, uint width)
+        {
+            byte[] rgbframe = YUV420ToRGB(plane.Y, plane.U, plane.V, (int)height, (int)width);
+
+            MediaVisionSource source = new MediaVisionSource(rgbframe, width, height, Tizen.Multimedia.ColorSpace.Rgb888);
+            return Tizen.Multimedia.Vision.PoseLandmarkDetector.DetectAsync(source, pdConfig);
+        }
+
+        // https://stackoverflow.com/questions/16107165/convert-from-yuv-420-to-imagebgr-byte
+        private byte[] YUV420ToRGB(byte[] planeY, byte[] planeU, byte[] planeV, int height, int width)
+        {
+            byte[] rgb = new byte[height * width * 3];
+            int indexRGB = 0;
+            int indexY = 0;
+            int indexUV = 0;
+            for (int r = 0; r < height; r++)
+            {
+                indexRGB = r * width * 3;
+                indexY = r * width;
+                indexUV = (r / 2) * width / 2;
+
+                // process two pixels at a time
+                for (int c = 0; c < width; c += 2)
+                {
+                    int c1 = planeY[indexY + c] - 16;
+                    int c2 = planeY[indexY + c + 1] - 16;
+                    int d1 = planeU[indexUV + (c / 2)] - 128;
+                    int e1 = planeV[indexUV + (c / 2)] - 128;
+
+                    int r1 = ((298 * c1) + (409 * e1) + 128) >> 8;
+                    int g1 = ((298 * c1) - (100 * d1) - (208 * e1) + 128) >> 8;
+                    int b1 = ((298 * c1) + (516 * d1) + 128) >> 8;
+                    int r2 = ((298 * c2) + (409 * e1) + 128) >> 8;
+                    int g2 = ((298 * c2) - (100 * d1) - (208 * e1) + 128) >> 8;
+                    int b2 = ((298 * c2) + (516 * d1) + 128) >> 8;
+
+                    rgb[indexRGB + (c * 3) + 0] = (byte)System.Math.Clamp(r1, 0, 255);
+                    rgb[indexRGB + (c * 3) + 1] = (byte)System.Math.Clamp(g1, 0, 255);
+                    rgb[indexRGB + (c * 3) + 2] = (byte)System.Math.Clamp(b1, 0, 255);
+                    rgb[indexRGB + (c * 3) + 3] = (byte)System.Math.Clamp(r2, 0, 255);
+                    rgb[indexRGB + (c * 3) + 4] = (byte)System.Math.Clamp(g2, 0, 255);
+                    rgb[indexRGB + (c * 3) + 5] = (byte)System.Math.Clamp(b2, 0, 255);
+                }
+            }
+
+            return rgb;
+        }
+    }
+}

diff --git a/Fitness/res/tflite/pld_tflite_model.tflite b/Fitness/res/tflite/pld_tflite_model.tflite
new file mode 100644 (file)
index 0000000..ff6044d
Binary files /dev/null and b/Fitness/res/tflite/pld_tflite_model.tflite differ