From: Piotr Czaja/Advanced Frameworks (PLT) /SRPOL/Engineer/Samsung Electronics Date: Wed, 21 Jul 2021 12:32:11 +0000 (+0200) Subject: Refactor squat service. X-Git-Tag: submit/trunk/20210914.122825~43 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=80c47c7c953cfb49b90c0147be9acd3bf69143ae;p=profile%2Fiot%2Fapps%2Fdotnet%2Ffitness.git Refactor squat service. --- diff --git a/Fitness/Services/Exercises/BaseExerciseService.cs b/Fitness/Services/Exercises/BaseExerciseService.cs new file mode 100644 index 0000000..1868fe8 --- /dev/null +++ b/Fitness/Services/Exercises/BaseExerciseService.cs @@ -0,0 +1,106 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Tizen.Multimedia; +using Tizen.Multimedia.Vision; + +namespace Fitness.Services +{ + /// + /// Common base class for all exercise services. + /// + public abstract class BaseExerciseService : INotifyPropertyChanged + { + private readonly PoseDetector poseDetector = new PoseDetector(); + private int isInferencing = 0; + private Landmark[,] poseLandmarks; + + /// + /// Occurs when a property value changes. + /// + public event PropertyChangedEventHandler PropertyChanged; + + /// + /// Gets the pose landmark property. + /// + public Landmark[,] PoseLandmarks + { + get => poseLandmarks; + private set + { + if (value != poseLandmarks) + { + poseLandmarks = value; + RaisePropertyChanged(); + } + } + } + + /// + /// Detects performing the exercise based on given preview image data. + /// + /// Preview image data. + /// A task that represents the exercise detection. + public async Task DetectExercise(Tizen.Multimedia.PreviewFrame previewFrame) + { + var stopwatch = new System.Diagnostics.Stopwatch(); + stopwatch.Start(); + + try + { + if (Interlocked.Exchange(ref isInferencing, 1) == 0) + { + var plane = previewFrame.Plane as TriplePlane; + if (plane is null) + { + throw new System.Exception($"Unsupported plane type: {previewFrame.Plane.GetType()}"); + } + + uint width = (uint)previewFrame.Resolution.Width; + uint height = (uint)previewFrame.Resolution.Height; + + var landmarks = await poseDetector.Detect(plane, height, width); + + DetectExercise(landmarks); + NUIContext.InvokeOnMainThread(() => + { + PoseLandmarks = landmarks; + }); + } + } + catch (System.Exception exception) + { + Services.Logger.Error(exception.Message + System.Environment.NewLine + exception.StackTrace); + } + finally + { + stopwatch.Stop(); + Services.Logger.Debug($"total time {stopwatch.ElapsedMilliseconds} msec"); + + Interlocked.Exchange(ref isInferencing, 0); + } + } + + /// + /// Detects performing the exercise based on given landmarks. + /// + /// Body landmarks. + protected virtual void DetectExercise(Landmark[,] landmarks) + { + } + + /// + /// Raises PropertyChanged event. + /// + /// Property name. + protected void RaisePropertyChanged([CallerMemberName] string propertyName = null) + { + PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(propertyName)); + } + } +} diff --git a/Fitness/Services/Exercises/IExerciseService.cs b/Fitness/Services/Exercises/IExerciseService.cs new file mode 100644 index 0000000..a6dfe3d --- /dev/null +++ b/Fitness/Services/Exercises/IExerciseService.cs @@ -0,0 +1,48 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Tizen.Multimedia; +using Tizen.Multimedia.Vision; + +namespace Fitness.Services +{ + /// + /// Interface for exercise service. + /// + public interface IExerciseService + { + /// + /// Gets the pose landmark property. + /// + Landmark[,] PoseLandmarks { get; } + + /// + /// Gets or sets the hold time threshold specifying the time of a single exercise. + /// + long HoldTimeThreshold { get; set; } + + /// + /// Gets the property specifying the time of the exercise performed - values ranging from 0 to 5. + /// + int Hold { get; } + + /// + /// Gets the property specifying the number of repetitions of the exercise. + /// + int Count { get; } + + /// + /// Gets the property specifying the correctness of the exercise - values ranging from 0 to 100. + /// + int Score { get; } + + /// + /// Detects performing the exercise based on given preview image data. + /// + /// Preview image data. + /// A task that represents the exercise detection. + Task DetectExercise(PreviewFrame preview); + } +} diff --git a/Fitness/Services/Exercises/PoseDetector.cs b/Fitness/Services/Exercises/PoseDetector.cs new file mode 100644 index 0000000..8d2f021 --- /dev/null +++ b/Fitness/Services/Exercises/PoseDetector.cs @@ -0,0 +1,97 @@ +using System.Threading; +using System.Threading.Tasks; +using Tizen.Applications; +using Tizen.Multimedia.Vision; + +namespace Fitness.Services +{ + public class PoseDetector + { + private InferenceModelConfiguration pdConfig; + + public PoseDetector() + { + pdConfig = new InferenceModelConfiguration(); + pdConfig.WeightFilePath = Application.Current.DirectoryInfo.Resource + "tflite/pld_tflite_model.tflite"; + pdConfig.MeanValue = 0.0; + pdConfig.StdValue = 1.0; + pdConfig.Backend = InferenceBackendType.TFLite; + pdConfig.Device = InferenceTargetDevice.CPU; + pdConfig.DataType = InferenceDataType.Float32; + pdConfig.TensorSize = new Tizen.Multimedia.Size(192, 192); + pdConfig.TensorChannels = 3; + pdConfig.InputNodeName = "image"; + pdConfig.OutputNodeName = new string[] + { + "Convolutional_Pose_Machine/stage_5_out", + }; + pdConfig.LoadInferenceModel(); + } + + public enum BodyPart + { + Head, + Neck, + RightUpArm, + RightLowArm, + RightHand, + LeftUpArm, + LeftLowArm, + LeftHand, + RightUpLeg, + RightLowLeg, + RightFoot, + LeftUpLeg, + LeftLowLeg, + LeftFoot, + } + + public Task Detect(Tizen.Multimedia.TriplePlane plane, uint height, uint width) + { + byte[] rgbframe = YUV420ToRGB(plane.Y, plane.U, plane.V, (int)height, (int)width); + + MediaVisionSource source = new MediaVisionSource(rgbframe, width, height, Tizen.Multimedia.ColorSpace.Rgb888); + return Tizen.Multimedia.Vision.PoseLandmarkDetector.DetectAsync(source, pdConfig); + } + + // https://stackoverflow.com/questions/16107165/convert-from-yuv-420-to-imagebgr-byte + private byte[] YUV420ToRGB(byte[] planeY, byte[] planeU, byte[] planeV, int height, int width) + { + byte[] rgb = new byte[height * width * 3]; + int indexRGB = 0; + int indexY = 0; + int indexUV = 0; + for (int r = 0; r < height; r++) + { + indexRGB = r * width * 3; + indexY = r * width; + indexUV = (r / 2) * width / 2; + + // process two pixels at a time + for (int c = 0; c < width; c += 2) + { + int c1 = planeY[indexY + c] - 16; + int c2 = planeY[indexY + c + 1] - 16; + int d1 = planeU[indexUV + (c / 2)] - 128; + int e1 = planeV[indexUV + (c / 2)] - 128; + + int r1 = ((298 * c1) + (409 * e1) + 128) >> 8; + int g1 = ((298 * c1) - (100 * d1) - (208 * e1) + 128) >> 8; + int b1 = ((298 * c1) + (516 * d1) + 128) >> 8; + int r2 = ((298 * c2) + (409 * e1) + 128) >> 8; + int g2 = ((298 * c2) - (100 * d1) - (208 * e1) + 128) >> 8; + int b2 = ((298 * c2) + (516 * d1) + 128) >> 8; + + rgb[indexRGB + (c * 3) + 0] = (byte)System.Math.Clamp(r1, 0, 255); + rgb[indexRGB + (c * 3) + 1] = (byte)System.Math.Clamp(g1, 0, 255); + rgb[indexRGB + (c * 3) + 2] = (byte)System.Math.Clamp(b1, 0, 255); + rgb[indexRGB + (c * 3) + 3] = (byte)System.Math.Clamp(r2, 0, 255); + rgb[indexRGB + (c * 3) + 4] = (byte)System.Math.Clamp(g2, 0, 255); + rgb[indexRGB + (c * 3) + 5] = (byte)System.Math.Clamp(b2, 0, 255); + } + } + + return rgb; + } + } +} diff --git a/Fitness/Services/Exercises/SquatDetector.cs b/Fitness/Services/Exercises/SquatDetector.cs new file mode 100644 index 0000000..2cb4c71 --- /dev/null +++ b/Fitness/Services/Exercises/SquatDetector.cs @@ -0,0 +1,65 @@ +using System.Collections.Generic; +using System.Numerics; +using Tizen.Multimedia; + +namespace Fitness.Services +{ + /// + /// SquatDetector. + /// + public static class SquatDetector + { + // SquatLowerBody is calculation result of squat picture from + // https://review.tizen.org/gerrit/gitweb?p=test/tct/native/api.git;a=blob;f=src/utc/capi-media-vision/res/inference/images/poseLandmark.jpg;h=4bead08b8dc3687f228b5ab885e7136b26331cfb;hb=refs/heads/tizen + // Each below vector is derived from two points respectively, + // 0. RightUpLeg => RightLowLeg, + // 1. RightLowLeg => RightFoot, + // 2. LeftUpLeg => LeftLowLeg, + // 4. LeftLowLeg => LeftFoot + private static readonly List SquatLowerBody = new List(new[] + { + new Vector2(0.6666667F, -0.33333334F), + new Vector2(0F, -1F), + new Vector2(-0.6666667F, -0.33333334F), + new Vector2(0F, -1F), + }); + + /// + /// Return Average of lower body Cosine Similarity between input user pose and ground truth pose. + /// + /// user body detection result. + /// Float score value (-1.0 ~ 1.0). + public static float Similarity(List userBody) + { + if (userBody.Count != System.Enum.GetNames(typeof(PoseDetector.BodyPart)).Length) + { + throw new System.ArgumentException("Must have all points which represent user body", "userBody"); + } + + List userSquat = new List(new[] + { + L1UnitVectorize(userBody[(int)PoseDetector.BodyPart.RightUpLeg], userBody[(int)PoseDetector.BodyPart.RightLowLeg]), + L1UnitVectorize(userBody[(int)PoseDetector.BodyPart.RightLowLeg], userBody[(int)PoseDetector.BodyPart.RightFoot]), + L1UnitVectorize(userBody[(int)PoseDetector.BodyPart.LeftUpLeg], userBody[(int)PoseDetector.BodyPart.LeftLowLeg]), + L1UnitVectorize(userBody[(int)PoseDetector.BodyPart.LeftLowLeg], userBody[(int)PoseDetector.BodyPart.LeftFoot]), + }); + + float score = 0F; + + for (int i = 0; i < SquatLowerBody.Count; i++) + { + score += Vector2.Dot(SquatLowerBody[i], userSquat[i]) / SquatLowerBody[i].Length() / userSquat[i].Length(); + } + + return score / SquatLowerBody.Count; + } + + private static Vector2 L1UnitVectorize(Point from, Point to) + { + int vx = from.X - to.X; + int vy = from.Y - to.Y; + float parent = System.Math.Abs(vx) + System.Math.Abs(vy); + return new Vector2(vx / parent, vy / parent); + } + } +} diff --git a/Fitness/Services/Exercises/SquatService.cs b/Fitness/Services/Exercises/SquatService.cs new file mode 100644 index 0000000..e43bafd --- /dev/null +++ b/Fitness/Services/Exercises/SquatService.cs @@ -0,0 +1,147 @@ +using System.ComponentModel; +using System.Diagnostics; +using System.Linq; +using System.Timers; +using Tizen.Multimedia.Vision; + +namespace Fitness.Services +{ + /// + /// Class handling the squat exercise detection. + /// + public class SquatService : BaseExerciseService, IExerciseService + { + private const float SquatDetectedThreshold = 0.92f; + private const int HoldCount = 5; + private int hold; + private int count; + private int score; + private int newHold; + private int newCount; + private int newScore; + private float fscore = 0; + private Stopwatch stopwatch = new Stopwatch(); + private System.Timers.Timer holdTimer; + private long holdTimeThreshold; + + public int Hold + { + get => hold; + set + { + if (value != hold) + { + hold = value; + RaisePropertyChanged(); + } + } + } + + public int Count + { + get => count; + set + { + if (value != count) + { + count = value; + RaisePropertyChanged(); + } + } + } + + public int Score + { + get => score; + set + { + if (value != score) + { + score = value; + RaisePropertyChanged(); + } + } + } + + public long HoldTimeThreshold + { + get => holdTimeThreshold; + set + { + if (value != holdTimeThreshold) + { + holdTimeThreshold = value; + InitializeTimer(); + } + } + } + + protected override void DetectExercise(Landmark[,] landmarks) + { + int numberOfBodyParts = landmarks.GetLength(1); + var range = Enumerable.Range(0, numberOfBodyParts); + + var locations = range.Select(x => landmarks[0, x].Location).ToList(); + var squatSimilarity = SquatDetector.Similarity(locations); + if (float.IsNaN(squatSimilarity)) + { + squatSimilarity = 0.0f; + } + + DetectSquat(squatSimilarity); + + NUIContext.InvokeOnMainThread(() => + { + Hold = newHold; + Count = newCount; + Score = newScore; + }); + } + + private void InitializeTimer() + { + long interval = holdTimeThreshold / HoldCount; + holdTimer = new System.Timers.Timer(interval); + holdTimer.Elapsed += HoldTimer_Elapsed; + holdTimer.AutoReset = true; + holdTimer.Enabled = true; + } + + private void HoldTimer_Elapsed(object sender, ElapsedEventArgs e) + { + if (Hold < HoldCount && fscore >= SquatDetectedThreshold) + { + newHold++; + } + } + + private void DetectSquat(float squatSimilarity) + { + var previousScore = fscore; + fscore = squatSimilarity; + newScore = (int)System.Math.Round((float)(100 * fscore)); + + if (previousScore < SquatDetectedThreshold) + { + if (fscore >= SquatDetectedThreshold) + { + stopwatch.Start(); + holdTimer.Start(); + } + } + else if (fscore < SquatDetectedThreshold) + { + stopwatch.Stop(); + holdTimer.Stop(); + + if (stopwatch.ElapsedMilliseconds >= holdTimeThreshold) + { + newCount++; + } + + newHold = 0; + stopwatch.Reset(); + } + } + } +} diff --git a/Fitness/Services/PoseDetector.cs b/Fitness/Services/PoseDetector.cs deleted file mode 100644 index 8d2f021..0000000 --- a/Fitness/Services/PoseDetector.cs +++ /dev/null @@ -1,97 +0,0 @@ -using System.Threading; -using System.Threading.Tasks; -using Tizen.Applications; -using Tizen.Multimedia.Vision; - -namespace Fitness.Services -{ - public class PoseDetector - { - private InferenceModelConfiguration pdConfig; - - public PoseDetector() - { - pdConfig = new InferenceModelConfiguration(); - pdConfig.WeightFilePath = Application.Current.DirectoryInfo.Resource + "tflite/pld_tflite_model.tflite"; - pdConfig.MeanValue = 0.0; - pdConfig.StdValue = 1.0; - pdConfig.Backend = InferenceBackendType.TFLite; - pdConfig.Device = InferenceTargetDevice.CPU; - pdConfig.DataType = InferenceDataType.Float32; - pdConfig.TensorSize = new Tizen.Multimedia.Size(192, 192); - pdConfig.TensorChannels = 3; - pdConfig.InputNodeName = "image"; - pdConfig.OutputNodeName = new string[] - { - "Convolutional_Pose_Machine/stage_5_out", - }; - pdConfig.LoadInferenceModel(); - } - - public enum BodyPart - { - Head, - Neck, - RightUpArm, - RightLowArm, - RightHand, - LeftUpArm, - LeftLowArm, - LeftHand, - RightUpLeg, - RightLowLeg, - RightFoot, - LeftUpLeg, - LeftLowLeg, - LeftFoot, - } - - public Task Detect(Tizen.Multimedia.TriplePlane plane, uint height, uint width) - { - byte[] rgbframe = YUV420ToRGB(plane.Y, plane.U, plane.V, (int)height, (int)width); - - MediaVisionSource source = new MediaVisionSource(rgbframe, width, height, Tizen.Multimedia.ColorSpace.Rgb888); - return Tizen.Multimedia.Vision.PoseLandmarkDetector.DetectAsync(source, pdConfig); - } - - // https://stackoverflow.com/questions/16107165/convert-from-yuv-420-to-imagebgr-byte - private byte[] YUV420ToRGB(byte[] planeY, byte[] planeU, byte[] planeV, int height, int width) - { - byte[] rgb = new byte[height * width * 3]; - int indexRGB = 0; - int indexY = 0; - int indexUV = 0; - for (int r = 0; r < height; r++) - { - indexRGB = r * width * 3; - indexY = r * width; - indexUV = (r / 2) * width / 2; - - // process two pixels at a time - for (int c = 0; c < width; c += 2) - { - int c1 = planeY[indexY + c] - 16; - int c2 = planeY[indexY + c + 1] - 16; - int d1 = planeU[indexUV + (c / 2)] - 128; - int e1 = planeV[indexUV + (c / 2)] - 128; - - int r1 = ((298 * c1) + (409 * e1) + 128) >> 8; - int g1 = ((298 * c1) - (100 * d1) - (208 * e1) + 128) >> 8; - int b1 = ((298 * c1) + (516 * d1) + 128) >> 8; - int r2 = ((298 * c2) + (409 * e1) + 128) >> 8; - int g2 = ((298 * c2) - (100 * d1) - (208 * e1) + 128) >> 8; - int b2 = ((298 * c2) + (516 * d1) + 128) >> 8; - - rgb[indexRGB + (c * 3) + 0] = (byte)System.Math.Clamp(r1, 0, 255); - rgb[indexRGB + (c * 3) + 1] = (byte)System.Math.Clamp(g1, 0, 255); - rgb[indexRGB + (c * 3) + 2] = (byte)System.Math.Clamp(b1, 0, 255); - rgb[indexRGB + (c * 3) + 3] = (byte)System.Math.Clamp(r2, 0, 255); - rgb[indexRGB + (c * 3) + 4] = (byte)System.Math.Clamp(g2, 0, 255); - rgb[indexRGB + (c * 3) + 5] = (byte)System.Math.Clamp(b2, 0, 255); - } - } - - return rgb; - } - } -} diff --git a/Fitness/Services/SquatDetector.cs b/Fitness/Services/SquatDetector.cs deleted file mode 100644 index 2cb4c71..0000000 --- a/Fitness/Services/SquatDetector.cs +++ /dev/null @@ -1,65 +0,0 @@ -using System.Collections.Generic; -using System.Numerics; -using Tizen.Multimedia; - -namespace Fitness.Services -{ - /// - /// SquatDetector. - /// - public static class SquatDetector - { - // SquatLowerBody is calculation result of squat picture from - // https://review.tizen.org/gerrit/gitweb?p=test/tct/native/api.git;a=blob;f=src/utc/capi-media-vision/res/inference/images/poseLandmark.jpg;h=4bead08b8dc3687f228b5ab885e7136b26331cfb;hb=refs/heads/tizen - // Each below vector is derived from two points respectively, - // 0. RightUpLeg => RightLowLeg, - // 1. RightLowLeg => RightFoot, - // 2. LeftUpLeg => LeftLowLeg, - // 4. LeftLowLeg => LeftFoot - private static readonly List SquatLowerBody = new List(new[] - { - new Vector2(0.6666667F, -0.33333334F), - new Vector2(0F, -1F), - new Vector2(-0.6666667F, -0.33333334F), - new Vector2(0F, -1F), - }); - - /// - /// Return Average of lower body Cosine Similarity between input user pose and ground truth pose. - /// - /// user body detection result. - /// Float score value (-1.0 ~ 1.0). - public static float Similarity(List userBody) - { - if (userBody.Count != System.Enum.GetNames(typeof(PoseDetector.BodyPart)).Length) - { - throw new System.ArgumentException("Must have all points which represent user body", "userBody"); - } - - List userSquat = new List(new[] - { - L1UnitVectorize(userBody[(int)PoseDetector.BodyPart.RightUpLeg], userBody[(int)PoseDetector.BodyPart.RightLowLeg]), - L1UnitVectorize(userBody[(int)PoseDetector.BodyPart.RightLowLeg], userBody[(int)PoseDetector.BodyPart.RightFoot]), - L1UnitVectorize(userBody[(int)PoseDetector.BodyPart.LeftUpLeg], userBody[(int)PoseDetector.BodyPart.LeftLowLeg]), - L1UnitVectorize(userBody[(int)PoseDetector.BodyPart.LeftLowLeg], userBody[(int)PoseDetector.BodyPart.LeftFoot]), - }); - - float score = 0F; - - for (int i = 0; i < SquatLowerBody.Count; i++) - { - score += Vector2.Dot(SquatLowerBody[i], userSquat[i]) / SquatLowerBody[i].Length() / userSquat[i].Length(); - } - - return score / SquatLowerBody.Count; - } - - private static Vector2 L1UnitVectorize(Point from, Point to) - { - int vx = from.X - to.X; - int vy = from.Y - to.Y; - float parent = System.Math.Abs(vx) + System.Math.Abs(vy); - return new Vector2(vx / parent, vy / parent); - } - } -} diff --git a/Fitness/Services/SquatService.cs b/Fitness/Services/SquatService.cs deleted file mode 100644 index 62e78ca..0000000 --- a/Fitness/Services/SquatService.cs +++ /dev/null @@ -1,239 +0,0 @@ -using System.ComponentModel; -using System.Diagnostics; -using System.Linq; -using System.Runtime.CompilerServices; -using System.Threading; -using System.Threading.Tasks; -using System.Timers; -using Tizen.Multimedia; -using Tizen.Multimedia.Vision; - -namespace Fitness.Services -{ - /// - /// Class handling the squat exercise detection. - /// - public class SquatService : INotifyPropertyChanged - { - private const float SquatDetectedThreshold = 0.92f; - private const int HoldCount = 5; - private readonly PoseDetector poseDetector = new PoseDetector(); - private int hold; - private int count; - private int score; - private int newHold; - private int newCount; - private int newScore; - private float fscore = 0; - private Stopwatch stopwatch = new Stopwatch(); - private System.Timers.Timer holdTimer; - private long holdTimeThreshold; - private int isInferencing = 0; - private Landmark[,] poseLandmarks; - - /// - /// Initializes a new instance of the class. - /// - public SquatService() - { - } - - /// - /// Occurs when a property value changes. - /// - public event PropertyChangedEventHandler PropertyChanged; - - /// - /// Gets or sets the pose landmark property. - /// - public Landmark[,] PoseLandmarks - { - get => poseLandmarks; - set - { - if (value != poseLandmarks) - { - poseLandmarks = value; - RaisePropertyChanged(); - } - } - } - - /// - /// Gets or sets the hold property specifying the time of the exercise performed - values ranging from 0 to 5. - /// - public int Hold - { - get => hold; - set - { - if (value != hold) - { - hold = value; - RaisePropertyChanged(); - } - } - } - - /// - /// Gets or sets the count property specifying the number of repetitions of the exercise. - /// - public int Count - { - get => count; - set - { - if (value != count) - { - count = value; - RaisePropertyChanged(); - } - } - } - - /// - /// Gets or sets the score property specifying the correctness of the exercise - values ranging from 0 to 100. - /// - public int Score - { - get => score; - set - { - if (value != score) - { - score = value; - RaisePropertyChanged(); - } - } - } - - /// - /// Gets or sets the hold time threshold specifying the time of a single exercise. - /// - public long HoldTimeThreshold - { - get => holdTimeThreshold; - set - { - if (value != holdTimeThreshold) - { - holdTimeThreshold = value; - InitializeTimer(); - } - } - } - - /// - /// Detects squat exercise based on given preview image data. - /// - /// Preview image data. - /// A task that represents the squat detection. - public async Task DetectSquat(Tizen.Multimedia.PreviewFrame previewFrame) - { - var stopwatch = new System.Diagnostics.Stopwatch(); - stopwatch.Start(); - - try - { - if (Interlocked.Exchange(ref isInferencing, 1) == 0) - { - var plane = previewFrame.Plane as TriplePlane; - if (plane == null) - { - throw new System.Exception($"Unsupported plane type: {previewFrame.Plane.GetType()}"); - } - - uint width = (uint)previewFrame.Resolution.Width; - uint height = (uint)previewFrame.Resolution.Height; - - var landmarks = await poseDetector.Detect(plane, height, width); - - int numberOfBodyParts = landmarks.GetLength(1); - var range = Enumerable.Range(0, numberOfBodyParts); - - var locations = range.Select(x => landmarks[0, x].Location).ToList(); - var squatSimilarity = SquatDetector.Similarity(locations); - if (float.IsNaN(squatSimilarity)) - { - squatSimilarity = 0.0f; - } - - DetectSquat(squatSimilarity); - - NUIContext.InvokeOnMainThread(() => - { - PoseLandmarks = landmarks; - Hold = newHold; - Count = newCount; - Score = newScore; - }); - } - } - catch (System.Exception exception) - { - Services.Logger.Error(exception.Message + System.Environment.NewLine + exception.StackTrace); - } - finally - { - stopwatch.Stop(); - Services.Logger.Debug($"total time {stopwatch.ElapsedMilliseconds} msec"); - - Interlocked.Exchange(ref isInferencing, 0); - } - } - - private void InitializeTimer() - { - long interval = holdTimeThreshold / HoldCount; - holdTimer = new System.Timers.Timer(interval); - holdTimer.Elapsed += HoldTimer_Elapsed; - holdTimer.AutoReset = true; - holdTimer.Enabled = true; - } - - private void HoldTimer_Elapsed(object sender, ElapsedEventArgs e) - { - if ((Hold < HoldCount) && (fscore >= SquatDetectedThreshold)) - { - newHold++; - } - } - - private void DetectSquat(float squatSimilarity) - { - var previousScore = fscore; - fscore = squatSimilarity; - newScore = (int)System.Math.Round((float)(100 * fscore)); - - if (previousScore < SquatDetectedThreshold) - { - if (fscore >= SquatDetectedThreshold) - { - stopwatch.Start(); - holdTimer.Start(); - } - } - else - { - if (fscore < SquatDetectedThreshold) - { - stopwatch.Stop(); - holdTimer.Stop(); - - if (stopwatch.ElapsedMilliseconds >= holdTimeThreshold) - { - newCount++; - } - - newHold = 0; - stopwatch.Reset(); - } - } - } - - private void RaisePropertyChanged([CallerMemberName] string propertyName = null) - { - PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(propertyName)); - } - } -} diff --git a/Fitness/Services/WorkoutRepository.cs b/Fitness/Services/WorkoutRepository.cs index 4b8cacd..4a2d98f 100644 --- a/Fitness/Services/WorkoutRepository.cs +++ b/Fitness/Services/WorkoutRepository.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.IO; using Fitness.Models; using Fitness.ViewModels; using Tizen.Applications; @@ -33,8 +34,8 @@ namespace Fitness.Services Difficulty = DifficultyLevel.Easy, Duration = new TimeSpan(0, 4, 30), Favourite = false, - VideoUrl = Application.Current.DirectoryInfo.Resource + "media/PlieSquat.avi", - ThumbnailUrl = Application.Current.DirectoryInfo.Resource + "media/PlieSquat.png", + VideoUrl = Path.Combine(Application.Current.DirectoryInfo.Resource, "media/PlieSquat.avi"), + ThumbnailUrl = Path.Combine(Application.Current.DirectoryInfo.Resource, "media/PlieSquat.png"), Id = "1", }, new WorkoutViewModel diff --git a/Fitness/ViewModels/ExercisingViewModel.cs b/Fitness/ViewModels/ExercisingViewModel.cs index f3a74d5..56524b3 100644 --- a/Fitness/ViewModels/ExercisingViewModel.cs +++ b/Fitness/ViewModels/ExercisingViewModel.cs @@ -9,7 +9,7 @@ namespace Fitness.ViewModels public class ExercisingViewModel : ChangeWorkoutViewModel { private WorkoutState state; - private SquatService squatService; + private IExerciseService squatService; public ExercisingViewModel(WorkoutViewModel workoutViewModel) { @@ -105,7 +105,7 @@ namespace Fitness.ViewModels /// /// Gets or sets the property. /// - public SquatService SquatService + public IExerciseService SquatService { get => squatService; set diff --git a/Fitness/ViewModels/ScanningViewModel.cs b/Fitness/ViewModels/ScanningViewModel.cs index 1885210..b8109de 100644 --- a/Fitness/ViewModels/ScanningViewModel.cs +++ b/Fitness/ViewModels/ScanningViewModel.cs @@ -9,7 +9,7 @@ namespace Fitness.ViewModels /// public class ScanningViewModel : BaseViewModel { - private SquatService squatService; + private IExerciseService squatService; /// /// Initializes a new instance of the class. @@ -34,7 +34,7 @@ namespace Fitness.ViewModels /// /// Gets or sets the property. /// - public SquatService SquatService + public IExerciseService SquatService { get => squatService; set diff --git a/Fitness/Views/ExercisingView.xaml.cs b/Fitness/Views/ExercisingView.xaml.cs index 077c5f0..3efc5f4 100644 --- a/Fitness/Views/ExercisingView.xaml.cs +++ b/Fitness/Views/ExercisingView.xaml.cs @@ -231,7 +231,7 @@ namespace Fitness.Views { if (BindingContext is ViewModels.ExercisingViewModel viewModel) { - _ = viewModel.SquatService.DetectSquat(e.Preview); + _ = viewModel.SquatService.DetectExercise(e.Preview); } } diff --git a/Fitness/Views/ScanningView.cs b/Fitness/Views/ScanningView.cs index 181e5e7..e750d2f 100644 --- a/Fitness/Views/ScanningView.cs +++ b/Fitness/Views/ScanningView.cs @@ -32,7 +32,7 @@ namespace Fitness.Views { if (BindingContext is ViewModels.ScanningViewModel viewModel) { - _ = viewModel.SquatService.DetectSquat(e.Preview); + _ = viewModel.SquatService.DetectExercise(e.Preview); } } }