[Camera] Fix preview gc issue (#3883)
authorHaesu Gwon <haesu.gwon@samsung.com>
Thu, 27 Jan 2022 06:45:56 +0000 (15:45 +0900)
committerGitHub <noreply@github.com>
Thu, 27 Jan 2022 06:45:56 +0000 (15:45 +0900)
* [Camera] Fix preview gc issue

src/Tizen.Multimedia.Camera/Camera/Camera.Events.cs
src/Tizen.Multimedia.Camera/Camera/Camera.cs
src/Tizen.Multimedia.Camera/Camera/DepthPlane.cs
src/Tizen.Multimedia.Camera/Camera/DoublePlane.cs
src/Tizen.Multimedia.Camera/Camera/EncodedPlane.cs
src/Tizen.Multimedia.Camera/Camera/PinnedPreviewBuffer.cs [new file with mode: 0644]
src/Tizen.Multimedia.Camera/Camera/PreviewFrame.cs
src/Tizen.Multimedia.Camera/Camera/RgbPlane.cs
src/Tizen.Multimedia.Camera/Camera/SinglePlane.cs
src/Tizen.Multimedia.Camera/Camera/TriplePlane.cs

index 54cf472..02dfdbe 100644 (file)
@@ -398,7 +398,12 @@ namespace Tizen.Multimedia
         {
             _previewCallback = (frame, _) =>
             {
-                _preview?.Invoke(this, new PreviewEventArgs(new PreviewFrame(frame)));
+                Log.Info(CameraLog.Tag, "Invoke C# Preview event - START");
+
+                _preview?.Invoke(this,
+                    new PreviewEventArgs(new PreviewFrame(frame, ref _previewBuffer)));
+
+                Log.Info(CameraLog.Tag, "Invoke C# Preview event - DONE");
             };
 
             Native.SetPreviewCallback(_handle, _previewCallback).
@@ -445,7 +450,12 @@ namespace Tizen.Multimedia
         {
             _extraPreviewCallback = (frame, streamId, _) =>
             {
-                _extraPreview?.Invoke(this, new ExtraPreviewEventArgs(new PreviewFrame(frame), streamId));
+                Log.Info(CameraLog.Tag, "Invoke C# Extra Preview event - START");
+
+                _extraPreview?.Invoke(this,
+                    new ExtraPreviewEventArgs(new PreviewFrame(frame, ref _previewBuffer), streamId));
+
+                Log.Info(CameraLog.Tag, "Invoke C# Extra Preview event - DONE");
             };
 
             Native.SetExtraPreviewCallback(_handle, _extraPreviewCallback).
index d0e0074..780d963 100644 (file)
@@ -45,6 +45,7 @@ namespace Tizen.Multimedia
         private IntPtr _handle = IntPtr.Zero;
         private bool _disposed = false;
         private CameraState _state = CameraState.None;
+        PinnedPreviewBuffer<byte> _previewBuffer;
 
         /// <summary>
         /// Initializes a new instance of the <see cref="Camera"/> class.
@@ -165,6 +166,7 @@ namespace Tizen.Multimedia
                 if (disposing)
                 {
                     // to be used if there are any other disposable objects
+                    _previewBuffer.Dispose();
                 }
 
                 if (_handle != IntPtr.Zero)
index fe169ba..1311118 100644 (file)
@@ -14,9 +14,6 @@
  * limitations under the License.
  */
 
-using System.Runtime.InteropServices;
-using static Interop.Camera;
-
 namespace Tizen.Multimedia
 {
     /// <summary>
@@ -25,10 +22,9 @@ namespace Tizen.Multimedia
     /// <since_tizen> 5 </since_tizen>
     public class DepthPlane : IPreviewPlane
     {
-        internal DepthPlane(DepthPlaneStruct unmanagedData)
+        internal DepthPlane(byte[] data)
         {
-            Data = new byte[unmanagedData.DataLength];
-            Marshal.Copy(unmanagedData.Data, Data, 0, (int)unmanagedData.DataLength);
+            Data = data;
         }
 
         /// <summary>
index 15f81fd..68cea70 100644 (file)
@@ -14,9 +14,6 @@
  * limitations under the License.
  */
 
-using System.Runtime.InteropServices;
-using static Interop.Camera;
-
 namespace Tizen.Multimedia
 {
     /// <summary>
@@ -25,12 +22,10 @@ namespace Tizen.Multimedia
     /// <since_tizen> 3 </since_tizen>
     public class DoublePlane : IPreviewPlane
     {
-        internal DoublePlane(DoublePlaneStruct unmanaged)
+        internal DoublePlane(params byte[][] data)
         {
-            Y = new byte[unmanaged.YLength];
-            UV = new byte[unmanaged.UVLength];
-            Marshal.Copy(unmanaged.Y, Y, 0, (int)unmanaged.YLength);
-            Marshal.Copy(unmanaged.UV, UV, 0, (int)unmanaged.UVLength);
+            Y = data[0];
+            UV = data[1];
         }
 
         /// <summary>
index fae8ea9..638c6b4 100644 (file)
@@ -14,9 +14,6 @@
  * limitations under the License.
  */
 
-using System.Runtime.InteropServices;
-using static Interop.Camera;
-
 namespace Tizen.Multimedia
 {
     /// <summary>
@@ -25,12 +22,10 @@ namespace Tizen.Multimedia
     /// <since_tizen> 3 </since_tizen>
     public class EncodedPlane : IPreviewPlane
     {
-        internal EncodedPlane(EncodedPlaneStruct unmanagedData)
+        internal EncodedPlane(byte[] data, bool isDeltaFrame)
         {
-            Data = new byte[unmanagedData.DataLength];
-            Marshal.Copy(unmanagedData.Data, Data, 0, (int)unmanagedData.DataLength);
-
-            IsDeltaFrame = unmanagedData.IsDeltaFrame;
+            Data = data;
+            IsDeltaFrame = isDeltaFrame;
         }
 
         /// <summary>
diff --git a/src/Tizen.Multimedia.Camera/Camera/PinnedPreviewBuffer.cs b/src/Tizen.Multimedia.Camera/Camera/PinnedPreviewBuffer.cs
new file mode 100644 (file)
index 0000000..a03c4cd
--- /dev/null
@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the License);
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+using System.Runtime.InteropServices;
+
+namespace Tizen.Multimedia
+{
+    internal class PinnedPreviewBuffer<T> : IDisposable
+    {
+        private readonly GCHandle[] _gcHandles;
+        private readonly T[][] _buffers;
+
+        internal PinnedPreviewBuffer(params uint[] sizes)
+        {
+            _buffers = new T[sizes.Length][];
+            _gcHandles = new GCHandle[sizes.Length];
+
+            for (int i = 0 ; i < sizes.Length; i++)
+            {
+                _buffers[i] = new T[sizes[i]];
+                _gcHandles[i] = GCHandle.Alloc(_buffers[i], GCHandleType.Pinned);
+            }
+        }
+
+        ~PinnedPreviewBuffer()
+        {
+            Dispose(false);
+        }
+
+        internal T[] this[int index] => _buffers[index];
+
+        private bool disposedValue = false;
+
+        protected virtual void Dispose(bool disposing)
+        {
+            if (!disposedValue)
+            {
+                foreach (var gcHandle in _gcHandles)
+                {
+                    gcHandle.Free();
+                }
+
+                Log.Info(CameraLog.Tag, $"Disposed : {disposing})");
+                disposedValue = true;
+            }
+        }
+
+        public void Dispose()
+        {
+            Dispose(true);
+            GC.SuppressFinalize(this);
+        }
+    }
+}
index 756e6f4..5b63ed8 100644 (file)
@@ -27,7 +27,7 @@ namespace Tizen.Multimedia
     /// <since_tizen> 4 </since_tizen>
     public class PreviewFrame
     {
-        internal PreviewFrame(IntPtr ptr)
+        internal PreviewFrame(IntPtr ptr, ref PinnedPreviewBuffer<byte> buffers)
         {
             var unmanagedStruct = Marshal.PtrToStructure<CameraPreviewDataStruct>(ptr);
 
@@ -35,7 +35,7 @@ namespace Tizen.Multimedia
             Resolution = new Size(unmanagedStruct.Width, unmanagedStruct.Height);
             TimeStamp = unmanagedStruct.TimeStamp;
             PlaneType = GetPlaneType(unmanagedStruct);
-            Plane = ConvertPlane(unmanagedStruct);
+            CreatePlane(unmanagedStruct, ref buffers);
         }
 
         private static PlaneType GetPlaneType(CameraPreviewDataStruct unmanagedStruct)
@@ -67,29 +67,149 @@ namespace Tizen.Multimedia
             return PlaneType.TriplePlane;
         }
 
-        private IPreviewPlane ConvertPlane(CameraPreviewDataStruct unmanagedStruct)
+        internal void CreatePlane(CameraPreviewDataStruct unmanagedStruct, ref PinnedPreviewBuffer<byte> buffers)
         {
+            Log.Info(CameraLog.Tag, "Enter");
             switch (PlaneType)
             {
                 case PlaneType.SinglePlane:
-                    return new SinglePlane(unmanagedStruct.Plane.SinglePlane);
+                    Log.Info(CameraLog.Tag, "SinglePlane - START");
+                    var singlePlane = unmanagedStruct.Plane.SinglePlane;
+
+                    if (buffers == null)
+                    {
+                        Log.Info(CameraLog.Tag, "SinglePlane - Alloc buffer");
+                        buffers = new PinnedPreviewBuffer<byte>(singlePlane.DataLength);
+                    }
+                    Log.Info(CameraLog.Tag, "SinglePlane - Marshal copy");
+                    Marshal.Copy(singlePlane.Data, buffers[0], 0, (int)singlePlane.DataLength);
+
+                    Log.Info(CameraLog.Tag, "SinglePlane - Create SinglePlane instance");
+                    Plane = new SinglePlane(buffers[0]);
+                    Log.Info(CameraLog.Tag, "SinglePlane - DONE");
+                    break;
+                case PlaneType.DoublePlane:
+                    Log.Info(CameraLog.Tag, "DoublePlane - START");
+                    var doublePlane = unmanagedStruct.Plane.DoublePlane;
+
+                    doublePlane.YLength = (uint)(Resolution.Width * Resolution.Height);
+                    doublePlane.UVLength = (uint)(Resolution.Width * Resolution.Height) / 2;
+
+                    if (buffers == null)
+                    {
+                        Log.Info(CameraLog.Tag, "DoublePlane - Alloc buffer");
+                        buffers = new PinnedPreviewBuffer<byte>(doublePlane.YLength, doublePlane.UVLength);
+                    }
+
+                    Log.Info(CameraLog.Tag, $"DoublePlane - Marshal copy");
+                    Marshal.Copy(doublePlane.Y, buffers[0], 0, (int)doublePlane.YLength);
+                    Marshal.Copy(doublePlane.UV, buffers[1], 0, (int)doublePlane.UVLength);
+
+                    Log.Info(CameraLog.Tag, "DoublePlane - Create DoublePlane instance");
+                    Plane =  new DoublePlane(buffers[0], buffers[1]);
+                    Log.Info(CameraLog.Tag, "DoublePlane - DONE");
+                    break;
+                case PlaneType.TriplePlane:
+                    Log.Info(CameraLog.Tag, "TriplePlane - START");
+                    var triplePlane = unmanagedStruct.Plane.TriplePlane;
+
+                    if (buffers == null)
+                    {
+                        Log.Info(CameraLog.Tag, "TriplePlane - Alloc buffer");
+                        buffers = new PinnedPreviewBuffer<byte>(triplePlane.YLength, triplePlane.ULength, triplePlane.VLength);
+                    }
+                    Log.Info(CameraLog.Tag, "TriplePlane - Marshal copy");
+                    Marshal.Copy(triplePlane.Y, buffers[0], 0, (int)triplePlane.YLength);
+                    Marshal.Copy(triplePlane.U, buffers[1], 0, (int)triplePlane.ULength);
+                    Marshal.Copy(triplePlane.V, buffers[2], 0, (int)triplePlane.VLength);
+
+                    Log.Info(CameraLog.Tag, "TriplePlane - Create TriplePlane instance");
+                    Plane =  new TriplePlane(buffers[0], buffers[1], buffers[2]);
+                    Log.Info(CameraLog.Tag, "TriplePlane - DONE");
+                    break;
+                case PlaneType.EncodedPlane:
+                    Log.Info(CameraLog.Tag, "EncodedPlane - START");
+                    var encodedPlane = unmanagedStruct.Plane.EncodedPlane;
+
+                    if (buffers == null)
+                    {
+                        Log.Info(CameraLog.Tag, "EncodedPlane - Alloc buffer");
+                        buffers = new PinnedPreviewBuffer<byte>(encodedPlane.DataLength * 2);
+                    }
+                    Log.Info(CameraLog.Tag, "EncodedPlane - Marshal copy");
+                    Marshal.Copy(encodedPlane.Data, buffers[0], 0, (int)encodedPlane.DataLength);
+
+                    Log.Info(CameraLog.Tag, "EncodedPlane - Create EncodedPlane instance");
+                    Plane = new EncodedPlane(buffers[0], encodedPlane.IsDeltaFrame);
+                    Log.Info(CameraLog.Tag, "EncodedPlane - DONE");
+                    break;
+                case PlaneType.DepthPlane:
+                    Log.Info(CameraLog.Tag, "DepthPlane - START");
+                    var depthPlane = unmanagedStruct.Plane.DepthPlane;
+
+                    if (buffers == null)
+                    {
+                        Log.Info(CameraLog.Tag, "DepthPlane - Alloc buffer");
+                        buffers = new PinnedPreviewBuffer<byte>(depthPlane.DataLength);
+                    }
+                    Log.Info(CameraLog.Tag, "DepthPlane - Marshal copy");
+                    Marshal.Copy(depthPlane.Data, buffers[0], 0, (int)depthPlane.DataLength);
+
+                    Log.Info(CameraLog.Tag, "DepthPlane - Create DepthPlane instance");
+                    Plane = new DepthPlane(buffers[0]);
+                    Log.Info(CameraLog.Tag, "DepthPlane - DONE");
+                    break;
+                case PlaneType.RgbPlane:
+                    var rgbPlane = unmanagedStruct.Plane.RgbPlane;
+
+                    if (buffers == null)
+                    {
+                        Log.Info(CameraLog.Tag, "RgbPlane - Alloc buffer");
+                        buffers = new PinnedPreviewBuffer<byte>(rgbPlane.DataLength);
+                    }
+                    Marshal.Copy(rgbPlane.Data, buffers[0], 0, (int)rgbPlane.DataLength);
+
+                    Plane = new RgbPlane(buffers[0]);
+                    break;
+                default:
+                    Debug.Fail("Unknown preview data!");
+                    break;
+            }
+            Log.Info(CameraLog.Tag, "Leave");
+        }
+
+        internal static uint GetMaxPreviewPlaneSize(IntPtr ptr)
+        {
+            uint size = 0;
+            var unmanagedStruct = Marshal.PtrToStructure<CameraPreviewDataStruct>(ptr);
+
+            switch (GetPlaneType(unmanagedStruct))
+            {
+                case PlaneType.SinglePlane:
+                    size = unmanagedStruct.Plane.SinglePlane.DataLength;
+                    break;
                 case PlaneType.DoublePlane:
-                    var size = Resolution.Width * Resolution.Height;
-                    unmanagedStruct.Plane.DoublePlane.YLength = (uint)size;
-                    unmanagedStruct.Plane.DoublePlane.UVLength = (uint)size / 2;
-                    return new DoublePlane(unmanagedStruct.Plane.DoublePlane);
+                    size = unmanagedStruct.Plane.DoublePlane.UVLength;
+                    Log.Info(CameraLog.Tag, $"Size : {size}");
+                    break;
                 case PlaneType.TriplePlane:
-                    return new TriplePlane(unmanagedStruct.Plane.TriplePlane);
+                    size = unmanagedStruct.Plane.TriplePlane.YLength;
+                    break;
                 case PlaneType.EncodedPlane:
-                    return new EncodedPlane(unmanagedStruct.Plane.EncodedPlane);
+                    size = unmanagedStruct.Plane.EncodedPlane.DataLength;
+                    break;
                 case PlaneType.DepthPlane:
-                    return new DepthPlane(unmanagedStruct.Plane.DepthPlane);
+                    size = unmanagedStruct.Plane.DepthPlane.DataLength;
+                    break;
                 case PlaneType.RgbPlane:
-                    return new RgbPlane(unmanagedStruct.Plane.RgbPlane);
+                    size = unmanagedStruct.Plane.RgbPlane.DataLength;
+                    break;
+                default:
+                    Debug.Fail("Unknown preview data!");
+                    break;
             }
 
-            Debug.Fail("Unknown preview data!");
-            return null;
+            return size;
         }
 
         /// <summary>
@@ -120,6 +240,6 @@ namespace Tizen.Multimedia
         /// The buffer including the preview frame.
         /// </summary>
         /// <since_tizen> 4 </since_tizen>
-        public IPreviewPlane Plane { get; }
+        public IPreviewPlane Plane { get; private set;}
     }
 }
index 0b76215..3e1dd1b 100644 (file)
@@ -14,9 +14,6 @@
  * limitations under the License.
  */
 
-using System.Runtime.InteropServices;
-using static Interop.Camera;
-
 namespace Tizen.Multimedia
 {
     /// <summary>
@@ -25,10 +22,9 @@ namespace Tizen.Multimedia
     /// <since_tizen> 5 </since_tizen>
     public class RgbPlane : IPreviewPlane
     {
-        internal RgbPlane(RgbPlaneStruct unmanagedData)
+        internal RgbPlane(byte[] data)
         {
-            Data = new byte[unmanagedData.DataLength];
-            Marshal.Copy(unmanagedData.Data, Data, 0, (int)unmanagedData.DataLength);
+            Data = data;
         }
 
         /// <summary>
index a9a3a68..bb0b0a9 100644 (file)
@@ -14,9 +14,6 @@
  * limitations under the License.
  */
 
-using System.Runtime.InteropServices;
-using static Interop.Camera;
-
 namespace Tizen.Multimedia
 {
     /// <summary>
@@ -25,10 +22,9 @@ namespace Tizen.Multimedia
     /// <since_tizen> 3 </since_tizen>
     public class SinglePlane : IPreviewPlane
     {
-        internal SinglePlane(SinglePlaneStruct unmanaged)
+        internal SinglePlane(byte[] y)
         {
-            Data = new byte[unmanaged.DataLength];
-            Marshal.Copy(unmanaged.Data, Data, 0, (int)unmanaged.DataLength);
+            Data = y;
         }
 
         /// <summary>
index 5563535..de2b18b 100644 (file)
@@ -25,14 +25,11 @@ namespace Tizen.Multimedia
     /// <since_tizen> 3 </since_tizen>
     public class TriplePlane : IPreviewPlane
     {
-        internal TriplePlane(TriplePlaneStruct unmanaged)
+        internal TriplePlane(params byte[][] data)
         {
-            Y = new byte[unmanaged.YLength];
-            U = new byte[unmanaged.ULength];
-            V = new byte[unmanaged.VLength];
-            Marshal.Copy(unmanaged.Y, Y, 0, (int)unmanaged.YLength);
-            Marshal.Copy(unmanaged.U, U, 0, (int)unmanaged.ULength);
-            Marshal.Copy(unmanaged.V, V, 0, (int)unmanaged.VLength);
+            Y = data[0];
+            U = data[1];
+            V = data[2];
         }
 
         /// <summary>