--- /dev/null
+<Project Sdk="Microsoft.NET.Sdk">
+
+ <PropertyGroup>
+ <TargetFramework>net6.0</TargetFramework>
+ <LangVersion>9.0</LangVersion>
+ <NoWarn>$(NoWarn);0618;CA1054;CA1056</NoWarn>
+ </PropertyGroup>
+
+ <ItemGroup>
+ <PackageReference Include="Newtonsoft.Json" Version="11.0.2" />
+ <ProjectReference Include="..\Tizen\Tizen.csproj" />
+ <ProjectReference Include="..\Tizen.Log\Tizen.Log.csproj" />
+ <ProjectReference Include="..\Tizen.Uix.Tts\Tizen.Uix.Tts.csproj" />
+ <ProjectReference Include="..\Tizen.NUI\Tizen.NUI.csproj" />
+ <ProjectReference Include="..\Tizen.NUI.Scene3D\Tizen.NUI.Scene3D.csproj" />
+ <ProjectReference Include="..\Tizen.MachineLearning.Inference\Tizen.MachineLearning.Inference.csproj" />
+ <ProjectReference Include="..\Tizen.Multimedia.AudioIO\Tizen.Multimedia.AudioIO.csproj" />
+ <ProjectReference Include="..\Tizen.Security\Tizen.Security.csproj" />
+ <ProjectReference Include="..\Tizen.Security.PrivacyPrivilegeManager\Tizen.Security.PrivacyPrivilegeManager.csproj" />
+ </ItemGroup>
+
+
+
+</Project>
+
--- /dev/null
+/*
+ * Copyright(c) 2023 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using System.Collections.Generic;
+using System.IO;
+
+using static Tizen.AIAvatar.AIAvatar;
+
+namespace Tizen.AIAvatar
+{
+ internal static class AvatarExtension
+ {
+ internal static List<AvatarInfo> GetDefaultAvatarList()
+ {
+ var list = new List<AvatarInfo>();
+ var avatarModelFolders = Directory.GetDirectories(ApplicationResourcePath + EmojiAvatarResourcePath);
+ foreach (var directoryInfo in avatarModelFolders)
+ {
+ Log.Info(LogTag, $"Directory Path : {directoryInfo}");
+ var avatarInfo = new AvatarInfo(directoryInfo);
+ list.Add(avatarInfo);
+ }
+
+ return list;
+ }
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2023 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using Tizen.NUI.Scene3D;
+
+namespace Tizen.AIAvatar
+{
+ internal class AvatarScene : SceneView
+ {
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2023 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Tizen.NUI.Scene3D;
+
+namespace Tizen.AIAvatar
+{
+ internal static class SceneViewExtension
+ {
+ internal static void SetAlphaMaskUrl(this SceneView sceneView, string url)
+ {
+ var setValue = new Tizen.NUI.PropertyValue(url);
+ sceneView.SetProperty(Interop.AlphaMaskURLGet(), setValue);
+ setValue.Dispose();
+ }
+
+ internal static string GetAlphaMaskUrl(this SceneView sceneView)
+ {
+ var returnValue = "";
+ var invertYAxis = sceneView.GetProperty(Interop.AlphaMaskURLGet());
+ invertYAxis?.Get(out returnValue);
+ invertYAxis?.Dispose();
+ return returnValue;
+ }
+
+ internal static void SetMaskContentScaleFactor(this SceneView sceneView, float factor)
+ {
+ var setValue = new Tizen.NUI.PropertyValue(factor);
+ sceneView.SetProperty(Interop.MaskContentScaleGet(), setValue);
+ setValue.Dispose();
+ }
+
+ internal static float GetMaskContentScaleFactor(this SceneView sceneView)
+ {
+ var returnValue = 0.0f;
+ var invertYAxis = sceneView.GetProperty(Interop.MaskContentScaleGet());
+ invertYAxis?.Get(out returnValue);
+ invertYAxis?.Dispose();
+ return returnValue;
+ }
+
+ internal static void EnableCropToMask(this SceneView sceneView, bool enableCropToMask)
+ {
+ var setValue = new Tizen.NUI.PropertyValue(enableCropToMask);
+ sceneView.SetProperty(Interop.CropToMaskGet(), setValue);
+ setValue.Dispose();
+ }
+
+ internal static bool IsEnabledCropToMask(this SceneView sceneView)
+ {
+ bool returnValue = false;
+ var invertYAxis = sceneView.GetProperty(Interop.CropToMaskGet());
+ invertYAxis?.Get(out returnValue);
+ invertYAxis?.Dispose();
+ return returnValue;
+ }
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2023 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using Tizen.Multimedia;
+
+namespace Tizen.AIAvatar
+{
+ internal static class AIAvatar
+ {
+ internal const string LogTag = "Tizen.AIAvatar";
+ internal static readonly string ApplicationResourcePath = "/usr/apps/org.tizen.default-avatar-resource/shared/res/";
+ internal static readonly string EmojiAvatarResourcePath = "/models/EmojiAvatar/";
+ internal static readonly string DefaultModelResourcePath = "/models/DefaultAvatar/";
+ internal static readonly string DefaultMotionResourcePath = "/animation/motion/";
+
+ internal static readonly string VisemeInfo = $"{ApplicationResourcePath}/viseme/emoji_viseme_info.json";
+ internal static readonly string DefaultModel = "DefaultAvatar.gltf";
+
+ internal static readonly string AREmojiDefaultAvatarPath = $"{ApplicationResourcePath}{DefaultModelResourcePath}{DefaultModel}";
+
+ internal static readonly string DefaultLowModelResourcePath = "/models/DefaultAvatar_Low/";
+ internal static readonly string ExternalModel = "model_external.gltf";
+ internal static readonly string AREmojiDefaultLowAvatarPath = $"{ApplicationResourcePath}{DefaultLowModelResourcePath}{ExternalModel}";
+
+ internal static AudioOptions DefaultAudioOptions = new AudioOptions(24000, AudioChannel.Mono, AudioSampleType.S16Le);
+ internal static AudioOptions CurrentAudioOptions = DefaultAudioOptions;
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2023 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+namespace Tizen.AIAvatar
+{
+ internal class Interop
+ {
+ internal static partial class Libraries
+ {
+ public const string Scene3D = "libdali2-csharp-binder-scene3d.so";
+ }
+
+ [global::System.Runtime.InteropServices.DllImport(Libraries.Scene3D, EntryPoint = "CSharp_Dali_SceneView_Property_AlphaMaskURL_get")]
+ public static extern int AlphaMaskURLGet();
+
+ [global::System.Runtime.InteropServices.DllImport(Libraries.Scene3D, EntryPoint = "CSharp_Dali_SceneView_Property_MaskContentScale_get")]
+ public static extern int MaskContentScaleGet();
+
+ [global::System.Runtime.InteropServices.DllImport(Libraries.Scene3D, EntryPoint = "CSharp_Dali_SceneView_Property_CropToMask_get")]
+ public static extern int CropToMaskGet();
+
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2023 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using System;
+using System.IO;
+using Tizen.Security;
+using Newtonsoft.Json;
+using System.Collections.Generic;
+using static Tizen.AIAvatar.AIAvatar;
+
+namespace Tizen.AIAvatar
+{
+ internal class Utils
+ {
+ internal static void ConvertAudioToFloat(in byte[] audioBytes, out float[] audioFloat)
+ {
+ audioFloat = new float[audioBytes.Length / 2];
+
+ for (int i = 0, j = 0; i < audioBytes.Length; i += 2, j++)
+ {
+ short sample = BitConverter.ToInt16(audioBytes, i);
+ audioFloat[j] = sample / 32768.0f;
+ }
+ }
+
+ internal static byte[] ReadAllBytes(string path)
+ {
+ try
+ {
+ var bytes = File.ReadAllBytes(path);
+ return bytes;
+ }
+ catch (Exception)
+ {
+ return null;
+ }
+ }
+
+ internal static void SaveFile(string path, string filename, byte[] array)
+ {
+ try
+ {
+ var file = new FileStream($"{path}/{filename}", FileMode.Create);
+ file.Write(array, 0, array.Length);
+ file.Close();
+ }
+ catch (Exception)
+ {
+ return;
+ }
+ }
+
+ internal static void CheckPrivilege(string privilege)
+ {
+ var result = PrivacyPrivilegeManager.CheckPermission(privilege);
+
+ switch (result)
+ {
+ case CheckResult.Allow:
+ Log.Info(LogTag, $"Privilege \"{privilege}\" : allowed.");
+ break;
+ case CheckResult.Deny:
+ Log.Info(LogTag, $"Privilege \"{privilege}\" : denied.");
+ /// Privilege can't be used
+ break;
+ case CheckResult.Ask:
+ /// Request permission to user
+ PrivacyPrivilegeManager.RequestPermission(privilege);
+ break;
+ }
+ }
+
+ internal static T ConvertJson<T>(string jsonString)
+ {
+ return JsonConvert.DeserializeObject<T>(jsonString);
+ }
+
+ internal int FindMaxValue<T>(List<T> list, Converter<T, int> projection)
+ {
+ if (list.Count == 0)
+ {
+ throw new InvalidOperationException("Empty list");
+ }
+ int maxValue = int.MinValue;
+ foreach (T item in list)
+ {
+ int value = projection(item);
+ if (value > maxValue)
+ {
+ maxValue = value;
+ }
+ }
+ return maxValue;
+ }
+ }
+}
+
+internal class SystemUtils
+{
+ internal static void ST()
+ {
+ Tizen.Log.Error("MYLOG", System.Environment.StackTrace);
+ }
+
+ internal static string GetFileName(string path)
+ {
+ return System.IO.Path.GetFileName(path);
+ }
+ internal static string GetFileNameWithoutExtension(string path)
+ {
+ return System.IO.Path.GetFileNameWithoutExtension(path);
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2023 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using System.Collections.Generic;
+using System.ComponentModel;
+using static Tizen.AIAvatar.AIAvatar;
+
+namespace Tizen.AIAvatar
+{
+ internal class BlendShapeInfo
+ {
+ internal BlendShapeModelInfo blendShape;
+ internal BlendShapeVisemeInfo[] visemes;
+
+ internal BlendShapeInfo() { }
+
+ internal string[] GetNodeNames()
+ {
+ if (blendShape == null)
+ {
+ Log.Error(LogTag, "blendShape is null");
+ return null;
+ }
+ return blendShape.nodeNames;
+ }
+
+ internal int[] GetBlendShapeCounts()
+ {
+ return blendShape.blendShapeCount;
+ }
+
+ internal Dictionary<Viseme, BlendShapeValue[]> GetVisemeMap()
+ {
+ Dictionary<Viseme, BlendShapeValue[]> visemeMap = new Dictionary<Viseme, BlendShapeValue[]>();
+
+ foreach (var visemeInfo in visemes)
+ {
+ visemeMap.Add(visemeInfo.name, visemeInfo.values);
+ }
+
+ return visemeMap;
+ }
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2023 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using System.ComponentModel;
+
+namespace Tizen.AIAvatar
+{
+ internal class BlendShapeModelInfo
+ {
+ internal string keyFormat;
+ internal string[] nodeNames;
+ internal int[] blendShapeCount;
+
+ internal BlendShapeModelInfo(string keyFormat, string[] nodeNames, int[] blendShapeCount)
+ {
+ this.keyFormat = keyFormat;
+ this.nodeNames = nodeNames;
+ this.blendShapeCount = blendShapeCount;
+ }
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2023 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using System.ComponentModel;
+
+namespace Tizen.AIAvatar
+{
+ internal class BlendShapeValue
+ {
+ internal string nodeName;
+ internal BlendShapeType blendIndex;
+ internal float blendValue;
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2023 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using System.ComponentModel;
+
+namespace Tizen.AIAvatar
+{
+ internal class BlendShapeVisemeInfo
+ {
+ internal Viseme name;
+ internal BlendShapeValue[] values;
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2024 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using System.Net.Http;
+using System.Threading.Tasks;
+
+namespace Tizen.AIAvatar
+{
+
+ internal interface IRestClient
+ {
+ Task<string> SendRequestAsync(HttpMethod method, string endpoint, string bearerToken = null, string jsonData = null);
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2024 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using System;
+using System.Net.Http.Headers;
+using System.Net.Http;
+using System.Text;
+using System.Threading.Tasks;
+using System.ComponentModel;
+
+namespace Tizen.AIAvatar
+{
+ internal class RestClient : IRestClient, IDisposable
+ {
+ private readonly HttpClient client;
+
+ internal RestClient(HttpClient httpClient)
+ {
+ client = httpClient;
+ }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public async Task<string> SendRequestAsync(HttpMethod method, string endpoint, string bearerToken = null, string jsonData = null)
+ {
+ AddBearerToken(bearerToken);
+
+ HttpRequestMessage request = new HttpRequestMessage(method, endpoint);
+
+ if (jsonData != null)
+ {
+ request.Content = new StringContent(jsonData, Encoding.UTF8, "application/json");
+ }
+
+ HttpResponseMessage response = await client.SendAsync(request);
+ return await HandleResponse(response);
+ }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public void Dispose()
+ {
+ client.Dispose();
+ }
+
+ private void AddBearerToken(string bearerToken)
+ {
+ if (!string.IsNullOrEmpty(bearerToken))
+ {
+ client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", bearerToken);
+ }
+ }
+
+ private async Task<string> HandleResponse(HttpResponseMessage response)
+ {
+ if (response.IsSuccessStatusCode)
+ {
+ return await response.Content.ReadAsStringAsync();
+ }
+ else
+ {
+ throw new HttpRequestException($"HTTP request failed with status code {response.StatusCode}");
+ }
+ }
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2024 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using System.Net.Http;
+using System;
+
+namespace Tizen.AIAvatar
+{
+ internal class RestClientFactory
+ {
+ internal IRestClient CreateClient(string baseUrl)
+ {
+ return new RestClient(new HttpClient { BaseAddress = new Uri(baseUrl) });
+ }
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2023 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Tizen.NUI;
+using Tizen.NUI.Scene3D;
+
+namespace Tizen.AIAvatar
+{
+ internal class DefaultAvatarProperties : AvatarProperties
+ {
+ private static AvatarPropertyMapper defaultJointMapper = new AvatarPropertyMapper();
+ private static AvatarPropertyMapper defaultBlendShapeNameMapper = new AvatarPropertyMapper();
+ private static AvatarPropertyMapper defaultNodeMapper = new AvatarPropertyMapper();
+
+
+ internal DefaultAvatarProperties() : base(defaultJointMapper, defaultBlendShapeNameMapper, defaultNodeMapper)
+ {
+ foreach (var indexNamePair in blendShapeList)
+ {
+ defaultBlendShapeNameMapper.SetPropertyName((uint)indexNamePair.Item1, indexNamePair.Item2);
+ }
+
+ foreach (var indexNamePair in jointList)
+ {
+ defaultJointMapper.SetPropertyName((uint)indexNamePair.Item1, indexNamePair.Item2);
+ }
+
+ foreach (var indexNamePair in nodeList)
+ {
+ defaultNodeMapper.SetPropertyName((uint)indexNamePair.Item1, indexNamePair.Item2);
+ }
+ }
+
+ #region AR Emoji BlendShape name list
+ private static readonly List<(BlendShapeType, string)> blendShapeList = new List<(BlendShapeType, string)>()
+ {
+ (BlendShapeType.EyeBlinkLeft, "EyeBlink_Left"),
+ (BlendShapeType.EyeSquintLeft, "EyeSquint_Left"),
+ (BlendShapeType.EyeDownLeft, "EyeDown_Left"),
+ (BlendShapeType.EyeInLeft, "EyeIn_Left"),
+ (BlendShapeType.EyeOpenLeft, "EyeOpen_Left"),
+ (BlendShapeType.EyeOutLeft, "EyeOut_Left"),
+ (BlendShapeType.EyeUpLeft, "EyeUp_Left"),
+
+ (BlendShapeType.EyeBlinkRight, "EyeBlink_Right"),
+ (BlendShapeType.EyeSquintRight, "EyeSquint_Right"),
+ (BlendShapeType.EyeDownRight, "EyeDown_Right"),
+ (BlendShapeType.EyeInRight, "EyeIn_Right"),
+ (BlendShapeType.EyeOpenRight, "EyeOpen_Right"),
+ (BlendShapeType.EyeOutRight, "EyeOut_Right"),
+ (BlendShapeType.EyeUpRight, "EyeUp_Right"),
+
+ (BlendShapeType.JawForward, "JawFwd"),
+ (BlendShapeType.JawOpen, "JawOpen"),
+ (BlendShapeType.JawChew, "JawChew"),
+ (BlendShapeType.JawLeft, "JawLeft"),
+ (BlendShapeType.JawRight, "JawRight"),
+
+ (BlendShapeType.MouthLeft, "MouthLeft"),
+ (BlendShapeType.MouthFrownLeft, "MouthFrown_Left"),
+ (BlendShapeType.MouthSmileLeft, "MouthSmile_Left"),
+ (BlendShapeType.MouthDimpleLeft, "MouthDimple_Left"),
+
+ (BlendShapeType.MouthRight, "MouthRight"),
+ (BlendShapeType.MouthFrownRight, "MouthFrown_Right"),
+ (BlendShapeType.MouthSmileRight, "MouthSmile_Right"),
+ (BlendShapeType.MouthDimpleRight, "MouthDimple_Right"),
+
+ (BlendShapeType.LipsStretchLeft, "LipsStretch_Left"),
+ (BlendShapeType.LipsStretchRight, "LipsStretch_Right"),
+ (BlendShapeType.LipsUpperClose, "LipsUpperClose"),
+ (BlendShapeType.LipsLowerClose, "LipsLowerClose"),
+ (BlendShapeType.LipsUpperUp, "LipsUpperUp"),
+ (BlendShapeType.LipsLowerDown, "LipsLowerDown"),
+ (BlendShapeType.LipsUpperOpen, "LipsUpperOpen"),
+ (BlendShapeType.LipsLowerOpen, "LipsLowerOpen"),
+ (BlendShapeType.LipsFunnel, "LipsFunnel"),
+ (BlendShapeType.LipsPucker, "LipsPucker"),
+
+ (BlendShapeType.BrowDownLeft, "BrowsDown_Left"),
+ (BlendShapeType.BrowDownRight, "BrowsDown_Right"),
+ (BlendShapeType.BrowUpCenter, "BrowsUp_Center"),
+ (BlendShapeType.BrowUpLeft, "BrowsUp_Left"),
+ (BlendShapeType.BrowUpRight, "BrowsUp_Right"),
+ (BlendShapeType.CheekSquintLeft, "CheekSquint_Left"),
+ (BlendShapeType.CheekSquintRight, "CheekSquint_Right"),
+ (BlendShapeType.ChinLowerRaise, "ChinLowerRaise"),
+ (BlendShapeType.ChinUpperRaise, "ChinUpperRaise"),
+
+ (BlendShapeType.TongueOut, "Tongue_Out"),
+ (BlendShapeType.TongueUp, "Tongue_Up"),
+ (BlendShapeType.TongueDown, "Tongue_Down"),
+ (BlendShapeType.TongueLeft, "Tongue_Left"),
+ (BlendShapeType.TongueRight, "Tongue_Right"),
+
+ (BlendShapeType.Sneer, "Sneer"),
+ (BlendShapeType.Puff, "Puff"),
+ (BlendShapeType.PuffLeft, "Puff_Left"),
+ (BlendShapeType.PuffRight, "Puff_Right"),
+ };
+ #endregion
+
+ #region AR Emoji Joint name list
+ private static readonly List<(JointType, string)> jointList = new List<(JointType, string)>
+ {
+ (JointType.Head, "head_JNT"),
+ (JointType.Neck, "neck_JNT"),
+ (JointType.EyeLeft, "l_eye_JNT"),
+ (JointType.EyeRight, "r_eye_JNT"),
+
+ (JointType.ShoulderLeft, "l_arm_JNT"),
+ (JointType.ElbowLeft, "l_forearm_JNT"),
+ (JointType.WristLeft, "l_hand_JNT"),
+
+ (JointType.ShoulderRight, "r_arm_JNT"),
+ (JointType.ElbowRight, "r_forearm_JNT"),
+ (JointType.WristRight, "r_hand_JNT"),
+
+ (JointType.HipLeft, "l_upleg_JNT"),
+ (JointType.KneeLeft, "l_leg_JNT"),
+ (JointType.AnkleLeft, "l_foot_JNT"),
+ (JointType.ForeFootLeft, "l_toebase_JNT"),
+
+ (JointType.HipRight, "r_upleg_JNT"),
+ (JointType.KneeRight, "r_leg_JNT"),
+ (JointType.AnkleRight, "r_foot_JNT"),
+ (JointType.ForeFootRight, "r_toebase_JNT"),
+
+ (JointType.FingerThumb1Left, "l_handThumb1_JNT"),
+ (JointType.FingerThumb2Left, "l_handThumb2_JNT"),
+ (JointType.FingerThumb3Left, "l_handThumb3_JNT"),
+ (JointType.FingerThumb4Left, "l_handThumb4_JNT"),
+ (JointType.FingerIndex1Left, "l_handIndex1_JNT"),
+ (JointType.FingerIndex2Left, "l_handIndex2_JNT"),
+ (JointType.FingerIndex3Left, "l_handIndex3_JNT"),
+ (JointType.FingerIndex4Left, "l_handIndex4_JNT"),
+ (JointType.FingerMiddle1Left, "l_handMiddle1_JNT"),
+ (JointType.FingerMiddle2Left, "l_handMiddle2_JNT"),
+ (JointType.FingerMiddle3Left, "l_handMiddle3_JNT"),
+ (JointType.FingerMiddle4Left, "l_handMiddle4_JNT"),
+ (JointType.FingerRing1Left, "l_handRing1_JNT"),
+ (JointType.FingerRing2Left, "l_handRing2_JNT"),
+ (JointType.FingerRing3Left, "l_handRing3_JNT"),
+ (JointType.FingerRing4Left, "l_handRing4_JNT"),
+ (JointType.FingerPinky1Left, "l_handPinky1_JNT"),
+ (JointType.FingerPinky2Left, "l_handPinky2_JNT"),
+ (JointType.FingerPinky3Left, "l_handPinky3_JNT"),
+ (JointType.FingerPinky4Left, "l_handPinky4_JNT"),
+
+ (JointType.FingerThumb1Right, "r_handThumb1_JNT"),
+ (JointType.FingerThumb2Right, "r_handThumb2_JNT"),
+ (JointType.FingerThumb3Right, "r_handThumb3_JNT"),
+ (JointType.FingerThumb4Right, "r_handThumb4_JNT"),
+ (JointType.FingerIndex1Right, "r_handIndex1_JNT"),
+ (JointType.FingerIndex2Right, "r_handIndex2_JNT"),
+ (JointType.FingerIndex3Right, "r_handIndex3_JNT"),
+ (JointType.FingerIndex4Right, "r_handIndex4_JNT"),
+ (JointType.FingerMiddle1Right, "r_handMiddle1_JNT"),
+ (JointType.FingerMiddle2Right, "r_handMiddle2_JNT"),
+ (JointType.FingerMiddle3Right, "r_handMiddle3_JNT"),
+ (JointType.FingerMiddle4Right, "r_handMiddle4_JNT"),
+ (JointType.FingerRing1Right, "r_handRing1_JNT"),
+ (JointType.FingerRing2Right, "r_handRing2_JNT"),
+ (JointType.FingerRing3Right, "r_handRing3_JNT"),
+ (JointType.FingerRing4Right, "r_handRing4_JNT"),
+ (JointType.FingerPinky1Right, "r_handPinky1_JNT"),
+ (JointType.FingerPinky2Right, "r_handPinky2_JNT"),
+ (JointType.FingerPinky3Right, "r_handPinky3_JNT"),
+ (JointType.FingerPinky4Right, "r_handPinky4_JNT"),
+ };
+ #endregion
+
+ #region AR Emoji Joint name list
+ private static readonly List<(NodeType, string)> nodeList = new List<(NodeType, string)>
+ {
+ (NodeType.HeadGeo, "head_GEO"),
+ (NodeType.MouthGeo, "mouth_GEO"),
+ (NodeType.EyelashGeo, "eyelash_GEO"),
+ };
+ #endregion
+
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2023 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using Tizen.NUI;
+
+namespace Tizen.AIAvatar
+{
+ internal class LipInfo
+ {
+ private Animatable animatable;
+ private string nodeName;
+ private int blendShapeCount;
+
+ internal Animatable Animatable { get => animatable; set => animatable = value; }
+ internal string NodeName { get => nodeName; set => nodeName = value; }
+ internal int BlendShapeCount { get => blendShapeCount; set => blendShapeCount = value; }
+ }
+}
--- /dev/null
+
+using System.ComponentModel;
+
+namespace Tizen.AIAvatar
+{
+ internal interface ISingleShotModel
+ {
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public void SetTensorData(int index, byte[] buffer);
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public byte[] GetTensorData(int index);
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public void Invoke();
+ }
+}
--- /dev/null
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace Tizen.AIAvatar
+{
+ internal static class SoftmaxLinqExtension
+ {
+ internal static IEnumerable<float> SoftMax(this IEnumerable<float> source)
+ {
+ var exp = source.Select(x => MathF.Exp(x)).ToArray();
+ var sum = exp.Sum();
+ return exp.Select(x => x / sum);
+ }
+ }
+}
--- /dev/null
+using System;
+using System.Collections.Generic;
+using System.ComponentModel;
+using System.Text;
+using Tizen.MachineLearning.Inference;
+using static Tizen.AIAvatar.AIAvatar;
+
+namespace Tizen.AIAvatar
+{
+ internal class TFVowel6 : ISingleShotModel
+ {
+ private SingleShot singleShot;
+ private TensorsInfo inputInfo;
+ private TensorsInfo outputInfo;
+ private TensorsData inputData;
+ private TensorsData outputData;
+
+ private readonly string modelPath = ApplicationResourcePath + "audio2vowel_7.tflite";
+
+ internal TFVowel6(int[] inputDimension, int[] outputDimension)
+ {
+ try
+ {
+ inputInfo = new TensorsInfo();
+ inputInfo.AddTensorInfo(TensorType.Float32, inputDimension);
+
+ outputInfo = new TensorsInfo();
+ outputInfo.AddTensorInfo(TensorType.Float32, outputDimension);
+
+ singleShot = new SingleShot(modelPath, inputInfo, outputInfo);
+ }
+ catch (Exception e)
+ {
+ if (e is NotSupportedException)
+ {
+ Log.Info(LogTag, "NotSupportedException occurs");
+ }
+ else
+ {
+ Log.Info(LogTag, e.Message);
+ }
+ }
+ }
+
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public void SetTensorData(int index, byte[] buffer)
+ {
+ try
+ {
+ inputData = inputInfo.GetTensorsData();
+ inputData.SetTensorData(index, buffer);
+ }
+ catch (Exception e)
+ {
+ Log.Info(LogTag, e.Message);
+ }
+ }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public void Invoke()
+ {
+ try
+ {
+ outputData = singleShot.Invoke(inputData);
+ }
+ catch (Exception e)
+ {
+ Log.Info(LogTag, e.Message);
+ }
+
+ }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public byte[] GetTensorData(int index)
+ {
+ return outputData.GetTensorData(index);
+ }
+
+
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2023 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+namespace Tizen.AIAvatar
+{
+ // Various visemes
+ internal enum Viseme
+ {
+ sil,
+ AE,
+ Ah,
+ B_M_P,
+ Ch_J,
+ EE,
+ Er,
+ IH,
+ Oh,
+ W_OO,
+ R,
+ };
+}
--- /dev/null
+/*
+ * Copyright(c) 2023 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using Tizen.Multimedia;
+using System.IO;
+using System;
+
+using static Tizen.AIAvatar.AIAvatar;
+
+namespace Tizen.AIAvatar
+{
+ internal class AudioPlayer
+ {
+ private AudioPlayback audioPlayback;
+ private MemoryStream audioStream;
+
+ internal AudioPlayer()
+ {
+ }
+
+ internal void PlayAsync(byte[] buffer, int sampleRate = 0)
+ {
+ if (audioPlayback == null)
+ {
+ Play(buffer, sampleRate);
+ }
+ else
+ {
+ audioPlayback.Write(buffer);
+ }
+ }
+
+ internal void Play(byte[] audioBytes, int sampleRate = 0)
+ {
+ if (audioBytes == null)
+ {
+ return;
+ }
+
+ try
+ {
+ if (audioPlayback != null)
+ {
+ DestroyAudioPlayback();
+ }
+ if (sampleRate == 0)
+ {
+ sampleRate = CurrentAudioOptions.SampleRate;
+ }
+ audioPlayback = new AudioPlayback(sampleRate, CurrentAudioOptions.Channel, CurrentAudioOptions.SampleType);
+ }
+ catch (Exception e)
+ {
+ Log.Error(LogTag, $"Failed to create AudioPlayback. {e.Message}");
+ return;
+ }
+
+ if (audioPlayback != null)
+ {
+ audioPlayback.Prepare();
+ audioPlayback.BufferAvailable += (sender, args) =>
+ {
+ if (audioStream.Position == audioStream.Length)
+ {
+ return;
+ }
+
+ try
+ {
+ var buffer = new byte[args.Length];
+ audioStream.Read(buffer, 0, args.Length);
+ audioPlayback.Write(buffer);
+ }
+ catch (Exception e)
+ {
+ Log.Error(LogTag, $"Failed to write. {e.Message}");
+ }
+ };
+
+ audioStream = new MemoryStream(audioBytes);
+ }
+ }
+
+ internal void Pause()
+ {
+ if (audioPlayback != null)
+ {
+ audioPlayback.Pause();
+ }
+ else
+ {
+ Log.Error(LogTag, $"audioPlayBack is null");
+ }
+ }
+
+ internal void Stop()
+ {
+ if (audioPlayback != null)
+ {
+ audioPlayback.Pause();
+ DestroyAudioPlayback();
+ }
+ else
+ {
+ Log.Error(LogTag, $"audioPlayBack is null");
+ }
+ }
+
+ internal void Destroy()
+ {
+ DestroyAudioPlayback();
+ }
+
+ private void DestroyAudioPlayback()
+ {
+ audioPlayback?.Unprepare();
+ audioPlayback?.Dispose();
+ audioPlayback = null;
+ }
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2023 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using System;
+using System.Linq;
+using System.Text;
+using Tizen.Multimedia;
+using Tizen.NUI;
+using static Tizen.AIAvatar.AIAvatar;
+
+namespace Tizen.AIAvatar
+{
+ internal class AudioRecorder
+ {
+ private const string privilegeForRecording = "http://tizen.org/privilege/recorder";
+
+ private AsyncAudioCapture asyncAudioCapture;
+
+ private byte[] recordedBuffer;
+ private float desiredBufferDuration = 0.16f;
+ private int desiredBufferLength;
+
+ private Timer audioRecordingTimer;
+
+ private Action audioRecdingAction;
+ private Action<byte[],int > bufferAction;
+
+ private static AudioRecorder instance;
+
+ internal static AudioRecorder Instance
+ {
+ get
+ {
+ if (instance == null)
+ {
+ instance = new AudioRecorder();
+ }
+ return instance;
+ }
+ }
+
+ internal event EventHandler<RecordBufferChangedEventArgs> BufferChanged;
+
+ internal AudioRecorder()
+ {
+ Utils.CheckPrivilege(privilegeForRecording);
+ desiredBufferLength = (int)(CurrentAudioOptions.SampleRate * desiredBufferDuration * 2);
+ }
+
+
+ internal void InitMic(BlendShapePlayer animator, uint recordingTime = 160)
+ {
+ audioRecordingTimer = new Timer(recordingTime);
+ if (animator != null)
+ {
+ //TODO : Connection MIC
+ var lipSyncer = (animator.GetAnimationModule(AnimationModuleType.LipSyncer) as LipSyncer);
+ if(lipSyncer != null)
+ {
+ Tizen.Log.Error(LogTag, "LipSyncer of animator is null");
+ return;
+ }
+ this.audioRecdingAction = lipSyncer.OnRecodingTick;
+ this.bufferAction = lipSyncer.OnRecordBufferChanged;
+
+ BufferChanged += OnRecordBufferChanged;
+ audioRecordingTimer.Tick += AudioRecordingTimerTick;
+ }
+ }
+
+
+ internal void DeinitMic()
+ {
+ StopRecording();
+ BufferChanged -= OnRecordBufferChanged;
+
+ if (audioRecordingTimer != null)
+ {
+ audioRecordingTimer.Stop();
+ audioRecordingTimer.Tick -= AudioRecordingTimerTick;
+
+ audioRecordingTimer.Dispose();
+ audioRecordingTimer = null;
+ }
+ audioRecdingAction = null;
+ }
+
+ internal void StartRecording()
+ {
+ audioRecordingTimer?.Start();
+ asyncAudioCapture = new AsyncAudioCapture(CurrentAudioOptions.SampleRate, CurrentAudioOptions.Channel, CurrentAudioOptions.SampleType);
+
+ recordedBuffer = new byte[0];
+ asyncAudioCapture.DataAvailable += (s, e) =>
+ {
+ recordedBuffer = recordedBuffer.Concat(e.Data).ToArray();
+ if (recordedBuffer.Length >= desiredBufferLength)
+ {
+ var recordedBuffer = this.recordedBuffer;
+ this.recordedBuffer = Array.Empty<byte>();
+
+ BufferChanged?.Invoke(this, new RecordBufferChangedEventArgs(recordedBuffer, CurrentAudioOptions.SampleRate));
+ }
+ };
+ asyncAudioCapture.Prepare();
+ Log.Info(LogTag, "Start Recording - Preapre AsyncAudioCapture");
+ }
+
+ internal void StopRecording()
+ {
+ audioRecordingTimer?.Stop();
+ asyncAudioCapture.Dispose();
+ }
+
+ internal void PauseRecording()
+ {
+ asyncAudioCapture.Pause();
+ }
+
+ internal void ResumeRecording()
+ {
+ asyncAudioCapture.Resume();
+ }
+
+ private void OnRecordBufferChanged(object sender, RecordBufferChangedEventArgs e)
+ {
+ bufferAction?.Invoke(e.RecordedBuffer, CurrentAudioOptions.SampleRate);
+ }
+
+ private bool AudioRecordingTimerTick(object source, Timer.TickEventArgs e)
+ {
+ Log.Info(LogTag, "TickTimer");
+ audioRecdingAction?.Invoke();
+ return true;
+ }
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2023 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using System;
+
+namespace Tizen.AIAvatar
+{
+ internal class RecordBufferChangedEventArgs : EventArgs
+ {
+ public byte[] RecordedBuffer { get; set; }
+ public int SampleRate { get; set; }
+
+ public RecordBufferChangedEventArgs(byte[] recordedBuffer, int SampleRate)
+ {
+ this.RecordedBuffer = recordedBuffer;
+ this.SampleRate = SampleRate;
+ }
+ }
+
+}
--- /dev/null
+/*
+ * Copyright(c) 2023 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using System;
+using System.Collections.Generic;
+using Tizen.Uix.Tts;
+using System.Runtime.InteropServices;
+
+using static Tizen.AIAvatar.AIAvatar;
+using System.Linq;
+using Tizen.NUI;
+
+namespace Tizen.AIAvatar
+{
+ internal class TTSLipSyncer
+ {
+
+ private Avatar currentAvatar;
+
+ private List<UtteranceText> textList;
+ private TtsClient ttsHandle;
+ private VoiceInfo voiceInfo;
+ private List<Byte> byteList;
+
+ private byte[] recordedBuffer;
+ private byte[] audioTailBuffer;
+
+ private int sampleRate;
+ private float desiredBufferDuration = 0.175f;
+ private float audioTailLengthFactor = 0.015f;
+ private float audioBufferMultiflier = 2f;
+
+ private int desiredBufferLength;
+ private int audioTailLength;
+
+ private bool isPrepared = false;
+ private bool isAsync = false;
+
+ private Action<byte[], int> bufferChangedAction;
+
+ private int audioLength;
+ private bool isAsyncLipStarting;
+
+ private AsyncLipSyncer lipSyncer;
+
+
+ internal TTSLipSyncer(Avatar avatar)
+ {
+ currentAvatar = avatar;
+ lipSyncer = (currentAvatar.AvatarAnimator.GetAnimationModule(AnimationModuleType.LipSyncer) as AsyncLipSyncer);
+ InitTts();
+ }
+
+ ~TTSLipSyncer()
+ {
+ DeinitTts();
+ }
+
+ internal event EventHandler PlayReadyCallback;
+
+ internal TtsClient TtsHandle
+ {
+ get { return ttsHandle; }
+ }
+
+ internal VoiceInfo VoiceInfo
+ {
+ get { return voiceInfo; }
+ set
+ {
+ voiceInfo = value;
+ }
+ }
+
+ internal List<VoiceInfo> GetSupportedVoices()
+ {
+ var voiceInfoList = new List<VoiceInfo>();
+
+ if (ttsHandle == null)
+ {
+ Log.Error(LogTag, $"ttsHandle is null");
+ return voiceInfoList;
+ }
+
+ var supportedVoices = ttsHandle.GetSupportedVoices();
+ foreach (var supportedVoice in supportedVoices)
+ {
+ Log.Info(LogTag, $"{supportedVoice.Language} & {supportedVoice.VoiceType} is supported");
+ voiceInfoList.Add(new VoiceInfo() { Lang = supportedVoice.Language, Type = supportedVoice.VoiceType });
+ }
+ return voiceInfoList;
+ }
+
+ internal bool IsSupportedVoice(string lang)
+ {
+ if (ttsHandle == null)
+ {
+ Log.Error(LogTag, $"ttsHandle is null");
+ return false;
+ }
+ var supportedVoices = ttsHandle.GetSupportedVoices();
+
+ foreach (var supportedVoice in supportedVoices)
+ {
+ if (supportedVoice.Language.Equals(lang))
+ {
+ Log.Info(LogTag, $"{lang} is supported");
+ return true;
+ }
+ }
+ return false;
+ }
+
+ internal bool IsSupportedVoice(VoiceInfo voiceInfo)
+ {
+ if (ttsHandle == null)
+ {
+ Log.Error(LogTag, $"ttsHandle is null");
+ return false;
+ }
+ var supportedVoices = ttsHandle.GetSupportedVoices();
+ foreach (var supportedVoice in supportedVoices)
+ {
+ if (supportedVoice.Language.Equals(voiceInfo.Lang) && (supportedVoice.VoiceType == voiceInfo.Type))
+ {
+ Log.Info(LogTag, $"{voiceInfo.Lang} & {voiceInfo.Type} is supported");
+ return true;
+ }
+ }
+ return false;
+ }
+
+
+ internal void AddText(string txt, VoiceInfo voiceInfo)
+ {
+ if (voiceInfo.Lang == null || voiceInfo.Type == null)
+ {
+ Log.Error(LogTag, "VoiceInfo's value is null");
+ }
+ if (ttsHandle == null)
+ {
+ Log.Error(LogTag, $"ttsHandle is null");
+ return;
+ }
+ var temp = new UtteranceText();
+ temp.Text = txt;
+ temp.UttID = ttsHandle.AddText(txt, voiceInfo.Lang, (int)voiceInfo.Type, 0);
+ try
+ {
+ textList.Add(temp);
+ }
+ catch (Exception e)
+ {
+ Log.Error(LogTag, $"Error AddText" + e.Message);
+ }
+ }
+
+ internal void AddText(string txt, string lang)
+ {
+ if (ttsHandle == null)
+ {
+ Log.Error(LogTag, $"ttsHandle is null");
+ return;
+ }
+ var temp = new UtteranceText();
+ temp.Text = txt;
+ temp.UttID = ttsHandle.AddText(txt, lang, (int)voiceInfo.Type, 0);
+ try
+ {
+ textList.Add(temp);
+ }
+ catch (Exception e)
+ {
+ Log.Error(LogTag, $"Error AddText" + e.Message);
+ }
+ }
+
+ internal void Prepare(EventHandler playReadyCallback)
+ {
+ if (ttsHandle == null)
+ {
+ Log.Error(LogTag, $"ttsHandle is null");
+ return;
+ }
+ Log.Info(LogTag, "Prepare TTS");
+ isPrepared = true;
+ isAsync = false;
+ PlayReadyCallback = playReadyCallback;
+ Play(true);
+ }
+
+ internal bool PlayPreparedText()
+ {
+ if (byteList != null && byteList.Count != 0)
+ {
+ Log.Info(LogTag, "PlayPreparedText TTS");
+ currentAvatar?.AvatarAnimator?.PlayLipSync(byteList.ToArray(), sampleRate);
+ return true;
+ }
+ return false;
+ }
+
+ internal void Play(bool isPrepared = false)
+ {
+ if (ttsHandle == null)
+ {
+ Log.Error(LogTag, $"ttsHandle is null");
+ return;
+ }
+
+ this.isPrepared = isPrepared;
+ isAsync = false;
+ ttsHandle.Play();
+ }
+
+ internal void PlayAsync(EventHandler playReadyCallback)
+ {
+ if (ttsHandle == null)
+ {
+ Log.Error(LogTag, $"ttsHandle is null");
+ return;
+ }
+
+ isPrepared = false;
+ isAsync = true;
+ PlayReadyCallback = playReadyCallback;
+ ttsHandle.Play();
+ }
+
+ public void Pause()
+ {
+ if (ttsHandle == null)
+ {
+ Log.Error(LogTag, $"ttsHandle is null");
+ return;
+ }
+ ttsHandle.Pause();
+ }
+
+ internal void Stop()
+ {
+ if (ttsHandle == null)
+ {
+ Log.Error(LogTag, $"ttsHandle is null");
+ return;
+ }
+ ttsHandle.Stop();
+ currentAvatar?.AvatarAnimator?.StopLipSync();
+ }
+
+ private void InitTts()
+ {
+ try
+ {
+ ttsHandle = new TtsClient();
+
+ // Register Callbacks
+ ttsHandle.DefaultVoiceChanged += TtsDefaultVoiceChangedCallback;
+ ttsHandle.EngineChanged += TtsEngineChangedCallback;
+ ttsHandle.ErrorOccurred += TtsErrorOccuredCallback;
+ ttsHandle.StateChanged += TtsStateChangedCallback;
+ ttsHandle.UtteranceCompleted += TtsUtteranceCompletedCallback;
+ ttsHandle.UtteranceStarted += TtsUtteranceStartedCallback;
+
+ ttsHandle.SynthesizedPcm += TtsSyntheiszedPCM;
+ ttsHandle.PlayingMode = PlayingMode.ByClient;
+
+ ttsHandle.Prepare();
+
+ voiceInfo = new VoiceInfo
+ {
+ Lang = ttsHandle.DefaultVoice.Language,
+ Type = ttsHandle.DefaultVoice.VoiceType
+ };
+
+ textList = new List<UtteranceText>();
+ Log.Info(LogTag, voiceInfo.Lang + ", " + voiceInfo.Type.ToString());
+
+ }
+ catch (Exception e)
+ {
+ Log.Error(LogTag, "[ERROR] Fail to prepare Tts");
+ Log.Error(LogTag, e.Message);
+ }
+ }
+
+ internal void DeinitTts()
+ {
+ try
+ {
+ if (ttsHandle != null)
+ {
+ ttsHandle.Unprepare();
+
+ // Unregister Callbacks
+ ttsHandle.DefaultVoiceChanged -= TtsDefaultVoiceChangedCallback;
+ ttsHandle.EngineChanged -= TtsEngineChangedCallback;
+ ttsHandle.ErrorOccurred -= TtsErrorOccuredCallback;
+ ttsHandle.StateChanged -= TtsStateChangedCallback;
+ ttsHandle.UtteranceCompleted -= TtsUtteranceCompletedCallback;
+ ttsHandle.UtteranceStarted -= TtsUtteranceStartedCallback;
+
+ ttsHandle.Dispose();
+ ttsHandle = null;
+ }
+
+ if (textList != null)
+ {
+ textList.Clear();
+ textList = null;
+ }
+
+ if (byteList != null)
+ {
+ byteList.Clear();
+ byteList = null;
+ }
+ currentAvatar = null;
+ }
+ catch (Exception e)
+ {
+ Log.Error(LogTag, "[ERROR] Fail to unprepare Tts");
+ Log.Error(LogTag, e.Message);
+ }
+ }
+
+ private void TtsSyntheiszedPCM(object sender, SynthesizedPcmEventArgs e)
+ {
+
+ var dataSize = e.Data.Length;
+ var audio = new byte[dataSize];
+ sampleRate = e.SampleRate;
+
+ //Marshal.Copy(e.Data, audio, 0, dataSize);
+ switch (e.EventType) //START
+ {
+ case SynthesizedPcmEvent.Start://start
+ Tizen.Log.Info(LogTag, "------------------Start : " + e.UtteranceId);
+ Tizen.Log.Info(LogTag, "Output audio Size : " + dataSize);
+ Tizen.Log.Info(LogTag, "SampleRate" + e.SampleRate);
+ if (byteList == null)
+ {
+ byteList = new List<byte>();
+ }
+ if (recordedBuffer == null)
+ {
+ recordedBuffer = new byte[0];
+ }
+ byteList.Clear();
+
+ if (isAsync)
+ {
+ recordedBuffer = Array.Empty<byte>();
+
+ desiredBufferLength = (int)(e.SampleRate * desiredBufferDuration * audioBufferMultiflier);
+ audioTailLength = (int)(sampleRate * audioTailLengthFactor * audioBufferMultiflier);
+ audioTailBuffer = new byte[audioTailLength];
+ PlayReadyCallback?.Invoke(null, EventArgs.Empty);
+ InitAsyncBuffer();
+ lipSyncer.SampleRate = sampleRate;
+ }
+ break;
+ case SynthesizedPcmEvent.Continue://continue
+ if (isAsync)
+ {
+ recordedBuffer = recordedBuffer.Concat(e.Data).ToArray();
+ //PlayAsync
+ if (recordedBuffer.Length >= desiredBufferLength)
+ {
+ Tizen.Log.Error(LogTag, "Current recordbuffer length :" + recordedBuffer.Length);
+ UpdateBuffer(recordedBuffer, sampleRate);
+
+ Buffer.BlockCopy(recordedBuffer, recordedBuffer.Length - audioTailLength, audioTailBuffer, 0, audioTailLength);
+
+ recordedBuffer = Array.Empty<byte>();
+ recordedBuffer = recordedBuffer.Concat(audioTailBuffer).ToArray();
+ Array.Clear(audioTailBuffer, 0, audioTailLength);
+ }
+ }
+ else
+ {
+ byteList.AddRange(e.Data);
+ }
+ break;
+ case SynthesizedPcmEvent.Finish://finish
+ Tizen.Log.Info(LogTag, "------------------Finish : " + e.UtteranceId);
+ if (!isAsync)
+ {
+ if (!isPrepared)
+ {
+ //Play voice immediately
+ //PlayPreparedText();
+ }
+ else
+ {
+ //Notify finished state
+ Log.Info(LogTag, "Notify finished state");
+ PlayReadyCallback?.Invoke(null, EventArgs.Empty);
+ }
+ }
+ else
+ {
+ lipSyncer.SetFinishAsyncLip(true);
+ }
+ break;
+ case SynthesizedPcmEvent.Fail: //fail
+ break;
+
+ }
+ }
+
+ private void TtsUtteranceStartedCallback(object sender, UtteranceEventArgs e)
+ {
+ Log.Debug(LogTag, "Utterance start now (" + e.UtteranceId + ")");
+ }
+
+ private void TtsUtteranceCompletedCallback(object sender, UtteranceEventArgs e)
+ {
+ Log.Debug(LogTag, "Utterance complete (" + e.UtteranceId + ")");
+
+ foreach (UtteranceText item in textList)
+ {
+ if (item.UttID == e.UtteranceId)
+ {
+ textList.Remove(item);
+ Log.Debug(LogTag, "TextList Count (" + textList.Count.ToString() + ")");
+ break;
+ }
+ }
+ }
+
+ private void TtsStateChangedCallback(object sender, StateChangedEventArgs e)
+ {
+ Log.Debug(LogTag, "Current state is changed from (" + e.Previous + ") to (" + e.Current + ")");
+ }
+
+ private void TtsErrorOccuredCallback(object sender, ErrorOccurredEventArgs e)
+ {
+ Log.Error(LogTag, "Error is occured (" + e.ErrorMessage + ")");
+ }
+
+ private void TtsEngineChangedCallback(object sender, EngineChangedEventArgs e)
+ {
+ Log.Debug(LogTag, "Prefered engine is changed (" + e.EngineId + ") (" + e.VoiceType.Language + ")");
+ }
+
+ private void TtsDefaultVoiceChangedCallback(object sender, DefaultVoiceChangedEventArgs e)
+ {
+ Log.Debug(LogTag, "Default voice is changed from (" + e.Previous + ") to (" + e.Current + ")");
+ }
+
+ internal void InitAsyncBuffer()
+ {
+ if (!lipSyncer.IsAsyncInit)
+ {
+ audioLength = (int)(sampleRate * 0.16f * 2f);
+
+ lipSyncer.InitAsyncLipsync();
+ lipSyncer.IsAsyncInit = true;
+
+ lipSyncer.SetFinishAsyncLip(false);
+ isAsyncLipStarting = false;
+ }
+ }
+
+ internal void UpdateBuffer(byte[] recordBuffer, int sampleRate)
+ {
+ if (lipSyncer != null)
+ {
+ Log.Error(LogTag, "OnTTSBufferChanged");
+ lipSyncer.EnqueueAnimation(recordBuffer, sampleRate, audioLength);
+ if (!isAsyncLipStarting)
+ {
+ lipSyncer.StartAsyncLipPlayTimer();
+ isAsyncLipStarting = true;
+ }
+ }
+ else
+ {
+ Log.Error(LogTag, "avatarLipSyncer is null");
+ }
+ }
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2023 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+namespace Tizen.AIAvatar
+{
+ internal struct UtteranceText
+ {
+ internal string Text;
+ internal int UttID;
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2024 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using System;
+using System.ComponentModel;
+using Tizen.NUI;
+using Tizen.NUI.Scene3D;
+
+using static Tizen.AIAvatar.AIAvatar;
+
+namespace Tizen.AIAvatar
+{
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public abstract class AnimationModule
+ {
+ private AvatarMotionState currentMotionState = AvatarMotionState.Unavailable;
+
+ private readonly Object motionChangedLock = new Object();
+ private event EventHandler<AvatarMotionChangedEventArgs> motionChanged;
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public AnimationModule()
+ {
+
+ }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public AvatarMotionState CurrentMotionState
+ {
+ get
+ {
+ return currentMotionState;
+ }
+ set
+ {
+ if (currentMotionState == value)
+ {
+ return;
+ }
+ var preState = currentMotionState;
+ currentMotionState = value;
+ if (motionChanged != null)
+ {
+ motionChanged?.Invoke(this, new AvatarMotionChangedEventArgs(preState, currentMotionState));
+ }
+ }
+ }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public abstract void Init(Animation animation);
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public abstract void Play(IAnimationModuleData data);
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public abstract void Stop();
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public abstract void Pause();
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public abstract void Destroy();
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ protected void SetCurrentState(AvatarMotionState state)
+ {
+ CurrentMotionState = state;
+ }
+
+
+ /// <summary>
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public event EventHandler<AvatarMotionChangedEventArgs> MotionStateChanged
+ {
+ add
+ {
+ lock (motionChangedLock)
+ {
+ motionChanged += value;
+ }
+
+ }
+
+ remove
+ {
+ lock (motionChangedLock)
+ {
+ if (motionChanged == null)
+ {
+ Log.Error(LogTag, "Remove StateChanged Failed : motionChanged is null");
+ return;
+ }
+ motionChanged -= value;
+ }
+ }
+ }
+ }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public enum AnimationModuleType
+ {
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ EyeBlinker,
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ LipSyncer,
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ MotionBehavior,
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ JointTransformer,
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2024 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using System;
+using System.Collections.Generic;
+using System.ComponentModel;
+using Tizen.NUI;
+
+using static Tizen.AIAvatar.AIAvatar;
+
+namespace Tizen.AIAvatar
+{
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal class AsyncLipSyncer : LipSyncer
+ {
+ private readonly uint AsyncPlayTime = 160;
+ private Queue<Animation> lipAnimations;
+ private Queue<byte[]> lipAudios;
+ private Timer asyncVowelTimer;
+
+ private bool isAsyncInit = false;
+ private bool isFinishAsyncLip = false;
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal AsyncLipSyncer()
+ {
+
+ }
+
+ internal int SampleRate { get; set; }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal bool IsAsyncInit { get=>isAsyncInit; set=>isAsyncInit = value; }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal void SetFinishAsyncLip(bool finished)
+ {
+ isFinishAsyncLip = finished;
+ }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ protected new void OnLipAnimationFinished(object sender, EventArgs e)
+ {
+ if (!isAsyncInit)
+ {
+ CurrentMotionState = AvatarMotionState.Stopped;
+ }
+ else
+ {
+ Tizen.Log.Error(LogTag, "OnLipAnimationFinished---------------c");
+ //Async State
+ if (isFinishAsyncLip && lipAnimations.Count == 0)
+ {
+ Tizen.Log.Error(LogTag, "Finish vowel lip sync");
+
+ AudioPlayer.Stop();
+ CurrentMotionState = AvatarMotionState.Stopped;
+ DestroyVowelTimer();
+ isAsyncInit = false;
+ }
+ }
+ }
+
+ internal void InitAsyncLipsync()
+ {
+ if (lipAnimations == null)
+ {
+ lipAnimations = new Queue<Animation>();
+ }
+ else
+ {
+ lipAnimations.Clear();
+ }
+
+ if (lipAudios == null)
+ {
+ lipAudios = new Queue<byte[]>();
+ }
+ else
+ {
+ lipAudios.Clear();
+ }
+ }
+
+ internal void EnqueueAnimation(byte[] recordBuffer, int sampleRate, int audioLength)
+ {
+ var createdAni = CreateAsyncLipAnimation(recordBuffer, sampleRate);
+ if (createdAni != null)
+ {
+ lipAnimations.Enqueue(createdAni);
+ }
+
+ //Use Audio Full File
+ ///var createdAni = avatarLipSyncer.CreateLipAnimation(recordBuffer, sampleRate);
+ //lipAnimations.Enqueue(createdAni);
+
+ var currentAudioBuffer = new byte[audioLength];
+ Buffer.BlockCopy(recordBuffer, 0, currentAudioBuffer, 0, audioLength);
+
+ lipAudios.Enqueue(currentAudioBuffer);
+ }
+
+ internal bool PlayAsyncLip(int sampleRate, bool isFinishAsyncLip)
+ {
+ try
+ {
+ if (lipAudios.Count <= 0 && lipAnimations.Count <= 0)
+ {
+ Tizen.Log.Info(LogTag, "Return lipaudio 0");
+ if (isFinishAsyncLip)
+ {
+ Tizen.Log.Info(LogTag, "Finish Async lipsync");
+ return false;
+ }
+ return true;
+ }
+ Tizen.Log.Info(LogTag, "Async timer tick lipAudios : " + lipAudios.Count);
+ Tizen.Log.Info(LogTag, "Async timer tick lipAnimations : " + lipAnimations.Count);
+
+ var lipAnimation = lipAnimations.Dequeue();
+ lipAnimation.Finished += OnLipAnimationFinished;
+
+ ResetLipAnimation(lipAnimation);
+ PlayLipAnimation();
+ var audioBuffer = lipAudios.Dequeue();
+ AudioPlayer.PlayAsync(audioBuffer, sampleRate);
+ return true;
+
+ }
+ catch (Exception ex)
+ {
+ Log.Error(LogTag, $"---Log Tick : {ex.StackTrace}");
+
+ return false;
+ }
+ }
+
+ internal void StartAsyncLipPlayTimer()
+ {
+ if (asyncVowelTimer == null)
+ {
+ Tizen.Log.Info(LogTag, "Start Async");
+ asyncVowelTimer = new Timer(AsyncPlayTime);
+ asyncVowelTimer.Tick += OnAsyncVowelTick;
+ asyncVowelTimer.Start();
+ }
+ return;
+ }
+
+ private void DestroyVowelTimer()
+ {
+ if (asyncVowelTimer != null)
+ {
+
+ asyncVowelTimer.Tick -= OnAsyncVowelTick;
+ asyncVowelTimer.Stop();
+ asyncVowelTimer.Dispose();
+ asyncVowelTimer = null;
+ }
+ }
+
+ private bool OnAsyncVowelTick(object source, Tizen.NUI.Timer.TickEventArgs e)
+ {
+ return PlayAsyncLip(SampleRate, isFinishAsyncLip);
+ }
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2024 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+using Tizen.NUI;
+using Tizen.NUI.Scene3D;
+
+namespace Tizen.AIAvatar
+{
+ internal class AvatarMotions
+ {
+ private MotionData eyeMotionData;
+ private const int blinkDuration = 200;
+
+ internal AvatarMotions(AvatarProperties properties)
+ {
+ CreateEyeBlinkMotionData(properties);
+ }
+
+ internal MotionData EyeMotionData { get { return eyeMotionData; } }
+
+ private void CreateEyeBlinkMotionData(AvatarProperties avatarProperties)
+ {
+ var keyFrames = new KeyFrames();
+ keyFrames.Add(0.1f, 0.0f);
+ keyFrames.Add(0.5f, 1.0f);
+ keyFrames.Add(0.9f, 0.0f);
+
+ var headBlendShapeEyeLeft = new AvatarBlendShapeIndex(avatarProperties.NodeMapper, NodeType.HeadGeo, avatarProperties.BlendShapeMapper, BlendShapeType.EyeBlinkLeft);
+ var headBlendShapeEyeRight = new AvatarBlendShapeIndex(avatarProperties.NodeMapper, NodeType.HeadGeo, avatarProperties.BlendShapeMapper, BlendShapeType.EyeBlinkRight);
+ var eyelashBlendShapeEyeLeft = new AvatarBlendShapeIndex(avatarProperties.NodeMapper, NodeType.EyelashGeo, avatarProperties.BlendShapeMapper, BlendShapeType.EyeBlinkLeft);
+ var eyelashBlendShapeEyeRight = new AvatarBlendShapeIndex(avatarProperties.NodeMapper, NodeType.EyelashGeo, avatarProperties.BlendShapeMapper, BlendShapeType.EyeBlinkRight);
+
+ eyeMotionData = new MotionData(blinkDuration);
+ if (headBlendShapeEyeLeft != null)
+ {
+ eyeMotionData.Add(headBlendShapeEyeLeft, new MotionValue(keyFrames));
+ }
+ if (headBlendShapeEyeRight != null)
+ {
+ eyeMotionData.Add(headBlendShapeEyeRight, new MotionValue(keyFrames));
+ }
+ if (eyelashBlendShapeEyeLeft != null)
+ {
+ eyeMotionData.Add(eyelashBlendShapeEyeLeft, new MotionValue(keyFrames));
+ }
+ if (eyelashBlendShapeEyeRight != null)
+ {
+ eyeMotionData.Add(eyelashBlendShapeEyeRight, new MotionValue(keyFrames));
+ }
+ }
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2023 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using System;
+using System.Collections.Generic;
+using System.ComponentModel;
+using System.Reflection;
+using Tizen.NUI.Scene3D;
+using Tizen.NUI;
+
+using static Tizen.AIAvatar.AIAvatar;
+
+namespace Tizen.AIAvatar
+{
+ internal class BlendShapePlayer
+ {
+ private Dictionary<AnimationModuleType, AnimationModule> animationModules = new Dictionary<AnimationModuleType, AnimationModule>();
+
+ private EyeBlinker blinker;
+ private AsyncLipSyncer lipSyncer;
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public BlendShapePlayer()
+ {
+ blinker = new EyeBlinker();
+ lipSyncer = new AsyncLipSyncer();
+
+ animationModules.Add(AnimationModuleType.EyeBlinker, blinker);
+ animationModules.Add(AnimationModuleType.LipSyncer, lipSyncer);
+ }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public void SetBlinkAnimation(Animation blinkerAnimation)
+ {
+ animationModules[AnimationModuleType.EyeBlinker].Init(blinkerAnimation);
+ }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public void SetLipSyncAnimation(Animation lipsyncAnimation)
+ {
+ animationModules[AnimationModuleType.LipSyncer].Init(lipsyncAnimation);
+ }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public AnimationModule GetAnimationModule(AnimationModuleType type)
+ {
+ return animationModules[type];
+ }
+
+ /// <summary>
+ /// Start eye blink animation
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public void StartEyeBlink()
+ {
+ animationModules[AnimationModuleType.EyeBlinker]?.Play(null);
+ }
+
+ /// <summary>
+ /// Stop eye blink animation
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public void StopEyeBlink()
+ {
+ animationModules[AnimationModuleType.EyeBlinker]?.Stop();
+ }
+
+ /// <summary>
+ /// Play synchronization avatar lip based on the voice file(byte[])
+ /// </summary>
+ /// <param name="audio">byte array of voice</param>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public void PlayLipSync(byte[] audio)
+ {
+ if (animationModules[AnimationModuleType.LipSyncer] == null)
+ {
+ Log.Error(LogTag, $"error : avatarLipSync is null");
+ return;
+ }
+ var lipData = new LipSyncData();
+
+ lipData.AudioFile = new byte[audio.Length];
+ Buffer.BlockCopy(audio, 0, lipData.AudioFile, 0, audio.Length);
+ animationModules[AnimationModuleType.LipSyncer].Play(lipData);
+ }
+
+ /// <summary>
+ /// Play synchronization avatar lip based on the voice file(byte[])
+ /// </summary>
+ /// <param name="audio">byte array of voice</param>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public void PlayLipSync(byte[] audio, int sampleRate)
+ {
+ if (animationModules[AnimationModuleType.LipSyncer] == null)
+ {
+ Log.Error(LogTag, $"error : avatarLipSync is null");
+ return;
+ }
+ var lipData = new LipSyncData();
+
+ lipData.AudioFile = new byte[audio.Length];
+ Buffer.BlockCopy(audio, 0, lipData.AudioFile, 0, audio.Length);
+ lipData.SampleRate = sampleRate;
+ animationModules[AnimationModuleType.LipSyncer].Play(lipData);
+ }
+
+ /// <summary>
+ /// Pause lip synchronization
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public void PauseLipSync()
+ {
+ if (animationModules[AnimationModuleType.LipSyncer] == null)
+ {
+ Log.Error(LogTag, $"error : avatarLipSync is null");
+ return;
+ }
+ animationModules[AnimationModuleType.LipSyncer].Pause();
+ }
+
+ /// <summary>
+ /// Stop lip synchronization
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public void StopLipSync()
+ {
+ if (animationModules[AnimationModuleType.LipSyncer] == null)
+ {
+ Log.Error(LogTag, $"error : avatarLipSync is null");
+ return;
+ }
+ animationModules[AnimationModuleType.LipSyncer].Stop();
+ }
+
+ internal void DestroyAnimations()
+ {
+ foreach ( var animationModule in animationModules.Values )
+ {
+ animationModule.Destroy();
+ }
+ }
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2023 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using System;
+using System.ComponentModel;
+using Tizen.NUI;
+using Tizen.NUI.Scene3D;
+
+using static Tizen.AIAvatar.AIAvatar;
+
+namespace Tizen.AIAvatar
+{
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal class EyeBlinker : AnimationModule
+ {
+ private const int blinkIntervalMinimum = 800;
+ private const int blinkIntervalMaximum = 3000;
+ private Animation eyeAnimation;
+
+ private Timer blinkTimer;
+
+ private bool isPlaying = false;
+ private const int blinkDuration = 200;
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal EyeBlinker()
+ {
+
+ }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public override void Init(Animation eyeAnimation)
+ {
+ this.eyeAnimation = eyeAnimation;
+ }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public override void Play(IAnimationModuleData data)
+ {
+ //data
+ StartEyeBlink();
+ }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public override void Stop()
+ {
+ StopEyeBlink();
+ }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public override void Pause()
+ {
+ eyeAnimation?.Pause();
+ }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public override void Destroy()
+ {
+ DestroyAnimation();
+ }
+
+ private void StartEyeBlink()
+ {
+ DestroyBlinkTimer();
+
+ blinkTimer = new Timer(blinkDuration);
+ if (blinkTimer != null)
+ {
+ blinkTimer.Tick += OnBlinkTimer;
+ blinkTimer?.Start();
+ isPlaying = true;
+ }
+ }
+
+ private void PauseEyeBlink()
+ {
+ blinkTimer?.Stop();
+ isPlaying = false;
+ }
+
+ private void StopEyeBlink()
+ {
+ blinkTimer?.Stop();
+ isPlaying = false;
+ }
+
+ private void DestroyAnimation()
+ {
+ DestroyBlinkTimer();
+ if (eyeAnimation != null)
+ {
+ eyeAnimation.Stop();
+ eyeAnimation.Dispose();
+ eyeAnimation = null;
+ }
+ isPlaying = false;
+ }
+
+ private bool OnBlinkTimer(object source, Timer.TickEventArgs e)
+ {
+ if (eyeAnimation == null)
+ {
+ Log.Error(LogTag, "eye animation is not ready");
+ return false;
+ }
+ eyeAnimation?.Play();
+
+ var random = new Random();
+ var fortimerinterval = (uint)random.Next(blinkIntervalMinimum, blinkIntervalMaximum);
+ blinkTimer.Interval = fortimerinterval;
+ return true;
+ }
+
+ private void DestroyBlinkTimer()
+ {
+ if (blinkTimer != null)
+ {
+ blinkTimer.Tick -= OnBlinkTimer;
+ blinkTimer.Stop();
+ blinkTimer.Dispose();
+ blinkTimer = null;
+ }
+ }
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2023 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using System.ComponentModel;
+using Tizen.NUI;
+using Tizen.NUI.Scene3D;
+
+namespace Tizen.AIAvatar
+{
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal class JointTransformer
+ {
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public JointTransformer()
+ {
+ }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public void Init(Animation animation)
+ {
+ }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public void Play(IAnimationModuleData data)
+ {
+ }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public void Pause()
+ {
+ }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public void Stop()
+ {
+ }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public void Destroy()
+ {
+ }
+
+ private void SetJointMotion(AvatarProperties properties, JointType jointType, MotionTransformIndex.TransformTypes type, Rotation rotation)
+ {
+ var motionTransform = new AvatarJointTransformIndex(properties.JointMapper, jointType, type);
+ var motionData = new MotionData();
+ motionData.Add(
+ motionTransform,
+ new MotionValue()
+ {
+ //TODO : Tizen_7.0에 pitch, yaw, roll patch 추가하기
+ //PropertyValue = new PropertyValue(new Rotation(new Radian(pitch), new Radian(yaw), new Radian(roll))),
+ }
+ );
+ //avatar.SetMotionData(motionData);
+ }
+
+ private void SetJointMotion(string keyValue, float pitch, float yaw, float roll)
+ {
+ var motionData = new MotionData();
+ motionData.Add(
+ new MotionTransformIndex()
+ {
+ ModelNodeId = new PropertyKey(keyValue),
+ TransformType = MotionTransformIndex.TransformTypes.Orientation,
+ },
+ new MotionValue()
+ {
+ //TODO : Tizen_7.0에 pitch, yaw, roll patch 추가하기
+ //PropertyValue = new PropertyValue(new Rotation(new Radian(pitch), new Radian(yaw), new Radian(roll))),
+ }
+ );
+ //avatar.SetMotionData(motionData);
+ }
+
+ private void SetJointMotion(string keyValue, MotionTransformIndex.TransformTypes type, Rotation rotation)
+ {
+ var motionData = new MotionData();
+ motionData.Add(
+ new MotionTransformIndex()
+ {
+ ModelNodeId = new PropertyKey(keyValue),
+ TransformType = type,
+ },
+ new MotionValue()
+ {
+ PropertyValue = new PropertyValue(rotation),
+ }
+ );
+ //avatar.SetMotionData(motionData);
+ }
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2023 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using System;
+using System.Collections.Generic;
+using System.ComponentModel;
+using Tizen.NUI;
+using Tizen.NUI.Scene3D;
+
+using static Tizen.AIAvatar.AIAvatar;
+
+namespace Tizen.AIAvatar
+{
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal class LipSyncer : AnimationModule
+ {
+ private Avatar avatar;
+ private Animation lipAnimation = null;
+
+ //private VowelConverter vowelConverter;
+ private AudioPlayer audioPlayer;
+
+ //Mic
+ private Queue<string[]> vowelPools = new Queue<string[]>();
+ private string prevVowel = "sil";
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal LipSyncer()
+ {
+ }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal AudioPlayer AudioPlayer { get { return audioPlayer; } }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public override void Init(Animation lipAnimation)
+ {
+ this.lipAnimation = lipAnimation;
+ //vowelConverter = new VowelConverter();
+ audioPlayer = new AudioPlayer();
+
+ CurrentMotionState = AvatarMotionState.Ready;
+ }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public override void Play(IAnimationModuleData data)
+ {
+ if (data is LipSyncData lipSyncData)
+ {
+ if (lipSyncData.AudioFile != null)
+ {
+ PlayLipSync(lipSyncData.AudioFile, lipSyncData.SampleRate);
+ }
+ }
+ }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public override void Stop()
+ {
+ StopLipSync();
+ }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public override void Pause()
+ {
+ PauseLipSync();
+ }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public override void Destroy()
+ {
+ DestroyLipAnimation();
+ }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ protected Animation CreateAsyncLipAnimation(byte[] buffer, int sampleRate)
+ {
+ EnqueueVowels(buffer, false);
+ return CreateLipAnimationByVowelsQueue(sampleRate);
+ }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ //TODO : lipAnimation 자체를 Animator안에서 생성하고 관리될 수 있게 수정.
+ protected void ResetLipAnimation(Animation lipAnimation)
+ {
+ DestroyLipAnimation();
+ this.lipAnimation = lipAnimation;
+ if (this.lipAnimation != null)
+ {
+ this.lipAnimation.Finished += OnLipAnimationFinished;
+ }
+ }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ protected void PlayLipAnimation()
+ {
+ if (lipAnimation == null)
+ {
+ Log.Error(LogTag, "Current Lip Animation is null");
+ }
+ lipAnimation?.Play();
+ }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ protected void OnLipAnimationFinished(object sender, EventArgs e)
+ {
+ CurrentMotionState = AvatarMotionState.Stopped;
+ }
+
+ private void PlayLipSync(byte[] audio)
+ {
+
+ Tizen.Log.Error("MYLOG", "Play Lip sync : " + audio.Length);
+ if(audio == null)
+ {
+ Tizen.Log.Error("MYLOG", "audi data is null");
+ }
+
+ DestroyLipAnimation();
+ var lipAnimation = CreateLipAnimation(audio, CurrentAudioOptions.SampleRate);
+ if (lipAnimation != null)
+ {
+ Tizen.Log.Error("MYLOG", "lipAnimation is not null");
+ ResetLipAnimation(lipAnimation);
+ PlayLipAnimation();
+ }
+ else
+ {
+
+ Tizen.Log.Error("MYLOG", "lipAnimation is null");
+ }
+ audioPlayer.Play(audio);
+ CurrentMotionState = AvatarMotionState.Playing;
+ }
+
+ private void PlayLipSync(byte[] audio, int sampleRate)
+ {
+ DestroyLipAnimation();
+ var lipAnimation = CreateLipAnimation(audio, sampleRate);
+ if (lipAnimation != null)
+ {
+ ResetLipAnimation(lipAnimation);
+ PlayLipAnimation();
+ }
+ audioPlayer.Play(audio, sampleRate);
+ CurrentMotionState = AvatarMotionState.Playing;
+ }
+
+ private void PlayLipSync(string path)
+ {
+ var bytes = Utils.ReadAllBytes(path);
+ if (bytes != null)
+ {
+ PlayLipSync(bytes);
+ }
+ else
+ {
+ Log.Error(LogTag, "Failed to load audio file");
+ }
+ }
+
+ private void PauseLipSync()
+ {
+ PauseLipAnimation();
+ audioPlayer.Pause();
+ CurrentMotionState = AvatarMotionState.Paused;
+ }
+
+ private void StopLipSync()
+ {
+ StopLipAnimation();
+ audioPlayer.Stop();
+ CurrentMotionState = AvatarMotionState.Stopped;
+ }
+
+ private void PauseLipAnimation()
+ {
+ if (lipAnimation != null)
+ {
+ lipAnimation?.Pause();
+ }
+ else
+ {
+ Log.Error(LogTag, "Current Lip Animation is null");
+ }
+ }
+
+ private void StopLipAnimation()
+ {
+ if (lipAnimation != null)
+ {
+ DestroyLipAnimation();
+
+ var lipAnimation = ResetLipAnimation();
+ if (lipAnimation != null)
+ {
+ ResetLipAnimation(lipAnimation);
+ PlayLipAnimation();
+ }
+ else
+ {
+ Log.Error(LogTag, "Current Lip Animation is null");
+ }
+ }
+ else
+ {
+ Log.Error(LogTag, "Current Lip Animation is null");
+ }
+ }
+
+ private void DestroyLipAnimation()
+ {
+ if (lipAnimation != null)
+ {
+ lipAnimation.Stop();
+ lipAnimation.Dispose();
+ lipAnimation = null;
+ }
+ }
+
+ private Animation CreateLipAnimation(byte[] array, int sampleRate)
+ {
+ Animation lipKeyframes = null;// CreateKeyFrame(array, sampleRate);
+ if (lipKeyframes != null)
+ {
+
+ return null;// CreateLipAnimation(lipKeyframes);
+ }
+ return null;
+ }
+
+ private void EnqueueVowels(byte[] buffer, bool deleteLast = false)
+ {
+ var vowels = PredictVowels(buffer);
+ if (vowels != null)
+ {
+ vowelPools.Enqueue(vowels);
+ if (deleteLast)
+ {
+ var vowelList = new List<string>(vowels);
+ vowelList.RemoveAt(vowelList.Count - 1);
+ vowels = vowelList.ToArray();
+ }
+ }
+ }
+
+ private Animation CreateLipAnimationByVowelsQueue(int sampleRate = 0)
+ {
+ if (sampleRate == 0)
+ {
+ sampleRate = CurrentAudioOptions.SampleRate;
+ }
+ if (vowelPools.Count > 0)
+ {
+ AttachPreviousVowel(vowelPools.Dequeue(), out var newVowels);
+ Log.Info(LogTag, $"vowelRecognition: {String.Join(", ", newVowels)}");
+
+ //var lipKeyFrames = vowelConverter?.CreateKeyFrames(newVowels, sampleRate, true);
+ return null;// CreateLipAnimation(lipKeyFrames, true);
+ }
+ return null;
+ }
+
+ private Animation ResetLipAnimation()
+ {
+ vowelPools.Clear();
+ var newVowels = new string[1];
+ newVowels[0] = prevVowel = "sil";
+ vowelPools.Enqueue(newVowels);
+ return CreateLipAnimationByVowelsQueue();
+ }
+
+ private string[] PredictVowels(byte[] audioData)
+ {
+ string[] vowels = null;// vowelConverter?.PredictVowels(audioData);
+ return vowels;
+ }
+
+ private void AttachPreviousVowel(in string[] vowels, out string[] newVowels)
+ {
+ newVowels = new string[vowels.Length + 1];
+ newVowels[0] = prevVowel;
+ Array.Copy(vowels, 0, newVowels, 1, vowels.Length);
+ prevVowel = vowels[vowels.Length - 1];
+ }
+
+ /*
+ private AnimationKeyFrame CreateKeyFrame(byte[] audio, int sampleRate)
+ {
+ var keyFrames = vowelConverter?.CreateKeyFrames(audio, sampleRate);
+ if (keyFrames == null)
+ {
+ Log.Error(LogTag, $"Failed to initialize KeyFrames");
+ }
+
+ return keyFrames;
+ }
+
+ private Animation CreateLipAnimation(AnimationKeyFrame animationKeyFrames, bool isMic = false)
+ {
+ var animationTime = (int)(animationKeyFrames.AnimationTime * 1000f);
+ var lipAnimation = new Animation(animationTime);
+
+ var motionData = new MotionData(animationTime);
+ for (var i = 0; i < animationKeyFrames.NodeNames.Length; i++)
+ {
+ var nodeName = animationKeyFrames.NodeNames[i];
+ for (var j = 0; j < animationKeyFrames.BlendShapeCounts[i]; j++)
+ {
+ var blendShapeIndex = new BlendShapeIndex(new PropertyKey(nodeName), new PropertyKey(j));
+ var keyFrameList = animationKeyFrames.GetKeyFrames(nodeName, j);
+ if (keyFrameList.Count == 0)
+ {
+ continue;
+ }
+
+ var keyFrames = new KeyFrames();
+ CreateKeyTimeFrames(ref keyFrames, keyFrameList, isMic);
+
+ motionData.Add(blendShapeIndex, new MotionValue(keyFrames));
+ lipAnimation = avatar.GenerateMotionDataAnimation(motionData);
+ }
+ }
+ return lipAnimation;
+ }
+ private KeyFrames CreateKeyTimeFrames(ref KeyFrames keyFrames, List<KeyFrame> keyFrameList, bool isMic = false)
+ {
+ foreach (var key in keyFrameList)
+ {
+ keyFrames.Add(key.time, key.value);
+ }
+ if (!isMic) keyFrames.Add(1.0f, 0.0f);
+
+ return keyFrames;
+ }
+ */
+
+ internal void OnRecordBufferChanged(byte[] recordBuffer, int sampleRate)
+ {
+ EnqueueVowels(recordBuffer);
+ }
+
+ internal void OnRecodingTick()
+ {
+ var lipAnimation = CreateLipAnimationByVowelsQueue();
+ if (lipAnimation != null)
+ {
+ ResetLipAnimation(lipAnimation);
+ PlayLipAnimation();
+ }
+ else
+ {
+ Log.Error(LogTag, "Current Lip Animation is null");
+ }
+ }
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2024 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using System.ComponentModel;
+using Tizen.NUI;
+using Tizen.NUI.Scene3D;
+
+namespace Tizen.AIAvatar
+{
+ internal class MotionPlayer
+ {
+ private Animation motionAnimation;
+ internal Animation MotionAnimation { get => motionAnimation; private set => motionAnimation = value; }
+
+ internal MotionPlayer()
+ {
+
+ }
+
+ /// <summary>
+ /// Play avatar animation by AnimationInfo
+ /// </summary>
+ /// <param name="index">index of default avatar animations</param>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal void PlayAnimation(Animation motionAnimation, int duration = 3000, bool isLooping = false, int loopCount = 1)
+ {
+ ResetAnimations();
+
+ MotionAnimation = this.motionAnimation;
+ MotionAnimation.Play();
+ }
+
+ /// <summary>
+ /// Pause avatar animation
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal void PauseMotionAnimation()
+ {
+ MotionAnimation?.Pause();
+ }
+
+ /// <summary>
+ /// Stop avatar animation
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal void StopMotionAnimation()
+ {
+ MotionAnimation?.Stop();
+ }
+
+ private void ResetAnimations()
+ {
+ if (MotionAnimation != null)
+ {
+ MotionAnimation.Stop();
+ MotionAnimation.Dispose();
+ MotionAnimation = null;
+ }
+ }
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2024 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using System.ComponentModel;
+
+namespace Tizen.AIAvatar
+{
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public interface IAnimationModuleData
+ {
+ }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public enum MotionDataType
+ {
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ AnimationInfo,
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ MotionData
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2024 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using System;
+using System.Collections.Generic;
+using System.ComponentModel;
+using System.Text;
+using Tizen.AIAvatar;
+
+namespace Tizen.AIAvatar
+{
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public class LipSyncData : IAnimationModuleData
+ {
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public byte[] AudioFile { get; set; }
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public int SampleRate { get; set; }
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2024 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using System.ComponentModel;
+using Tizen.NUI.Scene3D;
+
+namespace Tizen.AIAvatar
+{
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public class MotionBehaviorData : IAnimationModuleData
+ {
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public MotionDataType Type { get; set; } = MotionDataType.AnimationInfo;
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public AnimationInfo AnimationInfo { get; set; }
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public MotionData MotionData { get; set; }
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public int Duration { get; set; } = 3000;
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public bool IsLooping { get; set; } = false;
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public int LoopCount { get; set; } = 1;
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2023 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using System;
+using System.ComponentModel;
+
+namespace Tizen.AIAvatar
+{
+ /// <summary>
+ ///
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public class AvatarMotionChangedEventArgs : EventArgs
+ {
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public AvatarMotionChangedEventArgs(AvatarMotionState previous, AvatarMotionState current)
+ {
+ Previous = previous;
+ Current = current;
+ }
+
+ /// <summary>
+ /// The previous state.
+ /// </summary>
+ /// <since_tizen> 3 </since_tizen>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public AvatarMotionState Previous
+ {
+ get;
+ internal set;
+ }
+
+ /// <summary>
+ /// The current state.
+ /// </summary>
+ /// <since_tizen> 3 </since_tizen>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public AvatarMotionState Current
+ {
+ get;
+ internal set;
+ }
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2023 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using System.ComponentModel;
+
+namespace Tizen.AIAvatar
+{
+ /// <summary>
+ /// Enumeration for the states.
+ /// </summary>
+ /// <since_tizen> 3 </since_tizen>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public enum AvatarMotionState
+ {
+ /// <summary>
+ /// Created state.
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ Failed = -1,
+
+ /// <summary>
+ /// Ready state.
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ Ready = 0,
+
+ /// <summary>
+ /// Playing state.
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ Playing = 3,
+
+ /// <summary>
+ /// Paused state.
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ Paused = 4,
+
+ /// <summary>
+ /// Stopped state.
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ Stopped = 5,
+
+ /// <summary>
+ /// Unavailable state.
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ Unavailable
+ };
+}
--- /dev/null
+/*
+ * Copyright(c) 2023 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using System.ComponentModel;
+using Tizen.Multimedia;
+
+namespace Tizen.AIAvatar
+{
+ /// <summary>
+ /// Provides the ability to audio
+ /// </summary>
+ // This will be public opened after ACR done. (Before ACR, need to be hidden as Inhouse API)
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public class AudioOptions
+ {
+ private int sampleRate;
+ private AudioChannel channel;
+ private AudioSampleType sampleType;
+
+ /// <summary>
+ /// Initializes a new instance of the AudioOptions class with the specified sample rate, channel, and sampleType.
+ /// </summary>
+ /// <param name="sampleRate">the audio sample rate (8000 ~ 192000Hz)</param>
+ /// <param name="channel">the audio channel type.</param>
+ /// <param name="sampleType">the audio sample type.</param>
+ // This will be public opened after ACR done. (Before ACR, need to be hidden as Inhouse API)
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public AudioOptions(int sampleRate, AudioChannel channel, AudioSampleType sampleType)
+ {
+ this.sampleRate = sampleRate;
+ this.channel = channel;
+ this.sampleType = sampleType;
+ }
+
+ /// <summary>
+ /// The audio sample rate (8000 ~ 192000Hz)
+ /// </summary>
+ // This will be public opened after ACR done. (Before ACR, need to be hidden as Inhouse API)
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public int SampleRate { get => sampleRate; set => sampleRate = value; }
+
+ /// <summary>
+ /// The audio channel type
+ /// </summary>
+ // This will be public opened after ACR done. (Before ACR, need to be hidden as Inhouse API)
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public AudioChannel Channel { get => channel; set => channel = value; }
+
+ /// <summary>
+ /// The audio sample type
+ /// </summary>
+ // This will be public opened after ACR done. (Before ACR, need to be hidden as Inhouse API)
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public AudioSampleType SampleType { get => sampleType; set => sampleType = value; }
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2023 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using System.ComponentModel;
+using Tizen.NUI.Scene3D;
+using Tizen.NUI;
+
+using static Tizen.AIAvatar.AIAvatar;
+
+namespace Tizen.AIAvatar
+{
+ /// <summary>
+ /// Avatar is a Class to show 3D avatar objects.
+ /// It is subclass of Model s.t. we can control Avatar like models animation easly.
+ /// For example,
+ ///
+ /// Avatar supports AR Emoji.
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public class Avatar : Model
+ {
+ private AvatarProperties avatarProperties = new DefaultAvatarProperties();
+ private AvatarMotions avatarMotions;
+
+ private BlendShapePlayer avatarAnimator;
+ private MotionPlayer motionPlayer;
+ private JointTransformer jointTransformer;
+
+ /// <summary>
+ /// Create an initialized AvatarModel.
+ /// </summary>
+ // This will be public opened after ACR done. (Before ACR, need to be hidden as Inhouse API)
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public Avatar() : base()
+ {
+ InitAvatar();
+ }
+
+ /// <summary>
+ /// Create an initialized AREmojiDefaultAvatar.
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public Avatar(AvatarInfo avatarInfo) : base(avatarInfo.ResourcePath)
+ {
+ InitAvatar();
+ }
+
+ /// <summary>
+ /// Create an initialized Avatar.
+ /// </summary>
+ /// <param name="avatarlUrl">avatar file url.(e.g. glTF, and DLI).</param>
+ /// <param name="resourceDirectoryUrl"> The url to derectory containing resources: binary, image etc.</param>
+ /// <remarks>
+ /// If resourceDirectoryUrl is empty, the parent directory url of avatarUrl is used for resource url.
+ ///
+ /// http://tizen.org/privilege/mediastorage for local files in media storage.
+ /// http://tizen.org/privilege/externalstorage for local files in external storage.
+ /// </remarks>
+ // This will be public opened after ACR done. (Before ACR, need to be hidden as Inhouse API)
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public Avatar(string avatarUrl, string resourceDirectoryUrl = "") : base(avatarUrl, resourceDirectoryUrl)
+ {
+ InitAvatar();
+ }
+
+ /// <summary>
+ /// Copy constructor.
+ /// </summary>
+ /// <param name="avatar">Source object to copy.</param>
+ // This will be public opened after ACR done. (Before ACR, need to be hidden as Inhouse API)
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public Avatar(Avatar avatar) : base(avatar)
+ {
+ InitAvatar();
+ }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public AvatarProperties AvatarProperties
+ {
+ get => avatarProperties;
+ set => avatarProperties = value;
+ }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal BlendShapePlayer AvatarAnimator
+ {
+ get => avatarAnimator;
+ set => avatarAnimator = value;
+ }
+
+
+ /// <summary>
+ /// Play avatar animation by AnimationInfo
+ /// </summary>
+ /// <param name="index">index of default avatar animations</param>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ private void PlayAnimation(AnimationInfo animationInfo, int duration = 3000, bool isLooping = false, int loopCount = 1)
+ {
+ var motionAnimation = GenerateMotionDataAnimation(animationInfo.MotionData);
+ motionAnimation.Duration = duration;
+ motionAnimation.Looping = isLooping;
+ motionAnimation.LoopCount = loopCount;
+ motionAnimation.BlendPoint = 0.2f;
+ motionPlayer.PlayAnimation(motionAnimation);
+ }
+
+ /// <summary>
+ /// Play avatar animation by MotionData
+ /// </summary>
+ /// <param name="index">index of default avatar animations</param>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ private void PlayAnimation(MotionData motionData, int duration = 3000, bool isLooping = false, int loopCount = 1)
+ {
+
+ var motionAnimation = GenerateMotionDataAnimation(motionData);
+ motionAnimation.Duration = duration;
+ motionAnimation.Looping = isLooping;
+ motionAnimation.LoopCount = loopCount;
+ motionAnimation.BlendPoint = 0.2f;
+
+ motionPlayer.PlayAnimation(motionAnimation);
+ }
+
+ private void PlayAnimation(int index, int duration = 3000, bool isLooping = false, int loopCount = 1)
+ {
+ //TODO by index
+ //var motionAnimation = GenerateMotionDataAnimation(animationInfoList[index].MotionData);
+ /*motionAnimation.Duration = duration;
+ motionAnimation.Looping = isLooping;
+ motionAnimation.LoopCount = loopCount;
+ motionAnimation.BlendPoint = 0.2f;
+
+ motionPlayer.PlayAnimation(motionAnimation);*/
+ }
+
+ /// <summary>
+ /// Pause avatar animation
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ private void PauseMotionAnimation()
+ {
+ motionPlayer.PauseMotionAnimation();
+ }
+
+ /// <summary>
+ /// Stop avatar animation
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ private void StopMotionAnimation()
+ {
+ motionPlayer?.StopMotionAnimation();
+ }
+
+
+ private void InitAvatar()
+ {
+ avatarMotions = new AvatarMotions(avatarProperties);
+
+ AvatarAnimator = new BlendShapePlayer();
+
+ avatarProperties.AvatarPropertiesChanged += (s, e) =>
+ {
+ var eyeAnimation = CreateMotionAnimation(avatarMotions.EyeMotionData);
+ if (eyeAnimation != null)
+ {
+ avatarAnimator.SetBlinkAnimation(eyeAnimation);
+ }
+ };
+
+ ResourcesLoaded += (s, e) =>
+ {
+ var eyeAnimation = CreateMotionAnimation(avatarMotions.EyeMotionData);
+ if (eyeAnimation != null)
+ {
+ avatarAnimator.SetBlinkAnimation(eyeAnimation);
+ }
+ };
+ }
+
+ private Animation CreateMotionAnimation(MotionData data)
+ {
+ return GenerateMotionDataAnimation(data);
+ }
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2023 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using System.ComponentModel;
+
+namespace Tizen.AIAvatar
+{
+ /// <summary>
+ /// The type of predefined blendshape. We can customize each type name by "TODO_mapper"
+ /// TODO : Explain me
+ /// TODO : Need to check each joint exist in AR Emoji
+ /// Note : This is temperal name of joints.
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal enum BlendShapeType
+ {
+ #region Left Eyes
+ /// <summary>
+ /// EyeBlinkLeft blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ EyeBlinkLeft = 0,
+
+ /// <summary>
+ /// EyeSquintLeft blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ EyeSquintLeft,
+
+ /// <summary>
+ /// EyeDownLeft blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ EyeDownLeft,
+
+ /// <summary>
+ /// EyeInLeft blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ EyeInLeft,
+
+ /// <summary>
+ /// EyeOpenLeft blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ EyeOpenLeft,
+
+ /// <summary>
+ /// EyeOutLeft blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ EyeOutLeft,
+
+ /// <summary>
+ /// EyeUpLeft blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ EyeUpLeft,
+ #endregion
+
+ #region Right Eyes
+ /// <summary>
+ /// EyeBlinkRight blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ EyeBlinkRight,
+
+ /// <summary>
+ /// EyeSquintRight blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ EyeSquintRight,
+
+ /// <summary>
+ /// EyeDownRight blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ EyeDownRight,
+
+ /// <summary>
+ /// EyeInRight blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ EyeInRight,
+
+ /// <summary>
+ /// EyeOpenRight blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ EyeOpenRight,
+
+ /// <summary>
+ /// EyeOutRight blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ EyeOutRight,
+
+ /// <summary>
+ /// EyeUpRight blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ EyeUpRight,
+ #endregion
+
+ #region Mouth and Jaw
+ /// <summary>
+ /// JawForward blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ JawForward,
+
+ /// <summary>
+ /// JawOpen blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ JawOpen,
+
+ /// <summary>
+ /// JawChew blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ JawChew,
+
+ /// <summary>
+ /// JawLeft blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ JawLeft,
+
+ /// <summary>
+ /// JawRight blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ JawRight,
+
+ /// <summary>
+ /// MouthLeft blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ MouthLeft,
+
+ /// <summary>
+ /// MouthFrownLeft blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ MouthFrownLeft,
+
+ /// <summary>
+ /// MouthSmileLeft blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ MouthSmileLeft,
+
+ /// <summary>
+ /// MouthDimpleLeft blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ MouthDimpleLeft,
+
+ /// <summary>
+ /// MouthRight blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ MouthRight,
+
+ /// <summary>
+ /// MouthFrownRight blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ MouthFrownRight,
+
+ /// <summary>
+ /// MouthSmileRight blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ MouthSmileRight,
+
+ /// <summary>
+ /// MouthDimpleRight blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ MouthDimpleRight,
+ #endregion
+
+ #region Lips
+ /// <summary>
+ /// LipsStretchLeft blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ LipsStretchLeft,
+
+ /// <summary>
+ /// LipsStretchRight blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ LipsStretchRight,
+
+ /// <summary>
+ /// LipsUpperClose blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ LipsUpperClose,
+
+ /// <summary>
+ /// LipsLowerClose blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ LipsLowerClose,
+
+ /// <summary>
+ /// LipsUpperUp blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ LipsUpperUp,
+
+ /// <summary>
+ /// LipsLowerDown blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ LipsLowerDown,
+
+ /// <summary>
+ /// LipsUpperOpen blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ LipsUpperOpen,
+
+ /// <summary>
+ /// LipsLowerOpen blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ LipsLowerOpen,
+
+ /// <summary>
+ /// LipsFunnel blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ LipsFunnel,
+
+ /// <summary>
+ /// LipsPucker blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ LipsPucker,
+ #endregion
+
+ #region Eyebrows, Cheeks, and Chin
+ /// <summary>
+ /// BrowDownLeft blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ BrowDownLeft,
+
+ /// <summary>
+ /// BrowDownRight blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ BrowDownRight,
+
+ /// <summary>
+ /// BrowUpCenter blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ BrowUpCenter,
+
+ /// <summary>
+ /// BrowUpLeft blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ BrowUpLeft,
+
+ /// <summary>
+ /// BrowUpRight blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ BrowUpRight,
+
+ /// <summary>
+ /// CheekSquintLeft blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ CheekSquintLeft,
+
+ /// <summary>
+ /// CheekSquintRight blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ CheekSquintRight,
+
+ /// <summary>
+ /// ChinLowerRaise blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ ChinLowerRaise,
+
+ /// <summary>
+ /// ChinUpperRaise blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ ChinUpperRaise,
+ #endregion
+
+ #region Tongue
+ /// <summary>
+ /// TongueOut blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ TongueOut,
+
+ /// <summary>
+ /// TongueUp blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ TongueUp,
+
+ /// <summary>
+ /// TongueDown blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ TongueDown,
+
+ /// <summary>
+ /// TongueLeft blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ TongueLeft,
+
+ /// <summary>
+ /// TongueRight blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ TongueRight,
+ #endregion
+
+ #region ETC
+ /// <summary>
+ /// Sneer blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ Sneer,
+
+ /// <summary>
+ /// Puff blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ Puff,
+
+ /// <summary>
+ /// PuffLeft blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ PuffLeft,
+
+ /// <summary>
+ /// PuffRight blendshape
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ PuffRight,
+ #endregion
+ /// <summary>
+ /// Max value of default blendshape. It will be used when we determine the motion index is default or custom.
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ DefaultBlendShapeMax,
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2023 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using System.ComponentModel;
+
+namespace Tizen.AIAvatar
+{
+ /// <summary>
+ /// The type of predefined skeletal joint. We can customize each type name by "TODO_mapper"
+ /// TODO : Explain me
+ /// TODO : Need to check each joint exist in AR Emoji
+ /// Note : This is temperal name of joints.
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal enum JointType
+ {
+ #region Head
+ /// <summary>
+ /// Head joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ Head = 0,
+
+ /// <summary>
+ /// Neck joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ Neck,
+
+ /// <summary>
+ /// EyeLeft joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ EyeLeft,
+
+ /// <summary>
+ /// EyeRight joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ EyeRight,
+ #endregion
+
+ #region Left Upper Body
+ /// <summary>
+ /// ShoulderLeft joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ ShoulderLeft,
+
+ /// <summary>
+ /// ElbowLeft joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ ElbowLeft,
+
+ /// <summary>
+ /// WristLeft joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ WristLeft,
+ #endregion
+
+ #region Right Upper Body
+ /// <summary>
+ /// ShoulderRight joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ ShoulderRight,
+
+ /// <summary>
+ /// ElbowRight joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ ElbowRight,
+
+ /// <summary>
+ /// WristRight joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ WristRight,
+ #endregion
+
+ #region Left Lower Body
+ /// <summary>
+ /// HipLeft joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ HipLeft,
+
+ /// <summary>
+ /// KneeLeft joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ KneeLeft,
+
+ /// <summary>
+ /// AnkleLeft joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ AnkleLeft,
+
+ /// <summary>
+ /// ForeFootLeft joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ ForeFootLeft,
+ #endregion
+
+ #region Right Lower Body
+ /// <summary>
+ /// HipRight joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ HipRight,
+
+ /// <summary>
+ /// KneeRight joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ KneeRight,
+
+ /// <summary>
+ /// AnkleRight joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ AnkleRight,
+
+ /// <summary>
+ /// ForeFootRight joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ ForeFootRight,
+ #endregion
+
+ #region Left Hand Finger
+ /// <summary>
+ /// FingerThumb1Left joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ FingerThumb1Left,
+
+ /// <summary>
+ /// FingerThumb2Left joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ FingerThumb2Left,
+
+ /// <summary>
+ /// FingerThumb3Left joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ FingerThumb3Left,
+
+ /// <summary>
+ /// FingerThumb4Left joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ FingerThumb4Left,
+
+ /// <summary>
+ /// FingerIndex1Left joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ FingerIndex1Left,
+
+ /// <summary>
+ /// FingerIndex2Left joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ FingerIndex2Left,
+
+ /// <summary>
+ /// FingerIndex3Left joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ FingerIndex3Left,
+
+ /// <summary>
+ /// FingerIndex4Left joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ FingerIndex4Left,
+
+ /// <summary>
+ /// FingerMiddle1Left joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ FingerMiddle1Left,
+
+ /// <summary>
+ /// FingerMiddle2Left joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ FingerMiddle2Left,
+
+ /// <summary>
+ /// FingerMiddle3Left joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ FingerMiddle3Left,
+
+ /// <summary>
+ /// FingerMiddle4Left joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ FingerMiddle4Left,
+
+ /// <summary>
+ /// FingerRing1Left joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ FingerRing1Left,
+
+ /// <summary>
+ /// FingerRing2Left joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ FingerRing2Left,
+
+ /// <summary>
+ /// FingerRing3Left joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ FingerRing3Left,
+
+ /// <summary>
+ /// FingerRing4Left joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ FingerRing4Left,
+
+ /// <summary>
+ /// FingerPinky1Left joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ FingerPinky1Left,
+
+ /// <summary>
+ /// FingerPinky2Left joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ FingerPinky2Left,
+
+ /// <summary>
+ /// FingerPinky3Left joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ FingerPinky3Left,
+
+ /// <summary>
+ /// FingerPinky4Left joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ FingerPinky4Left,
+ #endregion
+
+ #region Right Hand Finger
+ /// <summary>
+ /// FingerThumb1Right joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ FingerThumb1Right,
+
+ /// <summary>
+ /// FingerThumb2Right joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ FingerThumb2Right,
+
+ /// <summary>
+ /// FingerThumb3Right joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ FingerThumb3Right,
+
+ /// <summary>
+ /// FingerThumb4Right joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ FingerThumb4Right,
+
+ /// <summary>
+ /// FingerIndex1Right joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ FingerIndex1Right,
+
+ /// <summary>
+ /// FingerIndex2Right joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ FingerIndex2Right,
+
+ /// <summary>
+ /// FingerIndex3Right joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ FingerIndex3Right,
+
+ /// <summary>
+ /// FingerIndex4Right joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ FingerIndex4Right,
+
+ /// <summary>
+ /// FingerMiddle1Right joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ FingerMiddle1Right,
+
+ /// <summary>
+ /// FingerMiddle2Right joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ FingerMiddle2Right,
+
+ /// <summary>
+ /// FingerMiddle3Right joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ FingerMiddle3Right,
+
+ /// <summary>
+ /// FingerMiddle4Right joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ FingerMiddle4Right,
+
+ /// <summary>
+ /// FingerRing1Right joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ FingerRing1Right,
+
+ /// <summary>
+ /// FingerRing2Right joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ FingerRing2Right,
+
+ /// <summary>
+ /// FingerRing3Right joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ FingerRing3Right,
+
+ /// <summary>
+ /// FingerRing4Right joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ FingerRing4Right,
+
+ /// <summary>
+ /// FingerPinky1Right joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ FingerPinky1Right,
+
+ /// <summary>
+ /// FingerPinky2Right joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ FingerPinky2Right,
+
+ /// <summary>
+ /// FingerPinky3Right joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ FingerPinky3Right,
+
+ /// <summary>
+ /// FingerPinky4Right joint
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ FingerPinky4Right,
+ #endregion
+
+ /// <summary>
+ /// Max value of default joint. It will be used when we determine the motion index is default or custom.
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ DefaultJointMax,
+ }
+
+}
--- /dev/null
+/*
+ * Copyright(c) 2023 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using System.ComponentModel;
+
+namespace Tizen.AIAvatar
+{
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal enum NodeType
+ {
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ HeadGeo,
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ MouthGeo,
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ EyelashGeo
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2024 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using Newtonsoft.Json;
+using System.Net.Http;
+using System;
+using System.Threading.Tasks;
+using Tizen.Uix.Tts;
+using System.ComponentModel;
+
+namespace Tizen.AIAvatar
+{
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal class AvatarLLM
+ {
+ private AvatarTTS avatarTTS;
+ private IRestClient restClient;
+ private const string playgroundURL = "https://playground-api.sec.samsung.net";
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal AvatarLLM()
+ {
+ }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal void InitAvatarLLM(AvatarTTS avatarTTS)
+ {
+ this.avatarTTS = avatarTTS;
+ // Setup RestClinet
+ var restClientFactory = new RestClientFactory();
+ restClient = restClientFactory.CreateClient(playgroundURL);
+ }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal async Task StartTTSWithLLMAsync(string text, string token)
+ {
+ var bearerToken = token;
+ var jsonData = "{\"messages\": [{\"role\": \"user\", \"content\": \"" + text + "\"}]}";
+
+ try
+ {
+ var postResponse = await restClient.SendRequestAsync(HttpMethod.Post, "/api/v1/chat/completions", bearerToken, jsonData);
+ var responseData = JsonConvert.DeserializeObject<dynamic>(postResponse);
+ string content = responseData["response"]["content"];
+ Log.Info("Tizen.AIAvatar", content);
+
+ //TTS 호출
+ var voiceInfo = new VoiceInfo()
+ {
+ Lang = "en_US",
+ Type = Voice.Female,
+ };
+
+ avatarTTS.PlayTTSAsync(content, voiceInfo, (o, e) => {
+
+ });
+
+ }
+ catch (Exception ex)
+ {
+ Log.Error("Tizen.AIAvatar", "에러 발생: " + ex.Message);
+ }
+ }
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2023 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using System.ComponentModel;
+
+using static Tizen.AIAvatar.AIAvatar;
+
+namespace Tizen.AIAvatar
+{
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal class AvatarMic
+ {
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal AvatarMic()
+ {
+ }
+
+ /// <summary>
+ /// Initialize Microphone for using live lip sync
+ /// </summary>
+ /// <privilege>http://tizen.org/privilege/recorder</privilege>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal bool InitAvatarMic(Avatar avatar)
+ {
+ if (AudioRecorder.Instance == null)
+ {
+ Log.Error(LogTag, $"Failed to initialize AudioRecorder");
+ return false;
+ }
+ AudioRecorder.Instance?.InitMic(avatar?.AvatarAnimator);
+
+ return true;
+ }
+
+ /// <summary>
+ /// Deinitialize Microphone
+ /// </summary>
+ /// <privilege>http://tizen.org/privilege/recorder</privilege>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal bool DeinitAvatarMIC()
+ {
+ AudioRecorder.Instance?.DeinitMic();
+
+ return true;
+ }
+
+ /// <summary>
+ /// Start synchronization avatar lip based on the microphone's voice
+ /// </summary>
+ /// <privilege>http://tizen.org/privilege/recorder</privilege>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal void StartMic()
+ {
+ AudioRecorder.Instance?.StartRecording();
+ }
+
+ /// <summary>
+ /// Stop microphone
+ /// </summary>
+ /// <privilege>http://tizen.org/privilege/recorder</privilege>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal void StopMic()
+ {
+ AudioRecorder.Instance?.StopRecording();
+ }
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2023 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using System.Collections.Generic;
+using System.ComponentModel;
+using System;
+using Tizen.Uix.Tts;
+
+using static Tizen.AIAvatar.AIAvatar;
+
+namespace Tizen.AIAvatar
+{
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal class AvatarTTS
+ {
+ private TTSLipSyncer ttsLipSyncer;
+
+ private event EventHandler ttsReadyFinished;
+ private readonly Object ttsReadyFinishedLock = new Object();
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal AvatarTTS()
+ {
+ }
+
+ /// <summary>
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal event EventHandler TTSReadyFinished
+ {
+ add
+ {
+ lock (ttsReadyFinishedLock)
+ {
+ ttsReadyFinished += value;
+ }
+
+ }
+
+ remove
+ {
+ lock (ttsReadyFinishedLock)
+ {
+ if (ttsReadyFinished == null)
+ {
+ Log.Error(LogTag, "ttsReadyFinished is null");
+ return;
+ }
+ ttsReadyFinished -= value;
+ }
+ }
+ }
+
+ /// <summary>
+ /// <param name=""></param>
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal TtsClient CurrentTTSClient => ttsLipSyncer?.TtsHandle;
+
+ /// <summary>
+ /// <param name=""></param>
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal void InitTTS(Avatar avatar)
+ {
+ if (ttsLipSyncer == null)
+ {
+ try
+ {
+ ttsLipSyncer = new TTSLipSyncer(avatar);
+ }
+ catch (Exception e)
+ {
+ Log.Error(LogTag, $"error :{e.Message}");
+ Log.Error(LogTag, $"{e.StackTrace}");
+ }
+ }
+ }
+
+ /// <summary>
+ /// <param name=""></param>
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal void DeinitTTS()
+ {
+ if (ttsLipSyncer != null)
+ {
+ try
+ {
+ ttsLipSyncer.DeinitTts();
+ ttsLipSyncer = null;
+ }
+ catch (Exception e)
+ {
+ Log.Error(LogTag, $"error :{e.Message}");
+ Log.Error(LogTag, $"{e.StackTrace}");
+ }
+ }
+ }
+
+ /// <summary>
+ /// <param name=""></param>
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal List<VoiceInfo> GetSupportedVoices()
+ {
+ return ttsLipSyncer.GetSupportedVoices();
+ }
+
+ /// <summary>
+ /// <param name=""></param>
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal bool PrepareTTS(string text, VoiceInfo voiceInfo, EventHandler ttsReadyFinishedCallback = null)
+ {
+ if (ttsLipSyncer == null || ttsLipSyncer.TtsHandle == null)
+ {
+ Log.Error(LogTag, "tts is null");
+ return false;
+ }
+
+
+ if (!ttsLipSyncer.IsSupportedVoice(voiceInfo))
+ {
+ Log.Info(LogTag, $"{voiceInfo.Lang} & {voiceInfo.Type} is not supported");
+ return false;
+ }
+
+ Log.Info(LogTag, "Current TTS State :" + ttsLipSyncer.TtsHandle.CurrentState);
+ if (ttsLipSyncer.TtsHandle.CurrentState != Tizen.Uix.Tts.State.Ready)
+ {
+ Log.Info(LogTag, "TTS is not ready");
+ return false;
+ }
+
+ try
+ {
+ ttsLipSyncer.AddText(text, voiceInfo);
+ ttsLipSyncer.Prepare(ttsReadyFinishedCallback);
+ }
+ catch (Exception e)
+ {
+ Log.Error(LogTag, $"error :{e.Message}");
+ Log.Error(LogTag, $"{e.StackTrace}");
+ return false;
+ }
+ return true;
+ }
+
+ /// <summary>
+ /// <param name=""></param>
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal bool PrepareTTS(string text, string lang = "ko_KR", EventHandler ttsReadyFinishedCallback = null)
+ {
+ if (ttsLipSyncer == null || ttsLipSyncer.TtsHandle == null)
+ {
+ Log.Error(LogTag, "tts is null");
+ return false;
+ }
+
+ if (!ttsLipSyncer.IsSupportedVoice(lang))
+ {
+ Log.Error(LogTag, $"{lang} is not supported");
+ return false;
+ }
+
+ Log.Info(LogTag, "Current TTS State :" + ttsLipSyncer.TtsHandle.CurrentState);
+ if (ttsLipSyncer.TtsHandle.CurrentState != Tizen.Uix.Tts.State.Ready)
+ {
+ Log.Error(LogTag, "TTS is not ready");
+ return false;
+ }
+ try
+ {
+ ttsLipSyncer.AddText(text, lang);
+ ttsLipSyncer.Prepare(ttsReadyFinishedCallback);
+ }
+ catch (Exception e)
+ {
+ Log.Error(LogTag, $"error :{e.Message}");
+ Log.Error(LogTag, $"{e.StackTrace}");
+ return false;
+ }
+ return true;
+ }
+
+ /// <summary>
+ /// <param name=""></param>
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal bool PlayPreparedTTS()
+ {
+ if (ttsLipSyncer == null || ttsLipSyncer.TtsHandle == null)
+ {
+ Log.Error(LogTag, "tts is null");
+ return false;
+ }
+
+ return ttsLipSyncer.PlayPreparedText();
+ }
+
+ /// <summary>
+ /// <param name=""></param>
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal bool PlayTTS(string text, VoiceInfo voiceInfo)
+ {
+ if (ttsLipSyncer == null || ttsLipSyncer.TtsHandle == null)
+ {
+ Log.Error(LogTag, "tts is null");
+ return false;
+ }
+
+
+ if (!ttsLipSyncer.IsSupportedVoice(voiceInfo))
+ {
+ Log.Info(LogTag, $"{voiceInfo.Lang} & {voiceInfo.Type} is not supported");
+ return false;
+ }
+
+ Log.Info(LogTag, "Current TTS State :" + ttsLipSyncer.TtsHandle.CurrentState);
+ if (ttsLipSyncer.TtsHandle.CurrentState != Tizen.Uix.Tts.State.Ready)
+ {
+ Log.Info(LogTag, "TTS is not ready");
+ return false;
+ }
+
+ try
+ {
+ ttsLipSyncer.AddText(text, voiceInfo);
+ ttsLipSyncer.Play();
+ }
+ catch (Exception e)
+ {
+ Log.Error(LogTag, $"error :{e.Message}");
+ Log.Error(LogTag, $"{e.StackTrace}");
+ return false;
+ }
+ return true;
+ }
+
+ /// <summary>
+ /// <param name=""></param>
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal bool PlayTTS(string text, string lang = "ko_KR")
+ {
+ if (ttsLipSyncer == null || ttsLipSyncer.TtsHandle == null)
+ {
+ Log.Error(LogTag, "tts is null");
+ return false;
+ }
+
+ if (!ttsLipSyncer.IsSupportedVoice(lang))
+ {
+ Log.Error(LogTag, $"{lang} is not supported");
+ return false;
+ }
+
+ Log.Info(LogTag, "Current TTS State :" + ttsLipSyncer.TtsHandle.CurrentState);
+ if (ttsLipSyncer.TtsHandle.CurrentState != Tizen.Uix.Tts.State.Ready)
+ {
+ Log.Error(LogTag, "TTS is not ready");
+ return false;
+ }
+ try
+ {
+ ttsLipSyncer.AddText(text, lang);
+ ttsLipSyncer.Play();
+ }
+ catch (Exception e)
+ {
+ Log.Error(LogTag, $"error :{e.Message}");
+ Log.Error(LogTag, $"{e.StackTrace}");
+ return false;
+ }
+ return true;
+ }
+
+ /// <summary>
+ /// <param name=""></param>
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal bool PlayTTSAsync(string text, VoiceInfo voiceInfo, EventHandler ttsReadyFinishedCallback = null)
+ {
+ if (ttsLipSyncer == null || ttsLipSyncer.TtsHandle == null)
+ {
+ Log.Error(LogTag, "tts is null");
+ return false;
+ }
+
+
+ if (!ttsLipSyncer.IsSupportedVoice(voiceInfo))
+ {
+ Log.Info(LogTag, $"{voiceInfo.Lang} & {voiceInfo.Type} is not supported");
+ return false;
+ }
+
+ Log.Info(LogTag, "Current TTS State :" + ttsLipSyncer.TtsHandle.CurrentState);
+ if (ttsLipSyncer.TtsHandle.CurrentState != Tizen.Uix.Tts.State.Ready)
+ {
+ Log.Info(LogTag, "TTS is not ready");
+ return false;
+ }
+
+ try
+ {
+ ttsLipSyncer.AddText(text, voiceInfo);
+ ttsLipSyncer.PlayAsync(ttsReadyFinishedCallback);
+ }
+ catch (Exception e)
+ {
+ Log.Error(LogTag, $"error :{e.Message}");
+ Log.Error(LogTag, $"{e.StackTrace}");
+ return false;
+ }
+ return true;
+ }
+
+ /// <summary>
+ /// <param name=""></param>
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal bool PlayTTS(string text, string lang = "ko_KR", EventHandler ttsReadyFinishedCallback = null)
+ {
+ if (ttsLipSyncer == null || ttsLipSyncer.TtsHandle == null)
+ {
+ Log.Error(LogTag, "tts is null");
+ return false;
+ }
+
+ if (!ttsLipSyncer.IsSupportedVoice(lang))
+ {
+ Log.Error(LogTag, $"{lang} is not supported");
+ return false;
+ }
+
+ Log.Info(LogTag, "Current TTS State :" + ttsLipSyncer.TtsHandle.CurrentState);
+ if (ttsLipSyncer.TtsHandle.CurrentState != Tizen.Uix.Tts.State.Ready)
+ {
+ Log.Error(LogTag, "TTS is not ready");
+ return false;
+ }
+ try
+ {
+ ttsLipSyncer.AddText(text, lang);
+ ttsLipSyncer.PlayAsync(ttsReadyFinishedCallback);
+ }
+ catch (Exception e)
+ {
+ Log.Error(LogTag, $"error :{e.Message}");
+ Log.Error(LogTag, $"{e.StackTrace}");
+ return false;
+ }
+ return true;
+ }
+
+ /// <summary>
+ /// <param name=""></param>
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal void PauseTTS()
+ {
+ if (ttsLipSyncer == null || ttsLipSyncer.TtsHandle == null)
+ {
+ Log.Error(LogTag, "tts is null");
+ return;
+ }
+
+ try
+ {
+ Log.Info(LogTag, "Current TTS State :" + ttsLipSyncer.TtsHandle.CurrentState);
+ ttsLipSyncer?.Pause();
+ }
+ catch (Exception e)
+ {
+ Log.Error(LogTag, $"error :{e.Message}");
+ Log.Error(LogTag, $"{e.StackTrace}");
+ }
+ }
+
+ /// <summary>
+ /// <param name=""></param>
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal void StopTTS()
+ {
+ if (ttsLipSyncer == null || ttsLipSyncer.TtsHandle == null)
+ {
+ Log.Error(LogTag, "tts is null");
+ return;
+ }
+
+ try
+ {
+ Log.Info(LogTag, "Current TTS State :" + ttsLipSyncer.TtsHandle.CurrentState);
+ ttsLipSyncer?.Stop();
+ }
+ catch (Exception e)
+ {
+ Log.Error(LogTag, $"error :{e.Message}");
+ Log.Error(LogTag, $"{e.StackTrace}");
+ }
+ }
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2023 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+using System.ComponentModel;
+using Tizen.NUI.Scene3D;
+
+namespace Tizen.AIAvatar
+{
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public class AnimationInfo
+ {
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public MotionData MotionData { get; internal set; }
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public string MotionName { get; internal set; }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public AnimationInfo(MotionData motionData, string motionName)
+ {
+ MotionData = motionData;
+ MotionName = motionName;
+ }
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2023 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using System;
+using System.ComponentModel;
+using static Tizen.AIAvatar.AIAvatar;
+
+namespace Tizen.AIAvatar
+{
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public class AvatarInfo
+ {
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public string Name { get; private set; }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public string ThumbnailPath { get; private set; }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal string ResourcePath { get; private set; }
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal AvatarInfoOption avatarInfoOption { get; private set; }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public AvatarInfo(string path, string Name, AvatarInfoOption info = AvatarInfoOption.Thumbnail)
+ {
+ this.Name = Name;
+ this.avatarInfoOption = info;
+
+ if (info == AvatarInfoOption.Thumbnail)
+ {
+ ThumbnailPath = path;
+ }
+ else
+ {
+ ResourcePath = path;
+ }
+ }
+
+ internal AvatarInfo(string directoryPath)
+ {
+ string path = ApplicationResourcePath + EmojiAvatarResourcePath;
+ Name = directoryPath.Substring(path.Length, directoryPath.Length - path.Length);
+ ResourcePath = $"{directoryPath}/{AIAvatar.ExternalModel}";
+ }
+ }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public enum AvatarInfoOption
+ {
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ Thumbnail = 0,
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ Resource = 1,
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2023 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+using System.ComponentModel;
+using Tizen.Uix.Tts;
+
+namespace Tizen.AIAvatar
+{
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public struct VoiceInfo
+ {
+ private string lang;
+ private Voice type;
+
+ /// <summary>
+ ///
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public string Lang { get => lang; set => lang = value; }
+
+ /// <summary>
+ ///
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public Voice Type { get => type; set => type = value; }
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2023 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using System.ComponentModel;
+using Tizen.NUI;
+using Tizen.NUI.Scene3D;
+
+namespace Tizen.AIAvatar
+{
+ /// <summary>
+ /// Specialized <see cref="BlendShapeIndex"/> to control avatar blend shape.
+ /// </summary>
+ /// <example>
+ /// <code>
+ /// AvatarBlendShapeIndex leftEyeBlink = new AvatarBlendShapeIndex(avatar.BlendShapeMapper, BlendShapeType.EyeBlinkLeft);
+ ///
+ /// // We can change the property later.
+ /// AVatarBlendShapeIndex rightEyeBlink = new AvatarJointTransformIndex(avatar.BlendShapeMapper);
+ /// rightEyeBlink.AvatarBlendShapeType = (uint)BlendShapeType.EyeBlinkRight;
+ /// </code>
+ /// </example>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public class AvatarBlendShapeIndex : BlendShapeIndex
+ {
+ internal AvatarPropertyMapper nameMapper = null;
+ internal uint blendShapeType;
+
+ /// <summary>
+ /// Create an initialized avatar blend shape index.
+ /// </summary>
+ /// <param name="mapper">Name mapper for this index</param>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public AvatarBlendShapeIndex(AvatarPropertyMapper mapper) : base()
+ {
+ nameMapper = mapper;
+ }
+
+ /// <summary>
+ /// Create an initialized avatar blend shape index with input blend shape ID.
+ /// </summary>
+ /// <param name="mapper">Name mapper for this index</param>
+ /// <param name="blendShapeId">Blend shape ID for this motion index</param>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public AvatarBlendShapeIndex(AvatarPropertyMapper mapper, PropertyKey blendShapeId) : base(new PropertyKey(GetPropertyNameFromMapper(mapper, blendShapeId)))
+ {
+ nameMapper = mapper;
+ }
+
+ /// <summary>
+ /// Create an initialized avatar blend shape index with blend shape type.
+ /// </summary>
+ /// <param name="mapper">Name mapper for this index</param>
+ /// <param name="blendShapeType">Type of blend shape for this motion index</param>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal AvatarBlendShapeIndex(AvatarPropertyMapper blendShapeMapper, BlendShapeType blendShapeType) : this(blendShapeMapper, (uint)blendShapeType)
+ {
+ }
+
+ /// <summary>
+ ///
+ /// </summary>
+ /// <param name="mapper">Name mapper for this index</param>
+ /// <param name="blendShapeType">Type of blend shape for this motion index</param>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public AvatarBlendShapeIndex(AvatarPropertyMapper mapper, uint blendShapeType) : base(new PropertyKey(GetPropertyNameFromMapper(mapper, blendShapeType)))
+ {
+ nameMapper = mapper;
+ this.blendShapeType = blendShapeType;
+ }
+
+ /// <summary>
+ ///
+ /// </summary>
+ /// <param name="mapper">Name mapper for this index</param>
+ /// <param name="blendShapeType">Type of blend shape for this motion index</param>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal AvatarBlendShapeIndex(AvatarPropertyMapper nodeMapper, NodeType nodeType, AvatarPropertyMapper blendShapeMapper, BlendShapeType blendShapeType) : base(new PropertyKey(GetPropertyNameFromMapper(nodeMapper, (uint)nodeType)), new PropertyKey(GetPropertyNameFromMapper(blendShapeMapper, (uint)blendShapeType)))
+ {
+ nameMapper = blendShapeMapper;
+ this.blendShapeType = (uint)blendShapeType;
+ }
+
+ /// <summary>
+ /// Get the name of given index.
+ /// </summary>
+ /// <param name="mapper">Name mapper for given index</param>
+ /// <param name="id">Target ID what we want to get name</param>
+ /// <returns>Name, or null if invalid</returns>
+ private static string GetPropertyNameFromMapper(AvatarPropertyMapper mapper, PropertyKey id)
+ {
+ if (id == null)
+ {
+ return "";
+ }
+ if (id.Type == PropertyKey.KeyType.String)
+ {
+ return id.StringKey;
+ }
+ return mapper?.GetPropertyName((uint)id.IndexKey) ?? "";
+ }
+
+ /// <summary>
+ /// Get the name of given JointType.
+ /// </summary>
+ /// <param name="mapper">Name mapper for given index</param>
+ /// <param name="blendShapeType">Type of joint what we want to get name</param>
+ /// <returns>Name, or null if invalid</returns>
+ private static string GetPropertyNameFromMapper(AvatarPropertyMapper mapper, uint blendShapeType)
+ {
+ return mapper?.GetPropertyName(blendShapeType) ?? "";
+ }
+
+ /// <summary>
+ /// TODO : Explain me
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public AvatarPropertyMapper NameMapper
+ {
+ get
+ {
+ return nameMapper;
+ }
+ set
+ {
+ nameMapper = value;
+
+ using PropertyKey blendShapeId = new(GetPropertyNameFromMapper(nameMapper, blendShapeType));
+ base.BlendShapeId = blendShapeId;
+ }
+ }
+
+ /// <summary>
+ /// TODO : Explain me
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public uint AvatarBlendShapeType
+ {
+ get
+ {
+ return blendShapeType;
+ }
+ set
+ {
+ blendShapeType = value;
+
+ using PropertyKey blendShapeId = new(GetPropertyNameFromMapper(nameMapper, blendShapeType));
+ base.BlendShapeId = blendShapeId;
+ }
+ }
+
+ /// <summary>
+ /// Hijack property to control Avatar specified logic.
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public new PropertyKey BlendShapeId
+ {
+ get
+ {
+ return base.BlendShapeId;
+ }
+ set
+ {
+ if (value != null)
+ {
+ if (value.Type == PropertyKey.KeyType.Index)
+ {
+ blendShapeType = (uint)value.IndexKey;
+ }
+ using PropertyKey blendShapeId = new(GetPropertyNameFromMapper(nameMapper, value));
+ base.BlendShapeId = blendShapeId;
+ }
+ else
+ {
+ base.BlendShapeId = value;
+ }
+ }
+ }
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2023 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using System.ComponentModel;
+using Tizen.NUI;
+using Tizen.NUI.Scene3D;
+
+namespace Tizen.AIAvatar
+{
+ /// <summary>
+ /// Specialized <see cref="MotionTransformIndex"/> to control avatar joint transform.
+ /// </summary>
+ /// <example>
+ /// <code>
+ /// AvatarJointTransformIndex position = new AvatarJointTransformIndex(avatar.JointMapper, JointType.Head, MotionTransformIndex.TransformTypes.Position);
+ ///
+ /// // We can change the property later.
+ /// AvatarJointTransformIndex orientation = new AvatarJointTransformIndex(avatar.JointMapper);
+ /// orientation.AvatarJointType = (uint)JointType.Neck;
+ /// orientation.TransformType = MotionTransformIndex.TransformTypes.Orientation;
+ /// </code>
+ /// </example>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public class AvatarJointTransformIndex : MotionTransformIndex
+ {
+ internal AvatarPropertyMapper nameMapper = null;
+ internal uint jointType;
+
+ /// <summary>
+ /// Create an initialized avatar joint transform index.
+ /// </summary>
+ /// <param name="mapper">Name mapper for this index</param>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public AvatarJointTransformIndex(AvatarPropertyMapper mapper) : base()
+ {
+ nameMapper = mapper;
+ }
+
+ /// <summary>
+ /// Create an initialized avatar joint transform index with input node id, and transform type.
+ /// </summary>
+ /// <param name="mapper">Name mapper for this index</param>
+ /// <param name="modelNodeId">Node ID for this motion index</param>
+ /// <param name="transformType">Transform property type for this motion index</param>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public AvatarJointTransformIndex(AvatarPropertyMapper mapper, PropertyKey modelNodeId, TransformTypes transformType) : base(new PropertyKey(GetPropertyNameFromMapper(mapper, modelNodeId)), transformType)
+ {
+ nameMapper = mapper;
+ }
+
+ /// <summary>
+ /// Create an initialized avatar joint transform index with input node id, and transform type.
+ /// </summary>
+ /// <param name="mapper">Name mapper for this index</param>
+ /// <param name="jointType">Type of joint for this motion index</param>
+ /// <param name="transformType">Transform property type for this motion index</param>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ internal AvatarJointTransformIndex(AvatarPropertyMapper mapper, JointType jointType, TransformTypes transformType) : this(mapper, (uint)jointType, transformType)
+ {
+ }
+
+ /// <summary>
+ /// Create an initialized avatar joint transform index with input node id, and transform type.
+ /// </summary>
+ /// <param name="mapper">Name mapper for this index</param>
+ /// <param name="jointType">Type of joint for this motion index</param>
+ /// <param name="transformType">Transform property type for this motion index</param>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public AvatarJointTransformIndex(AvatarPropertyMapper mapper, uint jointType, TransformTypes transformType) : base(new PropertyKey(GetPropertyNameFromMapper(mapper, jointType)), transformType)
+ {
+ nameMapper = mapper;
+ this.jointType = jointType;
+ }
+
+ private static string GetPropertyNameFromMapper(AvatarPropertyMapper mapper, PropertyKey id)
+ {
+ if (id == null)
+ {
+ return "";
+ }
+ if (id.Type == PropertyKey.KeyType.String)
+ {
+ return id.StringKey;
+ }
+ return mapper?.GetPropertyName((uint)id.IndexKey) ?? "";
+ }
+
+ private static string GetPropertyNameFromMapper(AvatarPropertyMapper mapper, uint jointType)
+ {
+ return mapper?.GetPropertyName(jointType) ?? "";
+ }
+
+ /// <summary>
+ /// TODO : Explain me
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public AvatarPropertyMapper NameMapper
+ {
+ get
+ {
+ return nameMapper;
+ }
+ set
+ {
+ nameMapper = value;
+
+ using PropertyKey nodeId = new(GetPropertyNameFromMapper(nameMapper, jointType));
+ base.ModelNodeId = nodeId;
+ }
+ }
+
+ /// <summary>
+ /// TODO : Explain me
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public uint AvatarJointType
+ {
+ get
+ {
+ return jointType;
+ }
+ set
+ {
+ jointType = value;
+
+ using PropertyKey nodeId = new(GetPropertyNameFromMapper(nameMapper, jointType));
+ base.ModelNodeId = nodeId;
+ }
+ }
+
+ /// <summary>
+ /// Hijack property to control Avatar specified logic.
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public new PropertyKey ModelNodeId
+ {
+ get
+ {
+ return base.ModelNodeId;
+ }
+ set
+ {
+ if (value != null)
+ {
+ if (value.Type == PropertyKey.KeyType.Index)
+ {
+ jointType = (uint)value.IndexKey;
+ }
+ using PropertyKey nodeId = new(GetPropertyNameFromMapper(nameMapper, value));
+ base.ModelNodeId = nodeId;
+ }
+ else
+ {
+ base.ModelNodeId = value;
+ }
+ }
+ }
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2023 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+using System;
+using System.ComponentModel;
+
+namespace Tizen.AIAvatar
+{
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public class AvatarProperties
+ {
+ private AvatarPropertyMapper jointMapper;
+ private AvatarPropertyMapper blendShapeMapper;
+ private AvatarPropertyMapper nodeMapper;
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public AvatarProperties(AvatarPropertyMapper jointMapper, AvatarPropertyMapper blendShapeMapper, AvatarPropertyMapper nodeMapper)
+ {
+ JointMapper = new AvatarPropertyMapper(jointMapper);
+ BlendShapeMapper = new AvatarPropertyMapper(blendShapeMapper);
+ NodeMapper = new AvatarPropertyMapper(nodeMapper);
+ }
+
+ internal event EventHandler<AvatarProperties> AvatarPropertiesChanged;
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public AvatarPropertyMapper JointMapper
+ {
+ get
+ {
+ return jointMapper;
+ }
+ set
+ {
+ jointMapper = value;
+ AvatarPropertiesChanged?.Invoke(jointMapper, this);
+ }
+ }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public AvatarPropertyMapper BlendShapeMapper
+ {
+ get
+ {
+ return blendShapeMapper;
+ }
+ set
+ {
+ blendShapeMapper = value;
+ AvatarPropertiesChanged?.Invoke(blendShapeMapper, this);
+ }
+ }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public AvatarPropertyMapper NodeMapper
+ {
+ get
+ {
+ return nodeMapper;
+ }
+ set
+ {
+ nodeMapper = value;
+ AvatarPropertiesChanged?.Invoke(nodeMapper, this);
+ }
+ }
+ }
+}
--- /dev/null
+/*
+ * Copyright(c) 2023 Samsung Electronics Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+using System.Collections.Generic;
+using System.ComponentModel;
+
+namespace Tizen.AIAvatar
+{
+ /// <summary>
+ /// TODO : Explain more detail
+ /// </summary>
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public class AvatarPropertyMapper
+ {
+ /// <summary>
+ /// Mapper between index and property name
+ /// </summary>
+ private Dictionary<uint, string> mapper = null;
+
+ /// <summary>
+ /// The counter of index. It will be increased one when we register custom index.
+ /// </summary>
+ private uint customIndexCounter = 0;
+
+ /// <summary>
+ /// Create an initialized AvatarPropertyNameMapper.
+ /// </summary>
+ // This will be public opened after ACR done. (Before ACR, need to be hidden as Inhouse API)
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public AvatarPropertyMapper()
+ {
+ mapper = new Dictionary<uint, string>();
+ customIndexCounter = 0u;
+ }
+
+ /// <summary>
+ /// Copy constructor.
+ /// </summary>
+ // This will be public opened after ACR done. (Before ACR, need to be hidden as Inhouse API)
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public AvatarPropertyMapper(AvatarPropertyMapper rhs)
+ {
+ if (rhs != null)
+ {
+ mapper = new Dictionary<uint, string>(rhs.Mapper);
+ customIndexCounter = rhs.customIndexCounter;
+ }
+ else
+ {
+ mapper = new Dictionary<uint, string>();
+ customIndexCounter = 0u;
+ }
+ }
+
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ protected uint CustomIndexCounter
+ {
+ get
+ {
+ return customIndexCounter;
+ }
+ set
+ {
+ customIndexCounter = value;
+ }
+ }
+
+ /// <summary>
+ /// Get current mapper information.
+ /// </summary>
+ // This will be public opened after ACR done. (Before ACR, need to be hidden as Inhouse API)
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public Dictionary<uint, string> Mapper
+ {
+ get
+ {
+ return mapper;
+ }
+ }
+
+ /// <summary>
+ /// TODO : Explain me.
+ /// </summary>
+ /// <param name="index">The index of property what we want to set</param>
+ /// <returns>The index of property, or uint.MaxValue if not exist</returns>
+ // This will be public opened after ACR done. (Before ACR, need to be hidden as Inhouse API)
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public string this[uint index]
+ {
+ set
+ {
+ SetPropertyName(index, value);
+ }
+ get
+ {
+ return GetPropertyName(index);
+ }
+ }
+
+ /// <summary>
+ /// TODO : Explain me.
+ /// </summary>
+ /// <param name="name">The name of custom property</param>
+ /// <returns>The index of property matched with name.</returns>
+ // This will be public opened after ACR done. (Before ACR, need to be hidden as Inhouse API)
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public uint RegisterCustomProperty(string name)
+ {
+ uint ret = GetPropertyIndexByName(name);
+ if (ret >= customIndexCounter)
+ {
+ ret = customIndexCounter++;
+ SetPropertyName(ret, name);
+ }
+ return ret;
+ }
+
+ /// <summary>
+ /// TODO : Explain me.
+ /// </summary>
+ /// <param name="name">The name of property what we want to get index</param>
+ /// <returns>The index of property, or uint.MaxValue if not exist</returns>
+ // This will be public opened after ACR done. (Before ACR, need to be hidden as Inhouse API)
+ [EditorBrowsable(EditorBrowsableState.Never)]
+ public uint GetPropertyIndexByName(string name)
+ {
+ // TODO : Do this without iteration
+ foreach (var pair in mapper)
+ {
+ if (pair.Value == name)
+ {
+ return pair.Key;
+ }
+ }
+ return uint.MaxValue;
+ }
+
+ /// <summary>
+ /// TODO : Explain me.
+ /// </summary>
+ /// <param name="index">The index of property what we want to set</param>
+ /// <param name="name">The name of property what we want to set</param>
+ /// <remark>
+ /// New property will be added if we use index that not exist in mapper.
+ /// </remark>
+ internal void SetPropertyName(uint index, string name)
+ {
+ mapper.TryAdd(index, name);
+ }
+
+ /// <summary>
+ /// TODO : Explain me.
+ /// </summary>
+ /// <param name="index">The index of property what we want to set</param>
+ /// <returns>The name of property, or null if not exist</returns>
+ internal string GetPropertyName(uint index)
+ {
+ string ret = null;
+ mapper.TryGetValue(index, out ret);
+ return ret;
+ }
+ }
+}
projects / platform / core / csapi / tizenfx.git / commitdiff