From 6145e8aa085f87546731d95543bb008a67a3fd31 Mon Sep 17 00:00:00 2001 From: huiyu <35286162+huiyueun@users.noreply.github.com> Date: Tue, 26 Mar 2024 16:04:42 +0900 Subject: [PATCH] Add Tizen.AIAvatar project (#6014) * Add Tizen.AIAvatar project Add new empty project of Tizen.AIAvatar Signed-off-by: huiyu.eun * Add hidden Signed-off-by: huiyu.eun * Modify apis Signed-off-by: huiyu.eun * Make internal Signed-off-by: huiyu.eun * remove lip module Signed-off-by: huiyu.eun --------- Signed-off-by: huiyu.eun --- src/Tizen.AIAvatar/Tizen.AIAvatar.csproj | 25 ++ .../src/Extensions/AvatarExtension.cs | 41 ++ src/Tizen.AIAvatar/src/Extensions/AvatarScene.cs | 25 ++ .../src/Extensions/SceneViewExtension.cs | 75 ++++ src/Tizen.AIAvatar/src/internal/Avatar/AIAvatar.cs | 42 ++ .../src/internal/Avatar/Interop.SceneView.cs | 37 ++ src/Tizen.AIAvatar/src/internal/Avatar/Utils.cs | 127 ++++++ .../src/internal/BlendShapeInfo/BlendShapeInfo.cs | 58 +++ .../internal/BlendShapeInfo/BlendShapeModelInfo.cs | 35 ++ .../src/internal/BlendShapeInfo/BlendShapeValue.cs | 28 ++ .../BlendShapeInfo/BlendShapeVisemeInfo.cs | 27 ++ .../src/internal/Common/IRestClient.cs | 28 ++ .../src/internal/Common/RestClient.cs | 78 ++++ .../src/internal/Common/RestClientFactory.cs | 30 ++ .../DefaultAvatar/DefaultAvatarProperties.cs | 200 +++++++++ src/Tizen.AIAvatar/src/internal/LipSync/LipInfo.cs | 32 ++ .../internal/LipSync/Models/ISingleShotModel.cs | 16 + .../LipSync/Models/SoftmaxLinqExtension.cs | 16 + .../src/internal/LipSync/Models/TFVowel6.cs | 82 ++++ src/Tizen.AIAvatar/src/internal/LipSync/Viseme.cs | 35 ++ .../src/internal/Multimedia/AudioPlayer.cs | 135 ++++++ .../src/internal/Multimedia/AudioRecorder.cs | 151 +++++++ .../Multimedia/RecordBufferChangedEventArgs.cs | 34 ++ .../src/internal/Uix/TTSLipSyncer.cs | 498 +++++++++++++++++++++ .../src/internal/Uix/UtteranceText.cs | 25 ++ .../Animations/AnimationModule/AnimationModule.cs | 126 ++++++ .../Animations/AnimationModule/AsyncLipSyncer.cs | 181 ++++++++ .../Animations/AnimationModule/AvatarMotions.cs | 65 +++ .../Animations/AnimationModule/BlendShapePlayer.cs | 157 +++++++ .../Animations/AnimationModule/EyeBlinker.cs | 139 ++++++ .../Animations/AnimationModule/JointTransformer.cs | 107 +++++ .../Avatar/Animations/AnimationModule/LipSyncer.cs | 371 +++++++++++++++ .../Animations/AnimationModule/MotionPlayer.cs | 75 ++++ .../AnimationModuleData/IAnimationModuleData.cs | 35 ++ .../Animations/AnimationModuleData/LipSyncData.cs | 34 ++ .../AnimationModuleData/MotionBehaviorData.cs | 39 ++ .../Animations/AvatarMotionChangedEventArgs.cs | 58 +++ .../public/Avatar/Animations/AvatarMotionState.cs | 65 +++ .../src/public/Avatar/AudioOptions.cs | 70 +++ src/Tizen.AIAvatar/src/public/Avatar/Avatar.cs | 199 ++++++++ .../src/public/Avatar/Common/BlendShapeType.cs | 380 ++++++++++++++++ .../src/public/Avatar/Common/JointType.cs | 400 +++++++++++++++++ .../src/public/Avatar/Common/NodeType.cs | 32 ++ .../src/public/Avatar/Controller/AvatarLLM.cs | 79 ++++ .../src/public/Avatar/Controller/AvatarMic.cs | 81 ++++ .../src/public/Avatar/Controller/AvatarTTS.cs | 421 +++++++++++++++++ .../src/public/Avatar/Info/AnimationInfo.cs | 37 ++ .../src/public/Avatar/Info/AvatarInfo.cs | 70 +++ .../src/public/Avatar/Info/VoiceInfo.cs | 40 ++ .../Avatar/Properties/AvatarBlendShapeIndex.cs | 193 ++++++++ .../Avatar/Properties/AvatarJointTransformIndex.cs | 173 +++++++ .../public/Avatar/Properties/AvatarProperties.cs | 82 ++++ .../Avatar/Properties/AvatarPropertyMapper.cs | 176 ++++++++ 53 files changed, 5765 insertions(+) create mode 100644 src/Tizen.AIAvatar/Tizen.AIAvatar.csproj create mode 100644 src/Tizen.AIAvatar/src/Extensions/AvatarExtension.cs create mode 100644 src/Tizen.AIAvatar/src/Extensions/AvatarScene.cs create mode 100644 src/Tizen.AIAvatar/src/Extensions/SceneViewExtension.cs create mode 100644 src/Tizen.AIAvatar/src/internal/Avatar/AIAvatar.cs create mode 100644 src/Tizen.AIAvatar/src/internal/Avatar/Interop.SceneView.cs create mode 100644 src/Tizen.AIAvatar/src/internal/Avatar/Utils.cs create mode 100644 src/Tizen.AIAvatar/src/internal/BlendShapeInfo/BlendShapeInfo.cs create mode 100644 src/Tizen.AIAvatar/src/internal/BlendShapeInfo/BlendShapeModelInfo.cs create mode 100644 src/Tizen.AIAvatar/src/internal/BlendShapeInfo/BlendShapeValue.cs create mode 100644 src/Tizen.AIAvatar/src/internal/BlendShapeInfo/BlendShapeVisemeInfo.cs create mode 100644 src/Tizen.AIAvatar/src/internal/Common/IRestClient.cs create mode 100644 src/Tizen.AIAvatar/src/internal/Common/RestClient.cs create mode 100644 src/Tizen.AIAvatar/src/internal/Common/RestClientFactory.cs create mode 100644 src/Tizen.AIAvatar/src/internal/DefaultAvatar/DefaultAvatarProperties.cs create mode 100644 src/Tizen.AIAvatar/src/internal/LipSync/LipInfo.cs create mode 100644 src/Tizen.AIAvatar/src/internal/LipSync/Models/ISingleShotModel.cs create mode 100644 src/Tizen.AIAvatar/src/internal/LipSync/Models/SoftmaxLinqExtension.cs create mode 100644 src/Tizen.AIAvatar/src/internal/LipSync/Models/TFVowel6.cs create mode 100644 src/Tizen.AIAvatar/src/internal/LipSync/Viseme.cs create mode 100644 src/Tizen.AIAvatar/src/internal/Multimedia/AudioPlayer.cs create mode 100644 src/Tizen.AIAvatar/src/internal/Multimedia/AudioRecorder.cs create mode 100644 src/Tizen.AIAvatar/src/internal/Multimedia/RecordBufferChangedEventArgs.cs create mode 100644 src/Tizen.AIAvatar/src/internal/Uix/TTSLipSyncer.cs create mode 100644 src/Tizen.AIAvatar/src/internal/Uix/UtteranceText.cs create mode 100644 src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModule/AnimationModule.cs create mode 100644 src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModule/AsyncLipSyncer.cs create mode 100644 src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModule/AvatarMotions.cs create mode 100644 src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModule/BlendShapePlayer.cs create mode 100644 src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModule/EyeBlinker.cs create mode 100644 src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModule/JointTransformer.cs create mode 100644 src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModule/LipSyncer.cs create mode 100644 src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModule/MotionPlayer.cs create mode 100644 src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModuleData/IAnimationModuleData.cs create mode 100644 src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModuleData/LipSyncData.cs create mode 100644 src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModuleData/MotionBehaviorData.cs create mode 100644 src/Tizen.AIAvatar/src/public/Avatar/Animations/AvatarMotionChangedEventArgs.cs create mode 100644 src/Tizen.AIAvatar/src/public/Avatar/Animations/AvatarMotionState.cs create mode 100644 src/Tizen.AIAvatar/src/public/Avatar/AudioOptions.cs create mode 100644 src/Tizen.AIAvatar/src/public/Avatar/Avatar.cs create mode 100644 src/Tizen.AIAvatar/src/public/Avatar/Common/BlendShapeType.cs create mode 100644 src/Tizen.AIAvatar/src/public/Avatar/Common/JointType.cs create mode 100644 src/Tizen.AIAvatar/src/public/Avatar/Common/NodeType.cs create mode 100644 src/Tizen.AIAvatar/src/public/Avatar/Controller/AvatarLLM.cs create mode 100644 src/Tizen.AIAvatar/src/public/Avatar/Controller/AvatarMic.cs create mode 100644 src/Tizen.AIAvatar/src/public/Avatar/Controller/AvatarTTS.cs create mode 100644 src/Tizen.AIAvatar/src/public/Avatar/Info/AnimationInfo.cs create mode 100644 src/Tizen.AIAvatar/src/public/Avatar/Info/AvatarInfo.cs create mode 100644 src/Tizen.AIAvatar/src/public/Avatar/Info/VoiceInfo.cs create mode 100644 src/Tizen.AIAvatar/src/public/Avatar/Properties/AvatarBlendShapeIndex.cs create mode 100644 src/Tizen.AIAvatar/src/public/Avatar/Properties/AvatarJointTransformIndex.cs create mode 100644 src/Tizen.AIAvatar/src/public/Avatar/Properties/AvatarProperties.cs create mode 100644 src/Tizen.AIAvatar/src/public/Avatar/Properties/AvatarPropertyMapper.cs diff --git a/src/Tizen.AIAvatar/Tizen.AIAvatar.csproj b/src/Tizen.AIAvatar/Tizen.AIAvatar.csproj new file mode 100644 index 0000000..6831414 --- /dev/null +++ b/src/Tizen.AIAvatar/Tizen.AIAvatar.csproj @@ -0,0 +1,25 @@ + + + + net6.0 + 9.0 + $(NoWarn);0618;CA1054;CA1056 + + + + + + + + + + + + + + + + + + + diff --git a/src/Tizen.AIAvatar/src/Extensions/AvatarExtension.cs b/src/Tizen.AIAvatar/src/Extensions/AvatarExtension.cs new file mode 100644 index 0000000..e2a219a --- /dev/null +++ b/src/Tizen.AIAvatar/src/Extensions/AvatarExtension.cs @@ -0,0 +1,41 @@ +/* + * Copyright(c) 2023 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using System.Collections.Generic; +using System.IO; + +using static Tizen.AIAvatar.AIAvatar; + +namespace Tizen.AIAvatar +{ + internal static class AvatarExtension + { + internal static List GetDefaultAvatarList() + { + var list = new List(); + var avatarModelFolders = Directory.GetDirectories(ApplicationResourcePath + EmojiAvatarResourcePath); + foreach (var directoryInfo in avatarModelFolders) + { + Log.Info(LogTag, $"Directory Path : {directoryInfo}"); + var avatarInfo = new AvatarInfo(directoryInfo); + list.Add(avatarInfo); + } + + return list; + } + } +} diff --git a/src/Tizen.AIAvatar/src/Extensions/AvatarScene.cs b/src/Tizen.AIAvatar/src/Extensions/AvatarScene.cs new file mode 100644 index 0000000..61a606d --- /dev/null +++ b/src/Tizen.AIAvatar/src/Extensions/AvatarScene.cs @@ -0,0 +1,25 @@ +/* + * Copyright(c) 2023 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using Tizen.NUI.Scene3D; + +namespace Tizen.AIAvatar +{ + internal class AvatarScene : SceneView + { + } +} diff --git a/src/Tizen.AIAvatar/src/Extensions/SceneViewExtension.cs b/src/Tizen.AIAvatar/src/Extensions/SceneViewExtension.cs new file mode 100644 index 0000000..b608205 --- /dev/null +++ b/src/Tizen.AIAvatar/src/Extensions/SceneViewExtension.cs @@ -0,0 +1,75 @@ +/* + * Copyright(c) 2023 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using System; +using System.Collections.Generic; +using System.Text; +using Tizen.NUI.Scene3D; + +namespace Tizen.AIAvatar +{ + internal static class SceneViewExtension + { + internal static void SetAlphaMaskUrl(this SceneView sceneView, string url) + { + var setValue = new Tizen.NUI.PropertyValue(url); + sceneView.SetProperty(Interop.AlphaMaskURLGet(), setValue); + setValue.Dispose(); + } + + internal static string GetAlphaMaskUrl(this SceneView sceneView) + { + var returnValue = ""; + var invertYAxis = sceneView.GetProperty(Interop.AlphaMaskURLGet()); + invertYAxis?.Get(out returnValue); + invertYAxis?.Dispose(); + return returnValue; + } + + internal static void SetMaskContentScaleFactor(this SceneView sceneView, float factor) + { + var setValue = new Tizen.NUI.PropertyValue(factor); + sceneView.SetProperty(Interop.MaskContentScaleGet(), setValue); + setValue.Dispose(); + } + + internal static float GetMaskContentScaleFactor(this SceneView sceneView) + { + var returnValue = 0.0f; + var invertYAxis = sceneView.GetProperty(Interop.MaskContentScaleGet()); + invertYAxis?.Get(out returnValue); + invertYAxis?.Dispose(); + return returnValue; + } + + internal static void EnableCropToMask(this SceneView sceneView, bool enableCropToMask) + { + var setValue = new Tizen.NUI.PropertyValue(enableCropToMask); + sceneView.SetProperty(Interop.CropToMaskGet(), setValue); + setValue.Dispose(); + } + + internal static bool IsEnabledCropToMask(this SceneView sceneView) + { + bool returnValue = false; + var invertYAxis = sceneView.GetProperty(Interop.CropToMaskGet()); + invertYAxis?.Get(out returnValue); + invertYAxis?.Dispose(); + return returnValue; + } + } +} diff --git a/src/Tizen.AIAvatar/src/internal/Avatar/AIAvatar.cs b/src/Tizen.AIAvatar/src/internal/Avatar/AIAvatar.cs new file mode 100644 index 0000000..0002bc3 --- /dev/null +++ b/src/Tizen.AIAvatar/src/internal/Avatar/AIAvatar.cs @@ -0,0 +1,42 @@ +/* + * Copyright(c) 2023 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using Tizen.Multimedia; + +namespace Tizen.AIAvatar +{ + internal static class AIAvatar + { + internal const string LogTag = "Tizen.AIAvatar"; + internal static readonly string ApplicationResourcePath = "/usr/apps/org.tizen.default-avatar-resource/shared/res/"; + internal static readonly string EmojiAvatarResourcePath = "/models/EmojiAvatar/"; + internal static readonly string DefaultModelResourcePath = "/models/DefaultAvatar/"; + internal static readonly string DefaultMotionResourcePath = "/animation/motion/"; + + internal static readonly string VisemeInfo = $"{ApplicationResourcePath}/viseme/emoji_viseme_info.json"; + internal static readonly string DefaultModel = "DefaultAvatar.gltf"; + + internal static readonly string AREmojiDefaultAvatarPath = $"{ApplicationResourcePath}{DefaultModelResourcePath}{DefaultModel}"; + + internal static readonly string DefaultLowModelResourcePath = "/models/DefaultAvatar_Low/"; + internal static readonly string ExternalModel = "model_external.gltf"; + internal static readonly string AREmojiDefaultLowAvatarPath = $"{ApplicationResourcePath}{DefaultLowModelResourcePath}{ExternalModel}"; + + internal static AudioOptions DefaultAudioOptions = new AudioOptions(24000, AudioChannel.Mono, AudioSampleType.S16Le); + internal static AudioOptions CurrentAudioOptions = DefaultAudioOptions; + } +} diff --git a/src/Tizen.AIAvatar/src/internal/Avatar/Interop.SceneView.cs b/src/Tizen.AIAvatar/src/internal/Avatar/Interop.SceneView.cs new file mode 100644 index 0000000..0e116ea --- /dev/null +++ b/src/Tizen.AIAvatar/src/internal/Avatar/Interop.SceneView.cs @@ -0,0 +1,37 @@ +/* + * Copyright(c) 2023 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +namespace Tizen.AIAvatar +{ + internal class Interop + { + internal static partial class Libraries + { + public const string Scene3D = "libdali2-csharp-binder-scene3d.so"; + } + + [global::System.Runtime.InteropServices.DllImport(Libraries.Scene3D, EntryPoint = "CSharp_Dali_SceneView_Property_AlphaMaskURL_get")] + public static extern int AlphaMaskURLGet(); + + [global::System.Runtime.InteropServices.DllImport(Libraries.Scene3D, EntryPoint = "CSharp_Dali_SceneView_Property_MaskContentScale_get")] + public static extern int MaskContentScaleGet(); + + [global::System.Runtime.InteropServices.DllImport(Libraries.Scene3D, EntryPoint = "CSharp_Dali_SceneView_Property_CropToMask_get")] + public static extern int CropToMaskGet(); + + } +} diff --git a/src/Tizen.AIAvatar/src/internal/Avatar/Utils.cs b/src/Tizen.AIAvatar/src/internal/Avatar/Utils.cs new file mode 100644 index 0000000..7f1acf9 --- /dev/null +++ b/src/Tizen.AIAvatar/src/internal/Avatar/Utils.cs @@ -0,0 +1,127 @@ +/* + * Copyright(c) 2023 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using System; +using System.IO; +using Tizen.Security; +using Newtonsoft.Json; +using System.Collections.Generic; +using static Tizen.AIAvatar.AIAvatar; + +namespace Tizen.AIAvatar +{ + internal class Utils + { + internal static void ConvertAudioToFloat(in byte[] audioBytes, out float[] audioFloat) + { + audioFloat = new float[audioBytes.Length / 2]; + + for (int i = 0, j = 0; i < audioBytes.Length; i += 2, j++) + { + short sample = BitConverter.ToInt16(audioBytes, i); + audioFloat[j] = sample / 32768.0f; + } + } + + internal static byte[] ReadAllBytes(string path) + { + try + { + var bytes = File.ReadAllBytes(path); + return bytes; + } + catch (Exception) + { + return null; + } + } + + internal static void SaveFile(string path, string filename, byte[] array) + { + try + { + var file = new FileStream($"{path}/{filename}", FileMode.Create); + file.Write(array, 0, array.Length); + file.Close(); + } + catch (Exception) + { + return; + } + } + + internal static void CheckPrivilege(string privilege) + { + var result = PrivacyPrivilegeManager.CheckPermission(privilege); + + switch (result) + { + case CheckResult.Allow: + Log.Info(LogTag, $"Privilege \"{privilege}\" : allowed."); + break; + case CheckResult.Deny: + Log.Info(LogTag, $"Privilege \"{privilege}\" : denied."); + /// Privilege can't be used + break; + case CheckResult.Ask: + /// Request permission to user + PrivacyPrivilegeManager.RequestPermission(privilege); + break; + } + } + + internal static T ConvertJson(string jsonString) + { + return JsonConvert.DeserializeObject(jsonString); + } + + internal int FindMaxValue(List list, Converter projection) + { + if (list.Count == 0) + { + throw new InvalidOperationException("Empty list"); + } + int maxValue = int.MinValue; + foreach (T item in list) + { + int value = projection(item); + if (value > maxValue) + { + maxValue = value; + } + } + return maxValue; + } + } +} + +internal class SystemUtils +{ + internal static void ST() + { + Tizen.Log.Error("MYLOG", System.Environment.StackTrace); + } + + internal static string GetFileName(string path) + { + return System.IO.Path.GetFileName(path); + } + internal static string GetFileNameWithoutExtension(string path) + { + return System.IO.Path.GetFileNameWithoutExtension(path); + } +} diff --git a/src/Tizen.AIAvatar/src/internal/BlendShapeInfo/BlendShapeInfo.cs b/src/Tizen.AIAvatar/src/internal/BlendShapeInfo/BlendShapeInfo.cs new file mode 100644 index 0000000..71e0ad1 --- /dev/null +++ b/src/Tizen.AIAvatar/src/internal/BlendShapeInfo/BlendShapeInfo.cs @@ -0,0 +1,58 @@ +/* + * Copyright(c) 2023 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using System.Collections.Generic; +using System.ComponentModel; +using static Tizen.AIAvatar.AIAvatar; + +namespace Tizen.AIAvatar +{ + internal class BlendShapeInfo + { + internal BlendShapeModelInfo blendShape; + internal BlendShapeVisemeInfo[] visemes; + + internal BlendShapeInfo() { } + + internal string[] GetNodeNames() + { + if (blendShape == null) + { + Log.Error(LogTag, "blendShape is null"); + return null; + } + return blendShape.nodeNames; + } + + internal int[] GetBlendShapeCounts() + { + return blendShape.blendShapeCount; + } + + internal Dictionary GetVisemeMap() + { + Dictionary visemeMap = new Dictionary(); + + foreach (var visemeInfo in visemes) + { + visemeMap.Add(visemeInfo.name, visemeInfo.values); + } + + return visemeMap; + } + } +} diff --git a/src/Tizen.AIAvatar/src/internal/BlendShapeInfo/BlendShapeModelInfo.cs b/src/Tizen.AIAvatar/src/internal/BlendShapeInfo/BlendShapeModelInfo.cs new file mode 100644 index 0000000..df8e125 --- /dev/null +++ b/src/Tizen.AIAvatar/src/internal/BlendShapeInfo/BlendShapeModelInfo.cs @@ -0,0 +1,35 @@ +/* + * Copyright(c) 2023 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using System.ComponentModel; + +namespace Tizen.AIAvatar +{ + internal class BlendShapeModelInfo + { + internal string keyFormat; + internal string[] nodeNames; + internal int[] blendShapeCount; + + internal BlendShapeModelInfo(string keyFormat, string[] nodeNames, int[] blendShapeCount) + { + this.keyFormat = keyFormat; + this.nodeNames = nodeNames; + this.blendShapeCount = blendShapeCount; + } + } +} diff --git a/src/Tizen.AIAvatar/src/internal/BlendShapeInfo/BlendShapeValue.cs b/src/Tizen.AIAvatar/src/internal/BlendShapeInfo/BlendShapeValue.cs new file mode 100644 index 0000000..3ce5c01 --- /dev/null +++ b/src/Tizen.AIAvatar/src/internal/BlendShapeInfo/BlendShapeValue.cs @@ -0,0 +1,28 @@ +/* + * Copyright(c) 2023 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using System.ComponentModel; + +namespace Tizen.AIAvatar +{ + internal class BlendShapeValue + { + internal string nodeName; + internal BlendShapeType blendIndex; + internal float blendValue; + } +} diff --git a/src/Tizen.AIAvatar/src/internal/BlendShapeInfo/BlendShapeVisemeInfo.cs b/src/Tizen.AIAvatar/src/internal/BlendShapeInfo/BlendShapeVisemeInfo.cs new file mode 100644 index 0000000..cc11c6a --- /dev/null +++ b/src/Tizen.AIAvatar/src/internal/BlendShapeInfo/BlendShapeVisemeInfo.cs @@ -0,0 +1,27 @@ +/* + * Copyright(c) 2023 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using System.ComponentModel; + +namespace Tizen.AIAvatar +{ + internal class BlendShapeVisemeInfo + { + internal Viseme name; + internal BlendShapeValue[] values; + } +} diff --git a/src/Tizen.AIAvatar/src/internal/Common/IRestClient.cs b/src/Tizen.AIAvatar/src/internal/Common/IRestClient.cs new file mode 100644 index 0000000..9a593b1 --- /dev/null +++ b/src/Tizen.AIAvatar/src/internal/Common/IRestClient.cs @@ -0,0 +1,28 @@ +/* + * Copyright(c) 2024 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using System.Net.Http; +using System.Threading.Tasks; + +namespace Tizen.AIAvatar +{ + + internal interface IRestClient + { + Task SendRequestAsync(HttpMethod method, string endpoint, string bearerToken = null, string jsonData = null); + } +} diff --git a/src/Tizen.AIAvatar/src/internal/Common/RestClient.cs b/src/Tizen.AIAvatar/src/internal/Common/RestClient.cs new file mode 100644 index 0000000..7d11291 --- /dev/null +++ b/src/Tizen.AIAvatar/src/internal/Common/RestClient.cs @@ -0,0 +1,78 @@ +/* + * Copyright(c) 2024 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using System; +using System.Net.Http.Headers; +using System.Net.Http; +using System.Text; +using System.Threading.Tasks; +using System.ComponentModel; + +namespace Tizen.AIAvatar +{ + internal class RestClient : IRestClient, IDisposable + { + private readonly HttpClient client; + + internal RestClient(HttpClient httpClient) + { + client = httpClient; + } + + [EditorBrowsable(EditorBrowsableState.Never)] + public async Task SendRequestAsync(HttpMethod method, string endpoint, string bearerToken = null, string jsonData = null) + { + AddBearerToken(bearerToken); + + HttpRequestMessage request = new HttpRequestMessage(method, endpoint); + + if (jsonData != null) + { + request.Content = new StringContent(jsonData, Encoding.UTF8, "application/json"); + } + + HttpResponseMessage response = await client.SendAsync(request); + return await HandleResponse(response); + } + + [EditorBrowsable(EditorBrowsableState.Never)] + public void Dispose() + { + client.Dispose(); + } + + private void AddBearerToken(string bearerToken) + { + if (!string.IsNullOrEmpty(bearerToken)) + { + client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", bearerToken); + } + } + + private async Task HandleResponse(HttpResponseMessage response) + { + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadAsStringAsync(); + } + else + { + throw new HttpRequestException($"HTTP request failed with status code {response.StatusCode}"); + } + } + } +} diff --git a/src/Tizen.AIAvatar/src/internal/Common/RestClientFactory.cs b/src/Tizen.AIAvatar/src/internal/Common/RestClientFactory.cs new file mode 100644 index 0000000..07b7fa1 --- /dev/null +++ b/src/Tizen.AIAvatar/src/internal/Common/RestClientFactory.cs @@ -0,0 +1,30 @@ +/* + * Copyright(c) 2024 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using System.Net.Http; +using System; + +namespace Tizen.AIAvatar +{ + internal class RestClientFactory + { + internal IRestClient CreateClient(string baseUrl) + { + return new RestClient(new HttpClient { BaseAddress = new Uri(baseUrl) }); + } + } +} diff --git a/src/Tizen.AIAvatar/src/internal/DefaultAvatar/DefaultAvatarProperties.cs b/src/Tizen.AIAvatar/src/internal/DefaultAvatar/DefaultAvatarProperties.cs new file mode 100644 index 0000000..d8e6ef3 --- /dev/null +++ b/src/Tizen.AIAvatar/src/internal/DefaultAvatar/DefaultAvatarProperties.cs @@ -0,0 +1,200 @@ +/* + * Copyright(c) 2023 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using System; +using System.Collections.Generic; +using System.Text; +using Tizen.NUI; +using Tizen.NUI.Scene3D; + +namespace Tizen.AIAvatar +{ + internal class DefaultAvatarProperties : AvatarProperties + { + private static AvatarPropertyMapper defaultJointMapper = new AvatarPropertyMapper(); + private static AvatarPropertyMapper defaultBlendShapeNameMapper = new AvatarPropertyMapper(); + private static AvatarPropertyMapper defaultNodeMapper = new AvatarPropertyMapper(); + + + internal DefaultAvatarProperties() : base(defaultJointMapper, defaultBlendShapeNameMapper, defaultNodeMapper) + { + foreach (var indexNamePair in blendShapeList) + { + defaultBlendShapeNameMapper.SetPropertyName((uint)indexNamePair.Item1, indexNamePair.Item2); + } + + foreach (var indexNamePair in jointList) + { + defaultJointMapper.SetPropertyName((uint)indexNamePair.Item1, indexNamePair.Item2); + } + + foreach (var indexNamePair in nodeList) + { + defaultNodeMapper.SetPropertyName((uint)indexNamePair.Item1, indexNamePair.Item2); + } + } + + #region AR Emoji BlendShape name list + private static readonly List<(BlendShapeType, string)> blendShapeList = new List<(BlendShapeType, string)>() + { + (BlendShapeType.EyeBlinkLeft, "EyeBlink_Left"), + (BlendShapeType.EyeSquintLeft, "EyeSquint_Left"), + (BlendShapeType.EyeDownLeft, "EyeDown_Left"), + (BlendShapeType.EyeInLeft, "EyeIn_Left"), + (BlendShapeType.EyeOpenLeft, "EyeOpen_Left"), + (BlendShapeType.EyeOutLeft, "EyeOut_Left"), + (BlendShapeType.EyeUpLeft, "EyeUp_Left"), + + (BlendShapeType.EyeBlinkRight, "EyeBlink_Right"), + (BlendShapeType.EyeSquintRight, "EyeSquint_Right"), + (BlendShapeType.EyeDownRight, "EyeDown_Right"), + (BlendShapeType.EyeInRight, "EyeIn_Right"), + (BlendShapeType.EyeOpenRight, "EyeOpen_Right"), + (BlendShapeType.EyeOutRight, "EyeOut_Right"), + (BlendShapeType.EyeUpRight, "EyeUp_Right"), + + (BlendShapeType.JawForward, "JawFwd"), + (BlendShapeType.JawOpen, "JawOpen"), + (BlendShapeType.JawChew, "JawChew"), + (BlendShapeType.JawLeft, "JawLeft"), + (BlendShapeType.JawRight, "JawRight"), + + (BlendShapeType.MouthLeft, "MouthLeft"), + (BlendShapeType.MouthFrownLeft, "MouthFrown_Left"), + (BlendShapeType.MouthSmileLeft, "MouthSmile_Left"), + (BlendShapeType.MouthDimpleLeft, "MouthDimple_Left"), + + (BlendShapeType.MouthRight, "MouthRight"), + (BlendShapeType.MouthFrownRight, "MouthFrown_Right"), + (BlendShapeType.MouthSmileRight, "MouthSmile_Right"), + (BlendShapeType.MouthDimpleRight, "MouthDimple_Right"), + + (BlendShapeType.LipsStretchLeft, "LipsStretch_Left"), + (BlendShapeType.LipsStretchRight, "LipsStretch_Right"), + (BlendShapeType.LipsUpperClose, "LipsUpperClose"), + (BlendShapeType.LipsLowerClose, "LipsLowerClose"), + (BlendShapeType.LipsUpperUp, "LipsUpperUp"), + (BlendShapeType.LipsLowerDown, "LipsLowerDown"), + (BlendShapeType.LipsUpperOpen, "LipsUpperOpen"), + (BlendShapeType.LipsLowerOpen, "LipsLowerOpen"), + (BlendShapeType.LipsFunnel, "LipsFunnel"), + (BlendShapeType.LipsPucker, "LipsPucker"), + + (BlendShapeType.BrowDownLeft, "BrowsDown_Left"), + (BlendShapeType.BrowDownRight, "BrowsDown_Right"), + (BlendShapeType.BrowUpCenter, "BrowsUp_Center"), + (BlendShapeType.BrowUpLeft, "BrowsUp_Left"), + (BlendShapeType.BrowUpRight, "BrowsUp_Right"), + (BlendShapeType.CheekSquintLeft, "CheekSquint_Left"), + (BlendShapeType.CheekSquintRight, "CheekSquint_Right"), + (BlendShapeType.ChinLowerRaise, "ChinLowerRaise"), + (BlendShapeType.ChinUpperRaise, "ChinUpperRaise"), + + (BlendShapeType.TongueOut, "Tongue_Out"), + (BlendShapeType.TongueUp, "Tongue_Up"), + (BlendShapeType.TongueDown, "Tongue_Down"), + (BlendShapeType.TongueLeft, "Tongue_Left"), + (BlendShapeType.TongueRight, "Tongue_Right"), + + (BlendShapeType.Sneer, "Sneer"), + (BlendShapeType.Puff, "Puff"), + (BlendShapeType.PuffLeft, "Puff_Left"), + (BlendShapeType.PuffRight, "Puff_Right"), + }; + #endregion + + #region AR Emoji Joint name list + private static readonly List<(JointType, string)> jointList = new List<(JointType, string)> + { + (JointType.Head, "head_JNT"), + (JointType.Neck, "neck_JNT"), + (JointType.EyeLeft, "l_eye_JNT"), + (JointType.EyeRight, "r_eye_JNT"), + + (JointType.ShoulderLeft, "l_arm_JNT"), + (JointType.ElbowLeft, "l_forearm_JNT"), + (JointType.WristLeft, "l_hand_JNT"), + + (JointType.ShoulderRight, "r_arm_JNT"), + (JointType.ElbowRight, "r_forearm_JNT"), + (JointType.WristRight, "r_hand_JNT"), + + (JointType.HipLeft, "l_upleg_JNT"), + (JointType.KneeLeft, "l_leg_JNT"), + (JointType.AnkleLeft, "l_foot_JNT"), + (JointType.ForeFootLeft, "l_toebase_JNT"), + + (JointType.HipRight, "r_upleg_JNT"), + (JointType.KneeRight, "r_leg_JNT"), + (JointType.AnkleRight, "r_foot_JNT"), + (JointType.ForeFootRight, "r_toebase_JNT"), + + (JointType.FingerThumb1Left, "l_handThumb1_JNT"), + (JointType.FingerThumb2Left, "l_handThumb2_JNT"), + (JointType.FingerThumb3Left, "l_handThumb3_JNT"), + (JointType.FingerThumb4Left, "l_handThumb4_JNT"), + (JointType.FingerIndex1Left, "l_handIndex1_JNT"), + (JointType.FingerIndex2Left, "l_handIndex2_JNT"), + (JointType.FingerIndex3Left, "l_handIndex3_JNT"), + (JointType.FingerIndex4Left, "l_handIndex4_JNT"), + (JointType.FingerMiddle1Left, "l_handMiddle1_JNT"), + (JointType.FingerMiddle2Left, "l_handMiddle2_JNT"), + (JointType.FingerMiddle3Left, "l_handMiddle3_JNT"), + (JointType.FingerMiddle4Left, "l_handMiddle4_JNT"), + (JointType.FingerRing1Left, "l_handRing1_JNT"), + (JointType.FingerRing2Left, "l_handRing2_JNT"), + (JointType.FingerRing3Left, "l_handRing3_JNT"), + (JointType.FingerRing4Left, "l_handRing4_JNT"), + (JointType.FingerPinky1Left, "l_handPinky1_JNT"), + (JointType.FingerPinky2Left, "l_handPinky2_JNT"), + (JointType.FingerPinky3Left, "l_handPinky3_JNT"), + (JointType.FingerPinky4Left, "l_handPinky4_JNT"), + + (JointType.FingerThumb1Right, "r_handThumb1_JNT"), + (JointType.FingerThumb2Right, "r_handThumb2_JNT"), + (JointType.FingerThumb3Right, "r_handThumb3_JNT"), + (JointType.FingerThumb4Right, "r_handThumb4_JNT"), + (JointType.FingerIndex1Right, "r_handIndex1_JNT"), + (JointType.FingerIndex2Right, "r_handIndex2_JNT"), + (JointType.FingerIndex3Right, "r_handIndex3_JNT"), + (JointType.FingerIndex4Right, "r_handIndex4_JNT"), + (JointType.FingerMiddle1Right, "r_handMiddle1_JNT"), + (JointType.FingerMiddle2Right, "r_handMiddle2_JNT"), + (JointType.FingerMiddle3Right, "r_handMiddle3_JNT"), + (JointType.FingerMiddle4Right, "r_handMiddle4_JNT"), + (JointType.FingerRing1Right, "r_handRing1_JNT"), + (JointType.FingerRing2Right, "r_handRing2_JNT"), + (JointType.FingerRing3Right, "r_handRing3_JNT"), + (JointType.FingerRing4Right, "r_handRing4_JNT"), + (JointType.FingerPinky1Right, "r_handPinky1_JNT"), + (JointType.FingerPinky2Right, "r_handPinky2_JNT"), + (JointType.FingerPinky3Right, "r_handPinky3_JNT"), + (JointType.FingerPinky4Right, "r_handPinky4_JNT"), + }; + #endregion + + #region AR Emoji Joint name list + private static readonly List<(NodeType, string)> nodeList = new List<(NodeType, string)> + { + (NodeType.HeadGeo, "head_GEO"), + (NodeType.MouthGeo, "mouth_GEO"), + (NodeType.EyelashGeo, "eyelash_GEO"), + }; + #endregion + + } +} diff --git a/src/Tizen.AIAvatar/src/internal/LipSync/LipInfo.cs b/src/Tizen.AIAvatar/src/internal/LipSync/LipInfo.cs new file mode 100644 index 0000000..8d785c9 --- /dev/null +++ b/src/Tizen.AIAvatar/src/internal/LipSync/LipInfo.cs @@ -0,0 +1,32 @@ +/* + * Copyright(c) 2023 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using Tizen.NUI; + +namespace Tizen.AIAvatar +{ + internal class LipInfo + { + private Animatable animatable; + private string nodeName; + private int blendShapeCount; + + internal Animatable Animatable { get => animatable; set => animatable = value; } + internal string NodeName { get => nodeName; set => nodeName = value; } + internal int BlendShapeCount { get => blendShapeCount; set => blendShapeCount = value; } + } +} diff --git a/src/Tizen.AIAvatar/src/internal/LipSync/Models/ISingleShotModel.cs b/src/Tizen.AIAvatar/src/internal/LipSync/Models/ISingleShotModel.cs new file mode 100644 index 0000000..cf2cb8e --- /dev/null +++ b/src/Tizen.AIAvatar/src/internal/LipSync/Models/ISingleShotModel.cs @@ -0,0 +1,16 @@ + +using System.ComponentModel; + +namespace Tizen.AIAvatar +{ + internal interface ISingleShotModel + { + + [EditorBrowsable(EditorBrowsableState.Never)] + public void SetTensorData(int index, byte[] buffer); + [EditorBrowsable(EditorBrowsableState.Never)] + public byte[] GetTensorData(int index); + [EditorBrowsable(EditorBrowsableState.Never)] + public void Invoke(); + } +} diff --git a/src/Tizen.AIAvatar/src/internal/LipSync/Models/SoftmaxLinqExtension.cs b/src/Tizen.AIAvatar/src/internal/LipSync/Models/SoftmaxLinqExtension.cs new file mode 100644 index 0000000..de471a3 --- /dev/null +++ b/src/Tizen.AIAvatar/src/internal/LipSync/Models/SoftmaxLinqExtension.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Tizen.AIAvatar +{ + internal static class SoftmaxLinqExtension + { + internal static IEnumerable SoftMax(this IEnumerable source) + { + var exp = source.Select(x => MathF.Exp(x)).ToArray(); + var sum = exp.Sum(); + return exp.Select(x => x / sum); + } + } +} diff --git a/src/Tizen.AIAvatar/src/internal/LipSync/Models/TFVowel6.cs b/src/Tizen.AIAvatar/src/internal/LipSync/Models/TFVowel6.cs new file mode 100644 index 0000000..1061a85 --- /dev/null +++ b/src/Tizen.AIAvatar/src/internal/LipSync/Models/TFVowel6.cs @@ -0,0 +1,82 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Text; +using Tizen.MachineLearning.Inference; +using static Tizen.AIAvatar.AIAvatar; + +namespace Tizen.AIAvatar +{ + internal class TFVowel6 : ISingleShotModel + { + private SingleShot singleShot; + private TensorsInfo inputInfo; + private TensorsInfo outputInfo; + private TensorsData inputData; + private TensorsData outputData; + + private readonly string modelPath = ApplicationResourcePath + "audio2vowel_7.tflite"; + + internal TFVowel6(int[] inputDimension, int[] outputDimension) + { + try + { + inputInfo = new TensorsInfo(); + inputInfo.AddTensorInfo(TensorType.Float32, inputDimension); + + outputInfo = new TensorsInfo(); + outputInfo.AddTensorInfo(TensorType.Float32, outputDimension); + + singleShot = new SingleShot(modelPath, inputInfo, outputInfo); + } + catch (Exception e) + { + if (e is NotSupportedException) + { + Log.Info(LogTag, "NotSupportedException occurs"); + } + else + { + Log.Info(LogTag, e.Message); + } + } + } + + + [EditorBrowsable(EditorBrowsableState.Never)] + public void SetTensorData(int index, byte[] buffer) + { + try + { + inputData = inputInfo.GetTensorsData(); + inputData.SetTensorData(index, buffer); + } + catch (Exception e) + { + Log.Info(LogTag, e.Message); + } + } + + [EditorBrowsable(EditorBrowsableState.Never)] + public void Invoke() + { + try + { + outputData = singleShot.Invoke(inputData); + } + catch (Exception e) + { + Log.Info(LogTag, e.Message); + } + + } + + [EditorBrowsable(EditorBrowsableState.Never)] + public byte[] GetTensorData(int index) + { + return outputData.GetTensorData(index); + } + + + } +} diff --git a/src/Tizen.AIAvatar/src/internal/LipSync/Viseme.cs b/src/Tizen.AIAvatar/src/internal/LipSync/Viseme.cs new file mode 100644 index 0000000..431c500 --- /dev/null +++ b/src/Tizen.AIAvatar/src/internal/LipSync/Viseme.cs @@ -0,0 +1,35 @@ +/* + * Copyright(c) 2023 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +namespace Tizen.AIAvatar +{ + // Various visemes + internal enum Viseme + { + sil, + AE, + Ah, + B_M_P, + Ch_J, + EE, + Er, + IH, + Oh, + W_OO, + R, + }; +} diff --git a/src/Tizen.AIAvatar/src/internal/Multimedia/AudioPlayer.cs b/src/Tizen.AIAvatar/src/internal/Multimedia/AudioPlayer.cs new file mode 100644 index 0000000..0c631d7 --- /dev/null +++ b/src/Tizen.AIAvatar/src/internal/Multimedia/AudioPlayer.cs @@ -0,0 +1,135 @@ +/* + * Copyright(c) 2023 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using Tizen.Multimedia; +using System.IO; +using System; + +using static Tizen.AIAvatar.AIAvatar; + +namespace Tizen.AIAvatar +{ + internal class AudioPlayer + { + private AudioPlayback audioPlayback; + private MemoryStream audioStream; + + internal AudioPlayer() + { + } + + internal void PlayAsync(byte[] buffer, int sampleRate = 0) + { + if (audioPlayback == null) + { + Play(buffer, sampleRate); + } + else + { + audioPlayback.Write(buffer); + } + } + + internal void Play(byte[] audioBytes, int sampleRate = 0) + { + if (audioBytes == null) + { + return; + } + + try + { + if (audioPlayback != null) + { + DestroyAudioPlayback(); + } + if (sampleRate == 0) + { + sampleRate = CurrentAudioOptions.SampleRate; + } + audioPlayback = new AudioPlayback(sampleRate, CurrentAudioOptions.Channel, CurrentAudioOptions.SampleType); + } + catch (Exception e) + { + Log.Error(LogTag, $"Failed to create AudioPlayback. {e.Message}"); + return; + } + + if (audioPlayback != null) + { + audioPlayback.Prepare(); + audioPlayback.BufferAvailable += (sender, args) => + { + if (audioStream.Position == audioStream.Length) + { + return; + } + + try + { + var buffer = new byte[args.Length]; + audioStream.Read(buffer, 0, args.Length); + audioPlayback.Write(buffer); + } + catch (Exception e) + { + Log.Error(LogTag, $"Failed to write. {e.Message}"); + } + }; + + audioStream = new MemoryStream(audioBytes); + } + } + + internal void Pause() + { + if (audioPlayback != null) + { + audioPlayback.Pause(); + } + else + { + Log.Error(LogTag, $"audioPlayBack is null"); + } + } + + internal void Stop() + { + if (audioPlayback != null) + { + audioPlayback.Pause(); + DestroyAudioPlayback(); + } + else + { + Log.Error(LogTag, $"audioPlayBack is null"); + } + } + + internal void Destroy() + { + DestroyAudioPlayback(); + } + + private void DestroyAudioPlayback() + { + audioPlayback?.Unprepare(); + audioPlayback?.Dispose(); + audioPlayback = null; + } + } +} diff --git a/src/Tizen.AIAvatar/src/internal/Multimedia/AudioRecorder.cs b/src/Tizen.AIAvatar/src/internal/Multimedia/AudioRecorder.cs new file mode 100644 index 0000000..e3e089a --- /dev/null +++ b/src/Tizen.AIAvatar/src/internal/Multimedia/AudioRecorder.cs @@ -0,0 +1,151 @@ +/* + * Copyright(c) 2023 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using System; +using System.Linq; +using System.Text; +using Tizen.Multimedia; +using Tizen.NUI; +using static Tizen.AIAvatar.AIAvatar; + +namespace Tizen.AIAvatar +{ + internal class AudioRecorder + { + private const string privilegeForRecording = "http://tizen.org/privilege/recorder"; + + private AsyncAudioCapture asyncAudioCapture; + + private byte[] recordedBuffer; + private float desiredBufferDuration = 0.16f; + private int desiredBufferLength; + + private Timer audioRecordingTimer; + + private Action audioRecdingAction; + private Action bufferAction; + + private static AudioRecorder instance; + + internal static AudioRecorder Instance + { + get + { + if (instance == null) + { + instance = new AudioRecorder(); + } + return instance; + } + } + + internal event EventHandler BufferChanged; + + internal AudioRecorder() + { + Utils.CheckPrivilege(privilegeForRecording); + desiredBufferLength = (int)(CurrentAudioOptions.SampleRate * desiredBufferDuration * 2); + } + + + internal void InitMic(BlendShapePlayer animator, uint recordingTime = 160) + { + audioRecordingTimer = new Timer(recordingTime); + if (animator != null) + { + //TODO : Connection MIC + var lipSyncer = (animator.GetAnimationModule(AnimationModuleType.LipSyncer) as LipSyncer); + if(lipSyncer != null) + { + Tizen.Log.Error(LogTag, "LipSyncer of animator is null"); + return; + } + this.audioRecdingAction = lipSyncer.OnRecodingTick; + this.bufferAction = lipSyncer.OnRecordBufferChanged; + + BufferChanged += OnRecordBufferChanged; + audioRecordingTimer.Tick += AudioRecordingTimerTick; + } + } + + + internal void DeinitMic() + { + StopRecording(); + BufferChanged -= OnRecordBufferChanged; + + if (audioRecordingTimer != null) + { + audioRecordingTimer.Stop(); + audioRecordingTimer.Tick -= AudioRecordingTimerTick; + + audioRecordingTimer.Dispose(); + audioRecordingTimer = null; + } + audioRecdingAction = null; + } + + internal void StartRecording() + { + audioRecordingTimer?.Start(); + asyncAudioCapture = new AsyncAudioCapture(CurrentAudioOptions.SampleRate, CurrentAudioOptions.Channel, CurrentAudioOptions.SampleType); + + recordedBuffer = new byte[0]; + asyncAudioCapture.DataAvailable += (s, e) => + { + recordedBuffer = recordedBuffer.Concat(e.Data).ToArray(); + if (recordedBuffer.Length >= desiredBufferLength) + { + var recordedBuffer = this.recordedBuffer; + this.recordedBuffer = Array.Empty(); + + BufferChanged?.Invoke(this, new RecordBufferChangedEventArgs(recordedBuffer, CurrentAudioOptions.SampleRate)); + } + }; + asyncAudioCapture.Prepare(); + Log.Info(LogTag, "Start Recording - Preapre AsyncAudioCapture"); + } + + internal void StopRecording() + { + audioRecordingTimer?.Stop(); + asyncAudioCapture.Dispose(); + } + + internal void PauseRecording() + { + asyncAudioCapture.Pause(); + } + + internal void ResumeRecording() + { + asyncAudioCapture.Resume(); + } + + private void OnRecordBufferChanged(object sender, RecordBufferChangedEventArgs e) + { + bufferAction?.Invoke(e.RecordedBuffer, CurrentAudioOptions.SampleRate); + } + + private bool AudioRecordingTimerTick(object source, Timer.TickEventArgs e) + { + Log.Info(LogTag, "TickTimer"); + audioRecdingAction?.Invoke(); + return true; + } + } +} diff --git a/src/Tizen.AIAvatar/src/internal/Multimedia/RecordBufferChangedEventArgs.cs b/src/Tizen.AIAvatar/src/internal/Multimedia/RecordBufferChangedEventArgs.cs new file mode 100644 index 0000000..70bed45 --- /dev/null +++ b/src/Tizen.AIAvatar/src/internal/Multimedia/RecordBufferChangedEventArgs.cs @@ -0,0 +1,34 @@ +/* + * Copyright(c) 2023 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using System; + +namespace Tizen.AIAvatar +{ + internal class RecordBufferChangedEventArgs : EventArgs + { + public byte[] RecordedBuffer { get; set; } + public int SampleRate { get; set; } + + public RecordBufferChangedEventArgs(byte[] recordedBuffer, int SampleRate) + { + this.RecordedBuffer = recordedBuffer; + this.SampleRate = SampleRate; + } + } + +} diff --git a/src/Tizen.AIAvatar/src/internal/Uix/TTSLipSyncer.cs b/src/Tizen.AIAvatar/src/internal/Uix/TTSLipSyncer.cs new file mode 100644 index 0000000..1049463 --- /dev/null +++ b/src/Tizen.AIAvatar/src/internal/Uix/TTSLipSyncer.cs @@ -0,0 +1,498 @@ +/* + * Copyright(c) 2023 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using System; +using System.Collections.Generic; +using Tizen.Uix.Tts; +using System.Runtime.InteropServices; + +using static Tizen.AIAvatar.AIAvatar; +using System.Linq; +using Tizen.NUI; + +namespace Tizen.AIAvatar +{ + internal class TTSLipSyncer + { + + private Avatar currentAvatar; + + private List textList; + private TtsClient ttsHandle; + private VoiceInfo voiceInfo; + private List byteList; + + private byte[] recordedBuffer; + private byte[] audioTailBuffer; + + private int sampleRate; + private float desiredBufferDuration = 0.175f; + private float audioTailLengthFactor = 0.015f; + private float audioBufferMultiflier = 2f; + + private int desiredBufferLength; + private int audioTailLength; + + private bool isPrepared = false; + private bool isAsync = false; + + private Action bufferChangedAction; + + private int audioLength; + private bool isAsyncLipStarting; + + private AsyncLipSyncer lipSyncer; + + + internal TTSLipSyncer(Avatar avatar) + { + currentAvatar = avatar; + lipSyncer = (currentAvatar.AvatarAnimator.GetAnimationModule(AnimationModuleType.LipSyncer) as AsyncLipSyncer); + InitTts(); + } + + ~TTSLipSyncer() + { + DeinitTts(); + } + + internal event EventHandler PlayReadyCallback; + + internal TtsClient TtsHandle + { + get { return ttsHandle; } + } + + internal VoiceInfo VoiceInfo + { + get { return voiceInfo; } + set + { + voiceInfo = value; + } + } + + internal List GetSupportedVoices() + { + var voiceInfoList = new List(); + + if (ttsHandle == null) + { + Log.Error(LogTag, $"ttsHandle is null"); + return voiceInfoList; + } + + var supportedVoices = ttsHandle.GetSupportedVoices(); + foreach (var supportedVoice in supportedVoices) + { + Log.Info(LogTag, $"{supportedVoice.Language} & {supportedVoice.VoiceType} is supported"); + voiceInfoList.Add(new VoiceInfo() { Lang = supportedVoice.Language, Type = supportedVoice.VoiceType }); + } + return voiceInfoList; + } + + internal bool IsSupportedVoice(string lang) + { + if (ttsHandle == null) + { + Log.Error(LogTag, $"ttsHandle is null"); + return false; + } + var supportedVoices = ttsHandle.GetSupportedVoices(); + + foreach (var supportedVoice in supportedVoices) + { + if (supportedVoice.Language.Equals(lang)) + { + Log.Info(LogTag, $"{lang} is supported"); + return true; + } + } + return false; + } + + internal bool IsSupportedVoice(VoiceInfo voiceInfo) + { + if (ttsHandle == null) + { + Log.Error(LogTag, $"ttsHandle is null"); + return false; + } + var supportedVoices = ttsHandle.GetSupportedVoices(); + foreach (var supportedVoice in supportedVoices) + { + if (supportedVoice.Language.Equals(voiceInfo.Lang) && (supportedVoice.VoiceType == voiceInfo.Type)) + { + Log.Info(LogTag, $"{voiceInfo.Lang} & {voiceInfo.Type} is supported"); + return true; + } + } + return false; + } + + + internal void AddText(string txt, VoiceInfo voiceInfo) + { + if (voiceInfo.Lang == null || voiceInfo.Type == null) + { + Log.Error(LogTag, "VoiceInfo's value is null"); + } + if (ttsHandle == null) + { + Log.Error(LogTag, $"ttsHandle is null"); + return; + } + var temp = new UtteranceText(); + temp.Text = txt; + temp.UttID = ttsHandle.AddText(txt, voiceInfo.Lang, (int)voiceInfo.Type, 0); + try + { + textList.Add(temp); + } + catch (Exception e) + { + Log.Error(LogTag, $"Error AddText" + e.Message); + } + } + + internal void AddText(string txt, string lang) + { + if (ttsHandle == null) + { + Log.Error(LogTag, $"ttsHandle is null"); + return; + } + var temp = new UtteranceText(); + temp.Text = txt; + temp.UttID = ttsHandle.AddText(txt, lang, (int)voiceInfo.Type, 0); + try + { + textList.Add(temp); + } + catch (Exception e) + { + Log.Error(LogTag, $"Error AddText" + e.Message); + } + } + + internal void Prepare(EventHandler playReadyCallback) + { + if (ttsHandle == null) + { + Log.Error(LogTag, $"ttsHandle is null"); + return; + } + Log.Info(LogTag, "Prepare TTS"); + isPrepared = true; + isAsync = false; + PlayReadyCallback = playReadyCallback; + Play(true); + } + + internal bool PlayPreparedText() + { + if (byteList != null && byteList.Count != 0) + { + Log.Info(LogTag, "PlayPreparedText TTS"); + currentAvatar?.AvatarAnimator?.PlayLipSync(byteList.ToArray(), sampleRate); + return true; + } + return false; + } + + internal void Play(bool isPrepared = false) + { + if (ttsHandle == null) + { + Log.Error(LogTag, $"ttsHandle is null"); + return; + } + + this.isPrepared = isPrepared; + isAsync = false; + ttsHandle.Play(); + } + + internal void PlayAsync(EventHandler playReadyCallback) + { + if (ttsHandle == null) + { + Log.Error(LogTag, $"ttsHandle is null"); + return; + } + + isPrepared = false; + isAsync = true; + PlayReadyCallback = playReadyCallback; + ttsHandle.Play(); + } + + public void Pause() + { + if (ttsHandle == null) + { + Log.Error(LogTag, $"ttsHandle is null"); + return; + } + ttsHandle.Pause(); + } + + internal void Stop() + { + if (ttsHandle == null) + { + Log.Error(LogTag, $"ttsHandle is null"); + return; + } + ttsHandle.Stop(); + currentAvatar?.AvatarAnimator?.StopLipSync(); + } + + private void InitTts() + { + try + { + ttsHandle = new TtsClient(); + + // Register Callbacks + ttsHandle.DefaultVoiceChanged += TtsDefaultVoiceChangedCallback; + ttsHandle.EngineChanged += TtsEngineChangedCallback; + ttsHandle.ErrorOccurred += TtsErrorOccuredCallback; + ttsHandle.StateChanged += TtsStateChangedCallback; + ttsHandle.UtteranceCompleted += TtsUtteranceCompletedCallback; + ttsHandle.UtteranceStarted += TtsUtteranceStartedCallback; + + ttsHandle.SynthesizedPcm += TtsSyntheiszedPCM; + ttsHandle.PlayingMode = PlayingMode.ByClient; + + ttsHandle.Prepare(); + + voiceInfo = new VoiceInfo + { + Lang = ttsHandle.DefaultVoice.Language, + Type = ttsHandle.DefaultVoice.VoiceType + }; + + textList = new List(); + Log.Info(LogTag, voiceInfo.Lang + ", " + voiceInfo.Type.ToString()); + + } + catch (Exception e) + { + Log.Error(LogTag, "[ERROR] Fail to prepare Tts"); + Log.Error(LogTag, e.Message); + } + } + + internal void DeinitTts() + { + try + { + if (ttsHandle != null) + { + ttsHandle.Unprepare(); + + // Unregister Callbacks + ttsHandle.DefaultVoiceChanged -= TtsDefaultVoiceChangedCallback; + ttsHandle.EngineChanged -= TtsEngineChangedCallback; + ttsHandle.ErrorOccurred -= TtsErrorOccuredCallback; + ttsHandle.StateChanged -= TtsStateChangedCallback; + ttsHandle.UtteranceCompleted -= TtsUtteranceCompletedCallback; + ttsHandle.UtteranceStarted -= TtsUtteranceStartedCallback; + + ttsHandle.Dispose(); + ttsHandle = null; + } + + if (textList != null) + { + textList.Clear(); + textList = null; + } + + if (byteList != null) + { + byteList.Clear(); + byteList = null; + } + currentAvatar = null; + } + catch (Exception e) + { + Log.Error(LogTag, "[ERROR] Fail to unprepare Tts"); + Log.Error(LogTag, e.Message); + } + } + + private void TtsSyntheiszedPCM(object sender, SynthesizedPcmEventArgs e) + { + + var dataSize = e.Data.Length; + var audio = new byte[dataSize]; + sampleRate = e.SampleRate; + + //Marshal.Copy(e.Data, audio, 0, dataSize); + switch (e.EventType) //START + { + case SynthesizedPcmEvent.Start://start + Tizen.Log.Info(LogTag, "------------------Start : " + e.UtteranceId); + Tizen.Log.Info(LogTag, "Output audio Size : " + dataSize); + Tizen.Log.Info(LogTag, "SampleRate" + e.SampleRate); + if (byteList == null) + { + byteList = new List(); + } + if (recordedBuffer == null) + { + recordedBuffer = new byte[0]; + } + byteList.Clear(); + + if (isAsync) + { + recordedBuffer = Array.Empty(); + + desiredBufferLength = (int)(e.SampleRate * desiredBufferDuration * audioBufferMultiflier); + audioTailLength = (int)(sampleRate * audioTailLengthFactor * audioBufferMultiflier); + audioTailBuffer = new byte[audioTailLength]; + PlayReadyCallback?.Invoke(null, EventArgs.Empty); + InitAsyncBuffer(); + lipSyncer.SampleRate = sampleRate; + } + break; + case SynthesizedPcmEvent.Continue://continue + if (isAsync) + { + recordedBuffer = recordedBuffer.Concat(e.Data).ToArray(); + //PlayAsync + if (recordedBuffer.Length >= desiredBufferLength) + { + Tizen.Log.Error(LogTag, "Current recordbuffer length :" + recordedBuffer.Length); + UpdateBuffer(recordedBuffer, sampleRate); + + Buffer.BlockCopy(recordedBuffer, recordedBuffer.Length - audioTailLength, audioTailBuffer, 0, audioTailLength); + + recordedBuffer = Array.Empty(); + recordedBuffer = recordedBuffer.Concat(audioTailBuffer).ToArray(); + Array.Clear(audioTailBuffer, 0, audioTailLength); + } + } + else + { + byteList.AddRange(e.Data); + } + break; + case SynthesizedPcmEvent.Finish://finish + Tizen.Log.Info(LogTag, "------------------Finish : " + e.UtteranceId); + if (!isAsync) + { + if (!isPrepared) + { + //Play voice immediately + //PlayPreparedText(); + } + else + { + //Notify finished state + Log.Info(LogTag, "Notify finished state"); + PlayReadyCallback?.Invoke(null, EventArgs.Empty); + } + } + else + { + lipSyncer.SetFinishAsyncLip(true); + } + break; + case SynthesizedPcmEvent.Fail: //fail + break; + + } + } + + private void TtsUtteranceStartedCallback(object sender, UtteranceEventArgs e) + { + Log.Debug(LogTag, "Utterance start now (" + e.UtteranceId + ")"); + } + + private void TtsUtteranceCompletedCallback(object sender, UtteranceEventArgs e) + { + Log.Debug(LogTag, "Utterance complete (" + e.UtteranceId + ")"); + + foreach (UtteranceText item in textList) + { + if (item.UttID == e.UtteranceId) + { + textList.Remove(item); + Log.Debug(LogTag, "TextList Count (" + textList.Count.ToString() + ")"); + break; + } + } + } + + private void TtsStateChangedCallback(object sender, StateChangedEventArgs e) + { + Log.Debug(LogTag, "Current state is changed from (" + e.Previous + ") to (" + e.Current + ")"); + } + + private void TtsErrorOccuredCallback(object sender, ErrorOccurredEventArgs e) + { + Log.Error(LogTag, "Error is occured (" + e.ErrorMessage + ")"); + } + + private void TtsEngineChangedCallback(object sender, EngineChangedEventArgs e) + { + Log.Debug(LogTag, "Prefered engine is changed (" + e.EngineId + ") (" + e.VoiceType.Language + ")"); + } + + private void TtsDefaultVoiceChangedCallback(object sender, DefaultVoiceChangedEventArgs e) + { + Log.Debug(LogTag, "Default voice is changed from (" + e.Previous + ") to (" + e.Current + ")"); + } + + internal void InitAsyncBuffer() + { + if (!lipSyncer.IsAsyncInit) + { + audioLength = (int)(sampleRate * 0.16f * 2f); + + lipSyncer.InitAsyncLipsync(); + lipSyncer.IsAsyncInit = true; + + lipSyncer.SetFinishAsyncLip(false); + isAsyncLipStarting = false; + } + } + + internal void UpdateBuffer(byte[] recordBuffer, int sampleRate) + { + if (lipSyncer != null) + { + Log.Error(LogTag, "OnTTSBufferChanged"); + lipSyncer.EnqueueAnimation(recordBuffer, sampleRate, audioLength); + if (!isAsyncLipStarting) + { + lipSyncer.StartAsyncLipPlayTimer(); + isAsyncLipStarting = true; + } + } + else + { + Log.Error(LogTag, "avatarLipSyncer is null"); + } + } + } +} diff --git a/src/Tizen.AIAvatar/src/internal/Uix/UtteranceText.cs b/src/Tizen.AIAvatar/src/internal/Uix/UtteranceText.cs new file mode 100644 index 0000000..0254bf9 --- /dev/null +++ b/src/Tizen.AIAvatar/src/internal/Uix/UtteranceText.cs @@ -0,0 +1,25 @@ +/* + * Copyright(c) 2023 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +namespace Tizen.AIAvatar +{ + internal struct UtteranceText + { + internal string Text; + internal int UttID; + } +} diff --git a/src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModule/AnimationModule.cs b/src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModule/AnimationModule.cs new file mode 100644 index 0000000..42f49ad --- /dev/null +++ b/src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModule/AnimationModule.cs @@ -0,0 +1,126 @@ +/* + * Copyright(c) 2024 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using System; +using System.ComponentModel; +using Tizen.NUI; +using Tizen.NUI.Scene3D; + +using static Tizen.AIAvatar.AIAvatar; + +namespace Tizen.AIAvatar +{ + [EditorBrowsable(EditorBrowsableState.Never)] + public abstract class AnimationModule + { + private AvatarMotionState currentMotionState = AvatarMotionState.Unavailable; + + private readonly Object motionChangedLock = new Object(); + private event EventHandler motionChanged; + + [EditorBrowsable(EditorBrowsableState.Never)] + public AnimationModule() + { + + } + + [EditorBrowsable(EditorBrowsableState.Never)] + public AvatarMotionState CurrentMotionState + { + get + { + return currentMotionState; + } + set + { + if (currentMotionState == value) + { + return; + } + var preState = currentMotionState; + currentMotionState = value; + if (motionChanged != null) + { + motionChanged?.Invoke(this, new AvatarMotionChangedEventArgs(preState, currentMotionState)); + } + } + } + + [EditorBrowsable(EditorBrowsableState.Never)] + public abstract void Init(Animation animation); + + [EditorBrowsable(EditorBrowsableState.Never)] + public abstract void Play(IAnimationModuleData data); + + [EditorBrowsable(EditorBrowsableState.Never)] + public abstract void Stop(); + + [EditorBrowsable(EditorBrowsableState.Never)] + public abstract void Pause(); + + [EditorBrowsable(EditorBrowsableState.Never)] + public abstract void Destroy(); + + [EditorBrowsable(EditorBrowsableState.Never)] + protected void SetCurrentState(AvatarMotionState state) + { + CurrentMotionState = state; + } + + + /// + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public event EventHandler MotionStateChanged + { + add + { + lock (motionChangedLock) + { + motionChanged += value; + } + + } + + remove + { + lock (motionChangedLock) + { + if (motionChanged == null) + { + Log.Error(LogTag, "Remove StateChanged Failed : motionChanged is null"); + return; + } + motionChanged -= value; + } + } + } + } + + [EditorBrowsable(EditorBrowsableState.Never)] + public enum AnimationModuleType + { + [EditorBrowsable(EditorBrowsableState.Never)] + EyeBlinker, + [EditorBrowsable(EditorBrowsableState.Never)] + LipSyncer, + [EditorBrowsable(EditorBrowsableState.Never)] + MotionBehavior, + [EditorBrowsable(EditorBrowsableState.Never)] + JointTransformer, + } +} diff --git a/src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModule/AsyncLipSyncer.cs b/src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModule/AsyncLipSyncer.cs new file mode 100644 index 0000000..7bab580 --- /dev/null +++ b/src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModule/AsyncLipSyncer.cs @@ -0,0 +1,181 @@ +/* + * Copyright(c) 2024 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using Tizen.NUI; + +using static Tizen.AIAvatar.AIAvatar; + +namespace Tizen.AIAvatar +{ + [EditorBrowsable(EditorBrowsableState.Never)] + internal class AsyncLipSyncer : LipSyncer + { + private readonly uint AsyncPlayTime = 160; + private Queue lipAnimations; + private Queue lipAudios; + private Timer asyncVowelTimer; + + private bool isAsyncInit = false; + private bool isFinishAsyncLip = false; + + [EditorBrowsable(EditorBrowsableState.Never)] + internal AsyncLipSyncer() + { + + } + + internal int SampleRate { get; set; } + + [EditorBrowsable(EditorBrowsableState.Never)] + internal bool IsAsyncInit { get=>isAsyncInit; set=>isAsyncInit = value; } + + [EditorBrowsable(EditorBrowsableState.Never)] + internal void SetFinishAsyncLip(bool finished) + { + isFinishAsyncLip = finished; + } + + [EditorBrowsable(EditorBrowsableState.Never)] + protected new void OnLipAnimationFinished(object sender, EventArgs e) + { + if (!isAsyncInit) + { + CurrentMotionState = AvatarMotionState.Stopped; + } + else + { + Tizen.Log.Error(LogTag, "OnLipAnimationFinished---------------c"); + //Async State + if (isFinishAsyncLip && lipAnimations.Count == 0) + { + Tizen.Log.Error(LogTag, "Finish vowel lip sync"); + + AudioPlayer.Stop(); + CurrentMotionState = AvatarMotionState.Stopped; + DestroyVowelTimer(); + isAsyncInit = false; + } + } + } + + internal void InitAsyncLipsync() + { + if (lipAnimations == null) + { + lipAnimations = new Queue(); + } + else + { + lipAnimations.Clear(); + } + + if (lipAudios == null) + { + lipAudios = new Queue(); + } + else + { + lipAudios.Clear(); + } + } + + internal void EnqueueAnimation(byte[] recordBuffer, int sampleRate, int audioLength) + { + var createdAni = CreateAsyncLipAnimation(recordBuffer, sampleRate); + if (createdAni != null) + { + lipAnimations.Enqueue(createdAni); + } + + //Use Audio Full File + ///var createdAni = avatarLipSyncer.CreateLipAnimation(recordBuffer, sampleRate); + //lipAnimations.Enqueue(createdAni); + + var currentAudioBuffer = new byte[audioLength]; + Buffer.BlockCopy(recordBuffer, 0, currentAudioBuffer, 0, audioLength); + + lipAudios.Enqueue(currentAudioBuffer); + } + + internal bool PlayAsyncLip(int sampleRate, bool isFinishAsyncLip) + { + try + { + if (lipAudios.Count <= 0 && lipAnimations.Count <= 0) + { + Tizen.Log.Info(LogTag, "Return lipaudio 0"); + if (isFinishAsyncLip) + { + Tizen.Log.Info(LogTag, "Finish Async lipsync"); + return false; + } + return true; + } + Tizen.Log.Info(LogTag, "Async timer tick lipAudios : " + lipAudios.Count); + Tizen.Log.Info(LogTag, "Async timer tick lipAnimations : " + lipAnimations.Count); + + var lipAnimation = lipAnimations.Dequeue(); + lipAnimation.Finished += OnLipAnimationFinished; + + ResetLipAnimation(lipAnimation); + PlayLipAnimation(); + var audioBuffer = lipAudios.Dequeue(); + AudioPlayer.PlayAsync(audioBuffer, sampleRate); + return true; + + } + catch (Exception ex) + { + Log.Error(LogTag, $"---Log Tick : {ex.StackTrace}"); + + return false; + } + } + + internal void StartAsyncLipPlayTimer() + { + if (asyncVowelTimer == null) + { + Tizen.Log.Info(LogTag, "Start Async"); + asyncVowelTimer = new Timer(AsyncPlayTime); + asyncVowelTimer.Tick += OnAsyncVowelTick; + asyncVowelTimer.Start(); + } + return; + } + + private void DestroyVowelTimer() + { + if (asyncVowelTimer != null) + { + + asyncVowelTimer.Tick -= OnAsyncVowelTick; + asyncVowelTimer.Stop(); + asyncVowelTimer.Dispose(); + asyncVowelTimer = null; + } + } + + private bool OnAsyncVowelTick(object source, Tizen.NUI.Timer.TickEventArgs e) + { + return PlayAsyncLip(SampleRate, isFinishAsyncLip); + } + } +} diff --git a/src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModule/AvatarMotions.cs b/src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModule/AvatarMotions.cs new file mode 100644 index 0000000..364cd3a --- /dev/null +++ b/src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModule/AvatarMotions.cs @@ -0,0 +1,65 @@ +/* + * Copyright(c) 2024 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +using Tizen.NUI; +using Tizen.NUI.Scene3D; + +namespace Tizen.AIAvatar +{ + internal class AvatarMotions + { + private MotionData eyeMotionData; + private const int blinkDuration = 200; + + internal AvatarMotions(AvatarProperties properties) + { + CreateEyeBlinkMotionData(properties); + } + + internal MotionData EyeMotionData { get { return eyeMotionData; } } + + private void CreateEyeBlinkMotionData(AvatarProperties avatarProperties) + { + var keyFrames = new KeyFrames(); + keyFrames.Add(0.1f, 0.0f); + keyFrames.Add(0.5f, 1.0f); + keyFrames.Add(0.9f, 0.0f); + + var headBlendShapeEyeLeft = new AvatarBlendShapeIndex(avatarProperties.NodeMapper, NodeType.HeadGeo, avatarProperties.BlendShapeMapper, BlendShapeType.EyeBlinkLeft); + var headBlendShapeEyeRight = new AvatarBlendShapeIndex(avatarProperties.NodeMapper, NodeType.HeadGeo, avatarProperties.BlendShapeMapper, BlendShapeType.EyeBlinkRight); + var eyelashBlendShapeEyeLeft = new AvatarBlendShapeIndex(avatarProperties.NodeMapper, NodeType.EyelashGeo, avatarProperties.BlendShapeMapper, BlendShapeType.EyeBlinkLeft); + var eyelashBlendShapeEyeRight = new AvatarBlendShapeIndex(avatarProperties.NodeMapper, NodeType.EyelashGeo, avatarProperties.BlendShapeMapper, BlendShapeType.EyeBlinkRight); + + eyeMotionData = new MotionData(blinkDuration); + if (headBlendShapeEyeLeft != null) + { + eyeMotionData.Add(headBlendShapeEyeLeft, new MotionValue(keyFrames)); + } + if (headBlendShapeEyeRight != null) + { + eyeMotionData.Add(headBlendShapeEyeRight, new MotionValue(keyFrames)); + } + if (eyelashBlendShapeEyeLeft != null) + { + eyeMotionData.Add(eyelashBlendShapeEyeLeft, new MotionValue(keyFrames)); + } + if (eyelashBlendShapeEyeRight != null) + { + eyeMotionData.Add(eyelashBlendShapeEyeRight, new MotionValue(keyFrames)); + } + } + } +} diff --git a/src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModule/BlendShapePlayer.cs b/src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModule/BlendShapePlayer.cs new file mode 100644 index 0000000..92906a0 --- /dev/null +++ b/src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModule/BlendShapePlayer.cs @@ -0,0 +1,157 @@ +/* + * Copyright(c) 2023 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Reflection; +using Tizen.NUI.Scene3D; +using Tizen.NUI; + +using static Tizen.AIAvatar.AIAvatar; + +namespace Tizen.AIAvatar +{ + internal class BlendShapePlayer + { + private Dictionary animationModules = new Dictionary(); + + private EyeBlinker blinker; + private AsyncLipSyncer lipSyncer; + + [EditorBrowsable(EditorBrowsableState.Never)] + public BlendShapePlayer() + { + blinker = new EyeBlinker(); + lipSyncer = new AsyncLipSyncer(); + + animationModules.Add(AnimationModuleType.EyeBlinker, blinker); + animationModules.Add(AnimationModuleType.LipSyncer, lipSyncer); + } + + [EditorBrowsable(EditorBrowsableState.Never)] + public void SetBlinkAnimation(Animation blinkerAnimation) + { + animationModules[AnimationModuleType.EyeBlinker].Init(blinkerAnimation); + } + + [EditorBrowsable(EditorBrowsableState.Never)] + public void SetLipSyncAnimation(Animation lipsyncAnimation) + { + animationModules[AnimationModuleType.LipSyncer].Init(lipsyncAnimation); + } + + [EditorBrowsable(EditorBrowsableState.Never)] + public AnimationModule GetAnimationModule(AnimationModuleType type) + { + return animationModules[type]; + } + + /// + /// Start eye blink animation + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public void StartEyeBlink() + { + animationModules[AnimationModuleType.EyeBlinker]?.Play(null); + } + + /// + /// Stop eye blink animation + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public void StopEyeBlink() + { + animationModules[AnimationModuleType.EyeBlinker]?.Stop(); + } + + /// + /// Play synchronization avatar lip based on the voice file(byte[]) + /// + /// byte array of voice + [EditorBrowsable(EditorBrowsableState.Never)] + public void PlayLipSync(byte[] audio) + { + if (animationModules[AnimationModuleType.LipSyncer] == null) + { + Log.Error(LogTag, $"error : avatarLipSync is null"); + return; + } + var lipData = new LipSyncData(); + + lipData.AudioFile = new byte[audio.Length]; + Buffer.BlockCopy(audio, 0, lipData.AudioFile, 0, audio.Length); + animationModules[AnimationModuleType.LipSyncer].Play(lipData); + } + + /// + /// Play synchronization avatar lip based on the voice file(byte[]) + /// + /// byte array of voice + [EditorBrowsable(EditorBrowsableState.Never)] + public void PlayLipSync(byte[] audio, int sampleRate) + { + if (animationModules[AnimationModuleType.LipSyncer] == null) + { + Log.Error(LogTag, $"error : avatarLipSync is null"); + return; + } + var lipData = new LipSyncData(); + + lipData.AudioFile = new byte[audio.Length]; + Buffer.BlockCopy(audio, 0, lipData.AudioFile, 0, audio.Length); + lipData.SampleRate = sampleRate; + animationModules[AnimationModuleType.LipSyncer].Play(lipData); + } + + /// + /// Pause lip synchronization + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public void PauseLipSync() + { + if (animationModules[AnimationModuleType.LipSyncer] == null) + { + Log.Error(LogTag, $"error : avatarLipSync is null"); + return; + } + animationModules[AnimationModuleType.LipSyncer].Pause(); + } + + /// + /// Stop lip synchronization + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public void StopLipSync() + { + if (animationModules[AnimationModuleType.LipSyncer] == null) + { + Log.Error(LogTag, $"error : avatarLipSync is null"); + return; + } + animationModules[AnimationModuleType.LipSyncer].Stop(); + } + + internal void DestroyAnimations() + { + foreach ( var animationModule in animationModules.Values ) + { + animationModule.Destroy(); + } + } + } +} diff --git a/src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModule/EyeBlinker.cs b/src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModule/EyeBlinker.cs new file mode 100644 index 0000000..c246732 --- /dev/null +++ b/src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModule/EyeBlinker.cs @@ -0,0 +1,139 @@ +/* + * Copyright(c) 2023 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using System; +using System.ComponentModel; +using Tizen.NUI; +using Tizen.NUI.Scene3D; + +using static Tizen.AIAvatar.AIAvatar; + +namespace Tizen.AIAvatar +{ + [EditorBrowsable(EditorBrowsableState.Never)] + internal class EyeBlinker : AnimationModule + { + private const int blinkIntervalMinimum = 800; + private const int blinkIntervalMaximum = 3000; + private Animation eyeAnimation; + + private Timer blinkTimer; + + private bool isPlaying = false; + private const int blinkDuration = 200; + + [EditorBrowsable(EditorBrowsableState.Never)] + internal EyeBlinker() + { + + } + + [EditorBrowsable(EditorBrowsableState.Never)] + public override void Init(Animation eyeAnimation) + { + this.eyeAnimation = eyeAnimation; + } + + [EditorBrowsable(EditorBrowsableState.Never)] + public override void Play(IAnimationModuleData data) + { + //data + StartEyeBlink(); + } + + [EditorBrowsable(EditorBrowsableState.Never)] + public override void Stop() + { + StopEyeBlink(); + } + + [EditorBrowsable(EditorBrowsableState.Never)] + public override void Pause() + { + eyeAnimation?.Pause(); + } + + [EditorBrowsable(EditorBrowsableState.Never)] + public override void Destroy() + { + DestroyAnimation(); + } + + private void StartEyeBlink() + { + DestroyBlinkTimer(); + + blinkTimer = new Timer(blinkDuration); + if (blinkTimer != null) + { + blinkTimer.Tick += OnBlinkTimer; + blinkTimer?.Start(); + isPlaying = true; + } + } + + private void PauseEyeBlink() + { + blinkTimer?.Stop(); + isPlaying = false; + } + + private void StopEyeBlink() + { + blinkTimer?.Stop(); + isPlaying = false; + } + + private void DestroyAnimation() + { + DestroyBlinkTimer(); + if (eyeAnimation != null) + { + eyeAnimation.Stop(); + eyeAnimation.Dispose(); + eyeAnimation = null; + } + isPlaying = false; + } + + private bool OnBlinkTimer(object source, Timer.TickEventArgs e) + { + if (eyeAnimation == null) + { + Log.Error(LogTag, "eye animation is not ready"); + return false; + } + eyeAnimation?.Play(); + + var random = new Random(); + var fortimerinterval = (uint)random.Next(blinkIntervalMinimum, blinkIntervalMaximum); + blinkTimer.Interval = fortimerinterval; + return true; + } + + private void DestroyBlinkTimer() + { + if (blinkTimer != null) + { + blinkTimer.Tick -= OnBlinkTimer; + blinkTimer.Stop(); + blinkTimer.Dispose(); + blinkTimer = null; + } + } + } +} diff --git a/src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModule/JointTransformer.cs b/src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModule/JointTransformer.cs new file mode 100644 index 0000000..4ff5e96 --- /dev/null +++ b/src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModule/JointTransformer.cs @@ -0,0 +1,107 @@ +/* + * Copyright(c) 2023 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using System.ComponentModel; +using Tizen.NUI; +using Tizen.NUI.Scene3D; + +namespace Tizen.AIAvatar +{ + [EditorBrowsable(EditorBrowsableState.Never)] + internal class JointTransformer + { + [EditorBrowsable(EditorBrowsableState.Never)] + public JointTransformer() + { + } + + [EditorBrowsable(EditorBrowsableState.Never)] + public void Init(Animation animation) + { + } + + [EditorBrowsable(EditorBrowsableState.Never)] + public void Play(IAnimationModuleData data) + { + } + + [EditorBrowsable(EditorBrowsableState.Never)] + public void Pause() + { + } + + [EditorBrowsable(EditorBrowsableState.Never)] + public void Stop() + { + } + + [EditorBrowsable(EditorBrowsableState.Never)] + public void Destroy() + { + } + + private void SetJointMotion(AvatarProperties properties, JointType jointType, MotionTransformIndex.TransformTypes type, Rotation rotation) + { + var motionTransform = new AvatarJointTransformIndex(properties.JointMapper, jointType, type); + var motionData = new MotionData(); + motionData.Add( + motionTransform, + new MotionValue() + { + //TODO : Tizen_7.0에 pitch, yaw, roll patch 추가하기 + //PropertyValue = new PropertyValue(new Rotation(new Radian(pitch), new Radian(yaw), new Radian(roll))), + } + ); + //avatar.SetMotionData(motionData); + } + + private void SetJointMotion(string keyValue, float pitch, float yaw, float roll) + { + var motionData = new MotionData(); + motionData.Add( + new MotionTransformIndex() + { + ModelNodeId = new PropertyKey(keyValue), + TransformType = MotionTransformIndex.TransformTypes.Orientation, + }, + new MotionValue() + { + //TODO : Tizen_7.0에 pitch, yaw, roll patch 추가하기 + //PropertyValue = new PropertyValue(new Rotation(new Radian(pitch), new Radian(yaw), new Radian(roll))), + } + ); + //avatar.SetMotionData(motionData); + } + + private void SetJointMotion(string keyValue, MotionTransformIndex.TransformTypes type, Rotation rotation) + { + var motionData = new MotionData(); + motionData.Add( + new MotionTransformIndex() + { + ModelNodeId = new PropertyKey(keyValue), + TransformType = type, + }, + new MotionValue() + { + PropertyValue = new PropertyValue(rotation), + } + ); + //avatar.SetMotionData(motionData); + } + } +} diff --git a/src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModule/LipSyncer.cs b/src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModule/LipSyncer.cs new file mode 100644 index 0000000..510be5c --- /dev/null +++ b/src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModule/LipSyncer.cs @@ -0,0 +1,371 @@ +/* + * Copyright(c) 2023 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using Tizen.NUI; +using Tizen.NUI.Scene3D; + +using static Tizen.AIAvatar.AIAvatar; + +namespace Tizen.AIAvatar +{ + [EditorBrowsable(EditorBrowsableState.Never)] + internal class LipSyncer : AnimationModule + { + private Avatar avatar; + private Animation lipAnimation = null; + + //private VowelConverter vowelConverter; + private AudioPlayer audioPlayer; + + //Mic + private Queue vowelPools = new Queue(); + private string prevVowel = "sil"; + + [EditorBrowsable(EditorBrowsableState.Never)] + internal LipSyncer() + { + } + + [EditorBrowsable(EditorBrowsableState.Never)] + internal AudioPlayer AudioPlayer { get { return audioPlayer; } } + + [EditorBrowsable(EditorBrowsableState.Never)] + public override void Init(Animation lipAnimation) + { + this.lipAnimation = lipAnimation; + //vowelConverter = new VowelConverter(); + audioPlayer = new AudioPlayer(); + + CurrentMotionState = AvatarMotionState.Ready; + } + + [EditorBrowsable(EditorBrowsableState.Never)] + public override void Play(IAnimationModuleData data) + { + if (data is LipSyncData lipSyncData) + { + if (lipSyncData.AudioFile != null) + { + PlayLipSync(lipSyncData.AudioFile, lipSyncData.SampleRate); + } + } + } + + [EditorBrowsable(EditorBrowsableState.Never)] + public override void Stop() + { + StopLipSync(); + } + + [EditorBrowsable(EditorBrowsableState.Never)] + public override void Pause() + { + PauseLipSync(); + } + + [EditorBrowsable(EditorBrowsableState.Never)] + public override void Destroy() + { + DestroyLipAnimation(); + } + + [EditorBrowsable(EditorBrowsableState.Never)] + protected Animation CreateAsyncLipAnimation(byte[] buffer, int sampleRate) + { + EnqueueVowels(buffer, false); + return CreateLipAnimationByVowelsQueue(sampleRate); + } + + [EditorBrowsable(EditorBrowsableState.Never)] + //TODO : lipAnimation 자체를 Animator안에서 생성하고 관리될 수 있게 수정. + protected void ResetLipAnimation(Animation lipAnimation) + { + DestroyLipAnimation(); + this.lipAnimation = lipAnimation; + if (this.lipAnimation != null) + { + this.lipAnimation.Finished += OnLipAnimationFinished; + } + } + + [EditorBrowsable(EditorBrowsableState.Never)] + protected void PlayLipAnimation() + { + if (lipAnimation == null) + { + Log.Error(LogTag, "Current Lip Animation is null"); + } + lipAnimation?.Play(); + } + + [EditorBrowsable(EditorBrowsableState.Never)] + protected void OnLipAnimationFinished(object sender, EventArgs e) + { + CurrentMotionState = AvatarMotionState.Stopped; + } + + private void PlayLipSync(byte[] audio) + { + + Tizen.Log.Error("MYLOG", "Play Lip sync : " + audio.Length); + if(audio == null) + { + Tizen.Log.Error("MYLOG", "audi data is null"); + } + + DestroyLipAnimation(); + var lipAnimation = CreateLipAnimation(audio, CurrentAudioOptions.SampleRate); + if (lipAnimation != null) + { + Tizen.Log.Error("MYLOG", "lipAnimation is not null"); + ResetLipAnimation(lipAnimation); + PlayLipAnimation(); + } + else + { + + Tizen.Log.Error("MYLOG", "lipAnimation is null"); + } + audioPlayer.Play(audio); + CurrentMotionState = AvatarMotionState.Playing; + } + + private void PlayLipSync(byte[] audio, int sampleRate) + { + DestroyLipAnimation(); + var lipAnimation = CreateLipAnimation(audio, sampleRate); + if (lipAnimation != null) + { + ResetLipAnimation(lipAnimation); + PlayLipAnimation(); + } + audioPlayer.Play(audio, sampleRate); + CurrentMotionState = AvatarMotionState.Playing; + } + + private void PlayLipSync(string path) + { + var bytes = Utils.ReadAllBytes(path); + if (bytes != null) + { + PlayLipSync(bytes); + } + else + { + Log.Error(LogTag, "Failed to load audio file"); + } + } + + private void PauseLipSync() + { + PauseLipAnimation(); + audioPlayer.Pause(); + CurrentMotionState = AvatarMotionState.Paused; + } + + private void StopLipSync() + { + StopLipAnimation(); + audioPlayer.Stop(); + CurrentMotionState = AvatarMotionState.Stopped; + } + + private void PauseLipAnimation() + { + if (lipAnimation != null) + { + lipAnimation?.Pause(); + } + else + { + Log.Error(LogTag, "Current Lip Animation is null"); + } + } + + private void StopLipAnimation() + { + if (lipAnimation != null) + { + DestroyLipAnimation(); + + var lipAnimation = ResetLipAnimation(); + if (lipAnimation != null) + { + ResetLipAnimation(lipAnimation); + PlayLipAnimation(); + } + else + { + Log.Error(LogTag, "Current Lip Animation is null"); + } + } + else + { + Log.Error(LogTag, "Current Lip Animation is null"); + } + } + + private void DestroyLipAnimation() + { + if (lipAnimation != null) + { + lipAnimation.Stop(); + lipAnimation.Dispose(); + lipAnimation = null; + } + } + + private Animation CreateLipAnimation(byte[] array, int sampleRate) + { + Animation lipKeyframes = null;// CreateKeyFrame(array, sampleRate); + if (lipKeyframes != null) + { + + return null;// CreateLipAnimation(lipKeyframes); + } + return null; + } + + private void EnqueueVowels(byte[] buffer, bool deleteLast = false) + { + var vowels = PredictVowels(buffer); + if (vowels != null) + { + vowelPools.Enqueue(vowels); + if (deleteLast) + { + var vowelList = new List(vowels); + vowelList.RemoveAt(vowelList.Count - 1); + vowels = vowelList.ToArray(); + } + } + } + + private Animation CreateLipAnimationByVowelsQueue(int sampleRate = 0) + { + if (sampleRate == 0) + { + sampleRate = CurrentAudioOptions.SampleRate; + } + if (vowelPools.Count > 0) + { + AttachPreviousVowel(vowelPools.Dequeue(), out var newVowels); + Log.Info(LogTag, $"vowelRecognition: {String.Join(", ", newVowels)}"); + + //var lipKeyFrames = vowelConverter?.CreateKeyFrames(newVowels, sampleRate, true); + return null;// CreateLipAnimation(lipKeyFrames, true); + } + return null; + } + + private Animation ResetLipAnimation() + { + vowelPools.Clear(); + var newVowels = new string[1]; + newVowels[0] = prevVowel = "sil"; + vowelPools.Enqueue(newVowels); + return CreateLipAnimationByVowelsQueue(); + } + + private string[] PredictVowels(byte[] audioData) + { + string[] vowels = null;// vowelConverter?.PredictVowels(audioData); + return vowels; + } + + private void AttachPreviousVowel(in string[] vowels, out string[] newVowels) + { + newVowels = new string[vowels.Length + 1]; + newVowels[0] = prevVowel; + Array.Copy(vowels, 0, newVowels, 1, vowels.Length); + prevVowel = vowels[vowels.Length - 1]; + } + + /* + private AnimationKeyFrame CreateKeyFrame(byte[] audio, int sampleRate) + { + var keyFrames = vowelConverter?.CreateKeyFrames(audio, sampleRate); + if (keyFrames == null) + { + Log.Error(LogTag, $"Failed to initialize KeyFrames"); + } + + return keyFrames; + } + + private Animation CreateLipAnimation(AnimationKeyFrame animationKeyFrames, bool isMic = false) + { + var animationTime = (int)(animationKeyFrames.AnimationTime * 1000f); + var lipAnimation = new Animation(animationTime); + + var motionData = new MotionData(animationTime); + for (var i = 0; i < animationKeyFrames.NodeNames.Length; i++) + { + var nodeName = animationKeyFrames.NodeNames[i]; + for (var j = 0; j < animationKeyFrames.BlendShapeCounts[i]; j++) + { + var blendShapeIndex = new BlendShapeIndex(new PropertyKey(nodeName), new PropertyKey(j)); + var keyFrameList = animationKeyFrames.GetKeyFrames(nodeName, j); + if (keyFrameList.Count == 0) + { + continue; + } + + var keyFrames = new KeyFrames(); + CreateKeyTimeFrames(ref keyFrames, keyFrameList, isMic); + + motionData.Add(blendShapeIndex, new MotionValue(keyFrames)); + lipAnimation = avatar.GenerateMotionDataAnimation(motionData); + } + } + return lipAnimation; + } + private KeyFrames CreateKeyTimeFrames(ref KeyFrames keyFrames, List keyFrameList, bool isMic = false) + { + foreach (var key in keyFrameList) + { + keyFrames.Add(key.time, key.value); + } + if (!isMic) keyFrames.Add(1.0f, 0.0f); + + return keyFrames; + } + */ + + internal void OnRecordBufferChanged(byte[] recordBuffer, int sampleRate) + { + EnqueueVowels(recordBuffer); + } + + internal void OnRecodingTick() + { + var lipAnimation = CreateLipAnimationByVowelsQueue(); + if (lipAnimation != null) + { + ResetLipAnimation(lipAnimation); + PlayLipAnimation(); + } + else + { + Log.Error(LogTag, "Current Lip Animation is null"); + } + } + } +} diff --git a/src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModule/MotionPlayer.cs b/src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModule/MotionPlayer.cs new file mode 100644 index 0000000..9797aff --- /dev/null +++ b/src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModule/MotionPlayer.cs @@ -0,0 +1,75 @@ +/* + * Copyright(c) 2024 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using System.ComponentModel; +using Tizen.NUI; +using Tizen.NUI.Scene3D; + +namespace Tizen.AIAvatar +{ + internal class MotionPlayer + { + private Animation motionAnimation; + internal Animation MotionAnimation { get => motionAnimation; private set => motionAnimation = value; } + + internal MotionPlayer() + { + + } + + /// + /// Play avatar animation by AnimationInfo + /// + /// index of default avatar animations + [EditorBrowsable(EditorBrowsableState.Never)] + internal void PlayAnimation(Animation motionAnimation, int duration = 3000, bool isLooping = false, int loopCount = 1) + { + ResetAnimations(); + + MotionAnimation = this.motionAnimation; + MotionAnimation.Play(); + } + + /// + /// Pause avatar animation + /// + [EditorBrowsable(EditorBrowsableState.Never)] + internal void PauseMotionAnimation() + { + MotionAnimation?.Pause(); + } + + /// + /// Stop avatar animation + /// + [EditorBrowsable(EditorBrowsableState.Never)] + internal void StopMotionAnimation() + { + MotionAnimation?.Stop(); + } + + private void ResetAnimations() + { + if (MotionAnimation != null) + { + MotionAnimation.Stop(); + MotionAnimation.Dispose(); + MotionAnimation = null; + } + } + } +} diff --git a/src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModuleData/IAnimationModuleData.cs b/src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModuleData/IAnimationModuleData.cs new file mode 100644 index 0000000..9b6110d --- /dev/null +++ b/src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModuleData/IAnimationModuleData.cs @@ -0,0 +1,35 @@ +/* + * Copyright(c) 2024 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using System.ComponentModel; + +namespace Tizen.AIAvatar +{ + [EditorBrowsable(EditorBrowsableState.Never)] + public interface IAnimationModuleData + { + } + + [EditorBrowsable(EditorBrowsableState.Never)] + public enum MotionDataType + { + [EditorBrowsable(EditorBrowsableState.Never)] + AnimationInfo, + [EditorBrowsable(EditorBrowsableState.Never)] + MotionData + } +} diff --git a/src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModuleData/LipSyncData.cs b/src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModuleData/LipSyncData.cs new file mode 100644 index 0000000..88b57cf --- /dev/null +++ b/src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModuleData/LipSyncData.cs @@ -0,0 +1,34 @@ +/* + * Copyright(c) 2024 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Text; +using Tizen.AIAvatar; + +namespace Tizen.AIAvatar +{ + [EditorBrowsable(EditorBrowsableState.Never)] + public class LipSyncData : IAnimationModuleData + { + [EditorBrowsable(EditorBrowsableState.Never)] + public byte[] AudioFile { get; set; } + [EditorBrowsable(EditorBrowsableState.Never)] + public int SampleRate { get; set; } + } +} diff --git a/src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModuleData/MotionBehaviorData.cs b/src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModuleData/MotionBehaviorData.cs new file mode 100644 index 0000000..a4bd49f --- /dev/null +++ b/src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModuleData/MotionBehaviorData.cs @@ -0,0 +1,39 @@ +/* + * Copyright(c) 2024 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using System.ComponentModel; +using Tizen.NUI.Scene3D; + +namespace Tizen.AIAvatar +{ + [EditorBrowsable(EditorBrowsableState.Never)] + public class MotionBehaviorData : IAnimationModuleData + { + [EditorBrowsable(EditorBrowsableState.Never)] + public MotionDataType Type { get; set; } = MotionDataType.AnimationInfo; + [EditorBrowsable(EditorBrowsableState.Never)] + public AnimationInfo AnimationInfo { get; set; } + [EditorBrowsable(EditorBrowsableState.Never)] + public MotionData MotionData { get; set; } + [EditorBrowsable(EditorBrowsableState.Never)] + public int Duration { get; set; } = 3000; + [EditorBrowsable(EditorBrowsableState.Never)] + public bool IsLooping { get; set; } = false; + [EditorBrowsable(EditorBrowsableState.Never)] + public int LoopCount { get; set; } = 1; + } +} diff --git a/src/Tizen.AIAvatar/src/public/Avatar/Animations/AvatarMotionChangedEventArgs.cs b/src/Tizen.AIAvatar/src/public/Avatar/Animations/AvatarMotionChangedEventArgs.cs new file mode 100644 index 0000000..77c497a --- /dev/null +++ b/src/Tizen.AIAvatar/src/public/Avatar/Animations/AvatarMotionChangedEventArgs.cs @@ -0,0 +1,58 @@ +/* + * Copyright(c) 2023 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using System; +using System.ComponentModel; + +namespace Tizen.AIAvatar +{ + /// + /// + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public class AvatarMotionChangedEventArgs : EventArgs + { + [EditorBrowsable(EditorBrowsableState.Never)] + public AvatarMotionChangedEventArgs(AvatarMotionState previous, AvatarMotionState current) + { + Previous = previous; + Current = current; + } + + /// + /// The previous state. + /// + /// 3 + [EditorBrowsable(EditorBrowsableState.Never)] + public AvatarMotionState Previous + { + get; + internal set; + } + + /// + /// The current state. + /// + /// 3 + [EditorBrowsable(EditorBrowsableState.Never)] + public AvatarMotionState Current + { + get; + internal set; + } + } +} diff --git a/src/Tizen.AIAvatar/src/public/Avatar/Animations/AvatarMotionState.cs b/src/Tizen.AIAvatar/src/public/Avatar/Animations/AvatarMotionState.cs new file mode 100644 index 0000000..4ad5692 --- /dev/null +++ b/src/Tizen.AIAvatar/src/public/Avatar/Animations/AvatarMotionState.cs @@ -0,0 +1,65 @@ +/* + * Copyright(c) 2023 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using System.ComponentModel; + +namespace Tizen.AIAvatar +{ + /// + /// Enumeration for the states. + /// + /// 3 + [EditorBrowsable(EditorBrowsableState.Never)] + public enum AvatarMotionState + { + /// + /// Created state. + /// + [EditorBrowsable(EditorBrowsableState.Never)] + Failed = -1, + + /// + /// Ready state. + /// + [EditorBrowsable(EditorBrowsableState.Never)] + Ready = 0, + + /// + /// Playing state. + /// + [EditorBrowsable(EditorBrowsableState.Never)] + Playing = 3, + + /// + /// Paused state. + /// + [EditorBrowsable(EditorBrowsableState.Never)] + Paused = 4, + + /// + /// Stopped state. + /// + [EditorBrowsable(EditorBrowsableState.Never)] + Stopped = 5, + + /// + /// Unavailable state. + /// + [EditorBrowsable(EditorBrowsableState.Never)] + Unavailable + }; +} diff --git a/src/Tizen.AIAvatar/src/public/Avatar/AudioOptions.cs b/src/Tizen.AIAvatar/src/public/Avatar/AudioOptions.cs new file mode 100644 index 0000000..f7587a6 --- /dev/null +++ b/src/Tizen.AIAvatar/src/public/Avatar/AudioOptions.cs @@ -0,0 +1,70 @@ +/* + * Copyright(c) 2023 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using System.ComponentModel; +using Tizen.Multimedia; + +namespace Tizen.AIAvatar +{ + /// + /// Provides the ability to audio + /// + // This will be public opened after ACR done. (Before ACR, need to be hidden as Inhouse API) + [EditorBrowsable(EditorBrowsableState.Never)] + public class AudioOptions + { + private int sampleRate; + private AudioChannel channel; + private AudioSampleType sampleType; + + /// + /// Initializes a new instance of the AudioOptions class with the specified sample rate, channel, and sampleType. + /// + /// the audio sample rate (8000 ~ 192000Hz) + /// the audio channel type. + /// the audio sample type. + // This will be public opened after ACR done. (Before ACR, need to be hidden as Inhouse API) + [EditorBrowsable(EditorBrowsableState.Never)] + public AudioOptions(int sampleRate, AudioChannel channel, AudioSampleType sampleType) + { + this.sampleRate = sampleRate; + this.channel = channel; + this.sampleType = sampleType; + } + + /// + /// The audio sample rate (8000 ~ 192000Hz) + /// + // This will be public opened after ACR done. (Before ACR, need to be hidden as Inhouse API) + [EditorBrowsable(EditorBrowsableState.Never)] + public int SampleRate { get => sampleRate; set => sampleRate = value; } + + /// + /// The audio channel type + /// + // This will be public opened after ACR done. (Before ACR, need to be hidden as Inhouse API) + [EditorBrowsable(EditorBrowsableState.Never)] + public AudioChannel Channel { get => channel; set => channel = value; } + + /// + /// The audio sample type + /// + // This will be public opened after ACR done. (Before ACR, need to be hidden as Inhouse API) + [EditorBrowsable(EditorBrowsableState.Never)] + public AudioSampleType SampleType { get => sampleType; set => sampleType = value; } + } +} diff --git a/src/Tizen.AIAvatar/src/public/Avatar/Avatar.cs b/src/Tizen.AIAvatar/src/public/Avatar/Avatar.cs new file mode 100644 index 0000000..10d6ea0 --- /dev/null +++ b/src/Tizen.AIAvatar/src/public/Avatar/Avatar.cs @@ -0,0 +1,199 @@ +/* + * Copyright(c) 2023 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using System.ComponentModel; +using Tizen.NUI.Scene3D; +using Tizen.NUI; + +using static Tizen.AIAvatar.AIAvatar; + +namespace Tizen.AIAvatar +{ + /// + /// Avatar is a Class to show 3D avatar objects. + /// It is subclass of Model s.t. we can control Avatar like models animation easly. + /// For example, + /// + /// Avatar supports AR Emoji. + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public class Avatar : Model + { + private AvatarProperties avatarProperties = new DefaultAvatarProperties(); + private AvatarMotions avatarMotions; + + private BlendShapePlayer avatarAnimator; + private MotionPlayer motionPlayer; + private JointTransformer jointTransformer; + + /// + /// Create an initialized AvatarModel. + /// + // This will be public opened after ACR done. (Before ACR, need to be hidden as Inhouse API) + [EditorBrowsable(EditorBrowsableState.Never)] + public Avatar() : base() + { + InitAvatar(); + } + + /// + /// Create an initialized AREmojiDefaultAvatar. + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public Avatar(AvatarInfo avatarInfo) : base(avatarInfo.ResourcePath) + { + InitAvatar(); + } + + /// + /// Create an initialized Avatar. + /// + /// avatar file url.(e.g. glTF, and DLI). + /// The url to derectory containing resources: binary, image etc. + /// + /// If resourceDirectoryUrl is empty, the parent directory url of avatarUrl is used for resource url. + /// + /// http://tizen.org/privilege/mediastorage for local files in media storage. + /// http://tizen.org/privilege/externalstorage for local files in external storage. + /// + // This will be public opened after ACR done. (Before ACR, need to be hidden as Inhouse API) + [EditorBrowsable(EditorBrowsableState.Never)] + public Avatar(string avatarUrl, string resourceDirectoryUrl = "") : base(avatarUrl, resourceDirectoryUrl) + { + InitAvatar(); + } + + /// + /// Copy constructor. + /// + /// Source object to copy. + // This will be public opened after ACR done. (Before ACR, need to be hidden as Inhouse API) + [EditorBrowsable(EditorBrowsableState.Never)] + public Avatar(Avatar avatar) : base(avatar) + { + InitAvatar(); + } + + [EditorBrowsable(EditorBrowsableState.Never)] + public AvatarProperties AvatarProperties + { + get => avatarProperties; + set => avatarProperties = value; + } + + [EditorBrowsable(EditorBrowsableState.Never)] + internal BlendShapePlayer AvatarAnimator + { + get => avatarAnimator; + set => avatarAnimator = value; + } + + + /// + /// Play avatar animation by AnimationInfo + /// + /// index of default avatar animations + [EditorBrowsable(EditorBrowsableState.Never)] + private void PlayAnimation(AnimationInfo animationInfo, int duration = 3000, bool isLooping = false, int loopCount = 1) + { + var motionAnimation = GenerateMotionDataAnimation(animationInfo.MotionData); + motionAnimation.Duration = duration; + motionAnimation.Looping = isLooping; + motionAnimation.LoopCount = loopCount; + motionAnimation.BlendPoint = 0.2f; + motionPlayer.PlayAnimation(motionAnimation); + } + + /// + /// Play avatar animation by MotionData + /// + /// index of default avatar animations + [EditorBrowsable(EditorBrowsableState.Never)] + private void PlayAnimation(MotionData motionData, int duration = 3000, bool isLooping = false, int loopCount = 1) + { + + var motionAnimation = GenerateMotionDataAnimation(motionData); + motionAnimation.Duration = duration; + motionAnimation.Looping = isLooping; + motionAnimation.LoopCount = loopCount; + motionAnimation.BlendPoint = 0.2f; + + motionPlayer.PlayAnimation(motionAnimation); + } + + private void PlayAnimation(int index, int duration = 3000, bool isLooping = false, int loopCount = 1) + { + //TODO by index + //var motionAnimation = GenerateMotionDataAnimation(animationInfoList[index].MotionData); + /*motionAnimation.Duration = duration; + motionAnimation.Looping = isLooping; + motionAnimation.LoopCount = loopCount; + motionAnimation.BlendPoint = 0.2f; + + motionPlayer.PlayAnimation(motionAnimation);*/ + } + + /// + /// Pause avatar animation + /// + [EditorBrowsable(EditorBrowsableState.Never)] + private void PauseMotionAnimation() + { + motionPlayer.PauseMotionAnimation(); + } + + /// + /// Stop avatar animation + /// + [EditorBrowsable(EditorBrowsableState.Never)] + private void StopMotionAnimation() + { + motionPlayer?.StopMotionAnimation(); + } + + + private void InitAvatar() + { + avatarMotions = new AvatarMotions(avatarProperties); + + AvatarAnimator = new BlendShapePlayer(); + + avatarProperties.AvatarPropertiesChanged += (s, e) => + { + var eyeAnimation = CreateMotionAnimation(avatarMotions.EyeMotionData); + if (eyeAnimation != null) + { + avatarAnimator.SetBlinkAnimation(eyeAnimation); + } + }; + + ResourcesLoaded += (s, e) => + { + var eyeAnimation = CreateMotionAnimation(avatarMotions.EyeMotionData); + if (eyeAnimation != null) + { + avatarAnimator.SetBlinkAnimation(eyeAnimation); + } + }; + } + + private Animation CreateMotionAnimation(MotionData data) + { + return GenerateMotionDataAnimation(data); + } + } +} diff --git a/src/Tizen.AIAvatar/src/public/Avatar/Common/BlendShapeType.cs b/src/Tizen.AIAvatar/src/public/Avatar/Common/BlendShapeType.cs new file mode 100644 index 0000000..3b55de0 --- /dev/null +++ b/src/Tizen.AIAvatar/src/public/Avatar/Common/BlendShapeType.cs @@ -0,0 +1,380 @@ +/* + * Copyright(c) 2023 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using System.ComponentModel; + +namespace Tizen.AIAvatar +{ + /// + /// The type of predefined blendshape. We can customize each type name by "TODO_mapper" + /// TODO : Explain me + /// TODO : Need to check each joint exist in AR Emoji + /// Note : This is temperal name of joints. + /// + [EditorBrowsable(EditorBrowsableState.Never)] + internal enum BlendShapeType + { + #region Left Eyes + /// + /// EyeBlinkLeft blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + EyeBlinkLeft = 0, + + /// + /// EyeSquintLeft blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + EyeSquintLeft, + + /// + /// EyeDownLeft blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + EyeDownLeft, + + /// + /// EyeInLeft blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + EyeInLeft, + + /// + /// EyeOpenLeft blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + EyeOpenLeft, + + /// + /// EyeOutLeft blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + EyeOutLeft, + + /// + /// EyeUpLeft blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + EyeUpLeft, + #endregion + + #region Right Eyes + /// + /// EyeBlinkRight blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + EyeBlinkRight, + + /// + /// EyeSquintRight blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + EyeSquintRight, + + /// + /// EyeDownRight blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + EyeDownRight, + + /// + /// EyeInRight blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + EyeInRight, + + /// + /// EyeOpenRight blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + EyeOpenRight, + + /// + /// EyeOutRight blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + EyeOutRight, + + /// + /// EyeUpRight blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + EyeUpRight, + #endregion + + #region Mouth and Jaw + /// + /// JawForward blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + JawForward, + + /// + /// JawOpen blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + JawOpen, + + /// + /// JawChew blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + JawChew, + + /// + /// JawLeft blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + JawLeft, + + /// + /// JawRight blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + JawRight, + + /// + /// MouthLeft blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + MouthLeft, + + /// + /// MouthFrownLeft blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + MouthFrownLeft, + + /// + /// MouthSmileLeft blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + MouthSmileLeft, + + /// + /// MouthDimpleLeft blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + MouthDimpleLeft, + + /// + /// MouthRight blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + MouthRight, + + /// + /// MouthFrownRight blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + MouthFrownRight, + + /// + /// MouthSmileRight blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + MouthSmileRight, + + /// + /// MouthDimpleRight blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + MouthDimpleRight, + #endregion + + #region Lips + /// + /// LipsStretchLeft blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + LipsStretchLeft, + + /// + /// LipsStretchRight blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + LipsStretchRight, + + /// + /// LipsUpperClose blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + LipsUpperClose, + + /// + /// LipsLowerClose blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + LipsLowerClose, + + /// + /// LipsUpperUp blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + LipsUpperUp, + + /// + /// LipsLowerDown blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + LipsLowerDown, + + /// + /// LipsUpperOpen blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + LipsUpperOpen, + + /// + /// LipsLowerOpen blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + LipsLowerOpen, + + /// + /// LipsFunnel blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + LipsFunnel, + + /// + /// LipsPucker blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + LipsPucker, + #endregion + + #region Eyebrows, Cheeks, and Chin + /// + /// BrowDownLeft blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + BrowDownLeft, + + /// + /// BrowDownRight blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + BrowDownRight, + + /// + /// BrowUpCenter blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + BrowUpCenter, + + /// + /// BrowUpLeft blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + BrowUpLeft, + + /// + /// BrowUpRight blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + BrowUpRight, + + /// + /// CheekSquintLeft blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + CheekSquintLeft, + + /// + /// CheekSquintRight blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + CheekSquintRight, + + /// + /// ChinLowerRaise blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + ChinLowerRaise, + + /// + /// ChinUpperRaise blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + ChinUpperRaise, + #endregion + + #region Tongue + /// + /// TongueOut blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + TongueOut, + + /// + /// TongueUp blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + TongueUp, + + /// + /// TongueDown blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + TongueDown, + + /// + /// TongueLeft blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + TongueLeft, + + /// + /// TongueRight blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + TongueRight, + #endregion + + #region ETC + /// + /// Sneer blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + Sneer, + + /// + /// Puff blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + Puff, + + /// + /// PuffLeft blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + PuffLeft, + + /// + /// PuffRight blendshape + /// + [EditorBrowsable(EditorBrowsableState.Never)] + PuffRight, + #endregion + /// + /// Max value of default blendshape. It will be used when we determine the motion index is default or custom. + /// + [EditorBrowsable(EditorBrowsableState.Never)] + DefaultBlendShapeMax, + } +} diff --git a/src/Tizen.AIAvatar/src/public/Avatar/Common/JointType.cs b/src/Tizen.AIAvatar/src/public/Avatar/Common/JointType.cs new file mode 100644 index 0000000..d8a87f4 --- /dev/null +++ b/src/Tizen.AIAvatar/src/public/Avatar/Common/JointType.cs @@ -0,0 +1,400 @@ +/* + * Copyright(c) 2023 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using System.ComponentModel; + +namespace Tizen.AIAvatar +{ + /// + /// The type of predefined skeletal joint. We can customize each type name by "TODO_mapper" + /// TODO : Explain me + /// TODO : Need to check each joint exist in AR Emoji + /// Note : This is temperal name of joints. + /// + [EditorBrowsable(EditorBrowsableState.Never)] + internal enum JointType + { + #region Head + /// + /// Head joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + Head = 0, + + /// + /// Neck joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + Neck, + + /// + /// EyeLeft joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + EyeLeft, + + /// + /// EyeRight joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + EyeRight, + #endregion + + #region Left Upper Body + /// + /// ShoulderLeft joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + ShoulderLeft, + + /// + /// ElbowLeft joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + ElbowLeft, + + /// + /// WristLeft joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + WristLeft, + #endregion + + #region Right Upper Body + /// + /// ShoulderRight joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + ShoulderRight, + + /// + /// ElbowRight joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + ElbowRight, + + /// + /// WristRight joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + WristRight, + #endregion + + #region Left Lower Body + /// + /// HipLeft joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + HipLeft, + + /// + /// KneeLeft joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + KneeLeft, + + /// + /// AnkleLeft joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + AnkleLeft, + + /// + /// ForeFootLeft joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + ForeFootLeft, + #endregion + + #region Right Lower Body + /// + /// HipRight joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + HipRight, + + /// + /// KneeRight joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + KneeRight, + + /// + /// AnkleRight joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + AnkleRight, + + /// + /// ForeFootRight joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + ForeFootRight, + #endregion + + #region Left Hand Finger + /// + /// FingerThumb1Left joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + FingerThumb1Left, + + /// + /// FingerThumb2Left joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + FingerThumb2Left, + + /// + /// FingerThumb3Left joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + FingerThumb3Left, + + /// + /// FingerThumb4Left joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + FingerThumb4Left, + + /// + /// FingerIndex1Left joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + FingerIndex1Left, + + /// + /// FingerIndex2Left joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + FingerIndex2Left, + + /// + /// FingerIndex3Left joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + FingerIndex3Left, + + /// + /// FingerIndex4Left joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + FingerIndex4Left, + + /// + /// FingerMiddle1Left joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + FingerMiddle1Left, + + /// + /// FingerMiddle2Left joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + FingerMiddle2Left, + + /// + /// FingerMiddle3Left joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + FingerMiddle3Left, + + /// + /// FingerMiddle4Left joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + FingerMiddle4Left, + + /// + /// FingerRing1Left joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + FingerRing1Left, + + /// + /// FingerRing2Left joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + FingerRing2Left, + + /// + /// FingerRing3Left joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + FingerRing3Left, + + /// + /// FingerRing4Left joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + FingerRing4Left, + + /// + /// FingerPinky1Left joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + FingerPinky1Left, + + /// + /// FingerPinky2Left joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + FingerPinky2Left, + + /// + /// FingerPinky3Left joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + FingerPinky3Left, + + /// + /// FingerPinky4Left joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + FingerPinky4Left, + #endregion + + #region Right Hand Finger + /// + /// FingerThumb1Right joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + FingerThumb1Right, + + /// + /// FingerThumb2Right joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + FingerThumb2Right, + + /// + /// FingerThumb3Right joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + FingerThumb3Right, + + /// + /// FingerThumb4Right joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + FingerThumb4Right, + + /// + /// FingerIndex1Right joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + FingerIndex1Right, + + /// + /// FingerIndex2Right joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + FingerIndex2Right, + + /// + /// FingerIndex3Right joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + FingerIndex3Right, + + /// + /// FingerIndex4Right joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + FingerIndex4Right, + + /// + /// FingerMiddle1Right joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + FingerMiddle1Right, + + /// + /// FingerMiddle2Right joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + FingerMiddle2Right, + + /// + /// FingerMiddle3Right joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + FingerMiddle3Right, + + /// + /// FingerMiddle4Right joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + FingerMiddle4Right, + + /// + /// FingerRing1Right joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + FingerRing1Right, + + /// + /// FingerRing2Right joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + FingerRing2Right, + + /// + /// FingerRing3Right joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + FingerRing3Right, + + /// + /// FingerRing4Right joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + FingerRing4Right, + + /// + /// FingerPinky1Right joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + FingerPinky1Right, + + /// + /// FingerPinky2Right joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + FingerPinky2Right, + + /// + /// FingerPinky3Right joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + FingerPinky3Right, + + /// + /// FingerPinky4Right joint + /// + [EditorBrowsable(EditorBrowsableState.Never)] + FingerPinky4Right, + #endregion + + /// + /// Max value of default joint. It will be used when we determine the motion index is default or custom. + /// + [EditorBrowsable(EditorBrowsableState.Never)] + DefaultJointMax, + } + +} diff --git a/src/Tizen.AIAvatar/src/public/Avatar/Common/NodeType.cs b/src/Tizen.AIAvatar/src/public/Avatar/Common/NodeType.cs new file mode 100644 index 0000000..e23c076 --- /dev/null +++ b/src/Tizen.AIAvatar/src/public/Avatar/Common/NodeType.cs @@ -0,0 +1,32 @@ +/* + * Copyright(c) 2023 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using System.ComponentModel; + +namespace Tizen.AIAvatar +{ + [EditorBrowsable(EditorBrowsableState.Never)] + internal enum NodeType + { + [EditorBrowsable(EditorBrowsableState.Never)] + HeadGeo, + [EditorBrowsable(EditorBrowsableState.Never)] + MouthGeo, + [EditorBrowsable(EditorBrowsableState.Never)] + EyelashGeo + } +} diff --git a/src/Tizen.AIAvatar/src/public/Avatar/Controller/AvatarLLM.cs b/src/Tizen.AIAvatar/src/public/Avatar/Controller/AvatarLLM.cs new file mode 100644 index 0000000..5bfed20 --- /dev/null +++ b/src/Tizen.AIAvatar/src/public/Avatar/Controller/AvatarLLM.cs @@ -0,0 +1,79 @@ +/* + * Copyright(c) 2024 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using Newtonsoft.Json; +using System.Net.Http; +using System; +using System.Threading.Tasks; +using Tizen.Uix.Tts; +using System.ComponentModel; + +namespace Tizen.AIAvatar +{ + [EditorBrowsable(EditorBrowsableState.Never)] + internal class AvatarLLM + { + private AvatarTTS avatarTTS; + private IRestClient restClient; + private const string playgroundURL = "https://playground-api.sec.samsung.net"; + + [EditorBrowsable(EditorBrowsableState.Never)] + internal AvatarLLM() + { + } + + [EditorBrowsable(EditorBrowsableState.Never)] + internal void InitAvatarLLM(AvatarTTS avatarTTS) + { + this.avatarTTS = avatarTTS; + // Setup RestClinet + var restClientFactory = new RestClientFactory(); + restClient = restClientFactory.CreateClient(playgroundURL); + } + + [EditorBrowsable(EditorBrowsableState.Never)] + internal async Task StartTTSWithLLMAsync(string text, string token) + { + var bearerToken = token; + var jsonData = "{\"messages\": [{\"role\": \"user\", \"content\": \"" + text + "\"}]}"; + + try + { + var postResponse = await restClient.SendRequestAsync(HttpMethod.Post, "/api/v1/chat/completions", bearerToken, jsonData); + var responseData = JsonConvert.DeserializeObject(postResponse); + string content = responseData["response"]["content"]; + Log.Info("Tizen.AIAvatar", content); + + //TTS 호출 + var voiceInfo = new VoiceInfo() + { + Lang = "en_US", + Type = Voice.Female, + }; + + avatarTTS.PlayTTSAsync(content, voiceInfo, (o, e) => { + + }); + + } + catch (Exception ex) + { + Log.Error("Tizen.AIAvatar", "에러 발생: " + ex.Message); + } + } + } +} diff --git a/src/Tizen.AIAvatar/src/public/Avatar/Controller/AvatarMic.cs b/src/Tizen.AIAvatar/src/public/Avatar/Controller/AvatarMic.cs new file mode 100644 index 0000000..8c8e8ba --- /dev/null +++ b/src/Tizen.AIAvatar/src/public/Avatar/Controller/AvatarMic.cs @@ -0,0 +1,81 @@ +/* + * Copyright(c) 2023 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using System.ComponentModel; + +using static Tizen.AIAvatar.AIAvatar; + +namespace Tizen.AIAvatar +{ + [EditorBrowsable(EditorBrowsableState.Never)] + internal class AvatarMic + { + [EditorBrowsable(EditorBrowsableState.Never)] + internal AvatarMic() + { + } + + /// + /// Initialize Microphone for using live lip sync + /// + /// http://tizen.org/privilege/recorder + [EditorBrowsable(EditorBrowsableState.Never)] + internal bool InitAvatarMic(Avatar avatar) + { + if (AudioRecorder.Instance == null) + { + Log.Error(LogTag, $"Failed to initialize AudioRecorder"); + return false; + } + AudioRecorder.Instance?.InitMic(avatar?.AvatarAnimator); + + return true; + } + + /// + /// Deinitialize Microphone + /// + /// http://tizen.org/privilege/recorder + [EditorBrowsable(EditorBrowsableState.Never)] + internal bool DeinitAvatarMIC() + { + AudioRecorder.Instance?.DeinitMic(); + + return true; + } + + /// + /// Start synchronization avatar lip based on the microphone's voice + /// + /// http://tizen.org/privilege/recorder + [EditorBrowsable(EditorBrowsableState.Never)] + internal void StartMic() + { + AudioRecorder.Instance?.StartRecording(); + } + + /// + /// Stop microphone + /// + /// http://tizen.org/privilege/recorder + [EditorBrowsable(EditorBrowsableState.Never)] + internal void StopMic() + { + AudioRecorder.Instance?.StopRecording(); + } + } +} diff --git a/src/Tizen.AIAvatar/src/public/Avatar/Controller/AvatarTTS.cs b/src/Tizen.AIAvatar/src/public/Avatar/Controller/AvatarTTS.cs new file mode 100644 index 0000000..9141fb5 --- /dev/null +++ b/src/Tizen.AIAvatar/src/public/Avatar/Controller/AvatarTTS.cs @@ -0,0 +1,421 @@ +/* + * Copyright(c) 2023 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using System.Collections.Generic; +using System.ComponentModel; +using System; +using Tizen.Uix.Tts; + +using static Tizen.AIAvatar.AIAvatar; + +namespace Tizen.AIAvatar +{ + [EditorBrowsable(EditorBrowsableState.Never)] + internal class AvatarTTS + { + private TTSLipSyncer ttsLipSyncer; + + private event EventHandler ttsReadyFinished; + private readonly Object ttsReadyFinishedLock = new Object(); + + [EditorBrowsable(EditorBrowsableState.Never)] + internal AvatarTTS() + { + } + + /// + /// + [EditorBrowsable(EditorBrowsableState.Never)] + internal event EventHandler TTSReadyFinished + { + add + { + lock (ttsReadyFinishedLock) + { + ttsReadyFinished += value; + } + + } + + remove + { + lock (ttsReadyFinishedLock) + { + if (ttsReadyFinished == null) + { + Log.Error(LogTag, "ttsReadyFinished is null"); + return; + } + ttsReadyFinished -= value; + } + } + } + + /// + /// + /// + [EditorBrowsable(EditorBrowsableState.Never)] + internal TtsClient CurrentTTSClient => ttsLipSyncer?.TtsHandle; + + /// + /// + /// + [EditorBrowsable(EditorBrowsableState.Never)] + internal void InitTTS(Avatar avatar) + { + if (ttsLipSyncer == null) + { + try + { + ttsLipSyncer = new TTSLipSyncer(avatar); + } + catch (Exception e) + { + Log.Error(LogTag, $"error :{e.Message}"); + Log.Error(LogTag, $"{e.StackTrace}"); + } + } + } + + /// + /// + /// + [EditorBrowsable(EditorBrowsableState.Never)] + internal void DeinitTTS() + { + if (ttsLipSyncer != null) + { + try + { + ttsLipSyncer.DeinitTts(); + ttsLipSyncer = null; + } + catch (Exception e) + { + Log.Error(LogTag, $"error :{e.Message}"); + Log.Error(LogTag, $"{e.StackTrace}"); + } + } + } + + /// + /// + /// + [EditorBrowsable(EditorBrowsableState.Never)] + internal List GetSupportedVoices() + { + return ttsLipSyncer.GetSupportedVoices(); + } + + /// + /// + /// + [EditorBrowsable(EditorBrowsableState.Never)] + internal bool PrepareTTS(string text, VoiceInfo voiceInfo, EventHandler ttsReadyFinishedCallback = null) + { + if (ttsLipSyncer == null || ttsLipSyncer.TtsHandle == null) + { + Log.Error(LogTag, "tts is null"); + return false; + } + + + if (!ttsLipSyncer.IsSupportedVoice(voiceInfo)) + { + Log.Info(LogTag, $"{voiceInfo.Lang} & {voiceInfo.Type} is not supported"); + return false; + } + + Log.Info(LogTag, "Current TTS State :" + ttsLipSyncer.TtsHandle.CurrentState); + if (ttsLipSyncer.TtsHandle.CurrentState != Tizen.Uix.Tts.State.Ready) + { + Log.Info(LogTag, "TTS is not ready"); + return false; + } + + try + { + ttsLipSyncer.AddText(text, voiceInfo); + ttsLipSyncer.Prepare(ttsReadyFinishedCallback); + } + catch (Exception e) + { + Log.Error(LogTag, $"error :{e.Message}"); + Log.Error(LogTag, $"{e.StackTrace}"); + return false; + } + return true; + } + + /// + /// + /// + [EditorBrowsable(EditorBrowsableState.Never)] + internal bool PrepareTTS(string text, string lang = "ko_KR", EventHandler ttsReadyFinishedCallback = null) + { + if (ttsLipSyncer == null || ttsLipSyncer.TtsHandle == null) + { + Log.Error(LogTag, "tts is null"); + return false; + } + + if (!ttsLipSyncer.IsSupportedVoice(lang)) + { + Log.Error(LogTag, $"{lang} is not supported"); + return false; + } + + Log.Info(LogTag, "Current TTS State :" + ttsLipSyncer.TtsHandle.CurrentState); + if (ttsLipSyncer.TtsHandle.CurrentState != Tizen.Uix.Tts.State.Ready) + { + Log.Error(LogTag, "TTS is not ready"); + return false; + } + try + { + ttsLipSyncer.AddText(text, lang); + ttsLipSyncer.Prepare(ttsReadyFinishedCallback); + } + catch (Exception e) + { + Log.Error(LogTag, $"error :{e.Message}"); + Log.Error(LogTag, $"{e.StackTrace}"); + return false; + } + return true; + } + + /// + /// + /// + [EditorBrowsable(EditorBrowsableState.Never)] + internal bool PlayPreparedTTS() + { + if (ttsLipSyncer == null || ttsLipSyncer.TtsHandle == null) + { + Log.Error(LogTag, "tts is null"); + return false; + } + + return ttsLipSyncer.PlayPreparedText(); + } + + /// + /// + /// + [EditorBrowsable(EditorBrowsableState.Never)] + internal bool PlayTTS(string text, VoiceInfo voiceInfo) + { + if (ttsLipSyncer == null || ttsLipSyncer.TtsHandle == null) + { + Log.Error(LogTag, "tts is null"); + return false; + } + + + if (!ttsLipSyncer.IsSupportedVoice(voiceInfo)) + { + Log.Info(LogTag, $"{voiceInfo.Lang} & {voiceInfo.Type} is not supported"); + return false; + } + + Log.Info(LogTag, "Current TTS State :" + ttsLipSyncer.TtsHandle.CurrentState); + if (ttsLipSyncer.TtsHandle.CurrentState != Tizen.Uix.Tts.State.Ready) + { + Log.Info(LogTag, "TTS is not ready"); + return false; + } + + try + { + ttsLipSyncer.AddText(text, voiceInfo); + ttsLipSyncer.Play(); + } + catch (Exception e) + { + Log.Error(LogTag, $"error :{e.Message}"); + Log.Error(LogTag, $"{e.StackTrace}"); + return false; + } + return true; + } + + /// + /// + /// + [EditorBrowsable(EditorBrowsableState.Never)] + internal bool PlayTTS(string text, string lang = "ko_KR") + { + if (ttsLipSyncer == null || ttsLipSyncer.TtsHandle == null) + { + Log.Error(LogTag, "tts is null"); + return false; + } + + if (!ttsLipSyncer.IsSupportedVoice(lang)) + { + Log.Error(LogTag, $"{lang} is not supported"); + return false; + } + + Log.Info(LogTag, "Current TTS State :" + ttsLipSyncer.TtsHandle.CurrentState); + if (ttsLipSyncer.TtsHandle.CurrentState != Tizen.Uix.Tts.State.Ready) + { + Log.Error(LogTag, "TTS is not ready"); + return false; + } + try + { + ttsLipSyncer.AddText(text, lang); + ttsLipSyncer.Play(); + } + catch (Exception e) + { + Log.Error(LogTag, $"error :{e.Message}"); + Log.Error(LogTag, $"{e.StackTrace}"); + return false; + } + return true; + } + + /// + /// + /// + [EditorBrowsable(EditorBrowsableState.Never)] + internal bool PlayTTSAsync(string text, VoiceInfo voiceInfo, EventHandler ttsReadyFinishedCallback = null) + { + if (ttsLipSyncer == null || ttsLipSyncer.TtsHandle == null) + { + Log.Error(LogTag, "tts is null"); + return false; + } + + + if (!ttsLipSyncer.IsSupportedVoice(voiceInfo)) + { + Log.Info(LogTag, $"{voiceInfo.Lang} & {voiceInfo.Type} is not supported"); + return false; + } + + Log.Info(LogTag, "Current TTS State :" + ttsLipSyncer.TtsHandle.CurrentState); + if (ttsLipSyncer.TtsHandle.CurrentState != Tizen.Uix.Tts.State.Ready) + { + Log.Info(LogTag, "TTS is not ready"); + return false; + } + + try + { + ttsLipSyncer.AddText(text, voiceInfo); + ttsLipSyncer.PlayAsync(ttsReadyFinishedCallback); + } + catch (Exception e) + { + Log.Error(LogTag, $"error :{e.Message}"); + Log.Error(LogTag, $"{e.StackTrace}"); + return false; + } + return true; + } + + /// + /// + /// + [EditorBrowsable(EditorBrowsableState.Never)] + internal bool PlayTTS(string text, string lang = "ko_KR", EventHandler ttsReadyFinishedCallback = null) + { + if (ttsLipSyncer == null || ttsLipSyncer.TtsHandle == null) + { + Log.Error(LogTag, "tts is null"); + return false; + } + + if (!ttsLipSyncer.IsSupportedVoice(lang)) + { + Log.Error(LogTag, $"{lang} is not supported"); + return false; + } + + Log.Info(LogTag, "Current TTS State :" + ttsLipSyncer.TtsHandle.CurrentState); + if (ttsLipSyncer.TtsHandle.CurrentState != Tizen.Uix.Tts.State.Ready) + { + Log.Error(LogTag, "TTS is not ready"); + return false; + } + try + { + ttsLipSyncer.AddText(text, lang); + ttsLipSyncer.PlayAsync(ttsReadyFinishedCallback); + } + catch (Exception e) + { + Log.Error(LogTag, $"error :{e.Message}"); + Log.Error(LogTag, $"{e.StackTrace}"); + return false; + } + return true; + } + + /// + /// + /// + [EditorBrowsable(EditorBrowsableState.Never)] + internal void PauseTTS() + { + if (ttsLipSyncer == null || ttsLipSyncer.TtsHandle == null) + { + Log.Error(LogTag, "tts is null"); + return; + } + + try + { + Log.Info(LogTag, "Current TTS State :" + ttsLipSyncer.TtsHandle.CurrentState); + ttsLipSyncer?.Pause(); + } + catch (Exception e) + { + Log.Error(LogTag, $"error :{e.Message}"); + Log.Error(LogTag, $"{e.StackTrace}"); + } + } + + /// + /// + /// + [EditorBrowsable(EditorBrowsableState.Never)] + internal void StopTTS() + { + if (ttsLipSyncer == null || ttsLipSyncer.TtsHandle == null) + { + Log.Error(LogTag, "tts is null"); + return; + } + + try + { + Log.Info(LogTag, "Current TTS State :" + ttsLipSyncer.TtsHandle.CurrentState); + ttsLipSyncer?.Stop(); + } + catch (Exception e) + { + Log.Error(LogTag, $"error :{e.Message}"); + Log.Error(LogTag, $"{e.StackTrace}"); + } + } + } +} diff --git a/src/Tizen.AIAvatar/src/public/Avatar/Info/AnimationInfo.cs b/src/Tizen.AIAvatar/src/public/Avatar/Info/AnimationInfo.cs new file mode 100644 index 0000000..17c7998 --- /dev/null +++ b/src/Tizen.AIAvatar/src/public/Avatar/Info/AnimationInfo.cs @@ -0,0 +1,37 @@ +/* + * Copyright(c) 2023 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +using System.ComponentModel; +using Tizen.NUI.Scene3D; + +namespace Tizen.AIAvatar +{ + [EditorBrowsable(EditorBrowsableState.Never)] + public class AnimationInfo + { + [EditorBrowsable(EditorBrowsableState.Never)] + public MotionData MotionData { get; internal set; } + [EditorBrowsable(EditorBrowsableState.Never)] + public string MotionName { get; internal set; } + + [EditorBrowsable(EditorBrowsableState.Never)] + public AnimationInfo(MotionData motionData, string motionName) + { + MotionData = motionData; + MotionName = motionName; + } + } +} diff --git a/src/Tizen.AIAvatar/src/public/Avatar/Info/AvatarInfo.cs b/src/Tizen.AIAvatar/src/public/Avatar/Info/AvatarInfo.cs new file mode 100644 index 0000000..7f7b95e --- /dev/null +++ b/src/Tizen.AIAvatar/src/public/Avatar/Info/AvatarInfo.cs @@ -0,0 +1,70 @@ +/* + * Copyright(c) 2023 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using System; +using System.ComponentModel; +using static Tizen.AIAvatar.AIAvatar; + +namespace Tizen.AIAvatar +{ + [EditorBrowsable(EditorBrowsableState.Never)] + public class AvatarInfo + { + [EditorBrowsable(EditorBrowsableState.Never)] + public string Name { get; private set; } + + [EditorBrowsable(EditorBrowsableState.Never)] + public string ThumbnailPath { get; private set; } + + [EditorBrowsable(EditorBrowsableState.Never)] + internal string ResourcePath { get; private set; } + [EditorBrowsable(EditorBrowsableState.Never)] + internal AvatarInfoOption avatarInfoOption { get; private set; } + + [EditorBrowsable(EditorBrowsableState.Never)] + public AvatarInfo(string path, string Name, AvatarInfoOption info = AvatarInfoOption.Thumbnail) + { + this.Name = Name; + this.avatarInfoOption = info; + + if (info == AvatarInfoOption.Thumbnail) + { + ThumbnailPath = path; + } + else + { + ResourcePath = path; + } + } + + internal AvatarInfo(string directoryPath) + { + string path = ApplicationResourcePath + EmojiAvatarResourcePath; + Name = directoryPath.Substring(path.Length, directoryPath.Length - path.Length); + ResourcePath = $"{directoryPath}/{AIAvatar.ExternalModel}"; + } + } + + [EditorBrowsable(EditorBrowsableState.Never)] + public enum AvatarInfoOption + { + [EditorBrowsable(EditorBrowsableState.Never)] + Thumbnail = 0, + [EditorBrowsable(EditorBrowsableState.Never)] + Resource = 1, + } +} diff --git a/src/Tizen.AIAvatar/src/public/Avatar/Info/VoiceInfo.cs b/src/Tizen.AIAvatar/src/public/Avatar/Info/VoiceInfo.cs new file mode 100644 index 0000000..7757da1 --- /dev/null +++ b/src/Tizen.AIAvatar/src/public/Avatar/Info/VoiceInfo.cs @@ -0,0 +1,40 @@ +/* + * Copyright(c) 2023 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +using System.ComponentModel; +using Tizen.Uix.Tts; + +namespace Tizen.AIAvatar +{ + [EditorBrowsable(EditorBrowsableState.Never)] + public struct VoiceInfo + { + private string lang; + private Voice type; + + /// + /// + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public string Lang { get => lang; set => lang = value; } + + /// + /// + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public Voice Type { get => type; set => type = value; } + } +} diff --git a/src/Tizen.AIAvatar/src/public/Avatar/Properties/AvatarBlendShapeIndex.cs b/src/Tizen.AIAvatar/src/public/Avatar/Properties/AvatarBlendShapeIndex.cs new file mode 100644 index 0000000..0264b9b --- /dev/null +++ b/src/Tizen.AIAvatar/src/public/Avatar/Properties/AvatarBlendShapeIndex.cs @@ -0,0 +1,193 @@ +/* + * Copyright(c) 2023 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using System.ComponentModel; +using Tizen.NUI; +using Tizen.NUI.Scene3D; + +namespace Tizen.AIAvatar +{ + /// + /// Specialized to control avatar blend shape. + /// + /// + /// + /// AvatarBlendShapeIndex leftEyeBlink = new AvatarBlendShapeIndex(avatar.BlendShapeMapper, BlendShapeType.EyeBlinkLeft); + /// + /// // We can change the property later. + /// AVatarBlendShapeIndex rightEyeBlink = new AvatarJointTransformIndex(avatar.BlendShapeMapper); + /// rightEyeBlink.AvatarBlendShapeType = (uint)BlendShapeType.EyeBlinkRight; + /// + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public class AvatarBlendShapeIndex : BlendShapeIndex + { + internal AvatarPropertyMapper nameMapper = null; + internal uint blendShapeType; + + /// + /// Create an initialized avatar blend shape index. + /// + /// Name mapper for this index + [EditorBrowsable(EditorBrowsableState.Never)] + public AvatarBlendShapeIndex(AvatarPropertyMapper mapper) : base() + { + nameMapper = mapper; + } + + /// + /// Create an initialized avatar blend shape index with input blend shape ID. + /// + /// Name mapper for this index + /// Blend shape ID for this motion index + [EditorBrowsable(EditorBrowsableState.Never)] + public AvatarBlendShapeIndex(AvatarPropertyMapper mapper, PropertyKey blendShapeId) : base(new PropertyKey(GetPropertyNameFromMapper(mapper, blendShapeId))) + { + nameMapper = mapper; + } + + /// + /// Create an initialized avatar blend shape index with blend shape type. + /// + /// Name mapper for this index + /// Type of blend shape for this motion index + [EditorBrowsable(EditorBrowsableState.Never)] + internal AvatarBlendShapeIndex(AvatarPropertyMapper blendShapeMapper, BlendShapeType blendShapeType) : this(blendShapeMapper, (uint)blendShapeType) + { + } + + /// + /// + /// + /// Name mapper for this index + /// Type of blend shape for this motion index + [EditorBrowsable(EditorBrowsableState.Never)] + public AvatarBlendShapeIndex(AvatarPropertyMapper mapper, uint blendShapeType) : base(new PropertyKey(GetPropertyNameFromMapper(mapper, blendShapeType))) + { + nameMapper = mapper; + this.blendShapeType = blendShapeType; + } + + /// + /// + /// + /// Name mapper for this index + /// Type of blend shape for this motion index + [EditorBrowsable(EditorBrowsableState.Never)] + internal AvatarBlendShapeIndex(AvatarPropertyMapper nodeMapper, NodeType nodeType, AvatarPropertyMapper blendShapeMapper, BlendShapeType blendShapeType) : base(new PropertyKey(GetPropertyNameFromMapper(nodeMapper, (uint)nodeType)), new PropertyKey(GetPropertyNameFromMapper(blendShapeMapper, (uint)blendShapeType))) + { + nameMapper = blendShapeMapper; + this.blendShapeType = (uint)blendShapeType; + } + + /// + /// Get the name of given index. + /// + /// Name mapper for given index + /// Target ID what we want to get name + /// Name, or null if invalid + private static string GetPropertyNameFromMapper(AvatarPropertyMapper mapper, PropertyKey id) + { + if (id == null) + { + return ""; + } + if (id.Type == PropertyKey.KeyType.String) + { + return id.StringKey; + } + return mapper?.GetPropertyName((uint)id.IndexKey) ?? ""; + } + + /// + /// Get the name of given JointType. + /// + /// Name mapper for given index + /// Type of joint what we want to get name + /// Name, or null if invalid + private static string GetPropertyNameFromMapper(AvatarPropertyMapper mapper, uint blendShapeType) + { + return mapper?.GetPropertyName(blendShapeType) ?? ""; + } + + /// + /// TODO : Explain me + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public AvatarPropertyMapper NameMapper + { + get + { + return nameMapper; + } + set + { + nameMapper = value; + + using PropertyKey blendShapeId = new(GetPropertyNameFromMapper(nameMapper, blendShapeType)); + base.BlendShapeId = blendShapeId; + } + } + + /// + /// TODO : Explain me + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public uint AvatarBlendShapeType + { + get + { + return blendShapeType; + } + set + { + blendShapeType = value; + + using PropertyKey blendShapeId = new(GetPropertyNameFromMapper(nameMapper, blendShapeType)); + base.BlendShapeId = blendShapeId; + } + } + + /// + /// Hijack property to control Avatar specified logic. + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public new PropertyKey BlendShapeId + { + get + { + return base.BlendShapeId; + } + set + { + if (value != null) + { + if (value.Type == PropertyKey.KeyType.Index) + { + blendShapeType = (uint)value.IndexKey; + } + using PropertyKey blendShapeId = new(GetPropertyNameFromMapper(nameMapper, value)); + base.BlendShapeId = blendShapeId; + } + else + { + base.BlendShapeId = value; + } + } + } + } +} diff --git a/src/Tizen.AIAvatar/src/public/Avatar/Properties/AvatarJointTransformIndex.cs b/src/Tizen.AIAvatar/src/public/Avatar/Properties/AvatarJointTransformIndex.cs new file mode 100644 index 0000000..6163e22 --- /dev/null +++ b/src/Tizen.AIAvatar/src/public/Avatar/Properties/AvatarJointTransformIndex.cs @@ -0,0 +1,173 @@ +/* + * Copyright(c) 2023 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using System.ComponentModel; +using Tizen.NUI; +using Tizen.NUI.Scene3D; + +namespace Tizen.AIAvatar +{ + /// + /// Specialized to control avatar joint transform. + /// + /// + /// + /// AvatarJointTransformIndex position = new AvatarJointTransformIndex(avatar.JointMapper, JointType.Head, MotionTransformIndex.TransformTypes.Position); + /// + /// // We can change the property later. + /// AvatarJointTransformIndex orientation = new AvatarJointTransformIndex(avatar.JointMapper); + /// orientation.AvatarJointType = (uint)JointType.Neck; + /// orientation.TransformType = MotionTransformIndex.TransformTypes.Orientation; + /// + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public class AvatarJointTransformIndex : MotionTransformIndex + { + internal AvatarPropertyMapper nameMapper = null; + internal uint jointType; + + /// + /// Create an initialized avatar joint transform index. + /// + /// Name mapper for this index + [EditorBrowsable(EditorBrowsableState.Never)] + public AvatarJointTransformIndex(AvatarPropertyMapper mapper) : base() + { + nameMapper = mapper; + } + + /// + /// Create an initialized avatar joint transform index with input node id, and transform type. + /// + /// Name mapper for this index + /// Node ID for this motion index + /// Transform property type for this motion index + [EditorBrowsable(EditorBrowsableState.Never)] + public AvatarJointTransformIndex(AvatarPropertyMapper mapper, PropertyKey modelNodeId, TransformTypes transformType) : base(new PropertyKey(GetPropertyNameFromMapper(mapper, modelNodeId)), transformType) + { + nameMapper = mapper; + } + + /// + /// Create an initialized avatar joint transform index with input node id, and transform type. + /// + /// Name mapper for this index + /// Type of joint for this motion index + /// Transform property type for this motion index + [EditorBrowsable(EditorBrowsableState.Never)] + internal AvatarJointTransformIndex(AvatarPropertyMapper mapper, JointType jointType, TransformTypes transformType) : this(mapper, (uint)jointType, transformType) + { + } + + /// + /// Create an initialized avatar joint transform index with input node id, and transform type. + /// + /// Name mapper for this index + /// Type of joint for this motion index + /// Transform property type for this motion index + [EditorBrowsable(EditorBrowsableState.Never)] + public AvatarJointTransformIndex(AvatarPropertyMapper mapper, uint jointType, TransformTypes transformType) : base(new PropertyKey(GetPropertyNameFromMapper(mapper, jointType)), transformType) + { + nameMapper = mapper; + this.jointType = jointType; + } + + private static string GetPropertyNameFromMapper(AvatarPropertyMapper mapper, PropertyKey id) + { + if (id == null) + { + return ""; + } + if (id.Type == PropertyKey.KeyType.String) + { + return id.StringKey; + } + return mapper?.GetPropertyName((uint)id.IndexKey) ?? ""; + } + + private static string GetPropertyNameFromMapper(AvatarPropertyMapper mapper, uint jointType) + { + return mapper?.GetPropertyName(jointType) ?? ""; + } + + /// + /// TODO : Explain me + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public AvatarPropertyMapper NameMapper + { + get + { + return nameMapper; + } + set + { + nameMapper = value; + + using PropertyKey nodeId = new(GetPropertyNameFromMapper(nameMapper, jointType)); + base.ModelNodeId = nodeId; + } + } + + /// + /// TODO : Explain me + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public uint AvatarJointType + { + get + { + return jointType; + } + set + { + jointType = value; + + using PropertyKey nodeId = new(GetPropertyNameFromMapper(nameMapper, jointType)); + base.ModelNodeId = nodeId; + } + } + + /// + /// Hijack property to control Avatar specified logic. + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public new PropertyKey ModelNodeId + { + get + { + return base.ModelNodeId; + } + set + { + if (value != null) + { + if (value.Type == PropertyKey.KeyType.Index) + { + jointType = (uint)value.IndexKey; + } + using PropertyKey nodeId = new(GetPropertyNameFromMapper(nameMapper, value)); + base.ModelNodeId = nodeId; + } + else + { + base.ModelNodeId = value; + } + } + } + } +} diff --git a/src/Tizen.AIAvatar/src/public/Avatar/Properties/AvatarProperties.cs b/src/Tizen.AIAvatar/src/public/Avatar/Properties/AvatarProperties.cs new file mode 100644 index 0000000..a6b8612 --- /dev/null +++ b/src/Tizen.AIAvatar/src/public/Avatar/Properties/AvatarProperties.cs @@ -0,0 +1,82 @@ +/* + * Copyright(c) 2023 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +using System; +using System.ComponentModel; + +namespace Tizen.AIAvatar +{ + [EditorBrowsable(EditorBrowsableState.Never)] + public class AvatarProperties + { + private AvatarPropertyMapper jointMapper; + private AvatarPropertyMapper blendShapeMapper; + private AvatarPropertyMapper nodeMapper; + + [EditorBrowsable(EditorBrowsableState.Never)] + public AvatarProperties(AvatarPropertyMapper jointMapper, AvatarPropertyMapper blendShapeMapper, AvatarPropertyMapper nodeMapper) + { + JointMapper = new AvatarPropertyMapper(jointMapper); + BlendShapeMapper = new AvatarPropertyMapper(blendShapeMapper); + NodeMapper = new AvatarPropertyMapper(nodeMapper); + } + + internal event EventHandler AvatarPropertiesChanged; + + [EditorBrowsable(EditorBrowsableState.Never)] + public AvatarPropertyMapper JointMapper + { + get + { + return jointMapper; + } + set + { + jointMapper = value; + AvatarPropertiesChanged?.Invoke(jointMapper, this); + } + } + + [EditorBrowsable(EditorBrowsableState.Never)] + public AvatarPropertyMapper BlendShapeMapper + { + get + { + return blendShapeMapper; + } + set + { + blendShapeMapper = value; + AvatarPropertiesChanged?.Invoke(blendShapeMapper, this); + } + } + + [EditorBrowsable(EditorBrowsableState.Never)] + public AvatarPropertyMapper NodeMapper + { + get + { + return nodeMapper; + } + set + { + nodeMapper = value; + AvatarPropertiesChanged?.Invoke(nodeMapper, this); + } + } + } +} diff --git a/src/Tizen.AIAvatar/src/public/Avatar/Properties/AvatarPropertyMapper.cs b/src/Tizen.AIAvatar/src/public/Avatar/Properties/AvatarPropertyMapper.cs new file mode 100644 index 0000000..bcd7115 --- /dev/null +++ b/src/Tizen.AIAvatar/src/public/Avatar/Properties/AvatarPropertyMapper.cs @@ -0,0 +1,176 @@ +/* + * Copyright(c) 2023 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +using System.Collections.Generic; +using System.ComponentModel; + +namespace Tizen.AIAvatar +{ + /// + /// TODO : Explain more detail + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public class AvatarPropertyMapper + { + /// + /// Mapper between index and property name + /// + private Dictionary mapper = null; + + /// + /// The counter of index. It will be increased one when we register custom index. + /// + private uint customIndexCounter = 0; + + /// + /// Create an initialized AvatarPropertyNameMapper. + /// + // This will be public opened after ACR done. (Before ACR, need to be hidden as Inhouse API) + [EditorBrowsable(EditorBrowsableState.Never)] + public AvatarPropertyMapper() + { + mapper = new Dictionary(); + customIndexCounter = 0u; + } + + /// + /// Copy constructor. + /// + // This will be public opened after ACR done. (Before ACR, need to be hidden as Inhouse API) + [EditorBrowsable(EditorBrowsableState.Never)] + public AvatarPropertyMapper(AvatarPropertyMapper rhs) + { + if (rhs != null) + { + mapper = new Dictionary(rhs.Mapper); + customIndexCounter = rhs.customIndexCounter; + } + else + { + mapper = new Dictionary(); + customIndexCounter = 0u; + } + } + + [EditorBrowsable(EditorBrowsableState.Never)] + protected uint CustomIndexCounter + { + get + { + return customIndexCounter; + } + set + { + customIndexCounter = value; + } + } + + /// + /// Get current mapper information. + /// + // This will be public opened after ACR done. (Before ACR, need to be hidden as Inhouse API) + [EditorBrowsable(EditorBrowsableState.Never)] + public Dictionary Mapper + { + get + { + return mapper; + } + } + + /// + /// TODO : Explain me. + /// + /// The index of property what we want to set + /// The index of property, or uint.MaxValue if not exist + // This will be public opened after ACR done. (Before ACR, need to be hidden as Inhouse API) + [EditorBrowsable(EditorBrowsableState.Never)] + public string this[uint index] + { + set + { + SetPropertyName(index, value); + } + get + { + return GetPropertyName(index); + } + } + + /// + /// TODO : Explain me. + /// + /// The name of custom property + /// The index of property matched with name. + // This will be public opened after ACR done. (Before ACR, need to be hidden as Inhouse API) + [EditorBrowsable(EditorBrowsableState.Never)] + public uint RegisterCustomProperty(string name) + { + uint ret = GetPropertyIndexByName(name); + if (ret >= customIndexCounter) + { + ret = customIndexCounter++; + SetPropertyName(ret, name); + } + return ret; + } + + /// + /// TODO : Explain me. + /// + /// The name of property what we want to get index + /// The index of property, or uint.MaxValue if not exist + // This will be public opened after ACR done. (Before ACR, need to be hidden as Inhouse API) + [EditorBrowsable(EditorBrowsableState.Never)] + public uint GetPropertyIndexByName(string name) + { + // TODO : Do this without iteration + foreach (var pair in mapper) + { + if (pair.Value == name) + { + return pair.Key; + } + } + return uint.MaxValue; + } + + /// + /// TODO : Explain me. + /// + /// The index of property what we want to set + /// The name of property what we want to set + /// + /// New property will be added if we use index that not exist in mapper. + /// + internal void SetPropertyName(uint index, string name) + { + mapper.TryAdd(index, name); + } + + /// + /// TODO : Explain me. + /// + /// The index of property what we want to set + /// The name of property, or null if not exist + internal string GetPropertyName(uint index) + { + string ret = null; + mapper.TryGetValue(index, out ret); + return ret; + } + } +} -- 2.7.4