src/Tizen.AIAvatar/src/public/Avatar/Animations/AnimationModule/AsyncLipSyncer.cs

   1 /*
   2  * Copyright(c) 2024 Samsung Electronics Co., Ltd.
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  * http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  *
  16  */
  17
  18 using System;
  19 using System.Collections.Generic;
  20 using System.ComponentModel;
  21 using Tizen.NUI;
  22
  23 using static Tizen.AIAvatar.AIAvatar;
  24
  25 namespace Tizen.AIAvatar
  26 {
  27     [EditorBrowsable(EditorBrowsableState.Never)]
  28     internal class AsyncLipSyncer : LipSyncer
  29     {
  30         private readonly uint AsyncPlayTime = 160;
  31         private Queue<Animation> lipAnimations;
  32         private Queue<byte[]> lipAudios;
  33         private Timer asyncVowelTimer;
  34
  35         private bool isAsyncInit = false;
  36         private bool isFinishAsyncLip = false;
  37
  38         [EditorBrowsable(EditorBrowsableState.Never)]
  39         internal AsyncLipSyncer()
  40         {
  41
  42         }
  43
  44         internal int SampleRate { get; set; }
  45
  46         [EditorBrowsable(EditorBrowsableState.Never)]
  47         internal bool IsAsyncInit { get=>isAsyncInit; set=>isAsyncInit = value; }
  48
  49         [EditorBrowsable(EditorBrowsableState.Never)]
  50         internal void SetFinishAsyncLip(bool finished)
  51         {
  52             isFinishAsyncLip = finished;
  53         }
  54
  55         [EditorBrowsable(EditorBrowsableState.Never)]
  56         protected new void OnLipAnimationFinished(object sender, EventArgs e)
  57         {
  58             if (!isAsyncInit)
  59             {
  60                 CurrentMotionState = AvatarMotionState.Stopped;
  61             }
  62             else
  63             {
  64                 Tizen.Log.Error(LogTag, "OnLipAnimationFinished---------------c");
  65                 //Async State
  66                 if (isFinishAsyncLip && lipAnimations.Count == 0)
  67                 {
  68                     Tizen.Log.Error(LogTag, "Finish vowel lip sync");
  69
  70                     AudioPlayer.Stop();
  71                     CurrentMotionState = AvatarMotionState.Stopped;
  72                     DestroyVowelTimer();
  73                     isAsyncInit = false;
  74                 }
  75             }
  76         }
  77
  78         internal void InitAsyncLipsync()
  79         {
  80             if (lipAnimations == null)
  81             {
  82                 lipAnimations = new Queue<Animation>();
  83             }
  84             else
  85             {
  86                 lipAnimations.Clear();
  87             }
  88
  89             if (lipAudios == null)
  90             {
  91                 lipAudios = new Queue<byte[]>();
  92             }
  93             else
  94             {
  95                 lipAudios.Clear();
  96             }
  97         }
  98
  99         internal void EnqueueAnimation(byte[] recordBuffer, int sampleRate, int audioLength)
 100         {
 101             var createdAni = CreateAsyncLipAnimation(recordBuffer, sampleRate);
 102             if (createdAni != null)
 103             {
 104                 lipAnimations.Enqueue(createdAni);
 105             }
 106
 107             //Use Audio Full File
 108             ///var createdAni = avatarLipSyncer.CreateLipAnimation(recordBuffer, sampleRate);
 109             //lipAnimations.Enqueue(createdAni);
 110
 111             var currentAudioBuffer = new byte[audioLength];
 112             Buffer.BlockCopy(recordBuffer, 0, currentAudioBuffer, 0, audioLength);
 113
 114             lipAudios.Enqueue(currentAudioBuffer);
 115         }
 116
 117         internal bool PlayAsyncLip(int sampleRate, bool isFinishAsyncLip)
 118         {
 119             try
 120             {
 121                 if (lipAudios.Count <= 0 && lipAnimations.Count <= 0)
 122                 {
 123                     Tizen.Log.Info(LogTag, "Return lipaudio 0");
 124                     if (isFinishAsyncLip)
 125                     {
 126                         Tizen.Log.Info(LogTag, "Finish Async lipsync");
 127                         return false;
 128                     }
 129                     return true;
 130                 }
 131                 Tizen.Log.Info(LogTag, "Async timer tick lipAudios : " + lipAudios.Count);
 132                 Tizen.Log.Info(LogTag, "Async timer tick lipAnimations : " + lipAnimations.Count);
 133
 134                 var lipAnimation = lipAnimations.Dequeue();
 135                 lipAnimation.Finished += OnLipAnimationFinished;
 136
 137                 ResetLipAnimation(lipAnimation);
 138                 PlayLipAnimation();
 139                 var audioBuffer = lipAudios.Dequeue();
 140                 AudioPlayer.PlayAsync(audioBuffer, sampleRate);
 141                 return true;
 142
 143             }
 144             catch (Exception ex)
 145             {
 146                 Log.Error(LogTag, $"---Log Tick : {ex.StackTrace}");
 147
 148                 return false;
 149             }
 150         }
 151
 152         internal void StartAsyncLipPlayTimer()
 153         {
 154             if (asyncVowelTimer == null)
 155             {
 156                 Tizen.Log.Info(LogTag, "Start Async");
 157                 asyncVowelTimer = new Timer(AsyncPlayTime);
 158                 asyncVowelTimer.Tick += OnAsyncVowelTick;
 159                 asyncVowelTimer.Start();
 160             }
 161             return;
 162         }
 163
 164         private void DestroyVowelTimer()
 165         {
 166             if (asyncVowelTimer != null)
 167             {
 168
 169                 asyncVowelTimer.Tick -= OnAsyncVowelTick;
 170                 asyncVowelTimer.Stop();
 171                 asyncVowelTimer.Dispose();
 172                 asyncVowelTimer = null;
 173             }
 174         }
 175
 176         private bool OnAsyncVowelTick(object source, Tizen.NUI.Timer.TickEventArgs e)
 177         {
 178             return PlayAsyncLip(SampleRate, isFinishAsyncLip);
 179         }
 180     }
 181 }