src/chrome/browser/speech/speech_recognition_bubble.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/speech/speech_recognition_bubble.h"
   6
   7 #include "base/bind.h"
   8 #include "base/lazy_instance.h"
   9 #include "base/message_loop/message_loop.h"
  10 #include "chrome/browser/tab_contents/tab_util.h"
  11 #include "content/public/browser/web_contents.h"
  12 #include "content/public/browser/web_contents_view.h"
  13 #include "grit/generated_resources.h"
  14 #include "grit/theme_resources.h"
  15 #include "ui/base/resource/resource_bundle.h"
  16 #include "ui/gfx/canvas.h"
  17 #include "ui/gfx/display.h"
  18 #include "ui/gfx/image/image_skia_operations.h"
  19 #include "ui/gfx/rect.h"
  20 #include "ui/gfx/screen.h"
  21
  22 using content::WebContents;
  23
  24 namespace {
  25
  26 const color_utils::HSL kGrayscaleShift = { -1, 0, 0.6 };
  27 const int kWarmingUpAnimationStartMs = 500;
  28 const int kWarmingUpAnimationStepMs = 100;
  29 const int kRecognizingAnimationStepMs = 100;
  30
  31 // A lazily initialized singleton to hold all the image used by the speech
  32 // recognition bubbles and safely destroy them on exit.
  33 class SpeechRecognitionBubbleImages {
  34  public:
  35   const std::vector<gfx::ImageSkia>& spinner() const { return spinner_; }
  36   const std::vector<gfx::ImageSkia>& warm_up() const { return warm_up_; }
  37   gfx::ImageSkia* mic_full() const { return mic_full_; }
  38   gfx::ImageSkia* mic_empty() const { return mic_empty_; }
  39   gfx::ImageSkia* mic_noise() const { return mic_noise_; }
  40   gfx::ImageSkia* mic_mask() const { return mic_mask_; }
  41
  42  private:
  43   // Private constructor to enforce singleton.
  44   friend struct base::DefaultLazyInstanceTraits<SpeechRecognitionBubbleImages>;
  45   SpeechRecognitionBubbleImages();
  46
  47   std::vector<gfx::ImageSkia> spinner_;  // Frames for the progress spinner.
  48   std::vector<gfx::ImageSkia> warm_up_;  // Frames for the warm up animation.
  49
  50   // These images are owned by ResourceBundle and need not be destroyed.
  51   gfx::ImageSkia* mic_full_;  // Mic image with full volume.
  52   gfx::ImageSkia* mic_noise_;  // Mic image with full noise volume.
  53   gfx::ImageSkia* mic_empty_;  // Mic image with zero volume.
  54   gfx::ImageSkia* mic_mask_;  // Gradient mask used by the volume indicator.
  55 };
  56
  57 SpeechRecognitionBubbleImages::SpeechRecognitionBubbleImages() {
  58   mic_empty_ = ui::ResourceBundle::GetSharedInstance().GetImageSkiaNamed(
  59       IDR_SPEECH_INPUT_MIC_EMPTY);
  60   mic_noise_ = ui::ResourceBundle::GetSharedInstance().GetImageSkiaNamed(
  61       IDR_SPEECH_INPUT_MIC_NOISE);
  62   mic_full_ = ui::ResourceBundle::GetSharedInstance().GetImageSkiaNamed(
  63       IDR_SPEECH_INPUT_MIC_FULL);
  64   mic_mask_ = ui::ResourceBundle::GetSharedInstance().GetImageSkiaNamed(
  65       IDR_SPEECH_INPUT_MIC_MASK);
  66
  67   // The sprite image consists of all the animation frames put together in one
  68   // horizontal/wide image. Each animation frame is square in shape within the
  69   // sprite.
  70   const gfx::ImageSkia* spinner_image = ui::ResourceBundle::GetSharedInstance().
  71       GetImageSkiaNamed(IDR_SPEECH_INPUT_SPINNER);
  72   int frame_size = spinner_image->height();
  73
  74   // When recording starts up, it may take a short while (few ms or even a
  75   // couple of seconds) before the audio device starts really capturing data.
  76   // This is more apparent on first use. To cover such cases we show a warming
  77   // up state in the bubble starting with a blank spinner image. If audio data
  78   // starts coming in within a couple hundred ms, we switch to the recording
  79   // UI and if it takes longer, we show the real warm up animation frames.
  80   // This reduces visual jank for the most part.
  81   SkBitmap empty_spinner;
  82   empty_spinner.setConfig(SkBitmap::kARGB_8888_Config, frame_size, frame_size);
  83   empty_spinner.allocPixels();
  84   empty_spinner.eraseRGB(255, 255, 255);
  85   // |empty_spinner| has solid color. Pixel doubling a solid color is ok.
  86   warm_up_.push_back(gfx::ImageSkia::CreateFrom1xBitmap(empty_spinner));
  87
  88   for (gfx::Rect src_rect(frame_size, frame_size);
  89        src_rect.x() < spinner_image->width();
  90        src_rect.Offset(frame_size, 0)) {
  91     gfx::ImageSkia frame = gfx::ImageSkiaOperations::ExtractSubset(
  92         *spinner_image, src_rect);
  93
  94     // The image created by ExtractSubset just points to the same pixels as
  95     // the original and adjusts rowBytes accordingly. However that doesn't
  96     // render properly and gets vertically squished in Linux due to a bug in
  97     // Skia. Until that gets fixed we work around by taking a real copy of it
  98     // below as the copied image has the correct rowBytes and renders fine.
  99     frame.EnsureRepsForSupportedScales();
 100     std::vector<gfx::ImageSkiaRep> image_reps = frame.image_reps();
 101     gfx::ImageSkia frame_copy;
 102     for (size_t i = 0; i < image_reps.size(); ++i) {
 103       const SkBitmap& copy_src = image_reps[i].sk_bitmap();
 104       SkBitmap copy_dst;
 105       copy_src.copyTo(&copy_dst, SkBitmap::kARGB_8888_Config);
 106       frame_copy.AddRepresentation(gfx::ImageSkiaRep(
 107           copy_dst, image_reps[i].scale()));
 108     }
 109     spinner_.push_back(frame_copy);
 110
 111     // The warm up spinner animation is a gray scale version of the real one.
 112     warm_up_.push_back(gfx::ImageSkiaOperations::CreateHSLShiftedImage(
 113         frame_copy, kGrayscaleShift));
 114   }
 115 }
 116
 117 base::LazyInstance<SpeechRecognitionBubbleImages> g_images =
 118     LAZY_INSTANCE_INITIALIZER;
 119
 120 }  // namespace
 121
 122 SpeechRecognitionBubble::FactoryMethod SpeechRecognitionBubble::factory_ = NULL;
 123 const int SpeechRecognitionBubble::kBubbleTargetOffsetX = 10;
 124
 125 SpeechRecognitionBubble* SpeechRecognitionBubble::Create(
 126     int render_process_id, int render_view_id, Delegate* delegate,
 127     const gfx::Rect& element_rect) {
 128   WebContents* web_contents =
 129       tab_util::GetWebContentsByID(render_process_id, render_view_id);
 130
 131   if (factory_)
 132     return (*factory_)(web_contents, delegate, element_rect);
 133
 134   // Has the tab already closed before bubble create request was processed?
 135   if (!web_contents)
 136     return NULL;
 137
 138   return CreateNativeBubble(render_process_id, render_view_id,
 139       delegate, element_rect);
 140 }
 141
 142 SpeechRecognitionBubbleBase::SpeechRecognitionBubbleBase(
 143     int render_process_id, int render_view_id)
 144     : weak_factory_(this),
 145       animation_step_(0),
 146       display_mode_(DISPLAY_MODE_RECORDING),
 147       render_process_id_(render_process_id),
 148       render_view_id_(render_view_id),
 149       scale_(1.0f) {
 150   WebContents* web_contents = GetWebContents();
 151   gfx::NativeView view =
 152       web_contents ? web_contents->GetView()->GetNativeView() : NULL;
 153   gfx::Screen* screen = gfx::Screen::GetScreenFor(view);
 154   gfx::Display display = screen->GetDisplayNearestWindow(view);
 155   scale_ = display.device_scale_factor();
 156
 157   const gfx::ImageSkiaRep& rep =
 158       g_images.Get().mic_empty()->GetRepresentation(scale_);
 159   mic_image_.reset(new SkBitmap());
 160   mic_image_->setConfig(SkBitmap::kARGB_8888_Config,
 161       rep.pixel_width(), rep.pixel_height());
 162   mic_image_->allocPixels();
 163
 164   buffer_image_.reset(new SkBitmap());
 165   buffer_image_->setConfig(SkBitmap::kARGB_8888_Config,
 166       rep.pixel_width(), rep.pixel_height());
 167   buffer_image_->allocPixels();
 168 }
 169
 170 SpeechRecognitionBubbleBase::~SpeechRecognitionBubbleBase() {
 171   // This destructor is added to make sure members such as the scoped_ptr
 172   // get destroyed here and the derived classes don't have to care about such
 173   // member variables which they don't use.
 174 }
 175
 176 void SpeechRecognitionBubbleBase::SetWarmUpMode() {
 177   weak_factory_.InvalidateWeakPtrs();
 178   display_mode_ = DISPLAY_MODE_WARM_UP;
 179   animation_step_ = 0;
 180   DoWarmingUpAnimationStep();
 181   UpdateLayout();
 182 }
 183
 184 void SpeechRecognitionBubbleBase::DoWarmingUpAnimationStep() {
 185   SetImage(g_images.Get().warm_up()[animation_step_]);
 186   base::MessageLoop::current()->PostDelayedTask(
 187       FROM_HERE,
 188       base::Bind(&SpeechRecognitionBubbleBase::DoWarmingUpAnimationStep,
 189           weak_factory_.GetWeakPtr()),
 190       base::TimeDelta::FromMilliseconds(
 191           animation_step_ == 0 ? kWarmingUpAnimationStartMs
 192                                : kWarmingUpAnimationStepMs));
 193   if (++animation_step_ >= static_cast<int>(g_images.Get().warm_up().size()))
 194     animation_step_ = 1;  // Frame 0 is skipped during the animation.
 195 }
 196
 197 void SpeechRecognitionBubbleBase::SetRecordingMode() {
 198   weak_factory_.InvalidateWeakPtrs();
 199   display_mode_ = DISPLAY_MODE_RECORDING;
 200   SetInputVolume(0, 0);
 201   UpdateLayout();
 202 }
 203
 204 void SpeechRecognitionBubbleBase::SetRecognizingMode() {
 205   display_mode_ = DISPLAY_MODE_RECOGNIZING;
 206   animation_step_ = 0;
 207   DoRecognizingAnimationStep();
 208   UpdateLayout();
 209 }
 210
 211 void SpeechRecognitionBubbleBase::DoRecognizingAnimationStep() {
 212   SetImage(g_images.Get().spinner()[animation_step_]);
 213   if (++animation_step_ >= static_cast<int>(g_images.Get().spinner().size()))
 214     animation_step_ = 0;
 215   base::MessageLoop::current()->PostDelayedTask(
 216       FROM_HERE,
 217       base::Bind(&SpeechRecognitionBubbleBase::DoRecognizingAnimationStep,
 218           weak_factory_.GetWeakPtr()),
 219       base::TimeDelta::FromMilliseconds(kRecognizingAnimationStepMs));
 220 }
 221
 222 void SpeechRecognitionBubbleBase::SetMessage(const base::string16& text) {
 223   weak_factory_.InvalidateWeakPtrs();
 224   message_text_ = text;
 225   display_mode_ = DISPLAY_MODE_MESSAGE;
 226   UpdateLayout();
 227 }
 228
 229 void SpeechRecognitionBubbleBase::DrawVolumeOverlay(SkCanvas* canvas,
 230                                                     const gfx::ImageSkia& image,
 231                                                     float volume) {
 232   buffer_image_->eraseARGB(0, 0, 0, 0);
 233
 234   int width = mic_image_->width();
 235   int height = mic_image_->height();
 236   SkCanvas buffer_canvas(*buffer_image_);
 237
 238   buffer_canvas.save();
 239   const int kVolumeSteps = 12;
 240   SkScalar clip_right =
 241       (((1.0f - volume) * (width * (kVolumeSteps + 1))) - width) / kVolumeSteps;
 242   buffer_canvas.clipRect(SkRect::MakeLTRB(0, 0,
 243       SkIntToScalar(width) - clip_right, SkIntToScalar(height)));
 244   buffer_canvas.drawBitmap(image.GetRepresentation(scale_).sk_bitmap(), 0, 0);
 245   buffer_canvas.restore();
 246   SkPaint multiply_paint;
 247   multiply_paint.setXfermodeMode(SkXfermode::kModulate_Mode);
 248   buffer_canvas.drawBitmap(
 249       g_images.Get().mic_mask()->GetRepresentation(scale_).sk_bitmap(),
 250       -clip_right, 0, &multiply_paint);
 251
 252   canvas->drawBitmap(*buffer_image_.get(), 0, 0);
 253 }
 254
 255 void SpeechRecognitionBubbleBase::SetInputVolume(float volume,
 256                                                  float noise_volume) {
 257   mic_image_->eraseARGB(0, 0, 0, 0);
 258   SkCanvas canvas(*mic_image_);
 259
 260   // Draw the empty volume image first and the current volume image on top,
 261   // and then the noise volume image on top of both.
 262   canvas.drawBitmap(
 263       g_images.Get().mic_empty()->GetRepresentation(scale_).sk_bitmap(),
 264       0, 0);
 265   DrawVolumeOverlay(&canvas, *g_images.Get().mic_full(), volume);
 266   DrawVolumeOverlay(&canvas, *g_images.Get().mic_noise(), noise_volume);
 267
 268   gfx::ImageSkia image(gfx::ImageSkiaRep(*mic_image_.get(), scale_));
 269   SetImage(image);
 270 }
 271
 272 WebContents* SpeechRecognitionBubbleBase::GetWebContents() {
 273   return tab_util::GetWebContentsByID(render_process_id_, render_view_id_);
 274 }
 275
 276 void SpeechRecognitionBubbleBase::SetImage(const gfx::ImageSkia& image) {
 277   icon_image_ = image;
 278   UpdateImage();
 279 }
 280
 281 gfx::ImageSkia SpeechRecognitionBubbleBase::icon_image() {
 282   return icon_image_;
 283 }