src/scanner-character-streams.h

   1 // Copyright 2011 the V8 project authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #ifndef V8_SCANNER_CHARACTER_STREAMS_H_
   6 #define V8_SCANNER_CHARACTER_STREAMS_H_
   7
   8 #include "src/scanner.h"
   9
  10 namespace v8 {
  11 namespace internal {
  12
  13 // A buffered character stream based on a random access character
  14 // source (ReadBlock can be called with pos_ pointing to any position,
  15 // even positions before the current).
  16 class BufferedUtf16CharacterStream: public Utf16CharacterStream {
  17  public:
  18   BufferedUtf16CharacterStream();
  19   virtual ~BufferedUtf16CharacterStream();
  20
  21   virtual void PushBack(uc32 character);
  22
  23  protected:
  24   static const size_t kBufferSize = 512;
  25   static const size_t kPushBackStepSize = 16;
  26
  27   virtual size_t SlowSeekForward(size_t delta);
  28   virtual bool ReadBlock();
  29   virtual void SlowPushBack(uc16 character);
  30
  31   virtual size_t BufferSeekForward(size_t delta) = 0;
  32   virtual size_t FillBuffer(size_t position) = 0;
  33
  34   const uc16* pushback_limit_;
  35   uc16 buffer_[kBufferSize];
  36 };
  37
  38
  39 // Generic string stream.
  40 class GenericStringUtf16CharacterStream: public BufferedUtf16CharacterStream {
  41  public:
  42   GenericStringUtf16CharacterStream(Handle<String> data, size_t start_position,
  43                                     size_t end_position);
  44   virtual ~GenericStringUtf16CharacterStream();
  45
  46   virtual bool SetBookmark();
  47   virtual void ResetToBookmark();
  48
  49  protected:
  50   static const size_t kNoBookmark = -1;
  51
  52   virtual size_t BufferSeekForward(size_t delta);
  53   virtual size_t FillBuffer(size_t position);
  54
  55   Handle<String> string_;
  56   size_t length_;
  57   size_t bookmark_;
  58 };
  59
  60
  61 // Utf16 stream based on a literal UTF-8 string.
  62 class Utf8ToUtf16CharacterStream: public BufferedUtf16CharacterStream {
  63  public:
  64   Utf8ToUtf16CharacterStream(const byte* data, size_t length);
  65   virtual ~Utf8ToUtf16CharacterStream();
  66
  67   static size_t CopyChars(uint16_t* dest, size_t length, const byte* src,
  68                           size_t* src_pos, size_t src_length);
  69
  70  protected:
  71   virtual size_t BufferSeekForward(size_t delta);
  72   virtual size_t FillBuffer(size_t char_position);
  73   void SetRawPosition(size_t char_position);
  74
  75   const byte* raw_data_;
  76   size_t raw_data_length_;  // Measured in bytes, not characters.
  77   size_t raw_data_pos_;
  78   // The character position of the character at raw_data[raw_data_pos_].
  79   // Not necessarily the same as pos_.
  80   size_t raw_character_position_;
  81 };
  82
  83
  84 // ExternalStreamingStream is a wrapper around an ExternalSourceStream (see
  85 // include/v8.h) subclass implemented by the embedder.
  86 class ExternalStreamingStream : public BufferedUtf16CharacterStream {
  87  public:
  88   ExternalStreamingStream(ScriptCompiler::ExternalSourceStream* source_stream,
  89                           v8::ScriptCompiler::StreamedSource::Encoding encoding)
  90       : source_stream_(source_stream),
  91         encoding_(encoding),
  92         current_data_(NULL),
  93         current_data_offset_(0),
  94         current_data_length_(0),
  95         utf8_split_char_buffer_length_(0),
  96         bookmark_(0),
  97         bookmark_utf8_split_char_buffer_length_(0) {}
  98
  99   virtual ~ExternalStreamingStream() {
 100     delete[] current_data_;
 101     bookmark_buffer_.Dispose();
 102     bookmark_data_.Dispose();
 103   }
 104
 105   size_t BufferSeekForward(size_t delta) override {
 106     // We never need to seek forward when streaming scripts. We only seek
 107     // forward when we want to parse a function whose location we already know,
 108     // and when streaming, we don't know the locations of anything we haven't
 109     // seen yet.
 110     UNREACHABLE();
 111     return 0;
 112   }
 113
 114   size_t FillBuffer(size_t position) override;
 115
 116   virtual bool SetBookmark() override;
 117   virtual void ResetToBookmark() override;
 118
 119  private:
 120   void HandleUtf8SplitCharacters(size_t* data_in_buffer);
 121   void FlushCurrent();
 122
 123   ScriptCompiler::ExternalSourceStream* source_stream_;
 124   v8::ScriptCompiler::StreamedSource::Encoding encoding_;
 125   const uint8_t* current_data_;
 126   size_t current_data_offset_;
 127   size_t current_data_length_;
 128   // For converting UTF-8 characters which are split across two data chunks.
 129   uint8_t utf8_split_char_buffer_[4];
 130   size_t utf8_split_char_buffer_length_;
 131
 132   // Bookmark support. See comments in ExternalStreamingStream::SetBookmark
 133   // for additional details.
 134   size_t bookmark_;
 135   Vector<uint16_t> bookmark_buffer_;
 136   Vector<uint8_t> bookmark_data_;
 137   uint8_t bookmark_utf8_split_char_buffer_[4];
 138   size_t bookmark_utf8_split_char_buffer_length_;
 139 };
 140
 141
 142 // UTF16 buffer to read characters from an external string.
 143 class ExternalTwoByteStringUtf16CharacterStream: public Utf16CharacterStream {
 144  public:
 145   ExternalTwoByteStringUtf16CharacterStream(Handle<ExternalTwoByteString> data,
 146                                             int start_position,
 147                                             int end_position);
 148   virtual ~ExternalTwoByteStringUtf16CharacterStream();
 149
 150   virtual void PushBack(uc32 character) {
 151     DCHECK(buffer_cursor_ > raw_data_);
 152     buffer_cursor_--;
 153     pos_--;
 154   }
 155
 156   virtual bool SetBookmark();
 157   virtual void ResetToBookmark();
 158
 159  protected:
 160   virtual size_t SlowSeekForward(size_t delta) {
 161     // Fast case always handles seeking.
 162     return 0;
 163   }
 164   virtual bool ReadBlock() {
 165     // Entire string is read at start.
 166     return false;
 167   }
 168   Handle<ExternalTwoByteString> source_;
 169   const uc16* raw_data_;  // Pointer to the actual array of characters.
 170
 171  private:
 172   static const size_t kNoBookmark = -1;
 173
 174   size_t bookmark_;
 175 };
 176
 177 } }  // namespace v8::internal
 178
 179 #endif  // V8_SCANNER_CHARACTER_STREAMS_H_