1 // This file is part of The New Aspell
2 // Copyright (C) 2002 by Kevin Atkinson under the GNU LGPL license
3 // version 2.0 or 2.1. You should have received a copy of the LGPL
4 // license along with this library if you did not you can find
5 // it at http://www.gnu.org/.
7 #include "tokenizer.hpp"
13 Tokenizer::Tokenizer()
14 : word_begin(0), word_end(0), end(0),
15 begin_pos(0), end_pos(0),
19 Tokenizer::~Tokenizer()
22 void Tokenizer::reset (FilterChar * begin, FilterChar * end)
24 bool can_encode = conv_->encode(begin, end, buf_);