1 // Boost token_functions.hpp ------------------------------------------------//
3 // Copyright John R. Bandela 2001.
5 // Distributed under the Boost Software License, Version 1.0. (See
6 // accompanying file LICENSE_1_0.txt or copy at
7 // http://www.boost.org/LICENSE_1_0.txt)
9 // See http://www.boost.org/libs/tokenizer/ for documentation.
12 // 01 Oct 2004 Joaquin M Lopez Munoz
13 // Workaround for a problem with string::assign in msvc-stlport
14 // 06 Apr 2004 John Bandela
15 // Fixed a bug involving using char_delimiter with a true input iterator
16 // 28 Nov 2003 Robert Zeh and John Bandela
17 // Converted into "fast" functions that avoid using += when
18 // the supplied iterator isn't an input_iterator; based on
19 // some work done at Archelon and a version that was checked into
20 // the boost CVS for a short period of time.
21 // 20 Feb 2002 John Maddock
22 // Removed using namespace std declarations and added
23 // workaround for BOOST_NO_STDC_NAMESPACE (the library
24 // can be safely mixed with regex).
25 // 06 Feb 2002 Jeremy Siek
26 // Added char_separator.
27 // 02 Feb 2002 Jeremy Siek
28 // Removed tabs and a little cleanup.
31 #ifndef BOOST_TOKEN_FUNCTIONS_JRB120303_HPP_
32 #define BOOST_TOKEN_FUNCTIONS_JRB120303_HPP_
38 #include <algorithm> // for find_if
39 #include <boost/config.hpp>
40 #include <boost/assert.hpp>
41 #include <boost/detail/workaround.hpp>
42 #include <boost/mpl/if.hpp>
43 #if !defined(BOOST_NO_CWCTYPE)
48 // the following must not be macros if we are to prefix them
49 // with std:: (they shouldn't be macros anyway...)
64 // fix namespace problems:
66 #ifdef BOOST_NO_STDC_NAMESPACE
70 #if !defined(BOOST_NO_CWCTYPE)
78 //===========================================================================
79 // The escaped_list_separator class. Which is a model of TokenizerFunction
80 // An escaped list is a super-set of what is commonly known as a comma
81 // separated value (csv) list.It is separated into fields by a comma or
82 // other character. If the delimiting character is inside quotes, then it is
83 // counted as a regular character.To allow for embedded quotes in a field,
84 // there can be escape sequences using the \ much like C.
85 // The role of the comma, the quotation mark, and the escape
86 // character (backslash \), can be assigned to other characters.
88 struct escaped_list_error : public std::runtime_error{
89 escaped_list_error(const std::string& what_arg):std::runtime_error(what_arg) { }
93 // The out of the box GCC 2.95 on cygwin does not have a char_traits class.
94 // MSVC does not like the following typename
96 class Traits = BOOST_DEDUCED_TYPENAME std::basic_string<Char>::traits_type >
97 class escaped_list_separator {
100 typedef std::basic_string<Char,Traits> string_type;
103 char_eq(Char e):e_(e) { }
104 bool operator()(Char c) {
105 return Traits::eq(e_,c);
113 bool is_escape(Char e) {
115 return std::find_if(escape_.begin(),escape_.end(),f)!=escape_.end();
119 return std::find_if(c_.begin(),c_.end(),f)!=c_.end();
121 bool is_quote(Char e) {
123 return std::find_if(quote_.begin(),quote_.end(),f)!=quote_.end();
125 template <typename iterator, typename Token>
126 void do_escape(iterator& next,iterator end,Token& tok) {
128 throw escaped_list_error(std::string("cannot end with escape"));
129 if (Traits::eq(*next,'n')) {
133 else if (is_quote(*next)) {
137 else if (is_c(*next)) {
141 else if (is_escape(*next)) {
146 throw escaped_list_error(std::string("unknown escape sequence"));
151 explicit escaped_list_separator(Char e = '\\',
152 Char c = ',',Char q = '\"')
153 : escape_(1,e), c_(1,c), quote_(1,q), last_(false) { }
155 escaped_list_separator(string_type e, string_type c, string_type q)
156 : escape_(e), c_(c), quote_(q), last_(false) { }
158 void reset() {last_=false;}
160 template <typename InputIterator, typename Token>
161 bool operator()(InputIterator& next,InputIterator end,Token& tok) {
162 bool bInQuote = false;
174 for (;next != end;++next) {
175 if (is_escape(*next)) {
176 do_escape(next,end,tok);
178 else if (is_c(*next)) {
180 // If we are not in quote, then we are done
182 // The last character was a c, that means there is
183 // 1 more blank field
189 else if (is_quote(*next)) {
200 //===========================================================================
201 // The classes here are used by offset_separator and char_separator to implement
202 // faster assigning of tokens using assign instead of +=
204 namespace tokenizer_detail {
205 //===========================================================================
206 // Tokenizer was broken for wide character separators, at least on Windows, since
207 // CRT functions isspace etc only expect values in [0, 0xFF]. Debug build asserts
208 // if higher values are passed in. The traits extension class should take care of this.
209 // Assuming that the conditional will always get optimized out in the function
210 // implementations, argument types are not a problem since both forms of character classifiers
213 #if !defined(BOOST_NO_CWCTYPE)
214 template<typename traits, int N>
215 struct traits_extension_details : public traits {
216 typedef typename traits::char_type char_type;
217 static bool isspace(char_type c)
219 return std::iswspace(c) != 0;
221 static bool ispunct(char_type c)
223 return std::iswpunct(c) != 0;
227 template<typename traits>
228 struct traits_extension_details<traits, 1> : public traits {
229 typedef typename traits::char_type char_type;
230 static bool isspace(char_type c)
232 return std::isspace(c) != 0;
234 static bool ispunct(char_type c)
236 return std::ispunct(c) != 0;
242 // In case there is no cwctype header, we implement the checks manually.
243 // We make use of the fact that the tested categories should fit in ASCII.
244 template<typename traits>
245 struct traits_extension : public traits {
246 typedef typename traits::char_type char_type;
247 static bool isspace(char_type c)
249 #if !defined(BOOST_NO_CWCTYPE)
250 return traits_extension_details<traits, sizeof(char_type)>::isspace(c);
252 return static_cast< unsigned >(c) <= 255 && std::isspace(c) != 0;
256 static bool ispunct(char_type c)
258 #if !defined(BOOST_NO_CWCTYPE)
259 return traits_extension_details<traits, sizeof(char_type)>::ispunct(c);
261 return static_cast< unsigned >(c) <= 255 && std::ispunct(c) != 0;
266 // The assign_or_plus_equal struct contains functions that implement
267 // assign, +=, and clearing based on the iterator type. The
268 // generic case does nothing for plus_equal and clearing, while
269 // passing through the call for assign.
271 // When an input iterator is being used, the situation is reversed.
272 // The assign method does nothing, plus_equal invokes operator +=,
273 // and the clearing method sets the supplied token to the default
274 // token constructor's result.
277 template<class IteratorTag>
278 struct assign_or_plus_equal {
279 template<class Iterator, class Token>
280 static void assign(Iterator b, Iterator e, Token &t) {
284 template<class Token, class Value>
285 static void plus_equal(Token &, const Value &) { }
287 // If we are doing an assign, there is no need for the
290 template<class Token>
291 static void clear(Token &) { }
295 struct assign_or_plus_equal<std::input_iterator_tag> {
296 template<class Iterator, class Token>
297 static void assign(Iterator , Iterator , Token &) { }
298 template<class Token, class Value>
299 static void plus_equal(Token &t, const Value &v) {
302 template<class Token>
303 static void clear(Token &t) {
309 template<class Iterator>
310 struct pointer_iterator_category{
311 typedef std::random_access_iterator_tag type;
315 template<class Iterator>
316 struct class_iterator_category{
317 typedef typename Iterator::iterator_category type;
322 // This portably gets the iterator_tag without partial template specialization
323 template<class Iterator>
324 struct get_iterator_category{
325 typedef typename mpl::if_<is_pointer<Iterator>,
326 pointer_iterator_category<Iterator>,
327 class_iterator_category<Iterator>
330 typedef typename cat::type iterator_category;
334 } // namespace tokenizer_detail
337 //===========================================================================
338 // The offset_separator class, which is a model of TokenizerFunction.
339 // Offset breaks a string into tokens based on a range of offsets
341 class offset_separator {
344 std::vector<int> offsets_;
345 unsigned int current_offset_;
347 bool return_partial_last_;
350 template <typename Iter>
351 offset_separator(Iter begin, Iter end, bool wrap_offsets = true,
352 bool return_partial_last = true)
353 : offsets_(begin,end), current_offset_(0),
354 wrap_offsets_(wrap_offsets),
355 return_partial_last_(return_partial_last) { }
358 : offsets_(1,1), current_offset_(),
359 wrap_offsets_(true), return_partial_last_(true) { }
365 template <typename InputIterator, typename Token>
366 bool operator()(InputIterator& next, InputIterator end, Token& tok)
368 typedef tokenizer_detail::assign_or_plus_equal<
369 BOOST_DEDUCED_TYPENAME tokenizer_detail::get_iterator_category<
374 BOOST_ASSERT(!offsets_.empty());
376 assigner::clear(tok);
377 InputIterator start(next);
382 if (current_offset_ == offsets_.size())
390 int c = offsets_[current_offset_];
393 if (next == end)break;
394 assigner::plus_equal(tok,*next++);
396 assigner::assign(start,next,tok);
398 if (!return_partial_last_)
408 //===========================================================================
409 // The char_separator class breaks a sequence of characters into
410 // tokens based on the character delimiters (very much like bad old
411 // strtok). A delimiter character can either be kept or dropped. A
412 // kept delimiter shows up as an output token, whereas a dropped
413 // delimiter does not.
415 // This class replaces the char_delimiters_separator class. The
416 // constructor for the char_delimiters_separator class was too
417 // confusing and needed to be deprecated. However, because of the
418 // default arguments to the constructor, adding the new constructor
419 // would cause ambiguity, so instead I deprecated the whole class.
420 // The implementation of the class was also simplified considerably.
422 enum empty_token_policy { drop_empty_tokens, keep_empty_tokens };
424 // The out of the box GCC 2.95 on cygwin does not have a char_traits class.
425 template <typename Char,
426 typename Tr = BOOST_DEDUCED_TYPENAME std::basic_string<Char>::traits_type >
429 typedef tokenizer_detail::traits_extension<Tr> Traits;
430 typedef std::basic_string<Char,Tr> string_type;
433 char_separator(const Char* dropped_delims,
434 const Char* kept_delims = 0,
435 empty_token_policy empty_tokens = drop_empty_tokens)
436 : m_dropped_delims(dropped_delims),
437 m_use_ispunct(false),
438 m_use_isspace(false),
439 m_empty_tokens(empty_tokens),
442 // Borland workaround
444 m_kept_delims = kept_delims;
447 // use ispunct() for kept delimiters and isspace for dropped.
450 : m_use_ispunct(true),
452 m_empty_tokens(drop_empty_tokens) { }
456 template <typename InputIterator, typename Token>
457 bool operator()(InputIterator& next, InputIterator end, Token& tok)
459 typedef tokenizer_detail::assign_or_plus_equal<
460 BOOST_DEDUCED_TYPENAME tokenizer_detail::get_iterator_category<
465 assigner::clear(tok);
467 // skip past all dropped_delims
468 if (m_empty_tokens == drop_empty_tokens)
469 for (; next != end && is_dropped(*next); ++next)
472 InputIterator start(next);
474 if (m_empty_tokens == drop_empty_tokens) {
480 // if we are on a kept_delims move past it and stop
481 if (is_kept(*next)) {
482 assigner::plus_equal(tok,*next);
485 // append all the non delim characters
486 for (; next != end && !is_dropped(*next) && !is_kept(*next); ++next)
487 assigner::plus_equal(tok,*next);
489 else { // m_empty_tokens == keep_empty_tokens
491 // Handle empty token at the end
494 if (m_output_done == false)
496 m_output_done = true;
497 assigner::assign(start,next,tok);
504 if (is_kept(*next)) {
505 if (m_output_done == false)
506 m_output_done = true;
508 assigner::plus_equal(tok,*next);
510 m_output_done = false;
513 else if (m_output_done == false && is_dropped(*next)) {
514 m_output_done = true;
517 if (is_dropped(*next))
519 for (; next != end && !is_dropped(*next) && !is_kept(*next); ++next)
520 assigner::plus_equal(tok,*next);
521 m_output_done = true;
524 assigner::assign(start,next,tok);
529 string_type m_kept_delims;
530 string_type m_dropped_delims;
533 empty_token_policy m_empty_tokens;
536 bool is_kept(Char E) const
538 if (m_kept_delims.length())
539 return m_kept_delims.find(E) != string_type::npos;
540 else if (m_use_ispunct) {
541 return Traits::ispunct(E) != 0;
545 bool is_dropped(Char E) const
547 if (m_dropped_delims.length())
548 return m_dropped_delims.find(E) != string_type::npos;
549 else if (m_use_isspace) {
550 return Traits::isspace(E) != 0;
556 //===========================================================================
557 // The following class is DEPRECATED, use class char_separators instead.
559 // The char_delimiters_separator class, which is a model of
560 // TokenizerFunction. char_delimiters_separator breaks a string
561 // into tokens based on character delimiters. There are 2 types of
562 // delimiters. returnable delimiters can be returned as
563 // tokens. These are often punctuation. nonreturnable delimiters
564 // cannot be returned as tokens. These are often whitespace
566 // The out of the box GCC 2.95 on cygwin does not have a char_traits class.
567 template <class Char,
568 class Tr = BOOST_DEDUCED_TYPENAME std::basic_string<Char>::traits_type >
569 class char_delimiters_separator {
572 typedef tokenizer_detail::traits_extension<Tr> Traits;
573 typedef std::basic_string<Char,Tr> string_type;
574 string_type returnable_;
575 string_type nonreturnable_;
580 bool is_ret(Char E)const
582 if (returnable_.length())
583 return returnable_.find(E) != string_type::npos;
585 if (no_ispunct_) {return false;}
587 int r = Traits::ispunct(E);
592 bool is_nonret(Char E)const
594 if (nonreturnable_.length())
595 return nonreturnable_.find(E) != string_type::npos;
597 if (no_isspace_) {return false;}
599 int r = Traits::isspace(E);
606 explicit char_delimiters_separator(bool return_delims = false,
607 const Char* returnable = 0,
608 const Char* nonreturnable = 0)
609 : returnable_(returnable ? returnable : string_type().c_str()),
610 nonreturnable_(nonreturnable ? nonreturnable:string_type().c_str()),
611 return_delims_(return_delims), no_ispunct_(returnable!=0),
612 no_isspace_(nonreturnable!=0) { }
618 template <typename InputIterator, typename Token>
619 bool operator()(InputIterator& next, InputIterator end,Token& tok) {
622 // skip past all nonreturnable delims
623 // skip past the returnable only if we are not returning delims
624 for (;next!=end && ( is_nonret(*next) || (is_ret(*next)
625 && !return_delims_ ) );++next) { }
631 // if we are to return delims and we are one a returnable one
632 // move past it and stop
633 if (is_ret(*next) && return_delims_) {
638 // append all the non delim characters
639 for (;next!=end && !is_nonret(*next) && !is_ret(*next);++next)