modules/test/include/dpl/test/value_separated_tokenizer.h

   1 /*
   2  * Copyright (c) 2013 Samsung Electronics Co., Ltd All Rights Reserved
   3  *
   4  *    Licensed under the Apache License, Version 2.0 (the "License");
   5  *    you may not use this file except in compliance with the License.
   6  *    You may obtain a copy of the License at
   7  *
   8  *        http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  *    Unless required by applicable law or agreed to in writing, software
  11  *    distributed under the License is distributed on an "AS IS" BASIS,
  12  *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  *    See the License for the specific language governing permissions and
  14  *    limitations under the License.
  15  */
  16 /*
  17  * @file        value_separated_tokenizer.h
  18  * @author      Tomasz Iwanek (t.iwanek@samsung.com)
  19  * @brief       Tokenizer for some value seperated files/data
  20  */
  21
  22 #ifndef VALUE_SEPARATED_TOKENIZER_H
  23 #define VALUE_SEPARATED_TOKENIZER_H
  24
  25 #include<dpl/test/abstract_input_tokenizer.h>
  26 #include<dpl/test/value_separated_tokens.h>
  27 #include<dpl/binary_queue.h>
  28
  29
  30 namespace DPL {
  31
  32 /**
  33  * Value Sperated tokenizer
  34  *
  35  * Requires following policy class:
  36  *
  37  * struct TokenizerPolicy
  38  * {
  39  *     static std::string GetSeperators();
  40  *     static bool SkipEmpty();
  41  *     static void PrepareValue(std::string & value);
  42  * };
  43  */
  44 template<class TokenizerPolicy>
  45 class VSTokenizer : public AbstractInputTokenizer<VSToken>
  46 {
  47 public:
  48     VSTokenizer() {}
  49
  50     void Reset(std::shared_ptr<AbstractInput> ia)
  51     {
  52         AbstractInputTokenizer<VSToken>::Reset(ia);
  53         m_queue.Clear();
  54         m_finished = false;
  55         m_newline = false;
  56     }
  57
  58     std::unique_ptr<VSToken> GetNextToken()
  59     {
  60         std::unique_ptr<VSToken> token;
  61         std::string data;
  62         char byte;
  63         int tryNumber = 0;
  64
  65         while(true)
  66         {
  67             //check if newline was approched
  68             if(m_newline)
  69             {
  70                 token.reset(new VSToken());
  71                 m_newline = false;
  72                 return token;
  73             }
  74
  75             //read next data
  76             if(m_queue.Empty())
  77             {
  78                 if(m_finished)
  79                 {
  80                     return token;
  81                 }
  82                 else
  83                 {
  84                     auto baptr = m_input->Read(4096);
  85                     if(baptr.get() == 0)
  86                     {
  87                         ThrowMsg(Exception::TokenizerError, "Input read failed");
  88                     }
  89                     if(baptr->Empty())
  90                     {
  91                         if(TokenizerPolicy::TryAgainAtEnd(tryNumber))
  92                         {
  93                             ++tryNumber;
  94                             continue;
  95                         }
  96                         m_finished = true;
  97                         return token;
  98                     }
  99                     m_queue.AppendMoveFrom(*baptr);
 100                 }
 101             }
 102
 103             //process
 104             m_queue.FlattenConsume(&byte, 1); //queue uses pointer to consume bytes, this do not causes reallocations
 105             if(byte == '\n')
 106             {
 107                 m_newline = true;
 108                 if(!data.empty() || !TokenizerPolicy::SkipEmpty())
 109                 {
 110                     ProduceString(token, data);
 111                     return token;
 112                 }
 113             }
 114             else if(TokenizerPolicy::GetSeperators().find(byte) != std::string::npos)
 115             {
 116                 if(!data.empty() || !TokenizerPolicy::SkipEmpty())
 117                 {
 118                     ProduceString(token, data);
 119                     return token;
 120                 }
 121             }
 122             else
 123             {
 124                 data += byte;
 125             }
 126         }
 127     }
 128
 129     bool IsStateValid()
 130     {
 131         if(!m_queue.Empty() && m_finished) return false;
 132         return true;
 133     }
 134
 135 protected:
 136     void ProduceString(std::unique_ptr<VSToken> & token, std::string & data)
 137     {
 138         TokenizerPolicy::PrepareValue(data);
 139         token.reset(new VSToken(data));
 140     }
 141
 142     BinaryQueue m_queue;
 143     bool m_finished;
 144     bool m_newline;
 145 };
 146
 147 }
 148
 149 #endif