Getting token based on delimiter
authordarpan.ka <darpan.ka@samsung.com>
Mon, 8 Jul 2013 09:24:14 +0000 (14:54 +0530)
committerdarpan.ka <darpan.ka@samsung.com>
Mon, 15 Jul 2013 07:07:19 +0000 (12:37 +0530)
Change-Id: Ic9c6ff3d88fae1a6e4b4bf60b6c193afd1d568a4
Signed-off-by: darpan.ka <darpan.ka@samsung.com>
src/base/utility/FBaseUtil_ScannerImpl.cpp
src/base/utility/FBaseUtil_ScannerImpl.h

index 4e6fe41..794d4dd 100644 (file)
@@ -22,7 +22,8 @@
 #include "FBase_NativeError.h"
 
 #define ANY_PATTERN L"\\s*[^\\s]*"
-#define DEFAULT_DELIMITER_PATTERN L"\\s*"
+#define DEFAULT_DELIMITER_PATTERN L"\\s+"
+#define DEFAULT_DELIMITER_PATTERN_ZERO_OR_MORE L"\\s*"
 #define COMPLETE_LINE_PATTERN L".*(\r\n|[\u000A\n\r\u2028\u2029\u0085])|.+$"
 #define DEFAULT_LINE_SEPARATOR_PATTERN L"[\u000A\n\r\u2028\u2029\u0085]*"
 
@@ -125,8 +126,13 @@ CATCH:
 
 bool _ScannerImpl::HasNextToken(void)
 {
-       String pattern(ANY_PATTERN);
-       return HasNextToken(pattern);
+       String token;
+       int length = 0;
+
+       result r = GetNextTokenWithoutPattern(token, length);
+       SysTryLogReturn(NID_BASE_UTIL, r == E_SUCCESS, false, "Can not get the next token");
+
+       return true;
 }
 
 bool _ScannerImpl::HasNextToken(const RegularExpression& pattern)
@@ -137,12 +143,24 @@ bool _ScannerImpl::HasNextToken(const RegularExpression& pattern)
 
 bool _ScannerImpl::HasNextToken(const String& pattern)
 {
-       result r = E_SUCCESS;
-       String token;
+       RegularExpression regex;
+       ArrayList matchedStrList;
        int length = 0;
+       String out;
 
-       r = GetNextToken(pattern, token, length);
-       SysTryReturn(NID_BASE_UTIL, r == E_SUCCESS, false, E_DATA_NOT_FOUND, "[%s] Can not get the next token", GetErrorMessage(E_DATA_NOT_FOUND));
+       result r = GetNextTokenWithoutPattern(out, length);
+       SysTryLogReturn(NID_BASE_UTIL, r != E_DATA_NOT_ENOUGH, false, "Input has no remaining tokens");
+
+       SysTryLogReturn(NID_BASE_UTIL, r == E_SUCCESS, false, "The next token does not match to the pattern");
+
+       r = regex.Construct(pattern, REGEX_UNICODE);
+       SysTryLogReturn(NID_BASE_UTIL, r == E_SUCCESS, false, "Regular expression construction failed");
+
+       r = matchedStrList.Construct();
+       SysTryLogReturn(NID_BASE_UTIL, r == E_SUCCESS, false, "Arraylist construction failed");
+
+       bool res = regex.Match(out, true, &matchedStrList);
+       SysTryLogReturn(NID_BASE_UTIL, res, res, "Match Failed");
 
        return true;
 }
@@ -150,11 +168,9 @@ bool _ScannerImpl::HasNextToken(const String& pattern)
 
 result _ScannerImpl::GetNextToken(String& nextStr)
 {
-       result r = E_SUCCESS;
-       String pattern(ANY_PATTERN);
        int length = 0;
 
-       r = GetNextToken(pattern, nextStr, length);
+       result r = GetNextTokenWithoutPattern(nextStr, length);
        SysTryReturnResult(NID_BASE_UTIL, r == E_SUCCESS, E_DATA_NOT_ENOUGH, "Can not get the next token");
 
        __position =  __position + length;
@@ -170,38 +186,28 @@ result _ScannerImpl::GetNextToken(const RegularExpression& pattern, String& next
 
 result _ScannerImpl::GetNextToken(const String& pattern, String& nextStr)
 {
-       result r = E_SUCCESS;
-       int length1 = 0;
-       int length2 = 0;
+       RegularExpression regex;
+       ArrayList matchedStrList;
+       int length = 0;
+       String out;
 
-       String str1;
-       String str2;
-       String anyPattern(ANY_PATTERN);
+       result r = GetNextTokenWithoutPattern(out, length);
+       SysTryReturnResult(NID_BASE_UTIL, r != E_DATA_NOT_ENOUGH, E_DATA_NOT_ENOUGH, "Input has no remaining tokens");
 
-       while (true)
-       {
-               r = GetNextToken(anyPattern, str1, length1);
-               SysTryReturnResult(NID_BASE_UTIL, r != E_DATA_NOT_ENOUGH, E_DATA_NOT_ENOUGH, "Input has no remaining tokens");
-               SysTryReturnResult(NID_BASE_UTIL, r == E_SUCCESS, E_DATA_NOT_FOUND, "The next token does not match to the pattern");
+       SysTryReturnResult(NID_BASE_UTIL, r == E_SUCCESS, E_DATA_NOT_FOUND, "The next token does not match to the pattern");
 
-               r = GetNextToken(pattern, str2, length2);
-               SysTryReturnResult(NID_BASE_UTIL, r != E_DATA_NOT_ENOUGH, E_DATA_NOT_ENOUGH, "Input has no remaining tokens");
-               SysTryReturnResult(NID_BASE_UTIL, r == E_SUCCESS, E_DATA_NOT_FOUND, "The next token does not match to the pattern");
+       r = regex.Construct(pattern, REGEX_UNICODE);
+       SysTryReturnResult(NID_BASE_UTIL, r == E_SUCCESS, E_DATA_NOT_FOUND, "Regular expression construction failed");
 
-               if(str1 == str2)
-               {
-                       nextStr = str1;
-                       __position =  __position + length2;
-                       break;
-               }
-               else
-               {
-                       nextStr = L"";
-                       __position = __position + length1;
-               }
-       }
+       r = matchedStrList.Construct();
+       SysTryReturnResult(NID_BASE_UTIL, r == E_SUCCESS, E_DATA_NOT_FOUND, "Arraylist construction failed");
 
-       return r;
+       bool res = regex.Match(out, true, &matchedStrList);
+       SysTryReturnResult(NID_BASE_UTIL, res, E_DATA_NOT_FOUND, "Match Failed");
+
+       nextStr = out;
+       __position =  __position + length;
+       return E_SUCCESS;
 }
 
 result _ScannerImpl::GetNextSignedChar(signed char& nextSignedChar)
@@ -211,13 +217,11 @@ result _ScannerImpl::GetNextSignedChar(signed char& nextSignedChar)
 
 result _ScannerImpl::GetNextSignedChar(signed char& nextSignedChar, int radix)
 {
-       result r = E_SUCCESS;
        String out;
        int length = 0;
        char ch = 0x00;
-       String pattern(ANY_PATTERN);
 
-       r = GetNextToken(pattern, out, length);
+       result r = GetNextTokenWithoutPattern(out, length);
        SysTryReturnResult(NID_BASE_UTIL, r != E_DATA_NOT_ENOUGH, E_DATA_NOT_ENOUGH, "Input has no remaining tokens");
        SysTryReturnResult(NID_BASE_UTIL, r == E_SUCCESS, E_NUM_FORMAT, "Can not parse the token to signed char");
 
@@ -237,13 +241,11 @@ bool _ScannerImpl::IsNextTokenConvertibleToSignedChar(void)
 
 bool _ScannerImpl::IsNextTokenConvertibleToSignedChar(int radix)
 {
-       result r = E_SUCCESS;
        String out;
        int length = 0;
-       String pattern(ANY_PATTERN);
        char ch = 0;
 
-       r = GetNextToken(pattern, out, length);
+       result r = GetNextTokenWithoutPattern(out, length);
        SysTryReturn(NID_BASE_UTIL, r == E_SUCCESS, false, E_NUM_FORMAT, "[%s] Can not get the next token as signed char", GetErrorMessage(E_NUM_FORMAT));
 
        r = Int8::Parse(out, radix, ch);
@@ -259,12 +261,10 @@ result _ScannerImpl::GetNextInt(int& nextInt)
 
 result _ScannerImpl::GetNextInt(int& nextInt, int radix)
 {
-       result r = E_SUCCESS;
        String out;
        int length = 0;
-       String pattern(ANY_PATTERN);
 
-       r = GetNextToken(pattern, out, length);
+       result r = GetNextTokenWithoutPattern(out, length);
        SysTryReturnResult(NID_BASE_UTIL, r != E_DATA_NOT_ENOUGH, E_DATA_NOT_ENOUGH, "Input has no remaining tokens");
        SysTryReturnResult(NID_BASE_UTIL, r == E_SUCCESS, E_NUM_FORMAT, "Can not parse the token to integer");
 
@@ -284,13 +284,11 @@ bool _ScannerImpl::IsNextTokenConvertibleToInt(void)
 
 bool _ScannerImpl::IsNextTokenConvertibleToInt(int radix)
 {
-       result r = E_SUCCESS;
        String out;
        int val = 0;
        int length = 0;
-       String pattern(ANY_PATTERN);
 
-       r = GetNextToken(pattern, out, length);
+       result r = GetNextTokenWithoutPattern(out, length);
        SysTryReturn(NID_BASE_UTIL, r == E_SUCCESS, false, E_NUM_FORMAT, "[%s] Can not get the next token as integer", GetErrorMessage(E_NUM_FORMAT));
 
        out.Replace(DEFAULT_GROUP_SEPARATOR, L"");
@@ -307,12 +305,10 @@ result _ScannerImpl::GetNextShort(short& nextShort)
 
 result _ScannerImpl::GetNextShort(short& nextShort, int radix)
 {
-       result r = E_SUCCESS;
        String out;
        int length = 0;
-       String pattern(ANY_PATTERN);
 
-       r = GetNextToken(pattern, out, length);
+       result r = GetNextTokenWithoutPattern(out, length);
        SysTryReturnResult(NID_BASE_UTIL, r != E_DATA_NOT_ENOUGH, E_DATA_NOT_ENOUGH, "Input has no remaining tokens");
        SysTryReturnResult(NID_BASE_UTIL, r == E_SUCCESS, E_NUM_FORMAT, "Can not parse the token to short");
 
@@ -332,13 +328,11 @@ bool _ScannerImpl::IsNextTokenConvertibleToShort(void)
 
 bool _ScannerImpl::IsNextTokenConvertibleToShort(int radix)
 {
-       result r = E_SUCCESS;
        String out;
        short val = 0;
        int length = 0;
-       String pattern(ANY_PATTERN);
 
-       r = GetNextToken(pattern, out, length);
+       result r = GetNextTokenWithoutPattern(out, length);
        SysTryReturn(NID_BASE_UTIL, r == E_SUCCESS, false, E_NUM_FORMAT, "[%s] Can not get next token as short", GetErrorMessage(E_NUM_FORMAT));
 
        out.Replace(DEFAULT_GROUP_SEPARATOR, L"");
@@ -355,12 +349,10 @@ result _ScannerImpl::GetNextLongLong(long long& nextLongLong)
 
 result _ScannerImpl::GetNextLongLong(long long& nextLongLong, int radix)
 {
-       result r = E_SUCCESS;
        String out;
        int length = 0;
-       String pattern(ANY_PATTERN);
 
-       r = GetNextToken(pattern, out, length);
+       result r = GetNextTokenWithoutPattern(out, length);
        SysTryReturnResult(NID_BASE_UTIL, r != E_DATA_NOT_ENOUGH, E_DATA_NOT_ENOUGH, "Input has no remaining tokens");
        SysTryReturnResult(NID_BASE_UTIL, r == E_SUCCESS, E_NUM_FORMAT, "Can not parse the token to long long");
 
@@ -375,13 +367,11 @@ result _ScannerImpl::GetNextLongLong(long long& nextLongLong, int radix)
 
 bool _ScannerImpl::IsNextTokenConvertibleToLongLong(void)
 {
-       result r = E_SUCCESS;
        String out;
        long long val = 0;
        int length = 0;
-       String pattern(ANY_PATTERN);
 
-       r = GetNextToken(pattern, out, length);
+       result r = GetNextTokenWithoutPattern(out, length);
        SysTryReturn(NID_BASE_UTIL, r == E_SUCCESS, false, E_NUM_FORMAT, "[E_NUM_FORMAT] Can not get next token as long long");
 
        r = LongLong::Parse(out, val);
@@ -392,12 +382,10 @@ bool _ScannerImpl::IsNextTokenConvertibleToLongLong(void)
 
 result _ScannerImpl::GetNextFloat(float& nextFloat)
 {
-       result r = E_SUCCESS;
        String out;
        int length = 0;
-       String pattern(ANY_PATTERN);
 
-       r = GetNextToken(pattern, out, length);
+       result r = GetNextTokenWithoutPattern(out, length);
        SysTryReturnResult(NID_BASE_UTIL, r != E_DATA_NOT_ENOUGH, E_DATA_NOT_ENOUGH, "Input has no remaining tokens");
        SysTryReturnResult(NID_BASE_UTIL, r == E_SUCCESS, E_NUM_FORMAT, "Can not parse the token to float");
 
@@ -413,14 +401,12 @@ result _ScannerImpl::GetNextFloat(float& nextFloat)
 
 bool _ScannerImpl::IsNextTokenConvertibleToFloat(void)
 {
-       result r = E_SUCCESS;
        String out;
        RegularExpression re;
        float val = 0.0f;
        int length = 0;
-       String pattern(ANY_PATTERN);
 
-       r = GetNextToken(pattern, out, length);
+       result r = GetNextTokenWithoutPattern(out, length);
        SysTryReturn(NID_BASE_UTIL, r == E_SUCCESS, false, E_NUM_FORMAT, "[%s] Can not get next token as float", GetErrorMessage(E_NUM_FORMAT));
 
        out.Replace(DEFAULT_GROUP_SEPARATOR, L"");
@@ -432,12 +418,10 @@ bool _ScannerImpl::IsNextTokenConvertibleToFloat(void)
 
 result _ScannerImpl::GetNextDouble(double& nextDouble)
 {
-       result r = E_SUCCESS;
        String out;
        int length = 0;
-       String pattern(ANY_PATTERN);
 
-       r = GetNextToken(pattern, out, length);
+       result r = GetNextTokenWithoutPattern(out, length);
        SysTryReturnResult(NID_BASE_UTIL, r != E_DATA_NOT_ENOUGH, E_DATA_NOT_ENOUGH, "Input has no remaining tokens");
        SysTryReturnResult(NID_BASE_UTIL, r == E_SUCCESS, E_NUM_FORMAT, "Can not parse the token to double");
 
@@ -452,13 +436,11 @@ result _ScannerImpl::GetNextDouble(double& nextDouble)
 
 bool _ScannerImpl::IsNextTokenConvertibleToDouble(void)
 {
-       result r = E_SUCCESS;
        String out;
        double val = 0.0f;
        int length = 0;
-       String pattern(ANY_PATTERN);
 
-       r = GetNextToken(pattern, out, length);
+       result r = GetNextTokenWithoutPattern(out, length);
        SysTryReturn(NID_BASE_UTIL, r == E_SUCCESS, false, E_NUM_FORMAT, "[%s] Can not get next token as double", GetErrorMessage(E_NUM_FORMAT));
 
        out.Replace(DEFAULT_GROUP_SEPARATOR, L"");
@@ -470,12 +452,10 @@ bool _ScannerImpl::IsNextTokenConvertibleToDouble(void)
 
 result _ScannerImpl::GetNextBool(bool& nextBool)
 {
-       result r = E_SUCCESS;
        String out;
        int length = 0;
-       String pattern(ANY_PATTERN);
 
-       r = GetNextToken(pattern, out, length);
+       result r = GetNextTokenWithoutPattern(out, length);
        SysTryReturnResult(NID_BASE_UTIL, r != E_DATA_NOT_ENOUGH, E_DATA_NOT_ENOUGH, "Input has no remaining tokens");
        SysTryReturnResult(NID_BASE_UTIL, r == E_SUCCESS, E_NUM_FORMAT, "Can not parse the token to bool");
 
@@ -503,44 +483,30 @@ result _ScannerImpl::GetNextBool(bool& nextBool)
 
 bool _ScannerImpl::IsNextTokenConvertibleToBool(void)
 {
-       result r = E_SUCCESS;
        String out;
-       bool ret = false;
        int length = 0;
-       String pattern(ANY_PATTERN);
 
-       r = GetNextToken(pattern, out, length);
+       result r = GetNextTokenWithoutPattern(out, length);
        SysTryReturn(NID_BASE_UTIL, r == E_SUCCESS, false, E_NUM_FORMAT, "[%s] Can not get next token as bool", GetErrorMessage(E_NUM_FORMAT));
 
        String temp;
        out.ToUpper(temp);
-       if((!temp.CompareTo(L"TRUE")) || (!temp.CompareTo(L"FALSE")))
-       {
-               ret = true;
-       }
-       else
-       {
-               ret = false;
-       }
-
-       return ret;
+       return ((!temp.CompareTo(L"TRUE")) || (!temp.CompareTo(L"FALSE")));
 }
 
 bool _ScannerImpl::HasNextLine(void)
 {
-       result r = E_SUCCESS;
        String pattern(DEFAULT_LINE_SEPARATOR_PATTERN);
        RegularExpression regex;
        ArrayList matchedStrList;
-       bool res = false;
 
-       r = regex.Construct(pattern, REGEX_UNICODE);
+       result r = regex.Construct(pattern, REGEX_UNICODE);
        SysTryReturn(NID_BASE_UTIL, r == E_SUCCESS, false, E_DATA_NOT_ENOUGH, "[%s] Regular expression construction failed", GetErrorMessage(E_DATA_NOT_ENOUGH));
 
        r = matchedStrList.Construct();
        SysTryReturn(NID_BASE_UTIL, r == E_SUCCESS, false, E_DATA_NOT_ENOUGH, "[%s] Arraylist construction failed", GetErrorMessage(E_DATA_NOT_ENOUGH));
 
-       res = regex.Match(String(__pParseStr + __position), false, &matchedStrList);
+       bool res = regex.Match(String(__pParseStr + __position), false, &matchedStrList);
        SysTryReturn(NID_BASE_UTIL, res == true, false, E_DATA_NOT_ENOUGH, "[%s] Match Failed", GetErrorMessage(E_DATA_NOT_ENOUGH));
 
        return res;
@@ -549,67 +515,57 @@ bool _ScannerImpl::HasNextLine(void)
 
 result _ScannerImpl::GetNextLine(String& nextLine)
 {
-       result r = E_SUCCESS;
        String pattern(COMPLETE_LINE_PATTERN);
        RegularExpression regex;
        ArrayList matchedStrList;
-       bool res = false;
-       int matches = 0;
 
-       r = regex.Construct(pattern, REGEX_UNICODE);
+       result r = regex.Construct(pattern, REGEX_UNICODE);
        SysTryReturnResult(NID_BASE_UTIL, r == E_SUCCESS, E_DATA_NOT_ENOUGH, "Regular expression construction failed");
 
        r = matchedStrList.Construct();
        SysTryReturnResult(NID_BASE_UTIL, r == E_SUCCESS, E_DATA_NOT_ENOUGH, "Arraylist construction failed");
 
-       res = regex.Match(String(__pParseStr + __position), false, &matchedStrList);
+       bool res = regex.Match(String(__pParseStr + __position), false, &matchedStrList);
        SysTryReturnResult(NID_BASE_UTIL, res == true, E_DATA_NOT_ENOUGH, "Match Failed");
 
-       matches = matchedStrList.GetCount();
-       if(matches > 0)
-       {
-               String temp = *(String *)matchedStrList.GetAt(0);
-               __position = __position + temp.GetLength();
-
-               RegularExpression re;
-               r = re.Construct(DEFAULT_LINE_SEPARATOR_PATTERN, REGEX_UNICODE);
-               re.Replace(temp, L"", true);
-               nextLine = temp;
+       int matches = matchedStrList.GetCount();
+       SysTryReturnResult(NID_BASE_UTIL,matches > 0, E_DATA_NOT_FOUND, "Match Failed");
+       
+       String temp = *(String *)matchedStrList.GetAt(0);
+       __position = __position + temp.GetLength();
 
-               matchedStrList.RemoveAll();
-       }
-
-       return r;
+       RegularExpression re;
+       r = re.Construct(DEFAULT_LINE_SEPARATOR_PATTERN, REGEX_UNICODE);
+       re.Replace(temp, L"", true);
+       nextLine = temp;
+       matchedStrList.RemoveAll();
 
+       return E_SUCCESS;
 }
 
 result _ScannerImpl::FindInLine(const String& str, String& MatchedStr)
 {
-       result r = E_SUCCESS;
        _RegularExpressionImpl regex;
        ArrayList matchedStrList;
-       bool res = false;
-       int matches = 0;
 
-       r = regex.Construct(str, REGEX_UNICODE);
+       result r = regex.Construct(str, REGEX_UNICODE);
        SysTryReturnResult(NID_BASE_UTIL, r == E_SUCCESS, E_DATA_NOT_FOUND, "Regular expression construction failed");
 
        r = matchedStrList.Construct();
        SysTryReturnResult(NID_BASE_UTIL, r == E_SUCCESS, E_DATA_NOT_FOUND, "Arraylist construction failed");
 
-       res = regex.Match(String(__pParseStr + __position), false, &matchedStrList);
+       bool res = regex.Match(String(__pParseStr + __position), false, &matchedStrList);
        SysTryReturnResult(NID_BASE_UTIL, res == true, E_DATA_NOT_FOUND, "Match Failed");
 
-       matches = matchedStrList.GetCount();
-       if(matches > 0)
-       {
-               MatchedStr = *(String *)matchedStrList.GetAt(0);
-               int matchEnd = regex.GetLastMatchEnd();
-               __position = __position + matchEnd;
-               matchedStrList.RemoveAll();
-       }
-       return r;
+       int matches = matchedStrList.GetCount();
+       SysTryReturnResult(NID_BASE_UTIL,matches > 0, E_DATA_NOT_FOUND, "Match Failed");
+
+       MatchedStr = *(String *)matchedStrList.GetAt(0);
+       int matchEnd = regex.GetLastMatchEnd();
+       __position = __position + matchEnd;
+       matchedStrList.RemoveAll();
 
+       return E_SUCCESS;
 }
 
 result _ScannerImpl::FindInLine(const RegularExpression& pattern, String& MatchedStr)
@@ -620,61 +576,63 @@ result _ScannerImpl::FindInLine(const RegularExpression& pattern, String& Matche
 
 void _ScannerImpl::Skip(const String& str)
 {
-       result r = E_SUCCESS;
        RegularExpression regex;
        ArrayList matchedStrList;
-       bool res = false;
-       int matches = 0;
+       String inputStr(__pParseStr + __position);
 
-       r = regex.Construct(str, REGEX_UNICODE);
-       SysTryCatch(NID_BASE_UTIL, r == E_SUCCESS, , r, "[%s] Regular expression construction failed", GetErrorMessage(r));
+       result r = regex.Construct(str, REGEX_UNICODE);
+       SysTryReturnVoidResult(NID_BASE_UTIL, r == E_SUCCESS, r,"Regular expression construction failed");
 
        r = matchedStrList.Construct();
-       SysTryCatch(NID_BASE_UTIL, r == E_SUCCESS, , r, "[%s] ArrayList construction failed", GetErrorMessage(r));
+       SysTryReturnVoidResult(NID_BASE_UTIL, r == E_SUCCESS, r, "ArrayList construction failed");
 
-       res = regex.Match(String(__pParseStr + __position), false, &matchedStrList);
-       SysTryCatch(NID_BASE_UTIL, res == true, , E_SYSTEM, "[%s] Regular expression match failed", GetErrorMessage(E_SYSTEM));
+       bool res = regex.Match(inputStr, false, &matchedStrList);
+       SysTryReturnVoidResult(NID_BASE_UTIL, res == true, E_SYSTEM, "Regular expression match failed");
 
-       matches = matchedStrList.GetCount();
-        if(matches > 0)
-        {
-                       String temp = *(String *)matchedStrList.GetAt(0);
-                       int length = temp.GetLength();
-                       const wchar_t* wstr1 = temp.GetPointer();
-                       const wchar_t* wstr2 = __pParseStr + __position;
-                       bool isSame = true;
-
-                       while(*wstr1 != 0x0000)
-                       {
-                               if(*wstr1++ != *wstr2++)
-                               {
-                                       isSame =  false;
-                                       break;
-                               }
-                       }
-                       if(isSame == true)
-                       {
-                               __position = __position + length;
-                       }
-                       matchedStrList.RemoveAll();
+       int matches = matchedStrList.GetCount();
+       SysTryReturnVoidResult(NID_BASE_UTIL, matches > 0, E_SYSTEM,"Regular expression construction failed");
 
-                       //Skip if there is any delimiters before the next token
-                       RegularExpression re;
-                       r = re.Construct(DEFAULT_DELIMITER_PATTERN, REGEX_UNICODE);
-                       res = re.Match(String(__pParseStr + __position), false, &matchedStrList);
-                       matches = matchedStrList.GetCount();
-                        if(matches > 0)
-                        {
-                                       String temp = *(String *)matchedStrList.GetAt(0);
-                                       int length = temp.GetLength();
-                                       __position = __position + length;
-                                       matchedStrList.RemoveAll();
-                        }
-
-        }
+       String skipToken = *(dynamic_cast< String* >(matchedStrList.GetAt(0)));
+       int length = skipToken.GetLength();
+       const wchar_t* wstr1 = skipToken.GetPointer();
+       const wchar_t* wstr2 = __pParseStr + __position;
+       bool isSame = true;
+
+       while (*wstr1 != 0x0000)
+       {
+               if (*wstr1++ != *wstr2++)
+               {
+                       isSame =  false;
+                       break;
+               }
+       }
+       if (isSame)
+       {
+               __position = __position + length;
+       }
+       matchedStrList.RemoveAll();
+
+       //Skip if there is any delimiters before the next token
+       RegularExpression re;
+       if (__delimiter == DEFAULT_DELIMITER_PATTERN)
+       {
+               r = re.Construct(DEFAULT_DELIMITER_PATTERN_ZERO_OR_MORE, REGEX_UNICODE);
+       }
+       else
+       {
+               r = re.Construct(__delimiter, REGEX_UNICODE);
+       }
+
+       res = re.Match(String(__pParseStr + __position), false, &matchedStrList);
+       matches = matchedStrList.GetCount();
+       if (matches > 0)
+       {
+               String temp = *(dynamic_cast< String* >(matchedStrList.GetAt(0)));
+               int length = temp.GetLength();
+               __position = __position + length;
+               matchedStrList.RemoveAll();
+       }
 
-CATCH:
-       return;
 }
 
 void _ScannerImpl::Skip(const RegularExpression& pattern)
@@ -712,39 +670,129 @@ void _ScannerImpl::SetRadix(int radix)
        }
 }
 
-result _ScannerImpl::GetNextToken(const String& pattern, String& ret, int& length)
+
+result _ScannerImpl::GetNextTokenWithoutPattern(String& ret, int& length)
 {
-       result r = E_SUCCESS;
-       SysTryReturnResult(NID_BASE_UTIL, __pParseStr + __position != null, E_DATA_NOT_ENOUGH, "Input has no remaining tokens.");
-       SysTryReturnResult(NID_BASE_UTIL, *(__pParseStr + __position) != 0x0000, E_DATA_NOT_ENOUGH, "Input has no remaining tokens.");
+       SysTryReturnResult(NID_BASE_UTIL, ((__pParseStr + __position) != null) && ((*(__pParseStr + __position) != 0x0000)),
+                       E_DATA_NOT_ENOUGH, "Input has no remaining tokens.");
+
+       RegularExpression regDelim;
+       result r = regDelim.Construct(__delimiter, REGEX_UNICODE);
+       SysTryReturnResult(NID_BASE_UTIL, r == E_SUCCESS, E_DATA_NOT_FOUND, "Regular expression construction failed");
 
-       RegularExpression regex;
        ArrayList matchedStrList;
-       bool res = false;
-       int matches = 0;
-       String regPattern(pattern);
+       r = matchedStrList.Construct();
+       SysTryReturnResult(NID_BASE_UTIL, r == E_SUCCESS, E_DATA_NOT_FOUND, "Arraylist construction failed");
 
-       regPattern.Append(__delimiter);
+       String inputStr(__pParseStr + __position);
+       bool res = regDelim.Match(inputStr, false, &matchedStrList);
 
-       r = regex.Construct(regPattern, REGEX_UNICODE);
-       SysTryReturnResult(NID_BASE_UTIL, r == E_SUCCESS, E_OPERATION_FAILED, "Regular expression construction failed");
+       int matches = matchedStrList.GetCount();
+       if (matches > 0)
+       {
+               String delimiter1 = *(dynamic_cast< String* >(matchedStrList.GetAt(0)));
 
-       r = matchedStrList.Construct();
-       SysTryReturnResult(NID_BASE_UTIL, r == E_SUCCESS, E_OPERATION_FAILED, "Arraylist construction failed");
+               if (!delimiter1.IsEmpty())      // 1st delimiter is not empty
+               {
+                       int delimLength1 = delimiter1.GetLength();
+                       int index1 = 0;
+                       inputStr.IndexOf(delimiter1, 0, index1);
 
-       res = regex.Match(String(__pParseStr + __position), false, &matchedStrList);
-       SysTryReturnResult(NID_BASE_UTIL, res == true, E_OPERATION_FAILED, "Match Failed");
+                       // When the delimiter exists in first position, find the 2nd delimiter
+                       if ((index1 == 0) && (__position == 0))
+                       {
+                               length += delimLength1;
+                               String tmpStr(__pParseStr + length);
 
-       matches = matchedStrList.GetCount();
-       if(matches > 0)
+                               matchedStrList.RemoveAll();
+                               res = regDelim.Match(tmpStr, false, &matchedStrList);
+                               matches = matchedStrList.GetCount();
+                               if (matches > 0)
+                               {
+                                       String delimiter2 = *(dynamic_cast< String* >(matchedStrList.GetAt(0)));
+
+                                       // If the 2nd delimiter is empty, the token will be one character followed by 1st delimiter.
+                                       if (delimiter2.IsEmpty())
+                                       {
+                                               ret.Clear();
+                                               ret.Append(*(__pParseStr + length));
+                                               length += 1;
+                                               // Though 2nd delimiter is empty, if 3rd non-empty delimiter exists right after the token, length to be moved should be added by 3rd delimiter.
+                                               matchedStrList.RemoveAll();
+                                               String tmpStr1(__pParseStr + length);
+                                               res = regDelim.Match(tmpStr1, false, &matchedStrList);
+                                               matches = matchedStrList.GetCount();
+                                               SysTryReturnResult(NID_BASE_UTIL, matches > 0, E_SUCCESS, "It's a last token.");
+
+                                               String delimiter3 = *(dynamic_cast< String* >(matchedStrList.GetAt(0)));
+                                               int index3 = 0;
+                                               tmpStr1.IndexOf(delimiter3, 0, index3);
+                                               if (!delimiter3.IsEmpty() && index3 == 0)
+                                               {
+                                                       length += delimiter3.GetLength();
+                                               }
+                                       }
+                                       else
+                                       {
+                                               int delimLength2 = delimiter2.GetLength();
+                                               int index2 = 0;
+                                               tmpStr.IndexOf(delimiter2, 0, index2);
+                                               tmpStr.SubString(0, index2, ret);       // Extract the string in between first and second delimiter
+                                               length += index2 + delimLength2;
+                                       }
+                               }
+                       }
+                       else
+                       {
+                               inputStr.SubString(0, index1, ret);
+                               length = index1 + delimLength1;
+                       }
+               }
+               else    // 1st delimiter is empty
+               {
+                       // Find 2nd delimiter from "__pParseStr + __position + 1"
+                       String tmpStr(__pParseStr + __position + 1);
+                       if (tmpStr == null)     // When InputStr is last token, extract the token.
+                       {
+                               ret.Clear();
+                               ret.Append(__pParseStr + __position);
+                               length = ret.GetLength();
+                       }
+                       else
+                       {
+                               matchedStrList.RemoveAll();
+                               res = regDelim.Match(tmpStr, false, &matchedStrList);
+                               matches = matchedStrList.GetCount();
+                               if (matches > 0)
+                               {
+                                       String delimiter2 = *(dynamic_cast< String* >(matchedStrList.GetAt(0)));
+                                       if (delimiter2.IsEmpty())       // If the 2nd delimiter is also empty, the token will be one character followed by 1st delimiter.
+                                       {
+                                               ret.Clear();
+                                               ret.Append(*(__pParseStr + __position));
+                                               length = 1;
+                                       }
+                                       else
+                                       {
+                                               int delimLength2 = delimiter2.GetLength();
+                                               int index2 = 0;
+                                               inputStr.IndexOf(delimiter2, 0, index2);
+                                               inputStr.SubString(0, index2, ret);
+                                               length = index2 + delimLength2;
+                                       }
+                               }
+                       }
+               }
+       }
+       else    //There are no delimiters
        {
-               String temp = *(String *)matchedStrList.GetAt(0);
-               length = temp.GetLength();
-               RegularExpression re;
-               r = re.Construct(__delimiter, REGEX_UNICODE);
-               re.Replace(temp, L"", true);
-               ret = temp;
-               matchedStrList.RemoveAll();
+               ret.Clear();
+               ret.Append(__pParseStr + __position);
+               length = ret.GetLength();
+               if (length <= 0)
+               {
+                       return E_DATA_NOT_FOUND;
+               }
        }
 
        return r;
index 4f26927..1820888 100644 (file)
@@ -89,6 +89,9 @@ public:
        result GetNextToken(const String& pattern, String& ret, int& matTokenLen);
        result GetNextMatchingString(const String& pattern, String& ret, int& length);
 
+private :
+       result GetNextTokenWithoutPattern(String& ret, int& length);
+
 public:
        int __radix;
        String __delimiter;