* Added support for the non-escaped print of utf-8 string.
* EscapeString: the first invalid symbol resets print_natural_utf8 flag to false.
* Move the test to ParseAndGenerateTextTest. Fixes.
* Removed dependence between `natural_utf8` and `allow_non_utf8` flags.
if (type_ == TYPE_STRING) {
String str(Indirect(), byte_width_);
if (strings_quoted) {
- flatbuffers::EscapeString(str.c_str(), str.length(), &s, true);
+ flatbuffers::EscapeString(str.c_str(), str.length(), &s, true, false);
} else {
s.append(str.c_str(), str.length());
}
} else if (IsKey()) {
auto str = AsKey();
if (keys_quoted) {
- flatbuffers::EscapeString(str, strlen(str), &s, true);
+ flatbuffers::EscapeString(str, strlen(str), &s, true, false);
} else {
s += str;
}
std::string object_suffix;
bool union_value_namespacing;
bool allow_non_utf8;
+ bool natural_utf8;
std::string include_prefix;
bool keep_include_path;
bool binary_schema_comments;
object_suffix("T"),
union_value_namespacing(true),
allow_non_utf8(false),
+ natural_utf8(false),
keep_include_path(false),
binary_schema_comments(false),
binary_schema_builtins(false),
void Float(float x) { s += NumToString(x); }
void Double(double x) { s += NumToString(x); }
void String(const struct String *str) {
- EscapeString(str->c_str(), str->size(), &s, true);
+ EscapeString(str->c_str(), str->size(), &s, true, false);
}
void Unknown(const uint8_t *) { s += "(?)"; }
void StartVector() { s += "[ "; }
}
inline bool EscapeString(const char *s, size_t length, std::string *_text,
- bool allow_non_utf8) {
+ bool allow_non_utf8, bool natural_utf8) {
std::string &text = *_text;
text += "\"";
for (uoffset_t i = 0; i < length; i++) {
return false;
}
} else {
- if (ucc <= 0xFFFF) {
+ if (natural_utf8) {
+ // utf8 points to past all utf-8 bytes parsed
+ text.append(s + i, static_cast<size_t>(utf8 - s - i));
+ } else if (ucc <= 0xFFFF) {
// Parses as Unicode within JSON's \uXXXX range, so use that.
text += "\\u";
text += IntToStringHex(ucc, 4);
" --allow-non-utf8 Pass non-UTF-8 input through parser and emit nonstandard\n"
" \\x escapes in JSON. (Default is to raise parse error on\n"
" non-UTF-8 input.)\n"
+ " --natural-utf8 Output strings with UTF-8 as human-readable strings.\n"
+ " By default, UTF-8 characters are printed as \\uXXXX escapes.\n"
" --defaults-json Output fields whose value is the default when\n"
" writing JSON\n"
" --unknown-json Allow fields in JSON that are not defined in the\n"
opts.strict_json = true;
} else if (arg == "--allow-non-utf8") {
opts.allow_non_utf8 = true;
+ } else if (arg == "--natural-utf8") {
+ opts.natural_utf8 = true;
} else if (arg == "--no-js-exports") {
opts.skip_js_exports = true;
} else if (arg == "--goog-js-export") {
break;
case BASE_TYPE_STRING: {
auto s = reinterpret_cast<const String *>(val);
- if (!EscapeString(s->c_str(), s->Length(), _text, opts.allow_non_utf8)) {
+ if (!EscapeString(s->c_str(), s->Length(), _text, opts.allow_non_utf8,
+ opts.natural_utf8)) {
return false;
}
break;
auto val = GetAnyFieldS(*table_field, fielddef, schema);
if (fielddef.type()->base_type() == reflection::String) {
std::string esc;
- flatbuffers::EscapeString(val.c_str(), val.length(), &esc, true);
+ flatbuffers::EscapeString(val.c_str(), val.length(), &esc, true,
+ false);
val = esc;
}
s += fielddef.name()->str();
-/*
+/*
* Copyright 2014 Google Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
// If this fails, check registry.lasterror_.
TEST_EQ(ok, true);
TEST_EQ_STR(text.c_str(), jsonfile.c_str());
+
+ // Generate text for UTF-8 strings without escapes.
+ std::string jsonfile_utf8;
+ TEST_EQ(flatbuffers::LoadFile((test_data_path + "unicode_test.json").c_str(),
+ false, &jsonfile_utf8),
+ true);
+ TEST_EQ(parser.Parse(jsonfile_utf8.c_str(), include_directories), true);
+ // To ensure it is correct, generate utf-8 text back from the binary.
+ std::string jsongen_utf8;
+ // request natural printing for utf-8 strings
+ parser.opts.natural_utf8 = true;
+ parser.opts.strict_json = true;
+ TEST_EQ(
+ GenerateText(parser, parser.builder_.GetBufferPointer(), &jsongen_utf8),
+ true);
+ TEST_EQ_STR(jsongen_utf8.c_str(), jsonfile_utf8.c_str());
}
void ReflectionTest(uint8_t *flatbuf, size_t length) {
{
"name": "unicode_test",
- "testarrayoftables": [
- { "name": "Цлїςσδε" },
- { "name": "フムアムカモケモ" },
- { "name": "フムヤムカモケモ" },
- { "name": "㊀㊁㊂㊃㊄" },
- { "name": "☳☶☲" },
- { "name": "𡇙𝌆" }
- ],
"testarrayofstring": [
"Цлїςσδε",
"フムアムカモケモ",
"㊀㊁㊂㊃㊄",
"☳☶☲",
"𡇙𝌆"
+ ],
+ "testarrayoftables": [
+ {
+ "name": "Цлїςσδε"
+ },
+ {
+ "name": "フムアムカモケモ"
+ },
+ {
+ "name": "フムヤムカモケモ"
+ },
+ {
+ "name": "㊀㊁㊂㊃㊄"
+ },
+ {
+ "name": "☳☶☲"
+ },
+ {
+ "name": "𡇙𝌆"
+ }
]
}