1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #ifndef URL_URL_CANON_INTERNAL_H_
6 #define URL_URL_CANON_INTERNAL_H_
8 // This file is intended to be included in another C++ file where the character
9 // types are defined. This allows us to write mostly generic code, but not have
10 // template bloat because everything is inlined when anybody calls any of our
16 #include "base/logging.h"
17 #include "url/url_canon.h"
21 // Character type handling -----------------------------------------------------
23 // Bits that identify different character types. These types identify different
24 // bits that are set for each 8-bit character in the kSharedCharTypeTable.
25 enum SharedCharTypes {
26 // Characters that do not require escaping in queries. Characters that do
27 // not have this flag will be escaped; see url_canon_query.cc
30 // Valid in the username/password field.
33 // Valid in a IPv4 address (digits plus dot and 'x' for hex).
36 // Valid in an ASCII-representation of a hex digit (as in %-escaped).
39 // Valid in an ASCII-representation of a decimal digit.
42 // Valid in an ASCII-representation of an octal digit.
45 // Characters that do not require escaping in encodeURIComponent. Characters
46 // that do not have this flag will be escaped; see url_util.cc.
50 // This table contains the flags in SharedCharTypes for each 8-bit character.
51 // Some canonicalization functions have their own specialized lookup table.
52 // For those with simple requirements, we have collected the flags in one
53 // place so there are fewer lookup tables to load into the CPU cache.
55 // Using an unsigned char type has a small but measurable performance benefit
56 // over using a 32-bit number.
57 extern const unsigned char kSharedCharTypeTable[0x100];
59 // More readable wrappers around the character type lookup table.
60 inline bool IsCharOfType(unsigned char c, SharedCharTypes type) {
61 return !!(kSharedCharTypeTable[c] & type);
63 inline bool IsQueryChar(unsigned char c) {
64 return IsCharOfType(c, CHAR_QUERY);
66 inline bool IsIPv4Char(unsigned char c) {
67 return IsCharOfType(c, CHAR_IPV4);
69 inline bool IsHexChar(unsigned char c) {
70 return IsCharOfType(c, CHAR_HEX);
72 inline bool IsComponentChar(unsigned char c) {
73 return IsCharOfType(c, CHAR_COMPONENT);
76 // Appends the given string to the output, escaping characters that do not
77 // match the given |type| in SharedCharTypes.
78 void AppendStringOfType(const char* source, int length,
81 void AppendStringOfType(const base::char16* source, int length,
85 // Maps the hex numerical values 0x0 to 0xf to the corresponding ASCII digit
86 // that will be used to represent it.
87 URL_EXPORT extern const char kHexCharLookup[0x10];
89 // This lookup table allows fast conversion between ASCII hex letters and their
90 // corresponding numerical value. The 8-bit range is divided up into 8
91 // regions of 0x20 characters each. Each of the three character types (numbers,
92 // uppercase, lowercase) falls into different regions of this range. The table
93 // contains the amount to subtract from characters in that range to get at
94 // the corresponding numerical value.
96 // See HexDigitToValue for the lookup.
97 extern const char kCharToHexLookup[8];
99 // Assumes the input is a valid hex digit! Call IsHexChar before using this.
100 inline unsigned char HexCharToValue(unsigned char c) {
101 return c - kCharToHexLookup[c / 0x20];
104 // Indicates if the given character is a dot or dot equivalent, returning the
105 // number of characters taken by it. This will be one for a literal dot, 3 for
106 // an escaped dot. If the character is not a dot, this will return 0.
107 template<typename CHAR>
108 inline int IsDot(const CHAR* spec, int offset, int end) {
109 if (spec[offset] == '.') {
111 } else if (spec[offset] == '%' && offset + 3 <= end &&
112 spec[offset + 1] == '2' &&
113 (spec[offset + 2] == 'e' || spec[offset + 2] == 'E')) {
120 // Returns the canonicalized version of the input character according to scheme
121 // rules. This is implemented alongside the scheme canonicalizer, and is
122 // required for relative URL resolving to test for scheme equality.
124 // Returns 0 if the input character is not a valid scheme character.
125 char CanonicalSchemeChar(base::char16 ch);
127 // Write a single character, escaped, to the output. This always escapes: it
128 // does no checking that thee character requires escaping.
129 // Escaping makes sense only 8 bit chars, so code works in all cases of
130 // input parameters (8/16bit).
131 template<typename UINCHAR, typename OUTCHAR>
132 inline void AppendEscapedChar(UINCHAR ch,
133 CanonOutputT<OUTCHAR>* output) {
134 output->push_back('%');
135 output->push_back(kHexCharLookup[(ch >> 4) & 0xf]);
136 output->push_back(kHexCharLookup[ch & 0xf]);
139 // The character we'll substitute for undecodable or invalid characters.
140 extern const base::char16 kUnicodeReplacementCharacter;
142 // UTF-8 functions ------------------------------------------------------------
144 // Reads one character in UTF-8 starting at |*begin| in |str| and places
145 // the decoded value into |*code_point|. If the character is valid, we will
146 // return true. If invalid, we'll return false and put the
147 // kUnicodeReplacementCharacter into |*code_point|.
149 // |*begin| will be updated to point to the last character consumed so it
150 // can be incremented in a loop and will be ready for the next character.
151 // (for a single-byte ASCII character, it will not be changed).
152 URL_EXPORT bool ReadUTFChar(const char* str, int* begin, int length,
153 unsigned* code_point_out);
155 // Generic To-UTF-8 converter. This will call the given append method for each
156 // character that should be appended, with the given output method. Wrappers
157 // are provided below for escaped and non-escaped versions of this.
159 // The char_value must have already been checked that it's a valid Unicode
161 template<class Output, void Appender(unsigned char, Output*)>
162 inline void DoAppendUTF8(unsigned char_value, Output* output) {
163 if (char_value <= 0x7f) {
164 Appender(static_cast<unsigned char>(char_value), output);
165 } else if (char_value <= 0x7ff) {
167 Appender(static_cast<unsigned char>(0xC0 | (char_value >> 6)),
169 Appender(static_cast<unsigned char>(0x80 | (char_value & 0x3f)),
171 } else if (char_value <= 0xffff) {
172 // 1110xxxx 10xxxxxx 10xxxxxx
173 Appender(static_cast<unsigned char>(0xe0 | (char_value >> 12)),
175 Appender(static_cast<unsigned char>(0x80 | ((char_value >> 6) & 0x3f)),
177 Appender(static_cast<unsigned char>(0x80 | (char_value & 0x3f)),
179 } else if (char_value <= 0x10FFFF) { // Max Unicode code point.
180 // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
181 Appender(static_cast<unsigned char>(0xf0 | (char_value >> 18)),
183 Appender(static_cast<unsigned char>(0x80 | ((char_value >> 12) & 0x3f)),
185 Appender(static_cast<unsigned char>(0x80 | ((char_value >> 6) & 0x3f)),
187 Appender(static_cast<unsigned char>(0x80 | (char_value & 0x3f)),
190 // Invalid UTF-8 character (>20 bits).
195 // Helper used by AppendUTF8Value below. We use an unsigned parameter so there
196 // are no funny sign problems with the input, but then have to convert it to
197 // a regular char for appending.
198 inline void AppendCharToOutput(unsigned char ch, CanonOutput* output) {
199 output->push_back(static_cast<char>(ch));
202 // Writes the given character to the output as UTF-8. This does NO checking
203 // of the validity of the Unicode characters; the caller should ensure that
204 // the value it is appending is valid to append.
205 inline void AppendUTF8Value(unsigned char_value, CanonOutput* output) {
206 DoAppendUTF8<CanonOutput, AppendCharToOutput>(char_value, output);
209 // Writes the given character to the output as UTF-8, escaping ALL
210 // characters (even when they are ASCII). This does NO checking of the
211 // validity of the Unicode characters; the caller should ensure that the value
212 // it is appending is valid to append.
213 inline void AppendUTF8EscapedValue(unsigned char_value, CanonOutput* output) {
214 DoAppendUTF8<CanonOutput, AppendEscapedChar>(char_value, output);
217 // UTF-16 functions -----------------------------------------------------------
219 // Reads one character in UTF-16 starting at |*begin| in |str| and places
220 // the decoded value into |*code_point|. If the character is valid, we will
221 // return true. If invalid, we'll return false and put the
222 // kUnicodeReplacementCharacter into |*code_point|.
224 // |*begin| will be updated to point to the last character consumed so it
225 // can be incremented in a loop and will be ready for the next character.
226 // (for a single-16-bit-word character, it will not be changed).
227 URL_EXPORT bool ReadUTFChar(const base::char16* str, int* begin, int length,
228 unsigned* code_point_out);
230 // Equivalent to U16_APPEND_UNSAFE in ICU but uses our output method.
231 inline void AppendUTF16Value(unsigned code_point,
232 CanonOutputT<base::char16>* output) {
233 if (code_point > 0xffff) {
234 output->push_back(static_cast<base::char16>((code_point >> 10) + 0xd7c0));
235 output->push_back(static_cast<base::char16>((code_point & 0x3ff) | 0xdc00));
237 output->push_back(static_cast<base::char16>(code_point));
241 // Escaping functions ---------------------------------------------------------
243 // Writes the given character to the output as UTF-8, escaped. Call this
244 // function only when the input is wide. Returns true on success. Failure
245 // means there was some problem with the encoding, we'll still try to
246 // update the |*begin| pointer and add a placeholder character to the
247 // output so processing can continue.
249 // We will append the character starting at ch[begin] with the buffer ch
250 // being |length|. |*begin| will be updated to point to the last character
251 // consumed (we may consume more than one for UTF-16) so that if called in
252 // a loop, incrementing the pointer will move to the next character.
254 // Every single output character will be escaped. This means that if you
255 // give it an ASCII character as input, it will be escaped. Some code uses
256 // this when it knows that a character is invalid according to its rules
257 // for validity. If you don't want escaping for ASCII characters, you will
258 // have to filter them out prior to calling this function.
260 // Assumes that ch[begin] is within range in the array, but does not assume
261 // that any following characters are.
262 inline bool AppendUTF8EscapedChar(const base::char16* str, int* begin,
263 int length, CanonOutput* output) {
264 // UTF-16 input. ReadUTFChar will handle invalid characters for us and give
265 // us the kUnicodeReplacementCharacter, so we don't have to do special
266 // checking after failure, just pass through the failure to the caller.
268 bool success = ReadUTFChar(str, begin, length, &char_value);
269 AppendUTF8EscapedValue(char_value, output);
273 // Handles UTF-8 input. See the wide version above for usage.
274 inline bool AppendUTF8EscapedChar(const char* str, int* begin, int length,
275 CanonOutput* output) {
276 // ReadUTF8Char will handle invalid characters for us and give us the
277 // kUnicodeReplacementCharacter, so we don't have to do special checking
278 // after failure, just pass through the failure to the caller.
280 bool success = ReadUTFChar(str, begin, length, &ch);
281 AppendUTF8EscapedValue(ch, output);
285 // Given a '%' character at |*begin| in the string |spec|, this will decode
286 // the escaped value and put it into |*unescaped_value| on success (returns
287 // true). On failure, this will return false, and will not write into
288 // |*unescaped_value|.
290 // |*begin| will be updated to point to the last character of the escape
291 // sequence so that when called with the index of a for loop, the next time
292 // through it will point to the next character to be considered. On failure,
293 // |*begin| will be unchanged.
294 inline bool Is8BitChar(char c) {
295 return true; // this case is specialized to avoid a warning
297 inline bool Is8BitChar(base::char16 c) {
301 template<typename CHAR>
302 inline bool DecodeEscaped(const CHAR* spec, int* begin, int end,
303 unsigned char* unescaped_value) {
304 if (*begin + 3 > end ||
305 !Is8BitChar(spec[*begin + 1]) || !Is8BitChar(spec[*begin + 2])) {
306 // Invalid escape sequence because there's not enough room, or the
307 // digits are not ASCII.
311 unsigned char first = static_cast<unsigned char>(spec[*begin + 1]);
312 unsigned char second = static_cast<unsigned char>(spec[*begin + 2]);
313 if (!IsHexChar(first) || !IsHexChar(second)) {
314 // Invalid hex digits, fail.
318 // Valid escape sequence.
319 *unescaped_value = (HexCharToValue(first) << 4) + HexCharToValue(second);
324 // Appends the given substring to the output, escaping "some" characters that
325 // it feels may not be safe. It assumes the input values are all contained in
326 // 8-bit although it allows any type.
328 // This is used in error cases to append invalid output so that it looks
329 // approximately correct. Non-error cases should not call this function since
330 // the escaping rules are not guaranteed!
331 void AppendInvalidNarrowString(const char* spec, int begin, int end,
332 CanonOutput* output);
333 void AppendInvalidNarrowString(const base::char16* spec, int begin, int end,
334 CanonOutput* output);
336 // Misc canonicalization helpers ----------------------------------------------
338 // Converts between UTF-8 and UTF-16, returning true on successful conversion.
339 // The output will be appended to the given canonicalizer output (so make sure
340 // it's empty if you want to replace).
342 // On invalid input, this will still write as much output as possible,
343 // replacing the invalid characters with the "invalid character". It will
344 // return false in the failure case, and the caller should not continue as
346 URL_EXPORT bool ConvertUTF16ToUTF8(const base::char16* input, int input_len,
347 CanonOutput* output);
348 URL_EXPORT bool ConvertUTF8ToUTF16(const char* input, int input_len,
349 CanonOutputT<base::char16>* output);
351 // Converts from UTF-16 to 8-bit using the character set converter. If the
352 // converter is NULL, this will use UTF-8.
353 void ConvertUTF16ToQueryEncoding(const base::char16* input,
354 const Component& query,
355 CharsetConverter* converter,
356 CanonOutput* output);
358 // Applies the replacements to the given component source. The component source
359 // should be pre-initialized to the "old" base. That is, all pointers will
360 // point to the spec of the old URL, and all of the Parsed components will
361 // be indices into that string.
363 // The pointers and components in the |source| for all non-NULL strings in the
364 // |repl| (replacements) will be updated to reference those strings.
365 // Canonicalizing with the new |source| and |parsed| can then combine URL
366 // components from many different strings.
367 void SetupOverrideComponents(const char* base,
368 const Replacements<char>& repl,
369 URLComponentSource<char>* source,
372 // Like the above 8-bit version, except that it additionally converts the
373 // UTF-16 input to UTF-8 before doing the overrides.
375 // The given utf8_buffer is used to store the converted components. They will
376 // be appended one after another, with the parsed structure identifying the
377 // appropriate substrings. This buffer is a parameter because the source has
378 // no storage, so the buffer must have the same lifetime as the source
379 // parameter owned by the caller.
381 // THE CALLER MUST NOT ADD TO THE |utf8_buffer| AFTER THIS CALL. Members of
382 // |source| will point into this buffer, which could be invalidated if
383 // additional data is added and the CanonOutput resizes its buffer.
385 // Returns true on success. False means that the input was not valid UTF-16,
386 // although we will have still done the override with "invalid characters" in
388 bool SetupUTF16OverrideComponents(const char* base,
389 const Replacements<base::char16>& repl,
390 CanonOutput* utf8_buffer,
391 URLComponentSource<char>* source,
394 // Implemented in url_canon_path.cc, these are required by the relative URL
395 // resolver as well, so we declare them here.
396 bool CanonicalizePartialPath(const char* spec,
397 const Component& path,
398 int path_begin_in_output,
399 CanonOutput* output);
400 bool CanonicalizePartialPath(const base::char16* spec,
401 const Component& path,
402 int path_begin_in_output,
403 CanonOutput* output);
407 // Implementations of Windows' int-to-string conversions
408 URL_EXPORT int _itoa_s(int value, char* buffer, size_t size_in_chars,
410 URL_EXPORT int _itow_s(int value, base::char16* buffer, size_t size_in_chars,
413 // Secure template overloads for these functions
415 inline int _itoa_s(int value, char (&buffer)[N], int radix) {
416 return _itoa_s(value, buffer, N, radix);
420 inline int _itow_s(int value, base::char16 (&buffer)[N], int radix) {
421 return _itow_s(value, buffer, N, radix);
424 // _strtoui64 and strtoull behave the same
425 inline unsigned long long _strtoui64(const char* nptr,
426 char** endptr, int base) {
427 return strtoull(nptr, endptr, base);
434 #endif // URL_URL_CANON_INTERNAL_H_