From f7190d43be29ec231c4f8b878748503f79056d91 Mon Sep 17 00:00:00 2001 From: JinWang An Date: Thu, 22 Jun 2023 10:41:38 +0900 Subject: [PATCH] [CVE-2022-0391] bpo-43882 - urllib.parse should sanitize urls containing ASCII newline and tabs. (GH-25595) (GH-25726) Co-authored-by: Gregory P. Smith Co-authored-by: Serhiy Storchaka (cherry picked from commit 76cd81d) Co-authored-by: Senthil Kumaran Co-authored-by: Senthil Kumaran (cherry picked from commit 515a7bc) Co-authored-by: Miss Islington (bot) <31488909+miss-islington@users.noreply.github.com> Change-Id: Ia736aa48623abda5b1f8d10c9512dcbb139db492 Signed-off-by: JinWang An --- Doc/library/urllib.rst | 7 +++++++ Doc/library/urlparse.rst | 5 +++++ Lib/test/test_urlparse.py | 49 +++++++++++++++++++++++++++++++++++++++++++++++ Lib/urlparse.py | 10 ++++++++++ 4 files changed, 71 insertions(+) diff --git a/Doc/library/urllib.rst b/Doc/library/urllib.rst index 084d567..aa1a388 100644 --- a/Doc/library/urllib.rst +++ b/Doc/library/urllib.rst @@ -284,6 +284,13 @@ Utility functions :func:`parse_qs` and :func:`parse_qsl` which are used to parse query strings into Python data structures. +.. seealso:: + + `WHATWG`_ - URL Living standard + Working Group for the URL Standard that defines URLs, domains, IP addresses, the + application/x-www-form-urlencoded format, and their API. + +.. _WHATWG: https://url.spec.whatwg.org/ .. function:: pathname2url(path) diff --git a/Doc/library/urlparse.rst b/Doc/library/urlparse.rst index c77fb89..1588d8a 100644 --- a/Doc/library/urlparse.rst +++ b/Doc/library/urlparse.rst @@ -247,6 +247,9 @@ The :mod:`urlparse` module defines the following functions: decomposed before parsing, or is not a Unicode string, no error will be raised. + Following the `WHATWG spec`_ that updates RFC 3986, ASCII newline + ``\n``, ``\r`` and tab ``\t`` characters are stripped from the URL. + .. versionadded:: 2.2 .. versionchanged:: 2.5 @@ -256,6 +259,8 @@ The :mod:`urlparse` module defines the following functions: Characters that affect netloc parsing under NFKC normalization will now raise :exc:`ValueError`. +.. _WHATWG spec: https://url.spec.whatwg.org/#concept-basic-url-parser + .. function:: urlunsplit(parts) diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 86c4a05..1bee8ad 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -504,6 +504,55 @@ class UrlParseTestCase(unittest.TestCase): p = urlparse.urlsplit(url) self.assertEqual(p.port, None) + def test_urlsplit_remove_unsafe_bytes(self): + # Remove ASCII tabs and newlines from input, for http common case scenario. + url = "h\nttp://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment" + p = urllib.parse.urlsplit(url) + self.assertEqual(p.scheme, "http") + self.assertEqual(p.netloc, "www.python.org") + self.assertEqual(p.path, "/javascript:alert('msg')/") + self.assertEqual(p.query, "query=something") + self.assertEqual(p.fragment, "fragment") + self.assertEqual(p.username, None) + self.assertEqual(p.password, None) + self.assertEqual(p.hostname, "www.python.org") + self.assertEqual(p.port, None) + self.assertEqual(p.geturl(), "http://www.python.org/javascript:alert('msg')/?query=something#fragment") + + # Remove ASCII tabs and newlines from input as bytes, for http common case scenario. + url = b"h\nttp://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment" + p = urllib.parse.urlsplit(url) + self.assertEqual(p.scheme, b"http") + self.assertEqual(p.netloc, b"www.python.org") + self.assertEqual(p.path, b"/javascript:alert('msg')/") + self.assertEqual(p.query, b"query=something") + self.assertEqual(p.fragment, b"fragment") + self.assertEqual(p.username, None) + self.assertEqual(p.password, None) + self.assertEqual(p.hostname, b"www.python.org") + self.assertEqual(p.port, None) + self.assertEqual(p.geturl(), b"http://www.python.org/javascript:alert('msg')/?query=something#fragment") + + # any scheme + url = "x-new-scheme\t://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment" + p = urllib.parse.urlsplit(url) + self.assertEqual(p.geturl(), "x-new-scheme://www.python.org/javascript:alert('msg')/?query=something#fragment") + + # Remove ASCII tabs and newlines from input as bytes, any scheme. + url = b"x-new-scheme\t://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment" + p = urllib.parse.urlsplit(url) + self.assertEqual(p.geturl(), b"x-new-scheme://www.python.org/javascript:alert('msg')/?query=something#fragment") + + # Unsafe bytes is not returned from urlparse cache. + # scheme is stored after parsing, sending an scheme with unsafe bytes *will not* return an unsafe scheme + url = "https://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment" + scheme = "htt\nps" + for _ in range(2): + p = urllib.parse.urlsplit(url, scheme=scheme) + self.assertEqual(p.scheme, "https") + self.assertEqual(p.geturl(), "https://www.python.org/javascript:alert('msg')/?query=something#fragment") + + def test_issue14072(self): p1 = urlparse.urlsplit('tel:+31-641044153') self.assertEqual(p1.scheme, 'tel') diff --git a/Lib/urlparse.py b/Lib/urlparse.py index 798b467..d015f2c 100644 --- a/Lib/urlparse.py +++ b/Lib/urlparse.py @@ -62,6 +62,9 @@ scheme_chars = ('abcdefghijklmnopqrstuvwxyz' '0123456789' '+-.') +# Unsafe bytes to be removed per WHATWG spec +_UNSAFE_URL_BYTES_TO_REMOVE = ['\t', '\r', '\n'] + MAX_CACHE_SIZE = 20 _parse_cache = {} @@ -184,12 +187,19 @@ def _checknetloc(netloc): "under NFKC normalization" % netloc) +def _remove_unsafe_bytes_from_url(url): + for b in _UNSAFE_URL_BYTES_TO_REMOVE: + url = url.replace(b, "") + return url + def urlsplit(url, scheme='', allow_fragments=True): """Parse a URL into 5 components: :///?# Return a 5-tuple: (scheme, netloc, path, query, fragment). Note that we don't break the components up in smaller bits (e.g. netloc is a single string) and we don't expand % escapes.""" + url = _remove_unsafe_bytes_from_url(url) + scheme = _remove_unsafe_bytes_from_url(scheme) allow_fragments = bool(allow_fragments) key = url, scheme, allow_fragments, type(url), type(scheme) cached = _parse_cache.get(key, None) -- 2.7.4