git.openwrt.org Git - feed/packages.git/blob

   1 From 3e3669c9c41a27e1466e2c28b3906e3dd0ce3e7e Mon Sep 17 00:00:00 2001
   2 From: Steve Dower <steve.dower@python.org>
   3 Date: Thu, 7 Mar 2019 08:25:22 -0800
   4 Subject: [PATCH] bpo-36216: Add check for characters in netloc that normalize
   5  to separators (GH-12201)
   6
   7 ---
   8  Doc/library/urlparse.rst                      | 20 ++++++++++++++++
   9  Lib/test/test_urlparse.py                     | 24 +++++++++++++++++++
  10  Lib/urlparse.py                               | 17 +++++++++++++
  11  .../2019-03-06-09-38-40.bpo-36216.6q1m4a.rst  |  3 +++
  12  4 files changed, 64 insertions(+)
  13  create mode 100644 Misc/NEWS.d/next/Security/2019-03-06-09-38-40.bpo-36216.6q1m4a.rst
  14
  15 diff --git a/Doc/library/urlparse.rst b/Doc/library/urlparse.rst
  16 index 22249da54fbb..0989c88c3022 100644
  17 --- a/Doc/library/urlparse.rst
  18 +++ b/Doc/library/urlparse.rst
  19 @@ -119,12 +119,22 @@ The :mod:`urlparse` module defines the following functions:
  20     See section :ref:`urlparse-result-object` for more information on the result
  21     object.
  22
  23 +   Characters in the :attr:`netloc` attribute that decompose under NFKC
  24 +   normalization (as used by the IDNA encoding) into any of ``/``, ``?``,
  25 +   ``#``, ``@``, or ``:`` will raise a :exc:`ValueError`. If the URL is
  26 +   decomposed before parsing, or is not a Unicode string, no error will be
  27 +   raised.
  28 +
  29     .. versionchanged:: 2.5
  30        Added attributes to return value.
  31
  32     .. versionchanged:: 2.7
  33        Added IPv6 URL parsing capabilities.
  34
  35 +   .. versionchanged:: 2.7.17
  36 +      Characters that affect netloc parsing under NFKC normalization will
  37 +      now raise :exc:`ValueError`.
  38 +
  39
  40  .. function:: parse_qs(qs[, keep_blank_values[, strict_parsing[, max_num_fields]]])
  41
  42 @@ -232,11 +242,21 @@ The :mod:`urlparse` module defines the following functions:
  43     See section :ref:`urlparse-result-object` for more information on the result
  44     object.
  45
  46 +   Characters in the :attr:`netloc` attribute that decompose under NFKC
  47 +   normalization (as used by the IDNA encoding) into any of ``/``, ``?``,
  48 +   ``#``, ``@``, or ``:`` will raise a :exc:`ValueError`. If the URL is
  49 +   decomposed before parsing, or is not a Unicode string, no error will be
  50 +   raised.
  51 +
  52     .. versionadded:: 2.2
  53
  54     .. versionchanged:: 2.5
  55        Added attributes to return value.
  56
  57 +   .. versionchanged:: 2.7.17
  58 +      Characters that affect netloc parsing under NFKC normalization will
  59 +      now raise :exc:`ValueError`.
  60 +
  61
  62  .. function:: urlunsplit(parts)
  63
  64 diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py
  65 index 4e1ded73c266..73b0228ea8e3 100644
  66 --- a/Lib/test/test_urlparse.py
  67 +++ b/Lib/test/test_urlparse.py
  68 @@ -1,4 +1,6 @@
  69  from test import test_support
  70 +import sys
  71 +import unicodedata
  72  import unittest
  73  import urlparse
  74
  75 @@ -624,6 +626,28 @@ def test_portseparator(self):
  76          self.assertEqual(urlparse.urlparse("http://www.python.org:80"),
  77                  ('http','www.python.org:80','','','',''))
  78
  79 +    def test_urlsplit_normalization(self):
  80 +        # Certain characters should never occur in the netloc,
  81 +        # including under normalization.
  82 +        # Ensure that ALL of them are detected and cause an error
  83 +        illegal_chars = u'/:#?@'
  84 +        hex_chars = {'{:04X}'.format(ord(c)) for c in illegal_chars}
  85 +        denorm_chars = [
  86 +            c for c in map(unichr, range(128, sys.maxunicode))
  87 +            if (hex_chars & set(unicodedata.decomposition(c).split()))
  88 +            and c not in illegal_chars
  89 +        ]
  90 +        # Sanity check that we found at least one such character
  91 +        self.assertIn(u'\u2100', denorm_chars)
  92 +        self.assertIn(u'\uFF03', denorm_chars)
  93 +
  94 +        for scheme in [u"http", u"https", u"ftp"]:
  95 +            for c in denorm_chars:
  96 +                url = u"{}://netloc{}false.netloc/path".format(scheme, c)
  97 +                print "Checking %r" % url
  98 +                with self.assertRaises(ValueError):
  99 +                    urlparse.urlsplit(url)
 100 +
 101  def test_main():
 102      test_support.run_unittest(UrlParseTestCase)
 103
 104 diff --git a/Lib/urlparse.py b/Lib/urlparse.py
 105 index f7c2b032b097..54eda08651ab 100644
 106 --- a/Lib/urlparse.py
 107 +++ b/Lib/urlparse.py
 108 @@ -165,6 +165,21 @@ def _splitnetloc(url, start=0):
 109              delim = min(delim, wdelim)     # use earliest delim position
 110      return url[start:delim], url[delim:]   # return (domain, rest)
 111
 112 +def _checknetloc(netloc):
 113 +    if not netloc or not isinstance(netloc, unicode):
 114 +        return
 115 +    # looking for characters like \u2100 that expand to 'a/c'
 116 +    # IDNA uses NFKC equivalence, so normalize for this check
 117 +    import unicodedata
 118 +    netloc2 = unicodedata.normalize('NFKC', netloc)
 119 +    if netloc == netloc2:
 120 +        return
 121 +    _, _, netloc = netloc.rpartition('@') # anything to the left of '@' is okay
 122 +    for c in '/?#@:':
 123 +        if c in netloc2:
 124 +            raise ValueError("netloc '" + netloc2 + "' contains invalid " +
 125 +                             "characters under NFKC normalization")
 126 +
 127  def urlsplit(url, scheme='', allow_fragments=True):
 128      """Parse a URL into 5 components:
 129      <scheme>://<netloc>/<path>?<query>#<fragment>
 130 @@ -193,6 +208,7 @@ def urlsplit(url, scheme='', allow_fragments=True):
 131                  url, fragment = url.split('#', 1)
 132              if '?' in url:
 133                  url, query = url.split('?', 1)
 134 +            _checknetloc(netloc)
 135              v = SplitResult(scheme, netloc, url, query, fragment)
 136              _parse_cache[key] = v
 137              return v
 138 @@ -216,6 +232,7 @@ def urlsplit(url, scheme='', allow_fragments=True):
 139          url, fragment = url.split('#', 1)
 140      if '?' in url:
 141          url, query = url.split('?', 1)
 142 +    _checknetloc(netloc)
 143      v = SplitResult(scheme, netloc, url, query, fragment)
 144      _parse_cache[key] = v
 145      return v
 146 diff --git a/Misc/NEWS.d/next/Security/2019-03-06-09-38-40.bpo-36216.6q1m4a.rst b/Misc/NEWS.d/next/Security/2019-03-06-09-38-40.bpo-36216.6q1m4a.rst
 147 new file mode 100644
 148 index 000000000000..1e1ad92c6feb
 149 --- /dev/null
 150 +++ b/Misc/NEWS.d/next/Security/2019-03-06-09-38-40.bpo-36216.6q1m4a.rst
 151 @@ -0,0 +1,3 @@
 152 +Changes urlsplit() to raise ValueError when the URL contains characters that
 153 +decompose under IDNA encoding (NFKC-normalization) into characters that
 154 +affect how the URL is parsed.
 155 \ No newline at end of file