|
| 1 | +diff --git a/Lib/http/client.py b/Lib/http/client.py |
| 2 | +index fb29923d942..70451d67d4c 100644 |
| 3 | +--- a/Lib/http/client.py |
| 4 | ++++ b/Lib/http/client.py |
| 5 | +@@ -111,6 +111,11 @@ |
| 6 | + _MAXLINE = 65536 |
| 7 | + _MAXHEADERS = 100 |
| 8 | + |
| 9 | ++# Data larger than this will be read in chunks, to prevent extreme |
| 10 | ++# overallocation. |
| 11 | ++_MIN_READ_BUF_SIZE = 1 << 20 |
| 12 | ++ |
| 13 | ++ |
| 14 | + # Header name/value ABNF (http://tools.ietf.org/html/rfc7230#section-3.2) |
| 15 | + # |
| 16 | + # VCHAR = %x21-7E |
| 17 | +@@ -639,10 +644,25 @@ def _safe_read(self, amt): |
| 18 | + reading. If the bytes are truly not available (due to EOF), then the |
| 19 | + IncompleteRead exception can be used to detect the problem. |
| 20 | + """ |
| 21 | +- data = self.fp.read(amt) |
| 22 | +- if len(data) < amt: |
| 23 | +- raise IncompleteRead(data, amt-len(data)) |
| 24 | +- return data |
| 25 | ++ cursize = min(amt, _MIN_READ_BUF_SIZE) |
| 26 | ++ data = self.fp.read(cursize) |
| 27 | ++ if len(data) >= amt: |
| 28 | ++ return data |
| 29 | ++ if len(data) < cursize: |
| 30 | ++ raise IncompleteRead(data, amt - len(data)) |
| 31 | ++ |
| 32 | ++ data = io.BytesIO(data) |
| 33 | ++ data.seek(0, 2) |
| 34 | ++ while True: |
| 35 | ++ # This is a geometric increase in read size (never more than |
| 36 | ++ # doubling out the current length of data per loop iteration). |
| 37 | ++ delta = min(cursize, amt - cursize) |
| 38 | ++ data.write(self.fp.read(delta)) |
| 39 | ++ if data.tell() >= amt: |
| 40 | ++ return data.getvalue() |
| 41 | ++ cursize += delta |
| 42 | ++ if data.tell() < cursize: |
| 43 | ++ raise IncompleteRead(data.getvalue(), amt - data.tell()) |
| 44 | + |
| 45 | + def _safe_readinto(self, b): |
| 46 | + """Same as _safe_read, but for reading into a buffer.""" |
| 47 | +diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py |
| 48 | +index 01f5a101901..10e0d6a2be0 100644 |
| 49 | +--- a/Lib/test/test_httplib.py |
| 50 | ++++ b/Lib/test/test_httplib.py |
| 51 | +@@ -1452,6 +1452,73 @@ def run_server(): |
| 52 | + thread.join() |
| 53 | + self.assertEqual(result, b"proxied data\n") |
| 54 | + |
| 55 | ++ def test_large_content_length(self): |
| 56 | ++ serv = socket.create_server((HOST, 0)) |
| 57 | ++ self.addCleanup(serv.close) |
| 58 | ++ |
| 59 | ++ def run_server(): |
| 60 | ++ [conn, address] = serv.accept() |
| 61 | ++ with conn: |
| 62 | ++ while conn.recv(1024): |
| 63 | ++ conn.sendall( |
| 64 | ++ b"HTTP/1.1 200 Ok\r\n" |
| 65 | ++ b"Content-Length: %d\r\n" |
| 66 | ++ b"\r\n" % size) |
| 67 | ++ conn.sendall(b'A' * (size//3)) |
| 68 | ++ conn.sendall(b'B' * (size - size//3)) |
| 69 | ++ |
| 70 | ++ thread = threading.Thread(target=run_server) |
| 71 | ++ thread.start() |
| 72 | ++ self.addCleanup(thread.join, 1.0) |
| 73 | ++ |
| 74 | ++ conn = client.HTTPConnection(*serv.getsockname()) |
| 75 | ++ try: |
| 76 | ++ for w in range(15, 27): |
| 77 | ++ size = 1 << w |
| 78 | ++ conn.request("GET", "/") |
| 79 | ++ with conn.getresponse() as response: |
| 80 | ++ self.assertEqual(len(response.read()), size) |
| 81 | ++ finally: |
| 82 | ++ conn.close() |
| 83 | ++ thread.join(1.0) |
| 84 | ++ |
| 85 | ++ def test_large_content_length_truncated(self): |
| 86 | ++ serv = socket.create_server((HOST, 0)) |
| 87 | ++ self.addCleanup(serv.close) |
| 88 | ++ |
| 89 | ++ def run_server(): |
| 90 | ++ while True: |
| 91 | ++ [conn, address] = serv.accept() |
| 92 | ++ with conn: |
| 93 | ++ conn.recv(1024) |
| 94 | ++ if not size: |
| 95 | ++ break |
| 96 | ++ conn.sendall( |
| 97 | ++ b"HTTP/1.1 200 Ok\r\n" |
| 98 | ++ b"Content-Length: %d\r\n" |
| 99 | ++ b"\r\n" |
| 100 | ++ b"Text" % size) |
| 101 | ++ |
| 102 | ++ thread = threading.Thread(target=run_server) |
| 103 | ++ thread.start() |
| 104 | ++ self.addCleanup(thread.join, 1.0) |
| 105 | ++ |
| 106 | ++ conn = client.HTTPConnection(*serv.getsockname()) |
| 107 | ++ try: |
| 108 | ++ for w in range(18, 65): |
| 109 | ++ size = 1 << w |
| 110 | ++ conn.request("GET", "/") |
| 111 | ++ with conn.getresponse() as response: |
| 112 | ++ self.assertRaises(client.IncompleteRead, response.read) |
| 113 | ++ conn.close() |
| 114 | ++ finally: |
| 115 | ++ conn.close() |
| 116 | ++ size = 0 |
| 117 | ++ conn.request("GET", "/") |
| 118 | ++ conn.close() |
| 119 | ++ thread.join(1.0) |
| 120 | ++ |
| 121 | ++ |
| 122 | + def test_putrequest_override_domain_validation(self): |
| 123 | + """ |
| 124 | + It should be possible to override the default validation |
0 commit comments