Attachment #160652 for bug #230640

Lines 255-260 Link Here

(-)Objects/unicodeobject.c (-15 / +43 lines)
255	}	255	}
256	}	256	}
257	else {	257	else {
		258	if (length > (PY_SSIZE_T_MAX - 1) / sizeof(Py_UNICODE))
		259	return NULL;
258	size_t new_size = sizeof(Py_UNICODE) * ((size_t)length + 1);	260	size_t new_size = sizeof(Py_UNICODE) * ((size_t)length + 1);
259	unicode->str = (Py_UNICODE*) PyObject_MALLOC(new_size);	261	unicode->str = (Py_UNICODE*) PyObject_MALLOC(new_size);
260	}	262	}
Lines 265-271 Link Here
265	unicode = PyObject_New(PyUnicodeObject, &PyUnicode_Type);	267	unicode = PyObject_New(PyUnicodeObject, &PyUnicode_Type);
266	if (unicode == NULL)	268	if (unicode == NULL)
267	return NULL;	269	return NULL;
268	new_size = sizeof(Py_UNICODE) * ((size_t)length + 1);	270	if (length > (PY_SSIZE_T_MAX - 1) / sizeof(Py_UNICODE))
		271	return NULL;
		272	new_size = sizeof(Py_UNICODE) * ((size_t)length + 1);
269	unicode->str = (Py_UNICODE*) PyObject_MALLOC(new_size);	273	unicode->str = (Py_UNICODE*) PyObject_MALLOC(new_size);
270	}	274	}
271		275
Lines 1095-1100 Link Here
1095	char * out;	1099	char * out;
1096	char * start;	1100	char * start;
1097		1101
		1102	if (cbAllocated / 5 != size)
		1103	return PyErr_NoMemory();
		1104
1098	if (size == 0)	1105	if (size == 0)
1099	return PyString_FromStringAndSize(NULL, 0);	1106	return PyString_FromStringAndSize(NULL, 0);
1100		1107
Lines 1693-1700 Link Here
1693	{	1700	{
1694	PyObject *v;	1701	PyObject *v;
1695	unsigned char *p;	1702	unsigned char *p;
		1703	Py_ssize_t nsize, bytesize;
1696	#ifdef Py_UNICODE_WIDE	1704	#ifdef Py_UNICODE_WIDE
1697	int i, pairs;	1705	Py_ssize_t i, pairs;
1698	#else	1706	#else
1699	const int pairs = 0;	1707	const int pairs = 0;
1700	#endif	1708	#endif
Lines 1717-1724 Link Here
1717	if (s[i] >= 0x10000)	1725	if (s[i] >= 0x10000)
1718	pairs++;	1726	pairs++;
1719	#endif	1727	#endif
1720	v = PyString_FromStringAndSize(NULL,	1728	/* 2 * (size + pairs + (byteorder == 0)) */
1721	2 * (size + pairs + (byteorder == 0)));	1729	if (size > PY_SSIZE_T_MAX \|\| size > PY_SSIZE_T_MAX - pairs - (byteorder == 0))
		1730	return PyErr_NoMemory();
		1731	nsize = (size + pairs + (byteorder == 0));
		1732	bytesize = nsize * 2;
		1733	if (bytesize / 2 != nsize)
		1734	return PyErr_NoMemory();
		1735	v = PyString_FromStringAndSize(NULL, bytesize);
1722	if (v == NULL)	1736	if (v == NULL)
1723	return NULL;	1737	return NULL;
1724		1738
Lines 2045-2051 Link Here
2045	PyObject *repr;	2059	PyObject *repr;
2046	char *p;	2060	char *p;
2047		2061
2048	static const char *hexdigit = "0123456789abcdef";	2062	static const char *hexdigit = "0123456789abcdef";
		2063	#ifdef Py_UNICODE_WIDE
		2064	const Py_ssize_t expandsize = 10;
		2065	#else
		2066	const Py_ssize_t expandsize = 6;
		2067	#endif
2049		2068
2050	/* Initial allocation is based on the longest-possible unichr	2069	/* Initial allocation is based on the longest-possible unichr
2051	escape.	2070	escape.
Lines 2061-2073 Link Here
2061	escape.	2080	escape.
2062	*/	2081	*/
2063		2082
		2083	if (size > (PY_SSIZE_T_MAX - 2 - 1) / expandsize )
		2084	return PyErr_NoMemory();
		2085
2064	repr = PyString_FromStringAndSize(NULL,	2086	repr = PyString_FromStringAndSize(NULL,
2065	2	2087	2
2066	#ifdef Py_UNICODE_WIDE	2088	+ expandsize*size
2067	+ 10*size
2068	#else
2069	+ 6*size
2070	#endif
2071	+ 1);	2089	+ 1);
2072	if (repr == NULL)	2090	if (repr == NULL)
2073	return NULL;	2091	return NULL;
Lines 2319-2331 Link Here
2319	char *p;	2337	char *p;
2320	char *q;	2338	char *q;
2321		2339
2322	static const char *hexdigit = "0123456789abcdef";	2340	static const char *hexdigit = "0123456789abcdef";
2323
2324	#ifdef Py_UNICODE_WIDE	2341	#ifdef Py_UNICODE_WIDE
2325	repr = PyString_FromStringAndSize(NULL, 10 * size);	2342	const Py_ssize_t expandsize = 10;
2326	#else	2343	#else
2327	repr = PyString_FromStringAndSize(NULL, 6 * size);	2344	const Py_ssize_t expandsize = 6;
2328	#endif	2345	#endif
		2346
		2347	if (size > PY_SSIZE_T_MAX / expandsize )
		2348	return PyErr_NoMemory();
		2349
		2350	repr = PyString_FromStringAndSize(NULL, expandsize * size);
2329	if (repr == NULL)	2351	if (repr == NULL)
2330	return NULL;	2352	return NULL;
2331	if (size == 0)	2353	if (size == 0)
Lines 4761-4767 Link Here
4761	return self;	4783	return self;
4762	}	4784	}
4763		4785
4764	u = _PyUnicode_New(left + self->length + right);	4786	if (left > PY_SSIZE_T_MAX - self->length \|\| right > PY_SSIZE_T_MAX - (left + self->length)
		4787	\|\|
		4788	!(u = _PyUnicode_New(left + self->length + right))
		4789	) {
		4790	PyErr_SetString(PyExc_OverflowError, "padded string is too long");
		4791	return NULL;
		4792	}
4765	if (u) {	4793	if (u) {
4766	if (left)	4794	if (left)
4767	Py_UNICODE_FILL(u->str, fill, left);	4795	Py_UNICODE_FILL(u->str, fill, left);




		Py_ssize_t nbytes = size * sizeof(PyObject *);
		/* Check for overflow */
		if (nbytes / sizeof(PyObject *) != (size_t)size ||
		    (nbytes > PY_SSIZE_T_MAX - sizeof(PyTupleObject) - sizeof(PyObject *)))
		    <= 0)
		{
			return PyErr_NoMemory();
		}
		nbytes += sizeof(PyTupleObject) - sizeof(PyObject *);

		op = PyObject_GC_NewVar(PyTupleObject, &PyTuple_Type, size);
		if (op == NULL)
			return NULL;

Lines 427-432 Link Here

(-)Objects/bufferobject.c (+5 lines)
427	count = 0;	427	count = 0;
428	if (!get_buf(self, &ptr, &size, ANY_BUFFER))	428	if (!get_buf(self, &ptr, &size, ANY_BUFFER))
429	return NULL;	429	return NULL;
		430	if (count > PY_SSIZE_T_MAX / size) {
		431	PyErr_SetString(PyExc_MemoryError,
		432	"result too large");
		433	return NULL;
		434	}
430	ob = PyString_FromStringAndSize(NULL, size * count);	435	ob = PyString_FromStringAndSize(NULL, size * count);
431	if ( ob == NULL )	436	if ( ob == NULL )
432	return NULL;	437	return NULL;

Lines 70-75 Link Here

(-)Objects/longobject.c (+2 lines)
70	PyErr_NoMemory();	70	PyErr_NoMemory();
71	return NULL;	71	return NULL;
72	}	72	}
		73	/* This can overflow -- PyObject_NEW_VAR / _PyObject_VAR_SIZE
		74	need to detect overflow */
73	return PyObject_NEW_VAR(PyLongObject, &PyLong_Type, size);	75	return PyObject_NEW_VAR(PyLongObject, &PyLong_Type, size);
74	}	76	}
75		77




		return (PyObject *)a;
	}
	size = a->ob_size + b->ob_size;
	if (a->ob_size < 0 || b->ob_size < 0 || a->ob_size > PY_SSIZE_T_MAX - b->ob_size) {
		PyErr_SetString(PyExc_OverflowError,
				"strings are too large to concat");
		return NULL;
	}
	  
	/* Inline PyObject_NewVar */
	if (size > PY_SSIZE_T_MAX - sizeof(PyStringObject)) {
		PyErr_SetString(PyExc_OverflowError,
				"strings are too large to concat");
		return NULL;
	}
	op = (PyStringObject *)PyObject_MALLOC(sizeof(PyStringObject) + size);
	if (op == NULL)
		return PyErr_NoMemory();

Lines 223-229 Link Here

(-)Modules/mmapmodule.c (-1 / +1 lines)
223	return(NULL);	223	return(NULL);
224		224
225	/* silently 'adjust' out-of-range requests */	225	/* silently 'adjust' out-of-range requests */
226	if ((self->pos + num_bytes) > self->size) {	226	if (num_bytes > self->size - self->pos) {
227	num_bytes -= (self->pos+num_bytes) - self->size;	227	num_bytes -= (self->pos+num_bytes) - self->size;
228	}	228	}
229	result = Py_BuildValue("s#", self->data+self->pos, num_bytes);	229	result = Py_BuildValue("s#", self->data+self->pos, num_bytes);

Lines 216-221 Link Here

(-)Modules/stropmodule.c (+11 lines)
216	return NULL;	216	return NULL;
217	}	217	}
218	slen = PyString_GET_SIZE(item);	218	slen = PyString_GET_SIZE(item);
		219	if (slen > PY_SSIZE_T_MAX - reslen \|\| seplen > PY_SSIZE_T_MAX - reslen - seplen) {
		220	PyErr_SetString(PyExc_OverflowError, "input too long");
		221	Py_DECREF(res);
		222	return NULL;
		223	}
219	while (reslen + slen + seplen >= sz) {	224	while (reslen + slen + seplen >= sz) {
220	if (_PyString_Resize(&res, sz * 2) < 0)	225	if (_PyString_Resize(&res, sz * 2) < 0)
221	return NULL;	226	return NULL;
Lines 253-258 Link Here
253	return NULL;	258	return NULL;
254	}	259	}
255	slen = PyString_GET_SIZE(item);	260	slen = PyString_GET_SIZE(item);
		261	if (slen > PY_SSIZE_T_MAX - reslen \|\| seplen > PY_SSIZE_T_MAX - reslen - seplen) {
		262	PyErr_SetString(PyExc_OverflowError, "input too long");
		263	Py_DECREF(res);
		264	Py_XDECREF(item);
		265	return NULL;
		266	}
256	while (reslen + slen + seplen >= sz) {	267	while (reslen + slen + seplen >= sz) {
257	if (_PyString_Resize(&res, sz * 2) < 0) {	268	if (_PyString_Resize(&res, sz * 2) < 0) {
258	Py_DECREF(item);	269	Py_DECREF(item);

Lines 1318-1324 Link Here

(-)Modules/gcmodule.c (-1 / +6 lines)
1318	_PyObject_GC_Malloc(size_t basicsize)	1318	_PyObject_GC_Malloc(size_t basicsize)
1319	{	1319	{
1320	PyObject *op;	1320	PyObject *op;
1321	PyGC_Head g = (PyGC_Head )PyObject_MALLOC(	1321	PyGC_Head *g;
		1322	if (basicsize > PY_SSIZE_T_MAX - sizeof(PyGC_Head))
		1323	return PyErr_NoMemory();
		1324	g = (PyGC_Head *)PyObject_MALLOC(
1322	sizeof(PyGC_Head) + basicsize);	1325	sizeof(PyGC_Head) + basicsize);
1323	if (g == NULL)	1326	if (g == NULL)
1324	return PyErr_NoMemory();	1327	return PyErr_NoMemory();
Lines 1361-1366 Link Here
1361	{	1364	{
1362	const size_t basicsize = _PyObject_VAR_SIZE(op->ob_type, nitems);	1365	const size_t basicsize = _PyObject_VAR_SIZE(op->ob_type, nitems);
1363	PyGC_Head *g = AS_GC(op);	1366	PyGC_Head *g = AS_GC(op);
		1367	if (basicsize > PY_SSIZE_T_MAX - sizeof(PyGC_Head))
		1368	return (PyVarObject *)PyErr_NoMemory();
1364	g = (PyGC_Head *)PyObject_REALLOC(g, sizeof(PyGC_Head) + basicsize);	1369	g = (PyGC_Head *)PyObject_REALLOC(g, sizeof(PyGC_Head) + basicsize);
1365	if (g == NULL)	1370	if (g == NULL)
1366	return (PyVarObject *)PyErr_NoMemory();	1371	return (PyVarObject *)PyErr_NoMemory();

Lines 115-121 Link Here

(-)Lib/test/test_strop.py (+19 lines)
115	strop.uppercase	115	strop.uppercase
116	strop.whitespace	116	strop.whitespace
117		117
		118	@test_support.precisionbigmemtest(size=test_support._2G - 1, memuse=5)
		119	def test_stropjoin_huge_list(self, size):
		120	a = "A" * size
		121	try:
		122	r = strop.join([a, a], a)
		123	except OverflowError:
		124	pass
		125	else:
		126	self.assertEquals(len(r), len(a) * 3)
118		127
		128	@test_support.precisionbigmemtest(size=test_support._2G - 1, memuse=1)
		129	def test_stropjoin_huge_tup(self, size):
		130	a = "A" * size
		131	try:
		132	r = strop.join((a, a), a)
		133	except OverflowError:
		134	pass # acceptable on 32-bit
		135	else:
		136	self.assertEquals(len(r), len(a) * 3)
		137
119	transtable = '\000\001\002\003\004\005\006\007\010\011\012\013\014\015\016\017\020\021\022\023\024\025\026\027\030\031\032\033\034\035\036\037 !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`xyzdefghijklmnopqrstuvwxyz{\|}~\177\200\201\202\203\204\205\206\207\210\211\212\213\214\215\216\217\220\221\222\223\224\225\226\227\230\231\232\233\234\235\236\237\240\241\242\243\244\245\246\247\250\251\252\253\254\255\256\257\260\261\262\263\264\265\266\267\270\271\272\273\274\275\276\277\300\301\302\303\304\305\306\307\310\311\312\313\314\315\316\317\320\321\322\323\324\325\326\327\330\331\332\333\334\335\336\337\340\341\342\343\344\345\346\347\350\351\352\353\354\355\356\357\360\361\362\363\364\365\366\367\370\371\372\373\374\375\376\377'	138	transtable = '\000\001\002\003\004\005\006\007\010\011\012\013\014\015\016\017\020\021\022\023\024\025\026\027\030\031\032\033\034\035\036\037 !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`xyzdefghijklmnopqrstuvwxyz{\|}~\177\200\201\202\203\204\205\206\207\210\211\212\213\214\215\216\217\220\221\222\223\224\225\226\227\230\231\232\233\234\235\236\237\240\241\242\243\244\245\246\247\250\251\252\253\254\255\256\257\260\261\262\263\264\265\266\267\270\271\272\273\274\275\276\277\300\301\302\303\304\305\306\307\310\311\312\313\314\315\316\317\320\321\322\323\324\325\326\327\330\331\332\333\334\335\336\337\340\341\342\343\344\345\346\347\350\351\352\353\354\355\356\357\360\361\362\363\364\365\366\367\370\371\372\373\374\375\376\377'
120		139
121		140




from test import test_support
from test.test_support import bigmemtest, _1G, _2G, _4G, precisionbigmemtest

import unittest
import operator

            lpadsize += 1
        self.assertEquals(s[lpadsize:-rpadsize], SUBSTR)
        self.assertEquals(s.strip(), SUBSTR.strip())
    
    @precisionbigmemtest(size=_2G - 1, memuse=1)
    def test_center_unicode(self, size):
        SUBSTR = u' abc def ghi'
        try:
            s = SUBSTR.center(size)
        except OverflowError:
            pass # acceptable on 32-bit
        else:
            self.assertEquals(len(s), size)
            lpadsize = rpadsize = (len(s) - len(SUBSTR)) // 2
            if len(s) % 2:
                lpadsize += 1
            self.assertEquals(s[lpadsize:-rpadsize], SUBSTR)
            self.assertEquals(s.strip(), SUBSTR.strip())
            del s
    
    @bigmemtest(minsize=_2G, memuse=2)
    def test_count(self, size):
        SUBSTR = ' abc def ghi'

    def test_decode(self, size):
        s = '.' * size
        self.assertEquals(len(s.decode('utf-8')), size)
    
    def basic_encode_test(self, size, enc, c=u'.', expectedsize=None):
        if expectedsize is None:
            expectedsize = size
        
        s = c * size
        self.assertEquals(len(s.encode(enc)), expectedsize)

    @bigmemtest(minsize=_2G + 2, memuse=3)
    def test_encode(self, size):
        return self.basic_encode_test(size, 'utf-8')
    
    @precisionbigmemtest(size=_4G / 6 + 2, memuse=2)
    def test_encode_raw_unicode_escape(self, size):
        try:
            return self.basic_encode_test(size, 'raw_unicode_escape')
        except MemoryError:
            pass # acceptable on 32-bit
    
    @precisionbigmemtest(size=_4G / 5 + 70, memuse=3)
    def test_encode_utf7(self, size):
        try:
            return self.basic_encode_test(size, 'utf7')
        except MemoryError:
            pass # acceptable on 32-bit
    
    @precisionbigmemtest(size=_2G-1, memuse=2)
    def test_decodeascii(self, size):
        return self.basic_encode_test(size, 'ascii', c='A')
    
    @precisionbigmemtest(size=_4G / 5, memuse=6+2)
    def test_unicode_repr_oflw(self, size):
        try:
            s = u"\uAAAA"*size
            r = repr(s)
        except MemoryError:
            pass # acceptable on 32-bit
        else:
            self.failUnless(s == eval(r))

    @bigmemtest(minsize=_2G, memuse=2)
    def test_endswith(self, size):

        self.assertEquals(s[-1], "'")
        self.assertEquals(s.count('\\'), size)
        self.assertEquals(s.count('0'), size * 2)
    
    @bigmemtest(minsize=2**32 / 5, memuse=6+2)
    def test_unicode_repr(self, size):
        s = u"\uAAAA" * size
        self.failUnless(len(repr(s)) > size)
    
    # This test is meaningful even with size < 2G, as long as the
    # doubled string is > 2G (but it tests more if both are > 2G :)
    @bigmemtest(minsize=_1G + 2, memuse=3)

    def test_repeat_large(self, size):
        return self.basic_test_repeat(size)

    @bigmemtest(minsize=_1G - 1, memuse=12)
    def test_repeat_large_2(self, size):
        return self.basic_test_repeat(size)
    
    @precisionbigmemtest(size=_1G - 1, memuse=9)
    def test_from_2G_generator(self, size):
        try:
            t = tuple(xrange(size))
        except MemoryError:
            pass # acceptable on 32-bit
        else:
            count = 0
            for item in t:
                self.assertEquals(item, count)
                count += 1
            self.assertEquals(count, size)
    
    @precisionbigmemtest(size=_1G - 25, memuse=9)
    def test_from_almost_2G_generator(self, size):
        try:
            t = tuple(xrange(size))
            count = 0
            for item in t:
                self.assertEquals(item, count)
                count += 1
            self.assertEquals(count, size)
        except MemoryError:
            pass # acceptable, expected on 32-bit

    # Like test_concat, split in two.
    def basic_test_repr(self, size):
        t = (0,) * size

        self.assertEquals(l[:10], [1] * 10)
        self.assertEquals(l[-10:], [5] * 10)

class BufferTest(unittest.TestCase):
    
    @precisionbigmemtest(size=_1G, memuse=4)
    def test_repeat(self, size):
        try:
            b = buffer("AAAA")*size
        except MemoryError:
            pass # acceptable on 32-bit
        else:
            count = 0
            for c in b:
                self.assertEquals(c, 'A')
                count += 1
            self.assertEquals(count, size*4)
    
def test_main():
    test_support.run_unittest(StrTest, TupleTest, ListTest, BufferTest)

# Expected failures (crashers)
# del StrTest.test_center_unicode
del StrTest.test_decodeascii
# del StrTest.test_encode_utf32
# del StrTest.test_encode_utf7
# del StrTest.test_encode_raw_unicode_escape
# 
# del TupleTest.test_from_2G_generator
#
# del BufferTest.test_repeat

if __name__ == '__main__':
    if len(sys.argv) > 1:
        test_support.set_memlimit(sys.argv[1])




use_resources = None     # Flag set to [] by regrtest.py
max_memuse = 0           # Disable bigmem tests (they will still be run with
                         # small sizes, to make sure they work.)
real_max_memuse = 0

# _original_stdout is meant to hold stdout at the time regrtest began.
# This may be "the real" stdout, or IDLE's emulation of stdout, or whatever.

_1M = 1024*1024
_1G = 1024 * _1M
_2G = 2 * _1G
_4G = 4 * _1G

# Hack to get at the maximum value an internal index can take.
class _Dummy:

def set_memlimit(limit):
    import re
    global max_memuse
    global real_max_memuse
    sizes = {
        'k': 1024,
        'm': _1M,

    if m is None:
        raise ValueError('Invalid memory limit %r' % (limit,))
    memlimit = int(float(m.group(1)) * sizes[m.group(3).lower()])
    real_max_memuse = memlimit
    if memlimit > MAX_Py_ssize_t:
        memlimit = MAX_Py_ssize_t
    if memlimit < _2G - 1:

        return wrapper
    return decorator

def precisionbigmemtest(size, memuse, overhead=5*_1M):
    def decorator(f):
        def wrapper(self):
            if not real_max_memuse:
                maxsize = 5147
            else:
                maxsize = size
            
                if real_max_memuse and real_max_memuse < maxsize * memuse:
                    if verbose:
                        sys.stderr.write("Skipping %s because of memory "
                                         "constraint\n" % (f.__name__,))
                    return
            
            return f(self, maxsize)
        wrapper.size = size
        wrapper.memuse = memuse
        wrapper.overhead = overhead
        return wrapper
    return decorator

def bigaddrspacetest(f):
    """Decorator for tests that fill the address space."""
    def wrapper(self):

Lines 307-312 Link Here

(-)Lib/test/seq_tests.py (+3 lines)
307	self.assertEqual(id(s), id(s*1))	307	self.assertEqual(id(s), id(s*1))
308		308
309	def test_bigrepeat(self):	309	def test_bigrepeat(self):
		310	from sys import maxint
		311	if not maxint <= 2147483647:
		312	return
310	x = self.type2test([0])	313	x = self.type2test([0])
311	x = 2*16	314	x = 2*16
312	self.assertRaises(MemoryError, x.__mul__, 2**16)	315	self.assertRaises(MemoryError, x.__mul__, 2**16)

Return to bug 230640

Lines 972-984 Link Here

(-)Objects/stringobject.c (-1 / +6 lines)
972	return (PyObject *)a;	972	return (PyObject *)a;
973	}	973	}
974	size = a->ob_size + b->ob_size;	974	size = a->ob_size + b->ob_size;
975	if (size < 0) {	975	if (a->ob_size < 0 \|\| b->ob_size < 0 \|\| a->ob_size > PY_SSIZE_T_MAX - b->ob_size) {
976	PyErr_SetString(PyExc_OverflowError,	976	PyErr_SetString(PyExc_OverflowError,
977	"strings are too large to concat");	977	"strings are too large to concat");
978	return NULL;	978	return NULL;
979	}	979	}
980		980
981	/* Inline PyObject_NewVar */	981	/* Inline PyObject_NewVar */
		982	if (size > PY_SSIZE_T_MAX - sizeof(PyStringObject)) {
		983	PyErr_SetString(PyExc_OverflowError,
		984	"strings are too large to concat");
		985	return NULL;
		986	}
982	op = (PyStringObject *)PyObject_MALLOC(sizeof(PyStringObject) + size);	987	op = (PyStringObject *)PyObject_MALLOC(sizeof(PyStringObject) + size);
983	if (op == NULL)	988	if (op == NULL)
984	return PyErr_NoMemory();	989	return PyErr_NoMemory();

Lines 60-70 Link Here

(-)Objects/tupleobject.c (-2 / +3 lines)
60	Py_ssize_t nbytes = size * sizeof(PyObject *);	60	Py_ssize_t nbytes = size * sizeof(PyObject *);
61	/* Check for overflow */	61	/* Check for overflow */
62	if (nbytes / sizeof(PyObject *) != (size_t)size \|\|	62	if (nbytes / sizeof(PyObject *) != (size_t)size \|\|
63	(nbytes += sizeof(PyTupleObject) - sizeof(PyObject *))	63	(nbytes > PY_SSIZE_T_MAX - sizeof(PyTupleObject) - sizeof(PyObject *)))
64	<= 0)
65	{	64	{
66	return PyErr_NoMemory();	65	return PyErr_NoMemory();
67	}	66	}
		67	nbytes += sizeof(PyTupleObject) - sizeof(PyObject *);
		68
68	op = PyObject_GC_NewVar(PyTupleObject, &PyTuple_Type, size);	69	op = PyObject_GC_NewVar(PyTupleObject, &PyTuple_Type, size);
69	if (op == NULL)	70	if (op == NULL)
70	return NULL;	71	return NULL;

Lines 1-5 Link Here

(-)Lib/test/test_bigmem.py (-6 / +116 lines)
1	from test import test_support	1	from test import test_support
2	from test.test_support import bigmemtest, _1G, _2G	2	from test.test_support import bigmemtest, _1G, _2G, _4G, precisionbigmemtest
3		3
4	import unittest	4	import unittest
5	import operator	5	import operator
Lines 53-59 Link Here
53	lpadsize += 1	53	lpadsize += 1
54	self.assertEquals(s[lpadsize:-rpadsize], SUBSTR)	54	self.assertEquals(s[lpadsize:-rpadsize], SUBSTR)
55	self.assertEquals(s.strip(), SUBSTR.strip())	55	self.assertEquals(s.strip(), SUBSTR.strip())
56		56
		57	@precisionbigmemtest(size=_2G - 1, memuse=1)
		58	def test_center_unicode(self, size):
		59	SUBSTR = u' abc def ghi'
		60	try:
		61	s = SUBSTR.center(size)
		62	except OverflowError:
		63	pass # acceptable on 32-bit
		64	else:
		65	self.assertEquals(len(s), size)
		66	lpadsize = rpadsize = (len(s) - len(SUBSTR)) // 2
		67	if len(s) % 2:
		68	lpadsize += 1
		69	self.assertEquals(s[lpadsize:-rpadsize], SUBSTR)
		70	self.assertEquals(s.strip(), SUBSTR.strip())
		71	del s
		72
57	@bigmemtest(minsize=_2G, memuse=2)	73	@bigmemtest(minsize=_2G, memuse=2)
58	def test_count(self, size):	74	def test_count(self, size):
59	SUBSTR = ' abc def ghi'	75	SUBSTR = ' abc def ghi'
Lines 69-79 Link Here
69	def test_decode(self, size):	85	def test_decode(self, size):
70	s = '.' * size	86	s = '.' * size
71	self.assertEquals(len(s.decode('utf-8')), size)	87	self.assertEquals(len(s.decode('utf-8')), size)
		88
		89	def basic_encode_test(self, size, enc, c=u'.', expectedsize=None):
		90	if expectedsize is None:
		91	expectedsize = size
		92
		93	s = c * size
		94	self.assertEquals(len(s.encode(enc)), expectedsize)
72		95
73	@bigmemtest(minsize=_2G + 2, memuse=3)	96	@bigmemtest(minsize=_2G + 2, memuse=3)
74	def test_encode(self, size):	97	def test_encode(self, size):
75	s = u'.' * size	98	return self.basic_encode_test(size, 'utf-8')
76	self.assertEquals(len(s.encode('utf-8')), size)	99
		100	@precisionbigmemtest(size=_4G / 6 + 2, memuse=2)
		101	def test_encode_raw_unicode_escape(self, size):
		102	try:
		103	return self.basic_encode_test(size, 'raw_unicode_escape')
		104	except MemoryError:
		105	pass # acceptable on 32-bit
		106
		107	@precisionbigmemtest(size=_4G / 5 + 70, memuse=3)
		108	def test_encode_utf7(self, size):
		109	try:
		110	return self.basic_encode_test(size, 'utf7')
		111	except MemoryError:
		112	pass # acceptable on 32-bit
		113
		114	@precisionbigmemtest(size=_2G-1, memuse=2)
		115	def test_decodeascii(self, size):
		116	return self.basic_encode_test(size, 'ascii', c='A')
		117
		118	@precisionbigmemtest(size=_4G / 5, memuse=6+2)
		119	def test_unicode_repr_oflw(self, size):
		120	try:
		121	s = u"\uAAAA"*size
		122	r = repr(s)
		123	except MemoryError:
		124	pass # acceptable on 32-bit
		125	else:
		126	self.failUnless(s == eval(r))
77		127
78	@bigmemtest(minsize=_2G, memuse=2)	128	@bigmemtest(minsize=_2G, memuse=2)
79	def test_endswith(self, size):	129	def test_endswith(self, size):
Lines 458-464 Link Here
458	self.assertEquals(s[-1], "'")	508	self.assertEquals(s[-1], "'")
459	self.assertEquals(s.count('\\'), size)	509	self.assertEquals(s.count('\\'), size)
460	self.assertEquals(s.count('0'), size * 2)	510	self.assertEquals(s.count('0'), size * 2)
461		511
		512	@bigmemtest(minsize=2**32 / 5, memuse=6+2)
		513	def test_unicode_repr(self, size):
		514	s = u"\uAAAA" * size
		515	self.failUnless(len(repr(s)) > size)
		516
462	# This test is meaningful even with size < 2G, as long as the	517	# This test is meaningful even with size < 2G, as long as the
463	# doubled string is > 2G (but it tests more if both are > 2G :)	518	# doubled string is > 2G (but it tests more if both are > 2G :)
464	@bigmemtest(minsize=_1G + 2, memuse=3)	519	@bigmemtest(minsize=_1G + 2, memuse=3)
Lines 642-647 Link Here
642	def test_repeat_large(self, size):	697	def test_repeat_large(self, size):
643	return self.basic_test_repeat(size)	698	return self.basic_test_repeat(size)
644		699
		700	@bigmemtest(minsize=_1G - 1, memuse=12)
		701	def test_repeat_large_2(self, size):
		702	return self.basic_test_repeat(size)
		703
		704	@precisionbigmemtest(size=_1G - 1, memuse=9)
		705	def test_from_2G_generator(self, size):
		706	try:
		707	t = tuple(xrange(size))
		708	except MemoryError:
		709	pass # acceptable on 32-bit
		710	else:
		711	count = 0
		712	for item in t:
		713	self.assertEquals(item, count)
		714	count += 1
		715	self.assertEquals(count, size)
		716
		717	@precisionbigmemtest(size=_1G - 25, memuse=9)
		718	def test_from_almost_2G_generator(self, size):
		719	try:
		720	t = tuple(xrange(size))
		721	count = 0
		722	for item in t:
		723	self.assertEquals(item, count)
		724	count += 1
		725	self.assertEquals(count, size)
		726	except MemoryError:
		727	pass # acceptable, expected on 32-bit
		728
645	# Like test_concat, split in two.	729	# Like test_concat, split in two.
646	def basic_test_repr(self, size):	730	def basic_test_repr(self, size):
647	t = (0,) * size	731	t = (0,) * size
Lines 957-965 Link Here
957	self.assertEquals(l[:10], [1] * 10)	1041	self.assertEquals(l[:10], [1] * 10)
958	self.assertEquals(l[-10:], [5] * 10)	1042	self.assertEquals(l[-10:], [5] * 10)
959		1043
		1044	class BufferTest(unittest.TestCase):
		1045
		1046	@precisionbigmemtest(size=_1G, memuse=4)
		1047	def test_repeat(self, size):
		1048	try:
		1049	b = buffer("AAAA")*size
		1050	except MemoryError:
		1051	pass # acceptable on 32-bit
		1052	else:
		1053	count = 0
		1054	for c in b:
		1055	self.assertEquals(c, 'A')
		1056	count += 1
		1057	self.assertEquals(count, size*4)
		1058
960	def test_main():	1059	def test_main():
961	test_support.run_unittest(StrTest, TupleTest, ListTest)	1060	test_support.run_unittest(StrTest, TupleTest, ListTest, BufferTest)
962		1061
		1062	# Expected failures (crashers)
		1063	# del StrTest.test_center_unicode
		1064	del StrTest.test_decodeascii
		1065	# del StrTest.test_encode_utf32
		1066	# del StrTest.test_encode_utf7
		1067	# del StrTest.test_encode_raw_unicode_escape
		1068	#
		1069	# del TupleTest.test_from_2G_generator
		1070	#
		1071	# del BufferTest.test_repeat
		1072
963	if __name__ == '__main__':	1073	if __name__ == '__main__':
964	if len(sys.argv) > 1:	1074	if len(sys.argv) > 1:
965	test_support.set_memlimit(sys.argv[1])	1075	test_support.set_memlimit(sys.argv[1])

Lines 33-38 Link Here

(-)Lib/test/test_support.py (+25 lines)
33	use_resources = None # Flag set to [] by regrtest.py	33	use_resources = None # Flag set to [] by regrtest.py
34	max_memuse = 0 # Disable bigmem tests (they will still be run with	34	max_memuse = 0 # Disable bigmem tests (they will still be run with
35	# small sizes, to make sure they work.)	35	# small sizes, to make sure they work.)
		36	real_max_memuse = 0
36		37
37	# _original_stdout is meant to hold stdout at the time regrtest began.	38	# _original_stdout is meant to hold stdout at the time regrtest began.
38	# This may be "the real" stdout, or IDLE's emulation of stdout, or whatever.	39	# This may be "the real" stdout, or IDLE's emulation of stdout, or whatever.
Lines 323-328 Link Here
323	_1M = 1024*1024	324	_1M = 1024*1024
324	_1G = 1024 * _1M	325	_1G = 1024 * _1M
325	_2G = 2 * _1G	326	_2G = 2 * _1G
		327	_4G = 4 * _1G
326		328
327	# Hack to get at the maximum value an internal index can take.	329	# Hack to get at the maximum value an internal index can take.
328	class _Dummy:	330	class _Dummy:
Lines 333-338 Link Here
333	def set_memlimit(limit):	335	def set_memlimit(limit):
334	import re	336	import re
335	global max_memuse	337	global max_memuse
		338	global real_max_memuse
336	sizes = {	339	sizes = {
337	'k': 1024,	340	'k': 1024,
338	'm': _1M,	341	'm': _1M,
Lines 344-349 Link Here
344	if m is None:	347	if m is None:
345	raise ValueError('Invalid memory limit %r' % (limit,))	348	raise ValueError('Invalid memory limit %r' % (limit,))
346	memlimit = int(float(m.group(1)) * sizes[m.group(3).lower()])	349	memlimit = int(float(m.group(1)) * sizes[m.group(3).lower()])
		350	real_max_memuse = memlimit
347	if memlimit > MAX_Py_ssize_t:	351	if memlimit > MAX_Py_ssize_t:
348	memlimit = MAX_Py_ssize_t	352	memlimit = MAX_Py_ssize_t
349	if memlimit < _2G - 1:	353	if memlimit < _2G - 1:
Lines 389-394 Link Here
389	return wrapper	393	return wrapper
390	return decorator	394	return decorator
391		395
		396	def precisionbigmemtest(size, memuse, overhead=5*_1M):
		397	def decorator(f):
		398	def wrapper(self):
		399	if not real_max_memuse:
		400	maxsize = 5147
		401	else:
		402	maxsize = size
		403
		404	if real_max_memuse and real_max_memuse < maxsize * memuse:
		405	if verbose:
		406	sys.stderr.write("Skipping %s because of memory "
		407	"constraint\n" % (f.__name__,))
		408	return
		409
		410	return f(self, maxsize)
		411	wrapper.size = size
		412	wrapper.memuse = memuse
		413	wrapper.overhead = overhead
		414	return wrapper
		415	return decorator
		416
392	def bigaddrspacetest(f):	417	def bigaddrspacetest(f):
393	"""Decorator for tests that fill the address space."""	418	"""Decorator for tests that fill the address space."""
394	def wrapper(self):	419	def wrapper(self):