Attachment #159418 for bug #230640

Lines 314-319 Link Here

(-)Objects/unicodeobject.c (-17 / +49 lines)
314	}	314	}
315	}	315	}
316	else {	316	else {
		317	if (length > (PY_SSIZE_T_MAX - 1) / sizeof(Py_UNICODE))
		318	return NULL;
317	size_t new_size = sizeof(Py_UNICODE) * ((size_t)length + 1);	319	size_t new_size = sizeof(Py_UNICODE) * ((size_t)length + 1);
318	unicode->str = (Py_UNICODE*) PyObject_MALLOC(new_size);	320	unicode->str = (Py_UNICODE*) PyObject_MALLOC(new_size);
319	}	321	}
Lines 324-329 Link Here
324	unicode = PyObject_New(PyUnicodeObject, &PyUnicode_Type);	326	unicode = PyObject_New(PyUnicodeObject, &PyUnicode_Type);
325	if (unicode == NULL)	327	if (unicode == NULL)
326	return NULL;	328	return NULL;
		329	if (length > (PY_SSIZE_T_MAX - 1) / sizeof(Py_UNICODE))
		330	return NULL;
327	new_size = sizeof(Py_UNICODE) * ((size_t)length + 1);	331	new_size = sizeof(Py_UNICODE) * ((size_t)length + 1);
328	unicode->str = (Py_UNICODE*) PyObject_MALLOC(new_size);	332	unicode->str = (Py_UNICODE*) PyObject_MALLOC(new_size);
329	}	333	}
Lines 1651-1656 Link Here
1651	char * out;	1655	char * out;
1652	char * start;	1656	char * start;
1653		1657
		1658	if (cbAllocated / 5 != size)
		1659	return PyErr_NoMemory();
		1660
1654	if (size == 0)	1661	if (size == 0)
1655	return PyString_FromStringAndSize(NULL, 0);	1662	return PyString_FromStringAndSize(NULL, 0);
1656		1663
Lines 2245-2252 Link Here
2245	{	2252	{
2246	PyObject *v;	2253	PyObject *v;
2247	unsigned char *p;	2254	unsigned char *p;
		2255	Py_ssize_t nsize, bytesize;
2248	#ifndef Py_UNICODE_WIDE	2256	#ifndef Py_UNICODE_WIDE
2249	int i, pairs;	2257	Py_ssize_t i, pairs;
2250	#else	2258	#else
2251	const int pairs = 0;	2259	const int pairs = 0;
2252	#endif	2260	#endif
Lines 2274-2281 Link Here
2274	0xDC00 <= s[i+1] && s[i+1] <= 0xDFFF)	2282	0xDC00 <= s[i+1] && s[i+1] <= 0xDFFF)
2275	pairs++;	2283	pairs++;
2276	#endif	2284	#endif
2277	v = PyString_FromStringAndSize(NULL,	2285	nsize = (size - pairs + (byteorder == 0));
2278	4 * (size - pairs + (byteorder == 0)));	2286	bytesize = nsize * 4;
		2287	if (bytesize / 4 != nsize)
		2288	return PyErr_NoMemory();
		2289	v = PyString_FromStringAndSize(NULL, bytesize);
2279	if (v == NULL)	2290	if (v == NULL)
2280	return NULL;	2291	return NULL;
2281		2292
Lines 2515-2522 Link Here
2515	{	2526	{
2516	PyObject *v;	2527	PyObject *v;
2517	unsigned char *p;	2528	unsigned char *p;
		2529	Py_ssize_t nsize, bytesize;
2518	#ifdef Py_UNICODE_WIDE	2530	#ifdef Py_UNICODE_WIDE
2519	int i, pairs;	2531	Py_ssize_t i, pairs;
2520	#else	2532	#else
2521	const int pairs = 0;	2533	const int pairs = 0;
2522	#endif	2534	#endif
Lines 2539-2546 Link Here
2539	if (s[i] >= 0x10000)	2551	if (s[i] >= 0x10000)
2540	pairs++;	2552	pairs++;
2541	#endif	2553	#endif
2542	v = PyString_FromStringAndSize(NULL,	2554	/* 2 * (size + pairs + (byteorder == 0)) */
2543	2 * (size + pairs + (byteorder == 0)));	2555	if (size > PY_SSIZE_T_MAX \|\| size > PY_SSIZE_T_MAX - pairs - (byteorder == 0))
		2556	return PyErr_NoMemory();
		2557	nsize = (size + pairs + (byteorder == 0));
		2558	bytesize = nsize * 2;
		2559	if (bytesize / 2 != nsize)
		2560	return PyErr_NoMemory();
		2561	v = PyString_FromStringAndSize(NULL, bytesize);
2544	if (v == NULL)	2562	if (v == NULL)
2545	return NULL;	2563	return NULL;
2546		2564
Lines 2867-2873 Link Here
2867	PyObject *repr;	2885	PyObject *repr;
2868	char *p;	2886	char *p;
2869		2887
2870	static const char *hexdigit = "0123456789abcdef";	2888	static const char *hexdigit = "0123456789abcdef";
		2889	#ifdef Py_UNICODE_WIDE
		2890	const Py_ssize_t expandsize = 10;
		2891	#else
		2892	const Py_ssize_t expandsize = 6;
		2893	#endif
2871		2894
2872	/* XXX(nnorwitz): rather than over-allocating, it would be	2895	/* XXX(nnorwitz): rather than over-allocating, it would be
2873	better to choose a different scheme. Perhaps scan the	2896	better to choose a different scheme. Perhaps scan the
Lines 2887-2899 Link Here
2887	escape.	2910	escape.
2888	*/	2911	*/
2889		2912
		2913	if (size > (PY_SSIZE_T_MAX - 2 - 1) / expandsize )
		2914	return PyErr_NoMemory();
		2915
2890	repr = PyString_FromStringAndSize(NULL,	2916	repr = PyString_FromStringAndSize(NULL,
2891	2	2917	2
2892	#ifdef Py_UNICODE_WIDE	2918	+ expandsize*size
2893	+ 10*size
2894	#else
2895	+ 6*size
2896	#endif
2897	+ 1);	2919	+ 1);
2898	if (repr == NULL)	2920	if (repr == NULL)
2899	return NULL;	2921	return NULL;
Lines 3145-3157 Link Here
3145	char *p;	3167	char *p;
3146	char *q;	3168	char *q;
3147		3169
3148	static const char *hexdigit = "0123456789abcdef";	3170	static const char *hexdigit = "0123456789abcdef";
3149
3150	#ifdef Py_UNICODE_WIDE	3171	#ifdef Py_UNICODE_WIDE
3151	repr = PyString_FromStringAndSize(NULL, 10 * size);	3172	const Py_ssize_t expandsize = 10;
3152	#else	3173	#else
3153	repr = PyString_FromStringAndSize(NULL, 6 * size);	3174	const Py_ssize_t expandsize = 6;
3154	#endif	3175	#endif
		3176
		3177	if (size > PY_SSIZE_T_MAX / expandsize )
		3178	return PyErr_NoMemory();
		3179
		3180	repr = PyString_FromStringAndSize(NULL, expandsize * size);
3155	if (repr == NULL)	3181	if (repr == NULL)
3156	return NULL;	3182	return NULL;
3157	if (size == 0)	3183	if (size == 0)
Lines 5574-5580 Link Here
5574	return self;	5600	return self;
5575	}	5601	}
5576		5602
5577	u = _PyUnicode_New(left + self->length + right);	5603	if (left > PY_SSIZE_T_MAX - self->length \|\| right > PY_SSIZE_T_MAX - (left + self->length)
		5604	\|\|
		5605	!(u = _PyUnicode_New(left + self->length + right))
		5606	) {
		5607	PyErr_SetString(PyExc_OverflowError, "padded string is too long");
		5608	return NULL;
		5609	}
5578	if (u) {	5610	if (u) {
5579	if (left)	5611	if (left)
5580	Py_UNICODE_FILL(u->str, fill, left);	5612	Py_UNICODE_FILL(u->str, fill, left);




		Py_ssize_t nbytes = size * sizeof(PyObject *);
		/* Check for overflow */
		if (nbytes / sizeof(PyObject *) != (size_t)size ||
		    (nbytes > PY_SSIZE_T_MAX - sizeof(PyTupleObject) - sizeof(PyObject *)))
		    <= 0)
		{
			return PyErr_NoMemory();
		}
		nbytes += sizeof(PyTupleObject) - sizeof(PyObject *);

		op = PyObject_GC_NewVar(PyTupleObject, &PyTuple_Type, size);
		if (op == NULL)
			return NULL;

Lines 431-436 Link Here

(-)Objects/bufferobject.c (+5 lines)
431	count = 0;	431	count = 0;
432	if (!get_buf(self, &ptr, &size, ANY_BUFFER))	432	if (!get_buf(self, &ptr, &size, ANY_BUFFER))
433	return NULL;	433	return NULL;
		434	if (count > PY_SSIZE_T_MAX / size) {
		435	PyErr_SetString(PyExc_MemoryError,
		436	"result too large");
		437	return NULL;
		438	}
434	ob = PyString_FromStringAndSize(NULL, size * count);	439	ob = PyString_FromStringAndSize(NULL, size * count);
435	if ( ob == NULL )	440	if ( ob == NULL )
436	return NULL;	441	return NULL;

Lines 70-75 Link Here

(-)Objects/longobject.c (+2 lines)
70	return NULL;	70	return NULL;
71	}	71	}
72	/* coverity[ampersand_in_size] */	72	/* coverity[ampersand_in_size] */
		73	/* This can overflow -- PyObject_NEW_VAR / _PyObject_VAR_SIZE
		74	need to detect overflow */
73	return PyObject_NEW_VAR(PyLongObject, &PyLong_Type, size);	75	return PyObject_NEW_VAR(PyLongObject, &PyLong_Type, size);
74	}	76	}
75		77




		return (PyObject *)a;
	}
	size = Py_SIZE(a) + Py_SIZE(b);
	if (Py_SIZE(a) < 0 || Py_SIZE(b) < 0 || Py_SIZE(a) > PY_SSIZE_T_MAX - Py_SIZE(b)) {
		PyErr_SetString(PyExc_OverflowError,
				"strings are too large to concat");
		return NULL;
	}
	  
	/* Inline PyObject_NewVar */
	if (size > PY_SSIZE_T_MAX - sizeof(PyStringObject)) {
		PyErr_SetString(PyExc_OverflowError,
				"strings are too large to concat");
		return NULL;
	}
	op = (PyStringObject *)PyObject_MALLOC(sizeof(PyStringObject) + size);
	if (op == NULL)
		return PyErr_NoMemory();

Lines 239-245 Link Here

(-)Modules/mmapmodule.c (-1 / +1 lines)
239	return(NULL);	239	return(NULL);
240		240
241	/* silently 'adjust' out-of-range requests */	241	/* silently 'adjust' out-of-range requests */
242	if ((self->pos + num_bytes) > self->size) {	242	if (num_bytes > self->size - self->pos) {
243	num_bytes -= (self->pos+num_bytes) - self->size;	243	num_bytes -= (self->pos+num_bytes) - self->size;
244	}	244	}
245	result = Py_BuildValue("s#", self->data+self->pos, num_bytes);	245	result = Py_BuildValue("s#", self->data+self->pos, num_bytes);

Lines 216-221 Link Here

(-)Modules/stropmodule.c (+11 lines)
216	return NULL;	216	return NULL;
217	}	217	}
218	slen = PyString_GET_SIZE(item);	218	slen = PyString_GET_SIZE(item);
		219	if (slen > PY_SSIZE_T_MAX - reslen \|\| seplen > PY_SSIZE_T_MAX - reslen - seplen) {
		220	PyErr_SetString(PyExc_OverflowError, "input too long");
		221	Py_DECREF(res);
		222	return NULL;
		223	}
219	while (reslen + slen + seplen >= sz) {	224	while (reslen + slen + seplen >= sz) {
220	if (_PyString_Resize(&res, sz * 2) < 0)	225	if (_PyString_Resize(&res, sz * 2) < 0)
221	return NULL;	226	return NULL;
Lines 253-258 Link Here
253	return NULL;	258	return NULL;
254	}	259	}
255	slen = PyString_GET_SIZE(item);	260	slen = PyString_GET_SIZE(item);
		261	if (slen > PY_SSIZE_T_MAX - reslen \|\| seplen > PY_SSIZE_T_MAX - reslen - seplen) {
		262	PyErr_SetString(PyExc_OverflowError, "input too long");
		263	Py_DECREF(res);
		264	Py_XDECREF(item);
		265	return NULL;
		266	}
256	while (reslen + slen + seplen >= sz) {	267	while (reslen + slen + seplen >= sz) {
257	if (_PyString_Resize(&res, sz * 2) < 0) {	268	if (_PyString_Resize(&res, sz * 2) < 0) {
258	Py_DECREF(item);	269	Py_DECREF(item);

Lines 1340-1346 Link Here

(-)Modules/gcmodule.c (-1 / +6 lines)
1340	_PyObject_GC_Malloc(size_t basicsize)	1340	_PyObject_GC_Malloc(size_t basicsize)
1341	{	1341	{
1342	PyObject *op;	1342	PyObject *op;
1343	PyGC_Head g = (PyGC_Head )PyObject_MALLOC(	1343	PyGC_Head *g;
		1344	if (basicsize > PY_SSIZE_T_MAX - sizeof(PyGC_Head))
		1345	return PyErr_NoMemory();
		1346	g = (PyGC_Head *)PyObject_MALLOC(
1344	sizeof(PyGC_Head) + basicsize);	1347	sizeof(PyGC_Head) + basicsize);
1345	if (g == NULL)	1348	if (g == NULL)
1346	return PyErr_NoMemory();	1349	return PyErr_NoMemory();
Lines 1383-1388 Link Here
1383	{	1386	{
1384	const size_t basicsize = _PyObject_VAR_SIZE(Py_TYPE(op), nitems);	1387	const size_t basicsize = _PyObject_VAR_SIZE(Py_TYPE(op), nitems);
1385	PyGC_Head *g = AS_GC(op);	1388	PyGC_Head *g = AS_GC(op);
		1389	if (basicsize > PY_SSIZE_T_MAX - sizeof(PyGC_Head))
		1390	return (PyVarObject *)PyErr_NoMemory();
1386	g = (PyGC_Head *)PyObject_REALLOC(g, sizeof(PyGC_Head) + basicsize);	1391	g = (PyGC_Head *)PyObject_REALLOC(g, sizeof(PyGC_Head) + basicsize);
1387	if (g == NULL)	1392	if (g == NULL)
1388	return (PyVarObject *)PyErr_NoMemory();	1393	return (PyVarObject *)PyErr_NoMemory();

Lines 115-121 Link Here

(-)Lib/test/test_strop.py (+19 lines)
115	strop.uppercase	115	strop.uppercase
116	strop.whitespace	116	strop.whitespace
117		117
		118	@test_support.precisionbigmemtest(size=test_support._2G - 1, memuse=5)
		119	def test_stropjoin_huge_list(self, size):
		120	a = "A" * size
		121	try:
		122	r = strop.join([a, a], a)
		123	except OverflowError:
		124	pass
		125	else:
		126	self.assertEquals(len(r), len(a) * 3)
118		127
		128	@test_support.precisionbigmemtest(size=test_support._2G - 1, memuse=1)
		129	def test_stropjoin_huge_tup(self, size):
		130	a = "A" * size
		131	try:
		132	r = strop.join((a, a), a)
		133	except OverflowError:
		134	pass # acceptable on 32-bit
		135	else:
		136	self.assertEquals(len(r), len(a) * 3)
		137
119	transtable = '\000\001\002\003\004\005\006\007\010\011\012\013\014\015\016\017\020\021\022\023\024\025\026\027\030\031\032\033\034\035\036\037 !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`xyzdefghijklmnopqrstuvwxyz{\|}~\177\200\201\202\203\204\205\206\207\210\211\212\213\214\215\216\217\220\221\222\223\224\225\226\227\230\231\232\233\234\235\236\237\240\241\242\243\244\245\246\247\250\251\252\253\254\255\256\257\260\261\262\263\264\265\266\267\270\271\272\273\274\275\276\277\300\301\302\303\304\305\306\307\310\311\312\313\314\315\316\317\320\321\322\323\324\325\326\327\330\331\332\333\334\335\336\337\340\341\342\343\344\345\346\347\350\351\352\353\354\355\356\357\360\361\362\363\364\365\366\367\370\371\372\373\374\375\376\377'	138	transtable = '\000\001\002\003\004\005\006\007\010\011\012\013\014\015\016\017\020\021\022\023\024\025\026\027\030\031\032\033\034\035\036\037 !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`xyzdefghijklmnopqrstuvwxyz{\|}~\177\200\201\202\203\204\205\206\207\210\211\212\213\214\215\216\217\220\221\222\223\224\225\226\227\230\231\232\233\234\235\236\237\240\241\242\243\244\245\246\247\250\251\252\253\254\255\256\257\260\261\262\263\264\265\266\267\270\271\272\273\274\275\276\277\300\301\302\303\304\305\306\307\310\311\312\313\314\315\316\317\320\321\322\323\324\325\326\327\330\331\332\333\334\335\336\337\340\341\342\343\344\345\346\347\350\351\352\353\354\355\356\357\360\361\362\363\364\365\366\367\370\371\372\373\374\375\376\377'
120		139
121		140




from test import test_support
from test.test_support import bigmemtest, _1G, _2G, _4G, precisionbigmemtest

import unittest
import operator

            lpadsize += 1
        self.assertEquals(s[lpadsize:-rpadsize], SUBSTR)
        self.assertEquals(s.strip(), SUBSTR.strip())
    
    @precisionbigmemtest(size=_2G - 1, memuse=1)
    def test_center_unicode(self, size):
        SUBSTR = u' abc def ghi'
        try:
            s = SUBSTR.center(size)
        except OverflowError:
            pass # acceptable on 32-bit
        else:
            self.assertEquals(len(s), size)
            lpadsize = rpadsize = (len(s) - len(SUBSTR)) // 2
            if len(s) % 2:
                lpadsize += 1
            self.assertEquals(s[lpadsize:-rpadsize], SUBSTR)
            self.assertEquals(s.strip(), SUBSTR.strip())
            del s
    
    @bigmemtest(minsize=_2G, memuse=2)
    def test_count(self, size):
        SUBSTR = ' abc def ghi'

    def test_decode(self, size):
        s = '.' * size
        self.assertEquals(len(s.decode('utf-8')), size)
    
    def basic_encode_test(self, size, enc, c=u'.', expectedsize=None):
        if expectedsize is None:
            expectedsize = size
        
        s = c * size
        self.assertEquals(len(s.encode(enc)), expectedsize)

    @bigmemtest(minsize=_2G + 2, memuse=3)
    def test_encode(self, size):
        return self.basic_encode_test(size, 'utf-8')
    
    @precisionbigmemtest(size=_4G / 6 + 2, memuse=2)
    def test_encode_raw_unicode_escape(self, size):
        try:
            return self.basic_encode_test(size, 'raw_unicode_escape')
        except MemoryError:
            pass # acceptable on 32-bit
    
    @precisionbigmemtest(size=_4G / 5 + 70, memuse=3)
    def test_encode_utf7(self, size):
        try:
            return self.basic_encode_test(size, 'utf7')
        except MemoryError:
            pass # acceptable on 32-bit
    
    @precisionbigmemtest(size=_4G / 4 + 5, memuse=6)
    def test_encode_utf32(self, size):
        try:
            return self.basic_encode_test(size, 'utf32', expectedsize=4*size+4)
        except MemoryError:
            pass # acceptable on 32-bit
    
    @precisionbigmemtest(size=_2G-1, memuse=2)
    def test_decodeascii(self, size):
        return self.basic_encode_test(size, 'ascii', c='A')
    
    @precisionbigmemtest(size=_4G / 5, memuse=6+2)
    def test_unicode_repr_oflw(self, size):
        try:
            s = u"\uAAAA"*size
            r = repr(s)
        except MemoryError:
            pass # acceptable on 32-bit
        else:
            self.failUnless(s == eval(r))

    @bigmemtest(minsize=_2G, memuse=2)
    def test_endswith(self, size):

        self.assertEquals(s[-1], "'")
        self.assertEquals(s.count('\\'), size)
        self.assertEquals(s.count('0'), size * 2)
    
    @bigmemtest(minsize=2**32 / 5, memuse=6+2)
    def test_unicode_repr(self, size):
        s = u"\uAAAA" * size
        self.failUnless(len(repr(s)) > size)
    
    # This test is meaningful even with size < 2G, as long as the
    # doubled string is > 2G (but it tests more if both are > 2G :)
    @bigmemtest(minsize=_1G + 2, memuse=3)

    def test_repeat_large(self, size):
        return self.basic_test_repeat(size)

    @bigmemtest(minsize=_1G - 1, memuse=12)
    def test_repeat_large_2(self, size):
        return self.basic_test_repeat(size)
    
    @precisionbigmemtest(size=_1G - 1, memuse=9)
    def test_from_2G_generator(self, size):
        try:
            t = tuple(xrange(size))
        except MemoryError:
            pass # acceptable on 32-bit
        else:
            count = 0
            for item in t:
                self.assertEquals(item, count)
                count += 1
            self.assertEquals(count, size)
    
    @precisionbigmemtest(size=_1G - 25, memuse=9)
    def test_from_almost_2G_generator(self, size):
        try:
            t = tuple(xrange(size))
            count = 0
            for item in t:
                self.assertEquals(item, count)
                count += 1
            self.assertEquals(count, size)
        except MemoryError:
            pass # acceptable, expected on 32-bit

    # Like test_concat, split in two.
    def basic_test_repr(self, size):
        t = (0,) * size

        self.assertEquals(l[:10], [1] * 10)
        self.assertEquals(l[-10:], [5] * 10)

class BufferTest(unittest.TestCase):
    
    @precisionbigmemtest(size=_1G, memuse=4)
    def test_repeat(self, size):
        try:
            b = buffer("AAAA")*size
        except MemoryError:
            pass # acceptable on 32-bit
        else:
            count = 0
            for c in b:
                self.assertEquals(c, 'A')
                count += 1
            self.assertEquals(count, size*4)
    
def test_main():
    test_support.run_unittest(StrTest, TupleTest, ListTest, BufferTest)

# Expected failures (crashers)
# del StrTest.test_center_unicode
del StrTest.test_decodeascii
# del StrTest.test_encode_utf32
# del StrTest.test_encode_utf7
# del StrTest.test_encode_raw_unicode_escape
# 
# del TupleTest.test_from_2G_generator
#
# del BufferTest.test_repeat

if __name__ == '__main__':
    if len(sys.argv) > 1:
        test_support.set_memlimit(sys.argv[1])




use_resources = None     # Flag set to [] by regrtest.py
max_memuse = 0           # Disable bigmem tests (they will still be run with
                         # small sizes, to make sure they work.)
real_max_memuse = 0

# _original_stdout is meant to hold stdout at the time regrtest began.
# This may be "the real" stdout, or IDLE's emulation of stdout, or whatever.

_1M = 1024*1024
_1G = 1024 * _1M
_2G = 2 * _1G
_4G = 4 * _1G

MAX_Py_ssize_t = sys.maxsize

def set_memlimit(limit):
    import re
    global max_memuse
    global real_max_memuse
    sizes = {
        'k': 1024,
        'm': _1M,

    if m is None:
        raise ValueError('Invalid memory limit %r' % (limit,))
    memlimit = int(float(m.group(1)) * sizes[m.group(3).lower()])
    real_max_memuse = memlimit
    if memlimit > MAX_Py_ssize_t:
        memlimit = MAX_Py_ssize_t
    if memlimit < _2G - 1:

        return wrapper
    return decorator

def precisionbigmemtest(size, memuse, overhead=5*_1M):
    def decorator(f):
        def wrapper(self):
            if not real_max_memuse:
                maxsize = 5147
            else:
                maxsize = size
            
                if real_max_memuse and real_max_memuse < maxsize * memuse:
                    if verbose:
                        sys.stderr.write("Skipping %s because of memory "
                                         "constraint\n" % (f.__name__,))
                    return
            
            return f(self, maxsize)
        wrapper.size = size
        wrapper.memuse = memuse
        wrapper.overhead = overhead
        return wrapper
    return decorator

def bigaddrspacetest(f):
    """Decorator for tests that fill the address space."""
    def wrapper(self):

Lines 307-312 Link Here

(-)Lib/test/seq_tests.py (+2 lines)
307	self.assertEqual(id(s), id(s*1))	307	self.assertEqual(id(s), id(s*1))
308		308
309	def test_bigrepeat(self):	309	def test_bigrepeat(self):
		310	import sys
		311	if sys.maxint <= 2147483647:
310	x = self.type2test([0])	312	x = self.type2test([0])
311	x = 2*16	313	x = 2*16
312	self.assertRaises(MemoryError, x.__mul__, 2**16)	314	self.assertRaises(MemoryError, x.__mul__, 2**16)

Return to bug 230640

Lines 977-989 Link Here

(-)Objects/stringobject.c (-1 / +6 lines)
977	return (PyObject *)a;	977	return (PyObject *)a;
978	}	978	}
979	size = Py_SIZE(a) + Py_SIZE(b);	979	size = Py_SIZE(a) + Py_SIZE(b);
980	if (size < 0) {	980	if (Py_SIZE(a) < 0 \|\| Py_SIZE(b) < 0 \|\| Py_SIZE(a) > PY_SSIZE_T_MAX - Py_SIZE(b)) {
981	PyErr_SetString(PyExc_OverflowError,	981	PyErr_SetString(PyExc_OverflowError,
982	"strings are too large to concat");	982	"strings are too large to concat");
983	return NULL;	983	return NULL;
984	}	984	}
985		985
986	/* Inline PyObject_NewVar */	986	/* Inline PyObject_NewVar */
		987	if (size > PY_SSIZE_T_MAX - sizeof(PyStringObject)) {
		988	PyErr_SetString(PyExc_OverflowError,
		989	"strings are too large to concat");
		990	return NULL;
		991	}
987	op = (PyStringObject *)PyObject_MALLOC(sizeof(PyStringObject) + size);	992	op = (PyStringObject *)PyObject_MALLOC(sizeof(PyStringObject) + size);
988	if (op == NULL)	993	if (op == NULL)
989	return PyErr_NoMemory();	994	return PyErr_NoMemory();

Lines 60-70 Link Here

(-)Objects/tupleobject.c (-2 / +3 lines)
60	Py_ssize_t nbytes = size * sizeof(PyObject *);	60	Py_ssize_t nbytes = size * sizeof(PyObject *);
61	/* Check for overflow */	61	/* Check for overflow */
62	if (nbytes / sizeof(PyObject *) != (size_t)size \|\|	62	if (nbytes / sizeof(PyObject *) != (size_t)size \|\|
63	(nbytes += sizeof(PyTupleObject) - sizeof(PyObject *))	63	(nbytes > PY_SSIZE_T_MAX - sizeof(PyTupleObject) - sizeof(PyObject *)))
64	<= 0)
65	{	64	{
66	return PyErr_NoMemory();	65	return PyErr_NoMemory();
67	}	66	}
		67	nbytes += sizeof(PyTupleObject) - sizeof(PyObject *);
		68
68	op = PyObject_GC_NewVar(PyTupleObject, &PyTuple_Type, size);	69	op = PyObject_GC_NewVar(PyTupleObject, &PyTuple_Type, size);
69	if (op == NULL)	70	if (op == NULL)
70	return NULL;	71	return NULL;

Lines 1-5 Link Here

(-)Lib/test/test_bigmem.py (-6 / +123 lines)
1	from test import test_support	1	from test import test_support
2	from test.test_support import bigmemtest, _1G, _2G	2	from test.test_support import bigmemtest, _1G, _2G, _4G, precisionbigmemtest
3		3
4	import unittest	4	import unittest
5	import operator	5	import operator
Lines 53-59 Link Here
53	lpadsize += 1	53	lpadsize += 1
54	self.assertEquals(s[lpadsize:-rpadsize], SUBSTR)	54	self.assertEquals(s[lpadsize:-rpadsize], SUBSTR)
55	self.assertEquals(s.strip(), SUBSTR.strip())	55	self.assertEquals(s.strip(), SUBSTR.strip())
56		56
		57	@precisionbigmemtest(size=_2G - 1, memuse=1)
		58	def test_center_unicode(self, size):
		59	SUBSTR = u' abc def ghi'
		60	try:
		61	s = SUBSTR.center(size)
		62	except OverflowError:
		63	pass # acceptable on 32-bit
		64	else:
		65	self.assertEquals(len(s), size)
		66	lpadsize = rpadsize = (len(s) - len(SUBSTR)) // 2
		67	if len(s) % 2:
		68	lpadsize += 1
		69	self.assertEquals(s[lpadsize:-rpadsize], SUBSTR)
		70	self.assertEquals(s.strip(), SUBSTR.strip())
		71	del s
		72
57	@bigmemtest(minsize=_2G, memuse=2)	73	@bigmemtest(minsize=_2G, memuse=2)
58	def test_count(self, size):	74	def test_count(self, size):
59	SUBSTR = ' abc def ghi'	75	SUBSTR = ' abc def ghi'
Lines 69-79 Link Here
69	def test_decode(self, size):	85	def test_decode(self, size):
70	s = '.' * size	86	s = '.' * size
71	self.assertEquals(len(s.decode('utf-8')), size)	87	self.assertEquals(len(s.decode('utf-8')), size)
		88
		89	def basic_encode_test(self, size, enc, c=u'.', expectedsize=None):
		90	if expectedsize is None:
		91	expectedsize = size
		92
		93	s = c * size
		94	self.assertEquals(len(s.encode(enc)), expectedsize)
72		95
73	@bigmemtest(minsize=_2G + 2, memuse=3)	96	@bigmemtest(minsize=_2G + 2, memuse=3)
74	def test_encode(self, size):	97	def test_encode(self, size):
75	s = u'.' * size	98	return self.basic_encode_test(size, 'utf-8')
76	self.assertEquals(len(s.encode('utf-8')), size)	99
		100	@precisionbigmemtest(size=_4G / 6 + 2, memuse=2)
		101	def test_encode_raw_unicode_escape(self, size):
		102	try:
		103	return self.basic_encode_test(size, 'raw_unicode_escape')
		104	except MemoryError:
		105	pass # acceptable on 32-bit
		106
		107	@precisionbigmemtest(size=_4G / 5 + 70, memuse=3)
		108	def test_encode_utf7(self, size):
		109	try:
		110	return self.basic_encode_test(size, 'utf7')
		111	except MemoryError:
		112	pass # acceptable on 32-bit
		113
		114	@precisionbigmemtest(size=_4G / 4 + 5, memuse=6)
		115	def test_encode_utf32(self, size):
		116	try:
		117	return self.basic_encode_test(size, 'utf32', expectedsize=4*size+4)
		118	except MemoryError:
		119	pass # acceptable on 32-bit
		120
		121	@precisionbigmemtest(size=_2G-1, memuse=2)
		122	def test_decodeascii(self, size):
		123	return self.basic_encode_test(size, 'ascii', c='A')
		124
		125	@precisionbigmemtest(size=_4G / 5, memuse=6+2)
		126	def test_unicode_repr_oflw(self, size):
		127	try:
		128	s = u"\uAAAA"*size
		129	r = repr(s)
		130	except MemoryError:
		131	pass # acceptable on 32-bit
		132	else:
		133	self.failUnless(s == eval(r))
77		134
78	@bigmemtest(minsize=_2G, memuse=2)	135	@bigmemtest(minsize=_2G, memuse=2)
79	def test_endswith(self, size):	136	def test_endswith(self, size):
Lines 458-464 Link Here
458	self.assertEquals(s[-1], "'")	515	self.assertEquals(s[-1], "'")
459	self.assertEquals(s.count('\\'), size)	516	self.assertEquals(s.count('\\'), size)
460	self.assertEquals(s.count('0'), size * 2)	517	self.assertEquals(s.count('0'), size * 2)
461		518
		519	@bigmemtest(minsize=2**32 / 5, memuse=6+2)
		520	def test_unicode_repr(self, size):
		521	s = u"\uAAAA" * size
		522	self.failUnless(len(repr(s)) > size)
		523
462	# This test is meaningful even with size < 2G, as long as the	524	# This test is meaningful even with size < 2G, as long as the
463	# doubled string is > 2G (but it tests more if both are > 2G :)	525	# doubled string is > 2G (but it tests more if both are > 2G :)
464	@bigmemtest(minsize=_1G + 2, memuse=3)	526	@bigmemtest(minsize=_1G + 2, memuse=3)
Lines 642-647 Link Here
642	def test_repeat_large(self, size):	704	def test_repeat_large(self, size):
643	return self.basic_test_repeat(size)	705	return self.basic_test_repeat(size)
644		706
		707	@bigmemtest(minsize=_1G - 1, memuse=12)
		708	def test_repeat_large_2(self, size):
		709	return self.basic_test_repeat(size)
		710
		711	@precisionbigmemtest(size=_1G - 1, memuse=9)
		712	def test_from_2G_generator(self, size):
		713	try:
		714	t = tuple(xrange(size))
		715	except MemoryError:
		716	pass # acceptable on 32-bit
		717	else:
		718	count = 0
		719	for item in t:
		720	self.assertEquals(item, count)
		721	count += 1
		722	self.assertEquals(count, size)
		723
		724	@precisionbigmemtest(size=_1G - 25, memuse=9)
		725	def test_from_almost_2G_generator(self, size):
		726	try:
		727	t = tuple(xrange(size))
		728	count = 0
		729	for item in t:
		730	self.assertEquals(item, count)
		731	count += 1
		732	self.assertEquals(count, size)
		733	except MemoryError:
		734	pass # acceptable, expected on 32-bit
		735
645	# Like test_concat, split in two.	736	# Like test_concat, split in two.
646	def basic_test_repr(self, size):	737	def basic_test_repr(self, size):
647	t = (0,) * size	738	t = (0,) * size
Lines 957-965 Link Here
957	self.assertEquals(l[:10], [1] * 10)	1048	self.assertEquals(l[:10], [1] * 10)
958	self.assertEquals(l[-10:], [5] * 10)	1049	self.assertEquals(l[-10:], [5] * 10)
959		1050
		1051	class BufferTest(unittest.TestCase):
		1052
		1053	@precisionbigmemtest(size=_1G, memuse=4)
		1054	def test_repeat(self, size):
		1055	try:
		1056	b = buffer("AAAA")*size
		1057	except MemoryError:
		1058	pass # acceptable on 32-bit
		1059	else:
		1060	count = 0
		1061	for c in b:
		1062	self.assertEquals(c, 'A')
		1063	count += 1
		1064	self.assertEquals(count, size*4)
		1065
960	def test_main():	1066	def test_main():
961	test_support.run_unittest(StrTest, TupleTest, ListTest)	1067	test_support.run_unittest(StrTest, TupleTest, ListTest, BufferTest)
962		1068
		1069	# Expected failures (crashers)
		1070	# del StrTest.test_center_unicode
		1071	del StrTest.test_decodeascii
		1072	# del StrTest.test_encode_utf32
		1073	# del StrTest.test_encode_utf7
		1074	# del StrTest.test_encode_raw_unicode_escape
		1075	#
		1076	# del TupleTest.test_from_2G_generator
		1077	#
		1078	# del BufferTest.test_repeat
		1079
963	if __name__ == '__main__':	1080	if __name__ == '__main__':
964	if len(sys.argv) > 1:	1081	if len(sys.argv) > 1:
965	test_support.set_memlimit(sys.argv[1])	1082	test_support.set_memlimit(sys.argv[1])

Lines 68-73 Link Here

(-)Lib/test/test_support.py (+25 lines)
68	use_resources = None # Flag set to [] by regrtest.py	68	use_resources = None # Flag set to [] by regrtest.py
69	max_memuse = 0 # Disable bigmem tests (they will still be run with	69	max_memuse = 0 # Disable bigmem tests (they will still be run with
70	# small sizes, to make sure they work.)	70	# small sizes, to make sure they work.)
		71	real_max_memuse = 0
71		72
72	# _original_stdout is meant to hold stdout at the time regrtest began.	73	# _original_stdout is meant to hold stdout at the time regrtest began.
73	# This may be "the real" stdout, or IDLE's emulation of stdout, or whatever.	74	# This may be "the real" stdout, or IDLE's emulation of stdout, or whatever.
Lines 596-607 Link Here
596	_1M = 1024*1024	597	_1M = 1024*1024
597	_1G = 1024 * _1M	598	_1G = 1024 * _1M
598	_2G = 2 * _1G	599	_2G = 2 * _1G
		600	_4G = 4 * _1G
599		601
600	MAX_Py_ssize_t = sys.maxsize	602	MAX_Py_ssize_t = sys.maxsize
601		603
602	def set_memlimit(limit):	604	def set_memlimit(limit):
603	import re	605	import re
604	global max_memuse	606	global max_memuse
		607	global real_max_memuse
605	sizes = {	608	sizes = {
606	'k': 1024,	609	'k': 1024,
607	'm': _1M,	610	'm': _1M,
Lines 613-618 Link Here
613	if m is None:	616	if m is None:
614	raise ValueError('Invalid memory limit %r' % (limit,))	617	raise ValueError('Invalid memory limit %r' % (limit,))
615	memlimit = int(float(m.group(1)) * sizes[m.group(3).lower()])	618	memlimit = int(float(m.group(1)) * sizes[m.group(3).lower()])
		619	real_max_memuse = memlimit
616	if memlimit > MAX_Py_ssize_t:	620	if memlimit > MAX_Py_ssize_t:
617	memlimit = MAX_Py_ssize_t	621	memlimit = MAX_Py_ssize_t
618	if memlimit < _2G - 1:	622	if memlimit < _2G - 1:
Lines 658-663 Link Here
658	return wrapper	662	return wrapper
659	return decorator	663	return decorator
660		664
		665	def precisionbigmemtest(size, memuse, overhead=5*_1M):
		666	def decorator(f):
		667	def wrapper(self):
		668	if not real_max_memuse:
		669	maxsize = 5147
		670	else:
		671	maxsize = size
		672
		673	if real_max_memuse and real_max_memuse < maxsize * memuse:
		674	if verbose:
		675	sys.stderr.write("Skipping %s because of memory "
		676	"constraint\n" % (f.__name__,))
		677	return
		678
		679	return f(self, maxsize)
		680	wrapper.size = size
		681	wrapper.memuse = memuse
		682	wrapper.overhead = overhead
		683	return wrapper
		684	return decorator
		685
661	def bigaddrspacetest(f):	686	def bigaddrspacetest(f):
662	"""Decorator for tests that fill the address space."""	687	"""Decorator for tests that fill the address space."""
663	def wrapper(self):	688	def wrapper(self):