Attachment #176475 for bug #252317

Line Link Here

(-)file_not_specified_in_diff (-88 / +342 lines)
0	-- Include/fileobject.h	0	++ Include/fileobject.h
Lines 25-30 Link Here
25	int f_skipnextlf; /* Skip next \n */	25	int f_skipnextlf; /* Skip next \n */
26	PyObject *f_encoding;	26	PyObject *f_encoding;
27	PyObject weakreflist; / List of weak references */	27	PyObject weakreflist; / List of weak references */
		28	int unlocked_count; /* Num. currently running sections of code
		29	using f_fp with the GIL released. */
28	} PyFileObject;	30	} PyFileObject;
29		31
30	PyAPI_DATA(PyTypeObject) PyFile_Type;	32	PyAPI_DATA(PyTypeObject) PyFile_Type;
Lines 38-43 Link Here
38	PyAPI_FUNC(PyObject ) PyFile_FromFile(FILE , char , char ,	40	PyAPI_FUNC(PyObject ) PyFile_FromFile(FILE , char , char ,
39	int ()(FILE ));	41	int ()(FILE ));
40	PyAPI_FUNC(FILE ) PyFile_AsFile(PyObject );	42	PyAPI_FUNC(FILE ) PyFile_AsFile(PyObject );
		43	PyAPI_FUNC(void) PyFile_IncUseCount(PyFileObject *);
		44	PyAPI_FUNC(void) PyFile_DecUseCount(PyFileObject *);
41	PyAPI_FUNC(PyObject ) PyFile_Name(PyObject );	45	PyAPI_FUNC(PyObject ) PyFile_Name(PyObject );
42	PyAPI_FUNC(PyObject ) PyFile_GetLine(PyObject , int);	46	PyAPI_FUNC(PyObject ) PyFile_GetLine(PyObject , int);
43	PyAPI_FUNC(int) PyFile_WriteObject(PyObject , PyObject , int);	47	PyAPI_FUNC(int) PyFile_WriteObject(PyObject , PyObject , int);
44	-- Lib/test/test_file.py	48	++ Lib/test/test_file.py
Lines 1-9 Link Here
		1	from __future__ import with_statement
		2
1	import sys	3	import sys
2	import os	4	import os
3	import unittest	5	import unittest
		6	import itertools
		7	import time
		8	import threading
4	from array import array	9	from array import array
5	from weakref import proxy	10	from weakref import proxy
6		11
		12	from test import test_support
7	from test.test_support import TESTFN, findfile, run_unittest	13	from test.test_support import TESTFN, findfile, run_unittest
8	from UserList import UserList	14	from UserList import UserList
9		15
Lines 357-368 Link Here
357	finally:	363	finally:
358	sys.stdout = save_stdout	364	sys.stdout = save_stdout
359		365
		366	class FileThreadingTests(unittest.TestCase):
		367	# These tests check the ability to call various methods of file objects
		368	# (including close()) concurrently without crashing the Python interpreter.
		369	# See #815646, #595601
		370
		371	def setUp(self):
		372	self.f = None
		373	self.filename = TESTFN
		374	with open(self.filename, "w") as f:
		375	f.write("\n".join("0123456789"))
		376	self._count_lock = threading.Lock()
		377	self.close_count = 0
		378	self.close_success_count = 0
		379
		380	def tearDown(self):
		381	if self.f:
		382	try:
		383	self.f.close()
		384	except (EnvironmentError, ValueError):
		385	pass
		386	try:
		387	os.remove(self.filename)
		388	except EnvironmentError:
		389	pass
		390
		391	def _create_file(self):
		392	self.f = open(self.filename, "w+")
		393
		394	def _close_file(self):
		395	with self._count_lock:
		396	self.close_count += 1
		397	self.f.close()
		398	with self._count_lock:
		399	self.close_success_count += 1
		400
		401	def _close_and_reopen_file(self):
		402	self._close_file()
		403	# if close raises an exception thats fine, self.f remains valid so
		404	# we don't need to reopen.
		405	self._create_file()
		406
		407	def _run_workers(self, func, nb_workers, duration=0.2):
		408	with self._count_lock:
		409	self.close_count = 0
		410	self.close_success_count = 0
		411	self.do_continue = True
		412	threads = []
		413	try:
		414	for i in range(nb_workers):
		415	t = threading.Thread(target=func)
		416	t.start()
		417	threads.append(t)
		418	for _ in xrange(100):
		419	time.sleep(duration/100)
		420	with self._count_lock:
		421	if self.close_count-self.close_success_count > nb_workers+1:
		422	if test_support.verbose:
		423	print 'Q',
		424	break
		425	time.sleep(duration)
		426	finally:
		427	self.do_continue = False
		428	for t in threads:
		429	t.join()
430
431	def _test_close_open_io(self, io_func, nb_workers=5):
432	def worker():
433	self._create_file()
434	funcs = itertools.cycle((
435	lambda: io_func(),
436	lambda: self._close_and_reopen_file(),
437	))
438	for f in funcs:
439	if not self.do_continue:
440	break
441	try:
442	f()
443	except (IOError, ValueError):
444	pass
445	self._run_workers(worker, nb_workers)
446	if test_support.verbose:
447	# Useful verbose statistics when tuning this test to take
448	# less time to run but still ensuring that its still useful.
449	#
450	# the percent of close calls that raised an error
451	percent = 100. - 100.*self.close_success_count/self.close_count
452	print self.close_count, ('%.4f ' % percent),
453
454	def test_close_open(self):
455	def io_func():
456	pass
457	self._test_close_open_io(io_func)
458
459	def test_close_open_flush(self):
460	def io_func():
461	self.f.flush()
462	self._test_close_open_io(io_func)
463
464	def test_close_open_iter(self):
465	def io_func():
466	list(iter(self.f))
467	self._test_close_open_io(io_func)
468
469	def test_close_open_isatty(self):
470	def io_func():
471	self.f.isatty()
472	self._test_close_open_io(io_func)
473
474	def test_close_open_print(self):
475	def io_func():
476	print >> self.f, ''
477	self._test_close_open_io(io_func)
478
479	def test_close_open_read(self):
480	def io_func():
481	self.f.read(0)
482	self._test_close_open_io(io_func)
483
484	def test_close_open_readinto(self):
485	def io_func():
486	a = array('c', 'xxxxx')
487	self.f.readinto(a)
488	self._test_close_open_io(io_func)
489
490	def test_close_open_readline(self):
491	def io_func():
492	self.f.readline()
493	self._test_close_open_io(io_func)
494
495	def test_close_open_readlines(self):
496	def io_func():
497	self.f.readlines()
498	self._test_close_open_io(io_func)
499
500	def test_close_open_seek(self):
501	def io_func():
502	self.f.seek(0, 0)
503	self._test_close_open_io(io_func)
504
505	def test_close_open_tell(self):
506	def io_func():
507	self.f.tell()
508	self._test_close_open_io(io_func)
509
510	def test_close_open_truncate(self):
511	def io_func():
512	self.f.truncate()
513	self._test_close_open_io(io_func)
514
515	def test_close_open_write(self):
516	def io_func():
517	self.f.write('')
518	self._test_close_open_io(io_func)
519
520	def test_close_open_writelines(self):
521	def io_func():
522	self.f.writelines('')
523	self._test_close_open_io(io_func)
360		524
361	def test_main():	525	def test_main():
362	# Historically, these tests have been sloppy about removing TESTFN.	526	# Historically, these tests have been sloppy about removing TESTFN.
363	# So get rid of it no matter what.	527	# So get rid of it no matter what.
364	try:	528	try:
365	run_unittest(AutoFileTests, OtherFileTests, StdoutTests)	529	run_unittest(AutoFileTests, OtherFileTests, StdoutTests, FileThreadingTests)
366	finally:	530	finally:
367	if os.path.exists(TESTFN):	531	if os.path.exists(TESTFN):
368	os.unlink(TESTFN)	532	os.unlink(TESTFN)
369	-- Misc/NEWS	533	++ Misc/NEWS
Lines 318-323 Link Here
318	Extension Modules	318	Extension Modules
319	-----------------	319	-----------------
320		320
		321	- Issue #815646: Individual file objects may now be used from multiple
		322	threads at once without fear of crashing the Python interpreter. If
		323	file.close() is called while an object is in use by another thread
		324	an IOError exception will be raised and the file will not be closed.
		325
321	- Fix deallocation of array objects when allocation ran out of memory.	326	- Fix deallocation of array objects when allocation ran out of memory.
322	Remove array test case that was incorrect on 64-bit systems.	327	Remove array test case that was incorrect on 64-bit systems.
323		328
324	-- Objects/fileobject.c	329	++ Objects/fileobject.c
Lines 48-53 Link Here
48	#define NEWLINE_LF 2 /* \n newline seen */	48	#define NEWLINE_LF 2 /* \n newline seen */
49	#define NEWLINE_CRLF 4 /* \r\n newline seen */	49	#define NEWLINE_CRLF 4 /* \r\n newline seen */
50		50
		51	/*
		52	* These macros release the GIL while preventing the f_close() function being
		53	* called in the interval between them. For that purpose, a running total of
		54	* the number of currently running unlocked code sections is kept in
		55	* the unlocked_count field of the PyFileObject. The close() method raises
		56	* an IOError if that field is non-zero. See issue #815646, #595601.
		57	*/
		58
		59	#define FILE_BEGIN_ALLOW_THREADS(fobj) \
		60	{ \
		61	fobj->unlocked_count++; \
		62	Py_BEGIN_ALLOW_THREADS
		63
		64	#define FILE_END_ALLOW_THREADS(fobj) \
		65	Py_END_ALLOW_THREADS \
		66	fobj->unlocked_count--; \
		67	assert(fobj->unlocked_count >= 0); \
		68	}
		69
		70	#define FILE_ABORT_ALLOW_THREADS(fobj) \
		71	Py_BLOCK_THREADS \
		72	fobj->unlocked_count--; \
		73	assert(fobj->unlocked_count >= 0);
		74
51	#ifdef __cplusplus	75	#ifdef __cplusplus
52	extern "C" {	76	extern "C" {
53	#endif	77	#endif
Lines 61-66 Link Here
61	return ((PyFileObject *)f)->f_fp;	85	return ((PyFileObject *)f)->f_fp;
62	}	86	}
63		87
		88	void PyFile_IncUseCount(PyFileObject *fobj)
		89	{
		90	fobj->unlocked_count++;
		91	}
		92
		93	void PyFile_DecUseCount(PyFileObject *fobj)
		94	{
		95	fobj->unlocked_count--;
		96	assert(fobj->unlocked_count >= 0);
		97	}
		98
64	PyObject *	99	PyObject *
65	PyFile_Name(PyObject *f)	100	PyFile_Name(PyObject *f)
66	{	101	{
Lines 70-75 Link Here
70	return ((PyFileObject *)f)->f_name;	105	return ((PyFileObject *)f)->f_name;
71	}	106	}
72		107
		108	/* This is a safe wrapper around PyObject_Print to print to the FILE
		109	of a PyFileObject. PyObject_Print releases the GIL but knows nothing
		110	about PyFileObject. */
		111	static int
		112	file_PyObject_Print(PyObject op, PyFileObject f, int flags)
		113	{
		114	int result;
		115	PyFile_IncUseCount(f);
		116	result = PyObject_Print(op, f->f_fp, flags);
		117	PyFile_DecUseCount(f);
		118	return result;
		119	}
		120
73	/* On Unix, fopen will succeed for directories.	121	/* On Unix, fopen will succeed for directories.
74	In Python, there should be no file objects referring to	122	In Python, there should be no file objects referring to
75	directories, so we need a check. */	123	directories, so we need a check. */
Lines 224-243 Link Here
224	PyObject *wmode;	272	PyObject *wmode;
225	wmode = PyUnicode_DecodeASCII(newmode, strlen(newmode), NULL);	273	wmode = PyUnicode_DecodeASCII(newmode, strlen(newmode), NULL);
226	if (f->f_name && wmode) {	274	if (f->f_name && wmode) {
227	Py_BEGIN_ALLOW_THREADS	275	FILE_BEGIN_ALLOW_THREADS(f)
228	/* PyUnicode_AS_UNICODE OK without thread	276	/* PyUnicode_AS_UNICODE OK without thread
229	lock as it is a simple dereference. */	277	lock as it is a simple dereference. */
230	f->f_fp = _wfopen(PyUnicode_AS_UNICODE(f->f_name),	278	f->f_fp = _wfopen(PyUnicode_AS_UNICODE(f->f_name),
231	PyUnicode_AS_UNICODE(wmode));	279	PyUnicode_AS_UNICODE(wmode));
232	Py_END_ALLOW_THREADS	280	FILE_END_ALLOW_THREADS(f)
233	}	281	}
234	Py_XDECREF(wmode);	282	Py_XDECREF(wmode);
235	}	283	}
236	#endif	284	#endif
237	if (NULL == f->f_fp && NULL != name) {	285	if (NULL == f->f_fp && NULL != name) {
238	Py_BEGIN_ALLOW_THREADS	286	FILE_BEGIN_ALLOW_THREADS(f)
239	f->f_fp = fopen(name, newmode);	287	f->f_fp = fopen(name, newmode);
240	Py_END_ALLOW_THREADS	288	FILE_END_ALLOW_THREADS(f)
241	}	289	}
242		290
243	if (f->f_fp == NULL) {	291	if (f->f_fp == NULL) {
Lines 275-280 Link Here
275	return (PyObject *)f;	323	return (PyObject *)f;
276	}	324	}
277		325
		326	static PyObject *
		327	close_the_file(PyFileObject *f)
		328	{
		329	int sts = 0;
		330	int (local_close)(FILE );
		331	FILE *local_fp = f->f_fp;
		332	if (local_fp != NULL) {
		333	local_close = f->f_close;
		334	if (local_close != NULL && f->unlocked_count > 0) {
		335	if (f->ob_refcnt > 0) {
		336	PyErr_SetString(PyExc_IOError,
		337	"close() called during concurrent "
		338	"operation on the same file object.");
		339	} else {
		340	/* This should not happen unless someone is
		341	* carelessly playing with the PyFileObject
		342	* struct fields and/or its associated FILE
		343	* pointer. */
		344	PyErr_SetString(PyExc_SystemError,
		345	"PyFileObject locking error in "
		346	"destructor (refcnt <= 0 at close).");
		347	}
		348	return NULL;
		349	}
		350	/* NULL out the FILE pointer before releasing the GIL, because
		351	* it will not be valid anymore after the close() function is
		352	* called. */
		353	f->f_fp = NULL;
		354	if (local_close != NULL) {
		355	Py_BEGIN_ALLOW_THREADS
		356	errno = 0;
		357	sts = (*local_close)(local_fp);
		358	Py_END_ALLOW_THREADS
		359	if (sts == EOF)
		360	return PyErr_SetFromErrno(PyExc_IOError);
		361	if (sts != 0)
		362	return PyInt_FromLong((long)sts);
		363	}
		364	}
		365	Py_RETURN_NONE;
		366	}
		367
278	PyObject *	368	PyObject *
279	PyFile_FromFile(FILE fp, char name, char mode, int (close)(FILE *))	369	PyFile_FromFile(FILE fp, char name, char mode, int (close)(FILE *))
280	{	370	{
Lines 390-408 Link Here
390	static void	480	static void
391	file_dealloc(PyFileObject *f)	481	file_dealloc(PyFileObject *f)
392	{	482	{
393	int sts = 0;	483	PyObject *ret;
394	if (f->weakreflist != NULL)	484	if (f->weakreflist != NULL)
395	PyObject_ClearWeakRefs((PyObject *) f);	485	PyObject_ClearWeakRefs((PyObject *) f);
396	if (f->f_fp != NULL && f->f_close != NULL) {	486	ret = close_the_file(f);
397	Py_BEGIN_ALLOW_THREADS	487	if (!ret) {
398	sts = (*f->f_close)(f->f_fp);	488	PySys_WriteStderr("close failed in file object destructor:\n");
399	Py_END_ALLOW_THREADS	489	PyErr_Print();
400	if (sts == EOF)	490	}
401	#ifdef HAVE_STRERROR	491	else {
402	PySys_WriteStderr("close failed: [Errno %d] %s\n", errno, strerror(errno));	492	Py_DECREF(ret);
403	#else
404	PySys_WriteStderr("close failed: [Errno %d]\n", errno);
405	#endif
406	}	493	}
407	PyMem_Free(f->f_setbuf);	494	PyMem_Free(f->f_setbuf);
408	Py_XDECREF(f->f_name);	495	Py_XDECREF(f->f_name);
Lines 440-463 Link Here
440	static PyObject *	527	static PyObject *
441	file_close(PyFileObject *f)	528	file_close(PyFileObject *f)
442	{	529	{
443	int sts = 0;	530	PyObject *sts = close_the_file(f);
444	if (f->f_fp != NULL) {
445	if (f->f_close != NULL) {
446	Py_BEGIN_ALLOW_THREADS
447	errno = 0;
448	sts = (*f->f_close)(f->f_fp);
449	Py_END_ALLOW_THREADS
450	}
451	f->f_fp = NULL;
452	}
453	PyMem_Free(f->f_setbuf);	531	PyMem_Free(f->f_setbuf);
454	f->f_setbuf = NULL;	532	f->f_setbuf = NULL;
455	if (sts == EOF)	533	return sts;
456	return PyErr_SetFromErrno(PyExc_IOError);
457	if (sts != 0)
458	return PyInt_FromLong((long)sts);
459	Py_INCREF(Py_None);
460	return Py_None;
461	}	534	}
462		535
463		536
Lines 561-570 Link Here
561	if (PyErr_Occurred())	634	if (PyErr_Occurred())
562	return NULL;	635	return NULL;
563		636
564	Py_BEGIN_ALLOW_THREADS	637	FILE_BEGIN_ALLOW_THREADS(f)
565	errno = 0;	638	errno = 0;
566	ret = _portable_fseek(f->f_fp, offset, whence);	639	ret = _portable_fseek(f->f_fp, offset, whence);
567	Py_END_ALLOW_THREADS	640	FILE_END_ALLOW_THREADS(f)
568		641
569	if (ret != 0) {	642	if (ret != 0) {
570	PyErr_SetFromErrno(PyExc_IOError);	643	PyErr_SetFromErrno(PyExc_IOError);
Lines 598-607 Link Here
598	* then at least on Windows). The easiest thing is to capture	671	* then at least on Windows). The easiest thing is to capture
599	* current pos now and seek back to it at the end.	672	* current pos now and seek back to it at the end.
600	*/	673	*/
601	Py_BEGIN_ALLOW_THREADS	674	FILE_BEGIN_ALLOW_THREADS(f)
602	errno = 0;	675	errno = 0;
603	initialpos = _portable_ftell(f->f_fp);	676	initialpos = _portable_ftell(f->f_fp);
604	Py_END_ALLOW_THREADS	677	FILE_END_ALLOW_THREADS(f)
605	if (initialpos == -1)	678	if (initialpos == -1)
606	goto onioerror;	679	goto onioerror;
607		680
Lines 626-635 Link Here
626	* I/O, and a flush may be necessary to synch both platform views	699	* I/O, and a flush may be necessary to synch both platform views
627	* of the current file state.	700	* of the current file state.
628	*/	701	*/
629	Py_BEGIN_ALLOW_THREADS	702	FILE_BEGIN_ALLOW_THREADS(f)
630	errno = 0;	703	errno = 0;
631	ret = fflush(f->f_fp);	704	ret = fflush(f->f_fp);
632	Py_END_ALLOW_THREADS	705	FILE_END_ALLOW_THREADS(f)
633	if (ret != 0)	706	if (ret != 0)
634	goto onioerror;	707	goto onioerror;
635		708
Lines 640-654 Link Here
640	HANDLE hFile;	713	HANDLE hFile;
641		714
642	/* Have to move current pos to desired endpoint on Windows. */	715	/* Have to move current pos to desired endpoint on Windows. */
643	Py_BEGIN_ALLOW_THREADS	716	FILE_BEGIN_ALLOW_THREADS(f)
644	errno = 0;	717	errno = 0;
645	ret = _portable_fseek(f->f_fp, newsize, SEEK_SET) != 0;	718	ret = _portable_fseek(f->f_fp, newsize, SEEK_SET) != 0;
646	Py_END_ALLOW_THREADS	719	FILE_END_ALLOW_THREADS(f)
647	if (ret)	720	if (ret)
648	goto onioerror;	721	goto onioerror;
649		722
650	/* Truncate. Note that this may grow the file! */	723	/* Truncate. Note that this may grow the file! */
651	Py_BEGIN_ALLOW_THREADS	724	FILE_BEGIN_ALLOW_THREADS(f)
652	errno = 0;	725	errno = 0;
653	hFile = (HANDLE)_get_osfhandle(fileno(f->f_fp));	726	hFile = (HANDLE)_get_osfhandle(fileno(f->f_fp));
654	ret = hFile == (HANDLE)-1;	727	ret = hFile == (HANDLE)-1;
Lines 657-680 Link Here
657	if (ret)	730	if (ret)
658	errno = EACCES;	731	errno = EACCES;
659	}	732	}
660	Py_END_ALLOW_THREADS	733	FILE_END_ALLOW_THREADS(f)
661	if (ret)	734	if (ret)
662	goto onioerror;	735	goto onioerror;
663	}	736	}
664	#else	737	#else
665	Py_BEGIN_ALLOW_THREADS	738	FILE_BEGIN_ALLOW_THREADS(f)
666	errno = 0;	739	errno = 0;
667	ret = ftruncate(fileno(f->f_fp), newsize);	740	ret = ftruncate(fileno(f->f_fp), newsize);
668	Py_END_ALLOW_THREADS	741	FILE_END_ALLOW_THREADS(f)
669	if (ret != 0)	742	if (ret != 0)
670	goto onioerror;	743	goto onioerror;
671	#endif /* !MS_WINDOWS */	744	#endif /* !MS_WINDOWS */
672		745
673	/* Restore original file position. */	746	/* Restore original file position. */
674	Py_BEGIN_ALLOW_THREADS	747	FILE_BEGIN_ALLOW_THREADS(f)
675	errno = 0;	748	errno = 0;
676	ret = _portable_fseek(f->f_fp, initialpos, SEEK_SET) != 0;	749	ret = _portable_fseek(f->f_fp, initialpos, SEEK_SET) != 0;
677	Py_END_ALLOW_THREADS	750	FILE_END_ALLOW_THREADS(f)
678	if (ret)	751	if (ret)
679	goto onioerror;	752	goto onioerror;
680		753
Lines 695-704 Link Here
695		768
696	if (f->f_fp == NULL)	769	if (f->f_fp == NULL)
697	return err_closed();	770	return err_closed();
698	Py_BEGIN_ALLOW_THREADS	771	FILE_BEGIN_ALLOW_THREADS(f)
699	errno = 0;	772	errno = 0;
700	pos = _portable_ftell(f->f_fp);	773	pos = _portable_ftell(f->f_fp);
701	Py_END_ALLOW_THREADS	774	FILE_END_ALLOW_THREADS(f)
		775
702	if (pos == -1) {	776	if (pos == -1) {
703	PyErr_SetFromErrno(PyExc_IOError);	777	PyErr_SetFromErrno(PyExc_IOError);
704	clearerr(f->f_fp);	778	clearerr(f->f_fp);
Lines 734-743 Link Here
734		808
735	if (f->f_fp == NULL)	809	if (f->f_fp == NULL)
736	return err_closed();	810	return err_closed();
737	Py_BEGIN_ALLOW_THREADS	811	FILE_BEGIN_ALLOW_THREADS(f)
738	errno = 0;	812	errno = 0;
739	res = fflush(f->f_fp);	813	res = fflush(f->f_fp);
740	Py_END_ALLOW_THREADS	814	FILE_END_ALLOW_THREADS(f)
741	if (res != 0) {	815	if (res != 0) {
742	PyErr_SetFromErrno(PyExc_IOError);	816	PyErr_SetFromErrno(PyExc_IOError);
743	clearerr(f->f_fp);	817	clearerr(f->f_fp);
Lines 753-761 Link Here
753	long res;	827	long res;
754	if (f->f_fp == NULL)	828	if (f->f_fp == NULL)
755	return err_closed();	829	return err_closed();
756	Py_BEGIN_ALLOW_THREADS	830	FILE_BEGIN_ALLOW_THREADS(f)
757	res = isatty((int)fileno(f->f_fp));	831	res = isatty((int)fileno(f->f_fp));
758	Py_END_ALLOW_THREADS	832	FILE_END_ALLOW_THREADS(f)
759	return PyBool_FromLong(res);	833	return PyBool_FromLong(res);
760	}	834	}
761		835
Lines 855-865 Link Here
855	return NULL;	929	return NULL;
856	bytesread = 0;	930	bytesread = 0;
857	for (;;) {	931	for (;;) {
858	Py_BEGIN_ALLOW_THREADS	932	FILE_BEGIN_ALLOW_THREADS(f)
859	errno = 0;	933	errno = 0;
860	chunksize = Py_UniversalNewlineFread(BUF(v) + bytesread,	934	chunksize = Py_UniversalNewlineFread(BUF(v) + bytesread,
861	buffersize - bytesread, f->f_fp, (PyObject *)f);	935	buffersize - bytesread, f->f_fp, (PyObject *)f);
862	Py_END_ALLOW_THREADS	936	FILE_END_ALLOW_THREADS(f)
863	if (chunksize == 0) {	937	if (chunksize == 0) {
864	if (!ferror(f->f_fp))	938	if (!ferror(f->f_fp))
865	break;	939	break;
Lines 911-921 Link Here
911	return NULL;	985	return NULL;
912	ndone = 0;	986	ndone = 0;
913	while (ntodo > 0) {	987	while (ntodo > 0) {
914	Py_BEGIN_ALLOW_THREADS	988	FILE_BEGIN_ALLOW_THREADS(f)
915	errno = 0;	989	errno = 0;
916	nnow = Py_UniversalNewlineFread(ptr+ndone, ntodo, f->f_fp,	990	nnow = Py_UniversalNewlineFread(ptr+ndone, ntodo, f->f_fp,
917	(PyObject *)f);	991	(PyObject *)f);
918	Py_END_ALLOW_THREADS	992	FILE_END_ALLOW_THREADS(f)
919	if (nnow == 0) {	993	if (nnow == 0) {
920	if (!ferror(f->f_fp))	994	if (!ferror(f->f_fp))
921	break;	995	break;
Lines 980-986 Link Here
980		1054
981	#ifdef USE_FGETS_IN_GETLINE	1055	#ifdef USE_FGETS_IN_GETLINE
982	static PyObject*	1056	static PyObject*
983	getline_via_fgets(FILE *fp)	1057	getline_via_fgets(PyFileObject f, FILE fp)
984	{	1058	{
985	/* INITBUFSIZE is the maximum line length that lets us get away with the fast	1059	/* INITBUFSIZE is the maximum line length that lets us get away with the fast
986	* no-realloc, one-fgets()-call path. Boosting it isn't free, because we have	1060	* no-realloc, one-fgets()-call path. Boosting it isn't free, because we have
Lines 1013-1025 Link Here
1013	total_v_size = INITBUFSIZE; /* start small and pray */	1087	total_v_size = INITBUFSIZE; /* start small and pray */
1014	pvfree = buf;	1088	pvfree = buf;
1015	for (;;) {	1089	for (;;) {
1016	Py_BEGIN_ALLOW_THREADS	1090	FILE_BEGIN_ALLOW_THREADS(f)
1017	pvend = buf + total_v_size;	1091	pvend = buf + total_v_size;
1018	nfree = pvend - pvfree;	1092	nfree = pvend - pvfree;
1019	memset(pvfree, '\n', nfree);	1093	memset(pvfree, '\n', nfree);
1020	assert(nfree < INT_MAX); /* Should be atmost MAXBUFSIZE */	1094	assert(nfree < INT_MAX); /* Should be atmost MAXBUFSIZE */
1021	p = fgets(pvfree, (int)nfree, fp);	1095	p = fgets(pvfree, (int)nfree, fp);
1022	Py_END_ALLOW_THREADS	1096	FILE_END_ALLOW_THREADS(f)
1023		1097
1024	if (p == NULL) {	1098	if (p == NULL) {
1025	clearerr(fp);	1099	clearerr(fp);
Lines 1088-1100 Link Here
1088	* the code above for detailed comments about the logic.	1162	* the code above for detailed comments about the logic.
1089	*/	1163	*/
1090	for (;;) {	1164	for (;;) {
1091	Py_BEGIN_ALLOW_THREADS	1165	FILE_BEGIN_ALLOW_THREADS(f)
1092	pvend = BUF(v) + total_v_size;	1166	pvend = BUF(v) + total_v_size;
1093	nfree = pvend - pvfree;	1167	nfree = pvend - pvfree;
1094	memset(pvfree, '\n', nfree);	1168	memset(pvfree, '\n', nfree);
1095	assert(nfree < INT_MAX);	1169	assert(nfree < INT_MAX);
1096	p = fgets(pvfree, (int)nfree, fp);	1170	p = fgets(pvfree, (int)nfree, fp);
1097	Py_END_ALLOW_THREADS	1171	FILE_END_ALLOW_THREADS(f)
1098		1172
1099	if (p == NULL) {	1173	if (p == NULL) {
1100	clearerr(fp);	1174	clearerr(fp);
Lines 1165-1171 Link Here
1165		1239
1166	#if defined(USE_FGETS_IN_GETLINE)	1240	#if defined(USE_FGETS_IN_GETLINE)
1167	if (n <= 0 && !univ_newline )	1241	if (n <= 0 && !univ_newline )
1168	return getline_via_fgets(fp);	1242	return getline_via_fgets(f, fp);
1169	#endif	1243	#endif
1170	total_v_size = n > 0 ? n : 100;	1244	total_v_size = n > 0 ? n : 100;
1171	v = PyString_FromStringAndSize((char *)NULL, total_v_size);	1245	v = PyString_FromStringAndSize((char *)NULL, total_v_size);
Lines 1175-1181 Link Here
1175	end = buf + total_v_size;	1249	end = buf + total_v_size;
1176		1250
1177	for (;;) {	1251	for (;;) {
1178	Py_BEGIN_ALLOW_THREADS	1252	FILE_BEGIN_ALLOW_THREADS(f)
1179	FLOCKFILE(fp);	1253	FLOCKFILE(fp);
1180	if (univ_newline) {	1254	if (univ_newline) {
1181	c = 'x'; /* Shut up gcc warning */	1255	c = 'x'; /* Shut up gcc warning */
Lines 1210-1216 Link Here
1210	buf != end)	1284	buf != end)
1211	;	1285	;
1212	FUNLOCKFILE(fp);	1286	FUNLOCKFILE(fp);
1213	Py_END_ALLOW_THREADS	1287	FILE_END_ALLOW_THREADS(f)
1214	f->f_newlinetypes = newlinetypes;	1288	f->f_newlinetypes = newlinetypes;
1215	f->f_skipnextlf = skipnextlf;	1289	f->f_skipnextlf = skipnextlf;
1216	if (c == '\n')	1290	if (c == '\n')
Lines 1375-1381 Link Here
1375	file_readlines(PyFileObject f, PyObject args)	1449	file_readlines(PyFileObject f, PyObject args)
1376	{	1450	{
1377	long sizehint = 0;	1451	long sizehint = 0;
1378	PyObject *list;	1452	PyObject *list = NULL;
1379	PyObject *line;	1453	PyObject *line;
1380	char small_buffer[SMALLCHUNK];	1454	char small_buffer[SMALLCHUNK];
1381	char *buffer = small_buffer;	1455	char *buffer = small_buffer;
Lines 1403-1413 Link Here
1403	if (shortread)	1477	if (shortread)
1404	nread = 0;	1478	nread = 0;
1405	else {	1479	else {
1406	Py_BEGIN_ALLOW_THREADS	1480	FILE_BEGIN_ALLOW_THREADS(f)
1407	errno = 0;	1481	errno = 0;
1408	nread = Py_UniversalNewlineFread(buffer+nfilled,	1482	nread = Py_UniversalNewlineFread(buffer+nfilled,
1409	buffersize-nfilled, f->f_fp, (PyObject *)f);	1483	buffersize-nfilled, f->f_fp, (PyObject *)f);
1410	Py_END_ALLOW_THREADS	1484	FILE_END_ALLOW_THREADS(f)
1411	shortread = (nread < buffersize-nfilled);	1485	shortread = (nread < buffersize-nfilled);
1412	}	1486	}
1413	if (nread == 0) {	1487	if (nread == 0) {
Lines 1416-1425 Link Here
1416	break;	1490	break;
1417	PyErr_SetFromErrno(PyExc_IOError);	1491	PyErr_SetFromErrno(PyExc_IOError);
1418	clearerr(f->f_fp);	1492	clearerr(f->f_fp);
1419	error:	1493	goto error;
1420	Py_DECREF(list);
1421	list = NULL;
1422	goto cleanup;
1423	}	1494	}
1424	totalread += nread;	1495	totalread += nread;
1425	p = (char *)memchr(buffer+nfilled, '\n', nread);	1496	p = (char *)memchr(buffer+nfilled, '\n', nread);
Lines 1493-1501 Link Here
1493	if (err != 0)	1564	if (err != 0)
1494	goto error;	1565	goto error;
1495	}	1566	}
1496	cleanup:	1567
		1568	cleanup:
1497	Py_XDECREF(big_buffer);	1569	Py_XDECREF(big_buffer);
1498	return list;	1570	return list;
		1571
		1572	error:
		1573	Py_CLEAR(list);
		1574	goto cleanup;
1499	}	1575	}
1500		1576
1501	static PyObject *	1577	static PyObject *
Lines 1508-1517 Link Here
1508	if (!PyArg_ParseTuple(args, f->f_binary ? "s#" : "t#", &s, &n))	1584	if (!PyArg_ParseTuple(args, f->f_binary ? "s#" : "t#", &s, &n))
1509	return NULL;	1585	return NULL;
1510	f->f_softspace = 0;	1586	f->f_softspace = 0;
1511	Py_BEGIN_ALLOW_THREADS	1587	FILE_BEGIN_ALLOW_THREADS(f)
1512	errno = 0;	1588	errno = 0;
1513	n2 = fwrite(s, 1, n, f->f_fp);	1589	n2 = fwrite(s, 1, n, f->f_fp);
1514	Py_END_ALLOW_THREADS	1590	FILE_END_ALLOW_THREADS(f)
1515	if (n2 != n) {	1591	if (n2 != n) {
1516	PyErr_SetFromErrno(PyExc_IOError);	1592	PyErr_SetFromErrno(PyExc_IOError);
1517	clearerr(f->f_fp);	1593	clearerr(f->f_fp);
Lines 1609-1615 Link Here
1609		1685
1610	/* Since we are releasing the global lock, the	1686	/* Since we are releasing the global lock, the
1611	following code may not execute Python code. */	1687	following code may not execute Python code. */
1612	Py_BEGIN_ALLOW_THREADS	1688	FILE_BEGIN_ALLOW_THREADS(f)
1613	f->f_softspace = 0;	1689	f->f_softspace = 0;
1614	errno = 0;	1690	errno = 0;
1615	for (i = 0; i < j; i++) {	1691	for (i = 0; i < j; i++) {
Lines 1618-1630 Link Here
1618	nwritten = fwrite(PyString_AS_STRING(line),	1694	nwritten = fwrite(PyString_AS_STRING(line),
1619	1, len, f->f_fp);	1695	1, len, f->f_fp);
1620	if (nwritten != len) {	1696	if (nwritten != len) {
1621	Py_BLOCK_THREADS	1697	FILE_ABORT_ALLOW_THREADS(f)
1622	PyErr_SetFromErrno(PyExc_IOError);	1698	PyErr_SetFromErrno(PyExc_IOError);
1623	clearerr(f->f_fp);	1699	clearerr(f->f_fp);
1624	goto error;	1700	goto error;
1625	}	1701	}
1626	}	1702	}
1627	Py_END_ALLOW_THREADS	1703	FILE_END_ALLOW_THREADS(f)
1628		1704
1629	if (j < CHUNKSIZE)	1705	if (j < CHUNKSIZE)
1630	break;	1706	break;
Lines 1857-1867 Link Here
1857	PyErr_NoMemory();	1933	PyErr_NoMemory();
1858	return -1;	1934	return -1;
1859	}	1935	}
1860	Py_BEGIN_ALLOW_THREADS	1936	FILE_BEGIN_ALLOW_THREADS(f)
1861	errno = 0;	1937	errno = 0;
1862	chunksize = Py_UniversalNewlineFread(	1938	chunksize = Py_UniversalNewlineFread(
1863	f->f_buf, bufsize, f->f_fp, (PyObject *)f);	1939	f->f_buf, bufsize, f->f_fp, (PyObject *)f);
1864	Py_END_ALLOW_THREADS	1940	FILE_END_ALLOW_THREADS(f)
1865	if (chunksize == 0) {	1941	if (chunksize == 0) {
1866	if (ferror(f->f_fp)) {	1942	if (ferror(f->f_fp)) {
1867	PyErr_SetFromErrno(PyExc_IOError);	1943	PyErr_SetFromErrno(PyExc_IOError);
Lines 1970-1975 Link Here
1970	Py_INCREF(Py_None);	2046	Py_INCREF(Py_None);
1971	((PyFileObject *)self)->f_encoding = Py_None;	2047	((PyFileObject *)self)->f_encoding = Py_None;
1972	((PyFileObject *)self)->weakreflist = NULL;	2048	((PyFileObject *)self)->weakreflist = NULL;
		2049	((PyFileObject *)self)->unlocked_count = 0;
1973	}	2050	}
1974	return self;	2051	return self;
1975	}	2052	}
Lines 2157-2168 Link Here
2157	return -1;	2234	return -1;
2158	}	2235	}
2159	else if (PyFile_Check(f)) {	2236	else if (PyFile_Check(f)) {
2160	FILE *fp = PyFile_AsFile(f);	2237	PyFileObject fobj = (PyFileObject ) f;
2161	#ifdef Py_USING_UNICODE	2238	#ifdef Py_USING_UNICODE
2162	PyObject enc = ((PyFileObject)f)->f_encoding;	2239	PyObject *enc = fobj->f_encoding;
2163	int result;	2240	int result;
2164	#endif	2241	#endif
2165	if (fp == NULL) {	2242	if (fobj->f_fp == NULL) {
2166	err_closed();	2243	err_closed();
2167	return -1;	2244	return -1;
2168	}	2245	}
Lines 2177-2187 Link Here
2177	value = v;	2254	value = v;
2178	Py_INCREF(value);	2255	Py_INCREF(value);
2179	}	2256	}
2180	result = PyObject_Print(value, fp, flags);	2257	result = file_PyObject_Print(value, fobj, flags);
2181	Py_DECREF(value);	2258	Py_DECREF(value);
2182	return result;	2259	return result;
2183	#else	2260	#else
2184	return PyObject_Print(v, fp, flags);	2261	return file_PyObject_Print(v, fobj, flags);
2185	#endif	2262	#endif
2186	}	2263	}
2187	writer = PyObject_GetAttrString(f, "write");	2264	writer = PyObject_GetAttrString(f, "write");
Lines 2219-2224 Link Here
2219	int	2296	int
2220	PyFile_WriteString(const char s, PyObject f)	2297	PyFile_WriteString(const char s, PyObject f)
2221	{	2298	{
		2299
2222	if (f == NULL) {	2300	if (f == NULL) {
2223	/* Should be caused by a pre-existing error */	2301	/* Should be caused by a pre-existing error */
2224	if (!PyErr_Occurred())	2302	if (!PyErr_Occurred())
Lines 2227-2238 Link Here
2227	return -1;	2305	return -1;
2228	}	2306	}
2229	else if (PyFile_Check(f)) {	2307	else if (PyFile_Check(f)) {
		2308	PyFileObject fobj = (PyFileObject ) f;
2230	FILE *fp = PyFile_AsFile(f);	2309	FILE *fp = PyFile_AsFile(f);
2231	if (fp == NULL) {	2310	if (fp == NULL) {
2232	err_closed();	2311	err_closed();
2233	return -1;	2312	return -1;
2234	}	2313	}
		2314	FILE_BEGIN_ALLOW_THREADS(fobj)
2235	fputs(s, fp);	2315	fputs(s, fp);
		2316	FILE_END_ALLOW_THREADS(fobj)
2236	return 0;	2317	return 0;
2237	}	2318	}
2238	else if (!PyErr_Occurred()) {	2319	else if (!PyErr_Occurred()) {

Return to bug 252317