1 /* File object implementation */
3 #define PY_SSIZE_T_CLEAN
5 #include "structmember.h"
7 #ifdef HAVE_SYS_TYPES_H
9 #endif /* HAVE_SYS_TYPES_H */
12 #define fileno _fileno
13 /* can simulate truncate with Win32 API functions; see file_truncate */
14 #define HAVE_FTRUNCATE
15 #define WIN32_LEAN_AND_MEAN
19 #if defined(PYOS_OS2) && defined(PYCC_GCC)
23 #define BUF(v) PyString_AS_STRING((PyStringObject *)v)
29 #ifdef HAVE_GETC_UNLOCKED
30 #define GETC(f) getc_unlocked(f)
31 #define FLOCKFILE(f) flockfile(f)
32 #define FUNLOCKFILE(f) funlockfile(f)
34 #define GETC(f) getc(f)
36 #define FUNLOCKFILE(f)
39 /* Bits in f_newlinetypes */
40 #define NEWLINE_UNKNOWN 0 /* No newline seen, yet */
41 #define NEWLINE_CR 1 /* \r newline seen */
42 #define NEWLINE_LF 2 /* \n newline seen */
43 #define NEWLINE_CRLF 4 /* \r\n newline seen */
46 * These macros release the GIL while preventing the f_close() function being
47 * called in the interval between them. For that purpose, a running total of
48 * the number of currently running unlocked code sections is kept in
49 * the unlocked_count field of the PyFileObject. The close() method raises
50 * an IOError if that field is non-zero. See issue #815646, #595601.
53 #define FILE_BEGIN_ALLOW_THREADS(fobj) \
55 fobj->unlocked_count++; \
56 Py_BEGIN_ALLOW_THREADS
58 #define FILE_END_ALLOW_THREADS(fobj) \
59 Py_END_ALLOW_THREADS \
60 fobj->unlocked_count--; \
61 assert(fobj->unlocked_count >= 0); \
64 #define FILE_ABORT_ALLOW_THREADS(fobj) \
66 fobj->unlocked_count--; \
67 assert(fobj->unlocked_count >= 0);
74 PyFile_AsFile(PyObject *f)
76 if (f == NULL || !PyFile_Check(f))
79 return ((PyFileObject *)f)->f_fp;
82 void PyFile_IncUseCount(PyFileObject *fobj)
84 fobj->unlocked_count++;
87 void PyFile_DecUseCount(PyFileObject *fobj)
89 fobj->unlocked_count--;
90 assert(fobj->unlocked_count >= 0);
94 PyFile_Name(PyObject *f)
96 if (f == NULL || !PyFile_Check(f))
99 return ((PyFileObject *)f)->f_name;
102 /* This is a safe wrapper around PyObject_Print to print to the FILE
103 of a PyFileObject. PyObject_Print releases the GIL but knows nothing
104 about PyFileObject. */
106 file_PyObject_Print(PyObject *op, PyFileObject *f, int flags)
109 PyFile_IncUseCount(f);
110 result = PyObject_Print(op, f->f_fp, flags);
111 PyFile_DecUseCount(f);
115 /* On Unix, fopen will succeed for directories.
116 In Python, there should be no file objects referring to
117 directories, so we need a check. */
120 dircheck(PyFileObject* f)
122 #if defined(HAVE_FSTAT) && defined(S_IFDIR) && defined(EISDIR)
126 if (fstat(fileno(f->f_fp), &buf) == 0 &&
127 S_ISDIR(buf.st_mode)) {
128 char *msg = strerror(EISDIR);
129 PyObject *exc = PyObject_CallFunction(PyExc_IOError, "(isO)",
130 EISDIR, msg, f->f_name);
131 PyErr_SetObject(PyExc_IOError, exc);
141 fill_file_fields(PyFileObject *f, FILE *fp, PyObject *name, char *mode,
142 int (*close)(FILE *))
144 assert(name != NULL);
146 assert(PyFile_Check(f));
147 assert(f->f_fp == NULL);
149 Py_DECREF(f->f_name);
150 Py_DECREF(f->f_mode);
151 Py_DECREF(f->f_encoding);
152 Py_DECREF(f->f_errors);
157 f->f_mode = PyString_FromString(mode);
161 f->f_binary = strchr(mode,'b') != NULL;
163 f->f_univ_newline = (strchr(mode, 'U') != NULL);
164 f->f_newlinetypes = NEWLINE_UNKNOWN;
167 f->f_encoding = Py_None;
169 f->f_errors = Py_None;
170 f->readable = f->writable = 0;
171 if (strchr(mode, 'r') != NULL || f->f_univ_newline)
173 if (strchr(mode, 'w') != NULL || strchr(mode, 'a') != NULL)
175 if (strchr(mode, '+') != NULL)
176 f->readable = f->writable = 1;
178 if (f->f_mode == NULL)
182 return (PyObject *) f;
185 #if defined _MSC_VER && _MSC_VER >= 1400 && defined(__STDC_SECURE_LIB__)
186 #define Py_VERIFY_WINNT
187 /* The CRT on windows compiled with Visual Studio 2005 and higher may
188 * assert if given invalid mode strings. This is all fine and well
189 * in static languages like C where the mode string is typcially hard
190 * coded. But in Python, were we pass in the mode string from the user,
191 * we need to verify it first manually
193 static int _PyVerify_Mode_WINNT(const char *mode)
195 /* See if mode string is valid on Windows to avoid hard assertions */
196 /* remove leading spacese */
202 while(*mode == ' ') /* strip initial spaces */
204 if (!strchr("rwa", *mode)) /* must start with one of these */
207 if (*mode == ' ' || *mode == 'N') /* ignore spaces and N */
209 s = "+TD"; /* each of this can appear only once */
210 c = strchr(s, *mode);
213 if (singles & (1<<idx))
218 s = "btcnSR"; /* only one of each letter in the pairs allowed */
219 c = strchr(s, *mode);
221 ptrdiff_t idx = (s-c)/2;
222 if (pairs & (1<<idx))
231 return 0; /* found an invalid char */
235 char *e[] = {"UTF-8", "UTF-16LE", "UNICODE"};
239 if (strncmp(mode, "ccs", 3))
248 for(encoding = 0; encoding<_countof(e); ++encoding) {
249 size_t l = strlen(e[encoding]);
250 if (!strncmp(mode, e[encoding], l)) {
251 mode += l; /* found a valid encoding */
255 if (encoding == _countof(e))
258 /* skip trailing spaces */
262 return *mode == '\0'; /* must be at the end of the string */
266 /* check for known incorrect mode strings - problem is, platforms are
267 free to accept any mode characters they like and are supposed to
268 ignore stuff they don't understand... write or append mode with
269 universal newline support is expressly forbidden by PEP 278.
270 Additionally, remove the 'U' from the mode string as platforms
271 won't know what it is. Non-zero return signals an exception */
273 _PyFile_SanitizeMode(char *mode)
276 size_t len = strlen(mode);
279 PyErr_SetString(PyExc_ValueError, "empty mode string");
283 upos = strchr(mode, 'U');
285 memmove(upos, upos+1, len-(upos-mode)); /* incl null char */
287 if (mode[0] == 'w' || mode[0] == 'a') {
288 PyErr_Format(PyExc_ValueError, "universal newline "
289 "mode can only be used with modes "
290 "starting with 'r'");
294 if (mode[0] != 'r') {
295 memmove(mode+1, mode, strlen(mode)+1);
299 if (!strchr(mode, 'b')) {
300 memmove(mode+2, mode+1, strlen(mode));
303 } else if (mode[0] != 'r' && mode[0] != 'w' && mode[0] != 'a') {
304 PyErr_Format(PyExc_ValueError, "mode string must begin with "
305 "one of 'r', 'w', 'a' or 'U', not '%.200s'", mode);
308 #ifdef Py_VERIFY_WINNT
309 /* additional checks on NT with visual studio 2005 and higher */
310 if (!_PyVerify_Mode_WINNT(mode)) {
311 PyErr_Format(PyExc_ValueError, "Invalid mode ('%.50s')", mode);
319 open_the_file(PyFileObject *f, char *name, char *mode)
323 assert(PyFile_Check(f));
325 /* windows ignores the passed name in order to support Unicode */
326 assert(f->f_name != NULL);
328 assert(name != NULL);
330 assert(mode != NULL);
331 assert(f->f_fp == NULL);
333 /* probably need to replace 'U' by 'rb' */
334 newmode = PyMem_MALLOC(strlen(mode) + 3);
339 strcpy(newmode, mode);
341 if (_PyFile_SanitizeMode(newmode)) {
346 /* rexec.py can't stop a user from getting the file() constructor --
347 all they have to do is get *any* file object f, and then do
348 type(f). Here we prevent them from doing damage with it. */
349 if (PyEval_GetRestricted()) {
350 PyErr_SetString(PyExc_IOError,
351 "file() constructor not accessible in restricted mode");
358 if (PyUnicode_Check(f->f_name)) {
360 wmode = PyUnicode_DecodeASCII(newmode, strlen(newmode), NULL);
361 if (f->f_name && wmode) {
362 FILE_BEGIN_ALLOW_THREADS(f)
363 /* PyUnicode_AS_UNICODE OK without thread
364 lock as it is a simple dereference. */
365 f->f_fp = _wfopen(PyUnicode_AS_UNICODE(f->f_name),
366 PyUnicode_AS_UNICODE(wmode));
367 FILE_END_ALLOW_THREADS(f)
372 if (NULL == f->f_fp && NULL != name) {
373 FILE_BEGIN_ALLOW_THREADS(f)
374 f->f_fp = fopen(name, newmode);
375 FILE_END_ALLOW_THREADS(f)
378 if (f->f_fp == NULL) {
379 #if defined _MSC_VER && (_MSC_VER < 1400 || !defined(__STDC_SECURE_LIB__))
380 /* MSVC 6 (Microsoft) leaves errno at 0 for bad mode strings,
381 * across all Windows flavors. When it sets EINVAL varies
382 * across Windows flavors, the exact conditions aren't
383 * documented, and the answer lies in the OS's implementation
384 * of Win32's CreateFile function (whose source is secret).
385 * Seems the best we can do is map EINVAL to ENOENT.
386 * Starting with Visual Studio .NET 2005, EINVAL is correctly
387 * set by our CRT error handler (set in exceptions.c.)
389 if (errno == 0) /* bad mode string */
391 else if (errno == EINVAL) /* unknown, but not a mode string */
394 /* EINVAL is returned when an invalid filename or
395 * an invalid mode is supplied. */
396 if (errno == EINVAL) {
399 PyOS_snprintf(message, 100,
400 "invalid mode ('%.50s') or filename", mode);
401 v = Py_BuildValue("(isO)", errno, message, f->f_name);
403 PyErr_SetObject(PyExc_IOError, v);
408 PyErr_SetFromErrnoWithFilenameObject(PyExc_IOError, f->f_name);
417 return (PyObject *)f;
421 close_the_file(PyFileObject *f)
424 int (*local_close)(FILE *);
425 FILE *local_fp = f->f_fp;
426 char *local_setbuf = f->f_setbuf;
427 if (local_fp != NULL) {
428 local_close = f->f_close;
429 if (local_close != NULL && f->unlocked_count > 0) {
430 if (f->ob_refcnt > 0) {
431 PyErr_SetString(PyExc_IOError,
432 "close() called during concurrent "
433 "operation on the same file object.");
435 /* This should not happen unless someone is
436 * carelessly playing with the PyFileObject
437 * struct fields and/or its associated FILE
439 PyErr_SetString(PyExc_SystemError,
440 "PyFileObject locking error in "
441 "destructor (refcnt <= 0 at close).");
445 /* NULL out the FILE pointer before releasing the GIL, because
446 * it will not be valid anymore after the close() function is
449 if (local_close != NULL) {
450 /* Issue #9295: must temporarily reset f_setbuf so that another
451 thread doesn't free it when running file_close() concurrently.
452 Otherwise this close() will crash when flushing the buffer. */
454 Py_BEGIN_ALLOW_THREADS
456 sts = (*local_close)(local_fp);
458 f->f_setbuf = local_setbuf;
460 return PyErr_SetFromErrno(PyExc_IOError);
462 return PyInt_FromLong((long)sts);
469 PyFile_FromFile(FILE *fp, char *name, char *mode, int (*close)(FILE *))
471 PyFileObject *f = (PyFileObject *)PyFile_Type.tp_new(&PyFile_Type,
474 PyObject *o_name = PyString_FromString(name);
477 if (fill_file_fields(f, fp, o_name, mode, close) == NULL) {
483 return (PyObject *) f;
487 PyFile_FromString(char *name, char *mode)
489 extern int fclose(FILE *);
492 f = (PyFileObject *)PyFile_FromFile((FILE *)NULL, name, mode, fclose);
494 if (open_the_file(f, name, mode) == NULL) {
499 return (PyObject *)f;
503 PyFile_SetBufSize(PyObject *f, int bufsize)
505 PyFileObject *file = (PyFileObject *)f;
526 if (type == _IONBF) {
527 PyMem_Free(file->f_setbuf);
528 file->f_setbuf = NULL;
530 file->f_setbuf = (char *)PyMem_Realloc(file->f_setbuf,
534 setvbuf(file->f_fp, file->f_setbuf, type, bufsize);
535 #else /* !HAVE_SETVBUF */
536 setbuf(file->f_fp, file->f_setbuf);
537 #endif /* !HAVE_SETVBUF */
541 /* Set the encoding used to output Unicode strings.
542 Return 1 on success, 0 on failure. */
545 PyFile_SetEncoding(PyObject *f, const char *enc)
547 return PyFile_SetEncodingAndErrors(f, enc, NULL);
551 PyFile_SetEncodingAndErrors(PyObject *f, const char *enc, char* errors)
553 PyFileObject *file = (PyFileObject*)f;
554 PyObject *str, *oerrors;
556 assert(PyFile_Check(f));
557 str = PyString_FromString(enc);
561 oerrors = PyString_FromString(errors);
570 Py_DECREF(file->f_encoding);
571 file->f_encoding = str;
572 Py_DECREF(file->f_errors);
573 file->f_errors = oerrors;
580 PyErr_SetString(PyExc_ValueError, "I/O operation on closed file");
585 err_mode(char *action)
587 PyErr_Format(PyExc_IOError, "File not open for %s", action);
591 /* Refuse regular file I/O if there's data in the iteration-buffer.
592 * Mixing them would cause data to arrive out of order, as the read*
593 * methods don't use the iteration buffer. */
595 err_iterbuffered(void)
597 PyErr_SetString(PyExc_ValueError,
598 "Mixing iteration and read methods would lose data");
602 static void drop_readahead(PyFileObject *);
607 file_dealloc(PyFileObject *f)
610 if (f->weakreflist != NULL)
611 PyObject_ClearWeakRefs((PyObject *) f);
612 ret = close_the_file(f);
614 PySys_WriteStderr("close failed in file object destructor:\n");
620 PyMem_Free(f->f_setbuf);
621 Py_XDECREF(f->f_name);
622 Py_XDECREF(f->f_mode);
623 Py_XDECREF(f->f_encoding);
624 Py_XDECREF(f->f_errors);
626 Py_TYPE(f)->tp_free((PyObject *)f);
630 file_repr(PyFileObject *f)
632 if (PyUnicode_Check(f->f_name)) {
633 #ifdef Py_USING_UNICODE
634 PyObject *ret = NULL;
635 PyObject *name = PyUnicode_AsUnicodeEscapeString(f->f_name);
636 const char *name_str = name ? PyString_AsString(name) : "?";
637 ret = PyString_FromFormat("<%s file u'%s', mode '%s' at %p>",
638 f->f_fp == NULL ? "closed" : "open",
640 PyString_AsString(f->f_mode),
646 return PyString_FromFormat("<%s file '%s', mode '%s' at %p>",
647 f->f_fp == NULL ? "closed" : "open",
648 PyString_AsString(f->f_name),
649 PyString_AsString(f->f_mode),
655 file_close(PyFileObject *f)
657 PyObject *sts = close_the_file(f);
659 PyMem_Free(f->f_setbuf);
666 /* Our very own off_t-like type, 64-bit if possible */
667 #if !defined(HAVE_LARGEFILE_SUPPORT)
668 typedef off_t Py_off_t;
669 #elif SIZEOF_OFF_T >= 8
670 typedef off_t Py_off_t;
671 #elif SIZEOF_FPOS_T >= 8
672 typedef fpos_t Py_off_t;
674 #error "Large file support, but neither off_t nor fpos_t is large enough."
678 /* a portable fseek() function
679 return 0 on success, non-zero on failure (with errno set) */
681 _portable_fseek(FILE *fp, Py_off_t offset, int whence)
683 #if !defined(HAVE_LARGEFILE_SUPPORT)
684 return fseek(fp, offset, whence);
685 #elif defined(HAVE_FSEEKO) && SIZEOF_OFF_T >= 8
686 return fseeko(fp, offset, whence);
687 #elif defined(HAVE_FSEEK64)
688 return fseek64(fp, offset, whence);
689 #elif defined(__BEOS__)
690 return _fseek(fp, offset, whence);
691 #elif SIZEOF_FPOS_T >= 8
692 /* lacking a 64-bit capable fseek(), use a 64-bit capable fsetpos()
693 and fgetpos() to implement fseek()*/
699 if (_lseeki64(fileno(fp), 0, 2) == -1)
702 if (fseek(fp, 0, SEEK_END) != 0)
707 if (fgetpos(fp, &pos) != 0)
711 /* case SEEK_SET: break; */
713 return fsetpos(fp, &offset);
715 #error "Large file support, but no way to fseek."
720 /* a portable ftell() function
721 Return -1 on failure with errno set appropriately, current file
722 position on success */
724 _portable_ftell(FILE* fp)
726 #if !defined(HAVE_LARGEFILE_SUPPORT)
728 #elif defined(HAVE_FTELLO) && SIZEOF_OFF_T >= 8
730 #elif defined(HAVE_FTELL64)
732 #elif SIZEOF_FPOS_T >= 8
734 if (fgetpos(fp, &pos) != 0)
738 #error "Large file support, but no way to ftell."
744 file_seek(PyFileObject *f, PyObject *args)
749 PyObject *offobj, *off_index;
755 if (!PyArg_ParseTuple(args, "O|i:seek", &offobj, &whence))
757 off_index = PyNumber_Index(offobj);
759 if (!PyFloat_Check(offobj))
761 /* Deprecated in 2.6 */
763 if (PyErr_WarnEx(PyExc_DeprecationWarning,
764 "integer argument expected, got float",
770 #if !defined(HAVE_LARGEFILE_SUPPORT)
771 offset = PyInt_AsLong(off_index);
773 offset = PyLong_Check(off_index) ?
774 PyLong_AsLongLong(off_index) : PyInt_AsLong(off_index);
776 Py_DECREF(off_index);
777 if (PyErr_Occurred())
780 FILE_BEGIN_ALLOW_THREADS(f)
782 ret = _portable_fseek(f->f_fp, offset, whence);
783 FILE_END_ALLOW_THREADS(f)
786 PyErr_SetFromErrno(PyExc_IOError);
796 #ifdef HAVE_FTRUNCATE
798 file_truncate(PyFileObject *f, PyObject *args)
801 PyObject *newsizeobj = NULL;
808 return err_mode("writing");
809 if (!PyArg_UnpackTuple(args, "truncate", 0, 1, &newsizeobj))
812 /* Get current file position. If the file happens to be open for
813 * update and the last operation was an input operation, C doesn't
814 * define what the later fflush() will do, but we promise truncate()
815 * won't change the current position (and fflush() *does* change it
816 * then at least on Windows). The easiest thing is to capture
817 * current pos now and seek back to it at the end.
819 FILE_BEGIN_ALLOW_THREADS(f)
821 initialpos = _portable_ftell(f->f_fp);
822 FILE_END_ALLOW_THREADS(f)
823 if (initialpos == -1)
826 /* Set newsize to current postion if newsizeobj NULL, else to the
829 if (newsizeobj != NULL) {
830 #if !defined(HAVE_LARGEFILE_SUPPORT)
831 newsize = PyInt_AsLong(newsizeobj);
833 newsize = PyLong_Check(newsizeobj) ?
834 PyLong_AsLongLong(newsizeobj) :
835 PyInt_AsLong(newsizeobj);
837 if (PyErr_Occurred())
840 else /* default to current position */
841 newsize = initialpos;
843 /* Flush the stream. We're mixing stream-level I/O with lower-level
844 * I/O, and a flush may be necessary to synch both platform views
845 * of the current file state.
847 FILE_BEGIN_ALLOW_THREADS(f)
849 ret = fflush(f->f_fp);
850 FILE_END_ALLOW_THREADS(f)
855 /* MS _chsize doesn't work if newsize doesn't fit in 32 bits,
856 so don't even try using it. */
860 /* Have to move current pos to desired endpoint on Windows. */
861 FILE_BEGIN_ALLOW_THREADS(f)
863 ret = _portable_fseek(f->f_fp, newsize, SEEK_SET) != 0;
864 FILE_END_ALLOW_THREADS(f)
868 /* Truncate. Note that this may grow the file! */
869 FILE_BEGIN_ALLOW_THREADS(f)
871 hFile = (HANDLE)_get_osfhandle(fileno(f->f_fp));
872 ret = hFile == (HANDLE)-1;
874 ret = SetEndOfFile(hFile) == 0;
878 FILE_END_ALLOW_THREADS(f)
883 FILE_BEGIN_ALLOW_THREADS(f)
885 ret = ftruncate(fileno(f->f_fp), newsize);
886 FILE_END_ALLOW_THREADS(f)
889 #endif /* !MS_WINDOWS */
891 /* Restore original file position. */
892 FILE_BEGIN_ALLOW_THREADS(f)
894 ret = _portable_fseek(f->f_fp, initialpos, SEEK_SET) != 0;
895 FILE_END_ALLOW_THREADS(f)
903 PyErr_SetFromErrno(PyExc_IOError);
907 #endif /* HAVE_FTRUNCATE */
910 file_tell(PyFileObject *f)
916 FILE_BEGIN_ALLOW_THREADS(f)
918 pos = _portable_ftell(f->f_fp);
919 FILE_END_ALLOW_THREADS(f)
922 PyErr_SetFromErrno(PyExc_IOError);
926 if (f->f_skipnextlf) {
930 f->f_newlinetypes |= NEWLINE_CRLF;
933 } else if (c != EOF) ungetc(c, f->f_fp);
935 #if !defined(HAVE_LARGEFILE_SUPPORT)
936 return PyInt_FromLong(pos);
938 return PyLong_FromLongLong(pos);
943 file_fileno(PyFileObject *f)
947 return PyInt_FromLong((long) fileno(f->f_fp));
951 file_flush(PyFileObject *f)
957 FILE_BEGIN_ALLOW_THREADS(f)
959 res = fflush(f->f_fp);
960 FILE_END_ALLOW_THREADS(f)
962 PyErr_SetFromErrno(PyExc_IOError);
971 file_isatty(PyFileObject *f)
976 FILE_BEGIN_ALLOW_THREADS(f)
977 res = isatty((int)fileno(f->f_fp));
978 FILE_END_ALLOW_THREADS(f)
979 return PyBool_FromLong(res);
984 #define SMALLCHUNK 8192
986 #define SMALLCHUNK BUFSIZ
990 #define BIGCHUNK (512 * 32)
992 #define BIGCHUNK (512 * 1024)
996 new_buffersize(PyFileObject *f, size_t currentsize)
1001 if (fstat(fileno(f->f_fp), &st) == 0) {
1003 /* The following is not a bug: we really need to call lseek()
1004 *and* ftell(). The reason is that some stdio libraries
1005 mistakenly flush their buffer when ftell() is called and
1006 the lseek() call it makes fails, thereby throwing away
1007 data that cannot be recovered in any way. To avoid this,
1008 we first test lseek(), and only call ftell() if lseek()
1009 works. We can't use the lseek() value either, because we
1010 need to take the amount of buffered data into account.
1011 (Yet another reason why stdio stinks. :-) */
1012 pos = lseek(fileno(f->f_fp), 0L, SEEK_CUR);
1014 pos = ftell(f->f_fp);
1018 if (end > pos && pos >= 0)
1019 return currentsize + end - pos + 1;
1020 /* Add 1 so if the file were to grow we'd notice. */
1023 if (currentsize > SMALLCHUNK) {
1024 /* Keep doubling until we reach BIGCHUNK;
1025 then keep adding BIGCHUNK. */
1026 if (currentsize <= BIGCHUNK)
1027 return currentsize + currentsize;
1029 return currentsize + BIGCHUNK;
1031 return currentsize + SMALLCHUNK;
1034 #if defined(EWOULDBLOCK) && defined(EAGAIN) && EWOULDBLOCK != EAGAIN
1035 #define BLOCKED_ERRNO(x) ((x) == EWOULDBLOCK || (x) == EAGAIN)
1038 #define BLOCKED_ERRNO(x) ((x) == EWOULDBLOCK)
1041 #define BLOCKED_ERRNO(x) ((x) == EAGAIN)
1043 #define BLOCKED_ERRNO(x) 0
1049 file_read(PyFileObject *f, PyObject *args)
1051 long bytesrequested = -1;
1052 size_t bytesread, buffersize, chunksize;
1055 if (f->f_fp == NULL)
1056 return err_closed();
1058 return err_mode("reading");
1059 /* refuse to mix with f.next() */
1060 if (f->f_buf != NULL &&
1061 (f->f_bufend - f->f_bufptr) > 0 &&
1062 f->f_buf[0] != '\0')
1063 return err_iterbuffered();
1064 if (!PyArg_ParseTuple(args, "|l:read", &bytesrequested))
1066 if (bytesrequested < 0)
1067 buffersize = new_buffersize(f, (size_t)0);
1069 buffersize = bytesrequested;
1070 if (buffersize > PY_SSIZE_T_MAX) {
1071 PyErr_SetString(PyExc_OverflowError,
1072 "requested number of bytes is more than a Python string can hold");
1075 v = PyString_FromStringAndSize((char *)NULL, buffersize);
1080 FILE_BEGIN_ALLOW_THREADS(f)
1082 chunksize = Py_UniversalNewlineFread(BUF(v) + bytesread,
1083 buffersize - bytesread, f->f_fp, (PyObject *)f);
1084 FILE_END_ALLOW_THREADS(f)
1085 if (chunksize == 0) {
1086 if (!ferror(f->f_fp))
1089 /* When in non-blocking mode, data shouldn't
1090 * be discarded if a blocking signal was
1091 * received. That will also happen if
1092 * chunksize != 0, but bytesread < buffersize. */
1093 if (bytesread > 0 && BLOCKED_ERRNO(errno))
1095 PyErr_SetFromErrno(PyExc_IOError);
1099 bytesread += chunksize;
1100 if (bytesread < buffersize) {
1104 if (bytesrequested < 0) {
1105 buffersize = new_buffersize(f, buffersize);
1106 if (_PyString_Resize(&v, buffersize) < 0)
1109 /* Got what was requested. */
1113 if (bytesread != buffersize && _PyString_Resize(&v, bytesread))
1119 file_readinto(PyFileObject *f, PyObject *args)
1123 Py_ssize_t ndone, nnow;
1126 if (f->f_fp == NULL)
1127 return err_closed();
1129 return err_mode("reading");
1130 /* refuse to mix with f.next() */
1131 if (f->f_buf != NULL &&
1132 (f->f_bufend - f->f_bufptr) > 0 &&
1133 f->f_buf[0] != '\0')
1134 return err_iterbuffered();
1135 if (!PyArg_ParseTuple(args, "w*", &pbuf))
1141 FILE_BEGIN_ALLOW_THREADS(f)
1143 nnow = Py_UniversalNewlineFread(ptr+ndone, ntodo, f->f_fp,
1145 FILE_END_ALLOW_THREADS(f)
1147 if (!ferror(f->f_fp))
1149 PyErr_SetFromErrno(PyExc_IOError);
1151 PyBuffer_Release(&pbuf);
1157 PyBuffer_Release(&pbuf);
1158 return PyInt_FromSsize_t(ndone);
1161 /**************************************************************************
1162 Routine to get next line using platform fgets().
1166 + MS threadsafe getc is very slow (multiple layers of function calls before+
1167 after each character, to lock+unlock the stream).
1168 + The stream-locking functions are MS-internal -- can't access them from user
1170 + There's nothing Tim could find in the MS C or platform SDK libraries that
1171 can worm around this.
1172 + MS fgets locks/unlocks only once per line; it's the only hook we have.
1174 So we use fgets for speed(!), despite that it's painful.
1176 MS realloc is also slow.
1178 Reports from other platforms on this method vs getc_unlocked (which MS doesn't
1182 Tru64 Unix getline_via_fgets significantly faster
1184 CAUTION: The C std isn't clear about this: in those cases where fgets
1185 writes something into the buffer, can it write into any position beyond the
1186 required trailing null byte? MSVC 6 fgets does not, and no platform is (yet)
1187 known on which it does; and it would be a strange way to code fgets. Still,
1188 getline_via_fgets may not work correctly if it does. The std test
1189 test_bufio.py should fail if platform fgets() routinely writes beyond the
1190 trailing null byte. #define DONT_USE_FGETS_IN_GETLINE to disable this code.
1191 **************************************************************************/
1193 /* Use this routine if told to, or by default on non-get_unlocked()
1194 * platforms unless told not to. Yikes! Let's spell that out:
1195 * On a platform with getc_unlocked():
1196 * By default, use getc_unlocked().
1197 * If you want to use fgets() instead, #define USE_FGETS_IN_GETLINE.
1198 * On a platform without getc_unlocked():
1199 * By default, use fgets().
1200 * If you don't want to use fgets(), #define DONT_USE_FGETS_IN_GETLINE.
1202 #if !defined(USE_FGETS_IN_GETLINE) && !defined(HAVE_GETC_UNLOCKED)
1203 #define USE_FGETS_IN_GETLINE
1206 #if defined(DONT_USE_FGETS_IN_GETLINE) && defined(USE_FGETS_IN_GETLINE)
1207 #undef USE_FGETS_IN_GETLINE
1210 #ifdef USE_FGETS_IN_GETLINE
1212 getline_via_fgets(PyFileObject *f, FILE *fp)
1214 /* INITBUFSIZE is the maximum line length that lets us get away with the fast
1215 * no-realloc, one-fgets()-call path. Boosting it isn't free, because we have
1216 * to fill this much of the buffer with a known value in order to figure out
1217 * how much of the buffer fgets() overwrites. So if INITBUFSIZE is larger
1218 * than "most" lines, we waste time filling unused buffer slots. 100 is
1219 * surely adequate for most peoples' email archives, chewing over source code,
1220 * etc -- "regular old text files".
1221 * MAXBUFSIZE is the maximum line length that lets us get away with the less
1222 * fast (but still zippy) no-realloc, two-fgets()-call path. See above for
1223 * cautions about boosting that. 300 was chosen because the worst real-life
1224 * text-crunching job reported on Python-Dev was a mail-log crawler where over
1225 * half the lines were 254 chars.
1227 #define INITBUFSIZE 100
1228 #define MAXBUFSIZE 300
1230 char buf[MAXBUFSIZE];
1231 PyObject* v; /* the string object result */
1232 char* pvfree; /* address of next free slot */
1233 char* pvend; /* address one beyond last free slot */
1234 size_t nfree; /* # of free buffer slots; pvend-pvfree */
1235 size_t total_v_size; /* total # of slots in buffer */
1236 size_t increment; /* amount to increment the buffer */
1239 /* Optimize for normal case: avoid _PyString_Resize if at all
1240 * possible via first reading into stack buffer "buf".
1242 total_v_size = INITBUFSIZE; /* start small and pray */
1245 FILE_BEGIN_ALLOW_THREADS(f)
1246 pvend = buf + total_v_size;
1247 nfree = pvend - pvfree;
1248 memset(pvfree, '\n', nfree);
1249 assert(nfree < INT_MAX); /* Should be atmost MAXBUFSIZE */
1250 p = fgets(pvfree, (int)nfree, fp);
1251 FILE_END_ALLOW_THREADS(f)
1255 if (PyErr_CheckSignals())
1257 v = PyString_FromStringAndSize(buf, pvfree - buf);
1260 /* fgets read *something* */
1261 p = memchr(pvfree, '\n', nfree);
1263 /* Did the \n come from fgets or from us?
1264 * Since fgets stops at the first \n, and then writes
1265 * \0, if it's from fgets a \0 must be next. But if
1266 * that's so, it could not have come from us, since
1267 * the \n's we filled the buffer with have only more
1268 * \n's to the right.
1270 if (p+1 < pvend && *(p+1) == '\0') {
1271 /* It's from fgets: we win! In particular,
1272 * we haven't done any mallocs yet, and can
1273 * build the final result on the first try.
1275 ++p; /* include \n from fgets */
1278 /* Must be from us: fgets didn't fill the
1279 * buffer and didn't find a newline, so it
1280 * must be the last and newline-free line of
1283 assert(p > pvfree && *(p-1) == '\0');
1284 --p; /* don't include \0 from fgets */
1286 v = PyString_FromStringAndSize(buf, p - buf);
1289 /* yuck: fgets overwrote all the newlines, i.e. the entire
1290 * buffer. So this line isn't over yet, or maybe it is but
1291 * we're exactly at EOF. If we haven't already, try using the
1292 * rest of the stack buffer.
1294 assert(*(pvend-1) == '\0');
1295 if (pvfree == buf) {
1296 pvfree = pvend - 1; /* overwrite trailing null */
1297 total_v_size = MAXBUFSIZE;
1303 /* The stack buffer isn't big enough; malloc a string object and read
1306 total_v_size = MAXBUFSIZE << 1;
1307 v = PyString_FromStringAndSize((char*)NULL, (int)total_v_size);
1310 /* copy over everything except the last null byte */
1311 memcpy(BUF(v), buf, MAXBUFSIZE-1);
1312 pvfree = BUF(v) + MAXBUFSIZE - 1;
1314 /* Keep reading stuff into v; if it ever ends successfully, break
1315 * after setting p one beyond the end of the line. The code here is
1316 * very much like the code above, except reads into v's buffer; see
1317 * the code above for detailed comments about the logic.
1320 FILE_BEGIN_ALLOW_THREADS(f)
1321 pvend = BUF(v) + total_v_size;
1322 nfree = pvend - pvfree;
1323 memset(pvfree, '\n', nfree);
1324 assert(nfree < INT_MAX);
1325 p = fgets(pvfree, (int)nfree, fp);
1326 FILE_END_ALLOW_THREADS(f)
1330 if (PyErr_CheckSignals()) {
1337 p = memchr(pvfree, '\n', nfree);
1339 if (p+1 < pvend && *(p+1) == '\0') {
1340 /* \n came from fgets */
1344 /* \n came from us; last line of file, no newline */
1345 assert(p > pvfree && *(p-1) == '\0');
1349 /* expand buffer and try again */
1350 assert(*(pvend-1) == '\0');
1351 increment = total_v_size >> 2; /* mild exponential growth */
1352 prev_v_size = total_v_size;
1353 total_v_size += increment;
1354 /* check for overflow */
1355 if (total_v_size <= prev_v_size ||
1356 total_v_size > PY_SSIZE_T_MAX) {
1357 PyErr_SetString(PyExc_OverflowError,
1358 "line is longer than a Python string can hold");
1362 if (_PyString_Resize(&v, (int)total_v_size) < 0)
1364 /* overwrite the trailing null byte */
1365 pvfree = BUF(v) + (prev_v_size - 1);
1367 if (BUF(v) + total_v_size != p && _PyString_Resize(&v, p - BUF(v)))
1373 #endif /* ifdef USE_FGETS_IN_GETLINE */
1375 /* Internal routine to get a line.
1376 Size argument interpretation:
1378 <= 0: read arbitrary line
1382 get_line(PyFileObject *f, int n)
1387 size_t total_v_size; /* total # of slots in buffer */
1388 size_t used_v_size; /* # used slots in buffer */
1389 size_t increment; /* amount to increment the buffer */
1391 int newlinetypes = f->f_newlinetypes;
1392 int skipnextlf = f->f_skipnextlf;
1393 int univ_newline = f->f_univ_newline;
1395 #if defined(USE_FGETS_IN_GETLINE)
1396 if (n <= 0 && !univ_newline )
1397 return getline_via_fgets(f, fp);
1399 total_v_size = n > 0 ? n : 100;
1400 v = PyString_FromStringAndSize((char *)NULL, total_v_size);
1404 end = buf + total_v_size;
1407 FILE_BEGIN_ALLOW_THREADS(f)
1410 c = 'x'; /* Shut up gcc warning */
1411 while ( buf != end && (c = GETC(fp)) != EOF ) {
1415 /* Seeing a \n here with
1416 * skipnextlf true means we
1419 newlinetypes |= NEWLINE_CRLF;
1421 if (c == EOF) break;
1423 newlinetypes |= NEWLINE_CR;
1429 } else if ( c == '\n')
1430 newlinetypes |= NEWLINE_LF;
1432 if (c == '\n') break;
1434 if ( c == EOF && skipnextlf )
1435 newlinetypes |= NEWLINE_CR;
1436 } else /* If not universal newlines use the normal loop */
1437 while ((c = GETC(fp)) != EOF &&
1438 (*buf++ = c) != '\n' &&
1442 FILE_END_ALLOW_THREADS(f)
1443 f->f_newlinetypes = newlinetypes;
1444 f->f_skipnextlf = skipnextlf;
1449 PyErr_SetFromErrno(PyExc_IOError);
1455 if (PyErr_CheckSignals()) {
1461 /* Must be because buf == end */
1464 used_v_size = total_v_size;
1465 increment = total_v_size >> 2; /* mild exponential growth */
1466 total_v_size += increment;
1467 if (total_v_size > PY_SSIZE_T_MAX) {
1468 PyErr_SetString(PyExc_OverflowError,
1469 "line is longer than a Python string can hold");
1473 if (_PyString_Resize(&v, total_v_size) < 0)
1475 buf = BUF(v) + used_v_size;
1476 end = BUF(v) + total_v_size;
1479 used_v_size = buf - BUF(v);
1480 if (used_v_size != total_v_size && _PyString_Resize(&v, used_v_size))
1485 /* External C interface */
1488 PyFile_GetLine(PyObject *f, int n)
1493 PyErr_BadInternalCall();
1497 if (PyFile_Check(f)) {
1498 PyFileObject *fo = (PyFileObject *)f;
1499 if (fo->f_fp == NULL)
1500 return err_closed();
1502 return err_mode("reading");
1503 /* refuse to mix with f.next() */
1504 if (fo->f_buf != NULL &&
1505 (fo->f_bufend - fo->f_bufptr) > 0 &&
1506 fo->f_buf[0] != '\0')
1507 return err_iterbuffered();
1508 result = get_line(fo, n);
1514 reader = PyObject_GetAttrString(f, "readline");
1518 args = PyTuple_New(0);
1520 args = Py_BuildValue("(i)", n);
1525 result = PyEval_CallObject(reader, args);
1528 if (result != NULL && !PyString_Check(result) &&
1529 !PyUnicode_Check(result)) {
1532 PyErr_SetString(PyExc_TypeError,
1533 "object.readline() returned non-string");
1537 if (n < 0 && result != NULL && PyString_Check(result)) {
1538 char *s = PyString_AS_STRING(result);
1539 Py_ssize_t len = PyString_GET_SIZE(result);
1543 PyErr_SetString(PyExc_EOFError,
1544 "EOF when reading a line");
1546 else if (s[len-1] == '\n') {
1547 if (result->ob_refcnt == 1) {
1548 if (_PyString_Resize(&result, len-1))
1553 v = PyString_FromStringAndSize(s, len-1);
1559 #ifdef Py_USING_UNICODE
1560 if (n < 0 && result != NULL && PyUnicode_Check(result)) {
1561 Py_UNICODE *s = PyUnicode_AS_UNICODE(result);
1562 Py_ssize_t len = PyUnicode_GET_SIZE(result);
1566 PyErr_SetString(PyExc_EOFError,
1567 "EOF when reading a line");
1569 else if (s[len-1] == '\n') {
1570 if (result->ob_refcnt == 1)
1571 PyUnicode_Resize(&result, len-1);
1574 v = PyUnicode_FromUnicode(s, len-1);
1587 file_readline(PyFileObject *f, PyObject *args)
1591 if (f->f_fp == NULL)
1592 return err_closed();
1594 return err_mode("reading");
1595 /* refuse to mix with f.next() */
1596 if (f->f_buf != NULL &&
1597 (f->f_bufend - f->f_bufptr) > 0 &&
1598 f->f_buf[0] != '\0')
1599 return err_iterbuffered();
1600 if (!PyArg_ParseTuple(args, "|i:readline", &n))
1603 return PyString_FromString("");
1606 return get_line(f, n);
1610 file_readlines(PyFileObject *f, PyObject *args)
1613 PyObject *list = NULL;
1615 char small_buffer[SMALLCHUNK];
1616 char *buffer = small_buffer;
1617 size_t buffersize = SMALLCHUNK;
1618 PyObject *big_buffer = NULL;
1621 size_t totalread = 0;
1626 if (f->f_fp == NULL)
1627 return err_closed();
1629 return err_mode("reading");
1630 /* refuse to mix with f.next() */
1631 if (f->f_buf != NULL &&
1632 (f->f_bufend - f->f_bufptr) > 0 &&
1633 f->f_buf[0] != '\0')
1634 return err_iterbuffered();
1635 if (!PyArg_ParseTuple(args, "|l:readlines", &sizehint))
1637 if ((list = PyList_New(0)) == NULL)
1643 FILE_BEGIN_ALLOW_THREADS(f)
1645 nread = Py_UniversalNewlineFread(buffer+nfilled,
1646 buffersize-nfilled, f->f_fp, (PyObject *)f);
1647 FILE_END_ALLOW_THREADS(f)
1648 shortread = (nread < buffersize-nfilled);
1652 if (!ferror(f->f_fp))
1654 PyErr_SetFromErrno(PyExc_IOError);
1659 p = (char *)memchr(buffer+nfilled, '\n', nread);
1661 /* Need a larger buffer to fit this line */
1664 if (buffersize > PY_SSIZE_T_MAX) {
1665 PyErr_SetString(PyExc_OverflowError,
1666 "line is longer than a Python string can hold");
1669 if (big_buffer == NULL) {
1670 /* Create the big buffer */
1671 big_buffer = PyString_FromStringAndSize(
1673 if (big_buffer == NULL)
1675 buffer = PyString_AS_STRING(big_buffer);
1676 memcpy(buffer, small_buffer, nfilled);
1679 /* Grow the big buffer */
1680 if ( _PyString_Resize(&big_buffer, buffersize) < 0 )
1682 buffer = PyString_AS_STRING(big_buffer);
1686 end = buffer+nfilled+nread;
1689 /* Process complete lines */
1691 line = PyString_FromStringAndSize(q, p-q);
1694 err = PyList_Append(list, line);
1699 p = (char *)memchr(q, '\n', end-q);
1700 } while (p != NULL);
1701 /* Move the remaining incomplete line to the start */
1703 memmove(buffer, q, nfilled);
1705 if (totalread >= (size_t)sizehint)
1709 /* Partial last line */
1710 line = PyString_FromStringAndSize(buffer, nfilled);
1714 /* Need to complete the last line */
1715 PyObject *rest = get_line(f, 0);
1720 PyString_Concat(&line, rest);
1725 err = PyList_Append(list, line);
1732 Py_XDECREF(big_buffer);
1741 file_write(PyFileObject *f, PyObject *args)
1746 PyObject *encoded = NULL;
1748 if (f->f_fp == NULL)
1749 return err_closed();
1751 return err_mode("writing");
1753 if (!PyArg_ParseTuple(args, "s*", &pbuf))
1759 const char *encoding, *errors;
1761 if (!PyArg_ParseTuple(args, "O", &text))
1764 if (PyString_Check(text)) {
1765 s = PyString_AS_STRING(text);
1766 n = PyString_GET_SIZE(text);
1767 } else if (PyUnicode_Check(text)) {
1768 if (f->f_encoding != Py_None)
1769 encoding = PyString_AS_STRING(f->f_encoding);
1771 encoding = PyUnicode_GetDefaultEncoding();
1772 if (f->f_errors != Py_None)
1773 errors = PyString_AS_STRING(f->f_errors);
1776 encoded = PyUnicode_AsEncodedString(text, encoding, errors);
1777 if (encoded == NULL)
1779 s = PyString_AS_STRING(encoded);
1780 n = PyString_GET_SIZE(encoded);
1782 if (PyObject_AsCharBuffer(text, &s, &n))
1787 FILE_BEGIN_ALLOW_THREADS(f)
1789 n2 = fwrite(s, 1, n, f->f_fp);
1790 FILE_END_ALLOW_THREADS(f)
1791 Py_XDECREF(encoded);
1793 PyBuffer_Release(&pbuf);
1795 PyErr_SetFromErrno(PyExc_IOError);
1804 file_writelines(PyFileObject *f, PyObject *seq)
1806 #define CHUNKSIZE 1000
1807 PyObject *list, *line;
1808 PyObject *it; /* iter(seq) */
1811 Py_ssize_t i, j, nwritten, len;
1813 assert(seq != NULL);
1814 if (f->f_fp == NULL)
1815 return err_closed();
1817 return err_mode("writing");
1821 islist = PyList_Check(seq);
1825 it = PyObject_GetIter(seq);
1827 PyErr_SetString(PyExc_TypeError,
1828 "writelines() requires an iterable argument");
1831 /* From here on, fail by going to error, to reclaim "it". */
1832 list = PyList_New(CHUNKSIZE);
1837 /* Strategy: slurp CHUNKSIZE lines into a private list,
1838 checking that they are all strings, then write that list
1839 without holding the interpreter lock, then come back for more. */
1840 for (index = 0; ; index += CHUNKSIZE) {
1843 list = PyList_GetSlice(seq, index, index+CHUNKSIZE);
1846 j = PyList_GET_SIZE(list);
1849 for (j = 0; j < CHUNKSIZE; j++) {
1850 line = PyIter_Next(it);
1852 if (PyErr_Occurred())
1856 PyList_SetItem(list, j, line);
1858 /* The iterator might have closed the file on us. */
1859 if (f->f_fp == NULL) {
1867 /* Check that all entries are indeed strings. If not,
1868 apply the same rules as for file.write() and
1869 convert the results to strings. This is slow, but
1870 seems to be the only way since all conversion APIs
1871 could potentially execute Python code. */
1872 for (i = 0; i < j; i++) {
1873 PyObject *v = PyList_GET_ITEM(list, i);
1874 if (!PyString_Check(v)) {
1876 if (((f->f_binary &&
1877 PyObject_AsReadBuffer(v,
1878 (const void**)&buffer,
1880 PyObject_AsCharBuffer(v,
1883 PyErr_SetString(PyExc_TypeError,
1884 "writelines() argument must be a sequence of strings");
1887 line = PyString_FromStringAndSize(buffer,
1892 PyList_SET_ITEM(list, i, line);
1896 /* Since we are releasing the global lock, the
1897 following code may *not* execute Python code. */
1899 FILE_BEGIN_ALLOW_THREADS(f)
1901 for (i = 0; i < j; i++) {
1902 line = PyList_GET_ITEM(list, i);
1903 len = PyString_GET_SIZE(line);
1904 nwritten = fwrite(PyString_AS_STRING(line),
1906 if (nwritten != len) {
1907 FILE_ABORT_ALLOW_THREADS(f)
1908 PyErr_SetFromErrno(PyExc_IOError);
1913 FILE_END_ALLOW_THREADS(f)
1929 file_self(PyFileObject *f)
1931 if (f->f_fp == NULL)
1932 return err_closed();
1934 return (PyObject *)f;
1938 file_xreadlines(PyFileObject *f)
1940 if (PyErr_WarnPy3k("f.xreadlines() not supported in 3.x, "
1941 "try 'for line in f' instead", 1) < 0)
1943 return file_self(f);
1947 file_exit(PyObject *f, PyObject *args)
1949 PyObject *ret = PyObject_CallMethod(f, "close", NULL);
1951 /* If error occurred, pass through */
1954 /* We cannot return the result of close since a true
1955 * value will be interpreted as "yes, swallow the
1956 * exception if one was raised inside the with block". */
1960 PyDoc_STRVAR(readline_doc,
1961 "readline([size]) -> next line from the file, as a string.\n"
1963 "Retain newline. A non-negative size argument limits the maximum\n"
1964 "number of bytes to return (an incomplete line may be returned then).\n"
1965 "Return an empty string at EOF.");
1967 PyDoc_STRVAR(read_doc,
1968 "read([size]) -> read at most size bytes, returned as a string.\n"
1970 "If the size argument is negative or omitted, read until EOF is reached.\n"
1971 "Notice that when in non-blocking mode, less data than what was requested\n"
1972 "may be returned, even if no size parameter was given.");
1974 PyDoc_STRVAR(write_doc,
1975 "write(str) -> None. Write string str to file.\n"
1977 "Note that due to buffering, flush() or close() may be needed before\n"
1978 "the file on disk reflects the data written.");
1980 PyDoc_STRVAR(fileno_doc,
1981 "fileno() -> integer \"file descriptor\".\n"
1983 "This is needed for lower-level file interfaces, such os.read().");
1985 PyDoc_STRVAR(seek_doc,
1986 "seek(offset[, whence]) -> None. Move to new file position.\n"
1988 "Argument offset is a byte count. Optional argument whence defaults to\n"
1989 "0 (offset from start of file, offset should be >= 0); other values are 1\n"
1990 "(move relative to current position, positive or negative), and 2 (move\n"
1991 "relative to end of file, usually negative, although many platforms allow\n"
1992 "seeking beyond the end of a file). If the file is opened in text mode,\n"
1993 "only offsets returned by tell() are legal. Use of other offsets causes\n"
1994 "undefined behavior."
1996 "Note that not all file objects are seekable.");
1998 #ifdef HAVE_FTRUNCATE
1999 PyDoc_STRVAR(truncate_doc,
2000 "truncate([size]) -> None. Truncate the file to at most size bytes.\n"
2002 "Size defaults to the current file position, as returned by tell().");
2005 PyDoc_STRVAR(tell_doc,
2006 "tell() -> current file position, an integer (may be a long integer).");
2008 PyDoc_STRVAR(readinto_doc,
2009 "readinto() -> Undocumented. Don't use this; it may go away.");
2011 PyDoc_STRVAR(readlines_doc,
2012 "readlines([size]) -> list of strings, each a line from the file.\n"
2014 "Call readline() repeatedly and return a list of the lines so read.\n"
2015 "The optional size argument, if given, is an approximate bound on the\n"
2016 "total number of bytes in the lines returned.");
2018 PyDoc_STRVAR(xreadlines_doc,
2019 "xreadlines() -> returns self.\n"
2021 "For backward compatibility. File objects now include the performance\n"
2022 "optimizations previously implemented in the xreadlines module.");
2024 PyDoc_STRVAR(writelines_doc,
2025 "writelines(sequence_of_strings) -> None. Write the strings to the file.\n"
2027 "Note that newlines are not added. The sequence can be any iterable object\n"
2028 "producing strings. This is equivalent to calling write() for each string.");
2030 PyDoc_STRVAR(flush_doc,
2031 "flush() -> None. Flush the internal I/O buffer.");
2033 PyDoc_STRVAR(close_doc,
2034 "close() -> None or (perhaps) an integer. Close the file.\n"
2036 "Sets data attribute .closed to True. A closed file cannot be used for\n"
2037 "further I/O operations. close() may be called more than once without\n"
2038 "error. Some kinds of file objects (for example, opened by popen())\n"
2039 "may return an exit status upon closing.");
2041 PyDoc_STRVAR(isatty_doc,
2042 "isatty() -> true or false. True if the file is connected to a tty device.");
2044 PyDoc_STRVAR(enter_doc,
2045 "__enter__() -> self.");
2047 PyDoc_STRVAR(exit_doc,
2048 "__exit__(*excinfo) -> None. Closes the file.");
2050 static PyMethodDef file_methods[] = {
2051 {"readline", (PyCFunction)file_readline, METH_VARARGS, readline_doc},
2052 {"read", (PyCFunction)file_read, METH_VARARGS, read_doc},
2053 {"write", (PyCFunction)file_write, METH_VARARGS, write_doc},
2054 {"fileno", (PyCFunction)file_fileno, METH_NOARGS, fileno_doc},
2055 {"seek", (PyCFunction)file_seek, METH_VARARGS, seek_doc},
2056 #ifdef HAVE_FTRUNCATE
2057 {"truncate", (PyCFunction)file_truncate, METH_VARARGS, truncate_doc},
2059 {"tell", (PyCFunction)file_tell, METH_NOARGS, tell_doc},
2060 {"readinto", (PyCFunction)file_readinto, METH_VARARGS, readinto_doc},
2061 {"readlines", (PyCFunction)file_readlines, METH_VARARGS, readlines_doc},
2062 {"xreadlines",(PyCFunction)file_xreadlines, METH_NOARGS, xreadlines_doc},
2063 {"writelines",(PyCFunction)file_writelines, METH_O, writelines_doc},
2064 {"flush", (PyCFunction)file_flush, METH_NOARGS, flush_doc},
2065 {"close", (PyCFunction)file_close, METH_NOARGS, close_doc},
2066 {"isatty", (PyCFunction)file_isatty, METH_NOARGS, isatty_doc},
2067 {"__enter__", (PyCFunction)file_self, METH_NOARGS, enter_doc},
2068 {"__exit__", (PyCFunction)file_exit, METH_VARARGS, exit_doc},
2069 {NULL, NULL} /* sentinel */
2072 #define OFF(x) offsetof(PyFileObject, x)
2074 static PyMemberDef file_memberlist[] = {
2075 {"mode", T_OBJECT, OFF(f_mode), RO,
2076 "file mode ('r', 'U', 'w', 'a', possibly with 'b' or '+' added)"},
2077 {"name", T_OBJECT, OFF(f_name), RO,
2079 {"encoding", T_OBJECT, OFF(f_encoding), RO,
2081 {"errors", T_OBJECT, OFF(f_errors), RO,
2082 "Unicode error handler"},
2083 /* getattr(f, "closed") is implemented without this table */
2084 {NULL} /* Sentinel */
2088 get_closed(PyFileObject *f, void *closure)
2090 return PyBool_FromLong((long)(f->f_fp == 0));
2093 get_newlines(PyFileObject *f, void *closure)
2095 switch (f->f_newlinetypes) {
2096 case NEWLINE_UNKNOWN:
2100 return PyString_FromString("\r");
2102 return PyString_FromString("\n");
2103 case NEWLINE_CR|NEWLINE_LF:
2104 return Py_BuildValue("(ss)", "\r", "\n");
2106 return PyString_FromString("\r\n");
2107 case NEWLINE_CR|NEWLINE_CRLF:
2108 return Py_BuildValue("(ss)", "\r", "\r\n");
2109 case NEWLINE_LF|NEWLINE_CRLF:
2110 return Py_BuildValue("(ss)", "\n", "\r\n");
2111 case NEWLINE_CR|NEWLINE_LF|NEWLINE_CRLF:
2112 return Py_BuildValue("(sss)", "\r", "\n", "\r\n");
2114 PyErr_Format(PyExc_SystemError,
2115 "Unknown newlines value 0x%x\n",
2122 get_softspace(PyFileObject *f, void *closure)
2124 if (PyErr_WarnPy3k("file.softspace not supported in 3.x", 1) < 0)
2126 return PyInt_FromLong(f->f_softspace);
2130 set_softspace(PyFileObject *f, PyObject *value)
2133 if (PyErr_WarnPy3k("file.softspace not supported in 3.x", 1) < 0)
2136 if (value == NULL) {
2137 PyErr_SetString(PyExc_TypeError,
2138 "can't delete softspace attribute");
2142 new = PyInt_AsLong(value);
2143 if (new == -1 && PyErr_Occurred())
2145 f->f_softspace = new;
2149 static PyGetSetDef file_getsetlist[] = {
2150 {"closed", (getter)get_closed, NULL, "True if the file is closed"},
2151 {"newlines", (getter)get_newlines, NULL,
2152 "end-of-line convention used in this file"},
2153 {"softspace", (getter)get_softspace, (setter)set_softspace,
2154 "flag indicating that a space needs to be printed; used by print"},
2159 drop_readahead(PyFileObject *f)
2161 if (f->f_buf != NULL) {
2162 PyMem_Free(f->f_buf);
2167 /* Make sure that file has a readahead buffer with at least one byte
2168 (unless at EOF) and no more than bufsize. Returns negative value on
2169 error, will set MemoryError if bufsize bytes cannot be allocated. */
2171 readahead(PyFileObject *f, int bufsize)
2173 Py_ssize_t chunksize;
2175 if (f->f_buf != NULL) {
2176 if( (f->f_bufend - f->f_bufptr) >= 1)
2181 if ((f->f_buf = (char *)PyMem_Malloc(bufsize)) == NULL) {
2185 FILE_BEGIN_ALLOW_THREADS(f)
2187 chunksize = Py_UniversalNewlineFread(
2188 f->f_buf, bufsize, f->f_fp, (PyObject *)f);
2189 FILE_END_ALLOW_THREADS(f)
2190 if (chunksize == 0) {
2191 if (ferror(f->f_fp)) {
2192 PyErr_SetFromErrno(PyExc_IOError);
2198 f->f_bufptr = f->f_buf;
2199 f->f_bufend = f->f_buf + chunksize;
2203 /* Used by file_iternext. The returned string will start with 'skip'
2204 uninitialized bytes followed by the remainder of the line. Don't be
2205 horrified by the recursive call: maximum recursion depth is limited by
2206 logarithmic buffer growth to about 50 even when reading a 1gb line. */
2208 static PyStringObject *
2209 readahead_get_line_skip(PyFileObject *f, int skip, int bufsize)
2216 if (f->f_buf == NULL)
2217 if (readahead(f, bufsize) < 0)
2220 len = f->f_bufend - f->f_bufptr;
2222 return (PyStringObject *)
2223 PyString_FromStringAndSize(NULL, skip);
2224 bufptr = (char *)memchr(f->f_bufptr, '\n', len);
2225 if (bufptr != NULL) {
2226 bufptr++; /* Count the '\n' */
2227 len = bufptr - f->f_bufptr;
2228 s = (PyStringObject *)
2229 PyString_FromStringAndSize(NULL, skip+len);
2232 memcpy(PyString_AS_STRING(s)+skip, f->f_bufptr, len);
2233 f->f_bufptr = bufptr;
2234 if (bufptr == f->f_bufend)
2237 bufptr = f->f_bufptr;
2239 f->f_buf = NULL; /* Force new readahead buffer */
2240 assert(skip+len < INT_MAX);
2241 s = readahead_get_line_skip(
2242 f, (int)(skip+len), bufsize + (bufsize>>2) );
2247 memcpy(PyString_AS_STRING(s)+skip, bufptr, len);
2253 /* A larger buffer size may actually decrease performance. */
2254 #define READAHEAD_BUFSIZE 8192
2257 file_iternext(PyFileObject *f)
2261 if (f->f_fp == NULL)
2262 return err_closed();
2264 return err_mode("reading");
2266 l = readahead_get_line_skip(f, 0, READAHEAD_BUFSIZE);
2267 if (l == NULL || PyString_GET_SIZE(l) == 0) {
2271 return (PyObject *)l;
2276 file_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
2279 static PyObject *not_yet_string;
2281 assert(type != NULL && type->tp_alloc != NULL);
2283 if (not_yet_string == NULL) {
2284 not_yet_string = PyString_InternFromString("<uninitialized file>");
2285 if (not_yet_string == NULL)
2289 self = type->tp_alloc(type, 0);
2291 /* Always fill in the name and mode, so that nobody else
2292 needs to special-case NULLs there. */
2293 Py_INCREF(not_yet_string);
2294 ((PyFileObject *)self)->f_name = not_yet_string;
2295 Py_INCREF(not_yet_string);
2296 ((PyFileObject *)self)->f_mode = not_yet_string;
2298 ((PyFileObject *)self)->f_encoding = Py_None;
2300 ((PyFileObject *)self)->f_errors = Py_None;
2301 ((PyFileObject *)self)->weakreflist = NULL;
2302 ((PyFileObject *)self)->unlocked_count = 0;
2308 file_init(PyObject *self, PyObject *args, PyObject *kwds)
2310 PyFileObject *foself = (PyFileObject *)self;
2312 static char *kwlist[] = {"name", "mode", "buffering", 0};
2316 int wideargument = 0;
2321 assert(PyFile_Check(self));
2322 if (foself->f_fp != NULL) {
2323 /* Have to close the existing file first. */
2324 PyObject *closeresult = file_close(foself);
2325 if (closeresult == NULL)
2327 Py_DECREF(closeresult);
2331 if (PyArg_ParseTupleAndKeywords(args, kwds, "U|si:file",
2332 kwlist, &po, &mode, &bufsize)) {
2334 if (fill_file_fields(foself, NULL, po, mode,
2338 /* Drop the argument parsing error as narrow
2339 strings are also valid. */
2344 if (!wideargument) {
2347 if (!PyArg_ParseTupleAndKeywords(args, kwds, "et|si:file", kwlist,
2348 Py_FileSystemDefaultEncoding,
2353 /* We parse again to get the name as a PyObject */
2354 if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|si:file",
2355 kwlist, &o_name, &mode,
2359 if (fill_file_fields(foself, NULL, o_name, mode,
2363 if (open_the_file(foself, name, mode) == NULL)
2365 foself->f_setbuf = NULL;
2366 PyFile_SetBufSize(self, bufsize);
2373 PyMem_Free(name); /* free the encoded string */
2377 PyDoc_VAR(file_doc) =
2379 "file(name[, mode[, buffering]]) -> file object\n"
2381 "Open a file. The mode can be 'r', 'w' or 'a' for reading (default),\n"
2382 "writing or appending. The file will be created if it doesn't exist\n"
2383 "when opened for writing or appending; it will be truncated when\n"
2384 "opened for writing. Add a 'b' to the mode for binary files.\n"
2385 "Add a '+' to the mode to allow simultaneous reading and writing.\n"
2386 "If the buffering argument is given, 0 means unbuffered, 1 means line\n"
2387 "buffered, and larger numbers specify the buffer size. The preferred way\n"
2388 "to open a file is with the builtin open() function.\n"
2391 "Add a 'U' to mode to open the file for input with universal newline\n"
2392 "support. Any line ending in the input file will be seen as a '\\n'\n"
2393 "in Python. Also, a file so opened gains the attribute 'newlines';\n"
2394 "the value for this attribute is one of None (no newline read yet),\n"
2395 "'\\r', '\\n', '\\r\\n' or a tuple containing all the newline types seen.\n"
2397 "'U' cannot be combined with 'w' or '+' mode.\n"
2400 PyTypeObject PyFile_Type = {
2401 PyVarObject_HEAD_INIT(&PyType_Type, 0)
2403 sizeof(PyFileObject),
2405 (destructor)file_dealloc, /* tp_dealloc */
2410 (reprfunc)file_repr, /* tp_repr */
2411 0, /* tp_as_number */
2412 0, /* tp_as_sequence */
2413 0, /* tp_as_mapping */
2417 PyObject_GenericGetAttr, /* tp_getattro */
2418 /* softspace is writable: we must supply tp_setattro */
2419 PyObject_GenericSetAttr, /* tp_setattro */
2420 0, /* tp_as_buffer */
2421 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_WEAKREFS, /* tp_flags */
2422 file_doc, /* tp_doc */
2423 0, /* tp_traverse */
2425 0, /* tp_richcompare */
2426 offsetof(PyFileObject, weakreflist), /* tp_weaklistoffset */
2427 (getiterfunc)file_self, /* tp_iter */
2428 (iternextfunc)file_iternext, /* tp_iternext */
2429 file_methods, /* tp_methods */
2430 file_memberlist, /* tp_members */
2431 file_getsetlist, /* tp_getset */
2434 0, /* tp_descr_get */
2435 0, /* tp_descr_set */
2436 0, /* tp_dictoffset */
2437 file_init, /* tp_init */
2438 PyType_GenericAlloc, /* tp_alloc */
2439 file_new, /* tp_new */
2440 PyObject_Del, /* tp_free */
2443 /* Interface for the 'soft space' between print items. */
2446 PyFile_SoftSpace(PyObject *f, int newflag)
2452 else if (PyFile_Check(f)) {
2453 oldflag = ((PyFileObject *)f)->f_softspace;
2454 ((PyFileObject *)f)->f_softspace = newflag;
2458 v = PyObject_GetAttrString(f, "softspace");
2463 oldflag = PyInt_AsLong(v);
2464 assert(oldflag < INT_MAX);
2467 v = PyInt_FromLong((long)newflag);
2471 if (PyObject_SetAttrString(f, "softspace", v) != 0)
2476 return (int)oldflag;
2479 /* Interfaces to write objects/strings to file-like objects */
2482 PyFile_WriteObject(PyObject *v, PyObject *f, int flags)
2484 PyObject *writer, *value, *args, *result;
2486 PyErr_SetString(PyExc_TypeError, "writeobject with NULL file");
2489 else if (PyFile_Check(f)) {
2490 PyFileObject *fobj = (PyFileObject *) f;
2491 #ifdef Py_USING_UNICODE
2492 PyObject *enc = fobj->f_encoding;
2495 if (fobj->f_fp == NULL) {
2499 #ifdef Py_USING_UNICODE
2500 if ((flags & Py_PRINT_RAW) &&
2501 PyUnicode_Check(v) && enc != Py_None) {
2502 char *cenc = PyString_AS_STRING(enc);
2503 char *errors = fobj->f_errors == Py_None ?
2504 "strict" : PyString_AS_STRING(fobj->f_errors);
2505 value = PyUnicode_AsEncodedString(v, cenc, errors);
2512 result = file_PyObject_Print(value, fobj, flags);
2516 return file_PyObject_Print(v, fobj, flags);
2519 writer = PyObject_GetAttrString(f, "write");
2522 if (flags & Py_PRINT_RAW) {
2523 if (PyUnicode_Check(v)) {
2527 value = PyObject_Str(v);
2530 value = PyObject_Repr(v);
2531 if (value == NULL) {
2535 args = PyTuple_Pack(1, value);
2541 result = PyEval_CallObject(writer, args);
2552 PyFile_WriteString(const char *s, PyObject *f)
2556 /* Should be caused by a pre-existing error */
2557 if (!PyErr_Occurred())
2558 PyErr_SetString(PyExc_SystemError,
2559 "null file for PyFile_WriteString");
2562 else if (PyFile_Check(f)) {
2563 PyFileObject *fobj = (PyFileObject *) f;
2564 FILE *fp = PyFile_AsFile(f);
2569 FILE_BEGIN_ALLOW_THREADS(fobj)
2571 FILE_END_ALLOW_THREADS(fobj)
2574 else if (!PyErr_Occurred()) {
2575 PyObject *v = PyString_FromString(s);
2579 err = PyFile_WriteObject(v, f, Py_PRINT_RAW);
2587 /* Try to get a file-descriptor from a Python object. If the object
2588 is an integer or long integer, its value is returned. If not, the
2589 object's fileno() method is called if it exists; the method must return
2590 an integer or long integer, which is returned as the file descriptor value.
2591 -1 is returned on failure.
2594 int PyObject_AsFileDescriptor(PyObject *o)
2599 if (PyInt_Check(o)) {
2600 fd = PyInt_AsLong(o);
2602 else if (PyLong_Check(o)) {
2603 fd = PyLong_AsLong(o);
2605 else if ((meth = PyObject_GetAttrString(o, "fileno")) != NULL)
2607 PyObject *fno = PyEval_CallObject(meth, NULL);
2612 if (PyInt_Check(fno)) {
2613 fd = PyInt_AsLong(fno);
2616 else if (PyLong_Check(fno)) {
2617 fd = PyLong_AsLong(fno);
2621 PyErr_SetString(PyExc_TypeError,
2622 "fileno() returned a non-integer");
2628 PyErr_SetString(PyExc_TypeError,
2629 "argument must be an int, or have a fileno() method.");
2634 PyErr_Format(PyExc_ValueError,
2635 "file descriptor cannot be a negative integer (%i)",
2642 /* From here on we need access to the real fgets and fread */
2647 ** Py_UniversalNewlineFgets is an fgets variation that understands
2648 ** all of \r, \n and \r\n conventions.
2649 ** The stream should be opened in binary mode.
2650 ** If fobj is NULL the routine always does newline conversion, and
2651 ** it may peek one char ahead to gobble the second char in \r\n.
2652 ** If fobj is non-NULL it must be a PyFileObject. In this case there
2653 ** is no readahead but in stead a flag is used to skip a following
2654 ** \n on the next read. Also, if the file is open in binary mode
2655 ** the whole conversion is skipped. Finally, the routine keeps track of
2656 ** the different types of newlines seen.
2657 ** Note that we need no error handling: fgets() treats error and eof
2661 Py_UniversalNewlineFgets(char *buf, int n, FILE *stream, PyObject *fobj)
2665 int newlinetypes = 0;
2667 int univ_newline = 1;
2670 if (!PyFile_Check(fobj)) {
2671 errno = ENXIO; /* What can you do... */
2674 univ_newline = ((PyFileObject *)fobj)->f_univ_newline;
2675 if ( !univ_newline )
2676 return fgets(buf, n, stream);
2677 newlinetypes = ((PyFileObject *)fobj)->f_newlinetypes;
2678 skipnextlf = ((PyFileObject *)fobj)->f_skipnextlf;
2681 c = 'x'; /* Shut up gcc warning */
2682 while (--n > 0 && (c = GETC(stream)) != EOF ) {
2686 /* Seeing a \n here with skipnextlf true
2687 ** means we saw a \r before.
2689 newlinetypes |= NEWLINE_CRLF;
2691 if (c == EOF) break;
2694 ** Note that c == EOF also brings us here,
2695 ** so we're okay if the last char in the file
2698 newlinetypes |= NEWLINE_CR;
2702 /* A \r is translated into a \n, and we skip
2703 ** an adjacent \n, if any. We don't set the
2704 ** newlinetypes flag until we've seen the next char.
2708 } else if ( c == '\n') {
2709 newlinetypes |= NEWLINE_LF;
2712 if (c == '\n') break;
2714 if ( c == EOF && skipnextlf )
2715 newlinetypes |= NEWLINE_CR;
2716 FUNLOCKFILE(stream);
2719 ((PyFileObject *)fobj)->f_newlinetypes = newlinetypes;
2720 ((PyFileObject *)fobj)->f_skipnextlf = skipnextlf;
2721 } else if ( skipnextlf ) {
2722 /* If we have no file object we cannot save the
2723 ** skipnextlf flag. We have to readahead, which
2724 ** will cause a pause if we're reading from an
2725 ** interactive stream, but that is very unlikely
2726 ** unless we're doing something silly like
2727 ** execfile("/dev/tty").
2739 ** Py_UniversalNewlineFread is an fread variation that understands
2740 ** all of \r, \n and \r\n conventions.
2741 ** The stream should be opened in binary mode.
2742 ** fobj must be a PyFileObject. In this case there
2743 ** is no readahead but in stead a flag is used to skip a following
2744 ** \n on the next read. Also, if the file is open in binary mode
2745 ** the whole conversion is skipped. Finally, the routine keeps track of
2746 ** the different types of newlines seen.
2749 Py_UniversalNewlineFread(char *buf, size_t n,
2750 FILE *stream, PyObject *fobj)
2753 PyFileObject *f = (PyFileObject *)fobj;
2754 int newlinetypes, skipnextlf;
2756 assert(buf != NULL);
2757 assert(stream != NULL);
2759 if (!fobj || !PyFile_Check(fobj)) {
2760 errno = ENXIO; /* What can you do... */
2763 if (!f->f_univ_newline)
2764 return fread(buf, 1, n, stream);
2765 newlinetypes = f->f_newlinetypes;
2766 skipnextlf = f->f_skipnextlf;
2767 /* Invariant: n is the number of bytes remaining to be filled
2775 nread = fread(dst, 1, n, stream);
2780 n -= nread; /* assuming 1 byte out for each in; will adjust */
2781 shortread = n != 0; /* true iff EOF or error */
2785 /* Save as LF and set flag to skip next LF. */
2789 else if (skipnextlf && c == '\n') {
2790 /* Skip LF, and remember we saw CR LF. */
2792 newlinetypes |= NEWLINE_CRLF;
2796 /* Normal char to be stored in buffer. Also
2797 * update the newlinetypes flag if either this
2798 * is an LF or the previous char was a CR.
2801 newlinetypes |= NEWLINE_LF;
2802 else if (skipnextlf)
2803 newlinetypes |= NEWLINE_CR;
2809 /* If this is EOF, update type flags. */
2810 if (skipnextlf && feof(stream))
2811 newlinetypes |= NEWLINE_CR;
2815 f->f_newlinetypes = newlinetypes;
2816 f->f_skipnextlf = skipnextlf;