suppress bogus ioctl on fd==INT_MAX when vasprintf() is called

libc/stdio/_fopen.c

@@ -121,11 +121,11 @@ FILE attribute_hidden *_stdio_fopen(intptr_t fname_or_mode,
 		i = (open_mode & (O_ACCMODE|O_LARGEFILE)) + 1;
 
 		/* NOTE: fopencookie needs changing if the basic check changes! */
-		if (((i & (((int) fname_or_mode) + 1)) != i) /* Basic agreement? */
-			|| (((open_mode & ~((__mode_t) fname_or_mode)) & O_APPEND)
-				&& fcntl(filedes, F_SETFL, O_APPEND))	/* Need O_APPEND. */
-			) {
+		if ((i & ((int)fname_or_mode + 1)) != i) /* Basic agreement? */
 			goto DO_EINVAL;
+		if ((open_mode & ~(__mode_t)fname_or_mode) & O_APPEND) {
+			if (fcntl(filedes, F_SETFL, O_APPEND))	/* Need O_APPEND. */
+				goto DO_EINVAL;
 		}
 		/* For later... to reflect largefile setting in stream flags. */
 		__STDIO_WHEN_LFS( open_mode |= (((__mode_t) fname_or_mode)
@@ -159,9 +159,15 @@ FILE attribute_hidden *_stdio_fopen(intptr_t fname_or_mode,
 		((((open_mode & O_ACCMODE) + 1) ^ 0x03) * __FLAG_WRITEONLY);
 
 #ifdef __STDIO_BUFFERS
-	i = errno;					/* Preserve errno against isatty call. */
-	stream->__modeflags |= (isatty(stream->__filedes) * __FLAG_LBF);
-	__set_errno(i);
+	if (stream->__filedes != INT_MAX) {
+		/* NB: fopencookie uses bogus filedes == INT_MAX,
+		 * avoiding isatty() in that case.
+		 */
+		i = errno; /* preserve errno against isatty call. */
+		if (isatty(stream->__filedes))
+			stream->__modeflags |= __FLAG_LBF;
+		__set_errno(i);
+	}
 
 	if (!stream->__bufstart) {
 		if ((stream->__bufstart = malloc(BUFSIZ)) != NULL) {

libc/stdio/_stdio.c

@@ -253,8 +253,10 @@ void attribute_hidden _stdio_init(void)
 #ifdef __STDIO_BUFFERS
 	int old_errno = errno;
 	/* stdin and stdout uses line buffering when connected to a tty. */
-	_stdio_streams[0].__modeflags ^= (1-isatty(0)) * __FLAG_LBF;
-	_stdio_streams[1].__modeflags ^= (1-isatty(1)) * __FLAG_LBF;
+	if (!isatty(0))
+		_stdio_streams[0].__modeflags ^= __FLAG_LBF;
+	if (!isatty(1))
+		_stdio_streams[1].__modeflags ^= __FLAG_LBF;
 	__set_errno(old_errno);
 #endif
 #ifndef __UCLIBC__

libc/stdio/fdopen.c

@@ -14,8 +14,9 @@ FILE *fdopen(int filedes, const char *mode)
 {
 	intptr_t cur_mode;
 
-	return (((cur_mode = fcntl(filedes, F_GETFL))) != -1)
-		? _stdio_fopen(cur_mode, mode, NULL, filedes)
-		: NULL;
+	cur_mode = fcntl(filedes, F_GETFL);
+	if (cur_mode != -1)
+		return _stdio_fopen(cur_mode, mode, NULL, filedes);
+	return NULL;
 }
 libc_hidden_def(fdopen)

libc/stdio/fopen.c

@@ -9,9 +9,9 @@
 
 #ifndef __DO_LARGEFILE
 # define FILEDES_ARG    (-1)
-#undef fopen
+# undef fopen
 #else
-#undef fopen64
+# undef fopen64
 #endif
 
 /* libc_hidden_proto(fopen) */