libcom_err: Fix file descriptor leak after an exec

Some applications repeatedly re-exec themselves, and if they use the
com_err library, they can leak a file descriptor for each re-exec.
Fix this by setting the close-on-exec flag on the debug file
descriptor.  In addition, if the COMERR_DEBUG environment variable
isn't set, don't open the file handle at all.

Addresses-Red-Hat-Bugzilla: #464689

Signed-off-by: "Theodore Ts'o" <tytso@mit.edu>

diff --git a/lib/et/error_message.c b/lib/et/error_message.c
index 11bda00..d0e90e1 100644 (file)
--- a/lib/et/error_message.c
+++ b/lib/et/error_message.c
@@ -34,6 +34,7 @@
 #if HAVE_UNISTD_H
 #include <unistd.h>
 #endif
+#include <fcntl.h>
 #if HAVE_SYS_TYPES_H
 #include <sys/types.h>
 #endif
@@ -201,25 +202,38 @@ static FILE *debug_f = 0;
 
 static void init_debug(void)
 {
-       char *dstr;
-       char *fn;
+       char    *dstr, *fn, *tmp;
+       int     fd, flags;
 
        if (debug_mask & DEBUG_INIT)
                return;
 
        dstr = getenv("COMERR_DEBUG");
-       if (dstr)
-               debug_mask = strtoul(dstr, 0, 0);
+       if (dstr) {
+               debug_mask = strtoul(dstr, &tmp, 0);
+               if (*tmp || errno)
+                       debug_mask = 0;
+       }
+
+       debug_mask |= DEBUG_INIT;
+       if (debug_mask == DEBUG_INIT)
+               return;
 
        fn = safe_getenv("COMERR_DEBUG_FILE");
        if (fn)
                debug_f = fopen(fn, "a");
        if (!debug_f)
                debug_f = fopen("/dev/tty", "a");
-       if (!debug_f)
-               debug_mask = 0;
+       if (debug_f) {
+               fd = fileno(debug_f);
+               if (fd >= 0) {
+                       flags = fcntl(fd, F_GETFD);
+                       if (flags >= 0)
+                               fcntl(fd, F_SETFD, flags | FD_CLOEXEC);
+               }
+       } else
+               debug_mask = DEBUG_INIT;
 
-       debug_mask |= DEBUG_INIT;
 }
 
 /*