Re: inotify back end for tail -f on linux

[Giuseppe Scrivano]

Hi Jim,

Jim Meyering <address@hidden> writes:

> I haven't fixed that last problem.
> Please fold the following patch into yours for the next iteration.

in this version I included all the problems you reported me, the "touch
k; chmod 0 k; ./tail -F k" test case is included in the tests suite
now.

Regards,
Giuseppe



>From 529e1c2ba2a74168995de9ae7f8b9efa0d2d71c4 Mon Sep 17 00:00:00 2001
From: Giuseppe Scrivano <address@hidden>
Date: Tue, 2 Jun 2009 08:28:23 +0200
Subject: [PATCH] tail: Use inotify if it is available.

* NEWS: Document the new feature.
* m4/jm-macros.m4: Check if inotify is present.
* src/tail.c (tail_forever_inotify): New function.
(main): Use the inotify-based function, if possible.
* tests/Makefile.am: Added new tests for tail.
* tests/tail-2/pid: New file.
* tests/tail-2/wait: New file.
---
 NEWS              |    2 +
 m4/jm-macros.m4   |    5 +
 src/tail.c        |  248 ++++++++++++++++++++++++++++++++++++++++++++++++++++-
 tests/Makefile.am |    2 +
 tests/tail-2/pid  |   81 +++++++++++++++++
 tests/tail-2/wait |  146 +++++++++++++++++++++++++++++++
 6 files changed, 483 insertions(+), 1 deletions(-)
 create mode 100755 tests/tail-2/pid
 create mode 100755 tests/tail-2/wait

diff --git a/NEWS b/NEWS
index 29b09a0..cc61bdc 100644
--- a/NEWS
+++ b/NEWS
@@ -14,6 +14,8 @@ GNU coreutils NEWS                                    -*- 
outline -*-
   sort accepts a new option, --human-numeric-sort (-h): sort numbers
   while honoring human readable suffixes like KiB and MB etc.
 
+  tail uses inotify when possible.
+
 
 * Noteworthy changes in release 7.4 (2009-05-07) [stable]
 
diff --git a/m4/jm-macros.m4 b/m4/jm-macros.m4
index 90c55bf..f14d6a3 100644
--- a/m4/jm-macros.m4
+++ b/m4/jm-macros.m4
@@ -52,6 +52,11 @@ AC_DEFUN([coreutils_MACROS],
   # Used by sort.c.
   AC_CHECK_FUNCS_ONCE([nl_langinfo])
 
+  # Used by tail.c.
+  AC_CHECK_FUNCS([inotify_init],
+    [AC_DEFINE([HAVE_INOTIFY], [1],
+     [Define to 1 if you have usable inotify support.])])
+
   AC_CHECK_FUNCS_ONCE( \
     endgrent \
     endpwent \
diff --git a/src/tail.c b/src/tail.c
index 9d416e1..3a7b022 100644
--- a/src/tail.c
+++ b/src/tail.c
@@ -46,6 +46,11 @@
 #include "xstrtol.h"
 #include "xstrtod.h"
 
+#ifdef HAVE_INOTIFY
+# include "hash.h"
+# include <sys/inotify.h>
+#endif
+
 /* The official name of this program (e.g., no `g' prefix).  */
 #define PROGRAM_NAME "tail"
 
@@ -125,6 +130,17 @@ struct File_spec
   /* The value of errno seen last time we checked this file.  */
   int errnum;
 
+#ifdef HAVE_INOTIFY
+  /* The watch descriptor used by inotify.  */
+  int wd;
+
+  /* The parent directory watch descriptor.  It is used only
+   * when Follow_name is used.  */
+  int parent_wd;
+
+  /* Offset in NAME of the basename part.  */
+  size_t basename_start;
+#endif
 };
 
 /* Keep trying to open a file even if it is inaccessible when tail starts
@@ -1117,6 +1133,219 @@ tail_forever (struct File_spec *f, int nfiles, double 
sleep_interval)
     }
 }
 
+#ifdef HAVE_INOTIFY
+
+static size_t
+wd_hasher (const void *entry, size_t tabsize)
+{
+  const struct File_spec *spec = entry;
+  return spec->wd % tabsize;
+}
+
+static bool
+wd_comparator (const void *e1, const void *e2)
+{
+  const struct File_spec *spec1 = e1;
+  const struct File_spec *spec2 = e2;
+  return spec1->wd == spec2->wd;
+}
+
+/* Tail NFILES files forever, or until killed.
+   Check modifications using the inotify events system.  */
+static void
+tail_forever_inotify (int wd, struct File_spec *f, int nfiles)
+{
+  unsigned int i;
+  unsigned int max_realloc = 3;
+  Hash_table *wd_table;
+
+  bool found_watchable = false;
+  size_t last;
+  size_t evlen = 0;
+  char *evbuff;
+  size_t evbuff_off = 0;
+  ssize_t len = 0;
+
+  wd_table = hash_initialize (nfiles, NULL, wd_hasher, wd_comparator, NULL);
+  if (! wd_table)
+    xalloc_die ();
+
+  for (i = 0; i < nfiles; i++)
+    {
+      if (!f[i].ignore)
+        {
+          int old_wd = f[i].wd;
+          size_t fnlen = strlen (f[i].name);
+          if (evlen < fnlen)
+            evlen = fnlen;
+
+          f[i].wd = 0;
+
+          if (follow_mode == Follow_name)
+            {
+              size_t dirlen = dir_len (f[i].name);
+              char prev = f[i].name[dirlen];
+              f[i].basename_start = last_component (f[i].name) - f[i].name;
+
+              f[i].name[dirlen] = '\0';
+
+               /* It's fine to add the same directory more than once.
+                  In that case the same watch descriptor is returned.  */
+              f[i].parent_wd = inotify_add_watch (wd, dirlen ? f[i].name : ".",
+                                                  IN_CREATE | IN_MOVED_TO);
+
+              f[i].name[dirlen] = prev;
+
+              if (f[i].parent_wd < 0)
+                {
+                  error (0, errno, _("cannot watch parent directory of %s"),
+                         quote (f[i].name));
+                  continue;
+                }
+            }
+
+          old_wd = f[i].wd;
+          f[i].wd = inotify_add_watch (wd, f[i].name,
+                                       (IN_MODIFY | IN_ATTRIB
+                                        | IN_DELETE_SELF | IN_MOVE_SELF));
+
+          if (last == old_wd)
+            last = f[i].wd;
+
+          if (f[i].wd < 0)
+            {
+              if (errno != f[i].errnum)
+                error (0, errno, _("cannot watch %s"), quote (f[i].name));
+              continue;
+            }
+
+          hash_insert (wd_table, &(f[i]));
+
+          if (follow_mode == Follow_name || f[i].wd)
+            found_watchable = true;
+        }
+    }
+
+  if (follow_mode == Follow_descriptor && !found_watchable)
+    return;
+
+  last = f[nfiles - 1].wd;
+
+  evlen += sizeof (struct inotify_event) + 1;
+  evbuff = xmalloc (evlen);
+
+  while (1)
+    {
+      char const *name;
+      struct File_spec *fspec;
+      uintmax_t bytes_read;
+      struct stat stats;
+
+      struct inotify_event *ev;
+
+      if (len <= evbuff_off)
+        {
+          len = safe_read (wd, evbuff, evlen);
+          evbuff_off = 0;
+
+          if (len == SAFE_READ_ERROR && errno == EINVAL && max_realloc--)
+            {
+              len = 0;
+              evlen *= 2;
+              evbuff = xrealloc (evbuff, evlen);
+              continue;
+            }
+
+          if (len == SAFE_READ_ERROR)
+            error (EXIT_FAILURE, errno, _("error reading inotify event"));
+        }
+
+      ev = (struct inotify_event *) (evbuff + evbuff_off);
+      evbuff_off += sizeof (*ev) + ev->len;
+
+      if (ev->mask & (IN_CREATE | IN_MOVED_TO))
+        {
+          for (i = 0; i < nfiles; i++)
+            {
+              if (f[i].parent_wd == ev->wd &&
+                  STREQ (ev->name, f[i].name + f[i].basename_start))
+                break;
+            }
+
+          /* It is not a watched file.  */
+          if (i == nfiles)
+            continue;
+
+          f[i].wd = inotify_add_watch (wd, f[i].name,
+                                       IN_MODIFY | IN_ATTRIB | IN_DELETE_SELF);
+
+          if (f[i].wd < 0)
+            {
+              error (0, errno, _("cannot watch %s"), quote (f[i].name));
+              continue;
+            }
+
+          fspec = &(f[i]);
+          hash_insert (wd_table, fspec);
+
+          if (follow_mode == Follow_name)
+            recheck (&(f[i]), false);
+        }
+      else
+        {
+          struct File_spec key;
+          key.wd = ev->wd;
+          fspec = hash_lookup (wd_table, &key);
+        }
+
+      if (! fspec)
+        continue;
+
+      if (ev->mask & (IN_ATTRIB | IN_DELETE_SELF | IN_MOVE_SELF))
+        {
+          if (ev->mask & (IN_DELETE_SELF | IN_MOVE_SELF))
+            {
+              inotify_rm_watch (wd, f[i].wd);
+              hash_delete (wd_table, &(f[i]));
+            }
+          if (follow_mode == Follow_name)
+            recheck (fspec, false);
+
+          continue;
+        }
+
+      name = pretty_name (fspec);
+
+      if (fstat (fspec->fd, &stats) != 0)
+        {
+          close_fd (fspec->fd, name);
+          fspec->fd = -1;
+          fspec->errnum = errno;
+          continue;
+        }
+
+      if (S_ISREG (fspec->mode) && stats.st_size < fspec->size)
+        {
+          error (0, 0, _("%s: file truncated"), name);
+          last = ev->wd;
+          xlseek (fspec->fd, stats.st_size, SEEK_SET, name);
+          fspec->size = stats.st_size;
+        }
+
+      if (ev->wd != last)
+        {
+          if (print_headers)
+            write_header (name);
+          last = ev->wd;
+        }
+
+      bytes_read = dump_remainder (name, fspec->fd, COPY_TO_EOF);
+      fspec->size += bytes_read;
+    }
+
+}
+#endif
+
 /* Output the last N_BYTES bytes of file FILENAME open for reading in FD.
    Return true if successful.  */
 
@@ -1691,7 +1920,24 @@ main (int argc, char **argv)
     ok &= tail_file (&F[i], n_units);
 
   if (forever)
-    tail_forever (F, n_files, sleep_interval);
+    {
+#ifdef HAVE_INOTIFY
+      if (pid == 0)
+        {
+          int wd = inotify_init ();
+          if (0 <= wd)
+            {
+              tail_forever_inotify (wd, F, n_files);
+
+              /* The only way the above returns is upon failure.  */
+              exit (EXIT_FAILURE);
+            }
+          else
+            error (0, errno, _("inotify cannot be used, reverting to 
polling"));
+        }
+#endif
+      tail_forever (F, n_files, sleep_interval);
+    }
 
   if (have_read_stdin && close (STDIN_FILENO) < 0)
     error (EXIT_FAILURE, errno, "-");
diff --git a/tests/Makefile.am b/tests/Makefile.am
index a0ed986..c108356 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -419,6 +419,8 @@ TESTS =                                             \
   tail-2/big-4gb                               \
   tail-2/proc-ksyms                            \
   tail-2/start-middle                          \
+  tail-2/pid                                   \
+  tail-2/wait                                  \
   touch/dangling-symlink                       \
   touch/dir-1                                  \
   touch/fail-diag                              \
diff --git a/tests/tail-2/pid b/tests/tail-2/pid
new file mode 100755
index 0000000..ed7bc7e
--- /dev/null
+++ b/tests/tail-2/pid
@@ -0,0 +1,81 @@
+#!/bin/sh
+# Test the --pid option of tail.
+
+# Copyright (C) 2003, 2006-2009 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+if test "$VERBOSE" = yes; then
+  set -x
+  tail --version
+fi
+
+. $srcdir/test-lib.sh
+
+sleep 2 &
+pid=$!
+sleep .5
+grep '^State:[  ]*[S]' /proc/$pid/status > /dev/null 2>&1 ||
+  skip_test_ "/proc/$pid/status: missing or 'different'"
+kill $pid
+
+if [ ! -e here ]; then
+  touch here || framework_failure
+fi
+
+fail=0
+
+
+# Use tail itself to create a background process.
+
+tail -f here &
+bg_pid=$!
+
+tail -f here --pid=$bg_pid &
+
+pid=$!
+
+sleep 2
+
+set _ `sed -n '/^State:[        ]*\([^  ]\)/s//\1/p' /proc/$pid/status`
+shift # Remove the leading `_'.
+state=$1
+
+if [ ! -z $state ]; then
+  case $state in
+    S*) ;;
+    *) echo $0: process dead 1>&2; fail=1 ;;
+  esac
+  kill $pid
+fi
+
+sleep 2
+
+set _ `sed -n '/^State:[        ]*\([^  ]\)/s//\1/p' /proc/$pid/status`
+shift
+state=$1
+
+if [ ! -z $state ]; then
+  case $state in
+    *) ;;
+    S*) echo $0: process still active 1>&2; fail=1 ;;
+  esac
+  kill $pid
+fi
+
+
+kill $bg_pid
+rm here
+
+Exit $fail
diff --git a/tests/tail-2/wait b/tests/tail-2/wait
new file mode 100755
index 0000000..e4cd9f2
--- /dev/null
+++ b/tests/tail-2/wait
@@ -0,0 +1,146 @@
+#!/bin/sh
+# Make sure that `tail -f' returns immediately if a file doesn't exist
+# while `tail -F' waits for it to appear.
+
+# Copyright (C) 2003, 2006-2009 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+if test "$VERBOSE" = yes; then
+  set -x
+  tail --version
+fi
+
+. $srcdir/test-lib.sh
+
+sleep 2 &
+pid=$!
+sleep .5
+grep '^State:[  ]*[S]' /proc/$pid/status > /dev/null 2>&1 ||
+  skip_test_ "/proc/$pid/status: missing or 'different'"
+kill $pid
+
+touch here || framework_failure
+
+(touch not_accessible && chmod 0 not_accessible) || framework_failure
+
+if [ -e not_here ]; then
+  rm -f not_here || framework_failure
+fi
+
+fail=0
+
+rm -f  tail.err
+
+(tail -f not_here 2> tail.err) &
+pid=$!
+sleep .5
+set _ `sed -n '/^State:[        ]*\([^  ]\)/s//\1/p' /proc/$pid/status`
+shift # Remove the leading `_'.
+state=$1
+
+if [ ! -z $state ]; then
+  case $state in
+    *) ;;
+    S*) echo $0: process still active 1>&2; fail=1 ;;
+  esac
+  kill $pid
+fi
+
+(tail -f not_accessible 2> tail.err) &
+pid=$!
+sleep .5
+set _ `sed -n '/^State:[        ]*\([^  ]\)/s//\1/p' /proc/$pid/status`
+shift
+state=$1
+
+if [ ! -z $state ]; then
+  case $state in
+    *) ;;
+    S*) echo $0: process still active 1>&2; fail=1 ;;
+  esac
+  kill $pid
+fi
+
+(tail -f here 2>> tail.err) &
+pid=$!
+sleep .5
+set _ `sed -n '/^State:[        ]*\([^  ]\)/s//\1/p' /proc/$pid/status`
+shift
+state=$1
+
+if [ ! -z $state ]; then
+  case $state in
+    S*) ;;
+    *)  echo $0: process died 1>&2; fail=1 ;;
+  esac
+  kill $pid
+fi
+
+
+# `tail -F' must wait in any case.
+
+(tail -F here 2>> tail.err) &
+pid=$!
+sleep .5
+set _ `sed -n '/^State:[        ]*\([^  ]\)/s//\1/p' /proc/$pid/status`
+shift
+state=$1
+
+if [ ! -z $state ]; then
+  case $state in
+    S*) ;;
+    *) echo $0: process died 1>&2; fail=1 ;;
+  esac
+  kill $pid
+fi
+
+(tail -F not_accessible 2>> tail.err) &
+pid=$!
+sleep .5
+set _ `sed -n '/^State:[        ]*\([^  ]\)/s//\1/p' /proc/$pid/status`
+shift
+state=$1
+
+if [ ! -z $state ]; then
+  case $state in
+    S*) ;;
+    *) echo $0: process died 1>&2; fail=1 ;;
+  esac
+  kill $pid
+fi
+
+(tail -F not_here 2>> tail.err) &
+pid=$!
+sleep .5
+set _ `sed -n '/^State:[        ]*\([^  ]\)/s//\1/p' /proc/$pid/status`
+shift
+state=$1
+
+if [ ! -z $state ]; then
+  case $state in
+    S*) ;;
+    *) echo $0: process died 1>&2; fail=1 ;;
+  esac
+  kill $pid
+fi
+
+test  "$(wc -w < tail.err) eq 0"  || fail=1
+
+rm -f here
+rm -f tail.err
+rm -f not_accessible
+
+
+Exit $fail
-- 
1.6.3.1