Re: inotify back end for tail -f on linux

[Giuseppe Scrivano]

Jim Meyering <address@hidden> writes:

> If we add inotify support, I'd like it to work for
> both tail-by-FD and tail-by-name.
>
> You can watch the parent directory for e.g., rename and creation events.
> Log rotation is so common, that it'd be a shame not to be able to take
> advantage of inotify when using "tail -F ...".
>
> For starters, you needn't worry about recovering from parent-dir-removal.
> I think that parent removal is rare enough in practice that we won't
> need to deal with it for some time.
>
> Eventually, the code could monitor the longest parent prefix that *does* 
> exist.
> When the missing subdir reappears, monitor $parent/$subdir, etc.
> However, there's no need to do that now.

Considering only the parent directory makes things easier but I don't
think that in future it will be difficult to remove completely this
limit.

The version I attached is a bit more complex than the last one.  Since
the watch descriptors are added at any time, I can't do any consideration
on their range and I am using a hash table to access the files
information in a costant time.

I need to stress my patch better but it seems already to cover all cases
you showed me.


> I haven't looked closely at the code below or even tried it,
> but here are a few superficial comments:

Thank you.  I included them except this one:

>> +      if (!len && errno == EINTR)
>> +        continue;
>> +
>> +      if (!len)
>> +        error (EXIT_FAILURE, errno, _("error reading inotify event"));
>
> This function must not exit upon a failure that is specific
> to a single file.  Just diagnose it and continue with any other files.
> Of course, if there are no other usable file descriptors, *then*
> you will need to exit the loop -- since this is currently used only
> when tailing-by-FD.
>
> You would not exit a similar loop when tailing by name.

I added some code to recover from errors when possible.  I don't think
there are other recoverable cases but if we want to force it to re-try
read then we will need to avoid an end-less loop when there is an
unrecoverable situation.  What do you think?

Regards,
Giuseppe


>From 706774fb68dde926343cc906dc627891e42504a9 Mon Sep 17 00:00:00 2001
From: Giuseppe Scrivano <address@hidden>
Date: Sat, 30 May 2009 13:31:58 +0200
Subject: [PATCH]     tail: Use inotify if it is available.

    * NEWS: Document the new feature
    * configure.ac: Check if inotify is present.
    * src/tail.c (main): Use the tail_forever inotify version if it
    is possible.
    (tail_forever_inotify): Added new function.
---
 NEWS         |    2 +
 configure.ac |    2 +
 src/tail.c   |  247 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
 3 files changed, 249 insertions(+), 2 deletions(-)

diff --git a/NEWS b/NEWS
index 29b09a0..c5a2ef5 100644
--- a/NEWS
+++ b/NEWS
@@ -14,6 +14,8 @@ GNU coreutils NEWS                                    -*- 
outline -*-
   sort accepts a new option, --human-numeric-sort (-h): sort numbers
   while honoring human readable suffixes like KiB and MB etc.
 
+  tail uses inotify if it is present.
+
 
 * Noteworthy changes in release 7.4 (2009-05-07) [stable]
 
diff --git a/configure.ac b/configure.ac
index 4eb640e..a63ac0c 100644
--- a/configure.ac
+++ b/configure.ac
@@ -410,6 +410,8 @@ AM_GNU_GETTEXT_VERSION([0.15])
 # For a test of uniq: it uses the $LOCALE_FR envvar.
 gt_LOCALE_FR
 
+AC_CHECK_FUNCS([inotify_init], [AC_DEFINE([HAVE_INOTIFY], [1], [Check for 
libinotify])])
+
 AC_CONFIG_FILES(
   Makefile
   doc/Makefile
diff --git a/src/tail.c b/src/tail.c
index fe34600..6335815 100644
--- a/src/tail.c
+++ b/src/tail.c
@@ -1,5 +1,5 @@
 /* tail -- output the last part of file(s)
-   Copyright (C) 1989, 90, 91, 1995-2006, 2008 Free Software Foundation, Inc.
+   Copyright (C) 1989, 90, 91, 1995-2006, 2008, 2009 Free Software Foundation, 
Inc.
 
    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -45,6 +45,12 @@
 #include "xstrtol.h"
 #include "xstrtod.h"
 
+#ifdef HAVE_INOTIFY
+# include "hash.h"
+# include "dirname.h"
+# include <sys/inotify.h>
+#endif
+
 /* The official name of this program (e.g., no `g' prefix).  */
 #define PROGRAM_NAME "tail"
 
@@ -124,6 +130,17 @@ struct File_spec
   /* The value of errno seen last time we checked this file.  */
   int errnum;
 
+#ifdef HAVE_INOTIFY
+  /* The watch descriptor used by inotify.  */
+  int wd;
+
+  /* The parent directory watch descriptor.  It is used only
+   * when Follow_name is used.  */
+  int parent_wd;
+
+  /* Offset in NAME of the basename part.  */
+  size_t basename_start;
+#endif
 };
 
 /* Keep trying to open a file even if it is inaccessible when tail starts
@@ -1116,6 +1133,225 @@ tail_forever (struct File_spec *f, int nfiles, double 
sleep_interval)
     }
 }
 
+#ifdef HAVE_INOTIFY
+
+static size_t
+wd_hasher (const void *entry, size_t tabsize)
+{
+  const struct File_spec *spec = entry;
+  return spec->wd % tabsize;
+}
+
+static bool
+wd_comparator (const void *e1, const void *e2)
+{
+  const struct File_spec *spec1 = e1;
+  const struct File_spec *spec2 = e2;
+  return spec1->wd == spec2->wd;
+}
+
+/* Tail NFILES files forever, or until killed.
+   Check modifications using the inotify events system.  */
+static void
+tail_forever_inotify (int wd, struct File_spec *f, int nfiles)
+{
+  /* Create an index to access File_spec by its watch descriptor
+   * in costant time.  */
+  struct File_spec **wd_index;
+  int i;
+  int max_realloc = 3;
+  Hash_table *wd_table;
+
+  int watched = 0;
+  size_t last;
+  size_t evlen = 0;
+  char *evbuff;
+  size_t evbuff_off = 0;
+  ssize_t len = 0;
+
+  wd_table = hash_initialize (nfiles, NULL, wd_hasher,
+                              wd_comparator,
+                              NULL);
+  if (! wd_table)
+    xalloc_die ();
+
+  for (i = 0; i < nfiles; i++)
+    {
+      if (!f[i].ignore)
+        {
+          size_t fnlen = strlen (f[i].name);
+          if (evlen < fnlen)
+            evlen = fnlen;
+
+          f[i].wd = 0;
+
+          if (follow_mode == Follow_name)
+            {
+              size_t dirlen = dir_len (f[i].name);
+              char prev = f[i].name[dirlen];
+              f[i].basename_start = last_component (f[i].name) - f[i].name;
+
+              f[i].name[dirlen] = '\0';
+
+              /* Do not care if the directory was already added, in this case 
the
+               * same watch descriptor will be returned.  */
+              f[i].parent_wd = inotify_add_watch (wd, f[i].name, IN_CREATE|
+                                                  IN_MOVED_TO);
+
+              f[i].name[dirlen] = prev;
+
+              if (f[i].parent_wd < 0)
+                {
+                  error (0, errno, _("cannot watch parent directory of %s"),
+                         quote (f[i].name));
+                  continue;
+                }
+            }
+
+          if (f[i].errnum != ENOENT)
+            {
+              f[i].wd = inotify_add_watch (wd, f[i].name, IN_MODIFY|
+                                           IN_ATTRIB|IN_DELETE_SELF|
+                                           IN_MOVE_SELF);
+
+              if (f[i].wd < 0)
+                {
+                  error (0, errno, _("cannot watch %s"), quote (f[i].name));
+                  continue;
+                }
+
+              hash_insert (wd_table, &(f[i]));
+            }
+
+            if (follow_mode == Follow_name || f[i].wd)
+                watched++;
+        }
+    }
+
+  if (!watched)
+    return;
+
+  last = f[nfiles - 1].wd;
+
+  evlen += sizeof (struct inotify_event) + 1;
+  evbuff = xmalloc (evlen);
+
+  while (1)
+    {
+      char inotify_buffer;
+      char const *name;
+      struct File_spec *fspec;
+      uintmax_t bytes_read;
+      struct stat stats;
+
+      struct inotify_event *ev;
+
+      evbuff_off += sizeof (*ev) + ev->len;
+
+      if (len <= evbuff_off)
+        {
+          len = read (wd, evbuff, evlen);
+          evbuff_off = 0;
+
+          if (len <= 0 && errno == EINVAL && max_realloc--)
+            {
+              len = 0;
+              evlen *= 2;
+              evbuff = xrealloc (evbuff, evlen);
+              continue;
+            }
+
+          if (len <= 0 && errno == EINTR)
+            {
+              len = 0;
+              continue;
+            }
+
+          if (len <= 0)
+            error (EXIT_FAILURE, errno, _("error reading inotify event"));
+        }
+
+      ev = (struct inotify_event*)(evbuff + evbuff_off);
+
+      if (ev->mask & (IN_CREATE|IN_MOVED_TO))
+        {
+          for (i = 0; i < nfiles; i++)
+            if (f[i].parent_wd == ev->wd &&
+                !strcmp (ev->name, f[i].name + f[i].basename_start))
+              break;
+
+          /* It is not a watched file.  */
+          if (i == nfiles)
+            continue;
+
+          f[i].wd = inotify_add_watch (wd, f[i].name, IN_MODIFY|IN_ATTRIB|
+                                       IN_DELETE_SELF);
+
+          if (f[i].wd < 0)
+            {
+              error (0, errno, _("cannot watch %s"), quote (f[i].name));
+              continue;
+            }
+
+          fspec = &(f[i]);
+          hash_insert (wd_table, fspec);
+
+          recheck (&(f[i]), false);
+        }
+      else
+        {
+          struct File_spec key;
+          key.wd = ev->wd;
+          fspec = hash_lookup (wd_table, &key);
+        }
+
+      if (! fspec)
+        continue;
+
+      if (ev->mask & (IN_ATTRIB | IN_DELETE_SELF | IN_MOVE_SELF))
+        {
+          if (ev->mask & (IN_DELETE_SELF | IN_MOVE_SELF))
+            {
+              inotify_rm_watch (wd, f[i].wd);
+              hash_delete (wd_table, &(f[i]));
+            }
+
+          recheck (fspec, false);
+          continue;
+        }
+
+      name = pretty_name (fspec);
+
+      if (fstat (fspec->fd, &stats) != 0)
+        {
+          close_fd (fspec->fd, name);
+          fspec->fd = -1;
+          fspec->errnum = errno;
+          continue;
+        }
+
+      if (S_ISREG (fspec->mode) && stats.st_size < fspec->size)
+        {
+          error (0, 0, _("%s: file truncated"), name);
+          last = ev->wd;
+          xlseek (fspec->fd, stats.st_size, SEEK_SET, name);
+          fspec->size = stats.st_size;
+        }
+
+      if (ev->wd != last)
+        {
+          if (print_headers)
+            write_header (name);
+          last = ev->wd;
+        }
+
+      bytes_read = dump_remainder (name, fspec->fd, COPY_TO_EOF);
+      fspec->size += bytes_read;
+    }
+
+}
+#endif
+
 /* Output the last N_BYTES bytes of file FILENAME open for reading in FD.
    Return true if successful.  */
 
@@ -1690,7 +1926,14 @@ main (int argc, char **argv)
     ok &= tail_file (&F[i], n_units);
 
   if (forever)
-    tail_forever (F, n_files, sleep_interval);
+    {
+#ifdef HAVE_INOTIFY
+      int wd = inotify_init ();
+      if (wd > 0)
+        tail_forever_inotify (wd, F, n_files);
+#endif
+      tail_forever (F, n_files, sleep_interval);
+    }
 
   if (have_read_stdin && close (STDIN_FILENO) < 0)
     error (EXIT_FAILURE, errno, "-");
-- 
1.6.3.1