[Rdiff-backup-commits] rdiff-backup CHANGELOG rdiff_backup/Security.py...

[Andrew Ferguson]

CVSROOT:        /sources/rdiff-backup
Module name:    rdiff-backup
Changes by:     Andrew Ferguson <owsla> 08/07/13 22:04:29

Modified files:
        .              : CHANGELOG 
        rdiff_backup   : Security.py rpath.py 

Log message:
        Move make_file_dict_python so that it is run on the remote end instead 
of
        the local end.

CVSWeb URLs:
http://cvs.savannah.gnu.org/viewcvs/rdiff-backup/CHANGELOG?cvsroot=rdiff-backup&r1=1.289&r2=1.290
http://cvs.savannah.gnu.org/viewcvs/rdiff-backup/rdiff_backup/Security.py?cvsroot=rdiff-backup&r1=1.33&r2=1.34
http://cvs.savannah.gnu.org/viewcvs/rdiff-backup/rdiff_backup/rpath.py?cvsroot=rdiff-backup&r1=1.124&r2=1.125

Patches:
Index: CHANGELOG
===================================================================
RCS file: /sources/rdiff-backup/rdiff-backup/CHANGELOG,v
retrieving revision 1.289
retrieving revision 1.290
diff -u -b -r1.289 -r1.290
--- CHANGELOG   13 Jul 2008 19:58:55 -0000      1.289
+++ CHANGELOG   13 Jul 2008 22:04:28 -0000      1.290
@@ -1,6 +1,12 @@
 New in v1.1.17 (????/??/??)
 ---------------------------
 
+Move make_file_dict_python so that it is run on the remote end instead of
+the local end. This improves performance for Windows hosts since it eliminates
+the lag due to checking os.name. It also makes the Windows design parallel
+to the Posix design, since the Windows method now returns a dictionary across
+the wire. (Andrew Ferguson)
+
 Catch EPERM error when trying to write extended attributes. (Andrew Ferguson)
 
 Allow rdiff-backup to be built into a single executable on Windows using

Index: rdiff_backup/Security.py
===================================================================
RCS file: /sources/rdiff-backup/rdiff-backup/rdiff_backup/Security.py,v
retrieving revision 1.33
retrieving revision 1.34
diff -u -b -r1.33 -r1.34
--- rdiff_backup/Security.py    9 Jul 2007 03:53:40 -0000       1.33
+++ rdiff_backup/Security.py    13 Jul 2008 22:04:29 -0000      1.34
@@ -41,7 +41,7 @@
 #
 # The keys are files request, the value is the index of the argument
 # taking a file.
-file_requests = {'os.listdir':0, 'C.make_file_dict':0, 'os.chmod':0,
+file_requests = {'os.listdir':0, 'rpath.make_file_dict':0, 'os.chmod':0,
                                 'os.chown':0, 'os.remove':0, 'os.removedirs':0,
                                 'os.rename':0, 'os.renames':0, 'os.rmdir':0, 
'os.unlink':0,
                                 'os.utime':0, 'os.lchown':0, 'os.link':1, 
'os.symlink':1,
@@ -136,7 +136,7 @@
                 "sys.stdout.write", "robust.install_signal_handlers"]
        if (sec_level == "read-only" or sec_level == "update-only" or
                sec_level == "all"):
-               l.extend(["C.make_file_dict", "os.listdir", "rpath.ea_get",
+               l.extend(["rpath.make_file_dict", "os.listdir", "rpath.ea_get",
                                  "rpath.acl_get", "rpath.setdata_local",
                                  "log.Log.log_to_file", "os.getuid", 
"Time.setcurtime_local",
                                  "rpath.gzip_open_local_read", 
"rpath.open_local_read",

Index: rdiff_backup/rpath.py
===================================================================
RCS file: /sources/rdiff-backup/rdiff-backup/rdiff_backup/rpath.py,v
retrieving revision 1.124
retrieving revision 1.125
diff -u -b -r1.124 -r1.125
--- rdiff_backup/rpath.py       5 Jul 2008 19:04:44 -0000       1.124
+++ rdiff_backup/rpath.py       13 Jul 2008 22:04:29 -0000      1.125
@@ -36,7 +36,7 @@
 """
 
 import os, stat, re, sys, shutil, gzip, socket, time, errno
-import Globals, Time, static, log, user_group
+import Globals, Time, static, log, user_group, C
 
 
 class SkipFileException(Exception):
@@ -264,16 +264,65 @@
                rp_dest.data = rp_source.data
                rp_source.data = {'type': None}
 
-def tupled_lstat(filename):
-       """Like os.lstat, but return only a tuple, or None if os.error
+def make_file_dict(filename):
+       """Generate the data dictionary for the given RPath
 
-       Later versions of os.lstat return a special lstat object,
-       which can confuse the pickler and cause errors in remote
-       operations.  This has been fixed in Python 2.2.1.
+       This is a global function so that os.name can be called locally,
+       thus avoiding network lag and so that we only need to send the
+       filename over the network, thus avoiding the need to pickle an
+       (incomplete) rpath object.
+       """
+       if os.name != 'nt':
+               return C.make_file_dict(filename)
+       else:
+               return make_file_dict_python(filename)
+
+def make_file_dict_python(filename):
+       """Create the data dictionary using a Python call to os.lstat
 
+       We do this on Windows since Python's implementation is much better
+       than the one in cmodule.c    Eventually, we will move to using
+       this on all platforms since CPUs have gotten much faster than
+       they were when it was necessary to write cmodule.c
        """
-       try: return tuple(os.lstat(filename))
-       except os.error: return None
+       try:
+               statblock = os.lstat(filename)
+       except os.error:
+               return {'type':None}
+       data = {}
+       mode = statblock[stat.ST_MODE]
+
+       if stat.S_ISREG(mode): type = 'reg'
+       elif stat.S_ISDIR(mode): type = 'dir'
+       elif stat.S_ISCHR(mode):
+               type = 'dev'
+               s = statblock.st_rdev
+               data['devnums'] = ('c',) + (s >> 8, s & 0xff)
+       elif stat.S_ISBLK(mode):
+               type = 'dev'
+               s = statblock.st_rdev
+               data['devnums'] = ('b',) + (s >> 8, s & 0xff)
+       elif stat.S_ISFIFO(mode): type = 'fifo'
+       elif stat.S_ISLNK(mode):
+               type = 'sym'
+               data['linkname'] = os.readlink(filename)
+       elif stat.S_ISSOCK(mode): type = 'sock'
+       else: raise C.UnknownFileError(filename)
+       data['type'] = type
+       data['size'] = statblock[stat.ST_SIZE]
+       data['perms'] = stat.S_IMODE(mode)
+       data['uid'] = statblock[stat.ST_UID]
+       data['gid'] = statblock[stat.ST_GID]
+       data['inode'] = statblock[stat.ST_INO]
+       data['devloc'] = statblock[stat.ST_DEV]
+       data['nlink'] = statblock[stat.ST_NLINK]
+
+       if not (type == 'sym' or type == 'dev'):
+               # mtimes on symlinks and dev files don't work consistently
+               data['mtime'] = long(statblock[stat.ST_MTIME])
+               data['atime'] = long(statblock[stat.ST_ATIME])
+               data['ctime'] = long(statblock[stat.ST_CTIME])
+       return data
 
 def make_socket_local(rpath):
        """Make a local socket at the given path
@@ -826,51 +875,10 @@
                self.path = "/".join((self.base,) + self.index)
 
        def setdata(self):
-               """Set data dictionary using C extension"""
-               try:
-                       self.data = self.conn.C.make_file_dict(self.path)
-               except AttributeError:
-                       self.data = self.make_file_dict_python()
+               """Set data dictionary using the wrapper"""
+               self.data = self.conn.rpath.make_file_dict(self.path)
                if self.lstat(): self.conn.rpath.setdata_local(self)
 
-       def make_file_dict_python(self):
-               """Create the data dictionary"""
-               statblock = self.conn.rpath.tupled_lstat(self.path)
-               if statblock is None:
-                       return {'type':None}
-               data = {}
-               mode = statblock[stat.ST_MODE]
-
-               if stat.S_ISREG(mode): type = 'reg'
-               elif stat.S_ISDIR(mode): type = 'dir'
-               elif stat.S_ISCHR(mode):
-                       type = 'dev'
-                       data['devnums'] = ('c',) + self._getdevnums()
-               elif stat.S_ISBLK(mode):
-                       type = 'dev'
-                       data['devnums'] = ('b',) + self._getdevnums()
-               elif stat.S_ISFIFO(mode): type = 'fifo'
-               elif stat.S_ISLNK(mode):
-                       type = 'sym'
-                       data['linkname'] = self.conn.os.readlink(self.path)
-               elif stat.S_ISSOCK(mode): type = 'sock'
-               else: raise C.UnknownFileError(self.path)
-               data['type'] = type
-               data['size'] = statblock[stat.ST_SIZE]
-               data['perms'] = stat.S_IMODE(mode)
-               data['uid'] = statblock[stat.ST_UID]
-               data['gid'] = statblock[stat.ST_GID]
-               data['inode'] = statblock[stat.ST_INO]
-               data['devloc'] = statblock[stat.ST_DEV]
-               data['nlink'] = statblock[stat.ST_NLINK]
-
-               if not (type == 'sym' or type == 'dev'):
-                       # mtimes on symlinks and dev files don't work 
consistently
-                       data['mtime'] = long(statblock[stat.ST_MTIME])
-                       data['atime'] = long(statblock[stat.ST_ATIME])
-                       data['ctime'] = long(statblock[stat.ST_CTIME])
-               return data
-
        def check_consistency(self):
                """Raise an error if consistency of rp broken
 
@@ -884,11 +892,6 @@
                           "\nName: %s\nOld: %s --> New: %s\n" % \
                           (self.path, temptype, self.data['type'])
 
-       def _getdevnums(self):
-               """Return tuple for special file (major, minor)"""
-               s = self.conn.reval("lambda path: os.lstat(path).st_rdev", 
self.path)
-               return (s >> 8, s & 0xff)
-
        def chmod(self, permissions):
                """Wrapper around os.chmod"""
                try: