[GNUnet-SVN] r17857 - in gnunet-update: doc gnunet_update

[gnunet]

Author: harsha
Date: 2011-10-30 04:46:55 +0100 (Sun, 30 Oct 2011)
New Revision: 17857

Modified:
   gnunet-update/doc/metadata.txt
   gnunet-update/gnunet_update/dependency.py
   gnunet-update/gnunet_update/install.py
   gnunet-update/gnunet_update/metadata.py
   gnunet-update/gnunet_update/package.py
Log:
metadata file reading

Modified: gnunet-update/doc/metadata.txt
===================================================================
--- gnunet-update/doc/metadata.txt      2011-10-29 11:09:39 UTC (rev 17856)
+++ gnunet-update/doc/metadata.txt      2011-10-30 03:46:55 UTC (rev 17857)
@@ -6,11 +6,15 @@
   containing `%%'. Currently there are 3 sections:
   1. The metadata header which consists of information about the host
      (the machine on which the package was built)
-  2. The metadata body which is a listing of the dependencies of
-     binary objects in the package
-  3. A listing of signatures of the binary objects to verify the
-     authenticity of each binary object in the package
+  2. The metadata body which is a list of the dependencies of binary objects in
+     the package 
+  3. A list of signatures of the binary objects to verify the authenticity of
+     each binary object in the package 
 
+  IMPORTANT: The lists in  metadata body and signatures should correspond to
+  each other. For every object listed in metadata body, there should be a
+  signature associated with that file in signature list
+
 ** Metadata header
    It consists of information about the host system. This is needed to
    identify the subset of machine which have the capability to execute
@@ -32,17 +36,17 @@
       identify their own release information
 
 ** Metadata body
-   This is a listing of dependencies for each binary object. 
+   This is a list of dependencies for each binary object. 
 
    If a binary object foo needs a shared library libbar.so.X where X
    is the major number for libbar, the dependency information for A is
    stored as follows:
-      A;libbar.so.X;X:Y:Z
+      A;libbar.so.X;X;Y;Z
    where Y, Z are the minor and revision numbers of libbar.so.X. In
    case minor and revision numbers for libbar cannot be determined,
    they are set to -1
 
-** Listing of signatures
+** List of signatures
    This is a list of sha512 digest of each file in the package
    expressed in hexadecimal format. For each file an entry begins with
    its file name followed by a `:' and then the sha512 digest.

Modified: gnunet-update/gnunet_update/dependency.py
===================================================================
--- gnunet-update/gnunet_update/dependency.py   2011-10-29 11:09:39 UTC (rev 
17856)
+++ gnunet-update/gnunet_update/dependency.py   2011-10-30 03:46:55 UTC (rev 
17857)
@@ -30,7 +30,7 @@
 class Dependency:
     """Class for holding data for a dependency"""
     major = minor = rev = None   
-    def __init__(self, name, path):
+    def __init__(self, name, path=None):
         """Creates a new dependency object with name and path."""
         self.name = name
         self.path = path
@@ -48,22 +48,24 @@
 class BinaryObject:
     """Class representing executable code."""
     
-    def __init__(self, path, name=None):
+    def __init__(self, path=None, name=None):
         """Returns am instance of BinaryObject."""
         self.name = name
         self.path = path
         self._deps = list()
         
-        #Calculate the hash of this binary object
-        hash_obj = sha512()
-        object_file = open(path, "rb")
-        while True:
-            #read 512 bytes - suitable for sha512 blocks
-            data = object_file.read(512)
-            if 0 == len(data): #End of file is reached
-                self.hash = hash_obj.hexdigest()
-                break;
-            hash_obj.update(data);
+        if path != None:
+            #Calculate the hash of this binary object
+            hash_obj = sha512()
+            object_file = open(path, "rb")
+            while True:
+                #read 512 bytes - suitable for sha512 blocksdepif
+                data = object_file.read(512)
+                if 0 == len(data): #End of file is reached
+                    self.hash = hash_obj.hexdigest()
+                    break
+                hash_obj.update(data)
+            object_file.close()
         
     def add_dependency(self, dep):
         """Adds dep object to the list of dependencies."""
@@ -76,12 +78,11 @@
     def _dependency_ascii(self, dep):
         """Given a dependency, returns an ascii line describing it."""
         dep_str = self.name + ";" + dep.name + ";"
-        dep_str += "-1:" if dep.major == None else dep.major + ":"
-        dep_str += "-1:" if dep.minor == None else dep.minor + ":"
+        dep_str += "-1;" if dep.major == None else dep.major + ";"
+        dep_str += "-1;" if dep.minor == None else dep.minor + ";"
         dep_str += "-1" if dep.rev == None else dep.rev
         return dep_str + "\n"
     
     def dependency_listlines(self):
         """Return list of lines which describe all dependencies."""
         return map(self._dependency_ascii, self._deps)
-

Modified: gnunet-update/gnunet_update/install.py
===================================================================
--- gnunet-update/gnunet_update/install.py      2011-10-29 11:09:39 UTC (rev 
17856)
+++ gnunet-update/gnunet_update/install.py      2011-10-30 03:46:55 UTC (rev 
17857)
@@ -31,13 +31,13 @@
 import sys
 import os
 import getopt
-import re
 import platform
+from metadata import Metadata
 
 def usage():
     """Print helpful usage information."""    
     print """
-Usage arguments: [options] <package_file> </path/to/install/location>
+Usage arguments: [options] <metadata_file> <package_file> </install/location>
 This script tries to install the contents in the package file in the given 
 location.
 
@@ -60,13 +60,15 @@
             usage()
             sys.exit(2)
     
-    if len(args) != 2:
-        print "insufficient number of arguments"
+    if len(args) != 3:
+        print "Incorrect number of arguments"
         usage()
         sys.exit(1)
     
-    package_tarfile = tarfile.open(args[0],'r')
-    install_dir = args[1]
+    metadata = Metadata()
+    metadata.readfromfile(args[0])
+    package_tarfile = tarfile.open(args[1],'r')
+    install_dir = args[2]
     
     try: 
         metadata_tarinfo = package_tarfile.getmember("metadata.dat")
@@ -76,39 +78,18 @@
         package_tarfile.close()
         sys.exit(2)
     
-    metadata_file = package_tarfile.extractfile(metadata_tarinfo)
-    regex = re.compile("(?P<key>OS|MACHINE):(?P<value>.*)\n")
-    
     #check whether the host os and machine architecture match    
-    OS_OK = MACHINE_OK = False
     host_os = platform.system()
     host_machine = platform.machine()
-    header_lines = 2
     
-    while not (OS_OK and MACHINE_OK):
-        if header_lines == 0:
-            print "The given package is not suited for this platform."
-            metadata_file.close()
-            package_tarfile.close()
-            sys.exit(1)
-            
-        match = regex.match(metadata_file.readline())
-        if not match:
-            print "Provided package doesn't conform to the expected format."
-            metadata_file.close()
-            package_tarfile.close()
-            sys.exit(1)
-            
-        if "OS" == match.group("key"):
-            OS_OK = host_os == match.group("value") 
-        if "MACHINE" == match.group("key"):
-            MACHINE_OK = host_machine == match.group("value")
-    metadata_file.close()
+    if metadata.os != host_os or metadata.machine != host_machine:
+        print "The given package is not suited for this platform."
+        sys.exit(1)
     
     #Platform check is done; now unpack the tarfile into destination directory
     try:
         os.stat(install_dir)
-    except OSError:
+    except OSError:             # Given directory not present 
         os.mkdir(install_dir, 0755)
     
     #FIXME: Security warning! Perhaps we should examin the contents of tarfile
@@ -121,4 +102,4 @@
     package_tarfile.close()
     
 if "__main__" == __name__:
-    main()
\ No newline at end of file
+    main()

Modified: gnunet-update/gnunet_update/metadata.py
===================================================================
--- gnunet-update/gnunet_update/metadata.py     2011-10-29 11:09:39 UTC (rev 
17856)
+++ gnunet-update/gnunet_update/metadata.py     2011-10-30 03:46:55 UTC (rev 
17857)
@@ -16,28 +16,28 @@
 # Free Software Foundation, Inc., 59 Temple Place - Suite 330,
 # Boston, MA 02111-1307, USA.
 #
-#File:     gnunet_update/install.py
+#File:     gnunet_update/metadata.py
 #Author:   Sree Harsha Totakura
 #
 #To handle metadata files.
 
 import tempfile
+from dependency import Dependency, BinaryObject
 
-
 class Metadata:
     """Class for holding metadata information."""
-    _machine = None
-    _os = None
-    _pkey = None
-    _release = None
+    machine = None
+    os = None
+    pkey = None
+    release = None
     _binary_objects = None
 
     def __init__(self, machine=None, os=None, pkey=None,
                  release=None):
-        self._machine = machine
-        self._os = os
-        self._pkey = pkey
-        self._release = release
+        self.machine = machine
+        self.os = os
+        self.pkey = pkey
+        self.release = release
 
     def setbinaryobjects(self, binary_objects):
         """Setter for _binary_objects."""
@@ -66,25 +66,24 @@
             f = open(path, "wb") #binary mode for compatibility
             file_name = path
 
-        def __writeln(str): #helper function to get writeln functionality
+        def writeln_(str): #helper function to get writeln functionality
             f.write(str + '\n')
         
         #write the header
-        if self._machine != None: __writeln("MACHINE:" + 
-                                             self._machine)
-        if self._os != None: __writeln("OS:" + self._os)
-        if self._pkey != None: __writeln("PKEY:" + self._pkey)
-        if self._release != None: __writeln("RELEASE:" + self._release)
+        if self.machine != None: writeln_("MACHINE:" + self.machine)
+        if self.os != None: writeln_("OS:" + self.os)
+        if self.pkey != None: writeln_("PKEY:" + self.pkey)
+        if self.release != None: writeln_("RELEASE:" + self.release)
 
         #write the metadata body
-        __writeln("%%") #section seperator
+        writeln_("%%") #section seperator
         for binary_object in self._binary_objects:
             f.writelines(binary_object.dependency_listlines())
 
         #write the signatures
-        __writeln("%%") #section seperator
+        writeln_("%%") #section seperator
         for binary_object in self._binary_objects:
-            __writeln(binary_object.name + ":" + binary_object.hash)
+            writeln_(binary_object.name + ":" + binary_object.hash)
 
         #close file
         if None == path:
@@ -96,3 +95,73 @@
         
     def readfromfile(self, path):
         """Reads metadata from the file at path."""
+        f = open(path, "rb") # binary for compatibility
+
+        def error_exit():
+            f.close()
+            exit(1)
+
+        # read until first `%%'
+        while True: # parses the header 
+            read_line = f.readline()
+            if len(read_line) == 0:
+                print "Unrecognized metadata file"
+                error_exit()
+            if "%%" == read_line[:2] : 
+                break
+            tokens = read_line.split(':')
+            # FIXME: Sanity check on file format
+            key = tokens[0]
+            value = tokens[1][:-1] # last character will be `\n'
+            if "MACHINE" == key:
+                self.machine = value
+            elif "OS" == key:
+                self.os = value
+            elif "PKEY" == key:
+                self.pkey = value
+            elif "RELEASE" == key:
+                self.release = value
+
+        # read until next `%%'
+        seen_bin_object = None
+        self._binary_objects = list()
+        while True:
+            read_line = f.readline()
+            if len(read_line) == 0:
+                print "Unrecognized metadata file"
+                error_exit()
+            if "%%" == read_line[:2]:
+                break
+            tokens = read_line.split(';')
+            bin_name = tokens[0]
+            dep_name = tokens[1]
+            dep_maj = int(tokens[2])
+            dep_min = int(tokens[3])
+            dep_rev = int(tokens[4][:-1])# last character will be `\n'
+            if None == seen_bin_object or seen_bin_object.name != bin_name:
+                seen_bin_object = BinaryObject(name=bin_name)
+                self._binary_objects.append(seen_bin_object)
+            dep = Dependency(dep_name)
+            if -1 != dep_maj: dep.maj = dep_maj
+            if -1 != dep_min: dep.min = dep_min
+            if -1 != dep_rev: dep.rev = dep_rev
+            seen_bin_object.add_dependency(dep)
+            
+        # read the hashes from signatures
+        for binary in self._binary_objects:
+            read_line = f.readline()
+            if len(read_line) == 0:
+                print "Unrecognized metadata file"
+                error_exit()
+            tokens = read_line.split(':')
+            bin_name = tokens[0]
+            hash = tokens[1][:-1] # last character will be `\n'
+            # FIXME: We are reading based on the assumption that the order of
+            # binary objects in the metadata body and signatures is
+            # similar. However this may not be the case according to the
+            # documentation.
+            if binary.name != bin_name:
+                print "Unrecognized file format"
+                error_exit()
+            binary.hash = hash
+        f.close()

Modified: gnunet-update/gnunet_update/package.py
===================================================================
--- gnunet-update/gnunet_update/package.py      2011-10-29 11:09:39 UTC (rev 
17856)
+++ gnunet-update/gnunet_update/package.py      2011-10-30 03:46:55 UTC (rev 
17857)
@@ -55,7 +55,7 @@
 Usage arguments: [options] /path/to/gnunet/source package-file
 This script compiles and builds given gnunet source tree. It then attempts to 
 install it and packs the installed files along with their dependencies into
-package-file
+package-file. It also generates a metadata file named <package-file>.meta
 
 Options:
     -h, --help        : prints this message
@@ -205,12 +205,9 @@
     tar_file.add(install_prefix, "install-prefix")
     
     #generate the metadata file and add it to tar
-    metadata_file = metadata.writetofile()
+    metadata_file = metadata.writetofile(package_file + ".meta")
     tar_file.add(metadata_file, "metadata.dat")
     
-    #delete the temporarily generated metadata file
-    os.remove(metadata_file)
-    
     print "Here are the dependencies:"
     for dep in dependencies:
         print dep.name