[Top][All Lists]
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[GNUnet-SVN] r23065 - in gnunet-planetlab/gplmt: . contrib contrib/taskl
|
From: |
gnunet |
|
Subject: |
[GNUnet-SVN] r23065 - in gnunet-planetlab/gplmt: . contrib contrib/tasklists gplmt |
|
Date: |
Fri, 3 Aug 2012 16:01:14 +0200 |
Author: wachs
Date: 2012-08-03 16:01:14 +0200 (Fri, 03 Aug 2012)
New Revision: 23065
Added:
gnunet-planetlab/gplmt/contrib/tasklist_schema.xsd
gnunet-planetlab/gplmt/gplmt/
gnunet-planetlab/gplmt/gplmt/Configuration.py
gnunet-planetlab/gplmt/gplmt/Nodes.py
gnunet-planetlab/gplmt/gplmt/Notifications.py
gnunet-planetlab/gplmt/gplmt/SCP.py
gnunet-planetlab/gplmt/gplmt/Tasks.py
gnunet-planetlab/gplmt/gplmt/Util.py
gnunet-planetlab/gplmt/gplmt/Worker.py
Removed:
gnunet-planetlab/gplmt/Configuration.py
gnunet-planetlab/gplmt/Nodes.py
gnunet-planetlab/gplmt/Notifications.py
gnunet-planetlab/gplmt/SCP.py
gnunet-planetlab/gplmt/Tasks.py
gnunet-planetlab/gplmt/Util.py
gnunet-planetlab/gplmt/Worker.py
gnunet-planetlab/gplmt/tasklist_schema.xsd
Modified:
gnunet-planetlab/gplmt/README
gnunet-planetlab/gplmt/contrib/tasklists/check_fedora_version.xml
gnunet-planetlab/gplmt/contrib/tasklists/check_node.xml
gnunet-planetlab/gplmt/contrib/tasklists/deploy_gnunet_fc12.xml
gnunet-planetlab/gplmt/contrib/tasklists/deploy_gnunet_fc8.xml
gnunet-planetlab/gplmt/contrib/tasklists/start_gnunet.xml
gnunet-planetlab/gplmt/contrib/tasklists/stop_gnunet.xml
gnunet-planetlab/gplmt/contrib/tasklists/test_sftp_tasks.xml
gnunet-planetlab/gplmt/contrib/tasklists/test_tasks.xml
gnunet-planetlab/gplmt/contrib/tasklists/update_gnunet.xml
gnunet-planetlab/gplmt/gplmt.py
Log:
cleanup
Deleted: gnunet-planetlab/gplmt/Configuration.py
===================================================================
--- gnunet-planetlab/gplmt/Configuration.py 2012-08-03 14:00:04 UTC (rev
23064)
+++ gnunet-planetlab/gplmt/Configuration.py 2012-08-03 14:01:14 UTC (rev
23065)
@@ -1,128 +0,0 @@
-#!/usr/bin/python
-#
-# This file is part of GNUnet.
-# (C) 2010 Christian Grothoff (and other contributing authors)
-#
-# GNUnet is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published
-# by the Free Software Foundation; either version 2, or (at your
-# option) any later version.
-#
-# GNUnet is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with GNUnet; see the file COPYING. If not, write to the
-# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-# Boston, MA 02111-1307, USA.
-#
-# GNUnet Planetlab deployment and automation toolset
-#
-# Configuration
-
-import ConfigParser
-import os
-
-class TransferMode:
- none=0
- sftp=1
- scp=2
-
-class Configuration:
- def __init__(self, filename, logger):
- assert (None != logger)
- self.filename = filename
- self.logger = logger
- self.notifications = ""
- self.pl_slicename = ""
- self.pl_api_url = ""
- self.pl_username = ""
- self.pl_password = ""
- self.pl_use_nodes = False
- self.taskfile = ""
- self.nodesfile = ""
- self.ssh_add_unkown_hostkeys = False
- self.ssh_keyfile = None
- self.ssh_password = ""
- self.ssh_use_known_hosts = False
- self.ssh_transfer = TransferMode.scp
- def load (self):
- self.logger.log ("Loading configuration file '" + self.filename + "'")
- if (False == os.path.exists (self.filename)):
- print "File does not exist: '" + self.filename + "'"
- return False
- config = ConfigParser.RawConfigParser()
- try:
- config.read(self.filename)
- except ConfigParser.Error as e:
- print "Error parsing configuration: " + str (e)
- return False
-
- # required values
- try:
- self.pl_slicename = config.get("planetlab", "slice")
- except ConfigParser.NoOptionError as e:
- print "Error parsing configuration: " + str (e)
- return False
- # optional values
- try:
- self.pl_use_nodes = config.getboolean ("planetlab", "use_pl_nodes")
- except ConfigParser.NoOptionError as e:
- pass
- try:
- self.pl_api_url = config.get("planetlab", "api_url")
- except ConfigParser.NoOptionError as e:
- pass
- try:
- self.pl_username = config.get("planetlab", "username")
- except ConfigParser.NoOptionError as e:
- pass
- try:
- self.pl_password = config.get("planetlab", "password")
- except ConfigParser.NoOptionError as e:
- pass
- try:
- # gplmt options
- self.taskfile = config.get("gplmt", "tasks")
- except ConfigParser.NoOptionError as e:
- pass
-
- try:
- self.nodesfile = config.get("gplmt", "nodes")
- except ConfigParser.NoOptionError as e:
- pass
-
- # ssh options
- try:
- self.ssh_add_unkown_hostkeys = config.getboolean ("ssh",
"add_unkown_hostkeys")
- except ConfigParser.NoOptionError as e:
- pass
- try:
- self.ssh_use_known_hosts = config.getboolean ("ssh",
"ssh_use_known_hosts")
- except ConfigParser.NoOptionError as e:
- pass
- try:
- self.ssh_keyfile = config.get("ssh", "ssh_keyfile")
- except ConfigParser.NoOptionError as e:
- pass
- try:
- self.ssh_password = config.get("ssh", "ssh_password")
- except ConfigParser.NoOptionError as e:
- pass
- try:
- transfer = config.get("ssh", "ssh_transfer")
- if (transfer == "scp"):
- self.ssh_transfer = TransferMode.scp
- elif (transfer == "sftp"):
- self.ssh_transfer = TransferMode.sftp
- else:
- print "Invalid ssh transfer mode: only SFTP or SCP are
supported"
- return False
- except ConfigParser.NoOptionError as e:
- pass
-
-
- return True
-
\ No newline at end of file
Deleted: gnunet-planetlab/gplmt/Nodes.py
===================================================================
--- gnunet-planetlab/gplmt/Nodes.py 2012-08-03 14:00:04 UTC (rev 23064)
+++ gnunet-planetlab/gplmt/Nodes.py 2012-08-03 14:01:14 UTC (rev 23065)
@@ -1,102 +0,0 @@
-#!/usr/bin/python
-#
-# This file is part of GNUnet.
-# (C) 2010 Christian Grothoff (and other contributing authors)
-#
-# GNUnet is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published
-# by the Free Software Foundation; either version 2, or (at your
-# option) any later version.
-#
-# GNUnet is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with GNUnet; see the file COPYING. If not, write to the
-# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-# Boston, MA 02111-1307, USA.
-#
-# GNUnet Planetlab deployment and automation toolset
-#
-# Tasks
-import sys, os, urllib, xmlrpclib, socket
-
-class Nodes:
- def __init__(self, filename, logger):
- assert (None != logger)
- self.logger = logger
- self.filename = filename
- self.nodes = list ()
- def load (self):
- self.logger.log ("Loading nodes file '" + self.filename + "'")
- try:
- fobj = open (self.filename, "r")
- for line in fobj:
- line = line.strip()
- self.logger.log ("Found node '" + line + "'")
- self.nodes.append(line)
- fobj.close()
- except IOError:
- print "File " + self.filename + " not found"
- return False
- self.logger.log ("Loaded " + str(len(self.nodes)) + " nodes")
- return True
-
-class StringNodes:
- def __init__(self, str, logger):
- assert (None != logger)
- self.str = str
- self.logger = logger
- self.nodes = list ()
- def load (self):
- self.logger.log ("Loading nodes '" + self.str + "'")
- self.nodes.append(self.str)
- self.logger.log ("Loaded " + str(len(self.nodes)) + " nodes")
- return True
-
-class PlanetLabNodes:
- def __init__(self, configuration, logger):
- assert (None != logger)
- self.logger = logger
- self.configuration = configuration
- self.nodes = list ()
- def load (self):
-
- if (self.configuration.pl_password == ""):
- print "No PlanetLab password given in configuration fail!"
- return False
- if (self.configuration.pl_username == ""):
- print "No PlanetLab username given in configuration, fail!"
- return False
- if (self.configuration.pl_api_url == ""):
- print "No PlanetLab API url given in configuration, fail!"
- return False
- self.logger.log ("Retrieving nodes assigned to slice '" +
self.configuration.pl_slicename + "' for user " +self.configuration.pl_username)
- try:
- server = xmlrpclib.ServerProxy(self.configuration.pl_api_url)
- except:
- print "Could not connect to PlanetLab API, fail!"
- return False
-
- slice_data = {}
- slice_data['name'] = self.configuration.pl_slicename
-
- auth = {}
- auth['Username'] = self.configuration.pl_username
- auth['AuthString'] = self.configuration.pl_password
- auth['AuthMethod'] = "password"
-
- try:
- node_ids = server.GetSlices(auth, [slice_data['name']],
['node_ids'])[0]['node_ids']
- node_hostnames = [node['hostname'] for node in
server.GetNodes(auth, node_ids, ['hostname'])]
- except Exception as e:
- print "Could not retrieve data from PlanetLab API: " + str(e)
- return False
-
- for node in node_hostnames:
- self.logger.log ("Planetlab API returned: " + node)
- self.nodes.append(node)
- self.logger.log ("Planetlab API returned " + str(len(self.nodes)) + "
nodes")
- return True
\ No newline at end of file
Deleted: gnunet-planetlab/gplmt/Notifications.py
===================================================================
--- gnunet-planetlab/gplmt/Notifications.py 2012-08-03 14:00:04 UTC (rev
23064)
+++ gnunet-planetlab/gplmt/Notifications.py 2012-08-03 14:01:14 UTC (rev
23065)
@@ -1,166 +0,0 @@
-#!/usr/bin/python
-#
-# This file is part of GNUnet.
-# (C) 2010 Christian Grothoff (and other contributing authors)
-#
-# GNUnet is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published
-# by the Free Software Foundation; either version 2, or (at your
-# option) any later version.
-#
-# GNUnet is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with GNUnet; see the file COPYING. If not, write to the
-# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-# Boston, MA 02111-1307, USA.
-#
-# GNUnet Planetlab deployment and automation toolset
-#
-# Notifications
-import Tasks
-import time
-
-class Notification:
- def __init__(self, logger):
- assert (None != logger)
- self.logger = logger
- def node_connected (self, node, success):
- assert (0)
- def node_disconnected (self, node, success):
- assert (0)
- def tasklist_started (self, node, tasks):
- assert (0)
- def tasklist_completed (self, node, tasks, success):
- assert (0)
- def task_started (self, node, tasks):
- assert (0)
- def task_completed (self, node, tasks, success):
- assert (0)
-
-class NodeCollection:
- def __init__(self):
- self.nodes = list ()
- def add (self, node):
- self.nodes.append (node)
- def get (self, name):
- for n in self.nodes:
- if (n.name == name):
- return n
- return None
-
-class Task:
- def __init__(self, task, result, fail):
- self.task = task
- self.result = result
- self.fail = fail
-
-class Node:
- def __init__(self, name):
- self.name = name
- self.start = time.time()
- self.end = 0
- self.tasks = list ()
-
-
-class FileLoggerNotification (Notification):
- def __init__(self, logger):
- assert (None != logger)
- self.logger = logger
- self.nodes = NodeCollection ()
- def node_connected (self, node, success):
- return
- def node_disconnected (self, node, success):
- return
- def tasklist_started (self, node, tasks):
- return
- def tasklist_completed (self, node, tasks, success):
- self.nodes.add (Node(node))
- if (success == True):
- print node + " : Tasklist '" + tasks.name + "' completed
successfully"
- else:
- print node + " : Tasklist '" + tasks.name + "' completed with
failure"
- def task_started (self, node, task):
- return
- def task_completed (self, node, task, result):
- return
-
-
-class TaskListResultNotification (Notification):
- def __init__(self, logger):
- assert (None != logger)
- self.logger = logger
- #self.nodes = NodeCollection ()
- def summarize (self):
- for n in self.nodes:
- tsk_str = ""
- for t in n.tasks:
- tsk_f = "[e]"
- if (t.fail == True):
- tsk_f = "[f]"
- else:
- tsk_f = "[s]"
- tsk_str += t.task.name + " " + tsk_f + " ->"
- print n.name
- def node_connected (self, node, success):
- return
- def node_disconnected (self, node, success):
- return
- def tasklist_started (self, node, tasks):
- return
- def tasklist_completed (self, node, tasks, success):
- if (success == True):
- print node + " : Tasklist '" + tasks.name + "' completed
successfully"
- else:
- print node + " : Tasklist '" + tasks.name + "' completed with
failure"
- def task_started (self, node, task):
- return
- def task_completed (self, node, task, result):
- return
-
-class SimpleNotification (Notification):
- def __init__(self, logger):
- assert (None != logger)
- self.logger = logger
- #self.nodes = NodeCollection ()
- def summarize (self):
- for n in self.nodes:
- tsk_str = ""
- for t in n.tasks:
- tsk_f = "[e]"
- if (t.fail == True):
- tsk_f = "[f]"
- else:
- tsk_f = "[s]"
- tsk_str += t.task.name + " " + tsk_f + " ->"
- print n.name
- def node_connected (self, node, success):
- if (success == True):
- print node + " : connected successfully"
- else:
- print node + " : connection failed"
- def node_disconnected (self, node, success):
- if (success == True):
- print node + " : disconnected"
- else:
- print node + " : disconnected with failure"
- def tasklist_started (self, node, tasks):
- print node + " : Tasklist '" + tasks.name + "' started"
- #self.nodes.add (Node(node))
- def tasklist_completed (self, node, tasks, success):
- if (success == True):
- print node + " : Tasklist '" + tasks.name + "' completed
successfully"
- else:
- print node + " : Tasklist '" + tasks.name + "' completed with
failure"
- def task_started (self, node, task):
- print node + " : Task '" + task.name + "' started"
- def task_completed (self, node, task, result):
- if (result == Tasks.Taskresult.success):
- print node + " : Task '" + task.name + "' completed successfully"
- elif (result == Tasks.Taskresult.src_file_not_found):
- print node + " : Task '" + task.name + "' failed : source file
not found"
- else:
- print node + " : Task '" + task.name + "' completed with failure"
\ No newline at end of file
Modified: gnunet-planetlab/gplmt/README
===================================================================
--- gnunet-planetlab/gplmt/README 2012-08-03 14:00:04 UTC (rev 23064)
+++ gnunet-planetlab/gplmt/README 2012-08-03 14:01:14 UTC (rev 23065)
@@ -34,6 +34,11 @@
- Task list validation against XML Schema
- Extensible logging functionality
+Content
+=============
+
+
+
Dependencies:
=============
Deleted: gnunet-planetlab/gplmt/SCP.py
===================================================================
--- gnunet-planetlab/gplmt/SCP.py 2012-08-03 14:00:04 UTC (rev 23064)
+++ gnunet-planetlab/gplmt/SCP.py 2012-08-03 14:01:14 UTC (rev 23065)
@@ -1,395 +0,0 @@
-# scp.py
-# Copyright (C) 2008 James Bardin <address@hidden>
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-
-"""
-Utilities for sending files over ssh using the scp1 protocol.
-"""
-
-import os
-from socket import timeout as SocketTimeout
-
-class SCPClient(object):
- """
- An scp1 implementation, compatible with openssh scp.
- Raises SCPException for all transport related errors. Local filesystem
- and OS errors pass through.
-
- Main public methods are .put and .get
- The get method is controlled by the remote scp instance, and behaves
- accordingly. This means that symlinks are resolved, and the transfer is
- halted after too many levels of symlinks are detected.
- The put method uses os.walk for recursion, and sends files accordingly.
- Since scp doesn't support symlinks, we send file symlinks as the file
- (matching scp behaviour), but we make no attempt at symlinked directories.
-
- Convenience methods:
- put_r: put with recursion
- put_p: put preserving times
- put_rp: put with recursion, preserving times
- get_r: get with recursion
- get_p: get preserving times
- get_rp: get with recursion, preserving times
- """
- def __init__(self, transport, buff_size = 16384, socket_timeout = 5.0,
- callback = None):
- """
- Create an scp1 client.
-
- @param transport: an existing paramiko L{Transport}
- @type transport: L{Transport}
- @param buff_size: size of the scp send buffer.
- @type buff_size: int
- @param socket_timeout: channel socket timeout in seconds
- @type socket_timeout: float
- @param callback: callback function for transfer status
- @type callback: func
- """
- self.transport = transport
- self.buff_size = buff_size
- self.socket_timeout = socket_timeout
- self.channel = None
- self.preserve_times = False
- self.callback = callback
- self._recv_dir = ''
- self._utime = None
- self._dirtimes = {}
-
- def put(self, files, remote_path = '.',
- recursive = False, preserve_times = False):
- """
- Transfer files to remote host.
-
- @param files: A single path, or a list of paths to be transfered.
- recursive must be True to transfer directories.
- @type files: string OR list of strings
- @param remote_path: path in which to receive the files on the remote
- host. defaults to '.'
- @type remote_path: str
- @param recursive: transfer files and directories recursively
- @type recursive: bool
- @param preserve_times: preserve mtime and atime of transfered files
- and directories.
- @type preserve_times: bool
- """
- self.preserve_times = preserve_times
- self.channel = self.transport.open_session()
- self.channel.settimeout(self.socket_timeout)
- scp_command = ('scp -t %s\n', 'scp -r -t %s\n')[recursive]
- self.channel.exec_command(scp_command % remote_path)
- self._recv_confirm()
-
- if not isinstance(files, (list, tuple)):
- files = [files]
-
- if recursive:
- self._send_recursive(files)
- else:
- self._send_files(files)
-
- if self.channel:
- self.channel.close()
-
- def put_r(self, files, remote_path = '.'):
- """
- Convenience function for a recursive put
-
- @param files: A single path, or a list of paths to be transfered.
- @type files: str, list
- @param remote_path: path in which to receive the files on the remote
- host. defaults to '.'
- @type remote_path: bool
- """
- self.put(files, remote_path, recursive = True)
-
- def put_p(self, files, remote_path = '.'):
- """
- Convenience function to put preserving times.
-
- @param files: A single path, or a list of paths to be transfered.
- @type files: str, list
- @param remote_path: path in which to receive the files on the remote
- host. defaults to '.'
- @type remote_path: bool
- """
- self.put(files, remote_path, preserve_times = True)
-
- def put_rp(self, files, remote_path = '.'):
- """
- Convenience function for a recursive put, preserving times.
-
- @param files: A single path, or a list of paths to be transfered.
- @type files: str, list
- @param remote_path: path in which to receive the files on the remote
- host. defaults to '.'
- @type remote_path: bool
- """
- self.put(files, remote_path, recursive = True, preserve_times = True)
-
- def get(self, remote_path, local_path = '',
- recursive = False, preserve_times = False):
- """
- Transfer files from remote host to localhost
-
- @param remote_path: path to retreive from remote host. since this is
- evaluated by scp on the remote host, shell wildcards and
- environment variables may be used.
- @type remote_path: str
- @param local_path: path in which to receive files locally
- @type local_path: str
- @param recursive: transfer files and directories recursively
- @type recursive: bool
- @param preserve_times: preserve mtime and atime of transfered files
- and directories.
- @type preserve_times: bool
- """
- self._recv_dir = local_path or os.getcwd()
- rcsv = ('', ' -r')[recursive]
- prsv = ('', ' -p')[preserve_times]
- self.channel = self.transport.open_session()
- self.channel.settimeout(self.socket_timeout)
- self.channel.exec_command('scp%s%s -f %s' % (rcsv, prsv, remote_path))
- self._recv_all()
-
- if self.channel:
- self.channel.close()
-
- def get_r(self, remote_path, local_path = '.'):
- """
- Convenience function for a recursive get
-
- @param remote_path: path to retrieve from server
- @type remote_path: str
- @param local_path: path in which to recieve files. default cwd
- @type local_path: str
- """
- self.get(remote_path, local_path, recursive = True)
-
- def get_p(self, remote_path, local_path = '.'):
- """
- Convenience function for get, preserving times.
-
- @param remote_path: path to retrieve from server
- @type remote_path: str
- @param local_path: path in which to recieve files. default cwd
- @type local_path: str
- """
- self.get(remote_path, local_path, preserve_times = True)
-
- def get_rp(self, remote_path, local_path = '.'):
- """
- Convenience function for a recursive get, preserving times.
-
- @param remote_path: path to retrieve from server
- @type remote_path: str
- @param local_path: path in which to recieve files. default cwd
- @type local_path: str
- """
- self.get(remote_path, local_path, recursive = True, preserve_times =
True)
-
- def _read_stats(self, name):
- """return just the file stats needed for scp"""
- stats = os.stat(name)
- mode = oct(stats.st_mode)[-4:]
- size = stats.st_size
- atime = int(stats.st_atime)
- mtime = int(stats.st_mtime)
- return (mode, size, mtime, atime)
-
- def _send_files(self, files):
- for name in files:
- basename = os.path.basename(name)
- (mode, size, mtime, atime) = self._read_stats(name)
- if self.preserve_times:
- self._send_time(mtime, atime)
- file_hdl = file(name, 'rb')
- self.channel.sendall('C%s %d %s\n' % (mode, size, basename))
- self._recv_confirm()
- file_pos = 0
- buff_size = self.buff_size
- chan = self.channel
- while file_pos < size:
- chan.sendall(file_hdl.read(buff_size))
- file_pos = file_hdl.tell()
- if self.callback:
- self.callback(file_pos, size)
- chan.sendall('\x00')
- file_hdl.close()
-
- def _send_recursive(self, files):
- for base in files:
- lastdir = base
- for root, dirs, fls in os.walk(base):
- # pop back out to the next dir in the walk
- while lastdir != os.path.commonprefix([lastdir, root]):
- self._send_popd()
- lastdir = os.path.split(lastdir)[0]
- self._send_pushd(root)
- lastdir = root
- self._send_files([os.path.join(root, f) for f in fls])
-
- def _send_pushd(self, directory):
- (mode, size, mtime, atime) = self._read_stats(directory)
- basename = os.path.basename(directory)
- if self.preserve_times:
- self._send_time(mtime, atime)
- self.channel.sendall('D%s 0 %s\n' % (mode, basename))
- self._recv_confirm()
-
- def _send_popd(self):
- self.channel.sendall('E\n')
- self._recv_confirm()
-
- def _send_time(self, mtime, atime):
- self.channel.sendall('T%d 0 %d 0\n' % (mtime, atime))
- self._recv_confirm()
-
- def _recv_confirm(self):
- # read scp response
- msg = ''
- try:
- msg = self.channel.recv(512)
- except SocketTimeout:
- raise SCPException('Timout waiting for scp response')
- if msg and msg[0] == '\x00':
- return
- elif msg and msg[0] == '\x01':
- raise SCPException(msg[1:])
- elif self.channel.recv_stderr_ready():
- msg = self.channel.recv_stderr(512)
- raise SCPException(msg)
- elif not msg:
- raise SCPException('No response from server')
- else:
- raise SCPException('Invalid response from server: ' + msg)
-
- def _recv_all(self):
- # loop over scp commands, and recive as necessary
- command = {'C': self._recv_file,
- 'T': self._set_time,
- 'D': self._recv_pushd,
- 'E': self._recv_popd}
- while not self.channel.closed:
- # wait for command as long as we're open
- self.channel.sendall('\x00')
- msg = self.channel.recv(1024)
- if not msg: # chan closed while recving
- break
- code = msg[0]
- try:
- command[code](msg[1:])
- except KeyError:
- raise SCPException(repr(msg))
- # directory times can't be set until we're done writing files
- self._set_dirtimes()
-
- def _set_time(self, cmd):
- try:
- times = cmd.split()
- mtime = int(times[0])
- atime = int(times[2]) or mtime
- except:
- self.channel.send('\x01')
- raise SCPException('Bad time format')
- # save for later
- self._utime = (mtime, atime)
-
- def _recv_file(self, cmd):
- chan = self.channel
- parts = cmd.split()
- try:
- mode = int(parts[0], 8)
- size = int(parts[1])
- path = os.path.join(self._recv_dir, parts[2])
- except:
- chan.send('\x01')
- chan.close()
- raise SCPException('Bad file format')
-
- try:
- file_hdl = file(path, 'wb')
- except IOError, e:
- chan.send('\x01'+e.message)
- chan.close()
- raise
-
- buff_size = self.buff_size
- pos = 0
- chan.send('\x00')
- try:
- while pos < size:
- # we have to make sure we don't read the final byte
- if size - pos <= buff_size:
- buff_size = size - pos
- file_hdl.write(chan.recv(buff_size))
- pos = file_hdl.tell()
- if self.callback:
- self.callback(pos, size)
-
- msg = chan.recv(512)
- if msg and msg[0] != '\x00':
- raise SCPException(msg[1:])
- except SocketTimeout:
- chan.close()
- raise SCPException('Error receiving, socket.timeout')
-
- file_hdl.truncate()
- try:
- os.utime(path, self._utime)
- self._utime = None
- os.chmod(path, mode)
- # should we notify the other end?
- finally:
- file_hdl.close()
- # '\x00' confirmation sent in _recv_all
-
- def _recv_pushd(self, cmd):
- parts = cmd.split()
- try:
- mode = int(parts[0], 8)
- path = os.path.join(self._recv_dir, parts[2])
- except:
- self.channel.send('\x01')
- raise SCPException('Bad directory format')
- try:
- if not os.path.exists(path):
- os.mkdir(path, mode)
- elif os.path.isdir(path):
- os.chmod(path, mode)
- else:
- raise SCPException('%s: Not a directory' % path)
- self._dirtimes[path] = (self._utime)
- self._utime = None
- self._recv_dir = path
- except (OSError, SCPException), e:
- self.channel.send('\x01'+e.message)
- raise
-
- def _recv_popd(self, *cmd):
- self._recv_dir = os.path.split(self._recv_dir)[0]
-
- def _set_dirtimes(self):
- try:
- for d in self._dirtimes:
- os.utime(d, self._dirtimes[d])
- finally:
- self._dirtimes = {}
-
-
-class SCPException(Exception):
- """SCP exception class"""
- pass
Deleted: gnunet-planetlab/gplmt/Tasks.py
===================================================================
--- gnunet-planetlab/gplmt/Tasks.py 2012-08-03 14:00:04 UTC (rev 23064)
+++ gnunet-planetlab/gplmt/Tasks.py 2012-08-03 14:01:14 UTC (rev 23065)
@@ -1,278 +0,0 @@
-#!/usr/bin/python
-#
-# This file is part of GNUnet.
-# (C) 2010 Christian Grothoff (and other contributing authors)
-#
-# GNUnet is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published
-# by the Free Software Foundation; either version 2, or (at your
-# option) any later version.
-#
-# GNUnet is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with GNUnet; see the file COPYING. If not, write to the
-# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-# Boston, MA 02111-1307, USA.
-#
-# GNUnet Planetlab deployment and automation toolset
-#
-# Nodes
-try:
- import xml
- from xml.parsers import expat
- from minixsv import pyxsval as xsv
- from elementtree import ElementTree
- elementtree_loaded = True
- from genxmlif import GenXmlIfError
-except ImportError:
- pass
-
-glogger = None
-
-supported_operations = ["run", "monitor", "get", "put"]
-
-class Taskresult:
- unspecified = -1
- success = 0
- timeout = 1
- fail = 2
- return_value_did_not_match = 3
- output_did_not_match = 4
- src_file_not_found = 5
-
-
-class Operation:
- none=0
- run=1
- monitor=2
- get=3
- put=4
-
-class Task:
- def __init__(self):
- self.id = -1
- self.name = ""
- self.type = Operation.none
- self.command = ""
- self.arguments = ""
- self.timeout = 0
- self.expected_return_code = -1
- self.expected_output = None
- self.stop_on_fail = False
- self.set = None
- self.src = None
- self.dest = None
- def log (self):
- glogger.log ("Task " + str(self.id) + ": " + self.name)
- def check (self):
- if (Operation.none):
- return False
- if (self.type == Operation.run):
- if ((self.id == -1) or (self.name == "") or (self.command == "")):
- return False
- if (self.type == Operation.put):
- if ((self.id == -1) or (self.name == "") or
- (self.src == None) or (self.dest == None)):
- return False
- if (self.type == Operation.get):
- if ((self.id == -1) or (self.name == "") or
- (self.src == None) or (self.dest == None)):
- return False
- return True
-
-class Taskset:
- def __init__(self):
- self.set = list()
-
-
-def handle_task (elem, tasks):
- t = Task ()
-
- if (None != elem.attrib.get("name")):
- t.name = elem.attrib.get("name")
-
- if (None != elem.attrib.get("id")):
- t.id = elem.attrib.get("id")
-
- if (tasks.startid != -1):
- if (tasks.startid == t.id):
- tasks.startid_found = True
- glogger.log ("Task " + str (t.id) + " '" + t.name + "' has start
ID")
- if (tasks.startid_found == False):
- glogger.log ("Task " + str (t.id) + " '" + t.name + "' skipped")
- return None;
-
- if (None != elem.attrib.get("enabled")):
- if ("FALSE" == str(elem.attrib.get("enabled")).upper()):
- glogger.log ("Task " + str (t.id) + " '" + t.name + "' is
disabled")
- return
-
- if (elem.tag == "run"):
- t.type = Operation.run
- elif (elem.tag == "monitor"):
- t.type = Operation.monitor
- elif (elem.tag == "get"):
- t.type = Operation.get
- elif (elem.tag == "put"):
- t.type = Operation.put
- else:
- t.type = Operation.none
-
- for child in elem:
- if ((child.tag == "command") and (child.text != None)):
- t.command = child.text
- if ((child.tag == "arguments") and (child.text != None)):
- t.arguments = child.text
- if (child.tag == "timeout"):
- try:
- t.timeout = int(child.text)
- except ValueError:
- print "Invalid timeout '"+child.text+"' for task id " + str
(t.id) + " name " + t.name
- if (child.tag == "expected_return_code"):
- try:
- t.expected_return_code = int(child.text)
- except ValueError:
- print "Invalid expected return code '" +child.text+ "' for
task id " + str (t.id) + " name " + t.name
- if ((child.tag == "expected_output") and (child.text != None)):
- t.expected_output = child.text
- if ((child.tag == "stop_on_fail") and (child.text != None)):
- if (str.upper(child.text) == "TRUE"):
- t.stop_on_fail = True
- else:
- t.stop_on_fail = False
- if ((child.tag == "source") and (child.text != None)):
- t.src = child.text
- if ((child.tag == "destination") and (child.text != None)):
- t.dest = child.text
-
- if (False == t.check()):
- print "Parsed invalid task with id " + str (t.id) + " name '" + t.name
+ "'"
- return None
- else:
- t.log ()
- return t
-
-def handle_sequence (elem, tasks):
- if (None != elem.attrib.get("enabled")):
- if ("FALSE" == str(elem.attrib.get("enabled")).upper()):
- glogger.log ("Element was disabled")
- return
- for child in elem:
- handle_child (child, tasks)
-
-def handle_parallel (elem, tasks):
- glogger.log ("Found parallel execution with " + str(len(elem)) + "
elements")
- if (None != elem.attrib.get("enabled")):
- if ("FALSE" == str(elem.attrib.get("enabled")).upper()):
- glogger.log ("Element was disabled")
- return
- ptask = Taskset ()
- for child in elem:
- if (elem.tag in supported_operations):
- t = handle_task (elem)
- if (None != t):
- ptask.set.append(t)
- print "Added " + t.name + " to set"
- elif (elem.tag == "parallel"):
- glogger.log ("x")
- # handle_parallel (elem, l)
- elif (elem.tag == "sequence"):
- glogger.log ("x")
- # handle_sequence (elem, l)
- else:
- print "Invalid element in task file: " + elem.tag
- tasks.l.append (ptask)
-
-def handle_child (elem, tasks):
- global support_operations
- if (elem.tag in supported_operations):
- t = handle_task (elem, tasks)
- if (None != t):
- tasks.l.append(t)
- elif (elem.tag == "parallel"):
- handle_parallel (elem, tasks)
- elif (elem.tag == "sequence"):
- handle_sequence (elem, tasks)
- else:
- print "Invalid element in task file: " + elem.tag
-
-def print_sequence (l):
- for i in l:
- print "->",
- if (i.__class__.__name__ == "Task"):
- print i.name,
- elif (i.__class__.__name__ == "Taskset"):
- print "{",
- print_sequence (i.set)
- print "}",
-
-
-
-class Tasks:
- def __init__(self, filename, logger, startid):
- assert (None != logger)
- global glogger
- glogger = logger
- self.logger = logger
- self.filename = filename
- self.name = "<Undefined>"
- self.l = list ()
- self.startid = startid
- self.startid_found = False
- def load (self):
- self.logger.log ("Loading tasks file '" + self.filename + "'")
- enabled = True
- try:
- xsv.parseAndValidate (self.filename)
- except xsv.xsvalErrorHandler.XsvalError as e:
- print "File '" + self.filename + "' does not validate against
schema: \n" + str(e)
- return False
- except GenXmlIfError as e:
- print "File '" + self.filename + "' is not well-formed: \n" +
str(e)
- return False
- except IOError:
- print "File '" + self.filename + "' not found \n"
- return False
-
- try:
- root = ElementTree.parse (self.filename).getroot()
- if (None != root.attrib.get("name")):
- self.name = root.attrib.get("name")
- if (None != root.attrib.get("enabled")):
- if (False == root.attrib.get("enabled")):
- enabled = False
- except expat.ExpatError as e:
- print "File '" + self.filename + "'is malformed: " + str(e)
- return False
- except IOError:
- print "File '" + self.filename + "' not found"
- return False
- if (enabled == True):
- for child in root:
- handle_child (child, self)
- else:
- print "Tasklist " + self.filename + " was disabled"
- #print_sequence (self.l)
-
- return True
-
-
- def copy (self):
- t = Tasks (self.filename, self.logger, -1)
- # Create a copy of the task list as described in
- # http://docs.python.org/library/copy.html
- t.filename = self.filename
- t.name = self.name
- t.l = self.l[:]
- return t
- def get (self):
- if (len (self.l) > 0):
- item = self.l[0]
- self.l.remove(item)
- return item
- else:
- return None
Deleted: gnunet-planetlab/gplmt/Util.py
===================================================================
--- gnunet-planetlab/gplmt/Util.py 2012-08-03 14:00:04 UTC (rev 23064)
+++ gnunet-planetlab/gplmt/Util.py 2012-08-03 14:01:14 UTC (rev 23065)
@@ -1,35 +0,0 @@
-#!/usr/bin/python
-#
-# This file is part of GNUnet.
-# (C) 2010 Christian Grothoff (and other contributing authors)
-#
-# GNUnet is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published
-# by the Free Software Foundation; either version 2, or (at your
-# option) any later version.
-#
-# GNUnet is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with GNUnet; see the file COPYING. If not, write to the
-# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-# Boston, MA 02111-1307, USA.
-#
-# GNUnet Planetlab deployment and automation toolset
-#
-# Utilities
-
-class Logger:
-
- def __init__(self, verbose):
- if (True == verbose):
- self.verbose = True
- else:
- self.verbose = False
- def log (self, message):
- global main
- if (True == self.verbose):
- print (message)
\ No newline at end of file
Deleted: gnunet-planetlab/gplmt/Worker.py
===================================================================
--- gnunet-planetlab/gplmt/Worker.py 2012-08-03 14:00:04 UTC (rev 23064)
+++ gnunet-planetlab/gplmt/Worker.py 2012-08-03 14:01:14 UTC (rev 23065)
@@ -1,288 +0,0 @@
-#!/usr/bin/python
-#
-# This file is part of GNUnet.
-# (C) 2010 Christian Grothoff (and other contributing authors)
-#
-# GNUnet is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published
-# by the Free Software Foundation; either version 2, or (at your
-# option) any later version.
-#
-# GNUnet is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with GNUnet; see the file COPYING. If not, write to the
-# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-# Boston, MA 02111-1307, USA.
-#
-# GNUnet Planetlab deployment and automation toolset
-#
-# Worker
-
-import Tasks
-import threading
-import socket
-import os
-import time
-import sys
-import select
-from SCP import SCPClient
-from Configuration import Configuration
-from Configuration import TransferMode
-
-try:
- import paramiko
- from paramiko.ssh_exception import SSHException
-except ImportError:
- pass
-
-# Global variables
-g_logger = None
-g_notifications = None
-g_configuration = None
-
-class NodeWorkerThread (threading.Thread):
- def __init__(self, threadID, node, tasks):
- threading.Thread.__init__(self)
- self.threadID = threadID
- self.node = node
- self.tasks = tasks
- def exec_run (self, task, transport):
- try:
- channel = transport.open_session()
- channel.settimeout(1.0)
- channel.get_pty ()
- #print "CMD: " + task.command + " " + task.arguments
- channel.exec_command(task.command + " " + task.arguments)
-
- except SSHException as e:
- print self.node + " : Error while trying to connect: " + str(e)
-
- if (task.timeout > 0):
- timeout = task.timeout
- else:
- timeout = -1
- result = Tasks.Taskresult.success
- exit_status = -1
- start_time = time.time ()
-
- stdout_data = ""
- stderr_data = ""
- while 1:
- if (timeout != -1):
- delta = time.time() - start_time
- if (delta > timeout):
- print "Timeout after " +str(delta) +" seconds"
- result = Tasks.Taskresult.timeout
- break
- (r, w, e) = select.select([channel], [], [], 1)
- if r:
- got_data = False
- if channel.recv_ready():
- data = r[0].recv(4096)
- if data:
- got_data = True
- #g_logger.log ('STDOUT: ' + data)
- stdout_data += data
- if channel.recv_stderr_ready():
- data = r[0].recv_stderr(4096)
- if data:
- got_data = True
- #g_logger.log ('STDERR: ' + data)
- stderr_data += data
- if not got_data:
- break
-
- if (result == Tasks.Taskresult.success):
- exit_status = channel.recv_exit_status ()
-
- if (result == Tasks.Taskresult.success):
- if (task.expected_return_code != -1):
- if (exit_status != task.expected_return_code):
- g_logger.log (self.node + " : Task '"+ task.name + "'
completed after "+ str(time.time() - start_time) +" sec, but exit code "
+str(exit_status)+ " was not as expected " + str(task.expected_return_code))
- g_logger.log (stdout_data)
- g_logger.log (stderr_data)
- result = Tasks.Taskresult.return_value_did_not_match
- else:
- g_logger.log (self.node + " : Task '"+ task.name + "'
completed after "+ str(time.time() - start_time) +" sec, exit code "
+str(exit_status)+ " was as expected " + str(task.expected_return_code))
-
- if (task.expected_output != None):
- output_contained = False
- if (task.expected_output in stdout_data):
- output_contained = True
- if (task.expected_output in stderr_data):
- output_contained = True
- if (output_contained == True):
- g_logger.log (self.node + " : Task '"+ task.name + "'
expected output '"+task.expected_output+"' was found")
- else:
- g_logger.log (self.node + " : Task '"+ task.name + "'
expected output '"+task.expected_output+"' was not found")
- result = Tasks.Taskresult.output_did_not_match
-
- if (result == Tasks.Taskresult.success):
- g_logger.log (self.node + " : Task '"+ task.name + "' successful")
- elif (result == Tasks.Taskresult.timeout):
- g_logger.log (self.node + " : Task '"+ task.name + "' with
timeout")
- else:
- g_logger.log (self.node + " : Task '"+ task.name + "' failed")
- return result
- def exec_put (self, task, transport):
- if (False == os.path.exists (task.src)):
- return Tasks.Taskresult.src_file_not_found
-
- result = Tasks.Taskresult.success
- try:
- if (g_configuration.ssh_transfer == TransferMode.scp):
- scp = SCPClient (transport)
- scp.put (task.src, task.dest)
- if (g_configuration.ssh_transfer == TransferMode.sftp):
- sftp = paramiko.SFTPClient.from_transport (transport)
- sftp.put(task.src, task.dest)
- sftp.close()
- except paramiko.SSHException as e:
- g_logger.log (self.node + " : Task '"+ task.name + "' :" + str(e))
- result = Tasks.Taskresult.fail
- pass
- except (OSError, IOError) as e:
- g_logger.log (self.node + " : Task '"+ task.name + "' : " + str(e))
- result = Tasks.Taskresult.src_file_not_found
- pass
- return result
-
- def exec_get (self, task, transport):
- result = Tasks.Taskresult.success
- try:
- if (g_configuration.ssh_transfer == TransferMode.scp):
- scp = SCPClient (transport)
- scp.get (task.src, task.dest)
- if (g_configuration.ssh_transfer == TransferMode.sftp):
- sftp = paramiko.SFTPClient.from_transport (transport)
- sftp.get (task.src, task.dest)
- sftp.close()
- except paramiko.SSHException as e:
- g_logger.log (self.node + " : Task '"+ task.name + "' :" + str(e))
- result = Tasks.Taskresult.fail
- pass
- except (OSError, IOError) as e:
- g_logger.log (self.node + " : Task '"+ task.name + "' : " + str(e))
- result = Tasks.Taskresult.src_file_not_found
- pass
- return result
-
- def run(self):
- g_logger.log (self.node + " : Starting tasklist " + self.tasks.name)
- task = self.tasks.get()
- try:
- ssh = paramiko.SSHClient()
- if (g_configuration.ssh_use_known_hosts):
- g_logger.log (self.node + " : Loading known hosts")
- ssh.load_system_host_keys ()
-
- # Automatically add new hostkeys
- if (g_configuration.ssh_add_unkown_hostkeys == True):
- ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
- # check for private key existance
- keyfile = None
- if (g_configuration.ssh_keyfile != None):
- if (os.path.exists (g_configuration.ssh_keyfile)):
- g_logger.log (self.node + " : Found " +
g_configuration.ssh_keyfile)
- keyfile = g_configuration.ssh_keyfile
- else:
- g_logger.log (self.node + " : Not found " +
g_configuration.ssh_keyfile)
-
- g_logger.log (self.node + " : Trying to connect to " +
- g_configuration.pl_slicename + "@" + self.node +
- " using password '" + g_configuration.ssh_password+
- "' and private keyfile '" +str(keyfile)+ "'")
- ssh.connect (self.node,
- username=g_configuration.pl_slicename,
- password=g_configuration.ssh_password,
- timeout=10,
- key_filename=keyfile)
- except (IOError,
- paramiko.SSHException,
- paramiko.BadHostKeyException,
- paramiko.AuthenticationException,
- socket.error) as e:
- print self.node + " : Error while trying to connect: " + str(e)
- g_notifications.node_connected (self.node, False)
- g_notifications.tasklist_completed (self.node, self.tasks, False)
- return
-
- g_notifications.node_connected (self.node, True)
-
- transport = ssh.get_transport()
- success = True
- result = Tasks.Taskresult.success
- while (None != task):
- g_logger.log (self.node + " : Running task id " +str(task.id)+" '"
+ task.name + "'")
- g_notifications.task_started (self.node, task)
- if (task.__class__.__name__ == "Task"):
- if (task.type == Tasks.Operation.run):
- result = self.exec_run (task, transport)
- g_notifications.task_completed (self.node, task, result)
- elif (task.type == Tasks.Operation.put):
- result = self.exec_put (task, transport)
- g_notifications.task_completed (self.node, task, result)
- elif (task.type == Tasks.Operation.get):
- result = self.exec_get (task, transport)
- g_notifications.task_completed (self.node, task, result)
- else:
- print "FAIL"
- elif (task.__class__.__name__ == "Taskset"):
- g_logger.log (self.node + " : Running task set")
- if ((task.stop_on_fail == True) and (result !=
Tasks.Taskresult.success)):
- g_logger.log (self.node + " : Task failed and therefore
execution is stopped")
- transport.close()
- success = False
- break
- task = self.tasks.get()
-
- ssh.close()
- g_notifications.node_disconnected (self.node, True)
- g_notifications.tasklist_completed (self.node, self.tasks, success)
- g_logger.log (self.node + " : All tasks done for " + self.node)
-
-
-
-class NodeWorker:
- def __init__(self, node, tasks):
- assert (None != node)
- assert (None != tasks)
- self.node = node
- self.tasks = tasks
- self.thread = None
- def start (self):
- g_logger.log ("Starting execution for node " + self.node)
- g_notifications.tasklist_started (self.node, self.tasks)
- self.thread = NodeWorkerThread (1, self.node, self.tasks)
- self.thread.start()
-
-class Worker:
- def __init__(self, logger, configuration, nodes, tasks, notifications):
- global g_logger;
- global g_configuration;
- global g_notifications;
- assert (None != logger)
- assert (None != nodes)
- assert (None != tasks)
- assert (None != notifications)
- assert (hasattr(notifications, 'node_connected'))
- assert (hasattr(notifications, 'node_disconnected'))
- assert (hasattr(notifications, 'tasklist_started'))
- assert (hasattr(notifications, 'tasklist_completed'))
- assert (hasattr(notifications, 'task_started'))
- assert (hasattr(notifications, 'task_completed'))
- self.nodes = nodes
- self.tasks = tasks
- g_configuration = configuration
- g_notifications = notifications
- g_logger = logger;
- def start (self):
- g_logger.log ("Starting execution")
- for n in self.nodes.nodes:
- nw = NodeWorker (n, self.tasks.copy())
- nw.start()
-
Copied: gnunet-planetlab/gplmt/contrib/tasklist_schema.xsd (from rev 22971,
gnunet-planetlab/gplmt/tasklist_schema.xsd)
===================================================================
--- gnunet-planetlab/gplmt/contrib/tasklist_schema.xsd
(rev 0)
+++ gnunet-planetlab/gplmt/contrib/tasklist_schema.xsd 2012-08-03 14:01:14 UTC
(rev 23065)
@@ -0,0 +1,98 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema";
elementFormDefault="qualified">
+<xs:element name="tasklist" type="tasklist" />
+
+<xs:complexType name="tasklist">
+ <xs:sequence>
+ <xs:choice minOccurs="0" maxOccurs="unbounded">
+ <xs:element name="sequence" type="sequence" />
+ <xs:element name="parallel" type="parallel" />
+ <xs:element name="run" type="run" />
+ <xs:element name="monitor" type="monitor" />
+ <xs:element name="put" type="put" />
+ <xs:element name="get" type="get" />
+ </xs:choice>
+ </xs:sequence>
+ <xs:attribute name="name" type="xs:string"/>
+ <xs:attribute name="enabled" type="xs:boolean"/>
+</xs:complexType>
+
+<xs:complexType name="sequence">
+ <xs:sequence>
+ <xs:choice minOccurs="0" maxOccurs="unbounded">
+ <xs:element name="sequence" type="sequence" />
+ <xs:element name="parallel" type="parallel" />
+ <xs:element name="run" type="run" />
+ <xs:element name="monitor" type="monitor" />
+ <xs:element name="put" type="put" />
+ <xs:element name="get" type="get" />
+ </xs:choice>
+ </xs:sequence>
+ <xs:attribute name="enabled" type="xs:boolean"/>
+</xs:complexType>
+
+<xs:complexType name="parallel">
+ <xs:sequence>
+ <xs:choice minOccurs="0" maxOccurs="unbounded">
+ <xs:element name="sequence" type="sequence" />
+ <xs:element name="parallel" type="parallel" />
+ <xs:element name="run" type="run" />
+ <xs:element name="monitor" type="monitor" />
+ <xs:element name="put" type="put" />
+ <xs:element name="get" type="get" />
+ </xs:choice>
+ </xs:sequence>
+ <xs:attribute name="enabled" type="xs:boolean"/>
+</xs:complexType>
+
+<xs:complexType name="run">
+ <xs:sequence>
+ <xs:element name="command" type="xs:string"/>
+ <xs:element name="arguments" type="xs:string"/>
+ <xs:element name="timeout" type="xs:integer"/>
+ <xs:element name="expected_return_code" type="xs:integer"/>
+ <xs:element name="expected_output" type="xs:string"/>
+ <xs:element name="stop_on_fail" type="xs:boolean"/>
+ </xs:sequence>
+ <xs:attribute name="id" type="xs:integer"/>
+ <xs:attribute name="name" type="xs:string"/>
+ <xs:attribute name="enabled" type="xs:boolean"/>
+</xs:complexType>
+
+<xs:complexType name="monitor">
+ <xs:sequence>
+ <xs:element name="command" type="xs:string"/>
+ <xs:element name="arguments" type="xs:string"/>
+ <xs:element name="timeout" type="xs:integer"/>
+ <xs:element name="expected_return_code" type="xs:integer"/>
+ <xs:element name="expected_output" type="xs:string"/>
+ <xs:element name="stop_on_fail" type="xs:boolean"/>
+ </xs:sequence>
+ <xs:attribute name="id" type="xs:integer"/>
+ <xs:attribute name="name" type="xs:string"/>
+ <xs:attribute name="enabled" type="xs:boolean"/>
+</xs:complexType>
+
+<xs:complexType name="put">
+ <xs:sequence>
+ <xs:element name="source" type="xs:string"/>
+ <xs:element name="destination" type="xs:string"/>
+ <xs:element name="stop_on_fail" type="xs:boolean"/>
+ </xs:sequence>
+ <xs:attribute name="id" type="xs:integer"/>
+ <xs:attribute name="name" type="xs:string"/>
+ <xs:attribute name="enabled" type="xs:boolean"/>
+</xs:complexType>
+
+<xs:complexType name="get">
+ <xs:sequence>
+ <xs:element name="source" type="xs:string"/>
+ <xs:element name="destination" type="xs:string"/>
+ <xs:element name="stop_on_fail" type="xs:boolean"/>
+ </xs:sequence>
+ <xs:attribute name="id" type="xs:integer"/>
+ <xs:attribute name="name" type="xs:string"/>
+ <xs:attribute name="enabled" type="xs:boolean"/>
+</xs:complexType>
+
+</xs:schema>
Modified: gnunet-planetlab/gplmt/contrib/tasklists/check_fedora_version.xml
===================================================================
--- gnunet-planetlab/gplmt/contrib/tasklists/check_fedora_version.xml
2012-08-03 14:00:04 UTC (rev 23064)
+++ gnunet-planetlab/gplmt/contrib/tasklists/check_fedora_version.xml
2012-08-03 14:01:14 UTC (rev 23065)
@@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
-<tasklist name="Check Fedora version on node"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:noNamespaceSchemaLocation="../../tasklist_schema.xsd">
+<tasklist name="Check Fedora version on node"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:noNamespaceSchemaLocation="../tasklist_schema.xsd">
<run id="0" name="check peer">
<!-- Check if peer is working -->
Modified: gnunet-planetlab/gplmt/contrib/tasklists/check_node.xml
===================================================================
--- gnunet-planetlab/gplmt/contrib/tasklists/check_node.xml 2012-08-03
14:00:04 UTC (rev 23064)
+++ gnunet-planetlab/gplmt/contrib/tasklists/check_node.xml 2012-08-03
14:01:14 UTC (rev 23065)
@@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
-<tasklist name="Check if peer can is accessible"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:noNamespaceSchemaLocation="../../tasklist_schema.xsd">
+<tasklist name="Check if peer can is accessible"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:noNamespaceSchemaLocation="../tasklist_schema.xsd">
<run id="0" name="check peer">
<!-- Check if peer is working -->
<command>sudo date</command>
Modified: gnunet-planetlab/gplmt/contrib/tasklists/deploy_gnunet_fc12.xml
===================================================================
--- gnunet-planetlab/gplmt/contrib/tasklists/deploy_gnunet_fc12.xml
2012-08-03 14:00:04 UTC (rev 23064)
+++ gnunet-planetlab/gplmt/contrib/tasklists/deploy_gnunet_fc12.xml
2012-08-03 14:01:14 UTC (rev 23065)
@@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
-<tasklist name="Deploy GNUnet"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:noNamespaceSchemaLocation="../../tasklist_schema.xsd">
+<tasklist name="Deploy GNUnet"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:noNamespaceSchemaLocation="../tasklist_schema.xsd">
<run id="0" name="check peer">
<!-- Check if peer is working -->
Modified: gnunet-planetlab/gplmt/contrib/tasklists/deploy_gnunet_fc8.xml
===================================================================
--- gnunet-planetlab/gplmt/contrib/tasklists/deploy_gnunet_fc8.xml
2012-08-03 14:00:04 UTC (rev 23064)
+++ gnunet-planetlab/gplmt/contrib/tasklists/deploy_gnunet_fc8.xml
2012-08-03 14:01:14 UTC (rev 23065)
@@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
-<tasklist name="Deploy GNUnet"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:noNamespaceSchemaLocation="../../tasklist_schema.xsd">
+<tasklist name="Deploy GNUnet"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:noNamespaceSchemaLocation="../tasklist_schema.xsd">
<run id="0" name="check peer">
<!-- Check if peer is working -->
Modified: gnunet-planetlab/gplmt/contrib/tasklists/start_gnunet.xml
===================================================================
--- gnunet-planetlab/gplmt/contrib/tasklists/start_gnunet.xml 2012-08-03
14:00:04 UTC (rev 23064)
+++ gnunet-planetlab/gplmt/contrib/tasklists/start_gnunet.xml 2012-08-03
14:01:14 UTC (rev 23065)
@@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
-<tasklist name="Start GNUnet"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:noNamespaceSchemaLocation="../../tasklist_schema.xsd">
+<tasklist name="Start GNUnet"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:noNamespaceSchemaLocation="../tasklist_schema.xsd">
<run id="0" name="check peer">
<!-- Check if peer is working -->
Modified: gnunet-planetlab/gplmt/contrib/tasklists/stop_gnunet.xml
===================================================================
--- gnunet-planetlab/gplmt/contrib/tasklists/stop_gnunet.xml 2012-08-03
14:00:04 UTC (rev 23064)
+++ gnunet-planetlab/gplmt/contrib/tasklists/stop_gnunet.xml 2012-08-03
14:01:14 UTC (rev 23065)
@@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
-<tasklist name="Start GNUnet"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:noNamespaceSchemaLocation="../../tasklist_schema.xsd">
+<tasklist name="Start GNUnet"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:noNamespaceSchemaLocation="../tasklist_schema.xsd">
<run id="0" name="check peer">
<!-- Check if peer is working -->
Modified: gnunet-planetlab/gplmt/contrib/tasklists/test_sftp_tasks.xml
===================================================================
--- gnunet-planetlab/gplmt/contrib/tasklists/test_sftp_tasks.xml
2012-08-03 14:00:04 UTC (rev 23064)
+++ gnunet-planetlab/gplmt/contrib/tasklists/test_sftp_tasks.xml
2012-08-03 14:01:14 UTC (rev 23065)
@@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
-<tasklist name="remote put/get test task list"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:noNamespaceSchemaLocation="../../tasklist_schema.xsd">
+<tasklist name="remote put/get test task list"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:noNamespaceSchemaLocation="../tasklist_schema.xsd">
<put id="0" name="put testfile">
<source>/tmp/test.file</source>
<destination>/tmp/put.file</destination>
Modified: gnunet-planetlab/gplmt/contrib/tasklists/test_tasks.xml
===================================================================
--- gnunet-planetlab/gplmt/contrib/tasklists/test_tasks.xml 2012-08-03
14:00:04 UTC (rev 23064)
+++ gnunet-planetlab/gplmt/contrib/tasklists/test_tasks.xml 2012-08-03
14:01:14 UTC (rev 23065)
@@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
-<tasklist name="Tasklist example"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:noNamespaceSchemaLocation="../../tasklist_schema.xsd">
+<tasklist name="Tasklist example"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:noNamespaceSchemaLocation="../tasklist_schema.xsd">
<task>
<id>0</id>
<name>run 1</name>
Modified: gnunet-planetlab/gplmt/contrib/tasklists/update_gnunet.xml
===================================================================
--- gnunet-planetlab/gplmt/contrib/tasklists/update_gnunet.xml 2012-08-03
14:00:04 UTC (rev 23064)
+++ gnunet-planetlab/gplmt/contrib/tasklists/update_gnunet.xml 2012-08-03
14:01:14 UTC (rev 23065)
@@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
-<tasklist name="Update GNUnet to SVN HEAD"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:noNamespaceSchemaLocation="../../tasklist_schema.xsd">
+<tasklist name="Update GNUnet to SVN HEAD"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:noNamespaceSchemaLocation="../tasklist_schema.xsd">
<run id="0" name="check peer">
<!-- Check if peer is working -->
<command>sudo date</command>
Added: gnunet-planetlab/gplmt/gplmt/Configuration.py
===================================================================
--- gnunet-planetlab/gplmt/gplmt/Configuration.py
(rev 0)
+++ gnunet-planetlab/gplmt/gplmt/Configuration.py 2012-08-03 14:01:14 UTC
(rev 23065)
@@ -0,0 +1,134 @@
+#!/usr/bin/python
+#
+# This file is part of GNUnet.
+# (C) 2010 Christian Grothoff (and other contributing authors)
+#
+# GNUnet is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published
+# by the Free Software Foundation; either version 2, or (at your
+# option) any later version.
+#
+# GNUnet is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with GNUnet; see the file COPYING. If not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+#
+# GNUnet Planetlab deployment and automation toolset
+#
+# Configuration
+
+
+try:
+ import ConfigParser
+ import os
+ import sys
+except ImportError:
+ print "That's a bug! please check README"
+ sys.exit (1)
+
+class TransferMode:
+ none=0
+ sftp=1
+ scp=2
+
+class Configuration:
+ def __init__(self, filename, logger):
+ assert (None != logger)
+ self.filename = filename
+ self.logger = logger
+ self.notifications = ""
+ self.pl_slicename = ""
+ self.pl_api_url = ""
+ self.pl_username = ""
+ self.pl_password = ""
+ self.pl_use_nodes = False
+ self.taskfile = ""
+ self.nodesfile = ""
+ self.ssh_add_unkown_hostkeys = False
+ self.ssh_keyfile = None
+ self.ssh_password = ""
+ self.ssh_use_known_hosts = False
+ self.ssh_transfer = TransferMode.scp
+ def load (self):
+ self.logger.log ("Loading configuration file '" + self.filename + "'")
+ if (False == os.path.exists (self.filename)):
+ print "File does not exist: '" + self.filename + "'"
+ return False
+ config = ConfigParser.RawConfigParser()
+ try:
+ config.read(self.filename)
+ except ConfigParser.Error as e:
+ print "Error parsing configuration: " + str (e)
+ return False
+
+ # required values
+ try:
+ self.pl_slicename = config.get("planetlab", "slice")
+ except ConfigParser.NoOptionError as e:
+ print "Error parsing configuration: " + str (e)
+ return False
+ # optional values
+ try:
+ self.pl_use_nodes = config.getboolean ("planetlab", "use_pl_nodes")
+ except ConfigParser.NoOptionError as e:
+ pass
+ try:
+ self.pl_api_url = config.get("planetlab", "api_url")
+ except ConfigParser.NoOptionError as e:
+ pass
+ try:
+ self.pl_username = config.get("planetlab", "username")
+ except ConfigParser.NoOptionError as e:
+ pass
+ try:
+ self.pl_password = config.get("planetlab", "password")
+ except ConfigParser.NoOptionError as e:
+ pass
+ try:
+ # gplmt options
+ self.taskfile = config.get("gplmt", "tasks")
+ except ConfigParser.NoOptionError as e:
+ pass
+
+ try:
+ self.nodesfile = config.get("gplmt", "nodes")
+ except ConfigParser.NoOptionError as e:
+ pass
+
+ # ssh options
+ try:
+ self.ssh_add_unkown_hostkeys = config.getboolean ("ssh",
"add_unkown_hostkeys")
+ except ConfigParser.NoOptionError as e:
+ pass
+ try:
+ self.ssh_use_known_hosts = config.getboolean ("ssh",
"ssh_use_known_hosts")
+ except ConfigParser.NoOptionError as e:
+ pass
+ try:
+ self.ssh_keyfile = config.get("ssh", "ssh_keyfile")
+ except ConfigParser.NoOptionError as e:
+ pass
+ try:
+ self.ssh_password = config.get("ssh", "ssh_password")
+ except ConfigParser.NoOptionError as e:
+ pass
+ try:
+ transfer = config.get("ssh", "ssh_transfer")
+ if (transfer == "scp"):
+ self.ssh_transfer = TransferMode.scp
+ elif (transfer == "sftp"):
+ self.ssh_transfer = TransferMode.sftp
+ else:
+ print "Invalid ssh transfer mode: only SFTP or SCP are
supported"
+ return False
+ except ConfigParser.NoOptionError as e:
+ pass
+
+
+ return True
+
\ No newline at end of file
Added: gnunet-planetlab/gplmt/gplmt/Nodes.py
===================================================================
--- gnunet-planetlab/gplmt/gplmt/Nodes.py (rev 0)
+++ gnunet-planetlab/gplmt/gplmt/Nodes.py 2012-08-03 14:01:14 UTC (rev
23065)
@@ -0,0 +1,111 @@
+#!/usr/bin/python
+#
+# This file is part of GNUnet.
+# (C) 2010 Christian Grothoff (and other contributing authors)
+#
+# GNUnet is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published
+# by the Free Software Foundation; either version 2, or (at your
+# option) any later version.
+#
+# GNUnet is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with GNUnet; see the file COPYING. If not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+#
+# GNUnet Planetlab deployment and automation toolset
+#
+# Nodes
+
+try:
+ import sys
+ import os
+ import urllib
+ import xmlrpclib
+ import socket
+except ImportError:
+ print "That's a bug! please check README"
+ sys.exit (1)
+
+class Nodes:
+ def __init__(self, filename, logger):
+ assert (None != logger)
+ self.logger = logger
+ self.filename = filename
+ self.nodes = list ()
+ def load (self):
+ self.logger.log ("Loading nodes file '" + self.filename + "'")
+ try:
+ fobj = open (self.filename, "r")
+ for line in fobj:
+ line = line.strip()
+ self.logger.log ("Found node '" + line + "'")
+ self.nodes.append(line)
+ fobj.close()
+ except IOError:
+ print "File " + self.filename + " not found"
+ return False
+ self.logger.log ("Loaded " + str(len(self.nodes)) + " nodes")
+ return True
+
+class StringNodes:
+ def __init__(self, str, logger):
+ assert (None != logger)
+ self.str = str
+ self.logger = logger
+ self.nodes = list ()
+ def load (self):
+ self.logger.log ("Loading nodes '" + self.str + "'")
+ self.nodes.append(self.str)
+ self.logger.log ("Loaded " + str(len(self.nodes)) + " nodes")
+ return True
+
+class PlanetLabNodes:
+ def __init__(self, configuration, logger):
+ assert (None != logger)
+ self.logger = logger
+ self.configuration = configuration
+ self.nodes = list ()
+ def load (self):
+
+ if (self.configuration.pl_password == ""):
+ print "No PlanetLab password given in configuration fail!"
+ return False
+ if (self.configuration.pl_username == ""):
+ print "No PlanetLab username given in configuration, fail!"
+ return False
+ if (self.configuration.pl_api_url == ""):
+ print "No PlanetLab API url given in configuration, fail!"
+ return False
+ self.logger.log ("Retrieving nodes assigned to slice '" +
self.configuration.pl_slicename + "' for user " +self.configuration.pl_username)
+ try:
+ server = xmlrpclib.ServerProxy(self.configuration.pl_api_url)
+ except:
+ print "Could not connect to PlanetLab API, fail!"
+ return False
+
+ slice_data = {}
+ slice_data['name'] = self.configuration.pl_slicename
+
+ auth = {}
+ auth['Username'] = self.configuration.pl_username
+ auth['AuthString'] = self.configuration.pl_password
+ auth['AuthMethod'] = "password"
+
+ try:
+ node_ids = server.GetSlices(auth, [slice_data['name']],
['node_ids'])[0]['node_ids']
+ node_hostnames = [node['hostname'] for node in
server.GetNodes(auth, node_ids, ['hostname'])]
+ except Exception as e:
+ print "Could not retrieve data from PlanetLab API: " + str(e)
+ return False
+
+ for node in node_hostnames:
+ self.logger.log ("Planetlab API returned: " + node)
+ self.nodes.append(node)
+ self.logger.log ("Planetlab API returned " + str(len(self.nodes)) + "
nodes")
+ return True
\ No newline at end of file
Added: gnunet-planetlab/gplmt/gplmt/Notifications.py
===================================================================
--- gnunet-planetlab/gplmt/gplmt/Notifications.py
(rev 0)
+++ gnunet-planetlab/gplmt/gplmt/Notifications.py 2012-08-03 14:01:14 UTC
(rev 23065)
@@ -0,0 +1,172 @@
+#!/usr/bin/python
+#
+# This file is part of GNUnet.
+# (C) 2010 Christian Grothoff (and other contributing authors)
+#
+# GNUnet is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published
+# by the Free Software Foundation; either version 2, or (at your
+# option) any later version.
+#
+# GNUnet is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with GNUnet; see the file COPYING. If not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+#
+# GNUnet Planetlab deployment and automation toolset
+#
+# Notifications
+
+try:
+ import gplmt.Tasks as Tasks
+ import time
+ import sys
+except ImportError:
+ print "That's a bug! please check README"
+
+
+class Notification:
+ def __init__(self, logger):
+ assert (None != logger)
+ self.logger = logger
+ def node_connected (self, node, success):
+ assert (0)
+ def node_disconnected (self, node, success):
+ assert (0)
+ def tasklist_started (self, node, tasks):
+ assert (0)
+ def tasklist_completed (self, node, tasks, success):
+ assert (0)
+ def task_started (self, node, tasks):
+ assert (0)
+ def task_completed (self, node, tasks, success):
+ assert (0)
+
+class NodeCollection:
+ def __init__(self):
+ self.nodes = list ()
+ def add (self, node):
+ self.nodes.append (node)
+ def get (self, name):
+ for n in self.nodes:
+ if (n.name == name):
+ return n
+ return None
+
+class Task:
+ def __init__(self, task, result, fail):
+ self.task = task
+ self.result = result
+ self.fail = fail
+
+class Node:
+ def __init__(self, name):
+ self.name = name
+ self.start = time.time()
+ self.end = 0
+ self.tasks = list ()
+
+
+class FileLoggerNotification (Notification):
+ def __init__(self, logger):
+ assert (None != logger)
+ self.logger = logger
+ self.nodes = NodeCollection ()
+ def node_connected (self, node, success):
+ return
+ def node_disconnected (self, node, success):
+ return
+ def tasklist_started (self, node, tasks):
+ return
+ def tasklist_completed (self, node, tasks, success):
+ self.nodes.add (Node(node))
+ if (success == True):
+ print node + " : Tasklist '" + tasks.name + "' completed
successfully"
+ else:
+ print node + " : Tasklist '" + tasks.name + "' completed with
failure"
+ def task_started (self, node, task):
+ return
+ def task_completed (self, node, task, result):
+ return
+
+
+class TaskListResultNotification (Notification):
+ def __init__(self, logger):
+ assert (None != logger)
+ self.logger = logger
+ #self.nodes = NodeCollection ()
+ def summarize (self):
+ for n in self.nodes:
+ tsk_str = ""
+ for t in n.tasks:
+ tsk_f = "[e]"
+ if (t.fail == True):
+ tsk_f = "[f]"
+ else:
+ tsk_f = "[s]"
+ tsk_str += t.task.name + " " + tsk_f + " ->"
+ print n.name
+ def node_connected (self, node, success):
+ return
+ def node_disconnected (self, node, success):
+ return
+ def tasklist_started (self, node, tasks):
+ return
+ def tasklist_completed (self, node, tasks, success):
+ if (success == True):
+ print node + " : Tasklist '" + tasks.name + "' completed
successfully"
+ else:
+ print node + " : Tasklist '" + tasks.name + "' completed with
failure"
+ def task_started (self, node, task):
+ return
+ def task_completed (self, node, task, result):
+ return
+
+class SimpleNotification (Notification):
+ def __init__(self, logger):
+ assert (None != logger)
+ self.logger = logger
+ #self.nodes = NodeCollection ()
+ def summarize (self):
+ for n in self.nodes:
+ tsk_str = ""
+ for t in n.tasks:
+ tsk_f = "[e]"
+ if (t.fail == True):
+ tsk_f = "[f]"
+ else:
+ tsk_f = "[s]"
+ tsk_str += t.task.name + " " + tsk_f + " ->"
+ print n.name
+ def node_connected (self, node, success):
+ if (success == True):
+ print node + " : connected successfully"
+ else:
+ print node + " : connection failed"
+ def node_disconnected (self, node, success):
+ if (success == True):
+ print node + " : disconnected"
+ else:
+ print node + " : disconnected with failure"
+ def tasklist_started (self, node, tasks):
+ print node + " : Tasklist '" + tasks.name + "' started"
+ #self.nodes.add (Node(node))
+ def tasklist_completed (self, node, tasks, success):
+ if (success == True):
+ print node + " : Tasklist '" + tasks.name + "' completed
successfully"
+ else:
+ print node + " : Tasklist '" + tasks.name + "' completed with
failure"
+ def task_started (self, node, task):
+ print node + " : Task '" + task.name + "' started"
+ def task_completed (self, node, task, result):
+ if (result == Tasks.Taskresult.success):
+ print node + " : Task '" + task.name + "' completed successfully"
+ elif (result == Tasks.Taskresult.src_file_not_found):
+ print node + " : Task '" + task.name + "' failed : source file
not found"
+ else:
+ print node + " : Task '" + task.name + "' completed with failure"
\ No newline at end of file
Added: gnunet-planetlab/gplmt/gplmt/SCP.py
===================================================================
--- gnunet-planetlab/gplmt/gplmt/SCP.py (rev 0)
+++ gnunet-planetlab/gplmt/gplmt/SCP.py 2012-08-03 14:01:14 UTC (rev 23065)
@@ -0,0 +1,395 @@
+# scp.py
+# Copyright (C) 2008 James Bardin <address@hidden>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+"""
+Utilities for sending files over ssh using the scp1 protocol.
+"""
+
+import os
+from socket import timeout as SocketTimeout
+
+class SCPClient(object):
+ """
+ An scp1 implementation, compatible with openssh scp.
+ Raises SCPException for all transport related errors. Local filesystem
+ and OS errors pass through.
+
+ Main public methods are .put and .get
+ The get method is controlled by the remote scp instance, and behaves
+ accordingly. This means that symlinks are resolved, and the transfer is
+ halted after too many levels of symlinks are detected.
+ The put method uses os.walk for recursion, and sends files accordingly.
+ Since scp doesn't support symlinks, we send file symlinks as the file
+ (matching scp behaviour), but we make no attempt at symlinked directories.
+
+ Convenience methods:
+ put_r: put with recursion
+ put_p: put preserving times
+ put_rp: put with recursion, preserving times
+ get_r: get with recursion
+ get_p: get preserving times
+ get_rp: get with recursion, preserving times
+ """
+ def __init__(self, transport, buff_size = 16384, socket_timeout = 5.0,
+ callback = None):
+ """
+ Create an scp1 client.
+
+ @param transport: an existing paramiko L{Transport}
+ @type transport: L{Transport}
+ @param buff_size: size of the scp send buffer.
+ @type buff_size: int
+ @param socket_timeout: channel socket timeout in seconds
+ @type socket_timeout: float
+ @param callback: callback function for transfer status
+ @type callback: func
+ """
+ self.transport = transport
+ self.buff_size = buff_size
+ self.socket_timeout = socket_timeout
+ self.channel = None
+ self.preserve_times = False
+ self.callback = callback
+ self._recv_dir = ''
+ self._utime = None
+ self._dirtimes = {}
+
+ def put(self, files, remote_path = '.',
+ recursive = False, preserve_times = False):
+ """
+ Transfer files to remote host.
+
+ @param files: A single path, or a list of paths to be transfered.
+ recursive must be True to transfer directories.
+ @type files: string OR list of strings
+ @param remote_path: path in which to receive the files on the remote
+ host. defaults to '.'
+ @type remote_path: str
+ @param recursive: transfer files and directories recursively
+ @type recursive: bool
+ @param preserve_times: preserve mtime and atime of transfered files
+ and directories.
+ @type preserve_times: bool
+ """
+ self.preserve_times = preserve_times
+ self.channel = self.transport.open_session()
+ self.channel.settimeout(self.socket_timeout)
+ scp_command = ('scp -t %s\n', 'scp -r -t %s\n')[recursive]
+ self.channel.exec_command(scp_command % remote_path)
+ self._recv_confirm()
+
+ if not isinstance(files, (list, tuple)):
+ files = [files]
+
+ if recursive:
+ self._send_recursive(files)
+ else:
+ self._send_files(files)
+
+ if self.channel:
+ self.channel.close()
+
+ def put_r(self, files, remote_path = '.'):
+ """
+ Convenience function for a recursive put
+
+ @param files: A single path, or a list of paths to be transfered.
+ @type files: str, list
+ @param remote_path: path in which to receive the files on the remote
+ host. defaults to '.'
+ @type remote_path: bool
+ """
+ self.put(files, remote_path, recursive = True)
+
+ def put_p(self, files, remote_path = '.'):
+ """
+ Convenience function to put preserving times.
+
+ @param files: A single path, or a list of paths to be transfered.
+ @type files: str, list
+ @param remote_path: path in which to receive the files on the remote
+ host. defaults to '.'
+ @type remote_path: bool
+ """
+ self.put(files, remote_path, preserve_times = True)
+
+ def put_rp(self, files, remote_path = '.'):
+ """
+ Convenience function for a recursive put, preserving times.
+
+ @param files: A single path, or a list of paths to be transfered.
+ @type files: str, list
+ @param remote_path: path in which to receive the files on the remote
+ host. defaults to '.'
+ @type remote_path: bool
+ """
+ self.put(files, remote_path, recursive = True, preserve_times = True)
+
+ def get(self, remote_path, local_path = '',
+ recursive = False, preserve_times = False):
+ """
+ Transfer files from remote host to localhost
+
+ @param remote_path: path to retreive from remote host. since this is
+ evaluated by scp on the remote host, shell wildcards and
+ environment variables may be used.
+ @type remote_path: str
+ @param local_path: path in which to receive files locally
+ @type local_path: str
+ @param recursive: transfer files and directories recursively
+ @type recursive: bool
+ @param preserve_times: preserve mtime and atime of transfered files
+ and directories.
+ @type preserve_times: bool
+ """
+ self._recv_dir = local_path or os.getcwd()
+ rcsv = ('', ' -r')[recursive]
+ prsv = ('', ' -p')[preserve_times]
+ self.channel = self.transport.open_session()
+ self.channel.settimeout(self.socket_timeout)
+ self.channel.exec_command('scp%s%s -f %s' % (rcsv, prsv, remote_path))
+ self._recv_all()
+
+ if self.channel:
+ self.channel.close()
+
+ def get_r(self, remote_path, local_path = '.'):
+ """
+ Convenience function for a recursive get
+
+ @param remote_path: path to retrieve from server
+ @type remote_path: str
+ @param local_path: path in which to recieve files. default cwd
+ @type local_path: str
+ """
+ self.get(remote_path, local_path, recursive = True)
+
+ def get_p(self, remote_path, local_path = '.'):
+ """
+ Convenience function for get, preserving times.
+
+ @param remote_path: path to retrieve from server
+ @type remote_path: str
+ @param local_path: path in which to recieve files. default cwd
+ @type local_path: str
+ """
+ self.get(remote_path, local_path, preserve_times = True)
+
+ def get_rp(self, remote_path, local_path = '.'):
+ """
+ Convenience function for a recursive get, preserving times.
+
+ @param remote_path: path to retrieve from server
+ @type remote_path: str
+ @param local_path: path in which to recieve files. default cwd
+ @type local_path: str
+ """
+ self.get(remote_path, local_path, recursive = True, preserve_times =
True)
+
+ def _read_stats(self, name):
+ """return just the file stats needed for scp"""
+ stats = os.stat(name)
+ mode = oct(stats.st_mode)[-4:]
+ size = stats.st_size
+ atime = int(stats.st_atime)
+ mtime = int(stats.st_mtime)
+ return (mode, size, mtime, atime)
+
+ def _send_files(self, files):
+ for name in files:
+ basename = os.path.basename(name)
+ (mode, size, mtime, atime) = self._read_stats(name)
+ if self.preserve_times:
+ self._send_time(mtime, atime)
+ file_hdl = file(name, 'rb')
+ self.channel.sendall('C%s %d %s\n' % (mode, size, basename))
+ self._recv_confirm()
+ file_pos = 0
+ buff_size = self.buff_size
+ chan = self.channel
+ while file_pos < size:
+ chan.sendall(file_hdl.read(buff_size))
+ file_pos = file_hdl.tell()
+ if self.callback:
+ self.callback(file_pos, size)
+ chan.sendall('\x00')
+ file_hdl.close()
+
+ def _send_recursive(self, files):
+ for base in files:
+ lastdir = base
+ for root, dirs, fls in os.walk(base):
+ # pop back out to the next dir in the walk
+ while lastdir != os.path.commonprefix([lastdir, root]):
+ self._send_popd()
+ lastdir = os.path.split(lastdir)[0]
+ self._send_pushd(root)
+ lastdir = root
+ self._send_files([os.path.join(root, f) for f in fls])
+
+ def _send_pushd(self, directory):
+ (mode, size, mtime, atime) = self._read_stats(directory)
+ basename = os.path.basename(directory)
+ if self.preserve_times:
+ self._send_time(mtime, atime)
+ self.channel.sendall('D%s 0 %s\n' % (mode, basename))
+ self._recv_confirm()
+
+ def _send_popd(self):
+ self.channel.sendall('E\n')
+ self._recv_confirm()
+
+ def _send_time(self, mtime, atime):
+ self.channel.sendall('T%d 0 %d 0\n' % (mtime, atime))
+ self._recv_confirm()
+
+ def _recv_confirm(self):
+ # read scp response
+ msg = ''
+ try:
+ msg = self.channel.recv(512)
+ except SocketTimeout:
+ raise SCPException('Timout waiting for scp response')
+ if msg and msg[0] == '\x00':
+ return
+ elif msg and msg[0] == '\x01':
+ raise SCPException(msg[1:])
+ elif self.channel.recv_stderr_ready():
+ msg = self.channel.recv_stderr(512)
+ raise SCPException(msg)
+ elif not msg:
+ raise SCPException('No response from server')
+ else:
+ raise SCPException('Invalid response from server: ' + msg)
+
+ def _recv_all(self):
+ # loop over scp commands, and recive as necessary
+ command = {'C': self._recv_file,
+ 'T': self._set_time,
+ 'D': self._recv_pushd,
+ 'E': self._recv_popd}
+ while not self.channel.closed:
+ # wait for command as long as we're open
+ self.channel.sendall('\x00')
+ msg = self.channel.recv(1024)
+ if not msg: # chan closed while recving
+ break
+ code = msg[0]
+ try:
+ command[code](msg[1:])
+ except KeyError:
+ raise SCPException(repr(msg))
+ # directory times can't be set until we're done writing files
+ self._set_dirtimes()
+
+ def _set_time(self, cmd):
+ try:
+ times = cmd.split()
+ mtime = int(times[0])
+ atime = int(times[2]) or mtime
+ except:
+ self.channel.send('\x01')
+ raise SCPException('Bad time format')
+ # save for later
+ self._utime = (mtime, atime)
+
+ def _recv_file(self, cmd):
+ chan = self.channel
+ parts = cmd.split()
+ try:
+ mode = int(parts[0], 8)
+ size = int(parts[1])
+ path = os.path.join(self._recv_dir, parts[2])
+ except:
+ chan.send('\x01')
+ chan.close()
+ raise SCPException('Bad file format')
+
+ try:
+ file_hdl = file(path, 'wb')
+ except IOError, e:
+ chan.send('\x01'+e.message)
+ chan.close()
+ raise
+
+ buff_size = self.buff_size
+ pos = 0
+ chan.send('\x00')
+ try:
+ while pos < size:
+ # we have to make sure we don't read the final byte
+ if size - pos <= buff_size:
+ buff_size = size - pos
+ file_hdl.write(chan.recv(buff_size))
+ pos = file_hdl.tell()
+ if self.callback:
+ self.callback(pos, size)
+
+ msg = chan.recv(512)
+ if msg and msg[0] != '\x00':
+ raise SCPException(msg[1:])
+ except SocketTimeout:
+ chan.close()
+ raise SCPException('Error receiving, socket.timeout')
+
+ file_hdl.truncate()
+ try:
+ os.utime(path, self._utime)
+ self._utime = None
+ os.chmod(path, mode)
+ # should we notify the other end?
+ finally:
+ file_hdl.close()
+ # '\x00' confirmation sent in _recv_all
+
+ def _recv_pushd(self, cmd):
+ parts = cmd.split()
+ try:
+ mode = int(parts[0], 8)
+ path = os.path.join(self._recv_dir, parts[2])
+ except:
+ self.channel.send('\x01')
+ raise SCPException('Bad directory format')
+ try:
+ if not os.path.exists(path):
+ os.mkdir(path, mode)
+ elif os.path.isdir(path):
+ os.chmod(path, mode)
+ else:
+ raise SCPException('%s: Not a directory' % path)
+ self._dirtimes[path] = (self._utime)
+ self._utime = None
+ self._recv_dir = path
+ except (OSError, SCPException), e:
+ self.channel.send('\x01'+e.message)
+ raise
+
+ def _recv_popd(self, *cmd):
+ self._recv_dir = os.path.split(self._recv_dir)[0]
+
+ def _set_dirtimes(self):
+ try:
+ for d in self._dirtimes:
+ os.utime(d, self._dirtimes[d])
+ finally:
+ self._dirtimes = {}
+
+
+class SCPException(Exception):
+ """SCP exception class"""
+ pass
Added: gnunet-planetlab/gplmt/gplmt/Tasks.py
===================================================================
--- gnunet-planetlab/gplmt/gplmt/Tasks.py (rev 0)
+++ gnunet-planetlab/gplmt/gplmt/Tasks.py 2012-08-03 14:01:14 UTC (rev
23065)
@@ -0,0 +1,280 @@
+#!/usr/bin/python
+#
+# This file is part of GNUnet.
+# (C) 2010 Christian Grothoff (and other contributing authors)
+#
+# GNUnet is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published
+# by the Free Software Foundation; either version 2, or (at your
+# option) any later version.
+#
+# GNUnet is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with GNUnet; see the file COPYING. If not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+#
+# GNUnet Planetlab deployment and automation toolset
+#
+# Nodes
+try:
+ import xml
+ import sys
+ from xml.parsers import expat
+ from minixsv import pyxsval as xsv
+ from elementtree import ElementTree
+ elementtree_loaded = True
+ from genxmlif import GenXmlIfError
+except ImportError:
+ print "That's a bug! please check README"
+ sys.exit(1)
+
+glogger = None
+
+supported_operations = ["run", "monitor", "get", "put"]
+
+class Taskresult:
+ unspecified = -1
+ success = 0
+ timeout = 1
+ fail = 2
+ return_value_did_not_match = 3
+ output_did_not_match = 4
+ src_file_not_found = 5
+
+
+class Operation:
+ none=0
+ run=1
+ monitor=2
+ get=3
+ put=4
+
+class Task:
+ def __init__(self):
+ self.id = -1
+ self.name = ""
+ self.type = Operation.none
+ self.command = ""
+ self.arguments = ""
+ self.timeout = 0
+ self.expected_return_code = -1
+ self.expected_output = None
+ self.stop_on_fail = False
+ self.set = None
+ self.src = None
+ self.dest = None
+ def log (self):
+ glogger.log ("Task " + str(self.id) + ": " + self.name)
+ def check (self):
+ if (Operation.none):
+ return False
+ if (self.type == Operation.run):
+ if ((self.id == -1) or (self.name == "") or (self.command == "")):
+ return False
+ if (self.type == Operation.put):
+ if ((self.id == -1) or (self.name == "") or
+ (self.src == None) or (self.dest == None)):
+ return False
+ if (self.type == Operation.get):
+ if ((self.id == -1) or (self.name == "") or
+ (self.src == None) or (self.dest == None)):
+ return False
+ return True
+
+class Taskset:
+ def __init__(self):
+ self.set = list()
+
+
+def handle_task (elem, tasks):
+ t = Task ()
+
+ if (None != elem.attrib.get("name")):
+ t.name = elem.attrib.get("name")
+
+ if (None != elem.attrib.get("id")):
+ t.id = elem.attrib.get("id")
+
+ if (tasks.startid != -1):
+ if (tasks.startid == t.id):
+ tasks.startid_found = True
+ glogger.log ("Task " + str (t.id) + " '" + t.name + "' has start
ID")
+ if (tasks.startid_found == False):
+ glogger.log ("Task " + str (t.id) + " '" + t.name + "' skipped")
+ return None;
+
+ if (None != elem.attrib.get("enabled")):
+ if ("FALSE" == str(elem.attrib.get("enabled")).upper()):
+ glogger.log ("Task " + str (t.id) + " '" + t.name + "' is
disabled")
+ return
+
+ if (elem.tag == "run"):
+ t.type = Operation.run
+ elif (elem.tag == "monitor"):
+ t.type = Operation.monitor
+ elif (elem.tag == "get"):
+ t.type = Operation.get
+ elif (elem.tag == "put"):
+ t.type = Operation.put
+ else:
+ t.type = Operation.none
+
+ for child in elem:
+ if ((child.tag == "command") and (child.text != None)):
+ t.command = child.text
+ if ((child.tag == "arguments") and (child.text != None)):
+ t.arguments = child.text
+ if (child.tag == "timeout"):
+ try:
+ t.timeout = int(child.text)
+ except ValueError:
+ print "Invalid timeout '"+child.text+"' for task id " + str
(t.id) + " name " + t.name
+ if (child.tag == "expected_return_code"):
+ try:
+ t.expected_return_code = int(child.text)
+ except ValueError:
+ print "Invalid expected return code '" +child.text+ "' for
task id " + str (t.id) + " name " + t.name
+ if ((child.tag == "expected_output") and (child.text != None)):
+ t.expected_output = child.text
+ if ((child.tag == "stop_on_fail") and (child.text != None)):
+ if (str.upper(child.text) == "TRUE"):
+ t.stop_on_fail = True
+ else:
+ t.stop_on_fail = False
+ if ((child.tag == "source") and (child.text != None)):
+ t.src = child.text
+ if ((child.tag == "destination") and (child.text != None)):
+ t.dest = child.text
+
+ if (False == t.check()):
+ print "Parsed invalid task with id " + str (t.id) + " name '" + t.name
+ "'"
+ return None
+ else:
+ t.log ()
+ return t
+
+def handle_sequence (elem, tasks):
+ if (None != elem.attrib.get("enabled")):
+ if ("FALSE" == str(elem.attrib.get("enabled")).upper()):
+ glogger.log ("Element was disabled")
+ return
+ for child in elem:
+ handle_child (child, tasks)
+
+def handle_parallel (elem, tasks):
+ glogger.log ("Found parallel execution with " + str(len(elem)) + "
elements")
+ if (None != elem.attrib.get("enabled")):
+ if ("FALSE" == str(elem.attrib.get("enabled")).upper()):
+ glogger.log ("Element was disabled")
+ return
+ ptask = Taskset ()
+ for child in elem:
+ if (elem.tag in supported_operations):
+ t = handle_task (elem)
+ if (None != t):
+ ptask.set.append(t)
+ print "Added " + t.name + " to set"
+ elif (elem.tag == "parallel"):
+ glogger.log ("x")
+ # handle_parallel (elem, l)
+ elif (elem.tag == "sequence"):
+ glogger.log ("x")
+ # handle_sequence (elem, l)
+ else:
+ print "Invalid element in task file: " + elem.tag
+ tasks.l.append (ptask)
+
+def handle_child (elem, tasks):
+ global support_operations
+ if (elem.tag in supported_operations):
+ t = handle_task (elem, tasks)
+ if (None != t):
+ tasks.l.append(t)
+ elif (elem.tag == "parallel"):
+ handle_parallel (elem, tasks)
+ elif (elem.tag == "sequence"):
+ handle_sequence (elem, tasks)
+ else:
+ print "Invalid element in task file: " + elem.tag
+
+def print_sequence (l):
+ for i in l:
+ print "->",
+ if (i.__class__.__name__ == "Task"):
+ print i.name,
+ elif (i.__class__.__name__ == "Taskset"):
+ print "{",
+ print_sequence (i.set)
+ print "}",
+
+
+
+class Tasks:
+ def __init__(self, filename, logger, startid):
+ assert (None != logger)
+ global glogger
+ glogger = logger
+ self.logger = logger
+ self.filename = filename
+ self.name = "<Undefined>"
+ self.l = list ()
+ self.startid = startid
+ self.startid_found = False
+ def load (self):
+ self.logger.log ("Loading tasks file '" + self.filename + "'")
+ enabled = True
+ try:
+ xsv.parseAndValidate (self.filename)
+ except xsv.xsvalErrorHandler.XsvalError as e:
+ print "File '" + self.filename + "' does not validate against
schema: \n" + str(e)
+ return False
+ except GenXmlIfError as e:
+ print "File '" + self.filename + "' is not well-formed: \n" +
str(e)
+ return False
+ except IOError:
+ print "File '" + self.filename + "' not found \n"
+ return False
+
+ try:
+ root = ElementTree.parse (self.filename).getroot()
+ if (None != root.attrib.get("name")):
+ self.name = root.attrib.get("name")
+ if (None != root.attrib.get("enabled")):
+ if (False == root.attrib.get("enabled")):
+ enabled = False
+ except expat.ExpatError as e:
+ print "File '" + self.filename + "'is malformed: " + str(e)
+ return False
+ except IOError:
+ print "File '" + self.filename + "' not found"
+ return False
+ if (enabled == True):
+ for child in root:
+ handle_child (child, self)
+ else:
+ print "Tasklist " + self.filename + " was disabled"
+ #print_sequence (self.l)
+
+ return True
+
+
+ def copy (self):
+ t = Tasks (self.filename, self.logger, -1)
+ # Create a copy of the task list as described in
+ # http://docs.python.org/library/copy.html
+ t.filename = self.filename
+ t.name = self.name
+ t.l = self.l[:]
+ return t
+ def get (self):
+ if (len (self.l) > 0):
+ item = self.l[0]
+ self.l.remove(item)
+ return item
+ else:
+ return None
Added: gnunet-planetlab/gplmt/gplmt/Util.py
===================================================================
--- gnunet-planetlab/gplmt/gplmt/Util.py (rev 0)
+++ gnunet-planetlab/gplmt/gplmt/Util.py 2012-08-03 14:01:14 UTC (rev
23065)
@@ -0,0 +1,35 @@
+#!/usr/bin/python
+#
+# This file is part of GNUnet.
+# (C) 2010 Christian Grothoff (and other contributing authors)
+#
+# GNUnet is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published
+# by the Free Software Foundation; either version 2, or (at your
+# option) any later version.
+#
+# GNUnet is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with GNUnet; see the file COPYING. If not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+#
+# GNUnet Planetlab deployment and automation toolset
+#
+# Utilities
+
+class Logger:
+
+ def __init__(self, verbose):
+ if (True == verbose):
+ self.verbose = True
+ else:
+ self.verbose = False
+ def log (self, message):
+ global main
+ if (True == self.verbose):
+ print (message)
\ No newline at end of file
Added: gnunet-planetlab/gplmt/gplmt/Worker.py
===================================================================
--- gnunet-planetlab/gplmt/gplmt/Worker.py (rev 0)
+++ gnunet-planetlab/gplmt/gplmt/Worker.py 2012-08-03 14:01:14 UTC (rev
23065)
@@ -0,0 +1,295 @@
+#!/usr/bin/python
+#
+# This file is part of GNUnet.
+# (C) 2010 Christian Grothoff (and other contributing authors)
+#
+# GNUnet is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published
+# by the Free Software Foundation; either version 2, or (at your
+# option) any later version.
+#
+# GNUnet is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with GNUnet; see the file COPYING. If not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+#
+# GNUnet Planetlab deployment and automation toolset
+#
+# Worker
+
+try:
+ import gplmt.Configuration as TransferMode
+ import gplmt.Tasks as Tasks
+
+ import threading
+ import socket
+ import os
+ import time
+ import sys
+ import select
+ from gplmt.SCP import SCPClient
+
+except ImportError:
+ print "That's a bug! please check README"
+ sys.exit(1)
+
+
+
+try:
+ import paramiko
+ from paramiko.ssh_exception import SSHException
+except ImportError:
+ pass
+
+# Global variables
+g_logger = None
+g_notifications = None
+g_configuration = None
+
+class NodeWorkerThread (threading.Thread):
+ def __init__(self, threadID, node, tasks):
+ threading.Thread.__init__(self)
+ self.threadID = threadID
+ self.node = node
+ self.tasks = tasks
+ def exec_run (self, task, transport):
+ try:
+ channel = transport.open_session()
+ channel.settimeout(1.0)
+ channel.get_pty ()
+ #print "CMD: " + task.command + " " + task.arguments
+ channel.exec_command(task.command + " " + task.arguments)
+
+ except SSHException as e:
+ print self.node + " : Error while trying to connect: " + str(e)
+
+ if (task.timeout > 0):
+ timeout = task.timeout
+ else:
+ timeout = -1
+ result = Tasks.Taskresult.success
+ exit_status = -1
+ start_time = time.time ()
+
+ stdout_data = ""
+ stderr_data = ""
+ while 1:
+ if (timeout != -1):
+ delta = time.time() - start_time
+ if (delta > timeout):
+ print "Timeout after " +str(delta) +" seconds"
+ result = Tasks.Taskresult.timeout
+ break
+ (r, w, e) = select.select([channel], [], [], 1)
+ if r:
+ got_data = False
+ if channel.recv_ready():
+ data = r[0].recv(4096)
+ if data:
+ got_data = True
+ #g_logger.log ('STDOUT: ' + data)
+ stdout_data += data
+ if channel.recv_stderr_ready():
+ data = r[0].recv_stderr(4096)
+ if data:
+ got_data = True
+ #g_logger.log ('STDERR: ' + data)
+ stderr_data += data
+ if not got_data:
+ break
+
+ if (result == Tasks.Taskresult.success):
+ exit_status = channel.recv_exit_status ()
+
+ if (result == Tasks.Taskresult.success):
+ if (task.expected_return_code != -1):
+ if (exit_status != task.expected_return_code):
+ g_logger.log (self.node + " : Task '"+ task.name + "'
completed after "+ str(time.time() - start_time) +" sec, but exit code "
+str(exit_status)+ " was not as expected " + str(task.expected_return_code))
+ g_logger.log (stdout_data)
+ g_logger.log (stderr_data)
+ result = Tasks.Taskresult.return_value_did_not_match
+ else:
+ g_logger.log (self.node + " : Task '"+ task.name + "'
completed after "+ str(time.time() - start_time) +" sec, exit code "
+str(exit_status)+ " was as expected " + str(task.expected_return_code))
+
+ if (task.expected_output != None):
+ output_contained = False
+ if (task.expected_output in stdout_data):
+ output_contained = True
+ if (task.expected_output in stderr_data):
+ output_contained = True
+ if (output_contained == True):
+ g_logger.log (self.node + " : Task '"+ task.name + "'
expected output '"+task.expected_output+"' was found")
+ else:
+ g_logger.log (self.node + " : Task '"+ task.name + "'
expected output '"+task.expected_output+"' was not found")
+ result = Tasks.Taskresult.output_did_not_match
+
+ if (result == Tasks.Taskresult.success):
+ g_logger.log (self.node + " : Task '"+ task.name + "' successful")
+ elif (result == Tasks.Taskresult.timeout):
+ g_logger.log (self.node + " : Task '"+ task.name + "' with
timeout")
+ else:
+ g_logger.log (self.node + " : Task '"+ task.name + "' failed")
+ return result
+ def exec_put (self, task, transport):
+ if (False == os.path.exists (task.src)):
+ return Tasks.Taskresult.src_file_not_found
+
+ result = Tasks.Taskresult.success
+ try:
+ if (g_configuration.ssh_transfer == TransferMode.scp):
+ scp = SCPClient (transport)
+ scp.put (task.src, task.dest)
+ if (g_configuration.ssh_transfer == TransferMode.sftp):
+ sftp = paramiko.SFTPClient.from_transport (transport)
+ sftp.put(task.src, task.dest)
+ sftp.close()
+ except paramiko.SSHException as e:
+ g_logger.log (self.node + " : Task '"+ task.name + "' :" + str(e))
+ result = Tasks.Taskresult.fail
+ pass
+ except (OSError, IOError) as e:
+ g_logger.log (self.node + " : Task '"+ task.name + "' : " + str(e))
+ result = Tasks.Taskresult.src_file_not_found
+ pass
+ return result
+
+ def exec_get (self, task, transport):
+ result = Tasks.Taskresult.success
+ try:
+ if (g_configuration.ssh_transfer == TransferMode.scp):
+ scp = SCPClient (transport)
+ scp.get (task.src, task.dest)
+ if (g_configuration.ssh_transfer == TransferMode.sftp):
+ sftp = paramiko.SFTPClient.from_transport (transport)
+ sftp.get (task.src, task.dest)
+ sftp.close()
+ except paramiko.SSHException as e:
+ g_logger.log (self.node + " : Task '"+ task.name + "' :" + str(e))
+ result = Tasks.Taskresult.fail
+ pass
+ except (OSError, IOError) as e:
+ g_logger.log (self.node + " : Task '"+ task.name + "' : " + str(e))
+ result = Tasks.Taskresult.src_file_not_found
+ pass
+ return result
+
+ def run(self):
+ g_logger.log (self.node + " : Starting tasklist " + self.tasks.name)
+ task = self.tasks.get()
+ try:
+ ssh = paramiko.SSHClient()
+ if (g_configuration.ssh_use_known_hosts):
+ g_logger.log (self.node + " : Loading known hosts")
+ ssh.load_system_host_keys ()
+
+ # Automatically add new hostkeys
+ if (g_configuration.ssh_add_unkown_hostkeys == True):
+ ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ # check for private key existance
+ keyfile = None
+ if (g_configuration.ssh_keyfile != None):
+ if (os.path.exists (g_configuration.ssh_keyfile)):
+ g_logger.log (self.node + " : Found " +
g_configuration.ssh_keyfile)
+ keyfile = g_configuration.ssh_keyfile
+ else:
+ g_logger.log (self.node + " : Not found " +
g_configuration.ssh_keyfile)
+
+ g_logger.log (self.node + " : Trying to connect to " +
+ g_configuration.pl_slicename + "@" + self.node +
+ " using password '" + g_configuration.ssh_password+
+ "' and private keyfile '" +str(keyfile)+ "'")
+ ssh.connect (self.node,
+ username=g_configuration.pl_slicename,
+ password=g_configuration.ssh_password,
+ timeout=10,
+ key_filename=keyfile)
+ except (IOError,
+ paramiko.SSHException,
+ paramiko.BadHostKeyException,
+ paramiko.AuthenticationException,
+ socket.error) as e:
+ print self.node + " : Error while trying to connect: " + str(e)
+ g_notifications.node_connected (self.node, False)
+ g_notifications.tasklist_completed (self.node, self.tasks, False)
+ return
+
+ g_notifications.node_connected (self.node, True)
+
+ transport = ssh.get_transport()
+ success = True
+ result = Tasks.Taskresult.success
+ while (None != task):
+ g_logger.log (self.node + " : Running task id " +str(task.id)+" '"
+ task.name + "'")
+ g_notifications.task_started (self.node, task)
+ if (task.__class__.__name__ == "Task"):
+ if (task.type == Tasks.Operation.run):
+ result = self.exec_run (task, transport)
+ g_notifications.task_completed (self.node, task, result)
+ elif (task.type == Tasks.Operation.put):
+ result = self.exec_put (task, transport)
+ g_notifications.task_completed (self.node, task, result)
+ elif (task.type == Tasks.Operation.get):
+ result = self.exec_get (task, transport)
+ g_notifications.task_completed (self.node, task, result)
+ else:
+ print "FAIL"
+ elif (task.__class__.__name__ == "Taskset"):
+ g_logger.log (self.node + " : Running task set")
+ if ((task.stop_on_fail == True) and (result !=
Tasks.Taskresult.success)):
+ g_logger.log (self.node + " : Task failed and therefore
execution is stopped")
+ transport.close()
+ success = False
+ break
+ task = self.tasks.get()
+
+ ssh.close()
+ g_notifications.node_disconnected (self.node, True)
+ g_notifications.tasklist_completed (self.node, self.tasks, success)
+ g_logger.log (self.node + " : All tasks done for " + self.node)
+
+
+
+class NodeWorker:
+ def __init__(self, node, tasks):
+ assert (None != node)
+ assert (None != tasks)
+ self.node = node
+ self.tasks = tasks
+ self.thread = None
+ def start (self):
+ g_logger.log ("Starting execution for node " + self.node)
+ g_notifications.tasklist_started (self.node, self.tasks)
+ self.thread = NodeWorkerThread (1, self.node, self.tasks)
+ self.thread.start()
+
+class Worker:
+ def __init__(self, logger, configuration, nodes, tasks, notifications):
+ global g_logger;
+ global g_configuration;
+ global g_notifications;
+ assert (None != logger)
+ assert (None != nodes)
+ assert (None != tasks)
+ assert (None != notifications)
+ assert (hasattr(notifications, 'node_connected'))
+ assert (hasattr(notifications, 'node_disconnected'))
+ assert (hasattr(notifications, 'tasklist_started'))
+ assert (hasattr(notifications, 'tasklist_completed'))
+ assert (hasattr(notifications, 'task_started'))
+ assert (hasattr(notifications, 'task_completed'))
+ self.nodes = nodes
+ self.tasks = tasks
+ g_configuration = configuration
+ g_notifications = notifications
+ g_logger = logger;
+ def start (self):
+ g_logger.log ("Starting execution")
+ for n in self.nodes.nodes:
+ nw = NodeWorker (n, self.tasks.copy())
+ nw.start()
+
Modified: gnunet-planetlab/gplmt/gplmt.py
===================================================================
--- gnunet-planetlab/gplmt/gplmt.py 2012-08-03 14:00:04 UTC (rev 23064)
+++ gnunet-planetlab/gplmt/gplmt.py 2012-08-03 14:01:14 UTC (rev 23065)
@@ -40,18 +40,22 @@
except ImportError:
minixsv_loaded = False
+try:
+ import gplmt.Util as Util
+ import gplmt.Configuration as Configuration
+ import gplmt.Nodes as Nodes
+ import gplmt.Tasks as Tasks
+ import gplmt.Worker as Worker
+ import gplmt.Notifications as Notifications
-import getopt;
-import sys;
-import Util;
-import Configuration;
-import Nodes;
-import Tasks;
-import Worker;
-import Notifications;
+ import getopt, sys
+except ImportError:
+ print "That's a bug! please check README"
+ sys.exit (1)
+
def main():
global main
main = Main ()
Deleted: gnunet-planetlab/gplmt/tasklist_schema.xsd
===================================================================
--- gnunet-planetlab/gplmt/tasklist_schema.xsd 2012-08-03 14:00:04 UTC (rev
23064)
+++ gnunet-planetlab/gplmt/tasklist_schema.xsd 2012-08-03 14:01:14 UTC (rev
23065)
@@ -1,98 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema";
elementFormDefault="qualified">
-<xs:element name="tasklist" type="tasklist" />
-
-<xs:complexType name="tasklist">
- <xs:sequence>
- <xs:choice minOccurs="0" maxOccurs="unbounded">
- <xs:element name="sequence" type="sequence" />
- <xs:element name="parallel" type="parallel" />
- <xs:element name="run" type="run" />
- <xs:element name="monitor" type="monitor" />
- <xs:element name="put" type="put" />
- <xs:element name="get" type="get" />
- </xs:choice>
- </xs:sequence>
- <xs:attribute name="name" type="xs:string"/>
- <xs:attribute name="enabled" type="xs:boolean"/>
-</xs:complexType>
-
-<xs:complexType name="sequence">
- <xs:sequence>
- <xs:choice minOccurs="0" maxOccurs="unbounded">
- <xs:element name="sequence" type="sequence" />
- <xs:element name="parallel" type="parallel" />
- <xs:element name="run" type="run" />
- <xs:element name="monitor" type="monitor" />
- <xs:element name="put" type="put" />
- <xs:element name="get" type="get" />
- </xs:choice>
- </xs:sequence>
- <xs:attribute name="enabled" type="xs:boolean"/>
-</xs:complexType>
-
-<xs:complexType name="parallel">
- <xs:sequence>
- <xs:choice minOccurs="0" maxOccurs="unbounded">
- <xs:element name="sequence" type="sequence" />
- <xs:element name="parallel" type="parallel" />
- <xs:element name="run" type="run" />
- <xs:element name="monitor" type="monitor" />
- <xs:element name="put" type="put" />
- <xs:element name="get" type="get" />
- </xs:choice>
- </xs:sequence>
- <xs:attribute name="enabled" type="xs:boolean"/>
-</xs:complexType>
-
-<xs:complexType name="run">
- <xs:sequence>
- <xs:element name="command" type="xs:string"/>
- <xs:element name="arguments" type="xs:string"/>
- <xs:element name="timeout" type="xs:integer"/>
- <xs:element name="expected_return_code" type="xs:integer"/>
- <xs:element name="expected_output" type="xs:string"/>
- <xs:element name="stop_on_fail" type="xs:boolean"/>
- </xs:sequence>
- <xs:attribute name="id" type="xs:integer"/>
- <xs:attribute name="name" type="xs:string"/>
- <xs:attribute name="enabled" type="xs:boolean"/>
-</xs:complexType>
-
-<xs:complexType name="monitor">
- <xs:sequence>
- <xs:element name="command" type="xs:string"/>
- <xs:element name="arguments" type="xs:string"/>
- <xs:element name="timeout" type="xs:integer"/>
- <xs:element name="expected_return_code" type="xs:integer"/>
- <xs:element name="expected_output" type="xs:string"/>
- <xs:element name="stop_on_fail" type="xs:boolean"/>
- </xs:sequence>
- <xs:attribute name="id" type="xs:integer"/>
- <xs:attribute name="name" type="xs:string"/>
- <xs:attribute name="enabled" type="xs:boolean"/>
-</xs:complexType>
-
-<xs:complexType name="put">
- <xs:sequence>
- <xs:element name="source" type="xs:string"/>
- <xs:element name="destination" type="xs:string"/>
- <xs:element name="stop_on_fail" type="xs:boolean"/>
- </xs:sequence>
- <xs:attribute name="id" type="xs:integer"/>
- <xs:attribute name="name" type="xs:string"/>
- <xs:attribute name="enabled" type="xs:boolean"/>
-</xs:complexType>
-
-<xs:complexType name="get">
- <xs:sequence>
- <xs:element name="source" type="xs:string"/>
- <xs:element name="destination" type="xs:string"/>
- <xs:element name="stop_on_fail" type="xs:boolean"/>
- </xs:sequence>
- <xs:attribute name="id" type="xs:integer"/>
- <xs:attribute name="name" type="xs:string"/>
- <xs:attribute name="enabled" type="xs:boolean"/>
-</xs:complexType>
-
-</xs:schema>
| [Prev in Thread] |
Current Thread |
[Next in Thread] |
- [GNUnet-SVN] r23065 - in gnunet-planetlab/gplmt: . contrib contrib/tasklists gplmt,
gnunet <=