From 9bec4d6bbab599bee72256c7e09fe214cb849a1b Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 5 Feb 2013 12:13:20 -0500 Subject: abstracted similar digit range classes in POSIXUsers/GroupPatterns into Bcfg2.Utils --- src/lib/Bcfg2/Utils.py | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 src/lib/Bcfg2/Utils.py (limited to 'src/lib/Bcfg2/Utils.py') diff --git a/src/lib/Bcfg2/Utils.py b/src/lib/Bcfg2/Utils.py new file mode 100644 index 000000000..ba17e1a63 --- /dev/null +++ b/src/lib/Bcfg2/Utils.py @@ -0,0 +1,67 @@ +""" Miscellaneous useful utility functions, classes, etc., that are +used by both client and server. Stuff that doesn't fit anywhere +else. """ + +from Bcfg2.Compat import any # pylint: disable=W0622 + + +class PackedDigitRange(object): + """ Representation of a set of integer ranges. A range is + described by a comma-delimited string of integers and ranges, + e.g.:: + + 1,10-12,15-20 + + Ranges are inclusive on both bounds, and may include 0. Negative + numbers are not supported.""" + + def __init__(self, *ranges): + """ May be instantiated in one of two ways:: + + PackedDigitRange() + + Or:: + + PackedDigitRange([, [, ...]]) + + E.g., both of the following are valid:: + + PackedDigitRange("1-5,7, 10-12") + PackedDigitRange("1-5", 7, "10-12") + """ + self.ranges = [] + self.ints = [] + self.str = ",".join(str(r) for r in ranges) + if len(ranges) == 1 and "," in ranges[0]: + ranges = ranges[0].split(",") + for item in ranges: + item = str(item).strip() + if item.endswith("-"): + self.ranges.append((int(item[:-1]), None)) + elif '-' in str(item): + self.ranges.append(tuple(int(x) for x in item.split('-'))) + else: + self.ints.append(int(item)) + + def includes(self, other): + """ Return True if ``other`` is included in this range. + Functionally equivalent to ``other in range``, which should be + used instead. """ + return other in self + + def __contains__(self, other): + other = int(other) + if other in self.ints: + return True + return any((end is None and other >= start) or + (end is not None and other >= start and other <= end) + for start, end in self.ranges) + + def __repr__(self): + return "%s:%s" % (self.__class__.__name__, str(self)) + + def __str__(self): + return "[%s]" % self.str + + def __len__(self): + return sum(r[1] - r[0] + 1 for r in self.ranges) + len(self.ints) -- cgit v1.2.3-1-g7c22 From fd67a2735ada342251cb6baaa4e678532566e975 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 7 Feb 2013 10:00:37 -0500 Subject: moved common file locking code into Bcfg2.Utils --- src/lib/Bcfg2/Utils.py | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'src/lib/Bcfg2/Utils.py') diff --git a/src/lib/Bcfg2/Utils.py b/src/lib/Bcfg2/Utils.py index ba17e1a63..247e4f16b 100644 --- a/src/lib/Bcfg2/Utils.py +++ b/src/lib/Bcfg2/Utils.py @@ -2,6 +2,7 @@ used by both client and server. Stuff that doesn't fit anywhere else. """ +import fcntl from Bcfg2.Compat import any # pylint: disable=W0622 @@ -65,3 +66,12 @@ class PackedDigitRange(object): def __len__(self): return sum(r[1] - r[0] + 1 for r in self.ranges) + len(self.ints) + + +def locked(fd): + """ Acquire a lock on a file """ + try: + fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except IOError: + return True + return False -- cgit v1.2.3-1-g7c22 From f91163abed4aa739f7f8b772eabb403f01b94a88 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 13 Feb 2013 16:08:08 -0500 Subject: extended usage of Executor class, added client-side timeout options --- src/lib/Bcfg2/Utils.py | 156 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 155 insertions(+), 1 deletion(-) (limited to 'src/lib/Bcfg2/Utils.py') diff --git a/src/lib/Bcfg2/Utils.py b/src/lib/Bcfg2/Utils.py index 247e4f16b..3b1559528 100644 --- a/src/lib/Bcfg2/Utils.py +++ b/src/lib/Bcfg2/Utils.py @@ -3,9 +3,25 @@ used by both client and server. Stuff that doesn't fit anywhere else. """ import fcntl +import logging +import threading +import subprocess from Bcfg2.Compat import any # pylint: disable=W0622 +class ClassName(object): + """ This very simple descriptor class exists only to get the name + of the owner class. This is used because, for historical reasons, + we expect every server plugin and every client tool to have a + ``name`` attribute that is in almost all cases the same as the + ``__class__.__name__`` attribute of the plugin object. This makes + that more dynamic so that each plugin and tool isn't repeating its own + name.""" + + def __get__(self, inst, owner): + return owner.__name__ + + class PackedDigitRange(object): """ Representation of a set of integer ranges. A range is described by a comma-delimited string of integers and ranges, @@ -69,9 +85,147 @@ class PackedDigitRange(object): def locked(fd): - """ Acquire a lock on a file """ + """ Acquire a lock on a file. + + :param fd: The file descriptor to lock + :type fd: int + :returns: bool - True if the file is already locked, False + otherwise """ try: fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError: return True return False + + +class ExecutorResult(object): + """ Returned as the result of a call to + :func:`Bcfg2.Utils.Executor.run`. The result can be accessed via + the instance variables, documented below, as a boolean (which is + equivalent to :attr:`Bcfg2.Utils.ExecutorResult.success`), or as a + tuple, which, for backwards compatibility, is equivalent to + ``(result.retval, result.stdout.splitlines())``.""" + + def __init__(self, stdout, stderr, retval): + #: The output of the command + self.stdout = stdout + + #: The error produced by the command + self.stderr = stderr + + #: The return value of the command. + self.retval = retval + + #: Whether or not the command was successful. If the + #: ExecutorResult is used as a boolean, ``success`` is + #: returned. + self.success = retval == 0 + + #: A friendly error message + self.error = None + if self.retval: + if self.stderr: + self.error = "%s (rv: %s)" % (self.stderr, self.retval) + elif self.stdout: + self.error = "%s (rv: %s)" % (self.stdout, self.retval) + else: + self.error = "No output or error; return value %s" % \ + self.retval + + def __repr__(self): + if self.error: + return "Errored command result: %s" % self.error + elif self.stdout: + return "Successful command result: %s" % self.stdout + else: + return "Successful command result: No output" + + def __getitem__(self, idx): + """ This provides compatibility with the old Executor, which + returned a tuple of (return value, stdout split by lines). """ + return (self.retval, self.stdout.splitlines())[idx] + + def __nonzero__(self): + return self.__bool__() + + def __bool__(self): + return self.success + + +class Executor(object): + """ A convenient way to run external commands with + :class:`subprocess.Popen` """ + + def __init__(self, timeout=None): + """ + :param timeout: Set a default timeout for all commands run by + this Executor object + :type timeout: float + """ + self.logger = logging.getLogger(self.__class__.__name__) + self.timeout = timeout + + def _timeout_callback(self, proc): + """ Get a callback (suitable for passing to + :class:`threading.Timer`) that kills the given process. + + :param proc: The process to kill upon timeout. + :type proc: subprocess.Popen + :returns: function """ + def _timeout(): + """ Callback that kills ``proc`` """ + if proc.poll() == None: + try: + proc.kill() + self.logger.warning("Process exceeeded timeout, killing") + except OSError: + pass + + return _timeout + + def run(self, command, inputdata=None, shell=False, timeout=None): + """ Run a command, given as a list, optionally giving it the + specified input data. + + :param command: The command to run, as a list (preferred) or + as a string. See :class:`subprocess.Popen` for + details. + :type command: list or string + :param inputdata: Data to pass to the command on stdin + :type inputdata: string + :param shell: Run the given command in a shell (not recommended) + :type shell: bool + :param timeout: Kill the command if it runs longer than this + many seconds. Set to 0 or -1 to explicitly + override a default timeout. + :type timeout: float + :returns: :class:`Bcfg2.Utils.ExecutorResult` + """ + if isinstance(command, str): + cmdstr = command + else: + cmdstr = " ".join(command) + self.logger.debug("Running: %s" % cmdstr) + proc = subprocess.Popen(command, shell=shell, bufsize=16384, + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, close_fds=True) + if timeout is None: + timeout = self.timeout + if timeout is not None: + timer = threading.Timer(float(timeout), + self._timeout_callback(proc), + [proc]) + timer.start() + try: + if inputdata: + for line in inputdata.splitlines(): + self.logger.debug('> %s' % line) + (stdout, stderr) = proc.communicate(input=inputdata) + for line in stdout.splitlines(): # pylint: disable=E1103 + self.logger.debug('< %s' % line) + for line in stderr.splitlines(): # pylint: disable=E1103 + self.logger.info(line) + return ExecutorResult(stdout, stderr, proc.wait()) + finally: + if timeout is not None: + timer.cancel() -- cgit v1.2.3-1-g7c22 From 3d06f311274d6b942ee89d8cdb13b2ecc99af1b0 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 14 Mar 2013 13:05:08 -0400 Subject: use Executor class for better subprocess calling on server --- src/lib/Bcfg2/Utils.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) (limited to 'src/lib/Bcfg2/Utils.py') diff --git a/src/lib/Bcfg2/Utils.py b/src/lib/Bcfg2/Utils.py index 3b1559528..29c27257a 100644 --- a/src/lib/Bcfg2/Utils.py +++ b/src/lib/Bcfg2/Utils.py @@ -183,9 +183,10 @@ class Executor(object): return _timeout - def run(self, command, inputdata=None, shell=False, timeout=None): + def run(self, command, inputdata=None, timeout=None, **kwargs): """ Run a command, given as a list, optionally giving it the - specified input data. + specified input data. All additional keyword arguments are + passed through to :class:`subprocess.Popen`. :param command: The command to run, as a list (preferred) or as a string. See :class:`subprocess.Popen` for @@ -193,8 +194,6 @@ class Executor(object): :type command: list or string :param inputdata: Data to pass to the command on stdin :type inputdata: string - :param shell: Run the given command in a shell (not recommended) - :type shell: bool :param timeout: Kill the command if it runs longer than this many seconds. Set to 0 or -1 to explicitly override a default timeout. @@ -206,9 +205,11 @@ class Executor(object): else: cmdstr = " ".join(command) self.logger.debug("Running: %s" % cmdstr) - proc = subprocess.Popen(command, shell=shell, bufsize=16384, - stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, close_fds=True) + args = dict(shell=False, bufsize=16384, close_fds=True) + args.update(kwargs) + args.update(stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + proc = subprocess.Popen(command, **args) if timeout is None: timeout = self.timeout if timeout is not None: -- cgit v1.2.3-1-g7c22