From c1e30f4bc8a03a9be535bf126e993ff45e8ab382 Mon Sep 17 00:00:00 2001 From: Nathan Olla Date: Mon, 14 Jul 2014 10:49:44 -0400 Subject: SYSV: Implement downloading and installing SYSV packages from HTTP pkgadd has different syntax for different sources (datastream and file system format) which makes using a single pkgtool variable difficult. Also, SYSV packages in datastream format don't necessarily have uniform names. Therefore, use the existing 'simplename' attribute to specify the datastream file name. --- src/lib/Bcfg2/Client/Tools/SYSV.py | 48 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/src/lib/Bcfg2/Client/Tools/SYSV.py b/src/lib/Bcfg2/Client/Tools/SYSV.py index aca7d593c..c17c3b712 100644 --- a/src/lib/Bcfg2/Client/Tools/SYSV.py +++ b/src/lib/Bcfg2/Client/Tools/SYSV.py @@ -4,6 +4,9 @@ import tempfile from Bcfg2.Compat import any # pylint: disable=W0622 import Bcfg2.Client.Tools import Bcfg2.Client.XML +import urllib +import copy + # pylint: disable=C0103 noask = ''' @@ -37,6 +40,8 @@ class SYSV(Bcfg2.Client.Tools.PkgTool): # noaskfile needs to live beyond __init__ otherwise file is removed self.noaskfile = tempfile.NamedTemporaryFile() self.noaskname = self.noaskfile.name + # for any pkg files downloaded + self.tmpfiles = [] try: self.noaskfile.write(noask) # flush admin file contents to disk @@ -46,6 +51,45 @@ class SYSV(Bcfg2.Client.Tools.PkgTool): except: # pylint: disable=W0702 self.pkgtool = (self.pkgtool[0] % "", self.pkgtool[1]) + def pkgmogrify(self, pkg): + """ Take a pkg object, check for a 'simplename' attribute. If found, + return a modified pkg object that points to to a temporary file + """ + if pkg.get('simplename'): + self.logger.debug("Pkgmogrifying %s because simplename %s found" % + (pkg.get('name'), pkg.get('simplename'))) + tmpfile = tempfile.NamedTemporaryFile() + self.tmpfiles.append(tmpfile) + self.logger.debug("URL: %s/%s" % + (pkg.get('url'), pkg.get('simplename'))) + urllib.urlretrieve("%s/%s" % + (pkg.get('url'), pkg.get('simplename')), + tmpfile.name) + newpkg = copy.copy(pkg) + newpkg.set('url', tmpfile.name) + return newpkg + return pkg + + def Install(self, packages, states): + for pkg in packages: + pkg = self.pkgmogrify(pkg) + if self.VerifyPackage(pkg, []): + self.logger.info("Forcing state to true for pkg %s" % + (pkg.get('name'))) + states[pkg] = True + else: + self.logger.info("Installing pkg %s version %s" % + (pkg.get('name'), pkg.get('version'))) + + if self.cmd.run(self._get_package_command([pkg])): + states[pkg] = True + else: + self.logger.error("Failed to install package %s" % + pkg.get('name')) + + self.RefreshPackages() + self.modified.extend(entry for entry in packages if states[entry]) + def RefreshPackages(self): """Refresh memory hashes of packages.""" self.installed = {} @@ -80,8 +124,8 @@ class SYSV(Bcfg2.Client.Tools.PkgTool): self.logger.debug("Package %s not installed" % entry.get("name")) else: - if (self.setup['quick'] or - entry.attrib.get('verify', 'true') == 'false'): + if self.setup['quick'] or \ + entry.attrib.get('verify', 'true') == 'false': return True rv = self.cmd.run("/usr/sbin/pkgchk -n %s" % entry.get('name')) if rv.success: -- cgit v1.2.3-1-g7c22 From a9fdf3d7e73c4aabc0b5e5710617c0788b9311c8 Mon Sep 17 00:00:00 2001 From: Nathan Olla Date: Mon, 14 Jul 2014 16:05:22 -0400 Subject: Add urlretrieve to Compat and document --- doc/development/compat.txt | 2 ++ src/lib/Bcfg2/Client/Tools/SYSV.py | 4 ++-- src/lib/Bcfg2/Compat.py | 4 +++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/doc/development/compat.txt b/doc/development/compat.txt index f90274ce5..8700c46d3 100644 --- a/doc/development/compat.txt +++ b/doc/development/compat.txt @@ -60,6 +60,8 @@ behavior (e.g., :func:`input`) do not cause unexpected side-effects. +---------------------------------+--------------------------------------------------+---------------------------------------------------------+ | urlparse | :func:`urlparse.urlparse` | :func:`urllib.parse.urlparse` | +---------------------------------+--------------------------------------------------+---------------------------------------------------------+ +| urlretrieve | :func:`urllib.urlretrieve` | :func:`urllib.request.urlretrieve` | ++---------------------------------+--------------------------------------------------+---------------------------------------------------------+ | HTTPBasicAuthHandler | :class:`urllib2.HTTPBasicAuthHandler` | :class:`urllib.request.HTTPBasicAuthHandler` | +---------------------------------+--------------------------------------------------+---------------------------------------------------------+ | HTTPPasswordMgrWithDefaultRealm | :class:`urllib2.HTTPPasswordMgrWithDefaultRealm` | :class:`urllib.request.HTTPPasswordMgrWithDefaultRealm` | diff --git a/src/lib/Bcfg2/Client/Tools/SYSV.py b/src/lib/Bcfg2/Client/Tools/SYSV.py index c17c3b712..2d835a8ae 100644 --- a/src/lib/Bcfg2/Client/Tools/SYSV.py +++ b/src/lib/Bcfg2/Client/Tools/SYSV.py @@ -4,8 +4,8 @@ import tempfile from Bcfg2.Compat import any # pylint: disable=W0622 import Bcfg2.Client.Tools import Bcfg2.Client.XML -import urllib import copy +from Bcfg2.Compat import urlretrieve # pylint: disable=C0103 @@ -62,7 +62,7 @@ class SYSV(Bcfg2.Client.Tools.PkgTool): self.tmpfiles.append(tmpfile) self.logger.debug("URL: %s/%s" % (pkg.get('url'), pkg.get('simplename'))) - urllib.urlretrieve("%s/%s" % + urlretrieve("%s/%s" % (pkg.get('url'), pkg.get('simplename')), tmpfile.name) newpkg = copy.copy(pkg) diff --git a/src/lib/Bcfg2/Compat.py b/src/lib/Bcfg2/Compat.py index 049236e03..88956e900 100644 --- a/src/lib/Bcfg2/Compat.py +++ b/src/lib/Bcfg2/Compat.py @@ -20,6 +20,7 @@ except ImportError: # urllib imports try: from urllib import quote_plus + from urllib import urlretrieve from urlparse import urljoin, urlparse from urllib2 import HTTPBasicAuthHandler, \ HTTPPasswordMgrWithDefaultRealm, build_opener, install_opener, \ @@ -27,7 +28,8 @@ try: except ImportError: from urllib.parse import urljoin, urlparse, quote_plus from urllib.request import HTTPBasicAuthHandler, \ - HTTPPasswordMgrWithDefaultRealm, build_opener, install_opener, urlopen + HTTPPasswordMgrWithDefaultRealm, build_opener, install_opener, \ + urlopen, urlretrieve from urllib.error import HTTPError, URLError try: -- cgit v1.2.3-1-g7c22 From 570fa88a17d87d4470ff54fe7f340d03edeefd52 Mon Sep 17 00:00:00 2001 From: Nathan Olla Date: Mon, 14 Jul 2014 16:49:31 -0400 Subject: Fix indent. --- src/lib/Bcfg2/Compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Compat.py b/src/lib/Bcfg2/Compat.py index 88956e900..b8a75a0c5 100644 --- a/src/lib/Bcfg2/Compat.py +++ b/src/lib/Bcfg2/Compat.py @@ -29,7 +29,7 @@ except ImportError: from urllib.parse import urljoin, urlparse, quote_plus from urllib.request import HTTPBasicAuthHandler, \ HTTPPasswordMgrWithDefaultRealm, build_opener, install_opener, \ - urlopen, urlretrieve + urlopen, urlretrieve from urllib.error import HTTPError, URLError try: -- cgit v1.2.3-1-g7c22 From f84558cbad0d35ef557f8b767b00f2c14f8907a1 Mon Sep 17 00:00:00 2001 From: Nathan Olla Date: Mon, 14 Jul 2014 16:52:37 -0400 Subject: Fix indenting --- src/lib/Bcfg2/Client/Tools/SYSV.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/lib/Bcfg2/Client/Tools/SYSV.py b/src/lib/Bcfg2/Client/Tools/SYSV.py index 2d835a8ae..811711723 100644 --- a/src/lib/Bcfg2/Client/Tools/SYSV.py +++ b/src/lib/Bcfg2/Client/Tools/SYSV.py @@ -60,11 +60,10 @@ class SYSV(Bcfg2.Client.Tools.PkgTool): (pkg.get('name'), pkg.get('simplename'))) tmpfile = tempfile.NamedTemporaryFile() self.tmpfiles.append(tmpfile) - self.logger.debug("URL: %s/%s" % - (pkg.get('url'), pkg.get('simplename'))) - urlretrieve("%s/%s" % - (pkg.get('url'), pkg.get('simplename')), - tmpfile.name) + self.logger.debug("URL: %s/%s" % (pkg.get('url'), + pkg.get('simplename'))) + urlretrieve("%s/%s" % (pkg.get('url'), pkg.get('simplename')), + tmpfile.name) newpkg = copy.copy(pkg) newpkg.set('url', tmpfile.name) return newpkg -- cgit v1.2.3-1-g7c22 From a4bf68d3c32e6654f53d8e151104850a54ad1e41 Mon Sep 17 00:00:00 2001 From: Nathan Olla Date: Wed, 16 Jul 2014 10:06:25 -0400 Subject: Implement _get_package_command and append _sysv_pkg_path attribute Instead of doing a partially complete Install() method for SYSV, implements a custom _get_package_command that will use the _sysv_pkg_path attribute added by the pkgmogrify call. This will allow the installs to complete. Unfortunately, the single-pass install will still fail if there are any packages with an http:// URL. The pkgadd invocation for 'device' sources doesn't take multiple packages and the 'datastream' invocation doesn't handle packages with an HTTP URL. Finally, there is no reliable standard naming convention for SYSV datastream files, so the simplename attribute is re-used. There is a known issue with this patch - if any packages specified in the PackageList have an http url, the single-pass install will produce an error like: Trying single pass package install for pkgtype sysv pkgadd: ERROR: Failure occurred with http(s) negotiation: <'Peername' doesn't match 'host' or no matching entry> pkgadd: ERROR: unable to download package datastream from . Single Pass Failed because the command that results isn't valid syntax for pkgadd. A workaround would be to add code to skip the single-pass install if any packages had the simplename attribute, or by checking the url for the presence of 'http'. I'm not sure if that should be fixed or if this is reasonable in this case. --- src/lib/Bcfg2/Client/Tools/SYSV.py | 60 +++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 33 deletions(-) diff --git a/src/lib/Bcfg2/Client/Tools/SYSV.py b/src/lib/Bcfg2/Client/Tools/SYSV.py index 811711723..4f3eea668 100644 --- a/src/lib/Bcfg2/Client/Tools/SYSV.py +++ b/src/lib/Bcfg2/Client/Tools/SYSV.py @@ -50,44 +50,38 @@ class SYSV(Bcfg2.Client.Tools.PkgTool): self.pkgtool[1]) except: # pylint: disable=W0702 self.pkgtool = (self.pkgtool[0] % "", self.pkgtool[1]) + self.origpkgtool = self.pkgtool - def pkgmogrify(self, pkg): - """ Take a pkg object, check for a 'simplename' attribute. If found, - return a modified pkg object that points to to a temporary file + def pkgmogrify(self, packages): + """ Take a list of pkg objects, check for a 'simplename' attribute. + If present, insert a _sysv_pkg_path attribute to the package and + download the datastream format SYSV package to a temporary file. """ - if pkg.get('simplename'): - self.logger.debug("Pkgmogrifying %s because simplename %s found" % - (pkg.get('name'), pkg.get('simplename'))) - tmpfile = tempfile.NamedTemporaryFile() - self.tmpfiles.append(tmpfile) - self.logger.debug("URL: %s/%s" % (pkg.get('url'), - pkg.get('simplename'))) - urlretrieve("%s/%s" % (pkg.get('url'), pkg.get('simplename')), - tmpfile.name) - newpkg = copy.copy(pkg) - newpkg.set('url', tmpfile.name) - return newpkg - return pkg - - def Install(self, packages, states): for pkg in packages: - pkg = self.pkgmogrify(pkg) - if self.VerifyPackage(pkg, []): - self.logger.info("Forcing state to true for pkg %s" % - (pkg.get('name'))) - states[pkg] = True - else: - self.logger.info("Installing pkg %s version %s" % - (pkg.get('name'), pkg.get('version'))) + if pkg.get('simplename'): + tmpfile = tempfile.NamedTemporaryFile() + self.tmpfiles.append(tmpfile) + urlretrieve("%s/%s" % (pkg.get('url'), pkg.get('simplename')), + tmpfile.name) + pkg.set('_sysv_pkg_path', tmpfile.name) - if self.cmd.run(self._get_package_command([pkg])): - states[pkg] = True - else: - self.logger.error("Failed to install package %s" % - pkg.get('name')) + def _get_package_command(self, packages): + """Override the default _get_package_command, replacing the attribute + 'url' if '_sysv_pkg_path' if necessary in the returned command + string""" + if len(packages) == 1 and '_sysv_pkg_path' in packages[0].keys(): + self.pkgtool = (self.pkgtool[0], ('%s %s', + ['_sysv_pkg_path', 'name'])) + else: + self.pkgtool = self.origpkgtool + + pkgcmd = super(SYSV, self)._get_package_command(packages) + self.logger.debug("Calling install command: %s" % pkgcmd) + return pkgcmd - self.RefreshPackages() - self.modified.extend(entry for entry in packages if states[entry]) + def Install(self, packages, states): + self.pkgmogrify(packages) + super(SYSV, self).Install(packages, states) def RefreshPackages(self): """Refresh memory hashes of packages.""" -- cgit v1.2.3-1-g7c22 From d5be1d7f9c9456ee0c6f43b3d05902b200f1dd54 Mon Sep 17 00:00:00 2001 From: Nathan Olla Date: Wed, 16 Jul 2014 10:49:48 -0400 Subject: Remove unused import of copy --- src/lib/Bcfg2/Client/Tools/SYSV.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lib/Bcfg2/Client/Tools/SYSV.py b/src/lib/Bcfg2/Client/Tools/SYSV.py index 4f3eea668..48d67c4f1 100644 --- a/src/lib/Bcfg2/Client/Tools/SYSV.py +++ b/src/lib/Bcfg2/Client/Tools/SYSV.py @@ -4,7 +4,6 @@ import tempfile from Bcfg2.Compat import any # pylint: disable=W0622 import Bcfg2.Client.Tools import Bcfg2.Client.XML -import copy from Bcfg2.Compat import urlretrieve -- cgit v1.2.3-1-g7c22 From c81ee4a99dfbd65b6ccde5a2d3fee9dd3caf5fc1 Mon Sep 17 00:00:00 2001 From: Nathan Olla Date: Wed, 16 Jul 2014 15:21:07 -0400 Subject: Add some basic documentation for SYSV packages over HTTP --- doc/client/tools.txt | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/doc/client/tools.txt b/doc/client/tools.txt index 1dbb33b1a..11f1d72cc 100644 --- a/doc/client/tools.txt +++ b/doc/client/tools.txt @@ -154,7 +154,32 @@ Systemd service support. SYSV ---- -Handles System V Packaging format that is available on Solaris. +Handles `System V Packaging `_ format that is available on Solaris. + +.. note:: + + If the Packages specified in the PackageList are datastream format packages distributed via HTTP, you must specify a simplename attribute. Such packages will be downloaded and installed from a local path. + + datastream format over HTTP: + + .. code-block:: xml + + + + + + + + File system format over NFS or local path: + + .. code-block:: xml + + + + + + + Upstart ------- -- cgit v1.2.3-1-g7c22 From b40dc8df20d8b1566da45ccea0620c60de476222 Mon Sep 17 00:00:00 2001 From: Nathan Olla Date: Wed, 16 Jul 2014 15:21:39 -0400 Subject: Check for origpkgtool attribute to prevent things that subclass SYSV from breaking --- src/lib/Bcfg2/Client/Tools/SYSV.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/lib/Bcfg2/Client/Tools/SYSV.py b/src/lib/Bcfg2/Client/Tools/SYSV.py index 48d67c4f1..712dcce32 100644 --- a/src/lib/Bcfg2/Client/Tools/SYSV.py +++ b/src/lib/Bcfg2/Client/Tools/SYSV.py @@ -67,12 +67,14 @@ class SYSV(Bcfg2.Client.Tools.PkgTool): def _get_package_command(self, packages): """Override the default _get_package_command, replacing the attribute 'url' if '_sysv_pkg_path' if necessary in the returned command - string""" - if len(packages) == 1 and '_sysv_pkg_path' in packages[0].keys(): - self.pkgtool = (self.pkgtool[0], ('%s %s', - ['_sysv_pkg_path', 'name'])) - else: - self.pkgtool = self.origpkgtool + string + """ + if hasattr(self, 'origpkgtool'): + if len(packages) == 1 and '_sysv_pkg_path' in packages[0].keys(): + self.pkgtool = (self.pkgtool[0], ('%s %s', + ['_sysv_pkg_path', 'name'])) + else: + self.pkgtool = self.origpkgtool pkgcmd = super(SYSV, self)._get_package_command(packages) self.logger.debug("Calling install command: %s" % pkgcmd) -- cgit v1.2.3-1-g7c22 From af0d30e03bcd7c40931aaa27536a75daa18f4134 Mon Sep 17 00:00:00 2001 From: Nathan Olla Date: Wed, 16 Jul 2014 15:27:42 -0400 Subject: Remove group tag from examples --- doc/client/tools.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/doc/client/tools.txt b/doc/client/tools.txt index 11f1d72cc..ce8732454 100644 --- a/doc/client/tools.txt +++ b/doc/client/tools.txt @@ -165,9 +165,7 @@ Handles `System V Packaging - - File system format over NFS or local path: @@ -175,9 +173,7 @@ Handles `System V Packaging - - -- cgit v1.2.3-1-g7c22 From d5b3c70dccd23f687c8194f6ea1807092370e586 Mon Sep 17 00:00:00 2001 From: Nathan Olla Date: Tue, 19 Aug 2014 17:58:20 -0400 Subject: Log when downloading packages via HTTP --- src/lib/Bcfg2/Client/Tools/SYSV.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib/Bcfg2/Client/Tools/SYSV.py b/src/lib/Bcfg2/Client/Tools/SYSV.py index 712dcce32..a29b49efa 100644 --- a/src/lib/Bcfg2/Client/Tools/SYSV.py +++ b/src/lib/Bcfg2/Client/Tools/SYSV.py @@ -60,6 +60,8 @@ class SYSV(Bcfg2.Client.Tools.PkgTool): if pkg.get('simplename'): tmpfile = tempfile.NamedTemporaryFile() self.tmpfiles.append(tmpfile) + self.logger.info("Downloading %s%s to %s" % (pkg.get('url'), + pkg.get('simplename'), tmpfile.name)) urlretrieve("%s/%s" % (pkg.get('url'), pkg.get('simplename')), tmpfile.name) pkg.set('_sysv_pkg_path', tmpfile.name) -- cgit v1.2.3-1-g7c22