From d543984b06796fd5c65def5e67d1dfd6cb068b6c Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Fri, 7 Jun 2013 04:24:30 +0200 Subject: Client/Tools/VCS: create directory for repository Older version of dulwich will not create the directory (even with mkdir=True) for the repository. So we cannot rely on it. Because we clean existing folders before, we have to create the directory in every case. --- src/lib/Bcfg2/Client/Tools/VCS.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/lib') diff --git a/src/lib/Bcfg2/Client/Tools/VCS.py b/src/lib/Bcfg2/Client/Tools/VCS.py index 1ab867215..23f04b5e3 100644 --- a/src/lib/Bcfg2/Client/Tools/VCS.py +++ b/src/lib/Bcfg2/Client/Tools/VCS.py @@ -71,7 +71,8 @@ class VCS(Bcfg2.Client.Tools.Tool): destname) return False - destr = dulwich.repo.Repo.init(destname, mkdir=True) + dulwich.file.ensure_dir_exists(destname) + destr = dulwich.repo.Repo.init(destname) cl, host_path = dulwich.client.get_transport_and_path(entry.get('sourceurl')) remote_refs = cl.fetch(host_path, destr, -- cgit v1.2.3-1-g7c22 From 3005879f1cbbfef2ec438a82a69f2fa0907d2dcf Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Fri, 7 Jun 2013 04:44:40 +0200 Subject: Client/Tools/VCS: add always on top feature You can specify a refname (like refs/heads/master) as revision and you will always get the current tree of this refname. During verify it is checked if the ref had changed in the remote repo and if an "update" is necessary. --- src/lib/Bcfg2/Client/Tools/VCS.py | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) (limited to 'src/lib') diff --git a/src/lib/Bcfg2/Client/Tools/VCS.py b/src/lib/Bcfg2/Client/Tools/VCS.py index 23f04b5e3..1f96ad5b4 100644 --- a/src/lib/Bcfg2/Client/Tools/VCS.py +++ b/src/lib/Bcfg2/Client/Tools/VCS.py @@ -47,11 +47,24 @@ class VCS(Bcfg2.Client.Tools.Tool): self.logger.info("Repository %s does not exist" % entry.get('name')) return False - cur_rev = repo.head() - if cur_rev != entry.get('revision'): + try: + expected_rev = entry.get('revision') + cur_rev = repo.head() + except: + return False + + try: + client, path = dulwich.client.get_transport_and_path(entry.get('sourceurl')) + remote_refs = client.fetch_pack(path, (lambda x: None), None, None, None) + if expected_rev in remote_refs: + expected_rev = remote_refs[expected_rev] + except: + pass + + if cur_rev != expected_rev: self.logger.info("At revision %s need to go to revision %s" % - (cur_rev, entry.get('revision'))) + (cur_rev.strip(), expected_rev.strip())) return False return True @@ -78,8 +91,13 @@ class VCS(Bcfg2.Client.Tools.Tool): destr, determine_wants=destr.object_store.determine_wants_all, progress=sys.stdout.write) - destr.refs['refs/heads/master'] = entry.get('revision') - dtree = destr[entry.get('revision')].tree + + if entry.get('revision') in remote_refs: + destr.refs['HEAD'] = remote_refs[entry.get('revision')] + else: + destr.refs['HEAD'] = entry.get('revision') + + dtree = destr['HEAD'].tree obj_store = destr.object_store for fname, mode, sha in obj_store.iter_tree_contents(dtree): fullpath = os.path.join(destname, fname) @@ -92,6 +110,7 @@ class VCS(Bcfg2.Client.Tools.Tool): f.write(destr[sha].data) f.close() os.chmod(os.path.join(destname, fname), mode) + return True # FIXME: figure out how to write the git index properly #iname = "%s/.git/index" % entry.get('name') -- cgit v1.2.3-1-g7c22 From a0612263ec30e7cb81dfd4bbd9bc73798630474d Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Fri, 7 Jun 2013 13:42:31 +0200 Subject: Client/Tools/VCS: some simplyfications using dulwich api --- src/lib/Bcfg2/Client/Tools/VCS.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) (limited to 'src/lib') diff --git a/src/lib/Bcfg2/Client/Tools/VCS.py b/src/lib/Bcfg2/Client/Tools/VCS.py index 1f96ad5b4..4a30d9d90 100644 --- a/src/lib/Bcfg2/Client/Tools/VCS.py +++ b/src/lib/Bcfg2/Client/Tools/VCS.py @@ -98,18 +98,14 @@ class VCS(Bcfg2.Client.Tools.Tool): destr.refs['HEAD'] = entry.get('revision') dtree = destr['HEAD'].tree - obj_store = destr.object_store - for fname, mode, sha in obj_store.iter_tree_contents(dtree): - fullpath = os.path.join(destname, fname) - try: - f = open(os.path.join(destname, fname), 'wb') - except IOError: - dir = os.path.split(fullpath)[0] - os.makedirs(dir) - f = open(os.path.join(destname, fname), 'wb') - f.write(destr[sha].data) - f.close() - os.chmod(os.path.join(destname, fname), mode) + for fname, mode, sha in destr.object_store.iter_tree_contents(dtree): + full_path = os.path.join(destname, fname) + dulwich.file.ensure_dir_exists(os.path.dirname(full_path)) + + file = open(full_path, 'wb') + file.write(destr[sha].as_raw_string()) + file.close() + os.chmod(full_path, mode) return True # FIXME: figure out how to write the git index properly -- cgit v1.2.3-1-g7c22 From 72e7bff7123e3639bf7da0866c633bdc123d1af4 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Fri, 7 Jun 2013 13:43:09 +0200 Subject: Client/Tools/VCS: add support for symlinks --- src/lib/Bcfg2/Client/Tools/VCS.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) (limited to 'src/lib') diff --git a/src/lib/Bcfg2/Client/Tools/VCS.py b/src/lib/Bcfg2/Client/Tools/VCS.py index 4a30d9d90..947cff11f 100644 --- a/src/lib/Bcfg2/Client/Tools/VCS.py +++ b/src/lib/Bcfg2/Client/Tools/VCS.py @@ -6,6 +6,7 @@ # * integrate properly with reports missing = [] +import errno import os import shutil import sys @@ -102,10 +103,22 @@ class VCS(Bcfg2.Client.Tools.Tool): full_path = os.path.join(destname, fname) dulwich.file.ensure_dir_exists(os.path.dirname(full_path)) - file = open(full_path, 'wb') - file.write(destr[sha].as_raw_string()) - file.close() - os.chmod(full_path, mode) + if dulwich.objects.S_ISGITLINK(mode): + src_path = destr[sha].as_raw_string() + try: + os.symlink(src_path, full_path) + except OSError: + e = sys.exc_info()[1] + if e.errno == errno.EEXIST: + os.unlink(full_path) + os.symlink(src_path, full_path) + else: + raise + else: + file = open(full_path, 'wb') + file.write(destr[sha].as_raw_string()) + file.close() + os.chmod(full_path, mode) return True # FIXME: figure out how to write the git index properly -- cgit v1.2.3-1-g7c22 From b78d3a3d2761a71ab0bc12ac2e53c13555e8ea16 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Fri, 7 Jun 2013 13:38:32 +0200 Subject: Client/Tools/VCS: build index after checkout --- src/lib/Bcfg2/Client/Tools/VCS.py | 47 +++++++++++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 9 deletions(-) (limited to 'src/lib') diff --git a/src/lib/Bcfg2/Client/Tools/VCS.py b/src/lib/Bcfg2/Client/Tools/VCS.py index 947cff11f..26e404cfa 100644 --- a/src/lib/Bcfg2/Client/Tools/VCS.py +++ b/src/lib/Bcfg2/Client/Tools/VCS.py @@ -1,7 +1,6 @@ """VCS support.""" # TODO: -# * git_write_index # * add svn support # * integrate properly with reports missing = [] @@ -10,6 +9,7 @@ import errno import os import shutil import sys +import stat # python-dulwich git imports try: @@ -27,6 +27,38 @@ except ImportError: import Bcfg2.Client.Tools +def cleanup_mode(mode): + """Cleanup a mode value. + + This will return a mode that can be stored in a tree object. + + :param mode: Mode to clean up. + """ + if stat.S_ISLNK(mode): + return stat.S_IFLNK + elif stat.S_ISDIR(mode): + return stat.S_IFDIR + elif dulwich.index.S_ISGITLINK(mode): + return dulwich.index.S_IFGITLINK + ret = stat.S_IFREG | int('644', 8) + ret |= (mode & int('111', 8)) + return ret + + +def index_entry_from_stat(stat_val, hex_sha, flags, mode=None): + """Create a new index entry from a stat value. + + :param stat_val: POSIX stat_result instance + :param hex_sha: Hex sha of the object + :param flags: Index flags + """ + if mode is None: + mode = cleanup_mode(stat_val.st_mode) + return (stat_val.st_ctime, stat_val.st_mtime, stat_val.st_dev, + stat_val.st_ino, mode, stat_val.st_uid, + stat_val.st_gid, stat_val.st_size, hex_sha, flags) + + class VCS(Bcfg2.Client.Tools.Tool): """VCS support.""" __handles__ = [('Path', 'vcs')] @@ -99,6 +131,7 @@ class VCS(Bcfg2.Client.Tools.Tool): destr.refs['HEAD'] = entry.get('revision') dtree = destr['HEAD'].tree + index = dulwich.index.Index(destr.index_path()) for fname, mode, sha in destr.object_store.iter_tree_contents(dtree): full_path = os.path.join(destname, fname) dulwich.file.ensure_dir_exists(os.path.dirname(full_path)) @@ -120,15 +153,11 @@ class VCS(Bcfg2.Client.Tools.Tool): file.close() os.chmod(full_path, mode) + st = os.lstat(full_path) + index[fname] = index_entry_from_stat(st, sha, 0) + + index.write() return True - # FIXME: figure out how to write the git index properly - #iname = "%s/.git/index" % entry.get('name') - #f = open(iname, 'w+') - #entries = obj_store[sha].iteritems() - #try: - # dulwich.index.write_index(f, entries) - #finally: - # f.close() def Verifysvn(self, entry, _): """Verify svn repositories""" -- cgit v1.2.3-1-g7c22