summaryrefslogtreecommitdiffstats
path: root/src/lib/Server/Plugins/Packages.py
diff options
context:
space:
mode:
authorNarayan Desai <desai@mcs.anl.gov>2009-06-18 18:16:13 +0000
committerNarayan Desai <desai@mcs.anl.gov>2009-06-18 18:16:13 +0000
commitbe8e36ce968a62ddda8619b2c17a614bf5868612 (patch)
treebc0ee6ed959e4fa26f647ec0451814c6e7c892cf /src/lib/Server/Plugins/Packages.py
parentcb03ec03515e6a442c94dae0f2ac56235c050e9f (diff)
downloadbcfg2-be8e36ce968a62ddda8619b2c17a614bf5868612.tar.gz
bcfg2-be8e36ce968a62ddda8619b2c17a614bf5868612.tar.bz2
bcfg2-be8e36ce968a62ddda8619b2c17a614bf5868612.zip
Packages: Improve YumSource performance dramatically
Track needed file paths so that the contents of all packages do not need to be tracked. This reduces memory consumption ~60% on my test repository (360MB -> 60MB for bcfg2-info). woo! git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@5283 ce84e21b-d406-0410-9b95-82705330c041
Diffstat (limited to 'src/lib/Server/Plugins/Packages.py')
-rw-r--r--src/lib/Server/Plugins/Packages.py19
1 files changed, 13 insertions, 6 deletions
diff --git a/src/lib/Server/Plugins/Packages.py b/src/lib/Server/Plugins/Packages.py
index d8e6d2004..b4bc01d2c 100644
--- a/src/lib/Server/Plugins/Packages.py
+++ b/src/lib/Server/Plugins/Packages.py
@@ -163,6 +163,7 @@ class YUMSource(Source):
self.deps = dict([('global', dict())])
self.provides = dict([('global', dict())])
self.filemap = dict([(x, dict()) for x in ['global'] + self.arches])
+ self.needed_paths = set()
def save_state(self):
cache = file(self.cachefile, 'wb')
@@ -195,13 +196,16 @@ class YUMSource(Source):
urls = property(get_urls)
def read_files(self):
- for fname in self.files:
+ for fname in [f for f in self.files if f.endswith('primary.xml.gz')]:
+ print fname
+ farch = fname.split('@')[-3]
+ fdata = lxml.etree.parse(fname).getroot()
+ self.parse_primary(fdata, farch)
+ for fname in [f for f in self.files if f.endswith('filelists.xml.gz')]:
+ print fname
farch = fname.split('@')[-3]
fdata = lxml.etree.parse(fname).getroot()
- if fname.endswith('primary.xml.gz'):
- self.parse_primary(fdata, farch)
- elif fname.endswith('filelists.xml.gz'):
- self.parse_filelist(fdata, farch)
+ self.parse_filelist(fdata, farch)
# merge data
sdata = self.packages.values()
self.packages['global'] = copy.deepcopy(sdata.pop())
@@ -216,7 +220,8 @@ class YUMSource(Source):
def parse_filelist(self, data, arch):
for pkg in data.findall(self.fl + 'package'):
for fentry in pkg.findall(self.fl + 'file'):
- self.filemap[arch][fentry.text] = pkg.get('name')
+ if fentry in self.needed_paths:
+ self.filemap[arch][fentry.text] = pkg.get('name')
def parse_primary(self, data, arch):
if arch not in self.packages:
@@ -236,6 +241,8 @@ class YUMSource(Source):
self.deps[arch][pkgname] = set()
for entry in pre.getchildren():
self.deps[arch][pkgname].add(entry.get('name'))
+ if entry.get('name').startswith('/'):
+ self.needed_paths.add(entry.get('name'))
pro = pdata.find(self.rp + 'provides')
if pro != None:
for entry in pro.getchildren():