From fcd35daad6e84fdaab69f4bd0c969b80daee590d Mon Sep 17 00:00:00 2001 From: dol-sen Date: Sat, 30 Apr 2011 15:36:54 -0700 Subject: Add checks and code to fetch the new list using 'If-Modified-Since' protocol. adds saving the 'last-modified' date of the same filename with a .timestamp extension. --- layman/api.py | 39 ++++++++++++++++------------- layman/db.py | 80 +++++++++++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 84 insertions(+), 35 deletions(-) diff --git a/layman/api.py b/layman/api.py index cbb9d99..b71dd72 100644 --- a/layman/api.py +++ b/layman/api.py @@ -120,7 +120,7 @@ class LaymanAPI(object): self._error(ERROR_INTERNAL_ERROR, "Failed to disable repository '"+ovl+"':\n"+str(e)) results.append(False) - self.get_installed(reload=True) + self.get_installed(dbreload=True) if False in results: return False return True @@ -151,7 +151,7 @@ class LaymanAPI(object): self._error(ERROR_INTERNAL_ERROR, "Failed to enable repository '"+ovl+"' : "+str(e)) results.append(False) - self.get_installed(reload=True) + self.get_installed(dbreload=True) if False in results: return False return True @@ -373,47 +373,52 @@ class LaymanAPI(object): def fetch_remote_list(self): """Fetches the latest remote overlay list""" try: - self._get_remote_db().cache() + dbreload = self._get_remote_db().cache() + self.output.debug( + 'LaymanAPI.fetch_remote_list(); cache updated = %s' + % str(dbreload),8) except Exception as error: self._error('Failed to fetch overlay list!\n Original Error was: ' + str(error)) return False - self.get_available(reload=True) + self.get_available(dbreload) return True - def get_available(self, reload=False): + def get_available(self, dbreload=False): """returns the list of available overlays""" - if self._available_ids is None or reload: - self._available_ids = self._get_remote_db(reload).list_ids() + self.output.info('LaymanAPI.get_available() dbreload = %s' + % str(dbreload)) + if self._available_ids is None or dbreload: + self._available_ids = self._get_remote_db(dbreload).list_ids() return self._available_ids[:] or ['None'] - def get_installed(self, reload=False): + def get_installed(self, dbreload=False): """returns the list of installed overlays""" - if self._installed_ids is None or reload: - self._installed_ids = self._get_installed_db(reload).list_ids() + if self._installed_ids is None or dbreload: + self._installed_ids = self._get_installed_db(dbreload).list_ids() return self._installed_ids[:] - def _get_installed_db(self, reload=False): + def _get_installed_db(self, dbreload=False): """returns the list of installed overlays""" - if not self._installed_db or reload: + if not self._installed_db or dbreload: self._installed_db = DB(self.config) return self._installed_db - def _get_remote_db(self, reload=False): + def _get_remote_db(self, dbreload=False): """returns the list of installed overlays""" - if self._available_db is None or reload: + if self._available_db is None or dbreload: self._available_db = RemoteDB(self.config) return self._available_db def reload(self): """reloads the installed and remote db's to the data on disk""" - result = self.get_available(reload=True) - result = self.get_installed(reload=True) + result = self.get_available(dbreload=True) + result = self.get_installed(dbreload=True) def _error(self, message): @@ -423,7 +428,7 @@ class LaymanAPI(object): """ self._error_messages.append(message) if self.report_errors: - print(message, file=stderr) + print(message, file=self.config['stderr']) def get_errors(self): diff --git a/layman/db.py b/layman/db.py index d078e46..3498223 100644 --- a/layman/db.py +++ b/layman/db.py @@ -24,7 +24,9 @@ __version__ = "$Id: db.py 309 2007-04-09 16:23:38Z wrobel $" # #------------------------------------------------------------------------------- -import os, os.path, urllib2, hashlib +import os, os.path +import urllib2 +import hashlib from layman.utils import path, delete_empty_directory from layman.dbbase import DbBase @@ -243,7 +245,7 @@ class RemoteDB(DbBase): self.urls = [i.strip() for i in config['overlays'].split('\n') if len(i)] - paths = [self.path(i) for i in self.urls] + paths = [self.filepath(i) + '.xml' for i in self.urls] if config['nocheck']: ignore = 2 @@ -283,22 +285,45 @@ class RemoteDB(DbBase): >>> a.overlays.keys() [u'wrobel', u'wrobel-stable'] ''' + has_updates = False for url in self.urls: - mpath = self.path(url) + filepath = self.filepath(url) + mpath = filepath + '.xml' + tpath = filepath + '.timestamp' - # Check for sufficient privileges - if os.path.exists(mpath) and not os.access(mpath, os.W_OK): - self.output.warn('You do not have permission to update the cache (%s).' % mpath) - import getpass - if getpass.getuser() != 'root': - self.output.warn('Hint: You are not root.\n') + # check when the cache was last updated + # and don't re-fetch it unless it has changed + request = urllib2.Request(url) + opener = urllib2.build_opener() + opener.addheaders = [('User-Agent', 'Layman-2.0-git')] + + if os.path.exists(tpath): + with open(tpath,'r') as previous: + last_time = previous.read() + request.add_header('If-Modified-Since', last_time) + + if not self.check_path([mpath]): continue try: - + connection = opener.open(request) + timestamp = connection.headers['last-modified'] + except urllib2.HTTPError as e: + if e.getcode() == 304: + self.output.info('Remote list already up to date: %s' + % url) + else: + self.output.info('RemoteDB.cache(); HTTPError was:\n %s' + % str(e)) + continue + except IOError as error: + self.output.warn('Failed to update the overlay list from: ' + + url + '\nError was:\n' + str(error)) + else: + self.output.info('Fetching new list...') # Fetch the remote list - olist = urllib2.urlopen(url).read() + olist = connection.read() # Create our storage directory if it is missing if not os.path.exists(os.path.dirname(mpath)): @@ -326,24 +351,43 @@ class RemoteDB(DbBase): out_file.write(olist) out_file.close() + out_file = open(tpath, 'w') + out_file.write(timestamp) + out_file.close() + + has_updates = True + except Exception as error: raise IOError('Failed to temporarily cache overlays list in' ' ' + mpath + '\nError was:\n' + str(error)) + return has_updates - except IOError as error: - self.output.warn('Failed to update the overlay list from: ' - + url + '\nError was:\n' + str(error)) - - def path(self, url): + def filepath(self, url): '''Return a unique file name for the url.''' base = self.config['cache'] self.output.debug('Generating cache path.', 6) - return base + '_' + hashlib.md5(url).hexdigest() + '.xml' - + return base + '_' + hashlib.md5(url).hexdigest() + + + def check_path(self, paths, hint=True): + '''Check for sufficient privileges''' + self.output.debug('RemoteDB.check_path; paths = ' + str(paths), 8) + is_ok = True + for path in paths: + if os.path.exists(path) and not os.access(path, os.W_OK): + if hint: + self.output.warn( + 'You do not have permission to update the cache (%s).' + % mpath) + import getpass + if getpass.getuser() != 'root': + self.output.warn('Hint: You are not root.\n') + is_ok = False + return is_ok #=============================================================================== # -- cgit v1.2.3-1-g7c22