Add checks and code to fetch the new list using 'If-Modified-Since' protocol.

adds saving the 'last-modified' date of the same filename with a .timestamp extension.

2 files changed, 84 insertions, 35 deletions

diff --git a/layman/api.py b/layman/api.py
index cbb9d99..b71dd72 100644
--- a/layman/api.py
+++ b/layman/api.py
@@ -120,7 +120,7 @@ class LaymanAPI(object):
                 self._error(ERROR_INTERNAL_ERROR,
                         "Failed to disable repository '"+ovl+"':\n"+str(e))
                 results.append(False)
-            self.get_installed(reload=True)
+            self.get_installed(dbreload=True)
         if False in results:
             return False
         return True
@@ -151,7 +151,7 @@ class LaymanAPI(object):
                 self._error(ERROR_INTERNAL_ERROR,
                         "Failed to enable repository '"+ovl+"' : "+str(e))
                 results.append(False)
-            self.get_installed(reload=True)
+            self.get_installed(dbreload=True)
         if False in results:
             return False
         return True
@@ -373,47 +373,52 @@ class LaymanAPI(object):
     def fetch_remote_list(self):
         """Fetches the latest remote overlay list"""
         try:
-            self._get_remote_db().cache()
+            dbreload = self._get_remote_db().cache()
+            self.output.debug(
+                'LaymanAPI.fetch_remote_list(); cache updated = %s'
+                % str(dbreload),8)
         except Exception as error:
             self._error('Failed to fetch overlay list!\n Original Error was: '
                     + str(error))
             return False
-        self.get_available(reload=True)
+        self.get_available(dbreload)
         return True
 
 
-    def get_available(self, reload=False):
+    def get_available(self, dbreload=False):
         """returns the list of available overlays"""
-        if self._available_ids is None or reload:
-            self._available_ids = self._get_remote_db(reload).list_ids()
+        self.output.info('LaymanAPI.get_available() dbreload = %s'
+            % str(dbreload))
+        if self._available_ids is None or dbreload:
+            self._available_ids = self._get_remote_db(dbreload).list_ids()
         return self._available_ids[:] or ['None']
 
 
-    def get_installed(self, reload=False):
+    def get_installed(self, dbreload=False):
         """returns the list of installed overlays"""
-        if self._installed_ids is None or reload:
-            self._installed_ids = self._get_installed_db(reload).list_ids()
+        if self._installed_ids is None or dbreload:
+            self._installed_ids = self._get_installed_db(dbreload).list_ids()
         return self._installed_ids[:]
 
 
-    def _get_installed_db(self, reload=False):
+    def _get_installed_db(self, dbreload=False):
         """returns the list of installed overlays"""
-        if not self._installed_db or reload:
+        if not self._installed_db or dbreload:
             self._installed_db = DB(self.config)
         return self._installed_db
 
 
-    def _get_remote_db(self, reload=False):
+    def _get_remote_db(self, dbreload=False):
         """returns the list of installed overlays"""
-        if self._available_db is None or reload:
+        if self._available_db is None or dbreload:
             self._available_db = RemoteDB(self.config)
         return self._available_db
 
 
     def reload(self):
         """reloads the installed and remote db's to the data on disk"""
-        result = self.get_available(reload=True)
-        result = self.get_installed(reload=True)
+        result = self.get_available(dbreload=True)
+        result = self.get_installed(dbreload=True)
 
 
     def _error(self, message):
@@ -423,7 +428,7 @@ class LaymanAPI(object):
         """
         self._error_messages.append(message)
         if self.report_errors:
-            print(message, file=stderr)
+            print(message, file=self.config['stderr'])
 
 
     def get_errors(self):
diff --git a/layman/db.py b/layman/db.py
index d078e46..3498223 100644
--- a/layman/db.py
+++ b/layman/db.py
@@ -24,7 +24,9 @@ __version__ = "$Id: db.py 309 2007-04-09 16:23:38Z wrobel $"
 #
 #-------------------------------------------------------------------------------
 
-import os, os.path, urllib2, hashlib
+import os, os.path
+import urllib2
+import hashlib
 
 from   layman.utils             import path, delete_empty_directory
 from   layman.dbbase            import DbBase
@@ -243,7 +245,7 @@ class RemoteDB(DbBase):
 
         self.urls  = [i.strip() for i in config['overlays'].split('\n') if len(i)]
 
-        paths = [self.path(i) for i in self.urls]
+        paths = [self.filepath(i) + '.xml' for i in self.urls]
 
         if config['nocheck']:
             ignore = 2
@@ -283,22 +285,45 @@ class RemoteDB(DbBase):
         >>> a.overlays.keys()
         [u'wrobel', u'wrobel-stable']
         '''
+        has_updates = False
         for url in self.urls:
 
-            mpath = self.path(url)
+            filepath = self.filepath(url)
+            mpath = filepath + '.xml'
+            tpath = filepath + '.timestamp'
 
-            # Check for sufficient privileges
-            if os.path.exists(mpath) and not os.access(mpath, os.W_OK):
-                self.output.warn('You do not have permission to update the cache (%s).' % mpath)
-                import getpass
-                if getpass.getuser() != 'root':
-                    self.output.warn('Hint: You are not root.\n')
+            # check when the cache was last updated
+            # and don't re-fetch it unless it has changed
+            request = urllib2.Request(url)
+            opener = urllib2.build_opener()
+            opener.addheaders = [('User-Agent', 'Layman-2.0-git')]
+
+            if os.path.exists(tpath):
+                with open(tpath,'r') as previous:
+                    last_time = previous.read()
+                request.add_header('If-Modified-Since', last_time)
+
+            if not self.check_path([mpath]):
                 continue
 
             try:
-
+                connection = opener.open(request)
+                timestamp = connection.headers['last-modified']
+            except urllib2.HTTPError as e:
+                if e.getcode() == 304:
+                    self.output.info('Remote list already up to date: %s'
+                        % url)
+                else:
+                    self.output.info('RemoteDB.cache(); HTTPError was:\n %s'
+                        % str(e))
+                continue
+            except IOError as error:
+                self.output.warn('Failed to update the overlay list from: '
+                         + url + '\nError was:\n' + str(error))
+            else:
+                self.output.info('Fetching new list...')
                 # Fetch the remote list
-                olist = urllib2.urlopen(url).read()
+                olist = connection.read()
 
                 # Create our storage directory if it is missing
                 if not os.path.exists(os.path.dirname(mpath)):
@@ -326,24 +351,43 @@ class RemoteDB(DbBase):
                     out_file.write(olist)
                     out_file.close()
 
+                    out_file = open(tpath, 'w')
+                    out_file.write(timestamp)
+                    out_file.close()
+
+                    has_updates = True
+
                 except Exception as error:
                     raise IOError('Failed to temporarily cache overlays list in'
                                   ' ' + mpath + '\nError was:\n' + str(error))
+        return has_updates
 
 
-            except IOError as error:
-                self.output.warn('Failed to update the overlay list from: '
-                         + url + '\nError was:\n' + str(error))
-
-    def path(self, url):
+    def filepath(self, url):
         '''Return a unique file name for the url.'''
 
         base = self.config['cache']
 
         self.output.debug('Generating cache path.', 6)
 
-        return base + '_' + hashlib.md5(url).hexdigest() + '.xml'
-
+        return base + '_' + hashlib.md5(url).hexdigest()
+
+
+    def check_path(self, paths, hint=True):
+        '''Check for sufficient privileges'''
+        self.output.debug('RemoteDB.check_path; paths = ' + str(paths), 8)
+        is_ok = True
+        for path in paths:
+            if os.path.exists(path) and not os.access(path, os.W_OK):
+                if hint:
+                    self.output.warn(
+                        'You do not have permission to update the cache (%s).'
+                        % mpath)
+                    import getpass
+                    if getpass.getuser() != 'root':
+                        self.output.warn('Hint: You are not root.\n')
+                is_ok = False
+        return is_ok
 
 #===============================================================================
 #