Performance improvements for bcfg2-admin reports scrub.

Switched object updates to executemany statements.
Added BatchFetch to retreive Django objects in groups.

git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@5891 ce84e21b-d406-0410-9b95-82705330c041

1 files changed, 30 insertions, 0 deletions

diff --git a/src/lib/Server/Reports/utils.py b/src/lib/Server/Reports/utils.py
new file mode 100755
index 000000000..2ef21e446
--- /dev/null
+++ b/src/lib/Server/Reports/utils.py
@@ -0,0 +1,30 @@
+'''Helper functions for reports'''
+
+class BatchFetch(object):
+    '''Fetch Django objects in smaller batches to save memory'''
+
+    def __init__(self, obj, step=10000):
+        self.count = 0
+        self.block_count = 0
+        self.obj = obj
+        self.data = None
+        self.step = step
+        self.max = obj.count()
+
+    def __iter__(self):
+        return self
+
+    def next(self):
+        '''Return the next object from our array and fetch from the
+           database when needed'''
+        if self.block_count + self.count - self.step == self.max:
+            raise StopIteration
+        if self.block_count == 0 or self.count == self.step:
+            # Without list() this turns into LIMIT 1 OFFSET x queries
+            self.data = list(self.obj.all()[self.block_count: \
+                                   (self.block_count + self.step)])
+            self.block_count += self.step
+            self.count = 0
+        self.count += 1
+        return self.data[self.count - 1]
+