From 8781c47d0ddac22192f1c233606d3a2923abd6f9 Mon Sep 17 00:00:00 2001
From: Tim Laszlo <tim.laszlo@gmail.com>
Date: Sat, 5 Jun 2010 13:41:23 +0000
Subject: Performance improvements for bcfg2-admin reports scrub.

Switched object updates to executemany statements.
Added BatchFetch to retreive Django objects in groups.

git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@5891 ce84e21b-d406-0410-9b95-82705330c041
---
 src/lib/Server/Reports/utils.py | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)
 create mode 100755 src/lib/Server/Reports/utils.py

(limited to 'src/lib/Server/Reports/utils.py')

diff --git a/src/lib/Server/Reports/utils.py b/src/lib/Server/Reports/utils.py
new file mode 100755
index 000000000..2ef21e446
--- /dev/null
+++ b/src/lib/Server/Reports/utils.py
@@ -0,0 +1,30 @@
+'''Helper functions for reports'''
+
+class BatchFetch(object):
+    '''Fetch Django objects in smaller batches to save memory'''
+
+    def __init__(self, obj, step=10000):
+        self.count = 0
+        self.block_count = 0
+        self.obj = obj
+        self.data = None
+        self.step = step
+        self.max = obj.count()
+
+    def __iter__(self):
+        return self
+
+    def next(self):
+        '''Return the next object from our array and fetch from the
+           database when needed'''
+        if self.block_count + self.count - self.step == self.max:
+            raise StopIteration
+        if self.block_count == 0 or self.count == self.step:
+            # Without list() this turns into LIMIT 1 OFFSET x queries
+            self.data = list(self.obj.all()[self.block_count: \
+                                   (self.block_count + self.step)])
+            self.block_count += self.step
+            self.count = 0
+        self.count += 1
+        return self.data[self.count - 1]
+
-- 
cgit v1.2.3-1-g7c22