Merge branch 'github-master'

17 files changed, 87 insertions, 25 deletions

diff --git a/askbot/doc/source/changelog.rst b/askbot/doc/source/changelog.rst
index 19ecfb2d..ac809dce 100644
--- a/askbot/doc/source/changelog.rst
+++ b/askbot/doc/source/changelog.rst
@@ -11,7 +11,9 @@ Development version (not released yet)
 * Added left sidebar option (Evgeny)
 * Added "help" page and links to in the header and the footer (Evgeny)
 * Removed url parameters and the hash fragment from uploaded files -
-  amazon S3 for some reason adds weird expiration parameters
+  amazon S3 for some reason adds weird expiration parameters (Evgeny)
+* Reduced memory usage in data migrations (Evgeny)
+* Added progress bars to slow data migrations (Evgeny)
 
 0.7.39 (Jan 11, 2012)
 ---------------------
diff --git a/askbot/migrations/0053_create_threads_for_questions.py b/askbot/migrations/0053_create_threads_for_questions.py
index 22365692..8c734571 100644
--- a/askbot/migrations/0053_create_threads_for_questions.py
+++ b/askbot/migrations/0053_create_threads_for_questions.py
@@ -9,6 +9,7 @@ class Migration(DataMigration):
 
     def forwards(self, orm):
         "Write your forwards methods here."
+        print "Converting question to threads:"
         num_questions = orm.Question.objects.count()
         for question in ProgressBar(orm.Question.objects.iterator(), num_questions):
             thread = orm.Thread.objects.create(favourite_count=question.favourite_count)
diff --git a/askbot/migrations/0057_transplant_answer_count_data.py b/askbot/migrations/0057_transplant_answer_count_data.py
index 654f9793..f83847cd 100644
--- a/askbot/migrations/0057_transplant_answer_count_data.py
+++ b/askbot/migrations/0057_transplant_answer_count_data.py
@@ -3,11 +3,14 @@ import datetime
 from south.db import db
 from south.v2 import DataMigration
 from django.db import models
+from askbot.utils.console import ProgressBar
 
 class Migration(DataMigration):
 
     def forwards(self, orm):
-        for question in orm.Question.objects.iterator():
+        message = "Adding answer counts to threads"
+        num_questions = orm.Question.objects.count() 
+        for question in ProgressBar(orm.Question.objects.iterator(), num_questions, message):
             thread = question.thread
             thread.answer_count = question.answer_count
             thread.save()
diff --git a/askbot/migrations/0060_view_count_transplant.py b/askbot/migrations/0060_view_count_transplant.py
index db3ab49b..e9538655 100644
--- a/askbot/migrations/0060_view_count_transplant.py
+++ b/askbot/migrations/0060_view_count_transplant.py
@@ -3,11 +3,14 @@ import datetime
 from south.db import db
 from south.v2 import DataMigration
 from django.db import models
+from askbot.utils.console import ProgressBar
 
 class Migration(DataMigration):
 
     def forwards(self, orm):
-        for question in orm.Question.objects.iterator():
+        message = "Adding view counts to threads"
+        num_questions = orm.Question.objects.count()
+        for question in ProgressBar(orm.Question.objects.iterator(), num_questions, message):
             thread = question.thread
             thread.view_count = question.view_count
             thread.save()
diff --git a/askbot/migrations/0063_transplant_question_closed_datas.py b/askbot/migrations/0063_transplant_question_closed_datas.py
index 052dc268..15626e48 100644
--- a/askbot/migrations/0063_transplant_question_closed_datas.py
+++ b/askbot/migrations/0063_transplant_question_closed_datas.py
@@ -3,11 +3,14 @@ import datetime
 from south.db import db
 from south.v2 import DataMigration
 from django.db import models
+from askbot.utils.console import ProgressBar
 
 class Migration(DataMigration):
 
     def forwards(self, orm):
-        for question in orm.Question.objects.iterator():
+        message = "Marking closed threads"
+        num_questions = orm.Question.objects.count()
+        for question in ProgressBar(orm.Question.objects.iterator(), num_questions, message):
             thread = question.thread
 
             thread.closed = question.closed
diff --git a/askbot/migrations/0066_transplant_accepted_answer_data.py b/askbot/migrations/0066_transplant_accepted_answer_data.py
index 6f5a81a0..f9c28b94 100644
--- a/askbot/migrations/0066_transplant_accepted_answer_data.py
+++ b/askbot/migrations/0066_transplant_accepted_answer_data.py
@@ -5,11 +5,14 @@ from south.v2 import DataMigration
 from django.db import models
 
 from askbot.migrations import TERM_YELLOW, TERM_RESET
+from askbot.utils.console import ProgressBar
 
 class Migration(DataMigration):
 
     def forwards(self, orm):
-        for question in orm.Question.objects.all():
+        message = "Adding accepted answers to threads"
+        num_questions = orm.Question.objects.count()
+        for question in ProgressBar(orm.Question.objects.iterator(), num_questions, message):
             thread = question.thread
 
             if question.answer_accepted:
@@ -27,7 +30,8 @@ class Migration(DataMigration):
                     thread.save()
 
         # Verify data integrity
-        for question in orm.Question.objects.all():
+        message = "Checking correctness of accepted answer data"
+        for question in ProgressBar(orm.Question.objects.iterator(), num_questions, message):
             accepted_answers = question.answers.filter(accepted=True)
             num_accepted_answers = len(accepted_answers)
 
diff --git a/askbot/migrations/0069_transplant_last_activity_data.py b/askbot/migrations/0069_transplant_last_activity_data.py
index a0756d68..bcd83ec9 100644
--- a/askbot/migrations/0069_transplant_last_activity_data.py
+++ b/askbot/migrations/0069_transplant_last_activity_data.py
@@ -3,11 +3,14 @@ import datetime
 from south.db import db
 from south.v2 import DataMigration
 from django.db import models
+from askbot.utils.console import ProgressBar
 
 class Migration(DataMigration):
 
     def forwards(self, orm):
-        for question in orm.Question.objects.iterator():
+        message = "Adding last activity data to threads"
+        num_questions = orm.Question.objects.count()
+        for question in ProgressBar(orm.Question.objects.iterator(), num_questions, message):
             thread = question.thread
             thread.last_activity_at = question.last_activity_at
             thread.last_activity_by = question.last_activity_by
diff --git a/askbot/migrations/0072_transplant_tagnames_data.py b/askbot/migrations/0072_transplant_tagnames_data.py
index 5f95fe07..caa5e6a6 100644
--- a/askbot/migrations/0072_transplant_tagnames_data.py
+++ b/askbot/migrations/0072_transplant_tagnames_data.py
@@ -3,11 +3,14 @@ import datetime
 from south.db import db
 from south.v2 import DataMigration
 from django.db import models
+from askbot.utils.console import ProgressBar
 
 class Migration(DataMigration):
 
     def forwards(self, orm):
-        for question in orm.Question.objects.iterator():
+        message = "Adding denormalized tags field to threads"
+        num_questions = orm.Question.objects.count()
+        for question in ProgressBar(orm.Question.objects.iterator(), num_questions, message):
             thread = question.thread
             thread.tagnames = question.tagnames
             thread.save()
diff --git a/askbot/migrations/0075_transplant_followed_by_data.py b/askbot/migrations/0075_transplant_followed_by_data.py
index 19529b2f..06afd155 100644
--- a/askbot/migrations/0075_transplant_followed_by_data.py
+++ b/askbot/migrations/0075_transplant_followed_by_data.py
@@ -3,11 +3,14 @@ import datetime
 from south.db import db
 from south.v2 import DataMigration
 from django.db import models
+from askbot.utils.console import ProgressBar
 
 class Migration(DataMigration):
 
     def forwards(self, orm):
-        for question in orm.Question.objects.iterator():
+        message = "Adding followers to threads"
+        num_questions = orm.Question.objects.count()
+        for question in ProgressBar(orm.Question.objects.iterator(), num_questions, message):
             question.thread.followed_by.clear() # just in case someone reversed this migration
             question.thread.followed_by.add(*list(question.followed_by.iterator()))
 
diff --git a/askbot/migrations/0079_transplant_favquestions_data.py b/askbot/migrations/0079_transplant_favquestions_data.py
index eb776f0d..67decb07 100644
--- a/askbot/migrations/0079_transplant_favquestions_data.py
+++ b/askbot/migrations/0079_transplant_favquestions_data.py
@@ -3,11 +3,14 @@ import datetime
 from south.db import db
 from south.v2 import DataMigration
 from django.db import models
+from askbot.utils.console import ProgressBar
 
 class Migration(DataMigration):
 
     def forwards(self, orm):
-        for fav in orm.FavoriteQuestion.objects.iterator():
+        message = "Connecting favorite questions to threads"
+        num_favs = orm.FavoriteQuestion.objects.count()
+        for fav in ProgressBar(orm.FavoriteQuestion.objects.iterator(), num_favs, message):
             fav.thread = fav.question.thread
             fav.save()
 
diff --git a/askbot/migrations/0082_transplant_title_data.py b/askbot/migrations/0082_transplant_title_data.py
index cb9f6c1c..ccdf1e16 100644
--- a/askbot/migrations/0082_transplant_title_data.py
+++ b/askbot/migrations/0082_transplant_title_data.py
@@ -3,11 +3,14 @@ import datetime
 from south.db import db
 from south.v2 import DataMigration
 from django.db import models
+from askbot.utils.console import ProgressBar
 
 class Migration(DataMigration):
 
     def forwards(self, orm):
-        for question in orm.Question.objects.iterator():
+        message = "Adding titles to threads"
+        num_questions = orm.Question.objects.count()
+        for question in ProgressBar(orm.Question.objects.iterator(), num_questions, message):
             question.thread.title = question.title
             question.thread.save()
 
diff --git a/askbot/migrations/0086_transplant_question_tags_data.py b/askbot/migrations/0086_transplant_question_tags_data.py
index bcb2a48f..c2d9fa76 100644
--- a/askbot/migrations/0086_transplant_question_tags_data.py
+++ b/askbot/migrations/0086_transplant_question_tags_data.py
@@ -3,11 +3,14 @@ import datetime
 from south.db import db
 from south.v2 import DataMigration
 from django.db import models
+from askbot.utils.console import ProgressBar
 
 class Migration(DataMigration):
 
     def forwards(self, orm):
-        for question in orm.Question.objects.iterator():
+        message = "Linking tag objects with threads"
+        num_questions = orm.Question.objects.count()
+        for question in ProgressBar(orm.Question.objects.iterator(), num_questions, message):
             question.thread.tags.add(*list(question.tags.iterator()))
 
         if orm.Question.objects.annotate(tag_num=models.Count('tags'), thread_tag_num=models.Count('thread__tags')).exclude(tag_num=models.F('thread_tag_num')).exists():
diff --git a/askbot/migrations/0090_postize_q_a_c.py b/askbot/migrations/0090_postize_q_a_c.py
index 3d3a7b44..7c85ed21 100644
--- a/askbot/migrations/0090_postize_q_a_c.py
+++ b/askbot/migrations/0090_postize_q_a_c.py
@@ -5,6 +5,7 @@ from south.db import db
 from south.v2 import DataMigration
 from django.db import models
 from django.conf import settings
+from askbot.utils.console import ProgressBar
 
 from askbot.migrations import TERM_RED_BOLD, TERM_GREEN, TERM_RESET
 
@@ -26,7 +27,9 @@ class Migration(DataMigration):
         if 'test' not in sys.argv: # Don't confuse users
             print TERM_GREEN, '[DEBUG] Initial Post.id ==', post_id + 1, TERM_RESET
 
-        for q in orm.Question.objects.iterator():
+        message = "Converting Questions -> Posts"
+        num_questions = orm.Question.objects.count()
+        for q in ProgressBar(orm.Question.objects.iterator(), num_questions, message):
             post_id += 1
             orm.Post.objects.create(
                 id=post_id,
@@ -67,7 +70,9 @@ class Migration(DataMigration):
                 is_anonymous=q.is_anonymous,
             )
 
-        for ans in orm.Answer.objects.iterator():
+        message = "Answers -> Posts"
+        num_answers = orm.Answer.objects.count()
+        for ans in ProgressBar(orm.Answer.objects.iterator(), num_answers, message):
             post_id += 1
             orm.Post.objects.create(
                 id=post_id,
@@ -108,7 +113,9 @@ class Migration(DataMigration):
                 is_anonymous=ans.is_anonymous,
             )
 
-        for cm in orm.Comment.objects.iterator():
+        message = "Comments -> Posts"
+        num_comments = orm.Comment.objects.count()
+        for cm in ProgressBar(orm.Comment.objects.iterator(), num_comments, message):
             # Workaround for a strange issue with: http://south.aeracode.org/docs/generics.html
             # No need to investigate that as this is as simple as the "proper" way
             if (cm.content_type.app_label, cm.content_type.model) == ('askbot', 'question'):
diff --git a/askbot/migrations/0092_postize_vote_and_activity.py b/askbot/migrations/0092_postize_vote_and_activity.py
index 45442f8f..9c77597d 100644
--- a/askbot/migrations/0092_postize_vote_and_activity.py
+++ b/askbot/migrations/0092_postize_vote_and_activity.py
@@ -6,13 +6,15 @@ from south.v2 import DataMigration
 from django.db import models
 
 from askbot.migrations import TERM_RED_BOLD, TERM_GREEN, TERM_RESET
+from askbot.utils.console import ProgressBar
 
 class Migration(DataMigration):
 
     def forwards(self, orm):
 
-        # TODO: Speed this up by prefetching all votes ?
-        for v in orm.Vote.objects.iterator():
+        message = "Connecting votes to posts"
+        num_votes = orm.Vote.objects.count()
+        for v in ProgressBar(orm.Vote.objects.iterator(), num_votes, message):
             if (v.content_type.app_label, v.content_type.model) == ('askbot', 'question'):
                 v.voted_post = orm.Post.objects.get(self_question__id=v.object_id)
             elif (v.content_type.app_label, v.content_type.model) == ('askbot', 'answer'):
@@ -31,7 +33,9 @@ class Migration(DataMigration):
 
         abandoned_activities = []
 
-        for a in orm.Activity.objects.iterator():
+        message = "Connecting activity objects to posts"
+        num_activities = orm.Activity.objects.count()
+        for a in ProgressBar(orm.Activity.objects.iterator(), num_activities, message):
             # test if content_object for this activity exists - there might be a bunch of "abandoned" activities
             #
             # NOTE  that if activity.content_object is gone then we cannot reliably recover it from activity.question
diff --git a/askbot/migrations/0095_postize_award_and_repute.py b/askbot/migrations/0095_postize_award_and_repute.py
index 801949c2..ea7e7064 100644
--- a/askbot/migrations/0095_postize_award_and_repute.py
+++ b/askbot/migrations/0095_postize_award_and_repute.py
@@ -4,6 +4,7 @@ import datetime
 from south.db import db
 from south.v2 import DataMigration
 from django.db import models
+from askbot.utils.console import ProgressBar
 
 class Migration(DataMigration):
 
@@ -11,7 +12,9 @@ class Migration(DataMigration):
         # ContentType for Post model should be created no later than in migration 0092
         ct_post = orm['contenttypes.ContentType'].objects.get(app_label='askbot', model='post')
 
-        for aw in orm.Award.objects.iterator():
+        message = "Connecting award objects to posts"
+        num_awards = orm.Award.objects.count()
+        for aw in ProgressBar(orm.Award.objects.iterator(), num_awards, message):
             ct = aw.content_type
             if ct.app_label == 'askbot' and ct.model in ('question', 'answer', 'comment'):
                 aw.content_type = ct_post
@@ -23,7 +26,9 @@ class Migration(DataMigration):
 
         ###
 
-        for rp in orm.Repute.objects.iterator():
+        message = "Connecting repute objects to posts"
+        num_reputes = orm.Repute.objects.count()
+        for rp in ProgressBar(orm.Repute.objects.iterator(), num_reputes, message):
             if rp.question:
                 rp.question_post = orm.Post.objects.get(self_question__id=rp.question.id)
                 rp.save()
diff --git a/askbot/migrations/0098_postize_thread_anonanswer_questionview_postrevision.py b/askbot/migrations/0098_postize_thread_anonanswer_questionview_postrevision.py
index 0fd73605..08e62dbb 100644
--- a/askbot/migrations/0098_postize_thread_anonanswer_questionview_postrevision.py
+++ b/askbot/migrations/0098_postize_thread_anonanswer_questionview_postrevision.py
@@ -3,26 +3,35 @@ import datetime
 from south.db import db
 from south.v2 import DataMigration
 from django.db import models
+from askbot.utils.console import ProgressBar
 
 class Migration(DataMigration):
 
     def forwards(self, orm):
         # TODO: Speed this migration up by prefetching data ?
 
-        for thread in orm.Thread.objects.iterator():
+        message = "Marking accepted answer in threads"
+        num_threads = orm.Thread.objects.count()
+        for thread in ProgressBar(orm.Thread.objects.iterator(), num_threads, message):
             if thread.accepted_answer:
                 thread.accepted_answer_post = orm.Post.objects.get(self_answer__id=thread.accepted_answer.id)
                 thread.save()
 
-        for qv in orm.QuestionView.objects.iterator():
+        message = "Connecting question view objects to posts"
+        num_question_views = orm.QuestionView.objects.count()
+        for qv in ProgressBar(orm.QuestionView.objects.iterator(), num_question_views, message):
             qv.question_post = orm.Post.objects.get(self_question__id=qv.question.id)
             qv.save()
 
-        for aa in orm.AnonymousAnswer.objects.iterator():
+        message = "Connecting anonymous answers to posts"
+        num_anon_answers = orm.AnonymousAnswer.objects.count()
+        for aa in ProgressBar(orm.AnonymousAnswer.objects.iterator(), num_anon_answers, message):
             aa.question_post = orm.Post.objects.get(self_question__id=aa.question.id)
             aa.save()
 
-        for rev in orm.PostRevision.objects.iterator():
+        message = "Connecting post revisions to posts"
+        num_post_revs = orm.PostRevision.objects.count()
+        for rev in ProgressBar(orm.PostRevision.objects.iterator(), num_post_revs, message):
             if rev.question:
                 assert not rev.answer
                 rev.post = orm.Post.objects.get(self_question__id=rev.question.id)
diff --git a/askbot/utils/console.py b/askbot/utils/console.py
index f132e7d0..a40de27c 100644
--- a/askbot/utils/console.py
+++ b/askbot/utils/console.py
@@ -79,12 +79,15 @@ class ProgressBar(object):
     """A wrapper for an iterator, that prints 
     a progress bar along the way of iteration
     """
-    def __init__(self, iterable, length):
+    def __init__(self, iterable, length, message = ''):
         self.iterable = iterable
         self.length = length
         self.counter = float(0)
         self.barlen = 60
         self.progress = ''
+        if message and length > 0:
+            print message
+ 
 
     def __iter__(self):
         return self