From b8674ff379c8a9ee2997020ab5ff13477170bb76 Mon Sep 17 00:00:00 2001 From: Evgeny Fadeev Date: Wed, 29 Feb 2012 01:32:04 -0300 Subject: about halfed the queries on the question page on warm cache, cut CPU time somewhat too --- askbot/conf/email.py | 9 ++ askbot/forms.py | 11 +-- .../commands/send_accept_answer_reminders.py | 2 + askbot/management/commands/send_email_alerts.py | 13 +-- .../commands/send_unanswered_question_reminders.py | 2 + askbot/models/__init__.py | 15 ++- askbot/models/post.py | 15 ++- askbot/models/question.py | 104 +++++++++++++++++++-- askbot/skins/default/templates/macros.html | 20 ++-- askbot/skins/default/templates/question.html | 11 ++- askbot/tasks.py | 12 ++- askbot/views/readers.py | 84 ++++++++--------- 12 files changed, 209 insertions(+), 89 deletions(-) diff --git a/askbot/conf/email.py b/askbot/conf/email.py index 3db80e7a..1f60c442 100644 --- a/askbot/conf/email.py +++ b/askbot/conf/email.py @@ -30,6 +30,15 @@ settings.register( ) ) +settings.register( + livesettings.BooleanValue( + EMAIL, + 'ENABLE_EMAIL_ALERTS', + default = True, + description = _('Enable email alerts'), + ) +) + settings.register( livesettings.IntegerValue( EMAIL, diff --git a/askbot/forms.py b/askbot/forms.py index 2aa37e75..4b72180c 100644 --- a/askbot/forms.py +++ b/askbot/forms.py @@ -685,17 +685,10 @@ class AnswerForm(forms.Form): openid = forms.CharField(required=False, max_length=255, widget=forms.TextInput(attrs={'size' : 40, 'class':'openid-input'})) user = forms.CharField(required=False, max_length=255, widget=forms.TextInput(attrs={'size' : 35})) email = forms.CharField(required=False, max_length=255, widget=forms.TextInput(attrs={'size' : 35})) - email_notify = EmailNotifyField() - def __init__(self, question, user, *args, **kwargs): + email_notify = EmailNotifyField(initial = False) + def __init__(self, *args, **kwargs): super(AnswerForm, self).__init__(*args, **kwargs) self.fields['email_notify'].widget.attrs['id'] = 'question-subscribe-updates' - if question.wiki and askbot_settings.WIKI_ON: - self.fields['wiki'].initial = True - if user.is_authenticated(): - if user in question.thread.followed_by.all(): - self.fields['email_notify'].initial = True - return - self.fields['email_notify'].initial = False class VoteForm(forms.Form): """form used in ajax vote view (only comment_upvote so far) diff --git a/askbot/management/commands/send_accept_answer_reminders.py b/askbot/management/commands/send_accept_answer_reminders.py index 54fdbed4..47358a99 100644 --- a/askbot/management/commands/send_accept_answer_reminders.py +++ b/askbot/management/commands/send_accept_answer_reminders.py @@ -13,6 +13,8 @@ DEBUG_THIS_COMMAND = False class Command(NoArgsCommand): def handle_noargs(self, **options): + if askbot_settings.ENABLE_EMAIL_ALERTS == False: + return if askbot_settings.ENABLE_ACCEPT_ANSWER_REMINDERS == False: return #get questions without answers, excluding closed and deleted diff --git a/askbot/management/commands/send_email_alerts.py b/askbot/management/commands/send_email_alerts.py index c1959885..8cb71859 100644 --- a/askbot/management/commands/send_email_alerts.py +++ b/askbot/management/commands/send_email_alerts.py @@ -75,13 +75,14 @@ def format_action_count(string, number, output): class Command(NoArgsCommand): def handle_noargs(self, **options): - try: + if askbot_settings.ENABLE_EMAIL_ALERTS: try: - self.send_email_alerts() - except Exception, e: - print e - finally: - connection.close() + try: + self.send_email_alerts() + except Exception, e: + print e + finally: + connection.close() def get_updated_questions_for_user(self,user): """ diff --git a/askbot/management/commands/send_unanswered_question_reminders.py b/askbot/management/commands/send_unanswered_question_reminders.py index ba21f7de..424e45cc 100644 --- a/askbot/management/commands/send_unanswered_question_reminders.py +++ b/askbot/management/commands/send_unanswered_question_reminders.py @@ -14,6 +14,8 @@ class Command(NoArgsCommand): about unanswered questions to all users """ def handle_noargs(self, **options): + if askbot_settings.ENABLE_EMAIL_ALERTS == False: + return if askbot_settings.ENABLE_UNANSWERED_REMINDERS == False: return #get questions without answers, excluding closed and deleted diff --git a/askbot/models/__init__.py b/askbot/models/__init__.py index a3644844..ae084845 100644 --- a/askbot/models/__init__.py +++ b/askbot/models/__init__.py @@ -1484,12 +1484,17 @@ def user_visit_question(self, question = None, timestamp = None): timestamp = datetime.datetime.now() try: - question_view = QuestionView.objects.get(who=self, question=question) + QuestionView.objects.filter( + who=self, question=question + ).update( + when = timestamp + ) except QuestionView.DoesNotExist: - question_view = QuestionView(who=self, question=question) - - question_view.when = timestamp - question_view.save() + QuestionView( + who=self, + question=question, + when = timestamp + ).save() #filter memo objects on response activities directed to the qurrent user #that refer to the children of the currently diff --git a/askbot/models/post.py b/askbot/models/post.py index dead32bc..d9cd9a5e 100644 --- a/askbot/models/post.py +++ b/askbot/models/post.py @@ -210,7 +210,7 @@ class PostManager(BaseQuerySetManager): for cm in comments: post_map[cm.parent_id].append(cm) for post in for_posts: - post._cached_comments = post_map[post.id] + post.set_cached_comments(post_map[post.id]) # Old Post.get_comment(self, visitor=None) method: # if visitor.is_anonymous(): @@ -540,6 +540,19 @@ class Post(models.Model): """ return html_utils.strip_tags(self.html)[:120] + ' ...' + def set_cached_comments(self, comments): + """caches comments in the lifetime of the object + does not talk to the actual cache system + """ + self._cached_comments = comments + + def get_cached_comments(self): + try: + return self._cached_comments + except AttributeError: + self._cached_comments = list() + return self._cached_comments + def add_comment(self, comment=None, user=None, added_at=None): if added_at is None: added_at = datetime.datetime.now() diff --git a/askbot/models/question.py b/askbot/models/question.py index 65dd6db0..f6a7e69b 100644 --- a/askbot/models/question.py +++ b/askbot/models/question.py @@ -423,7 +423,8 @@ class Thread(models.Model): def get_cached_answer_list(self, sort_method = None): """get all answer posts as a list for the Thread, and a given - user. This list is cached.""" + user. This list is cached. + """ key = self.ANSWER_LIST_KEY_TPL % self.id answer_list = cache.cache.get(key) if not answer_list: @@ -440,6 +441,75 @@ class Thread(models.Model): cache.cache.set(key, answer_list) return answer_list + def get_cached_post_data(self, sort_method = None): + """returns cached post data, as calculated by + the method get_post_data()""" + key = 'thread-data-%s-%s' % (self.id, sort_method) + post_data = cache.cache.get(key) + if not post_data: + post_data = self.get_post_data(sort_method) + cache.cache.set(key, post_data) + return post_data + + def get_post_data(self, sort_method = None): + """returns question, answers as list and a list of post ids + for the given thread + the returned posts are pre-stuffed with the comments + all (both posts and the comments sorted in the correct + order) + """ + print "cache miss!!!" + thread_posts = self.posts.all().order_by( + { + "latest":"-added_at", + "oldest":"added_at", + "votes":"-score" + }[sort_method] + ) + #1) collect question, answer and comment posts and list of post id's + answers = list() + post_map = dict() + comment_map = dict() + post_id_list = list() + question_post = None + for post in thread_posts: + #pass through only deleted question posts + if post.deleted and post.post_type != 'question': + continue + + post_id_list.append(post.id) + + if post.post_type == 'answer': + answers.append(post) + post_map[post.id] = post + elif post.post_type == 'comment': + if post.parent_id not in comment_map: + comment_map[post.parent_id] = list() + comment_map[post.parent_id].append(post) + elif post.post_type == 'question': + assert(question_post == None) + post_map[post.id] = post + question_post = post + + #2) sort comments in the temporal order + for comment_list in comment_map.values(): + comment_list.sort(key=operator.attrgetter('added_at')) + + #3) attach comments to question and the answers + for post_id, comment_list in comment_map.items(): + try: + post_map[post_id].set_cached_comments(comment_list) + except KeyError: + pass#comment to deleted answer - don't want it + + if self.accepted_answer and self.accepted_answer.deleted == False: + #Put the accepted answer to front + #the second check is for the case when accepted answer is deleted + answers.remove(self.accepted_answer) + answers.insert(0, self.accepted_answer) + + return (question_post, answers, post_id_list) + def get_similarity(self, other_thread = None): """return number of tags in the other question @@ -462,9 +532,15 @@ class Thread(models.Model): """ def get_data(): - tags_list = self.tags.all() - similar_threads = Thread.objects.filter(tags__in=tags_list).\ - exclude(id = self.id).exclude(posts__post_type='question', posts__deleted = True).distinct()[:100] + tags_list = self.get_tag_names() + similar_threads = Thread.objects.filter( + tags__name__in=tags_list + ).exclude( + id = self.id + ).exclude( + posts__post_type='question', + posts__deleted = True + ).distinct()[:100] similar_threads = list(similar_threads) for thread in similar_threads: @@ -475,7 +551,8 @@ class Thread(models.Model): # Denormalize questions to speed up template rendering thread_map = dict([(thread.id, thread) for thread in similar_threads]) - questions = Post.objects.get_questions().select_related('thread').filter(thread__in=similar_threads) + questions = Post.objects.get_questions() + questions = questions.select_related('thread').filter(thread__in=similar_threads) for q in questions: thread_map[q.thread_id].question_denorm = q @@ -486,10 +563,17 @@ class Thread(models.Model): 'title': thread.get_title(thread.question_denorm) } for thread in similar_threads ] - return similar_threads - return LazyList(get_data) + def get_cached_data(): + key = 'similar-threads-%s' % self.id + data = cache.cache.get(key) + if not data: + data = get_data() + cache.cache.set(key, data) + return data + + return LazyList(get_cached_data) def remove_author_anonymity(self): """removes anonymous flag from the question @@ -504,6 +588,12 @@ class Thread(models.Model): Post.objects.filter(id=thread_question.id).update(is_anonymous=False) thread_question.revisions.all().update(is_anonymous=False) + def is_followed_by(self, user = None): + """True if thread is followed by user""" + if user and user.is_authenticated(): + return self.followed_by.filter(id = user.id).count() > 0 + return False + def update_tags(self, tagnames = None, user = None, timestamp = None): """ Updates Tag associations for a thread to match the given diff --git a/askbot/skins/default/templates/macros.html b/askbot/skins/default/templates/macros.html index e1945aba..2f3ea879 100644 --- a/askbot/skins/default/templates/macros.html +++ b/askbot/skins/default/templates/macros.html @@ -277,9 +277,9 @@ for the purposes of the AJAX comment editor #}
{% if show_post == post and show_comment and show_comment_position > max_comments %} - {% set comments = post._cached_comments[:show_comment_position] %} + {% set comments = post.get_cached_comments()[:show_comment_position] %} {% else %} - {% set comments = post._cached_comments[:max_comments] %} + {% set comments = post.get_cached_comments()[:max_comments] %} {% endif %} {% for comment in comments %} {# Warning! Any changes to the comment markup IN THIS `FOR` LOOP must be duplicated in post.js @@ -287,29 +287,31 @@ for the purposes of the AJAX comment editor #}
{% if comment.score > 0 %} -
{{comment.score}}
+
{{comment.score}}
+ {% else %}
{% endif %}
- {% if user|can_delete_comment(comment) %} - - {% endif %} +
{{comment.html}} {{comment.author.username}}  ({{comment.added_at|diff_date}}) - {% if user|can_edit_comment(comment) %}  {% trans %}edit{% endtrans %} - {% endif %}
{% endfor %}
- {% set can_post = user.is_authenticated() and user.can_post_comment(post) %} + {% set can_post = True %} {% if show_post == post and show_comment %} {% if show_comment_position > max_comments %} {{ diff --git a/askbot/skins/default/templates/question.html b/askbot/skins/default/templates/question.html index 29301e30..f25496b8 100644 --- a/askbot/skins/default/templates/question.html +++ b/askbot/skins/default/templates/question.html @@ -15,9 +15,8 @@ var data = askbot['data']; if (data['userIsAuthenticated']){ var votes = {}; - votes['{{question.id}}'] = {{user_question_vote}}; - {% for answer_id in user_answer_votes %} - votes['{{answer_id}}'] = {{user_answer_votes[answer_id]}}; + {% for post_id in user_votes %} + votes['{{post_id}}'] = {{user_votes[post_id]}}; {% endfor %} data['user_votes'] = votes; } @@ -39,7 +38,11 @@ } else { return; } - btn.className = btn.className + ' on'; + if (post_type == 'comment'){ + btn.className = btn.className + ' upvoted'; + } else { + btn.className = btn.className + ' on'; + } } } } diff --git a/askbot/tasks.py b/askbot/tasks.py index 634befb9..5509c613 100644 --- a/askbot/tasks.py +++ b/askbot/tasks.py @@ -22,6 +22,7 @@ import traceback from django.contrib.contenttypes.models import ContentType from celery.decorators import task +from askbot.conf import settings as askbot_settings from askbot.models import Activity, Post, Thread, User from askbot.models import send_instant_notifications_about_activity_in_post from askbot.models.badges import award_badges_signal @@ -133,6 +134,10 @@ def record_post_update( for user in (set(recipients) | set(newly_mentioned_users)): user.update_response_counts() + #shortcircuit if the email alerts are disabled + if askbot_settings.ENABLE_EMAIL_ALERTS == False: + return + #todo: weird thing is that only comments need the recipients #todo: debug these calls and then uncomment in the repo #argument to this call @@ -164,10 +169,15 @@ def record_question_visit( question visit """ #1) maybe update the view count - question_post = Post.objects.get(id = question_post_id) + question_post = Post.objects.filter( + id = question_post_id + ).select_related('thread')[0] if update_view_count: question_post.thread.increase_view_count() + if user_id == None: + return + #2) question view count per user and clear response displays user = User.objects.get(id = user_id) if user.is_authenticated(): diff --git a/askbot/views/readers.py b/askbot/views/readers.py index c2818ac8..22f60efa 100644 --- a/askbot/views/readers.py +++ b/askbot/views/readers.py @@ -354,6 +354,16 @@ def question(request, id):#refactor - long subroutine. display question body, an request.user.message_set.create(message = unicode(error)) return HttpResponseRedirect(reverse('index')) + #redirect if slug in the url is wrong + if request.path.split('/')[-1] != question_post.slug: + logging.debug('no slug match!') + question_url = '?'.join(( + question_post.get_absolute_url(), + urllib.urlencode(request.GET) + )) + return HttpResponseRedirect(question_url) + + #resolve comment and answer permalinks #they go first because in theory both can be moved to another question #this block "returns" show_post and assigns actual comment and answer @@ -411,50 +421,35 @@ def question(request, id):#refactor - long subroutine. display question body, an thread = question_post.thread - #redirect if slug in the url is wrong - if request.path.split('/')[-1] != question_post.slug: - logging.debug('no slug match!') - question_url = '?'.join(( - question_post.get_absolute_url(), - urllib.urlencode(request.GET) - )) - return HttpResponseRedirect(question_url) - logging.debug('answer_sort_method=' + unicode(answer_sort_method)) - #load answers - answers = thread.get_cached_answer_list(sort_method = answer_sort_method) - - # TODO: Add unit test to catch the bug where precache_comments() is called above (before) reordering the accepted answer to the top - #Post.objects.precache_comments(for_posts=[question_post] + answers, visitor=request.user) + #load answers and post id's + updated_question_post, answers, post_id_list = thread.get_cached_post_data( + sort_method = answer_sort_method, + ) + question_post.set_cached_comments(updated_question_post.get_cached_comments()) - if thread.accepted_answer and thread.accepted_answer.deleted == False: - #Put the accepted answer to front - #the second check is for the case when accepted answer is deleted - answers.remove(thread.accepted_answer) - answers.insert(0, thread.accepted_answer) - #maybe remove the personalized per-visitor caching? - Post.objects.precache_comments(for_posts=[question_post] + answers, visitor=request.user) + #Post.objects.precache_comments(for_posts=[question_post] + answers, visitor=request.user) - user_answer_votes = {} + user_votes = {} + #todo: cache this query set, but again takes only 3ms! if request.user.is_authenticated(): - votes = Vote.objects.filter(user=request.user, voted_post__in=answers) - for vote in votes: - user_answer_votes[vote.voted_post.id] = int(vote) - - #not necessary any more? - filtered_answers = [answer for answer in answers if ((not answer.deleted) or (answer.deleted and answer.author_id == request.user.id))] - + user_votes = Vote.objects.filter( + user=request.user, + voted_post__id__in = post_id_list + ).values_list('voted_post_id', 'vote') + user_votes = dict(user_votes) + #resolve page number and comment number for permalinks show_comment_position = None if show_comment: - show_page = show_comment.get_page_number(answer_posts=filtered_answers) + show_page = show_comment.get_page_number(answer_posts=answers) show_comment_position = show_comment.get_order_number() elif show_answer: - show_page = show_post.get_page_number(answer_posts=filtered_answers) + show_page = show_post.get_page_number(answer_posts=answers) - objects_list = Paginator(filtered_answers, const.ANSWERS_PAGE_SIZE) + objects_list = Paginator(answers, const.ANSWERS_PAGE_SIZE) if show_page > objects_list.num_pages: return HttpResponseRedirect(question_post.get_absolute_url()) page_objects = objects_list.page(show_page) @@ -471,7 +466,7 @@ def question(request, id):#refactor - long subroutine. display question body, an last_seen = request.session['question_view_times'].get(question_post.id, None) - if thread.last_activity_by != request.user: + if thread.last_activity_by_id != request.user.id: if last_seen: if last_seen < thread.last_activity_at: update_view_count = True @@ -504,21 +499,19 @@ def question(request, id):#refactor - long subroutine. display question body, an #todo: maybe consolidate all activity in the thread #for the user into just one query? favorited = thread.has_favorite_by_user(request.user) - user_question_vote = 0 - if request.user.is_authenticated(): - #todo: narrow scope of the "select related" call here? - #todo: maybe consolidate this query with the answer vote query? - votes = question_post.votes.select_related().filter(user=request.user) - try: - user_question_vote = int(votes[0]) - except IndexError: - user_question_vote = 0 is_cacheable = True if show_page != 1: is_cacheable = False elif show_comment_position > askbot_settings.MAX_COMMENTS_TO_SHOW: is_cacheable = False + + answer_form = AnswerForm( + initial = { + 'wiki': question_post.wiki and askbot_settings.WIKI_ON, + 'email_notify': thread.is_followed_by(request.user) + } + ) data = { 'is_cacheable': is_cacheable, @@ -526,12 +519,9 @@ def question(request, id):#refactor - long subroutine. display question body, an 'active_tab': 'questions', 'question' : question_post, 'thread': thread, - 'user_question_vote' : user_question_vote, - 'question_comment_count': question_post.comments.count(), - 'answer' : AnswerForm(question_post, request.user), + 'answer' : answer_form, 'answers' : page_objects.object_list, - 'user_answer_votes': user_answer_votes, - 'tags' : thread.tags.all(),#todo: do we need actual tags here? + 'user_votes': user_votes, 'tab_id' : answer_sort_method, 'favorited' : favorited, 'similar_threads' : thread.get_similar_threads(),#todo: cache this? -- cgit v1.2.3-1-g7c22