diff options
| -rwxr-xr-x | forum/forms.py | 3 | ||||
| -rwxr-xr-x | forum/models/answer.py | 70 | ||||
| -rwxr-xr-x | forum/models/question.py | 126 | ||||
| -rwxr-xr-x | forum/views/writers.py | 142 | ||||
| -rw-r--r-- | stackexchange/management/commands/load_stackexchange.py | 89 |
5 files changed, 282 insertions, 148 deletions
diff --git a/forum/forms.py b/forum/forms.py index 6f456184..2fcbb4a4 100755 --- a/forum/forms.py +++ b/forum/forms.py @@ -13,6 +13,7 @@ from django.conf import settings from django.contrib.contenttypes.models import ContentType import logging + class TitleField(forms.CharField): def __init__(self, *args, **kwargs): super(TitleField, self).__init__(*args, **kwargs) @@ -41,7 +42,6 @@ class EditorField(forms.CharField): def clean(self, value): if len(value) < 10: raise forms.ValidationError(_('question content must be > 10 characters')) - return value class TagNamesField(forms.CharField): @@ -185,6 +185,7 @@ class EditQuestionForm(forms.Form): tags = TagNamesField() summary = SummaryField() + #todo: this is odd that this form takes question as an argument def __init__(self, question, revision, *args, **kwargs): super(EditQuestionForm, self).__init__(*args, **kwargs) self.fields['title'].initial = revision.title diff --git a/forum/models/answer.py b/forum/models/answer.py index 9d11bb5f..e7847f32 100755 --- a/forum/models/answer.py +++ b/forum/models/answer.py @@ -1,4 +1,11 @@ from base import * +#todo: take care of copy-past markdowner stuff maybe make html automatic field? +from forum.const import CONST +from markdown2 import Markdown +from django.utils.html import strip_tags +from forum.utils.html import sanitize_html +import datetime +markdowner = Markdown(html4tags=True) from question import Question @@ -9,7 +16,7 @@ class AnswerManager(models.Manager): author = author, added_at = added_at, wiki = wiki, - html = text + html = sanitize_html(markdowner.convert(text)), ) if answer.wiki: answer.last_edited_by = answer.author @@ -18,21 +25,19 @@ class AnswerManager(models.Manager): answer.save() + answer.add_revision( + revised_by=author, + revised_at=added_at, + text=text, + comment=CONST['default_version'], + ) + #update question data question.last_activity_at = added_at question.last_activity_by = author question.save() Question.objects.update_answer_count(question) - AnswerRevision.objects.create( - answer = answer, - revision = 1, - author = author, - revised_at = added_at, - summary = CONST['default_version'], - text = text - ) - #set notification/delete if email_notify: if author not in question.followed_by.all(): @@ -43,6 +48,7 @@ class AnswerManager(models.Manager): question.followed_by.remove(author) except: pass + return answer #GET_ANSWERS_FROM_USER_QUESTIONS = u'SELECT answer.* FROM answer INNER JOIN question ON answer.question_id = question.id WHERE question.author_id =%s AND answer.author_id <> %s' def get_answers_from_question(self, question, user=None): @@ -76,6 +82,50 @@ class Answer(Content, DeletableContent): class Meta(Content.Meta): db_table = u'answer' + def apply_edit(self, edited_at=None, edited_by=None, text=None, comment=None, wiki=False): + + if text is None: + text = self.get_latest_revision().text + if edited_at is None: + edited_at = datetime.datetime.now() + if edited_by is None: + raise Exception('edited_by is required') + + self.last_edited_at = edited_at + self.last_edited_by = edited_by + self.html = sanitize_html(markdowner.convert(text)) + #todo: bug wiki has no effect here + self.save() + + self.add_revision( + revised_by=edited_by, + revised_at=edited_at, + text=text, + comment=comment + ) + + self.question.last_activity_at = edited_at + self.question.last_activity_by = edited_by + self.question.save() + + def add_revision(self, revised_by=None, revised_at=None, text=None, comment=None): + if None in (revised_by, revised_at, text): + raise Exception('arguments revised_by, revised_at and text are required') + rev_no = self.revisions.all().count() + 1 + if comment in (None, ''): + if rev_no == 1: + comment = CONST['default_version'] + else: + comment = 'No.%s Revision' % rev_no + return AnswerRevision.objects.create( + answer=self, + author=revised_by, + revised_at=revised_at, + text=text, + summary=comment, + revision=rev_no + ) + def get_user_vote(self, user): if user.__class__.__name__ == "AnonymousUser": return None diff --git a/forum/models/question.py b/forum/models/question.py index ea26cace..f4d4ac2e 100755 --- a/forum/models/question.py +++ b/forum/models/question.py @@ -1,8 +1,16 @@ from base import * from tag import Tag +from forum.const import CONST +from forum.utils.html import sanitize_html +from markdown2 import Markdown +from django.utils.html import strip_tags +import datetime +markdowner = Markdown(html4tags=True) class QuestionManager(models.Manager): - def create_new(cls, title=None,author=None,added_at=None, wiki=False,tagnames=None,summary=None, text=None): + def create_new(cls, title=None,author=None,added_at=None, wiki=False,tagnames=None, text=None): + html = sanitize_html(markdowner.convert(text)) + summary = strip_tags(html)[:120] question = Question( title = title, author = author, @@ -11,7 +19,7 @@ class QuestionManager(models.Manager): last_activity_by = author, wiki = wiki, tagnames = tagnames, - html = text, + html = html, summary = summary ) if question.wiki: @@ -21,16 +29,11 @@ class QuestionManager(models.Manager): question.save() - # create the first revision - QuestionRevision.objects.create( - question = question, - revision = 1, - title = question.title, - author = author, - revised_at = added_at, - tagnames = question.tagnames, - summary = CONST['default_version'], - text = text + question.add_revision( + author=author, + text=text, + comment=CONST['default_version'], + revised_at=added_at, ) return question @@ -143,6 +146,105 @@ class Question(Content, DeletableContent): except Exception: logging.debug('problem pinging google did you register you sitemap with google?') + def retag(self, retagged_by=None, retagged_at=None, tagnames=None): + if None in (retagged_by, retagged_at, tagnames): + raise Exception('arguments retagged_at, retagged_by and tagnames are required') + # Update the Question itself + self.tagnames = tagnames + self.last_edited_at = retagged_at + self.last_activity_at = retagged_at + self.last_edited_by = retagged_by + self.last_activity_by = retagged_by + + # Update the Question's tag associations + tags_updated = self.objects.update_tags(self, + form.cleaned_data['tags'], request.user) + + # Create a new revision + latest_revision = self.get_latest_revision() + QuestionRevision.objects.create( + question = self, + title = latest_revision.title, + author = retagged_by, + revised_at = retagged_at, + tagnames = tagnames, + summary = CONST['retagged'], + text = latest_revision.text + ) + # send tags updated singal + tags_updated.send(sender=question.__class__, question=self) + + def apply_edit(self, edited_at=None, edited_by=None, title=None,\ + text=None, comment=None, tags=None, wiki=False): + + latest_revision = self.get_latest_revision() + #a hack to allow partial edits - important for SE loader + if title is None: + title = self.title + if text is None: + text = latest_revision.text + if tags is None: + tags = latest_revision.tagnames + + if edited_by is None: + raise Exception('parameter edited_by is required') + + if edited_at is None: + edited_at = datetime.datetime.now() + + #todo: have this copy-paste in few places + html = sanitize_html(markdowner.convert(text)) + question_summary = strip_tags(html)[:120] + + # Update the Question itself + self.title = title + self.last_edited_at = edited_at + self.last_activity_at = edited_at + self.last_edited_by = edited_by + self.last_activity_by = edited_by + self.tagnames = tags + self.summary = question_summary + self.html = html + + #wiki is an eternal trap whence there is no exit + if self.wiki == False and wiki == True: + self.wiki = True + + self.save() + + # Update the Question tag associations + if latest_revision.tagnames != tags: + tags_updated = Question.objects.update_tags(self, tags, edited_by) + + # Create a new revision + self.add_revision( + author = edited_by, + text = text, + revised_at = edited_at, + comment = comment, + ) + + def add_revision(self,author=None, text=None, comment=None, revised_at=None): + if None in (author, text, comment): + raise Exception('author, text and revised_at are required arguments') + rev_no = self.revisions.all().count() + 1 + if comment in (None, ''): + if rev_no == 1: + comment = CONST['default_version'] + else: + comment = 'No.%s Revision' % rev_no + + return QuestionRevision.objects.create( + question = self, + revision = rev_no, + title = self.title, + author = author, + revised_at = revised_at, + tagnames = self.tagnames, + summary = comment, + text = text + ) + def save(self, **kwargs): """ Overridden to manually manage addition of tags when the object diff --git a/forum/views/writers.py b/forum/views/writers.py index 2b2461de..6db39aaf 100755 --- a/forum/views/writers.py +++ b/forum/views/writers.py @@ -13,8 +13,6 @@ from django.utils.translation import ugettext as _ from django.core.urlresolvers import reverse from django.core.exceptions import PermissionDenied -from forum.utils.html import sanitize_html -from markdown2 import Markdown from forum.forms import * from forum.models import * from forum.auth import * @@ -34,8 +32,6 @@ QUESTIONS_PAGE_SIZE = 10 # used in answers ANSWERS_PAGE_SIZE = 10 -markdowner = Markdown(html4tags=True) - def upload(request):#ajax upload file to a question or answer class FileTypeNotAllow(Exception): pass @@ -94,12 +90,16 @@ def ask(request):#view used to ask a new question if form.is_valid(): added_at = datetime.datetime.now() + #todo: move this to clean_title title = strip_tags(form.cleaned_data['title'].strip()) wiki = form.cleaned_data['wiki'] + #todo: move this to clean_tagnames tagnames = form.cleaned_data['tags'].strip() text = form.cleaned_data['text'] - html = sanitize_html(markdowner.convert(text)) - summary = strip_tags(html)[:120] + + #todo: move this to AskForm.clean_text + #todo: make custom MarkDownField + text = form.cleaned_data['text'] if request.user.is_authenticated(): author = request.user @@ -110,14 +110,14 @@ def ask(request):#view used to ask a new question added_at = added_at, wiki = wiki, tagnames = tagnames, - summary = summary, - text = sanitize_html(markdowner.convert(text)) + text = text, ) return HttpResponseRedirect(question.get_absolute_url()) else: request.session.flush() session_key = request.session.session_key + summary = strip_tags(text)[:120] question = AnonymousQuestion( session_key = session_key, title = title, @@ -162,32 +162,11 @@ def _retag_question(request, question):#non-url subview of edit question - just form = RetagQuestionForm(question, request.POST) if form.is_valid(): if form.has_changed(): - latest_revision = question.get_latest_revision() - retagged_at = datetime.datetime.now() - # Update the Question itself - Question.objects.filter(id=question.id).update( - tagnames = form.cleaned_data['tags'], - last_edited_at = retagged_at, - last_edited_by = request.user, - last_activity_at = retagged_at, - last_activity_by = request.user - ) - # Update the Question's tag associations - tags_updated = Question.objects.update_tags(question, - form.cleaned_data['tags'], request.user) - # Create a new revision - QuestionRevision.objects.create( - question = question, - title = latest_revision.title, - author = request.user, - revised_at = retagged_at, - tagnames = form.cleaned_data['tags'], - summary = CONST['retagged'], - text = latest_revision.text + question.retag( + retagged_by = request.user, + retagged_at = datetime.datetime.now(), + tagnames = form.cleaned_data['tags'], ) - # send tags updated singal - tags_updated.send(sender=question.__class__, question=question) - return HttpResponseRedirect(question.get_absolute_url()) else: form = RetagQuestionForm(question) @@ -201,7 +180,7 @@ def _edit_question(request, question):#non-url subview of edit_question - just e latest_revision = question.get_latest_revision() revision_form = None if request.method == 'POST': - if 'select_revision' in request.POST: + if 'select_revision' in request.POST:#revert-type edit # user has changed revistion number revision_form = RevisionForm(question, latest_revision, request.POST) if revision_form.is_valid(): @@ -211,60 +190,27 @@ def _edit_question(request, question):#non-url subview of edit_question - just e revision=revision_form.cleaned_data['revision'])) else: form = EditQuestionForm(question, latest_revision, request.POST) - else: + else:#new content edit # Always check modifications against the latest revision form = EditQuestionForm(question, latest_revision, request.POST) if form.is_valid(): - html = sanitize_html(markdowner.convert(form.cleaned_data['text'])) + html = form.cleaned_data['text']#markdown this if form.has_changed(): edited_at = datetime.datetime.now() - tags_changed = (latest_revision.tagnames != - form.cleaned_data['tags']) - tags_updated = False - # Update the Question itself - updated_fields = { - 'title': form.cleaned_data['title'], - 'last_edited_at': edited_at, - 'last_edited_by': request.user, - 'last_activity_at': edited_at, - 'last_activity_by': request.user, - 'tagnames': form.cleaned_data['tags'], - 'summary': strip_tags(html)[:120], - 'html': html, - } - - # only save when it's checked - # because wiki doesn't allow to be edited if last version has been enabled already - # and we make sure this in forms. - if ('wiki' in form.cleaned_data and - form.cleaned_data['wiki']): - updated_fields['wiki'] = True - updated_fields['wikified_at'] = edited_at - - Question.objects.filter( - id=question.id).update(**updated_fields) - # Update the Question's tag associations - if tags_changed: - tags_updated = Question.objects.update_tags( - question, form.cleaned_data['tags'], request.user) - # Create a new revision - revision = QuestionRevision( - question = question, - title = form.cleaned_data['title'], - author = request.user, - revised_at = edited_at, - tagnames = form.cleaned_data['tags'], - text = form.cleaned_data['text'], + edited_by = request.user + question.apply_edit( + edited_at = edited_at, + edited_by = edited_by, + title = form.cleaned_data['title'], + text = form.cleaned_data['text'], + #todo: summary name clash in question and question revision + comment = form.cleaned_data['summary'], + tags = form.cleaned_data['tags'], + wiki = form.cleaned_data.get('wiki',False), ) - if form.cleaned_data['summary']: - revision.summary = form.cleaned_data['summary'] - else: - revision.summary = 'No.%s Revision' % latest_revision.revision - revision.save() return HttpResponseRedirect(question.get_absolute_url()) else: - revision_form = RevisionForm(question, latest_revision) form = EditQuestionForm(question, latest_revision) return render_to_response('question_edit.html', { @@ -297,33 +243,15 @@ def edit_answer(request, id): else: form = EditAnswerForm(answer, latest_revision, request.POST) if form.is_valid(): - html = sanitize_html(markdowner.convert(form.cleaned_data['text'])) if form.has_changed(): edited_at = datetime.datetime.now() - updated_fields = { - 'last_edited_at': edited_at, - 'last_edited_by': request.user, - 'html': html, - } - Answer.objects.filter(id=answer.id).update(**updated_fields) - - revision = AnswerRevision( - answer=answer, - author=request.user, - revised_at=edited_at, - text=form.cleaned_data['text'] - ) - - if form.cleaned_data['summary']: - revision.summary = form.cleaned_data['summary'] - else: - revision.summary = 'No.%s Revision' % latest_revision.revision - revision.save() - - answer.question.last_activity_at = edited_at - answer.question.last_activity_by = request.user - answer.question.save() - + answer.apply_edit( + edited_at = edited_at, + edited_by = request.user, + text = form.cleaned_data['text'], + comment = form.cleaned_data['summary'], + wiki = False,#todo: fix this there is no "wiki" field on "edit answer" + ) return HttpResponseRedirect(answer.get_absolute_url()) else: revision_form = RevisionForm(answer, latest_revision) @@ -349,18 +277,16 @@ def answer(request, id):#process a new answer author=request.user, added_at=update_time, wiki=wiki, - text=sanitize_html(markdowner.convert(text)), + text=text, email_notify=form.cleaned_data['email_notify'] ) else: request.session.flush() - html = sanitize_html(markdowner.convert(text)) - summary = strip_tags(html)[:120] anon = AnonymousAnswer( question=question, wiki=wiki, text=text, - summary=summary, + summary=strip_tags(text)[:120], session_key=request.session.session_key, ip_addr=request.META['REMOTE_ADDR'], ) diff --git a/stackexchange/management/commands/load_stackexchange.py b/stackexchange/management/commands/load_stackexchange.py index e1e10dd7..7d47870e 100644 --- a/stackexchange/management/commands/load_stackexchange.py +++ b/stackexchange/management/commands/load_stackexchange.py @@ -27,6 +27,8 @@ xml_read_order = ( #association tables SE item id --> OSQA item id #table associations are implied USER = {}#SE User.id --> django(OSQA) User.id +QUESTION = {} +ANSWER = {} NAMESAKE_COUNT = {}# number of times user name was used - for X.get_screen_name class X(object):# @@ -195,9 +197,7 @@ class Command(BaseCommand): #transfer data into OSQA tables self.transfer_users() - self.transfer_questions() - self.transfer_answers() - self.transfer_close_decisions() + self.transfer_question_and_answer_activity() self.transfer_comments() self.transfer_badges() self.transfer_votes() @@ -225,20 +225,77 @@ class Command(BaseCommand): tags = X.clean_tags(rev.text) elif rev_type == 'Community Owned': wiki = True + else: + raise Exception('unexpected revision type %s' % rev_type) - if rev_group[0].post.post_type.name == 'Question': - osqa.Question.objects.create_new( + post_type = rev_group[0].post.post_type.name + if post_type == 'Question': + q = osqa.Question.objects.create_new( title = title, author = author, added_at = added_at, wiki = wiki, tagnames = tags, - summary = text[:120], text = text ) + QUESTION[rev_group[0].post.id] = q + elif post_type == 'Answer': + q = QUESTION[rev_group[0].post.parent.id] + a = osqa.Answer.objects.create_new( + question = q, + author = author, + added_at = added_at, + wiki = wiki, + text = text, + ) + ANSWER[rev_group[0].post.id] = a + else: + post_id = rev_group[0].post.id + raise Exception('unknown post type %s for id=%d' % (post_type, post_id)) def _process_post_edit_revision_group(self, rev_group): - pass + #question apply edit + (title, text, tags, wiki) = (None, None, None, False) + for rev in rev_group: + rev_type = rev.post_history_type.name + if rev_type == 'Edit Title': + title = rev.text + elif rev_type == 'Edit Body': + text = rev.text + elif rev_type == 'Edit Tags': + tags = X.clean_tags(rev.text) + elif rev_type == 'Community Owned': + wiki = True + else: + raise Exception('unexpected revision type %s' % rev_type) + + rev0 = rev_group[0] + edited_by = USER[rev0.user.id] + edited_at = rev0.creation_date + comment = ';'.join([rev.comment for rev in rev_group if rev.comment]) + post_type = rev0.post.post_type.name + + if post_type == 'Question': + q = QUESTION[rev0.post.id] + q.apply_edit( + edited_at = edited_at, + edited_by = edited_by, + title = title, + text = text, + comment = comment, + tags = tags, + wiki = wiki + ) + elif post_type == 'Answer': + a = ANSWER[rev0.post.id] + #todo: wiki will probably be lost here + a.apply_edit( + edited_at = edited_at, + edited_by = edited_by, + text = text, + comment = comment, + wiki = wiki + ) def _process_post_action_revision_group(self, rev_group): pass @@ -255,8 +312,14 @@ class Command(BaseCommand): else: self._process_post_action_revision_group(rev_group) - def transfer_questions(self): - se_revs = se.PostHistory.objects.filter(post__post_type__name='Question') + def transfer_question_and_answer_activity(self): + """transfers all question and answer + edits and related status changes + """ + #assuming that there are only two post types + se_revs = se.PostHistory.objects.all() + #assuming that chronologial order is correct and there + #will be no problems of data integrity upon insertion of records se_revs = se_revs.order_by('creation_date','revision_guid') #todo: ignored fringe case - no revisions c_guid = se_revs[0].revision_guid @@ -272,14 +335,6 @@ class Command(BaseCommand): c_group.append(se_rev) c_guid = se_rev.revision_guid - def transfer_answers(self): - pass - - def transfer_close_decisions(self): - #this is not necessary, b/c close/reopen decisions - #are parts of revisions so this will stay noop - pass - def transfer_comments(self): pass |