From 710574352dabcab4c710f2795a26501e54329e79 Mon Sep 17 00:00:00 2001 From: hrcerqueira Date: Thu, 18 Feb 2010 00:14:01 +0000 Subject: Fixed some errors and moved some folders to inside the forum folder. --- .gitignore | 2 + django_authopenid/forms.py | 4 +- django_authopenid/util.py | 2 +- django_authopenid/views.py | 2 +- forum/forms.py | 2 +- forum/management/commands/send_email_alerts.py | 2 +- forum/middleware/__init__.py | 0 forum/middleware/anon_user.py | 34 + forum/middleware/cancel.py | 15 + forum/middleware/pagesize.py | 33 + forum/models/answer.py | 27 +- forum/models/base.py | 3 +- forum/models/question.py | 10 +- forum/utils/__init__.py | 0 forum/utils/cache.py | 92 ++ forum/utils/decorators.py | 25 + forum/utils/forms.py | 151 +++ forum/utils/html.py | 51 + forum/utils/lists.py | 86 ++ forum/utils/odict.py | 1399 ++++++++++++++++++++++++ forum/views/commands.py | 2 +- forum/views/content.py | 6 +- forum/views/meta.py | 2 +- forum/views/readers.py | 4 +- forum/views/writers.py | 11 +- middleware/__init__.py | 0 middleware/anon_user.py | 34 - middleware/cancel.py | 15 - middleware/pagesize.py | 33 - settings.py | 6 +- settings_local.py.dist | 110 ++ utils/__init__.py | 0 utils/cache.py | 92 -- utils/decorators.py | 25 - utils/forms.py | 151 --- utils/html.py | 51 - utils/lists.py | 86 -- utils/odict.py | 1399 ------------------------ 38 files changed, 2038 insertions(+), 1929 deletions(-) create mode 100644 forum/middleware/__init__.py create mode 100644 forum/middleware/anon_user.py create mode 100644 forum/middleware/cancel.py create mode 100644 forum/middleware/pagesize.py create mode 100644 forum/utils/__init__.py create mode 100644 forum/utils/cache.py create mode 100644 forum/utils/decorators.py create mode 100644 forum/utils/forms.py create mode 100644 forum/utils/html.py create mode 100644 forum/utils/lists.py create mode 100644 forum/utils/odict.py delete mode 100644 middleware/__init__.py delete mode 100644 middleware/anon_user.py delete mode 100644 middleware/cancel.py delete mode 100644 middleware/pagesize.py create mode 100755 settings_local.py.dist delete mode 100644 utils/__init__.py delete mode 100644 utils/cache.py delete mode 100644 utils/decorators.py delete mode 100644 utils/forms.py delete mode 100644 utils/html.py delete mode 100644 utils/lists.py delete mode 100644 utils/odict.py diff --git a/.gitignore b/.gitignore index c02c6f1a..11c8905f 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ osqa.wsgi nbproject settings_local.py +.idea +*.iml diff --git a/django_authopenid/forms.py b/django_authopenid/forms.py index 5ec21c1c..2f34986c 100644 --- a/django_authopenid/forms.py +++ b/django_authopenid/forms.py @@ -39,7 +39,7 @@ import types import re from django.utils.safestring import mark_safe from recaptcha_django import ReCaptchaField -from utils.forms import NextUrlField, UserNameField, UserEmailField, SetPasswordForm +from forum.utils.forms import NextUrlField, UserNameField, UserEmailField, SetPasswordForm EXTERNAL_LOGIN_APP = settings.LOAD_EXTERNAL_LOGIN_APP() # needed for some linux distributions like debian @@ -48,7 +48,7 @@ try: except ImportError: from yadis import xri -from utils.forms import clean_next +from forum.utils.forms import clean_next from django_authopenid.models import ExternalLoginData __all__ = ['OpenidSigninForm', 'ClassicLoginForm', 'OpenidVerifyForm', diff --git a/django_authopenid/util.py b/django_authopenid/util.py index 969af0b9..cd2c2e2c 100644 --- a/django_authopenid/util.py +++ b/django_authopenid/util.py @@ -15,7 +15,7 @@ except: from yadis import xri import time, base64, hashlib, operator, logging -from utils.forms import clean_next, get_next_url +from forum.utils.forms import clean_next, get_next_url from models import Association, Nonce diff --git a/django_authopenid/views.py b/django_authopenid/views.py index 16a78864..7c7d9e07 100755 --- a/django_authopenid/views.py +++ b/django_authopenid/views.py @@ -67,7 +67,7 @@ from django_authopenid.forms import OpenidSigninForm, ClassicLoginForm, OpenidRe OpenidVerifyForm, ClassicRegisterForm, ChangePasswordForm, ChangeEmailForm, \ ChangeopenidForm, DeleteForm, EmailPasswordForm import logging -from utils.forms import get_next_url +from forum.utils.forms import get_next_url EXTERNAL_LOGIN_APP = settings.LOAD_EXTERNAL_LOGIN_APP() diff --git a/forum/forms.py b/forum/forms.py index 5796e2c1..f22763f7 100644 --- a/forum/forms.py +++ b/forum/forms.py @@ -5,7 +5,7 @@ from models import * from const import * from django.utils.translation import ugettext as _ from django.contrib.auth.models import User -from utils.forms import NextUrlField, UserNameField +from forum.utils.forms import NextUrlField, UserNameField from recaptcha_django import ReCaptchaField from django.conf import settings import logging diff --git a/forum/management/commands/send_email_alerts.py b/forum/management/commands/send_email_alerts.py index 62f13d69..26eb7790 100644 --- a/forum/management/commands/send_email_alerts.py +++ b/forum/management/commands/send_email_alerts.py @@ -9,7 +9,7 @@ from django.utils.translation import ungettext import datetime from django.conf import settings import logging -from utils.odict import OrderedDict +from forum.utils.odict import OrderedDict class Command(NoArgsCommand): def handle_noargs(self,**options): diff --git a/forum/middleware/__init__.py b/forum/middleware/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/forum/middleware/anon_user.py b/forum/middleware/anon_user.py new file mode 100644 index 00000000..5295d9c0 --- /dev/null +++ b/forum/middleware/anon_user.py @@ -0,0 +1,34 @@ +from django.http import HttpResponseRedirect +from forum.utils.forms import get_next_url +from django.utils.translation import ugettext as _ +from user_messages import create_message, get_and_delete_messages +from django.conf import settings +import logging + +class AnonymousMessageManager(object): + def __init__(self,request): + self.request = request + def create(self,message=''): + create_message(self.request,message) + def get_and_delete(self): + messages = get_and_delete_messages(self.request) + return messages + +def dummy_deepcopy(*arg): + """this is necessary to prevent deepcopy() on anonymous user object + that now contains reference to request, which cannot be deepcopied + """ + return None + +class ConnectToSessionMessagesMiddleware(object): + def process_request(self, request): + if not request.user.is_authenticated(): + request.user.__deepcopy__ = dummy_deepcopy #plug on deepcopy which may be called by django db "driver" + request.user.message_set = AnonymousMessageManager(request) #here request is linked to anon user + request.user.get_and_delete_messages = request.user.message_set.get_and_delete + + #also set the first greeting one time per session only + if 'greeting_set' not in request.session: + request.session['greeting_set'] = True + msg = _('first time greeting with %(url)s') % {'url':settings.GREETING_URL} + request.user.message_set.create(message=msg) diff --git a/forum/middleware/cancel.py b/forum/middleware/cancel.py new file mode 100644 index 00000000..15a4371d --- /dev/null +++ b/forum/middleware/cancel.py @@ -0,0 +1,15 @@ +from django.http import HttpResponseRedirect +from forum.utils.forms import get_next_url +import logging +class CancelActionMiddleware(object): + def process_view(self, request, view_func, view_args, view_kwargs): + if 'cancel' in request.REQUEST: + #todo use session messages for the anonymous users + try: + msg = getattr(view_func,'CANCEL_MESSAGE') + except AttributeError: + msg = 'action canceled' + request.user.message_set.create(message=msg) + return HttpResponseRedirect(get_next_url(request)) + else: + return None diff --git a/forum/middleware/pagesize.py b/forum/middleware/pagesize.py new file mode 100644 index 00000000..f6e6fcfd --- /dev/null +++ b/forum/middleware/pagesize.py @@ -0,0 +1,33 @@ +# used in questions +QUESTIONS_PAGE_SIZE = 10 +class QuestionsPageSizeMiddleware(object): + def process_request(self, request): + # Set flag to False by default. If it is equal to True, then need to be saved. + pagesize_changed = False + # get pagesize from session, if failed then get default value + user_page_size = request.session.get("pagesize", QUESTIONS_PAGE_SIZE) + # set pagesize equal to logon user specified value in database + if request.user.is_authenticated() and request.user.questions_per_page > 0: + user_page_size = request.user.questions_per_page + + try: + # get new pagesize from UI selection + pagesize = int(request.GET.get('pagesize', user_page_size)) + if pagesize <> user_page_size: + pagesize_changed = True + + except ValueError: + pagesize = user_page_size + + # save this pagesize to user database + if pagesize_changed: + if request.user.is_authenticated(): + user = request.user + user.questions_per_page = pagesize + user.save() + # put pagesize into session + request.session["pagesize"] = pagesize + + def process_exception(self,request,exception): + import logging + logging.debug('have exception %s' % str(exception)) diff --git a/forum/models/answer.py b/forum/models/answer.py index 4a44bd49..a1580828 100755 --- a/forum/models/answer.py +++ b/forum/models/answer.py @@ -24,9 +24,6 @@ class AnswerManager(models.Manager): question.save() Question.objects.update_answer_count(question) - #update revision - from models import AnswerRevision - AnswerRevision.objects.create( answer = answer, revision = 1, @@ -47,7 +44,7 @@ class AnswerManager(models.Manager): except: pass - GET_ANSWERS_FROM_USER_QUESTIONS = u'SELECT answer.* FROM answer INNER JOIN question ON answer.question_id = question.id WHERE question.author_id =%s AND answer.author_id <> %s' + #GET_ANSWERS_FROM_USER_QUESTIONS = u'SELECT answer.* FROM answer INNER JOIN question ON answer.question_id = question.id WHERE question.author_id =%s AND answer.author_id <> %s' def get_answers_from_question(self, question, user=None): """ Retrieves visibile answers for the given question. Delete answers @@ -57,16 +54,17 @@ class AnswerManager(models.Manager): if user is None or not user.is_authenticated(): return self.filter(question=question, deleted=False) else: - return self.filter(Q(question=question), - Q(deleted=False) | Q(deleted_by=user)) - - def get_answers_from_questions(self, user_id): - """ - Retrieves visibile answers for the given question. Which are not included own answers - """ - cursor = connection.cursor() - cursor.execute(self.GET_ANSWERS_FROM_USER_QUESTIONS, [user_id, user_id]) - return cursor.fetchall() + return self.filter(models.Q(question=question), + models.Q(deleted=False) | models.Q(deleted_by=user)) + + #todo: I think this method is not being used anymore, I'll just comment it for now + #def get_answers_from_questions(self, user_id): + # """ + # Retrieves visibile answers for the given question. Which are not included own answers + # """ + # cursor = connection.cursor() + # cursor.execute(self.GET_ANSWERS_FROM_USER_QUESTIONS, [user_id, user_id]) + # return cursor.fetchall() class Answer(Content, DeletableContent): question = models.ForeignKey('Question', related_name='answers') @@ -99,6 +97,7 @@ class Answer(Content, DeletableContent): def __unicode__(self): return self.html + class AnswerRevision(ContentRevision): """A revision of an Answer.""" diff --git a/forum/models/base.py b/forum/models/base.py index 24fe2b0d..2c28a470 100755 --- a/forum/models/base.py +++ b/forum/models/base.py @@ -1,7 +1,7 @@ import datetime import hashlib from urllib import quote_plus, urlencode -from django.db import models, IntegrityError +from django.db import models, IntegrityError, connection, transaction from django.utils.http import urlquote as django_urlquote from django.utils.html import strip_tags from django.core.urlresolvers import reverse @@ -20,7 +20,6 @@ import logging if settings.USE_SPHINX_SEARCH == True: from djangosphinx.models import SphinxSearch -from forum.managers import * from forum.const import * class MetaContent(models.Model): diff --git a/forum/models/question.py b/forum/models/question.py index cfa2f6be..20779d26 100755 --- a/forum/models/question.py +++ b/forum/models/question.py @@ -21,8 +21,6 @@ class QuestionManager(models.Manager): question.save() - from models import QuestionRevision - # create the first revision QuestionRevision.objects.create( question = question, @@ -77,7 +75,7 @@ class QuestionManager(models.Manager): # for some reasons, this Answer class failed to be imported, # although we have imported all classes from models on top. - from models import Answer + from answer import Answer self.filter(id=question.id).update( answer_count=Answer.objects.get_answers_from_question(question).filter(deleted=False).count()) @@ -91,7 +89,6 @@ class QuestionManager(models.Manager): """ update favourite_count for given question """ - from models import FavoriteQuestion self.filter(id=question.id).update(favourite_count = FavoriteQuestion.objects.filter(question=question).count()) def get_similar_questions(self, question): @@ -176,11 +173,10 @@ class Question(Content, DeletableContent): if not user.is_authenticated(): return False - from models import FavoriteQuestion return FavoriteQuestion.objects.filter(question=self, user=user).count() > 0 def get_answer_count_by_user(self, user_id): - from models import Answer + from answer import Answer query_set = Answer.objects.filter(author__id=user_id) return query_set.filter(question=self).count() @@ -273,6 +269,8 @@ class Question(Content, DeletableContent): def __unicode__(self): return self.title + + class QuestionView(models.Model): question = models.ForeignKey(Question, related_name='viewed') who = models.ForeignKey(User, related_name='question_views') diff --git a/forum/utils/__init__.py b/forum/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/forum/utils/cache.py b/forum/utils/cache.py new file mode 100644 index 00000000..410c0662 --- /dev/null +++ b/forum/utils/cache.py @@ -0,0 +1,92 @@ +"""Utilities for working with Django Models.""" +import itertools + +from django.contrib.contenttypes.models import ContentType + +from lanai.utils.lists import flatten + +def fetch_model_dict(model, ids, fields=None): + """ + Fetches a dict of model details for model instances with the given + ids, keyed by their id. + + If a fields list is given, a dict of details will be retrieved for + each model, otherwise complete model instances will be retrieved. + + Any fields list given shouldn't contain the primary key attribute for + the model, as this can be determined from its Options. + """ + if fields is None: + return model._default_manager.in_bulk(ids) + else: + id_attr = model._meta.pk.attname + return dict((obj[id_attr], obj) for obj + in model._default_manager.filter(id__in=ids).values( + *itertools.chain((id_attr,), fields))) + +def populate_foreign_key_caches(model, objects_to_populate, fields=None): + """ + Populates caches for the given related Model in instances of objects + which have a ForeignKey relationship to it, specified as a list of + (object list, related attribute name list) two-tuples. + + If a list of field names is given, only the given fields will be + looked up and related object caches will be populated with a dict of + the specified fields. Otherwise, complete model instances will be + retrieved. + """ + # Get all related object ids for the appropriate fields + related_object_ids = [] + for objects, attrs in objects_to_populate: + related_object_ids.append(tuple(tuple(getattr(obj, '%s_id' % attr) + for attr in attrs) + for obj in objects)) + unique_ids = tuple(set(pk for pk in flatten(related_object_ids) if pk)) + related_objects = fetch_model_dict(model, unique_ids, fields) + + # Fill related object caches + for (objects, attrs), related_ids in itertools.izip(objects_to_populate, + related_object_ids): + for obj, related_ids_for_obj in itertools.izip(objects, + related_ids): + for attr, related_object in itertools.izip(attrs, (related_objects.get(pk, None) + for pk in related_ids_for_obj)): + setattr(obj, '_%s_cache' % attr, related_object) + +def populate_content_object_caches(generic_related_objects, model_fields=None): + """ + Retrieves ``ContentType`` and content objects for the given list of + items which use a generic relation, grouping the retrieval of content + objects by model to reduce the number of queries executed. + + This results in ``number_of_content_types + 1`` queries rather than + the ``number_of_generic_reL_objects * 2`` queries you'd get by + iterating over the list and accessing each item's object attribute. + + If a dict mapping model classes to field names is given, only the + given fields will be looked up for each model specified and the + object cache will be populated with a dict of the specified fields. + Otherwise, complete model instances will be retrieved. + """ + if model_fields is None: + model_fields = {} + + # Group content object ids by their content type ids + ids_by_content_type = {} + for obj in generic_related_objects: + ids_by_content_type.setdefault(obj.content_type_id, + []).append(obj.object_id) + + # Retrieve content types and content objects in bulk + content_types = ContentType.objects.in_bulk(ids_by_content_type.keys()) + for content_type_id, ids in ids_by_content_type.iteritems(): + model = content_types[content_type_id].model_class() + objects[content_type_id] = fetch_model_dict( + model, tuple(set(ids)), model_fields.get(model, None)) + + # Set content types and content objects in the appropriate cache + # attributes, so accessing the 'content_type' and 'object' attributes + # on each object won't result in further database hits. + for obj in generic_related_objects: + obj._object_cache = objects[obj.content_type_id][obj.object_id] + obj._content_type_cache = content_types[obj.content_type_id] diff --git a/forum/utils/decorators.py b/forum/utils/decorators.py new file mode 100644 index 00000000..e4e7acb3 --- /dev/null +++ b/forum/utils/decorators.py @@ -0,0 +1,25 @@ +from django.http import HttpResponse, HttpResponseForbidden, Http404 +from django.utils import simplejson + +def ajax_login_required(view_func): + def wrap(request,*args,**kwargs): + if request.user.is_authenticated(): + return view_func(request,*args,**kwargs) + else: + json = simplejson.dumps({'login_required':True}) + return HttpResponseForbidden(json,mimetype='application/json') + return wrap + +def ajax_method(view_func): + def wrap(request,*args,**kwargs): + if not request.is_ajax(): + raise Http404 + retval = view_func(request,*args,**kwargs) + if isinstance(retval, HttpResponse): + retval.mimetype = 'application/json' + return retval + else: + json = simplejson.dumps(retval) + return HttpResponse(json,mimetype='application/json') + return wrap + diff --git a/forum/utils/forms.py b/forum/utils/forms.py new file mode 100644 index 00000000..c54056ca --- /dev/null +++ b/forum/utils/forms.py @@ -0,0 +1,151 @@ +from django import forms +import re +from django.utils.translation import ugettext as _ +from django.utils.safestring import mark_safe +from django.conf import settings +from django.http import str_to_unicode +from django.contrib.auth.models import User +import urllib + +DEFAULT_NEXT = '/' + getattr(settings, 'FORUM_SCRIPT_ALIAS') +def clean_next(next): + if next is None: + return DEFAULT_NEXT + next = str_to_unicode(urllib.unquote(next), 'utf-8') + next = next.strip() + if next.startswith('/'): + return next + return DEFAULT_NEXT + +def get_next_url(request): + return clean_next(request.REQUEST.get('next')) + +class StrippedNonEmptyCharField(forms.CharField): + def clean(self,value): + value = value.strip() + if self.required and value == '': + raise forms.ValidationError(_('this field is required')) + return value + +class NextUrlField(forms.CharField): + def __init__(self): + super(NextUrlField,self).__init__(max_length = 255,widget = forms.HiddenInput(),required = False) + def clean(self,value): + return clean_next(value) + +login_form_widget_attrs = { 'class': 'required login' } +username_re = re.compile(r'^[\w ]+$') + +class UserNameField(StrippedNonEmptyCharField): + RESERVED_NAMES = (u'fuck', u'shit', u'ass', u'sex', u'add', + u'edit', u'save', u'delete', u'manage', u'update', 'remove', 'new') + def __init__(self,db_model=User, db_field='username', must_exist=False,skip_clean=False,label=_('choose a username'),**kw): + self.must_exist = must_exist + self.skip_clean = skip_clean + self.db_model = db_model + self.db_field = db_field + error_messages={'required':_('user name is required'), + 'taken':_('sorry, this name is taken, please choose another'), + 'forbidden':_('sorry, this name is not allowed, please choose another'), + 'missing':_('sorry, there is no user with this name'), + 'multiple-taken':_('sorry, we have a serious error - user name is taken by several users'), + 'invalid':_('user name can only consist of letters, empty space and underscore'), + } + if 'error_messages' in kw: + error_messages.update(kw['error_messages']) + del kw['error_messages'] + super(UserNameField,self).__init__(max_length=30, + widget=forms.TextInput(attrs=login_form_widget_attrs), + label=label, + error_messages=error_messages, + **kw + ) + + def clean(self,username): + """ validate username """ + if self.skip_clean == True: + return username + if hasattr(self, 'user_instance') and isinstance(self.user_instance, User): + if username == self.user_instance.username: + return username + try: + username = super(UserNameField, self).clean(username) + except forms.ValidationError: + raise forms.ValidationError(self.error_messages['required']) + if self.required and not username_re.search(username): + raise forms.ValidationError(self.error_messages['invalid']) + if username in self.RESERVED_NAMES: + raise forms.ValidationError(self.error_messages['forbidden']) + try: + user = self.db_model.objects.get( + **{'%s' % self.db_field : username} + ) + if user: + if self.must_exist: + return username + else: + raise forms.ValidationError(self.error_messages['taken']) + except self.db_model.DoesNotExist: + if self.must_exist: + raise forms.ValidationError(self.error_messages['missing']) + else: + return username + except self.db_model.MultipleObjectsReturned: + raise forms.ValidationError(self.error_messages['multiple-taken']) + +class UserEmailField(forms.EmailField): + def __init__(self,skip_clean=False,**kw): + self.skip_clean = skip_clean + super(UserEmailField,self).__init__(widget=forms.TextInput(attrs=dict(login_form_widget_attrs, + maxlength=200)), label=mark_safe(_('your email address')), + error_messages={'required':_('email address is required'), + 'invalid':_('please enter a valid email address'), + 'taken':_('this email is already used by someone else, please choose another'), + }, + **kw + ) + + def clean(self,email): + """ validate if email exist in database + from legacy register + return: raise error if it exist """ + email = super(UserEmailField,self).clean(email.strip()) + if self.skip_clean: + return email + if settings.EMAIL_UNIQUE == True: + try: + user = User.objects.get(email = email) + raise forms.ValidationError(self.error_messsages['taken']) + except User.DoesNotExist: + return email + except User.MultipleObjectsReturned: + raise forms.ValidationError(self.error_messages['taken']) + else: + return email + +class SetPasswordForm(forms.Form): + password1 = forms.CharField(widget=forms.PasswordInput(attrs=login_form_widget_attrs), + label=_('choose password'), + error_messages={'required':_('password is required')}, + ) + password2 = forms.CharField(widget=forms.PasswordInput(attrs=login_form_widget_attrs), + label=mark_safe(_('retype password')), + error_messages={'required':_('please, retype your password'), + 'nomatch':_('sorry, entered passwords did not match, please try again')}, + ) + def clean_password2(self): + """ + Validates that the two password inputs match. + + """ + if 'password1' in self.cleaned_data: + if self.cleaned_data['password1'] == self.cleaned_data['password2']: + self.password = self.cleaned_data['password2'] + self.cleaned_data['password'] = self.cleaned_data['password2'] + return self.cleaned_data['password2'] + else: + del self.cleaned_data['password2'] + raise forms.ValidationError(self.fields['password2'].error_messages['nomatch']) + else: + return self.cleaned_data['password2'] + diff --git a/forum/utils/html.py b/forum/utils/html.py new file mode 100644 index 00000000..25a74a4a --- /dev/null +++ b/forum/utils/html.py @@ -0,0 +1,51 @@ +"""Utilities for working with HTML.""" +import html5lib +from html5lib import sanitizer, serializer, tokenizer, treebuilders, treewalkers + +class HTMLSanitizerMixin(sanitizer.HTMLSanitizerMixin): + acceptable_elements = ('a', 'abbr', 'acronym', 'address', 'b', 'big', + 'blockquote', 'br', 'caption', 'center', 'cite', 'code', 'col', + 'colgroup', 'dd', 'del', 'dfn', 'dir', 'div', 'dl', 'dt', 'em', 'font', + 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'hr', 'i', 'img', 'ins', 'kbd', + 'li', 'ol', 'p', 'pre', 'q', 's', 'samp', 'small', 'span', 'strike', + 'strong', 'sub', 'sup', 'table', 'tbody', 'td', 'tfoot', 'th', 'thead', + 'tr', 'tt', 'u', 'ul', 'var') + + acceptable_attributes = ('abbr', 'align', 'alt', 'axis', 'border', + 'cellpadding', 'cellspacing', 'char', 'charoff', 'charset', 'cite', + 'cols', 'colspan', 'datetime', 'dir', 'frame', 'headers', 'height', + 'href', 'hreflang', 'hspace', 'lang', 'longdesc', 'name', 'nohref', + 'noshade', 'nowrap', 'rel', 'rev', 'rows', 'rowspan', 'rules', 'scope', + 'span', 'src', 'start', 'summary', 'title', 'type', 'valign', 'vspace', + 'width') + + allowed_elements = acceptable_elements + allowed_attributes = acceptable_attributes + allowed_css_properties = () + allowed_css_keywords = () + allowed_svg_properties = () + +class HTMLSanitizer(tokenizer.HTMLTokenizer, HTMLSanitizerMixin): + def __init__(self, stream, encoding=None, parseMeta=True, useChardet=True, + lowercaseElementName=True, lowercaseAttrName=True): + tokenizer.HTMLTokenizer.__init__(self, stream, encoding, parseMeta, + useChardet, lowercaseElementName, + lowercaseAttrName) + + def __iter__(self): + for token in tokenizer.HTMLTokenizer.__iter__(self): + token = self.sanitize_token(token) + if token: + yield token + +def sanitize_html(html): + """Sanitizes an HTML fragment.""" + p = html5lib.HTMLParser(tokenizer=HTMLSanitizer, + tree=treebuilders.getTreeBuilder("dom")) + dom_tree = p.parseFragment(html) + walker = treewalkers.getTreeWalker("dom") + stream = walker(dom_tree) + s = serializer.HTMLSerializer(omit_optional_tags=False, + quote_attr_values=True) + output_generator = s.serialize(stream) + return u''.join(output_generator) diff --git a/forum/utils/lists.py b/forum/utils/lists.py new file mode 100644 index 00000000..bbcfae98 --- /dev/null +++ b/forum/utils/lists.py @@ -0,0 +1,86 @@ +"""Utilities for working with lists and sequences.""" + +def flatten(x): + """ + Returns a single, flat list which contains all elements retrieved + from the sequence and all recursively contained sub-sequences + (iterables). + + Examples: + >>> [1, 2, [3, 4], (5, 6)] + [1, 2, [3, 4], (5, 6)] + + From http://kogs-www.informatik.uni-hamburg.de/~meine/python_tricks + """ + result = [] + for el in x: + if hasattr(el, '__iter__') and not isinstance(el, basestring): + result.extend(flatten(el)) + else: + result.append(el) + return result + +def batch_size(items, size): + """ + Retrieves items in batches of the given size. + + >>> l = range(1, 11) + >>> batch_size(l, 3) + [[1, 2, 3], [4, 5, 6], [7, 8, 9], [10]] + >>> batch_size(l, 5) + [[1, 2, 3, 4, 5], [6, 7, 8, 9, 10]] + """ + return [items[i:i+size] for i in xrange(0, len(items), size)] + +def batches(items, number): + """ + Retrieves items in the given number of batches. + + >>> l = range(1, 11) + >>> batches(l, 1) + [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]] + >>> batches(l, 2) + [[1, 2, 3, 4, 5], [6, 7, 8, 9, 10]] + >>> batches(l, 3) + [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10]] + >>> batches(l, 4) + [[1, 2, 3], [4, 5, 6], [7, 8, 9], [10]] + >>> batches(l, 5) + [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]] + + Initial batches will contain as many items as possible in cases where + there are not enough items to be distributed evenly. + + >>> batches(l, 6) + [[1, 2], [3, 4], [5, 6], [7, 8], [9], [10]] + >>> batches(l, 7) + [[1, 2], [3, 4], [5, 6], [7], [8], [9], [10]] + >>> batches(l, 8) + [[1, 2], [3, 4], [5], [6], [7], [8], [9], [10]] + >>> batches(l, 9) + [[1, 2], [3], [4], [5], [6], [7], [8], [9], [10]] + >>> batches(l, 10) + [[1], [2], [3], [4], [5], [6], [7], [8], [9], [10]] + + If there are more batches than items, empty batches will be appended + to the batch list. + + >>> batches(l, 11) + [[1], [2], [3], [4], [5], [6], [7], [8], [9], [10], []] + >>> batches(l, 12) + [[1], [2], [3], [4], [5], [6], [7], [8], [9], [10], [], []] + """ + div, mod= divmod(len(items), number) + if div > 1: + if mod: + div += 1 + return batch_size(items, div) + else: + if not div: + return [[item] for item in items] + [[]] * (number - mod) + elif div == 1 and not mod: + return [[item] for item in items] + else: + # mod now tells you how many lists of 2 you can fit in + return ([items[i*2:(i*2)+2] for i in xrange(0, mod)] + + [[item] for item in items[mod*2:]]) diff --git a/forum/utils/odict.py b/forum/utils/odict.py new file mode 100644 index 00000000..2c8391d7 --- /dev/null +++ b/forum/utils/odict.py @@ -0,0 +1,1399 @@ +# odict.py +# An Ordered Dictionary object +# Copyright (C) 2005 Nicola Larosa, Michael Foord +# E-mail: nico AT tekNico DOT net, fuzzyman AT voidspace DOT org DOT uk + +# This software is licensed under the terms of the BSD license. +# http://www.voidspace.org.uk/python/license.shtml +# Basically you're free to copy, modify, distribute and relicense it, +# So long as you keep a copy of the license with it. + +# Documentation at http://www.voidspace.org.uk/python/odict.html +# For information about bugfixes, updates and support, please join the +# Pythonutils mailing list: +# http://groups.google.com/group/pythonutils/ +# Comments, suggestions and bug reports welcome. + +"""A dict that keeps keys in insertion order""" +from __future__ import generators + +__author__ = ('Nicola Larosa ,' + 'Michael Foord ') + +__docformat__ = "restructuredtext en" + +__revision__ = '$Id: odict.py 129 2005-09-12 18:15:28Z teknico $' + +__version__ = '0.2.2' + +__all__ = ['OrderedDict', 'SequenceOrderedDict'] + +import sys +INTP_VER = sys.version_info[:2] +if INTP_VER < (2, 2): + raise RuntimeError("Python v.2.2 or later required") + +import types, warnings + +class OrderedDict(dict): + """ + A class of dictionary that keeps the insertion order of keys. + + All appropriate methods return keys, items, or values in an ordered way. + + All normal dictionary methods are available. Update and comparison is + restricted to other OrderedDict objects. + + Various sequence methods are available, including the ability to explicitly + mutate the key ordering. + + __contains__ tests: + + >>> d = OrderedDict(((1, 3),)) + >>> 1 in d + 1 + >>> 4 in d + 0 + + __getitem__ tests: + + >>> OrderedDict(((1, 3), (3, 2), (2, 1)))[2] + 1 + >>> OrderedDict(((1, 3), (3, 2), (2, 1)))[4] + Traceback (most recent call last): + KeyError: 4 + + __len__ tests: + + >>> len(OrderedDict()) + 0 + >>> len(OrderedDict(((1, 3), (3, 2), (2, 1)))) + 3 + + get tests: + + >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) + >>> d.get(1) + 3 + >>> d.get(4) is None + 1 + >>> d.get(4, 5) + 5 + >>> d + OrderedDict([(1, 3), (3, 2), (2, 1)]) + + has_key tests: + + >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) + >>> d.has_key(1) + 1 + >>> d.has_key(4) + 0 + """ + + def __init__(self, init_val=(), strict=False): + """ + Create a new ordered dictionary. Cannot init from a normal dict, + nor from kwargs, since items order is undefined in those cases. + + If the ``strict`` keyword argument is ``True`` (``False`` is the + default) then when doing slice assignment - the ``OrderedDict`` you are + assigning from *must not* contain any keys in the remaining dict. + + >>> OrderedDict() + OrderedDict([]) + >>> OrderedDict({1: 1}) + Traceback (most recent call last): + TypeError: undefined order, cannot get items from dict + >>> OrderedDict({1: 1}.items()) + OrderedDict([(1, 1)]) + >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) + >>> d + OrderedDict([(1, 3), (3, 2), (2, 1)]) + >>> OrderedDict(d) + OrderedDict([(1, 3), (3, 2), (2, 1)]) + """ + self.strict = strict + dict.__init__(self) + if isinstance(init_val, OrderedDict): + self._sequence = init_val.keys() + dict.update(self, init_val) + elif isinstance(init_val, dict): + # we lose compatibility with other ordered dict types this way + raise TypeError('undefined order, cannot get items from dict') + else: + self._sequence = [] + self.update(init_val) + +### Special methods ### + + def __delitem__(self, key): + """ + >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) + >>> del d[3] + >>> d + OrderedDict([(1, 3), (2, 1)]) + >>> del d[3] + Traceback (most recent call last): + KeyError: 3 + >>> d[3] = 2 + >>> d + OrderedDict([(1, 3), (2, 1), (3, 2)]) + >>> del d[0:1] + >>> d + OrderedDict([(2, 1), (3, 2)]) + """ + if isinstance(key, types.SliceType): + # FIXME: efficiency? + keys = self._sequence[key] + for entry in keys: + dict.__delitem__(self, entry) + del self._sequence[key] + else: + # do the dict.__delitem__ *first* as it raises + # the more appropriate error + dict.__delitem__(self, key) + self._sequence.remove(key) + + def __eq__(self, other): + """ + >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) + >>> d == OrderedDict(d) + True + >>> d == OrderedDict(((1, 3), (2, 1), (3, 2))) + False + >>> d == OrderedDict(((1, 0), (3, 2), (2, 1))) + False + >>> d == OrderedDict(((0, 3), (3, 2), (2, 1))) + False + >>> d == dict(d) + False + >>> d == False + False + """ + if isinstance(other, OrderedDict): + # FIXME: efficiency? + # Generate both item lists for each compare + return (self.items() == other.items()) + else: + return False + + def __lt__(self, other): + """ + >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) + >>> c = OrderedDict(((0, 3), (3, 2), (2, 1))) + >>> c < d + True + >>> d < c + False + >>> d < dict(c) + Traceback (most recent call last): + TypeError: Can only compare with other OrderedDicts + """ + if not isinstance(other, OrderedDict): + raise TypeError('Can only compare with other OrderedDicts') + # FIXME: efficiency? + # Generate both item lists for each compare + return (self.items() < other.items()) + + def __le__(self, other): + """ + >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) + >>> c = OrderedDict(((0, 3), (3, 2), (2, 1))) + >>> e = OrderedDict(d) + >>> c <= d + True + >>> d <= c + False + >>> d <= dict(c) + Traceback (most recent call last): + TypeError: Can only compare with other OrderedDicts + >>> d <= e + True + """ + if not isinstance(other, OrderedDict): + raise TypeError('Can only compare with other OrderedDicts') + # FIXME: efficiency? + # Generate both item lists for each compare + return (self.items() <= other.items()) + + def __ne__(self, other): + """ + >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) + >>> d != OrderedDict(d) + False + >>> d != OrderedDict(((1, 3), (2, 1), (3, 2))) + True + >>> d != OrderedDict(((1, 0), (3, 2), (2, 1))) + True + >>> d == OrderedDict(((0, 3), (3, 2), (2, 1))) + False + >>> d != dict(d) + True + >>> d != False + True + """ + if isinstance(other, OrderedDict): + # FIXME: efficiency? + # Generate both item lists for each compare + return not (self.items() == other.items()) + else: + return True + + def __gt__(self, other): + """ + >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) + >>> c = OrderedDict(((0, 3), (3, 2), (2, 1))) + >>> d > c + True + >>> c > d + False + >>> d > dict(c) + Traceback (most recent call last): + TypeError: Can only compare with other OrderedDicts + """ + if not isinstance(other, OrderedDict): + raise TypeError('Can only compare with other OrderedDicts') + # FIXME: efficiency? + # Generate both item lists for each compare + return (self.items() > other.items()) + + def __ge__(self, other): + """ + >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) + >>> c = OrderedDict(((0, 3), (3, 2), (2, 1))) + >>> e = OrderedDict(d) + >>> c >= d + False + >>> d >= c + True + >>> d >= dict(c) + Traceback (most recent call last): + TypeError: Can only compare with other OrderedDicts + >>> e >= d + True + """ + if not isinstance(other, OrderedDict): + raise TypeError('Can only compare with other OrderedDicts') + # FIXME: efficiency? + # Generate both item lists for each compare + return (self.items() >= other.items()) + + def __repr__(self): + """ + Used for __repr__ and __str__ + + >>> r1 = repr(OrderedDict((('a', 'b'), ('c', 'd'), ('e', 'f')))) + >>> r1 + "OrderedDict([('a', 'b'), ('c', 'd'), ('e', 'f')])" + >>> r2 = repr(OrderedDict((('a', 'b'), ('e', 'f'), ('c', 'd')))) + >>> r2 + "OrderedDict([('a', 'b'), ('e', 'f'), ('c', 'd')])" + >>> r1 == str(OrderedDict((('a', 'b'), ('c', 'd'), ('e', 'f')))) + True + >>> r2 == str(OrderedDict((('a', 'b'), ('e', 'f'), ('c', 'd')))) + True + """ + return '%s([%s])' % (self.__class__.__name__, ', '.join( + ['(%r, %r)' % (key, self[key]) for key in self._sequence])) + + def __setitem__(self, key, val): + """ + Allows slice assignment, so long as the slice is an OrderedDict + >>> d = OrderedDict() + >>> d['a'] = 'b' + >>> d['b'] = 'a' + >>> d[3] = 12 + >>> d + OrderedDict([('a', 'b'), ('b', 'a'), (3, 12)]) + >>> d[:] = OrderedDict(((1, 2), (2, 3), (3, 4))) + >>> d + OrderedDict([(1, 2), (2, 3), (3, 4)]) + >>> d[::2] = OrderedDict(((7, 8), (9, 10))) + >>> d + OrderedDict([(7, 8), (2, 3), (9, 10)]) + >>> d = OrderedDict(((0, 1), (1, 2), (2, 3), (3, 4))) + >>> d[1:3] = OrderedDict(((1, 2), (5, 6), (7, 8))) + >>> d + OrderedDict([(0, 1), (1, 2), (5, 6), (7, 8), (3, 4)]) + >>> d = OrderedDict(((0, 1), (1, 2), (2, 3), (3, 4)), strict=True) + >>> d[1:3] = OrderedDict(((1, 2), (5, 6), (7, 8))) + >>> d + OrderedDict([(0, 1), (1, 2), (5, 6), (7, 8), (3, 4)]) + + >>> a = OrderedDict(((0, 1), (1, 2), (2, 3)), strict=True) + >>> a[3] = 4 + >>> a + OrderedDict([(0, 1), (1, 2), (2, 3), (3, 4)]) + >>> a[::1] = OrderedDict([(0, 1), (1, 2), (2, 3), (3, 4)]) + >>> a + OrderedDict([(0, 1), (1, 2), (2, 3), (3, 4)]) + >>> a[:2] = OrderedDict([(0, 1), (1, 2), (2, 3), (3, 4), (4, 5)]) + Traceback (most recent call last): + ValueError: slice assignment must be from unique keys + >>> a = OrderedDict(((0, 1), (1, 2), (2, 3))) + >>> a[3] = 4 + >>> a + OrderedDict([(0, 1), (1, 2), (2, 3), (3, 4)]) + >>> a[::1] = OrderedDict([(0, 1), (1, 2), (2, 3), (3, 4)]) + >>> a + OrderedDict([(0, 1), (1, 2), (2, 3), (3, 4)]) + >>> a[:2] = OrderedDict([(0, 1), (1, 2), (2, 3), (3, 4)]) + >>> a + OrderedDict([(0, 1), (1, 2), (2, 3), (3, 4)]) + >>> a[::-1] = OrderedDict([(0, 1), (1, 2), (2, 3), (3, 4)]) + >>> a + OrderedDict([(3, 4), (2, 3), (1, 2), (0, 1)]) + + >>> d = OrderedDict([(0, 1), (1, 2), (2, 3), (3, 4)]) + >>> d[:1] = 3 + Traceback (most recent call last): + TypeError: slice assignment requires an OrderedDict + + >>> d = OrderedDict([(0, 1), (1, 2), (2, 3), (3, 4)]) + >>> d[:1] = OrderedDict([(9, 8)]) + >>> d + OrderedDict([(9, 8), (1, 2), (2, 3), (3, 4)]) + """ + if isinstance(key, types.SliceType): + if not isinstance(val, OrderedDict): + # FIXME: allow a list of tuples? + raise TypeError('slice assignment requires an OrderedDict') + keys = self._sequence[key] + # NOTE: Could use ``range(*key.indices(len(self._sequence)))`` + indexes = range(len(self._sequence))[key] + if key.step is None: + # NOTE: new slice may not be the same size as the one being + # overwritten ! + # NOTE: What is the algorithm for an impossible slice? + # e.g. d[5:3] + pos = key.start or 0 + del self[key] + newkeys = val.keys() + for k in newkeys: + if k in self: + if self.strict: + raise ValueError('slice assignment must be from ' + 'unique keys') + else: + # NOTE: This removes duplicate keys *first* + # so start position might have changed? + del self[k] + self._sequence = (self._sequence[:pos] + newkeys + + self._sequence[pos:]) + dict.update(self, val) + else: + # extended slice - length of new slice must be the same + # as the one being replaced + if len(keys) != len(val): + raise ValueError('attempt to assign sequence of size %s ' + 'to extended slice of size %s' % (len(val), len(keys))) + # FIXME: efficiency? + del self[key] + item_list = zip(indexes, val.items()) + # smallest indexes first - higher indexes not guaranteed to + # exist + item_list.sort() + for pos, (newkey, newval) in item_list: + if self.strict and newkey in self: + raise ValueError('slice assignment must be from unique' + ' keys') + self.insert(pos, newkey, newval) + else: + if key not in self: + self._sequence.append(key) + dict.__setitem__(self, key, val) + + def __getitem__(self, key): + """ + Allows slicing. Returns an OrderedDict if you slice. + >>> b = OrderedDict([(7, 0), (6, 1), (5, 2), (4, 3), (3, 4), (2, 5), (1, 6)]) + >>> b[::-1] + OrderedDict([(1, 6), (2, 5), (3, 4), (4, 3), (5, 2), (6, 1), (7, 0)]) + >>> b[2:5] + OrderedDict([(5, 2), (4, 3), (3, 4)]) + >>> type(b[2:4]) + + """ + if isinstance(key, types.SliceType): + # FIXME: does this raise the error we want? + keys = self._sequence[key] + # FIXME: efficiency? + return OrderedDict([(entry, self[entry]) for entry in keys]) + else: + return dict.__getitem__(self, key) + + __str__ = __repr__ + + def __setattr__(self, name, value): + """ + Implemented so that accesses to ``sequence`` raise a warning and are + diverted to the new ``setkeys`` method. + """ + if name == 'sequence': + warnings.warn('Use of the sequence attribute is deprecated.' + ' Use the keys method instead.', DeprecationWarning) + # NOTE: doesn't return anything + self.setkeys(value) + else: + # FIXME: do we want to allow arbitrary setting of attributes? + # Or do we want to manage it? + object.__setattr__(self, name, value) + + def __getattr__(self, name): + """ + Implemented so that access to ``sequence`` raises a warning. + + >>> d = OrderedDict() + >>> d.sequence + [] + """ + if name == 'sequence': + warnings.warn('Use of the sequence attribute is deprecated.' + ' Use the keys method instead.', DeprecationWarning) + # NOTE: Still (currently) returns a direct reference. Need to + # because code that uses sequence will expect to be able to + # mutate it in place. + return self._sequence + else: + # raise the appropriate error + raise AttributeError("OrderedDict has no '%s' attribute" % name) + + def __deepcopy__(self, memo): + """ + To allow deepcopy to work with OrderedDict. + + >>> from copy import deepcopy + >>> a = OrderedDict([(1, 1), (2, 2), (3, 3)]) + >>> a['test'] = {} + >>> b = deepcopy(a) + >>> b == a + True + >>> b is a + False + >>> a['test'] is b['test'] + False + """ + from copy import deepcopy + return self.__class__(deepcopy(self.items(), memo), self.strict) + + +### Read-only methods ### + + def copy(self): + """ + >>> OrderedDict(((1, 3), (3, 2), (2, 1))).copy() + OrderedDict([(1, 3), (3, 2), (2, 1)]) + """ + return OrderedDict(self) + + def items(self): + """ + ``items`` returns a list of tuples representing all the + ``(key, value)`` pairs in the dictionary. + + >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) + >>> d.items() + [(1, 3), (3, 2), (2, 1)] + >>> d.clear() + >>> d.items() + [] + """ + return zip(self._sequence, self.values()) + + def keys(self): + """ + Return a list of keys in the ``OrderedDict``. + + >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) + >>> d.keys() + [1, 3, 2] + """ + return self._sequence[:] + + def values(self, values=None): + """ + Return a list of all the values in the OrderedDict. + + Optionally you can pass in a list of values, which will replace the + current list. The value list must be the same len as the OrderedDict. + + >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) + >>> d.values() + [3, 2, 1] + """ + return [self[key] for key in self._sequence] + + def iteritems(self): + """ + >>> ii = OrderedDict(((1, 3), (3, 2), (2, 1))).iteritems() + >>> ii.next() + (1, 3) + >>> ii.next() + (3, 2) + >>> ii.next() + (2, 1) + >>> ii.next() + Traceback (most recent call last): + StopIteration + """ + def make_iter(self=self): + keys = self.iterkeys() + while True: + key = keys.next() + yield (key, self[key]) + return make_iter() + + def iterkeys(self): + """ + >>> ii = OrderedDict(((1, 3), (3, 2), (2, 1))).iterkeys() + >>> ii.next() + 1 + >>> ii.next() + 3 + >>> ii.next() + 2 + >>> ii.next() + Traceback (most recent call last): + StopIteration + """ + return iter(self._sequence) + + __iter__ = iterkeys + + def itervalues(self): + """ + >>> iv = OrderedDict(((1, 3), (3, 2), (2, 1))).itervalues() + >>> iv.next() + 3 + >>> iv.next() + 2 + >>> iv.next() + 1 + >>> iv.next() + Traceback (most recent call last): + StopIteration + """ + def make_iter(self=self): + keys = self.iterkeys() + while True: + yield self[keys.next()] + return make_iter() + +### Read-write methods ### + + def clear(self): + """ + >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) + >>> d.clear() + >>> d + OrderedDict([]) + """ + dict.clear(self) + self._sequence = [] + + def pop(self, key, *args): + """ + No dict.pop in Python 2.2, gotta reimplement it + + >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) + >>> d.pop(3) + 2 + >>> d + OrderedDict([(1, 3), (2, 1)]) + >>> d.pop(4) + Traceback (most recent call last): + KeyError: 4 + >>> d.pop(4, 0) + 0 + >>> d.pop(4, 0, 1) + Traceback (most recent call last): + TypeError: pop expected at most 2 arguments, got 3 + """ + if len(args) > 1: + raise TypeError, ('pop expected at most 2 arguments, got %s' % + (len(args) + 1)) + if key in self: + val = self[key] + del self[key] + else: + try: + val = args[0] + except IndexError: + raise KeyError(key) + return val + + def popitem(self, i=-1): + """ + Delete and return an item specified by index, not a random one as in + dict. The index is -1 by default (the last item). + + >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) + >>> d.popitem() + (2, 1) + >>> d + OrderedDict([(1, 3), (3, 2)]) + >>> d.popitem(0) + (1, 3) + >>> OrderedDict().popitem() + Traceback (most recent call last): + KeyError: 'popitem(): dictionary is empty' + >>> d.popitem(2) + Traceback (most recent call last): + IndexError: popitem(): index 2 not valid + """ + if not self._sequence: + raise KeyError('popitem(): dictionary is empty') + try: + key = self._sequence[i] + except IndexError: + raise IndexError('popitem(): index %s not valid' % i) + return (key, self.pop(key)) + + def setdefault(self, key, defval = None): + """ + >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) + >>> d.setdefault(1) + 3 + >>> d.setdefault(4) is None + True + >>> d + OrderedDict([(1, 3), (3, 2), (2, 1), (4, None)]) + >>> d.setdefault(5, 0) + 0 + >>> d + OrderedDict([(1, 3), (3, 2), (2, 1), (4, None), (5, 0)]) + """ + if key in self: + return self[key] + else: + self[key] = defval + return defval + + def update(self, from_od): + """ + Update from another OrderedDict or sequence of (key, value) pairs + + >>> d = OrderedDict(((1, 0), (0, 1))) + >>> d.update(OrderedDict(((1, 3), (3, 2), (2, 1)))) + >>> d + OrderedDict([(1, 3), (0, 1), (3, 2), (2, 1)]) + >>> d.update({4: 4}) + Traceback (most recent call last): + TypeError: undefined order, cannot get items from dict + >>> d.update((4, 4)) + Traceback (most recent call last): + TypeError: cannot convert dictionary update sequence element "4" to a 2-item sequence + """ + if isinstance(from_od, OrderedDict): + for key, val in from_od.items(): + self[key] = val + elif isinstance(from_od, dict): + # we lose compatibility with other ordered dict types this way + raise TypeError('undefined order, cannot get items from dict') + else: + # FIXME: efficiency? + # sequence of 2-item sequences, or error + for item in from_od: + try: + key, val = item + except TypeError: + raise TypeError('cannot convert dictionary update' + ' sequence element "%s" to a 2-item sequence' % item) + self[key] = val + + def rename(self, old_key, new_key): + """ + Rename the key for a given value, without modifying sequence order. + + For the case where new_key already exists this raise an exception, + since if new_key exists, it is ambiguous as to what happens to the + associated values, and the position of new_key in the sequence. + + >>> od = OrderedDict() + >>> od['a'] = 1 + >>> od['b'] = 2 + >>> od.items() + [('a', 1), ('b', 2)] + >>> od.rename('b', 'c') + >>> od.items() + [('a', 1), ('c', 2)] + >>> od.rename('c', 'a') + Traceback (most recent call last): + ValueError: New key already exists: 'a' + >>> od.rename('d', 'b') + Traceback (most recent call last): + KeyError: 'd' + """ + if new_key == old_key: + # no-op + return + if new_key in self: + raise ValueError("New key already exists: %r" % new_key) + # rename sequence entry + value = self[old_key] + old_idx = self._sequence.index(old_key) + self._sequence[old_idx] = new_key + # rename internal dict entry + dict.__delitem__(self, old_key) + dict.__setitem__(self, new_key, value) + + def setitems(self, items): + """ + This method allows you to set the items in the dict. + + It takes a list of tuples - of the same sort returned by the ``items`` + method. + + >>> d = OrderedDict() + >>> d.setitems(((3, 1), (2, 3), (1, 2))) + >>> d + OrderedDict([(3, 1), (2, 3), (1, 2)]) + """ + self.clear() + # FIXME: this allows you to pass in an OrderedDict as well :-) + self.update(items) + + def setkeys(self, keys): + """ + ``setkeys`` all ows you to pass in a new list of keys which will + replace the current set. This must contain the same set of keys, but + need not be in the same order. + + If you pass in new keys that don't match, a ``KeyError`` will be + raised. + + >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) + >>> d.keys() + [1, 3, 2] + >>> d.setkeys((1, 2, 3)) + >>> d + OrderedDict([(1, 3), (2, 1), (3, 2)]) + >>> d.setkeys(['a', 'b', 'c']) + Traceback (most recent call last): + KeyError: 'Keylist is not the same as current keylist.' + """ + # FIXME: Efficiency? (use set for Python 2.4 :-) + # NOTE: list(keys) rather than keys[:] because keys[:] returns + # a tuple, if keys is a tuple. + kcopy = list(keys) + kcopy.sort() + self._sequence.sort() + if kcopy != self._sequence: + raise KeyError('Keylist is not the same as current keylist.') + # NOTE: This makes the _sequence attribute a new object, instead + # of changing it in place. + # FIXME: efficiency? + self._sequence = list(keys) + + def setvalues(self, values): + """ + You can pass in a list of values, which will replace the + current list. The value list must be the same len as the OrderedDict. + + (Or a ``ValueError`` is raised.) + + >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) + >>> d.setvalues((1, 2, 3)) + >>> d + OrderedDict([(1, 1), (3, 2), (2, 3)]) + >>> d.setvalues([6]) + Traceback (most recent call last): + ValueError: Value list is not the same length as the OrderedDict. + """ + if len(values) != len(self): + # FIXME: correct error to raise? + raise ValueError('Value list is not the same length as the ' + 'OrderedDict.') + self.update(zip(self, values)) + +### Sequence Methods ### + + def index(self, key): + """ + Return the position of the specified key in the OrderedDict. + + >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) + >>> d.index(3) + 1 + >>> d.index(4) + Traceback (most recent call last): + ValueError: list.index(x): x not in list + """ + return self._sequence.index(key) + + def insert(self, index, key, value): + """ + Takes ``index``, ``key``, and ``value`` as arguments. + + Sets ``key`` to ``value``, so that ``key`` is at position ``index`` in + the OrderedDict. + + >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) + >>> d.insert(0, 4, 0) + >>> d + OrderedDict([(4, 0), (1, 3), (3, 2), (2, 1)]) + >>> d.insert(0, 2, 1) + >>> d + OrderedDict([(2, 1), (4, 0), (1, 3), (3, 2)]) + >>> d.insert(8, 8, 1) + >>> d + OrderedDict([(2, 1), (4, 0), (1, 3), (3, 2), (8, 1)]) + """ + if key in self: + # FIXME: efficiency? + del self[key] + self._sequence.insert(index, key) + dict.__setitem__(self, key, value) + + def reverse(self): + """ + Reverse the order of the OrderedDict. + + >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) + >>> d.reverse() + >>> d + OrderedDict([(2, 1), (3, 2), (1, 3)]) + """ + self._sequence.reverse() + + def sort(self, *args, **kwargs): + """ + Sort the key order in the OrderedDict. + + This method takes the same arguments as the ``list.sort`` method on + your version of Python. + + >>> d = OrderedDict(((4, 1), (2, 2), (3, 3), (1, 4))) + >>> d.sort() + >>> d + OrderedDict([(1, 4), (2, 2), (3, 3), (4, 1)]) + """ + self._sequence.sort(*args, **kwargs) + +class Keys(object): + # FIXME: should this object be a subclass of list? + """ + Custom object for accessing the keys of an OrderedDict. + + Can be called like the normal ``OrderedDict.keys`` method, but also + supports indexing and sequence methods. + """ + + def __init__(self, main): + self._main = main + + def __call__(self): + """Pretend to be the keys method.""" + return self._main._keys() + + def __getitem__(self, index): + """Fetch the key at position i.""" + # NOTE: this automatically supports slicing :-) + return self._main._sequence[index] + + def __setitem__(self, index, name): + """ + You cannot assign to keys, but you can do slice assignment to re-order + them. + + You can only do slice assignment if the new set of keys is a reordering + of the original set. + """ + if isinstance(index, types.SliceType): + # FIXME: efficiency? + # check length is the same + indexes = range(len(self._main._sequence))[index] + if len(indexes) != len(name): + raise ValueError('attempt to assign sequence of size %s ' + 'to slice of size %s' % (len(name), len(indexes))) + # check they are the same keys + # FIXME: Use set + old_keys = self._main._sequence[index] + new_keys = list(name) + old_keys.sort() + new_keys.sort() + if old_keys != new_keys: + raise KeyError('Keylist is not the same as current keylist.') + orig_vals = [self._main[k] for k in name] + del self._main[index] + vals = zip(indexes, name, orig_vals) + vals.sort() + for i, k, v in vals: + if self._main.strict and k in self._main: + raise ValueError('slice assignment must be from ' + 'unique keys') + self._main.insert(i, k, v) + else: + raise ValueError('Cannot assign to keys') + + ### following methods pinched from UserList and adapted ### + def __repr__(self): return repr(self._main._sequence) + + # FIXME: do we need to check if we are comparing with another ``Keys`` + # object? (like the __cast method of UserList) + def __lt__(self, other): return self._main._sequence < other + def __le__(self, other): return self._main._sequence <= other + def __eq__(self, other): return self._main._sequence == other + def __ne__(self, other): return self._main._sequence != other + def __gt__(self, other): return self._main._sequence > other + def __ge__(self, other): return self._main._sequence >= other + # FIXME: do we need __cmp__ as well as rich comparisons? + def __cmp__(self, other): return cmp(self._main._sequence, other) + + def __contains__(self, item): return item in self._main._sequence + def __len__(self): return len(self._main._sequence) + def __iter__(self): return self._main.iterkeys() + def count(self, item): return self._main._sequence.count(item) + def index(self, item, *args): return self._main._sequence.index(item, *args) + def reverse(self): self._main._sequence.reverse() + def sort(self, *args, **kwds): self._main._sequence.sort(*args, **kwds) + def __mul__(self, n): return self._main._sequence*n + __rmul__ = __mul__ + def __add__(self, other): return self._main._sequence + other + def __radd__(self, other): return other + self._main._sequence + + ## following methods not implemented for keys ## + def __delitem__(self, i): raise TypeError('Can\'t delete items from keys') + def __iadd__(self, other): raise TypeError('Can\'t add in place to keys') + def __imul__(self, n): raise TypeError('Can\'t multiply keys in place') + def append(self, item): raise TypeError('Can\'t append items to keys') + def insert(self, i, item): raise TypeError('Can\'t insert items into keys') + def pop(self, i=-1): raise TypeError('Can\'t pop items from keys') + def remove(self, item): raise TypeError('Can\'t remove items from keys') + def extend(self, other): raise TypeError('Can\'t extend keys') + +class Items(object): + """ + Custom object for accessing the items of an OrderedDict. + + Can be called like the normal ``OrderedDict.items`` method, but also + supports indexing and sequence methods. + """ + + def __init__(self, main): + self._main = main + + def __call__(self): + """Pretend to be the items method.""" + return self._main._items() + + def __getitem__(self, index): + """Fetch the item at position i.""" + if isinstance(index, types.SliceType): + # fetching a slice returns an OrderedDict + return self._main[index].items() + key = self._main._sequence[index] + return (key, self._main[key]) + + def __setitem__(self, index, item): + """Set item at position i to item.""" + if isinstance(index, types.SliceType): + # NOTE: item must be an iterable (list of tuples) + self._main[index] = OrderedDict(item) + else: + # FIXME: Does this raise a sensible error? + orig = self._main.keys[index] + key, value = item + if self._main.strict and key in self and (key != orig): + raise ValueError('slice assignment must be from ' + 'unique keys') + # delete the current one + del self._main[self._main._sequence[index]] + self._main.insert(index, key, value) + + def __delitem__(self, i): + """Delete the item at position i.""" + key = self._main._sequence[i] + if isinstance(i, types.SliceType): + for k in key: + # FIXME: efficiency? + del self._main[k] + else: + del self._main[key] + + ### following methods pinched from UserList and adapted ### + def __repr__(self): return repr(self._main.items()) + + # FIXME: do we need to check if we are comparing with another ``Items`` + # object? (like the __cast method of UserList) + def __lt__(self, other): return self._main.items() < other + def __le__(self, other): return self._main.items() <= other + def __eq__(self, other): return self._main.items() == other + def __ne__(self, other): return self._main.items() != other + def __gt__(self, other): return self._main.items() > other + def __ge__(self, other): return self._main.items() >= other + def __cmp__(self, other): return cmp(self._main.items(), other) + + def __contains__(self, item): return item in self._main.items() + def __len__(self): return len(self._main._sequence) # easier :-) + def __iter__(self): return self._main.iteritems() + def count(self, item): return self._main.items().count(item) + def index(self, item, *args): return self._main.items().index(item, *args) + def reverse(self): self._main.reverse() + def sort(self, *args, **kwds): self._main.sort(*args, **kwds) + def __mul__(self, n): return self._main.items()*n + __rmul__ = __mul__ + def __add__(self, other): return self._main.items() + other + def __radd__(self, other): return other + self._main.items() + + def append(self, item): + """Add an item to the end.""" + # FIXME: this is only append if the key isn't already present + key, value = item + self._main[key] = value + + def insert(self, i, item): + key, value = item + self._main.insert(i, key, value) + + def pop(self, i=-1): + key = self._main._sequence[i] + return (key, self._main.pop(key)) + + def remove(self, item): + key, value = item + try: + assert value == self._main[key] + except (KeyError, AssertionError): + raise ValueError('ValueError: list.remove(x): x not in list') + else: + del self._main[key] + + def extend(self, other): + # FIXME: is only a true extend if none of the keys already present + for item in other: + key, value = item + self._main[key] = value + + def __iadd__(self, other): + self.extend(other) + + ## following methods not implemented for items ## + + def __imul__(self, n): raise TypeError('Can\'t multiply items in place') + +class Values(object): + """ + Custom object for accessing the values of an OrderedDict. + + Can be called like the normal ``OrderedDict.values`` method, but also + supports indexing and sequence methods. + """ + + def __init__(self, main): + self._main = main + + def __call__(self): + """Pretend to be the values method.""" + return self._main._values() + + def __getitem__(self, index): + """Fetch the value at position i.""" + if isinstance(index, types.SliceType): + return [self._main[key] for key in self._main._sequence[index]] + else: + return self._main[self._main._sequence[index]] + + def __setitem__(self, index, value): + """ + Set the value at position i to value. + + You can only do slice assignment to values if you supply a sequence of + equal length to the slice you are replacing. + """ + if isinstance(index, types.SliceType): + keys = self._main._sequence[index] + if len(keys) != len(value): + raise ValueError('attempt to assign sequence of size %s ' + 'to slice of size %s' % (len(name), len(keys))) + # FIXME: efficiency? Would be better to calculate the indexes + # directly from the slice object + # NOTE: the new keys can collide with existing keys (or even + # contain duplicates) - these will overwrite + for key, val in zip(keys, value): + self._main[key] = val + else: + self._main[self._main._sequence[index]] = value + + ### following methods pinched from UserList and adapted ### + def __repr__(self): return repr(self._main.values()) + + # FIXME: do we need to check if we are comparing with another ``Values`` + # object? (like the __cast method of UserList) + def __lt__(self, other): return self._main.values() < other + def __le__(self, other): return self._main.values() <= other + def __eq__(self, other): return self._main.values() == other + def __ne__(self, other): return self._main.values() != other + def __gt__(self, other): return self._main.values() > other + def __ge__(self, other): return self._main.values() >= other + def __cmp__(self, other): return cmp(self._main.values(), other) + + def __contains__(self, item): return item in self._main.values() + def __len__(self): return len(self._main._sequence) # easier :-) + def __iter__(self): return self._main.itervalues() + def count(self, item): return self._main.values().count(item) + def index(self, item, *args): return self._main.values().index(item, *args) + + def reverse(self): + """Reverse the values""" + vals = self._main.values() + vals.reverse() + # FIXME: efficiency + self[:] = vals + + def sort(self, *args, **kwds): + """Sort the values.""" + vals = self._main.values() + vals.sort(*args, **kwds) + self[:] = vals + + def __mul__(self, n): return self._main.values()*n + __rmul__ = __mul__ + def __add__(self, other): return self._main.values() + other + def __radd__(self, other): return other + self._main.values() + + ## following methods not implemented for values ## + def __delitem__(self, i): raise TypeError('Can\'t delete items from values') + def __iadd__(self, other): raise TypeError('Can\'t add in place to values') + def __imul__(self, n): raise TypeError('Can\'t multiply values in place') + def append(self, item): raise TypeError('Can\'t append items to values') + def insert(self, i, item): raise TypeError('Can\'t insert items into values') + def pop(self, i=-1): raise TypeError('Can\'t pop items from values') + def remove(self, item): raise TypeError('Can\'t remove items from values') + def extend(self, other): raise TypeError('Can\'t extend values') + +class SequenceOrderedDict(OrderedDict): + """ + Experimental version of OrderedDict that has a custom object for ``keys``, + ``values``, and ``items``. + + These are callable sequence objects that work as methods, or can be + manipulated directly as sequences. + + Test for ``keys``, ``items`` and ``values``. + + >>> d = SequenceOrderedDict(((1, 2), (2, 3), (3, 4))) + >>> d + SequenceOrderedDict([(1, 2), (2, 3), (3, 4)]) + >>> d.keys + [1, 2, 3] + >>> d.keys() + [1, 2, 3] + >>> d.setkeys((3, 2, 1)) + >>> d + SequenceOrderedDict([(3, 4), (2, 3), (1, 2)]) + >>> d.setkeys((1, 2, 3)) + >>> d.keys[0] + 1 + >>> d.keys[:] + [1, 2, 3] + >>> d.keys[-1] + 3 + >>> d.keys[-2] + 2 + >>> d.keys[0:2] = [2, 1] + >>> d + SequenceOrderedDict([(2, 3), (1, 2), (3, 4)]) + >>> d.keys.reverse() + >>> d.keys + [3, 1, 2] + >>> d.keys = [1, 2, 3] + >>> d + SequenceOrderedDict([(1, 2), (2, 3), (3, 4)]) + >>> d.keys = [3, 1, 2] + >>> d + SequenceOrderedDict([(3, 4), (1, 2), (2, 3)]) + >>> a = SequenceOrderedDict() + >>> b = SequenceOrderedDict() + >>> a.keys == b.keys + 1 + >>> a['a'] = 3 + >>> a.keys == b.keys + 0 + >>> b['a'] = 3 + >>> a.keys == b.keys + 1 + >>> b['b'] = 3 + >>> a.keys == b.keys + 0 + >>> a.keys > b.keys + 0 + >>> a.keys < b.keys + 1 + >>> 'a' in a.keys + 1 + >>> len(b.keys) + 2 + >>> 'c' in d.keys + 0 + >>> 1 in d.keys + 1 + >>> [v for v in d.keys] + [3, 1, 2] + >>> d.keys.sort() + >>> d.keys + [1, 2, 3] + >>> d = SequenceOrderedDict(((1, 2), (2, 3), (3, 4)), strict=True) + >>> d.keys[::-1] = [1, 2, 3] + >>> d + SequenceOrderedDict([(3, 4), (2, 3), (1, 2)]) + >>> d.keys[:2] + [3, 2] + >>> d.keys[:2] = [1, 3] + Traceback (most recent call last): + KeyError: 'Keylist is not the same as current keylist.' + + >>> d = SequenceOrderedDict(((1, 2), (2, 3), (3, 4))) + >>> d + SequenceOrderedDict([(1, 2), (2, 3), (3, 4)]) + >>> d.values + [2, 3, 4] + >>> d.values() + [2, 3, 4] + >>> d.setvalues((4, 3, 2)) + >>> d + SequenceOrderedDict([(1, 4), (2, 3), (3, 2)]) + >>> d.values[::-1] + [2, 3, 4] + >>> d.values[0] + 4 + >>> d.values[-2] + 3 + >>> del d.values[0] + Traceback (most recent call last): + TypeError: Can't delete items from values + >>> d.values[::2] = [2, 4] + >>> d + SequenceOrderedDict([(1, 2), (2, 3), (3, 4)]) + >>> 7 in d.values + 0 + >>> len(d.values) + 3 + >>> [val for val in d.values] + [2, 3, 4] + >>> d.values[-1] = 2 + >>> d.values.count(2) + 2 + >>> d.values.index(2) + 0 + >>> d.values[-1] = 7 + >>> d.values + [2, 3, 7] + >>> d.values.reverse() + >>> d.values + [7, 3, 2] + >>> d.values.sort() + >>> d.values + [2, 3, 7] + >>> d.values.append('anything') + Traceback (most recent call last): + TypeError: Can't append items to values + >>> d.values = (1, 2, 3) + >>> d + SequenceOrderedDict([(1, 1), (2, 2), (3, 3)]) + + >>> d = SequenceOrderedDict(((1, 2), (2, 3), (3, 4))) + >>> d + SequenceOrderedDict([(1, 2), (2, 3), (3, 4)]) + >>> d.items() + [(1, 2), (2, 3), (3, 4)] + >>> d.setitems([(3, 4), (2 ,3), (1, 2)]) + >>> d + SequenceOrderedDict([(3, 4), (2, 3), (1, 2)]) + >>> d.items[0] + (3, 4) + >>> d.items[:-1] + [(3, 4), (2, 3)] + >>> d.items[1] = (6, 3) + >>> d.items + [(3, 4), (6, 3), (1, 2)] + >>> d.items[1:2] = [(9, 9)] + >>> d + SequenceOrderedDict([(3, 4), (9, 9), (1, 2)]) + >>> del d.items[1:2] + >>> d + SequenceOrderedDict([(3, 4), (1, 2)]) + >>> (3, 4) in d.items + 1 + >>> (4, 3) in d.items + 0 + >>> len(d.items) + 2 + >>> [v for v in d.items] + [(3, 4), (1, 2)] + >>> d.items.count((3, 4)) + 1 + >>> d.items.index((1, 2)) + 1 + >>> d.items.index((2, 1)) + Traceback (most recent call last): + ValueError: list.index(x): x not in list + >>> d.items.reverse() + >>> d.items + [(1, 2), (3, 4)] + >>> d.items.reverse() + >>> d.items.sort() + >>> d.items + [(1, 2), (3, 4)] + >>> d.items.append((5, 6)) + >>> d.items + [(1, 2), (3, 4), (5, 6)] + >>> d.items.insert(0, (0, 0)) + >>> d.items + [(0, 0), (1, 2), (3, 4), (5, 6)] + >>> d.items.insert(-1, (7, 8)) + >>> d.items + [(0, 0), (1, 2), (3, 4), (7, 8), (5, 6)] + >>> d.items.pop() + (5, 6) + >>> d.items + [(0, 0), (1, 2), (3, 4), (7, 8)] + >>> d.items.remove((1, 2)) + >>> d.items + [(0, 0), (3, 4), (7, 8)] + >>> d.items.extend([(1, 2), (5, 6)]) + >>> d.items + [(0, 0), (3, 4), (7, 8), (1, 2), (5, 6)] + """ + + def __init__(self, init_val=(), strict=True): + OrderedDict.__init__(self, init_val, strict=strict) + self._keys = self.keys + self._values = self.values + self._items = self.items + self.keys = Keys(self) + self.values = Values(self) + self.items = Items(self) + self._att_dict = { + 'keys': self.setkeys, + 'items': self.setitems, + 'values': self.setvalues, + } + + def __setattr__(self, name, value): + """Protect keys, items, and values.""" + if not '_att_dict' in self.__dict__: + object.__setattr__(self, name, value) + else: + try: + fun = self._att_dict[name] + except KeyError: + OrderedDict.__setattr__(self, name, value) + else: + fun(value) + +if __name__ == '__main__': + if INTP_VER < (2, 3): + raise RuntimeError("Tests require Python v.2.3 or later") + # turn off warnings for tests + warnings.filterwarnings('ignore') + # run the code tests in doctest format + import doctest + m = sys.modules.get('__main__') + globs = m.__dict__.copy() + globs.update({ + 'INTP_VER': INTP_VER, + }) + doctest.testmod(m, globs=globs) + diff --git a/forum/views/commands.py b/forum/views/commands.py index 38915728..65121862 100644 --- a/forum/views/commands.py +++ b/forum/views/commands.py @@ -9,7 +9,7 @@ from forum.models import * from forum.forms import CloseForm from forum import auth from django.contrib.auth.decorators import login_required -from utils.decorators import ajax_method, ajax_login_required +from forum.utils.decorators import ajax_method, ajax_login_required import logging def vote(request, id):#refactor - pretty incomprehensible view used by various ajax calls diff --git a/forum/views/content.py b/forum/views/content.py index 9506fe3a..84fed42b 100644 --- a/forum/views/content.py +++ b/forum/views/content.py @@ -22,8 +22,8 @@ from django.template.defaultfilters import slugify from django.core.exceptions import PermissionDenied from django.core.urlresolvers import reverse -from utils.html import sanitize_html -from utils.decorators import ajax_method, ajax_login_required +from forum.utils.html import sanitize_html +from forum.utils.decorators import ajax_method, ajax_login_required from markdown2 import Markdown #from lxml.html.diff import htmldiff from forum.diff import textDiff as htmldiff @@ -32,7 +32,7 @@ from forum.models import * from forum.auth import * from forum.const import * from forum import auth -from utils.forms import get_next_url +from forum.utils.forms import get_next_url # used in index page INDEX_PAGE_SIZE = 20 diff --git a/forum/views/meta.py b/forum/views/meta.py index 07e222b0..bb42f022 100644 --- a/forum/views/meta.py +++ b/forum/views/meta.py @@ -5,7 +5,7 @@ from django.http import HttpResponseRedirect, HttpResponse from forum.forms import FeedbackForm from django.core.mail import mail_admins from django.utils.translation import ugettext as _ -from utils.forms import get_next_url +from forum.utils.forms import get_next_url from forum.models import Badge, Award def about(request): diff --git a/forum/views/readers.py b/forum/views/readers.py index 88b786aa..46e2885c 100644 --- a/forum/views/readers.py +++ b/forum/views/readers.py @@ -14,7 +14,7 @@ from django.utils.translation import ugettext as _ from django.template.defaultfilters import slugify from django.utils.datastructures import SortedDict -from utils.html import sanitize_html +from forum.utils.html import sanitize_html from markdown2 import Markdown #from lxml.html.diff import htmldiff from forum.diff import textDiff as htmldiff @@ -23,7 +23,7 @@ from forum.models import * from forum.auth import * from forum.const import * from forum import auth -from utils.forms import get_next_url +from forum.utils.forms import get_next_url # used in index page #refactor - move these numbers somewhere? diff --git a/forum/views/writers.py b/forum/views/writers.py index bb966b9f..666b936a 100644 --- a/forum/views/writers.py +++ b/forum/views/writers.py @@ -10,16 +10,17 @@ from django.template import RequestContext from django.utils.html import * from django.utils import simplejson from django.utils.translation import ugettext as _ +from django.core.urlresolvers import reverse from django.core.exceptions import PermissionDenied -from utils.html import sanitize_html +from forum.utils.html import sanitize_html from markdown2 import Markdown from forum.forms import * from forum.models import * from forum.auth import * from forum.const import * from forum import auth -from utils.forms import get_next_url +from forum.utils.forms import get_next_url from forum.views.readers import _get_tags_cache_json # used in index page @@ -103,14 +104,14 @@ def ask(request):#view used to ask a new question if request.user.is_authenticated(): author = request.user - Question.objects.create_new( + question = Question.objects.create_new( title = title, author = author, added_at = added_at, wiki = wiki, tagnames = tagnames, summary = summary, - text = text + text = sanitize_html(markdowner.convert(text)) ) return HttpResponseRedirect(question.get_absolute_url()) @@ -348,7 +349,7 @@ def answer(request, id):#process a new answer author=request.user, added_at=update_time, wiki=wiki, - text=text, + text=sanitize_html(markdowner.convert(text)), email_notify=form.cleaned_data['email_notify'] ) else: diff --git a/middleware/__init__.py b/middleware/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/middleware/anon_user.py b/middleware/anon_user.py deleted file mode 100644 index fa2686f0..00000000 --- a/middleware/anon_user.py +++ /dev/null @@ -1,34 +0,0 @@ -from django.http import HttpResponseRedirect -from utils.forms import get_next_url -from django.utils.translation import ugettext as _ -from user_messages import create_message, get_and_delete_messages -from django.conf import settings -import logging - -class AnonymousMessageManager(object): - def __init__(self,request): - self.request = request - def create(self,message=''): - create_message(self.request,message) - def get_and_delete(self): - messages = get_and_delete_messages(self.request) - return messages - -def dummy_deepcopy(*arg): - """this is necessary to prevent deepcopy() on anonymous user object - that now contains reference to request, which cannot be deepcopied - """ - return None - -class ConnectToSessionMessagesMiddleware(object): - def process_request(self, request): - if not request.user.is_authenticated(): - request.user.__deepcopy__ = dummy_deepcopy #plug on deepcopy which may be called by django db "driver" - request.user.message_set = AnonymousMessageManager(request) #here request is linked to anon user - request.user.get_and_delete_messages = request.user.message_set.get_and_delete - - #also set the first greeting one time per session only - if 'greeting_set' not in request.session: - request.session['greeting_set'] = True - msg = _('first time greeting with %(url)s') % {'url':settings.GREETING_URL} - request.user.message_set.create(message=msg) diff --git a/middleware/cancel.py b/middleware/cancel.py deleted file mode 100644 index 51e1b253..00000000 --- a/middleware/cancel.py +++ /dev/null @@ -1,15 +0,0 @@ -from django.http import HttpResponseRedirect -from utils.forms import get_next_url -import logging -class CancelActionMiddleware(object): - def process_view(self, request, view_func, view_args, view_kwargs): - if 'cancel' in request.REQUEST: - #todo use session messages for the anonymous users - try: - msg = getattr(view_func,'CANCEL_MESSAGE') - except AttributeError: - msg = 'action canceled' - request.user.message_set.create(message=msg) - return HttpResponseRedirect(get_next_url(request)) - else: - return None diff --git a/middleware/pagesize.py b/middleware/pagesize.py deleted file mode 100644 index f6e6fcfd..00000000 --- a/middleware/pagesize.py +++ /dev/null @@ -1,33 +0,0 @@ -# used in questions -QUESTIONS_PAGE_SIZE = 10 -class QuestionsPageSizeMiddleware(object): - def process_request(self, request): - # Set flag to False by default. If it is equal to True, then need to be saved. - pagesize_changed = False - # get pagesize from session, if failed then get default value - user_page_size = request.session.get("pagesize", QUESTIONS_PAGE_SIZE) - # set pagesize equal to logon user specified value in database - if request.user.is_authenticated() and request.user.questions_per_page > 0: - user_page_size = request.user.questions_per_page - - try: - # get new pagesize from UI selection - pagesize = int(request.GET.get('pagesize', user_page_size)) - if pagesize <> user_page_size: - pagesize_changed = True - - except ValueError: - pagesize = user_page_size - - # save this pagesize to user database - if pagesize_changed: - if request.user.is_authenticated(): - user = request.user - user.questions_per_page = pagesize - user.save() - # put pagesize into session - request.session["pagesize"] = pagesize - - def process_exception(self,request,exception): - import logging - logging.debug('have exception %s' % str(exception)) diff --git a/settings.py b/settings.py index b384e3fa..426a12ff 100755 --- a/settings.py +++ b/settings.py @@ -24,9 +24,9 @@ MIDDLEWARE_CLASSES = [ #'django.middleware.cache.FetchFromCacheMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', #'django.middleware.sqlprint.SqlPrintingMiddleware', - 'middleware.anon_user.ConnectToSessionMessagesMiddleware', - 'middleware.pagesize.QuestionsPageSizeMiddleware', - 'middleware.cancel.CancelActionMiddleware', + 'forum.middleware.anon_user.ConnectToSessionMessagesMiddleware', + 'forum.middleware.pagesize.QuestionsPageSizeMiddleware', + 'forum.middleware.cancel.CancelActionMiddleware', 'debug_toolbar.middleware.DebugToolbarMiddleware', 'recaptcha_django.middleware.ReCaptchaMiddleware', 'django.middleware.transaction.TransactionMiddleware', diff --git a/settings_local.py.dist b/settings_local.py.dist new file mode 100755 index 00000000..3a71b389 --- /dev/null +++ b/settings_local.py.dist @@ -0,0 +1,110 @@ +# encoding:utf-8 +import os.path +from django.utils.translation import ugettext as _ + +def check_local_setting(name, value): + local_vars = locals() + if name in local_vars and local_vars[name] == value: + return True + else: + return False + +SITE_SRC_ROOT = os.path.dirname(__file__) +LOG_FILENAME = 'django.osqa.log' + +#for logging +import logging +logging.basicConfig( + filename=os.path.join(SITE_SRC_ROOT, 'log', LOG_FILENAME), + level=logging.DEBUG, + format='%(pathname)s TIME: %(asctime)s MSG: %(filename)s:%(funcName)s:%(lineno)d %(message)s', +) + +#ADMINS and MANAGERS +ADMINS = (('Forum Admin', 'forum@example.com'),) +MANAGERS = ADMINS + +#DEBUG SETTINGS +DEBUG = False +TEMPLATE_DEBUG = DEBUG +INTERNAL_IPS = ('127.0.0.1',) + +DATABASE_NAME = '' # Or path to database file if using sqlite3. +DATABASE_USER = '' # Not used with sqlite3. +DATABASE_PASSWORD = '' # Not used with sqlite3. +DATABASE_ENGINE = '' #mysql, etc +DATABASE_HOST = '' +DATABASE_PORT = '' + +#Moved from settings.py for better organization. (please check it up to clean up settings.py) + +#email server settings +SERVER_EMAIL = '' +DEFAULT_FROM_EMAIL = '' +EMAIL_HOST_USER = '' +EMAIL_HOST_PASSWORD = '' +EMAIL_SUBJECT_PREFIX = '[OSQA] ' +EMAIL_HOST='osqa.net' +EMAIL_PORT='25' +EMAIL_USE_TLS=False + +#LOCALIZATIONS +TIME_ZONE = 'America/New_York' + +########################### +# +# this will allow running your forum with url like http://site.com/forum +# +# FORUM_SCRIPT_ALIAS = 'forum/' +# +FORUM_SCRIPT_ALIAS = '' #no leading slash, default = '' empty string + + +#OTHER SETTINGS +APP_TITLE = u'OSQA: Open Source Q&A Forum' +APP_SHORT_NAME = u'OSQA' +APP_KEYWORDS = u'OSQA,CNPROG,forum,community' +APP_DESCRIPTION = u'Ask and answer questions.' +APP_INTRO = u'

Ask and answer questions, make the world better!

' +APP_COPYRIGHT = 'Copyright OSQA, 2009. Some rights reserved under creative commons license.' +LOGIN_URL = '/%s%s%s' % (FORUM_SCRIPT_ALIAS,'account/','signin/') +GREETING_URL = LOGIN_URL #may be url of "faq" page or "about", etc + +USE_I18N = True +LANGUAGE_CODE = 'en' +EMAIL_VALIDATION = 'off' #string - on|off +MIN_USERNAME_LENGTH = 1 +EMAIL_UNIQUE = False +APP_URL = 'http://osqa.net' #used by email notif system and RSS +GOOGLE_SITEMAP_CODE = '' +GOOGLE_ANALYTICS_KEY = '' +BOOKS_ON = False +WIKI_ON = True +USE_EXTERNAL_LEGACY_LOGIN = False +EXTERNAL_LEGACY_LOGIN_HOST = 'login.osqa.net' +EXTERNAL_LEGACY_LOGIN_PORT = 80 +EXTERNAL_LEGACY_LOGIN_PROVIDER_NAME = 'OSQA' +FEEDBACK_SITE_URL = None #None or url +EDITABLE_SCREEN_NAME = False #True or False - can user change screen name? + +DJANGO_VERSION = 1.1 +RESOURCE_REVISION=4 + +USE_SPHINX_SEARCH = False #if True all SPHINX_* settings are required +#also sphinx search engine and djangosphinxs app must be installed +#sample sphinx configuration file is /sphinx/sphinx.conf +SPHINX_API_VERSION = 0x113 #refer to djangosphinx documentation +SPHINX_SEARCH_INDICES=('osqa',) #a tuple of index names remember about a comma after the +#last item, especially if you have just one :) +SPHINX_SERVER='localhost' +SPHINX_PORT=3312 + +#please get these at recaptcha.net +RECAPTCHA_PRIVATE_KEY='...' +RECAPTCHA_PUBLIC_KEY='...' +OSQA_DEFAULT_SKIN = 'default' + +#Facebook settings +USE_FB_CONNECT=False +FB_API_KEY='' #your api key from facebook +FB_SECRET='' #your application secret diff --git a/utils/__init__.py b/utils/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/utils/cache.py b/utils/cache.py deleted file mode 100644 index 410c0662..00000000 --- a/utils/cache.py +++ /dev/null @@ -1,92 +0,0 @@ -"""Utilities for working with Django Models.""" -import itertools - -from django.contrib.contenttypes.models import ContentType - -from lanai.utils.lists import flatten - -def fetch_model_dict(model, ids, fields=None): - """ - Fetches a dict of model details for model instances with the given - ids, keyed by their id. - - If a fields list is given, a dict of details will be retrieved for - each model, otherwise complete model instances will be retrieved. - - Any fields list given shouldn't contain the primary key attribute for - the model, as this can be determined from its Options. - """ - if fields is None: - return model._default_manager.in_bulk(ids) - else: - id_attr = model._meta.pk.attname - return dict((obj[id_attr], obj) for obj - in model._default_manager.filter(id__in=ids).values( - *itertools.chain((id_attr,), fields))) - -def populate_foreign_key_caches(model, objects_to_populate, fields=None): - """ - Populates caches for the given related Model in instances of objects - which have a ForeignKey relationship to it, specified as a list of - (object list, related attribute name list) two-tuples. - - If a list of field names is given, only the given fields will be - looked up and related object caches will be populated with a dict of - the specified fields. Otherwise, complete model instances will be - retrieved. - """ - # Get all related object ids for the appropriate fields - related_object_ids = [] - for objects, attrs in objects_to_populate: - related_object_ids.append(tuple(tuple(getattr(obj, '%s_id' % attr) - for attr in attrs) - for obj in objects)) - unique_ids = tuple(set(pk for pk in flatten(related_object_ids) if pk)) - related_objects = fetch_model_dict(model, unique_ids, fields) - - # Fill related object caches - for (objects, attrs), related_ids in itertools.izip(objects_to_populate, - related_object_ids): - for obj, related_ids_for_obj in itertools.izip(objects, - related_ids): - for attr, related_object in itertools.izip(attrs, (related_objects.get(pk, None) - for pk in related_ids_for_obj)): - setattr(obj, '_%s_cache' % attr, related_object) - -def populate_content_object_caches(generic_related_objects, model_fields=None): - """ - Retrieves ``ContentType`` and content objects for the given list of - items which use a generic relation, grouping the retrieval of content - objects by model to reduce the number of queries executed. - - This results in ``number_of_content_types + 1`` queries rather than - the ``number_of_generic_reL_objects * 2`` queries you'd get by - iterating over the list and accessing each item's object attribute. - - If a dict mapping model classes to field names is given, only the - given fields will be looked up for each model specified and the - object cache will be populated with a dict of the specified fields. - Otherwise, complete model instances will be retrieved. - """ - if model_fields is None: - model_fields = {} - - # Group content object ids by their content type ids - ids_by_content_type = {} - for obj in generic_related_objects: - ids_by_content_type.setdefault(obj.content_type_id, - []).append(obj.object_id) - - # Retrieve content types and content objects in bulk - content_types = ContentType.objects.in_bulk(ids_by_content_type.keys()) - for content_type_id, ids in ids_by_content_type.iteritems(): - model = content_types[content_type_id].model_class() - objects[content_type_id] = fetch_model_dict( - model, tuple(set(ids)), model_fields.get(model, None)) - - # Set content types and content objects in the appropriate cache - # attributes, so accessing the 'content_type' and 'object' attributes - # on each object won't result in further database hits. - for obj in generic_related_objects: - obj._object_cache = objects[obj.content_type_id][obj.object_id] - obj._content_type_cache = content_types[obj.content_type_id] diff --git a/utils/decorators.py b/utils/decorators.py deleted file mode 100644 index e4e7acb3..00000000 --- a/utils/decorators.py +++ /dev/null @@ -1,25 +0,0 @@ -from django.http import HttpResponse, HttpResponseForbidden, Http404 -from django.utils import simplejson - -def ajax_login_required(view_func): - def wrap(request,*args,**kwargs): - if request.user.is_authenticated(): - return view_func(request,*args,**kwargs) - else: - json = simplejson.dumps({'login_required':True}) - return HttpResponseForbidden(json,mimetype='application/json') - return wrap - -def ajax_method(view_func): - def wrap(request,*args,**kwargs): - if not request.is_ajax(): - raise Http404 - retval = view_func(request,*args,**kwargs) - if isinstance(retval, HttpResponse): - retval.mimetype = 'application/json' - return retval - else: - json = simplejson.dumps(retval) - return HttpResponse(json,mimetype='application/json') - return wrap - diff --git a/utils/forms.py b/utils/forms.py deleted file mode 100644 index c54056ca..00000000 --- a/utils/forms.py +++ /dev/null @@ -1,151 +0,0 @@ -from django import forms -import re -from django.utils.translation import ugettext as _ -from django.utils.safestring import mark_safe -from django.conf import settings -from django.http import str_to_unicode -from django.contrib.auth.models import User -import urllib - -DEFAULT_NEXT = '/' + getattr(settings, 'FORUM_SCRIPT_ALIAS') -def clean_next(next): - if next is None: - return DEFAULT_NEXT - next = str_to_unicode(urllib.unquote(next), 'utf-8') - next = next.strip() - if next.startswith('/'): - return next - return DEFAULT_NEXT - -def get_next_url(request): - return clean_next(request.REQUEST.get('next')) - -class StrippedNonEmptyCharField(forms.CharField): - def clean(self,value): - value = value.strip() - if self.required and value == '': - raise forms.ValidationError(_('this field is required')) - return value - -class NextUrlField(forms.CharField): - def __init__(self): - super(NextUrlField,self).__init__(max_length = 255,widget = forms.HiddenInput(),required = False) - def clean(self,value): - return clean_next(value) - -login_form_widget_attrs = { 'class': 'required login' } -username_re = re.compile(r'^[\w ]+$') - -class UserNameField(StrippedNonEmptyCharField): - RESERVED_NAMES = (u'fuck', u'shit', u'ass', u'sex', u'add', - u'edit', u'save', u'delete', u'manage', u'update', 'remove', 'new') - def __init__(self,db_model=User, db_field='username', must_exist=False,skip_clean=False,label=_('choose a username'),**kw): - self.must_exist = must_exist - self.skip_clean = skip_clean - self.db_model = db_model - self.db_field = db_field - error_messages={'required':_('user name is required'), - 'taken':_('sorry, this name is taken, please choose another'), - 'forbidden':_('sorry, this name is not allowed, please choose another'), - 'missing':_('sorry, there is no user with this name'), - 'multiple-taken':_('sorry, we have a serious error - user name is taken by several users'), - 'invalid':_('user name can only consist of letters, empty space and underscore'), - } - if 'error_messages' in kw: - error_messages.update(kw['error_messages']) - del kw['error_messages'] - super(UserNameField,self).__init__(max_length=30, - widget=forms.TextInput(attrs=login_form_widget_attrs), - label=label, - error_messages=error_messages, - **kw - ) - - def clean(self,username): - """ validate username """ - if self.skip_clean == True: - return username - if hasattr(self, 'user_instance') and isinstance(self.user_instance, User): - if username == self.user_instance.username: - return username - try: - username = super(UserNameField, self).clean(username) - except forms.ValidationError: - raise forms.ValidationError(self.error_messages['required']) - if self.required and not username_re.search(username): - raise forms.ValidationError(self.error_messages['invalid']) - if username in self.RESERVED_NAMES: - raise forms.ValidationError(self.error_messages['forbidden']) - try: - user = self.db_model.objects.get( - **{'%s' % self.db_field : username} - ) - if user: - if self.must_exist: - return username - else: - raise forms.ValidationError(self.error_messages['taken']) - except self.db_model.DoesNotExist: - if self.must_exist: - raise forms.ValidationError(self.error_messages['missing']) - else: - return username - except self.db_model.MultipleObjectsReturned: - raise forms.ValidationError(self.error_messages['multiple-taken']) - -class UserEmailField(forms.EmailField): - def __init__(self,skip_clean=False,**kw): - self.skip_clean = skip_clean - super(UserEmailField,self).__init__(widget=forms.TextInput(attrs=dict(login_form_widget_attrs, - maxlength=200)), label=mark_safe(_('your email address')), - error_messages={'required':_('email address is required'), - 'invalid':_('please enter a valid email address'), - 'taken':_('this email is already used by someone else, please choose another'), - }, - **kw - ) - - def clean(self,email): - """ validate if email exist in database - from legacy register - return: raise error if it exist """ - email = super(UserEmailField,self).clean(email.strip()) - if self.skip_clean: - return email - if settings.EMAIL_UNIQUE == True: - try: - user = User.objects.get(email = email) - raise forms.ValidationError(self.error_messsages['taken']) - except User.DoesNotExist: - return email - except User.MultipleObjectsReturned: - raise forms.ValidationError(self.error_messages['taken']) - else: - return email - -class SetPasswordForm(forms.Form): - password1 = forms.CharField(widget=forms.PasswordInput(attrs=login_form_widget_attrs), - label=_('choose password'), - error_messages={'required':_('password is required')}, - ) - password2 = forms.CharField(widget=forms.PasswordInput(attrs=login_form_widget_attrs), - label=mark_safe(_('retype password')), - error_messages={'required':_('please, retype your password'), - 'nomatch':_('sorry, entered passwords did not match, please try again')}, - ) - def clean_password2(self): - """ - Validates that the two password inputs match. - - """ - if 'password1' in self.cleaned_data: - if self.cleaned_data['password1'] == self.cleaned_data['password2']: - self.password = self.cleaned_data['password2'] - self.cleaned_data['password'] = self.cleaned_data['password2'] - return self.cleaned_data['password2'] - else: - del self.cleaned_data['password2'] - raise forms.ValidationError(self.fields['password2'].error_messages['nomatch']) - else: - return self.cleaned_data['password2'] - diff --git a/utils/html.py b/utils/html.py deleted file mode 100644 index 25a74a4a..00000000 --- a/utils/html.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Utilities for working with HTML.""" -import html5lib -from html5lib import sanitizer, serializer, tokenizer, treebuilders, treewalkers - -class HTMLSanitizerMixin(sanitizer.HTMLSanitizerMixin): - acceptable_elements = ('a', 'abbr', 'acronym', 'address', 'b', 'big', - 'blockquote', 'br', 'caption', 'center', 'cite', 'code', 'col', - 'colgroup', 'dd', 'del', 'dfn', 'dir', 'div', 'dl', 'dt', 'em', 'font', - 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'hr', 'i', 'img', 'ins', 'kbd', - 'li', 'ol', 'p', 'pre', 'q', 's', 'samp', 'small', 'span', 'strike', - 'strong', 'sub', 'sup', 'table', 'tbody', 'td', 'tfoot', 'th', 'thead', - 'tr', 'tt', 'u', 'ul', 'var') - - acceptable_attributes = ('abbr', 'align', 'alt', 'axis', 'border', - 'cellpadding', 'cellspacing', 'char', 'charoff', 'charset', 'cite', - 'cols', 'colspan', 'datetime', 'dir', 'frame', 'headers', 'height', - 'href', 'hreflang', 'hspace', 'lang', 'longdesc', 'name', 'nohref', - 'noshade', 'nowrap', 'rel', 'rev', 'rows', 'rowspan', 'rules', 'scope', - 'span', 'src', 'start', 'summary', 'title', 'type', 'valign', 'vspace', - 'width') - - allowed_elements = acceptable_elements - allowed_attributes = acceptable_attributes - allowed_css_properties = () - allowed_css_keywords = () - allowed_svg_properties = () - -class HTMLSanitizer(tokenizer.HTMLTokenizer, HTMLSanitizerMixin): - def __init__(self, stream, encoding=None, parseMeta=True, useChardet=True, - lowercaseElementName=True, lowercaseAttrName=True): - tokenizer.HTMLTokenizer.__init__(self, stream, encoding, parseMeta, - useChardet, lowercaseElementName, - lowercaseAttrName) - - def __iter__(self): - for token in tokenizer.HTMLTokenizer.__iter__(self): - token = self.sanitize_token(token) - if token: - yield token - -def sanitize_html(html): - """Sanitizes an HTML fragment.""" - p = html5lib.HTMLParser(tokenizer=HTMLSanitizer, - tree=treebuilders.getTreeBuilder("dom")) - dom_tree = p.parseFragment(html) - walker = treewalkers.getTreeWalker("dom") - stream = walker(dom_tree) - s = serializer.HTMLSerializer(omit_optional_tags=False, - quote_attr_values=True) - output_generator = s.serialize(stream) - return u''.join(output_generator) diff --git a/utils/lists.py b/utils/lists.py deleted file mode 100644 index bbcfae98..00000000 --- a/utils/lists.py +++ /dev/null @@ -1,86 +0,0 @@ -"""Utilities for working with lists and sequences.""" - -def flatten(x): - """ - Returns a single, flat list which contains all elements retrieved - from the sequence and all recursively contained sub-sequences - (iterables). - - Examples: - >>> [1, 2, [3, 4], (5, 6)] - [1, 2, [3, 4], (5, 6)] - - From http://kogs-www.informatik.uni-hamburg.de/~meine/python_tricks - """ - result = [] - for el in x: - if hasattr(el, '__iter__') and not isinstance(el, basestring): - result.extend(flatten(el)) - else: - result.append(el) - return result - -def batch_size(items, size): - """ - Retrieves items in batches of the given size. - - >>> l = range(1, 11) - >>> batch_size(l, 3) - [[1, 2, 3], [4, 5, 6], [7, 8, 9], [10]] - >>> batch_size(l, 5) - [[1, 2, 3, 4, 5], [6, 7, 8, 9, 10]] - """ - return [items[i:i+size] for i in xrange(0, len(items), size)] - -def batches(items, number): - """ - Retrieves items in the given number of batches. - - >>> l = range(1, 11) - >>> batches(l, 1) - [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]] - >>> batches(l, 2) - [[1, 2, 3, 4, 5], [6, 7, 8, 9, 10]] - >>> batches(l, 3) - [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10]] - >>> batches(l, 4) - [[1, 2, 3], [4, 5, 6], [7, 8, 9], [10]] - >>> batches(l, 5) - [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]] - - Initial batches will contain as many items as possible in cases where - there are not enough items to be distributed evenly. - - >>> batches(l, 6) - [[1, 2], [3, 4], [5, 6], [7, 8], [9], [10]] - >>> batches(l, 7) - [[1, 2], [3, 4], [5, 6], [7], [8], [9], [10]] - >>> batches(l, 8) - [[1, 2], [3, 4], [5], [6], [7], [8], [9], [10]] - >>> batches(l, 9) - [[1, 2], [3], [4], [5], [6], [7], [8], [9], [10]] - >>> batches(l, 10) - [[1], [2], [3], [4], [5], [6], [7], [8], [9], [10]] - - If there are more batches than items, empty batches will be appended - to the batch list. - - >>> batches(l, 11) - [[1], [2], [3], [4], [5], [6], [7], [8], [9], [10], []] - >>> batches(l, 12) - [[1], [2], [3], [4], [5], [6], [7], [8], [9], [10], [], []] - """ - div, mod= divmod(len(items), number) - if div > 1: - if mod: - div += 1 - return batch_size(items, div) - else: - if not div: - return [[item] for item in items] + [[]] * (number - mod) - elif div == 1 and not mod: - return [[item] for item in items] - else: - # mod now tells you how many lists of 2 you can fit in - return ([items[i*2:(i*2)+2] for i in xrange(0, mod)] + - [[item] for item in items[mod*2:]]) diff --git a/utils/odict.py b/utils/odict.py deleted file mode 100644 index 2c8391d7..00000000 --- a/utils/odict.py +++ /dev/null @@ -1,1399 +0,0 @@ -# odict.py -# An Ordered Dictionary object -# Copyright (C) 2005 Nicola Larosa, Michael Foord -# E-mail: nico AT tekNico DOT net, fuzzyman AT voidspace DOT org DOT uk - -# This software is licensed under the terms of the BSD license. -# http://www.voidspace.org.uk/python/license.shtml -# Basically you're free to copy, modify, distribute and relicense it, -# So long as you keep a copy of the license with it. - -# Documentation at http://www.voidspace.org.uk/python/odict.html -# For information about bugfixes, updates and support, please join the -# Pythonutils mailing list: -# http://groups.google.com/group/pythonutils/ -# Comments, suggestions and bug reports welcome. - -"""A dict that keeps keys in insertion order""" -from __future__ import generators - -__author__ = ('Nicola Larosa ,' - 'Michael Foord ') - -__docformat__ = "restructuredtext en" - -__revision__ = '$Id: odict.py 129 2005-09-12 18:15:28Z teknico $' - -__version__ = '0.2.2' - -__all__ = ['OrderedDict', 'SequenceOrderedDict'] - -import sys -INTP_VER = sys.version_info[:2] -if INTP_VER < (2, 2): - raise RuntimeError("Python v.2.2 or later required") - -import types, warnings - -class OrderedDict(dict): - """ - A class of dictionary that keeps the insertion order of keys. - - All appropriate methods return keys, items, or values in an ordered way. - - All normal dictionary methods are available. Update and comparison is - restricted to other OrderedDict objects. - - Various sequence methods are available, including the ability to explicitly - mutate the key ordering. - - __contains__ tests: - - >>> d = OrderedDict(((1, 3),)) - >>> 1 in d - 1 - >>> 4 in d - 0 - - __getitem__ tests: - - >>> OrderedDict(((1, 3), (3, 2), (2, 1)))[2] - 1 - >>> OrderedDict(((1, 3), (3, 2), (2, 1)))[4] - Traceback (most recent call last): - KeyError: 4 - - __len__ tests: - - >>> len(OrderedDict()) - 0 - >>> len(OrderedDict(((1, 3), (3, 2), (2, 1)))) - 3 - - get tests: - - >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) - >>> d.get(1) - 3 - >>> d.get(4) is None - 1 - >>> d.get(4, 5) - 5 - >>> d - OrderedDict([(1, 3), (3, 2), (2, 1)]) - - has_key tests: - - >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) - >>> d.has_key(1) - 1 - >>> d.has_key(4) - 0 - """ - - def __init__(self, init_val=(), strict=False): - """ - Create a new ordered dictionary. Cannot init from a normal dict, - nor from kwargs, since items order is undefined in those cases. - - If the ``strict`` keyword argument is ``True`` (``False`` is the - default) then when doing slice assignment - the ``OrderedDict`` you are - assigning from *must not* contain any keys in the remaining dict. - - >>> OrderedDict() - OrderedDict([]) - >>> OrderedDict({1: 1}) - Traceback (most recent call last): - TypeError: undefined order, cannot get items from dict - >>> OrderedDict({1: 1}.items()) - OrderedDict([(1, 1)]) - >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) - >>> d - OrderedDict([(1, 3), (3, 2), (2, 1)]) - >>> OrderedDict(d) - OrderedDict([(1, 3), (3, 2), (2, 1)]) - """ - self.strict = strict - dict.__init__(self) - if isinstance(init_val, OrderedDict): - self._sequence = init_val.keys() - dict.update(self, init_val) - elif isinstance(init_val, dict): - # we lose compatibility with other ordered dict types this way - raise TypeError('undefined order, cannot get items from dict') - else: - self._sequence = [] - self.update(init_val) - -### Special methods ### - - def __delitem__(self, key): - """ - >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) - >>> del d[3] - >>> d - OrderedDict([(1, 3), (2, 1)]) - >>> del d[3] - Traceback (most recent call last): - KeyError: 3 - >>> d[3] = 2 - >>> d - OrderedDict([(1, 3), (2, 1), (3, 2)]) - >>> del d[0:1] - >>> d - OrderedDict([(2, 1), (3, 2)]) - """ - if isinstance(key, types.SliceType): - # FIXME: efficiency? - keys = self._sequence[key] - for entry in keys: - dict.__delitem__(self, entry) - del self._sequence[key] - else: - # do the dict.__delitem__ *first* as it raises - # the more appropriate error - dict.__delitem__(self, key) - self._sequence.remove(key) - - def __eq__(self, other): - """ - >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) - >>> d == OrderedDict(d) - True - >>> d == OrderedDict(((1, 3), (2, 1), (3, 2))) - False - >>> d == OrderedDict(((1, 0), (3, 2), (2, 1))) - False - >>> d == OrderedDict(((0, 3), (3, 2), (2, 1))) - False - >>> d == dict(d) - False - >>> d == False - False - """ - if isinstance(other, OrderedDict): - # FIXME: efficiency? - # Generate both item lists for each compare - return (self.items() == other.items()) - else: - return False - - def __lt__(self, other): - """ - >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) - >>> c = OrderedDict(((0, 3), (3, 2), (2, 1))) - >>> c < d - True - >>> d < c - False - >>> d < dict(c) - Traceback (most recent call last): - TypeError: Can only compare with other OrderedDicts - """ - if not isinstance(other, OrderedDict): - raise TypeError('Can only compare with other OrderedDicts') - # FIXME: efficiency? - # Generate both item lists for each compare - return (self.items() < other.items()) - - def __le__(self, other): - """ - >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) - >>> c = OrderedDict(((0, 3), (3, 2), (2, 1))) - >>> e = OrderedDict(d) - >>> c <= d - True - >>> d <= c - False - >>> d <= dict(c) - Traceback (most recent call last): - TypeError: Can only compare with other OrderedDicts - >>> d <= e - True - """ - if not isinstance(other, OrderedDict): - raise TypeError('Can only compare with other OrderedDicts') - # FIXME: efficiency? - # Generate both item lists for each compare - return (self.items() <= other.items()) - - def __ne__(self, other): - """ - >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) - >>> d != OrderedDict(d) - False - >>> d != OrderedDict(((1, 3), (2, 1), (3, 2))) - True - >>> d != OrderedDict(((1, 0), (3, 2), (2, 1))) - True - >>> d == OrderedDict(((0, 3), (3, 2), (2, 1))) - False - >>> d != dict(d) - True - >>> d != False - True - """ - if isinstance(other, OrderedDict): - # FIXME: efficiency? - # Generate both item lists for each compare - return not (self.items() == other.items()) - else: - return True - - def __gt__(self, other): - """ - >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) - >>> c = OrderedDict(((0, 3), (3, 2), (2, 1))) - >>> d > c - True - >>> c > d - False - >>> d > dict(c) - Traceback (most recent call last): - TypeError: Can only compare with other OrderedDicts - """ - if not isinstance(other, OrderedDict): - raise TypeError('Can only compare with other OrderedDicts') - # FIXME: efficiency? - # Generate both item lists for each compare - return (self.items() > other.items()) - - def __ge__(self, other): - """ - >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) - >>> c = OrderedDict(((0, 3), (3, 2), (2, 1))) - >>> e = OrderedDict(d) - >>> c >= d - False - >>> d >= c - True - >>> d >= dict(c) - Traceback (most recent call last): - TypeError: Can only compare with other OrderedDicts - >>> e >= d - True - """ - if not isinstance(other, OrderedDict): - raise TypeError('Can only compare with other OrderedDicts') - # FIXME: efficiency? - # Generate both item lists for each compare - return (self.items() >= other.items()) - - def __repr__(self): - """ - Used for __repr__ and __str__ - - >>> r1 = repr(OrderedDict((('a', 'b'), ('c', 'd'), ('e', 'f')))) - >>> r1 - "OrderedDict([('a', 'b'), ('c', 'd'), ('e', 'f')])" - >>> r2 = repr(OrderedDict((('a', 'b'), ('e', 'f'), ('c', 'd')))) - >>> r2 - "OrderedDict([('a', 'b'), ('e', 'f'), ('c', 'd')])" - >>> r1 == str(OrderedDict((('a', 'b'), ('c', 'd'), ('e', 'f')))) - True - >>> r2 == str(OrderedDict((('a', 'b'), ('e', 'f'), ('c', 'd')))) - True - """ - return '%s([%s])' % (self.__class__.__name__, ', '.join( - ['(%r, %r)' % (key, self[key]) for key in self._sequence])) - - def __setitem__(self, key, val): - """ - Allows slice assignment, so long as the slice is an OrderedDict - >>> d = OrderedDict() - >>> d['a'] = 'b' - >>> d['b'] = 'a' - >>> d[3] = 12 - >>> d - OrderedDict([('a', 'b'), ('b', 'a'), (3, 12)]) - >>> d[:] = OrderedDict(((1, 2), (2, 3), (3, 4))) - >>> d - OrderedDict([(1, 2), (2, 3), (3, 4)]) - >>> d[::2] = OrderedDict(((7, 8), (9, 10))) - >>> d - OrderedDict([(7, 8), (2, 3), (9, 10)]) - >>> d = OrderedDict(((0, 1), (1, 2), (2, 3), (3, 4))) - >>> d[1:3] = OrderedDict(((1, 2), (5, 6), (7, 8))) - >>> d - OrderedDict([(0, 1), (1, 2), (5, 6), (7, 8), (3, 4)]) - >>> d = OrderedDict(((0, 1), (1, 2), (2, 3), (3, 4)), strict=True) - >>> d[1:3] = OrderedDict(((1, 2), (5, 6), (7, 8))) - >>> d - OrderedDict([(0, 1), (1, 2), (5, 6), (7, 8), (3, 4)]) - - >>> a = OrderedDict(((0, 1), (1, 2), (2, 3)), strict=True) - >>> a[3] = 4 - >>> a - OrderedDict([(0, 1), (1, 2), (2, 3), (3, 4)]) - >>> a[::1] = OrderedDict([(0, 1), (1, 2), (2, 3), (3, 4)]) - >>> a - OrderedDict([(0, 1), (1, 2), (2, 3), (3, 4)]) - >>> a[:2] = OrderedDict([(0, 1), (1, 2), (2, 3), (3, 4), (4, 5)]) - Traceback (most recent call last): - ValueError: slice assignment must be from unique keys - >>> a = OrderedDict(((0, 1), (1, 2), (2, 3))) - >>> a[3] = 4 - >>> a - OrderedDict([(0, 1), (1, 2), (2, 3), (3, 4)]) - >>> a[::1] = OrderedDict([(0, 1), (1, 2), (2, 3), (3, 4)]) - >>> a - OrderedDict([(0, 1), (1, 2), (2, 3), (3, 4)]) - >>> a[:2] = OrderedDict([(0, 1), (1, 2), (2, 3), (3, 4)]) - >>> a - OrderedDict([(0, 1), (1, 2), (2, 3), (3, 4)]) - >>> a[::-1] = OrderedDict([(0, 1), (1, 2), (2, 3), (3, 4)]) - >>> a - OrderedDict([(3, 4), (2, 3), (1, 2), (0, 1)]) - - >>> d = OrderedDict([(0, 1), (1, 2), (2, 3), (3, 4)]) - >>> d[:1] = 3 - Traceback (most recent call last): - TypeError: slice assignment requires an OrderedDict - - >>> d = OrderedDict([(0, 1), (1, 2), (2, 3), (3, 4)]) - >>> d[:1] = OrderedDict([(9, 8)]) - >>> d - OrderedDict([(9, 8), (1, 2), (2, 3), (3, 4)]) - """ - if isinstance(key, types.SliceType): - if not isinstance(val, OrderedDict): - # FIXME: allow a list of tuples? - raise TypeError('slice assignment requires an OrderedDict') - keys = self._sequence[key] - # NOTE: Could use ``range(*key.indices(len(self._sequence)))`` - indexes = range(len(self._sequence))[key] - if key.step is None: - # NOTE: new slice may not be the same size as the one being - # overwritten ! - # NOTE: What is the algorithm for an impossible slice? - # e.g. d[5:3] - pos = key.start or 0 - del self[key] - newkeys = val.keys() - for k in newkeys: - if k in self: - if self.strict: - raise ValueError('slice assignment must be from ' - 'unique keys') - else: - # NOTE: This removes duplicate keys *first* - # so start position might have changed? - del self[k] - self._sequence = (self._sequence[:pos] + newkeys + - self._sequence[pos:]) - dict.update(self, val) - else: - # extended slice - length of new slice must be the same - # as the one being replaced - if len(keys) != len(val): - raise ValueError('attempt to assign sequence of size %s ' - 'to extended slice of size %s' % (len(val), len(keys))) - # FIXME: efficiency? - del self[key] - item_list = zip(indexes, val.items()) - # smallest indexes first - higher indexes not guaranteed to - # exist - item_list.sort() - for pos, (newkey, newval) in item_list: - if self.strict and newkey in self: - raise ValueError('slice assignment must be from unique' - ' keys') - self.insert(pos, newkey, newval) - else: - if key not in self: - self._sequence.append(key) - dict.__setitem__(self, key, val) - - def __getitem__(self, key): - """ - Allows slicing. Returns an OrderedDict if you slice. - >>> b = OrderedDict([(7, 0), (6, 1), (5, 2), (4, 3), (3, 4), (2, 5), (1, 6)]) - >>> b[::-1] - OrderedDict([(1, 6), (2, 5), (3, 4), (4, 3), (5, 2), (6, 1), (7, 0)]) - >>> b[2:5] - OrderedDict([(5, 2), (4, 3), (3, 4)]) - >>> type(b[2:4]) - - """ - if isinstance(key, types.SliceType): - # FIXME: does this raise the error we want? - keys = self._sequence[key] - # FIXME: efficiency? - return OrderedDict([(entry, self[entry]) for entry in keys]) - else: - return dict.__getitem__(self, key) - - __str__ = __repr__ - - def __setattr__(self, name, value): - """ - Implemented so that accesses to ``sequence`` raise a warning and are - diverted to the new ``setkeys`` method. - """ - if name == 'sequence': - warnings.warn('Use of the sequence attribute is deprecated.' - ' Use the keys method instead.', DeprecationWarning) - # NOTE: doesn't return anything - self.setkeys(value) - else: - # FIXME: do we want to allow arbitrary setting of attributes? - # Or do we want to manage it? - object.__setattr__(self, name, value) - - def __getattr__(self, name): - """ - Implemented so that access to ``sequence`` raises a warning. - - >>> d = OrderedDict() - >>> d.sequence - [] - """ - if name == 'sequence': - warnings.warn('Use of the sequence attribute is deprecated.' - ' Use the keys method instead.', DeprecationWarning) - # NOTE: Still (currently) returns a direct reference. Need to - # because code that uses sequence will expect to be able to - # mutate it in place. - return self._sequence - else: - # raise the appropriate error - raise AttributeError("OrderedDict has no '%s' attribute" % name) - - def __deepcopy__(self, memo): - """ - To allow deepcopy to work with OrderedDict. - - >>> from copy import deepcopy - >>> a = OrderedDict([(1, 1), (2, 2), (3, 3)]) - >>> a['test'] = {} - >>> b = deepcopy(a) - >>> b == a - True - >>> b is a - False - >>> a['test'] is b['test'] - False - """ - from copy import deepcopy - return self.__class__(deepcopy(self.items(), memo), self.strict) - - -### Read-only methods ### - - def copy(self): - """ - >>> OrderedDict(((1, 3), (3, 2), (2, 1))).copy() - OrderedDict([(1, 3), (3, 2), (2, 1)]) - """ - return OrderedDict(self) - - def items(self): - """ - ``items`` returns a list of tuples representing all the - ``(key, value)`` pairs in the dictionary. - - >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) - >>> d.items() - [(1, 3), (3, 2), (2, 1)] - >>> d.clear() - >>> d.items() - [] - """ - return zip(self._sequence, self.values()) - - def keys(self): - """ - Return a list of keys in the ``OrderedDict``. - - >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) - >>> d.keys() - [1, 3, 2] - """ - return self._sequence[:] - - def values(self, values=None): - """ - Return a list of all the values in the OrderedDict. - - Optionally you can pass in a list of values, which will replace the - current list. The value list must be the same len as the OrderedDict. - - >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) - >>> d.values() - [3, 2, 1] - """ - return [self[key] for key in self._sequence] - - def iteritems(self): - """ - >>> ii = OrderedDict(((1, 3), (3, 2), (2, 1))).iteritems() - >>> ii.next() - (1, 3) - >>> ii.next() - (3, 2) - >>> ii.next() - (2, 1) - >>> ii.next() - Traceback (most recent call last): - StopIteration - """ - def make_iter(self=self): - keys = self.iterkeys() - while True: - key = keys.next() - yield (key, self[key]) - return make_iter() - - def iterkeys(self): - """ - >>> ii = OrderedDict(((1, 3), (3, 2), (2, 1))).iterkeys() - >>> ii.next() - 1 - >>> ii.next() - 3 - >>> ii.next() - 2 - >>> ii.next() - Traceback (most recent call last): - StopIteration - """ - return iter(self._sequence) - - __iter__ = iterkeys - - def itervalues(self): - """ - >>> iv = OrderedDict(((1, 3), (3, 2), (2, 1))).itervalues() - >>> iv.next() - 3 - >>> iv.next() - 2 - >>> iv.next() - 1 - >>> iv.next() - Traceback (most recent call last): - StopIteration - """ - def make_iter(self=self): - keys = self.iterkeys() - while True: - yield self[keys.next()] - return make_iter() - -### Read-write methods ### - - def clear(self): - """ - >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) - >>> d.clear() - >>> d - OrderedDict([]) - """ - dict.clear(self) - self._sequence = [] - - def pop(self, key, *args): - """ - No dict.pop in Python 2.2, gotta reimplement it - - >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) - >>> d.pop(3) - 2 - >>> d - OrderedDict([(1, 3), (2, 1)]) - >>> d.pop(4) - Traceback (most recent call last): - KeyError: 4 - >>> d.pop(4, 0) - 0 - >>> d.pop(4, 0, 1) - Traceback (most recent call last): - TypeError: pop expected at most 2 arguments, got 3 - """ - if len(args) > 1: - raise TypeError, ('pop expected at most 2 arguments, got %s' % - (len(args) + 1)) - if key in self: - val = self[key] - del self[key] - else: - try: - val = args[0] - except IndexError: - raise KeyError(key) - return val - - def popitem(self, i=-1): - """ - Delete and return an item specified by index, not a random one as in - dict. The index is -1 by default (the last item). - - >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) - >>> d.popitem() - (2, 1) - >>> d - OrderedDict([(1, 3), (3, 2)]) - >>> d.popitem(0) - (1, 3) - >>> OrderedDict().popitem() - Traceback (most recent call last): - KeyError: 'popitem(): dictionary is empty' - >>> d.popitem(2) - Traceback (most recent call last): - IndexError: popitem(): index 2 not valid - """ - if not self._sequence: - raise KeyError('popitem(): dictionary is empty') - try: - key = self._sequence[i] - except IndexError: - raise IndexError('popitem(): index %s not valid' % i) - return (key, self.pop(key)) - - def setdefault(self, key, defval = None): - """ - >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) - >>> d.setdefault(1) - 3 - >>> d.setdefault(4) is None - True - >>> d - OrderedDict([(1, 3), (3, 2), (2, 1), (4, None)]) - >>> d.setdefault(5, 0) - 0 - >>> d - OrderedDict([(1, 3), (3, 2), (2, 1), (4, None), (5, 0)]) - """ - if key in self: - return self[key] - else: - self[key] = defval - return defval - - def update(self, from_od): - """ - Update from another OrderedDict or sequence of (key, value) pairs - - >>> d = OrderedDict(((1, 0), (0, 1))) - >>> d.update(OrderedDict(((1, 3), (3, 2), (2, 1)))) - >>> d - OrderedDict([(1, 3), (0, 1), (3, 2), (2, 1)]) - >>> d.update({4: 4}) - Traceback (most recent call last): - TypeError: undefined order, cannot get items from dict - >>> d.update((4, 4)) - Traceback (most recent call last): - TypeError: cannot convert dictionary update sequence element "4" to a 2-item sequence - """ - if isinstance(from_od, OrderedDict): - for key, val in from_od.items(): - self[key] = val - elif isinstance(from_od, dict): - # we lose compatibility with other ordered dict types this way - raise TypeError('undefined order, cannot get items from dict') - else: - # FIXME: efficiency? - # sequence of 2-item sequences, or error - for item in from_od: - try: - key, val = item - except TypeError: - raise TypeError('cannot convert dictionary update' - ' sequence element "%s" to a 2-item sequence' % item) - self[key] = val - - def rename(self, old_key, new_key): - """ - Rename the key for a given value, without modifying sequence order. - - For the case where new_key already exists this raise an exception, - since if new_key exists, it is ambiguous as to what happens to the - associated values, and the position of new_key in the sequence. - - >>> od = OrderedDict() - >>> od['a'] = 1 - >>> od['b'] = 2 - >>> od.items() - [('a', 1), ('b', 2)] - >>> od.rename('b', 'c') - >>> od.items() - [('a', 1), ('c', 2)] - >>> od.rename('c', 'a') - Traceback (most recent call last): - ValueError: New key already exists: 'a' - >>> od.rename('d', 'b') - Traceback (most recent call last): - KeyError: 'd' - """ - if new_key == old_key: - # no-op - return - if new_key in self: - raise ValueError("New key already exists: %r" % new_key) - # rename sequence entry - value = self[old_key] - old_idx = self._sequence.index(old_key) - self._sequence[old_idx] = new_key - # rename internal dict entry - dict.__delitem__(self, old_key) - dict.__setitem__(self, new_key, value) - - def setitems(self, items): - """ - This method allows you to set the items in the dict. - - It takes a list of tuples - of the same sort returned by the ``items`` - method. - - >>> d = OrderedDict() - >>> d.setitems(((3, 1), (2, 3), (1, 2))) - >>> d - OrderedDict([(3, 1), (2, 3), (1, 2)]) - """ - self.clear() - # FIXME: this allows you to pass in an OrderedDict as well :-) - self.update(items) - - def setkeys(self, keys): - """ - ``setkeys`` all ows you to pass in a new list of keys which will - replace the current set. This must contain the same set of keys, but - need not be in the same order. - - If you pass in new keys that don't match, a ``KeyError`` will be - raised. - - >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) - >>> d.keys() - [1, 3, 2] - >>> d.setkeys((1, 2, 3)) - >>> d - OrderedDict([(1, 3), (2, 1), (3, 2)]) - >>> d.setkeys(['a', 'b', 'c']) - Traceback (most recent call last): - KeyError: 'Keylist is not the same as current keylist.' - """ - # FIXME: Efficiency? (use set for Python 2.4 :-) - # NOTE: list(keys) rather than keys[:] because keys[:] returns - # a tuple, if keys is a tuple. - kcopy = list(keys) - kcopy.sort() - self._sequence.sort() - if kcopy != self._sequence: - raise KeyError('Keylist is not the same as current keylist.') - # NOTE: This makes the _sequence attribute a new object, instead - # of changing it in place. - # FIXME: efficiency? - self._sequence = list(keys) - - def setvalues(self, values): - """ - You can pass in a list of values, which will replace the - current list. The value list must be the same len as the OrderedDict. - - (Or a ``ValueError`` is raised.) - - >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) - >>> d.setvalues((1, 2, 3)) - >>> d - OrderedDict([(1, 1), (3, 2), (2, 3)]) - >>> d.setvalues([6]) - Traceback (most recent call last): - ValueError: Value list is not the same length as the OrderedDict. - """ - if len(values) != len(self): - # FIXME: correct error to raise? - raise ValueError('Value list is not the same length as the ' - 'OrderedDict.') - self.update(zip(self, values)) - -### Sequence Methods ### - - def index(self, key): - """ - Return the position of the specified key in the OrderedDict. - - >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) - >>> d.index(3) - 1 - >>> d.index(4) - Traceback (most recent call last): - ValueError: list.index(x): x not in list - """ - return self._sequence.index(key) - - def insert(self, index, key, value): - """ - Takes ``index``, ``key``, and ``value`` as arguments. - - Sets ``key`` to ``value``, so that ``key`` is at position ``index`` in - the OrderedDict. - - >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) - >>> d.insert(0, 4, 0) - >>> d - OrderedDict([(4, 0), (1, 3), (3, 2), (2, 1)]) - >>> d.insert(0, 2, 1) - >>> d - OrderedDict([(2, 1), (4, 0), (1, 3), (3, 2)]) - >>> d.insert(8, 8, 1) - >>> d - OrderedDict([(2, 1), (4, 0), (1, 3), (3, 2), (8, 1)]) - """ - if key in self: - # FIXME: efficiency? - del self[key] - self._sequence.insert(index, key) - dict.__setitem__(self, key, value) - - def reverse(self): - """ - Reverse the order of the OrderedDict. - - >>> d = OrderedDict(((1, 3), (3, 2), (2, 1))) - >>> d.reverse() - >>> d - OrderedDict([(2, 1), (3, 2), (1, 3)]) - """ - self._sequence.reverse() - - def sort(self, *args, **kwargs): - """ - Sort the key order in the OrderedDict. - - This method takes the same arguments as the ``list.sort`` method on - your version of Python. - - >>> d = OrderedDict(((4, 1), (2, 2), (3, 3), (1, 4))) - >>> d.sort() - >>> d - OrderedDict([(1, 4), (2, 2), (3, 3), (4, 1)]) - """ - self._sequence.sort(*args, **kwargs) - -class Keys(object): - # FIXME: should this object be a subclass of list? - """ - Custom object for accessing the keys of an OrderedDict. - - Can be called like the normal ``OrderedDict.keys`` method, but also - supports indexing and sequence methods. - """ - - def __init__(self, main): - self._main = main - - def __call__(self): - """Pretend to be the keys method.""" - return self._main._keys() - - def __getitem__(self, index): - """Fetch the key at position i.""" - # NOTE: this automatically supports slicing :-) - return self._main._sequence[index] - - def __setitem__(self, index, name): - """ - You cannot assign to keys, but you can do slice assignment to re-order - them. - - You can only do slice assignment if the new set of keys is a reordering - of the original set. - """ - if isinstance(index, types.SliceType): - # FIXME: efficiency? - # check length is the same - indexes = range(len(self._main._sequence))[index] - if len(indexes) != len(name): - raise ValueError('attempt to assign sequence of size %s ' - 'to slice of size %s' % (len(name), len(indexes))) - # check they are the same keys - # FIXME: Use set - old_keys = self._main._sequence[index] - new_keys = list(name) - old_keys.sort() - new_keys.sort() - if old_keys != new_keys: - raise KeyError('Keylist is not the same as current keylist.') - orig_vals = [self._main[k] for k in name] - del self._main[index] - vals = zip(indexes, name, orig_vals) - vals.sort() - for i, k, v in vals: - if self._main.strict and k in self._main: - raise ValueError('slice assignment must be from ' - 'unique keys') - self._main.insert(i, k, v) - else: - raise ValueError('Cannot assign to keys') - - ### following methods pinched from UserList and adapted ### - def __repr__(self): return repr(self._main._sequence) - - # FIXME: do we need to check if we are comparing with another ``Keys`` - # object? (like the __cast method of UserList) - def __lt__(self, other): return self._main._sequence < other - def __le__(self, other): return self._main._sequence <= other - def __eq__(self, other): return self._main._sequence == other - def __ne__(self, other): return self._main._sequence != other - def __gt__(self, other): return self._main._sequence > other - def __ge__(self, other): return self._main._sequence >= other - # FIXME: do we need __cmp__ as well as rich comparisons? - def __cmp__(self, other): return cmp(self._main._sequence, other) - - def __contains__(self, item): return item in self._main._sequence - def __len__(self): return len(self._main._sequence) - def __iter__(self): return self._main.iterkeys() - def count(self, item): return self._main._sequence.count(item) - def index(self, item, *args): return self._main._sequence.index(item, *args) - def reverse(self): self._main._sequence.reverse() - def sort(self, *args, **kwds): self._main._sequence.sort(*args, **kwds) - def __mul__(self, n): return self._main._sequence*n - __rmul__ = __mul__ - def __add__(self, other): return self._main._sequence + other - def __radd__(self, other): return other + self._main._sequence - - ## following methods not implemented for keys ## - def __delitem__(self, i): raise TypeError('Can\'t delete items from keys') - def __iadd__(self, other): raise TypeError('Can\'t add in place to keys') - def __imul__(self, n): raise TypeError('Can\'t multiply keys in place') - def append(self, item): raise TypeError('Can\'t append items to keys') - def insert(self, i, item): raise TypeError('Can\'t insert items into keys') - def pop(self, i=-1): raise TypeError('Can\'t pop items from keys') - def remove(self, item): raise TypeError('Can\'t remove items from keys') - def extend(self, other): raise TypeError('Can\'t extend keys') - -class Items(object): - """ - Custom object for accessing the items of an OrderedDict. - - Can be called like the normal ``OrderedDict.items`` method, but also - supports indexing and sequence methods. - """ - - def __init__(self, main): - self._main = main - - def __call__(self): - """Pretend to be the items method.""" - return self._main._items() - - def __getitem__(self, index): - """Fetch the item at position i.""" - if isinstance(index, types.SliceType): - # fetching a slice returns an OrderedDict - return self._main[index].items() - key = self._main._sequence[index] - return (key, self._main[key]) - - def __setitem__(self, index, item): - """Set item at position i to item.""" - if isinstance(index, types.SliceType): - # NOTE: item must be an iterable (list of tuples) - self._main[index] = OrderedDict(item) - else: - # FIXME: Does this raise a sensible error? - orig = self._main.keys[index] - key, value = item - if self._main.strict and key in self and (key != orig): - raise ValueError('slice assignment must be from ' - 'unique keys') - # delete the current one - del self._main[self._main._sequence[index]] - self._main.insert(index, key, value) - - def __delitem__(self, i): - """Delete the item at position i.""" - key = self._main._sequence[i] - if isinstance(i, types.SliceType): - for k in key: - # FIXME: efficiency? - del self._main[k] - else: - del self._main[key] - - ### following methods pinched from UserList and adapted ### - def __repr__(self): return repr(self._main.items()) - - # FIXME: do we need to check if we are comparing with another ``Items`` - # object? (like the __cast method of UserList) - def __lt__(self, other): return self._main.items() < other - def __le__(self, other): return self._main.items() <= other - def __eq__(self, other): return self._main.items() == other - def __ne__(self, other): return self._main.items() != other - def __gt__(self, other): return self._main.items() > other - def __ge__(self, other): return self._main.items() >= other - def __cmp__(self, other): return cmp(self._main.items(), other) - - def __contains__(self, item): return item in self._main.items() - def __len__(self): return len(self._main._sequence) # easier :-) - def __iter__(self): return self._main.iteritems() - def count(self, item): return self._main.items().count(item) - def index(self, item, *args): return self._main.items().index(item, *args) - def reverse(self): self._main.reverse() - def sort(self, *args, **kwds): self._main.sort(*args, **kwds) - def __mul__(self, n): return self._main.items()*n - __rmul__ = __mul__ - def __add__(self, other): return self._main.items() + other - def __radd__(self, other): return other + self._main.items() - - def append(self, item): - """Add an item to the end.""" - # FIXME: this is only append if the key isn't already present - key, value = item - self._main[key] = value - - def insert(self, i, item): - key, value = item - self._main.insert(i, key, value) - - def pop(self, i=-1): - key = self._main._sequence[i] - return (key, self._main.pop(key)) - - def remove(self, item): - key, value = item - try: - assert value == self._main[key] - except (KeyError, AssertionError): - raise ValueError('ValueError: list.remove(x): x not in list') - else: - del self._main[key] - - def extend(self, other): - # FIXME: is only a true extend if none of the keys already present - for item in other: - key, value = item - self._main[key] = value - - def __iadd__(self, other): - self.extend(other) - - ## following methods not implemented for items ## - - def __imul__(self, n): raise TypeError('Can\'t multiply items in place') - -class Values(object): - """ - Custom object for accessing the values of an OrderedDict. - - Can be called like the normal ``OrderedDict.values`` method, but also - supports indexing and sequence methods. - """ - - def __init__(self, main): - self._main = main - - def __call__(self): - """Pretend to be the values method.""" - return self._main._values() - - def __getitem__(self, index): - """Fetch the value at position i.""" - if isinstance(index, types.SliceType): - return [self._main[key] for key in self._main._sequence[index]] - else: - return self._main[self._main._sequence[index]] - - def __setitem__(self, index, value): - """ - Set the value at position i to value. - - You can only do slice assignment to values if you supply a sequence of - equal length to the slice you are replacing. - """ - if isinstance(index, types.SliceType): - keys = self._main._sequence[index] - if len(keys) != len(value): - raise ValueError('attempt to assign sequence of size %s ' - 'to slice of size %s' % (len(name), len(keys))) - # FIXME: efficiency? Would be better to calculate the indexes - # directly from the slice object - # NOTE: the new keys can collide with existing keys (or even - # contain duplicates) - these will overwrite - for key, val in zip(keys, value): - self._main[key] = val - else: - self._main[self._main._sequence[index]] = value - - ### following methods pinched from UserList and adapted ### - def __repr__(self): return repr(self._main.values()) - - # FIXME: do we need to check if we are comparing with another ``Values`` - # object? (like the __cast method of UserList) - def __lt__(self, other): return self._main.values() < other - def __le__(self, other): return self._main.values() <= other - def __eq__(self, other): return self._main.values() == other - def __ne__(self, other): return self._main.values() != other - def __gt__(self, other): return self._main.values() > other - def __ge__(self, other): return self._main.values() >= other - def __cmp__(self, other): return cmp(self._main.values(), other) - - def __contains__(self, item): return item in self._main.values() - def __len__(self): return len(self._main._sequence) # easier :-) - def __iter__(self): return self._main.itervalues() - def count(self, item): return self._main.values().count(item) - def index(self, item, *args): return self._main.values().index(item, *args) - - def reverse(self): - """Reverse the values""" - vals = self._main.values() - vals.reverse() - # FIXME: efficiency - self[:] = vals - - def sort(self, *args, **kwds): - """Sort the values.""" - vals = self._main.values() - vals.sort(*args, **kwds) - self[:] = vals - - def __mul__(self, n): return self._main.values()*n - __rmul__ = __mul__ - def __add__(self, other): return self._main.values() + other - def __radd__(self, other): return other + self._main.values() - - ## following methods not implemented for values ## - def __delitem__(self, i): raise TypeError('Can\'t delete items from values') - def __iadd__(self, other): raise TypeError('Can\'t add in place to values') - def __imul__(self, n): raise TypeError('Can\'t multiply values in place') - def append(self, item): raise TypeError('Can\'t append items to values') - def insert(self, i, item): raise TypeError('Can\'t insert items into values') - def pop(self, i=-1): raise TypeError('Can\'t pop items from values') - def remove(self, item): raise TypeError('Can\'t remove items from values') - def extend(self, other): raise TypeError('Can\'t extend values') - -class SequenceOrderedDict(OrderedDict): - """ - Experimental version of OrderedDict that has a custom object for ``keys``, - ``values``, and ``items``. - - These are callable sequence objects that work as methods, or can be - manipulated directly as sequences. - - Test for ``keys``, ``items`` and ``values``. - - >>> d = SequenceOrderedDict(((1, 2), (2, 3), (3, 4))) - >>> d - SequenceOrderedDict([(1, 2), (2, 3), (3, 4)]) - >>> d.keys - [1, 2, 3] - >>> d.keys() - [1, 2, 3] - >>> d.setkeys((3, 2, 1)) - >>> d - SequenceOrderedDict([(3, 4), (2, 3), (1, 2)]) - >>> d.setkeys((1, 2, 3)) - >>> d.keys[0] - 1 - >>> d.keys[:] - [1, 2, 3] - >>> d.keys[-1] - 3 - >>> d.keys[-2] - 2 - >>> d.keys[0:2] = [2, 1] - >>> d - SequenceOrderedDict([(2, 3), (1, 2), (3, 4)]) - >>> d.keys.reverse() - >>> d.keys - [3, 1, 2] - >>> d.keys = [1, 2, 3] - >>> d - SequenceOrderedDict([(1, 2), (2, 3), (3, 4)]) - >>> d.keys = [3, 1, 2] - >>> d - SequenceOrderedDict([(3, 4), (1, 2), (2, 3)]) - >>> a = SequenceOrderedDict() - >>> b = SequenceOrderedDict() - >>> a.keys == b.keys - 1 - >>> a['a'] = 3 - >>> a.keys == b.keys - 0 - >>> b['a'] = 3 - >>> a.keys == b.keys - 1 - >>> b['b'] = 3 - >>> a.keys == b.keys - 0 - >>> a.keys > b.keys - 0 - >>> a.keys < b.keys - 1 - >>> 'a' in a.keys - 1 - >>> len(b.keys) - 2 - >>> 'c' in d.keys - 0 - >>> 1 in d.keys - 1 - >>> [v for v in d.keys] - [3, 1, 2] - >>> d.keys.sort() - >>> d.keys - [1, 2, 3] - >>> d = SequenceOrderedDict(((1, 2), (2, 3), (3, 4)), strict=True) - >>> d.keys[::-1] = [1, 2, 3] - >>> d - SequenceOrderedDict([(3, 4), (2, 3), (1, 2)]) - >>> d.keys[:2] - [3, 2] - >>> d.keys[:2] = [1, 3] - Traceback (most recent call last): - KeyError: 'Keylist is not the same as current keylist.' - - >>> d = SequenceOrderedDict(((1, 2), (2, 3), (3, 4))) - >>> d - SequenceOrderedDict([(1, 2), (2, 3), (3, 4)]) - >>> d.values - [2, 3, 4] - >>> d.values() - [2, 3, 4] - >>> d.setvalues((4, 3, 2)) - >>> d - SequenceOrderedDict([(1, 4), (2, 3), (3, 2)]) - >>> d.values[::-1] - [2, 3, 4] - >>> d.values[0] - 4 - >>> d.values[-2] - 3 - >>> del d.values[0] - Traceback (most recent call last): - TypeError: Can't delete items from values - >>> d.values[::2] = [2, 4] - >>> d - SequenceOrderedDict([(1, 2), (2, 3), (3, 4)]) - >>> 7 in d.values - 0 - >>> len(d.values) - 3 - >>> [val for val in d.values] - [2, 3, 4] - >>> d.values[-1] = 2 - >>> d.values.count(2) - 2 - >>> d.values.index(2) - 0 - >>> d.values[-1] = 7 - >>> d.values - [2, 3, 7] - >>> d.values.reverse() - >>> d.values - [7, 3, 2] - >>> d.values.sort() - >>> d.values - [2, 3, 7] - >>> d.values.append('anything') - Traceback (most recent call last): - TypeError: Can't append items to values - >>> d.values = (1, 2, 3) - >>> d - SequenceOrderedDict([(1, 1), (2, 2), (3, 3)]) - - >>> d = SequenceOrderedDict(((1, 2), (2, 3), (3, 4))) - >>> d - SequenceOrderedDict([(1, 2), (2, 3), (3, 4)]) - >>> d.items() - [(1, 2), (2, 3), (3, 4)] - >>> d.setitems([(3, 4), (2 ,3), (1, 2)]) - >>> d - SequenceOrderedDict([(3, 4), (2, 3), (1, 2)]) - >>> d.items[0] - (3, 4) - >>> d.items[:-1] - [(3, 4), (2, 3)] - >>> d.items[1] = (6, 3) - >>> d.items - [(3, 4), (6, 3), (1, 2)] - >>> d.items[1:2] = [(9, 9)] - >>> d - SequenceOrderedDict([(3, 4), (9, 9), (1, 2)]) - >>> del d.items[1:2] - >>> d - SequenceOrderedDict([(3, 4), (1, 2)]) - >>> (3, 4) in d.items - 1 - >>> (4, 3) in d.items - 0 - >>> len(d.items) - 2 - >>> [v for v in d.items] - [(3, 4), (1, 2)] - >>> d.items.count((3, 4)) - 1 - >>> d.items.index((1, 2)) - 1 - >>> d.items.index((2, 1)) - Traceback (most recent call last): - ValueError: list.index(x): x not in list - >>> d.items.reverse() - >>> d.items - [(1, 2), (3, 4)] - >>> d.items.reverse() - >>> d.items.sort() - >>> d.items - [(1, 2), (3, 4)] - >>> d.items.append((5, 6)) - >>> d.items - [(1, 2), (3, 4), (5, 6)] - >>> d.items.insert(0, (0, 0)) - >>> d.items - [(0, 0), (1, 2), (3, 4), (5, 6)] - >>> d.items.insert(-1, (7, 8)) - >>> d.items - [(0, 0), (1, 2), (3, 4), (7, 8), (5, 6)] - >>> d.items.pop() - (5, 6) - >>> d.items - [(0, 0), (1, 2), (3, 4), (7, 8)] - >>> d.items.remove((1, 2)) - >>> d.items - [(0, 0), (3, 4), (7, 8)] - >>> d.items.extend([(1, 2), (5, 6)]) - >>> d.items - [(0, 0), (3, 4), (7, 8), (1, 2), (5, 6)] - """ - - def __init__(self, init_val=(), strict=True): - OrderedDict.__init__(self, init_val, strict=strict) - self._keys = self.keys - self._values = self.values - self._items = self.items - self.keys = Keys(self) - self.values = Values(self) - self.items = Items(self) - self._att_dict = { - 'keys': self.setkeys, - 'items': self.setitems, - 'values': self.setvalues, - } - - def __setattr__(self, name, value): - """Protect keys, items, and values.""" - if not '_att_dict' in self.__dict__: - object.__setattr__(self, name, value) - else: - try: - fun = self._att_dict[name] - except KeyError: - OrderedDict.__setattr__(self, name, value) - else: - fun(value) - -if __name__ == '__main__': - if INTP_VER < (2, 3): - raise RuntimeError("Tests require Python v.2.3 or later") - # turn off warnings for tests - warnings.filterwarnings('ignore') - # run the code tests in doctest format - import doctest - m = sys.modules.get('__main__') - globs = m.__dict__.copy() - globs.update({ - 'INTP_VER': INTP_VER, - }) - doctest.testmod(m, globs=globs) - -- cgit v1.2.3-1-g7c22