implemented unicode aware `slugify` with unidecode library

11 files changed, 29 insertions, 9 deletions

diff --git a/askbot/importers/stackexchange/management/commands/load_stackexchange.py b/askbot/importers/stackexchange/management/commands/load_stackexchange.py
index 53e5e029..a813716a 100644
--- a/askbot/importers/stackexchange/management/commands/load_stackexchange.py
+++ b/askbot/importers/stackexchange/management/commands/load_stackexchange.py
@@ -1,5 +1,4 @@
 from django.core.management.base import BaseCommand
-from django.template.defaultfilters import slugify #todo: adopt unicode-aware slugify
 #todo: http://stackoverflow.com/questions/837828/how-to-use-a-slug-in-django 
 import os
 import re
@@ -14,6 +13,7 @@ from askbot.forms import EditUserEmailFeedsForm
 from askbot.conf import settings as askbot_settings
 from django.contrib.auth.models import Message as DjangoMessage
 from django.utils.translation import ugettext as _
+from askbot.utils.slug import slugify
 #from markdown2 import Markdown
 #markdowner = Markdown(html4tags=True)
 
diff --git a/askbot/migrations/0009_calculate_html_field_for_comments.py b/askbot/migrations/0009_calculate_html_field_for_comments.py
index 3a3fc8d0..53de1832 100644
--- a/askbot/migrations/0009_calculate_html_field_for_comments.py
+++ b/askbot/migrations/0009_calculate_html_field_for_comments.py
@@ -5,8 +5,8 @@ from south.db import db
 from south.v2 import DataMigration
 from django.db import models
 from django.utils.html import urlize
-from django.template.defaultfilters import slugify
 from django.core.urlresolvers import reverse
+from askbot.utils.slug import slugify
 try:
     from forum import const
 except ImportError:
diff --git a/askbot/models/__init__.py b/askbot/models/__init__.py
index 16bc3032..e99672d2 100644
--- a/askbot/models/__init__.py
+++ b/askbot/models/__init__.py
@@ -9,7 +9,6 @@ from django.template import loader, Context
 from django.utils.translation import ugettext as _
 from django.utils.translation import ungettext
 from django.contrib.auth.models import User
-from django.template.defaultfilters import slugify
 from django.utils.safestring import mark_safe
 from django.db import models
 from django.conf import settings as django_settings
@@ -31,6 +30,7 @@ from askbot.models import signals
 from askbot.models.repute import Badge, Award, Repute
 from askbot import auth
 from askbot.utils.decorators import auto_now_timestamp
+from askbot.utils.slug import slugify
 from askbot.startup_tests import run_startup_tests
 
 run_startup_tests()
diff --git a/askbot/models/answer.py b/askbot/models/answer.py
index 9afab656..9f48e720 100644
--- a/askbot/models/answer.py
+++ b/askbot/models/answer.py
@@ -1,7 +1,6 @@
 import datetime
 from django.db import models
 from django.utils.http import urlquote  as django_urlquote
-from django.template.defaultfilters import slugify
 from django.core.urlresolvers import reverse
 from askbot.models.base import AnonymousContent, DeletableContent
 from askbot.models.base import ContentRevision
@@ -9,6 +8,7 @@ from askbot.models.base import parse_post_text, parse_and_save_post
 from askbot.models import content
 from askbot.models.question import Question
 from askbot import const
+from askbot.utils.slug import slugify
 
 
 class AnswerManager(models.Manager):
diff --git a/askbot/models/question.py b/askbot/models/question.py
index aecfd794..265d9138 100644
--- a/askbot/models/question.py
+++ b/askbot/models/question.py
@@ -6,7 +6,6 @@ from django.utils.datastructures import SortedDict
 from django.db import models
 from django.contrib.auth.models import User
 from django.utils.http import urlquote as django_urlquote
-from django.template.defaultfilters import slugify
 from django.core.urlresolvers import reverse
 from django.contrib.sitemaps import ping_google
 from django.utils.translation import ugettext as _
@@ -17,6 +16,7 @@ from askbot.models.base import parse_post_text, parse_and_save_post
 from askbot.models import content
 from askbot import const
 from askbot.utils.lists import LazyList
+from askbot.utils.slug import slugify
 
 #todo: too bad keys are duplicated see const sort methods
 QUESTION_ORDER_BY_MAP = {
diff --git a/askbot/templatetags/extra_tags.py b/askbot/templatetags/extra_tags.py
index 770ffe0b..d7730677 100644
--- a/askbot/templatetags/extra_tags.py
+++ b/askbot/templatetags/extra_tags.py
@@ -15,11 +15,11 @@ from django.utils.translation import ungettext
 from django.conf import settings
 from askbot.conf import settings as askbot_settings
 from django.template.defaulttags import url as default_url
-from django.template.defaultfilters import slugify
 from django.core.urlresolvers import reverse
 from askbot.skins import utils as skin_utils
 from askbot.utils import colors
 from askbot.utils.functions import get_from_dict_or_object
+from askbot.utils.slug import slugify
 from askbot.templatetags import extra_filters
 
 register = template.Library()
diff --git a/askbot/tests/page_load_tests.py b/askbot/tests/page_load_tests.py
index 0938be18..21b897c1 100644
--- a/askbot/tests/page_load_tests.py
+++ b/askbot/tests/page_load_tests.py
@@ -2,6 +2,7 @@ from django.test import TestCase
 from django.template import defaultfilters
 from django.core.urlresolvers import reverse
 from askbot import models
+from askbot.utils.slug import slugify
 
 class PageLoadTestCase(TestCase):
     def try_url(
@@ -223,7 +224,7 @@ class PageLoadTests(PageLoadTestCase):
 
     def test_user_urls(self):
         user = models.User.objects.get(id=2)
-        name_slug = defaultfilters.slugify(user.username)
+        name_slug = slugify(user.username)
         self.try_url(
             'user_profile', 
             kwargs={'id': 2, 'slug': name_slug},
diff --git a/askbot/utils/slug.py b/askbot/utils/slug.py
new file mode 100644
index 00000000..e89c667e
--- /dev/null
+++ b/askbot/utils/slug.py
@@ -0,0 +1,18 @@
+from unidecode import unidecode
+from django.template.defaultfilters import slugify as slgfy
+
+
+def slugify(s, max_length=50):
+    if s == '':
+        return s
+    slug = slgfy(unidecode(s))
+    while len(slug) > max_length:
+        # try to shorten word by word until len(slug) <= max_length
+        temp = slug[:slug.rfind('-')]
+        if len(temp) > 0:
+            slug = temp
+        else:
+            # we have nothing left, do not apply the last crop, apply the cut-off directly
+            slug = slug[:max_length]
+            break
+    return slug
diff --git a/askbot/views/readers.py b/askbot/views/readers.py
index e2380e85..49a42ad8 100644
--- a/askbot/views/readers.py
+++ b/askbot/views/readers.py
@@ -18,11 +18,11 @@ from django.utils.html import *
 from django.utils import simplejson
 from django.db.models import Q
 from django.utils.translation import ugettext as _
-from django.template.defaultfilters import slugify
 from django.core.urlresolvers import reverse
 from django.views.decorators.cache import cache_page
 from django.core import exceptions as django_exceptions
 
+from askbot.utils.slug import slugify
 from askbot.utils.html import sanitize_html
 #from lxml.html.diff import htmldiff
 from askbot.utils.diff import textDiff as htmldiff
diff --git a/askbot/views/users.py b/askbot/views/users.py
index 18dd52fb..9b130fd2 100644
--- a/askbot/views/users.py
+++ b/askbot/views/users.py
@@ -13,7 +13,6 @@ import logging
 from django.db.models import Sum
 from django.contrib.auth.decorators import login_required
 from django.core.paginator import Paginator, EmptyPage, InvalidPage
-from django.template.defaultfilters import slugify
 from django.contrib.contenttypes.models import ContentType
 from django.core.urlresolvers import reverse
 from django.shortcuts import render_to_response, get_object_or_404
@@ -25,6 +24,7 @@ from django.utils.html import strip_tags
 from django.utils import simplejson
 from django.conf import settings as django_settings
 import askbot
+from askbot.utils.slug import slugify
 from askbot.utils.html import sanitize_html
 from askbot import auth
 from askbot import forms
diff --git a/setup.py b/setup.py
index bb63ebd3..be0d4268 100644
--- a/setup.py
+++ b/setup.py
@@ -14,6 +14,7 @@ install_requires = [
     'django-keyedcache',
     'django-threaded-multihost',
     'django-robots',
+    'unidecode',
 ]
 
 import askbot