diff options
| -rw-r--r-- | askbot/doc/source/solr.rst | 113 | ||||
| -rw-r--r-- | askbot/management/commands/askbot_rebuild_index.py | 26 | ||||
| -rw-r--r-- | askbot/models/question.py | 36 | ||||
| -rw-r--r-- | askbot/search/haystack/__init__.py | 108 | ||||
| -rw-r--r-- | askbot/search/haystack/routers.py | 20 | ||||
| -rw-r--r-- | askbot/search/haystack/searchquery.py | 53 | ||||
| -rw-r--r-- | askbot/search_indexes.py | 4 | ||||
| -rw-r--r-- | askbot/setup_templates/settings.py | 2 | ||||
| -rw-r--r-- | askbot/setup_templates/settings.py.mustache | 2 | ||||
| -rw-r--r-- | askbot/startup_procedures.py | 12 | ||||
| -rw-r--r-- | askbot/templates/search/indexes/askbot/post_text.txt | 1 | ||||
| -rw-r--r-- | askbot/templates/search/indexes/askbot/thread_text.txt | 6 | ||||
| -rw-r--r-- | askbot/templates/search/indexes/auth/user_text.txt | 5 | ||||
| -rw-r--r-- | askbot/tests/haystack_search_tests.py | 3 |
14 files changed, 316 insertions, 75 deletions
diff --git a/askbot/doc/source/solr.rst b/askbot/doc/source/solr.rst new file mode 100644 index 00000000..0b76aa94 --- /dev/null +++ b/askbot/doc/source/solr.rst @@ -0,0 +1,113 @@ +.. _solr: + +=========================================================== +Installing Apache Solr with Apache Tomcat 7 in Ubuntu 12.04 +=========================================================== + + +This document describes the process of instalation of Apache Solr search engine in Ubuntu Server 12.04 +for askbot use. To follow this steps you must have already askbot installed and running. + +Getting the requirements +------------------------ + +We need the following packages installed:: + + sudo apt-get install tomcat7 tomcat7-admin + +We need to download Apache Solr from the `official site <http://lucene.apache.org/solr/downloads.html>`_:: + + wget http://www.bizdirusa.com/mirrors/apache/lucene/solr/3.6.2/apache-solr-3.6.2.tgz + +Then we decompress it:: + + tar -xzf solr-3.6.2.tgz + +Setting up Tomcat +----------------- + +After installing tomcat there are some configuration required to make it work. First we are going to add +Tomcat users. Edit /etc/tomcat7/tomcat-users.xml and add the following:: + + <?xml version='1.0' encoding='utf-8'?> + <tomcat-users> + <role rolename="manager"/> + <role rolename="admin"/> + <role rolename="admin-gui"/> + <role rolename="manager-gui"/> + <user username="tomcat" password="tomcat" roles="manager,admin,manager-gui,admin-gui"/> + </tomcat-users> + +This will allow you to connect to the web management interface. After doing it restart the service: + + service tomcat7 restart + +To make see if it works go to: http://youripaddress:8080/manager it will ask for your tomcat user password +described in the tomcat-users.xml + +Installing Solr under Tomcat +---------------------------- + +Extract the solr tar archive from the previous download:: + + tar -xzf solr-4.3.0.tgz + +Copy the example/ directory from the source to /opt/solr/. Open the file /opt/solr/example/conf/solrconfig.xml +and Modify the dataDir parameter as:: + + <dataDir>${solr.data.dir:/opt/solr/example/solr/data}</dataDir> + +Copy the .war file in dist directory to /opt/solr:: + + cp dist/apache-solr-3.6.2.war /opt/solr + +Create solr.xml inside of /etc/tomcat/Catalina/localhost/ with the following contents:: + + <?xml version="1.0" encoding="utf-8"?> + <Context docBase="/opt/solr/apache-solr-3.6.2.war" debug="0" crossContext="true"> + <Environment name="solr/home" type="java.lang.String" value="/opt/solr/example/solr" override="true"/> + </Context> + +Restart tomcat server:: + + service tomcat7 restart + +By now you should be able to see the "solr" application in the tomcat manager and also access it in /solr/admin. + + +Configuring Askbot with Solr +---------------------------- + +Open settings.py file and configure the following:: + + ENABLE_HAYSTACK_SEARCH = True + HAYSTACK_SEARCH_ENGINE = 'solr' + HAYSTACK_SOLR_URL = 'http://127.0.0.1:8080/solr' + +After that create the solr schema and store the output to your solr instalation:: + + python manage.py build_solr_schema > /opt/solr/example/solr/conf/schema.xml + +Restart tomcat server:: + + service tomcat7 restart + +Build the Index for the first time:: + + python manage.py rebuild_index + +The output should be something like:: + + All documents removed. + Indexing 43 people. + Indexing 101 posts. + Indexing 101 threads. + +You must be good to go after this, just restart the askbot application and test the search with haystack and solr + + +Keeping the index fresh +----------------------- + +For this we recommend to use one of haystack `third party apps <http://django-haystack.readthedocs.org/en/latest/other_apps.html>`_ that use celery, +plese check this `link <http://django-haystack.readthedocs.org/en/latest/other_apps.html>`_ for more info. diff --git a/askbot/management/commands/askbot_rebuild_index.py b/askbot/management/commands/askbot_rebuild_index.py new file mode 100644 index 00000000..204f3b56 --- /dev/null +++ b/askbot/management/commands/askbot_rebuild_index.py @@ -0,0 +1,26 @@ +from optparse import make_option + +from django.core.management import call_command +from django.utils.translation import activate as activate_language +from django.core.management.base import BaseCommand +from django.conf import settings + +try: + from haystack.management.commands.clear_index import Command as ClearCommand + from haystack.management.commands.update_index import Command as UpdateCommand + haystack_option_list = [option for option in UpdateCommand.base_options if option.get_opt_string() != '--verbosity'] + \ + [option for option in ClearCommand.base_options if not option.get_opt_string() in ['--using', '--verbosity']] +except ImportError: + haystack_option_list = [] + +class Command(BaseCommand): + help = "Completely rebuilds the search index by removing the old data and then updating." + base_options = [make_option("-l", "--language", action="store", type="string", dest="language", + help='Language to user, in language code format'),] + option_list = list(BaseCommand.option_list) + haystack_option_list + base_options + + def handle(self, **options): + lang_code = options.get('language', settings.LANGUAGE_CODE.lower()) + activate_language(lang_code) + options['using'] = ['default_%s' % lang_code[:2],] + call_command('rebuild_index', **options) diff --git a/askbot/models/question.py b/askbot/models/question.py index 3dd9fc6b..5c6814bb 100644 --- a/askbot/models/question.py +++ b/askbot/models/question.py @@ -48,24 +48,30 @@ class ThreadQuerySet(models.query.QuerySet): todo: possibly add tags todo: implement full text search on relevant fields """ - db_engine_name = askbot.get_database_engine_name() - filter_parameters = {'deleted': False} - if 'postgresql_psycopg2' in db_engine_name: - from askbot.search import postgresql - return postgresql.run_title_search( - self, search_query - ).filter( - **filter_parameters - ).order_by('-relevance') - elif 'mysql' in db_engine_name and mysql.supports_full_text_search(): - filter_parameters['title__search'] = search_query + + if getattr(django_settings, 'ENABLE_HAYSTACK_SEARCH', False): + from askbot.search.haystack import AskbotSearchQuerySet + hs_qs = AskbotSearchQuerySet().filter(content=search_query).models(self.model) + return hs_qs.get_django_queryset() else: - filter_parameters['title__icontains'] = search_query + db_engine_name = askbot.get_database_engine_name() + filter_parameters = {'deleted': False} + if 'postgresql_psycopg2' in db_engine_name: + from askbot.search import postgresql + return postgresql.run_title_search( + self, search_query + ).filter( + **filter_parameters + ).order_by('-relevance') + elif 'mysql' in db_engine_name and mysql.supports_full_text_search(): + filter_parameters['title__search'] = search_query + else: + filter_parameters['title__icontains'] = search_query - if getattr(django_settings, 'ASKBOT_MULTILINGUAL', False): - filter_parameters['language_code'] = get_language() + if getattr(django_settings, 'ASKBOT_MULTILINGUAL', False): + filter_parameters['language_code'] = get_language() - return self.filter(**filter_parameters) + return self.filter(**filter_parameters) class ThreadManager(BaseQuerySetManager): diff --git a/askbot/search/haystack/__init__.py b/askbot/search/haystack/__init__.py index 71f04d00..45a037c9 100644 --- a/askbot/search/haystack/__init__.py +++ b/askbot/search/haystack/__init__.py @@ -1,59 +1,57 @@ -try: - from haystack import indexes, site - from haystack.query import SearchQuerySet - from askbot.models import Post, Thread, User - - - class ThreadIndex(indexes.SearchIndex): - text = indexes.CharField(document=True, use_template=True) - title = indexes.CharField(model_attr='title') - post_text = indexes.CharField(model_attr='posts__text__search') - - def index_queryset(self): - return Thread.objects.filter(posts__deleted=False) - - def prepare(self, obj): - self.prepared_data = super(ThreadIndex, self).prepare(object) - - self.prepared_data['tags'] = [tag.name for tag in objects.tags.all()] - - class PostIndex(indexes.SearchIndex): - text = indexes.CharField(document=True, use_template=True) - post_text = indexes.CharField(model_attr='text') - author = indexes.CharField(model_attr='user') - thread_id = indexes.CharField(model_attr='thread') - - def index_queryset(self): - return Post.objects.filter(deleted=False) +from django.conf import settings +from django.utils.translation import get_language - class UserIndex(indexes.SearchIndex): - text = indexes.CharField(document=True, use_template=True) - - def index_queryset(self): - return User.objects.all() - - site.register(Post, PostIndex) - site.register(Thread, ThreadIndex) - site.register(User, UserIndex) - - class AskbotSearchQuerySet(SearchQuerySet): - - def get_django_queryset(self, model_klass=Thread): - '''dirty hack because models() method from the - SearchQuerySet does not work </3''' - id_list = [] - for r in self: - if r.model_name in ['thread','post'] \ - and model_klass._meta.object_name.lower() == 'thread': - if getattr(r, 'thread_id'): - id_list.append(r.thread_id) - else: - id_list.append(r.pk) - elif r.model_name == model_klass._meta.object_name.lower(): - #FIXME: add a highlight here? - id_list.append(r.pk) - - return model_klass.objects.filter(id__in=set(id_list)) +from haystack import indexes +try: + from searchquery import AskbotSearchQuerySet except: pass + +class ThreadIndex(indexes.SearchIndex, indexes.Indexable): + text = indexes.CharField(document=True, use_template=True) + title = indexes.CharField() + tags = indexes.MultiValueField() + + def get_model(self): + from askbot.models import Thread + return Thread + + def index_queryset(self, using=None): + if getattr(settings, 'ASKBOT_MULTILINGUAL', True): + lang_code = get_language()[:2] + return self.get_model().objects.filter(language_code=lang_code, + posts__deleted=False) + else: + return self.get_model().objects.filter(posts__deleted=False) + + def prepare_tags(self, obj): + return [tag.name for tag in obj.tags.all()] + +class PostIndex(indexes.SearchIndex, indexes.Indexable): + text = indexes.CharField(document=True, use_template=True) + post_text = indexes.CharField(model_attr='text') + author = indexes.CharField() + thread_id = indexes.IntegerField(model_attr='thread__pk') + + def get_model(self): + from askbot.models import Post + return Post + + def index_queryset(self, using=None): + if getattr(settings, 'ASKBOT_MULTILINGUAL', True): + lang_code = get_language()[:2] + return self.get_model().objects.filter(language_code=lang_code, + deleted=False) + else: + return self.get_model().objects.filter(deleted=False) + +class UserIndex(indexes.SearchIndex, indexes.Indexable): + text = indexes.CharField(document=True, use_template=True) + + def get_model(self): + from askbot.models import User + return User + + def index_queryset(self, using=None): + return self.get_model().objects.all() diff --git a/askbot/search/haystack/routers.py b/askbot/search/haystack/routers.py new file mode 100644 index 00000000..7b7e0a4b --- /dev/null +++ b/askbot/search/haystack/routers.py @@ -0,0 +1,20 @@ +from django.utils.translation import get_language + +from haystack.routers import BaseRouter +from haystack.constants import DEFAULT_ALIAS + +class LanguageRouter(BaseRouter): + + def for_read(self, **hints): + from django.conf import settings + if getattr(settings, 'ASKBOT_MULTILINGUAL'): + return 'default_' + get_language()[:2] + else: + return DEFAULT_ALIAS + + def for_write(self, **hints): + from django.conf import settings + if getattr(settings, 'ASKBOT_MULTILINGUAL'): + return 'default_' + get_language()[:2] + else: + return DEFAULT_ALIAS diff --git a/askbot/search/haystack/searchquery.py b/askbot/search/haystack/searchquery.py new file mode 100644 index 00000000..116f4990 --- /dev/null +++ b/askbot/search/haystack/searchquery.py @@ -0,0 +1,53 @@ +from askbot.models import Thread, User +from haystack.query import SearchQuerySet + +class AskbotSearchQuerySet(SearchQuerySet): + + def _determine_backend(self): + '''This is a hack somehow connection_router got wrong values + from setting and did not loaded the LanguageRouter''' + + from haystack import connections, connection_router + # A backend has been manually selected. Use it instead. + if self._using is not None: + self.query = connections[self._using].get_query() + return + + # No backend, so rely on the routers to figure out what's right. + hints = {} + + if self.query: + hints['models'] = self.query.models + + backend_alias = connection_router.for_read(**hints) + + if isinstance(backend_alias, (list, tuple)) and len(backend_alias): + # We can only effectively read from one engine. + backend_alias = backend_alias[0] + + # The ``SearchQuery`` might swap itself out for a different variant + # here. + if self.query: + self.query = self.query.using(backend_alias) + else: + self.query = connections[backend_alias].get_query() + + def get_django_queryset(self, model_klass=Thread): + '''dirty hack because models() method from the + SearchQuerySet does not work </3''' + id_list = [] + for r in self: + if r.model_name in ['thread','post'] \ + and model_klass._meta.object_name.lower() == 'thread': + if getattr(r, 'thread_id'): + id_list.append(r.thread_id) + else: + id_list.append(r.pk) + elif r.model_name == model_klass._meta.object_name.lower(): + #FIXME: add a highlight here? + id_list.append(r.pk) + + if model_klass == User: + return model_klass.objects.filter(id__in=set(id_list)) + else: + return model_klass.objects.filter(id__in=set(id_list)) diff --git a/askbot/search_indexes.py b/askbot/search_indexes.py new file mode 100644 index 00000000..f2674996 --- /dev/null +++ b/askbot/search_indexes.py @@ -0,0 +1,4 @@ +from django.conf import settings + +if getattr(settings, 'ENABLE_HAYSTACK_SEARCH'): + from askbot.search.haystack import UserIndex, ThreadIndex, PostIndex diff --git a/askbot/setup_templates/settings.py b/askbot/setup_templates/settings.py index 1427e506..e5b21d92 100644 --- a/askbot/setup_templates/settings.py +++ b/askbot/setup_templates/settings.py @@ -258,6 +258,8 @@ RECAPTCHA_USE_SSL = True #HAYSTACK_SETTINGS ENABLE_HAYSTACK_SEARCH = False HAYSTACK_SITECONF = 'askbot.search.haystack' +#if you set this to True it can fail +HAYSTACK_ENABLE_REGISTRATIONS = False #more information #http://django-haystack.readthedocs.org/en/v1.2.7/settings.html HAYSTACK_SEARCH_ENGINE = 'simple' diff --git a/askbot/setup_templates/settings.py.mustache b/askbot/setup_templates/settings.py.mustache index f30297d7..f59dd735 100644 --- a/askbot/setup_templates/settings.py.mustache +++ b/askbot/setup_templates/settings.py.mustache @@ -256,6 +256,8 @@ RECAPTCHA_USE_SSL = True #HAYSTACK_SETTINGS ENABLE_HAYSTACK_SEARCH = False HAYSTACK_SITECONF = 'askbot.search.haystack' +#if you set this to True it can fail +HAYSTACK_ENABLE_REGISTRATIONS = False #more information #http://django-haystack.readthedocs.org/en/v1.2.7/settings.html HAYSTACK_SEARCH_ENGINE = 'simple' diff --git a/askbot/startup_procedures.py b/askbot/startup_procedures.py index b7ad47bf..8f4ea2ac 100644 --- a/askbot/startup_procedures.py +++ b/askbot/startup_procedures.py @@ -622,6 +622,12 @@ def test_haystack(): if not hasattr(django_settings, 'HAYSTACK_SITECONF'): message = 'Please add HAYSTACK_SITECONF = "askbot.search.haystack"' errors.append(message) + if not hasattr(django_settings, 'HAYSTACK_ENABLE_REGISTRATIONS'): + message = 'Please add "HAYSTACK_ENABLE_REGISTRATIONS = False"' + errors.append(message) + elif getattr(django_settings, 'HAYSTACK_ENABLE_REGISTRATIONS'): + message = 'Please set "HAYSTACK_ENABLE_REGISTRATIONS = False"' + errors.append(message) footer = 'Please refer to haystack documentation at http://django-haystack.readthedocs.org/en/v1.2.7/settings.html#haystack-search-engine' print_errors(errors, footer=footer) @@ -919,7 +925,7 @@ def run_startup_tests(): test_compressor() test_custom_user_profile_tab() test_group_messaging() - test_haystack() + #test_haystack() test_jinja2() test_longerusername() test_new_skins() @@ -965,10 +971,6 @@ def run_startup_tests(): 'value': True, 'message': 'Please add: RECAPTCHA_USE_SSL = True' }, - 'HAYSTACK_SITECONF': { - 'value': 'askbot.search.haystack', - 'message': 'Please add: HAYSTACK_SITECONF = "askbot.search.haystack"' - } }) settings_tester.run() if 'manage.py test' in ' '.join(sys.argv): diff --git a/askbot/templates/search/indexes/askbot/post_text.txt b/askbot/templates/search/indexes/askbot/post_text.txt new file mode 100644 index 00000000..b373ccf7 --- /dev/null +++ b/askbot/templates/search/indexes/askbot/post_text.txt @@ -0,0 +1 @@ +{{ object.text }} diff --git a/askbot/templates/search/indexes/askbot/thread_text.txt b/askbot/templates/search/indexes/askbot/thread_text.txt new file mode 100644 index 00000000..0db5dba0 --- /dev/null +++ b/askbot/templates/search/indexes/askbot/thread_text.txt @@ -0,0 +1,6 @@ +{{ object.title }} +{{ object.tags }} + +{% for tag in object.tags.all() %} +{{ tag.name }} +{% endfor %} diff --git a/askbot/templates/search/indexes/auth/user_text.txt b/askbot/templates/search/indexes/auth/user_text.txt new file mode 100644 index 00000000..b0baa3a6 --- /dev/null +++ b/askbot/templates/search/indexes/auth/user_text.txt @@ -0,0 +1,5 @@ +{{ object.username }} +{{ object.first_name }} +{{ object.last_name }} +{{ object.email }} +{{ object.full_name }} diff --git a/askbot/tests/haystack_search_tests.py b/askbot/tests/haystack_search_tests.py index 7a8bfcfd..c256c533 100644 --- a/askbot/tests/haystack_search_tests.py +++ b/askbot/tests/haystack_search_tests.py @@ -12,7 +12,9 @@ class HaystackSearchTests(AskbotTestCase): """ def setUp(self): self._old_value = getattr(settings, 'ENABLE_HAYSTACK_SEARCH', False) + self._old_search_engine = getattr(settings, 'HAYSTACK_SEARCH_ENGINE', 'simple') setattr(settings, "ENABLE_HAYSTACK_SEARCH", True) + setattr(settings, "HAYSTACK_SEARCH_ENGINE", 'simple') self.user = self.create_user(username='gepeto') self.other_user = self.create_user(username = 'pinocho') @@ -53,6 +55,7 @@ class HaystackSearchTests(AskbotTestCase): def tearDown(self): setattr(settings, "ENABLE_HAYSTACK_SEARCH", self._old_value) + setattr(settings, "HAYSTACK_SEARCH_ENGINE", self._old_search_engine) @skipIf('haystack' not in settings.INSTALLED_APPS, 'Haystack not setup') |