diff options
-rw-r--r-- | askbot/doc/source/solr.rst | 113 | ||||
-rw-r--r-- | askbot/management/commands/askbot_build_solr_schema.py | 69 | ||||
-rw-r--r-- | askbot/management/commands/askbot_clear_index.py | 87 | ||||
-rw-r--r-- | askbot/management/commands/askbot_rebuild_index.py | 92 | ||||
-rw-r--r-- | askbot/management/commands/askbot_update_index.py | 86 | ||||
-rw-r--r-- | askbot/models/question.py | 36 | ||||
-rw-r--r-- | askbot/search/haystack/__init__.py | 108 | ||||
-rw-r--r-- | askbot/search/haystack/routers.py | 20 | ||||
-rw-r--r-- | askbot/search/haystack/searchquery.py | 53 | ||||
-rw-r--r-- | askbot/search_indexes.py | 4 | ||||
-rw-r--r-- | askbot/setup_templates/settings.py | 2 | ||||
-rw-r--r-- | askbot/setup_templates/settings.py.mustache | 2 | ||||
-rw-r--r-- | askbot/startup_procedures.py | 12 | ||||
-rw-r--r-- | askbot/templates/search/indexes/askbot/post_text.txt | 1 | ||||
-rw-r--r-- | askbot/templates/search/indexes/askbot/thread_text.txt | 6 | ||||
-rw-r--r-- | askbot/templates/search/indexes/auth/user_text.txt | 5 | ||||
-rw-r--r-- | askbot/templates/search_configuration/askbotsolr.xml | 201 | ||||
-rw-r--r-- | askbot/tests/haystack_search_tests.py | 3 |
18 files changed, 825 insertions, 75 deletions
diff --git a/askbot/doc/source/solr.rst b/askbot/doc/source/solr.rst new file mode 100644 index 00000000..0b76aa94 --- /dev/null +++ b/askbot/doc/source/solr.rst @@ -0,0 +1,113 @@ +.. _solr: + +=========================================================== +Installing Apache Solr with Apache Tomcat 7 in Ubuntu 12.04 +=========================================================== + + +This document describes the process of instalation of Apache Solr search engine in Ubuntu Server 12.04 +for askbot use. To follow this steps you must have already askbot installed and running. + +Getting the requirements +------------------------ + +We need the following packages installed:: + + sudo apt-get install tomcat7 tomcat7-admin + +We need to download Apache Solr from the `official site <http://lucene.apache.org/solr/downloads.html>`_:: + + wget http://www.bizdirusa.com/mirrors/apache/lucene/solr/3.6.2/apache-solr-3.6.2.tgz + +Then we decompress it:: + + tar -xzf solr-3.6.2.tgz + +Setting up Tomcat +----------------- + +After installing tomcat there are some configuration required to make it work. First we are going to add +Tomcat users. Edit /etc/tomcat7/tomcat-users.xml and add the following:: + + <?xml version='1.0' encoding='utf-8'?> + <tomcat-users> + <role rolename="manager"/> + <role rolename="admin"/> + <role rolename="admin-gui"/> + <role rolename="manager-gui"/> + <user username="tomcat" password="tomcat" roles="manager,admin,manager-gui,admin-gui"/> + </tomcat-users> + +This will allow you to connect to the web management interface. After doing it restart the service: + + service tomcat7 restart + +To make see if it works go to: http://youripaddress:8080/manager it will ask for your tomcat user password +described in the tomcat-users.xml + +Installing Solr under Tomcat +---------------------------- + +Extract the solr tar archive from the previous download:: + + tar -xzf solr-4.3.0.tgz + +Copy the example/ directory from the source to /opt/solr/. Open the file /opt/solr/example/conf/solrconfig.xml +and Modify the dataDir parameter as:: + + <dataDir>${solr.data.dir:/opt/solr/example/solr/data}</dataDir> + +Copy the .war file in dist directory to /opt/solr:: + + cp dist/apache-solr-3.6.2.war /opt/solr + +Create solr.xml inside of /etc/tomcat/Catalina/localhost/ with the following contents:: + + <?xml version="1.0" encoding="utf-8"?> + <Context docBase="/opt/solr/apache-solr-3.6.2.war" debug="0" crossContext="true"> + <Environment name="solr/home" type="java.lang.String" value="/opt/solr/example/solr" override="true"/> + </Context> + +Restart tomcat server:: + + service tomcat7 restart + +By now you should be able to see the "solr" application in the tomcat manager and also access it in /solr/admin. + + +Configuring Askbot with Solr +---------------------------- + +Open settings.py file and configure the following:: + + ENABLE_HAYSTACK_SEARCH = True + HAYSTACK_SEARCH_ENGINE = 'solr' + HAYSTACK_SOLR_URL = 'http://127.0.0.1:8080/solr' + +After that create the solr schema and store the output to your solr instalation:: + + python manage.py build_solr_schema > /opt/solr/example/solr/conf/schema.xml + +Restart tomcat server:: + + service tomcat7 restart + +Build the Index for the first time:: + + python manage.py rebuild_index + +The output should be something like:: + + All documents removed. + Indexing 43 people. + Indexing 101 posts. + Indexing 101 threads. + +You must be good to go after this, just restart the askbot application and test the search with haystack and solr + + +Keeping the index fresh +----------------------- + +For this we recommend to use one of haystack `third party apps <http://django-haystack.readthedocs.org/en/latest/other_apps.html>`_ that use celery, +plese check this `link <http://django-haystack.readthedocs.org/en/latest/other_apps.html>`_ for more info. diff --git a/askbot/management/commands/askbot_build_solr_schema.py b/askbot/management/commands/askbot_build_solr_schema.py new file mode 100644 index 00000000..0d44f922 --- /dev/null +++ b/askbot/management/commands/askbot_build_solr_schema.py @@ -0,0 +1,69 @@ +from optparse import make_option +import sys + +from django.core.exceptions import ImproperlyConfigured +from django.core.management.base import BaseCommand +from django.template import loader, Context +from haystack.backends.solr_backend import SolrSearchBackend +from haystack.constants import ID, DJANGO_CT, DJANGO_ID, DEFAULT_OPERATOR, DEFAULT_ALIAS + + +class Command(BaseCommand): + help = "Generates a Solr schema that reflects the indexes." + base_options = ( + make_option("-f", "--filename", action="store", type="string", dest="filename", + help='If provided, directs output to a file instead of stdout.'), + make_option("-u", "--using", action="store", type="string", dest="using", default=DEFAULT_ALIAS, + help='If provided, chooses a connection to work with.'), + make_option("-l", "--language", action="store", type="string", dest="language", default='en', + help='Language to user, in language code format') + ) + option_list = BaseCommand.option_list + base_options + + def handle(self, **options): + """Generates a Solr schema that reflects the indexes.""" + using = options.get('using') + language = options.get('language') + schema_xml = self.build_template(using=using, language=language) + + if options.get('filename'): + self.write_file(options.get('filename'), schema_xml) + else: + self.print_stdout(schema_xml) + + def build_context(self, using, language='en'): + from haystack import connections, connection_router + backend = connections[using].get_backend() + + if not isinstance(backend, SolrSearchBackend): + raise ImproperlyConfigured("'%s' isn't configured as a SolrEngine)." % backend.connection_alias) + + content_field_name, fields = backend.build_schema(connections[using].get_unified_index().all_searchfields()) + return Context({ + 'content_field_name': content_field_name, + 'fields': fields, + 'default_operator': DEFAULT_OPERATOR, + 'ID': ID, + 'DJANGO_CT': DJANGO_CT, + 'DJANGO_ID': DJANGO_ID, + 'language': language, + }) + + def build_template(self, using, language='en'): + t = loader.get_template('search_configuration/askbotsolr.xml') + c = self.build_context(using=using, language=language) + return t.render(c) + + def print_stdout(self, schema_xml): + sys.stderr.write("\n") + sys.stderr.write("\n") + sys.stderr.write("\n") + sys.stderr.write("Save the following output to 'schema.xml' and place it in your Solr configuration directory.\n") + sys.stderr.write("--------------------------------------------------------------------------------------------\n") + sys.stderr.write("\n") + print schema_xml + + def write_file(self, filename, schema_xml): + schema_file = open(filename, 'w') + schema_file.write(schema_xml) + schema_file.close() diff --git a/askbot/management/commands/askbot_clear_index.py b/askbot/management/commands/askbot_clear_index.py new file mode 100644 index 00000000..bb3b04b9 --- /dev/null +++ b/askbot/management/commands/askbot_clear_index.py @@ -0,0 +1,87 @@ +import sys +from optparse import make_option + +from django.core.management import get_commands, load_command_class +from django.utils.translation import activate as activate_language +from django.core.management.base import BaseCommand, CommandError +from django.conf import settings + +try: + from haystack.management.commands.clear_index import Command as ClearCommand + haystack_option_list = [option for option in ClearCommand.base_options if not option.get_opt_string() != '--verbosity'] +except ImportError: + haystack_option_list = [] + +class Command(BaseCommand): + help = "Completely rebuilds the search index by removing the old data and then updating." + base_options = [make_option("-l", "--language", action="store", type="string", dest="language", + help='Language to user, in language code format'),] + option_list = list(BaseCommand.option_list) + haystack_option_list + base_options + + def handle(self, **options): + lang_code = options.get('language', settings.LANGUAGE_CODE.lower()) + options['using'] = ['default_%s' % lang_code[:2],] + activate_language(lang_code) + + klass = self._get_command_class('clear_index') + klass.handle(*args, **options) + + def _get_command_class(self, name): + try: + app_name = get_commands()[name] + if isinstance(app_name, BaseCommand): + # If the command is already loaded, use it directly. + klass = app_name + else: + klass = load_command_class(app_name, name) + except KeyError: + raise CommandError("Unknown command: %r" % name) + return klass + + + def execute(self, *args, **options): + """ + Try to execute this command, performing model validation if + needed (as controlled by the attribute + ``self.requires_model_validation``). If the command raises a + ``CommandError``, intercept it and print it sensibly to + stderr. + """ + show_traceback = options.get('traceback', False) + + if self.can_import_settings: + try: + #language part used to be here + pass + except ImportError, e: + # If settings should be available, but aren't, + # raise the error and quit. + if show_traceback: + traceback.print_exc() + else: + sys.stderr.write(smart_str(self.style.ERROR('Error: %s\n' % e))) + sys.exit(1) + + try: + self.stdout = options.get('stdout', sys.stdout) + self.stderr = options.get('stderr', sys.stderr) + if self.requires_model_validation: + self.validate() + output = self.handle(*args, **options) + if output: + if self.output_transaction: + # This needs to be imported here, because it relies on + # settings. + from django.db import connections, DEFAULT_DB_ALIAS + connection = connections[options.get('database', DEFAULT_DB_ALIAS)] + if connection.ops.start_transaction_sql(): + self.stdout.write(self.style.SQL_KEYWORD(connection.ops.start_transaction_sql()) + '\n') + self.stdout.write(output) + if self.output_transaction: + self.stdout.write('\n' + self.style.SQL_KEYWORD("COMMIT;") + '\n') + except CommandError, e: + if show_traceback: + traceback.print_exc() + else: + self.stderr.write(smart_str(self.style.ERROR('Error: %s\n' % e))) + sys.exit(1) diff --git a/askbot/management/commands/askbot_rebuild_index.py b/askbot/management/commands/askbot_rebuild_index.py new file mode 100644 index 00000000..366bad53 --- /dev/null +++ b/askbot/management/commands/askbot_rebuild_index.py @@ -0,0 +1,92 @@ +import sys +from optparse import make_option + +from django.core.management import get_commands, load_command_class +from django.utils.translation import activate as activate_language +from django.core.management.base import BaseCommand, CommandError +from django.conf import settings + +try: + from haystack.management.commands.clear_index import Command as ClearCommand + from haystack.management.commands.update_index import Command as UpdateCommand + haystack_option_list = [option for option in UpdateCommand.base_options if option.get_opt_string() != '--verbosity'] + \ + [option for option in ClearCommand.base_options if not option.get_opt_string() in ['--using', '--verbosity']] +except ImportError: + haystack_option_list = [] + +class Command(BaseCommand): + help = "Completely rebuilds the search index by removing the old data and then updating." + base_options = [make_option("-l", "--language", action="store", type="string", dest="language", + help='Language to user, in language code format'),] + option_list = list(BaseCommand.option_list) + haystack_option_list + base_options + + def handle(self, *args, **options): + lang_code = options.get('language', settings.LANGUAGE_CODE.lower()) + options['using'] = ['default_%s' % lang_code[:2],] + activate_language(lang_code) + + klass = self._get_command_class('clear_index') + klass.handle(*args, **options) + + klass = self._get_command_class('update_index') + klass.handle(*args, **options) + + def _get_command_class(self, name): + try: + app_name = get_commands()[name] + if isinstance(app_name, BaseCommand): + # If the command is already loaded, use it directly. + klass = app_name + else: + klass = load_command_class(app_name, name) + except KeyError: + raise CommandError("Unknown command: %r" % name) + return klass + + + def execute(self, *args, **options): + """ + Try to execute this command, performing model validation if + needed (as controlled by the attribute + ``self.requires_model_validation``). If the command raises a + ``CommandError``, intercept it and print it sensibly to + stderr. + """ + show_traceback = options.get('traceback', False) + + if self.can_import_settings: + try: + #language part used to be here + pass + except ImportError, e: + # If settings should be available, but aren't, + # raise the error and quit. + if show_traceback: + traceback.print_exc() + else: + sys.stderr.write(smart_str(self.style.ERROR('Error: %s\n' % e))) + sys.exit(1) + + try: + self.stdout = options.get('stdout', sys.stdout) + self.stderr = options.get('stderr', sys.stderr) + if self.requires_model_validation: + self.validate() + output = self.handle(*args, **options) + if output: + if self.output_transaction: + # This needs to be imported here, because it relies on + # settings. + from django.db import connections, DEFAULT_DB_ALIAS + connection = connections[options.get('database', DEFAULT_DB_ALIAS)] + if connection.ops.start_transaction_sql(): + self.stdout.write(self.style.SQL_KEYWORD(connection.ops.start_transaction_sql()) + '\n') + self.stdout.write(output) + if self.output_transaction: + self.stdout.write('\n' + self.style.SQL_KEYWORD("COMMIT;") + '\n') + except CommandError, e: + if show_traceback: + traceback.print_exc() + else: + self.stderr.write(smart_str(self.style.ERROR('Error: %s\n' % e))) + sys.exit(1) diff --git a/askbot/management/commands/askbot_update_index.py b/askbot/management/commands/askbot_update_index.py new file mode 100644 index 00000000..2499878d --- /dev/null +++ b/askbot/management/commands/askbot_update_index.py @@ -0,0 +1,86 @@ +import sys +from optparse import make_option + +from django.core.management import get_commands, load_command_class +from django.utils.translation import activate as activate_language +from django.core.management.base import BaseCommand, CommandError +from django.conf import settings + +try: + from haystack.management.commands.update_index import Command as UpdateCommand + haystack_option_list = [option for option in UpdateCommand.base_options if option.get_opt_string() != '--verbosity'] +except ImportError: + haystack_option_list = [] + +class Command(BaseCommand): + help = "Completely rebuilds the search index by removing the old data and then updating." + base_options = [make_option("-l", "--language", action="store", type="string", dest="language", + help='Language to user, in language code format'),] + option_list = list(BaseCommand.option_list) + haystack_option_list + base_options + + def handle(self, **options): + lang_code = options.get('language', settings.LANGUAGE_CODE.lower()) + activate_language(lang_code) + options['using'] = ['default_%s' % lang_code[:2],] + klass = self._get_command_class('update_index') + klass.handle(*args, **options) + + def _get_command_class(self, name): + try: + app_name = get_commands()[name] + if isinstance(app_name, BaseCommand): + # If the command is already loaded, use it directly. + klass = app_name + else: + klass = load_command_class(app_name, name) + except KeyError: + raise CommandError("Unknown command: %r" % name) + return klass + + + def execute(self, *args, **options): + """ + Try to execute this command, performing model validation if + needed (as controlled by the attribute + ``self.requires_model_validation``). If the command raises a + ``CommandError``, intercept it and print it sensibly to + stderr. + """ + show_traceback = options.get('traceback', False) + + if self.can_import_settings: + try: + #language part used to be here + pass + except ImportError, e: + # If settings should be available, but aren't, + # raise the error and quit. + if show_traceback: + traceback.print_exc() + else: + sys.stderr.write(smart_str(self.style.ERROR('Error: %s\n' % e))) + sys.exit(1) + + try: + self.stdout = options.get('stdout', sys.stdout) + self.stderr = options.get('stderr', sys.stderr) + if self.requires_model_validation: + self.validate() + output = self.handle(*args, **options) + if output: + if self.output_transaction: + # This needs to be imported here, because it relies on + # settings. + from django.db import connections, DEFAULT_DB_ALIAS + connection = connections[options.get('database', DEFAULT_DB_ALIAS)] + if connection.ops.start_transaction_sql(): + self.stdout.write(self.style.SQL_KEYWORD(connection.ops.start_transaction_sql()) + '\n') + self.stdout.write(output) + if self.output_transaction: + self.stdout.write('\n' + self.style.SQL_KEYWORD("COMMIT;") + '\n') + except CommandError, e: + if show_traceback: + traceback.print_exc() + else: + self.stderr.write(smart_str(self.style.ERROR('Error: %s\n' % e))) + sys.exit(1) diff --git a/askbot/models/question.py b/askbot/models/question.py index 70060eb2..ed30c5f5 100644 --- a/askbot/models/question.py +++ b/askbot/models/question.py @@ -48,24 +48,30 @@ class ThreadQuerySet(models.query.QuerySet): todo: possibly add tags todo: implement full text search on relevant fields """ - db_engine_name = askbot.get_database_engine_name() - filter_parameters = {'deleted': False} - if 'postgresql_psycopg2' in db_engine_name: - from askbot.search import postgresql - return postgresql.run_title_search( - self, search_query - ).filter( - **filter_parameters - ).order_by('-relevance') - elif 'mysql' in db_engine_name and mysql.supports_full_text_search(): - filter_parameters['title__search'] = search_query + + if getattr(django_settings, 'ENABLE_HAYSTACK_SEARCH', False): + from askbot.search.haystack import AskbotSearchQuerySet + hs_qs = AskbotSearchQuerySet().filter(content=search_query).models(self.model) + return hs_qs.get_django_queryset() else: - filter_parameters['title__icontains'] = search_query + db_engine_name = askbot.get_database_engine_name() + filter_parameters = {'deleted': False} + if 'postgresql_psycopg2' in db_engine_name: + from askbot.search import postgresql + return postgresql.run_title_search( + self, search_query + ).filter( + **filter_parameters + ).order_by('-relevance') + elif 'mysql' in db_engine_name and mysql.supports_full_text_search(): + filter_parameters['title__search'] = search_query + else: + filter_parameters['title__icontains'] = search_query - if getattr(django_settings, 'ASKBOT_MULTILINGUAL', False): - filter_parameters['language_code'] = get_language() + if getattr(django_settings, 'ASKBOT_MULTILINGUAL', False): + filter_parameters['language_code'] = get_language() - return self.filter(**filter_parameters) + return self.filter(**filter_parameters) class ThreadManager(BaseQuerySetManager): diff --git a/askbot/search/haystack/__init__.py b/askbot/search/haystack/__init__.py index 71f04d00..45a037c9 100644 --- a/askbot/search/haystack/__init__.py +++ b/askbot/search/haystack/__init__.py @@ -1,59 +1,57 @@ -try: - from haystack import indexes, site - from haystack.query import SearchQuerySet - from askbot.models import Post, Thread, User - - - class ThreadIndex(indexes.SearchIndex): - text = indexes.CharField(document=True, use_template=True) - title = indexes.CharField(model_attr='title') - post_text = indexes.CharField(model_attr='posts__text__search') - - def index_queryset(self): - return Thread.objects.filter(posts__deleted=False) - - def prepare(self, obj): - self.prepared_data = super(ThreadIndex, self).prepare(object) - - self.prepared_data['tags'] = [tag.name for tag in objects.tags.all()] - - class PostIndex(indexes.SearchIndex): - text = indexes.CharField(document=True, use_template=True) - post_text = indexes.CharField(model_attr='text') - author = indexes.CharField(model_attr='user') - thread_id = indexes.CharField(model_attr='thread') - - def index_queryset(self): - return Post.objects.filter(deleted=False) +from django.conf import settings +from django.utils.translation import get_language - class UserIndex(indexes.SearchIndex): - text = indexes.CharField(document=True, use_template=True) - - def index_queryset(self): - return User.objects.all() - - site.register(Post, PostIndex) - site.register(Thread, ThreadIndex) - site.register(User, UserIndex) - - class AskbotSearchQuerySet(SearchQuerySet): - - def get_django_queryset(self, model_klass=Thread): - '''dirty hack because models() method from the - SearchQuerySet does not work </3''' - id_list = [] - for r in self: - if r.model_name in ['thread','post'] \ - and model_klass._meta.object_name.lower() == 'thread': - if getattr(r, 'thread_id'): - id_list.append(r.thread_id) - else: - id_list.append(r.pk) - elif r.model_name == model_klass._meta.object_name.lower(): - #FIXME: add a highlight here? - id_list.append(r.pk) - - return model_klass.objects.filter(id__in=set(id_list)) +from haystack import indexes +try: + from searchquery import AskbotSearchQuerySet except: pass + +class ThreadIndex(indexes.SearchIndex, indexes.Indexable): + text = indexes.CharField(document=True, use_template=True) + title = indexes.CharField() + tags = indexes.MultiValueField() + + def get_model(self): + from askbot.models import Thread + return Thread + + def index_queryset(self, using=None): + if getattr(settings, 'ASKBOT_MULTILINGUAL', True): + lang_code = get_language()[:2] + return self.get_model().objects.filter(language_code=lang_code, + posts__deleted=False) + else: + return self.get_model().objects.filter(posts__deleted=False) + + def prepare_tags(self, obj): + return [tag.name for tag in obj.tags.all()] + +class PostIndex(indexes.SearchIndex, indexes.Indexable): + text = indexes.CharField(document=True, use_template=True) + post_text = indexes.CharField(model_attr='text') + author = indexes.CharField() + thread_id = indexes.IntegerField(model_attr='thread__pk') + + def get_model(self): + from askbot.models import Post + return Post + + def index_queryset(self, using=None): + if getattr(settings, 'ASKBOT_MULTILINGUAL', True): + lang_code = get_language()[:2] + return self.get_model().objects.filter(language_code=lang_code, + deleted=False) + else: + return self.get_model().objects.filter(deleted=False) + +class UserIndex(indexes.SearchIndex, indexes.Indexable): + text = indexes.CharField(document=True, use_template=True) + + def get_model(self): + from askbot.models import User + return User + + def index_queryset(self, using=None): + return self.get_model().objects.all() diff --git a/askbot/search/haystack/routers.py b/askbot/search/haystack/routers.py new file mode 100644 index 00000000..7b7e0a4b --- /dev/null +++ b/askbot/search/haystack/routers.py @@ -0,0 +1,20 @@ +from django.utils.translation import get_language + +from haystack.routers import BaseRouter +from haystack.constants import DEFAULT_ALIAS + +class LanguageRouter(BaseRouter): + + def for_read(self, **hints): + from django.conf import settings + if getattr(settings, 'ASKBOT_MULTILINGUAL'): + return 'default_' + get_language()[:2] + else: + return DEFAULT_ALIAS + + def for_write(self, **hints): + from django.conf import settings + if getattr(settings, 'ASKBOT_MULTILINGUAL'): + return 'default_' + get_language()[:2] + else: + return DEFAULT_ALIAS diff --git a/askbot/search/haystack/searchquery.py b/askbot/search/haystack/searchquery.py new file mode 100644 index 00000000..116f4990 --- /dev/null +++ b/askbot/search/haystack/searchquery.py @@ -0,0 +1,53 @@ +from askbot.models import Thread, User +from haystack.query import SearchQuerySet + +class AskbotSearchQuerySet(SearchQuerySet): + + def _determine_backend(self): + '''This is a hack somehow connection_router got wrong values + from setting and did not loaded the LanguageRouter''' + + from haystack import connections, connection_router + # A backend has been manually selected. Use it instead. + if self._using is not None: + self.query = connections[self._using].get_query() + return + + # No backend, so rely on the routers to figure out what's right. + hints = {} + + if self.query: + hints['models'] = self.query.models + + backend_alias = connection_router.for_read(**hints) + + if isinstance(backend_alias, (list, tuple)) and len(backend_alias): + # We can only effectively read from one engine. + backend_alias = backend_alias[0] + + # The ``SearchQuery`` might swap itself out for a different variant + # here. + if self.query: + self.query = self.query.using(backend_alias) + else: + self.query = connections[backend_alias].get_query() + + def get_django_queryset(self, model_klass=Thread): + '''dirty hack because models() method from the + SearchQuerySet does not work </3''' + id_list = [] + for r in self: + if r.model_name in ['thread','post'] \ + and model_klass._meta.object_name.lower() == 'thread': + if getattr(r, 'thread_id'): + id_list.append(r.thread_id) + else: + id_list.append(r.pk) + elif r.model_name == model_klass._meta.object_name.lower(): + #FIXME: add a highlight here? + id_list.append(r.pk) + + if model_klass == User: + return model_klass.objects.filter(id__in=set(id_list)) + else: + return model_klass.objects.filter(id__in=set(id_list)) diff --git a/askbot/search_indexes.py b/askbot/search_indexes.py new file mode 100644 index 00000000..f2674996 --- /dev/null +++ b/askbot/search_indexes.py @@ -0,0 +1,4 @@ +from django.conf import settings + +if getattr(settings, 'ENABLE_HAYSTACK_SEARCH'): + from askbot.search.haystack import UserIndex, ThreadIndex, PostIndex diff --git a/askbot/setup_templates/settings.py b/askbot/setup_templates/settings.py index 1427e506..e5b21d92 100644 --- a/askbot/setup_templates/settings.py +++ b/askbot/setup_templates/settings.py @@ -258,6 +258,8 @@ RECAPTCHA_USE_SSL = True #HAYSTACK_SETTINGS ENABLE_HAYSTACK_SEARCH = False HAYSTACK_SITECONF = 'askbot.search.haystack' +#if you set this to True it can fail +HAYSTACK_ENABLE_REGISTRATIONS = False #more information #http://django-haystack.readthedocs.org/en/v1.2.7/settings.html HAYSTACK_SEARCH_ENGINE = 'simple' diff --git a/askbot/setup_templates/settings.py.mustache b/askbot/setup_templates/settings.py.mustache index f30297d7..f59dd735 100644 --- a/askbot/setup_templates/settings.py.mustache +++ b/askbot/setup_templates/settings.py.mustache @@ -256,6 +256,8 @@ RECAPTCHA_USE_SSL = True #HAYSTACK_SETTINGS ENABLE_HAYSTACK_SEARCH = False HAYSTACK_SITECONF = 'askbot.search.haystack' +#if you set this to True it can fail +HAYSTACK_ENABLE_REGISTRATIONS = False #more information #http://django-haystack.readthedocs.org/en/v1.2.7/settings.html HAYSTACK_SEARCH_ENGINE = 'simple' diff --git a/askbot/startup_procedures.py b/askbot/startup_procedures.py index b7ad47bf..8f4ea2ac 100644 --- a/askbot/startup_procedures.py +++ b/askbot/startup_procedures.py @@ -622,6 +622,12 @@ def test_haystack(): if not hasattr(django_settings, 'HAYSTACK_SITECONF'): message = 'Please add HAYSTACK_SITECONF = "askbot.search.haystack"' errors.append(message) + if not hasattr(django_settings, 'HAYSTACK_ENABLE_REGISTRATIONS'): + message = 'Please add "HAYSTACK_ENABLE_REGISTRATIONS = False"' + errors.append(message) + elif getattr(django_settings, 'HAYSTACK_ENABLE_REGISTRATIONS'): + message = 'Please set "HAYSTACK_ENABLE_REGISTRATIONS = False"' + errors.append(message) footer = 'Please refer to haystack documentation at http://django-haystack.readthedocs.org/en/v1.2.7/settings.html#haystack-search-engine' print_errors(errors, footer=footer) @@ -919,7 +925,7 @@ def run_startup_tests(): test_compressor() test_custom_user_profile_tab() test_group_messaging() - test_haystack() + #test_haystack() test_jinja2() test_longerusername() test_new_skins() @@ -965,10 +971,6 @@ def run_startup_tests(): 'value': True, 'message': 'Please add: RECAPTCHA_USE_SSL = True' }, - 'HAYSTACK_SITECONF': { - 'value': 'askbot.search.haystack', - 'message': 'Please add: HAYSTACK_SITECONF = "askbot.search.haystack"' - } }) settings_tester.run() if 'manage.py test' in ' '.join(sys.argv): diff --git a/askbot/templates/search/indexes/askbot/post_text.txt b/askbot/templates/search/indexes/askbot/post_text.txt new file mode 100644 index 00000000..b373ccf7 --- /dev/null +++ b/askbot/templates/search/indexes/askbot/post_text.txt @@ -0,0 +1 @@ +{{ object.text }} diff --git a/askbot/templates/search/indexes/askbot/thread_text.txt b/askbot/templates/search/indexes/askbot/thread_text.txt new file mode 100644 index 00000000..0db5dba0 --- /dev/null +++ b/askbot/templates/search/indexes/askbot/thread_text.txt @@ -0,0 +1,6 @@ +{{ object.title }} +{{ object.tags }} + +{% for tag in object.tags.all() %} +{{ tag.name }} +{% endfor %} diff --git a/askbot/templates/search/indexes/auth/user_text.txt b/askbot/templates/search/indexes/auth/user_text.txt new file mode 100644 index 00000000..b0baa3a6 --- /dev/null +++ b/askbot/templates/search/indexes/auth/user_text.txt @@ -0,0 +1,5 @@ +{{ object.username }} +{{ object.first_name }} +{{ object.last_name }} +{{ object.email }} +{{ object.full_name }} diff --git a/askbot/templates/search_configuration/askbotsolr.xml b/askbot/templates/search_configuration/askbotsolr.xml new file mode 100644 index 00000000..9eb61d43 --- /dev/null +++ b/askbot/templates/search_configuration/askbotsolr.xml @@ -0,0 +1,201 @@ +<?xml version="1.0" ?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<schema name="default" version="1.4"> + <types> + <fieldtype name="string" class="solr.StrField" sortMissingLast="true" omitNorms="true"/> + <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true" omitNorms="true"/> + <fieldtype name="binary" class="solr.BinaryField"/> + + <!-- Numeric field types that manipulate the value into + a string value that isn't human-readable in its internal form, + but with a lexicographic ordering the same as the numeric ordering, + so that range queries work correctly. --> + <fieldType name="int" class="solr.TrieIntField" precisionStep="0" omitNorms="true" sortMissingLast="true" positionIncrementGap="0"/> + <fieldType name="float" class="solr.TrieFloatField" precisionStep="0" omitNorms="true" sortMissingLast="true" positionIncrementGap="0"/> + <fieldType name="long" class="solr.TrieLongField" precisionStep="0" omitNorms="true" sortMissingLast="true" positionIncrementGap="0"/> + <fieldType name="double" class="solr.TrieDoubleField" precisionStep="0" omitNorms="true" sortMissingLast="true" positionIncrementGap="0"/> + + <fieldType name="tint" class="solr.TrieIntField" precisionStep="8" omitNorms="true" positionIncrementGap="0"/> + <fieldType name="tfloat" class="solr.TrieFloatField" precisionStep="8" omitNorms="true" positionIncrementGap="0"/> + <fieldType name="tlong" class="solr.TrieLongField" precisionStep="8" omitNorms="true" positionIncrementGap="0"/> + <fieldType name="tdouble" class="solr.TrieDoubleField" precisionStep="8" omitNorms="true" positionIncrementGap="0"/> + + <fieldType name="date" class="solr.TrieDateField" omitNorms="true" precisionStep="0" positionIncrementGap="0"/> + <!-- A Trie based date field for faster date range queries and date faceting. --> + <fieldType name="tdate" class="solr.TrieDateField" omitNorms="true" precisionStep="6" positionIncrementGap="0"/> + + <fieldType name="point" class="solr.PointType" dimension="2" subFieldSuffix="_d"/> + <fieldType name="location" class="solr.LatLonType" subFieldSuffix="_coordinate"/> + <fieldtype name="geohash" class="solr.GeoHashField"/> + + <fieldType name="text_general" class="solr.TextField" positionIncrementGap="100"> + <analyzer type="index"> + {% if language == 'en' %} + <tokenizer class="solr.StandardTokenizerFactory"/> + <filter class="solr.LowerCaseFilterFactory"/> + <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" /> + {% elif language == 'es' %} + <filter class="solr.LowerCaseFilterFactory"/> + <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_es.txt" enablePositionIncrements="true" /> + <filter class="solr.SnowballPorterFilterFactory" language="Spanish" /> + {% elif language == 'es' %} + <filter class="solr.LowerCaseFilterFactory"/> + <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_ru.txt" enablePositionIncrements="true" /> + <filter class="solr.SnowballPorterFilterFactory" language="Russian" /> + {% elif language == 'fr' %} + <filter class="solr.LowerCaseFilterFactory"/> + <filter class="solr.ElisionFilterFactory"/> + <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_fr.txt" enablePositionIncrements="true" /> + <filter class="solr.SnowballPorterFilterFactory" language="French" /> + {% elif language == 'cn' %} + <tokenizer class="solr.SmartChineseSentenceTokenizerFactory"/> + <filter class="solr.SmartChineseWordTokenFilterFactory"/> + <filter class="solr.LowerCaseFilterFactory"/> + <filter class="solr.PositionFilterFactory" /> + {% elif language in ('jp', 'ko') %} + <tokenizer class="solr.CJKTokenizerFactory"/> + {% endif %} + </analyzer> + <analyzer type="query"> + {% if language == 'en' %} + <tokenizer class="solr.StandardTokenizerFactory"/> + <filter class="solr.LowerCaseFilterFactory"/> + <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" /> + {% elif language == 'es' %} + <filter class="solr.LowerCaseFilterFactory"/> + <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_es.txt" enablePositionIncrements="true" /> + <filter class="solr.SnowballPorterFilterFactory" language="Spanish" /> + {% elif language == 'es' %} + <filter class="solr.LowerCaseFilterFactory"/> + <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_ru.txt" enablePositionIncrements="true" /> + <filter class="solr.SnowballPorterFilterFactory" language="Russian" /> + {% elif language == 'fr' %} + <filter class="solr.LowerCaseFilterFactory"/> + <filter class="solr.ElisionFilterFactory"/> + <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_fr.txt" enablePositionIncrements="true" /> + <filter class="solr.SnowballPorterFilterFactory" language="French" /> + {% elif language == 'cn' %} + <tokenizer class="solr.SmartChineseSentenceTokenizerFactory"/> + <filter class="solr.SmartChineseWordTokenFilterFactory"/> + <filter class="solr.LowerCaseFilterFactory"/> + <filter class="solr.PositionFilterFactory" /> + {% elif language in ('jp', 'ko') %} + <tokenizer class="solr.CJKTokenizerFactory"/> + {% endif %} + </analyzer> + </fieldType> + + <fieldType name="text_en" class="solr.TextField" positionIncrementGap="100"> + <analyzer type="index"> + <tokenizer class="solr.StandardTokenizerFactory"/> + <filter class="solr.StopFilterFactory" + ignoreCase="true" + words="stopwords_en.txt" + enablePositionIncrements="true" + /> + <filter class="solr.LowerCaseFilterFactory"/> + <filter class="solr.EnglishPossessiveFilterFactory"/> + <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/> + <!-- Optionally you may want to use this less aggressive stemmer instead of PorterStemFilterFactory: + <filter class="solr.EnglishMinimalStemFilterFactory"/> + --> + <filter class="solr.PorterStemFilterFactory"/> + </analyzer> + <analyzer type="query"> + <tokenizer class="solr.StandardTokenizerFactory"/> + <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/> + <filter class="solr.StopFilterFactory" + ignoreCase="true" + words="stopwords_en.txt" + enablePositionIncrements="true" + /> + <filter class="solr.LowerCaseFilterFactory"/> + <filter class="solr.EnglishPossessiveFilterFactory"/> + <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/> + <!-- Optionally you may want to use this less aggressive stemmer instead of PorterStemFilterFactory: + <filter class="solr.EnglishMinimalStemFilterFactory"/> + --> + <filter class="solr.PorterStemFilterFactory"/> + </analyzer> + </fieldType> + + <fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100"> + <analyzer> + <tokenizer class="solr.WhitespaceTokenizerFactory"/> + </analyzer> + </fieldType> + + <fieldType name="ngram" class="solr.TextField" > + <analyzer type="index"> + <tokenizer class="solr.KeywordTokenizerFactory"/> + <filter class="solr.LowerCaseFilterFactory"/> + <filter class="solr.NGramFilterFactory" minGramSize="3" maxGramSize="15" /> + </analyzer> + <analyzer type="query"> + <tokenizer class="solr.KeywordTokenizerFactory"/> + <filter class="solr.LowerCaseFilterFactory"/> + </analyzer> + </fieldType> + + <fieldType name="edge_ngram" class="solr.TextField" positionIncrementGap="1"> + <analyzer type="index"> + <tokenizer class="solr.WhitespaceTokenizerFactory" /> + <filter class="solr.LowerCaseFilterFactory" /> + <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"/> + <filter class="solr.EdgeNGramFilterFactory" minGramSize="2" maxGramSize="15" side="front" /> + </analyzer> + <analyzer type="query"> + <tokenizer class="solr.WhitespaceTokenizerFactory" /> + <filter class="solr.LowerCaseFilterFactory" /> + <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"/> + </analyzer> + </fieldType> + </types> + + <fields> + <!-- general --> + <field name="{{ ID }}" type="string" indexed="true" stored="true" multiValued="false" required="true"/> + <field name="{{ DJANGO_CT }}" type="string" indexed="true" stored="true" multiValued="false"/> + <field name="{{ DJANGO_ID }}" type="string" indexed="true" stored="true" multiValued="false"/> + + <dynamicField name="*_i" type="int" indexed="true" stored="true"/> + <dynamicField name="*_s" type="string" indexed="true" stored="true"/> + <dynamicField name="*_l" type="long" indexed="true" stored="true"/> + <dynamicField name="*_t" type="text_en" indexed="true" stored="true"/> + <dynamicField name="*_b" type="boolean" indexed="true" stored="true"/> + <dynamicField name="*_f" type="float" indexed="true" stored="true"/> + <dynamicField name="*_d" type="double" indexed="true" stored="true"/> + <dynamicField name="*_dt" type="date" indexed="true" stored="true"/> + <dynamicField name="*_p" type="location" indexed="true" stored="true"/> + <dynamicField name="*_coordinate" type="tdouble" indexed="true" stored="false"/> + +{% for field in fields %} + <field name="{{ field.field_name }}" type="{{ field.type }}" indexed="{{ field.indexed }}" stored="{{ field.stored }}" multiValued="{{ field.multi_valued }}" /> +{% endfor %} + </fields> + + <!-- field to use to determine and enforce document uniqueness. --> + <uniqueKey>{{ ID }}</uniqueKey> + + <!-- field for the QueryParser to use when an explicit fieldname is absent --> + <defaultSearchField>{{ content_field_name }}</defaultSearchField> + + <!-- SolrQueryParser configuration: defaultOperator="AND|OR" --> + <solrQueryParser defaultOperator="{{ default_operator }}"/> +</schema> diff --git a/askbot/tests/haystack_search_tests.py b/askbot/tests/haystack_search_tests.py index 7a8bfcfd..c256c533 100644 --- a/askbot/tests/haystack_search_tests.py +++ b/askbot/tests/haystack_search_tests.py @@ -12,7 +12,9 @@ class HaystackSearchTests(AskbotTestCase): """ def setUp(self): self._old_value = getattr(settings, 'ENABLE_HAYSTACK_SEARCH', False) + self._old_search_engine = getattr(settings, 'HAYSTACK_SEARCH_ENGINE', 'simple') setattr(settings, "ENABLE_HAYSTACK_SEARCH", True) + setattr(settings, "HAYSTACK_SEARCH_ENGINE", 'simple') self.user = self.create_user(username='gepeto') self.other_user = self.create_user(username = 'pinocho') @@ -53,6 +55,7 @@ class HaystackSearchTests(AskbotTestCase): def tearDown(self): setattr(settings, "ENABLE_HAYSTACK_SEARCH", self._old_value) + setattr(settings, "HAYSTACK_SEARCH_ENGINE", self._old_search_engine) @skipIf('haystack' not in settings.INSTALLED_APPS, 'Haystack not setup') |