From c14bae74d5e7da70bce2163c8c81d511ea4b1d03 Mon Sep 17 00:00:00 2001 From: Georg Gadinger Date: Sun, 22 Oct 2023 18:13:26 +0200 Subject: [PATCH 1/3] attempt to add pg_search this commit is an attempt at building a search functionality using only the stuff provided by postgresql it is, unfortunately, painfully slow: PgSearch::Document Load (7869.1ms) SELECT "pg_search_documents".* FROM "pg_search_documents" INNER JOIN (SELECT "pg_search_documents"."id" AS pg_search_id, (ts_rank((to_tsvector('simple', coalesce("pg_search_documents"."content"::text, ''))), (to_tsquery('simple', ''' ' || 'awoo' || ' ''')), 0)) AS rank FROM "pg_search_documents" WHERE ((to_tsvector('simple', coalesce("pg_search_documents"."content"::text, ''))) @@ (to_tsquery('simple', ''' ' || 'awoo' || ' ''')))) AS pg_search_ce9b9dd18c5c0023f2116f ON "pg_search_documents"."id" = pg_search_ce9b9dd18c5c0023f2116f.pg_search_id ORDER BY pg_search_ce9b9dd18c5c0023f2116f.rank DESC, "pg_search_documents"."id" ASC LIMIT $1 [["LIMIT", 10]] --- Gemfile | 2 ++ Gemfile.lock | 4 ++++ app/assets/stylesheets/search.scss | 3 +++ app/controllers/search_controller.rb | 9 ++++++++ app/helpers/search_helper.rb | 2 ++ app/models/answer.rb | 3 +++ app/models/question.rb | 3 +++ app/views/search/index.haml | 23 +++++++++++++++++++ config/routes.rb | 2 ++ ...231022155815_create_pg_search_documents.rb | 17 ++++++++++++++ db/schema.rb | 12 +++++++++- spec/helpers/search_helper_spec.rb | 15 ++++++++++++ spec/requests/search_spec.rb | 11 +++++++++ spec/views/search/index.html.erb_spec.rb | 5 ++++ 14 files changed, 110 insertions(+), 1 deletion(-) create mode 100644 app/assets/stylesheets/search.scss create mode 100644 app/controllers/search_controller.rb create mode 100644 app/helpers/search_helper.rb create mode 100644 app/views/search/index.haml create mode 100644 db/migrate/20231022155815_create_pg_search_documents.rb create mode 100644 spec/helpers/search_helper_spec.rb create mode 100644 spec/requests/search_spec.rb create mode 100644 spec/views/search/index.html.erb_spec.rb diff --git a/Gemfile b/Gemfile index 63e2ae23c..b6e3712be 100644 --- a/Gemfile +++ b/Gemfile @@ -118,3 +118,5 @@ gem "openssl", "~> 3.2" gem "mail", "~> 2.7.1" gem "prometheus-client", "~> 4.2" + +gem "pg_search", "~> 2.3" diff --git a/Gemfile.lock b/Gemfile.lock index c9dfd28f0..6fdf5200c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -289,6 +289,9 @@ GEM ast (~> 2.4.1) racc pg (1.5.4) + pg_search (2.3.6) + activerecord (>= 5.2) + activesupport (>= 5.2) pghero (3.3.4) activerecord (>= 6) prometheus-client (4.2.1) @@ -542,6 +545,7 @@ DEPENDENCIES oj openssl (~> 3.2) pg + pg_search (~> 2.3) pghero prometheus-client (~> 4.2) puma diff --git a/app/assets/stylesheets/search.scss b/app/assets/stylesheets/search.scss new file mode 100644 index 000000000..1b26d4a36 --- /dev/null +++ b/app/assets/stylesheets/search.scss @@ -0,0 +1,3 @@ +// Place all the styles related to the search controller here. +// They will automatically be included in application.css. +// You can use Sass (SCSS) here: https://sass-lang.com/ diff --git a/app/controllers/search_controller.rb b/app/controllers/search_controller.rb new file mode 100644 index 000000000..bdb9a9ca5 --- /dev/null +++ b/app/controllers/search_controller.rb @@ -0,0 +1,9 @@ +class SearchController < ApplicationController + def index + @results = [] + query = params[:q] + return if query.blank? + + @results = PgSearch.multisearch(query).limit(10) + end +end diff --git a/app/helpers/search_helper.rb b/app/helpers/search_helper.rb new file mode 100644 index 000000000..b3ce20acb --- /dev/null +++ b/app/helpers/search_helper.rb @@ -0,0 +1,2 @@ +module SearchHelper +end diff --git a/app/models/answer.rb b/app/models/answer.rb index 71417252e..9f76153ac 100644 --- a/app/models/answer.rb +++ b/app/models/answer.rb @@ -1,6 +1,9 @@ class Answer < ApplicationRecord extend Answer::TimelineMethods + include PgSearch::Model + multisearchable against: [:content] + belongs_to :user, counter_cache: :answered_count belongs_to :question, counter_cache: :answer_count has_many :comments, dependent: :destroy diff --git a/app/models/question.rb b/app/models/question.rb index 0a6adec04..0ecf14be9 100644 --- a/app/models/question.rb +++ b/app/models/question.rb @@ -1,6 +1,9 @@ class Question < ApplicationRecord include Question::AnswerMethods + include PgSearch::Model + multisearchable against: [:content] + belongs_to :user, optional: true has_many :answers, dependent: :destroy has_many :inboxes, dependent: :destroy diff --git a/app/views/search/index.haml b/app/views/search/index.haml new file mode 100644 index 000000000..6f8280c93 --- /dev/null +++ b/app/views/search/index.haml @@ -0,0 +1,23 @@ +.container-lg.container--main + .row + .col-sm-10.col-md-10.col-lg-9.mx-auto + .card + .card-body + = bootstrap_form_tag layout: :inline, method: :get do |f| + = f.text_field :q, skip_label: true, append: f.primary("Search") + - unless @results.blank? + .container-lg.container--main + .row + .col-sm-10.col-md-10.col-lg-9.mx-auto + - @results.each do |result| + - case result.searchable + - when Answer + = render "answerbox", a: result.searchable, display_all: false, subscribed_answer_ids: [] + - when Question + = render "shared/question", q: result.searchable, type: nil + += render 'shared/links' + +:ruby + provide(:title, generate_title('Search')) + parent_layout 'base' diff --git a/config/routes.rb b/config/routes.rb index e5fcf8999..e3c4f3a4f 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -148,6 +148,8 @@ post "/inbox/create", to: "inbox#create", as: :inbox_create get "/inbox", to: "inbox#show", as: :inbox + get "/search", to: "search#index" + get "/user/:username", to: "user#show" get "/@:username", to: "user#show", as: :user get "/@:username/a/:id", to: "answer#show", as: :answer diff --git a/db/migrate/20231022155815_create_pg_search_documents.rb b/db/migrate/20231022155815_create_pg_search_documents.rb new file mode 100644 index 000000000..628721916 --- /dev/null +++ b/db/migrate/20231022155815_create_pg_search_documents.rb @@ -0,0 +1,17 @@ +class CreatePgSearchDocuments < ActiveRecord::Migration[6.1] + def up + say_with_time("Creating table for pg_search multisearch") do + create_table :pg_search_documents do |t| + t.text :content + t.belongs_to :searchable, polymorphic: true, index: true + t.timestamps null: false + end + end + end + + def down + say_with_time("Dropping table for pg_search multisearch") do + drop_table :pg_search_documents + end + end +end diff --git a/db/schema.rb b/db/schema.rb index ea268655c..26d80e0f1 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2023_10_18_172518) do +ActiveRecord::Schema.define(version: 2023_10_22_155815) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -127,6 +127,15 @@ t.index ["type"], name: "index_notifications_on_type" end + create_table "pg_search_documents", force: :cascade do |t| + t.text "content" + t.string "searchable_type" + t.bigint "searchable_id" + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.index ["searchable_type", "searchable_id"], name: "index_pg_search_documents_on_searchable" + end + create_table "profiles", force: :cascade do |t| t.bigint "user_id" t.string "display_name" @@ -388,5 +397,6 @@ t.index ["user_id"], name: "index_web_push_subscriptions_on_user_id" end + add_foreign_key "anonymous_blocks", "users", column: "target_user_id" add_foreign_key "profiles", "users" end diff --git a/spec/helpers/search_helper_spec.rb b/spec/helpers/search_helper_spec.rb new file mode 100644 index 000000000..74b6daf02 --- /dev/null +++ b/spec/helpers/search_helper_spec.rb @@ -0,0 +1,15 @@ +require 'rails_helper' + +# Specs in this file have access to a helper object that includes +# the SearchHelper. For example: +# +# describe SearchHelper do +# describe "string concat" do +# it "concats two strings with spaces" do +# expect(helper.concat_strings("this","that")).to eq("this that") +# end +# end +# end +RSpec.describe SearchHelper, type: :helper do + pending "add some examples to (or delete) #{__FILE__}" +end diff --git a/spec/requests/search_spec.rb b/spec/requests/search_spec.rb new file mode 100644 index 000000000..5e3640acc --- /dev/null +++ b/spec/requests/search_spec.rb @@ -0,0 +1,11 @@ +require 'rails_helper' + +RSpec.describe "Searches", type: :request do + describe "GET /index" do + it "returns http success" do + get "/search/index" + expect(response).to have_http_status(:success) + end + end + +end diff --git a/spec/views/search/index.html.erb_spec.rb b/spec/views/search/index.html.erb_spec.rb new file mode 100644 index 000000000..bdeb4dc21 --- /dev/null +++ b/spec/views/search/index.html.erb_spec.rb @@ -0,0 +1,5 @@ +require 'rails_helper' + +RSpec.describe "search/index.html.erb", type: :view do + pending "add some examples to (or delete) #{__FILE__}" +end From d5c1e66b1e69cb240a4b207fecc926c21e4f93c5 Mon Sep 17 00:00:00 2001 From: Georg Gadinger Date: Sun, 22 Oct 2023 18:34:05 +0200 Subject: [PATCH 2/3] remove pg_search --- Gemfile | 1 - Gemfile.lock | 4 ---- app/controllers/search_controller.rb | 2 +- app/models/answer.rb | 3 --- app/models/question.rb | 3 --- ...20231022155815_create_pg_search_documents.rb | 17 ----------------- 6 files changed, 1 insertion(+), 29 deletions(-) delete mode 100644 db/migrate/20231022155815_create_pg_search_documents.rb diff --git a/Gemfile b/Gemfile index b6e3712be..552a6fd5d 100644 --- a/Gemfile +++ b/Gemfile @@ -119,4 +119,3 @@ gem "mail", "~> 2.7.1" gem "prometheus-client", "~> 4.2" -gem "pg_search", "~> 2.3" diff --git a/Gemfile.lock b/Gemfile.lock index 6fdf5200c..c9dfd28f0 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -289,9 +289,6 @@ GEM ast (~> 2.4.1) racc pg (1.5.4) - pg_search (2.3.6) - activerecord (>= 5.2) - activesupport (>= 5.2) pghero (3.3.4) activerecord (>= 6) prometheus-client (4.2.1) @@ -545,7 +542,6 @@ DEPENDENCIES oj openssl (~> 3.2) pg - pg_search (~> 2.3) pghero prometheus-client (~> 4.2) puma diff --git a/app/controllers/search_controller.rb b/app/controllers/search_controller.rb index bdb9a9ca5..38cb84c65 100644 --- a/app/controllers/search_controller.rb +++ b/app/controllers/search_controller.rb @@ -4,6 +4,6 @@ def index query = params[:q] return if query.blank? - @results = PgSearch.multisearch(query).limit(10) + @results = [] end end diff --git a/app/models/answer.rb b/app/models/answer.rb index 9f76153ac..71417252e 100644 --- a/app/models/answer.rb +++ b/app/models/answer.rb @@ -1,9 +1,6 @@ class Answer < ApplicationRecord extend Answer::TimelineMethods - include PgSearch::Model - multisearchable against: [:content] - belongs_to :user, counter_cache: :answered_count belongs_to :question, counter_cache: :answer_count has_many :comments, dependent: :destroy diff --git a/app/models/question.rb b/app/models/question.rb index 0ecf14be9..0a6adec04 100644 --- a/app/models/question.rb +++ b/app/models/question.rb @@ -1,9 +1,6 @@ class Question < ApplicationRecord include Question::AnswerMethods - include PgSearch::Model - multisearchable against: [:content] - belongs_to :user, optional: true has_many :answers, dependent: :destroy has_many :inboxes, dependent: :destroy diff --git a/db/migrate/20231022155815_create_pg_search_documents.rb b/db/migrate/20231022155815_create_pg_search_documents.rb deleted file mode 100644 index 628721916..000000000 --- a/db/migrate/20231022155815_create_pg_search_documents.rb +++ /dev/null @@ -1,17 +0,0 @@ -class CreatePgSearchDocuments < ActiveRecord::Migration[6.1] - def up - say_with_time("Creating table for pg_search multisearch") do - create_table :pg_search_documents do |t| - t.text :content - t.belongs_to :searchable, polymorphic: true, index: true - t.timestamps null: false - end - end - end - - def down - say_with_time("Dropping table for pg_search multisearch") do - drop_table :pg_search_documents - end - end -end From 44871cbf4a99fd21f991009c53412f1b938bb940 Mon Sep 17 00:00:00 2001 From: Georg Gadinger Date: Sun, 22 Oct 2023 19:39:18 +0200 Subject: [PATCH 3/3] add meilisearch this thing is way too fast! only downside is that indexing takes a bit longer, and the search indexes are big (16Gi for 2.7 million records) i have no idea how to properly integrate it in the UI, but it seems promising :^) --- Gemfile | 1 + Gemfile.lock | 5 +++++ Procfile.dev | 1 + app/controllers/search_controller.rb | 32 +++++++++++++++++++++++++--- app/models/answer.rb | 6 ++++++ app/models/question.rb | 6 ++++++ app/views/search/index.haml | 11 +++++----- config/initializers/meilisearch.rb | 8 +++++++ db/schema.rb | 12 +---------- 9 files changed, 63 insertions(+), 19 deletions(-) create mode 100644 config/initializers/meilisearch.rb diff --git a/Gemfile b/Gemfile index 552a6fd5d..b66cc87d1 100644 --- a/Gemfile +++ b/Gemfile @@ -119,3 +119,4 @@ gem "mail", "~> 2.7.1" gem "prometheus-client", "~> 4.2" +gem "meilisearch-rails", "~> 0.10.1" diff --git a/Gemfile.lock b/Gemfile.lock index c9dfd28f0..ea379f909 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -249,6 +249,10 @@ GEM mail (2.7.1) mini_mime (>= 0.1.1) marcel (1.0.2) + meilisearch (0.25.1) + httparty (>= 0.17.1, < 0.22.0) + meilisearch-rails (0.10.1) + meilisearch (~> 0.25.0) method_source (1.0.0) mime-types (3.5.1) mime-types-data (~> 3.2015) @@ -535,6 +539,7 @@ DEPENDENCIES letter_opener lograge mail (~> 2.7.1) + meilisearch-rails (~> 0.10.1) mini_magick net-imap net-pop diff --git a/Procfile.dev b/Procfile.dev index bd494ae84..201c9d43a 100644 --- a/Procfile.dev +++ b/Procfile.dev @@ -2,3 +2,4 @@ web: unset PORT && bin/rails server worker: bundle exec sidekiq css: yarn build:css --watch js: yarn build --watch +search: meilisearch --no-analytics --env development --db-path tmp/meilisearch/db --dump-dir tmp/meilisearch/dump --master-key justfordev42069e621 diff --git a/app/controllers/search_controller.rb b/app/controllers/search_controller.rb index 38cb84c65..a2cdf81c7 100644 --- a/app/controllers/search_controller.rb +++ b/app/controllers/search_controller.rb @@ -1,9 +1,35 @@ class SearchController < ApplicationController def index @results = [] - query = params[:q] - return if query.blank? + @query = params[:q] + return if @query.blank? - @results = [] + @results = if params[:multi_search] == "1" + multi_search_experiment + else + [*Answer.search(@query), *Question.search(@query)] + end + end + + private + + def multi_search_experiment + MeiliSearch::Rails.client.multi_search( + [Answer, Question].map do |klass| + { + q: @query, + index_uid: klass.name.to_s, + show_ranking_score: true, + } + end + )["results"].flat_map do |h| + model = h["indexUid"].constantize # bad practice! + results = model.find(h["hits"].pluck("id")).map { |r| [r.id.to_s, r] }.to_h + h["hits"].map { |hit| [hit["_rankingScore"], results[hit["id"]]] } + end + .sort_by(&:first) + .reverse + .tap { |results| Rails.logger.debug(results) } + .map(&:last) end end diff --git a/app/models/answer.rb b/app/models/answer.rb index 71417252e..e0be24751 100644 --- a/app/models/answer.rb +++ b/app/models/answer.rb @@ -1,6 +1,12 @@ class Answer < ApplicationRecord extend Answer::TimelineMethods + include MeiliSearch::Rails + + meilisearch do + attribute :content + end + belongs_to :user, counter_cache: :answered_count belongs_to :question, counter_cache: :answer_count has_many :comments, dependent: :destroy diff --git a/app/models/question.rb b/app/models/question.rb index 0a6adec04..505ec13e9 100644 --- a/app/models/question.rb +++ b/app/models/question.rb @@ -1,6 +1,12 @@ class Question < ApplicationRecord include Question::AnswerMethods + include MeiliSearch::Rails + + meilisearch do + attribute :content + end + belongs_to :user, optional: true has_many :answers, dependent: :destroy has_many :inboxes, dependent: :destroy diff --git a/app/views/search/index.haml b/app/views/search/index.haml index 6f8280c93..a1e162f1e 100644 --- a/app/views/search/index.haml +++ b/app/views/search/index.haml @@ -3,18 +3,19 @@ .col-sm-10.col-md-10.col-lg-9.mx-auto .card .card-body - = bootstrap_form_tag layout: :inline, method: :get do |f| - = f.text_field :q, skip_label: true, append: f.primary("Search") + = bootstrap_form_with url: search_path, layout: :inline, method: :get do |f| + = f.text_field :q, skip_label: true, append: f.primary("Search"), value: params[:q] + = f.check_box :multi_search, label: "Multisearch" - unless @results.blank? .container-lg.container--main .row .col-sm-10.col-md-10.col-lg-9.mx-auto - @results.each do |result| - - case result.searchable + - case result - when Answer - = render "answerbox", a: result.searchable, display_all: false, subscribed_answer_ids: [] + = render "answerbox", a: result, display_all: false, subscribed_answer_ids: [] - when Question - = render "shared/question", q: result.searchable, type: nil + = render "shared/question", q: result, type: nil = render 'shared/links' diff --git a/config/initializers/meilisearch.rb b/config/initializers/meilisearch.rb new file mode 100644 index 000000000..ac167bb17 --- /dev/null +++ b/config/initializers/meilisearch.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +return unless ENV["SEARCH_ENABLED"] == "true" + +MeiliSearch::Rails.configuration = { + meilisearch_url: ENV.fetch("MEILISEARCH_HOST", "http://localhost:7700"), + meilisearch_api_key: ENV.fetch("MEILISEARCH_API_KEY", "justfordev42069e621") +} diff --git a/db/schema.rb b/db/schema.rb index 26d80e0f1..ea268655c 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2023_10_22_155815) do +ActiveRecord::Schema.define(version: 2023_10_18_172518) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -127,15 +127,6 @@ t.index ["type"], name: "index_notifications_on_type" end - create_table "pg_search_documents", force: :cascade do |t| - t.text "content" - t.string "searchable_type" - t.bigint "searchable_id" - t.datetime "created_at", precision: 6, null: false - t.datetime "updated_at", precision: 6, null: false - t.index ["searchable_type", "searchable_id"], name: "index_pg_search_documents_on_searchable" - end - create_table "profiles", force: :cascade do |t| t.bigint "user_id" t.string "display_name" @@ -397,6 +388,5 @@ t.index ["user_id"], name: "index_web_push_subscriptions_on_user_id" end - add_foreign_key "anonymous_blocks", "users", column: "target_user_id" add_foreign_key "profiles", "users" end