From 5d6cb6a4591978a767716abbc816158da5e24cc7 Mon Sep 17 00:00:00 2001 From: Oneric Date: Thu, 2 May 2024 22:44:41 +0200 Subject: [PATCH 1/4] meilisearch: remove duplicate preload --- lib/pleroma/search/meilisearch.ex | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 8fcf9310a..cecb7bd00 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -93,7 +93,6 @@ defmodule Pleroma.Search.Meilisearch do hits |> Activity.create_by_object_ap_id() |> Activity.with_preloaded_object() - |> Activity.with_preloaded_object() |> Activity.restrict_deactivated_users() |> maybe_restrict_local(user) |> maybe_restrict_author(author) From 65aeaefa417d971d05f660a18f683b65d9a7bffb Mon Sep 17 00:00:00 2001 From: Oneric Date: Thu, 2 May 2024 22:44:48 +0200 Subject: [PATCH 2/4] =?UTF-8?q?meilisearch:=20respect=20meili=E2=80=99s=20?= =?UTF-8?q?result=20ranking?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Meilisearch is already configured to return results sorted by a particular ranking configured in the meilisearch CLI task. Resorting the returned top results by date partially negates this and runs counter to what someone with tweaked settings expects. Issue and fix identified by AdamK2003 in https://akkoma.dev/AkkomaGang/akkoma/pulls/579 But instead of using a O(n^2) resorting, this commit directly retrieves results in the correct order from the database. Closes: https://akkoma.dev/AkkomaGang/akkoma/pulls/579 --- CHANGELOG.md | 3 +++ lib/pleroma/activity.ex | 21 +++++++++++++++++++++ lib/pleroma/search/meilisearch.ex | 4 +--- test/pleroma/activity_test.exs | 20 ++++++++++++++++++++ 4 files changed, 45 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d3034464..c7a838612 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Added - Implement [FEP-67ff](https://codeberg.org/fediverse/fep/src/branch/main/fep/67ff/fep-67ff.md) (federation documentation) +## Fixed +- Meilisearch: order of results returned from our REST API now actually matches how Meilisearch ranks results + ## 2024.04 ## Added diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index bf851f808..f820cbdae 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -258,6 +258,27 @@ defmodule Pleroma.Activity do def get_create_by_object_ap_id(_), do: nil + @doc """ + Accepts a list of `ap__id`. + Returns a query yielding Create activities for the given objects, + in the same order as they were specified in the input list. + """ + @spec get_presorted_create_by_object_ap_id([String.t()]) :: Ecto.Queryable.t() + def get_presorted_create_by_object_ap_id(ap_ids) do + from( + a in Activity, + join: + ids in fragment( + "SELECT * FROM UNNEST(?::text[]) WITH ORDINALITY AS ids(ap_id, ord)", + ^ap_ids + ), + on: + ids.ap_id == fragment("?->>'object'", a.data) and + fragment("?->>'type'", a.data) == "Create", + order_by: [asc: ids.ord] + ) + end + @doc """ Accepts `ap_id` or list of `ap_id`. Returns a query. diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index cecb7bd00..822c95b4a 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -5,7 +5,6 @@ defmodule Pleroma.Search.Meilisearch do alias Pleroma.Activity import Pleroma.Search.DatabaseSearch - import Ecto.Query @behaviour Pleroma.Search.SearchBackend @@ -91,14 +90,13 @@ defmodule Pleroma.Search.Meilisearch do try do hits - |> Activity.create_by_object_ap_id() + |> Activity.get_presorted_create_by_object_ap_id() |> Activity.with_preloaded_object() |> Activity.restrict_deactivated_users() |> maybe_restrict_local(user) |> maybe_restrict_author(author) |> maybe_restrict_blocked(user) |> maybe_fetch(user, query) - |> order_by([object: obj], desc: obj.data["published"]) |> Pleroma.Repo.all() rescue _ -> maybe_fetch([], user, query) diff --git a/test/pleroma/activity_test.exs b/test/pleroma/activity_test.exs index 4f9144f91..1943746cb 100644 --- a/test/pleroma/activity_test.exs +++ b/test/pleroma/activity_test.exs @@ -41,6 +41,26 @@ defmodule Pleroma.ActivityTest do assert activity == found_activity end + test "returns activities by object's AP id in requested presorted order" do + a1 = insert(:note_activity) + o1 = Object.normalize(a1, fetch: false).data["id"] + + a2 = insert(:note_activity) + o2 = Object.normalize(a2, fetch: false).data["id"] + + a3 = insert(:note_activity) + o3 = Object.normalize(a3, fetch: false).data["id"] + + a4 = insert(:note_activity) + o4 = Object.normalize(a4, fetch: false).data["id"] + + found_activities = + Activity.get_presorted_create_by_object_ap_id([o3, o2, o4, o1]) + |> Repo.all() + + assert found_activities == [a3, a2, a4, a1] + end + test "preloading a bookmark" do user = insert(:user) user2 = insert(:user) From 59685e25d216c682815c8f7ccf10d577f2c735fb Mon Sep 17 00:00:00 2001 From: Oneric Date: Wed, 15 May 2024 06:46:47 +0200 Subject: [PATCH 3/4] meilisearch: show keys by name not description MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This makes show-key’s output match our documentation as of Meilisearch 1.8.0-8-g4d5971f343c00d45c11ef0cfb6f61e83a8508208. Since I’m not sure if older versions maybe only provided description, it will fallback to the latter if no name parameter exists. --- lib/mix/tasks/pleroma/search/meilisearch.ex | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 299fb5b14..e4dc616b4 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -126,8 +126,12 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do decoded = Jason.decode!(result.body) if decoded["results"] do - Enum.each(decoded["results"], fn %{"description" => desc, "key" => key} -> - IO.puts("#{desc}: #{key}") + Enum.each(decoded["results"], fn + %{"name" => name, "key" => key} -> + IO.puts("#{name}: #{key}") + + %{"description" => desc, "key" => key} -> + IO.puts("#{desc}: #{key}") end) else IO.puts("Error fetching the keys, check the master key is correct: #{inspect(decoded)}") From fc7e07f42467c971cfa1c8211932f89003c826c3 Mon Sep 17 00:00:00 2001 From: Oneric Date: Wed, 15 May 2024 07:05:28 +0200 Subject: [PATCH 4/4] meilisearch: enable using search_key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Using only the admin key works as well currently and Akkoma needs to know the admin key to be able to add new entries etc. However the Meilisearch key descriptions suggest the admin key is not supposed to be used for searches, so let’s not. For compatibility with existings configs, search_key remains optional. --- CHANGELOG.md | 3 +++ docs/docs/configuration/search.md | 9 ++++++-- lib/pleroma/search/meilisearch.ex | 38 +++++++++++++++++++------------ 3 files changed, 34 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c7a838612..9a272d4a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Added - Implement [FEP-67ff](https://codeberg.org/fediverse/fep/src/branch/main/fep/67ff/fep-67ff.md) (federation documentation) +## Added +- Meilisearch: it is now possible to use separate keys for search and admin actions + ## Fixed - Meilisearch: order of results returned from our REST API now actually matches how Meilisearch ranks results diff --git a/docs/docs/configuration/search.md b/docs/docs/configuration/search.md index 1e343032f..4c6bc412f 100644 --- a/docs/docs/configuration/search.md +++ b/docs/docs/configuration/search.md @@ -33,6 +33,7 @@ indexes faster when it can process many posts in a single batch. > config :pleroma, Pleroma.Search.Meilisearch, > url: "http://127.0.0.1:7700/", > private_key: "private key", +> search_key: "search key", > initial_indexing_chunk_size: 100_000 Information about setting up meilisearch can be found in the @@ -45,7 +46,7 @@ is hardly usable on a somewhat big instance. ### Private key authentication (optional) To set the private key, use the `MEILI_MASTER_KEY` environment variable when starting. After setting the _master key_, -you have to get the _private key_, which is actually used for authentication. +you have to get the _private key_ and possibly _search key_, which are actually used for authentication. === "OTP" ```sh @@ -57,7 +58,11 @@ you have to get the _private key_, which is actually used for authentication. mix pleroma.search.meilisearch show-keys ``` -You will see a "Default Admin API Key", this is the key you actually put into your configuration file. +You will see a "Default Admin API Key", this is the key you actually put into +your configuration file as `private_key`. You should also see a +"Default Search API key", put this into your config as `search_key`. +If your version of Meilisearch only showed the former, +just leave `search_key` completely unset in Akkoma's config. ### Initial indexing diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 822c95b4a..e6213d37f 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -8,11 +8,24 @@ defmodule Pleroma.Search.Meilisearch do @behaviour Pleroma.Search.SearchBackend - defp meili_headers do - private_key = Pleroma.Config.get([Pleroma.Search.Meilisearch, :private_key]) + defp meili_headers(key) do + key_header = + if is_nil(key), do: [], else: [{"Authorization", "Bearer #{key}"}] - [{"Content-Type", "application/json"}] ++ - if is_nil(private_key), do: [], else: [{"Authorization", "Bearer #{private_key}"}] + [{"Content-Type", "application/json"} | key_header] + end + + defp meili_headers_admin do + private_key = Pleroma.Config.get([Pleroma.Search.Meilisearch, :private_key]) + meili_headers(private_key) + end + + defp meili_headers_search do + search_key = + Pleroma.Config.get([Pleroma.Search.Meilisearch, :search_key]) || + Pleroma.Config.get([Pleroma.Search.Meilisearch, :private_key]) + + meili_headers(search_key) end def meili_get(path) do @@ -21,7 +34,7 @@ defmodule Pleroma.Search.Meilisearch do result = Pleroma.HTTP.get( Path.join(endpoint, path), - meili_headers() + meili_headers_admin() ) with {:ok, res} <- result do @@ -29,14 +42,14 @@ defmodule Pleroma.Search.Meilisearch do end end - def meili_post(path, params) do + defp meili_search(params) do endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) result = Pleroma.HTTP.post( - Path.join(endpoint, path), + Path.join(endpoint, "/indexes/objects/search"), Jason.encode!(params), - meili_headers() + meili_headers_search() ) with {:ok, res} <- result do @@ -52,7 +65,7 @@ defmodule Pleroma.Search.Meilisearch do :put, Path.join(endpoint, path), Jason.encode!(params), - meili_headers(), + meili_headers_admin(), [] ) @@ -69,7 +82,7 @@ defmodule Pleroma.Search.Meilisearch do :delete, Path.join(endpoint, path), "", - meili_headers(), + meili_headers_admin(), [] ) end @@ -80,10 +93,7 @@ defmodule Pleroma.Search.Meilisearch do author = Keyword.get(options, :author) res = - meili_post( - "/indexes/objects/search", - %{q: query, offset: offset, limit: limit} - ) + meili_search(%{q: query, offset: offset, limit: limit}) with {:ok, result} <- res do hits = result["hits"] |> Enum.map(& &1["ap"])