diff --git a/config/benchmark.exs b/config/benchmark.exs index b6a0115c4..3ae8350d3 100644 --- a/config/benchmark.exs +++ b/config/benchmark.exs @@ -71,8 +71,14 @@ config :pleroma, :rate_limit, config :pleroma, :http_security, report_uri: "https://endpoint.com" rum_enabled = System.get_env("RUM_ENABLED") == "true" -config :pleroma, :database, rum_enabled: rum_enabled +pgroonga_enabled = System.get_env("PGROONGA_ENABLED") == "true" + +config :pleroma, :database, + rum_enabled: rum_enabled, + pgroonga_enabled: pgroonga_enabled + IO.puts("RUM enabled: #{rum_enabled}") +IO.puts("PGroonga enabled: #{pgroonga_enabled}") config :pleroma, Pleroma.ReverseProxy.Client, Pleroma.ReverseProxy.ClientMock diff --git a/config/config.exs b/config/config.exs index c02b9cacb..eee7dd56b 100644 --- a/config/config.exs +++ b/config/config.exs @@ -689,7 +689,9 @@ config :pleroma, :oauth2, issue_new_refresh_token: true, clean_expired_tokens: false -config :pleroma, :database, rum_enabled: false +config :pleroma, :database, + rum_enabled: false, + pgroonga_enabled: false config :pleroma, :features, improved_hashtag_timeline: :auto diff --git a/config/docker.exs b/config/docker.exs index fc24a4d67..dfb40cf5c 100644 --- a/config/docker.exs +++ b/config/docker.exs @@ -23,7 +23,10 @@ config :pleroma, Pleroma.Repo, # Configure web push notifications config :web_push_encryption, :vapid_details, subject: "mailto:#{System.get_env("NOTIFY_EMAIL")}" -config :pleroma, :database, rum_enabled: false +config :pleroma, :database, + rum_enabled: false, + pgroonga_enabled: false + config :pleroma, :instance, static_dir: "/var/lib/akkoma/static" config :pleroma, Pleroma.Uploaders.Local, uploads: "/var/lib/akkoma/uploads" diff --git a/config/test.exs b/config/test.exs index 4448eeb73..7453b39a2 100644 --- a/config/test.exs +++ b/config/test.exs @@ -98,8 +98,14 @@ config :pleroma, :http_security, report_uri: "https://endpoint.com" config :pleroma, :http, send_user_agent: false rum_enabled = System.get_env("RUM_ENABLED") == "true" -config :pleroma, :database, rum_enabled: rum_enabled +pgroonga_enabled = System.get_env("PGROONGA_ENABLED") == "true" + +config :pleroma, :database, + rum_enabled: rum_enabled, + pgroonga_enabled: pgroonga_enabled + IO.puts("RUM enabled: #{rum_enabled}") +IO.puts("PGroonga enabled: #{pgroonga_enabled}") config :joken, default_signer: "yU8uHKq+yyAkZ11Hx//jcdacWc8yQ1bxAAGrplzB0Zwwjkp35v0RK9SO8WTPr6QZ" diff --git a/docker-compose.yml b/docker-compose.yml index 394415ee1..b6dbbd459 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,6 +4,7 @@ version: "3.7" services: db: image: postgres:14-alpine + # image: groonga/pgroonga:3.1.1-alpine-14 # Use this one if you want to use PGroonga restart: unless-stopped environment: POSTGRES_DB: akkoma diff --git a/docs/docs/configuration/howto_database_config.md b/docs/docs/configuration/howto_database_config.md index a6dea5640..1d5111a11 100644 --- a/docs/docs/configuration/howto_database_config.md +++ b/docs/docs/configuration/howto_database_config.md @@ -54,6 +54,7 @@ The configuration of Akkoma (and Pleroma) has traditionally been managed with a * config :pleroma, Pleroma.Repo * config :pleroma, configurable\_from\_database * config :pleroma, :database, rum_enabled + * config :pleroma, :database, pgroonga_enabled * config :pleroma, :connections_pool Here is an example of a server config stripped down after migration: diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index 55d1c8ddc..fca1e6d9c 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -326,37 +326,50 @@ defmodule Mix.Tasks.Pleroma.Database do "ALTER DATABASE #{db} SET default_text_search_config = '#{tsconfig}';" ) - # non-exist config will not raise excpetion but only give >0 messages + # non-exist config will not raise exception but only give >0 messages if length(msg) > 0 do shell_info("Error: #{inspect(msg, pretty: true)}") else rum_enabled = Pleroma.Config.get([:database, :rum_enabled]) - shell_info("Recreate index, RUM: #{rum_enabled}") + pgroonga_enabled = Pleroma.Config.get([:database, :pgroonga_enabled]) + shell_info("Recreate index, RUM: #{rum_enabled}, PGroonga: #{pgroonga_enabled}") # Note SQL below needs to be kept up-to-date with latest GIN or RUM index definition in future - if rum_enabled do - Ecto.Adapters.SQL.query!( - Pleroma.Repo, - "CREATE OR REPLACE FUNCTION objects_fts_update() RETURNS trigger AS $$ BEGIN + cond do + rum_enabled -> + Ecto.Adapters.SQL.query!( + Pleroma.Repo, + "CREATE OR REPLACE FUNCTION objects_fts_update() RETURNS trigger AS $$ BEGIN new.fts_content := to_tsvector(new.data->>'content'); RETURN new; END $$ LANGUAGE plpgsql", - [], - timeout: :infinity - ) + [], + timeout: :infinity + ) - shell_info("Refresh RUM index") - Ecto.Adapters.SQL.query!(Pleroma.Repo, "UPDATE objects SET updated_at = NOW();") - else - Ecto.Adapters.SQL.query!(Pleroma.Repo, "DROP INDEX IF EXISTS objects_fts;") + shell_info("Refresh RUM index") + Ecto.Adapters.SQL.query!(Pleroma.Repo, "UPDATE objects SET updated_at = NOW();") - Ecto.Adapters.SQL.query!( - Pleroma.Repo, - "CREATE INDEX CONCURRENTLY objects_fts ON objects USING gin(to_tsvector('#{tsconfig}', data->>'content')); ", - [], - timeout: :infinity - ) + pgroonga_enabled -> + Ecto.Adapters.SQL.query!(Pleroma.Repo, "DROP INDEX IF EXISTS object_content_pgroonga;") + + Ecto.Adapters.SQL.query!( + Pleroma.Repo, + "CREATE INDEX object_content_pgroonga ON objects USING pgroonga ((data->'content')) WITH (tokenizer='TokenMecab');", + [], + timeout: :infinity + ) + + true -> + Ecto.Adapters.SQL.query!(Pleroma.Repo, "DROP INDEX IF EXISTS objects_fts;") + + Ecto.Adapters.SQL.query!( + Pleroma.Repo, + "CREATE INDEX CONCURRENTLY objects_fts ON objects USING gin(to_tsvector('#{tsconfig}', data->>'content'));", + [], + timeout: :infinity + ) end shell_info('Done.') diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex index 52fd184b5..2361b23f4 100644 --- a/lib/mix/tasks/pleroma/instance.ex +++ b/lib/mix/tasks/pleroma/instance.ex @@ -28,6 +28,7 @@ defmodule Mix.Tasks.Pleroma.Instance do dbuser: :string, dbpass: :string, rum: :string, + pgroonga: :string, indexable: :string, db_configurable: :string, uploads_dir: :string, @@ -127,6 +128,18 @@ defmodule Mix.Tasks.Pleroma.Instance do "n" ) === "y" + pgroonga_enabled = + if rum_enabled do + false + else + get_option( + options, + :pgroonga, + "Would you like to use PGroonga to index and search?", + "n" + ) === "y" + end + listen_port = get_option( options, @@ -225,6 +238,7 @@ defmodule Mix.Tasks.Pleroma.Instance do static_dir: static_dir, uploads_dir: uploads_dir, rum_enabled: rum_enabled, + pgroonga_enabled: pgroonga_enabled, listen_ip: listen_ip, listen_port: listen_port, upload_filters: diff --git a/lib/pleroma/application_requirements.ex b/lib/pleroma/application_requirements.ex index 19236aaa2..f1460a7f2 100644 --- a/lib/pleroma/application_requirements.ex +++ b/lib/pleroma/application_requirements.ex @@ -24,6 +24,7 @@ defmodule Pleroma.ApplicationRequirements do |> check_migrations_applied!() |> check_welcome_message_config!() |> check_rum!() + |> check_pgroonga!() |> check_repo_pool_size!() |> handle_result() end @@ -162,6 +163,55 @@ defmodule Pleroma.ApplicationRequirements do end end + # Checks for settings of PGroonga. + # + defp check_pgroonga!(:ok) do + {_, res, _} = + Ecto.Migrator.with_repo(Pleroma.Repo, fn repo -> + # TODO: check migrate + migrate = + from(o in "pg_indexes", + where: o.indexname == "object_content_pgroonga" + ) + |> repo.exists?() + + setting = Pleroma.Config.get([:database, :pgroonga_enabled], false) + + do_check_pgroonga!(setting, migrate) + end) + + res + end + + defp check_pgroonga!(result), do: result + + defp do_check_pgroonga!(setting, migrate) do + case {setting, migrate} do + {true, false} -> + Logger.error( + "PGroonga is enabled, but no migrations have been applied for it.\n" <> + "If you want to start Akkoma without using PGroonga, set `config :pleroma, :database, pgroonga_enabled: false`.\n" <> + "Otherwise apply the following migrations:\n" <> + "`mix ecto.migrate --migrations-path priv/repo/optional_migrations/pgroonga/`" + ) + + {:error, "Unapplied PGroonga Migrations detected"} + + {false, true} -> + Logger.error( + "Detected applied migrations to use PGroonga to search, but it is not enabled in the settings.\n" <> + "If you want to use PGroonga, set `config :pleroma, :database, pgroonga_enabled: true`.\n" <> + "Otherwise apply the following migrations:\n" <> + "`mix ecto.rollback --migrations-path priv/repo/optional_migrations/pgroonga/`" + ) + + {:error, "PGroonga Migrations detected"} + + _ -> + :ok + end + end + defp check_system_commands!(:ok) do filter_commands_statuses = [ check_filter(Pleroma.Upload.Filter.Exiftool, "exiftool"), diff --git a/lib/pleroma/search/database_search.ex b/lib/pleroma/search/database_search.ex index 3735a5fab..5b9be4bed 100644 --- a/lib/pleroma/search/database_search.ex +++ b/lib/pleroma/search/database_search.ex @@ -16,7 +16,13 @@ defmodule Pleroma.Search.DatabaseSearch do @behaviour Pleroma.Search.SearchBackend def search(user, search_query, options \\ []) do - index_type = if Pleroma.Config.get([:database, :rum_enabled]), do: :rum, else: :gin + index_type = + cond do + Pleroma.Config.get([:database, :rum_enabled]) -> :rum + Pleroma.Config.get([:database, :pgroonga_enabled]) -> :pgroonga + true -> :gin + end + limit = Enum.min([Keyword.get(options, :limit), 40]) offset = Keyword.get(options, :offset, 0) author = Keyword.get(options, :author) @@ -132,6 +138,18 @@ defmodule Pleroma.Search.DatabaseSearch do ) end + defp query_with(q, :pgroonga, search_query, _) do + # PGroonga only supports PostgreSQL version 11 or newer + from([a, o] in q, + where: + fragment( + "?->'content' &@~ ?", + o.data, + ^search_query + ) + ) + end + def maybe_restrict_local(q, user) do limit = Pleroma.Config.get([:instance, :limit_to_local_content], :unauthenticated) diff --git a/priv/repo/optional_migrations/pgroonga/20230802195431_create_pgroonga_index.exs b/priv/repo/optional_migrations/pgroonga/20230802195431_create_pgroonga_index.exs new file mode 100644 index 000000000..0e006b58f --- /dev/null +++ b/priv/repo/optional_migrations/pgroonga/20230802195431_create_pgroonga_index.exs @@ -0,0 +1,29 @@ +defmodule Pleroma.Repo.Migrations.CreatePgroongaIndex do + use Ecto.Migration + + def up do + execute("CREATE EXTENSION IF NOT EXISTS pgroonga") + + drop_if_exists( + index(:objects, ["(to_tsvector('english', data->>'content'))"], + using: :gin, + name: :objects_fts + ) + ) + + execute( + "CREATE INDEX object_content_pgroonga ON objects USING pgroonga ((data->'content')) WITH (tokenizer='TokenMecab')" + ) + end + + def down do + execute("DROP INDEX IF EXISTS object_content_pgroonga") + + create_if_not_exists( + index(:objects, ["(to_tsvector('english', data->>'content'))"], + using: :gin, + name: :objects_fts + ) + ) + end +end diff --git a/priv/templates/sample_config.eex b/priv/templates/sample_config.eex index 0068969ac..dad06e108 100644 --- a/priv/templates/sample_config.eex +++ b/priv/templates/sample_config.eex @@ -41,7 +41,10 @@ config :web_push_encryption, :vapid_details, public_key: "<%= web_push_public_key %>", private_key: "<%= web_push_private_key %>" -config :pleroma, :database, rum_enabled: <%= rum_enabled %> +config :pleroma, :database, + rum_enabled: <%= rum_enabled %>, + pgroonga_enabled: <%= pgroonga_enabled %> + config :pleroma, :instance, static_dir: "<%= static_dir %>" config :pleroma, Pleroma.Uploaders.Local, uploads: "<%= uploads_dir %>" diff --git a/priv/templates/sample_psql.eex b/priv/templates/sample_psql.eex index 627839a68..3daa8594b 100644 --- a/priv/templates/sample_psql.eex +++ b/priv/templates/sample_psql.eex @@ -10,3 +10,8 @@ CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; else "" end %> +<%= if pgroonga_enabled do + "CREATE EXTENSION IF NOT EXISTS pgroonga;" +else +"" +end %>