This commit is contained in:
Alexander Strizhakov 2020-03-12 18:28:54 +03:00
parent f39e1b9eff
commit 1306b92997
No known key found for this signature in database
GPG Key ID: 022896A53AEF1381
10 changed files with 94 additions and 155 deletions

View File

@ -42,7 +42,9 @@ defmodule Pleroma.Application do
setup_instrumenters() setup_instrumenters()
load_custom_modules() load_custom_modules()
if adapter() == Tesla.Adapter.Gun do adapter = Application.get_env(:tesla, :adapter)
if adapter == Tesla.Adapter.Gun do
if version = Pleroma.OTPVersion.version() do if version = Pleroma.OTPVersion.version() do
[major, minor] = [major, minor] =
version version
@ -74,7 +76,7 @@ defmodule Pleroma.Application do
Pleroma.Plugs.RateLimiter.Supervisor Pleroma.Plugs.RateLimiter.Supervisor
] ++ ] ++
cachex_children() ++ cachex_children() ++
http_pools_children(Config.get(:env)) ++ http_children(adapter, @env) ++
[ [
Pleroma.Stats, Pleroma.Stats,
Pleroma.JobQueueMonitor, Pleroma.JobQueueMonitor,
@ -206,15 +208,13 @@ defmodule Pleroma.Application do
end end
# start hackney and gun pools in tests # start hackney and gun pools in tests
defp http_pools_children(:test) do defp http_children(_, :test) do
hackney_options = Config.get([:hackney_pools, :federation]) hackney_options = Config.get([:hackney_pools, :federation])
hackney_pool = :hackney_pool.child_spec(:federation, hackney_options) hackney_pool = :hackney_pool.child_spec(:federation, hackney_options)
[hackney_pool, Pleroma.Pool.Supervisor] [hackney_pool, Pleroma.Pool.Supervisor]
end end
defp http_pools_children(_), do: http_pools(adapter()) defp http_children(Tesla.Adapter.Hackney, _) do
defp http_pools(Tesla.Adapter.Hackney) do
pools = [:federation, :media] pools = [:federation, :media]
pools = pools =
@ -230,9 +230,7 @@ defmodule Pleroma.Application do
end end
end end
defp http_pools(Tesla.Adapter.Gun), do: [Pleroma.Pool.Supervisor] defp http_children(Tesla.Adapter.Gun, _), do: [Pleroma.Pool.Supervisor]
defp http_pools(_), do: [] defp http_children(_, _), do: []
defp adapter, do: Application.get_env(:tesla, :adapter)
end end

View File

@ -5,6 +5,7 @@
defmodule Pleroma.Config.TransferTask do defmodule Pleroma.Config.TransferTask do
use Task use Task
alias Pleroma.Config
alias Pleroma.ConfigDB alias Pleroma.ConfigDB
alias Pleroma.Repo alias Pleroma.Repo
@ -36,36 +37,31 @@ defmodule Pleroma.Config.TransferTask do
def start_link(_) do def start_link(_) do
load_and_update_env() load_and_update_env()
if Pleroma.Config.get(:env) == :test, do: Ecto.Adapters.SQL.Sandbox.checkin(Repo) if Config.get(:env) == :test, do: Ecto.Adapters.SQL.Sandbox.checkin(Repo)
:ignore :ignore
end end
@spec load_and_update_env([ConfigDB.t()]) :: :ok | false @spec load_and_update_env([ConfigDB.t()], boolean()) :: :ok
def load_and_update_env(deleted \\ [], restart_pleroma? \\ true) do def load_and_update_env(deleted_settings \\ [], restart_pleroma? \\ true) do
with {_, true} <- {:configurable, Pleroma.Config.get(:configurable_from_database)} do with {_, true} <- {:configurable, Config.get(:configurable_from_database)} do
# We need to restart applications for loaded settings take effect # We need to restart applications for loaded settings take effect
in_db = Repo.all(ConfigDB)
with_deleted = in_db ++ deleted
# TODO: some problem with prometheus after restart! # TODO: some problem with prometheus after restart!
reject = [nil, :prometheus] reject_restart =
reject_for_restart =
if restart_pleroma? do if restart_pleroma? do
reject [nil, :prometheus]
else else
[:pleroma | reject] [:pleroma, nil, :prometheus]
end end
started_applications = Application.started_applications() started_applications = Application.started_applications()
with_deleted (Repo.all(ConfigDB) ++ deleted_settings)
|> Enum.map(&merge_and_update(&1)) |> Enum.map(&merge_and_update/1)
|> Enum.uniq() |> Enum.uniq()
|> Enum.reject(&(&1 in reject_for_restart)) |> Enum.reject(&(&1 in reject_restart))
|> maybe_set_pleroma_last() |> maybe_set_pleroma_last()
|> Enum.each(&restart(started_applications, &1, Pleroma.Config.get(:env))) |> Enum.each(&restart(started_applications, &1, Config.get(:env)))
:ok :ok
else else
@ -108,18 +104,14 @@ defmodule Pleroma.Config.TransferTask do
key = ConfigDB.from_string(setting.key) key = ConfigDB.from_string(setting.key)
group = ConfigDB.from_string(setting.group) group = ConfigDB.from_string(setting.group)
default = Pleroma.Config.Holder.config(group, key) default = Config.Holder.config(group, key)
value = ConfigDB.from_binary(setting.value) value = ConfigDB.from_binary(setting.value)
merged_value = merged_value =
if Ecto.get_meta(setting, :state) == :deleted do cond do
default Ecto.get_meta(setting, :state) == :deleted -> default
else can_be_merged?(default, value) -> ConfigDB.merge_group(group, key, default, value)
if can_be_merged?(default, value) do true -> value
ConfigDB.merge_group(group, key, default, value)
else
value
end
end end
:ok = update_env(group, key, merged_value) :ok = update_env(group, key, merged_value)

View File

@ -49,8 +49,6 @@ defmodule Pleroma.Gun.Conn do
key = "#{uri.scheme}:#{uri.host}:#{uri.port}" key = "#{uri.scheme}:#{uri.host}:#{uri.port}"
Logger.debug("opening new connection #{Connections.compose_uri_log(uri)}")
conn_pid = conn_pid =
if Connections.count(name) < opts[:max_connection] do if Connections.count(name) < opts[:max_connection] do
do_open(uri, opts) do_open(uri, opts)
@ -109,9 +107,9 @@ defmodule Pleroma.Gun.Conn do
else else
error -> error ->
Logger.warn( Logger.warn(
"Received error on opening connection with http proxy #{ "Opening proxied connection to #{compose_uri_log(uri)} failed with error #{
Connections.compose_uri_log(uri) inspect(error)
} #{inspect(error)}" }"
) )
error error
@ -145,9 +143,9 @@ defmodule Pleroma.Gun.Conn do
else else
error -> error ->
Logger.warn( Logger.warn(
"Received error on opening connection with socks proxy #{ "Opening socks proxied connection to #{compose_uri_log(uri)} failed with error #{
Connections.compose_uri_log(uri) inspect(error)
} #{inspect(error)}" }"
) )
error error
@ -163,9 +161,7 @@ defmodule Pleroma.Gun.Conn do
else else
error -> error ->
Logger.warn( Logger.warn(
"Received error on opening connection #{Connections.compose_uri_log(uri)} #{ "Opening connection to #{compose_uri_log(uri)} failed with error #{inspect(error)}"
inspect(error)
}"
) )
error error
@ -184,16 +180,17 @@ defmodule Pleroma.Gun.Conn do
defp add_http2_opts(opts, _, _), do: opts defp add_http2_opts(opts, _, _), do: opts
defp close_least_used_and_do_open(name, uri, opts) do defp close_least_used_and_do_open(name, uri, opts) do
Logger.debug("try to open conn #{Connections.compose_uri_log(uri)}") with [{key, conn} | _conns] <- Connections.get_unused_conns(name),
:ok <- Gun.close(conn.conn) do
with [{close_key, least_used} | _conns] <- Connections.remove_conn(name, key)
Connections.get_unused_conns(name),
:ok <- Gun.close(least_used.conn) do
Connections.remove_conn(name, close_key)
do_open(uri, opts) do_open(uri, opts)
else else
[] -> {:error, :pool_overflowed} [] -> {:error, :pool_overflowed}
end end
end end
def compose_uri_log(%URI{scheme: scheme, host: host, path: path}) do
"#{scheme}://#{host}#{path}"
end
end end

View File

@ -7,7 +7,7 @@ defmodule Pleroma.HTTP.AdapterHelper do
@type proxy :: @type proxy ::
{Connection.host(), pos_integer()} {Connection.host(), pos_integer()}
| {Connection.proxy_type(), pos_integer()} | {Connection.proxy_type(), Connection.host(), pos_integer()}
@callback options(keyword(), URI.t()) :: keyword() @callback options(keyword(), URI.t()) :: keyword()
@callback after_request(keyword()) :: :ok @callback after_request(keyword()) :: :ok

View File

@ -20,8 +20,8 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
] ]
@spec options(keyword(), URI.t()) :: keyword() @spec options(keyword(), URI.t()) :: keyword()
def options(connection_opts \\ [], %URI{} = uri) do def options(incoming_opts \\ [], %URI{} = uri) do
formatted_proxy = proxy =
Pleroma.Config.get([:http, :proxy_url], nil) Pleroma.Config.get([:http, :proxy_url], nil)
|> AdapterHelper.format_proxy() |> AdapterHelper.format_proxy()
@ -30,8 +30,8 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
@defaults @defaults
|> Keyword.merge(config_opts) |> Keyword.merge(config_opts)
|> add_scheme_opts(uri) |> add_scheme_opts(uri)
|> AdapterHelper.maybe_add_proxy(formatted_proxy) |> AdapterHelper.maybe_add_proxy(proxy)
|> maybe_get_conn(uri, connection_opts) |> maybe_get_conn(uri, incoming_opts)
end end
@spec after_request(keyword()) :: :ok @spec after_request(keyword()) :: :ok
@ -43,44 +43,35 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
:ok :ok
end end
defp add_scheme_opts(opts, %URI{scheme: "http"}), do: opts defp add_scheme_opts(opts, %{scheme: "http"}), do: opts
defp add_scheme_opts(opts, %URI{scheme: "https"}) do defp add_scheme_opts(opts, %{scheme: "https"}) do
opts opts
|> Keyword.put(:certificates_verification, true) |> Keyword.put(:certificates_verification, true)
|> Keyword.put(:transport, :tls)
|> Keyword.put(:tls_opts, log_level: :warning) |> Keyword.put(:tls_opts, log_level: :warning)
end end
defp maybe_get_conn(adapter_opts, uri, connection_opts) do defp maybe_get_conn(adapter_opts, uri, incoming_opts) do
{receive_conn?, opts} = {receive_conn?, opts} =
adapter_opts adapter_opts
|> Keyword.merge(connection_opts) |> Keyword.merge(incoming_opts)
|> Keyword.pop(:receive_conn, true) |> Keyword.pop(:receive_conn, true)
if Connections.alive?(:gun_connections) and receive_conn? do if Connections.alive?(:gun_connections) and receive_conn? do
try_to_get_conn(uri, opts) checkin_conn(uri, opts)
else else
opts opts
end end
end end
defp try_to_get_conn(uri, opts) do defp checkin_conn(uri, opts) do
case Connections.checkin(uri, :gun_connections) do case Connections.checkin(uri, :gun_connections) do
nil -> nil ->
Logger.debug( Task.start(Pleroma.Gun.Conn, :open, [uri, :gun_connections, opts])
"Gun connections pool checkin was not successful. Trying to open conn for next request."
)
Task.start(fn -> Pleroma.Gun.Conn.open(uri, :gun_connections, opts) end)
opts opts
conn when is_pid(conn) -> conn when is_pid(conn) ->
Logger.debug("received conn #{inspect(conn)} #{Connections.compose_uri_log(uri)}") Keyword.merge(opts, conn: conn, close_conn: false)
opts
|> Keyword.put(:conn, conn)
|> Keyword.put(:close_conn, false)
end end
end end
end end

View File

@ -71,15 +71,15 @@ defmodule Pleroma.HTTP.Connection do
{:ok, parse_host(host), port} {:ok, parse_host(host), port}
else else
{_, _} -> {_, _} ->
Logger.warn("parsing port in proxy fail #{inspect(proxy)}") Logger.warn("Parsing port failed #{inspect(proxy)}")
{:error, :invalid_proxy_port} {:error, :invalid_proxy_port}
:error -> :error ->
Logger.warn("parsing port in proxy fail #{inspect(proxy)}") Logger.warn("Parsing port failed #{inspect(proxy)}")
{:error, :invalid_proxy_port} {:error, :invalid_proxy_port}
_ -> _ ->
Logger.warn("parsing proxy fail #{inspect(proxy)}") Logger.warn("Parsing proxy failed #{inspect(proxy)}")
{:error, :invalid_proxy} {:error, :invalid_proxy}
end end
end end
@ -89,7 +89,7 @@ defmodule Pleroma.HTTP.Connection do
{:ok, type, parse_host(host), port} {:ok, type, parse_host(host), port}
else else
_ -> _ ->
Logger.warn("parsing proxy fail #{inspect(proxy)}") Logger.warn("Parsing proxy failed #{inspect(proxy)}")
{:error, :invalid_proxy} {:error, :invalid_proxy}
end end
end end

View File

@ -56,10 +56,9 @@ defmodule Pleroma.HTTP do
{:ok, Env.t()} | {:error, any()} {:ok, Env.t()} | {:error, any()}
def request(method, url, body, headers, options) when is_binary(url) do def request(method, url, body, headers, options) when is_binary(url) do
uri = URI.parse(url) uri = URI.parse(url)
received_adapter_opts = Keyword.get(options, :adapter, []) adapter_opts = Connection.options(uri, options[:adapter] || [])
adapter_opts = Connection.options(uri, received_adapter_opts)
options = put_in(options[:adapter], adapter_opts) options = put_in(options[:adapter], adapter_opts)
params = Keyword.get(options, :params, []) params = options[:params] || []
request = build_request(method, headers, options, url, body, params) request = build_request(method, headers, options, url, body, params)
adapter = Application.get_env(:tesla, :adapter) adapter = Application.get_env(:tesla, :adapter)

View File

@ -87,18 +87,11 @@ defmodule Pleroma.Pool.Connections do
@impl true @impl true
def handle_cast({:checkout, conn_pid, pid}, state) do def handle_cast({:checkout, conn_pid, pid}, state) do
Logger.debug("checkout #{inspect(conn_pid)}")
state = state =
with true <- Process.alive?(conn_pid), with true <- Process.alive?(conn_pid),
{key, conn} <- find_conn(state.conns, conn_pid), {key, conn} <- find_conn(state.conns, conn_pid),
used_by <- List.keydelete(conn.used_by, pid, 0) do used_by <- List.keydelete(conn.used_by, pid, 0) do
conn_state = conn_state = if used_by == [], do: :idle, else: conn.conn_state
if used_by == [] do
:idle
else
conn.conn_state
end
put_in(state.conns[key], %{conn | conn_state: conn_state, used_by: used_by}) put_in(state.conns[key], %{conn | conn_state: conn_state, used_by: used_by})
else else
@ -123,26 +116,23 @@ defmodule Pleroma.Pool.Connections do
@impl true @impl true
def handle_call({:checkin, uri}, from, state) do def handle_call({:checkin, uri}, from, state) do
key = "#{uri.scheme}:#{uri.host}:#{uri.port}" key = "#{uri.scheme}:#{uri.host}:#{uri.port}"
Logger.debug("checkin #{key}")
case state.conns[key] do case state.conns[key] do
%{conn: conn, gun_state: :up} = current_conn -> %{conn: pid, gun_state: :up} = conn ->
Logger.debug("reusing conn #{key}")
time = :os.system_time(:second) time = :os.system_time(:second)
last_reference = time - current_conn.last_reference last_reference = time - conn.last_reference
current_crf = crf(last_reference, 100, current_conn.crf) crf = crf(last_reference, 100, conn.crf)
state = state =
put_in(state.conns[key], %{ put_in(state.conns[key], %{
current_conn conn
| last_reference: time, | last_reference: time,
crf: current_crf, crf: crf,
conn_state: :active, conn_state: :active,
used_by: [from | current_conn.used_by] used_by: [from | conn.used_by]
}) })
{:reply, conn, state} {:reply, pid, state}
%{gun_state: :down} -> %{gun_state: :down} ->
{:reply, nil, state} {:reply, nil, state}
@ -164,50 +154,48 @@ defmodule Pleroma.Pool.Connections do
def handle_call(:unused_conns, _from, state) do def handle_call(:unused_conns, _from, state) do
unused_conns = unused_conns =
state.conns state.conns
|> Enum.filter(fn {_k, v} -> |> Enum.filter(&filter_conns/1)
v.conn_state == :idle and v.used_by == [] |> Enum.sort(&sort_conns/2)
end)
|> Enum.sort(fn {_x_k, x}, {_y_k, y} ->
x.crf <= y.crf and x.last_reference <= y.last_reference
end)
{:reply, unused_conns, state} {:reply, unused_conns, state}
end end
defp filter_conns({_, %{conn_state: :idle, used_by: []}}), do: true
defp filter_conns(_), do: false
defp sort_conns({_, c1}, {_, c2}) do
c1.crf <= c2.crf and c1.last_reference <= c2.last_reference
end
@impl true @impl true
def handle_info({:gun_up, conn_pid, _protocol}, state) do def handle_info({:gun_up, conn_pid, _protocol}, state) do
state = %{origin_host: host, origin_scheme: scheme, origin_port: port} = Gun.info(conn_pid)
with conn_key when is_binary(conn_key) <- compose_key_gun_info(conn_pid),
{key, conn} <- find_conn(state.conns, conn_pid, conn_key),
{true, key} <- {Process.alive?(conn_pid), key} do
time = :os.system_time(:second)
last_reference = time - conn.last_reference
current_crf = crf(last_reference, 100, conn.crf)
host =
case :inet.ntoa(host) do
{:error, :einval} -> host
ip -> ip
end
key = "#{scheme}:#{host}:#{port}"
state =
with {_key, conn} <- find_conn(state.conns, conn_pid, key),
{true, key} <- {Process.alive?(conn_pid), key} do
put_in(state.conns[key], %{ put_in(state.conns[key], %{
conn conn
| gun_state: :up, | gun_state: :up,
last_reference: time,
crf: current_crf,
conn_state: :active, conn_state: :active,
retries: 0 retries: 0
}) })
else else
:error_gun_info ->
Logger.debug(":gun.info caused error")
state
{false, key} -> {false, key} ->
Logger.debug(":gun_up message for closed conn #{inspect(conn_pid)}")
put_in( put_in(
state.conns, state.conns,
Map.delete(state.conns, key) Map.delete(state.conns, key)
) )
nil -> nil ->
Logger.debug(":gun_up message for conn which is not found in state")
:ok = Gun.close(conn_pid) :ok = Gun.close(conn_pid)
state state
@ -224,7 +212,6 @@ defmodule Pleroma.Pool.Connections do
with {key, conn} <- find_conn(state.conns, conn_pid), with {key, conn} <- find_conn(state.conns, conn_pid),
{true, key} <- {Process.alive?(conn_pid), key} do {true, key} <- {Process.alive?(conn_pid), key} do
if conn.retries == retries do if conn.retries == retries do
Logger.debug("closing conn if retries is eq #{inspect(conn_pid)}")
:ok = Gun.close(conn.conn) :ok = Gun.close(conn.conn)
put_in( put_in(
@ -240,18 +227,13 @@ defmodule Pleroma.Pool.Connections do
end end
else else
{false, key} -> {false, key} ->
# gun can send gun_down for closed conn, maybe connection is not closed yet
Logger.debug(":gun_down message for closed conn #{inspect(conn_pid)}")
put_in( put_in(
state.conns, state.conns,
Map.delete(state.conns, key) Map.delete(state.conns, key)
) )
nil -> nil ->
Logger.debug(":gun_down message for conn which is not found in state") Logger.debug(":gun_down for conn which isn't found in state")
:ok = Gun.close(conn_pid)
state state
end end
@ -275,7 +257,7 @@ defmodule Pleroma.Pool.Connections do
) )
else else
nil -> nil ->
Logger.debug(":DOWN message for conn which is not found in state") Logger.debug(":DOWN for conn which isn't found in state")
state state
end end
@ -283,18 +265,6 @@ defmodule Pleroma.Pool.Connections do
{:noreply, state} {:noreply, state}
end end
defp compose_key_gun_info(pid) do
%{origin_host: origin_host, origin_scheme: scheme, origin_port: port} = Gun.info(pid)
host =
case :inet.ntoa(origin_host) do
{:error, :einval} -> origin_host
ip -> ip
end
"#{scheme}:#{host}:#{port}"
end
defp find_conn(conns, conn_pid) do defp find_conn(conns, conn_pid) do
Enum.find(conns, fn {_key, conn} -> Enum.find(conns, fn {_key, conn} ->
conn.conn == conn_pid conn.conn == conn_pid
@ -310,8 +280,4 @@ defmodule Pleroma.Pool.Connections do
def crf(current, steps, crf) do def crf(current, steps, crf) do
1 + :math.pow(0.5, current / steps) * crf 1 + :math.pow(0.5, current / steps) * crf
end end
def compose_uri_log(%URI{scheme: scheme, host: host, path: path}) do
"#{scheme}://#{host}#{path}"
end
end end

View File

@ -6,7 +6,6 @@ defmodule Pleroma.HTTP.AdapterHelper.GunTest do
use ExUnit.Case, async: true use ExUnit.Case, async: true
use Pleroma.Tests.Helpers use Pleroma.Tests.Helpers
import ExUnit.CaptureLog
import Mox import Mox
alias Pleroma.Config alias Pleroma.Config
@ -63,7 +62,6 @@ defmodule Pleroma.HTTP.AdapterHelper.GunTest do
opts = Gun.options([receive_conn: false], uri) opts = Gun.options([receive_conn: false], uri)
assert opts[:certificates_verification] assert opts[:certificates_verification]
assert opts[:transport] == :tls
end end
test "get conn on next request" do test "get conn on next request" do
@ -73,14 +71,12 @@ defmodule Pleroma.HTTP.AdapterHelper.GunTest do
on_exit(fn -> Logger.configure(level: level) end) on_exit(fn -> Logger.configure(level: level) end)
uri = URI.parse("http://some-domain2.com") uri = URI.parse("http://some-domain2.com")
assert capture_log(fn -> opts = Gun.options(uri)
opts = Gun.options(uri)
assert opts[:conn] == nil assert opts[:conn] == nil
assert opts[:close_conn] == nil assert opts[:close_conn] == nil
end) =~
"Gun connections pool checkin was not successful. Trying to open conn for next request."
Process.sleep(50)
opts = Gun.options(uri) opts = Gun.options(uri)
assert is_pid(opts[:conn]) assert is_pid(opts[:conn])

View File

@ -355,7 +355,7 @@ defmodule Pleroma.Pool.ConnectionsTest do
refute Conn.open(url, name) refute Conn.open(url, name)
refute Connections.checkin(url, name) refute Connections.checkin(url, name)
end) =~ end) =~
"Received error on opening connection http://gun-not-up.com {:error, :timeout}" "Opening connection to http://gun-not-up.com failed with error {:error, :timeout}"
end end
test "process gun_down message and then gun_up", %{name: name} do test "process gun_down message and then gun_up", %{name: name} do