[#2497] Adjusted media proxy preview invalidation. Allowed client-side caching for media preview. Adjusted prewarmer to fetch only proxiable URIs.

Removed :preview pool in favor of existing :media one. Misc. refactoring.
This commit is contained in:
Ivan Tashkinov 2020-09-05 20:19:09 +03:00
parent c3b02341bf
commit f170d47130
7 changed files with 34 additions and 32 deletions

View File

@ -754,11 +754,6 @@ config :pleroma, :pools,
timeout: 10_000 timeout: 10_000
], ],
media: [ media: [
size: 50,
max_waiting: 10,
timeout: 10_000
],
preview: [
size: 50, size: 50,
max_waiting: 20, max_waiting: 20,
timeout: 15_000 timeout: 15_000

View File

@ -14,7 +14,7 @@ defmodule Pleroma.Helpers.MediaHelper do
def image_resize(url, options) do def image_resize(url, options) do
with executable when is_binary(executable) <- System.find_executable("convert"), with executable when is_binary(executable) <- System.find_executable("convert"),
{:ok, args} <- prepare_image_resize_args(options), {:ok, args} <- prepare_image_resize_args(options),
{:ok, env} <- HTTP.get(url, [], adapter: [pool: :preview]), {:ok, env} <- HTTP.get(url, [], adapter: [pool: :media]),
{:ok, fifo_path} <- mkfifo() do {:ok, fifo_path} <- mkfifo() do
args = List.flatten([fifo_path, args]) args = List.flatten([fifo_path, args])
run_fifo(fifo_path, env, executable, args) run_fifo(fifo_path, env, executable, args)
@ -62,7 +62,7 @@ defmodule Pleroma.Helpers.MediaHelper do
def video_framegrab(url) do def video_framegrab(url) do
with executable when is_binary(executable) <- System.find_executable("ffmpeg"), with executable when is_binary(executable) <- System.find_executable("ffmpeg"),
{:ok, env} <- HTTP.get(url, [], adapter: [pool: :preview]), {:ok, env} <- HTTP.get(url, [], adapter: [pool: :media]),
{:ok, fifo_path} <- mkfifo(), {:ok, fifo_path} <- mkfifo(),
args = [ args = [
"-y", "-y",

View File

@ -18,6 +18,7 @@ defmodule Pleroma.ReverseProxy do
@methods ~w(GET HEAD) @methods ~w(GET HEAD)
def max_read_duration_default, do: @max_read_duration def max_read_duration_default, do: @max_read_duration
def default_cache_control_header, do: @default_cache_control_header
@moduledoc """ @moduledoc """
A reverse proxy. A reverse proxy.

View File

@ -12,23 +12,28 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
require Logger require Logger
@options [ @adapter_options [
pool: :media pool: :media
] ]
def perform(:prefetch, url) do def perform(:prefetch, url) do
Logger.debug("Prefetching #{inspect(url)}") # Fetching only proxiable resources
if MediaProxy.enabled?() and MediaProxy.url_proxiable?(url) do
# If preview proxy is enabled, it'll also hit media proxy (so we're caching both requests)
prefetch_url = MediaProxy.preview_url(url)
opts = Logger.debug("Prefetching #{inspect(url)} as #{inspect(prefetch_url)}")
if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
Keyword.put(@options, :recv_timeout, 10_000) HTTP.get(prefetch_url, [], adapter: adapter_options())
else end
@options
end end
url defp adapter_options do
|> MediaProxy.preview_url() if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
|> HTTP.get([], adapter: opts) Keyword.put(@adapter_options, :recv_timeout, 10_000)
else
@adapter_options
end
end end
def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do

View File

@ -33,6 +33,8 @@ defmodule Pleroma.Web.MediaProxy.Invalidation do
def prepare_urls(urls) do def prepare_urls(urls) do
urls urls
|> List.wrap() |> List.wrap()
|> Enum.map(&MediaProxy.url/1) |> Enum.map(fn url -> [MediaProxy.url(url), MediaProxy.preview_url(url)] end)
|> List.flatten()
|> Enum.uniq()
end end
end end

View File

@ -41,20 +41,16 @@ defmodule Pleroma.Web.MediaProxy do
def url("/" <> _ = url), do: url def url("/" <> _ = url), do: url
def url(url) do def url(url) do
if not enabled?() or not url_proxiable?(url) do if enabled?() and url_proxiable?(url) do
url
else
encode_url(url) encode_url(url)
else
url
end end
end end
@spec url_proxiable?(String.t()) :: boolean() @spec url_proxiable?(String.t()) :: boolean()
def url_proxiable?(url) do def url_proxiable?(url) do
if local?(url) or whitelisted?(url) do not local?(url) and not whitelisted?(url)
false
else
true
end
end end
def preview_url(url, preview_params \\ []) do def preview_url(url, preview_params \\ []) do
@ -69,7 +65,7 @@ defmodule Pleroma.Web.MediaProxy do
# Note: media proxy must be enabled for media preview proxy in order to load all # Note: media proxy must be enabled for media preview proxy in order to load all
# non-local non-whitelisted URLs through it and be sure that body size constraint is preserved. # non-local non-whitelisted URLs through it and be sure that body size constraint is preserved.
def preview_enabled?, do: enabled?() and Config.get([:media_preview_proxy, :enabled], false) def preview_enabled?, do: enabled?() and !!Config.get([:media_preview_proxy, :enabled])
def local?(url), do: String.starts_with?(url, Pleroma.Web.base_url()) def local?(url), do: String.starts_with?(url, Pleroma.Web.base_url())
@ -138,9 +134,13 @@ defmodule Pleroma.Web.MediaProxy do
if path = URI.parse(url_or_path).path, do: Path.basename(path) if path = URI.parse(url_or_path).path, do: Path.basename(path)
end end
def base_url do
Config.get([:media_proxy, :base_url], Web.base_url())
end
defp proxy_url(path, sig_base64, url_base64, filename) do defp proxy_url(path, sig_base64, url_base64, filename) do
[ [
Config.get([:media_proxy, :base_url], Web.base_url()), base_url(),
path, path,
sig_base64, sig_base64,
url_base64, url_base64,

View File

@ -51,7 +51,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
media_proxy_url = MediaProxy.url(url) media_proxy_url = MediaProxy.url(url)
with {:ok, %{status: status} = head_response} when status in 200..299 <- with {:ok, %{status: status} = head_response} when status in 200..299 <-
Pleroma.HTTP.request("head", media_proxy_url, [], [], adapter: [pool: :preview]) do Pleroma.HTTP.request("head", media_proxy_url, [], [], adapter: [pool: :media]) do
content_type = Tesla.get_header(head_response, "content-type") content_type = Tesla.get_header(head_response, "content-type")
handle_preview(content_type, conn, media_proxy_url) handle_preview(content_type, conn, media_proxy_url)
else else
@ -154,8 +154,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
conn conn
|> put_resp_header("content-type", content_type) |> put_resp_header("content-type", content_type)
|> put_resp_header("content-disposition", "inline; filename=\"#{filename}\"") |> put_resp_header("content-disposition", "inline; filename=\"#{filename}\"")
# TODO: enable caching |> put_resp_header("cache-control", ReverseProxy.default_cache_control_header())
|> put_resp_header("cache-control", "max-age=0, private, must-revalidate")
end end
defp thumbnail_max_dimensions(params) do defp thumbnail_max_dimensions(params) do