add user import
This commit is contained in:
		
							parent
							
								
									72e22a6dae
								
							
						
					
					
						commit
						6bb19454fd
					
				
					 5 changed files with 54 additions and 12 deletions
				
			
		| 
						 | 
				
			
			@ -8,21 +8,28 @@ defmodule Mix.Tasks.Pleroma.Search do
 | 
			
		|||
  import Ecto.Query
 | 
			
		||||
  alias Pleroma.Activity
 | 
			
		||||
  alias Pleroma.Pagination
 | 
			
		||||
  alias Pleroma.User
 | 
			
		||||
 | 
			
		||||
  @shortdoc "Manages elasticsearch"
 | 
			
		||||
 | 
			
		||||
  def run(["import" | _rest]) do
 | 
			
		||||
  def run(["import", "activities" | _rest]) do
 | 
			
		||||
    start_pleroma()
 | 
			
		||||
 | 
			
		||||
    from(a in Activity, where: not ilike(a.actor, "%/relay"))
 | 
			
		||||
    |> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data))
 | 
			
		||||
    |> Activity.with_preloaded_object()
 | 
			
		||||
    |> Activity.with_preloaded_user_actor()
 | 
			
		||||
    |> get_all
 | 
			
		||||
    |> get_all(:activities)
 | 
			
		||||
  end
 | 
			
		||||
 | 
			
		||||
  defp get_all(query, max_id \\ nil) do
 | 
			
		||||
    IO.puts(max_id)
 | 
			
		||||
  def run(["import", "users" | _rest]) do
 | 
			
		||||
    start_pleroma()  
 | 
			
		||||
                     
 | 
			
		||||
    from(u in User, where: not ilike(u.ap_id, "%/relay"))
 | 
			
		||||
    |> get_all(:users)
 | 
			
		||||
  end
 | 
			
		||||
 | 
			
		||||
  defp get_all(query, index, max_id \\ nil) do
 | 
			
		||||
    params = %{limit: 2000}
 | 
			
		||||
 | 
			
		||||
    params =
 | 
			
		||||
| 
						 | 
				
			
			@ -40,15 +47,9 @@ defp get_all(query, max_id \\ nil) do
 | 
			
		|||
      :ok
 | 
			
		||||
    else
 | 
			
		||||
      res
 | 
			
		||||
      |> Enum.filter(fn x -> 
 | 
			
		||||
        t = x.object
 | 
			
		||||
	|> Map.get(:data, %{})
 | 
			
		||||
	|> Map.get("type", "")
 | 
			
		||||
	t == "Note"
 | 
			
		||||
      end)
 | 
			
		||||
      |> Pleroma.Elasticsearch.bulk_post(:activities)
 | 
			
		||||
      |> Pleroma.Elasticsearch.bulk_post(index)
 | 
			
		||||
 | 
			
		||||
      get_all(query, List.last(res).id)
 | 
			
		||||
      get_all(query, index, List.last(res).id)
 | 
			
		||||
    end
 | 
			
		||||
  end
 | 
			
		||||
end
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
							
								
								
									
										12
									
								
								lib/pleroma/elasticsearch/document_mappings/user.ex
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										12
									
								
								lib/pleroma/elasticsearch/document_mappings/user.ex
									
									
									
									
									
										Normal file
									
								
							| 
						 | 
				
			
			@ -0,0 +1,12 @@
 | 
			
		|||
defmodule Pleroma.Elasticsearch.DocumentMappings.User do
 | 
			
		||||
  def id(obj), do: obj.id
 | 
			
		||||
 | 
			
		||||
  def encode(%{actor_type: "Person"} = user) do
 | 
			
		||||
    %{
 | 
			
		||||
      timestamp: user.inserted_at,
 | 
			
		||||
      instance: URI.parse(user.ap_id).host,
 | 
			
		||||
      nickname: user.nickname,
 | 
			
		||||
      bio: user.bio
 | 
			
		||||
    }
 | 
			
		||||
  end
 | 
			
		||||
end
 | 
			
		||||
| 
						 | 
				
			
			@ -44,6 +44,12 @@ def put(%Activity{} = activity) do
 | 
			
		|||
  def bulk_post(data, :activities) do
 | 
			
		||||
    d =
 | 
			
		||||
      data
 | 
			
		||||
      |> Enum.filter(fn x ->
 | 
			
		||||
        t = x.object
 | 
			
		||||
        |> Map.get(:data, %{})
 | 
			
		||||
        |> Map.get("type", "")
 | 
			
		||||
        t == "Note"
 | 
			
		||||
      end)
 | 
			
		||||
      |> Enum.map(fn d ->
 | 
			
		||||
        [
 | 
			
		||||
          %{index: %{_id: DocumentMappings.Activity.id(d)}},
 | 
			
		||||
| 
						 | 
				
			
			@ -60,6 +66,25 @@ def bulk_post(data, :activities) do
 | 
			
		|||
    )
 | 
			
		||||
  end
 | 
			
		||||
 | 
			
		||||
  def bulk_post(data, :users) do
 | 
			
		||||
    d =
 | 
			
		||||
      data
 | 
			
		||||
      |> Enum.map(fn d ->
 | 
			
		||||
        [
 | 
			
		||||
          %{index: %{_id: DocumentMappings.User.id(d)}},
 | 
			
		||||
          DocumentMappings.User.encode(d)
 | 
			
		||||
        ]
 | 
			
		||||
      end)
 | 
			
		||||
      |> List.flatten()
 | 
			
		||||
 | 
			
		||||
    Elastix.Bulk.post(
 | 
			
		||||
      url(),
 | 
			
		||||
      d,
 | 
			
		||||
      index: "users",
 | 
			
		||||
      type: "user"
 | 
			
		||||
    )
 | 
			
		||||
  end
 | 
			
		||||
 | 
			
		||||
  def search_activities(q) do
 | 
			
		||||
    Elastix.Search.search(
 | 
			
		||||
      url(),
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
| 
						 | 
				
			
			@ -1,5 +1,9 @@
 | 
			
		|||
{
 | 
			
		||||
  "properties": {
 | 
			
		||||
    "timestamp": {
 | 
			
		||||
      "type": "date",
 | 
			
		||||
      "index": true
 | 
			
		||||
    },
 | 
			
		||||
    "instance": {
 | 
			
		||||
      "type": "keyword"
 | 
			
		||||
    },
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
		Loading…
	
		Reference in a new issue