POC: Rollup view (#5719)

* POC rollup dashboard

* add rollups to stats api

* remove the irrelevant new controller action

* use team_identifier instead of team_id

Author: RobertJoonas

lib/plausible/site.ex

@@ -62,9 +62,23 @@ defmodule Plausible.Site do
 
     has_many :completed_imports, Plausible.Imported.SiteImport, where: [status: :completed]
 
+    field :rollup, :boolean, virtual: true, default: false
+
     timestamps()
   end
 
+  def rollup(team) do
+    %Plausible.Site{
+      id: 0,
+      native_stats_start_at: ~N[2018-01-01 00:00:00],
+      stats_start_date: ~D[2018-01-01],
+      rollup: true,
+      domain: "rollup:#{team.identifier}",
+      team: team,
+      team_id: team.id
+    }
+  end
+
   def new_for_team(team, params) do
     params
     |> new()

lib/plausible/stats/filters/query_parser.ex

@@ -47,8 +47,10 @@ defmodule Plausible.Stats.Filters.QueryParser do
          {:ok, pagination} <- parse_pagination(Map.get(params, "pagination", %{})),
          {preloaded_goals, revenue_warning, revenue_currencies} <-
            preload_goals_and_revenue(site, metrics, filters, dimensions),
+         rollup_site_ids = get_rollup_site_ids(site),
          query = %{
            now: now,
+           rollup_site_ids: rollup_site_ids,
            input_date_range: Map.get(params, "date_range"),
            metrics: metrics,
            filters: filters,
@@ -75,6 +77,12 @@ defmodule Plausible.Stats.Filters.QueryParser do
     end
   end
 
+  def get_rollup_site_ids(%Plausible.Site{rollup: true} = site) do
+    Plausible.Teams.owned_sites_ids(site.team)
+  end
+
+  def get_rollup_site_ids(_site), do: nil
+
   def parse_date_range_pair(site, [from, to]) when is_binary(from) and is_binary(to) do
     with {:ok, date_range} <- date_range_from_date_strings(site, from, to) do
       {:ok, date_range |> DateTimeRange.to_timezone("Etc/UTC")}

lib/plausible/stats/legacy/legacy_query_builder.ex

@@ -28,6 +28,7 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do
       |> put_parsed_filters(params)
       |> resolve_segments(site)
       |> preload_goals_and_revenue(site)
+      |> put_rollup_site_ids(site)
       |> put_order_by(params)
       |> put_include(site, params)
       |> Query.put_comparison_utc_time_range()
@@ -41,6 +42,13 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do
     query
   end
 
+  defp put_rollup_site_ids(query, %Plausible.Site{rollup: true} = site) do
+    site_ids = Plausible.Teams.owned_sites_ids(site.team)
+    struct!(query, rollup_site_ids: site_ids)
+  end
+
+  defp put_rollup_site_ids(query, _site), do: query
+
   defp resolve_segments(query, site) do
     with {:ok, preloaded_segments} <-
            Plausible.Segments.Filters.preload_needed_segments(site, query.filters),

lib/plausible/stats/query.ex

@@ -26,6 +26,7 @@ defmodule Plausible.Stats.Query do
             revenue_warning: nil,
             remove_unavailable_revenue_metrics: false,
             site_id: nil,
+            rollup_site_ids: nil,
             site_native_stats_start_at: nil,
             # Contains information to determine how to combine legacy and new time on page metrics
             time_on_page_data: %{},

lib/plausible/stats/sql/where_builder.ex

@@ -41,6 +41,19 @@ defmodule Plausible.Stats.SQL.WhereBuilder do
     end
   end
 
+  defp filter_site_time_range(
+         :events,
+         %Plausible.Stats.Query{rollup_site_ids: [_ | _] = site_ids} = query
+       ) do
+    {first_datetime, last_datetime} = utc_boundaries(query)
+
+    dynamic(
+      [e],
+      e.site_id in ^site_ids and e.timestamp >= ^first_datetime and
+        e.timestamp <= ^last_datetime
+    )
+  end
+
   defp filter_site_time_range(:events, query) do
     {first_datetime, last_datetime} = utc_boundaries(query)
 
@@ -51,6 +64,21 @@ defmodule Plausible.Stats.SQL.WhereBuilder do
     )
   end
 
+  defp filter_site_time_range(
+         :sessions,
+         %Plausible.Stats.Query{rollup_site_ids: [_ | _] = site_ids} = query
+       ) do
+    {first_datetime, last_datetime} = utc_boundaries(query)
+
+    dynamic(
+      [s],
+      s.site_id in ^site_ids and
+        s.start >= ^NaiveDateTime.add(first_datetime, -7, :day) and
+        s.timestamp >= ^first_datetime and
+        s.start <= ^last_datetime
+    )
+  end
+
   defp filter_site_time_range(:sessions, query) do
     {first_datetime, last_datetime} = utc_boundaries(query)
 

lib/plausible_web/plugs/authorize_public_api.ex

@@ -121,7 +121,7 @@ defmodule PlausibleWeb.Plugs.AuthorizePublicAPI do
 
   defp verify_by_scope(conn, api_key, "stats:read:" <> _ = scope) do
     with :ok <- check_scope(api_key, scope),
-         {:ok, site} <- find_site(conn.params["site_id"]),
+         {:ok, site} <- find_site(conn.params["site_id"], api_key),
          :ok <- verify_site_access(api_key, site) do
       Plausible.OpenTelemetry.add_site_attributes(site)
       site = Plausible.Repo.preload(site, :completed_imports)
@@ -173,9 +173,18 @@ defmodule PlausibleWeb.Plugs.AuthorizePublicAPI do
     end
   end
 
-  defp find_site(nil), do: {:error, :missing_site_id}
+  defp find_site(nil, _api_key), do: {:error, :missing_site_id}
 
-  defp find_site(site_id) do
+  defp find_site("rollup:" <> team_identifier, api_key) do
+    with true <- Plausible.Auth.is_super_admin?(api_key.user),
+         %Plausible.Teams.Team{} = team <- Plausible.Teams.get(team_identifier) do
+      {:ok, Plausible.Site.rollup(team)}
+    else
+      _ -> {:error, :invalid_api_key}
+    end
+  end
+
+  defp find_site(site_id, _api_key) do
     domain_based_search =
       from s in Plausible.Site, where: s.domain == ^site_id or s.domain_changed_from == ^site_id
 

lib/plausible_web/plugs/authorize_site_access.ex

@@ -172,6 +172,15 @@ defmodule PlausibleWeb.Plugs.AuthorizeSiteAccess do
     end
   end
 
+  defp get_site_with_role(conn, current_user, "rollup:" <> team_identifier) do
+    with true <- Plausible.Auth.is_super_admin?(current_user),
+         %Plausible.Teams.Team{} = team <- Plausible.Teams.get(team_identifier) do
+      {:ok, %{site: Plausible.Site.rollup(team), role: nil, member_type: nil}}
+    else
+      _ -> error_not_found(conn)
+    end
+  end
+
   defp get_site_with_role(conn, current_user, domain) do
     site = Repo.get_by(Plausible.Site, domain: domain)
 

test/plausible/stats/query_parser_test.exs

@@ -77,7 +77,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
         :input_date_range,
         :preloaded_goals,
         :revenue_warning,
-        :revenue_currencies
+        :revenue_currencies,
+        :rollup_site_ids
       ])
 
     assert result == expected_result