Stats: Rebuild session smearing for timeseries (#5694)

* Refactor table_decider#partition_metrics

* Refactor query pipeline to return a list of subqueries after splitting

* Move order_by out of join logic

* Refactor joining logic in query_builder

1. JOIN type is now set in QueryOptimizer
2. JOIN logic is now table and list-size agnostic

* Comment an edge case

* Rebuild session/visit smearing

Previously, whenever graphing any visit metric hourly/realtime, visit_duration and other
visit metrics would be way higher than expected, due to long sessions
dragging each bucket up and up. Now visits/visitors metrics are still
smeared and other visit metrics are counted under last bucket user was
active in.

visits metric was also overcounted (see new tests).

* Remove unneeded case

* Unit test for smearing in tabledecider

Author: macobo

CHANGELOG.md

@@ -27,6 +27,9 @@ All notable changes to this project will be documented in this file.
 - Make clicking Compare / Disable Comparison in period picker menu close the menu
 - Do not log page views for hidden pages (prerendered pages and new tabs), until pages are viewed
 - Password-authenticated shared links now carry over dashboard params properly
+- Realtime and hourly graphs of visit duration, views per visit no longer overcount due to long-lasting sessions, instead showing each visit
+  when they occurred.
+- Fixed realtime and hourly graphs of visits overcounting
 
 ## v3.0.0 - 2025-04-11
 

lib/plausible/stats/query.ex

@@ -28,7 +28,9 @@ defmodule Plausible.Stats.Query do
             site_id: nil,
             site_native_stats_start_at: nil,
             # Contains information to determine how to combine legacy and new time on page metrics
-            time_on_page_data: %{}
+            time_on_page_data: %{},
+            sql_join_type: :left,
+            smear_session_metrics: false
 
   require OpenTelemetry.Tracer, as: Tracer
   alias Plausible.Stats.{DateTimeRange, Filters, Imported, Legacy, Comparisons}

lib/plausible/stats/query_optimizer.ex

@@ -26,6 +26,7 @@ defmodule Plausible.Stats.QueryOptimizer do
     4. Updates event:hostname filters to also apply on visit level for sane results.
     5. Removes revenue metrics from dashboard queries if not requested, present or unavailable for the site.
     6. Trims the date range to the current time if query.include.trim_relative_date_range is true.
+    7. Sets the join_type for the query based on the query.
 
   """
   def optimize(query) do
@@ -40,18 +41,12 @@ defmodule Plausible.Stats.QueryOptimizer do
   for sessions.
   """
   def split(query) do
-    {event_metrics, sessions_metrics, _other_metrics} =
-      query.metrics
-      |> Util.maybe_add_visitors_metric()
-      |> TableDecider.partition_metrics(query)
-
-    {
-      Query.set(query,
-        metrics: event_metrics,
-        include_imported: query.include_imported
-      ),
-      split_sessions_query(query, sessions_metrics)
-    }
+    query.metrics
+    |> Util.maybe_add_visitors_metric()
+    |> TableDecider.partition_metrics(query)
+    |> Enum.map(fn {table_type, metrics} ->
+      build_split_query(table_type, metrics, query)
+    end)
   end
 
   defp pipeline() do
@@ -62,7 +57,8 @@ defmodule Plausible.Stats.QueryOptimizer do
       &extend_hostname_filters_to_visit/1,
       &remove_revenue_metrics_if_unavailable/1,
       &set_time_on_page_data/1,
-      &trim_relative_date_range/1
+      &trim_relative_date_range/1,
+      &set_sql_join_type/1
     ]
   end
 
@@ -162,7 +158,17 @@ defmodule Plausible.Stats.QueryOptimizer do
     Enum.find(query.dimensions, &Time.time_dimension?/1)
   end
 
-  defp split_sessions_query(query, session_metrics) do
+  defp build_split_query(:events, metrics, query) do
+    {
+      :events,
+      Query.set(query,
+        metrics: metrics,
+        include_imported: query.include_imported
+      )
+    }
+  end
+
+  defp build_split_query(:sessions, metrics, query) do
     dimensions =
       query.dimensions
       |> Enum.map(fn
@@ -179,12 +185,21 @@ defmodule Plausible.Stats.QueryOptimizer do
         query.filters
       end
 
-    Query.set(query,
-      filters: filters,
-      metrics: session_metrics,
-      dimensions: dimensions,
-      include_imported: query.include_imported
-    )
+    {
+      :sessions,
+      Query.set(query,
+        filters: filters,
+        metrics: metrics,
+        dimensions: dimensions,
+        include_imported: query.include_imported
+      )
+    }
+  end
+
+  defp build_split_query(:sessions_smeared, metrics, query) do
+    {_, query} = build_split_query(:sessions, metrics, query)
+
+    {:sessions, Query.set(query, smear_session_metrics: true)}
   end
 
   on_ee do
@@ -299,4 +314,19 @@ defmodule Plausible.Stats.QueryOptimizer do
       |> DateTimeRange.to_timezone("Etc/UTC")
     end
   end
+
+  # Normally we can always LEFT JOIN as this is more performant and tables
+  # are expected to contain the same dimensions.
+
+  # The only exception is using the "time:minute" dimension where the sessions
+  # subquery might return more rows than the events one. That's because we're
+  # counting sessions in all time buckets they were active in even if no event
+  # occurred during that particular minute.
+  defp set_sql_join_type(query) do
+    if "time:minute" in query.dimensions do
+      Query.set(query, sql_join_type: :full)
+    else
+      query
+    end
+  end
 end

lib/plausible/stats/sql/expression.ex

@@ -64,7 +64,7 @@ defmodule Plausible.Stats.SQL.Expression do
     })
   end
 
-  def select_dimension(q, key, "time:hour", :sessions, query) do
+  def select_dimension(q, key, "time:hour", :sessions, query) when query.smear_session_metrics do
     # :TRICKY: ClickHouse timeSlots works off of unix epoch and is not
     #   timezone-aware. This means that for e.g. Asia/Katmandu (GMT+5:45)
     #   to work, we divide time into 15-minute buckets and later combine these
@@ -87,7 +87,8 @@ defmodule Plausible.Stats.SQL.Expression do
   end
 
   # :NOTE: This is not exposed in Query APIv2
-  def select_dimension(q, key, "time:minute", :sessions, query) do
+  def select_dimension(q, key, "time:minute", :sessions, query)
+      when query.smear_session_metrics do
     q
     |> join(:inner, [s], time_slot in time_slots(query, 60),
       as: :time_slot,
@@ -338,6 +339,12 @@ defmodule Plausible.Stats.SQL.Expression do
     })
   end
 
+  def session_metric(:visits, query) when query.smear_session_metrics do
+    wrap_alias([s], %{
+      visits: scale_sample(fragment("uniq(?)", s.session_id))
+    })
+  end
+
   def session_metric(:visits, _query) do
     wrap_alias([s], %{
       visits: scale_sample(fragment("greatest(sum(?), 0)", s.sign))

lib/plausible/stats/sql/query_builder.ex

@@ -15,15 +15,14 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
   require Plausible.Stats.SQL.Expression
 
   def build(query, site) do
-    {event_query, sessions_query} = QueryOptimizer.split(query)
-
-    event_q = build_events_query(site, event_query)
-    sessions_q = build_sessions_query(site, sessions_query)
-
-    join_query_results(
-      {event_q, event_query},
-      {sessions_q, sessions_query}
-    )
+    query
+    |> QueryOptimizer.split()
+    |> Enum.map(fn {table_type, table_query} ->
+      q = build_table_query(table_type, site, table_query)
+      {table_type, table_query, q}
+    end)
+    |> join_query_results(query)
+    |> build_order_by(query)
     |> paginate(query.pagination)
     |> select_total_rows(query.include.total_rows)
   end
@@ -32,9 +31,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
     Enum.reduce(query.order_by || [], q, &build_order_by(&2, query, &1))
   end
 
-  defp build_events_query(_site, %Query{metrics: []}), do: nil
-
-  defp build_events_query(site, events_query) do
+  defp build_table_query(:events, site, events_query) do
     q =
       from(
         e in "events_v2",
@@ -54,6 +51,25 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
     |> TimeOnPage.merge_legacy_time_on_page(events_query)
   end
 
+  defp build_table_query(:sessions, site, sessions_query) do
+    q =
+      from(
+        e in "sessions_v2",
+        where: ^SQL.WhereBuilder.build(:sessions, sessions_query),
+        select: ^select_session_metrics(sessions_query)
+      )
+
+    on_ee do
+      q = Plausible.Stats.Sampling.add_query_hint(q, sessions_query)
+    end
+
+    q
+    |> join_events_if_needed(sessions_query)
+    |> build_group_by(:sessions, sessions_query)
+    |> merge_imported(site, sessions_query)
+    |> SQL.SpecialMetrics.add(site, sessions_query)
+  end
+
   defp join_sessions_if_needed(q, query) do
     if TableDecider.events_join_sessions?(query) do
       sessions_q =
@@ -79,27 +95,6 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
     end
   end
 
-  defp build_sessions_query(_site, %Query{metrics: []}), do: nil
-
-  defp build_sessions_query(site, sessions_query) do
-    q =
-      from(
-        e in "sessions_v2",
-        where: ^SQL.WhereBuilder.build(:sessions, sessions_query),
-        select: ^select_session_metrics(sessions_query)
-      )
-
-    on_ee do
-      q = Plausible.Stats.Sampling.add_query_hint(q, sessions_query)
-    end
-
-    q
-    |> join_events_if_needed(sessions_query)
-    |> build_group_by(:sessions, sessions_query)
-    |> merge_imported(site, sessions_query)
-    |> SQL.SpecialMetrics.add(site, sessions_query)
-  end
-
   def join_events_if_needed(q, query) do
     if TableDecider.sessions_join_events?(query) do
       events_q =
@@ -173,24 +168,24 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
     )
   end
 
-  defp join_query_results({nil, _}, {nil, _}), do: nil
-
-  defp join_query_results({events_q, events_query}, {nil, _}),
-    do: events_q |> build_order_by(events_query)
+  # Only one table is being queried - skip joining!
+  defp join_query_results([{_table_type, _query, q}], _main_query), do: q
 
-  defp join_query_results({nil, events_query}, {sessions_q, _}),
-    do: sessions_q |> build_order_by(events_query)
+  # Multiple tables: join results based on dimensions, select metrics from each and the appropriate dimensions.
+  defp join_query_results(queries, main_query) do
+    queries
+    |> Enum.reduce(nil, fn
+      {_table_type, query, q}, nil ->
+        from(e in subquery(q))
+        |> select_join_metrics(query, query.metrics)
 
-  defp join_query_results({events_q, events_query}, {sessions_q, sessions_query}) do
-    {join_type, events_q_fields, sessions_q_fields} =
-      TableDecider.join_options(events_query, sessions_query)
-
-    join(subquery(events_q), join_type, [e], s in subquery(sessions_q),
-      on: ^build_group_by_join(events_query)
-    )
-    |> select_join_fields(events_query, events_q_fields, e)
-    |> select_join_fields(sessions_query, sessions_q_fields, s)
-    |> build_order_by(events_query)
+      {_table_type, query, q}, acc ->
+        join(acc, main_query.sql_join_type, [], s in subquery(q),
+          on: ^build_group_by_join(main_query)
+        )
+        |> select_join_metrics(query, query.metrics -- [:sample_percent])
+    end)
+    |> select_dimensions(main_query)
   end
 
   # NOTE: Old queries do their own pagination
@@ -214,8 +209,33 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
   def build_group_by_join(query) do
     query.dimensions
     |> Enum.map(fn dim ->
-      dynamic([e, s], field(e, ^shortname(query, dim)) == field(s, ^shortname(query, dim)))
+      dynamic([a, ..., b], field(a, ^shortname(query, dim)) == field(b, ^shortname(query, dim)))
     end)
     |> Enum.reduce(fn condition, acc -> dynamic([], ^acc and ^condition) end)
   end
+
+  defp select_join_metrics(q, query, metrics) do
+    Enum.reduce(metrics, q, fn
+      metric, q ->
+        select_merge_as(q, [..., x], %{
+          shortname(query, metric) => field(x, ^shortname(query, metric))
+        })
+    end)
+  end
+
+  defp select_dimensions(q, query) do
+    Enum.reduce(query.dimensions, q, fn dimension, q ->
+      # We generally select dimensions from the left-most table. Only exception is time:minute where
+      # we use sessions table as sessions are considered on-going during the whole period.
+      if query.sql_join_type == :full and "time:minute" == dimension do
+        select_merge_as(q, [..., x], %{
+          shortname(query, dimension) => field(x, ^shortname(query, dimension))
+        })
+      else
+        select_merge_as(q, [x], %{
+          shortname(query, dimension) => field(x, ^shortname(query, dimension))
+        })
+      end
+    end)
+  end
 end