Merge branch 'master' into remove-goals-page-path-unique-index-migration

.formatter.exs

@@ -7,5 +7,6 @@
     "{config,lib,test,extra}/**/*.{heex,ex,exs}",
     "priv/*/seeds.exs",
     "storybook/**/*.exs"
-  ]
+  ],
+  locals_without_parens: [assert_matches: 1]
 ]

CHANGELOG.md

@@ -19,6 +19,7 @@ All notable changes to this project will be documented in this file.
 - Filters appear in the search bar as ?f=is,page,/docs,/blog&f=... instead of ?filters=((is,page,(/docs,/blog)),...) for Plausible links sent on various platforms to work reliably.
 - Details modal search inputs are now case-insensitive.
 - Improved report performance in cases where site has a lot of unique pathnames
+- Plausible script now uses `fetch` with keepalive flag as default over `XMLHttpRequest`. This will ensure more reliable tracking. Reminder to use `compat` script variant if tracking Internet Explorer is required.
 
 ### Fixed
 

lib/plausible/application.ex

@@ -120,6 +120,7 @@ defmodule Plausible.Application do
     setup_geolocation()
     Location.load_all()
     Plausible.Ingestion.Source.init()
+    Plausible.Ingestion.ScrollDepthVisibleAt.init()
     Plausible.Geo.await_loader()
 
     Supervisor.start_link(List.flatten(children), opts)

lib/plausible/exports.ex

@@ -418,8 +418,6 @@ defmodule Plausible.Exports do
     site = Plausible.Repo.get(Plausible.Site, site_id)
     current_user = current_user_id && Plausible.Repo.get(Plausible.Auth.User, current_user_id)
 
-    include_scroll_depth? = Plausible.Stats.ScrollDepth.check_feature_visible!(site, current_user)
-
     base_q =
       from(e in sampled("events_v2"),
         where: ^export_filter(site_id, date_range),
@@ -428,7 +426,7 @@ defmodule Plausible.Exports do
         order_by: selected_as(:date)
       )
 
-    if include_scroll_depth? do
+    if Plausible.Stats.ScrollDepth.feature_visible?(site, current_user) do
       max_scroll_depth_per_visitor_q =
         from(e in "events_v2",
           where: ^export_filter(site_id, date_range),

lib/plausible/ingestion/event.ex

@@ -130,7 +130,8 @@ defmodule Plausible.Ingestion.Event do
       put_user_id: &put_user_id/2,
       validate_clickhouse_event: &validate_clickhouse_event/2,
       register_session: &register_session/2,
-      write_to_buffer: &write_to_buffer/2
+      write_to_buffer: &write_to_buffer/2,
+      register_scroll_depth_visible: &register_scroll_depth_visible/2
     ]
   end
 
@@ -405,6 +406,15 @@ defmodule Plausible.Ingestion.Event do
     event
   end
 
+  defp register_scroll_depth_visible(%__MODULE__{clickhouse_event: ev} = event, _context) do
+    if is_nil(event.site.scroll_depth_visible_at) and ev.name in ["pageleave", "engagement"] and
+         not is_nil(ev.scroll_depth) and ev.scroll_depth >= 0 and ev.scroll_depth <= 100 do
+      Plausible.Ingestion.ScrollDepthVisibleAt.mark_scroll_depth_visible(event.site.id)
+    end
+
+    event
+  end
+
   @click_id_params ["gclid", "gbraid", "wbraid", "msclkid", "fbclid", "twclid"]
 
   defp get_click_id_param(nil), do: nil

lib/plausible/ingestion/scroll_depth_visible_at.ex

@@ -0,0 +1,44 @@
+defmodule Plausible.Ingestion.ScrollDepthVisibleAt do
+  @moduledoc """
+  Module that updates the `scroll_depth_visible_at` column for a site when needed.
+
+  This is called in a hot loop in ingestion, so it:
+  1. Only updates the database once per site async (if SiteCache doesn't know about visibility yet)
+  2. Does not retry the update if it fails, to be retried on server restart
+  """
+
+  require Logger
+  alias Plausible.Repo
+
+  import Ecto.Query
+
+  def init() do
+    :ets.new(__MODULE__, [
+      :named_table,
+      :set,
+      :public,
+      {:read_concurrency, true},
+      {:write_concurrency, true}
+    ])
+  end
+
+  def mark_scroll_depth_visible(site_id) do
+    if :ets.insert_new(__MODULE__, {site_id}) do
+      Task.start(fn -> attempt_update_repo(site_id) end)
+    end
+  end
+
+  defp attempt_update_repo(site_id) do
+    Repo.update_all(
+      from(s in Plausible.Site, where: s.id == ^site_id and is_nil(s.scroll_depth_visible_at)),
+      set: [
+        scroll_depth_visible_at: DateTime.utc_now()
+      ]
+    )
+  rescue
+    error ->
+      Logger.error(
+        "Error updating scroll_depth_visible_at for site #{site_id}: #{inspect(error)}. This will not be retried until server restart."
+      )
+  end
+end

lib/plausible/site/cache.ex

@@ -31,6 +31,7 @@ defmodule Plausible.Site.Cache do
     domain_changed_from
     ingest_rate_limit_scale_seconds
     ingest_rate_limit_threshold
+    scroll_depth_visible_at
    )a
 
   @impl true

lib/plausible/stats/scroll_depth.ex

@@ -3,11 +3,6 @@ defmodule Plausible.Stats.ScrollDepth do
   Module to check whether the scroll depth metric is available and visible for a site.
   """
 
-  import Ecto.Query
-  require Logger
-
-  alias Plausible.ClickhouseRepo
-
   def feature_available?(site, user) do
     FunWithFlags.enabled?(:scroll_depth, for: user) ||
       FunWithFlags.enabled?(:scroll_depth, for: site)
@@ -16,48 +11,4 @@ defmodule Plausible.Stats.ScrollDepth do
   def feature_visible?(site, user) do
     feature_available?(site, user) && not is_nil(site.scroll_depth_visible_at)
   end
-
-  @doc """
-  Checks whether the scroll depth feature is visible for a site and updates the site record if it is.
-
-  Note this function queries ClickHouse and may take a while to complete.
-  """
-  def check_feature_visible!(site, user) do
-    cond do
-      not feature_available?(site, user) ->
-        false
-
-      not is_nil(site.scroll_depth_visible_at) ->
-        true
-
-      is_nil(site.scroll_depth_visible_at) ->
-        visible? = has_scroll_data_last_30d?(site)
-
-        if visible? do
-          Plausible.Sites.set_scroll_depth_visible_at(site)
-        end
-
-        visible?
-    end
-  end
-
-  defp has_scroll_data_last_30d?(site) do
-    try do
-      ClickhouseRepo.exists?(
-        from(e in "events_v2",
-          where:
-            e.site_id == ^site.id and
-              e.name == "pageleave" and
-              e.timestamp >= fragment("toStartOfDay(now() - toIntervalDay(30))") and
-              e.scroll_depth > 0 and e.scroll_depth <= 100
-        )
-      )
-    rescue
-      # Avoid propagating error to the user, bringing down the site.
-      error ->
-        Logger.error("Error checking scroll data for site #{site.id}: #{inspect(error)}")
-
-        false
-    end
-  end
 end

lib/plausible_web/components/billing/plan_benefits.ex

@@ -110,12 +110,12 @@ defmodule PlausibleWeb.Components.Billing.PlanBenefits do
   defp site_limit_benefit(%Plan{} = plan), do: "Up to #{plan.site_limit} sites"
 
   defp feature_benefits(%Plan{} = plan) do
-    Enum.map(plan.features, fn feature_mod ->
+    Enum.flat_map(plan.features, fn feature_mod ->
       case feature_mod.name() do
-        :goals -> "Goals and custom events"
-        :stats_api -> "Stats API (600 requests per hour)"
-        :revenue_goals -> "Ecommerce revenue attribution"
-        _ -> feature_mod.display_name()
+        :goals -> ["Goals and custom events"]
+        :stats_api -> ["Stats API (600 requests per hour)", "Looker Studio Connector"]
+        :revenue_goals -> ["Ecommerce revenue attribution"]
+        _ -> [feature_mod.display_name()]
       end
     end)
   end

lib/plausible_web/controllers/stats_controller.ex

@@ -45,7 +45,7 @@ defmodule PlausibleWeb.StatsController do
   use Plausible.Repo
 
   alias Plausible.Sites
-  alias Plausible.Stats.{Filters, Query}
+  alias Plausible.Stats.{Filters, Query, ScrollDepth}
   alias PlausibleWeb.Api
 
   plug(PlausibleWeb.Plugs.AuthorizeSiteAccess when action in [:stats, :csv_export])
@@ -59,9 +59,6 @@ defmodule PlausibleWeb.StatsController do
     dogfood_page_path = if demo, do: "/#{site.domain}", else: "/:dashboard"
     skip_to_dashboard? = conn.params["skip_to_dashboard"] == "true"
 
-    scroll_depth_visible? =
-      Plausible.Stats.ScrollDepth.check_feature_visible!(site, current_user)
-
     cond do
       (stats_start_date && can_see_stats?) || (can_see_stats? && skip_to_dashboard?) ->
         conn
@@ -72,7 +69,7 @@ defmodule PlausibleWeb.StatsController do
           revenue_goals: list_revenue_goals(site),
           funnels: list_funnels(site),
           has_props: Plausible.Props.configured?(site),
-          scroll_depth_visible: scroll_depth_visible?,
+          scroll_depth_visible: ScrollDepth.feature_visible?(site, current_user),
           stats_start_date: stats_start_date,
           native_stats_start_date: NaiveDateTime.to_date(site.native_stats_start_at),
           title: title(conn, site),
@@ -350,9 +347,6 @@ defmodule PlausibleWeb.StatsController do
         shared_link = Plausible.Repo.preload(shared_link, site: :owner)
         stats_start_date = Plausible.Sites.stats_start_date(shared_link.site)
 
-        scroll_depth_visible? =
-          Plausible.Stats.ScrollDepth.check_feature_visible!(shared_link.site, current_user)
-
         conn
         |> put_resp_header("x-robots-tag", "noindex, nofollow")
         |> delete_resp_header("x-frame-options")
@@ -362,7 +356,7 @@ defmodule PlausibleWeb.StatsController do
           revenue_goals: list_revenue_goals(shared_link.site),
           funnels: list_funnels(shared_link.site),
           has_props: Plausible.Props.configured?(shared_link.site),
-          scroll_depth_visible: scroll_depth_visible?,
+          scroll_depth_visible: ScrollDepth.feature_visible?(shared_link.site, current_user),
           stats_start_date: stats_start_date,
           native_stats_start_date: NaiveDateTime.to_date(shared_link.site.native_stats_start_at),
           title: title(conn, shared_link.site),

lib/plausible_web/templates/layout/_tracking.html.heex

@@ -5,7 +5,6 @@
       data-api={PlausibleWeb.Dogfood.api_destination()}
       data-domain={PlausibleWeb.Dogfood.domain(@conn)}
       src={PlausibleWeb.Dogfood.script_url()}
-      data-allow-fetch
     >
     </script>
     <script>