diff --git a/assets/js/dashboard/extra/funnel-exploration.js b/assets/js/dashboard/extra/funnel-exploration.js new file mode 100644 index 000000000000..59031868bfff --- /dev/null +++ b/assets/js/dashboard/extra/funnel-exploration.js @@ -0,0 +1,183 @@ +import React, { useState, useEffect } from 'react' +import * as api from '../api' +import { useDashboardStateContext } from '../dashboard-state-context' +import { useSiteContext } from '../site-context' +import { createStatsQuery } from '../stats-query' +import { numberShortFormatter } from '../util/number-formatter' + +const PAGE_FILTER_KEYS = ['page', 'entry_page', 'exit_page'] + +function fetchColumnData(site, dashboardState, steps) { + // Page filters only apply to the first step — strip them for subsequent columns + const stateToUse = + steps.length > 0 + ? { + ...dashboardState, + filters: dashboardState.filters.filter( + ([_op, key]) => !PAGE_FILTER_KEYS.includes(key) + ) + } + : dashboardState + + const query = createStatsQuery(stateToUse, { + dimensions: ['event:label'], + metrics: ['visitors'] + }) + + if (steps.length > 0) { + const seqFilter = ['sequence', steps.map((s) => ['is', 'event:label', [s]])] + query.filters = [...query.filters, seqFilter] + } + + return api.stats(site, query) +} + +function ExplorationColumn({ header, steps, selected, onSelect, dashboardState }) { + const site = useSiteContext() + const [loading, setLoading] = useState(steps !== null) + const [results, setResults] = useState([]) + + useEffect(() => { + if (steps === null) { + setResults([]) + setLoading(false) + return + } + + setLoading(true) + setResults([]) + + fetchColumnData(site, dashboardState, steps) + .then((response) => { + setResults(response.results || []) + }) + .catch(() => { + setResults([]) + }) + .finally(() => { + setLoading(false) + }) + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [dashboardState, steps === null ? null : steps.join('|||')]) + + const maxVisitors = results.length > 0 ? results[0].metrics[0] : 1 + + return ( +
+
+ + {header} + + {selected && ( + + )} +
+ + {loading ? ( +
+
+
+
+
+ ) : results.length === 0 ? ( +
+ {steps === null ? 'Select an event to continue' : 'No data'} +
+ ) : ( + + )} +
+ ) +} + +function columnHeader(index) { + if (index === 0) return 'Start' + return `${index} step${index === 1 ? '' : 's'} after` +} + +export function FunnelExploration() { + const { dashboardState } = useDashboardStateContext() + const [steps, setSteps] = useState([]) + + function handleSelect(columnIndex, label) { + if (label === null) { + setSteps(steps.slice(0, columnIndex)) + } else { + setSteps([...steps.slice(0, columnIndex), label]) + } + } + + // Show 3 columns by default; add a new column each time the last column gets a selection + const numColumns = Math.max(3, steps.length + 1) + + return ( +
+

+ Explore user journeys +

+
+ {Array.from({ length: numColumns }, (_, i) => ( + = i ? steps.slice(0, i) : null} + selected={steps[i] || null} + onSelect={(label) => handleSelect(i, label)} + dashboardState={dashboardState} + /> + ))} +
+
+ ) +} + +export default FunnelExploration diff --git a/assets/js/dashboard/site-context.tsx b/assets/js/dashboard/site-context.tsx index ac6213b21c16..fec00ba96cc2 100644 --- a/assets/js/dashboard/site-context.tsx +++ b/assets/js/dashboard/site-context.tsx @@ -27,7 +27,9 @@ export function parseSiteFromDataset(dataset: DOMStringMap): PlausibleSite { } // Update this object when new feature flags are added to the frontend. -type FeatureFlags = Record +type FeatureFlags = { + funnel_exploration?: boolean +} export const siteContextDefaultValue = { domain: '', diff --git a/assets/js/dashboard/stats/behaviours/index.js b/assets/js/dashboard/stats/behaviours/index.js index 0ca81e0b1f70..43946099716a 100644 --- a/assets/js/dashboard/stats/behaviours/index.js +++ b/assets/js/dashboard/stats/behaviours/index.js @@ -32,16 +32,14 @@ import { getSpecialGoal, isPageViewGoal, isSpecialGoal } from '../../util/goals' /*global BUILD_EXTRA*/ /*global require*/ -function maybeRequire() { - if (BUILD_EXTRA) { - // eslint-disable-next-line @typescript-eslint/no-require-imports - return require('../../extra/funnel') - } else { - return { default: null } - } -} +// eslint-disable-next-line @typescript-eslint/no-require-imports +const Funnel = BUILD_EXTRA ? require('../../extra/funnel').default : null +// eslint-disable-next-line @typescript-eslint/no-require-imports +const FunnelExploration = BUILD_EXTRA + ? (require('../../extra/funnel-exploration').FunnelExploration ?? null) + : null -const Funnel = maybeRequire().default +const EXPLORE_MODE = '__explore__' function singleGoalFilterApplied(dashboardState) { const goalFilter = getGoalFilter(dashboardState) @@ -94,12 +92,21 @@ function storePropKey({ site, propKey, dashboardState }) { } } +const funnelExplorationAvailable = (site) => + FunnelExploration !== null && site.flags.funnel_exploration + function getDefaultSelectedFunnel({ site }) { const stored = storage.getItem(STORAGE_KEYS.getForFunnel({ site })) - const storedExists = stored && site.funnels.some((f) => f.name === stored) + const storedExists = + stored === EXPLORE_MODE + ? funnelExplorationAvailable(site) + : stored && site.funnels.some((f) => f.name === stored) if (storedExists) { return stored + } else if (funnelExplorationAvailable(site)) { + storage.setItem(STORAGE_KEYS.getForFunnel({ site }), EXPLORE_MODE) + return EXPLORE_MODE } else if (site.funnels.length > 0) { const firstAvailable = site.funnels[0].name storage.setItem(STORAGE_KEYS.getForFunnel({ site }), firstAvailable) @@ -291,7 +298,9 @@ function Behaviours({ importedDataInView, setMode, mode }) { } function renderFunnels() { - if (Funnel === null) { + if (selectedFunnel === EXPLORE_MODE && funnelExplorationAvailable(site)) { + return + } else if (Funnel === null) { return featureUnavailable() } else if (Funnel && selectedFunnel && site.funnelsAvailable) { return @@ -496,16 +505,28 @@ function Behaviours({ importedDataInView, setMode, mode }) { {!site.isConsolidatedView && isEnabled(Mode.FUNNELS) && Funnel && - (site.funnels.length > 0 && site.funnelsAvailable ? ( + (site.funnels.length > 0 && site.funnelsAvailable || + funnelExplorationAvailable(site) ? ( ({ - label: name, - onClick: setFunnelFactory(name), - selected: mode === Mode.FUNNELS && selectedFunnel === name - }))} + options={[ + ...(funnelExplorationAvailable(site) + ? [ + { + label: 'Explore', + onClick: setFunnelFactory(EXPLORE_MODE), + selected: mode === Mode.FUNNELS && selectedFunnel === EXPLORE_MODE + } + ] + : []), + ...site.funnels.map(({ name }) => ({ + label: name, + onClick: setFunnelFactory(name), + selected: mode === Mode.FUNNELS && selectedFunnel === name + })) + ]} searchable={true} > Funnels diff --git a/assets/js/types/query-api.d.ts b/assets/js/types/query-api.d.ts index 3dce7f674337..3ba323894017 100644 --- a/assets/js/types/query-api.d.ts +++ b/assets/js/types/query-api.d.ts @@ -43,6 +43,7 @@ export type SimpleFilterDimensions = | "event:name" | "event:page" | "event:hostname" + | "event:label" | "visit:source" | "visit:channel" | "visit:referrer" @@ -70,7 +71,7 @@ export type SimpleFilterDimensions = export type CustomPropertyFilterDimensions = string; export type GoalDimension = "event:goal"; export type TimeDimensions = "time" | "time:month" | "time:week" | "time:day" | "time:hour"; -export type FilterTree = FilterEntry | FilterAndOr | FilterNot | FilterHasDone; +export type FilterTree = FilterEntry | FilterAndOr | FilterNot | FilterHasDone | FilterSequence; export type FilterEntry = FilterWithoutGoals | FilterWithIs | FilterWithContains | FilterWithPattern; /** * @minItems 3 @@ -147,6 +148,11 @@ export type FilterNot = ["not", FilterTree]; * @maxItems 2 */ export type FilterHasDone = ["has_done" | "has_not_done", FilterTree]; +/** + * @minItems 2 + * @maxItems 2 + */ +export type FilterSequence = ["sequence", [FilterTree, ...FilterTree[]]]; /** * @minItems 2 * @maxItems 2 diff --git a/lib/plausible/stats/api_query_parser.ex b/lib/plausible/stats/api_query_parser.ex index 1aea69dcb437..bb2af5218642 100644 --- a/lib/plausible/stats/api_query_parser.ex +++ b/lib/plausible/stats/api_query_parser.ex @@ -87,13 +87,14 @@ defmodule Plausible.Stats.ApiQueryParser do defp parse_operator(["not" | _rest]), do: {:ok, :not} defp parse_operator(["has_done" | _rest]), do: {:ok, :has_done} defp parse_operator(["has_not_done" | _rest]), do: {:ok, :has_not_done} + defp parse_operator(["sequence" | _rest]), do: {:ok, :sequence} defp parse_operator(filter), do: {:error, %QueryError{code: :invalid_filters, message: "Unknown operator for filter '#{i(filter)}'."}} - def parse_filter_second(operator, [_, filters | _rest]) when operator in [:and, :or], + def parse_filter_second(operator, [_, filters | _rest]) when operator in [:and, :or, :sequence], do: parse_filters(filters) def parse_filter_second(operator, [_, filter | _rest]) @@ -127,7 +128,7 @@ defmodule Plausible.Stats.ApiQueryParser do end defp parse_filter_rest(operator, _filter) - when operator in [:not, :and, :or, :has_done, :has_not_done], + when operator in [:not, :and, :or, :has_done, :has_not_done, :sequence], do: {:ok, []} defp parse_clauses_list([operator, dimension, list | _rest] = filter) when is_list(list) do @@ -221,14 +222,14 @@ defmodule Plausible.Stats.ApiQueryParser do end end - defp parse_dimensions(dimensions) when is_list(dimensions) do + def parse_dimensions(dimensions) when is_list(dimensions) do parse_list( dimensions, &parse_dimension_entry(&1, "Invalid dimensions '#{i(dimensions)}'") ) end - defp parse_dimensions(nil), do: {:ok, []} + def parse_dimensions(nil), do: {:ok, []} def parse_order_by(order_by) when is_list(order_by) do parse_list(order_by, &parse_order_by_entry/1) diff --git a/lib/plausible/stats/dashboard/query_parser.ex b/lib/plausible/stats/dashboard/query_parser.ex index a969224ba1c4..0f12e9862228 100644 --- a/lib/plausible/stats/dashboard/query_parser.ex +++ b/lib/plausible/stats/dashboard/query_parser.ex @@ -18,6 +18,7 @@ defmodule Plausible.Stats.Dashboard.QueryParser do with {:ok, input_date_range} <- parse_input_date_range(params), {:ok, relative_date} <- parse_relative_date(params), {:ok, filters} <- parse_filters(params), + {:ok, dimensions} <- ApiQueryParser.parse_dimensions(params["dimensions"]), {:ok, metrics} <- parse_metrics(params), {:ok, include} <- parse_include(params) do {:ok, @@ -25,6 +26,7 @@ defmodule Plausible.Stats.Dashboard.QueryParser do input_date_range: input_date_range, relative_date: relative_date, filters: filters, + dimensions: dimensions, metrics: metrics, include: include, skip_goal_existence_check: true diff --git a/lib/plausible/stats/filters/filters.ex b/lib/plausible/stats/filters/filters.ex index 9bcfa2f545dc..643ee63b0512 100644 --- a/lib/plausible/stats/filters/filters.ex +++ b/lib/plausible/stats/filters/filters.ex @@ -44,7 +44,7 @@ defmodule Plausible.Stats.Filters do ] def event_table_visit_props(), do: @event_table_visit_props |> Enum.map(&to_string/1) - @event_props [:name, :page, :goal, :hostname] + @event_props [:name, :page, :goal, :hostname, :label] def event_props(), do: @event_props |> Enum.map(&to_string/1) @@ -98,7 +98,7 @@ defmodule Plausible.Stats.Filters do |> traverse( {0, false}, fn {depth, is_behavioral_filter}, operator -> - {depth + 1, is_behavioral_filter or operator in [:has_done, :has_not_done]} + {depth + 1, is_behavioral_filter or operator in [:has_done, :has_not_done, :sequence]} end ) |> Enum.filter(fn {_filter, {depth, is_behavioral_filter}} -> @@ -180,7 +180,7 @@ defmodule Plausible.Stats.Filters do [transformed_child] = transform_tree(child_filter, transformer) [[operator, transformed_child]] - {nil, [operator, filters]} when operator in [:and, :or] -> + {nil, [operator, filters]} when operator in [:and, :or, :sequence] -> [[operator, transform_filters(filters, transformer)]] # Reached a leaf node, return existing value @@ -207,7 +207,7 @@ defmodule Plausible.Stats.Filters do when operation in [:not, :ignore_in_totals_query, :has_done, :has_not_done] -> traverse_tree(child_filter, state_transformer.(state, operation), state_transformer) - [operation, filters] when operation in [:and, :or] -> + [operation, filters] when operation in [:and, :or, :sequence] -> traverse(filters, state_transformer.(state, operation), state_transformer) # Leaf node diff --git a/lib/plausible/stats/goals.ex b/lib/plausible/stats/goals.ex index 806635f3509e..8468043c50a4 100644 --- a/lib/plausible/stats/goals.ex +++ b/lib/plausible/stats/goals.ex @@ -13,7 +13,9 @@ defmodule Plausible.Stats.Goals do """ def preload_needed_goals(site, dimensions, filters) do if Enum.member?(dimensions, "event:goal") or - Filters.filtering_on_dimension?(filters, "event:goal") do + Enum.member?(dimensions, "event:label") or + Filters.filtering_on_dimension?(filters, "event:goal") or + Filters.filtering_on_dimension?(filters, "event:label") do site = Plausible.Repo.preload(site, :team) props_available? = Plausible.Billing.Feature.Props.check_availability(site.team) == :ok goals = Plausible.Goals.for_site(site, include_goals_with_custom_props?: props_available?) @@ -22,7 +24,7 @@ defmodule Plausible.Stats.Goals do # When grouping by event:goal, later pipeline needs to know which goals match filters exactly. # This can affect both calculations whether all goals have the same revenue currency and # whether we should skip imports. - matching_toplevel_filters: goals_matching_toplevel_filters(goals, filters), + matching_toplevel_filters: goals_matching_toplevel_filters(goals, dimensions, filters), all: goals } else @@ -143,14 +145,19 @@ defmodule Plausible.Stats.Goals do Enum.filter(goals, fn goal -> matches?(goal, filter, clause) end) end - defp goals_matching_toplevel_filters(goals, filters) do - Enum.reduce(filters, goals, fn - [_, "event:goal" | _] = filter, goals -> - goals_matching_any_clause(goals, filter) + defp goals_matching_toplevel_filters(goals, dimensions, filters) do + if Enum.member?(dimensions, "event:goal") or + Filters.filtering_on_dimension?(filters, "event:goal") do + Enum.reduce(filters, goals, fn + [_, "event:goal" | _] = filter, goals -> + goals_matching_any_clause(goals, filter) - _filter, goals -> - goals - end) + _filter, goals -> + goals + end) + else + [] + end end defp goals_matching_any_clause(goals, [_, _, clauses | _] = filter) do @@ -274,4 +281,17 @@ defmodule Plausible.Stats.Goals do def page_path_db_field(true = _imported?), do: :page def page_path_db_field(false = _imported?), do: :pathname + + @doc """ + Returns two parallel lists of event names and their display names for all + custom event goals. Used to build the `event:label` computed dimension via + ClickHouse's `transform` function. + """ + @spec event_name_display_name_arrays(Plausible.Stats.Query.t()) :: {[String.t()], [String.t()]} + def event_name_display_name_arrays(query) do + query.preloaded_goals.all + |> Enum.filter(&(&1.event_name != nil)) + |> Enum.map(&{&1.event_name, Plausible.Goal.display_name(&1)}) + |> Enum.unzip() + end end diff --git a/lib/plausible/stats/query_builder.ex b/lib/plausible/stats/query_builder.ex index ac98c174050b..82ee057acebe 100644 --- a/lib/plausible/stats/query_builder.ex +++ b/lib/plausible/stats/query_builder.ex @@ -344,44 +344,86 @@ defmodule Plausible.Stats.QueryBuilder do end defp validate_behavioral_filters(query) do - query.filters - |> Filters.traverse(0, fn behavioral_depth, operator -> - if operator in [:has_done, :has_not_done] do - behavioral_depth + 1 - else - behavioral_depth - end - end) - |> Enum.reduce_while(:ok, fn {[_operator, dimension | _rest], behavioral_depth}, :ok -> - cond do - behavioral_depth == 0 -> - # ignore non-behavioral filters - {:cont, :ok} - - behavioral_depth > 1 -> - {:halt, - {:error, - %QueryError{ - code: :invalid_filters, - message: - "Invalid filters. Behavioral filters (has_done, has_not_done) cannot be nested." - }}} - - not String.starts_with?(dimension, "event:") -> - {:halt, - {:error, - %QueryError{ - code: :invalid_filters, - message: - "Invalid filters. Behavioral filters (has_done, has_not_done) can only be used with event dimension filters." - }}} - - true -> - {:cont, :ok} - end - end) + with :ok <- validate_not_wrapping_sequence(query) do + query.filters + |> Filters.traverse(0, fn behavioral_depth, operator -> + if operator in [:has_done, :has_not_done, :sequence] do + behavioral_depth + 1 + else + behavioral_depth + end + end) + |> Enum.reduce_while(:ok, fn {[_operator, dimension | _rest], behavioral_depth}, :ok -> + cond do + behavioral_depth == 0 -> + # ignore non-behavioral filters + {:cont, :ok} + + behavioral_depth > 1 -> + {:halt, + {:error, + %QueryError{ + code: :invalid_filters, + message: + "Invalid filters. Behavioral filters (has_done, has_not_done, sequence) cannot be nested." + }}} + + not String.starts_with?(dimension, "event:") -> + {:halt, + {:error, + %QueryError{ + code: :invalid_filters, + message: + "Invalid filters. Behavioral filters (has_done, has_not_done, sequence) can only be used with event dimension filters." + }}} + + true -> + {:cont, :ok} + end + end) + end end + defp validate_not_wrapping_sequence(query) do + if sequence_inside_not?(query.filters) do + {:error, + %QueryError{ + code: :invalid_filters, + message: "Invalid filters. sequence filters cannot be wrapped in not." + }} + else + :ok + end + end + + defp sequence_inside_not?(filters) when is_list(filters) do + Enum.any?(filters, &filter_sequence_inside_not?/1) + end + + defp filter_sequence_inside_not?([:not, child]), do: subtree_has_sequence?(child) + + defp filter_sequence_inside_not?([op, children]) when op in [:and, :or], + do: sequence_inside_not?(children) + + defp filter_sequence_inside_not?([op, child]) + when op in [:has_done, :has_not_done, :ignore_in_totals_query], + do: filter_sequence_inside_not?(child) + + defp filter_sequence_inside_not?([:sequence, steps]), do: sequence_inside_not?(steps) + defp filter_sequence_inside_not?(_), do: false + + defp subtree_has_sequence?([:sequence | _]), do: true + + defp subtree_has_sequence?([op, children]) when op in [:and, :or] do + is_list(children) and Enum.any?(children, &subtree_has_sequence?/1) + end + + defp subtree_has_sequence?([op, child]) + when op in [:not, :has_done, :has_not_done, :ignore_in_totals_query], + do: subtree_has_sequence?(child) + + defp subtree_has_sequence?(_), do: false + defp validate_filtered_goals_exist(_query, %ParsedQueryParams{skip_goal_existence_check: true}), do: :ok diff --git a/lib/plausible/stats/sql/expression.ex b/lib/plausible/stats/sql/expression.ex index c48ae9e0d09f..0424b1bf4933 100644 --- a/lib/plausible/stats/sql/expression.ex +++ b/lib/plausible/stats/sql/expression.ex @@ -107,6 +107,26 @@ defmodule Plausible.Stats.SQL.Expression do }) end + def select_dimension(q, key, "event:label", :events, query) do + {event_names, display_names} = Plausible.Stats.Goals.event_name_display_name_arrays(query) + + select_merge_as(q, [t], %{ + key => + fragment( + "if(? = 'pageview', concat('Visit ', ?), transform(?, ?, ?, ?))", + t.name, + t.pathname, + t.name, + ^event_names, + ^display_names, + t.name + ) + }) + end + + def select_dimension(q, key, "event:label", :sessions, _query), + do: select_merge_as(q, [t], %{key => fragment("''")}) + def select_dimension(q, key, "event:name", _table, _query), do: select_merge_as(q, [t], %{key => t.name}) diff --git a/lib/plausible/stats/sql/where_builder.ex b/lib/plausible/stats/sql/where_builder.ex index 18353d34cba0..d667d81fba2e 100644 --- a/lib/plausible/stats/sql/where_builder.ex +++ b/lib/plausible/stats/sql/where_builder.ex @@ -153,10 +153,68 @@ defmodule Plausible.Stats.SQL.WhereBuilder do dynamic([], not (^add_filter(table, query, [:has_done, filter]))) end + defp add_filter(:sessions, query, [:sequence, steps]) do + completion_q = build_sequence_sessions_q(steps, query) + dynamic([t], t.session_id in subquery(completion_q)) + end + + defp add_filter(:events, query, [:sequence, steps]) do + if Enum.any?(query.dimensions, &String.starts_with?(&1, "event:")) do + completion_q = build_sequence_completion_q(steps, query) + + next_event_cond = + dynamic( + [e, {:completion, c}], + ^filter_site_id(query) and ^filter_time_range(:events, query) and + e.timestamp > c.step_ts + ) + + next_event_q = + from(e in "events_v2", + join: seq in subquery(completion_q), + as: :completion, + on: e.session_id == seq.session_id, + where: ^next_event_cond, + group_by: e.session_id, + select: %{session_id: e.session_id, timestamp: min(e.timestamp)} + ) + + dynamic( + [t], + fragment("(?, ?) IN ?", t.session_id, t.timestamp, subquery(next_event_q)) + ) + else + sessions_q = build_sequence_sessions_q(steps, query) + dynamic([t], t.session_id in subquery(sessions_q)) + end + end + defp add_filter(:events, _query, [:is, "event:name" | _rest] = filter) do in_clause(col_value(:name), filter) end + defp add_filter(:events, query, [_, "event:label" | _rest] = filter) do + {event_names, display_names} = Plausible.Stats.Goals.event_name_display_name_arrays(query) + + label_expr = + dynamic( + [t], + fragment( + "if(? = 'pageview', concat('Visit ', ?), transform(?, ?, ?, ?))", + t.name, + t.pathname, + t.name, + ^event_names, + ^display_names, + t.name + ) + ) + + filter_field_dynamic(label_expr, filter) + end + + defp add_filter(:sessions, _query, [_, "event:label" | _rest]), do: true + defp add_filter(:events, query, [_, "event:goal" | _rest] = filter) do Plausible.Stats.Goals.add_filter(query, filter) end @@ -296,6 +354,96 @@ defmodule Plausible.Stats.SQL.WhereBuilder do ) end + defp build_sequence_sessions_q(steps, query) do + completion_q = build_sequence_completion_q(steps, query) + from(q in subquery(completion_q), select: q.session_id) + end + + defp build_sequence_completion_q(steps, query) do + Enum.reduce(steps, nil, fn step, prev_q -> + if prev_q == nil do + step_cond = build_sequence_step_cond(step, query) + + from(e in "events_v2", + where: ^step_cond, + group_by: e.session_id, + select: %{session_id: e.session_id, step_ts: min(e.timestamp)} + ) + else + first_after_cond = + dynamic( + [e, {:prev_step, p}], + ^filter_site_id(query) and ^filter_time_range(:events, query) and + e.timestamp > p.step_ts + ) + + first_after_prev_q = + from(e in "events_v2", + join: prev in subquery(prev_q), + as: :prev_step, + on: e.session_id == prev.session_id, + where: ^first_after_cond, + group_by: e.session_id, + select: %{session_id: e.session_id, first_ts: min(e.timestamp)} + ) + + step_cond = build_sequence_step_cond(step, query) + + from(e in "events_v2", + join: first in subquery(first_after_prev_q), + on: e.session_id == first.session_id and e.timestamp == first.first_ts, + where: ^step_cond, + group_by: e.session_id, + select: %{session_id: e.session_id, step_ts: min(e.timestamp)} + ) + end + end) + end + + defp build_sequence_step_cond(step, query, ordering_cond \\ nil) do + site_time_cond = dynamic([], ^filter_site_id(query) and ^filter_time_range(:events, query)) + step_filter_cond = add_filter(:events, query, step) + + if ordering_cond do + dynamic([], ^site_time_cond and ^step_filter_cond and ^ordering_cond) + else + dynamic([], ^site_time_cond and ^step_filter_cond) + end + end + + defp filter_field_dynamic(value_expr, [:is | _] = filter) do + in_clause(value_expr, filter) + end + + defp filter_field_dynamic(value_expr, [:is_not | rest]) do + dynamic([], not (^filter_field_dynamic(value_expr, [:is | rest]))) + end + + defp filter_field_dynamic(value_expr, [:matches_wildcard, dim, glob_exprs | rest]) do + regexes = Enum.map(glob_exprs, &page_regex/1) + filter_field_dynamic(value_expr, [:matches, dim, regexes | rest]) + end + + defp filter_field_dynamic(value_expr, [:matches_wildcard_not | rest]) do + dynamic([], not (^filter_field_dynamic(value_expr, [:matches_wildcard | rest]))) + end + + defp filter_field_dynamic(value_expr, [:contains | _] = filter) do + contains_clause(value_expr, filter) + end + + defp filter_field_dynamic(value_expr, [:contains_not | rest]) do + dynamic([], not (^filter_field_dynamic(value_expr, [:contains | rest]))) + end + + defp filter_field_dynamic(value_expr, [:matches, _, clauses | _]) do + dynamic([x], fragment("multiMatchAny(?, ?)", ^value_expr, ^clauses)) + end + + defp filter_field_dynamic(value_expr, [:matches_not | rest]) do + dynamic([], not (^filter_field_dynamic(value_expr, [:matches | rest]))) + end + defp filter_field(db_field, [:matches_wildcard, _dimension, glob_exprs | _rest]) do page_regexes = Enum.map(glob_exprs, &page_regex/1) diff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.ex index d2c7b43f5cba..b8183fda3e8c 100644 --- a/lib/plausible_web/controllers/stats_controller.ex +++ b/lib/plausible_web/controllers/stats_controller.ex @@ -540,7 +540,7 @@ defmodule PlausibleWeb.StatsController do defp get_flags(user, site), do: - [] + [:funnel_exploration] |> Enum.map(fn flag -> {flag, FunWithFlags.enabled?(flag, for: user) || FunWithFlags.enabled?(flag, for: site)} end) diff --git a/priv/json-schemas/query-api-schema.json b/priv/json-schemas/query-api-schema.json index 99a50baddeae..1644ba6e3aed 100644 --- a/priv/json-schemas/query-api-schema.json +++ b/priv/json-schemas/query-api-schema.json @@ -262,6 +262,7 @@ "event:name", "event:page", "event:hostname", + "event:label", "visit:source", "visit:channel", "visit:referrer", @@ -455,7 +456,8 @@ { "$ref": "#/definitions/filter_entry" }, { "$ref": "#/definitions/filter_and_or" }, { "$ref": "#/definitions/filter_not" }, - { "$ref": "#/definitions/filter_has_done" } + { "$ref": "#/definitions/filter_has_done" }, + { "$ref": "#/definitions/filter_sequence" } ] }, "filter_not": { @@ -501,6 +503,23 @@ { "$ref": "#/definitions/filter_tree" } ] }, + "filter_sequence": { + "type": "array", + "additionalItems": false, + "minItems": 2, + "maxItems": 2, + "items": [ + { + "type": "string", + "enum": ["sequence"] + }, + { + "type": "array", + "items": { "$ref": "#/definitions/filter_tree" }, + "minItems": 1 + } + ] + }, "order_by_entry": { "type": "array", "additionalItems": false, diff --git a/test/plausible/stats/query/query_parse_and_build_test.exs b/test/plausible/stats/query/query_parse_and_build_test.exs index ef705cc883d1..2f72f276adea 100644 --- a/test/plausible/stats/query/query_parse_and_build_test.exs +++ b/test/plausible/stats/query/query_parse_and_build_test.exs @@ -641,7 +641,7 @@ defmodule Plausible.Stats.Query.QueryParseAndBuildTest do Query.parse_and_build(site, params, now: @now) assert error == - "Invalid filters. Behavioral filters (has_done, has_not_done) can only be used with event dimension filters." + "Invalid filters. Behavioral filters (has_done, has_not_done, sequence) can only be used with event dimension filters." end test "fails when nesting behavioral filters", %{site: site} do @@ -658,7 +658,7 @@ defmodule Plausible.Stats.Query.QueryParseAndBuildTest do Query.parse_and_build(site, params, now: @now) assert error == - "Invalid filters. Behavioral filters (has_done, has_not_done) cannot be nested." + "Invalid filters. Behavioral filters (has_done, has_not_done, sequence) cannot be nested." end for operator <- ["not", "or", "has_done", "has_not_done"] do diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_goal_dimension_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_goal_dimension_test.exs index 9deb1ee2ce75..498436cf827c 100644 --- a/test/plausible_web/controllers/api/external_stats_controller/query_goal_dimension_test.exs +++ b/test/plausible_web/controllers/api/external_stats_controller/query_goal_dimension_test.exs @@ -651,4 +651,35 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryGoalDimensionTest do ] end end + + describe "breakdown by event:label" do + test "does not return engagement events even when site has scroll goals", %{ + conn: conn, + site: site + } do + insert(:goal, %{site: site, page_path: "/blog", scroll_threshold: 50}) + + populate_stats(site, [ + build(:pageview, user_id: 234, pathname: "/blog", timestamp: ~N[2021-01-01 00:00:00]), + build(:engagement, + user_id: 234, + pathname: "/blog", + scroll_depth: 60, + timestamp: ~N[2021-01-01 00:00:01] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors"], + "dimensions" => ["event:label"] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["Visit /blog"], "metrics" => [1]} + ] + end + end end diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs index 810d7771f852..3a0b5fc99313 100644 --- a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs +++ b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs @@ -5084,7 +5084,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do assert %{"error" => error} = json_response(conn, 400) assert error =~ - "Invalid filters. Behavioral filters (has_done, has_not_done) can only be used with event dimension filters." + "Invalid filters. Behavioral filters (has_done, has_not_done, sequence) can only be used with event dimension filters." end end diff --git a/test/plausible_web/controllers/api/external_stats_controller/sequence_test.exs b/test/plausible_web/controllers/api/external_stats_controller/sequence_test.exs new file mode 100644 index 000000000000..a66c70094fa6 --- /dev/null +++ b/test/plausible_web/controllers/api/external_stats_controller/sequence_test.exs @@ -0,0 +1,1438 @@ +defmodule PlausibleWeb.Api.ExternalStatsController.SequenceTest do + use PlausibleWeb.ConnCase + + setup [:create_user, :create_site, :create_api_key, :use_api_key] + + # + # Events table path ("next event after sequence" semantics) + # + # When querying with event-specific dimensions (event:page, event:name, etc.), + # the sequence filter pins the result to the single immediate next event per + # session after the sequence completed. + # + + describe "sequence filter on events query" do + test "returns next event after 2-step sequence", %{conn: conn, site: site} do + populate_stats(site, [ + # Session 1: /pricing -> Signup -> /dashboard (sequence matches, next = /dashboard) + build(:event, + name: "pageview", + pathname: "/pricing", + user_id: 1, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Signup", + pathname: "/signup", + user_id: 1, + timestamp: ~N[2021-01-01 00:01:00] + ), + build(:event, + name: "pageview", + pathname: "/dashboard", + user_id: 1, + timestamp: ~N[2021-01-01 00:02:00] + ), + # Session 2: /pricing only — no Signup, no match + build(:event, + name: "pageview", + pathname: "/pricing", + user_id: 2, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "pageview", + pathname: "/other", + user_id: 2, + timestamp: ~N[2021-01-01 00:01:00] + ), + # Session 3: Signup only — step 1 never satisfied + build(:event, + name: "Signup", + pathname: "/signup", + user_id: 3, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "pageview", + pathname: "/thanks", + user_id: 3, + timestamp: ~N[2021-01-01 00:01:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:page"], + "filters" => [ + ["sequence", [["is", "event:page", ["/pricing"]], ["is", "event:name", ["Signup"]]]] + ] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["/dashboard"], "metrics" => [1]} + ] + end + + test "returns next event after 3-step sequence", %{conn: conn, site: site} do + populate_stats(site, [ + # Session 1: /a -> /b -> /c -> /result (full 3-step sequence, next = /result) + build(:event, + name: "pageview", + pathname: "/a", + user_id: 1, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "pageview", + pathname: "/b", + user_id: 1, + timestamp: ~N[2021-01-01 00:01:00] + ), + build(:event, + name: "pageview", + pathname: "/c", + user_id: 1, + timestamp: ~N[2021-01-01 00:02:00] + ), + build(:event, + name: "pageview", + pathname: "/result", + user_id: 1, + timestamp: ~N[2021-01-01 00:03:00] + ), + # Session 2: /a -> /b only — misses step 3 + build(:event, + name: "pageview", + pathname: "/a", + user_id: 2, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "pageview", + pathname: "/b", + user_id: 2, + timestamp: ~N[2021-01-01 00:01:00] + ), + build(:event, + name: "pageview", + pathname: "/other", + user_id: 2, + timestamp: ~N[2021-01-01 00:02:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:page"], + "filters" => [ + [ + "sequence", + [ + ["is", "event:page", ["/a"]], + ["is", "event:page", ["/b"]], + ["is", "event:page", ["/c"]] + ] + ] + ] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["/result"], "metrics" => [1]} + ] + end + + test "order is enforced — steps in wrong order do not match", %{conn: conn, site: site} do + populate_stats(site, [ + # Signup before /pricing — wrong order, should not match + build(:event, + name: "Signup", + pathname: "/signup", + user_id: 1, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "pageview", + pathname: "/pricing", + user_id: 1, + timestamp: ~N[2021-01-01 00:01:00] + ), + build(:event, + name: "pageview", + pathname: "/dashboard", + user_id: 1, + timestamp: ~N[2021-01-01 00:02:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:page"], + "filters" => [ + ["sequence", [["is", "event:page", ["/pricing"]], ["is", "event:name", ["Signup"]]]] + ] + }) + + assert json_response(conn, 200)["results"] == [] + end + + test "sessions with no event after sequence are excluded", %{conn: conn, site: site} do + populate_stats(site, [ + # Sequence completes but session ends — no next event to return + build(:event, + name: "pageview", + pathname: "/pricing", + user_id: 1, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Signup", + pathname: "/signup", + user_id: 1, + timestamp: ~N[2021-01-01 00:01:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:page"], + "filters" => [ + ["sequence", [["is", "event:page", ["/pricing"]], ["is", "event:name", ["Signup"]]]] + ] + }) + + assert json_response(conn, 200)["results"] == [] + end + + test "returns IMMEDIATE next event, not any future event", %{conn: conn, site: site} do + populate_stats(site, [ + # Session: /pricing -> Signup -> /step1 -> /step2 + # Next event after sequence is /step1, not /step2 + build(:event, + name: "pageview", + pathname: "/pricing", + user_id: 1, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Signup", + pathname: "/signup", + user_id: 1, + timestamp: ~N[2021-01-01 00:01:00] + ), + build(:event, + name: "pageview", + pathname: "/step1", + user_id: 1, + timestamp: ~N[2021-01-01 00:02:00] + ), + build(:event, + name: "pageview", + pathname: "/step2", + user_id: 1, + timestamp: ~N[2021-01-01 00:03:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:page"], + "filters" => [ + ["sequence", [["is", "event:page", ["/pricing"]], ["is", "event:name", ["Signup"]]]] + ] + }) + + # Only /step1 (the immediate next), not /step2 + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["/step1"], "metrics" => [1]} + ] + end + + test "multiple sessions each contribute their own next event", %{conn: conn, site: site} do + populate_stats(site, [ + # User 1: /pricing -> Signup -> /dashboard + build(:event, + name: "pageview", + pathname: "/pricing", + user_id: 1, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Signup", + pathname: "/signup", + user_id: 1, + timestamp: ~N[2021-01-01 00:01:00] + ), + build(:event, + name: "pageview", + pathname: "/dashboard", + user_id: 1, + timestamp: ~N[2021-01-01 00:02:00] + ), + # User 2: /pricing -> Signup -> /settings + build(:event, + name: "pageview", + pathname: "/pricing", + user_id: 2, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Signup", + pathname: "/signup", + user_id: 2, + timestamp: ~N[2021-01-01 00:01:00] + ), + build(:event, + name: "pageview", + pathname: "/settings", + user_id: 2, + timestamp: ~N[2021-01-01 00:02:00] + ), + # User 3: /pricing -> Signup -> /dashboard (same next page as user 1) + build(:event, + name: "pageview", + pathname: "/pricing", + user_id: 3, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Signup", + pathname: "/signup", + user_id: 3, + timestamp: ~N[2021-01-01 00:01:00] + ), + build(:event, + name: "pageview", + pathname: "/dashboard", + user_id: 3, + timestamp: ~N[2021-01-01 00:02:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:page"], + "filters" => [ + ["sequence", [["is", "event:page", ["/pricing"]], ["is", "event:name", ["Signup"]]]] + ] + }) + + results = json_response(conn, 200)["results"] + assert length(results) == 2 + assert %{"dimensions" => ["/dashboard"], "metrics" => [2]} in results + assert %{"dimensions" => ["/settings"], "metrics" => [1]} in results + end + + test "single-step sequence returns next event", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, + name: "pageview", + pathname: "/landing", + user_id: 1, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "pageview", + pathname: "/next", + user_id: 1, + timestamp: ~N[2021-01-01 00:01:00] + ), + # User 2: same landing page but different next page + build(:event, + name: "pageview", + pathname: "/landing", + user_id: 2, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "pageview", + pathname: "/other-next", + user_id: 2, + timestamp: ~N[2021-01-01 00:01:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:page"], + "filters" => [["sequence", [["is", "event:page", ["/landing"]]]]] + }) + + results = json_response(conn, 200)["results"] + assert length(results) == 2 + assert %{"dimensions" => ["/next"], "metrics" => [1]} in results + assert %{"dimensions" => ["/other-next"], "metrics" => [1]} in results + end + + test "intervening events between steps do not count as a sequence match", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + # User 1: /a -> /b -> /c + build(:event, + name: "pageview", + pathname: "/a", + user_id: 1, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "pageview", + pathname: "/b", + user_id: 1, + timestamp: ~N[2021-01-01 00:01:00] + ), + build(:event, + name: "pageview", + pathname: "/c", + user_id: 1, + timestamp: ~N[2021-01-01 00:02:00] + ), + # User 2: /a -> /x -> /b -> /c + build(:event, + name: "pageview", + pathname: "/a", + user_id: 2, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "pageview", + pathname: "/x", + user_id: 2, + timestamp: ~N[2021-01-01 00:01:00] + ), + build(:event, + name: "pageview", + pathname: "/b", + user_id: 2, + timestamp: ~N[2021-01-01 00:02:00] + ), + build(:event, + name: "pageview", + pathname: "/c", + user_id: 2, + timestamp: ~N[2021-01-01 00:03:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:page"], + "filters" => [ + ["sequence", [["is", "event:page", ["/a"]], ["is", "event:page", ["/b"]]]] + ] + }) + + # Only user 1 should match + # User 2 should not match because /x intervened between /a and /b. + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["/c"], "metrics" => [1]} + ] + end + + test "step matching on event:name works", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, name: "AddToCart", user_id: 1, timestamp: ~N[2021-01-01 00:00:00]), + build(:event, name: "Checkout", user_id: 1, timestamp: ~N[2021-01-01 00:01:00]), + build(:event, + name: "pageview", + pathname: "/confirmation", + user_id: 1, + timestamp: ~N[2021-01-01 00:02:00] + ), + # No AddToCart -> Checkout sequence + build(:event, name: "Checkout", user_id: 2, timestamp: ~N[2021-01-01 00:00:00]), + build(:event, + name: "pageview", + pathname: "/confirmation", + user_id: 2, + timestamp: ~N[2021-01-01 00:01:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:page"], + "filters" => [ + [ + "sequence", + [["is", "event:name", ["AddToCart"]], ["is", "event:name", ["Checkout"]]] + ] + ] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["/confirmation"], "metrics" => [1]} + ] + end + + test "step with `and` operator matches compound conditions on the same event", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + # Step 1: pageview on /pricing AND step 2: Signup + build(:event, + name: "pageview", + pathname: "/pricing", + user_id: 1, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, name: "Signup", user_id: 1, timestamp: ~N[2021-01-01 00:01:00]), + build(:event, + name: "pageview", + pathname: "/done", + user_id: 1, + timestamp: ~N[2021-01-01 00:02:00] + ), + # /other pageview then Signup — step 1 not satisfied (wrong page) + build(:event, + name: "pageview", + pathname: "/other", + user_id: 2, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, name: "Signup", user_id: 2, timestamp: ~N[2021-01-01 00:01:00]), + build(:event, + name: "pageview", + pathname: "/done", + user_id: 2, + timestamp: ~N[2021-01-01 00:02:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:page"], + "filters" => [ + [ + "sequence", + [ + ["and", [["is", "event:page", ["/pricing"]], ["is", "event:name", ["pageview"]]]], + ["is", "event:name", ["Signup"]] + ] + ] + ] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["/done"], "metrics" => [1]} + ] + end + + test "sequence does not cross session boundaries", %{conn: conn, site: site} do + populate_stats(site, [ + # User 1 session 1: /pricing (step 1 satisfied) + build(:event, + name: "pageview", + pathname: "/pricing", + user_id: 1, + timestamp: ~N[2021-01-01 00:00:00] + ), + # User 1 session 2 (31+ minutes later): Signup (should NOT chain with previous session's step 1) + build(:event, name: "Signup", user_id: 1, timestamp: ~N[2021-01-01 01:00:00]), + build(:event, + name: "pageview", + pathname: "/next", + user_id: 1, + timestamp: ~N[2021-01-01 01:01:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:page"], + "filters" => [ + ["sequence", [["is", "event:page", ["/pricing"]], ["is", "event:name", ["Signup"]]]] + ] + }) + + assert json_response(conn, 200)["results"] == [] + end + + test "sequence with event:name dimension works", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, + name: "pageview", + pathname: "/pricing", + user_id: 1, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, name: "Signup", user_id: 1, timestamp: ~N[2021-01-01 00:01:00]), + build(:event, name: "Purchase", user_id: 1, timestamp: ~N[2021-01-01 00:02:00]), + # No /pricing first + build(:event, name: "Signup", user_id: 2, timestamp: ~N[2021-01-01 00:00:00]), + build(:event, name: "Purchase", user_id: 2, timestamp: ~N[2021-01-01 00:01:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:name"], + "filters" => [ + ["sequence", [["is", "event:page", ["/pricing"]], ["is", "event:name", ["Signup"]]]] + ] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["Purchase"], "metrics" => [1]} + ] + end + end + + # + # Sessions table path (session membership semantics) + # + # When querying with non-event dimensions (visit:source, visit:country, etc.) + # or no dimensions at all, the sequence filter restricts to sessions where + # the sequence occurred — regardless of whether there is a next event. + # + + describe "sequence filter on sessions query" do + test "restricts sessions to those where sequence occurred", %{conn: conn, site: site} do + populate_stats(site, [ + # Session 1: sequence completed — counts + build(:event, + name: "pageview", + pathname: "/pricing", + user_id: 1, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, name: "Signup", user_id: 1, timestamp: ~N[2021-01-01 00:01:00]), + # Session 2: /pricing only — no Signup, no match + build(:event, + name: "pageview", + pathname: "/pricing", + user_id: 2, + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:source"], + "filters" => [ + ["sequence", [["is", "event:page", ["/pricing"]], ["is", "event:name", ["Signup"]]]] + ] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["Direct / None"], "metrics" => [1]} + ] + end + + test "counts session even when there is no next event after the sequence", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + # Sequence completes but session ends — sessions query should still count it + build(:event, + name: "pageview", + pathname: "/pricing", + user_id: 1, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, name: "Signup", user_id: 1, timestamp: ~N[2021-01-01 00:01:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:source"], + "filters" => [ + ["sequence", [["is", "event:page", ["/pricing"]], ["is", "event:name", ["Signup"]]]] + ] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["Direct / None"], "metrics" => [1]} + ] + end + + test "session membership is not affected by events after the sequence", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:event, + name: "pageview", + pathname: "/pricing", + user_id: 1, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, name: "Signup", user_id: 1, timestamp: ~N[2021-01-01 00:01:00]), + build(:event, + name: "pageview", + pathname: "/dashboard", + user_id: 1, + timestamp: ~N[2021-01-01 00:02:00] + ), + build(:event, + name: "pageview", + pathname: "/settings", + user_id: 1, + timestamp: ~N[2021-01-01 00:03:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:source"], + "filters" => [ + ["sequence", [["is", "event:page", ["/pricing"]], ["is", "event:name", ["Signup"]]]] + ] + }) + + # Only 1 visitor, not duplicated for each subsequent event + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["Direct / None"], "metrics" => [1]} + ] + end + + test "order is enforced on sessions query", %{conn: conn, site: site} do + populate_stats(site, [ + # Signup before /pricing — wrong order + build(:event, name: "Signup", user_id: 1, timestamp: ~N[2021-01-01 00:00:00]), + build(:event, + name: "pageview", + pathname: "/pricing", + user_id: 1, + timestamp: ~N[2021-01-01 00:01:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:source"], + "filters" => [ + ["sequence", [["is", "event:page", ["/pricing"]], ["is", "event:name", ["Signup"]]]] + ] + }) + + assert json_response(conn, 200)["results"] == [] + end + + test "3-step sequence on sessions query", %{conn: conn, site: site} do + populate_stats(site, [ + # Full 3-step sequence — counts + build(:event, + name: "pageview", + pathname: "/a", + user_id: 1, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "pageview", + pathname: "/b", + user_id: 1, + timestamp: ~N[2021-01-01 00:01:00] + ), + build(:event, + name: "pageview", + pathname: "/c", + user_id: 1, + timestamp: ~N[2021-01-01 00:02:00] + ), + # Only 2 out of 3 steps — no match + build(:event, + name: "pageview", + pathname: "/a", + user_id: 2, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "pageview", + pathname: "/b", + user_id: 2, + timestamp: ~N[2021-01-01 00:01:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:source"], + "filters" => [ + [ + "sequence", + [ + ["is", "event:page", ["/a"]], + ["is", "event:page", ["/b"]], + ["is", "event:page", ["/c"]] + ] + ] + ] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["Direct / None"], "metrics" => [1]} + ] + end + + test "aggregate visitors with no dimensions", %{conn: conn, site: site} do + populate_stats(site, [ + # User 1: completes sequence + build(:event, + name: "pageview", + pathname: "/pricing", + user_id: 1, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, name: "Signup", user_id: 1, timestamp: ~N[2021-01-01 00:01:00]), + # User 2: completes sequence + build(:event, + name: "pageview", + pathname: "/pricing", + user_id: 2, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, name: "Signup", user_id: 2, timestamp: ~N[2021-01-01 00:01:00]), + # User 3: no sequence + build(:event, + name: "pageview", + pathname: "/pricing", + user_id: 3, + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + ["sequence", [["is", "event:page", ["/pricing"]], ["is", "event:name", ["Signup"]]]] + ] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => [], "metrics" => [2]} + ] + end + + test "visits metric counts sessions with sequence", %{conn: conn, site: site} do + populate_stats(site, [ + # User 1: completes sequence + build(:event, + name: "pageview", + pathname: "/pricing", + user_id: 1, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, name: "Signup", user_id: 1, timestamp: ~N[2021-01-01 00:01:00]), + # User 2: no sequence + build(:event, + name: "pageview", + pathname: "/pricing", + user_id: 2, + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors", "visits"], + "date_range" => "all", + "filters" => [ + ["sequence", [["is", "event:page", ["/pricing"]], ["is", "event:name", ["Signup"]]]] + ] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => [], "metrics" => [1, 1]} + ] + end + + test "multiple users completing sequence are all counted", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, + name: "pageview", + pathname: "/pricing", + user_id: 1, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, name: "Signup", user_id: 1, timestamp: ~N[2021-01-01 00:01:00]), + build(:event, + name: "pageview", + pathname: "/pricing", + user_id: 2, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, name: "Signup", user_id: 2, timestamp: ~N[2021-01-01 00:01:00]), + build(:event, + name: "pageview", + pathname: "/pricing", + user_id: 3, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, name: "Signup", user_id: 3, timestamp: ~N[2021-01-01 00:01:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + ["sequence", [["is", "event:page", ["/pricing"]], ["is", "event:name", ["Signup"]]]] + ] + }) + + assert json_response(conn, 200)["results"] == [%{"dimensions" => [], "metrics" => [3]}] + end + + test "sequence steps satisfied multiple times in one session count visitor once", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + # User 1 visits /pricing -> Signup -> /pricing -> Signup twice in same session + build(:event, + name: "pageview", + pathname: "/pricing", + user_id: 1, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, name: "Signup", user_id: 1, timestamp: ~N[2021-01-01 00:01:00]), + build(:event, + name: "pageview", + pathname: "/pricing", + user_id: 1, + timestamp: ~N[2021-01-01 00:02:00] + ), + build(:event, name: "Signup", user_id: 1, timestamp: ~N[2021-01-01 00:03:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + ["sequence", [["is", "event:page", ["/pricing"]], ["is", "event:name", ["Signup"]]]] + ] + }) + + assert json_response(conn, 200)["results"] == [%{"dimensions" => [], "metrics" => [1]}] + end + + test "sequence combined with a top-level event filter", %{conn: conn, site: site} do + populate_stats(site, [ + # User 1: /pricing -> Signup -> /dashboard (sequence + has dashboard visit) + build(:event, + name: "pageview", + pathname: "/pricing", + user_id: 1, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, name: "Signup", user_id: 1, timestamp: ~N[2021-01-01 00:01:00]), + build(:event, + name: "pageview", + pathname: "/dashboard", + user_id: 1, + timestamp: ~N[2021-01-01 00:02:00] + ), + # User 2: /pricing -> Signup but no /dashboard + build(:event, + name: "pageview", + pathname: "/pricing", + user_id: 2, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, name: "Signup", user_id: 2, timestamp: ~N[2021-01-01 00:01:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + ["sequence", [["is", "event:page", ["/pricing"]], ["is", "event:name", ["Signup"]]]], + ["is", "event:page", ["/dashboard"]] + ] + }) + + assert json_response(conn, 200)["results"] == [%{"dimensions" => [], "metrics" => [1]}] + end + + test "sequence does not cross session boundaries on sessions query", %{conn: conn, site: site} do + populate_stats(site, [ + # User 1 session 1: /pricing (step 1) + build(:event, + name: "pageview", + pathname: "/pricing", + user_id: 1, + timestamp: ~N[2021-01-01 00:00:00] + ), + # User 1 session 2 (new session, 31+ minutes later): Signup — should NOT chain + build(:event, name: "Signup", user_id: 1, timestamp: ~N[2021-01-01 01:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + ["sequence", [["is", "event:page", ["/pricing"]], ["is", "event:name", ["Signup"]]]] + ] + }) + + assert json_response(conn, 200)["results"] == [%{"dimensions" => [], "metrics" => [0]}] + end + end + + # + # Validation errors + # + + describe "sequence filter validation" do + test "rejects sequence with visit dimension in steps", %{conn: conn, site: site} do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + ["sequence", [["is", "visit:source", ["Google"]], ["is", "event:name", ["Signup"]]]] + ] + }) + + assert json_response(conn, 400)["error"] =~ + "Behavioral filters (has_done, has_not_done, sequence) can only be used with event dimension filters" + end + + test "rejects sequence nested inside has_done", %{conn: conn, site: site} do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + [ + "has_done", + ["sequence", [["is", "event:page", ["/pricing"]], ["is", "event:name", ["Signup"]]]] + ] + ] + }) + + assert json_response(conn, 400)["error"] =~ + "Behavioral filters (has_done, has_not_done, sequence) cannot be nested" + end + + test "rejects sequence wrapped in not", %{conn: conn, site: site} do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + [ + "not", + ["sequence", [["is", "event:page", ["/pricing"]], ["is", "event:name", ["Signup"]]]] + ] + ] + }) + + assert json_response(conn, 400)["error"] =~ + "sequence filters cannot be wrapped in not" + end + + test "rejects has_done nested inside sequence steps", %{conn: conn, site: site} do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + [ + "sequence", + [ + ["has_done", ["is", "event:page", ["/pricing"]]], + ["is", "event:name", ["Signup"]] + ] + ] + ] + }) + + assert json_response(conn, 400)["error"] =~ + "Behavioral filters (has_done, has_not_done, sequence) cannot be nested" + end + + test "rejects sequence nested inside sequence", %{conn: conn, site: site} do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + [ + "sequence", + [ + ["sequence", [["is", "event:page", ["/a"]], ["is", "event:page", ["/b"]]]], + ["is", "event:name", ["Signup"]] + ] + ] + ] + }) + + assert json_response(conn, 400)["error"] =~ + "Behavioral filters (has_done, has_not_done, sequence) cannot be nested" + end + + test "rejects sequence with non-list steps (schema validation)", %{conn: conn, site: site} do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [["sequence", "not-a-list"]] + }) + + assert json_response(conn, 400)["error"] =~ "Invalid filter" + end + end + + # + # Edge cases and specific scenarios + # + + describe "sequence filter edge cases" do + test "step 1 event that also satisfies step 2 does not cause self-join match", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + # Single /pricing event — step 2 (also /pricing) must come AFTER step 1 + # so this single event should not match both steps + build(:event, + name: "pageview", + pathname: "/pricing", + user_id: 1, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "pageview", + pathname: "/next", + user_id: 1, + timestamp: ~N[2021-01-01 00:01:00] + ), + # Two /pricing events — step 1 = first, step 2 = second, next = /done + build(:event, + name: "pageview", + pathname: "/pricing", + user_id: 2, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "pageview", + pathname: "/pricing", + user_id: 2, + timestamp: ~N[2021-01-01 00:01:00] + ), + build(:event, + name: "pageview", + pathname: "/done", + user_id: 2, + timestamp: ~N[2021-01-01 00:02:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:page"], + "filters" => [ + ["sequence", [["is", "event:page", ["/pricing"]], ["is", "event:page", ["/pricing"]]]] + ] + }) + + # Only user 2 matches (two /pricing events), next = /done + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["/done"], "metrics" => [1]} + ] + end + + test "sequence with multiple possible step1 anchors uses the earliest one", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + # Two /pricing events, Signup comes after the first + # The earliest /pricing is the anchor, Signup is the step 2, /done is next + build(:event, + name: "pageview", + pathname: "/pricing", + user_id: 1, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, name: "Signup", user_id: 1, timestamp: ~N[2021-01-01 00:01:00]), + build(:event, + name: "pageview", + pathname: "/pricing", + user_id: 1, + timestamp: ~N[2021-01-01 00:02:00] + ), + build(:event, + name: "pageview", + pathname: "/done", + user_id: 1, + timestamp: ~N[2021-01-01 00:03:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:page"], + "filters" => [ + ["sequence", [["is", "event:page", ["/pricing"]], ["is", "event:name", ["Signup"]]]] + ] + }) + + # Next event after Signup (first completion) is /pricing (00:02), not /done + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["/pricing"], "metrics" => [1]} + ] + end + + test "sequence with or operator in step", %{conn: conn, site: site} do + populate_stats(site, [ + # Step 1: /pricing OR /plans, step 2: Signup + build(:event, + name: "pageview", + pathname: "/plans", + user_id: 1, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, name: "Signup", user_id: 1, timestamp: ~N[2021-01-01 00:01:00]), + build(:event, + name: "pageview", + pathname: "/dashboard", + user_id: 1, + timestamp: ~N[2021-01-01 00:02:00] + ), + build(:event, + name: "pageview", + pathname: "/pricing", + user_id: 2, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, name: "Signup", user_id: 2, timestamp: ~N[2021-01-01 00:01:00]), + build(:event, + name: "pageview", + pathname: "/dashboard", + user_id: 2, + timestamp: ~N[2021-01-01 00:02:00] + ), + # /other -> Signup — step 1 not satisfied + build(:event, + name: "pageview", + pathname: "/other", + user_id: 3, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, name: "Signup", user_id: 3, timestamp: ~N[2021-01-01 00:01:00]), + build(:event, + name: "pageview", + pathname: "/dashboard", + user_id: 3, + timestamp: ~N[2021-01-01 00:02:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:page"], + "filters" => [ + [ + "sequence", + [ + ["or", [["is", "event:page", ["/pricing"]], ["is", "event:page", ["/plans"]]]], + ["is", "event:name", ["Signup"]] + ] + ] + ] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["/dashboard"], "metrics" => [2]} + ] + end + + test "sequence combined with has_done at top level", %{conn: conn, site: site} do + populate_stats(site, [ + # User 1: /pricing -> Signup AND has done Purchase at some point + build(:event, + name: "pageview", + pathname: "/pricing", + user_id: 1, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, name: "Signup", user_id: 1, timestamp: ~N[2021-01-01 00:01:00]), + build(:event, name: "Purchase", user_id: 1, timestamp: ~N[2021-01-01 00:02:00]), + # User 2: /pricing -> Signup but no Purchase + build(:event, + name: "pageview", + pathname: "/pricing", + user_id: 2, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, name: "Signup", user_id: 2, timestamp: ~N[2021-01-01 00:01:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + ["sequence", [["is", "event:page", ["/pricing"]], ["is", "event:name", ["Signup"]]]], + ["has_done", ["is", "event:name", ["Purchase"]]] + ] + }) + + assert json_response(conn, 200)["results"] == [%{"dimensions" => [], "metrics" => [1]}] + end + + test "sequence with event:props step filter", %{conn: conn, site: site} do + populate_stats(site, [ + # User 1: FileDownload with type=pdf -> Signup + build(:event, + name: "FileDownload", + "meta.key": ["type"], + "meta.value": ["pdf"], + user_id: 1, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, name: "Signup", user_id: 1, timestamp: ~N[2021-01-01 00:01:00]), + # User 2: FileDownload with type=doc (not pdf) -> Signup — step 1 not satisfied + build(:event, + name: "FileDownload", + "meta.key": ["type"], + "meta.value": ["doc"], + user_id: 2, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, name: "Signup", user_id: 2, timestamp: ~N[2021-01-01 00:01:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + [ + "sequence", + [ + [ + "and", + [ + ["is", "event:name", ["FileDownload"]], + ["is", "event:props:type", ["pdf"]] + ] + ], + ["is", "event:name", ["Signup"]] + ] + ] + ] + }) + + assert json_response(conn, 200)["results"] == [%{"dimensions" => [], "metrics" => [1]}] + end + + test "returns imports warning when sequence filter is used with imported data", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:event, + name: "pageview", + pathname: "/pricing", + user_id: 1, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, name: "Signup", user_id: 1, timestamp: ~N[2021-01-01 00:01:00]) + ]) + + site_import = insert(:site_import, site: site) + + populate_stats(site, site_import.id, [ + build(:imported_visitors, date: ~D[2021-01-01]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "include" => %{"imports" => true}, + "filters" => [ + ["sequence", [["is", "event:page", ["/pricing"]], ["is", "event:name", ["Signup"]]]] + ] + }) + + assert json_response(conn, 200)["meta"]["imports_warning"] != nil + end + end +end