Prometheus + JSON logging for grafana

compose.saas.yml

@@ -35,6 +35,15 @@ services:
       - 'traefik.http.routers.ethui-stacks-secure.tls.domains[0].main=stacks.ethui.dev'
       - 'traefik.http.routers.ethui-stacks-secure.tls.domains[0].sans=*.stacks.ethui.dev'
 
+      # block external access to /metrics
+      - "traefik.http.routesr.ethui-stacks-secure.middlewares=block-metrics"
+      - "traefik.http.middlewares.block-metrics.replacepathregex.regex=^/metrics.*"
+      - "traefik.http.middlewares.block-metrics.replacepathregex.replacement=/404-not-found"
+
+      # enabling scraping from alloy/prometheus
+      - "prometheus.scrape=true"
+      - "prometheus.port=4000"
+
 volumes:
   data:
 

server/config/runtime.exs

@@ -81,6 +81,31 @@ if config_env() == :prod do
 
   config :ethui, :jwt_secret, jwt_secret
 
+  # JSON logging configuration for production
+  config :logger, :default_handler,
+    formatter:
+      {LoggerJSON.Formatters.Basic,
+       metadata: :all, exclude_metadata: [:domain, :erl_level, :gl, :time]}
+
+  config :logger, :console,
+    format: {LoggerJSON.Formatters.Basic, :format},
+    metadata: [
+      :request_id,
+      :user_id,
+      :stack_slug,
+      :remote_ip,
+      :method,
+      :path,
+      :status,
+      :duration,
+      :pid,
+      :application,
+      :module,
+      :function,
+      :file,
+      :line
+    ]
+
   if is_saas? do
     config :ethui, Ethui.Mailer,
       adapter: Swoosh.Adapters.Mua,

server/lib/ethui/telemetry.ex

@@ -0,0 +1,10 @@
+defmodule Ethui.Telemetry do
+  @moduledoc """
+  Helper module for executing telemetry events with the [:ethui_stacks] prefix.
+  """
+
+  @spec exec(event :: [atom()], metadata :: map()) :: :ok
+  def exec(event, metadata \\ %{}) when is_list(event) do
+    :telemetry.execute([:ethui_stacks | event], %{count: 1}, metadata)
+  end
+end

server/lib/ethui_web/controllers/api/auth_controller.ex

@@ -12,6 +12,8 @@ defmodule EthuiWeb.Api.AuthController do
   """
   def send_code(conn, %{"email" => email}) do
     with {:ok, _user} <- Accounts.send_verification_code(email) do
+      Ethui.Telemetry.exec([:auth, :code_sent], %{email: email})
+
       render(conn, :send_code, message: "Verification code sent")
     end
   end
@@ -24,12 +26,18 @@ defmodule EthuiWeb.Api.AuthController do
   def verify_code(conn, %{"email" => email, "code" => code}) do
     case Accounts.verify_code_and_generate_token(email, code) do
       {:ok, token} ->
+        Ethui.Telemetry.exec([:auth, :code_verified], %{status: :success, email: email})
+
         render(conn, :verify_code, token: token)
 
       {:error, :invalid_code} ->
+        Ethui.Telemetry.exec([:auth, :code_verified], %{status: :invalid_code, email: email})
+
         {:error, "Invalid or expired verification code"}
 
       {:error, _reason} ->
+        Ethui.Telemetry.exec([:auth, :code_verified], %{status: :error, email: email})
+
         {:error, "Verification failed"}
     end
   end

server/lib/ethui_web/controllers/api/stack_controller.ex

@@ -32,6 +32,8 @@ defmodule EthuiWeb.Api.StackController do
 
     with {:ok, stack} <- Stacks.create_stack(user, params),
          _ <- Server.create(stack) do
+      Ethui.Telemetry.exec([:stacks, :created], %{user_id: user && user.id, stack_slug: stack.slug})
+
       conn
       |> put_status(:created)
       |> render(:create, stack: stack)
@@ -45,6 +47,8 @@ defmodule EthuiWeb.Api.StackController do
          :ok <- authorize_user_access(user, stack),
          _ <- Server.destroy(stack),
          {:ok, _} <- Stacks.delete_stack(stack) do
+      Ethui.Telemetry.exec([:stacks, :deleted], %{user_id: user && user.id, stack_slug: slug})
+
       send_resp(conn, :no_content, "")
     else
       nil ->

server/lib/ethui_web/controllers/metrics_controller.ex

@@ -0,0 +1,19 @@
+defmodule EthuiWeb.MetricsController do
+  use EthuiWeb, :controller
+
+  @moduledoc """
+  Exposes Prometheus metrics for scraping.
+
+  This endpoint is designed to be accessible only within the internal Docker/Dokploy network,
+  not via external Traefik routing. Prometheus should scrape metrics directly from the
+  application on port 4000.
+  """
+
+  def index(conn, _params) do
+    metrics = TelemetryMetricsPrometheus.Core.scrape(EthuiWeb.Telemetry.Prometheus)
+
+    conn
+    |> put_resp_content_type("text/plain")
+    |> send_resp(200, metrics)
+  end
+end

server/lib/ethui_web/endpoint.ex

@@ -34,6 +34,7 @@ defmodule EthuiWeb.Endpoint do
     cookie_key: "request_logger"
 
   plug Plug.RequestId
+  plug EthuiWeb.Plugs.LogMetadata
   plug Plug.Telemetry, event_prefix: [:phoenix, :endpoint]
 
   plug CORSPlug, origin: ["*"]

server/lib/ethui_web/plugs/log_metadata.ex

@@ -0,0 +1,68 @@
+defmodule EthuiWeb.Plugs.LogMetadata do
+  @moduledoc """
+  Adds request-specific metadata to logs for structured JSON logging.
+
+  This plug enriches logs with contextual information including:
+  - Request ID, remote IP, HTTP method, and path
+  - User ID (if authenticated)
+  - Stack slug (if available from subdomain routing)
+  - Response status and duration (added before sending response)
+  """
+
+  import Plug.Conn
+  require Logger
+
+  def init(opts), do: opts
+
+  def call(conn, _opts) do
+    Logger.metadata(
+      request_id: get_request_id(conn),
+      remote_ip: format_ip(conn.remote_ip),
+      method: conn.method,
+      path: conn.request_path
+    )
+
+    # Add user_id if authenticated
+    case conn.assigns[:current_user] do
+      %{id: user_id} -> Logger.metadata(user_id: user_id)
+      _ -> :ok
+    end
+
+    # Add stack_slug if available (from subdomain routing)
+    case conn.assigns[:stack] do
+      %{slug: slug} -> Logger.metadata(stack_slug: slug)
+      _ -> :ok
+    end
+
+    register_before_send(conn, fn conn ->
+      # Add response metadata before sending
+      Logger.metadata(
+        status: conn.status,
+        duration: calculate_duration(conn)
+      )
+      conn
+    end)
+  end
+
+  defp get_request_id(conn) do
+    case get_resp_header(conn, "x-request-id") do
+      [request_id] -> request_id
+      _ -> Logger.metadata()[:request_id]
+    end
+  end
+
+  defp format_ip({a, b, c, d}), do: "#{a}.#{b}.#{c}.#{d}"
+  defp format_ip(ip), do: inspect(ip)
+
+  defp calculate_duration(conn) do
+    case conn.private[:phoenix_endpoint_start] do
+      %{system: start} ->
+        System.monotonic_time()
+        |> Kernel.-(start)
+        |> System.convert_time_unit(:native, :microsecond)
+
+      _ ->
+        nil
+    end
+  end
+end

server/lib/ethui_web/router.ex

@@ -77,6 +77,11 @@ defmodule EthuiWeb.Router do
     end
   end
 
+  # Metrics endpoint - accessible only within internal Docker network
+  scope "/" do
+    get "/metrics", EthuiWeb.MetricsController, :index
+  end
+
   scope "/", EthuiWeb do
     pipe_through :proxy
 

server/lib/ethui_web/telemetry.ex

@@ -11,9 +11,9 @@ defmodule EthuiWeb.Telemetry do
     children = [
       # Telemetry poller will execute the given period measurements
       # every 10_000ms. Learn more here: https://hexdocs.pm/telemetry_metrics
-      {:telemetry_poller, measurements: periodic_measurements(), period: 10_000}
-      # Add reporters as children of your supervision tree.
-      # {Telemetry.Metrics.ConsoleReporter, metrics: metrics()}
+      {:telemetry_poller, measurements: periodic_measurements(), period: 10_000},
+      # Prometheus metrics reporter
+      {TelemetryMetricsPrometheus.Core, metrics: metrics(), name: __MODULE__.Prometheus}
     ]
 
     Supervisor.init(children, strategy: :one_for_one)
@@ -22,64 +22,110 @@ defmodule EthuiWeb.Telemetry do
   def metrics do
     [
       # Phoenix Metrics
-      summary("phoenix.endpoint.start.system_time",
-        unit: {:native, :millisecond}
-      ),
-      summary("phoenix.endpoint.stop.duration",
-        unit: {:native, :millisecond}
+      distribution("phoenix.endpoint.stop.duration",
+        unit: {:native, :millisecond},
+        description: "Phoenix endpoint response time",
+        reporter_options: [buckets: [10, 50, 100, 250, 500, 1000, 2500, 5000, 10000]]
       ),
-      summary("phoenix.router_dispatch.start.system_time",
+      distribution("phoenix.router_dispatch.stop.duration",
         tags: [:route],
-        unit: {:native, :millisecond}
+        unit: {:native, :millisecond},
+        description: "Phoenix router dispatch time by route",
+        reporter_options: [buckets: [10, 50, 100, 250, 500, 1000, 2500, 5000, 10000]]
       ),
-      summary("phoenix.router_dispatch.exception.duration",
+      distribution("phoenix.router_dispatch.exception.duration",
         tags: [:route],
-        unit: {:native, :millisecond}
+        unit: {:native, :millisecond},
+        description: "Phoenix router exception duration",
+        reporter_options: [buckets: [10, 50, 100, 250, 500, 1000, 2500, 5000, 10000]]
       ),
-      summary("phoenix.router_dispatch.stop.duration",
-        tags: [:route],
-        unit: {:native, :millisecond}
+      distribution("phoenix.socket_connected.duration",
+        unit: {:native, :millisecond},
+        description: "WebSocket connection time",
+        reporter_options: [buckets: [100, 250, 500, 1000, 2500, 5000]]
       ),
-      summary("phoenix.socket_connected.duration",
-        unit: {:native, :millisecond}
+      sum("phoenix.socket_drain.count",
+        description: "WebSocket drain count"
       ),
-      sum("phoenix.socket_drain.count"),
-      summary("phoenix.channel_joined.duration",
-        unit: {:native, :millisecond}
+      distribution("phoenix.channel_joined.duration",
+        unit: {:native, :millisecond},
+        description: "Channel join duration",
+        reporter_options: [buckets: [100, 250, 500, 1000, 2500, 5000]]
       ),
-      summary("phoenix.channel_handled_in.duration",
+      distribution("phoenix.channel_handled_in.duration",
         tags: [:event],
-        unit: {:native, :millisecond}
+        unit: {:native, :millisecond},
+        description: "Channel message handling duration",
+        reporter_options: [buckets: [10, 50, 100, 250, 500, 1000]]
       ),
 
       # Database Metrics
-      summary("ethui.repo.query.total_time",
+      distribution("ethui.repo.query.total_time",
         unit: {:native, :millisecond},
-        description: "The sum of the other measurements"
+        description: "Total database query time",
+        reporter_options: [buckets: [1, 5, 10, 25, 50, 100, 250, 500, 1000]]
       ),
-      summary("ethui.repo.query.decode_time",
+      distribution("ethui.repo.query.decode_time",
         unit: {:native, :millisecond},
-        description: "The time spent decoding the data received from the database"
+        description: "Time spent decoding database results",
+        reporter_options: [buckets: [1, 5, 10, 25, 50, 100, 250]]
       ),
-      summary("ethui.repo.query.query_time",
+      distribution("ethui.repo.query.query_time",
         unit: {:native, :millisecond},
-        description: "The time spent executing the query"
+        description: "Time spent executing database query",
+        reporter_options: [buckets: [1, 5, 10, 25, 50, 100, 250, 500, 1000]]
       ),
-      summary("ethui.repo.query.queue_time",
+      distribution("ethui.repo.query.queue_time",
         unit: {:native, :millisecond},
-        description: "The time spent waiting for a database connection"
+        description: "Time spent waiting for database connection",
+        reporter_options: [buckets: [1, 5, 10, 25, 50, 100, 250]]
       ),
-      summary("ethui.repo.query.idle_time",
+      distribution("ethui.repo.query.idle_time",
         unit: {:native, :millisecond},
-        description:
-          "The time the connection spent waiting before being checked out for the query"
+        description: "Database connection idle time before query",
+        reporter_options: [buckets: [10, 50, 100, 250, 500, 1000]]
       ),
 
       # VM Metrics
-      summary("vm.memory.total", unit: {:byte, :kilobyte}),
-      summary("vm.total_run_queue_lengths.total"),
-      summary("vm.total_run_queue_lengths.cpu"),
-      summary("vm.total_run_queue_lengths.io")
+      last_value("vm.memory.total",
+        unit: {:byte, :kilobyte},
+        description: "Total VM memory usage"
+      ),
+      last_value("vm.total_run_queue_lengths.total",
+        description: "Total run queue length"
+      ),
+      last_value("vm.total_run_queue_lengths.cpu",
+        description: "CPU run queue length"
+      ),
+      last_value("vm.total_run_queue_lengths.io",
+        description: "IO run queue length"
+      ),
+
+      # Application Metrics
+      counter("ethui.stacks.created.count",
+        description: "Total number of stacks created"
+      ),
+      counter("ethui.stacks.deleted.count",
+        description: "Total number of stacks deleted"
+      ),
+      last_value("ethui.stacks.active.count",
+        description: "Current number of active stacks"
+      ),
+      counter("ethui.api.requests.count",
+        tags: [:method, :path, :status],
+        description: "API request count by method, path, and status"
+      ),
+      counter("ethui.auth.code_sent.count",
+        description: "Number of authentication codes sent"
+      ),
+      counter("ethui.auth.code_verified.count",
+        tags: [:status],
+        description: "Number of authentication verification attempts"
+      ),
+      counter("ethui.errors.count",
+        tags: [:type],
+        description: "Application errors by type"
+      )
     ]
   end
 

server/mix.exs

@@ -57,6 +57,8 @@ defmodule Ethui.MixProject do
       {:finch, "~> 0.13"},
       {:telemetry_metrics, "~> 1.0"},
       {:telemetry_poller, "~> 1.0"},
+      {:telemetry_metrics_prometheus_core, "~> 1.2"},
+      {:logger_json, "~> 6.2"},
       {:jason, "~> 1.2"},
       {:dns_cluster, "~> 0.1.1"},
       {:plug_cowboy, "~> 2.7"},

server/mix.lock

@@ -44,6 +44,7 @@
   "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"},
   "joken": {:hex, :joken, "2.6.2", "5daaf82259ca603af4f0b065475099ada1b2b849ff140ccd37f4b6828ca6892a", [:mix], [{:jose, "~> 1.11.10", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "5134b5b0a6e37494e46dbf9e4dad53808e5e787904b7c73972651b51cce3d72b"},
   "jose": {:hex, :jose, "1.11.10", "a903f5227417bd2a08c8a00a0cbcc458118be84480955e8d251297a425723f83", [:mix, :rebar3], [], "hexpm", "0d6cd36ff8ba174db29148fc112b5842186b68a90ce9fc2b3ec3afe76593e614"},
+  "logger_json": {:hex, :logger_json, "6.2.1", "a1db30e1164e6057f2328a1e4d6b632b9583c015574fdf6c38cf73721128edcb", [:mix], [{:decimal, ">= 0.0.0", [hex: :decimal, repo: "hexpm", optional: true]}, {:ecto, "~> 3.11", [hex: :ecto, repo: "hexpm", optional: true]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:plug, "~> 1.15", [hex: :plug, repo: "hexpm", optional: true]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "34acd0bfd419d5fcf08c4108a8a4b59b695fcc60409dc1dd1a868b70c42e1d1f"},
   "mail": {:hex, :mail, "0.3.1", "cb0a14e4ed8904e4e5a08214e686ccf6f9099346885db17d8c309381f865cc5c", [:mix], [], "hexpm", "1db701e89865c1d5fa296b2b57b1cd587587cca8d8a1a22892b35ef5a8e352a6"},
   "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"},
   "mime": {:hex, :mime, "2.0.6", "8f18486773d9b15f95f4f4f1e39b710045fa1de891fada4516559967276e4dc2", [:mix], [], "hexpm", "c9945363a6b26d747389aac3643f8e0e09d30499a138ad64fe8fd1d13d9b153e"},
@@ -81,6 +82,7 @@
   "tailwind": {:hex, :tailwind, "0.3.1", "a89d2835c580748c7a975ad7dd3f2ea5e63216dc16d44f9df492fbd12c094bed", [:mix], [], "hexpm", "98a45febdf4a87bc26682e1171acdedd6317d0919953c353fcd1b4f9f4b676a2"},
   "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"},
   "telemetry_metrics": {:hex, :telemetry_metrics, "1.1.0", "5bd5f3b5637e0abea0426b947e3ce5dd304f8b3bc6617039e2b5a008adc02f8f", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e7b79e8ddfde70adb6db8a6623d1778ec66401f366e9a8f5dd0955c56bc8ce67"},
+  "telemetry_metrics_prometheus_core": {:hex, :telemetry_metrics_prometheus_core, "1.2.1", "c9755987d7b959b557084e6990990cb96a50d6482c683fb9622a63837f3cd3d8", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "5e2c599da4983c4f88a33e9571f1458bf98b0cf6ba930f1dc3a6e8cf45d5afb6"},
   "telemetry_poller": {:hex, :telemetry_poller, "1.2.0", "ba82e333215aed9dd2096f93bd1d13ae89d249f82760fcada0850ba33bac154b", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7216e21a6c326eb9aa44328028c34e9fd348fb53667ca837be59d0aa2a0156e8"},
   "tesla": {:hex, :tesla, "1.14.1", "71c5b031b4e089c0fbfb2b362e24b4478465773ae4ef569760a8c2899ad1e73c", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:finch, "~> 0.13", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, ">= 1.0.0", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.21", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "4.4.2", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:mox, "~> 1.0", [hex: :mox, repo: "hexpm", optional: true]}, {:msgpax, "~> 2.3", [hex: :msgpax, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "c1dde8140a49a3bef5bb622356e77ac5a24ad0c8091f12c3b7fc1077ce797155"},
   "thousand_island": {:hex, :thousand_island, "1.3.12", "590ff651a6d2a59ed7eabea398021749bdc664e2da33e0355e6c64e7e1a2ef93", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "55d0b1c868b513a7225892b8a8af0234d7c8981a51b0740369f3125f7c99a549"},