From b1fca46b389647f8730ce3feca1b2b8d71ee683d Mon Sep 17 00:00:00 2001 From: sarhiri Date: Mon, 29 Jun 2026 16:46:27 -0500 Subject: [PATCH 01/49] Replace the docs-style Overview homepage with a structured landing page built on Fumadocs components and custom JSX, keeping the existing sidebar and on-this-page TOC. Signed-off-by: sarhiri --- website/content/index.mdx | 159 ++++++++++++++------ website/src/app/(docs)/[[...slug]]/page.tsx | 8 +- 2 files changed, 122 insertions(+), 45 deletions(-) diff --git a/website/content/index.mdx b/website/content/index.mdx index 9fd552922cc..febbf51489b 100644 --- a/website/content/index.mdx +++ b/website/content/index.mdx @@ -1,61 +1,134 @@ --- -title: Overview +title: Armada --- import { ArmadaIcon, ArmadaText } from '@/components/logo'; - -
- - -
- - CircleCI - - - Go Report Card - - + +
+ + + + {/* blue glow behind the logo */} + + + {/* Hero Section/What is Armada? */} + +
+ + ## What is Armada? + + + {/* headline */} + + One API.
+ Any number of clusters.
+ Millions of jobs. + + + {/* Armada description */} + + Armada is the open-source batch job meta-scheduler that makes Kubernetes + handle massive-scale workloads, with fair queuing, gang scheduling, and multi-cluster + orchestration built in. + + + {/* buttons */} +
+ + Get started → + + - Artifact Hub - - - LFX Health Score - + rel='noreferrer' + className='rounded-lg border border-fd-border px-3.5 py-2 font-medium text-base no-underline transition-all hover:bg-fd-muted hover:shadow-lg hover:shadow-black/20 dark:hover:shadow-white/20' + > + View on GitHub + +
+ + {/* divider */} +
+
-## What is Armada? +{/* ## What is Armada? */} -Armada is a multi-Kubernetes cluster batch job meta-scheduler designed to handle massive-scale workloads. Built on top of Kubernetes, Armada enables organizations to distribute millions of batch jobs per day across tens of thousands of nodes spanning multiple clusters, making it an ideal solution for high-throughput computational workloads. +{/* Armada is a multi-Kubernetes cluster batch job meta-scheduler designed to handle massive-scale workloads. Built on top of Kubernetes, Armada enables organizations to distribute millions of batch jobs per day across tens of thousands of nodes spanning multiple clusters, making it an ideal solution for high-throughput computational workloads. Armada serves as middleware that transforms Kubernetes into a powerful batch processing platform while maintaining compatibility with service workloads. It addresses the fundamental limitations of running batch workloads at scale on Kubernetes by providing: - **Multi-cluster orchestration**: Schedule jobs across many Kubernetes clusters seamlessly - **High-throughput queueing**: Handle millions of queued jobs - **Advanced batch scheduling**: Fair queuing, gang scheduling, preemption, and resource limits -- **Enterprise-grade reliability**: Secure, highly available components designed for production use +- **Enterprise-grade reliability**: Secure, highly available components designed for production use */} As a [CNCF Sandbox project](https://www.cncf.io/), Armada is actively maintained and used in production environments, including at [G-Research](https://www.gresearch.com/) where it processes millions of jobs daily. diff --git a/website/src/app/(docs)/[[...slug]]/page.tsx b/website/src/app/(docs)/[[...slug]]/page.tsx index 741b39c38b5..aaa3dd1d64f 100644 --- a/website/src/app/(docs)/[[...slug]]/page.tsx +++ b/website/src/app/(docs)/[[...slug]]/page.tsx @@ -16,6 +16,8 @@ export default async function Page(props: { }) { const params = await props.params; const page = source.getPage(params.slug); + const isHome = !params.slug || params.slug.length === 0; + if (!page) notFound(); const MDXContent = page.data.body; @@ -41,8 +43,10 @@ export default async function Page(props: { editOnGithub={editOnGithub} lastUpdate={lastModified} > - {page.data.title} - {page.data.description} + {/* {page.data.title} + {page.data.description} */} + {!isHome && {page.data.title}} + {!isHome && {page.data.description}} Date: Mon, 29 Jun 2026 16:51:59 -0500 Subject: [PATCH 02/49] Updating website readme to match structure of code Signed-off-by: sarhiri --- website/README.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/website/README.md b/website/README.md index b1a39c6743b..8bf82616665 100644 --- a/website/README.md +++ b/website/README.md @@ -49,6 +49,44 @@ yarn format:fix yarn lint:fix ``` +## How the site works + +Every page on the site is an `.mdx` file under `content/`. MDX is Markdown that +can also use React components. You write normal Markdown, and you can drop in +components (cards, callouts, custom JSX) where you need them. + +The homepage is `content/index.mdx`. It is a content page like any other — there +is no separate "landing page" route. See "The homepage is a docs page" below for +the one way it is treated specially. + +### The left sidebar nav comes from `meta.json` + +The left-hand navigation is the **page tree**, built by the Fumadocs source +loader from `meta.json` files inside `content/`. + +- The root `content/meta.json` defines the top-level nav and has `"root": true`. +- A `meta.json` inside a folder controls that folder's title, order, and which + pages appear. +- A folder with **no** `meta.json` is invisible to the nav — the pages exist but + are not listed. (Handy to know if a page you created isn't showing up: check + whether its folder is declared in a `meta.json`.) + +So to change what appears in the sidebar — order, grouping, labels — you edit +`meta.json`, not the page files. + +### The right "On this page" TOC comes from Markdown headings + +The table of contents on the right is generated automatically from the Markdown +headings in each page — the `##` and `###` lines. Nothing else feeds it. + +This has one important consequence when you write custom layouts: + +- A real Markdown heading (`## Features` on its own line) **becomes a TOC entry.** +- A heading-looking `` or `

` you styled to look like a heading does + **not** appear in the TOC; it's just text to Fumadocs. + + + ## Learn More To learn more about Next.js, take a look at the following resources: From c34089ab4334a8d7e589c92b3308939078ae56cf Mon Sep 17 00:00:00 2001 From: sarhiri Date: Tue, 30 Jun 2026 15:52:47 -0500 Subject: [PATCH 03/49] Refine What is Armada section Signed-off-by: sarhiri --- website/content/index.mdx | 61 ++++++++++++++++++++++++++++++++------- 1 file changed, 50 insertions(+), 11 deletions(-) diff --git a/website/content/index.mdx b/website/content/index.mdx index febbf51489b..a75bd8e6140 100644 --- a/website/content/index.mdx +++ b/website/content/index.mdx @@ -4,6 +4,8 @@ title: Armada import { ArmadaIcon, ArmadaText } from '@/components/logo'; import Link from 'next/link'; +import { Cards, Card } from 'fumadocs-ui/components/card'; +import { Boxes, Scale, Zap, MoveUpRight, } from 'lucide-react';

@@ -14,7 +16,7 @@ import Link from 'next/link'; {/* blue glow behind the logo */}
@@ -61,7 +63,7 @@ import Link from 'next/link';
{/* Badges row 2 */}
-
+
CNCF Sandbox @@ -80,9 +82,9 @@ import Link from 'next/link'; {/* Hero Section/What is Armada? */}
- - ## What is Armada? - + {/* + ## Armada + */} {/* headline */} @@ -119,7 +121,7 @@ import Link from 'next/link';
-{/* ## What is Armada? */} +{/* ## What is Armada? - OLD */} {/* Armada is a multi-Kubernetes cluster batch job meta-scheduler designed to handle massive-scale workloads. Built on top of Kubernetes, Armada enables organizations to distribute millions of batch jobs per day across tens of thousands of nodes spanning multiple clusters, making it an ideal solution for high-throughput computational workloads. @@ -131,10 +133,49 @@ Armada serves as middleware that transforms Kubernetes into a powerful batch pro - **Enterprise-grade reliability**: Secure, highly available components designed for production use */} As a [CNCF Sandbox project](https://www.cncf.io/), Armada is actively maintained and used in production environments, including at [G-Research](https://www.gresearch.com/) where it processes millions of jobs daily. +{/* divider */} +
+ + +{/* ## Why use Armada? */} +
+ + ## Why use Armada? + + + {/* heading */} + + The batch scheduler Kubernetes was missing. + + + {/* description */} + + Armada sits above your Kubernetes clusters as a control plane. It doesn't replace Kubernetes, it allows Kubernetes to handle millions of jobs a day across tens of thousands of nodes. + + + {/* cards — left-accent, content stays left-aligned */} + + } title='Multi-cluster native' className='border-0 border-l-2 border-fd-primary rounded-sm shadow-md shadow-black/20 dark:shadow-white/10'> + One API across unlimited clusters. Add or remove capacity without downtime or disrupting running jobs. + + } title='Fair-share queuing' className='border-0 border-l-2 border-fd-primary rounded-sm shadow-md shadow-black/20 dark:shadow-white/10'> + Every team gets a proportional share of resources over time. Heavy users don't permanently crowd out everyone else. + + } title='Gang scheduling' className='border-0 border-l-2 border-fd-primary rounded-sm shadow-md shadow-black/20 dark:shadow-white/10'> + All workers in a job start together or not at all. Essential for MPI, PyTorch, and Spark. + + } title='Preemption' className='border-0 border-l-2 border-fd-primary rounded-sm shadow-md shadow-black/20 dark:shadow-white/10'> + Urgent jobs preempt lower-priority work automatically. Configurable per queue. + + +
+{/* divider */} +
-## Why Use Armada? -### Kubernetes Limitations for Batch Workloads +{/* ## Why use Armada? - OLD */} + +{/* ### Kubernetes Limitations for Batch Workloads Traditional Kubernetes faces several challenges when running batch workloads at scale: @@ -150,7 +191,7 @@ Armada overcomes these limitations by: - **Distributing across multiple clusters**: Manage thousands of nodes across many Kubernetes clusters - **Partial Out-of-cluster scheduling**: Leverage external storage backends (e.g., PostgreSQL and Redis) for high-throughput batch job queueing and scheduling -- **Purpose-built batch scheduler**: Include advanced scheduling features designed specifically for batch workloads +- **Purpose-built batch scheduler**: Include advanced scheduling features designed specifically for batch workloads */} ## Key Features and Benefits @@ -255,8 +296,6 @@ G-Research, a leading quantitative research company, uses Armada in production t Ready to explore Armada? Here are your next steps: -import { Cards, Card } from 'fumadocs-ui/components/card'; - Date: Tue, 30 Jun 2026 16:14:47 -0500 Subject: [PATCH 04/49] combining what armada does with features into one section Signed-off-by: sarhiri --- website/content/index.mdx | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/website/content/index.mdx b/website/content/index.mdx index a75bd8e6140..5b5bb7dbda1 100644 --- a/website/content/index.mdx +++ b/website/content/index.mdx @@ -5,7 +5,7 @@ title: Armada import { ArmadaIcon, ArmadaText } from '@/components/logo'; import Link from 'next/link'; import { Cards, Card } from 'fumadocs-ui/components/card'; -import { Boxes, Scale, Zap, MoveUpRight, } from 'lucide-react'; +import { Boxes, Scale, Zap, MoveUpRight, Activity, LayoutGrid } from 'lucide-react';
@@ -138,9 +138,9 @@ As a [CNCF Sandbox project](https://www.cncf.io/), Armada is actively maintained {/* ## Why use Armada? */} -
+
- ## Why use Armada? + ## What is Armada? {/* heading */} @@ -156,17 +156,23 @@ As a [CNCF Sandbox project](https://www.cncf.io/), Armada is actively maintained {/* cards — left-accent, content stays left-aligned */} } title='Multi-cluster native' className='border-0 border-l-2 border-fd-primary rounded-sm shadow-md shadow-black/20 dark:shadow-white/10'> - One API across unlimited clusters. Add or remove capacity without downtime or disrupting running jobs. - - } title='Fair-share queuing' className='border-0 border-l-2 border-fd-primary rounded-sm shadow-md shadow-black/20 dark:shadow-white/10'> - Every team gets a proportional share of resources over time. Heavy users don't permanently crowd out everyone else. - - } title='Gang scheduling' className='border-0 border-l-2 border-fd-primary rounded-sm shadow-md shadow-black/20 dark:shadow-white/10'> - All workers in a job start together or not at all. Essential for MPI, PyTorch, and Spark. - - } title='Preemption' className='border-0 border-l-2 border-fd-primary rounded-sm shadow-md shadow-black/20 dark:shadow-white/10'> - Urgent jobs preempt lower-priority work automatically. Configurable per queue. - + Run jobs across many clusters through one API, and add or remove capacity without disrupting what's already running. + + } title='Fair-share scheduling' className='border-0 border-l-2 border-fd-primary rounded-sm shadow-md shadow-black/20 dark:shadow-white/10'> + Every team gets a fair share of resources over time, so heavy users can't crowd everyone else out. + + } title='Gang scheduling' className='border-0 border-l-2 border-fd-primary rounded-sm shadow-md shadow-black/20 dark:shadow-white/10'> + All the workers in a job start together or not at all, which is what frameworks like MPI, PyTorch, and Spark need. + + } title='Intelligent preemption' className='border-0 border-l-2 border-fd-primary rounded-sm shadow-md shadow-black/20 dark:shadow-white/10'> + Urgent work can preempt lower-priority jobs to run in time, and you decide how that works per queue. + + } title='High throughput' className='border-0 border-l-2 border-fd-primary rounded-sm shadow-md shadow-black/20 dark:shadow-white/10'> + Handle millions of queued jobs by moving queueing onto PostgreSQL and Redis instead of leaning on etcd. + + } title='Built for production' className='border-0 border-l-2 border-fd-primary rounded-sm shadow-md shadow-black/20 dark:shadow-white/10'> + Prometheus metrics, Lookout web UI, secure auth, and automatic handling of failed nodes, all come built in. +
{/* divider */} @@ -193,7 +199,7 @@ Armada overcomes these limitations by: - **Partial Out-of-cluster scheduling**: Leverage external storage backends (e.g., PostgreSQL and Redis) for high-throughput batch job queueing and scheduling - **Purpose-built batch scheduler**: Include advanced scheduling features designed specifically for batch workloads */} -## Key Features and Benefits +{/* ## Key Features and Benefits ### Core Scheduling Features @@ -244,7 +250,7 @@ Armada overcomes these limitations by: - Secure authentication and authorization - High availability architecture -- Automatic node failure handling +- Automatic node failure handling */} ## Use Cases and Success Stories From 8499f99dad332c6af771245eca00f6d377f17a5b Mon Sep 17 00:00:00 2001 From: sarhiri Date: Tue, 30 Jun 2026 16:37:30 -0500 Subject: [PATCH 05/49] added use cases section Signed-off-by: sarhiri --- website/content/index.mdx | 107 +++++++++++++++++++++++++++++++++++--- 1 file changed, 101 insertions(+), 6 deletions(-) diff --git a/website/content/index.mdx b/website/content/index.mdx index 5b5bb7dbda1..5f2bf1c0dc4 100644 --- a/website/content/index.mdx +++ b/website/content/index.mdx @@ -82,9 +82,9 @@ import { Boxes, Scale, Zap, MoveUpRight, Activity, LayoutGrid } from 'lucide-rea {/* Hero Section/What is Armada? */}
- {/* + ## Armada - */} + {/* headline */} @@ -252,7 +252,102 @@ Armada overcomes these limitations by: - High availability architecture - Automatic node failure handling */} -## Use Cases and Success Stories + +{/* ## Use cases */} +
+ + ## Use Cases + + + {/* heading */} + + Is Armada Right for you? + + + {/* description */} + + + + {/* numbered list — divide-y draws the lines between rows; + font-mono primary number on the left, content on the right */} +
+ + {/* 01 */} +
+ 01 +
+ Machine learning training at scale + + Your workers need to start together or not at all. Armada's gang scheduling makes sure they do — across however many clusters have the GPU capacity you need. + +
+
+ + {/* 02 */} +
+ 02 +
+ Quantitative research and financial modelling + + Millions of short-lived jobs, every day. Armada keeps them moving fairly and fast, with priority controls for the calculations that can't wait. + +
+
+ + {/* 03 */} +
+ 03 +
+ High-performance computing + + MPI workloads in containers, scheduled across clusters, with hardware-aware placement. Cloud-native tooling without giving up the reproducibility HPC teams depend on. + +
+
+ + {/* 04 */} +
+ 04 +
+ Multi-tenant compute environments + + Multiple teams, one infrastructure. Fair-share scheduling means no single team can quietly consume everything while others wait. + +
+
+ + {/* 05 */} +
+ 05 +
+ CI/CD build and test + + Critical merges go first. Large test suites don't block urgent builds. Priority and fairness built in, no manual queue management. + +
+
+ +
+ + Armada's niche is multi-cluster. If you're not there yet, another project may be a better fit. + + + See how Armada compares → + +
+ +
+
+{/* divider */} +
+ + +{/* ## Use Cases and Success Stories ### High-Performance Computing (HPC) @@ -290,13 +385,13 @@ G-Research, a leading quantitative research company, uses Armada in production t - **Batch Features**: Purpose-built for batch vs. service-oriented design - **Fair Scheduling**: Advanced fair-use policies vs. basic priority classes -### vs. Traditional HPC Schedulers (SLURM, PBS) +### vs. Traditional HPC Schedulers (SLURM, PBS) */} -- **Container Native**: Built for containerized workloads vs. traditional HPC +{/* - **Container Native**: Built for containerized workloads vs. traditional HPC - **Kubernetes Integration**: Leverages Kubernetes ecosystem vs. isolated systems - **Cloud Ready**: Designed for cloud and hybrid environments - **Modern APIs**: REST/gRPC APIs vs. command-line interfaces -- **Rich Client Support**: Client libraries available for multiple languages (Go, Java, Scala, Python and .NET) +- **Rich Client Support**: Client libraries available for multiple languages (Go, Java, Scala, Python and .NET) */} ## Next Steps From 6d6709d59fb592f99feeb1192722aa1b095df8f1 Mon Sep 17 00:00:00 2001 From: sarhiri Date: Tue, 30 Jun 2026 17:01:06 -0500 Subject: [PATCH 06/49] Add cncf logo color svg, use cases section on landing page Signed-off-by: sarhiri --- website/content/index.mdx | 5 ++++- website/src/components/logo.tsx | 40 +++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/website/content/index.mdx b/website/content/index.mdx index 5f2bf1c0dc4..2571b95b0da 100644 --- a/website/content/index.mdx +++ b/website/content/index.mdx @@ -133,6 +133,9 @@ Armada serves as middleware that transforms Kubernetes into a powerful batch pro - **Enterprise-grade reliability**: Secure, highly available components designed for production use */} As a [CNCF Sandbox project](https://www.cncf.io/), Armada is actively maintained and used in production environments, including at [G-Research](https://www.gresearch.com/) where it processes millions of jobs daily. + + + {/* divider */}
@@ -329,7 +332,7 @@ Armada overcomes these limitations by:
- Armada's niche is multi-cluster. If you're not there yet, another project may be a better fit. + Armada's niche is multi-cluster. If you're not there yet, another project may be a better fit. ) => ( ); + +export const CncfLogo = (props: SVGProps) => ( + + Cloud Native Computing Foundation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +); \ No newline at end of file From 0f6e544d88c49ecf37f4412a4fa934117671951c Mon Sep 17 00:00:00 2001 From: sarhiri Date: Tue, 30 Jun 2026 17:06:59 -0500 Subject: [PATCH 07/49] added transitions to cards on homepage Signed-off-by: sarhiri --- website/content/index.mdx | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/website/content/index.mdx b/website/content/index.mdx index 2571b95b0da..c6869d8afb6 100644 --- a/website/content/index.mdx +++ b/website/content/index.mdx @@ -158,22 +158,22 @@ As a [CNCF Sandbox project](https://www.cncf.io/), Armada is actively maintained {/* cards — left-accent, content stays left-aligned */} - } title='Multi-cluster native' className='border-0 border-l-2 border-fd-primary rounded-sm shadow-md shadow-black/20 dark:shadow-white/10'> + } title='Multi-cluster native' className='border-0 border-l-2 border-fd-primary rounded-sm shadow-md shadow-black/20 dark:shadow-white/10 transition-colors hover:bg-fd-muted'> Run jobs across many clusters through one API, and add or remove capacity without disrupting what's already running. - } title='Fair-share scheduling' className='border-0 border-l-2 border-fd-primary rounded-sm shadow-md shadow-black/20 dark:shadow-white/10'> + } title='Fair-share scheduling' className='border-0 border-l-2 border-fd-primary rounded-sm shadow-md shadow-black/20 dark:shadow-white/10 transition-colors hover:bg-fd-muted'> Every team gets a fair share of resources over time, so heavy users can't crowd everyone else out. - } title='Gang scheduling' className='border-0 border-l-2 border-fd-primary rounded-sm shadow-md shadow-black/20 dark:shadow-white/10'> + } title='Gang scheduling' className='border-0 border-l-2 border-fd-primary rounded-sm shadow-md shadow-black/20 dark:shadow-white/10 transition-colors hover:bg-fd-muted'> All the workers in a job start together or not at all, which is what frameworks like MPI, PyTorch, and Spark need. - } title='Intelligent preemption' className='border-0 border-l-2 border-fd-primary rounded-sm shadow-md shadow-black/20 dark:shadow-white/10'> + } title='Intelligent preemption' className='border-0 border-l-2 border-fd-primary rounded-sm shadow-md shadow-black/20 dark:shadow-white/10 transition-colors hover:bg-fd-muted'> Urgent work can preempt lower-priority jobs to run in time, and you decide how that works per queue. - } title='High throughput' className='border-0 border-l-2 border-fd-primary rounded-sm shadow-md shadow-black/20 dark:shadow-white/10'> + } title='High throughput' className='border-0 border-l-2 border-fd-primary rounded-sm shadow-md shadow-black/20 dark:shadow-white/10 transition-colors hover:bg-fd-muted'> Handle millions of queued jobs by moving queueing onto PostgreSQL and Redis instead of leaning on etcd. - } title='Built for production' className='border-0 border-l-2 border-fd-primary rounded-sm shadow-md shadow-black/20 dark:shadow-white/10'> + } title='Built for production' className='border-0 border-l-2 border-fd-primary rounded-sm shadow-md shadow-black/20 dark:shadow-white/10 transition-colors hover:bg-fd-muted'> Prometheus metrics, Lookout web UI, secure auth, and automatic handling of failed nodes, all come built in. From 29c763c833daa5629c03595154648c3206af8cc5 Mon Sep 17 00:00:00 2001 From: sarhiri Date: Tue, 30 Jun 2026 18:27:44 -0500 Subject: [PATCH 08/49] formatted GR section Signed-off-by: sarhiri --- website/content/index.mdx | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/website/content/index.mdx b/website/content/index.mdx index c6869d8afb6..875ec259939 100644 --- a/website/content/index.mdx +++ b/website/content/index.mdx @@ -132,7 +132,9 @@ Armada serves as middleware that transforms Kubernetes into a powerful batch pro - **Advanced batch scheduling**: Fair queuing, gang scheduling, preemption, and resource limits - **Enterprise-grade reliability**: Secure, highly available components designed for production use */} -As a [CNCF Sandbox project](https://www.cncf.io/), Armada is actively maintained and used in production environments, including at [G-Research](https://www.gresearch.com/) where it processes millions of jobs daily. +
+ As a [CNCF Sandbox project](https://www.cncf.io/), Armada is actively maintained and used in production environments, including at [G-Research](https://www.gresearch.com/) where it processes millions of jobs daily. +
@@ -141,7 +143,7 @@ As a [CNCF Sandbox project](https://www.cncf.io/), Armada is actively maintained {/* ## Why use Armada? */} -
+
## What is Armada? @@ -257,7 +259,7 @@ Armada overcomes these limitations by: {/* ## Use cases */} -
+
## Use Cases From 6dd7a4d088caeab19c387a2cf7dff76a7c9e46f0 Mon Sep 17 00:00:00 2001 From: sarhiri Date: Tue, 30 Jun 2026 19:04:00 -0500 Subject: [PATCH 09/49] Landing page UI update MVP, old docs commented out until I get aproval on design and content Signed-off-by: sarhiri --- website/content/index.mdx | 49 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/website/content/index.mdx b/website/content/index.mdx index 875ec259939..e5812a825a7 100644 --- a/website/content/index.mdx +++ b/website/content/index.mdx @@ -398,6 +398,53 @@ G-Research, a leading quantitative research company, uses Armada in production t - **Modern APIs**: REST/gRPC APIs vs. command-line interfaces - **Rich Client Support**: Client libraries available for multiple languages (Go, Java, Scala, Python and .NET) */} + + +{/* ── CTA ── */} +
+ + + ## Next Steps + + + +{/* divider */} +
+ + + + + +{/* ## Next Steps Ready to explore Armada? Here are your next steps: @@ -413,4 +460,4 @@ Ready to explore Armada? Here are your next steps: title='Core Concepts' description='Learn about jobs, queues, and scheduling' /> - + */} From f598f851259b6ccde7a13d75d40f93ed7e2d94ae Mon Sep 17 00:00:00 2001 From: sarhiri Date: Mon, 6 Jul 2026 19:38:28 -0500 Subject: [PATCH 10/49] Finalized code structure, ran in dev environment. Needs content review for PR. Readme also updated Signed-off-by: sarhiri --- website/README.md | 43 +++++++++++++++------------- website/content/index.mdx | 60 ++++++++++++++++++--------------------- 2 files changed, 50 insertions(+), 53 deletions(-) diff --git a/website/README.md b/website/README.md index 8bf82616665..e016a10ac74 100644 --- a/website/README.md +++ b/website/README.md @@ -1,42 +1,45 @@ -# Armada - documentation website +# Armada Website Readme -This is a [Next.js](https://nextjs.org) project, based on the [Fumadocs](https://fumadocs.dev) framework, bootstrapped -using: - -- [`npx create-next-app@latest`](https://nextjs.org/docs/app/api-reference/cli/create-next-app). -- [`yarn create fumadocs-app`](https://github.com/fuma-nama/fumadocs). +The Armada documentation site is built with [Next.js](https://nextjs.org) and [Fumadocs](https://fumadocs.dev). All content is written in MDX, Markdown that can include React components. ## Requirements -- Node.js >= 20.x -- Yarn ~1.22.22 +Before you start, make sure you have the following installed: + +- **Node.js** >= 20.x — [Download](https://nodejs.org) +- **Yarn** ~1.22.22 — install with `npm install -g yarn` if you don't have it + +## Getting started + +All commands should be run from inside the `website/` folder. If you are at the root of the `armada` repository, navigate there first: + +```bash +cd website +``` + -## Installation +### Install dependencies -```shell +```bash yarn install ``` -## Local Development +### Start the local dev server ```bash yarn dev ``` -Open http://localhost:3000 on your browser to see the result. +Open [http://localhost:3000](http://localhost:3000) in your browser. The page will hot-reload as you edit files — you do not need to restart the server after making changes to content or components. -## Build and Preview +### GitHub Pages base path -To build the project for production and preview it, run: +The live site is deployed to GitHub Pages under a base path. If your changes involve links, images, or assets and you want to make sure they resolve correctly in that environment, copy `.env.example` to `.env.local` and set the base path before running the preview: ```bash -yarn build -# then -yarn preview +cp .env.example .env.local ``` - -The preview server will start on http://localhost:3000 by default. It also supports base path configuration to mimic -the GitHub Pages environment. Check the `.env.example` file to see how to set it up. +Then open `.env.local` and follow the instructions inside. You do not need this for most content changes: only if you are working on routing, assets, or the Next.js config itself. ## Format, Lint Content, Lint Code and Spell Check diff --git a/website/content/index.mdx b/website/content/index.mdx index e5812a825a7..95e27818f49 100644 --- a/website/content/index.mdx +++ b/website/content/index.mdx @@ -31,14 +31,8 @@ import { Boxes, Scale, Zap, MoveUpRight, Activity, LayoutGrid } from 'lucide-rea Latest release - - Go Report Card + + GitHub stars - Is Armada Right for you? + Is Armada right for you? {/* description */} @@ -407,33 +401,33 @@ G-Research, a leading quantitative research company, uses Armada in production t ## Next Steps -
+
-

- Ready to run batch at scale? -

+
+ Ready to run batch at scale? +
-

- Get Armada running locally in minutes. Join the community of organisations running batch workloads on Kubernetes. -

+
+ Get Armada running locally in minutes. Join the community of organisations running batch workloads on Kubernetes. +
- From e0039e5d73ee076a018de06102dc10c7e6a54b6b Mon Sep 17 00:00:00 2001 From: sarhiri Date: Tue, 7 Jul 2026 13:27:40 -0500 Subject: [PATCH 11/49] Navigation: Moved all documentation into designated 'docs' section Signed-off-by: sarhiri --- website/content/{user-guide => docs}/api.mdx | 0 .../architecture.mdx | 0 website/content/{user-guide => docs}/cli.mdx | 0 .../content/{user-guide => docs}/clients.mdx | 0 .../core-concepts.mdx | 0 .../content/{ => docs}/developer-guide.mdx | 0 .../{user-guide => docs}/integrations.mdx | 0 website/content/docs/meta.json | 9 ++++ website/content/{ => docs}/operator-guide.mdx | 0 website/content/docs/user-guide.mdx | 52 +++++++++++++++++++ website/content/meta.json | 5 +- 11 files changed, 62 insertions(+), 4 deletions(-) rename website/content/{user-guide => docs}/api.mdx (100%) rename website/content/{understanding-armada => docs}/architecture.mdx (100%) rename website/content/{user-guide => docs}/cli.mdx (100%) rename website/content/{user-guide => docs}/clients.mdx (100%) rename website/content/{understanding-armada => docs}/core-concepts.mdx (100%) rename website/content/{ => docs}/developer-guide.mdx (100%) rename website/content/{user-guide => docs}/integrations.mdx (100%) create mode 100644 website/content/docs/meta.json rename website/content/{ => docs}/operator-guide.mdx (100%) create mode 100644 website/content/docs/user-guide.mdx diff --git a/website/content/user-guide/api.mdx b/website/content/docs/api.mdx similarity index 100% rename from website/content/user-guide/api.mdx rename to website/content/docs/api.mdx diff --git a/website/content/understanding-armada/architecture.mdx b/website/content/docs/architecture.mdx similarity index 100% rename from website/content/understanding-armada/architecture.mdx rename to website/content/docs/architecture.mdx diff --git a/website/content/user-guide/cli.mdx b/website/content/docs/cli.mdx similarity index 100% rename from website/content/user-guide/cli.mdx rename to website/content/docs/cli.mdx diff --git a/website/content/user-guide/clients.mdx b/website/content/docs/clients.mdx similarity index 100% rename from website/content/user-guide/clients.mdx rename to website/content/docs/clients.mdx diff --git a/website/content/understanding-armada/core-concepts.mdx b/website/content/docs/core-concepts.mdx similarity index 100% rename from website/content/understanding-armada/core-concepts.mdx rename to website/content/docs/core-concepts.mdx diff --git a/website/content/developer-guide.mdx b/website/content/docs/developer-guide.mdx similarity index 100% rename from website/content/developer-guide.mdx rename to website/content/docs/developer-guide.mdx diff --git a/website/content/user-guide/integrations.mdx b/website/content/docs/integrations.mdx similarity index 100% rename from website/content/user-guide/integrations.mdx rename to website/content/docs/integrations.mdx diff --git a/website/content/docs/meta.json b/website/content/docs/meta.json new file mode 100644 index 00000000000..8a4e1c6fbaf --- /dev/null +++ b/website/content/docs/meta.json @@ -0,0 +1,9 @@ +{ + "title": "Docs", + "pages": [ + "core-concepts", + "developer-guide", + "architecture", + "..." + ] +} \ No newline at end of file diff --git a/website/content/operator-guide.mdx b/website/content/docs/operator-guide.mdx similarity index 100% rename from website/content/operator-guide.mdx rename to website/content/docs/operator-guide.mdx diff --git a/website/content/docs/user-guide.mdx b/website/content/docs/user-guide.mdx new file mode 100644 index 00000000000..351ae936516 --- /dev/null +++ b/website/content/docs/user-guide.mdx @@ -0,0 +1,52 @@ +--- +title: 'User Guide' +--- + +**Practical guidance for job submission and management.** + +Armada provides multiple ways to interact with the system, allowing you to choose the method that best fits your workflow. Whether you're submitting jobs interactively, integrating with existing pipelines, or building custom applications, Armada has the right interface for you. + +## Ways to Interact with Armada + +**Command-Line Interface (CLI) - Recommended for Most Users** + +`armadactl` is the official command-line tool and the **best way to interact with Armada** for most users. It provides a simple, intuitive interface for submitting jobs, managing queues, monitoring job status, and performing common operations. Built on the [Cobra](https://github.com/spf13/cobra) framework, `armadactl` offers a familiar command-line experience with comprehensive functionality. + +**Client Libraries - For Programmatic Access** + +For applications and scripts that need to programmatically interact with Armada, client libraries are available in multiple programming languages including Go, Java, Scala, Python, and .NET. These libraries wrap the Armada gRPC APIs and provide type-safe interfaces for job submission and management. + +**REST and gRPC APIs - For Custom Integrations** + +For custom integrations or when client libraries aren't available in your preferred language, Armada exposes REST and gRPC APIs. These APIs provide full access to all Armada functionality and are ideal for building custom tooling or integrating with existing systems. + +**Integrations - For Workflow Orchestration** + +Armada integrates seamlessly with popular workflow orchestration tools including Apache Airflow, Metaflow, Jenkins, and Apache Spark. These integrations allow you to leverage Armada's powerful scheduling capabilities within your existing workflows. + +## Getting Started + +import { Cards, Card } from 'fumadocs-ui/components/card'; + + + + + + + diff --git a/website/content/meta.json b/website/content/meta.json index 580f5373247..13c4d5fb4c2 100644 --- a/website/content/meta.json +++ b/website/content/meta.json @@ -3,10 +3,7 @@ "pages": [ "index", "getting-started", - "understanding-armada", - "operator-guide", - "user-guide", - "developer-guide", + "docs", "contribute", "community" ] From 8934d396d7e1b7422472daf7b148abb7314c7949 Mon Sep 17 00:00:00 2001 From: sarhiri Date: Tue, 7 Jul 2026 13:31:01 -0500 Subject: [PATCH 12/49] Updated navigation nesting for community Signed-off-by: sarhiri --- website/content/contribute/community.mdx | 41 ++++++++++++++++++++++++ website/content/meta.json | 3 +- 2 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 website/content/contribute/community.mdx diff --git a/website/content/contribute/community.mdx b/website/content/contribute/community.mdx new file mode 100644 index 00000000000..91bb3ac3795 --- /dev/null +++ b/website/content/contribute/community.mdx @@ -0,0 +1,41 @@ +--- +title: 'Community & Adopters' +description: 'Get help, connect with the community, and see who is using Armada.' +--- + +## Connect with us + +We'd love to hear from you! Whether you need help, want to contribute, or just want to say hello, there are several ways to connect with our community. + +### Slack + +Real-time interactions between Armada developers and users occurs primarily in [CNCF Slack](https://cloud-native.slack.com/archives/C03T9CBCEMC). This is where we gather to ask questions, share ideas, and connect with other users and maintainers. + +- If you already have an account on CNCF Slack, join #armada on [https://cloud-native.slack.com](https://cloud-native.slack.com) +- If you need an invitation to CNCF Slack, you can get one at [https://slack.cncf.io](https://slack.cncf.io) + +Don't hesitate to reach out if you need help getting started or have questions about using Armada in your environment. If you're wondering whether Armada is right for your use case, we'd love to hear about your requirements and help you evaluate. Jump into Slack and let's talk! + +### GitHub Discussions + +Armada uses GitHub Discussions for long-form communication and design discussions. To join the conversation there, go to: [https://github.com/armadaproject/armada/discussions](https://github.com/armadaproject/armada/discussions) + +This is the best place for brainstorming potential new features, sharing ideas, and having detailed technical discussions. + +### GitHub Issues & Pull Requests + +Found a bug or have a feature request? Open an issue on our [GitHub repository](https://github.com/armadaproject/armada/issues) and tell us about it. We're always looking for ways to improve. + +Have a fix or enhancement ready? We'd love to see your contribution! Submit a pull request on our [GitHub repository](https://github.com/armadaproject/armada/pulls). Whether it's code, documentation, or improvements, every contribution helps make Armada better for everyone. + +Interested in contributing but not sure where to start? Check out our [Contributor Guide](./contribute/contributor-guide.mdx) and browse open issues on GitHub. There's always something you can help with! + +## Community Meetings + +We host bi-weekly Armada Outreach meetings where we discuss project updates, gather feedback, and plan future developments. Join us to stay in the loop and share your thoughts! To receive an invitation link, reach out through our Slack channel. + +## Adopters + +Organizations around the world run Armada in production. See who's using Armada and how they're using it in our **[adopters list on GitHub](https://github.com/armadaproject/armada/blob/master/ADOPTERS.md)**. + +Using Armada at your organization? We'd love to hear your story—[add your organization](https://github.com/armadaproject/armada/blob/master/ADOPTERS.md) with a pull request. diff --git a/website/content/meta.json b/website/content/meta.json index 13c4d5fb4c2..10e3935a806 100644 --- a/website/content/meta.json +++ b/website/content/meta.json @@ -4,7 +4,6 @@ "index", "getting-started", "docs", - "contribute", - "community" + "contribute" ] } From 726e09f6ecd247b8a36e1c760d0875459dfeb911 Mon Sep 17 00:00:00 2001 From: sarhiri Date: Tue, 7 Jul 2026 13:38:31 -0500 Subject: [PATCH 13/49] updated Readme with nav instructions Signed-off-by: sarhiri --- website/README.md | 102 +++++++++++++++++++++++++++++++--------------- 1 file changed, 69 insertions(+), 33 deletions(-) diff --git a/website/README.md b/website/README.md index e016a10ac74..89ae0f56bbc 100644 --- a/website/README.md +++ b/website/README.md @@ -41,54 +41,90 @@ cp .env.example .env.local ``` Then open `.env.local` and follow the instructions inside. You do not need this for most content changes: only if you are working on routing, assets, or the Next.js config itself. -## Format, Lint Content, Lint Code and Spell Check +## How the site works -Please make sure to format and lint your code before committing: +Every page on the site is an `.mdx` file under `content/`. MDX is Markdown that can also use React components. You write normal Markdown and drop in components (cards, callouts, custom JSX) where you need them. -```bash -yarn content:check -yarn spell:check -yarn format:fix -yarn lint:fix -``` +### Content structure -## How the site works +content/ +├── index.mdx # Homepage / landing page +├── getting-started.mdx # Quickstart guide +├── meta.json # Root nav configuration +├── docs/ # All documentation pages +│ ├── meta.json +│ ├── core-concepts.mdx +│ ├── developer-guide.mdx +│ └── ... +└── contribute/ # Contributing section +├── meta.json +└── ... + +### How the left sidebar nav works + +The left sidebar is driven entirely by `meta.json` files — Fumadocs reads them at build time and constructs the page tree from them. + +**Root nav — `content/meta.json`** + + +**Section nav — `content/docs/meta.json`** + +A `meta.json` inside a folder controls that section's title, page order, and which pages appear. + +**To add a page to the nav:** +1. Create the `.mdx` file in the right folder +2. Add the filename (without `.mdx`) to the relevant `meta.json` pages array -Every page on the site is an `.mdx` file under `content/`. MDX is Markdown that -can also use React components. You write normal Markdown, and you can drop in -components (cards, callouts, custom JSX) where you need them. +**To remove a page from the nav:** +Remove it from `meta.json`. -The homepage is `content/index.mdx`. It is a content page like any other — there -is no separate "landing page" route. See "The homepage is a docs page" below for -the one way it is treated specially. +**To reorder pages:** +Change the order in the `meta.json` pages array. Top to bottom = top to bottom in the sidebar. -### The left sidebar nav comes from `meta.json` -The left-hand navigation is the **page tree**, built by the Fumadocs source -loader from `meta.json` files inside `content/`. +**Important:** A folder with no `meta.json` is completely invisible to the nav. The pages exist and are routable URLs but won't appear in the sidebar. If a page you created isn't showing up, check whether its folder has a `meta.json` and whether that file is listed in it. -- The root `content/meta.json` defines the top-level nav and has `"root": true`. -- A `meta.json` inside a folder controls that folder's title, order, and which - pages appear. -- A folder with **no** `meta.json` is invisible to the nav — the pages exist but - are not listed. (Handy to know if a page you created isn't showing up: check - whether its folder is declared in a `meta.json`.) +### Right TOC -So to change what appears in the sidebar — order, grouping, labels — you edit -`meta.json`, not the page files. +Generated automatically from `##` and `###` Markdown headings only. JSX elements and styled `
`/`` tags do not appear in the TOC regardless of how they look visually. -### The right "On this page" TOC comes from Markdown headings +### The homepage -The table of contents on the right is generated automatically from the Markdown -headings in each page — the `##` and `###` lines. Nothing else feeds it. +The homepage is `content/index.mdx` — a content page like any other, served by the `[[...slug]]` catch-all route. The `_(home)` route group in `src/app/` handles the root `/` path and renders `index.mdx` directly. -This has one important consequence when you write custom layouts: +### The `not-prose` rule -- A real Markdown heading (`## Features` on its own line) **becomes a TOC entry.** -- A heading-looking `` or `

` you styled to look like a heading does - **not** appear in the TOC; it's just text to Fumadocs. +Fumadocs applies typography styles to all MDX content by default. Any custom JSX layout block — a hero section, a card grid, a CTA — needs the `not-prose` class on its outermost element, otherwise Fumadocs' prose styles will override your custom styles: +```mdx +

+ {/* your custom layout here */} +
+``` + +### Markdown links inside JSX + +Markdown link syntax (`[text](url)`) does not render inside JSX elements. Use anchor tags instead: + +```mdx +{/* This won't work inside a JSX div */} +[CNCF](https://cncf.io) + +{/* Use this instead */} +CNCF +``` + + +## Format, Lint Content, Lint Code and Spell Check +Please make sure to format and lint your code before committing: + +```bash +yarn content:check +yarn spell:check +yarn format:fix +yarn lint:fix +``` ## Learn More From 41dca1d9ec9af29333f69b4f4eafafe853f7c12a Mon Sep 17 00:00:00 2001 From: sarhiri Date: Mon, 13 Jul 2026 14:20:07 -0500 Subject: [PATCH 14/49] docs: restructure developer guide into local development setup - Rename developer-guide to Local Development Setup - Remove deprecated sections per Dejan review: VS Code debugging, Delve, extending Armada, UI development, mage localdev - Keep and expand Goreman setup as the recommended approach - Add mage dev:up, dev:full, dev:down commands - Add Steps components for Goreman and auth setup flows - Add fake executor section for Kubernetes-free testing - Add debug port mappings table - Add troubleshooting section for port 6443 and Arm/M1 Mac issues - Remove operator-guide.mdx deprecated per Dejan - Remove orphaned redirect shims pointing to operator-guide - Add CTA block matching site-wide style Signed-off-by: sarhiri --- website/content/docs/developer-guide.mdx | 323 +++++++++++++++++- website/content/index.mdx | 2 +- website/content/{docs => }/operator-guide.mdx | 0 3 files changed, 322 insertions(+), 3 deletions(-) rename website/content/{docs => }/operator-guide.mdx (100%) diff --git a/website/content/docs/developer-guide.mdx b/website/content/docs/developer-guide.mdx index 22a2473bda0..4be0335c535 100644 --- a/website/content/docs/developer-guide.mdx +++ b/website/content/docs/developer-guide.mdx @@ -1,8 +1,327 @@ --- -title: 'Developer Guide' +title: 'Local Development Setup' description: 'Set up your development environment and start contributing to Armada' --- +import { Callout } from 'fumadocs-ui/components/callout'; +import { Step, Steps } from 'fumadocs-ui/components/steps'; +This guide walks you through setting up a local Armada development environment using Goreman our recommended approach for contributing to Armada. + +## Prerequisites + +Before you begin, make sure you have the following installed: + +- `Go` — [go.dev](https://go.dev/doc/install) +- `gcc` — C compiler required by some Go packages +- `mage` — [magefile.org](https://magefile.org/) +- `Docker` — [docs.docker.com](https://docs.docker.com/get-docker/) +- `kubectl` — [kubernetes.io](https://kubernetes.io/docs/tasks/tools/) +- `protobuf` — Protocol buffer compiler +- `kind` — [kind.sigs.k8s.io](https://kind.sigs.k8s.io/) +- `yarn` — [yarnpkg.com](https://yarnpkg.com/getting-started/install) — required for Lookout UI development + + + Additional tools are automatically installed via `mage BootstrapTools` from + `tools.yaml`, including golangci-lint, sqlc, go-swagger, and others. + + +## Using Goreman + +[Goreman](https://github.com/mattn/goreman) is a Go-based clone of +[Foreman](https://github.com/ddollar/foreman) that manages Procfile-based +applications, allowing you to run multiple processes with a single command. +Components are built from source and run on the host, so iteration is fast +and debuggers attach directly. + + + + + +**Clone the repository** + +```bash +git clone https://github.com/armadaproject/armada.git +cd armada +``` + + + + + +**Create a local Kind cluster** + +```bash +mage kind +``` + +This is a one-time setup step. + + + + + +**Start all Armada services** + +```bash +mage dev:up +``` + +This starts Redis, PostgreSQL, and Pulsar in containers, then runs the Armada +server, scheduler, executor, Lookout, and all ingesters as local processes via +Goreman. + + + + + +**Verify everything is running** + +```bash +goreman run status +``` + +Running processes are prefixed with `*`: +*server
+*scheduler
+*scheduleringester
+*eventingester
+*executor
+*lookout
+*lookoutingester
+*binoculars
+*lookoutui + + + + + + + Restart individual processes without stopping everything: + +```bash + goreman restart server +``` + + +**Useful mage commands** + +```bash +mage dev:up # start dependencies + Armada components via Goreman +mage dev:full # run the entire stack in containers against Kind (what CI uses) +mage dev:down # stop dependency containers (after mage dev:up) +mage dev:fullDown # stop containerised stack and tear down Kind (after mage dev:full) +mage -l # list all available mage commands +``` + + Use `mage dev:full` to replicate what CI runs. Use `mage dev:up` for day-to-day + development. It's faster since components run as host processes. + + +## Running with authentication + + + + + +**Start dependencies with the auth profile** + +```bash +docker compose -f _local/compose/stack.yaml --profile auth up -d +``` + +This starts Redis, PostgreSQL, Pulsar, and Keycloak with a pre-configured realm. + + + + + +**Initialise databases and Kubernetes resources** + +```bash +_local/scripts/init.sh +``` + + + + + +**Start Armada components with auth configuration** + +```bash +goreman -f _local/procfiles/auth.Procfile start +``` + + + The first run compiles all Armada components from source which can take + several minutes. Subsequent runs are faster as Go caches build artifacts. + + + + + + +**Use armadactl with OIDC authentication** + +```bash +armadactl --config _local/.armadactl.yaml --context auth-oidc get queues +``` + + + + + + + Default Keycloak credentials — Admin: `admin` / `admin` · User: `user` / `password` + + +## Running without a Kubernetes cluster + +For testing Armada without a real Kubernetes cluster, use the fake executor +which simulates a Kubernetes environment: + +```bash +goreman -f _local/procfiles/fake-executor.Procfile start +``` + +The fake executor simulates: + +- 2 virtual nodes with 8 CPUs and 32Gi memory each +- Pod lifecycle management without actual container execution +- Resource allocation and job state transitions + +Useful for testing scheduling logic, development when Kubernetes is unavailable, +and integration testing of job flows. + +## Code structure +Armada
+├── cmd/ # Entry points for all components
+│ ├── server/ # Armada server (API server)
+│ ├── executor/ # Executor (runs in each K8s cluster)
+│ ├── scheduler/ # Scheduler (job scheduling logic)
+│ ├── lookout/ # Lookout (job monitoring/UI backend)
+│ └── armadactl/ # Command-line interface
+├── internal/ # Internal packages (not for external use)
+│ ├── server/ # Server implementation
+│ ├── executor/ # Executor implementation
+│ ├── scheduler/ # Scheduler implementation
+│ ├── lookout/ # Lookout implementation
+│ └── common/ # Shared utilities
+├── pkg/ # Public packages (for external use)
+│ ├── api/ # gRPC API definitions
+│ └── client/ # Client libraries
+├── config/ # Configuration files for components
+├── magefiles/ # Build automation (mage targets)
+└── testsuite/ # Integration test cases
+ +## Testing your setup + +Run the full test suite: + +```bash +mage testsuite +``` + +Or manually: + +```bash +go run cmd/armadactl/main.go create queue e2e-test-queue +export ARMADA_EXECUTOR_INGRESS_URL="http://localhost" +export ARMADA_EXECUTOR_INGRESS_PORT=5001 +go run cmd/testsuite/main.go test --tests "testsuite/testcases/basic/*" --junit junit.xml +``` + +## Profiling with pprof + +Enable profiling in your component config: + +```yaml +profiling: + port: 6060 +``` + +Then connect: + +```bash +go tool pprof http://localhost:6060/debug/pprof/profile +``` + +## Debug port mappings + +| Component | Debug host | +| ----------------- | ---------------- | +| `server` | `localhost:4000` | +| `executor` | `localhost:4001` | +| `binoculars` | `localhost:4002` | +| `eventingester` | `localhost:4003` | +| `lookoutui` | `localhost:4004` | +| `lookout` | `localhost:4005` | +| `lookoutingester` | `localhost:4007` | + +## Troubleshooting + +**Port 6443 already in use** + +Modify `_local/kind/cluster.yaml` to use a different port: + +```yaml +- containerPort: 6443 + hostPort: 6444 + protocol: TCP +``` + +**Arm/M1 Mac issues** + +```bash +export PULSAR_IMAGE=richgross/pulsar:2.11.0 +``` + +See [Arm issue #2493](https://github.com/armadaproject/armada/issues/2493) and +[Windows issue #2492](https://github.com/armadaproject/armada/issues/2492) for +more details. + +**Need help?** + +Ask in the [#armada channel on CNCF Slack](https://cloud-native.slack.com/archives/C03T9CBCEMC). + + +{/* ── CTA ── */} +
+ +
+ +
+ Need help? +
+ +
+ Ask in the Armada Slack channel or find us on Github! +
+ + + +
+{/* This guide helps you set up a development environment for contributing to Armada or customizing it with new features. For contribution guidelines, see the [Contributor Guide](./contribute/contributor-guide.mdx). ## Prerequisites @@ -355,4 +674,4 @@ For more information on known issues: - [API Reference](./user-guide/api.mdx) - API reference for REST and gRPC - [Contributor Guide](./contribute/contributor-guide.mdx) - Contribution guidelines and PR process - [Understanding Armada](./understanding-armada) - Core concepts and architecture -- [Community](./community.mdx) - Get help, connect with the community, and find support resources +- [Community](./community.mdx) - Get help, connect with the community, and find support resources */} diff --git a/website/content/index.mdx b/website/content/index.mdx index 95e27818f49..62d33121db5 100644 --- a/website/content/index.mdx +++ b/website/content/index.mdx @@ -424,7 +424,7 @@ G-Research, a leading quantitative research company, uses Armada in production t Join #armada on Slack diff --git a/website/content/docs/operator-guide.mdx b/website/content/operator-guide.mdx similarity index 100% rename from website/content/docs/operator-guide.mdx rename to website/content/operator-guide.mdx From ac3de01801a0f96f1118ba2ab37a9ea4b2d32071 Mon Sep 17 00:00:00 2001 From: Maurice Yap Date: Tue, 30 Jun 2026 09:57:43 +0100 Subject: [PATCH 15/49] Fix unhandled clipboard rejection in CopyIconButton (#4988) `navigator.clipboard.writeText()` in `CopyIconButton` was called without awaiting or catching its returned promise. When the document isn't focused during the click, the API rejects with `NotAllowedError: Document is not focused`. This PR fixes it by wrapping the call in a try-catch and surfacing the error via a snackbar. --------- Signed-off-by: Maurice Yap Signed-off-by: sarhiri --- .../lookoutui/src/components/CopyIconButton.tsx | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/internal/lookoutui/src/components/CopyIconButton.tsx b/internal/lookoutui/src/components/CopyIconButton.tsx index de3f1fe2a5d..0e75a866651 100644 --- a/internal/lookoutui/src/components/CopyIconButton.tsx +++ b/internal/lookoutui/src/components/CopyIconButton.tsx @@ -3,6 +3,8 @@ import { useState } from "react" import { ContentCopy } from "@mui/icons-material" import { IconButton, IconButtonProps, styled, SvgIcon, Tooltip } from "@mui/material" +import { useCustomSnackbar } from "./hooks/useCustomSnackbar" + const LEAVE_DELAY_MS = 1_000 const StyledIconButton = styled(IconButton)(({ hidden }) => ({ @@ -27,6 +29,7 @@ export const CopyIconButton = ({ copiedTooltipTitle = "Copied!", }: CopyIconButtonProps) => { const [tooltipOpen, setTooltipOpen] = useState(false) + const openSnackbar = useCustomSnackbar() return ( { + onClick={async (e) => { onClick?.(e) - navigator.clipboard.writeText(content) - setTooltipOpen(true) + try { + await navigator.clipboard.writeText(content) + setTooltipOpen(true) + } catch (error) { + openSnackbar( + `Failed to copy to clipboard: ${error instanceof Error ? error.message : String(error)}`, + "error", + ) + } }} aria-label="copy" hidden={hidden && !tooltipOpen} From 25e67f296f23fc29c26742281c9f03813af2df89 Mon Sep 17 00:00:00 2001 From: Maurice Yap Date: Tue, 30 Jun 2026 10:56:04 +0100 Subject: [PATCH 16/49] Fix unhandled rejection on the Job Sets page; migrate JobSetsContainer to a functional component (#4987) Our monitoring picked up a runtime error on the Job Sets page: > UnhandledRejection: Non-Error promise rejection captured with value: TypeError: Failed to fetch `JobSetsContainer` was a class component that fetched data via an async `loadJobSets()` method. That method had no error handling and was wired into `setInterval` (auto-refresh), a refresh button, and a result callback. None of these handle a rejected promise. When a fetch failed at the network level, the rejection went unhandled. This PR migrates `JobSetsContainer` from a class component to a functional component using hooks and TanStack Query. TanStack Query now owns the fetch promise lifecycle, so an app-managed promise can no longer reject unhandled, and loading/error state is reactive. User-facing behaviour (queue selection, ordering, active-only filter, auto-refresh, cancel/reprioritize dialogs) is unchanged. --------- Signed-off-by: Maurice Yap Signed-off-by: sarhiri --- internal/lookoutui/eslint.config.mjs | 5 + internal/lookoutui/src/common/utils.tsx | 31 +- .../components/CancelJobSetsDialog.tsx | 41 +- .../components/JobSetsContainer.test.tsx | 131 +++++ .../jobSets/components/JobSetsContainer.tsx | 549 ++++++------------ .../components/ReprioritizeJobSetsDialog.tsx | 37 +- .../jobs/components/JobsTableContainer.tsx | 9 +- .../services/JobSetsLocalStorageService.ts | 13 +- .../src/services/JobSetsQueryParamsService.ts | 7 +- .../src/services/lookout/mocks/mockServer.ts | 6 +- .../src/services/lookout/useGetJobSets.ts | 58 ++ 11 files changed, 468 insertions(+), 419 deletions(-) create mode 100644 internal/lookoutui/src/pages/jobSets/components/JobSetsContainer.test.tsx create mode 100644 internal/lookoutui/src/services/lookout/useGetJobSets.ts diff --git a/internal/lookoutui/eslint.config.mjs b/internal/lookoutui/eslint.config.mjs index 909619fb20c..5e906abe012 100644 --- a/internal/lookoutui/eslint.config.mjs +++ b/internal/lookoutui/eslint.config.mjs @@ -76,6 +76,11 @@ export default tseslint.config( "aggregatable", "ingester", + // TanStack Query terminology + "refetch", + "refetches", + "refetching", + // Use the American spelling of these words for consistency with the Armada API "reprioritize", "reprioritized", diff --git a/internal/lookoutui/src/common/utils.tsx b/internal/lookoutui/src/common/utils.tsx index 94245bea1c3..1a29fa0ab4a 100644 --- a/internal/lookoutui/src/common/utils.tsx +++ b/internal/lookoutui/src/common/utils.tsx @@ -1,4 +1,4 @@ -import { Component, FC } from "react" +import { Component, FC, useRef } from "react" import { Location, NavigateFunction, Params, useLocation, useNavigate, useParams } from "react-router-dom" @@ -119,4 +119,33 @@ export function withRouter(Component: FC): FC> } +// Returns a referentially-stable Router whose location, navigate and params always +// reflect the current render. Use this when passing a router to a service that is +// instantiated once (e.g. via useMemo with an empty dependency array), so the service +// reads the live location rather than a snapshot captured at mount. +export function useStableRouter(): Router { + const location = useLocation() + const navigate = useNavigate() + const params = useParams() + + const ref = useRef({ location, navigate, params }) + ref.current.location = location + ref.current.navigate = navigate + ref.current.params = params + + const stableRouter = useRef({ + get location() { + return ref.current.location + }, + get navigate() { + return ref.current.navigate + }, + get params() { + return ref.current.params + }, + }) + + return stableRouter.current +} + export const PlatformCancelReason = "Platform error marked by user" diff --git a/internal/lookoutui/src/pages/jobSets/components/CancelJobSetsDialog.tsx b/internal/lookoutui/src/pages/jobSets/components/CancelJobSetsDialog.tsx index 40ffe6913b7..31854646938 100644 --- a/internal/lookoutui/src/pages/jobSets/components/CancelJobSetsDialog.tsx +++ b/internal/lookoutui/src/pages/jobSets/components/CancelJobSetsDialog.tsx @@ -3,8 +3,9 @@ import { useState } from "react" import { Dialog, DialogContent, DialogTitle } from "@mui/material" import { ErrorBoundary } from "react-error-boundary" -import { ApiResult, PlatformCancelReason, RequestStatus } from "../../../common/utils" +import { ApiResult, getErrorMessage, PlatformCancelReason, RequestStatus } from "../../../common/utils" import { AlertErrorFallback } from "../../../components/AlertErrorFallback" +import { useCustomSnackbar } from "../../../components/hooks/useCustomSnackbar" import { JobSet } from "../../../models/lookoutModels" import { ApiJobState } from "../../../openapi/armada" import { CancelJobSetsResponse, useCancelJobSets } from "../../../services/lookout/useCancelJobSets" @@ -54,6 +55,7 @@ export default function CancelJobSetsDialog(props: CancelJobSetsDialogProps) { const statesToCancel = getStatesToCancel(includeQueued, includeRunning) const cancelJobSetsMutation = useCancelJobSets() + const openSnackbar = useCustomSnackbar() async function cancelJobSets() { if (requestStatus === "Loading") { @@ -62,22 +64,27 @@ export default function CancelJobSetsDialog(props: CancelJobSetsDialogProps) { setRequestStatus("Loading") const reason = isPlatformCancel ? PlatformCancelReason : "" - const cancelJobSetsResponse = await cancelJobSetsMutation.mutateAsync({ - queue: props.queue, - jobSets: jobSetsToCancel, - states: statesToCancel, - reason, - }) - setRequestStatus("Idle") - - setResponse(cancelJobSetsResponse) - setState("CancelJobSetsResult") - if (cancelJobSetsResponse.failedJobSetCancellations.length === 0) { - props.onResult("Success") - } else if (cancelJobSetsResponse.cancelledJobSets.length === 0) { - props.onResult("Failure") - } else { - props.onResult("Partial success") + try { + const cancelJobSetsResponse = await cancelJobSetsMutation.mutateAsync({ + queue: props.queue, + jobSets: jobSetsToCancel, + states: statesToCancel, + reason, + }) + + setResponse(cancelJobSetsResponse) + setState("CancelJobSetsResult") + if (cancelJobSetsResponse.failedJobSetCancellations.length === 0) { + props.onResult("Success") + } else if (cancelJobSetsResponse.cancelledJobSets.length === 0) { + props.onResult("Failure") + } else { + props.onResult("Partial success") + } + } catch (e) { + openSnackbar(`Failed to cancel job sets: ${await getErrorMessage(e)}`, "error") + } finally { + setRequestStatus("Idle") } } diff --git a/internal/lookoutui/src/pages/jobSets/components/JobSetsContainer.test.tsx b/internal/lookoutui/src/pages/jobSets/components/JobSetsContainer.test.tsx new file mode 100644 index 00000000000..b246c4f8ad9 --- /dev/null +++ b/internal/lookoutui/src/pages/jobSets/components/JobSetsContainer.test.tsx @@ -0,0 +1,131 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query" +import { render } from "@testing-library/react" +import userEvent from "@testing-library/user-event" +import { http, HttpResponse } from "msw" +import { SnackbarProvider } from "notistack" +import { createMemoryRouter, RouterProvider } from "react-router-dom" +import { v4 as uuidV4 } from "uuid" +import { vi } from "vitest" + +import { Job, JobState } from "../../../models/lookoutModels" +import { JOB_SETS } from "../../../pathnames" +import { ApiClientsProvider } from "../../../services/apiClients" +import { MockServer } from "../../../services/lookout/mocks/mockServer" + +import JobSetsContainer from "./JobSetsContainer" + +const mockServer = new MockServer() + +function makeTestJobs(n: number, queue: string, jobSet: string, state: JobState): Job[] { + const jobs: Job[] = [] + for (let i = 0; i < n; i++) { + jobs.push({ + annotations: {}, + cpu: 1, + ephemeralStorage: 8192, + gpu: 0, + jobId: uuidV4(), + jobSet, + lastTransitionTime: "2024-01-01T00:00:00Z", + memory: 8192, + owner: queue, + namespace: queue, + priority: 1000, + priorityClass: "armada-default", + queue, + runs: [], + state, + submitted: "2024-01-01T00:00:00Z", + }) + } + return jobs +} + +describe("JobSetsContainer", () => { + beforeAll(() => { + mockServer.listen() + }) + + beforeEach(() => { + localStorage.clear() + }) + + afterEach(() => { + localStorage.clear() + mockServer.reset() + }) + + afterAll(() => { + mockServer.close() + }) + + const renderComponent = (search: string) => { + const testQueryClient = new QueryClient({ + defaultOptions: { + queries: { retry: false, gcTime: 0 }, + }, + }) + + const element = ( + + + + + + + + ) + + const router = createMemoryRouter( + [ + { + path: JOB_SETS, + element, + }, + ], + { initialEntries: [JOB_SETS + search], initialIndex: 0 }, + ) + + return render() + } + + it("renders the job sets table for the queue in the URL", async () => { + mockServer.setPostJobsResponse([ + ...makeTestJobs(2, "queue-1", "job-set-a", JobState.Running), + ...makeTestJobs(3, "queue-1", "job-set-b", JobState.Queued), + ]) + + const { findByLabelText, queryByText } = renderComponent("?queue=queue-1") + + // The "select all" header checkbox only renders when at least one job set has loaded. + expect(await findByLabelText("select all job sets")).toBeInTheDocument() + expect(queryByText("No job sets found for this queue.")).not.toBeInTheDocument() + }) + + it("refetches when the active-only filter changes", async () => { + // Only finished jobs: visible by default, but excluded once "Active only" is on. + mockServer.setPostJobsResponse(makeTestJobs(2, "queue-1", "finished-set", JobState.Succeeded)) + + const { findByLabelText, findByText, getByLabelText } = renderComponent("?queue=queue-1") + + expect(await findByLabelText("select all job sets")).toBeInTheDocument() + + await userEvent.click(getByLabelText("Active only")) + + expect(await findByText("No job sets found for this queue.")).toBeInTheDocument() + }) + + it("surfaces a fetch failure as a snackbar without an unhandled rejection", async () => { + const unhandledRejection = vi.fn() + window.addEventListener("unhandledrejection", unhandledRejection) + + mockServer.use(http.post("/api/v1/jobGroups", () => HttpResponse.error())) + + const { findByText } = renderComponent("?queue=queue-1") + + expect(await findByText(/Failed to load job sets for queue queue-1/)).toBeInTheDocument() + expect(unhandledRejection).not.toHaveBeenCalled() + + window.removeEventListener("unhandledrejection", unhandledRejection) + }) +}) diff --git a/internal/lookoutui/src/pages/jobSets/components/JobSetsContainer.tsx b/internal/lookoutui/src/pages/jobSets/components/JobSetsContainer.tsx index e4c80c44d05..09ddefe465b 100644 --- a/internal/lookoutui/src/pages/jobSets/components/JobSetsContainer.tsx +++ b/internal/lookoutui/src/pages/jobSets/components/JobSetsContainer.tsx @@ -1,21 +1,14 @@ -import { Component, FC } from "react" +import { useCallback, useEffect, useMemo, useState } from "react" import { ErrorBoundary } from "react-error-boundary" import { StandardColumnId } from "../../../common/jobsTableColumns" -import { - ApiResult, - debounced, - PropsWithRouter, - RequestStatus, - selectItem, - setStateAsync, - withRouter, -} from "../../../common/utils" +import { ApiResult, RequestStatus, selectItem, useStableRouter } from "../../../common/utils" import { AlertErrorFallback } from "../../../components/AlertErrorFallback" -import { GetJobSetsRequest, JobSet, JobSetsOrderByColumn, JobState, Match } from "../../../models/lookoutModels" +import { useCustomSnackbar } from "../../../components/hooks/useCustomSnackbar" +import { JobSet, JobSetsOrderByColumn } from "../../../models/lookoutModels" import { JOBS } from "../../../pathnames" -import JobSetsLocalStorageService from "../../../services/JobSetsLocalStorageService" +import JobSetsLocalStorageService, { JobSetsPrefs } from "../../../services/JobSetsLocalStorageService" import JobSetsQueryParamsService from "../../../services/JobSetsQueryParamsService" import { DEFAULT_PREFERENCES, @@ -23,382 +16,192 @@ import { stringifyQueryParams, toQueryStringSafe, } from "../../../services/lookout/JobsTablePreferencesService" -import { useGroupJobs } from "../../../services/lookout/useGroupJobs" +import { useGetJobSets } from "../../../services/lookout/useGetJobSets" import CancelJobSetsDialog, { getCancellableJobSets } from "./CancelJobSetsDialog" import JobSets from "./JobSets" import ReprioritizeJobSetsDialog, { getReprioritizableJobSets } from "./ReprioritizeJobSetsDialog" -interface JobSetsContainerProps extends PropsWithRouter { - groupJobs: ReturnType +export interface JobSetsContainerProps { jobSetsAutoRefreshMs: number | undefined } -type JobSetsContainerParams = { - queue: string +const DEFAULT_PREFS: JobSetsPrefs = { + queue: "", + autoRefresh: true, + orderByColumn: "submitted", + orderByDesc: true, + activeOnly: false, } -export type JobSetsContainerState = { - jobSets: JobSet[] - selectedJobSets: Map - getJobSetsRequestStatus: RequestStatus - autoRefresh: boolean - lastSelectedIndex: number - orderByColumn: JobSetsOrderByColumn - orderByDesc: boolean - activeOnly: boolean - cancelJobSetsIsOpen: boolean - reprioritizeJobSetsIsOpen: boolean -} & JobSetsContainerParams - -class JobSetsContainer extends Component { - autoRefreshInterval: NodeJS.Timeout | undefined - autoRefreshMs: number | undefined - localStorageService: JobSetsLocalStorageService - queryParamsService: JobSetsQueryParamsService - - constructor(props: JobSetsContainerProps) { - super(props) - - this.autoRefreshMs = props.jobSetsAutoRefreshMs - this.localStorageService = new JobSetsLocalStorageService() - this.queryParamsService = new JobSetsQueryParamsService(this.props.router) - - this.state = { - queue: "", - jobSets: [], - selectedJobSets: new Map(), - getJobSetsRequestStatus: "Idle", - autoRefresh: true, - lastSelectedIndex: 0, - cancelJobSetsIsOpen: false, - reprioritizeJobSetsIsOpen: false, - orderByColumn: "submitted", - orderByDesc: true, - activeOnly: false, - } - - this.setQueue = this.setQueue.bind(this) - this.orderChange = this.orderChange.bind(this) - this.activeOnlyChange = this.activeOnlyChange.bind(this) - this.selectJobSet = this.selectJobSet.bind(this) - this.shiftSelectJobSet = this.shiftSelectJobSet.bind(this) - this.deselectAll = this.deselectAll.bind(this) - this.selectAll = this.selectAll.bind(this) - - this.openCancelJobSets = this.openCancelJobSets.bind(this) - this.openReprioritizeJobSets = this.openReprioritizeJobSets.bind(this) - this.handleApiResult = this.handleApiResult.bind(this) - - this.fetchJobSets = debounced(this.fetchJobSets.bind(this), 100) - this.loadJobSets = this.loadJobSets.bind(this) - this.toggleAutoRefresh = this.toggleAutoRefresh.bind(this) - this.onJobSetStateClick = this.onJobSetStateClick.bind(this) - } - - async componentDidMount() { - const newState = { ...this.state } - - this.localStorageService.updateState(newState) - this.queryParamsService.updateState(newState) - - this.localStorageService.saveState(newState) - // queryParamsService.saveState calls navigate, which should only be called in useEffect - // actual fix is migrating this component to a functional one with hooks - setTimeout(() => this.queryParamsService.saveState(newState)) - - await setStateAsync(this, { - ...newState, - }) - - await this.loadJobSets() - - this.tryStartAutoRefresh() - } - - componentWillUnmount() { - this.stopAutoRefresh() - } - - async setQueue(queue: string) { - await this.updateState({ - ...this.state, - queue: queue, - }) - - // Performed separately because debounced - await this.loadJobSets() - } - - async orderChange(orderByColumn: JobSetsOrderByColumn, orderByDesc: boolean) { - await this.updateState({ - ...this.state, - orderByColumn, - orderByDesc, - }) - await this.loadJobSets() - } - - async activeOnlyChange(activeOnly: boolean) { - await this.updateState({ - ...this.state, - activeOnly: activeOnly, - }) - - await this.loadJobSets() - } - - selectJobSet(index: number, selected: boolean) { - if (index < 0 || index >= this.state.jobSets.length) { - return +export default function JobSetsContainer({ jobSetsAutoRefreshMs }: JobSetsContainerProps) { + const router = useStableRouter() + const openSnackbar = useCustomSnackbar() + + const localStorageService = useMemo(() => new JobSetsLocalStorageService(), []) + const queryParamsService = useMemo(() => new JobSetsQueryParamsService(router), [router]) + + const [prefs] = useState(() => { + const initial = { ...DEFAULT_PREFS } + localStorageService.updateState(initial) + queryParamsService.updateState(initial) + return initial + }) + + const [queue, setQueueState] = useState(prefs.queue) + const [orderByColumn, setOrderByColumn] = useState(prefs.orderByColumn) + const [orderByDesc, setOrderByDesc] = useState(prefs.orderByDesc) + const [activeOnly, setActiveOnly] = useState(prefs.activeOnly) + const [autoRefresh, setAutoRefresh] = useState(prefs.autoRefresh) + + const [selectedJobSets, setSelectedJobSets] = useState>(new Map()) + const [lastSelectedIndex, setLastSelectedIndex] = useState(0) + const [cancelJobSetsIsOpen, setCancelJobSetsIsOpen] = useState(false) + const [reprioritizeJobSetsIsOpen, setReprioritizeJobSetsIsOpen] = useState(false) + + const { data, error, errorUpdatedAt, isFetching, refetch } = useGetJobSets({ + queue, + activeOnly, + orderByColumn, + orderByDesc, + autoRefresh, + autoRefreshMs: jobSetsAutoRefreshMs, + }) + const jobSets = data ?? [] + + useEffect(() => { + const currentPrefs: JobSetsPrefs = { queue, autoRefresh, orderByColumn, orderByDesc, activeOnly } + localStorageService.saveState(currentPrefs) + queryParamsService.saveState(currentPrefs) + }, [queue, autoRefresh, orderByColumn, orderByDesc, activeOnly, localStorageService, queryParamsService]) + + useEffect(() => { + if (error !== null) { + openSnackbar(`Failed to load job sets for queue ${queue}: ${error}`, "error") } - const jobSet = this.state.jobSets[index] - - const selectedJobSets = new Map(this.state.selectedJobSets) - selectItem(jobSet.jobSetId, jobSet, selectedJobSets, selected) - - this.setState({ - ...this.state, - selectedJobSets: selectedJobSets, - lastSelectedIndex: index, - }) - } - - shiftSelectJobSet(index: number, selected: boolean) { - if (index >= this.state.jobSets.length || index < 0) { - return - } - - const [start, end] = [this.state.lastSelectedIndex, index].sort((a, b) => a - b) - - const selectedJobSets = new Map(this.state.selectedJobSets) - for (let i = start; i <= end; i++) { - const jobSet = this.state.jobSets[i] - selectItem(jobSet.jobSetId, jobSet, selectedJobSets, selected) - } - - this.setState({ - ...this.state, - selectedJobSets: selectedJobSets, - lastSelectedIndex: index, - }) - } + // Keyed on errorUpdatedAt (not error) so repeated identical failures, such as + // retrying refresh while offline, each surface a fresh snackbar. + }, [errorUpdatedAt]) - deselectAll() { - this.setState({ - ...this.state, - selectedJobSets: new Map(), - lastSelectedIndex: 0, - }) - } + const deselectAll = useCallback(() => { + setSelectedJobSets(new Map()) + setLastSelectedIndex(0) + }, []) - selectAll() { + const selectAll = useCallback(() => { const selected = new Map() - this.state.jobSets.forEach((jobSet) => selected.set(jobSet.jobSetId, jobSet)) - - this.setState({ - ...this.state, - selectedJobSets: selected, - lastSelectedIndex: 0, - }) - } - - openCancelJobSets(isOpen: boolean) { - this.setState({ - ...this.state, - cancelJobSetsIsOpen: isOpen, - }) - } - - openReprioritizeJobSets(isOpen: boolean) { - this.setState({ - ...this.state, - reprioritizeJobSetsIsOpen: isOpen, - }) - } - - handleApiResult(result: ApiResult) { - if (result === "Success") { - this.deselectAll() - return this.loadJobSets() - } else if (result === "Partial success") { - return this.loadJobSets() - } - } - - async toggleAutoRefresh(autoRefresh: boolean) { - await this.updateState({ - ...this.state, - autoRefresh: autoRefresh, - }) - this.tryStartAutoRefresh() - } - - tryStartAutoRefresh() { - this.stopAutoRefresh() - if (this.state.autoRefresh && this.autoRefreshMs !== undefined) { - this.autoRefreshInterval = setInterval(this.loadJobSets, this.autoRefreshMs) - } - } - - stopAutoRefresh() { - if (this.autoRefreshInterval) { - clearInterval(this.autoRefreshInterval) - this.autoRefreshInterval = undefined - } - } - - private async onJobSetStateClick(rowIndex: number, state: string) { - const jobSet = this.state.jobSets[rowIndex] - - const prefs: JobsTablePreferences = { - ...DEFAULT_PREFERENCES, - filters: [ - { - id: StandardColumnId.Queue, - value: jobSet.queue, - }, - { - id: StandardColumnId.State, - value: [state], - }, - { - id: StandardColumnId.JobSet, - value: jobSet.jobSetId, - }, - ], - } - - this.props.router.navigate({ - pathname: JOBS, - search: stringifyQueryParams(toQueryStringSafe(prefs)), - }) - } - - private async updateState(updatedState: JobSetsContainerState) { - this.localStorageService.saveState(updatedState) - this.queryParamsService.saveState(updatedState) - await setStateAsync(this, updatedState) - } - - private async loadJobSets() { - if (this.state.queue === "") { - return - } - await setStateAsync(this, { - ...this.state, - getJobSetsRequestStatus: "Loading", - }) - const jobSets = await this.fetchJobSets({ - queue: this.state.queue, - orderByColumn: this.state.orderByColumn, - orderByDesc: this.state.orderByDesc, - activeOnly: this.state.activeOnly, - }) - this.setState({ - ...this.state, - jobSets: jobSets, - getJobSetsRequestStatus: "Idle", - }) - } - - private async fetchJobSets(getJobSetsRequest: GetJobSetsRequest): Promise { - const response = await this.props.groupJobs( - [ - { - isAnnotation: false, - field: "queue", - value: getJobSetsRequest.queue, - match: Match.Exact, - }, - ], - getJobSetsRequest.activeOnly, - { - field: getJobSetsRequest.orderByColumn, - direction: getJobSetsRequest.orderByDesc ? "DESC" : "ASC", - }, - { - field: "jobSet", - isAnnotation: false, - }, - ["state", "submitted"], - 0, - 0, - ) - - return response.groups.map((group) => { - const state = group.aggregates.state as Record - return { - jobSetId: group.name, - queue: getJobSetsRequest.queue, - jobsQueued: state[JobState.Queued] || 0, - jobsPending: state[JobState.Pending] || 0, - jobsRunning: state[JobState.Running] || 0, - jobsSucceeded: state[JobState.Succeeded] || 0, - jobsFailed: state[JobState.Failed] || 0, - jobsCancelled: state[JobState.Cancelled] || 0, - latestSubmissionTime: group.aggregates.submitted as string, + jobSets.forEach((jobSet) => selected.set(jobSet.jobSetId, jobSet)) + setSelectedJobSets(selected) + setLastSelectedIndex(0) + }, [jobSets]) + + const selectJobSet = useCallback( + (index: number, selected: boolean) => { + if (index < 0 || index >= jobSets.length) { + return } - }) - } - - render() { - const selectedJobSets = Array.from(this.state.selectedJobSets.values()) - return ( - <> - this.openCancelJobSets(false)} - /> - { + if (index < 0 || index >= jobSets.length) { + return + } + const [start, end] = [lastSelectedIndex, index].sort((a, b) => a - b) + const newSelected = new Map(selectedJobSets) + for (let i = start; i <= end; i++) { + const jobSet = jobSets[i] + selectItem(jobSet.jobSetId, jobSet, newSelected, selected) + } + setSelectedJobSets(newSelected) + setLastSelectedIndex(index) + }, + [jobSets, lastSelectedIndex, selectedJobSets], + ) + + const handleApiResult = useCallback( + (result: ApiResult) => { + if (result === "Success") { + deselectAll() + refetch() + } else if (result === "Partial success") { + refetch() + } + }, + [deselectAll, refetch], + ) + + const onJobSetStateClick = useCallback( + (rowIndex: number, state: string) => { + const jobSet = jobSets[rowIndex] + const prefs: JobsTablePreferences = { + ...DEFAULT_PREFERENCES, + filters: [ + { id: StandardColumnId.Queue, value: jobSet.queue }, + { id: StandardColumnId.State, value: [state] }, + { id: StandardColumnId.JobSet, value: jobSet.jobSetId }, + ], + } + router.navigate({ pathname: JOBS, search: stringifyQueryParams(toQueryStringSafe(prefs)) }) + }, + [jobSets, router], + ) + + const selectedJobSetsArray = Array.from(selectedJobSets.values()) + const getJobSetsRequestStatus: RequestStatus = isFetching ? "Loading" : "Idle" + + return ( + <> + setCancelJobSetsIsOpen(false)} + /> + setReprioritizeJobSetsIsOpen(false)} + /> + + 0} + canReprioritize={getReprioritizableJobSets(selectedJobSetsArray).length > 0} + queue={queue} + jobSets={jobSets} selectedJobSets={selectedJobSets} - onResult={this.handleApiResult} - onClose={() => this.openReprioritizeJobSets(false)} + getJobSetsRequestStatus={getJobSetsRequestStatus} + autoRefresh={autoRefresh} + orderByColumn={orderByColumn} + orderByDesc={orderByDesc} + activeOnly={activeOnly} + onQueueChange={setQueueState} + onOrderChange={(column: JobSetsOrderByColumn, desc: boolean) => { + setOrderByColumn(column) + setOrderByDesc(desc) + }} + onActiveOnlyChange={setActiveOnly} + onRefresh={() => refetch()} + onSelectJobSet={selectJobSet} + onShiftSelectJobSet={shiftSelectJobSet} + onDeselectAllClick={deselectAll} + onSelectAllClick={selectAll} + onCancelJobSetsClick={() => setCancelJobSetsIsOpen(true)} + onToggleAutoRefresh={jobSetsAutoRefreshMs !== undefined ? setAutoRefresh : undefined} + onReprioritizeJobSetsClick={() => setReprioritizeJobSetsIsOpen(true)} + onJobSetStateClick={onJobSetStateClick} /> - - 0} - canReprioritize={getReprioritizableJobSets(selectedJobSets).length > 0} - queue={this.state.queue} - jobSets={this.state.jobSets} - selectedJobSets={this.state.selectedJobSets} - getJobSetsRequestStatus={this.state.getJobSetsRequestStatus} - autoRefresh={this.state.autoRefresh} - orderByColumn={this.state.orderByColumn} - orderByDesc={this.state.orderByDesc} - activeOnly={this.state.activeOnly} - onQueueChange={this.setQueue} - onOrderChange={this.orderChange} - onActiveOnlyChange={this.activeOnlyChange} - onRefresh={this.loadJobSets} - onSelectJobSet={this.selectJobSet} - onShiftSelectJobSet={this.shiftSelectJobSet} - onDeselectAllClick={this.deselectAll} - onSelectAllClick={this.selectAll} - onCancelJobSetsClick={() => this.openCancelJobSets(true)} - onToggleAutoRefresh={this.autoRefreshMs !== undefined ? this.toggleAutoRefresh : undefined} - onReprioritizeJobSetsClick={() => this.openReprioritizeJobSets(true)} - onJobSetStateClick={this.onJobSetStateClick} - /> - - - ) - } + + + ) } - -const withGroupJobs = }>( - Component: FC, -): FC> => { - function ComponentWithGroupJobs(props: T) { - const groupJobs = useGroupJobs() - return - } - return ComponentWithGroupJobs as FC> -} - -export default withGroupJobs(withRouter((props: JobSetsContainerProps) => )) diff --git a/internal/lookoutui/src/pages/jobSets/components/ReprioritizeJobSetsDialog.tsx b/internal/lookoutui/src/pages/jobSets/components/ReprioritizeJobSetsDialog.tsx index 0e634b19d32..24995fbe62a 100644 --- a/internal/lookoutui/src/pages/jobSets/components/ReprioritizeJobSetsDialog.tsx +++ b/internal/lookoutui/src/pages/jobSets/components/ReprioritizeJobSetsDialog.tsx @@ -3,8 +3,9 @@ import { useState } from "react" import { Dialog, DialogContent, DialogTitle } from "@mui/material" import { ErrorBoundary } from "react-error-boundary" -import { ApiResult, priorityIsValid, RequestStatus } from "../../../common/utils" +import { ApiResult, getErrorMessage, priorityIsValid, RequestStatus } from "../../../common/utils" import { AlertErrorFallback } from "../../../components/AlertErrorFallback" +import { useCustomSnackbar } from "../../../components/hooks/useCustomSnackbar" import { JobSet } from "../../../models/lookoutModels" import { ReprioritizeJobSetsResponse, useReprioritizeJobSets } from "../../../services/lookout/useReprioritizeJobSets" @@ -39,6 +40,7 @@ export default function ReprioritizeJobSetsDialog(props: ReprioritizeJobSetsDial const jobSetsToReprioritize = getReprioritizableJobSets(props.selectedJobSets) const reprioritizeJobSetsMutation = useReprioritizeJobSets() + const openSnackbar = useCustomSnackbar() async function reprioritizeJobSets() { if (requestStatus == "Loading" || !priorityIsValid(priority)) { @@ -46,21 +48,26 @@ export default function ReprioritizeJobSetsDialog(props: ReprioritizeJobSetsDial } setRequestStatus("Loading") - const reprioritizeJobSetsResponse = await reprioritizeJobSetsMutation.mutateAsync({ - queue: props.queue, - jobSets: jobSetsToReprioritize, - newPriority: Number(priority), - }) - setRequestStatus("Idle") + try { + const reprioritizeJobSetsResponse = await reprioritizeJobSetsMutation.mutateAsync({ + queue: props.queue, + jobSets: jobSetsToReprioritize, + newPriority: Number(priority), + }) - setResponse(reprioritizeJobSetsResponse) - setState("ReprioritizeJobSetsResult") - if (reprioritizeJobSetsResponse.failedJobSetReprioritizations.length === 0) { - props.onResult("Success") - } else if (reprioritizeJobSetsResponse.reprioritizedJobSets.length === 0) { - props.onResult("Failure") - } else { - props.onResult("Partial success") + setResponse(reprioritizeJobSetsResponse) + setState("ReprioritizeJobSetsResult") + if (reprioritizeJobSetsResponse.failedJobSetReprioritizations.length === 0) { + props.onResult("Success") + } else if (reprioritizeJobSetsResponse.reprioritizedJobSets.length === 0) { + props.onResult("Failure") + } else { + props.onResult("Partial success") + } + } catch (e) { + openSnackbar(`Failed to reprioritize job sets: ${await getErrorMessage(e)}`, "error") + } finally { + setRequestStatus("Idle") } } diff --git a/internal/lookoutui/src/pages/jobs/components/JobsTableContainer.tsx b/internal/lookoutui/src/pages/jobs/components/JobsTableContainer.tsx index 78d24c1385a..3bcfd6e48e9 100644 --- a/internal/lookoutui/src/pages/jobs/components/JobsTableContainer.tsx +++ b/internal/lookoutui/src/pages/jobs/components/JobsTableContainer.tsx @@ -34,7 +34,6 @@ import { } from "@tanstack/react-table" import _ from "lodash" import { ErrorBoundary } from "react-error-boundary" -import { useLocation, useNavigate, useParams } from "react-router-dom" import { buildViewEventData } from "../../../analytics/viewMetadata" import { @@ -62,7 +61,7 @@ import { } from "../../../common/jobsTableUtils" import { fromRowId, RowId } from "../../../common/reactTableUtils" import { EmptyInputError, ParseError } from "../../../common/resourceUtils" -import { getErrorMessage, waitMs } from "../../../common/utils" +import { getErrorMessage, useStableRouter, waitMs } from "../../../common/utils" import { AlertErrorFallback } from "../../../components/AlertErrorFallback" import { useFormatNumberWithUserSettings } from "../../../components/hooks/formatNumberWithUserSettings" import { @@ -121,10 +120,8 @@ export const JobsTableContainer = ({ debug, autoRefreshMs, commandSpecs }: JobsT const openSnackbar = useCustomSnackbar() const groupJobs = useGroupJobs() - const location = useLocation() - const navigate = useNavigate() - const params = useParams() - const jobsTablePreferencesService = useMemo(() => new JobsTablePreferencesService({ location, navigate, params }), []) + const router = useStableRouter() + const jobsTablePreferencesService = useMemo(() => new JobsTablePreferencesService(router), [router]) const customViewsService = useMemo(() => new CustomViewsService(), []) const initialPrefs = useMemo(() => jobsTablePreferencesService.getUserPrefs(), []) diff --git a/internal/lookoutui/src/services/JobSetsLocalStorageService.ts b/internal/lookoutui/src/services/JobSetsLocalStorageService.ts index 3ad80477500..5b2f40474ea 100644 --- a/internal/lookoutui/src/services/JobSetsLocalStorageService.ts +++ b/internal/lookoutui/src/services/JobSetsLocalStorageService.ts @@ -1,9 +1,16 @@ import { tryParseJson } from "../common/utils" import { isJobSetsOrderByColumn, JobSetsOrderByColumn } from "../models/lookoutModels" -import { JobSetsContainerState } from "../pages/jobSets/components/JobSetsContainer" const LOCAL_STORAGE_KEY = "armada_lookout_job_sets_user_settings" +export interface JobSetsPrefs { + queue: string + autoRefresh: boolean + orderByColumn: JobSetsOrderByColumn + orderByDesc: boolean + activeOnly: boolean +} + export type JobSetsLocalStorageState = { autoRefresh?: boolean queue?: string @@ -36,7 +43,7 @@ function convertToLocalStorageState(loadedData: Record): JobSet } export default class JobSetsLocalStorageService { - saveState(state: JobSetsContainerState) { + saveState(state: JobSetsPrefs) { const localStorageState = { autoRefresh: state.autoRefresh, queue: state.queue, @@ -47,7 +54,7 @@ export default class JobSetsLocalStorageService { localStorage.setItem(LOCAL_STORAGE_KEY, JSON.stringify(localStorageState)) } - updateState(state: JobSetsContainerState) { + updateState(state: JobSetsPrefs) { const stateJson = localStorage.getItem(LOCAL_STORAGE_KEY) if (stateJson == undefined) { return diff --git a/internal/lookoutui/src/services/JobSetsQueryParamsService.ts b/internal/lookoutui/src/services/JobSetsQueryParamsService.ts index 62ef53a9a93..19de80d6a7c 100644 --- a/internal/lookoutui/src/services/JobSetsQueryParamsService.ts +++ b/internal/lookoutui/src/services/JobSetsQueryParamsService.ts @@ -2,7 +2,8 @@ import queryString, { ParseOptions, StringifyOptions } from "query-string" import { Router } from "../common/utils" import { JobSetsOrderByColumn } from "../models/lookoutModels" -import { JobSetsContainerState } from "../pages/jobSets/components/JobSetsContainer" + +import { type JobSetsPrefs } from "./JobSetsLocalStorageService" const QUERY_STRING_OPTIONS: ParseOptions | StringifyOptions = { arrayFormat: "comma", @@ -20,7 +21,7 @@ type JobSetsQueryParams = { export default class JobSetsQueryParamsService { constructor(private router: Router) {} - saveState(state: JobSetsContainerState) { + saveState(state: JobSetsPrefs) { const params = queryString.parse(this.router.location.search, QUERY_STRING_OPTIONS) as Record if (state.queue) { @@ -36,7 +37,7 @@ export default class JobSetsQueryParamsService { }) } - updateState(state: JobSetsContainerState) { + updateState(state: JobSetsPrefs) { const params = queryString.parse(this.router.location.search, QUERY_STRING_OPTIONS) as JobSetsQueryParams if (params.queue) state.queue = params.queue diff --git a/internal/lookoutui/src/services/lookout/mocks/mockServer.ts b/internal/lookoutui/src/services/lookout/mocks/mockServer.ts index 66a31afbe99..15fb0dff7d8 100644 --- a/internal/lookoutui/src/services/lookout/mocks/mockServer.ts +++ b/internal/lookoutui/src/services/lookout/mocks/mockServer.ts @@ -1,4 +1,4 @@ -import { http, HttpResponse, PathParams } from "msw" +import { http, HttpResponse, PathParams, RequestHandler } from "msw" import { setupServer, SetupServerApi } from "msw/node" import { compareValues, getActiveJobSets, mergeFilters } from "../../../common/fakeJobsUtils" @@ -130,6 +130,10 @@ export class MockServer { return this.server.close() } + use(...handlers: RequestHandler[]) { + return this.server.use(...handlers) + } + setGetQueuesResponse(queueNames: string[]) { this.server.use( http.get(GET_QUEUES_ENDPOINT, () => diff --git a/internal/lookoutui/src/services/lookout/useGetJobSets.ts b/internal/lookoutui/src/services/lookout/useGetJobSets.ts new file mode 100644 index 00000000000..ddf98d5e6c7 --- /dev/null +++ b/internal/lookoutui/src/services/lookout/useGetJobSets.ts @@ -0,0 +1,58 @@ +import { useQuery } from "@tanstack/react-query" + +import { JobSet, JobSetsOrderByColumn, JobState, Match } from "../../models/lookoutModels" + +import { useGroupJobs } from "./useGroupJobs" + +export interface UseGetJobSetsParams { + queue: string + activeOnly: boolean + orderByColumn: JobSetsOrderByColumn + orderByDesc: boolean + autoRefresh: boolean + autoRefreshMs: number | undefined +} + +export const useGetJobSets = ({ + queue, + activeOnly, + orderByColumn, + orderByDesc, + autoRefresh, + autoRefreshMs, +}: UseGetJobSetsParams) => { + const groupJobs = useGroupJobs() + + return useQuery({ + queryKey: ["getJobSets", queue, activeOnly, orderByColumn, orderByDesc], + queryFn: async ({ signal }) => { + const response = await groupJobs( + [{ isAnnotation: false, field: "queue", value: queue, match: Match.Exact }], + activeOnly, + { field: orderByColumn, direction: orderByDesc ? "DESC" : "ASC" }, + { field: "jobSet", isAnnotation: false }, + ["state", "submitted"], + 0, + 0, + signal, + ) + + return response.groups.map((group) => { + const state = group.aggregates.state as Record + return { + jobSetId: group.name, + queue, + jobsQueued: state[JobState.Queued] || 0, + jobsPending: state[JobState.Pending] || 0, + jobsRunning: state[JobState.Running] || 0, + jobsSucceeded: state[JobState.Succeeded] || 0, + jobsFailed: state[JobState.Failed] || 0, + jobsCancelled: state[JobState.Cancelled] || 0, + latestSubmissionTime: group.aggregates.submitted as string, + } + }) + }, + enabled: Boolean(queue), + refetchInterval: autoRefresh && autoRefreshMs !== undefined ? autoRefreshMs : false, + }) +} From 3738e5fb971ddcd5d000beacfbadc20899be6ccc Mon Sep 17 00:00:00 2001 From: Nikola Jokic Date: Thu, 2 Jul 2026 11:26:16 +0200 Subject: [PATCH 17/49] Create observability package introducing OTEL (#4975) #### What type of PR is this? Enhancement #### What this PR does / why we need it Introuce base implementation for opentelemetry integration in Armada Signed-off-by: sarhiri --- README.md | 3 + go.mod | 53 +- go.sum | 146 ++-- .../common/observability/attribute_policy.go | 303 ++++++++ .../observability/attribute_policy_test.go | 189 +++++ internal/common/observability/config.go | 115 ++++ internal/common/observability/config_test.go | 117 ++++ internal/common/observability/lifecycle.go | 221 ++++++ .../observability/lifecycle_bootstrap_test.go | 150 ++++ .../common/observability/lifecycle_test.go | 651 ++++++++++++++++++ 10 files changed, 1864 insertions(+), 84 deletions(-) create mode 100644 internal/common/observability/attribute_policy.go create mode 100644 internal/common/observability/attribute_policy_test.go create mode 100644 internal/common/observability/config.go create mode 100644 internal/common/observability/config_test.go create mode 100644 internal/common/observability/lifecycle.go create mode 100644 internal/common/observability/lifecycle_bootstrap_test.go create mode 100644 internal/common/observability/lifecycle_test.go diff --git a/README.md b/README.md index 224481b1a64..54ecea279ad 100644 --- a/README.md +++ b/README.md @@ -189,6 +189,9 @@ Goreman exposes services on the following ports: | Redis | 6379 | Cache & events | | PostgreSQL | 5432 | Database | | Pulsar | 6650 | Message broker | +| OTEL Collector gRPC | 4317 | OTLP ingest | +| OTEL Collector HTTP | 4318 | OTLP ingest | +| Jaeger UI | 16686 | Trace visualization | ## Documentation diff --git a/go.mod b/go.mod index 295fca8526a..826823dd072 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/armadaproject/armada go 1.26.1 require ( - github.com/apache/pulsar-client-go v0.18.0 + github.com/apache/pulsar-client-go v0.15.1-candidate-1 github.com/coreos/go-oidc/v3 v3.17.0 github.com/go-openapi/analysis v0.24.2 github.com/go-openapi/jsonreference v0.21.4 @@ -32,11 +32,11 @@ require ( github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 golang.org/x/exp v0.0.0-20260112195511-716be5621a96 - golang.org/x/net v0.49.0 - golang.org/x/oauth2 v0.34.0 - golang.org/x/sync v0.19.0 + golang.org/x/net v0.55.0 + golang.org/x/oauth2 v0.36.0 + golang.org/x/sync v0.20.0 google.golang.org/genproto v0.0.0-20260122232226-8e98ce8d340d // indirect - google.golang.org/grpc v1.78.0 + google.golang.org/grpc v1.81.1 gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.32.11 k8s.io/apimachinery v0.32.11 @@ -59,7 +59,7 @@ require ( github.com/go-playground/validator/v10 v10.30.1 github.com/go-viper/mapstructure/v2 v2.5.0 github.com/gogo/status v1.1.1 - github.com/goreleaser/goreleaser/v2 v2.13.3 + github.com/goreleaser/goreleaser/v2 v2.8.2 github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0 github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3 github.com/jackc/pgx/v5 v5.8.0 @@ -75,12 +75,18 @@ require ( github.com/segmentio/fasthash v1.0.3 github.com/xitongsys/parquet-go v1.6.2 github.com/zalando/go-keyring v0.2.6 + go.opentelemetry.io/otel v1.44.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.33.0 + go.opentelemetry.io/otel/metric v1.44.0 + go.opentelemetry.io/otel/sdk v1.44.0 + go.opentelemetry.io/otel/trace v1.44.0 go.uber.org/atomic v1.11.0 go.uber.org/mock v0.6.0 - golang.org/x/term v0.39.0 - golang.org/x/text v0.33.0 + golang.org/x/term v0.43.0 + golang.org/x/text v0.37.0 golang.org/x/time v0.14.0 - google.golang.org/genproto/googleapis/api v0.0.0-20260122232226-8e98ce8d340d + google.golang.org/genproto/googleapis/api v0.0.0-20260226221140-a57be14db171 gopkg.in/inf.v0 v0.9.1 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v3 v3.0.1 @@ -91,15 +97,16 @@ require ( al.essio.dev/pkg/shellescape v1.6.0 // indirect charm.land/lipgloss/v2 v2.0.0-beta.3.0.20251120230642-dcccabe2cd63 // indirect dario.cat/mergo v1.0.2 // indirect + github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect + github.com/99designs/keyring v1.2.1 // indirect github.com/AlekSi/pointer v1.2.0 // indirect - github.com/AthenZ/athenz v1.12.14 // indirect + github.com/AthenZ/athenz v1.12.12 // indirect github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect github.com/DataDog/zstd v1.5.7 // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/sprig/v3 v3.3.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/ProtonMail/go-crypto v1.3.0 // indirect - github.com/RoaringBitmap/roaring/v2 v2.14.4 // indirect github.com/alecthomas/chroma/v2 v2.23.1 // indirect github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 // indirect github.com/apache/thrift v0.22.0 // indirect @@ -112,6 +119,7 @@ require ( github.com/blang/semver/v4 v4.0.0 // indirect github.com/buger/jsonparser v1.1.1 // indirect github.com/caarlos0/log v0.5.4 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/charmbracelet/colorprofile v0.4.1 // indirect github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834 // indirect @@ -132,6 +140,7 @@ require ( github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dlclark/regexp2 v1.11.5 // indirect github.com/docker/go-units v0.5.0 // indirect + github.com/dvsekhvalnov/jose2go v1.6.0 // indirect github.com/emicklei/go-restful/v3 v3.13.0 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/fatih/camelcase v1.0.0 // indirect @@ -142,8 +151,9 @@ require ( github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.7.0 // indirect github.com/go-git/go-git/v5 v5.16.4 // indirect - github.com/go-jose/go-jose/v4 v4.1.3 // indirect + github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.22.4 // indirect github.com/go-openapi/swag/cmdutils v0.25.4 // indirect github.com/go-openapi/swag/conv v0.25.4 // indirect @@ -159,6 +169,7 @@ require ( github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/gobwas/glob v0.2.3 // indirect + github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect github.com/godbus/dbus/v5 v5.2.2 // indirect github.com/gogo/googleapis v1.4.1 // indirect github.com/golang-jwt/jwt/v5 v5.3.0 // indirect @@ -173,6 +184,8 @@ require ( github.com/goreleaser/nfpm/v2 v2.44.1 // indirect github.com/gorilla/css v1.0.1 // indirect github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect + github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect github.com/hamba/avro/v2 v2.31.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect @@ -199,12 +212,13 @@ require ( github.com/microcosm-cc/bluemonday v1.0.27 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect - github.com/moby/go-archive v0.2.0 // indirect + github.com/moby/sys/sequential v0.6.0 // indirect + github.com/moby/sys/user v0.4.0 // indirect github.com/moby/term v0.5.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect - github.com/mschoch/smat v0.2.0 // indirect + github.com/mtibben/percent v0.2.1 // indirect github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/reflow v0.3.0 // indirect github.com/muesli/termenv v0.16.0 // indirect @@ -235,13 +249,16 @@ require ( github.com/yuin/goldmark-emoji v1.0.6 // indirect gitlab.com/digitalxero/go-conventional-commit v1.0.7 // indirect go.mongodb.org/mongo-driver v1.17.7 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 // indirect + go.opentelemetry.io/proto/otlp v1.9.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/crypto v0.47.0 // indirect - golang.org/x/mod v0.32.0 // indirect - golang.org/x/sys v0.40.0 // indirect + golang.org/x/crypto v0.51.0 // indirect + golang.org/x/mod v0.35.0 // indirect + golang.org/x/sys v0.45.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260122232226-8e98ce8d340d // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect diff --git a/go.sum b/go.sum index f45e1a98563..eeb6deef37a 100644 --- a/go.sum +++ b/go.sum @@ -32,7 +32,7 @@ cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Ud cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= cloud.google.com/go v0.100.1/go.mod h1:fs4QogzfH5n2pBXBP9vRiU+eCny7lD2vmFZy79Iuw1U= cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= -cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= +cloud.google.com/go v0.115.1 h1:Jo0SM9cQnSkYfp44+v+NQXHpcHqlnRJk2qxh6yvxxxQ= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -78,10 +78,14 @@ dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8= +github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMbk2FiG/kXiLl8BRyzTWDw7gX/Hz7Dd5eDMs= +github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4/go.mod h1:hN7oaIRCjzsZ2dE+yG5k+rsdt3qcwykqK6HVGcKwsw4= +github.com/99designs/keyring v1.2.1 h1:tYLp1ULvO7i3fI5vE21ReQuj99QFSs7lGm0xWyJo87o= +github.com/99designs/keyring v1.2.1/go.mod h1:fc+wB5KTk9wQ9sDx0kFXB3A0MaeGHM9AwRStKOQ5vOA= github.com/AlekSi/pointer v1.2.0 h1:glcy/gc4h8HnG2Z3ZECSzZ1IX1x2JxRVuDzaJwQE0+w= github.com/AlekSi/pointer v1.2.0/go.mod h1:gZGfd3dpW4vEc/UlyfKKi1roIqcCgwOIvb0tSNSBle0= -github.com/AthenZ/athenz v1.12.14 h1:y/SbWMBU1CejnkLSWgGOJuQEBEcCGDn9bsPGDEPAqDc= -github.com/AthenZ/athenz v1.12.14/go.mod h1:syp1M8L/dB9KimW+VKgpAWZIart3HVTbWgm0smRXLVI= +github.com/AthenZ/athenz v1.12.12 h1:Upf5Zx96GAgOGRwnGZN2YdgNGd52p+yyLZ85WvHpdC8= +github.com/AthenZ/athenz v1.12.12/go.mod h1:tepNDlRtQPpJ0f8C1WNx8T/L/C/D3fbA7FrGVI2fbFc= github.com/Azure/azure-amqp-common-go/v3 v3.2.1/go.mod h1:O6X1iYHP7s2x7NjUKsXVhkwWrQhxrd+d8/3rRadj4CI= github.com/Azure/azure-amqp-common-go/v3 v3.2.2/go.mod h1:O6X1iYHP7s2x7NjUKsXVhkwWrQhxrd+d8/3rRadj4CI= github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= @@ -144,8 +148,6 @@ github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/ProtonMail/go-crypto v1.3.0 h1:ILq8+Sf5If5DCpHQp4PbZdS1J7HDFRXz/+xKBiRGFrw= github.com/ProtonMail/go-crypto v1.3.0/go.mod h1:9whxjD8Rbs29b4XWbB8irEcE8KHMqaR2e7GWU1R+/PE= -github.com/RoaringBitmap/roaring/v2 v2.14.4 h1:4aKySrrg9G/5oRtJ3TrZLObVqxgQ9f1znCRBwEwjuVw= -github.com/RoaringBitmap/roaring/v2 v2.14.4/go.mod h1:oMvV6omPWr+2ifRdeZvVJyaz+aoEUopyv5iH0u/+wbY= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= @@ -159,8 +161,8 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516/go.mod h1:QNYViu/X0HXDHw7m3KXzWSVXIbfUvJqBFe6Gj8/pYA0= github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 h1:q4dksr6ICHXqG5hm0ZW5IHyeEJXoIJSOZeBLmWPNeIQ= github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40/go.mod h1:Q7yQnSMnLvcXlZ8RV+jwz/6y1rQTqbX6C82SndT52Zs= -github.com/apache/pulsar-client-go v0.18.0 h1:YsySoOds7WCXkRcOKHb85gk/v1Jndp+2oCkkRQEowUA= -github.com/apache/pulsar-client-go v0.18.0/go.mod h1:GKmTD1u5YLuhUnoVTNGdhdGNAYhoglWNWgwLJZTljAw= +github.com/apache/pulsar-client-go v0.15.1-candidate-1 h1:5LFEXv7goIO3XbQHpNWXjxp7xlduOVascUY4s90JNmc= +github.com/apache/pulsar-client-go v0.15.1-candidate-1/go.mod h1:HyzPvgO7Nc48/Mzk7Coo1YaZY+SN63F+nNwkHmjXkSI= github.com/apache/thrift v0.0.0-20181112125854-24918abba929/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.14.2/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.22.0 h1:r7mTJdj51TMDe6RtcmNdQxgn9XcyfGDOzegMDRg47uc= @@ -247,7 +249,6 @@ github.com/caarlos0/testfs v0.4.4 h1:3PHvzHi5Lt+g332CiShwS8ogTgS3HjrmzZxCm6JCDr8 github.com/caarlos0/testfs v0.4.4/go.mod h1:bRN55zgG4XCUVVHZCeU+/Tz1Q6AxEJOEJTliBy+1DMk= github.com/cavaliergopher/cpio v1.0.1 h1:KQFSeKmZhv0cr+kawA3a0xTQCU4QxXF1vhU7P7av2KM= github.com/cavaliergopher/cpio v1.0.1/go.mod h1:pBdaqQjnvXxdS/6CvNDwIANIFSP0xRKI16PX4xejRQc= -github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -302,10 +303,6 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/colinmarc/hdfs/v2 v2.1.1/go.mod h1:M3x+k8UKKmxtFu++uAZ0OtDU8jR3jnaZIAc6yK4Ue0c= -github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= -github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= -github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= -github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= @@ -344,13 +341,15 @@ github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZ github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= -github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM= -github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= -github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= +github.com/docker/docker v27.5.0+incompatible h1:um++2NcQtGRTz5eEgO6aJimo6/JxrTXC941hd05JO6U= +github.com/docker/docker v27.5.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dvsekhvalnov/jose2go v1.6.0 h1:Y9gnSnP4qEI0+/uQkHvFXeD2PLPJeXEL+ySMEA2EjTY= +github.com/dvsekhvalnov/jose2go v1.6.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU= github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= @@ -408,11 +407,12 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= -github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= -github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= +github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -492,6 +492,8 @@ github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJA github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= +github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0= +github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.2.2 h1:TUR3TgtSVDmjiXOgAAyaZbYmIeP3DPkld3jgKGV8mXQ= github.com/godbus/dbus/v5 v5.2.2/go.mod h1:3AAv2+hPq5rdnr5txxxRwiGjPXamgoIHgz9FPBfOp3c= @@ -633,8 +635,8 @@ github.com/goreleaser/chglog v0.7.4 h1:3pnNt/XCrUcAOq+KC91Azlgp5CRv4GHo1nl8Aws7O github.com/goreleaser/chglog v0.7.4/go.mod h1:dTVoZZagTz7hHdWaZ9OshHntKiF44HbWIHWxYJQ/h0Y= github.com/goreleaser/fileglob v1.4.0 h1:Y7zcUnzQjT1gbntacGAkIIfLv+OwojxTXBFxjSFoBBs= github.com/goreleaser/fileglob v1.4.0/go.mod h1:1pbHx7hhmJIxNZvm6fi6WVrnP0tndq6p3ayWdLn1Yf8= -github.com/goreleaser/goreleaser/v2 v2.13.3 h1:S8d13YgzzFXxoUJ9NJInuyq3lPNCXTcuW8wSvM+rXnQ= -github.com/goreleaser/goreleaser/v2 v2.13.3/go.mod h1:Rj+yhhXrO6WHc6cNh1GggpxzhhHXv9lczL5M4cSV3oA= +github.com/goreleaser/goreleaser/v2 v2.8.2 h1:S7fQyaumFjJKkUKQ2yHLKanfs2Uc1JK+P9mzDAc5hsE= +github.com/goreleaser/goreleaser/v2 v2.8.2/go.mod h1:dqm6yLhjxeROOrM+Y9LvBToheVcgJSd1oqShSJcR+dQ= github.com/goreleaser/nfpm/v2 v2.44.1 h1:g+QNjkEx+C2Zu8dB48t9da/VfV0CWS5TMjxT8HG1APY= github.com/goreleaser/nfpm/v2 v2.44.1/go.mod h1:drIYLqkla9SaOLbSnaFOmSIv5LXGfhHcbK54st97b4s= github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= @@ -648,6 +650,10 @@ github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3 h1:B+8ClL/kCQkRiU82d9xajR github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3/go.mod h1:NbCUVmiS4foBGBHOYlCT25+YmGpJ32dZPi75pGEUpj4= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 h1:NmZ1PKzSTQbuGHw9DGPFomqkkLWMC+vZCkfs+FHv1Vg= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3/go.mod h1:zQrxl1YP88HQlA6i9c63DSVPFklWpGX4OWAc9bFuaH4= +github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= +github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= github.com/hamba/avro/v2 v2.31.0 h1:wv3nmua7lCEIwWsb6vqsTS3pXktTxcKg5eoyNu0VhrU= github.com/hamba/avro/v2 v2.31.0/go.mod h1:t6lJYAGE5Mswfn17zjtyQsssRQgnqO6TXLBCHHWRqrw= github.com/hanwen/go-fuse v1.0.0/go.mod h1:unqXarDXqzAk0rt98O2tVndEPIpUgLD9+rwFisZH3Ok= @@ -839,8 +845,6 @@ github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zx github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= -github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8= -github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU= github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= @@ -865,8 +869,8 @@ github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt github.com/montanaflynn/stats v0.7.0/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= -github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= +github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs= +github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns= github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= @@ -877,6 +881,7 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/ncw/swift v1.0.52/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= github.com/nhooyr/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= @@ -885,14 +890,14 @@ github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= -github.com/onsi/gomega v1.37.0 h1:CdEG8g0S133B4OswTDC/5XPSzE1OeP29QOioj2PID2Y= -github.com/onsi/gomega v1.37.0/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0= +github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= +github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= github.com/openconfig/goyang v1.6.3 h1:9nWXBwd6b4+nZr8ni7O4zUXVhrVMXCLFz8os5YWFuo4= github.com/openconfig/goyang v1.6.3/go.mod h1:5WolITjek1NF8yrNERyVZ7jqjOClJTpO8p/+OwmETM4= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= -github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pborman/getopt v0.0.0-20180729010549-6fdd0a2c7117/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= @@ -1065,19 +1070,27 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.64.0 h1:ssfIgGNANqpVFCndZvcuyKbl0g+UAVcbBcqGkG28H0Y= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.64.0/go.mod h1:GQ/474YrbE4Jx8gZ4q5I4hrhUzM6UPzyrqJYV2AqPoQ= -go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= -go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= -go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= -go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= -go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= -go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= -go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= -go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= -go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= -go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 h1:yd02MEjBdJkG3uabWP9apV+OuWRIXGDuJEUJbOHmCFU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0/go.mod h1:umTcuxiv1n/s/S6/c2AT/g2CQ7u5C59sHDNmfSwgz7Q= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 h1:f0cb2XPmrqn4XMy9PNliTgRKJgS5WcL/u0/WRYGz4t0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0/go.mod h1:vnakAaFckOMiMtOIhFI2MNH4FYrZzXCYxmb1LlhoGz8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 h1:5pojmb1U1AogINhN3SurB+zm/nIcusopeBNp42f45QM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0/go.mod h1:57gTHJSE5S1tqg+EKsLPlTWhpHMsWlVmer+LA926XiA= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.33.0 h1:wpMfgF8E1rkrT1Z6meFh1NDtownE9Ii3n3X2GJYjsaU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.33.0/go.mod h1:wAy0T/dUbs468uOlkT31xjvqQgEVXv58BRFWEgn5v/0= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= +go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -1127,8 +1140,8 @@ golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= -golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8= -golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A= +golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= +golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1180,8 +1193,8 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c= -golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU= +golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= +golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1235,8 +1248,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= -golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1257,8 +1270,8 @@ golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= -golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1272,8 +1285,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= -golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1356,16 +1369,16 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= -golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY= -golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww= +golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= +golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1378,8 +1391,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= -golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1454,8 +1467,8 @@ golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc= -golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg= +golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= +golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1467,8 +1480,8 @@ golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6f gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= gonum.org/v1/gonum v0.9.3/go.mod h1:TZumC3NeyVQskjXqmyWt4S3bINhy7B4eYwW69EbyX+0= -gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= -gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= gonum.org/v1/plot v0.9.0/go.mod h1:3Pcqqmp6RHvJI72kgb8fThyUnav364FOsdDo2aGW5lY= @@ -1612,10 +1625,10 @@ google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2I google.golang.org/genproto v0.0.0-20220401170504-314d38edb7de/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/genproto v0.0.0-20260122232226-8e98ce8d340d h1:hUplc9kLwH374NIY3PreRUK3Unc0xLm/W7MDsm0gCNo= google.golang.org/genproto v0.0.0-20260122232226-8e98ce8d340d/go.mod h1:SpjiK7gGN2j/djoQMxLl3QOe/J/XxNzC5M+YLecVVWU= -google.golang.org/genproto/googleapis/api v0.0.0-20260122232226-8e98ce8d340d h1:tUKoKfdZnSjTf5LW7xpG4c6SZ3Ozisn5eumcoTuMEN4= -google.golang.org/genproto/googleapis/api v0.0.0-20260122232226-8e98ce8d340d/go.mod h1:p3MLuOwURrGBRoEyFHBT3GjUwaCQVKeNqqWxlcISGdw= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260122232226-8e98ce8d340d h1:xXzuihhT3gL/ntduUZwHECzAn57E8dA6l8SOtYWdD8Q= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260122232226-8e98ce8d340d/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= +google.golang.org/genproto/googleapis/api v0.0.0-20260226221140-a57be14db171 h1:tu/dtnW1o3wfaxCOjSLn5IRX4YDcJrtlpzYkhHhGaC4= +google.golang.org/genproto/googleapis/api v0.0.0-20260226221140-a57be14db171/go.mod h1:M5krXqk4GhBKvB596udGL3UyjL4I1+cTbK0orROM9ng= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -1646,8 +1659,8 @@ google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= -google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= +google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ= +google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -1668,6 +1681,7 @@ google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= diff --git a/internal/common/observability/attribute_policy.go b/internal/common/observability/attribute_policy.go new file mode 100644 index 00000000000..e7985e59516 --- /dev/null +++ b/internal/common/observability/attribute_policy.go @@ -0,0 +1,303 @@ +package observability + +import ( + "context" + "fmt" + "strings" + "sync" + + "go.opentelemetry.io/otel/attribute" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + + "github.com/armadaproject/armada/internal/common/logging" +) + +const ( + // DefaultAttributeCardinalityLimit is the maximum number of unique values + // allowed per custom span attribute key before values are redacted. + DefaultAttributeCardinalityLimit = 1000 + + // AttributeRedactedValue is used when an attribute key is denied for PII/sensitive content. + AttributeRedactedValue = "[REDACTED]" + // AttributeDisallowedValue is used when an attribute key is not in the allow-list. + AttributeDisallowedValue = "[DISALLOWED]" + // AttributeHighCardinalityValue is used once cardinality guardrails are exceeded. + AttributeHighCardinalityValue = "[HIGH_CARDINALITY]" +) + +// SpanAttributePolicy defines guardrails for span attribute keys and values. +// +// Policy summary: +// - Allow-list: rpc.*, http.*, net.*, server.*, service.*, armada.*, trace_id, span_id +// - Deny-list: explicit sensitive keys and key-name patterns (password/secret/token/api_key/apikey) +// - Cardinality: custom (non-standard) keys capped at DefaultAttributeCardinalityLimit unique values +// +// Important: OTel SDK span processors cannot delete already-set attributes from an active span. +// To prevent raw PII leakage, denied/disallowed keys are overwritten with marker values. +type SpanAttributePolicy struct { + allowedPrefixes []string + allowedExact map[string]struct{} + deniedExact map[string]struct{} + deniedContains []string + + cardinalityExemptPrefixes []string + cardinalityExemptExact map[string]struct{} + cardinality *attributeCardinalityTracker +} + +type SpanAttributeViolationReason string + +const ( + SpanAttributeViolationDenied SpanAttributeViolationReason = "denied" + SpanAttributeViolationDisallowed SpanAttributeViolationReason = "disallowed" + SpanAttributeViolationHighCardinality SpanAttributeViolationReason = "high_cardinality" +) + +type SpanAttributeViolation struct { + Key string + Reason SpanAttributeViolationReason +} + +// NewDefaultSpanAttributePolicy returns the default attribute policy for Armada traces. +func NewDefaultSpanAttributePolicy() *SpanAttributePolicy { + return &SpanAttributePolicy{ + allowedPrefixes: []string{"rpc.", "http.", "net.", "server.", "service.", "armada."}, + allowedExact: map[string]struct{}{ + "trace_id": {}, + "span_id": {}, + }, + // Explicit deny-list for common PII/sensitive payload fields. + deniedExact: map[string]struct{}{ + "user_id": {}, + "user_email": {}, + "user_name": {}, + "password": {}, + "api_key": {}, + "token": {}, + "secret": {}, + "job_payload": {}, + "request_body": {}, + "response_body": {}, + }, + deniedContains: []string{"password", "secret", "token", "api_key", "apikey"}, + cardinalityExemptPrefixes: []string{"rpc.", "http.", "net.", "server.", "service."}, + cardinalityExemptExact: map[string]struct{}{ + "trace_id": {}, + "span_id": {}, + "http.user_agent": {}, + }, + cardinality: newAttributeCardinalityTracker(DefaultAttributeCardinalityLimit), + } +} + +// SanitizeForSpan returns attributes safe for emission by applying deny-list, +// allow-list, and cardinality guardrails. +// +// This function only sanitizes the attributes passed to it. In the OTel +// SDK processor model, OnStart can sanitize initial attributes, but attributes added +// later via span.SetAttributes(...) cannot be rewritten at OnEnd. See +// ViolationsForSpan/OnEnd guardrails for post-start detection. +func (p *SpanAttributePolicy) SanitizeForSpan(attrs []attribute.KeyValue) []attribute.KeyValue { + out := make([]attribute.KeyValue, 0, len(attrs)) + for _, kv := range attrs { + key := string(kv.Key) + keyLower := strings.ToLower(key) + + switch { + case p.isDenied(keyLower): + out = append(out, attribute.String(key, AttributeRedactedValue)) + continue + case !p.isAllowed(keyLower): + out = append(out, attribute.String(key, AttributeDisallowedValue)) + continue + } + + if p.shouldGuardCardinality(keyLower) && p.cardinality.ExceedsLimit(keyLower, attributeValueFingerprint(kv.Value)) { + out = append(out, attribute.String(key, AttributeHighCardinalityValue)) + continue + } + + out = append(out, kv) + } + return out +} + +// ViolationsForSpan returns policy violations present in the provided attributes. +// +// Use this as a guardrail for ended spans where mutation is no longer possible, +// e.g. to detect attributes that were added after OnStart and therefore bypassed +// processor-time sanitization. +func (p *SpanAttributePolicy) ViolationsForSpan(attrs []attribute.KeyValue) []SpanAttributeViolation { + violations := make([]SpanAttributeViolation, 0) + for _, kv := range attrs { + if isSanitizedMarkerValue(kv.Value) { + continue + } + + key := string(kv.Key) + keyLower := strings.ToLower(key) + + switch { + case p.isDenied(keyLower): + violations = append(violations, SpanAttributeViolation{Key: key, Reason: SpanAttributeViolationDenied}) + case !p.isAllowed(keyLower): + violations = append(violations, SpanAttributeViolation{Key: key, Reason: SpanAttributeViolationDisallowed}) + case p.shouldGuardCardinality(keyLower) && p.cardinality.IsOverLimit(keyLower, attributeValueFingerprint(kv.Value)): + violations = append(violations, SpanAttributeViolation{Key: key, Reason: SpanAttributeViolationHighCardinality}) + } + } + return violations +} + +func isSanitizedMarkerValue(value attribute.Value) bool { + if value.Type() != attribute.STRING { + return false + } + + s := value.AsString() + return s == AttributeRedactedValue || s == AttributeDisallowedValue || s == AttributeHighCardinalityValue +} + +func (p *SpanAttributePolicy) isAllowed(key string) bool { + if _, ok := p.allowedExact[key]; ok { + return true + } + for _, prefix := range p.allowedPrefixes { + if strings.HasPrefix(key, prefix) { + return true + } + } + return false +} + +func (p *SpanAttributePolicy) isDenied(key string) bool { + if _, ok := p.deniedExact[key]; ok { + return true + } + for _, disallowedPart := range p.deniedContains { + if strings.Contains(key, disallowedPart) { + return true + } + } + return false +} + +func (p *SpanAttributePolicy) shouldGuardCardinality(key string) bool { + if _, ok := p.cardinalityExemptExact[key]; ok { + return false + } + for _, prefix := range p.cardinalityExemptPrefixes { + if strings.HasPrefix(key, prefix) { + return false + } + } + return true +} + +func attributeValueFingerprint(value attribute.Value) string { + return fmt.Sprintf("%d:%v", value.Type(), value.AsInterface()) +} + +type attributeCardinalityTracker struct { + limit int + + mu sync.Mutex + seen map[string]map[string]struct{} + overLimit map[string]bool +} + +func newAttributeCardinalityTracker(limit int) *attributeCardinalityTracker { + return &attributeCardinalityTracker{ + limit: limit, + seen: make(map[string]map[string]struct{}), + overLimit: make(map[string]bool), + } +} + +func (c *attributeCardinalityTracker) ExceedsLimit(key, value string) bool { + c.mu.Lock() + defer c.mu.Unlock() + + knownValues, ok := c.seen[key] + if !ok { + knownValues = make(map[string]struct{}) + c.seen[key] = knownValues + } + + if _, seen := knownValues[value]; seen { + return false + } + + if len(knownValues) >= c.limit { + c.overLimit[key] = true + return true + } + + knownValues[value] = struct{}{} + return false +} + +func (c *attributeCardinalityTracker) IsOverLimit(key, value string) bool { + c.mu.Lock() + defer c.mu.Unlock() + + knownValues, ok := c.seen[key] + if !ok { + return false + } + + if _, seen := knownValues[value]; seen { + return false + } + + return c.overLimit[key] || len(knownValues) >= c.limit +} + +type spanAttributePolicyProcessor struct { + policy *SpanAttributePolicy +} + +// NewSpanAttributePolicyProcessor creates a span processor that sanitizes span attributes +// at span start using the configured policy and applies OnEnd guardrails for attributes +// added after span start. +// +// Constraint: OTel SDK processors cannot mutate attributes on ended spans. Therefore, +// attributes set via span.SetAttributes(...) after OnStart cannot be rewritten at +// processor level. OnEnd logs policy violations as an operational guardrail. +func NewSpanAttributePolicyProcessor(policy *SpanAttributePolicy) sdktrace.SpanProcessor { + if policy == nil { + policy = NewDefaultSpanAttributePolicy() + } + return &spanAttributePolicyProcessor{policy: policy} +} + +func (p *spanAttributePolicyProcessor) OnStart(_ context.Context, span sdktrace.ReadWriteSpan) { + attrs := span.Attributes() + if len(attrs) == 0 { + return + } + sanitized := p.policy.SanitizeForSpan(attrs) + span.SetAttributes(sanitized...) +} + +func (p *spanAttributePolicyProcessor) OnEnd(span sdktrace.ReadOnlySpan) { + violations := p.policy.ViolationsForSpan(span.Attributes()) + if len(violations) == 0 { + return + } + + keys := make([]string, 0, len(violations)) + for _, violation := range violations { + keys = append(keys, fmt.Sprintf("%s(%s)", violation.Key, violation.Reason)) + } + + logging.Warnf( + "Span %q contains policy-violating attributes that could not be rewritten after span start: %s", + span.Name(), + strings.Join(keys, ", "), + ) +} + +func (p *spanAttributePolicyProcessor) Shutdown(context.Context) error { return nil } + +func (p *spanAttributePolicyProcessor) ForceFlush(context.Context) error { return nil } diff --git a/internal/common/observability/attribute_policy_test.go b/internal/common/observability/attribute_policy_test.go new file mode 100644 index 00000000000..e1495dd07ba --- /dev/null +++ b/internal/common/observability/attribute_policy_test.go @@ -0,0 +1,189 @@ +package observability + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + semconv "go.opentelemetry.io/otel/semconv/v1.26.0" + oteltrace "go.opentelemetry.io/otel/trace" +) + +func TestSpanAttributeAllowListPreservesStandardKeys(t *testing.T) { + policy := NewDefaultSpanAttributePolicy() + + attrs := []attribute.KeyValue{ + attribute.String("rpc.system.name", "grpc"), + attribute.String("http.method", "GET"), + attribute.Int("http.status_code", 200), + attribute.String("net.peer.name", "localhost"), + attribute.String("server.address", "armada.local"), + attribute.String("service.name", "server"), + attribute.String("trace_id", "abc"), + attribute.String("span_id", "def"), + } + + sanitized := policy.SanitizeForSpan(attrs) + require.Len(t, sanitized, len(attrs)) + + for i := range attrs { + assert.Equal(t, attrs[i], sanitized[i]) + } +} + +func TestSpanAttributePolicyDropsPII(t *testing.T) { + policy := NewDefaultSpanAttributePolicy() + + attrs := []attribute.KeyValue{ + attribute.String("user_email", "user@example.com"), + attribute.String("password", "super-secret"), + attribute.String("request_body", `{"sensitive":true}`), + attribute.String("armada.custom_token_field", "token-value"), + attribute.String("rpc.system.name", "grpc"), + } + + sanitized := policy.SanitizeForSpan(attrs) + + assert.Equal(t, attribute.String("user_email", AttributeRedactedValue), sanitized[0]) + assert.Equal(t, attribute.String("password", AttributeRedactedValue), sanitized[1]) + assert.Equal(t, attribute.String("request_body", AttributeRedactedValue), sanitized[2]) + assert.Equal(t, attribute.String("armada.custom_token_field", AttributeRedactedValue), sanitized[3]) + assert.Equal(t, attribute.String("rpc.system.name", "grpc"), sanitized[4]) +} + +func TestSpanAttributePolicyAllowsNonSensitiveKeyAttributes(t *testing.T) { + policy := NewDefaultSpanAttributePolicy() + + attrs := []attribute.KeyValue{ + attribute.String("api_key", "sensitive"), + attribute.String("armada.queue_key_type", "priority-class"), + attribute.String("rpc.request.key_field", "queue"), + } + + sanitized := policy.SanitizeForSpan(attrs) + + assert.Equal(t, attribute.String("api_key", AttributeRedactedValue), sanitized[0]) + assert.Equal(t, attrs[1], sanitized[1]) + assert.Equal(t, attrs[2], sanitized[2]) +} + +func TestSpanAttributePolicyBoundsCardinality(t *testing.T) { + policy := NewDefaultSpanAttributePolicy() + + customKey := "armada.user_agent_variant" + for i := range DefaultAttributeCardinalityLimit { + in := []attribute.KeyValue{attribute.String(customKey, fmt.Sprintf("ua-%d", i))} + out := policy.SanitizeForSpan(in) + require.Len(t, out, 1) + assert.Equal(t, in[0], out[0], "attribute should be preserved before limit") + } + + overLimit := []attribute.KeyValue{attribute.String(customKey, "ua-over-limit")} + sanitized := policy.SanitizeForSpan(overLimit) + require.Len(t, sanitized, 1) + assert.Equal(t, attribute.String(customKey, AttributeHighCardinalityValue), sanitized[0]) + + httpUserAgent := []attribute.KeyValue{attribute.String("http.user_agent", "ua-allowed")} + httpUserAgentSanitized := policy.SanitizeForSpan(httpUserAgent) + assert.Equal(t, httpUserAgent[0], httpUserAgentSanitized[0], "http.user_agent should be cardinality-exempt") +} + +func TestResourceAttributesSet(t *testing.T) { + spanRecorder := tracetest.NewSpanRecorder() + + res, err := resource.New(context.Background(), + resource.WithAttributes( + semconv.ServiceName("resource-service"), + semconv.ServiceNamespace("armada"), + semconv.ServiceVersion("1.2.3"), + ), + ) + require.NoError(t, err) + + tp := sdktrace.NewTracerProvider( + sdktrace.WithResource(res), + sdktrace.WithSpanProcessor(NewSpanAttributePolicyProcessor(NewDefaultSpanAttributePolicy())), + sdktrace.WithSpanProcessor(spanRecorder), + ) + defer func() { + _ = tp.Shutdown(context.Background()) + }() + + otel.SetTracerProvider(tp) + + tracer := tp.Tracer("resource-test") + _, span := tracer.Start(context.Background(), "resource-span") + span.SetAttributes(attribute.String("user_id", "pii-user")) + span.End() + + spans := spanRecorder.Ended() + require.Len(t, spans, 1) + + resourceAttrs := spans[0].Resource().Attributes() + assert.Contains(t, resourceAttrs, semconv.ServiceName("resource-service")) + assert.Contains(t, resourceAttrs, semconv.ServiceNamespace("armada")) + assert.Contains(t, resourceAttrs, semconv.ServiceVersion("1.2.3")) + + // Policy processor must not mutate resource attributes. + for _, kv := range resourceAttrs { + assert.NotEqual(t, "user_id", string(kv.Key)) + } +} + +func TestSpanAttributePolicySanitizesAttributesProvidedAtSpanStart(t *testing.T) { + spanRecorder := tracetest.NewSpanRecorder() + tp := sdktrace.NewTracerProvider( + sdktrace.WithSpanProcessor(NewSpanAttributePolicyProcessor(NewDefaultSpanAttributePolicy())), + sdktrace.WithSpanProcessor(spanRecorder), + ) + defer func() { + _ = tp.Shutdown(context.Background()) + }() + + tracer := tp.Tracer("start-attrs-test") + _, span := tracer.Start(context.Background(), "start-attrs", + oteltrace.WithAttributes(attribute.String("user_id", "pii-user-at-start")), + ) + span.End() + + spans := spanRecorder.Ended() + require.Len(t, spans, 1) + endedAttrs := spans[0].Attributes() + assert.Contains(t, endedAttrs, attribute.String("user_id", AttributeRedactedValue)) + + violations := NewDefaultSpanAttributePolicy().ViolationsForSpan(endedAttrs) + assert.NotContains(t, violations, SpanAttributeViolation{Key: "user_id", Reason: SpanAttributeViolationDenied}) +} + +func TestSpanAttributePolicyGuardrailDetectsDeniedAttributesSetAfterStart(t *testing.T) { + policy := NewDefaultSpanAttributePolicy() + spanRecorder := tracetest.NewSpanRecorder() + tp := sdktrace.NewTracerProvider( + sdktrace.WithSpanProcessor(NewSpanAttributePolicyProcessor(policy)), + sdktrace.WithSpanProcessor(spanRecorder), + ) + defer func() { + _ = tp.Shutdown(context.Background()) + }() + + tracer := tp.Tracer("late-attrs-test") + _, span := tracer.Start(context.Background(), "late-attrs") + span.SetAttributes(attribute.String("user_id", "late-pii-user")) + span.End() + + spans := spanRecorder.Ended() + require.Len(t, spans, 1) + + endedAttrs := spans[0].Attributes() + assert.Contains(t, endedAttrs, attribute.String("user_id", "late-pii-user"), "OTel processor cannot rewrite post-start attributes") + + violations := policy.ViolationsForSpan(endedAttrs) + assert.Contains(t, violations, SpanAttributeViolation{Key: "user_id", Reason: SpanAttributeViolationDenied}) +} diff --git a/internal/common/observability/config.go b/internal/common/observability/config.go new file mode 100644 index 00000000000..4a9ab7eeaa9 --- /dev/null +++ b/internal/common/observability/config.go @@ -0,0 +1,115 @@ +package observability + +import ( + "fmt" + "maps" + "net/url" + "strings" +) + +const ( + ConfigOtelExporterOtlpEndpoint = "observability.exporter.endpoint" + ConfigOtelExporterOtlpProtocol = "observability.exporter.protocol" + ConfigOtelTracesSampler = "observability.traces.sampler" + ConfigOtelTracesSamplerArg = "observability.traces.samplerArg" +) + +const ( + ResourceAttributeServiceName = "service.name" + ResourceAttributeServiceNamespace = "service.namespace" + ResourceAttributeServiceVersion = "service.version" + ResourceAttributeServiceInstance = "service.instance.id" +) + +var validSamplers = map[string]bool{ + "parent_based_trace_id_ratio": true, + "trace_id_ratio": true, + "always_on": true, + "always_off": true, +} + +var validOTLPProtocols = map[string]struct{}{ + "http/protobuf": {}, + "grpc": {}, +} + +// ResourceAttributes define the required OpenTelemetry service identity contract. +type ResourceAttributes struct { + // Service name is the name of the service. + // Required attribute, with key `service.name`. + ServiceName string + // Service version is the version of the service. + // Required attribute, with key `service.version`. + ServiceVersion string + // Service instance is a unique identifier for the service instance. + // Required attribute, with key `service.instance.id`. + ServiceInstance string + // Extra attributes if specified will be added to the resource attributes. + // These can be used to add additional metadata about the service instance. + Extra map[string]string +} + +type OTLPExporterConfig struct { + Endpoint string + Protocol string +} + +type TracesConfig struct { + // Sampler controls root trace sampling policy. + // Supported values: + // - always_on + // - always_off + // - trace_id_ratio + // - parent_based_trace_id_ratio + Sampler string + // SamplerArg is the sampler parameter used by ratio samplers. + // Valid range for ratio samplers is 0.0 to 1.0. + SamplerArg float64 +} + +type ObservabilityConfig struct { + Enabled bool + Exporter OTLPExporterConfig + Traces TracesConfig + Resource ResourceAttributes +} + +func (c ObservabilityConfig) Validate() error { + if strings.TrimSpace(c.Exporter.Endpoint) == "" { + return fmt.Errorf("%s must not be empty", ConfigOtelExporterOtlpEndpoint) + } + + parsedURL, err := url.Parse(c.Exporter.Endpoint) + if err != nil || parsedURL.Scheme == "" || parsedURL.Host == "" { + return fmt.Errorf("%s must be a valid absolute URL", ConfigOtelExporterOtlpEndpoint) + } + if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" { + return fmt.Errorf("%s scheme must be http or https", ConfigOtelExporterOtlpEndpoint) + } + + if _, ok := validOTLPProtocols[c.Exporter.Protocol]; !ok { + return fmt.Errorf("%s=%q is invalid: supported values are %v", ConfigOtelExporterOtlpProtocol, c.Exporter.Protocol, maps.Keys(validOTLPProtocols)) + } + + if _, ok := validSamplers[c.Traces.Sampler]; !ok { + return fmt.Errorf("%s=%q is invalid: supported values are %v", ConfigOtelTracesSampler, c.Traces.Sampler, maps.Keys(validSamplers)) + } + + if c.Traces.Sampler == "parent_based_trace_id_ratio" || c.Traces.Sampler == "trace_id_ratio" { + if c.Traces.SamplerArg < 0 || c.Traces.SamplerArg > 1 { + return fmt.Errorf("%s must be between 0 and 1 for sampler %q", ConfigOtelTracesSamplerArg, c.Traces.Sampler) + } + } + + if strings.TrimSpace(c.Resource.ServiceName) == "" { + return fmt.Errorf("resource attribute %q must not be empty", ResourceAttributeServiceName) + } + if strings.TrimSpace(c.Resource.ServiceVersion) == "" { + return fmt.Errorf("resource attribute %q must not be empty", ResourceAttributeServiceVersion) + } + if strings.TrimSpace(c.Resource.ServiceInstance) == "" { + return fmt.Errorf("resource attribute %q must not be empty", ResourceAttributeServiceInstance) + } + + return nil +} diff --git a/internal/common/observability/config_test.go b/internal/common/observability/config_test.go new file mode 100644 index 00000000000..18f4dab9cc6 --- /dev/null +++ b/internal/common/observability/config_test.go @@ -0,0 +1,117 @@ +package observability + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + testOTLPHTTPProtocol = "http/protobuf" + testParentBasedTraceIDRatioSampler = "parent_based_trace_id_ratio" +) + +func TestObservabilityConfig(t *testing.T) { + t.Run("rejects unset config", func(t *testing.T) { + err := (ObservabilityConfig{}).Validate() + require.Error(t, err) + + assert.ErrorContains(t, err, ConfigOtelExporterOtlpEndpoint) + }) + + t.Run("preserves config values", func(t *testing.T) { + cfg := ObservabilityConfig{ + Enabled: true, + Exporter: OTLPExporterConfig{ + Endpoint: "http://otel-collector:4318", + Protocol: testOTLPHTTPProtocol, + }, + Traces: TracesConfig{ + Sampler: testParentBasedTraceIDRatioSampler, + SamplerArg: 0.25, + }, + Resource: ResourceAttributes{ + ServiceName: "armada-executor", + ServiceVersion: "v2.0.0", + ServiceInstance: "pod-xyz", + }, + } + require.NoError(t, cfg.Validate()) + + assert.True(t, cfg.Enabled) + assert.Equal(t, "http://otel-collector:4318", cfg.Exporter.Endpoint) + assert.Equal(t, "http/protobuf", cfg.Exporter.Protocol) + assert.Equal(t, testParentBasedTraceIDRatioSampler, cfg.Traces.Sampler) + assert.Equal(t, 0.25, cfg.Traces.SamplerArg) + assert.Equal(t, "armada-executor", cfg.Resource.ServiceName) + assert.Equal(t, "v2.0.0", cfg.Resource.ServiceVersion) + assert.Equal(t, "pod-xyz", cfg.Resource.ServiceInstance) + }) + + t.Run("preserves explicit extra resource attributes", func(t *testing.T) { + cfg := ObservabilityConfig{ + Exporter: OTLPExporterConfig{ + Endpoint: "http://otel-collector:4318", + Protocol: testOTLPHTTPProtocol, + }, + Traces: TracesConfig{ + Sampler: testParentBasedTraceIDRatioSampler, + SamplerArg: 1.0, + }, + Resource: ResourceAttributes{ + ServiceName: "armada-executor", + ServiceVersion: "v2.0.0", + ServiceInstance: "pod-xyz", + Extra: map[string]string{ + "deployment.environment": "prod", + }, + }, + } + require.NoError(t, cfg.Validate()) + + assert.Equal(t, map[string]string{"deployment.environment": "prod"}, cfg.Resource.Extra) + }) +} + +func TestObservabilityConfigRejectsInvalidSampler(t *testing.T) { + err := (ObservabilityConfig{ + Exporter: OTLPExporterConfig{ + Endpoint: "http://otel-collector:4318", + Protocol: testOTLPHTTPProtocol, + }, + Traces: TracesConfig{ + Sampler: "invalid_sampler_name", + SamplerArg: 1.0, + }, + Resource: ResourceAttributes{ + ServiceName: "armada-server", + ServiceVersion: "1.2.3", + ServiceInstance: "instance-1", + }, + }).Validate() + require.Error(t, err) + assert.ErrorContains(t, err, ConfigOtelTracesSampler) + assert.ErrorContains(t, err, "invalid_sampler_name") +} + +func TestObservabilityConfigRejectsUnsupportedEndpointScheme(t *testing.T) { + err := (ObservabilityConfig{ + Exporter: OTLPExporterConfig{ + Endpoint: "grpc://otel-collector:4317", + Protocol: testOTLPHTTPProtocol, + }, + Traces: TracesConfig{ + Sampler: testParentBasedTraceIDRatioSampler, + SamplerArg: 1.0, + }, + Resource: ResourceAttributes{ + ServiceName: "armada-server", + ServiceVersion: "1.2.3", + ServiceInstance: "instance-1", + }, + }).Validate() + require.Error(t, err) + assert.ErrorContains(t, err, ConfigOtelExporterOtlpEndpoint) + assert.ErrorContains(t, err, "http or https") +} diff --git a/internal/common/observability/lifecycle.go b/internal/common/observability/lifecycle.go new file mode 100644 index 00000000000..ff6b18a37f7 --- /dev/null +++ b/internal/common/observability/lifecycle.go @@ -0,0 +1,221 @@ +package observability + +import ( + "context" + "fmt" + "net/url" + "sync" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + metricnoop "go.opentelemetry.io/otel/metric/noop" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + tracenoop "go.opentelemetry.io/otel/trace/noop" + + "github.com/armadaproject/armada/internal/common/logging" +) + +const ( + exportTimeout = 10 * time.Second + batchTimeout = 5 * time.Second + maxExportBatch = 512 + maxBatchQueue = 2048 + shutdownTimeout = 5 * time.Second +) + +var ( + globalTracerProvider *sdktrace.TracerProvider + globalTracerProviderMu sync.RWMutex +) + +func setNoopOTelLocked() { + otel.SetTracerProvider(tracenoop.NewTracerProvider()) + otel.SetMeterProvider(metricnoop.NewMeterProvider()) + otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator()) + globalTracerProvider = nil +} + +func setNoopOTel() { + globalTracerProviderMu.Lock() + tp := globalTracerProvider + setNoopOTelLocked() + globalTracerProviderMu.Unlock() + + if err := shutdownTracerProvider(context.Background(), tp); err != nil { + logging.WithError(err).Warn("Failed to shutdown previous OTel tracer provider") + } +} + +func init() { + setNoopOTel() +} + +// InitOTel initializes the global OpenTelemetry tracer provider with the given configuration. +// This function is fail-open: if the OTLP collector is unreachable, it logs the error but +// returns success to allow the service to start. The tracer provider is set globally via +// otel.SetTracerProvider() and W3C propagators are registered via otel.SetTextMapPropagator(). +// +// Returns an error only if configuration is invalid or critical setup fails (not collector reachability). +func InitOTel(cfg ObservabilityConfig) error { + if !cfg.Enabled { + setNoopOTel() + logging.Info("OpenTelemetry disabled by config") + return nil + } + + attrs := []attribute.KeyValue{ + attribute.String(ResourceAttributeServiceName, cfg.Resource.ServiceName), + attribute.String(ResourceAttributeServiceVersion, cfg.Resource.ServiceVersion), + attribute.String(ResourceAttributeServiceInstance, cfg.Resource.ServiceInstance), + } + for key, value := range cfg.Resource.Extra { + attrs = append(attrs, attribute.String(key, value)) + } + res, err := resource.New( + context.Background(), + resource.WithAttributes( + attrs..., + ), + ) + if err != nil { + return fmt.Errorf("failed to create OTel resource: %w", err) + } + + // Create OTLP exporter with bounded timeout + ctx, cancel := context.WithTimeout(context.Background(), exportTimeout) + defer cancel() + + exporter, err := newTraceExporter(ctx, cfg) + if err != nil { + // Fail-open: log error but continue with noop provider + logging.WithError(err).Warnf( + "Failed to create OTLP trace exporter (endpoint=%s, protocol=%s). Service will start without tracing.", + cfg.Exporter.Endpoint, + cfg.Exporter.Protocol, + ) + setNoopOTel() + return nil + } + + // Create sampler based on config + var sampler sdktrace.Sampler + switch cfg.Traces.Sampler { + case "parent_based_trace_id_ratio": + sampler = sdktrace.ParentBased(sdktrace.TraceIDRatioBased(cfg.Traces.SamplerArg)) + case "trace_id_ratio": + sampler = sdktrace.TraceIDRatioBased(cfg.Traces.SamplerArg) + case "always_on": + sampler = sdktrace.AlwaysSample() + case "always_off": + sampler = sdktrace.NeverSample() + default: + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), shutdownTimeout) + defer shutdownCancel() + _ = exporter.Shutdown(shutdownCtx) + return fmt.Errorf("unsupported sampler: %s", cfg.Traces.Sampler) + } + + // Create tracer provider with bounded batch processing guardrails: + // - max export batch size: 512 spans + // - max queue size: 2048 spans (drop-on-backpressure above this bound) + // - batch timeout: 5s + // - export timeout: 10s + tp := sdktrace.NewTracerProvider( + sdktrace.WithSpanProcessor(NewSpanAttributePolicyProcessor(NewDefaultSpanAttributePolicy())), + sdktrace.WithBatcher( + exporter, + sdktrace.WithMaxExportBatchSize(maxExportBatch), + sdktrace.WithMaxQueueSize(maxBatchQueue), + sdktrace.WithExportTimeout(exportTimeout), + sdktrace.WithBatchTimeout(batchTimeout), + ), + sdktrace.WithResource(res), + sdktrace.WithSampler(sampler), + ) + + // Set global tracer provider and propagators. + globalTracerProviderMu.Lock() + otel.SetTracerProvider(tp) + otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator( + propagation.TraceContext{}, + propagation.Baggage{}, + )) + globalTracerProvider = tp + globalTracerProviderMu.Unlock() + + logging.Infof( + "OpenTelemetry initialized: endpoint=%s, sampler=%s, ratio=%.2f, service=%s", + cfg.Exporter.Endpoint, + cfg.Traces.Sampler, + cfg.Traces.SamplerArg, + cfg.Resource.ServiceName, + ) + + return nil +} + +// ShutdownWithDefaultTimeout gracefully shuts down the global tracer provider with a default timeout. +// The default timeout is 5s. +func ShutdownWithDefaultTimeout() error { + ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout) + defer cancel() + return ShutdownOTel(ctx) +} + +// ShutdownOTel gracefully shuts down the global tracer provider, flushing any +// pending spans to the collector. +func ShutdownOTel(ctx context.Context) error { + globalTracerProviderMu.Lock() + tp := globalTracerProvider + setNoopOTelLocked() + globalTracerProviderMu.Unlock() + + return shutdownTracerProvider(ctx, tp) +} + +func shutdownTracerProvider(ctx context.Context, tp *sdktrace.TracerProvider) error { + if tp == nil { + return nil + } + + shutdownCtx, cancel := context.WithTimeout(ctx, shutdownTimeout) + defer cancel() + if err := tp.Shutdown(shutdownCtx); err != nil { + return fmt.Errorf("failed to shutdown OTel tracer provider: %w", err) + } + + logging.Info("OpenTelemetry tracer provider shut down successfully") + return nil +} + +func newTraceExporter(ctx context.Context, cfg ObservabilityConfig) (sdktrace.SpanExporter, error) { + parsedURL, err := url.Parse(cfg.Exporter.Endpoint) + if err != nil { + return nil, fmt.Errorf("failed to parse exporter endpoint: %w", err) + } + if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" { + return nil, fmt.Errorf("unsupported endpoint scheme %q: must be http or https", parsedURL.Scheme) + } + + switch cfg.Exporter.Protocol { + case "http/protobuf": + exporterOpts := []otlptracehttp.Option{ + otlptracehttp.WithEndpointURL(cfg.Exporter.Endpoint), + otlptracehttp.WithTimeout(exportTimeout), + } + return otlptracehttp.New(ctx, exporterOpts...) + case "grpc": + exporterOpts := []otlptracegrpc.Option{ + otlptracegrpc.WithEndpointURL(cfg.Exporter.Endpoint), + otlptracegrpc.WithTimeout(exportTimeout), + } + return otlptracegrpc.New(ctx, exporterOpts...) + default: + return nil, fmt.Errorf("unsupported OTLP protocol: %s", cfg.Exporter.Protocol) + } +} diff --git a/internal/common/observability/lifecycle_bootstrap_test.go b/internal/common/observability/lifecycle_bootstrap_test.go new file mode 100644 index 00000000000..bc96623f5fb --- /dev/null +++ b/internal/common/observability/lifecycle_bootstrap_test.go @@ -0,0 +1,150 @@ +package observability + +import ( + "context" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel" + + "github.com/armadaproject/armada/internal/lookout/version" +) + +func TestServiceBootstrapPatternServerConfig(t *testing.T) { + cfg := testBootstrapConfig("server", version.Version, uuid.New().String()) + require.NoError(t, cfg.Validate()) + require.Equal(t, "server", cfg.Resource.ServiceName) + require.NotEmpty(t, cfg.Resource.ServiceInstance) +} + +func TestServiceBootstrapPatternExecutorConfig(t *testing.T) { + cfg := testBootstrapConfig("executor", version.Version, uuid.New().String()) + require.NoError(t, cfg.Validate()) + require.Equal(t, "executor", cfg.Resource.ServiceName) +} + +func TestServiceBootstrapPatternSchedulerConfig(t *testing.T) { + cfg := testBootstrapConfig("scheduler", version.Version, uuid.New().String()) + require.NoError(t, cfg.Validate()) + require.Equal(t, "scheduler", cfg.Resource.ServiceName) +} + +func TestBootstrapWithDisabledOTel(t *testing.T) { + cfg := testBootstrapConfig("test-service", "test", uuid.New().String()) + require.False(t, cfg.Enabled) + + err := InitOTel(cfg) + require.NoError(t, err) + + tp := otel.GetTracerProvider() + tracer := tp.Tracer("test") + _, span := tracer.Start(context.Background(), "test-span") + defer span.End() + + spanCtx := span.SpanContext() + require.False(t, spanCtx.IsValid(), "Span should not be valid when OTel is disabled") + + err = ShutdownOTel(context.Background()) + require.NoError(t, err) +} + +func TestBootstrapWithEnabledOTelAndInvalidCollector(t *testing.T) { + cfg := testBootstrapConfig("test-service", "test", uuid.New().String()) + cfg.Enabled = true + cfg.Exporter.Endpoint = "http://localhost:19999" + require.NoError(t, cfg.Validate()) + require.True(t, cfg.Enabled) + + err := InitOTel(cfg) + require.NoError(t, err) + + tp := otel.GetTracerProvider() + require.NotNil(t, tp) + + tracer := tp.Tracer("test-bootstrap") + _, span := tracer.Start(context.Background(), "bootstrap-test-span") + + spanCtx := span.SpanContext() + require.True(t, spanCtx.IsValid(), "Span should be valid even with unreachable collector (fail-open)") + require.NotEmpty(t, spanCtx.TraceID().String()) + require.NotEmpty(t, spanCtx.SpanID().String()) + + span.End() + + err = ShutdownOTel(context.Background()) + require.NoError(t, err) +} + +func TestBootstrapResourceAttributesFromConfig(t *testing.T) { + cfg := ObservabilityConfig{ + Exporter: OTLPExporterConfig{ + Endpoint: "http://otel-collector:4318", + Protocol: testOTLPHTTPProtocol, + }, + Traces: TracesConfig{ + Sampler: testParentBasedTraceIDRatioSampler, + SamplerArg: 1.0, + }, + Resource: ResourceAttributes{ + ServiceName: "configured-service", + ServiceVersion: "configured-v1", + ServiceInstance: uuid.New().String(), + }, + } + require.NoError(t, cfg.Validate()) + require.Equal(t, "configured-service", cfg.Resource.ServiceName) + require.Equal(t, "configured-v1", cfg.Resource.ServiceVersion) +} + +func TestBootstrapInitShutdownMultipleTimes(t *testing.T) { + for i := 0; i < 3; i++ { + cfg := ObservabilityConfig{ + Enabled: true, + Exporter: OTLPExporterConfig{ + Endpoint: "http://localhost:19999", + Protocol: testOTLPHTTPProtocol, + }, + Traces: TracesConfig{ + Sampler: testParentBasedTraceIDRatioSampler, + SamplerArg: 1.0, + }, + Resource: ResourceAttributes{ + ServiceName: "test-service", + ServiceVersion: "test", + ServiceInstance: uuid.New().String(), + }, + } + require.NoError(t, cfg.Validate()) + + err := InitOTel(cfg) + require.NoError(t, err) + + tp := otel.GetTracerProvider() + tracer := tp.Tracer("test") + _, span := tracer.Start(context.Background(), "test-span") + require.True(t, span.SpanContext().IsValid()) + span.End() + + err = ShutdownOTel(context.Background()) + require.NoError(t, err) + } +} + +func testBootstrapConfig(serviceName, serviceVersion, serviceInstance string) ObservabilityConfig { + return ObservabilityConfig{ + Exporter: OTLPExporterConfig{ + Endpoint: "http://otel-collector:4318", + Protocol: testOTLPHTTPProtocol, + }, + Traces: TracesConfig{ + Sampler: testParentBasedTraceIDRatioSampler, + SamplerArg: 1.0, + }, + Resource: ResourceAttributes{ + ServiceName: serviceName, + ServiceVersion: serviceVersion, + ServiceInstance: serviceInstance, + }, + } +} diff --git a/internal/common/observability/lifecycle_test.go b/internal/common/observability/lifecycle_test.go new file mode 100644 index 00000000000..9d299bb0643 --- /dev/null +++ b/internal/common/observability/lifecycle_test.go @@ -0,0 +1,651 @@ +package observability + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel" + metricnoop "go.opentelemetry.io/otel/metric/noop" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + tracenoop "go.opentelemetry.io/otel/trace/noop" +) + +func TestOtelLifecycleCollectorReachable(t *testing.T) { + spanRecorder := tracetest.NewSpanRecorder() + exportedSpans := make(chan []sdktrace.ReadOnlySpan, 1) + + mockCollector := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + spans := spanRecorder.Ended() + if len(spans) > 0 { + exportedSpans <- spans + } + w.WriteHeader(http.StatusOK) + })) + defer mockCollector.Close() + + cfg := ObservabilityConfig{ + Enabled: true, + Exporter: OTLPExporterConfig{ + Endpoint: mockCollector.URL, + Protocol: testOTLPHTTPProtocol, + }, + Traces: TracesConfig{ + Sampler: testParentBasedTraceIDRatioSampler, + SamplerArg: 1.0, + }, + Resource: ResourceAttributes{ + ServiceName: "test-service", + ServiceVersion: "1.0.0", + ServiceInstance: "test-instance-1", + }, + } + + err := InitOTel(cfg) + require.NoError(t, err, "InitOTel should succeed with reachable collector") + + tp := otel.GetTracerProvider() + assert.NotNil(t, tp, "Global tracer provider should be set") + + tracer := tp.Tracer("test-tracer") + assert.NotNil(t, tracer, "Tracer should not be nil") + + ctx := context.Background() + _, span := tracer.Start(ctx, "test-span") + span.End() + + err = ShutdownOTel(context.Background()) + assert.NoError(t, err, "ShutdownOTel should succeed") +} + +func TestOtelAPIsAreSafeBeforeInit(t *testing.T) { + setNoopOTel() + + tracer := otel.Tracer("pre-init-tracer") + meter := otel.Meter("pre-init-meter") + require.NotNil(t, tracer) + require.NotNil(t, meter) + + _, span := tracer.Start(context.Background(), "pre-init-span") + require.NotPanics(t, func() { span.End() }) + + _, err := meter.Int64Counter("pre_init_counter") + require.NoError(t, err) +} + +func TestOtelLifecycleFailOpenWhenCollectorDown(t *testing.T) { + cfg := ObservabilityConfig{ + Enabled: true, + Exporter: OTLPExporterConfig{ + Endpoint: "http://localhost:19999", + Protocol: testOTLPHTTPProtocol, + }, + Traces: TracesConfig{ + Sampler: testParentBasedTraceIDRatioSampler, + SamplerArg: 1.0, + }, + Resource: ResourceAttributes{ + ServiceName: "test-service", + ServiceVersion: "1.0.0", + ServiceInstance: "test-instance-2", + }, + } + + err := InitOTel(cfg) + require.NoError(t, err, "InitOTel should succeed even when collector is unreachable (fail-open)") + + tp := otel.GetTracerProvider() + assert.NotNil(t, tp, "Global tracer provider should be set even with unreachable collector") + + tracer := tp.Tracer("test-tracer") + assert.NotNil(t, tracer, "Tracer should not be nil") + + ctx := context.Background() + _, span := tracer.Start(ctx, "test-span") + span.End() + + err = ShutdownOTel(context.Background()) + assert.NoError(t, err, "ShutdownOTel should succeed") +} + +func TestOtelShutdownFlushes(t *testing.T) { + mockCollector := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer mockCollector.Close() + + cfg := ObservabilityConfig{ + Enabled: true, + Exporter: OTLPExporterConfig{ + Endpoint: mockCollector.URL, + Protocol: testOTLPHTTPProtocol, + }, + Traces: TracesConfig{ + Sampler: "always_on", + SamplerArg: 1.0, + }, + Resource: ResourceAttributes{ + ServiceName: "test-service", + ServiceVersion: "1.0.0", + ServiceInstance: "test-instance-3", + }, + } + + err := InitOTel(cfg) + require.NoError(t, err) + + tracer := otel.GetTracerProvider().Tracer("test-tracer") + ctx := context.Background() + _, span := tracer.Start(ctx, "test-span") + span.End() + + start := time.Now() + err = ShutdownOTel(context.Background()) + elapsed := time.Since(start) + + assert.NoError(t, err, "ShutdownOTel should succeed") + assert.LessOrEqual(t, elapsed, shutdownTimeout+2*time.Second, "Shutdown should complete within timeout bounds") +} + +func TestOtelDisabledWhenConfigDisabled(t *testing.T) { + cfg := ObservabilityConfig{ + Enabled: false, + Exporter: OTLPExporterConfig{ + Endpoint: "http://localhost:4318", + Protocol: testOTLPHTTPProtocol, + }, + Traces: TracesConfig{ + Sampler: testParentBasedTraceIDRatioSampler, + SamplerArg: 1.0, + }, + Resource: ResourceAttributes{ + ServiceName: "test-service", + ServiceVersion: "1.0.0", + ServiceInstance: "test-instance-4", + }, + } + + err := InitOTel(cfg) + require.NoError(t, err, "InitOTel should succeed when disabled") + assert.IsType(t, tracenoop.TracerProvider{}, otel.GetTracerProvider()) + assert.IsType(t, metricnoop.MeterProvider{}, otel.GetMeterProvider()) + + err = ShutdownOTel(context.Background()) + assert.NoError(t, err, "ShutdownOTel should succeed when OTel was disabled") +} + +func TestSetNoopOTelShutsDownPreviousProvider(t *testing.T) { + processor := &shutdownObservingSpanProcessor{shutdown: make(chan struct{})} + tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(processor)) + + globalTracerProviderMu.Lock() + otel.SetTracerProvider(tp) + globalTracerProvider = tp + globalTracerProviderMu.Unlock() + + setNoopOTel() + + select { + case <-processor.shutdown: + case <-time.After(time.Second): + t.Fatal("setNoopOTel did not shut down the previous tracer provider") + } + assert.IsType(t, tracenoop.TracerProvider{}, otel.GetTracerProvider()) +} + +type shutdownObservingSpanProcessor struct { + shutdown chan struct{} +} + +func (p *shutdownObservingSpanProcessor) OnStart(parent context.Context, span sdktrace.ReadWriteSpan) { +} + +func (p *shutdownObservingSpanProcessor) OnEnd(span sdktrace.ReadOnlySpan) {} + +func (p *shutdownObservingSpanProcessor) Shutdown(ctx context.Context) error { + close(p.shutdown) + return nil +} + +func (p *shutdownObservingSpanProcessor) ForceFlush(ctx context.Context) error { + return nil +} + +func TestOtelShutdownResetsToNoopProviders(t *testing.T) { + mockCollector := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer mockCollector.Close() + + cfg := ObservabilityConfig{ + Enabled: true, + Exporter: OTLPExporterConfig{ + Endpoint: mockCollector.URL, + Protocol: testOTLPHTTPProtocol, + }, + Traces: TracesConfig{ + Sampler: testParentBasedTraceIDRatioSampler, + SamplerArg: 1.0, + }, + Resource: ResourceAttributes{ + ServiceName: "test-service", + ServiceVersion: "1.0.0", + ServiceInstance: "test-instance-noop-reset", + }, + } + + require.NoError(t, InitOTel(cfg)) + require.NoError(t, ShutdownOTel(context.Background())) + assert.IsType(t, tracenoop.TracerProvider{}, otel.GetTracerProvider()) + assert.IsType(t, metricnoop.MeterProvider{}, otel.GetMeterProvider()) +} + +func TestOtelPropagatorSetup(t *testing.T) { + mockCollector := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer mockCollector.Close() + + cfg := ObservabilityConfig{ + Enabled: true, + Exporter: OTLPExporterConfig{ + Endpoint: mockCollector.URL, + Protocol: testOTLPHTTPProtocol, + }, + Traces: TracesConfig{ + Sampler: testParentBasedTraceIDRatioSampler, + SamplerArg: 1.0, + }, + Resource: ResourceAttributes{ + ServiceName: "test-service", + ServiceVersion: "1.0.0", + ServiceInstance: "test-instance-5", + }, + } + + err := InitOTel(cfg) + require.NoError(t, err) + + propagator := otel.GetTextMapPropagator() + assert.NotNil(t, propagator, "Global propagator should be set") + + carrier := make(map[string]string) + ctx := context.Background() + + tracer := otel.GetTracerProvider().Tracer("test-tracer") + ctx, span := tracer.Start(ctx, "test-span") + defer span.End() + + propagator.Inject(ctx, &testCarrier{data: carrier}) + + assert.Contains(t, carrier, "traceparent", "Propagator should inject W3C traceparent header") + + err = ShutdownOTel(context.Background()) + assert.NoError(t, err) +} + +func TestOtelSamplerConfiguration(t *testing.T) { + tests := []struct { + name string + sampler string + samplerArg float64 + shouldFail bool + }{ + { + name: "parent_based_trace_id_ratio", + sampler: "parent_based_trace_id_ratio", + samplerArg: 0.5, + shouldFail: false, + }, + { + name: "always_on", + sampler: "always_on", + samplerArg: 1.0, + shouldFail: false, + }, + { + name: "always_off", + sampler: "always_off", + samplerArg: 0.0, + shouldFail: false, + }, + { + name: "trace_id_ratio", + sampler: "trace_id_ratio", + samplerArg: 0.1, + shouldFail: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockCollector := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer mockCollector.Close() + + cfg := ObservabilityConfig{ + Enabled: true, + Exporter: OTLPExporterConfig{ + Endpoint: mockCollector.URL, + Protocol: testOTLPHTTPProtocol, + }, + Traces: TracesConfig{ + Sampler: tt.sampler, + SamplerArg: tt.samplerArg, + }, + Resource: ResourceAttributes{ + ServiceName: "test-service", + ServiceVersion: "1.0.0", + ServiceInstance: "test-instance-sampler", + }, + } + + err := InitOTel(cfg) + if tt.shouldFail { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + + if err == nil { + tp := otel.GetTracerProvider() + assert.NotNil(t, tp) + _ = ShutdownOTel(context.Background()) + } + }) + } +} + +func TestSamplerMustBeConfigured(t *testing.T) { + cfg := ObservabilityConfig{ + Exporter: OTLPExporterConfig{ + Endpoint: "http://otel-collector:4318", + Protocol: testOTLPHTTPProtocol, + }, + Traces: TracesConfig{ + SamplerArg: 1.0, + }, + Resource: ResourceAttributes{ + ServiceName: "server", + ServiceVersion: "1.0.0", + ServiceInstance: "test-sampler-required", + }, + } + + err := cfg.Validate() + require.Error(t, err) + assert.ErrorContains(t, err, ConfigOtelTracesSampler) +} + +func TestSamplerOverrideByConfig(t *testing.T) { + tests := []struct { + name string + sampler string + samplerArg float64 + wantArg float64 + }{ + {name: "always_on", sampler: "always_on", samplerArg: 0.5, wantArg: 0.5}, + {name: "always_off", sampler: "always_off", samplerArg: 0.5, wantArg: 0.5}, + {name: "trace_id_ratio", sampler: "trace_id_ratio", samplerArg: 0.10, wantArg: 0.10}, + {name: "parent_based_trace_id_ratio", sampler: testParentBasedTraceIDRatioSampler, samplerArg: 0.25, wantArg: 0.25}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := ObservabilityConfig{ + Exporter: OTLPExporterConfig{ + Endpoint: "http://otel-collector:4318", + Protocol: testOTLPHTTPProtocol, + }, + Traces: TracesConfig{ + Sampler: tt.sampler, + SamplerArg: tt.samplerArg, + }, + Resource: ResourceAttributes{ + ServiceName: "server", + ServiceVersion: "1.0.0", + ServiceInstance: "test-overrides", + }, + } + require.NoError(t, cfg.Validate()) + + assert.Equal(t, tt.sampler, cfg.Traces.Sampler) + assert.Equal(t, tt.wantArg, cfg.Traces.SamplerArg) + }) + } +} + +func TestSamplingRatioEnforcesValidRange(t *testing.T) { + tests := []struct { + name string + sampler string + samplerArg float64 + shouldFail bool + }{ + {name: "trace_id_ratio below 0", sampler: "trace_id_ratio", samplerArg: -0.01, shouldFail: true}, + {name: "trace_id_ratio above 1", sampler: "trace_id_ratio", samplerArg: 1.01, shouldFail: true}, + {name: "parent_based_trace_id_ratio below 0", sampler: testParentBasedTraceIDRatioSampler, samplerArg: -0.5, shouldFail: true}, + {name: "parent_based_trace_id_ratio above 1", sampler: testParentBasedTraceIDRatioSampler, samplerArg: 2.0, shouldFail: true}, + {name: "trace_id_ratio at lower bound", sampler: "trace_id_ratio", samplerArg: 0.0, shouldFail: false}, + {name: "trace_id_ratio at upper bound", sampler: "trace_id_ratio", samplerArg: 1.0, shouldFail: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := ObservabilityConfig{ + Exporter: OTLPExporterConfig{ + Endpoint: "http://otel-collector:4318", + Protocol: testOTLPHTTPProtocol, + }, + Traces: TracesConfig{ + Sampler: tt.sampler, + SamplerArg: tt.samplerArg, + }, + Resource: ResourceAttributes{ + ServiceName: "server", + ServiceVersion: "1.0.0", + ServiceInstance: "test-ratio-range", + }, + } + err := cfg.Validate() + if tt.shouldFail { + require.Error(t, err) + assert.ErrorContains(t, err, ConfigOtelTracesSamplerArg) + return + } + + require.NoError(t, err) + assert.Equal(t, tt.sampler, cfg.Traces.Sampler) + assert.Equal(t, tt.samplerArg, cfg.Traces.SamplerArg) + }) + } +} + +func TestExporterBackpressureSafetyBounds(t *testing.T) { + assert.Equal(t, 512, maxExportBatch) + assert.Equal(t, 2048, maxBatchQueue) + assert.Equal(t, 10*time.Second, exportTimeout) + assert.Equal(t, 5*time.Second, batchTimeout) + + // Slow collector to induce exporter pressure without failing InitOTel. + mockCollector := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(200 * time.Millisecond) + w.WriteHeader(http.StatusOK) + })) + defer mockCollector.Close() + + cfg := ObservabilityConfig{ + Enabled: true, + Exporter: OTLPExporterConfig{ + Endpoint: mockCollector.URL, + Protocol: testOTLPHTTPProtocol, + }, + Traces: TracesConfig{ + Sampler: "always_on", + SamplerArg: 1.0, + }, + Resource: ResourceAttributes{ + ServiceName: "test-service", + ServiceVersion: "1.0.0", + ServiceInstance: "test-backpressure", + }, + } + + err := InitOTel(cfg) + require.NoError(t, err) + t.Cleanup(func() { + _ = ShutdownOTel(context.Background()) + }) + + tracer := otel.GetTracerProvider().Tracer("backpressure-test") + start := time.Now() + for range maxBatchQueue * 2 { + _, span := tracer.Start(context.Background(), "backpressure-span") + span.End() + } + elapsed := time.Since(start) + + // If queue is bounded and drop-on-backpressure is active, span creation/end should remain fast. + assert.Less(t, elapsed, 3*time.Second) +} + +type testCarrier struct { + data map[string]string +} + +func (c *testCarrier) Get(key string) string { + return c.data[key] +} + +func (c *testCarrier) Set(key, value string) { + c.data[key] = value +} + +func (c *testCarrier) Keys() []string { + keys := make([]string, 0, len(c.data)) + for k := range c.data { + keys = append(keys, k) + } + return keys +} + +func TestOtelMultipleInitShutdownCycles(t *testing.T) { + mockCollector := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer mockCollector.Close() + + cfg := ObservabilityConfig{ + Enabled: true, + Exporter: OTLPExporterConfig{ + Endpoint: mockCollector.URL, + Protocol: testOTLPHTTPProtocol, + }, + Traces: TracesConfig{ + Sampler: testParentBasedTraceIDRatioSampler, + SamplerArg: 1.0, + }, + Resource: ResourceAttributes{ + ServiceName: "test-service", + ServiceVersion: "1.0.0", + ServiceInstance: "test-instance-multi", + }, + } + + for i := range 3 { + err := InitOTel(cfg) + require.NoError(t, err, "InitOTel cycle %d should succeed", i) + + tracer := otel.GetTracerProvider().Tracer("test-tracer") + ctx := context.Background() + _, span := tracer.Start(ctx, "test-span") + span.End() + + err = ShutdownOTel(context.Background()) + assert.NoError(t, err, "ShutdownOTel cycle %d should succeed", i) + } +} + +func TestOtelInitWithInvalidResourceAttributes(t *testing.T) { + mockCollector := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer mockCollector.Close() + + cfg := ObservabilityConfig{ + Enabled: true, + Exporter: OTLPExporterConfig{ + Endpoint: mockCollector.URL, + Protocol: testOTLPHTTPProtocol, + }, + Traces: TracesConfig{ + Sampler: testParentBasedTraceIDRatioSampler, + SamplerArg: 1.0, + }, + Resource: ResourceAttributes{ + ServiceName: "", + ServiceVersion: "1.0.0", + ServiceInstance: "test-instance-invalid", + }, + } + + err := cfg.Validate() + assert.Error(t, err, "Config validation should fail with empty service name") +} + +func TestOtelShutdownWithoutInit(t *testing.T) { + setNoopOTel() + err := ShutdownOTel(context.Background()) + assert.NoError(t, err, "ShutdownOTel should handle nil provider gracefully") +} + +func TestOtelTracerProviderIsGloballySet(t *testing.T) { + mockCollector := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer mockCollector.Close() + + cfg := ObservabilityConfig{ + Enabled: true, + Exporter: OTLPExporterConfig{ + Endpoint: mockCollector.URL, + Protocol: testOTLPHTTPProtocol, + }, + Traces: TracesConfig{ + Sampler: testParentBasedTraceIDRatioSampler, + SamplerArg: 1.0, + }, + Resource: ResourceAttributes{ + ServiceName: "test-service", + ServiceVersion: "1.0.0", + ServiceInstance: "test-instance-global", + }, + } + + err := InitOTel(cfg) + require.NoError(t, err) + + tp := otel.GetTracerProvider() + _, ok := tp.(*sdktrace.TracerProvider) + assert.True(t, ok, "Global tracer provider should be of type *sdktrace.TracerProvider") + + tracer := tp.Tracer("integration-test") + ctx := context.Background() + _, span := tracer.Start(ctx, "global-span-test") + + spanContext := span.SpanContext() + assert.True(t, spanContext.IsValid(), "Span context should be valid") + assert.True(t, spanContext.TraceID().IsValid(), "Trace ID should be valid") + assert.True(t, spanContext.SpanID().IsValid(), "Span ID should be valid") + + span.End() + + err = ShutdownOTel(context.Background()) + assert.NoError(t, err) +} From 6e8e24262237d2db9d9ed5fc01ae11afa149a271 Mon Sep 17 00:00:00 2001 From: JamesMurkin Date: Fri, 3 Jul 2026 10:30:49 +0100 Subject: [PATCH 18/49] Add tgucks to Armada maintainers list (#4991) Signed-off-by: sarhiri --- .mergify.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.mergify.yml b/.mergify.yml index d47534f1a63..8e5ecda058a 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -26,5 +26,5 @@ pull_request_rules: - "#approved-reviews-by>=2" - and: - "#approved-reviews-by>=1" - - "author~=^(d80tb7|dave[-]gantenbein|dejanzele|eleanorpratt|geaere|JamesMurkin|mauriceyap|masipauskas|MustafaI|zuqq|richscott|robertdavidsmith|samclark|suprjinx|EnricoMi|nikola-jokic|j8169|sarhiri)$" + - "author~=^(d80tb7|dave[-]gantenbein|dejanzele|eleanorpratt|geaere|JamesMurkin|mauriceyap|masipauskas|MustafaI|zuqq|richscott|robertdavidsmith|samclark|suprjinx|EnricoMi|nikola-jokic|j8169|sarhiri|tgucks)$" title: Two approvals required, or one if author is a maintainer. From 6cc0ea27de6baa380fa294f200c562bcec273a46 Mon Sep 17 00:00:00 2001 From: Trey Guckian <24757349+tgucks@users.noreply.github.com> Date: Fri, 3 Jul 2026 05:06:00 -0500 Subject: [PATCH 19/49] Add flags to disable scheduling preemption by type (#4989) #### What type of PR is this? Feature #### What this PR does / why we need it Adds two new `PoolConfig` flags, `DisableFairshareScheduling` and `DisableUrgencyScheduling`, that let an operator turn off either strategy per pool. In `NodeDb.selectNodeForJobWithTxnAtPriority` each pass is now guarded by its flag, so a pool can run with one, both, or neither strategy enabled. As part of this, the per-setting `NodeDb` mutators (`DisableAwayScheduling`/`EnableAwayScheduling`, `DisableHomeScheduling`, `DisableGangAwayScheduling`, `SetDisallowedJobResources`, and their `Enable*` counterparts) are replaced by a single `ConfigureScheduling(SchedulingOptions)` call. The scheduling algo and submitcheck now configure the `NodeDb` in one shot, which removes the previous enable/disable toggling and the need to reset state between pools. #### Special notes for your reviewer - `ConfigureScheduling` sets every field from the options struct on each call, so it implicitly resets state between pools. This replaces the explicit `Enable*`/`Disable*` toggling that `submitcheck.go` did per synthetic pool - worth confirming that "set everything every time" is the intended semantics there. - Both flags default to `false`, so existing deployments keep both strategies enabled with no config change. --------- Signed-off-by: Trey Guckian <24757349+tgucks@users.noreply.github.com> Co-authored-by: JamesMurkin Signed-off-by: sarhiri --- .../scheduler/configuration/configuration.go | 2 + internal/scheduler/nodedb/nodedb.go | 78 ++++++------- internal/scheduler/nodedb/nodedb_test.go | 110 ++++++++++++++++-- .../scheduler/scheduling/scheduling_algo.go | 21 ++-- .../scheduling/scheduling_algo_test.go | 70 +++++++++++ internal/scheduler/submitcheck.go | 27 ++--- .../scheduler/testfixtures/testfixtures.go | 9 ++ 7 files changed, 235 insertions(+), 82 deletions(-) diff --git a/internal/scheduler/configuration/configuration.go b/internal/scheduler/configuration/configuration.go index 42fe76c2877..470d0e03f7f 100644 --- a/internal/scheduler/configuration/configuration.go +++ b/internal/scheduler/configuration/configuration.go @@ -409,6 +409,8 @@ type PoolConfig struct { DisableHomeScheduling bool DisableAwayScheduling bool DisableGangAwayScheduling bool + DisableFairshareScheduling bool + DisableUrgencyScheduling bool } func (p PoolConfig) GetSubmissionGroup() string { diff --git a/internal/scheduler/nodedb/nodedb.go b/internal/scheduler/nodedb/nodedb.go index c5b9c56c0ce..1b5de1b6478 100644 --- a/internal/scheduler/nodedb/nodedb.go +++ b/internal/scheduler/nodedb/nodedb.go @@ -154,9 +154,11 @@ type NodeDb struct { // it will not be scheduled onto any node regardless of if the nodes have enough resource disallowedJobResources []string - disableHomeScheduling bool - disableAwayScheduling bool - disableGangAwayScheduling bool + disableHomeScheduling bool + disableAwayScheduling bool + disableGangAwayScheduling bool + disableFairshareScheduling bool + disableUrgencyScheduling bool } func NewNodeDb( @@ -334,32 +336,22 @@ func (nodeDb *NodeDb) GetNodeWithTxn(txn *memdb.Txn, id string) (*internaltypes. return obj.(*internaltypes.Node), nil } -func (nodeDb *NodeDb) DisableAwayScheduling() { - nodeDb.disableAwayScheduling = true +type SchedulingOptions struct { + DisableHomeScheduling bool + DisableAwayScheduling bool + DisableGangAwayScheduling bool + DisableFairshareScheduling bool + DisableUrgencyScheduling bool + DisallowedJobResources []string } -func (nodeDb *NodeDb) EnableAwayScheduling() { - nodeDb.disableAwayScheduling = false -} - -func (nodeDb *NodeDb) DisableHomeScheduling() { - nodeDb.disableHomeScheduling = true -} - -func (nodeDb *NodeDb) EnableHomeScheduling() { - nodeDb.disableHomeScheduling = false -} - -func (nodeDb *NodeDb) DisableGangAwayScheduling() { - nodeDb.disableGangAwayScheduling = true -} - -func (nodeDb *NodeDb) EnableGangAwayScheduling() { - nodeDb.disableGangAwayScheduling = false -} - -func (nodeDb *NodeDb) SetDisallowedJobResources(resources []string) { - nodeDb.disallowedJobResources = resources +func (nodeDb *NodeDb) ConfigureScheduling(opts SchedulingOptions) { + nodeDb.disableHomeScheduling = opts.DisableHomeScheduling + nodeDb.disableAwayScheduling = opts.DisableAwayScheduling + nodeDb.disableGangAwayScheduling = opts.DisableGangAwayScheduling + nodeDb.disableFairshareScheduling = opts.DisableFairshareScheduling + nodeDb.disableUrgencyScheduling = opts.DisableUrgencyScheduling + nodeDb.disallowedJobResources = opts.DisallowedJobResources } func (nodeDb *NodeDb) GetNodes() ([]*internaltypes.Node, error) { @@ -639,13 +631,15 @@ func (nodeDb *NodeDb) selectNodeForJobWithTxnAtPriority( // Schedule by preventing evicted jobs from being re-scheduled. // This method respect fairness by preventing from re-scheduling jobs that appear as far back in the total order as possible. - if node, err := nodeDb.selectNodeForJobWithFairPreemption(txn, jctx); err != nil { - return nil, err - } else if err := assertPodSchedulingContextNode(pctx, node); err != nil { - return nil, err - } else if node != nil { - pctx.SchedulingMethod = context.ScheduledWithFairSharePreemption - return node, nil + if !nodeDb.disableFairshareScheduling { + if node, err := nodeDb.selectNodeForJobWithFairPreemption(txn, jctx); err != nil { + return nil, err + } else if err := assertPodSchedulingContextNode(pctx, node); err != nil { + return nil, err + } else if node != nil { + pctx.SchedulingMethod = context.ScheduledWithFairSharePreemption + return node, nil + } } pctx.NodeId = "" @@ -653,13 +647,15 @@ func (nodeDb *NodeDb) selectNodeForJobWithTxnAtPriority( // Schedule by kicking off jobs currently bound to a node. // This method does not respect fairness when choosing on which node to schedule the job. - if node, err := nodeDb.selectNodeForJobWithUrgencyPreemption(txn, jctx, matchingNodeTypeIds); err != nil { - return nil, err - } else if err := assertPodSchedulingContextNode(pctx, node); err != nil { - return nil, err - } else if node != nil { - pctx.SchedulingMethod = context.ScheduledWithUrgencyBasedPreemption - return node, nil + if !nodeDb.disableUrgencyScheduling { + if node, err := nodeDb.selectNodeForJobWithUrgencyPreemption(txn, jctx, matchingNodeTypeIds); err != nil { + return nil, err + } else if err := assertPodSchedulingContextNode(pctx, node); err != nil { + return nil, err + } else if node != nil { + pctx.SchedulingMethod = context.ScheduledWithUrgencyBasedPreemption + return node, nil + } } return nil, nil diff --git a/internal/scheduler/nodedb/nodedb_test.go b/internal/scheduler/nodedb/nodedb_test.go index 43510a25298..fb2a8229bcb 100644 --- a/internal/scheduler/nodedb/nodedb_test.go +++ b/internal/scheduler/nodedb/nodedb_test.go @@ -704,7 +704,7 @@ func TestDisallowedJobResources(t *testing.T) { } nodeDbTxn := nodeDb.Txn(true) - nodeDb.SetDisallowedJobResources(tc.disallowedJobResources) + nodeDb.ConfigureScheduling(SchedulingOptions{DisallowedJobResources: tc.disallowedJobResources}) node, err := nodeDb.SelectNodeForJobWithTxn(nodeDbTxn, jctx) require.NoError(t, err) @@ -767,7 +767,7 @@ func TestHomeNodeScheduling(t *testing.T) { ) require.NoError(t, err) if tc.disableHomeScheduling { - nodeDb.DisableHomeScheduling() + nodeDb.ConfigureScheduling(SchedulingOptions{DisableHomeScheduling: true}) } nodeDbTxn := nodeDb.Txn(true) @@ -820,6 +820,101 @@ func TestHomeNodeScheduling(t *testing.T) { } } +func TestPreemptionScheduling(t *testing.T) { + tests := map[string]struct { + registerEvictedJobs bool + disableFairshareScheduling bool + disableUrgencyScheduling bool + expectSuccess bool + expectedSchedulingMethod context.SchedulingType + }{ + "urgency-based preemption by default": { + expectSuccess: true, + expectedSchedulingMethod: context.ScheduledWithUrgencyBasedPreemption, + }, + "no urgency-based preemption when disabled": { + disableUrgencyScheduling: true, + expectSuccess: false, + }, + "fair-share preemption by default": { + registerEvictedJobs: true, + expectSuccess: true, + expectedSchedulingMethod: context.ScheduledWithFairSharePreemption, + }, + "falls through to urgency-based preemption when fair-share disabled": { + registerEvictedJobs: true, + disableFairshareScheduling: true, + expectSuccess: true, + expectedSchedulingMethod: context.ScheduledWithUrgencyBasedPreemption, + }, + "fair-share preemption when urgency disabled": { + registerEvictedJobs: true, + disableUrgencyScheduling: true, + expectSuccess: true, + expectedSchedulingMethod: context.ScheduledWithFairSharePreemption, + }, + "no preemption when both strategies disabled": { + registerEvictedJobs: true, + disableFairshareScheduling: true, + disableUrgencyScheduling: true, + expectSuccess: false, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + nodeDb, err := newNodeDbWithNodes(nil) + require.NoError(t, err) + nodeDb.ConfigureScheduling(SchedulingOptions{ + DisableFairshareScheduling: tc.disableFairshareScheduling, + DisableUrgencyScheduling: tc.disableUrgencyScheduling, + }) + + // Fully allocate the node with low-priority jobs. + txn := nodeDb.Txn(true) + node := testfixtures.Test32CpuNode(testfixtures.TestPriorities) + boundJobs := testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 32) + require.NoError(t, nodeDb.CreateAndInsertWithJobDbJobsWithTxn(txn, boundJobs, node)) + txn.Commit() + + if tc.registerEvictedJobs { + node, err = nodeDb.GetNode(node.GetId()) + require.NoError(t, err) + evictedNode, err := nodeDb.EvictJobsFromNode(boundJobs, node) + require.NoError(t, err) + + txn = nodeDb.Txn(true) + require.NoError(t, nodeDb.UpsertWithTxn(txn, evictedNode)) + for i, job := range boundJobs { + evictedJctx := context.JobSchedulingContextFromJob(job) + evictedJctx.SetAssignedNode(evictedNode) + require.NoError(t, nodeDb.AddEvictedJobSchedulingContextWithTxn(txn, i, evictedJctx)) + } + txn.Commit() + } + + incoming := testfixtures.N1Cpu4GiJobs("B", testfixtures.PriorityClass1, 1)[0] + jctx := context.JobSchedulingContextFromJob(incoming) + gctx := context.NewGangSchedulingContext([]*context.JobSchedulingContext{jctx}) + + txn = nodeDb.Txn(true) + ok, err := nodeDb.ScheduleManyWithTxn(txn, gctx) + require.NoError(t, err) + + require.Equal(t, tc.expectSuccess, ok) + require.NotNil(t, jctx.PodSchedulingContext) + if tc.expectSuccess { + assert.True(t, jctx.PodSchedulingContext.IsSuccessful()) + assert.Equal(t, node.GetId(), jctx.PodSchedulingContext.NodeId) + assert.Equal(t, tc.expectedSchedulingMethod, jctx.PodSchedulingContext.SchedulingMethod) + } else { + assert.False(t, jctx.PodSchedulingContext.IsSuccessful()) + assert.Empty(t, jctx.PodSchedulingContext.NodeId) + } + }) + } +} + func TestMatchesConditions(t *testing.T) { cpu0 := resource.MustParse("0") cpu2 := resource.MustParse("2") @@ -1041,13 +1136,10 @@ func TestAwayNodeScheduling(t *testing.T) { testfixtures.TestResourceListFactory, ) require.NoError(t, err) - if tc.disableAwayScheduling { - nodeDb.DisableAwayScheduling() - } - - if tc.disableGangAwayScheduling { - nodeDb.DisableGangAwayScheduling() - } + nodeDb.ConfigureScheduling(SchedulingOptions{ + DisableAwayScheduling: tc.disableAwayScheduling, + DisableGangAwayScheduling: tc.disableGangAwayScheduling, + }) nodeDbTxn := nodeDb.Txn(true) node := testfixtures.Test32CpuNode([]int32{29000, 30000}) diff --git a/internal/scheduler/scheduling/scheduling_algo.go b/internal/scheduler/scheduling/scheduling_algo.go index 130e476e3f1..c9f4cf5b1a5 100644 --- a/internal/scheduler/scheduling/scheduling_algo.go +++ b/internal/scheduler/scheduling/scheduling_algo.go @@ -235,19 +235,14 @@ func (l *FairSchedulingAlgo) runPoolSchedulingRound( }, nil } - if pool.DisableAwayScheduling { - fsctx.nodeDb.DisableAwayScheduling() - } - - if pool.DisableHomeScheduling { - fsctx.nodeDb.DisableHomeScheduling() - } - - if pool.DisableGangAwayScheduling { - fsctx.nodeDb.DisableGangAwayScheduling() - } - - fsctx.nodeDb.SetDisallowedJobResources(pool.ExperimentalUnscheduledResources) + fsctx.nodeDb.ConfigureScheduling(nodedb.SchedulingOptions{ + DisableHomeScheduling: pool.DisableHomeScheduling, + DisableAwayScheduling: pool.DisableAwayScheduling, + DisableGangAwayScheduling: pool.DisableGangAwayScheduling, + DisableFairshareScheduling: pool.DisableFairshareScheduling, + DisableUrgencyScheduling: pool.DisableUrgencyScheduling, + DisallowedJobResources: pool.ExperimentalUnscheduledResources, + }) start := time.Now() schedulingResult, sctx, err := l.SchedulePool(ctx, fsctx, pool) diff --git a/internal/scheduler/scheduling/scheduling_algo_test.go b/internal/scheduler/scheduling/scheduling_algo_test.go index 4bbdc464499..f204c900d6c 100644 --- a/internal/scheduler/scheduling/scheduling_algo_test.go +++ b/internal/scheduler/scheduling/scheduling_algo_test.go @@ -27,6 +27,7 @@ import ( "github.com/armadaproject/armada/internal/scheduler/priorityoverride" "github.com/armadaproject/armada/internal/scheduler/reports" "github.com/armadaproject/armada/internal/scheduler/schedulerobjects" + schedulercontext "github.com/armadaproject/armada/internal/scheduler/scheduling/context" "github.com/armadaproject/armada/internal/scheduler/testfixtures" "github.com/armadaproject/armada/pkg/api" ) @@ -299,6 +300,8 @@ func TestSchedule(t *testing.T) { expectedScheduledIndices []int // Number of jobs expected to be scheduled by pool expectedScheduledByPool map[string]int + // If set, at least one scheduled job is expected to have been placed using this scheduling method. + expectedSchedulingMethod schedulercontext.SchedulingType }{ "scheduling": { schedulingConfig: testfixtures.TestSchedulingConfig(), @@ -748,6 +751,64 @@ func TestSchedule(t *testing.T) { }, expectedScheduledIndices: []int{0}, }, + "fair-share preemption still applies when only urgency-based preemption disabled": { + schedulingConfig: testfixtures.WithPreemptionDisabled(false, true, testfixtures.TestSchedulingConfig()), + executors: []*schedulerobjects.Executor{test1Node32CoreExecutor("executor1")}, + queues: []*api.Queue{{Name: "A", PriorityFactor: 0.01}, {Name: "B", PriorityFactor: 0.01}}, + queuedJobs: testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 2), + scheduledJobsByExecutorIndexAndNodeIndex: map[int]map[int]scheduledJobs{ + 0: { + 0: scheduledJobs{ + jobs: testfixtures.N16Cpu128GiJobs("B", testfixtures.PriorityClass0, 2), + acknowledged: true, + }, + }, + }, + expectedPreemptedJobIndicesByExecutorIndexAndNodeIndex: map[int]map[int][]int{ + 0: { + 0: {1}, + }, + }, + expectedScheduledIndices: []int{0}, + expectedSchedulingMethod: schedulercontext.ScheduledWithFairSharePreemption, + }, + "urgency-based preemption still applies when only fair-share preemption disabled": { + schedulingConfig: testfixtures.WithPreemptionDisabled(true, false, testfixtures.TestSchedulingConfig()), + executors: []*schedulerobjects.Executor{test1Node32CoreExecutor("executor1")}, + queues: []*api.Queue{{Name: "A"}}, + queuedJobs: testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass1, 2), + scheduledJobsByExecutorIndexAndNodeIndex: map[int]map[int]scheduledJobs{ + 0: { + 0: scheduledJobs{ + jobs: testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1), + acknowledged: true, + }, + }, + }, + expectedPreemptedJobIndicesByExecutorIndexAndNodeIndex: map[int]map[int][]int{ + 0: { + 0: {0}, + }, + }, + expectedScheduledIndices: []int{0, 1}, + expectedSchedulingMethod: schedulercontext.ScheduledWithUrgencyBasedPreemption, + }, + "no preemption when both strategies disabled": { + schedulingConfig: testfixtures.WithPreemptionDisabled(true, true, testfixtures.TestSchedulingConfig()), + executors: []*schedulerobjects.Executor{test1Node32CoreExecutor("executor1")}, + queues: []*api.Queue{{Name: "A", PriorityFactor: 0.01}, {Name: "B", PriorityFactor: 0.01}}, + queuedJobs: testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 2), + scheduledJobsByExecutorIndexAndNodeIndex: map[int]map[int]scheduledJobs{ + 0: { + 0: scheduledJobs{ + jobs: testfixtures.N16Cpu128GiJobs("B", testfixtures.PriorityClass0, 2), + acknowledged: true, + }, + }, + }, + expectedPreemptedJobIndicesByExecutorIndexAndNodeIndex: map[int]map[int][]int{}, + expectedScheduledIndices: []int{}, + }, "gang scheduling successful": { schedulingConfig: testfixtures.TestSchedulingConfig(), executors: []*schedulerobjects.Executor{test1Node32CoreExecutor("executor1")}, @@ -1048,6 +1109,15 @@ func TestSchedule(t *testing.T) { assert.Len(t, jobsSchedulerOnPool, expectedScheduledCount) } + if tc.expectedSchedulingMethod != "" { + actualSchedulingMethods := make([]schedulercontext.SchedulingType, 0) + for _, jctx := range schedulerResult.GetAllScheduledJobs() { + require.NotNil(t, jctx.PodSchedulingContext) + actualSchedulingMethods = append(actualSchedulingMethods, jctx.PodSchedulingContext.SchedulingMethod) + } + assert.Contains(t, actualSchedulingMethods, tc.expectedSchedulingMethod) + } + // Check that preempted jobs are marked as such consistently. for _, job := range preemptedJobs { dbJob := txn.GetById(job.Id()) diff --git a/internal/scheduler/submitcheck.go b/internal/scheduler/submitcheck.go index e813cff705e..c6ab4211285 100644 --- a/internal/scheduler/submitcheck.go +++ b/internal/scheduler/submitcheck.go @@ -350,25 +350,14 @@ poolStart: gctx := copyGangContext(originalGangCtx) // TODO construct nodedb per synthetic pool to avoid needing to set this dynamically - if pool.DisableAwayScheduling { - ex.nodeDb.DisableAwayScheduling() - } else { - ex.nodeDb.EnableAwayScheduling() - } - - if pool.DisableHomeScheduling { - ex.nodeDb.DisableHomeScheduling() - } else { - ex.nodeDb.EnableHomeScheduling() - } - - if pool.DisableGangAwayScheduling { - ex.nodeDb.DisableGangAwayScheduling() - } else { - ex.nodeDb.EnableGangAwayScheduling() - } - - ex.nodeDb.SetDisallowedJobResources(pool.ExperimentalUnscheduledResources) + ex.nodeDb.ConfigureScheduling(nodedb.SchedulingOptions{ + DisableHomeScheduling: pool.DisableHomeScheduling, + DisableAwayScheduling: pool.DisableAwayScheduling, + DisableGangAwayScheduling: pool.DisableGangAwayScheduling, + DisableFairshareScheduling: pool.DisableFairshareScheduling, + DisableUrgencyScheduling: pool.DisableUrgencyScheduling, + DisallowedJobResources: pool.ExperimentalUnscheduledResources, + }) txn := ex.nodeDb.Txn(true) ok, err := ex.nodeDb.ScheduleManyWithTxn(txn, gctx) diff --git a/internal/scheduler/testfixtures/testfixtures.go b/internal/scheduler/testfixtures/testfixtures.go index b61a023fa5f..b43b0c05140 100644 --- a/internal/scheduler/testfixtures/testfixtures.go +++ b/internal/scheduler/testfixtures/testfixtures.go @@ -285,6 +285,15 @@ func WithGangAwaySchedulingDisabled(config schedulerconfiguration.SchedulingConf return config } +func WithPreemptionDisabled(disableFairshareScheduling bool, disableUrgencyScheduling bool, config schedulerconfiguration.SchedulingConfig) schedulerconfiguration.SchedulingConfig { + for i, pool := range config.Pools { + pool.DisableFairshareScheduling = disableFairshareScheduling + pool.DisableUrgencyScheduling = disableUrgencyScheduling + config.Pools[i] = pool + } + return config +} + func WithMarketBasedSchedulingEnabled(config schedulerconfiguration.SchedulingConfig) schedulerconfiguration.SchedulingConfig { for i, pool := range config.Pools { pool.ExperimentalMarketScheduling = &schedulerconfiguration.MarketSchedulingConfig{ From 346b6e244cb73e4a65e695ee7091f200b64f19b0 Mon Sep 17 00:00:00 2001 From: Trey Guckian <24757349+tgucks@users.noreply.github.com> Date: Fri, 3 Jul 2026 08:32:25 -0500 Subject: [PATCH 20/49] feat(scheduler): Extract short job penalty service (#4969) #### What type of PR is this? Feature/refactor #### What this PR does / why we need it Extracts short-job-penalty tracking out of the scheduling hot path and the jobDb retention logic into a dedicated, self-contained ShortJobPenalty service. Previously the penalty was recomputed every scheduling cycle by scanning all terminal jobs held in the jobDb. To make those jobs available for the scan, terminal short jobs were deliberately kept in the jobDb while their penalty was active. This required an occasional full GC to clean them up periodically. This PR makes ShortJobPenalty own its own state: - Terminal jobs are reported to the service once each via `ReportFinishedJob` at each point where a job can go terminal. This records the job's resources keyed by (pool, queue). - Penalties are snapshotted once per scheduling cycle via `Snapshot()` and read back per-pool from that immutable `ShortJobPenaltySnapshot` via `GetPenaltiesForPool`, replacing the inline per-job `ShouldApplyPenalty` accumulation in `calculateJobSchedulingInfo.` - Entries expire automatically via a deadline-ordered min-heap (`runStart + cutoff[pool]`), with a derived per-(pool, queue) running total cache kept in sync as entries are added and expired. Access is guarded by a mutex. This removes the need for the periodic full GC of terminal jobs from the jobDb. They're deleted as they go terminal now. #### Special notes for your reviewer - `ShortJobPenalty` is now stateful and should be concurrency-safe (sync.Mutex); the entries are the source of truth and sums is a derived cache. A `Snapshot()` taken once per cycle gives every pool a consistent point-in-time view. - `syncState`'s signature changed (dropped the `fullJobGc` bool); call sites in cycle and initialize were updated accordingly. - Tests in short_job_penalty_test.go were substantially expanded to cover reporting, dedup, expiry, and per-pool reads. --------- Signed-off-by: Trey Guckian <24757349+tgucks@users.noreply.github.com> Co-authored-by: JamesMurkin Signed-off-by: sarhiri --- internal/scheduler/jobdb/jobdb.go | 49 +-- internal/scheduler/jobdb/jobdb_test.go | 62 --- internal/scheduler/scheduler.go | 35 +- internal/scheduler/scheduler_test.go | 187 +++++++-- .../scheduler/scheduling/scheduling_algo.go | 29 +- .../scheduler/scheduling/short_job_penalty.go | 103 ++++- .../scheduling/short_job_penalty_test.go | 375 +++++++++++++++--- .../scheduling/short_job_penalty_types.go | 69 ++++ 8 files changed, 664 insertions(+), 245 deletions(-) create mode 100644 internal/scheduler/scheduling/short_job_penalty_types.go diff --git a/internal/scheduler/jobdb/jobdb.go b/internal/scheduler/jobdb/jobdb.go index 220a0c56bd3..c95c1425eb8 100644 --- a/internal/scheduler/jobdb/jobdb.go +++ b/internal/scheduler/jobdb/jobdb.go @@ -72,7 +72,6 @@ type JobDb struct { jobsByQueue map[string]immutable.SortedSet[*Job] jobsByPoolAndQueue map[string]map[string]immutable.SortedSet[*Job] leasedJobs *immutable.Set[*Job] - terminalJobs *immutable.Set[*Job] unvalidatedJobs *immutable.Set[*Job] // Configured priority classes. priorityClasses map[string]types.PriorityClass @@ -137,7 +136,6 @@ func NewJobDbWithSchedulingKeyGenerator( } unvalidatedJobs := immutable.NewSet[*Job](JobHasher{}) leasedJobs := immutable.NewSet[*Job](JobHasher{}) - terminalJobs := immutable.NewSet[*Job](JobHasher{}) return &JobDb{ jobsById: immutable.NewMap[string, *Job](nil), jobsByRunId: immutable.NewMap[string, string](nil), @@ -145,7 +143,6 @@ func NewJobDbWithSchedulingKeyGenerator( jobsByQueue: map[string]immutable.SortedSet[*Job]{}, jobsByPoolAndQueue: map[string]map[string]immutable.SortedSet[*Job]{}, leasedJobs: &leasedJobs, - terminalJobs: &terminalJobs, unvalidatedJobs: &unvalidatedJobs, priorityClasses: priorityClasses, defaultPriorityClass: defaultPriorityClass, @@ -178,7 +175,6 @@ func (jobDb *JobDb) Clone() *JobDb { jobsByQueue: maps.Clone(jobDb.jobsByQueue), jobsByPoolAndQueue: deepClone(jobDb.jobsByPoolAndQueue), leasedJobs: jobDb.leasedJobs, - terminalJobs: jobDb.terminalJobs, unvalidatedJobs: jobDb.unvalidatedJobs, priorityClasses: jobDb.priorityClasses, defaultPriorityClass: jobDb.defaultPriorityClass, @@ -353,7 +349,6 @@ func (jobDb *JobDb) ReadTxn() *Txn { jobsByQueue: jobDb.jobsByQueue, jobsByPoolAndQueue: jobDb.jobsByPoolAndQueue, leasedJobs: jobDb.leasedJobs, - terminalJobs: jobDb.terminalJobs, unvalidatedJobs: jobDb.unvalidatedJobs, bidPriceSnapshot: jobDb.bidPriceSnapshot, active: true, @@ -376,7 +371,6 @@ func (jobDb *JobDb) WriteTxn() *Txn { jobsByQueue: maps.Clone(jobDb.jobsByQueue), jobsByPoolAndQueue: deepClone(jobDb.jobsByPoolAndQueue), leasedJobs: jobDb.leasedJobs, - terminalJobs: jobDb.terminalJobs, unvalidatedJobs: jobDb.unvalidatedJobs, bidPriceSnapshot: jobDb.bidPriceSnapshot, active: true, @@ -399,7 +393,6 @@ func (jobDb *JobDb) DryRunTxn() *Txn { jobsByQueue: maps.Clone(jobDb.jobsByQueue), jobsByPoolAndQueue: deepClone(jobDb.jobsByPoolAndQueue), leasedJobs: jobDb.leasedJobs, - terminalJobs: jobDb.terminalJobs, unvalidatedJobs: jobDb.unvalidatedJobs, bidPriceSnapshot: jobDb.bidPriceSnapshot, active: true, @@ -444,8 +437,6 @@ type Txn struct { jobsByPoolAndQueue map[string]map[string]immutable.SortedSet[*Job] // Jobs that are currently leased leasedJobs *immutable.Set[*Job] - // Jobs that are currently in a terminal state - terminalJobs *immutable.Set[*Job] // Jobs that require submit checking unvalidatedJobs *immutable.Set[*Job] // The current snapshot of bid prices - allowing look up of bidding prices on job creation @@ -473,7 +464,6 @@ func (txn *Txn) Commit() { txn.jobDb.jobsByQueue = txn.jobsByQueue txn.jobDb.jobsByPoolAndQueue = txn.jobsByPoolAndQueue txn.jobDb.leasedJobs = txn.leasedJobs - txn.jobDb.terminalJobs = txn.terminalJobs txn.jobDb.unvalidatedJobs = txn.unvalidatedJobs txn.jobDb.bidPriceSnapshot = txn.bidPriceSnapshot @@ -614,11 +604,6 @@ func (txn *Txn) Upsert(jobs []*Job) error { txn.leasedJobs = &newLeasedJobs } - if existingJob.InTerminalState() { - newTerminalJobs := txn.terminalJobs.Delete(existingJob) - txn.terminalJobs = &newTerminalJobs - } - if !existingJob.Validated() { newUnvalidatedJobs := txn.unvalidatedJobs.Delete(existingJob) txn.unvalidatedJobs = &newUnvalidatedJobs @@ -629,7 +614,7 @@ func (txn *Txn) Upsert(jobs []*Job) error { // Now need to insert jobs, runs and queuedJobs. This can be done in parallel. wg := sync.WaitGroup{} - wg.Add(7) + wg.Add(6) // jobs go func() { @@ -792,30 +777,6 @@ func (txn *Txn) Upsert(jobs []*Job) error { } }() - // Terminal jobs - go func() { - defer wg.Done() - if hasJobs { - for _, job := range jobs { - if job.InTerminalState() { - terminalJobs := txn.terminalJobs.Add(job) - txn.terminalJobs = &terminalJobs - } - } - } else { - terminalJobs := map[*Job]bool{} - - for _, job := range jobs { - if job.InTerminalState() { - terminalJobs[job] = true - } - } - - terminalJobsImmutable := immutable.NewSet[*Job](JobHasher{}, maps.Keys(terminalJobs)...) - txn.terminalJobs = &terminalJobsImmutable - } - }() - // Unvalidated jobs go func() { defer wg.Done() @@ -957,11 +918,6 @@ func (txn *Txn) GetAllLeasedJobs() []*Job { return txn.leasedJobs.Items() } -// GetAllTerminalJobs returns all terminal jobs in the database -func (txn *Txn) GetAllTerminalJobs() []*Job { - return txn.terminalJobs.Items() -} - // GetAll returns all jobs in the database. func (txn *Txn) GetAll() []*Job { allJobs := make([]*Job, 0, txn.jobsById.Len()) @@ -1034,9 +990,6 @@ func (txn *Txn) delete(jobId string) { newLeasedJobs := txn.leasedJobs.Delete(job) txn.leasedJobs = &newLeasedJobs - newTerminalJobs := txn.terminalJobs.Delete(job) - txn.terminalJobs = &newTerminalJobs - newUnvalidatedJobs := txn.unvalidatedJobs.Delete(job) txn.unvalidatedJobs = &newUnvalidatedJobs } diff --git a/internal/scheduler/jobdb/jobdb_test.go b/internal/scheduler/jobdb/jobdb_test.go index d763e559ce5..defd59081f9 100644 --- a/internal/scheduler/jobdb/jobdb_test.go +++ b/internal/scheduler/jobdb/jobdb_test.go @@ -147,68 +147,6 @@ func TestJobDb_LeasedJobs_Deleted(t *testing.T) { assert.Empty(t, txn.GetAllLeasedJobs()) } -func TestJobDb_TestGetTerminalJobs(t *testing.T) { - jobDb := NewTestJobDb() - job1 := newJob().WithQueued(false).WithNewRun("executor", "nodeId", "nodeName", "pool", 5) - job2 := newJob().WithQueued(true) - job3 := newJob().WithQueued(false).WithSucceeded(true) - job4 := newJob().WithQueued(false).WithCancelled(true) - job5 := newJob().WithQueued(false).WithFailed(true) - job6 := newJob().WithQueued(true).WithFailed(true) - txn := jobDb.WriteTxn() - - err := txn.Upsert([]*Job{job1, job2, job3, job4, job5, job6}) - require.NoError(t, err) - - expected := []*Job{job3, job4, job5, job6} - actual := txn.GetAllTerminalJobs() - sort.SliceStable(actual, func(i, j int) bool { return actual[i].id < actual[j].id }) - sort.SliceStable(expected, func(i, j int) bool { return expected[i].id < expected[j].id }) - assert.Equal(t, expected, actual) -} - -func TestJobDb_TerminalJobs_Lifecycle(t *testing.T) { - jobDb := NewTestJobDb() - - upsert := func(jobDb *JobDb, job *Job) { - txn := jobDb.WriteTxn() - err := txn.Upsert([]*Job{job}) - require.NoError(t, err) - txn.Commit() - } - - job1 := newJob().WithQueued(true) - upsert(jobDb, job1) - assert.Empty(t, jobDb.ReadTxn().GetAllTerminalJobs()) - - // leased - job1 = job1.WithQueued(false).WithNewRun("executor", "nodeId", "nodeName", "pool", 5) - upsert(jobDb, job1) - assert.Empty(t, jobDb.ReadTxn().GetAllTerminalJobs()) - - // finished - job1 = job1.WithSucceeded(true) - upsert(jobDb, job1) - assert.NotEmpty(t, jobDb.ReadTxn().GetAllTerminalJobs()) -} - -func TestJobDb_TerminalJobs_Deleted(t *testing.T) { - jobDb := NewTestJobDb() - job1 := newJob().WithFailed(true) - txn := jobDb.WriteTxn() - - err := txn.Upsert([]*Job{job1}) - require.NoError(t, err) - - expected := []*Job{job1} - actual := txn.GetAllTerminalJobs() - assert.Equal(t, expected, actual) - - err = txn.BatchDelete([]string{job1.Id()}) - require.NoError(t, err) - assert.Empty(t, txn.GetAllTerminalJobs()) -} - func TestJobDb_TestGetUnvalidated(t *testing.T) { jobDb := NewTestJobDb() job1 := newJob().WithValidated(false) diff --git a/internal/scheduler/scheduler.go b/internal/scheduler/scheduler.go index 66875ef8aa7..2c8df7d0113 100644 --- a/internal/scheduler/scheduler.go +++ b/internal/scheduler/scheduler.go @@ -285,7 +285,7 @@ func (s *Scheduler) cycle(ctx *armadacontext.Context, updateAll bool, leaderToke }(ctx) // Update job state. ctx.Info("Syncing internal state with database") - updatedJobs, jsts, newJobsSerial, newRunsSerial, err := s.syncState(ctx, false, cycleNumber%10 == 0) + updatedJobs, jsts, newJobsSerial, newRunsSerial, err := s.syncState(ctx, false) if err != nil { return false, err } @@ -438,7 +438,7 @@ func (s *Scheduler) cycle(ctx *armadacontext.Context, updateAll bool, leaderToke // syncState updates jobs in jobDb to match state in postgres and returns all updated jobs along with // the new jobsSerial and runsSerial cursor values that should be applied once the resulting events // have been published successfully. -func (s *Scheduler) syncState(ctx *armadacontext.Context, initial, fullJobGc bool) ([]*jobdb.Job, []jobdb.JobStateTransitions, int64, int64, error) { +func (s *Scheduler) syncState(ctx *armadacontext.Context, initial bool) ([]*jobdb.Job, []jobdb.JobStateTransitions, int64, int64, error) { txn := s.jobDb.WriteTxn() defer txn.Abort() @@ -501,25 +501,12 @@ func (s *Scheduler) syncState(ctx *armadacontext.Context, initial, fullJobGc boo // Delete jobs in a terminal state. idsOfJobsToDelete := make([]string, 0) - deletionCandidates := jobDbJobs - if fullJobGc { - // Occasional full gc so jobs that were not deleted - // earlier as ShortJobPenalty was being applied - // eventually get deleted. - deletionCandidates = txn.GetAll() - } - shortJobCount := 0 - for _, j := range deletionCandidates { - if !j.InTerminalState() { - continue - } - if s.shortJobPenalty.ShouldApplyPenalty(j) { - shortJobCount++ - continue + for _, j := range jobDbJobs { + if j.InTerminalState() { + idsOfJobsToDelete = append(idsOfJobsToDelete, j.Id()) } - idsOfJobsToDelete = append(idsOfJobsToDelete, j.Id()) } - ctx.Logger().Infof("Deleting %d jobs out of %d considered for deletion (%d short jobs, full job gc=%t)", len(idsOfJobsToDelete), len(deletionCandidates), shortJobCount, fullJobGc) + ctx.Logger().Infof("Deleting %d terminal jobs out of %d updated jobs", len(idsOfJobsToDelete), len(jobDbJobs)) if err := txn.BatchDelete(idsOfJobsToDelete); err != nil { return nil, nil, 0, 0, err } @@ -1075,6 +1062,9 @@ func (s *Scheduler) generateUpdateMessagesFromJob(ctx *armadacontext.Context, jo } if !origJob.Equal(job) { + if job.InTerminalState() { + s.shortJobPenalty.ReportFinishedJob(job) + } if err := txn.Upsert([]*jobdb.Job{job}); err != nil { return nil, err } @@ -1133,7 +1123,9 @@ func (s *Scheduler) expireJobsIfNecessary(ctx *armadacontext.Context, txn *jobdb run := job.LatestRun() if run != nil && !job.Queued() && staleExecutors[run.Executor()] { ctx.Warnf("Cancelling job %s as it is running on lost executor %s", job.Id(), run.Executor()) - jobsToUpdate = append(jobsToUpdate, job.WithQueued(false).WithFailed(true).WithUpdatedRun(run.WithFailed(true))) + expiredJob := job.WithQueued(false).WithFailed(true).WithUpdatedRun(run.WithFailed(true)) + s.shortJobPenalty.ReportFinishedJob(expiredJob) + jobsToUpdate = append(jobsToUpdate, expiredJob) leaseExpiredError := &armadaevents.Error{ Terminal: true, @@ -1241,6 +1233,7 @@ func (s *Scheduler) submitCheck(ctx *armadacontext.Context, txn *jobdb.Txn) ([]* } } else { job = job.WithFailed(true).WithQueued(false) + s.shortJobPenalty.ReportFinishedJob(job) jobsToUpdate = append(jobsToUpdate, job) es.Events[0].Event = &armadaevents.EventSequence_Event_JobErrors{ @@ -1284,7 +1277,7 @@ func (s *Scheduler) initialise(ctx *armadacontext.Context) error { case <-ctx.Done(): return nil default: - if _, _, newJobsSerial, newRunsSerial, err := s.syncState(ctx, true, false); err != nil { + if _, _, newJobsSerial, newRunsSerial, err := s.syncState(ctx, true); err != nil { ctx.Logger(). WithStacktrace(err). Error("failed to initialise; trying again in 1 second") diff --git a/internal/scheduler/scheduler_test.go b/internal/scheduler/scheduler_test.go index a4abc648632..72a37dbf186 100644 --- a/internal/scheduler/scheduler_test.go +++ b/internal/scheduler/scheduler_test.go @@ -15,6 +15,7 @@ import ( "github.com/stretchr/testify/require" "golang.org/x/exp/slices" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" clock "k8s.io/utils/clock/testing" "k8s.io/utils/pointer" @@ -116,7 +117,26 @@ var ( }, Version: 1, } - schedulingInfoBytes = protoutil.MustMarshall(schedulingInfo) + schedulingInfoBytes = protoutil.MustMarshall(schedulingInfo) + shortJobSchedulingInfo = &schedulerobjects.JobSchedulingInfo{ + AtMostOnce: true, + PriorityClassName: testfixtures.PriorityClass2NonPreemptible, + ObjectRequirements: []*schedulerobjects.ObjectRequirements{ + { + Requirements: &schedulerobjects.ObjectRequirements_PodRequirements{ + PodRequirements: &schedulerobjects.PodRequirements{ + ResourceRequirements: &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + "cpu": resource.MustParse("1"), + "memory": resource.MustParse("1Gi"), + }, + }, + }, + }, + }, + }, + Version: 1, + } updatedSchedulingInfo = &schedulerobjects.JobSchedulingInfo{ AtMostOnce: true, ObjectRequirements: []*schedulerobjects.ObjectRequirements{ @@ -310,6 +330,46 @@ var leasedFailFastJob = testfixtures.NewJob( true, ).WithNewRun("testExecutor", "test-node", "node", "pool", 5) +var shortJobRunningTime = time.Now() + +func shortJobRun(job *jobdb.Job) *jobdb.Job { + job = job.WithNewRun("testExecutor", "test-node", "node", "pool", 5) + return job.WithUpdatedRun(job.LatestRun().WithRunningTime(&shortJobRunningTime)) +} + +var shortLeasedJob = shortJobRun(testfixtures.NewJob( + util.NewULID(), + "testJobset", + "testQueue", + uint32(10), + toInternalSchedulingInfo(shortJobSchedulingInfo), + false, + 1, + false, + false, + false, + 1, + true, +)) + +var shortUnschedulableJob = func() *jobdb.Job { + job := shortJobRun(testfixtures.NewJob( + util.NewULID(), + "testJobset", + "testQueue", + uint32(10), + toInternalSchedulingInfo(shortJobSchedulingInfo), + false, + 1, + false, + false, + false, + 1, + false, + )) + return job.WithUpdatedRun(job.LatestRun().WithFailed(true).WithReturned(true)) +}() + var ( preemptibleGangJob1 = createPreemptibleGangJob() preemptibleGangJob2 = createPreemptibleGangJob() @@ -371,40 +431,41 @@ type jobRunId struct { // Test a single scheduler cycle func TestScheduler_TestCycle(t *testing.T) { tests := map[string]struct { - initialJobs []*jobdb.Job // jobs in the jobDb at the start of the cycle - jobUpdates []database.Job // job updates from the database - runUpdates []database.Run // run updates from the database - jobRunErrors map[string]*armadaevents.Error // job run errors in the database - staleExecutor bool // if true then the executorRepository will report the executor as stale - fetchError bool // if true then the jobRepository will throw an error - scheduleError error // if true then the scheduling algo will throw an error - publishError bool // if true the publisher will throw an error - submitCheckerFailure bool // if true the submit checker will say the job is unschedulable - submitGangValidateFailure bool // if true the gang validator will say the gang job is invalid - jobIdsToFailDueToReconciliation []string // job ids that will be failed by the scheduler due to reconciliation issues - jobIdsToPreemptDueToReconciliation []string // job ids that will be preempted by the scheduler due to reconciliation issues - expectedJobRunLeased []string // ids of jobs we expect to have produced leased messages - expectedJobRunErrors []jobRunId // ids of jobs we expect to have produced jobRunErrors messages - expectedJobErrors []string // ids of jobs we expect to have produced jobErrors messages - expectedJobsRunsToPreempt []string // ids of jobs we expect to be preempted by the scheduler - expectedJobRunPreempted []jobRunId // ids of jobs we expect to have produced jobRunPreempted messages - expectedJobRunCancelled []jobRunId // ids of jobs we expect to have produced jobRunPreempted messages - expectedJobCancelled []string // ids of jobs we expect to have produced cancelled messages - expectedJobRequestCancel []string // ids of jobs we expect to have produced request cancel - expectedJobReprioritised []string // ids of jobs we expect to have produced reprioritised messages - expectedQueued []string // ids of jobs we expect to have produced requeued messages - expectedJobSucceeded []string // ids of jobs we expect to have produced succeeded messages - expectedLeased []string // ids of jobs we expected to be leased in jobdb at the end of the cycle - expectedRequeued []string // ids of jobs we expected to be requeued in jobdb at the end of the cycle - expectedValidated []string // ids of jobs we expected to have produced submit checked messages - expectedTerminal []string // ids of jobs we expected to be terminal in jobdb at the end of the cycle - expectedJobPriority map[string]uint32 // expected priority of jobs at the end of the cycle - expectedNodeAntiAffinities []string // list of nodes there is expected to be anti affinities for on job scheduling info - expectedJobSchedulingInfoVersion int // expected scheduling info version of jobs at the end of the cycle - expectedQueuedVersion int32 // expected queued version of jobs at the end of the cycle - cordonedQueues []string // queues that are cordoned - queueCacheError bool // if true then the queue cache will throw an error - expectedPreemptReasons map[string]string // map of job id to expected preempt reason on the latest run + initialJobs []*jobdb.Job // jobs in the jobDb at the start of the cycle + jobUpdates []database.Job // job updates from the database + runUpdates []database.Run // run updates from the database + jobRunErrors map[string]*armadaevents.Error // job run errors in the database + staleExecutor bool // if true then the executorRepository will report the executor as stale + fetchError bool // if true then the jobRepository will throw an error + scheduleError error // if true then the scheduling algo will throw an error + publishError bool // if true the publisher will throw an error + submitCheckerFailure bool // if true the submit checker will say the job is unschedulable + submitGangValidateFailure bool // if true the gang validator will say the gang job is invalid + jobIdsToFailDueToReconciliation []string // job ids that will be failed by the scheduler due to reconciliation issues + jobIdsToPreemptDueToReconciliation []string // job ids that will be preempted by the scheduler due to reconciliation issues + expectedJobRunLeased []string // ids of jobs we expect to have produced leased messages + expectedJobRunErrors []jobRunId // ids of jobs we expect to have produced jobRunErrors messages + expectedJobErrors []string // ids of jobs we expect to have produced jobErrors messages + expectedJobsRunsToPreempt []string // ids of jobs we expect to be preempted by the scheduler + expectedJobRunPreempted []jobRunId // ids of jobs we expect to have produced jobRunPreempted messages + expectedJobRunCancelled []jobRunId // ids of jobs we expect to have produced jobRunPreempted messages + expectedJobCancelled []string // ids of jobs we expect to have produced cancelled messages + expectedJobRequestCancel []string // ids of jobs we expect to have produced request cancel + expectedJobReprioritised []string // ids of jobs we expect to have produced reprioritised messages + expectedQueued []string // ids of jobs we expect to have produced requeued messages + expectedJobSucceeded []string // ids of jobs we expect to have produced succeeded messages + expectedLeased []string // ids of jobs we expected to be leased in jobdb at the end of the cycle + expectedRequeued []string // ids of jobs we expected to be requeued in jobdb at the end of the cycle + expectedValidated []string // ids of jobs we expected to have produced submit checked messages + expectedTerminal []string // ids of jobs we expected to be terminal in jobdb at the end of the cycle + expectedJobPriority map[string]uint32 // expected priority of jobs at the end of the cycle + expectedNodeAntiAffinities []string // list of nodes there is expected to be anti affinities for on job scheduling info + expectedJobSchedulingInfoVersion int // expected scheduling info version of jobs at the end of the cycle + expectedQueuedVersion int32 // expected queued version of jobs at the end of the cycle + cordonedQueues []string // queues that are cordoned + queueCacheError bool // if true then the queue cache will throw an error + expectedPreemptReasons map[string]string // map of job id to expected preempt reason on the latest run + expectedShortJobPenalties map[string]internaltypes.ResourceList // map of queue to the short-job penalty resources expected for testPool }{ "Lease a single job already in the db": { initialJobs: []*jobdb.Job{queuedJob}, @@ -990,6 +1051,46 @@ func TestScheduler_TestCycle(t *testing.T) { expectedLeased: []string{leasedJob.Id()}, expectedQueuedVersion: leasedJob.QueuedVersion(), }, + "Short job succeeded reports a penalty": { + initialJobs: []*jobdb.Job{shortLeasedJob}, + runUpdates: []database.Run{ + { + RunID: shortLeasedJob.LatestRun().Id(), + JobID: shortLeasedJob.Id(), + JobSet: "testJobSet", + Executor: "testExecutor", + Succeeded: true, + Serial: 1, + }, + }, + expectedJobSucceeded: []string{shortLeasedJob.Id()}, + expectedTerminal: []string{shortLeasedJob.Id()}, + expectedQueuedVersion: shortLeasedJob.QueuedVersion(), + expectedShortJobPenalties: map[string]internaltypes.ResourceList{ + "testQueue": shortLeasedJob.AllResourceRequirements(), + }, + }, + "Short job expired reports a penalty": { + initialJobs: []*jobdb.Job{shortLeasedJob}, + staleExecutor: true, + expectedJobRunErrors: []jobRunId{{jobId: shortLeasedJob.Id(), runId: shortLeasedJob.LatestRun().Id()}}, + expectedJobErrors: []string{shortLeasedJob.Id()}, + expectedTerminal: []string{shortLeasedJob.Id()}, + expectedQueuedVersion: shortLeasedJob.QueuedVersion(), + expectedShortJobPenalties: map[string]internaltypes.ResourceList{ + "testQueue": shortLeasedJob.AllResourceRequirements(), + }, + }, + "Short job failing submit check reports a penalty": { + initialJobs: []*jobdb.Job{shortUnschedulableJob}, + submitCheckerFailure: true, + expectedJobErrors: []string{shortUnschedulableJob.Id()}, + expectedTerminal: []string{shortUnschedulableJob.Id()}, + expectedQueuedVersion: shortUnschedulableJob.QueuedVersion(), + expectedShortJobPenalties: map[string]internaltypes.ResourceList{ + "testQueue": shortUnschedulableJob.AllResourceRequirements(), + }, + }, } for name, tc := range tests { t.Run(name, func(t *testing.T) { @@ -1026,6 +1127,8 @@ func TestScheduler_TestCycle(t *testing.T) { queues = append(queues, &api.Queue{Name: name, Cordoned: true}) } queueCache := &testQueueCache{queues: queues, shouldError: tc.queueCacheError} + shortJobPenalty := scheduling.NewShortJobPenalty(map[string]time.Duration{"pool": time.Minute}) + shortJobPenalty.SetNow(shortJobRunningTime.Add(time.Second)) sched, err := NewScheduler( testfixtures.NewJobDb(testfixtures.TestResourceListFactory), jobRepo, @@ -1038,7 +1141,7 @@ func TestScheduler_TestCycle(t *testing.T) { 1*time.Second, 5*time.Second, clusterTimeout, - nil, + shortJobPenalty, maxNumberOfAttempts, nodeIdLabel, schedulerMetrics, @@ -1168,6 +1271,14 @@ func TestScheduler_TestCycle(t *testing.T) { } } } + + // assert short-job penalties reported during terminalisation + penalties := shortJobPenalty.Snapshot().GetPenaltiesForPool("pool") + for queue, expectedResources := range tc.expectedShortJobPenalties { + assert.True(t, penalties[queue].Equal(expectedResources), + "expected short-job penalty for queue %s to equal %s, got %s", queue, expectedResources, penalties[queue]) + } + assert.Len(t, penalties, len(tc.expectedShortJobPenalties)) cancel() }) } @@ -2019,7 +2130,7 @@ func TestScheduler_TestSyncInitialState(t *testing.T) { // which must be consistent within tests. sched.jobDb = testfixtures.NewJobDb(testfixtures.TestResourceListFactory) - initialJobs, _, newJobsSerial, newRunsSerial, err := sched.syncState(ctx, true, false) + initialJobs, _, newJobsSerial, newRunsSerial, err := sched.syncState(ctx, true) require.NoError(t, err) sched.jobsSerial = newJobsSerial sched.runsSerial = newRunsSerial @@ -2242,7 +2353,7 @@ func TestScheduler_TestSyncState(t *testing.T) { require.NoError(t, err) txn.Commit() - updatedJobs, _, _, _, err := sched.syncState(ctx, false, false) + updatedJobs, _, _, _, err := sched.syncState(ctx, false) require.NoError(t, err) expectedJobDb := testfixtures.NewJobDbWithJobs(tc.expectedUpdatedJobs) diff --git a/internal/scheduler/scheduling/scheduling_algo.go b/internal/scheduler/scheduling/scheduling_algo.go index c9f4cf5b1a5..3a4126360da 100644 --- a/internal/scheduler/scheduling/scheduling_algo.go +++ b/internal/scheduler/scheduling/scheduling_algo.go @@ -137,6 +137,8 @@ func (l *FairSchedulingAlgo) Schedule( return nil, err } + shortJobPenalty := l.shortJobPenalty.Snapshot() + reconciliationByPool, err := l.reconcilePools(ctx, txn, executors) if err != nil { return nil, err @@ -153,7 +155,7 @@ func (l *FairSchedulingAlgo) Schedule( if reconciliation.Err() != nil { outcome = reconciliation.Outcome() } else { - outcome, schedulingResult, err = l.runPoolSchedulingRound(ctx, pool, txn, executors) + outcome, schedulingResult, err = l.runPoolSchedulingRound(ctx, pool, txn, executors, shortJobPenalty) if err != nil { return nil, err } @@ -207,6 +209,7 @@ func (l *FairSchedulingAlgo) runPoolSchedulingRound( pool configuration.PoolConfig, txn *jobdb.Txn, executors []*schedulerobjects.Executor, + shortJobPenalty *ShortJobPenaltySnapshot, ) (*PoolSchedulingOutcome, *SchedulingResult, error) { select { case <-ctx.Done(): @@ -217,7 +220,7 @@ func (l *FairSchedulingAlgo) runPoolSchedulingRound( // It is important to pass the validated executors here // This is because the validation ensures those nodes are inline with the jobs // If we use a different copy of nodes (possibly more to date copy) it may no longer align with the jobs/runs - fsctx, err := l.newFairSchedulingAlgoContext(ctx, txn, executors, pool) + fsctx, err := l.newFairSchedulingAlgoContext(ctx, txn, executors, pool, shortJobPenalty) if err != nil { return NewPoolSchedulingOutcome(PoolSchedulingTerminationReasonSchedulingDisabled, errors.WithMessagef(err, "failed to create scheduling algo context")), nil, nil } @@ -405,7 +408,7 @@ func markAsFailedReconciliation(clock clock.Clock, job *jobdb.Job) *jobdb.Job { return job } -func (l *FairSchedulingAlgo) newFairSchedulingAlgoContext(ctx *armadacontext.Context, txn *jobdb.Txn, executors []*schedulerobjects.Executor, currentPool configuration.PoolConfig) (*FairSchedulingAlgoContext, error) { +func (l *FairSchedulingAlgo) newFairSchedulingAlgoContext(ctx *armadacontext.Context, txn *jobdb.Txn, executors []*schedulerobjects.Executor, currentPool configuration.PoolConfig, shortJobPenalty *ShortJobPenaltySnapshot) (*FairSchedulingAlgoContext, error) { queues, err := l.queueCache.GetAll(ctx) if err != nil { return nil, err @@ -430,16 +433,12 @@ func (l *FairSchedulingAlgo) newFairSchedulingAlgoContext(ctx *armadacontext.Con // - Jobs active on the nodes of this pool // - These are used to populate the jobdb, calculate demand/fairshare // - This may include nodes from other pools, especially if the nodes pool has changed - // - Terminal jobs of this pool - // - For calculating short job penalty // - Jobs queued against home/away pools relevant to the pool being computed // - This is to calculate demand on both home and away pools leasedJobs := txn.GetAllLeasedJobs() - terminalJobs := txn.GetAllTerminalJobs() queuedJobs := getQueuedJobs(txn, allPools) - allJobs := make([]*jobdb.Job, 0, len(leasedJobs)+len(terminalJobs)+len(queuedJobs)) + allJobs := make([]*jobdb.Job, 0, len(leasedJobs)+len(queuedJobs)) allJobs = append(allJobs, leasedJobs...) - allJobs = append(allJobs, terminalJobs...) allJobs = append(allJobs, queuedJobs...) jobSchedulingInfo, err := l.calculateJobSchedulingInfo(ctx, @@ -450,7 +449,8 @@ func (l *FairSchedulingAlgo) newFairSchedulingAlgoContext(ctx *armadacontext.Con allJobs, currentPool.Name, awayAllocationPools, - allPools) + allPools, + shortJobPenalty) if err != nil { return nil, err } @@ -575,13 +575,13 @@ type jobSchedulingInfo struct { func (l *FairSchedulingAlgo) calculateJobSchedulingInfo(ctx *armadacontext.Context, activeExecutorsSet map[string]bool, queues map[string]*api.Queue, jobs []*jobdb.Job, currentPool string, awayAllocationPools []string, allPools []string, + shortJobPenalty *ShortJobPenaltySnapshot, ) (*jobSchedulingInfo, error) { jobsByExecutorId := make(map[string][]*jobdb.Job) jobsByPool := make(map[string][]*jobdb.Job) demandByQueueAndPriorityClass := make(map[string]map[string]internaltypes.ResourceList) allocatedByQueueAndPriorityClass := make(map[string]map[string]internaltypes.ResourceList) awayAllocatedByQueueAndPriorityClass := make(map[string]map[string]internaltypes.ResourceList) - shortJobPenaltyByQueue := make(map[string]internaltypes.ResourceList) for _, job := range jobs { queue, present := queues[job.Queue()] @@ -590,14 +590,6 @@ func (l *FairSchedulingAlgo) calculateJobSchedulingInfo(ctx *armadacontext.Conte continue } - if l.shortJobPenalty.ShouldApplyPenalty(job) { - jobPool := job.LatestRun().Pool() - jobRequirements := job.AllResourceRequirements() - if jobPool == currentPool { - shortJobPenaltyByQueue[queue.Name] = shortJobPenaltyByQueue[queue.Name].Add(jobRequirements) - } - } - if job.InTerminalState() { continue } @@ -675,6 +667,7 @@ func (l *FairSchedulingAlgo) calculateJobSchedulingInfo(ctx *armadacontext.Conte jobsByExecutorId[executorId] = append(jobsByExecutorId[executorId], job) } + shortJobPenaltyByQueue := shortJobPenalty.GetPenaltiesForPool(currentPool) return &jobSchedulingInfo{ jobsByExecutorId: jobsByExecutorId, jobsByPool: jobsByPool, diff --git a/internal/scheduler/scheduling/short_job_penalty.go b/internal/scheduler/scheduling/short_job_penalty.go index bfd9563e2f7..f9c45d8fa6d 100644 --- a/internal/scheduler/scheduling/short_job_penalty.go +++ b/internal/scheduler/scheduling/short_job_penalty.go @@ -1,21 +1,20 @@ package scheduling import ( + "container/heap" + "maps" "time" + "github.com/armadaproject/armada/internal/scheduler/internaltypes" "github.com/armadaproject/armada/internal/scheduler/jobdb" ) -// Used to penalize short-running jobs by pretending they -// ran for some minimum length when calculating costs. -type ShortJobPenalty struct { - cutoffDurationByPool map[string]time.Duration - now time.Time -} - func NewShortJobPenalty(cutoffDurationByPool map[string]time.Duration) *ShortJobPenalty { return &ShortJobPenalty{ cutoffDurationByPool: cutoffDurationByPool, + penaltyByJobID: map[string]*penaltyEntry{}, + expiry: &entryHeap{}, + sums: map[string]map[string]internaltypes.ResourceList{}, } } @@ -23,10 +22,12 @@ func (sjp *ShortJobPenalty) SetNow(now time.Time) { if sjp == nil { return } + sjp.mu.Lock() + defer sjp.mu.Unlock() sjp.now = now } -func (sjp *ShortJobPenalty) ShouldApplyPenalty(job *jobdb.Job) bool { +func (sjp *ShortJobPenalty) shouldApplyPenalty(job *jobdb.Job) bool { if sjp == nil || sjp.now.IsZero() { return false } @@ -51,3 +52,89 @@ func (sjp *ShortJobPenalty) ShouldApplyPenalty(job *jobdb.Job) bool { return sjp.now.Sub(*jobStart) < sjp.cutoffDurationByPool[jobRun.Pool()] } + +// ReportFinishedJob applies a terminal short job's resources to its (pool, queue) sums once. +// Non-terminal and duplicate jobs are ignored. +func (sjp *ShortJobPenalty) ReportFinishedJob(job *jobdb.Job) { + if sjp == nil { + return + } + sjp.mu.Lock() + defer sjp.mu.Unlock() + sjp.expireUpTo(sjp.now) + + if _, alreadyCounted := sjp.penaltyByJobID[job.Id()]; alreadyCounted { + return + } + if !sjp.shouldApplyPenalty(job) { + return + } + + run := job.LatestRun() + pool := run.Pool() + queue := job.Queue() + resources := job.AllResourceRequirements() + deadline := run.RunningTime().Add(sjp.cutoffDurationByPool[pool]) + + e := &penaltyEntry{ + jobID: job.Id(), + pool: pool, + queue: queue, + resources: resources, + deadline: deadline, + } + sjp.penaltyByJobID[job.Id()] = e + heap.Push(sjp.expiry, e) + sjp.addToSums(pool, queue, resources) +} + +// Snapshot expires entries up to the current now and returns an immutable, +// deep-copied view of the per-(pool,queue) penalty sums. +func (sjp *ShortJobPenalty) Snapshot() *ShortJobPenaltySnapshot { + if sjp == nil { + return &ShortJobPenaltySnapshot{} + } + sjp.mu.Lock() + defer sjp.mu.Unlock() + sjp.expireUpTo(sjp.now) + + sums := make(map[string]map[string]internaltypes.ResourceList, len(sjp.sums)) + for pool, queueSums := range sjp.sums { + inner := make(map[string]internaltypes.ResourceList, len(queueSums)) + maps.Copy(inner, queueSums) + sums[pool] = inner + } + return &ShortJobPenaltySnapshot{sums: sums} +} + +// expireUpTo pops every entry whose deadline is at or before now, +// subtracting its penalty contribution +func (sjp *ShortJobPenalty) expireUpTo(now time.Time) { + for sjp.expiry.Len() > 0 && !sjp.expiry.peek().deadline.After(now) { + e := heap.Pop(sjp.expiry).(*penaltyEntry) + sjp.subtractFromSums(e.pool, e.queue, e.resources) + delete(sjp.penaltyByJobID, e.jobID) + } +} + +func (sjp *ShortJobPenalty) addToSums(pool, queue string, resources internaltypes.ResourceList) { + queueSums, ok := sjp.sums[pool] + if !ok { + queueSums = map[string]internaltypes.ResourceList{} + sjp.sums[pool] = queueSums + } + queueSums[queue] = queueSums[queue].Add(resources) +} + +func (sjp *ShortJobPenalty) subtractFromSums(pool, queue string, resources internaltypes.ResourceList) { + queueSums := sjp.sums[pool] + remaining := queueSums[queue].Subtract(resources) + if remaining.AllZero() { + delete(queueSums, queue) + if len(queueSums) == 0 { + delete(sjp.sums, pool) + } + return + } + queueSums[queue] = remaining +} diff --git a/internal/scheduler/scheduling/short_job_penalty_test.go b/internal/scheduler/scheduling/short_job_penalty_test.go index f995927c016..ef7c9ca733a 100644 --- a/internal/scheduler/scheduling/short_job_penalty_test.go +++ b/internal/scheduler/scheduling/short_job_penalty_test.go @@ -1,81 +1,171 @@ package scheduling import ( + "container/heap" + "slices" + "sync" "testing" "time" "github.com/stretchr/testify/assert" + "github.com/armadaproject/armada/internal/scheduler/internaltypes" "github.com/armadaproject/armada/internal/scheduler/jobdb" "github.com/armadaproject/armada/internal/scheduler/testfixtures" ) -func TestNilSjpReturnsFalse(t *testing.T) { - var nilSjp *ShortJobPenalty = nil - job := shortTestJob(time.Now()).WithSucceeded(true) - assert.False(t, nilSjp.ShouldApplyPenalty(job)) -} +func TestEntryHeapLess(t *testing.T) { + base := time.Now() + earlier := &penaltyEntry{deadline: base} + later := &penaltyEntry{deadline: base.Add(time.Second)} -func TestTimeNotSetReturnsFalse(t *testing.T) { - job := shortTestJob(time.Now()).WithSucceeded(true) - assert.False(t, makeSut().ShouldApplyPenalty(job)) + tests := map[string]struct { + h entryHeap + expected bool + }{ + "earlier deadline is less than later": {h: entryHeap{earlier, later}, expected: true}, + "later deadline is not less than earlier": {h: entryHeap{later, earlier}, expected: false}, + "equal deadlines are not less": {h: entryHeap{earlier, earlier}, expected: false}, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + assert.Equal(t, tc.expected, tc.h.Less(0, 1)) + }) + } } -func TestLongSucceededJobReturnsFalse(t *testing.T) { - now := time.Now() - sut := makeSut() - sut.SetNow(now) +func TestEntryHeapPopsInDeadlineOrder(t *testing.T) { + base := time.Now() + offsets := []time.Duration{ + 40 * time.Second, + -10 * time.Second, + 0, + 90 * time.Second, + 10 * time.Second, + -10 * time.Second, + 30 * time.Second, + } - job := longTestJob(now).WithSucceeded(true) - assert.False(t, sut.ShouldApplyPenalty(job)) -} + h := &entryHeap{} + for _, off := range offsets { + heap.Push(h, &penaltyEntry{deadline: base.Add(off)}) + } -func TestShortRunningJobReturnsFalse(t *testing.T) { - sut := makeSut() - now := time.Now() - sut.SetNow(now) + sortedOffsets := slices.Clone(offsets) + slices.Sort(sortedOffsets) - job := shortTestJob(now) - assert.False(t, sut.ShouldApplyPenalty(job)) -} + var popped []time.Duration + for h.Len() > 0 { + assert.Equal(t, base.Add(sortedOffsets[len(popped)]), h.peek().deadline, + "peek must always expose the earliest remaining deadline") + e := heap.Pop(h).(*penaltyEntry) + popped = append(popped, e.deadline.Sub(base)) + } -func TestShortSucceededJobReturnsTrue(t *testing.T) { - sut := makeSut() - now := time.Now() - sut.SetNow(now) + assert.Equal(t, sortedOffsets, popped, "entries must pop in ascending deadline order") +} - job := shortTestJob(now).WithSucceeded(true) - assert.True(t, sut.ShouldApplyPenalty(job)) +func TestNilSjpIsSafeToCall(t *testing.T) { + var nilSjp *ShortJobPenalty = nil + job := shortTestJob(time.Now()).WithSucceeded(true) + assert.NotPanics(t, func() { + nilSjp.SetNow(time.Now()) + nilSjp.ReportFinishedJob(job) + }) + assert.Nil(t, nilSjp.Snapshot().GetPenaltiesForPool(testfixtures.TestPool)) } -func TestShortPreemptedJobReturnsFalse(t *testing.T) { - sut := makeSut() +func TestShouldApplyPenalty(t *testing.T) { now := time.Now() - sut.SetNow(now) - job := shortTestJob(now).WithSucceeded(true) - job = job.WithUpdatedRun(job.LatestRun().WithPreempted(true)) - assert.False(t, sut.ShouldApplyPenalty(job)) -} + withNow := func() *ShortJobPenalty { + sut := makeSut() + sut.SetNow(now) + return sut + } -func TestShortJobWithPreemptRequestedReturnsFalse(t *testing.T) { - sut := makeSut() - now := time.Now() - sut.SetNow(now) + shortPreemptedJob := shortTestJob(now).WithSucceeded(true) + shortPreemptedJob = shortPreemptedJob.WithUpdatedRun(shortPreemptedJob.LatestRun().WithPreempted(true)) - job := shortTestJob(now).WithSucceeded(true) - job = job.WithUpdatedRun(job.LatestRun().WithPreemptRequested(true)) - assert.False(t, sut.ShouldApplyPenalty(job)) -} + shortPreemptRequestedJob := shortTestJob(now).WithSucceeded(true) + shortPreemptRequestedJob = shortPreemptRequestedJob.WithUpdatedRun(shortPreemptRequestedJob.LatestRun().WithPreemptRequested(true)) -func TestShortJobWithPreemptedTimeSetReturnsFalse(t *testing.T) { - sut := makeSut() - now := time.Now() - sut.SetNow(now) + shortPreemptedTimeJob := shortTestJob(now).WithSucceeded(true) + shortPreemptedTimeJob = shortPreemptedTimeJob.WithUpdatedRun(shortPreemptedTimeJob.LatestRun().WithPreemptedTime(&now)) - job := shortTestJob(now).WithSucceeded(true) - job = job.WithUpdatedRun(job.LatestRun().WithPreemptedTime(&now)) - assert.False(t, sut.ShouldApplyPenalty(job)) + tests := map[string]struct { + sut *ShortJobPenalty + job *jobdb.Job + expected bool + }{ + "nil penalty returns false": { + sut: nil, + job: shortTestJob(now).WithSucceeded(true), + expected: false, + }, + "now not set returns false": { + sut: makeSut(), + job: shortTestJob(now).WithSucceeded(true), + expected: false, + }, + "job with no run returns false": { + sut: withNow(), + job: testfixtures.Test32Cpu256GiJob("q", testfixtures.PriorityClass2).WithSucceeded(true), + expected: false, + }, + "job with no running time returns false": { + sut: withNow(), + job: testfixtures.Test32Cpu256GiJob("q", testfixtures.PriorityClass2).WithNewRun("testExecutor", "test-node", "node", testfixtures.TestPool, 5).WithSucceeded(true), + expected: false, + }, + "long succeeded job returns false": { + sut: withNow(), + job: longTestJob(now).WithSucceeded(true), + expected: false, + }, + "short running (non-terminal) job returns false": { + sut: withNow(), + job: shortTestJob(now), + expected: false, + }, + "short succeeded job returns true": { + sut: withNow(), + job: shortTestJob(now).WithSucceeded(true), + expected: true, + }, + "short cancelled job returns true": { + sut: withNow(), + job: shortTestJob(now).WithCancelled(true), + expected: true, + }, + "short failed job returns true": { + sut: withNow(), + job: shortTestJob(now).WithFailed(true), + expected: true, + }, + "short preempted job returns false": { + sut: withNow(), + job: shortPreemptedJob, + expected: false, + }, + "short job with preempt requested returns false": { + sut: withNow(), + job: shortPreemptRequestedJob, + expected: false, + }, + "short job with preempted time set returns false": { + sut: withNow(), + job: shortPreemptedTimeJob, + expected: false, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + assert.Equal(t, tc.expected, tc.sut.shouldApplyPenalty(tc.job)) + }) + } } func makeSut() *ShortJobPenalty { @@ -95,3 +185,188 @@ func testJob(runningTime time.Time) *jobdb.Job { run := job.LatestRun() return job.WithUpdatedRun(run.WithRunningTime(&runningTime)) } + +type sjpStep func(t *testing.T, sut *ShortJobPenalty) + +func setNow(now time.Time) sjpStep { + return func(_ *testing.T, sut *ShortJobPenalty) { sut.SetNow(now) } +} + +func report(job *jobdb.Job) sjpStep { + return func(_ *testing.T, sut *ShortJobPenalty) { sut.ReportFinishedJob(job) } +} + +func expectPenalty(pool, queue string, resources internaltypes.ResourceList) sjpStep { + return func(t *testing.T, sut *ShortJobPenalty) { + assert.True(t, sut.Snapshot().GetPenaltiesForPool(pool)[queue].Equal(resources)) + } +} + +func expectPoolEmpty(pool string) sjpStep { + return func(t *testing.T, sut *ShortJobPenalty) { + assert.Empty(t, sut.Snapshot().GetPenaltiesForPool(pool)) + } +} + +func TestPenaltyAccounting(t *testing.T) { + now := time.Now() + + accumA := testJobForQueue("q1", now.Add(-30*time.Second)).WithSucceeded(true) + accumB := testJobForQueue("q1", now.Add(-20*time.Second)).WithSucceeded(true) + accumC := testJobForQueue("q2", now.Add(-20*time.Second)).WithSucceeded(true) + + dedupJob := testJobForQueue("q1", now.Add(-30*time.Second)).WithSucceeded(true) + + expiringJob := testJobForQueue("q1", now).WithSucceeded(true) + + early := testJobForQueue("q1", now.Add(-50*time.Second)).WithSucceeded(true) + late := testJobForQueue("q1", now.Add(-20*time.Second)).WithSucceeded(true) + + reReportJob := testJobForQueue("q1", now).WithSucceeded(true) + + poolAJob := testJobForPool("q1", "poolA", now.Add(-30*time.Minute)).WithSucceeded(true) + poolBJob := testJobForPool("q1", "poolB", now.Add(-30*time.Minute)).WithSucceeded(true) + poolCJob := testJobForPool("q1", "poolC", now.Add(-1*time.Second)).WithSucceeded(true) + + tests := map[string]struct { + newSut func() *ShortJobPenalty + steps []sjpStep + }{ + "accumulates per queue": { + steps: []sjpStep{ + setNow(now), + report(accumA), + report(accumB), + report(accumC), + expectPenalty(testfixtures.TestPool, "q1", accumA.AllResourceRequirements().Add(accumB.AllResourceRequirements())), + expectPenalty(testfixtures.TestPool, "q2", accumC.AllResourceRequirements()), + }, + }, + "non-qualifying job is not charged": { + steps: []sjpStep{ + setNow(now), + report(longTestJob(now).WithSucceeded(true)), + expectPoolEmpty(testfixtures.TestPool), + }, + }, + "dedup same job reported twice": { + steps: []sjpStep{ + setNow(now), + report(dedupJob), + report(dedupJob), + expectPenalty(testfixtures.TestPool, "q1", dedupJob.AllResourceRequirements()), + }, + }, + "entry expires exactly at deadline": { + steps: []sjpStep{ + setNow(now.Add(30 * time.Second)), + report(expiringJob), + expectPenalty(testfixtures.TestPool, "q1", expiringJob.AllResourceRequirements()), + setNow(now.Add(time.Minute)), + expectPoolEmpty(testfixtures.TestPool), + }, + }, + "partial expiry leaves remainder": { + steps: []sjpStep{ + setNow(now), + report(early), + report(late), + setNow(now.Add(20 * time.Second)), + expectPenalty(testfixtures.TestPool, "q1", late.AllResourceRequirements()), + }, + }, + "post expiry re-report never re-qualifies": { + steps: []sjpStep{ + setNow(now.Add(10 * time.Second)), + report(reReportJob), + setNow(now.Add(2 * time.Minute)), + expectPoolEmpty(testfixtures.TestPool), + report(reReportJob), + expectPoolEmpty(testfixtures.TestPool), + }, + }, + "per-pool cutoff and pool isolation": { + newSut: func() *ShortJobPenalty { + return NewShortJobPenalty(map[string]time.Duration{ + "poolA": time.Minute, + "poolB": time.Hour, + }) + }, + steps: []sjpStep{ + setNow(now), + report(poolAJob), + report(poolBJob), + report(poolCJob), + expectPoolEmpty("poolA"), + expectPenalty("poolB", "q1", poolBJob.AllResourceRequirements()), + expectPoolEmpty("poolC"), + }, + }, + "penalties for unknown pool is empty": { + steps: []sjpStep{ + setNow(now), + expectPoolEmpty("does-not-exist"), + }, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + newSut := tc.newSut + if newSut == nil { + newSut = makeSut + } + sut := newSut() + for _, step := range tc.steps { + step(t, sut) + } + }) + } +} + +func TestShortJobPenalty_ConcurrentReportsAreCorrect(t *testing.T) { + now := time.Now() + sut := makeSut() + sut.SetNow(now) + + const numWorkers = 8 + const jobsPerWorker = 500 + runningTime := now.Add(-10 * time.Second) + + batches := make([][]*jobdb.Job, numWorkers) + expected := internaltypes.ResourceList{} + for w := range batches { + batch := make([]*jobdb.Job, jobsPerWorker) + for i := range batch { + job := testJobForQueue("q1", runningTime).WithSucceeded(true) + batch[i] = job + expected = expected.Add(job.AllResourceRequirements()) + } + batches[w] = batch + } + + var writers sync.WaitGroup + for _, batch := range batches { + writers.Add(1) + go func(batch []*jobdb.Job) { + defer writers.Done() + for _, job := range batch { + sut.ReportFinishedJob(job) + } + }(batch) + } + writers.Wait() + + penalties := sut.Snapshot().GetPenaltiesForPool(testfixtures.TestPool) + assert.True(t, penalties["q1"].Equal(expected)) +} + +func testJobForQueue(queue string, runningTime time.Time) *jobdb.Job { + return testJobForPool(queue, testfixtures.TestPool, runningTime) +} + +func testJobForPool(queue string, pool string, runningTime time.Time) *jobdb.Job { + job := testfixtures.Test32Cpu256GiJob(queue, testfixtures.PriorityClass2).WithNewRun("testExecutor", "test-node", "node", pool, 5) + run := job.LatestRun() + return job.WithUpdatedRun(run.WithRunningTime(&runningTime)) +} diff --git a/internal/scheduler/scheduling/short_job_penalty_types.go b/internal/scheduler/scheduling/short_job_penalty_types.go new file mode 100644 index 00000000000..025c2413dd3 --- /dev/null +++ b/internal/scheduler/scheduling/short_job_penalty_types.go @@ -0,0 +1,69 @@ +package scheduling + +import ( + "container/heap" + "sync" + "time" + + "github.com/armadaproject/armada/internal/scheduler/internaltypes" +) + +type penaltyEntry struct { + jobID string + pool string + queue string + resources internaltypes.ResourceList + // deadline is runStart + cutoffDurationByPool[pool], fixed at insert time + deadline time.Time +} + +// ShortJobPenalty owns job penalty state keyed by (pool, queue). +type ShortJobPenalty struct { + mu sync.Mutex + cutoffDurationByPool map[string]time.Duration + now time.Time + + penaltyByJobID map[string]*penaltyEntry + expiry *entryHeap + // Derived cache of the per-(pool,queue) running total + sums map[string]map[string]internaltypes.ResourceList +} + +type ShortJobPenaltySnapshot struct { + sums map[string]map[string]internaltypes.ResourceList +} + +func (s *ShortJobPenaltySnapshot) GetPenaltiesForPool(pool string) map[string]internaltypes.ResourceList { + if s == nil { + return nil + } + return s.sums[pool] +} + +// entryHeap is a min-heap of penaltyEntry ordered by deadline. +type entryHeap []*penaltyEntry + +func (h entryHeap) Len() int { return len(h) } +func (h entryHeap) Less(i, j int) bool { return h[i].deadline.Before(h[j].deadline) } +func (h entryHeap) Swap(i, j int) { + h[i], h[j] = h[j], h[i] +} + +func (h *entryHeap) Push(x any) { + *h = append(*h, x.(*penaltyEntry)) +} + +func (h *entryHeap) Pop() any { + old := *h + n := len(old) + e := old[n-1] + old[n-1] = nil + *h = old[:n-1] + return e +} + +func (h entryHeap) peek() *penaltyEntry { + return h[0] +} + +var _ heap.Interface = (*entryHeap)(nil) From fa1295ac704c064e096c6f8d05c06602adab7df0 Mon Sep 17 00:00:00 2001 From: Nikola Jokic Date: Mon, 6 Jul 2026 13:46:07 +0200 Subject: [PATCH 21/49] Add OTEL config and wrappers (#4985) #### What type of PR is this? Enhancement #### What this PR does / why we need it Wire the observability packages to each service #### Special notes for your reviewer Depends on #4975 --------- Co-authored-by: JamesMurkin Signed-off-by: sarhiri --- .goreleaser.yml | 15 ++++++ _local/binoculars/config-auth.yaml | 8 +++ _local/binoculars/config.yaml | 8 +++ _local/compose/full.yaml | 53 ++++++++++++------ _local/compose/stack.yaml | 24 ++++++++- _local/eventingester/config.yaml | 8 +++ _local/executor/config-auth.yaml | 8 +++ _local/executor/config.yaml | 8 +++ _local/lookout/config-auth.yaml | 8 +++ _local/lookout/config.yaml | 8 +++ _local/lookoutingester/config.yaml | 8 +++ _local/otel/collector-config.yaml | 23 ++++++++ _local/scheduler/config-auth.yaml | 8 +++ _local/scheduler/config.yaml | 8 +++ _local/scheduleringester/config.yaml | 8 +++ _local/server/config-auth.yaml | 10 +++- _local/server/config.yaml | 10 +++- cmd/binoculars/main.go | 11 ++++ cmd/eventingester/main.go | 20 ++++++- cmd/executor/main.go | 10 ++++ cmd/lookout/main.go | 11 ++++ cmd/lookoutingester/main.go | 11 ++++ cmd/scheduleringester/main.go | 11 ++++ cmd/server/main.go | 11 ++++ go.mod | 3 ++ go.sum | 2 + internal/binoculars/configuration/types.go | 10 ++-- .../binoculars/configuration/validation.go | 3 +- internal/common/build/build.go | 3 ++ internal/common/grpc/gateway.go | 12 +++-- internal/common/grpc/grpc.go | 4 +- internal/common/logging/logger.go | 35 ++++++++++++ internal/common/observability/config.go | 38 +++++++++++++ internal/common/observability/config_test.go | 45 ++++++++++++++++ internal/common/observability/lifecycle.go | 4 ++ .../observability/lifecycle_bootstrap_test.go | 16 +++--- .../common/observability/lifecycle_test.go | 5 +- internal/eventingester/configuration/types.go | 3 ++ .../eventingester/configuration/validation.go | 3 +- internal/executor/configuration/types.go | 2 + internal/executor/configuration/validation.go | 3 +- internal/lookout/configuration/types.go | 8 +-- internal/lookout/configuration/validation.go | 3 +- .../lookoutingester/configuration/types.go | 3 ++ .../configuration/validation.go | 3 +- .../scheduler/configuration/configuration.go | 3 ++ .../scheduler/configuration/validation.go | 2 + .../configuration/validation_test.go | 54 +++++++++++++++++++ internal/scheduler/schedulerapp.go | 13 +++++ internal/scheduleringester/config.go | 6 ++- internal/server/configuration/types.go | 10 ++-- internal/server/configuration/validation.go | 3 +- pkg/client/connection.go | 2 + website/content/docs/developer-guide.mdx | 2 +- 54 files changed, 557 insertions(+), 54 deletions(-) create mode 100644 _local/otel/collector-config.yaml create mode 100644 internal/common/build/build.go diff --git a/.goreleaser.yml b/.goreleaser.yml index 4ae4e47c0b9..8e708f25f2b 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -27,6 +27,8 @@ builds: binary: server main: ./cmd/server/main.go mod_timestamp: '{{ .CommitTimestamp }}' + ldflags: + - -X github.com/armadaproject/armada/internal/common/build.ReleaseVersion={{.Version}} goos: - linux goarch: @@ -36,6 +38,8 @@ builds: binary: executor main: ./cmd/executor/main.go mod_timestamp: '{{ .CommitTimestamp }}' + ldflags: + - -X github.com/armadaproject/armada/internal/common/build.ReleaseVersion={{.Version}} goos: - linux goarch: @@ -63,6 +67,8 @@ builds: binary: binoculars main: ./cmd/binoculars/main.go mod_timestamp: '{{ .CommitTimestamp }}' + ldflags: + - -X github.com/armadaproject/armada/internal/common/build.ReleaseVersion={{.Version}} goos: - linux goarch: @@ -76,6 +82,7 @@ builds: - -X github.com/armadaproject/armada/internal/lookout/version.Version={{.Version}} - -X github.com/armadaproject/armada/internal/lookout/version.Commit={{.FullCommit}} - -X github.com/armadaproject/armada/internal/lookout/version.BuildTime={{.Date}} + - -X github.com/armadaproject/armada/internal/common/build.ReleaseVersion={{.Version}} goos: - linux goarch: @@ -85,6 +92,8 @@ builds: binary: lookoutingester main: ./cmd/lookoutingester/main.go mod_timestamp: '{{ .CommitTimestamp }}' + ldflags: + - -X github.com/armadaproject/armada/internal/common/build.ReleaseVersion={{.Version}} goos: - linux goarch: @@ -94,6 +103,8 @@ builds: binary: eventingester main: ./cmd/eventingester/main.go mod_timestamp: '{{ .CommitTimestamp }}' + ldflags: + - -X github.com/armadaproject/armada/internal/common/build.ReleaseVersion={{.Version}} goos: - linux goarch: @@ -103,6 +114,8 @@ builds: binary: scheduler main: ./cmd/scheduler/main.go mod_timestamp: '{{ .CommitTimestamp }}' + ldflags: + - -X github.com/armadaproject/armada/internal/common/build.ReleaseVersion={{.Version}} goos: - linux goarch: @@ -112,6 +125,8 @@ builds: binary: scheduleringester main: ./cmd/scheduleringester/main.go mod_timestamp: '{{ .CommitTimestamp }}' + ldflags: + - -X github.com/armadaproject/armada/internal/common/build.ReleaseVersion={{.Version}} goos: - linux goarch: diff --git a/_local/binoculars/config-auth.yaml b/_local/binoculars/config-auth.yaml index 2fc00b4a62b..59c1fa8361b 100644 --- a/_local/binoculars/config-auth.yaml +++ b/_local/binoculars/config-auth.yaml @@ -1,6 +1,14 @@ httpPort: 8084 grpcPort: 50053 metricsPort: 9007 +observability: + enabled: true + exporter: + endpoint: "http://localhost:4318" + protocol: "http/protobuf" + traces: + sampler: "parent_based_trace_id_ratio" + samplerArg: 1.0 application: clusterId: local-cluster kubernetes: diff --git a/_local/binoculars/config.yaml b/_local/binoculars/config.yaml index ea659fbf9c2..dcd6973e9e8 100644 --- a/_local/binoculars/config.yaml +++ b/_local/binoculars/config.yaml @@ -1,6 +1,14 @@ httpPort: 8084 grpcPort: 50053 metricsPort: 9007 +observability: + enabled: true + exporter: + endpoint: "http://localhost:4318" + protocol: "http/protobuf" + traces: + sampler: "parent_based_trace_id_ratio" + samplerArg: 1.0 application: clusterId: local-cluster kubernetes: diff --git a/_local/compose/full.yaml b/_local/compose/full.yaml index ec4fb276646..6a0e5b4d9a9 100644 --- a/_local/compose/full.yaml +++ b/_local/compose/full.yaml @@ -75,7 +75,8 @@ services: PULSAR_PREFIX_allowAutoTopicCreation: "true" PULSAR_PREFIX_allowAutoTopicCreationType: non-partitioned PULSAR_PREFIX_autoSkipNonRecoverableData: "true" - entrypoint: sh -c "bin/apply-config-from-env.py conf/standalone.conf && bin/pulsar standalone" + entrypoint: sh -c "bin/apply-config-from-env.py conf/standalone.conf && + bin/pulsar standalone" ports: - "6650:6650" - "8090:8080" @@ -88,6 +89,28 @@ services: retries: 10 start_period: 30s + jaeger: + container_name: jaeger + image: ${JAEGER_IMAGE:-jaegertracing/all-in-one:1.76.0} + environment: + - COLLECTOR_OTLP_ENABLED=true + ports: + - "16686:16686" # Jaeger UI + restart: unless-stopped + + otel-collector: + container_name: otel-collector + image: ${OTEL_IMAGE:-otel/opentelemetry-collector-contrib:0.154.0} + command: ["--config=/etc/otelcol-contrib/config.yaml"] + volumes: + - ../otel/collector-config.yaml:/etc/otelcol-contrib/config.yaml:ro + ports: + - "4317:4317" # OTLP gRPC + - "4318:4318" # OTLP HTTP + restart: unless-stopped + depends_on: + - jaeger + # ========================================================= # Database migrations (run once, then exit) # ========================================================= @@ -96,7 +119,7 @@ services: container_name: scheduler-migration image: ${ARMADA_IMAGE:-gresearch/armada-bundle}:${ARMADA_IMAGE_TAG:-latest} depends_on: - postgres: { condition: service_healthy } + postgres: {condition: service_healthy} volumes: - ../scheduler/config.yaml:/config/config.yaml:ro environment: @@ -108,7 +131,7 @@ services: container_name: lookout-migration image: ${ARMADA_IMAGE:-gresearch/armada-lookout-bundle}:${ARMADA_IMAGE_TAG:-latest} depends_on: - postgres: { condition: service_healthy } + postgres: {condition: service_healthy} volumes: - ../lookout/config.yaml:/config/config.yaml:ro environment: @@ -123,7 +146,7 @@ services: container_name: scheduler image: ${ARMADA_IMAGE:-gresearch/armada-bundle}:${ARMADA_IMAGE_TAG:-latest} depends_on: - scheduler-migration: { condition: service_completed_successfully } + scheduler-migration: {condition: service_completed_successfully} volumes: - ../scheduler/config.yaml:/config/config.yaml:ro environment: @@ -140,8 +163,8 @@ services: container_name: scheduleringester image: ${ARMADA_IMAGE:-gresearch/armada-bundle}:${ARMADA_IMAGE_TAG:-latest} depends_on: - scheduler-migration: { condition: service_completed_successfully } - pulsar: { condition: service_healthy } + scheduler-migration: {condition: service_completed_successfully} + pulsar: {condition: service_healthy} volumes: - ../scheduleringester/config.yaml:/config/config.yaml:ro environment: @@ -161,9 +184,9 @@ services: - "50051:50051" - "8081:8081" depends_on: - lookout-migration: { condition: service_completed_successfully } - pulsar: { condition: service_healthy } - redis: { condition: service_healthy } + lookout-migration: {condition: service_completed_successfully} + pulsar: {condition: service_healthy} + redis: {condition: service_healthy} volumes: - ../server/config.yaml:/config/config.yaml:ro environment: @@ -182,7 +205,7 @@ services: extra_hosts: - "host.docker.internal:host-gateway" depends_on: - scheduler: { condition: service_started } + scheduler: {condition: service_started} volumes: - ../executor/config.yaml:/config/config.yaml:ro - ../../.kube/internal:/.kube:ro @@ -196,8 +219,8 @@ services: container_name: eventingester image: ${ARMADA_IMAGE:-gresearch/armada-bundle}:${ARMADA_IMAGE_TAG:-latest} depends_on: - pulsar: { condition: service_healthy } - redis: { condition: service_healthy } + pulsar: {condition: service_healthy} + redis: {condition: service_healthy} volumes: - ../eventingester/config.yaml:/config/config.yaml:ro environment: @@ -211,8 +234,8 @@ services: container_name: lookoutingester image: ${ARMADA_IMAGE:-gresearch/armada-lookout-bundle}:${ARMADA_IMAGE_TAG:-latest} depends_on: - lookout-migration: { condition: service_completed_successfully } - pulsar: { condition: service_healthy } + lookout-migration: {condition: service_completed_successfully} + pulsar: {condition: service_healthy} volumes: - ../lookoutingester/config.yaml:/config/config.yaml:ro environment: @@ -229,7 +252,7 @@ services: ports: - "8089:8089" depends_on: - lookout-migration: { condition: service_completed_successfully } + lookout-migration: {condition: service_completed_successfully} volumes: - ../lookout/config.yaml:/config/config.yaml:ro environment: diff --git a/_local/compose/stack.yaml b/_local/compose/stack.yaml index 06bea6a218d..a6ce32a0fd2 100644 --- a/_local/compose/stack.yaml +++ b/_local/compose/stack.yaml @@ -61,7 +61,8 @@ services: PULSAR_PREFIX_allowAutoTopicCreation: "true" PULSAR_PREFIX_allowAutoTopicCreationType: non-partitioned PULSAR_PREFIX_autoSkipNonRecoverableData: "true" - entrypoint: sh -c "bin/apply-config-from-env.py conf/standalone.conf && bin/pulsar standalone" + entrypoint: sh -c "bin/apply-config-from-env.py conf/standalone.conf && + bin/pulsar standalone" ports: - "6650:6650" - "8090:8080" @@ -96,6 +97,27 @@ services: # `docker compose up --wait` is willing to block for. `mage dev:up auth` polls the # realm endpoint via waitForKeycloak before starting goreman. + jaeger: + container_name: jaeger + image: ${JAEGER_IMAGE:-jaegertracing/all-in-one:1.76.0} + environment: + - COLLECTOR_OTLP_ENABLED=true + ports: + - "16686:16686" # Jaeger UI + restart: unless-stopped + + otel-collector: + container_name: otel-collector + image: ${OTEL_IMAGE:-otel/opentelemetry-collector-contrib:0.154.0} + command: ["--config=/etc/otelcol-contrib/config.yaml"] + volumes: + - ../otel/collector-config.yaml:/etc/otelcol-contrib/config.yaml:ro + ports: + - "4317:4317" # OTLP gRPC + - "4318:4318" # OTLP HTTP + restart: unless-stopped + depends_on: + - jaeger prometheus: container_name: prometheus image: ${PROMETHEUS_IMAGE:-prom/prometheus:v3.11.3} diff --git a/_local/eventingester/config.yaml b/_local/eventingester/config.yaml index fecd76092aa..22eb2f0b2ab 100644 --- a/_local/eventingester/config.yaml +++ b/_local/eventingester/config.yaml @@ -9,6 +9,14 @@ pulsar: jobsetEventsTopic: "events" subscriptionName: "events-ingester" metricsPort: 9004 +observability: + enabled: true + exporter: + endpoint: "http://localhost:4318" + protocol: "http/protobuf" + traces: + sampler: "parent_based_trace_id_ratio" + samplerArg: 1.0 metrics: redis: enabled: true diff --git a/_local/executor/config-auth.yaml b/_local/executor/config-auth.yaml index de2f10a1d38..98da8547fc5 100644 --- a/_local/executor/config-auth.yaml +++ b/_local/executor/config-auth.yaml @@ -7,6 +7,14 @@ executorApiConnection: clientId: "armada-executor" clientSecret: "executor-secret" scopes: ["profile", "email"] +observability: + enabled: true + exporter: + endpoint: "http://localhost:4318" + protocol: "http/protobuf" + traces: + sampler: "parent_based_trace_id_ratio" + samplerArg: 1.0 metric: port: 9002 application: diff --git a/_local/executor/config.yaml b/_local/executor/config.yaml index ac38aad66e4..9fd905fc424 100644 --- a/_local/executor/config.yaml +++ b/_local/executor/config.yaml @@ -2,6 +2,14 @@ httpPort: 8082 executorApiConnection: armadaUrl: "localhost:50052" forceNoTls: true +observability: + enabled: true + exporter: + endpoint: "http://localhost:4318" + protocol: "http/protobuf" + traces: + sampler: "parent_based_trace_id_ratio" + samplerArg: 1.0 metric: port: 9002 application: diff --git a/_local/lookout/config-auth.yaml b/_local/lookout/config-auth.yaml index b8ac6dfa349..67682a080da 100644 --- a/_local/lookout/config-auth.yaml +++ b/_local/lookout/config-auth.yaml @@ -1,5 +1,13 @@ apiPort: 8089 metricsPort: 9003 +observability: + enabled: true + exporter: + endpoint: "http://localhost:4318" + protocol: "http/protobuf" + traces: + sampler: "parent_based_trace_id_ratio" + samplerArg: 1.0 corsAllowedOrigins: - "http://localhost:3000" - "http://localhost:8089" diff --git a/_local/lookout/config.yaml b/_local/lookout/config.yaml index c7e374ad73d..028b12f39e2 100644 --- a/_local/lookout/config.yaml +++ b/_local/lookout/config.yaml @@ -1,5 +1,13 @@ apiPort: 8089 metricsPort: 9003 +observability: + enabled: true + exporter: + endpoint: "http://localhost:4318" + protocol: "http/protobuf" + traces: + sampler: "parent_based_trace_id_ratio" + samplerArg: 1.0 corsAllowedOrigins: - "http://localhost:3000" - "http://localhost:8089" diff --git a/_local/lookoutingester/config.yaml b/_local/lookoutingester/config.yaml index fba04adaa74..fffb575434d 100644 --- a/_local/lookoutingester/config.yaml +++ b/_local/lookoutingester/config.yaml @@ -1,4 +1,12 @@ metricsPort: 9005 +observability: + enabled: true + exporter: + endpoint: "http://localhost:4318" + protocol: "http/protobuf" + traces: + sampler: "parent_based_trace_id_ratio" + samplerArg: 1.0 postgres: connection: host: localhost diff --git a/_local/otel/collector-config.yaml b/_local/otel/collector-config.yaml new file mode 100644 index 00000000000..a3b7316f796 --- /dev/null +++ b/_local/otel/collector-config.yaml @@ -0,0 +1,23 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + batch: + +exporters: + otlp/jaeger: + endpoint: jaeger:4317 + tls: + insecure: true + +service: + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [otlp/jaeger] diff --git a/_local/scheduler/config-auth.yaml b/_local/scheduler/config-auth.yaml index 71f7c125169..753cc9b98ac 100644 --- a/_local/scheduler/config-auth.yaml +++ b/_local/scheduler/config-auth.yaml @@ -2,6 +2,14 @@ grpc: port: 50052 tls: enabled: false +observability: + enabled: true + exporter: + endpoint: "http://localhost:4318" + protocol: "http/protobuf" + traces: + sampler: "parent_based_trace_id_ratio" + samplerArg: 1.0 http: port: 8080 auth: diff --git a/_local/scheduler/config.yaml b/_local/scheduler/config.yaml index 8e33796d806..308aaba0165 100644 --- a/_local/scheduler/config.yaml +++ b/_local/scheduler/config.yaml @@ -2,6 +2,14 @@ grpc: port: 50052 tls: enabled: false +observability: + enabled: true + exporter: + endpoint: "http://localhost:4318" + protocol: "http/protobuf" + traces: + sampler: "parent_based_trace_id_ratio" + samplerArg: 1.0 http: port: 8080 auth: diff --git a/_local/scheduleringester/config.yaml b/_local/scheduleringester/config.yaml index d9a25c2da25..0e15f53981c 100644 --- a/_local/scheduleringester/config.yaml +++ b/_local/scheduleringester/config.yaml @@ -1,3 +1,11 @@ +observability: + enabled: true + exporter: + endpoint: "http://localhost:4318" + protocol: "http/protobuf" + traces: + sampler: "parent_based_trace_id_ratio" + samplerArg: 1.0 metricsPort: 9006 postgres: connection: diff --git a/_local/server/config-auth.yaml b/_local/server/config-auth.yaml index 32ee6ef49a2..0a6f1f38f81 100644 --- a/_local/server/config-auth.yaml +++ b/_local/server/config-auth.yaml @@ -1,6 +1,14 @@ httpPort: 8081 grpcPort: 50051 -metricsPort: 9009 +metricsPort: 9000 +observability: + enabled: true + exporter: + endpoint: "http://localhost:4318" + protocol: "http/protobuf" + traces: + sampler: "parent_based_trace_id_ratio" + samplerArg: 1.0 corsAllowedOrigins: - "http://localhost:3000" - "http://localhost:8089" diff --git a/_local/server/config.yaml b/_local/server/config.yaml index 62c5b6c16a7..212b54066a9 100644 --- a/_local/server/config.yaml +++ b/_local/server/config.yaml @@ -1,6 +1,14 @@ httpPort: 8081 grpcPort: 50051 -metricsPort: 9009 +metricsPort: 9000 +observability: + enabled: true + exporter: + endpoint: "http://localhost:4318" + protocol: "http/protobuf" + traces: + sampler: "parent_based_trace_id_ratio" + samplerArg: 1.0 corsAllowedOrigins: - "http://localhost:3000" - "http://localhost:8089" diff --git a/cmd/binoculars/main.go b/cmd/binoculars/main.go index cd4e0221766..7a80c4cfb37 100644 --- a/cmd/binoculars/main.go +++ b/cmd/binoculars/main.go @@ -19,6 +19,7 @@ import ( gateway "github.com/armadaproject/armada/internal/common/grpc" "github.com/armadaproject/armada/internal/common/health" log "github.com/armadaproject/armada/internal/common/logging" + "github.com/armadaproject/armada/internal/common/observability" "github.com/armadaproject/armada/internal/common/profiling" api "github.com/armadaproject/armada/pkg/api/binoculars" ) @@ -44,6 +45,16 @@ func main() { log.Info("Starting...") + // Initialize OpenTelemetry + if err := observability.InitOTel(config.Observability); err != nil { + log.Fatalf("Failed to initialize OTel: %v", err) + } + defer func() { + if err := observability.ShutdownWithDefaultTimeout(); err != nil { + log.Warnf("Failed to shutdown OTel: %v", err) + } + }() + // Expose profiling endpoints if enabled. err := profiling.SetupPprof(config.Profiling, armadacontext.Background(), nil) if err != nil { diff --git a/cmd/eventingester/main.go b/cmd/eventingester/main.go index c6cca9d2791..6141fc081ab 100644 --- a/cmd/eventingester/main.go +++ b/cmd/eventingester/main.go @@ -1,13 +1,16 @@ package main import ( - "github.com/armadaproject/armada/internal/common/logging" - "github.com/armadaproject/armada/internal/eventingester" + "context" + "time" "github.com/spf13/pflag" "github.com/spf13/viper" "github.com/armadaproject/armada/internal/common" + "github.com/armadaproject/armada/internal/common/logging" + "github.com/armadaproject/armada/internal/common/observability" + "github.com/armadaproject/armada/internal/eventingester" "github.com/armadaproject/armada/internal/eventingester/configuration" ) @@ -32,5 +35,18 @@ func main() { userSpecifiedConfigs := viper.GetStringSlice(CustomConfigLocation) common.LoadConfig(&config, "./config/eventingester", userSpecifiedConfigs) + + // Initialize OpenTelemetry + if err := observability.InitOTel(config.Observability); err != nil { + logging.Warnf("Failed to initialize OTel: %v", err) + } + defer func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := observability.ShutdownOTel(ctx); err != nil { + logging.Warnf("Failed to shutdown OTel: %v", err) + } + }() + eventingester.Run(&config) } diff --git a/cmd/executor/main.go b/cmd/executor/main.go index 75567416a1f..d1476099c53 100644 --- a/cmd/executor/main.go +++ b/cmd/executor/main.go @@ -14,6 +14,7 @@ import ( "github.com/armadaproject/armada/internal/common/armadacontext" "github.com/armadaproject/armada/internal/common/health" log "github.com/armadaproject/armada/internal/common/logging" + "github.com/armadaproject/armada/internal/common/observability" "github.com/armadaproject/armada/internal/common/profiling" "github.com/armadaproject/armada/internal/executor" "github.com/armadaproject/armada/internal/executor/configuration" @@ -38,6 +39,15 @@ func main() { userSpecifiedConfigs := viper.GetStringSlice(CustomConfigLocation) common.LoadConfig(&config, "./config/executor", userSpecifiedConfigs) + if err := observability.InitOTel(config.Observability); err != nil { + log.Fatalf("Failed to initialize OTel: %v", err) + } + defer func() { + if err := observability.ShutdownWithDefaultTimeout(); err != nil { + log.Warnf("Failed to shutdown OTel: %v", err) + } + }() + // Expose profiling endpoints if enabled. err := profiling.SetupPprof(config.Profiling, armadacontext.Background(), nil) if err != nil { diff --git a/cmd/lookout/main.go b/cmd/lookout/main.go index af1eaaa6962..85f85143a8c 100644 --- a/cmd/lookout/main.go +++ b/cmd/lookout/main.go @@ -14,6 +14,7 @@ import ( "github.com/armadaproject/armada/internal/common/armadacontext" "github.com/armadaproject/armada/internal/common/database" log "github.com/armadaproject/armada/internal/common/logging" + "github.com/armadaproject/armada/internal/common/observability" "github.com/armadaproject/armada/internal/common/profiling" "github.com/armadaproject/armada/internal/lookout" "github.com/armadaproject/armada/internal/lookout/configuration" @@ -163,6 +164,16 @@ func main() { userSpecifiedConfigs := viper.GetStringSlice(CustomConfigLocation) common.LoadConfig(&config, "./config/lookout", userSpecifiedConfigs) + // Initialize OpenTelemetry + if err := observability.InitOTel(config.Observability); err != nil { + log.Fatalf("Failed to initialize OTel: %v", err) + } + defer func() { + if err := observability.ShutdownWithDefaultTimeout(); err != nil { + log.Warnf("Failed to shutdown OTel: %v", err) + } + }() + // Expose profiling endpoints if enabled. err := profiling.SetupPprof(config.Profiling, armadacontext.Background(), nil) if err != nil { diff --git a/cmd/lookoutingester/main.go b/cmd/lookoutingester/main.go index 5f35380d8d8..501a7d8f806 100644 --- a/cmd/lookoutingester/main.go +++ b/cmd/lookoutingester/main.go @@ -6,6 +6,7 @@ import ( "github.com/armadaproject/armada/internal/common" log "github.com/armadaproject/armada/internal/common/logging" + "github.com/armadaproject/armada/internal/common/observability" "github.com/armadaproject/armada/internal/lookoutingester" "github.com/armadaproject/armada/internal/lookoutingester/benchmark" "github.com/armadaproject/armada/internal/lookoutingester/configuration" @@ -35,6 +36,16 @@ func main() { common.LoadConfig(&config, "./config/lookoutingester", userSpecifiedConfigs) + // Initialize OpenTelemetry + if err := observability.InitOTel(config.Observability); err != nil { + log.Fatalf("Failed to initialize OTel: %v", err) + } + defer func() { + if err := observability.ShutdownWithDefaultTimeout(); err != nil { + log.Warnf("Failed to shutdown OTel: %v", err) + } + }() + runBenchmarks := viper.GetBool(Benchmark) if runBenchmarks { log.Info("Running Lookout Ingester benchmarks") diff --git a/cmd/scheduleringester/main.go b/cmd/scheduleringester/main.go index 625b2a97da6..9327f6700e2 100644 --- a/cmd/scheduleringester/main.go +++ b/cmd/scheduleringester/main.go @@ -9,6 +9,7 @@ import ( "github.com/armadaproject/armada/internal/common" "github.com/armadaproject/armada/internal/common/logging" + "github.com/armadaproject/armada/internal/common/observability" "github.com/armadaproject/armada/internal/scheduleringester" ) @@ -32,6 +33,16 @@ func main() { common.LoadConfig(&config, "./config/scheduleringester", userSpecifiedConfigs) + // Initialize OpenTelemetry + if err := observability.InitOTel(config.Observability); err != nil { + logging.Fatalf("Failed to initialize OTel: %v", err) + } + defer func() { + if err := observability.ShutdownWithDefaultTimeout(); err != nil { + logging.Warnf("Failed to shutdown OTel: %v", err) + } + }() + if err := scheduleringester.Run(config); err != nil { fmt.Println(err) os.Exit(-1) diff --git a/cmd/server/main.go b/cmd/server/main.go index 38286f393b8..cab9c7bc0b7 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -17,6 +17,7 @@ import ( "github.com/armadaproject/armada/internal/common/health" "github.com/armadaproject/armada/internal/common/logging" log "github.com/armadaproject/armada/internal/common/logging" + "github.com/armadaproject/armada/internal/common/observability" "github.com/armadaproject/armada/internal/common/profiling" "github.com/armadaproject/armada/internal/server" "github.com/armadaproject/armada/internal/server/configuration" @@ -44,6 +45,16 @@ func main() { userSpecifiedConfigs := viper.GetStringSlice(CustomConfigLocation) common.LoadConfig(&config, "./config/server", userSpecifiedConfigs) + // Initialize OpenTelemetry + if err := observability.InitOTel(config.Observability); err != nil { + log.Fatalf("Failed to initialize OTel: %v", err) + } + defer func() { + if err := observability.ShutdownWithDefaultTimeout(); err != nil { + log.Warnf("Failed to shutdown OTel: %v", err) + } + }() + log.Info("Starting...") // Run services within an errgroup to propagate errors between services. diff --git a/go.mod b/go.mod index 826823dd072..9c0ed2da926 100644 --- a/go.mod +++ b/go.mod @@ -75,6 +75,8 @@ require ( github.com/segmentio/fasthash v1.0.3 github.com/xitongsys/parquet-go v1.6.2 github.com/zalando/go-keyring v0.2.6 + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.58.0 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 go.opentelemetry.io/otel v1.44.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.33.0 @@ -144,6 +146,7 @@ require ( github.com/emicklei/go-restful/v3 v3.13.0 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/fatih/camelcase v1.0.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/gabriel-vasile/mimetype v1.4.12 // indirect diff --git a/go.sum b/go.sum index eeb6deef37a..4e68a9215dc 100644 --- a/go.sum +++ b/go.sum @@ -1070,6 +1070,8 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.58.0 h1:PS8wXpbyaDJQ2VDHHncMe9Vct0Zn1fEjpsjrLxGJoSc= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.58.0/go.mod h1:HDBUsEjOuRC0EzKZ1bSaRGZWUBAzo+MhAcUUORSr4D0= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 h1:yd02MEjBdJkG3uabWP9apV+OuWRIXGDuJEUJbOHmCFU= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0/go.mod h1:umTcuxiv1n/s/S6/c2AT/g2CQ7u5C59sHDNmfSwgz7Q= go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= diff --git a/internal/binoculars/configuration/types.go b/internal/binoculars/configuration/types.go index a10c81f59c4..6ba8c1b1c18 100644 --- a/internal/binoculars/configuration/types.go +++ b/internal/binoculars/configuration/types.go @@ -3,6 +3,7 @@ package configuration import ( "github.com/armadaproject/armada/internal/common/auth/configuration" grpcconfig "github.com/armadaproject/armada/internal/common/grpc/configuration" + "github.com/armadaproject/armada/internal/common/observability" profilingconfig "github.com/armadaproject/armada/internal/common/profiling/configuration" ) @@ -10,10 +11,11 @@ type BinocularsConfig struct { Cordon CordonConfiguration Auth configuration.AuthConfig - GrpcPort uint16 - HttpPort uint16 - MetricsPort uint16 - Profiling *profilingconfig.ProfilingConfig + GrpcPort uint16 + HttpPort uint16 + MetricsPort uint16 + Profiling *profilingconfig.ProfilingConfig + Observability observability.ObservabilityConfig CorsAllowedOrigins []string diff --git a/internal/binoculars/configuration/validation.go b/internal/binoculars/configuration/validation.go index cd0d321ffd2..8597eed0efb 100644 --- a/internal/binoculars/configuration/validation.go +++ b/internal/binoculars/configuration/validation.go @@ -11,6 +11,7 @@ func (c BinocularsConfig) Validate() error { return validate.Struct(c) } -func (c BinocularsConfig) Mutate() (commonconfig.Config, error) { +func (c *BinocularsConfig) Mutate() (commonconfig.Config, error) { + c.Observability.ApplyResourceDefaults("binoculars") return c, nil } diff --git a/internal/common/build/build.go b/internal/common/build/build.go new file mode 100644 index 00000000000..deb7604e05e --- /dev/null +++ b/internal/common/build/build.go @@ -0,0 +1,3 @@ +package build + +var ReleaseVersion = "unknown" diff --git a/internal/common/grpc/gateway.go b/internal/common/grpc/gateway.go index 9ae6aeddd09..233b0fd28a6 100644 --- a/internal/common/grpc/gateway.go +++ b/internal/common/grpc/gateway.go @@ -10,6 +10,7 @@ import ( "github.com/go-openapi/runtime/middleware" "github.com/grpc-ecosystem/grpc-gateway/runtime" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "golang.org/x/exp/slices" "google.golang.org/grpc" "google.golang.org/grpc/credentials" @@ -59,12 +60,15 @@ func CreateGatewayHandler( } } + handler := otelhttp.NewHandler( + logRestRequests(allowCORS(gw, corsAllowedOrigins)), + "grpc-gateway", + ) + if stripPrefix { - prefixToStrip := strings.TrimSuffix(apiBasePath, "/") - mux.Handle(apiBasePath, http.StripPrefix(prefixToStrip, logRestRequests(allowCORS(gw, corsAllowedOrigins)))) - } else { - mux.Handle(apiBasePath, logRestRequests(allowCORS(gw, corsAllowedOrigins))) + handler = http.StripPrefix(strings.TrimSuffix(apiBasePath, "/"), handler) } + mux.Handle(apiBasePath, handler) mux.Handle(path.Join(apiBasePath, "swagger.json"), middleware.Spec(apiBasePath, []byte(spec), nil)) return func() { diff --git a/internal/common/grpc/grpc.go b/internal/common/grpc/grpc.go index d678580ccf5..2bd5c400afd 100644 --- a/internal/common/grpc/grpc.go +++ b/internal/common/grpc/grpc.go @@ -15,6 +15,7 @@ import ( grpc_recovery "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/recovery" "github.com/prometheus/client_golang/prometheus" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" @@ -51,6 +52,7 @@ func CreateGrpcServer( grpc.KeepaliveParams(keepaliveParams), grpc.KeepaliveEnforcementPolicy(keepaliveEnforcementPolicy), setupTls(tlsConfig), + grpc.StatsHandler(otelgrpc.NewServerHandler()), grpc.ChainUnaryInterceptor( requestid.UnaryServerInterceptor(false), grpc_auth.UnaryServerInterceptor(authFunction), @@ -81,7 +83,7 @@ func setupPromMetrics() *grpc_prometheus.ServerMetrics { ), grpc_prometheus.WithContextLabels("user"), ) - prometheus.MustRegister(srvMetrics) + prometheus.DefaultRegisterer.MustRegister(srvMetrics) return srvMetrics } diff --git a/internal/common/logging/logger.go b/internal/common/logging/logger.go index 3763fe02c47..427032c7ca3 100644 --- a/internal/common/logging/logger.go +++ b/internal/common/logging/logger.go @@ -1,10 +1,14 @@ package logging import ( + "context" "fmt" "github.com/pkg/errors" "github.com/rs/zerolog" + "go.opentelemetry.io/otel/trace" + + "github.com/armadaproject/armada/internal/common/requestid" ) // Logger wraps a zerolog.Logger so that the rest of the code doesn't depend directly on zerolog @@ -148,6 +152,37 @@ func (l *Logger) WithFields(args map[string]any) *Logger { } } +// WithContext returns a new Logger that extracts and adds trace context fields (trace_id, span_id) +// from the provided context. It also preserves the x-request-id if present. +// If no trace context is available or the span is invalid, returns the logger unchanged. +func (l *Logger) WithContext(ctx context.Context) *Logger { + if ctx == nil { + return l + } + + fields := make(map[string]any) + + // Extract trace context from OTel + span := trace.SpanFromContext(ctx) + spanCtx := span.SpanContext() + if spanCtx.IsValid() { + fields["trace_id"] = spanCtx.TraceID().String() + fields["span_id"] = spanCtx.SpanID().String() + } + + // Preserve existing x-request-id if present + if reqID, ok := requestid.FromContext(ctx); ok { + fields["x-request-id"] = reqID + } + + // If no fields were extracted, return the logger unchanged + if len(fields) == 0 { + return l + } + + return l.WithFields(fields) +} + // WithCallerSkip returns a new Logger with the number of callers skipped increased by the skip amount. // This is needed when building wrappers around the Logger so as to prevent us from always reporting the // wrapper code as the caller. diff --git a/internal/common/observability/config.go b/internal/common/observability/config.go index 4a9ab7eeaa9..495f5fadce2 100644 --- a/internal/common/observability/config.go +++ b/internal/common/observability/config.go @@ -4,7 +4,10 @@ import ( "fmt" "maps" "net/url" + "os" "strings" + + "github.com/armadaproject/armada/internal/common/build" ) const ( @@ -33,6 +36,8 @@ var validOTLPProtocols = map[string]struct{}{ "grpc": {}, } +const unknownResourceValue = "unknown" + // ResourceAttributes define the required OpenTelemetry service identity contract. type ResourceAttributes struct { // Service name is the name of the service. @@ -74,6 +79,39 @@ type ObservabilityConfig struct { Resource ResourceAttributes } +func (c *ObservabilityConfig) ApplyResourceDefaults(serviceName string) { + if !c.Enabled { + return + } + + serviceName = strings.TrimSpace(serviceName) + if strings.TrimSpace(c.Resource.ServiceName) == "" && serviceName != "" { + c.Resource.ServiceName = "armada-" + strings.TrimPrefix(serviceName, "armada-") + } + if strings.TrimSpace(c.Resource.ServiceVersion) == "" { + c.Resource.ServiceVersion = defaultServiceVersion() + } + if strings.TrimSpace(c.Resource.ServiceInstance) == "" { + c.Resource.ServiceInstance = defaultServiceInstance() + } +} + +func defaultServiceVersion() string { + version := strings.TrimSpace(build.ReleaseVersion) + if version != "" { + return version + } + return unknownResourceValue +} + +func defaultServiceInstance() string { + hostname, err := os.Hostname() + if err == nil && strings.TrimSpace(hostname) != "" { + return hostname + } + return unknownResourceValue +} + func (c ObservabilityConfig) Validate() error { if strings.TrimSpace(c.Exporter.Endpoint) == "" { return fmt.Errorf("%s must not be empty", ConfigOtelExporterOtlpEndpoint) diff --git a/internal/common/observability/config_test.go b/internal/common/observability/config_test.go index 18f4dab9cc6..8ee8bd58947 100644 --- a/internal/common/observability/config_test.go +++ b/internal/common/observability/config_test.go @@ -5,6 +5,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/armadaproject/armada/internal/common/build" ) const ( @@ -74,6 +76,49 @@ func TestObservabilityConfig(t *testing.T) { }) } +func TestObservabilityConfigApplyResourceDefaults(t *testing.T) { + t.Run("defaults missing resource attributes when enabled", func(t *testing.T) { + previousReleaseVersion := build.ReleaseVersion + build.ReleaseVersion = "test-version" + defer func() { build.ReleaseVersion = previousReleaseVersion }() + + cfg := ObservabilityConfig{Enabled: true} + + cfg.ApplyResourceDefaults("server") + + assert.Equal(t, "armada-server", cfg.Resource.ServiceName) + assert.Equal(t, "test-version", cfg.Resource.ServiceVersion) + assert.NotEmpty(t, cfg.Resource.ServiceInstance) + }) + + t.Run("preserves configured resource attributes", func(t *testing.T) { + cfg := ObservabilityConfig{ + Enabled: true, + Resource: ResourceAttributes{ + ServiceName: "custom-service", + ServiceVersion: "custom-version", + ServiceInstance: "custom-instance", + }, + } + + cfg.ApplyResourceDefaults("server") + + assert.Equal(t, "custom-service", cfg.Resource.ServiceName) + assert.Equal(t, "custom-version", cfg.Resource.ServiceVersion) + assert.Equal(t, "custom-instance", cfg.Resource.ServiceInstance) + }) + + t.Run("leaves disabled config untouched", func(t *testing.T) { + cfg := ObservabilityConfig{} + + cfg.ApplyResourceDefaults("server") + + assert.Empty(t, cfg.Resource.ServiceName) + assert.Empty(t, cfg.Resource.ServiceVersion) + assert.Empty(t, cfg.Resource.ServiceInstance) + }) +} + func TestObservabilityConfigRejectsInvalidSampler(t *testing.T) { err := (ObservabilityConfig{ Exporter: OTLPExporterConfig{ diff --git a/internal/common/observability/lifecycle.go b/internal/common/observability/lifecycle.go index ff6b18a37f7..f8bd6867402 100644 --- a/internal/common/observability/lifecycle.go +++ b/internal/common/observability/lifecycle.go @@ -68,6 +68,10 @@ func InitOTel(cfg ObservabilityConfig) error { return nil } + if err := cfg.Validate(); err != nil { + return err + } + attrs := []attribute.KeyValue{ attribute.String(ResourceAttributeServiceName, cfg.Resource.ServiceName), attribute.String(ResourceAttributeServiceVersion, cfg.Resource.ServiceVersion), diff --git a/internal/common/observability/lifecycle_bootstrap_test.go b/internal/common/observability/lifecycle_bootstrap_test.go index bc96623f5fb..10bd00bebaa 100644 --- a/internal/common/observability/lifecycle_bootstrap_test.go +++ b/internal/common/observability/lifecycle_bootstrap_test.go @@ -7,25 +7,25 @@ import ( "github.com/google/uuid" "github.com/stretchr/testify/require" "go.opentelemetry.io/otel" - - "github.com/armadaproject/armada/internal/lookout/version" ) +const ServiceVersion = "0.1.0" + func TestServiceBootstrapPatternServerConfig(t *testing.T) { - cfg := testBootstrapConfig("server", version.Version, uuid.New().String()) + cfg := testBootstrapConfig("server", ServiceVersion, uuid.New().String()) require.NoError(t, cfg.Validate()) require.Equal(t, "server", cfg.Resource.ServiceName) require.NotEmpty(t, cfg.Resource.ServiceInstance) } func TestServiceBootstrapPatternExecutorConfig(t *testing.T) { - cfg := testBootstrapConfig("executor", version.Version, uuid.New().String()) + cfg := testBootstrapConfig("executor", ServiceVersion, uuid.New().String()) require.NoError(t, cfg.Validate()) require.Equal(t, "executor", cfg.Resource.ServiceName) } func TestServiceBootstrapPatternSchedulerConfig(t *testing.T) { - cfg := testBootstrapConfig("scheduler", version.Version, uuid.New().String()) + cfg := testBootstrapConfig("scheduler", ServiceVersion, uuid.New().String()) require.NoError(t, cfg.Validate()) require.Equal(t, "scheduler", cfg.Resource.ServiceName) } @@ -98,7 +98,7 @@ func TestBootstrapResourceAttributesFromConfig(t *testing.T) { } func TestBootstrapInitShutdownMultipleTimes(t *testing.T) { - for i := 0; i < 3; i++ { + for range 3 { cfg := ObservabilityConfig{ Enabled: true, Exporter: OTLPExporterConfig{ @@ -131,7 +131,7 @@ func TestBootstrapInitShutdownMultipleTimes(t *testing.T) { } } -func testBootstrapConfig(serviceName, serviceVersion, serviceInstance string) ObservabilityConfig { +func testBootstrapConfig(serviceName, serviceServiceVersion, serviceInstance string) ObservabilityConfig { return ObservabilityConfig{ Exporter: OTLPExporterConfig{ Endpoint: "http://otel-collector:4318", @@ -143,7 +143,7 @@ func testBootstrapConfig(serviceName, serviceVersion, serviceInstance string) Ob }, Resource: ResourceAttributes{ ServiceName: serviceName, - ServiceVersion: serviceVersion, + ServiceVersion: serviceServiceVersion, ServiceInstance: serviceInstance, }, } diff --git a/internal/common/observability/lifecycle_test.go b/internal/common/observability/lifecycle_test.go index 9d299bb0643..a6a80b1935c 100644 --- a/internal/common/observability/lifecycle_test.go +++ b/internal/common/observability/lifecycle_test.go @@ -595,8 +595,9 @@ func TestOtelInitWithInvalidResourceAttributes(t *testing.T) { }, } - err := cfg.Validate() - assert.Error(t, err, "Config validation should fail with empty service name") + err := InitOTel(cfg) + assert.Error(t, err, "InitOTel should reject empty service name") + assert.ErrorContains(t, err, ResourceAttributeServiceName) } func TestOtelShutdownWithoutInit(t *testing.T) { diff --git a/internal/eventingester/configuration/types.go b/internal/eventingester/configuration/types.go index c7fa60c77dd..d0100fbb8c8 100644 --- a/internal/eventingester/configuration/types.go +++ b/internal/eventingester/configuration/types.go @@ -6,6 +6,7 @@ import ( "github.com/redis/go-redis/v9" commonconfig "github.com/armadaproject/armada/internal/common/config" + "github.com/armadaproject/armada/internal/common/observability" profilingconfig "github.com/armadaproject/armada/internal/common/profiling/configuration" "github.com/armadaproject/armada/internal/leaderelection" ) @@ -20,6 +21,8 @@ type EventIngesterConfiguration struct { MetricsPort uint16 // Metrics configuration for Redis memory metrics collection Metrics MetricsConfig + // Configuration controlling OpenTelemetry observability + Observability observability.ObservabilityConfig // General Pulsar configuration Pulsar commonconfig.PulsarConfig // Pulsar subscription name diff --git a/internal/eventingester/configuration/validation.go b/internal/eventingester/configuration/validation.go index cc347cb1f9a..03940bb2251 100644 --- a/internal/eventingester/configuration/validation.go +++ b/internal/eventingester/configuration/validation.go @@ -11,6 +11,7 @@ func (c EventIngesterConfiguration) Validate() error { return validate.Struct(c) } -func (c EventIngesterConfiguration) Mutate() (commonconfig.Config, error) { +func (c *EventIngesterConfiguration) Mutate() (commonconfig.Config, error) { + c.Observability.ApplyResourceDefaults("eventingester") return c, nil } diff --git a/internal/executor/configuration/types.go b/internal/executor/configuration/types.go index c8665aae75d..be0bf128e9d 100644 --- a/internal/executor/configuration/types.go +++ b/internal/executor/configuration/types.go @@ -5,6 +5,7 @@ import ( "google.golang.org/grpc/keepalive" + "github.com/armadaproject/armada/internal/common/observability" profilingconfig "github.com/armadaproject/armada/internal/common/profiling/configuration" armadaresource "github.com/armadaproject/armada/internal/common/resource" "github.com/armadaproject/armada/internal/executor/categorizer" @@ -178,6 +179,7 @@ type ExecutorConfiguration struct { HttpPort uint16 // If non-nil, net/http/pprof endpoints are exposed on localhost on this port. Profiling *profilingconfig.ProfilingConfig + Observability observability.ObservabilityConfig Metric MetricConfiguration Application ApplicationConfiguration ExecutorApiConnection client.ApiConnectionDetails diff --git a/internal/executor/configuration/validation.go b/internal/executor/configuration/validation.go index 8106459223e..92e98b22841 100644 --- a/internal/executor/configuration/validation.go +++ b/internal/executor/configuration/validation.go @@ -11,6 +11,7 @@ func (c ExecutorConfiguration) Validate() error { return validate.Struct(c) } -func (c ExecutorConfiguration) Mutate() (commonconfig.Config, error) { +func (c *ExecutorConfiguration) Mutate() (commonconfig.Config, error) { + c.Observability.ApplyResourceDefaults("executor") return c, nil } diff --git a/internal/lookout/configuration/types.go b/internal/lookout/configuration/types.go index 4dff8a42fec..77ba9a13baf 100644 --- a/internal/lookout/configuration/types.go +++ b/internal/lookout/configuration/types.go @@ -5,6 +5,7 @@ import ( authconfig "github.com/armadaproject/armada/internal/common/auth/configuration" "github.com/armadaproject/armada/internal/common/database" + "github.com/armadaproject/armada/internal/common/observability" profilingconfig "github.com/armadaproject/armada/internal/common/profiling/configuration" "github.com/armadaproject/armada/internal/server/configuration" ) @@ -12,9 +13,10 @@ import ( type LookoutConfig struct { Auth authconfig.AuthConfig - ApiPort int - Profiling *profilingconfig.ProfilingConfig - MetricsPort int + ApiPort int + Profiling *profilingconfig.ProfilingConfig + MetricsPort int + Observability observability.ObservabilityConfig CorsAllowedOrigins []string Tls TlsConfig diff --git a/internal/lookout/configuration/validation.go b/internal/lookout/configuration/validation.go index a8060d5ed31..f53a8807d48 100644 --- a/internal/lookout/configuration/validation.go +++ b/internal/lookout/configuration/validation.go @@ -11,6 +11,7 @@ func (c LookoutConfig) Validate() error { return validate.Struct(c) } -func (c LookoutConfig) Mutate() (commonconfig.Config, error) { +func (c *LookoutConfig) Mutate() (commonconfig.Config, error) { + c.Observability.ApplyResourceDefaults("lookout") return c, nil } diff --git a/internal/lookoutingester/configuration/types.go b/internal/lookoutingester/configuration/types.go index 1402454c513..7bebcadf7c6 100644 --- a/internal/lookoutingester/configuration/types.go +++ b/internal/lookoutingester/configuration/types.go @@ -6,6 +6,7 @@ import ( log "github.com/armadaproject/armada/internal/common/logging" commonconfig "github.com/armadaproject/armada/internal/common/config" + "github.com/armadaproject/armada/internal/common/observability" profilingconfig "github.com/armadaproject/armada/internal/common/profiling/configuration" "github.com/armadaproject/armada/internal/server/configuration" ) @@ -15,6 +16,8 @@ type LookoutIngesterConfiguration struct { Postgres configuration.PostgresConfig // Metrics configuration MetricsPort uint16 + // Configuration controlling OpenTelemetry observability + Observability observability.ObservabilityConfig // General Pulsar configuration Pulsar commonconfig.PulsarConfig // Pulsar subscription name diff --git a/internal/lookoutingester/configuration/validation.go b/internal/lookoutingester/configuration/validation.go index 5fe6ace8da3..288be88ebe4 100644 --- a/internal/lookoutingester/configuration/validation.go +++ b/internal/lookoutingester/configuration/validation.go @@ -11,6 +11,7 @@ func (c LookoutIngesterConfiguration) Validate() error { return validate.Struct(c) } -func (c LookoutIngesterConfiguration) Mutate() (commonconfig.Config, error) { +func (c *LookoutIngesterConfiguration) Mutate() (commonconfig.Config, error) { + c.Observability.ApplyResourceDefaults("lookoutingester") return c, nil } diff --git a/internal/scheduler/configuration/configuration.go b/internal/scheduler/configuration/configuration.go index 470d0e03f7f..eb937508a65 100644 --- a/internal/scheduler/configuration/configuration.go +++ b/internal/scheduler/configuration/configuration.go @@ -10,6 +10,7 @@ import ( commonconfig "github.com/armadaproject/armada/internal/common/config" "github.com/armadaproject/armada/internal/common/database" grpcconfig "github.com/armadaproject/armada/internal/common/grpc/configuration" + "github.com/armadaproject/armada/internal/common/observability" profilingconfig "github.com/armadaproject/armada/internal/common/profiling/configuration" armadaresource "github.com/armadaproject/armada/internal/common/resource" "github.com/armadaproject/armada/internal/common/types" @@ -38,6 +39,8 @@ type Configuration struct { Leader leaderelection.Config // Configuration controlling metrics Metrics MetricsConfig + // Configuration controlling OpenTelemetry observability + Observability observability.ObservabilityConfig // Scheduler configuration (this is shared with the old scheduler) Scheduling SchedulingConfig Auth authconfig.AuthConfig diff --git a/internal/scheduler/configuration/validation.go b/internal/scheduler/configuration/validation.go index a6f8268cf6f..133f7f4d73a 100644 --- a/internal/scheduler/configuration/validation.go +++ b/internal/scheduler/configuration/validation.go @@ -10,6 +10,8 @@ import ( ) func (c *Configuration) Mutate() (config.Config, error) { + c.Observability.ApplyResourceDefaults("scheduler") + if c.MaxSchedulingDuration > 0 { log.Warnf("use of top level MaxSchedulingDuration has been deprecated - please use scheduling.MaxSchedulingDuration. Applying MaxSchedulingDuration to scheduling.MaxSchedulingDuration") c.Scheduling.MaxSchedulingDuration = c.MaxSchedulingDuration diff --git a/internal/scheduler/configuration/validation_test.go b/internal/scheduler/configuration/validation_test.go index c42e3026345..f7961e8f2eb 100644 --- a/internal/scheduler/configuration/validation_test.go +++ b/internal/scheduler/configuration/validation_test.go @@ -8,6 +8,7 @@ import ( v1 "k8s.io/api/core/v1" commonconfig "github.com/armadaproject/armada/internal/common/config" + "github.com/armadaproject/armada/internal/common/observability" "github.com/armadaproject/armada/internal/common/types" "github.com/armadaproject/armada/internal/leaderelection" schedulerdb "github.com/armadaproject/armada/internal/scheduler/database" @@ -68,17 +69,70 @@ func TestMutate(t *testing.T) { }, }, }, + "Observability - preserves configured value": { + input: &Configuration{ + Observability: observability.ObservabilityConfig{ + Enabled: true, + Exporter: observability.OTLPExporterConfig{ + Endpoint: "http://otel-collector:4318", + Protocol: "http/protobuf", + }, + Traces: observability.TracesConfig{ + Sampler: "parent_based_trace_id_ratio", + SamplerArg: 0.25, + }, + Resource: observability.ResourceAttributes{ + ServiceName: "scheduler", + ServiceVersion: "configured-version", + ServiceInstance: "configured-instance", + }, + }, + }, + expected: &Configuration{ + Observability: observability.ObservabilityConfig{ + Enabled: true, + Exporter: observability.OTLPExporterConfig{ + Endpoint: "http://otel-collector:4318", + Protocol: "http/protobuf", + }, + Traces: observability.TracesConfig{ + Sampler: "parent_based_trace_id_ratio", + SamplerArg: 0.25, + }, + Resource: observability.ResourceAttributes{ + ServiceName: "scheduler", + ServiceVersion: "configured-version", + ServiceInstance: "configured-instance", + }, + }, + }, + }, } for name, tc := range tests { t.Run(name, func(t *testing.T) { result, err := tc.input.Mutate() assert.NoError(t, err) + assert.Equal(t, tc.expected, result) }) } } +func TestMutateAppliesObservabilityResourceDefaults(t *testing.T) { + config := &Configuration{ + Observability: observability.ObservabilityConfig{Enabled: true}, + } + + result, err := config.Mutate() + assert.NoError(t, err) + + mutated := result.(*Configuration) + assert.Equal(t, "armada-scheduler", mutated.Observability.Resource.ServiceName) + assert.NotEmpty(t, mutated.Observability.Resource.ServiceVersion) + assert.NotEmpty(t, mutated.Observability.Resource.ServiceInstance) +} + func TestValidate_SchedulingTimeoutConfig(t *testing.T) { tests := map[string]struct { config func(c Configuration) Configuration diff --git a/internal/scheduler/schedulerapp.go b/internal/scheduler/schedulerapp.go index c12f4420006..3213956d6ad 100644 --- a/internal/scheduler/schedulerapp.go +++ b/internal/scheduler/schedulerapp.go @@ -25,6 +25,7 @@ import ( grpcCommon "github.com/armadaproject/armada/internal/common/grpc" "github.com/armadaproject/armada/internal/common/health" log "github.com/armadaproject/armada/internal/common/logging" + "github.com/armadaproject/armada/internal/common/observability" "github.com/armadaproject/armada/internal/common/profiling" "github.com/armadaproject/armada/internal/common/pulsarutils" "github.com/armadaproject/armada/internal/common/pulsarutils/jobsetevents" @@ -56,6 +57,18 @@ import ( func Run(config schedulerconfig.Configuration) error { g, ctx := armadacontext.ErrGroup(app.CreateContextWithShutdown()) + // //////////////////////////////////////////////////////////////////////// + // OpenTelemetry + // //////////////////////////////////////////////////////////////////////// + if err := observability.InitOTel(config.Observability); err != nil { + log.Fatalf("Failed to initialize OTel: %v", err) + } + defer func() { + if err := observability.ShutdownWithDefaultTimeout(); err != nil { + log.Warnf("Failed to shutdown OTel: %v", err) + } + }() + // //////////////////////////////////////////////////////////////////////// // Expose profiling endpoints if enabled. // //////////////////////////////////////////////////////////////////////// diff --git a/internal/scheduleringester/config.go b/internal/scheduleringester/config.go index 7ae94a45c44..09d1d6a57b8 100644 --- a/internal/scheduleringester/config.go +++ b/internal/scheduleringester/config.go @@ -6,6 +6,7 @@ import ( "github.com/go-playground/validator/v10" commonconfig "github.com/armadaproject/armada/internal/common/config" + "github.com/armadaproject/armada/internal/common/observability" profilingconfig "github.com/armadaproject/armada/internal/common/profiling/configuration" schedulerdb "github.com/armadaproject/armada/internal/scheduler/database" "github.com/armadaproject/armada/internal/server/configuration" @@ -16,6 +17,8 @@ type Configuration struct { Postgres configuration.PostgresConfig // Metrics Port MetricsPort uint16 + // Configuration controlling OpenTelemetry observability + Observability observability.ObservabilityConfig // General Pulsar configuration Pulsar commonconfig.PulsarConfig // Pulsar subscription name @@ -32,7 +35,8 @@ type Configuration struct { JobMetadataMigrationPhase schedulerdb.JobMetadataMigrationPhase `validate:"required,oneof=legacy dualWrite cutover"` } -func (c Configuration) Mutate() (commonconfig.Config, error) { +func (c *Configuration) Mutate() (commonconfig.Config, error) { + c.Observability.ApplyResourceDefaults("scheduleringester") return c, nil } diff --git a/internal/server/configuration/types.go b/internal/server/configuration/types.go index 509c0ed330e..c76f654d008 100644 --- a/internal/server/configuration/types.go +++ b/internal/server/configuration/types.go @@ -9,6 +9,7 @@ import ( authconfig "github.com/armadaproject/armada/internal/common/auth/configuration" commonconfig "github.com/armadaproject/armada/internal/common/config" grpcconfig "github.com/armadaproject/armada/internal/common/grpc/configuration" + "github.com/armadaproject/armada/internal/common/observability" profilingconfig "github.com/armadaproject/armada/internal/common/profiling/configuration" armadaresource "github.com/armadaproject/armada/internal/common/resource" "github.com/armadaproject/armada/pkg/client" @@ -17,10 +18,11 @@ import ( type ArmadaConfig struct { Auth authconfig.AuthConfig - GrpcPort uint16 - HttpPort uint16 - MetricsPort uint16 - Profiling *profilingconfig.ProfilingConfig + GrpcPort uint16 + HttpPort uint16 + MetricsPort uint16 + Profiling *profilingconfig.ProfilingConfig + Observability observability.ObservabilityConfig CorsAllowedOrigins []string GrpcGatewayPath string diff --git a/internal/server/configuration/validation.go b/internal/server/configuration/validation.go index ca7134db28c..392f166e8f7 100644 --- a/internal/server/configuration/validation.go +++ b/internal/server/configuration/validation.go @@ -11,6 +11,7 @@ func (c ArmadaConfig) Validate() error { return validate.Struct(c) } -func (c ArmadaConfig) Mutate() (commonconfig.Config, error) { +func (c *ArmadaConfig) Mutate() (commonconfig.Config, error) { + c.Observability.ApplyResourceDefaults("server") return c, nil } diff --git a/pkg/client/connection.go b/pkg/client/connection.go index 2b9dacc244b..a7ff9c80b2f 100644 --- a/pkg/client/connection.go +++ b/pkg/client/connection.go @@ -8,6 +8,7 @@ import ( grpc_retry "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/retry" "github.com/pkg/errors" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" @@ -69,6 +70,7 @@ func CreateApiConnectionWithCallOptions( unuaryInterceptors := grpc.WithChainUnaryInterceptor(grpc_retry.UnaryClientInterceptor(retryOpts...)) streamInterceptors := grpc.WithChainStreamInterceptor(grpc_retry.StreamClientInterceptor(retryOpts...)) dialOpts := append(additionalDialOptions, + grpc.WithStatsHandler(otelgrpc.NewClientHandler()), defaultCallOptions, unuaryInterceptors, streamInterceptors, diff --git a/website/content/docs/developer-guide.mdx b/website/content/docs/developer-guide.mdx index 4be0335c535..39393d0cd96 100644 --- a/website/content/docs/developer-guide.mdx +++ b/website/content/docs/developer-guide.mdx @@ -387,7 +387,7 @@ mage dev:fullDown # stop the containerized stack and tear down Kind (after `ma docker compose -f _local/compose/stack.yaml up -d ``` - **Note:** Images can be overridden using environment variables: `REDIS_IMAGE`, `POSTGRES_IMAGE`, `PULSAR_IMAGE`, `KEYCLOAK_IMAGE` + **Note:** Images can be overridden using environment variables: `REDIS_IMAGE`, `POSTGRES_IMAGE`, `PULSAR_IMAGE`, `KEYCLOAK_IMAGE`, `OTEL_IMAGE`, `JAEGER_IMAGE`, `GRAFANA_IMAGE` 3. Initialize databases and Kubernetes resources: From 02ba2d2ab638a1df6e4d270df647ac8ae2e877fd Mon Sep 17 00:00:00 2001 From: JamesMurkin Date: Mon, 6 Jul 2026 14:02:18 +0100 Subject: [PATCH 22/49] Improve the main scheduler loop (#4993) Main changes: - Scheduling cycles are now triggered on a regular cadence where possible - Previously schedulePeriod would determine how long we'd wait between cycles. Measured since last cycle finished - The issue with this is for long cycles, ideally we'd trigger again immediately, instead we leave a gap. - This was slowing scheduling down when it already slow - Now we trigger the next cycle based on when the last cycle started - For quick cycles, we'll still get a gap between them - For long cycles we'll trigger again immediately - Make the main loop check for an async scheduling result every cycle - This makes us more responsive to async scheduler results - Make UpdateJobPrices be called every cycle - Update job prices has now been made much more efficient (most calls are a noop) - The ensures we keep as up to date as we can, while also simplifying logic of when to call it --------- Signed-off-by: JamesMurkin Signed-off-by: sarhiri --- internal/scheduler/scheduler.go | 73 +++++++++---------- internal/scheduler/scheduler_test.go | 3 +- internal/scheduler/scheduling/runner/async.go | 5 +- .../scheduler/scheduling/runner/async_test.go | 22 ++++++ internal/scheduler/scheduling/runner/sync.go | 2 +- internal/scheduler/scheduling/runner/types.go | 6 +- 6 files changed, 68 insertions(+), 43 deletions(-) diff --git a/internal/scheduler/scheduler.go b/internal/scheduler/scheduler.go index 2c8df7d0113..0e362982c45 100644 --- a/internal/scheduler/scheduler.go +++ b/internal/scheduler/scheduler.go @@ -159,7 +159,7 @@ func (s *Scheduler) Run(ctx *armadacontext.Context) error { ticker := s.clock.NewTicker(s.cyclePeriod) prevLeaderToken := leaderelection.InvalidLeaderToken() - previousSchedulingRoundEnd := time.Time{} + lastScheduleStart := time.Time{} cycleNumber := 0 for { select { @@ -199,29 +199,19 @@ func (s *Scheduler) Run(ctx *armadacontext.Context) error { // Run a scheduler cycle. // - // If there is an error, we can't guarantee that the scheduler-internal state is consistent with what was published - // (scheduling decisions may have been partially published) - // and we must invalidate the held leader token to trigger flushing Pulsar at the next cycle. - // - // TODO: Once the Pulsar client supports transactions, we can guarantee consistency even in case of errors. - shouldSchedule := s.clock.Now().Sub(previousSchedulingRoundEnd) > s.schedulePeriod - if !shouldSchedule { - ctx.Info("Won't schedule this cycle as still within schedulePeriod") - } - - schedulingAttempted, err := s.cycle(ctx, fullUpdate, leaderToken, shouldSchedule, cycleNumber) - if shouldSchedule { - previousSchedulingRoundEnd = s.clock.Now() + // We trigger a new cycle when the elapsed time since last cycle start is > schedulePeriod + // The reason we measure since start time, is so when the scheduling cycles are long, + // we retrigger a new cycle immediately without gap + shouldTriggerScheduling := s.clock.Now().Sub(lastScheduleStart) >= s.schedulePeriod + if !shouldTriggerScheduling { + ctx.Info("Won't start scheduling this cycle; still within schedulePeriod") } + shouldGetSchedulerResult := s.runner.IsAsync() || shouldTriggerScheduling + schedulingAttempted, err := s.cycle(ctx, fullUpdate, leaderToken, shouldGetSchedulerResult, cycleNumber) cycleTime := s.clock.Since(start) - isSchedulingCycle := shouldSchedule && leaderToken.Leader() - if s.runner.IsAsync() { - isSchedulingCycle = leaderToken.Leader() && schedulingAttempted - } - - if isSchedulingCycle { + if schedulingAttempted { // Only the leader does real scheduling rounds. s.metrics.ReportScheduleCycleTime(cycleTime) s.metrics.ReportScheduleCycleOutcome(err == nil) @@ -232,16 +222,26 @@ func (s *Scheduler) Run(ctx *armadacontext.Context) error { } if err != nil { + // If there is an error, we can't guarantee that the scheduler-internal state is consistent + // with what was published (scheduling decisions may have been partially published) and we + // must invalidate the held leader token to trigger flushing Pulsar at the next cycle. + // + // TODO: Once the Pulsar client supports transactions, we can guarantee consistency even in case of errors. ctx.Logger().WithStacktrace(err).Error("cycle failure") leaderToken = leaderelection.InvalidLeaderToken() } - // Kick off the next async run only after a clean cycle as leader, - // so the background run schedules against committed state. On error - // leaderToken was invalidated above, so a failed cycle won't trigger - // a run that would be discarded. In sync mode Trigger is a no-op. - if shouldSchedule && err == nil && leaderToken.Leader() { - s.runner.Trigger() + if !s.runner.IsAsync() { + if shouldTriggerScheduling { + lastScheduleStart = start + } + } else { + if shouldTriggerScheduling && s.leaderController.ValidateToken(leaderToken) { + triggered := s.runner.Trigger() + if triggered { + lastScheduleStart = start + } + } } prevLeaderToken = leaderToken @@ -278,7 +278,7 @@ func (s *Scheduler) Run(ctx *armadacontext.Context) error { // This means we can start the next cycle immediately after one cycle finishes. // As state transitions are persisted and read back from the schedulerDb over later cycles, // there is no change to the jobDb, since the correct changes have already been made. -func (s *Scheduler) cycle(ctx *armadacontext.Context, updateAll bool, leaderToken leaderelection.LeaderToken, shouldSchedule bool, cycleNumber int) (bool, error) { +func (s *Scheduler) cycle(ctx *armadacontext.Context, updateAll bool, leaderToken leaderelection.LeaderToken, shouldGetSchedulingResult bool, cycleNumber int) (bool, error) { ctx.Logger().Infof("starting cycle") defer func(ctx *armadacontext.Context) { ctx.Logger().Infof("finished cycle") @@ -363,16 +363,15 @@ func (s *Scheduler) cycle(ctx *armadacontext.Context, updateAll bool, leaderToke ctx.Infof("Finished looking for jobs to expire, generating %d events", len(expirationEvents)) events = append(events, expirationEvents...) - var schedulerResult *scheduling.SchedulerResult - // Schedule jobs. - if shouldSchedule { - start := time.Now() - err = s.updateJobPrices(ctx, txn) - if err != nil { - return false, err - } - ctx.Logger().Infof("updating job prices in %s", time.Now().Sub(start)) + start := s.clock.Now() + err = s.updateJobPrices(ctx, txn) + if err != nil { + return false, err + } + ctx.Logger().Infof("updating job prices in %s", s.clock.Now().Sub(start)) + var schedulerResult *scheduling.SchedulerResult + if shouldGetSchedulingResult { var result *scheduling.SchedulerResult result, err = s.runner.GetSchedulerResult(ctx, txn) if err != nil { @@ -393,7 +392,7 @@ func (s *Scheduler) cycle(ctx *armadacontext.Context, updateAll bool, leaderToke isLeader := func() bool { return s.leaderController.ValidateToken(leaderToken) } - start := s.clock.Now() + start = s.clock.Now() ctx.Infof("Starting to publish %d eventSequences to pulsar", len(events)) if err = s.publisher.PublishMessages(ctx, events, isLeader); err != nil { return schedulerResult != nil, err diff --git a/internal/scheduler/scheduler_test.go b/internal/scheduler/scheduler_test.go index 72a37dbf186..65db467b204 100644 --- a/internal/scheduler/scheduler_test.go +++ b/internal/scheduler/scheduler_test.go @@ -1688,10 +1688,11 @@ type recordingRunner struct { triggerCalls int } -func (r *recordingRunner) Trigger() { +func (r *recordingRunner) Trigger() bool { r.mu.Lock() defer r.mu.Unlock() r.triggerCalls++ + return true } func (r *recordingRunner) GetSchedulerResult(_ *armadacontext.Context, _ *jobdb.Txn) (*scheduling.SchedulerResult, error) { diff --git a/internal/scheduler/scheduling/runner/async.go b/internal/scheduler/scheduling/runner/async.go index 0163f485bff..193b9ce1f28 100644 --- a/internal/scheduler/scheduling/runner/async.go +++ b/internal/scheduler/scheduling/runner/async.go @@ -78,13 +78,13 @@ func NewAsyncSchedulingRunner(ctx *armadacontext.Context, schedulingAlgo schedul return r } -func (r *AsyncSchedulingRunner) Trigger() { +func (r *AsyncSchedulingRunner) Trigger() bool { r.mu.Lock() defer r.mu.Unlock() if r.state != Idle { // A run is requested, in flight, or holding an unread result. // Drop this Trigger rather than overwrite pending work. - return + return false } r.state = RunRequested @@ -93,6 +93,7 @@ func (r *AsyncSchedulingRunner) Trigger() { default: // wake already buffered; the goroutine will see state on its next read. } + return true } func (r *AsyncSchedulingRunner) resetResult() { diff --git a/internal/scheduler/scheduling/runner/async_test.go b/internal/scheduler/scheduling/runner/async_test.go index 469f2e7101d..c290d8ad409 100644 --- a/internal/scheduler/scheduling/runner/async_test.go +++ b/internal/scheduler/scheduling/runner/async_test.go @@ -120,6 +120,28 @@ func TestAsyncSchedulingRunner_DropsTriggersWhileResultPending(t *testing.T) { assert.Equal(t, 2, algo.calls()) } +func TestAsyncSchedulingRunner_TriggerReportsWhetherRunStarted(t *testing.T) { + algo := &fakeSchedulingAlgo{result: &scheduling.SchedulerResult{}} + runner, jobDb := newTestRunner(t, algo) + + assert.True(t, runner.Trigger(), "Trigger should return true when starting a run from Idle") + waitForResultReady(t, runner) + + // A result is now pending and unread: further Triggers are dropped. + assert.False(t, runner.Trigger(), "Trigger should return false while a result is pending") + + // Draining the result returns the runner to Idle. + txn := jobDb.WriteTxn() + _, err := runner.GetSchedulerResult(armadacontext.Background(), txn) + require.NoError(t, err) + txn.Abort() + + // Idle again: the next Trigger starts a fresh run. + assert.True(t, runner.Trigger(), "Trigger should return true again once the result is drained") + waitForResultReady(t, runner) + assert.Equal(t, 2, algo.calls()) +} + func TestAsyncSchedulingRunner_AlgoErrorPropagated(t *testing.T) { schedulingErr := fmt.Errorf("scheduling failed") algo := &fakeSchedulingAlgo{err: schedulingErr} diff --git a/internal/scheduler/scheduling/runner/sync.go b/internal/scheduler/scheduling/runner/sync.go index 38846cd20ff..7ea647372aa 100644 --- a/internal/scheduler/scheduling/runner/sync.go +++ b/internal/scheduler/scheduling/runner/sync.go @@ -14,7 +14,7 @@ func NewSyncSchedulingRunner(schedulingAlgo scheduling.SchedulingAlgo) Schedulin return &syncSchedulingRunner{schedulingAlgo: schedulingAlgo} } -func (r *syncSchedulingRunner) Trigger() {} +func (r *syncSchedulingRunner) Trigger() bool { return false } func (r *syncSchedulingRunner) GetSchedulerResult(ctx *armadacontext.Context, txn *jobdb.Txn) (*scheduling.SchedulerResult, error) { return r.schedulingAlgo.Schedule(ctx, txn) diff --git a/internal/scheduler/scheduling/runner/types.go b/internal/scheduler/scheduling/runner/types.go index 3665c1c8c9d..a591e75fd46 100644 --- a/internal/scheduler/scheduling/runner/types.go +++ b/internal/scheduler/scheduling/runner/types.go @@ -11,8 +11,10 @@ import ( // It is expected the jobdb is updated with the latest result of GetSchedulerResult before calling Trigger again // - Failing to do so will result in scheduling on stale data. type SchedulingRunner interface { - // Trigger starts the next scheduling run in async runners. - Trigger() + // Trigger requests the next scheduling run in async runners. + // Returns true if a new run was actually started, false if a run was already + // in flight (async) or the runner never starts background runs (sync). + Trigger() bool // GetSchedulerResult // Applies the result of the current scheduling cycle to the provided txn. From fee86416f2e60b5b98b6bd625d0128cd9be836c1 Mon Sep 17 00:00:00 2001 From: David Slear <48934402+dslear@users.noreply.github.com> Date: Tue, 7 Jul 2026 10:52:25 -0500 Subject: [PATCH 23/49] Hot cold local dev (#4899) Adding local development configurations to run the Lookout hot/cold partitioned database in parallel with the existing Lookout database. This includes the VSCode configurations in the `.vscode` directory, the goreman procfiles and config files under `_local`, and the JetBrains configurations under `.run` --------- Signed-off-by: David Slear Signed-off-by: sarhiri --- .run/Armada HC.run.xml | 20 ++++++++ .run/Armada.run.xml | 3 ++ .run/Binoculars.run.xml | 20 ++++++++ .run/Lookout HC Ingester.run.xml | 21 ++++++++ .run/Lookout HC UI.run.xml | 20 ++++++++ .run/Lookout HC.run.xml | 26 ++++++++++ .run/Lookout UI.run.xml | 2 +- .run/lookouthcPostgresMigration.run.xml | 20 ++++++++ .vscode/launch.json | 68 +++++++++++++++++++++++++ .vscode/tasks.json | 37 ++++++++++++++ README.md | 1 + _local/compose/postgres-init.sql | 1 + _local/lookouthc/config.yaml | 24 +++++++++ _local/lookouthcingester/config.yaml | 14 +++++ _local/procfiles/fake-executor.Procfile | 2 +- _local/procfiles/hot-cold-dap.Procfile | 10 ++++ _local/procfiles/hot-cold.Procfile | 12 +++++ _local/scripts/check-port-conflicts.sh | 2 +- _local/scripts/init.sh | 30 +++++++++++ _local/scripts/kill-port-conflicts.sh | 2 +- _local/scripts/prebuild-services.sh | 61 ++++++++++++++++++++-- _local/scripts/wait-for-dlv.sh | 7 +++ _local/server/config.yaml | 2 +- config/lookout/config.yaml | 1 + config/lookouthc/config.yaml | 51 +++++++++++++++++++ docs/developer_guide.md | 7 ++- internal/lookoutui/.gitignore | 2 + internal/lookoutui/vite.config.mts | 3 +- magefiles/dev.go | 11 ++-- 29 files changed, 466 insertions(+), 14 deletions(-) create mode 100644 .run/Armada HC.run.xml create mode 100644 .run/Binoculars.run.xml create mode 100644 .run/Lookout HC Ingester.run.xml create mode 100644 .run/Lookout HC UI.run.xml create mode 100644 .run/Lookout HC.run.xml create mode 100644 .run/lookouthcPostgresMigration.run.xml create mode 100644 _local/lookouthc/config.yaml create mode 100644 _local/lookouthcingester/config.yaml create mode 100644 _local/procfiles/hot-cold-dap.Procfile create mode 100644 _local/procfiles/hot-cold.Procfile create mode 100644 config/lookouthc/config.yaml diff --git a/.run/Armada HC.run.xml b/.run/Armada HC.run.xml new file mode 100644 index 00000000000..9aafb34724a --- /dev/null +++ b/.run/Armada HC.run.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/.run/Armada.run.xml b/.run/Armada.run.xml index 8774d76305b..073a2d441c1 100644 --- a/.run/Armada.run.xml +++ b/.run/Armada.run.xml @@ -1,8 +1,11 @@ + + + diff --git a/.run/Binoculars.run.xml b/.run/Binoculars.run.xml new file mode 100644 index 00000000000..c7853164459 --- /dev/null +++ b/.run/Binoculars.run.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/.run/Lookout HC Ingester.run.xml b/.run/Lookout HC Ingester.run.xml new file mode 100644 index 00000000000..a85d2691c35 --- /dev/null +++ b/.run/Lookout HC Ingester.run.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/.run/Lookout HC UI.run.xml b/.run/Lookout HC UI.run.xml new file mode 100644 index 00000000000..41a838a0f24 --- /dev/null +++ b/.run/Lookout HC UI.run.xml @@ -0,0 +1,20 @@ + + + + diff --git a/.run/Lookout HC.run.xml b/.run/Lookout HC.run.xml new file mode 100644 index 00000000000..bb9d7798d27 --- /dev/null +++ b/.run/Lookout HC.run.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/.run/Lookout UI.run.xml b/.run/Lookout UI.run.xml index c60a44572ee..0a514808c52 100644 --- a/.run/Lookout UI.run.xml +++ b/.run/Lookout UI.run.xml @@ -1,6 +1,6 @@ -
@@ -204,11 +204,11 @@ export const TimeRangeSelector = ({ value: { startIsoString, endIsoString }, onC actionBar: { actions: ["cancel", "today", "clear", "accept"] }, textField: { size: "small", margin: "dense", fullWidth: true }, }} - defaultValue={endDate ? endDateDayJs : undefined} + defaultValue={endDateDayJs} onAccept={(value: Dayjs | null) => - onChange({ startIsoString, endIsoString: value?.toISOString() ?? null }) + onChange({ startIsoString, endIsoString: value?.isValid() ? value.toISOString() : null }) } - minDateTime={startDate ? startDateDayJs : undefined} + minDateTime={startDateDayJs} maxDateTime={nowDayjs} />
From a8bef5d259d81bcf14c39cfc447785e66d7f66c9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Jul 2026 12:49:43 +0000 Subject: [PATCH 28/49] Bump next from 15.5.10 to 15.5.18 in /website (#5003) Bumps [next](https://github.com/vercel/next.js) from 15.5.10 to 15.5.18.
Release notes

Sourced from next's releases.

v15.5.18

This release contains security fixes for the following advisories:

High:

Moderate:

Low:

v15.5.16

This release contains security fixes for the following advisories:

High:

Moderate:

Low:

Commits
  • 9ff92ce v15.5.18
  • 00ebe23 [backport] Disable build caches for production/staging/force-preview deploys ...
  • 62c97ab v15.5.17
  • 423623a Turbopack: Match proxy matchers with webpack implementation (#93594)
  • fa78739 Turbopack: Fix middleware matcher suffix (#93590)
  • 36e62c6 [backport] Turbopack: more strict vergen setup (#93588)
  • 36589b5 [backport][test] Pin package manager to patch versions (#93596)
  • ad6fd4e v15.5.16
  • 79d7dff Ignore malformed CSP nonce headers (#103)
  • c4f6908 router-server: guard upgrade proxy against absolute-url SSRF (#77) (#102)
  • Additional commits viewable in compare view
Maintainer changes

This version was pushed to npm by GitHub Actions, a new releaser for next since your current version.


[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=next&package-manager=npm_and_yarn&previous-version=15.5.10&new-version=15.5.18)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/armadaproject/armada/network/alerts).
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: sarhiri --- website/package.json | 2 +- website/yarn.lock | 112 +++++++++++++++++++++---------------------- 2 files changed, 57 insertions(+), 57 deletions(-) diff --git a/website/package.json b/website/package.json index dc406cc29d1..8656e433cc1 100644 --- a/website/package.json +++ b/website/package.json @@ -30,7 +30,7 @@ "fumadocs-ui": "15.5.5", "lucide-react": "^0.515.0", "mermaid": "^11.6.0", - "next": "15.5.10", + "next": "15.5.18", "react": "^19.1.0", "react-dom": "^19.1.0", "title": "^4.0.1" diff --git a/website/yarn.lock b/website/yarn.lock index f6ad72afd42..ad9500133b9 100644 --- a/website/yarn.lock +++ b/website/yarn.lock @@ -1125,10 +1125,10 @@ "@emnapi/runtime" "^1.4.3" "@tybys/wasm-util" "^0.9.0" -"@next/env@15.5.10", "@next/env@^15.3.3": - version "15.5.10" - resolved "https://registry.yarnpkg.com/@next/env/-/env-15.5.10.tgz#3b0506c57d0977e60726a1663f36bc96d42c295b" - integrity sha512-plg+9A/KoZcTS26fe15LHg+QxReTazrIOoKKUC3Uz4leGGeNPgLHdevVraAAOX0snnUs3WkRx3eUQpj9mreG6A== +"@next/env@15.5.18", "@next/env@^15.3.3": + version "15.5.18" + resolved "https://registry.yarnpkg.com/@next/env/-/env-15.5.18.tgz#207ab150d3f1c787ac343946155392d478506c1b" + integrity sha512-hAV85Ckd9QR6RvH04MEKwsfLTksvFpO47j9xwtoIuvuPnlwecpSi+uZTtm8HirVbtlI2Fnz//xpcSTjFdyJk+g== "@next/eslint-plugin-next@15.3.3": version "15.3.3" @@ -1137,45 +1137,45 @@ dependencies: fast-glob "3.3.1" -"@next/swc-darwin-arm64@15.5.7": - version "15.5.7" - resolved "https://registry.yarnpkg.com/@next/swc-darwin-arm64/-/swc-darwin-arm64-15.5.7.tgz#f0c9ccfec2cd87cbd4b241ce4c779a7017aed958" - integrity sha512-IZwtxCEpI91HVU/rAUOOobWSZv4P2DeTtNaCdHqLcTJU4wdNXgAySvKa/qJCgR5m6KI8UsKDXtO2B31jcaw1Yw== - -"@next/swc-darwin-x64@15.5.7": - version "15.5.7" - resolved "https://registry.yarnpkg.com/@next/swc-darwin-x64/-/swc-darwin-x64-15.5.7.tgz#18009e9fcffc5c0687cc9db24182ddeac56280d9" - integrity sha512-UP6CaDBcqaCBuiq/gfCEJw7sPEoX1aIjZHnBWN9v9qYHQdMKvCKcAVs4OX1vIjeE+tC5EIuwDTVIoXpUes29lg== - -"@next/swc-linux-arm64-gnu@15.5.7": - version "15.5.7" - resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-15.5.7.tgz#fe7c7e08264cf522d4e524299f6d3e63d68d579a" - integrity sha512-NCslw3GrNIw7OgmRBxHtdWFQYhexoUCq+0oS2ccjyYLtcn1SzGzeM54jpTFonIMUjNbHmpKpziXnpxhSWLcmBA== - -"@next/swc-linux-arm64-musl@15.5.7": - version "15.5.7" - resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-15.5.7.tgz#94228fe293475ec34a5a54284e1056876f43a3cf" - integrity sha512-nfymt+SE5cvtTrG9u1wdoxBr9bVB7mtKTcj0ltRn6gkP/2Nu1zM5ei8rwP9qKQP0Y//umK+TtkKgNtfboBxRrw== - -"@next/swc-linux-x64-gnu@15.5.7": - version "15.5.7" - resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-15.5.7.tgz#078c71201dfe7fcfb8fa6dc92aae6c94bc011cdc" - integrity sha512-hvXcZvCaaEbCZcVzcY7E1uXN9xWZfFvkNHwbe/n4OkRhFWrs1J1QV+4U1BN06tXLdaS4DazEGXwgqnu/VMcmqw== - -"@next/swc-linux-x64-musl@15.5.7": - version "15.5.7" - resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-15.5.7.tgz#72947f5357f9226292353e0bb775643da3c7a182" - integrity sha512-4IUO539b8FmF0odY6/SqANJdgwn1xs1GkPO5doZugwZ3ETF6JUdckk7RGmsfSf7ws8Qb2YB5It33mvNL/0acqA== - -"@next/swc-win32-arm64-msvc@15.5.7": - version "15.5.7" - resolved "https://registry.yarnpkg.com/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-15.5.7.tgz#397b912cd51c6a80e32b9c0507ecd82514353941" - integrity sha512-CpJVTkYI3ZajQkC5vajM7/ApKJUOlm6uP4BknM3XKvJ7VXAvCqSjSLmM0LKdYzn6nBJVSjdclx8nYJSa3xlTgQ== - -"@next/swc-win32-x64-msvc@15.5.7": - version "15.5.7" - resolved "https://registry.yarnpkg.com/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-15.5.7.tgz#e02b543d9dc6c1631d4ac239cb1177245dfedfe4" - integrity sha512-gMzgBX164I6DN+9/PGA+9dQiwmTkE4TloBNx8Kv9UiGARsr9Nba7IpcBRA1iTV9vwlYnrE3Uy6I7Aj6qLjQuqw== +"@next/swc-darwin-arm64@15.5.18": + version "15.5.18" + resolved "https://registry.yarnpkg.com/@next/swc-darwin-arm64/-/swc-darwin-arm64-15.5.18.tgz#d4376e2d31f90679128c5d83aecfb7aa244e475c" + integrity sha512-w0WvQf1n+txiwns/9pwIQteCJpZTbxzO2SE0FLcwuD4v0WEh1JPOjdyxWL21XwJsdpx8cFRjyzxzCS/siP7HcQ== + +"@next/swc-darwin-x64@15.5.18": + version "15.5.18" + resolved "https://registry.yarnpkg.com/@next/swc-darwin-x64/-/swc-darwin-x64-15.5.18.tgz#35e179f2863d0ea1b249d5f8a1616593431f1645" + integrity sha512-znn71QmDuxm+BOaglihMZfvyySMnNljkVIY5Z2TCssBmm+WqL6c19VhtH5ktFkHa8EZ2bnTUpcNcmNSQsg67og== + +"@next/swc-linux-arm64-gnu@15.5.18": + version "15.5.18" + resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-15.5.18.tgz#5c97bd8422cc0d43b6b07329514bfaceb04b8014" + integrity sha512-yPPe5MNL+igZUa+OsqQJisqSfh6oarIuA1Q0BDxljGJhRQyZeP+WRHh7rs/jZUGMh5aY0YdIjXZG0VohkKkUdw== + +"@next/swc-linux-arm64-musl@15.5.18": + version "15.5.18" + resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-15.5.18.tgz#497dda6be6e905e3ab7b02624a5493ccf08f06e7" + integrity sha512-glaCczEWIrHsokFZ3pP08U4BpKxwIdnT+txdOM32OBgpL9Yw4aqx8NejmgtZQZOdstQ5f0L3CasIZudzCuD+nw== + +"@next/swc-linux-x64-gnu@15.5.18": + version "15.5.18" + resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-15.5.18.tgz#c5863100284c0182ff546d212c74dc9348d16564" + integrity sha512-oUfg2EgJmU3R0OCOWiokGFUTvZiPfXtriXiuF3YNxRoROCdgvTedHIzYoeKH34gsZxS/V7mHbfq2hpAHwhH1/A== + +"@next/swc-linux-x64-musl@15.5.18": + version "15.5.18" + resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-15.5.18.tgz#26f3fdc26707458481d40649a1420584bf1f3bf7" + integrity sha512-JLxSP3KTd9iu/bvUMQxH7RJo9xKSHf55/6RPE4a6FTSZygGn7uvZbCej0AHXydwkggQGSD9UddSjwv6Xz5ESfA== + +"@next/swc-win32-arm64-msvc@15.5.18": + version "15.5.18" + resolved "https://registry.yarnpkg.com/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-15.5.18.tgz#8acdb2aeec6d768bffa3043485d22ecdf48a6e4c" + integrity sha512-ir1v7enP52K2HNz3tQQvwF+x7VNxBk1ciiZ18WBPvxf4C59IqdfmHPJYK3vH7rSxpuCVw/8C712wTXNAtEp+NA== + +"@next/swc-win32-x64-msvc@15.5.18": + version "15.5.18" + resolved "https://registry.yarnpkg.com/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-15.5.18.tgz#beac6228e60e3ee08ce7a20b7f61b3dc516d4b10" + integrity sha512-LIu5me6QTANCd25E7I5uIEfvgQ06RK7tvHAbYo3zCb3VpxQEPvMcSpd87NwUABDT6MbGPdEGR5VRiK4PPTJhQg== "@next/third-parties@^16.0.1": version "16.0.1" @@ -6402,25 +6402,25 @@ next-themes@^0.4.6: resolved "https://registry.yarnpkg.com/next-themes/-/next-themes-0.4.6.tgz#8d7e92d03b8fea6582892a50a928c9b23502e8b6" integrity sha512-pZvgD5L0IEvX5/9GWyHMf3m8BKiVQwsCMHfoFosXtXBMnaS0ZnIJ9ST4b4NqLVKDEm8QBxoNNGNaBv2JNF6XNA== -next@15.5.10: - version "15.5.10" - resolved "https://registry.yarnpkg.com/next/-/next-15.5.10.tgz#5e3824d8f00dcd66ca4e79c38834f766976116bd" - integrity sha512-r0X65PNwyDDyOrWNKpQoZvOatw7BcsTPRKdwEqtc9cj3wv7mbBIk9tKed4klRaFXJdX0rugpuMTHslDrAU1bBg== +next@15.5.18: + version "15.5.18" + resolved "https://registry.yarnpkg.com/next/-/next-15.5.18.tgz#b0a9d82763f7358938fe4732bb6f605f7e941815" + integrity sha512-eKL8zUJkX9Y5lE+RX/2YJoItVdGlIscyVyboeD9wSpp0PaGqjoA4tTpT2qPqz9ax+5IzGESyLSeZ/RCwbSZ2uQ== dependencies: - "@next/env" "15.5.10" + "@next/env" "15.5.18" "@swc/helpers" "0.5.15" caniuse-lite "^1.0.30001579" postcss "8.4.31" styled-jsx "5.1.6" optionalDependencies: - "@next/swc-darwin-arm64" "15.5.7" - "@next/swc-darwin-x64" "15.5.7" - "@next/swc-linux-arm64-gnu" "15.5.7" - "@next/swc-linux-arm64-musl" "15.5.7" - "@next/swc-linux-x64-gnu" "15.5.7" - "@next/swc-linux-x64-musl" "15.5.7" - "@next/swc-win32-arm64-msvc" "15.5.7" - "@next/swc-win32-x64-msvc" "15.5.7" + "@next/swc-darwin-arm64" "15.5.18" + "@next/swc-darwin-x64" "15.5.18" + "@next/swc-linux-arm64-gnu" "15.5.18" + "@next/swc-linux-arm64-musl" "15.5.18" + "@next/swc-linux-x64-gnu" "15.5.18" + "@next/swc-linux-x64-musl" "15.5.18" + "@next/swc-win32-arm64-msvc" "15.5.18" + "@next/swc-win32-x64-msvc" "15.5.18" sharp "^0.34.3" node-emoji@^2.1.3: From a7155545784461fe5821df0378052aae3da019a1 Mon Sep 17 00:00:00 2001 From: Rob Smith <34475852+robertdavidsmith@users.noreply.github.com> Date: Mon, 13 Jul 2026 17:49:19 +0100 Subject: [PATCH 29/49] Scheduler: for testing, support not publishing to pulsar (#5013) Signed-off-by: sarhiri --- internal/common/config/pulsar.go | 2 +- .../scheduler/publisher/dummy_publisher.go | 23 ++++ internal/scheduler/publisher/publisher.go | 20 ++++ .../pulsar_publisher.go} | 14 +-- .../pulsar_publisher_test.go} | 2 +- internal/scheduler/scheduler.go | 11 +- internal/scheduler/scheduler_test.go | 6 +- internal/scheduler/schedulerapp.go | 109 ++++++++++-------- 8 files changed, 117 insertions(+), 70 deletions(-) create mode 100644 internal/scheduler/publisher/dummy_publisher.go create mode 100644 internal/scheduler/publisher/publisher.go rename internal/scheduler/{publisher.go => publisher/pulsar_publisher.go} (86%) rename internal/scheduler/{publisher_test.go => publisher/pulsar_publisher_test.go} (99%) diff --git a/internal/common/config/pulsar.go b/internal/common/config/pulsar.go index 08782003e6e..9930af0377a 100644 --- a/internal/common/config/pulsar.go +++ b/internal/common/config/pulsar.go @@ -8,7 +8,7 @@ import ( type PulsarConfig struct { // Pulsar URL - URL string `validate:"required"` + URL string // Pulsar REST API URL (Pulsar admin API) // If not set, event latency metrics will not be published RestURL string diff --git a/internal/scheduler/publisher/dummy_publisher.go b/internal/scheduler/publisher/dummy_publisher.go new file mode 100644 index 00000000000..0b831bc12ba --- /dev/null +++ b/internal/scheduler/publisher/dummy_publisher.go @@ -0,0 +1,23 @@ +package publisher + +import ( + "github.com/google/uuid" + + "github.com/armadaproject/armada/internal/common/armadacontext" + "github.com/armadaproject/armada/pkg/armadaevents" +) + +// Dummy publisher just drops all messages, can be useful for testing the scheduler +type DummyPublisher struct{} + +func NewDummyPublisher() *DummyPublisher { + return &DummyPublisher{} +} + +func (p *DummyPublisher) PublishMessages(ctx *armadacontext.Context, events []*armadaevents.EventSequence, shouldPublish func() bool) error { + return nil +} + +func (p *DummyPublisher) PublishMarkers(ctx *armadacontext.Context, groupId uuid.UUID) (uint32, error) { + return 0, nil +} diff --git a/internal/scheduler/publisher/publisher.go b/internal/scheduler/publisher/publisher.go new file mode 100644 index 00000000000..7428bbd5248 --- /dev/null +++ b/internal/scheduler/publisher/publisher.go @@ -0,0 +1,20 @@ +package publisher + +import ( + "github.com/google/uuid" + + "github.com/armadaproject/armada/internal/common/armadacontext" + "github.com/armadaproject/armada/pkg/armadaevents" +) + +// Publisher is an interface to be implemented by structs that handle publishing messages to pulsar +type Publisher interface { + // PublishMessages will publish the supplied messages. A LeaderToken is provided and the + // implementor may decide whether to publish based on the status of this token + PublishMessages(ctx *armadacontext.Context, events []*armadaevents.EventSequence, shouldPublish func() bool) error + + // PublishMarkers publishes a single marker message for each Pulsar partition. Each marker + // massage contains the supplied group id, which allows all marker messages for a given call + // to be identified. The uint32 returned is the number of messages published + PublishMarkers(ctx *armadacontext.Context, groupId uuid.UUID) (uint32, error) +} diff --git a/internal/scheduler/publisher.go b/internal/scheduler/publisher/pulsar_publisher.go similarity index 86% rename from internal/scheduler/publisher.go rename to internal/scheduler/publisher/pulsar_publisher.go index 67cd20e6ff0..46f82da41c4 100644 --- a/internal/scheduler/publisher.go +++ b/internal/scheduler/publisher/pulsar_publisher.go @@ -1,4 +1,4 @@ -package scheduler +package publisher import ( "fmt" @@ -21,18 +21,6 @@ const ( explicitPartitionKey = "armada_pulsar_partition" ) -// Publisher is an interface to be implemented by structs that handle publishing messages to pulsar -type Publisher interface { - // PublishMessages will publish the supplied messages. A LeaderToken is provided and the - // implementor may decide whether to publish based on the status of this token - PublishMessages(ctx *armadacontext.Context, events []*armadaevents.EventSequence, shouldPublish func() bool) error - - // PublishMarkers publishes a single marker message for each Pulsar partition. Each marker - // massage contains the supplied group id, which allows all marker messages for a given call - // to be identified. The uint32 returned is the number of messages published - PublishMarkers(ctx *armadacontext.Context, groupId uuid.UUID) (uint32, error) -} - // PulsarPublisher is the default implementation of Publisher type PulsarPublisher struct { // Used to send events sequences to pulsar diff --git a/internal/scheduler/publisher_test.go b/internal/scheduler/publisher/pulsar_publisher_test.go similarity index 99% rename from internal/scheduler/publisher_test.go rename to internal/scheduler/publisher/pulsar_publisher_test.go index 8ccd2d3f68d..b40bf96500d 100644 --- a/internal/scheduler/publisher_test.go +++ b/internal/scheduler/publisher/pulsar_publisher_test.go @@ -1,4 +1,4 @@ -package scheduler +package publisher import ( "fmt" diff --git a/internal/scheduler/scheduler.go b/internal/scheduler/scheduler.go index 0e362982c45..e3d685e31b1 100644 --- a/internal/scheduler/scheduler.go +++ b/internal/scheduler/scheduler.go @@ -23,6 +23,7 @@ import ( "github.com/armadaproject/armada/internal/scheduler/kubernetesobjects/affinity" "github.com/armadaproject/armada/internal/scheduler/metrics" "github.com/armadaproject/armada/internal/scheduler/pricing" + "github.com/armadaproject/armada/internal/scheduler/publisher" "github.com/armadaproject/armada/internal/scheduler/queue" "github.com/armadaproject/armada/internal/scheduler/schedulerobjects" "github.com/armadaproject/armada/internal/scheduler/scheduling" @@ -54,7 +55,7 @@ type Scheduler struct { // This is used to check if gangs jobs are valid before considering their jobs validated gangValidator SubmitGangValidator // Responsible for publishing messages to Pulsar. Only the leader publishes. - publisher Publisher + publisher publisher.Publisher // Minimum duration between scheduler cycles. cyclePeriod time.Duration // Minimum duration between Schedule() calls - calls that actually schedule new jobs. @@ -100,7 +101,7 @@ func NewScheduler( executorRepository database.ExecutorRepository, runner runner.SchedulingRunner, leaderController leaderelection.LeaderController, - publisher Publisher, + publisher publisher.Publisher, submitChecker SubmitScheduleChecker, gangValidator SubmitGangValidator, cyclePeriod time.Duration, @@ -1318,6 +1319,12 @@ func (s *Scheduler) ensureDbUpToDate(ctx *armadacontext.Context, pollInterval ti } } + // We're using a dummy publisher for testing + if numSent == 0 { + ctx.Infof("No pulsar partitions configured, skipping checking for database up to date") + return nil + } + // Try to read these messages back from postgres. for { select { diff --git a/internal/scheduler/scheduler_test.go b/internal/scheduler/scheduler_test.go index 7a94baa7c3b..f7a2787b8d3 100644 --- a/internal/scheduler/scheduler_test.go +++ b/internal/scheduler/scheduler_test.go @@ -19,8 +19,6 @@ import ( clock "k8s.io/utils/clock/testing" "k8s.io/utils/pointer" - "github.com/armadaproject/armada/internal/scheduler/scheduling/runner" - "github.com/armadaproject/armada/internal/common/armadacontext" "github.com/armadaproject/armada/internal/common/armadaerrors" apiconfig "github.com/armadaproject/armada/internal/common/constants" @@ -37,9 +35,11 @@ import ( "github.com/armadaproject/armada/internal/scheduler/kubernetesobjects/affinity" "github.com/armadaproject/armada/internal/scheduler/metrics" "github.com/armadaproject/armada/internal/scheduler/pricing" + "github.com/armadaproject/armada/internal/scheduler/publisher" "github.com/armadaproject/armada/internal/scheduler/schedulerobjects" "github.com/armadaproject/armada/internal/scheduler/scheduling" schedulercontext "github.com/armadaproject/armada/internal/scheduler/scheduling/context" + "github.com/armadaproject/armada/internal/scheduler/scheduling/runner" "github.com/armadaproject/armada/internal/scheduler/testfixtures" "github.com/armadaproject/armada/internal/scheduleringester" "github.com/armadaproject/armada/pkg/api" @@ -3723,7 +3723,7 @@ func TestCycleConsistency(t *testing.T) { return nil } - eventsFromTestPublisher := func(p Publisher) []*armadaevents.EventSequence { + eventsFromTestPublisher := func(p publisher.Publisher) []*armadaevents.EventSequence { return p.(*testPublisher).eventSequences } diff --git a/internal/scheduler/schedulerapp.go b/internal/scheduler/schedulerapp.go index 2a2a84fca4e..fc724657580 100644 --- a/internal/scheduler/schedulerapp.go +++ b/internal/scheduler/schedulerapp.go @@ -15,8 +15,6 @@ import ( "github.com/prometheus/client_golang/prometheus" "google.golang.org/grpc/codes" - "github.com/armadaproject/armada/internal/scheduler/scheduling/runner" - "github.com/armadaproject/armada/internal/common" "github.com/armadaproject/armada/internal/common/app" "github.com/armadaproject/armada/internal/common/armadacontext" @@ -42,9 +40,11 @@ import ( "github.com/armadaproject/armada/internal/scheduler/metrics" "github.com/armadaproject/armada/internal/scheduler/pricing" "github.com/armadaproject/armada/internal/scheduler/priorityoverride" + "github.com/armadaproject/armada/internal/scheduler/publisher" "github.com/armadaproject/armada/internal/scheduler/queue" "github.com/armadaproject/armada/internal/scheduler/reports" "github.com/armadaproject/armada/internal/scheduler/scheduling" + "github.com/armadaproject/armada/internal/scheduler/scheduling/runner" "github.com/armadaproject/armada/pkg/api" "github.com/armadaproject/armada/pkg/api/schedulerobjects" "github.com/armadaproject/armada/pkg/armadaevents" @@ -200,47 +200,74 @@ func Run(config schedulerconfig.Configuration) error { // //////////////////////////////////////////////////////////////////////// // Pulsar // //////////////////////////////////////////////////////////////////////// - ctx.Infof("Setting up Pulsar connectivity") - pulsarClient, err := pulsarutils.NewPulsarClient(&config.Pulsar) - if err != nil { - return errors.WithMessage(err, "Error creating pulsar client") - } - defer pulsarClient.Close() - - jobsetEventPublisher, err := NewPulsarPublisher(pulsarClient, pulsar.ProducerOptions{ - Name: fmt.Sprintf("armada-scheduler-%s", uuid.NewString()), - CompressionType: config.Pulsar.CompressionType, - CompressionLevel: config.Pulsar.CompressionLevel, - BatchingMaxSize: config.Pulsar.MaxAllowedMessageSize, - Topic: config.Pulsar.JobsetEventsTopic, - }, config.Pulsar.MaxAllowedEventsPerMessage, config.Pulsar.MaxAllowedMessageSize, config.Pulsar.SendTimeout) - if err != nil { - return errors.WithMessage(err, "error creating jobset event pulsar publisher") - } - // Publishing metrics to pulsar is experimental. We default to a no-op publisher and only enable a pulsar publisher - // if the feature flag is set in config + var jobSetEventPublisher publisher.Publisher var metricPublisher pulsarutils.Publisher[*metricevents.Event] = pulsarutils.NoOpPublisher[*metricevents.Event]{} - if config.PublishMetricsToPulsar { - metricPublisher, err = pulsarutils.NewPulsarPublisher[*metricevents.Event]( + var apiPublisher pulsarutils.Publisher[*armadaevents.EventSequence] = pulsarutils.NoOpPublisher[*armadaevents.EventSequence]{} + + if config.Pulsar.URL == "" { + ctx.Warn("Pulsar URL not configured so won't publish to pulsar, this can be useful for testing but has no legitimate production use") + jobSetEventPublisher = publisher.NewDummyPublisher() + } else { + ctx.Infof("Setting up Pulsar connectivity") + pulsarClient, err := pulsarutils.NewPulsarClient(&config.Pulsar) + if err != nil { + return errors.WithMessage(err, "Error creating pulsar client") + } + defer pulsarClient.Close() + + jobSetEventPublisher, err = publisher.NewPulsarPublisher(pulsarClient, pulsar.ProducerOptions{ + Name: fmt.Sprintf("armada-scheduler-%s", uuid.NewString()), + CompressionType: config.Pulsar.CompressionType, + CompressionLevel: config.Pulsar.CompressionLevel, + BatchingMaxSize: config.Pulsar.MaxAllowedMessageSize, + Topic: config.Pulsar.JobsetEventsTopic, + }, config.Pulsar.MaxAllowedEventsPerMessage, config.Pulsar.MaxAllowedMessageSize, config.Pulsar.SendTimeout) + if err != nil { + return errors.WithMessage(err, "error creating jobset event pulsar publisher") + } + + // Publishing metrics to pulsar is experimental. We default to a no-op publisher and only enable a pulsar publisher + // if the feature flag is set in config + if config.PublishMetricsToPulsar { + metricPublisher, err = pulsarutils.NewPulsarPublisher[*metricevents.Event]( + pulsarClient, + pulsar.ProducerOptions{ + Name: fmt.Sprintf("armada-scheduler-metrics-%s", uuid.NewString()), + CompressionType: config.Pulsar.CompressionType, + CompressionLevel: config.Pulsar.CompressionLevel, + BatchingMaxSize: config.Pulsar.MaxAllowedMessageSize, + Topic: config.Pulsar.MetricEventsTopic, + }, + utils.NoOpPreProcessor, + // Metrics are sent to an unpartitioned pulsar topic so there is no key needed + func(event *metricevents.Event) string { + return "" + }, + config.Pulsar.SendTimeout, + ) + if err != nil { + return errors.WithMessage(err, "error creating metric event pulsar publisher") + } + } + preProcessor := jobsetevents.NewPreProcessor(config.Pulsar.MaxAllowedEventsPerMessage, config.Pulsar.MaxAllowedMessageSize) + apiPublisher, err = pulsarutils.NewPulsarPublisher[*armadaevents.EventSequence]( pulsarClient, pulsar.ProducerOptions{ - Name: fmt.Sprintf("armada-scheduler-metrics-%s", uuid.NewString()), + Name: fmt.Sprintf("armada-executor-api-%s", uuid.NewString()), CompressionType: config.Pulsar.CompressionType, CompressionLevel: config.Pulsar.CompressionLevel, BatchingMaxSize: config.Pulsar.MaxAllowedMessageSize, - Topic: config.Pulsar.MetricEventsTopic, - }, - utils.NoOpPreProcessor, - // Metrics are sent to an unpartitioned pulsar topic so there is no key needed - func(event *metricevents.Event) string { - return "" + Topic: config.Pulsar.JobsetEventsTopic, }, + preProcessor, + jobsetevents.RetrieveKey, config.Pulsar.SendTimeout, ) if err != nil { - return errors.WithMessage(err, "error creating metric event pulsar publisher") + return errors.Wrapf(err, "error creating pulsar publisher for executor api") } + defer apiPublisher.Close() } // //////////////////////////////////////////////////////////////////////// @@ -260,24 +287,6 @@ func Run(config schedulerconfig.Configuration) error { // Executor Api // //////////////////////////////////////////////////////////////////////// ctx.Infof("Setting up executor api") - preProcessor := jobsetevents.NewPreProcessor(config.Pulsar.MaxAllowedEventsPerMessage, config.Pulsar.MaxAllowedMessageSize) - apiPublisher, err := pulsarutils.NewPulsarPublisher[*armadaevents.EventSequence]( - pulsarClient, - pulsar.ProducerOptions{ - Name: fmt.Sprintf("armada-executor-api-%s", uuid.NewString()), - CompressionType: config.Pulsar.CompressionType, - CompressionLevel: config.Pulsar.CompressionLevel, - BatchingMaxSize: config.Pulsar.MaxAllowedMessageSize, - Topic: config.Pulsar.JobsetEventsTopic, - }, - preProcessor, - jobsetevents.RetrieveKey, - config.Pulsar.SendTimeout, - ) - if err != nil { - return errors.Wrapf(err, "error creating pulsar publisher for executor api") - } - defer apiPublisher.Close() authServices, err := auth.ConfigureAuth(config.Auth) if err != nil { @@ -412,7 +421,7 @@ func Run(config schedulerconfig.Configuration) error { executorRepository, schedulingRunner, leaderController, - jobsetEventPublisher, + jobSetEventPublisher, submitChecker, NewGangValidator(), config.CyclePeriod, From 4afefea4b64fa392ece7280f566a9ba64f746dc1 Mon Sep 17 00:00:00 2001 From: Maurice Yap Date: Mon, 13 Jul 2026 18:07:21 +0100 Subject: [PATCH 30/49] Add configurable Query API query mirroring to a second database (#5016) This adds opt-in, fire-and-forget mirroring that replays each Query API query against a second Lookout database with the result discarded, never affecting the primary path. Signed-off-by: Maurice Yap Signed-off-by: sarhiri --- config/server/config.yaml | 17 +++ internal/server/configuration/types.go | 17 +++ internal/server/queryapi/doc.go | 14 +++ internal/server/queryapi/mirror.go | 133 ++++++++++++++++++++++ internal/server/queryapi/mirror_test.go | 141 ++++++++++++++++++++++++ internal/server/queryapi/query_api.go | 5 +- internal/server/server.go | 17 ++- 7 files changed, 340 insertions(+), 4 deletions(-) create mode 100644 internal/server/queryapi/doc.go create mode 100644 internal/server/queryapi/mirror.go create mode 100644 internal/server/queryapi/mirror_test.go diff --git a/config/server/config.yaml b/config/server/config.yaml index c3d1d6ccadd..592a242f275 100644 --- a/config/server/config.yaml +++ b/config/server/config.yaml @@ -87,3 +87,20 @@ postgres: sslmode: disable queryapi: maxQueryItems: 500 + # Optionally replay each Query API database query against a second database + # for performance evaluation. Fire-and-forget: mirror results are discarded + # and never affect the primary query path. + mirror: + enabled: false + # Maximum concurrently replayed mirror queries. When this bound is reached + # further queries are dropped (logged at debug level), so the mirrored load + # under-counts real Query API traffic if the bound is hit. + maxInFlight: 100 + postgres: + connection: + host: postgres + port: 5432 + user: postgres + password: psw + dbname: lookout_mirror + sslmode: disable diff --git a/internal/server/configuration/types.go b/internal/server/configuration/types.go index c76f654d008..38d88f43fce 100644 --- a/internal/server/configuration/types.go +++ b/internal/server/configuration/types.go @@ -108,4 +108,21 @@ type PostgresConfig struct { type QueryApiConfig struct { MaxQueryItems int + // Mirror optionally duplicates each Query API database query against a + // second database for performance evaluation under real query patterns. + Mirror QueryApiMirrorConfig +} + +// QueryApiMirrorConfig configures fire-and-forget server-side mirroring of +// Query API database queries to a second database, so it can be evaluated +// under real production query patterns without affecting the primary query +// path. +type QueryApiMirrorConfig struct { + Enabled bool + // Postgres connection pointed at the database to mirror queries to. + Postgres PostgresConfig + // MaxInFlight bounds the number of concurrently replayed mirror queries. + // When the bound is reached, further mirror queries are dropped. If zero, + // a sensible default is used. + MaxInFlight int } diff --git a/internal/server/queryapi/doc.go b/internal/server/queryapi/doc.go new file mode 100644 index 00000000000..b312e305805 --- /dev/null +++ b/internal/server/queryapi/doc.go @@ -0,0 +1,14 @@ +// Package queryapi implements the Armada server's Jobs gRPC service (the +// "Query API"), which serves programmatic job-status queries by reading the +// Lookout database (the job, job_run, job_spec and job_error tables) directly +// via the server's own connection pool. +// +// The QueryApi type in query_api.go implements each RPC by issuing type-safe +// queries generated by sqlc (see the database subpackage). +// +// mirror.go provides optional, opt-in server-side query mirroring: when +// enabled, a QueryDB wrapper replays each query against a second database in a +// bounded, fire-and-forget manner so that database can be evaluated under real +// Query API load. Mirror results and errors are discarded and never affect the +// primary query path. +package queryapi diff --git a/internal/server/queryapi/mirror.go b/internal/server/queryapi/mirror.go new file mode 100644 index 00000000000..331c92a7999 --- /dev/null +++ b/internal/server/queryapi/mirror.go @@ -0,0 +1,133 @@ +package queryapi + +import ( + "context" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + + log "github.com/armadaproject/armada/internal/common/logging" + "github.com/armadaproject/armada/internal/server/queryapi/database" +) + +// defaultMirrorMaxInFlight bounds concurrently replayed mirror queries when no +// limit is configured. +const defaultMirrorMaxInFlight = 100 + +// mirrorTimeout bounds how long a single replayed mirror query may run before +// it is abandoned. Mirror work uses a detached context so it is not cancelled +// when the originating request returns. +const mirrorTimeout = 30 * time.Second + +// QueryDB is the subset of *pgxpool.Pool that QueryApi depends on. Both the +// real pool and the mirroring wrapper satisfy it. +type QueryDB interface { + database.DBTX + BeginTx(ctx context.Context, opts pgx.TxOptions) (pgx.Tx, error) +} + +// mirroringDB wraps a primary QueryDB and asynchronously replays every query +// against a mirror pool (a second database) for performance evaluation. +// Results from the primary are returned unchanged; mirror results and errors +// are discarded. Replaying is fire-and-forget and never blocks or affects the +// primary query path. +type mirroringDB struct { + primary QueryDB + mirror *pgxpool.Pool + inFlight chan struct{} +} + +// NewMirroringDB returns a QueryDB that delegates to primary and additionally +// replays each query against mirror in a bounded, fire-and-forget manner. +func NewMirroringDB(primary QueryDB, mirror *pgxpool.Pool, maxInFlight int) QueryDB { + if maxInFlight <= 0 { + maxInFlight = defaultMirrorMaxInFlight + } + return &mirroringDB{ + primary: primary, + mirror: mirror, + inFlight: make(chan struct{}, maxInFlight), + } +} + +func (m *mirroringDB) Exec(ctx context.Context, sql string, args ...any) (pgconn.CommandTag, error) { + m.replay(sql, args) + return m.primary.Exec(ctx, sql, args...) +} + +func (m *mirroringDB) Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) { + m.replay(sql, args) + return m.primary.Query(ctx, sql, args...) +} + +func (m *mirroringDB) QueryRow(ctx context.Context, sql string, args ...any) pgx.Row { + m.replay(sql, args) + return m.primary.QueryRow(ctx, sql, args...) +} + +// BeginTx returns the real primary transaction wrapped so that statements +// issued within it are also mirrored. The mirror replays each statement +// non-transactionally; it only needs representative read load, not +// transactional fidelity. +func (m *mirroringDB) BeginTx(ctx context.Context, opts pgx.TxOptions) (pgx.Tx, error) { + tx, err := m.primary.BeginTx(ctx, opts) + if err != nil { + return nil, err + } + return &mirroringTx{Tx: tx, db: m}, nil +} + +// replay schedules the given query to run against the mirror pool in the +// background. It is dropped if the in-flight bound is reached, so a slow mirror +// database can never leak goroutines or slow the primary path. +func (m *mirroringDB) replay(sql string, args []any) { + select { + case m.inFlight <- struct{}{}: + default: + log.Debug("dropping mirrored query; in-flight bound reached") + return + } + // Copy the args slice so the background goroutine never shares mutable + // state with the synchronous primary call. + argsCopy := append([]any(nil), args...) + go func() { + defer func() { <-m.inFlight }() + ctx, cancel := context.WithTimeout(context.Background(), mirrorTimeout) + defer cancel() + rows, err := m.mirror.Query(ctx, sql, argsCopy...) + if err != nil { + log.WithError(err).Debug("mirrored query failed") + return + } + // Drain and close so the connection is returned to the pool; results + // are intentionally discarded. + for rows.Next() { + } + rows.Close() + }() +} + +// mirroringTx wraps a primary pgx.Tx, mirroring statements issued within it +// while delegating all other transaction behaviour (Commit, Rollback, etc.) to +// the embedded transaction. +type mirroringTx struct { + pgx.Tx + db *mirroringDB +} + +func (t *mirroringTx) Exec(ctx context.Context, sql string, args ...any) (pgconn.CommandTag, error) { + t.db.replay(sql, args) + return t.Tx.Exec(ctx, sql, args...) +} + +func (t *mirroringTx) Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) { + t.db.replay(sql, args) + return t.Tx.Query(ctx, sql, args...) +} + +func (t *mirroringTx) QueryRow(ctx context.Context, sql string, args ...any) pgx.Row { + t.db.replay(sql, args) + return t.Tx.QueryRow(ctx, sql, args...) +} diff --git a/internal/server/queryapi/mirror_test.go b/internal/server/queryapi/mirror_test.go new file mode 100644 index 00000000000..80331ab9fac --- /dev/null +++ b/internal/server/queryapi/mirror_test.go @@ -0,0 +1,141 @@ +package queryapi + +import ( + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/armadaproject/armada/internal/common/armadacontext" + dbcommon "github.com/armadaproject/armada/internal/common/database" + "github.com/armadaproject/armada/internal/common/database/lookout" + lookoutschema "github.com/armadaproject/armada/internal/lookout/schema" + lookouthcschema "github.com/armadaproject/armada/internal/lookouthc/schema" + "github.com/armadaproject/armada/internal/server/queryapi/database" + "github.com/armadaproject/armada/pkg/api" +) + +// withPrimaryAndMirrorDbs runs action with two independent test databases: a +// primary carrying the standard lookout schema and a mirror carrying the +// experimental hot-cold partitioned schema. +func withPrimaryAndMirrorDbs(t *testing.T, action func(primary, mirror *pgxpool.Pool)) { + t.Helper() + migrations, err := lookoutschema.LookoutMigrations() + require.NoError(t, err) + + err = dbcommon.WithTestDb(migrations, func(primary *pgxpool.Pool) error { + return dbcommon.WithTestDb(migrations, func(mirror *pgxpool.Pool) error { + require.NoError(t, lookouthcschema.ApplyPartitioner(armadacontext.Background(), mirror)) + action(primary, mirror) + return nil + }) + }) + require.NoError(t, err) +} + +// waitForMirrorQuery blocks until the mirror pool's acquire count has advanced +// past baseline (i.e. a mirrored query has run) or the deadline elapses. +func waitForMirrorQuery(t *testing.T, mirror *pgxpool.Pool, baseline int64) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + if mirror.Stat().AcquireCount() > baseline { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatal("mirror pool was not queried within the deadline") +} + +func TestMirroringDB_ReplaysQueryAndPreservesPrimaryResult(t *testing.T) { + ctx, cancel := armadacontext.WithTimeout(armadacontext.Background(), 30*time.Second) + defer cancel() + + testJobs := []database.Job{ + newJob("job1", lookout.JobRunningOrdinal, ""), + newJob("job2", lookout.JobSucceededOrdinal, ""), + } + + withPrimaryAndMirrorDbs(t, func(primary, mirror *pgxpool.Pool) { + require.NoError(t, dbcommon.UpsertPartitionedWithTransaction(ctx, primary, "job", []string{"job_id"}, testJobs)) + require.NoError(t, dbcommon.UpsertPartitionedWithTransaction(ctx, mirror, "job", []string{"job_id"}, testJobs)) + + request := &api.JobStatusRequest{JobIds: []string{"job1", "job2"}} + + baseline := New(primary, defaultMaxQueryItems, testDecompressor) + baselineResp, err := baseline.GetJobStatus(ctx, request) + require.NoError(t, err) + + mirrorAcquiresBefore := mirror.Stat().AcquireCount() + mirrored := New(NewMirroringDB(primary, mirror, 0), defaultMaxQueryItems, testDecompressor) + mirroredResp, err := mirrored.GetJobStatus(ctx, request) + require.NoError(t, err) + + assert.Equal(t, baselineResp, mirroredResp) + waitForMirrorQuery(t, mirror, mirrorAcquiresBefore) + }) +} + +func TestMirroringDB_ReplaysTransactionalQuery(t *testing.T) { + ctx, cancel := armadacontext.WithTimeout(armadacontext.Background(), 30*time.Second) + defer cancel() + + testJobs := []database.Job{ + newJob("job1", lookout.JobRunningOrdinal, "run1"), + } + testJobRuns := []database.JobRun{ + newJobRun("job1", "run1", lookout.JobRunRunningOrdinal, baseTime, "", testIngressAddresses), + } + + withPrimaryAndMirrorDbs(t, func(primary, mirror *pgxpool.Pool) { + for _, db := range []*pgxpool.Pool{primary, mirror} { + require.NoError(t, dbcommon.UpsertPartitionedWithTransaction(ctx, db, "job", []string{"job_id"}, testJobs)) + require.NoError(t, dbcommon.UpsertPartitionedWithTransaction(ctx, db, "job_run", []string{"run_id"}, testJobRuns)) + } + + request := &api.JobDetailsRequest{JobIds: []string{"job1"}, ExpandJobRun: true} + + baseline := New(primary, defaultMaxQueryItems, testDecompressor) + baselineResp, err := baseline.GetJobDetails(ctx, request) + require.NoError(t, err) + + mirrorAcquiresBefore := mirror.Stat().AcquireCount() + mirrored := New(NewMirroringDB(primary, mirror, 0), defaultMaxQueryItems, testDecompressor) + mirroredResp, err := mirrored.GetJobDetails(ctx, request) + require.NoError(t, err) + + assert.Equal(t, baselineResp, mirroredResp) + // GetJobDetails runs inside a read-only transaction, so this exercises + // the mirroringTx replay path. + waitForMirrorQuery(t, mirror, mirrorAcquiresBefore) + }) +} + +func TestMirroringDB_BrokenMirrorDoesNotAffectPrimary(t *testing.T) { + ctx, cancel := armadacontext.WithTimeout(armadacontext.Background(), 30*time.Second) + defer cancel() + + testJobs := []database.Job{ + newJob("job1", lookout.JobRunningOrdinal, ""), + } + + withPrimaryAndMirrorDbs(t, func(primary, mirror *pgxpool.Pool) { + require.NoError(t, dbcommon.UpsertPartitionedWithTransaction(ctx, primary, "job", []string{"job_id"}, testJobs)) + // Close the mirror pool so every replayed query fails. + mirror.Close() + + request := &api.JobStatusRequest{JobIds: []string{"job1"}} + + baseline := New(primary, defaultMaxQueryItems, testDecompressor) + baselineResp, err := baseline.GetJobStatus(ctx, request) + require.NoError(t, err) + + mirrored := New(NewMirroringDB(primary, mirror, 0), defaultMaxQueryItems, testDecompressor) + mirroredResp, err := mirrored.GetJobStatus(ctx, request) + require.NoError(t, err) + + assert.Equal(t, baselineResp, mirroredResp) + }) +} diff --git a/internal/server/queryapi/query_api.go b/internal/server/queryapi/query_api.go index cf7b0485066..2c922e0ff6c 100644 --- a/internal/server/queryapi/query_api.go +++ b/internal/server/queryapi/query_api.go @@ -6,7 +6,6 @@ import ( "fmt" "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/pgxpool" "github.com/armadaproject/armada/internal/common/compress" "github.com/armadaproject/armada/internal/common/database/lookout" @@ -43,12 +42,12 @@ var JobRunStateMap = map[int16]api.JobRunState{ } type QueryApi struct { - db *pgxpool.Pool + db QueryDB decompressorFactory func() compress.Decompressor maxQueryItems int } -func New(db *pgxpool.Pool, maxQueryItems int, decompressorFactory func() compress.Decompressor) *QueryApi { +func New(db QueryDB, maxQueryItems int, decompressorFactory func() compress.Decompressor) *QueryApi { return &QueryApi{ db: db, maxQueryItems: maxQueryItems, diff --git a/internal/server/server.go b/internal/server/server.go index f6411234aa2..b0932998b4b 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -89,8 +89,23 @@ func Serve(ctx *armadacontext.Context, config *configuration.ArmadaConfig, healt return errors.WithMessage(err, "error creating postgres pool") } defer dbPool.Close() + + var queryDb queryapi.QueryDB = dbPool + if config.QueryApi.Mirror.Enabled { + mirrorPool, err := database.OpenPgxPool(config.QueryApi.Mirror.Postgres) + if err != nil { + // Query mirroring is an evaluation aid; it must never prevent the + // server from starting. Log and continue with mirroring disabled. + log.WithError(err).Error("failed to open query api mirror database; continuing with mirroring disabled") + } else { + // In-flight mirror queries use a detached context, so shutdown may + // block here for up to the mirror query timeout while they drain. + defer mirrorPool.Close() + queryDb = queryapi.NewMirroringDB(dbPool, mirrorPool, config.QueryApi.Mirror.MaxInFlight) + } + } queryapiServer := queryapi.New( - dbPool, + queryDb, config.QueryApi.MaxQueryItems, func() compress.Decompressor { return compress.NewZlibDecompressor() }) api.RegisterJobsServer(grpcServer, queryapiServer) From f78d0320dd9205296c4bad813c4adc5453038d9c Mon Sep 17 00:00:00 2001 From: sarhiri Date: Mon, 13 Jul 2026 14:43:13 -0500 Subject: [PATCH 31/49] fix: add missing meta.json for understanding-armada section Signed-off-by: sarhiri --- website/content/understanding-armada/meta.json | 1 + 1 file changed, 1 insertion(+) create mode 100644 website/content/understanding-armada/meta.json diff --git a/website/content/understanding-armada/meta.json b/website/content/understanding-armada/meta.json new file mode 100644 index 00000000000..40688565eeb --- /dev/null +++ b/website/content/understanding-armada/meta.json @@ -0,0 +1 @@ +{"title": "Understanding Armada", "pages": ["index", "architecture", "core-concepts"]} From 6d41761097cc3a415e69f1fba6b5456dc5a7bb37 Mon Sep 17 00:00:00 2001 From: sarhiri Date: Fri, 17 Jul 2026 13:57:52 -0500 Subject: [PATCH 32/49] Docs: updated local development guide to focus on goreman installation Signed-off-by: sarhiri --- README.md | 174 ++++++++++++++++++++++- website/content/docs/developer-guide.mdx | 33 ++++- 2 files changed, 203 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 494ad04ae7b..1bfce653d3c 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,176 @@
+ Armada logo + +

One API. Any number of clusters. Millions of jobs.

+

The open-source batch job meta-scheduler that makes Kubernetes work at scale.

+ +

+ CircleCI + Go Report Card + Artifact Hub + LFX Health Score + OpenSSF Best Practices +

+ +

+ Website · + Quickstart · + Documentation · + Slack +

+
+ +--- + +## What is Armada? + +Kubernetes was built for services. Armada was built for batch. + +When your job volume exceeds what a single cluster can handle, you need a control plane that sits above your fleet — routing jobs intelligently, fairly, and at scale. Armada is that layer. + +**Armada solves the problems Kubernetes wasn't designed to handle:** + +- **No job queue** — Kubernetes has no concept of ordering. Jobs compete for resources with no fairness guarantees. Armada adds a proper queue with priority, fair-share, and rate limiting. +- **No multi-cluster coordination** — Each Kubernetes cluster is an island. Armada routes jobs across as many clusters as you need from a single API. +- **No gang scheduling** — Distributed jobs that need all workers to start simultaneously (MPI, PyTorch, Spark) have no atomic startup guarantee in vanilla Kubernetes. Armada either starts the whole group or holds it. +- **No fairness across teams** — One team can starve everyone else. Armada enforces fair-share scheduling so heavy users don't permanently dominate shared infrastructure. + +Armada is used in production at [G-Research](https://www.gresearch.co.uk/) since 2020, processing **millions of batch jobs per day** across tens of thousands of nodes. + +--- + +## Features + +| Feature | Description | +|---|---| +| 🌐 **Multi-cluster scheduling** | One API across unlimited Kubernetes clusters | +| ⚖️ **Fair-share queuing** | Dominant resource fairness across teams and queues | +| 🔗 **Gang scheduling** | Atomic startup for distributed workloads | +| ⚡ **Preemption** | Urgent jobs bump lower-priority work automatically | +| 📊 **Prometheus metrics** | Full observability into queue health and cluster utilisation | +| 🔭 **Lookout UI** | Web interface for monitoring jobs, queues, and clusters | +| 🔒 **Enterprise-ready** | Secure, highly available, OIDC authentication support | + +--- + +## Getting started + +The fastest way to get Armada running locally is with the [Armada Operator](https://github.com/armadaproject/armada-operator): + +```bash +git clone https://github.com/armadaproject/armada-operator.git +cd armada-operator +make kind-all +``` + +→ **[Full quickstart guide](https://armadaproject.io/quickstart)** — get up and running in under 15 minutes. + +### armadactl + +Armada's CLI for interacting with the system: + +```bash +# download via script +scripts/get-armadactl.sh + +# or grab the binary from the releases page +https://github.com/armadaproject/armada/releases/latest +``` + +--- + +## Local development + +Armada runs locally via [Goreman](https://github.com/mattn/goreman) — dependencies (Redis, Postgres, Pulsar) run in containers, Armada components run as host processes built from source. Iteration is fast and debuggers attach directly. + +```bash +mage kind # one-time: create local Kubernetes cluster +export KUBECONFIG=.kube/external/config + +mage dev:up # default — no auth +mage dev:up auth # with OIDC via Keycloak +mage dev:up fake-executor # no Kubernetes cluster needed +mage dev:down # stop dependency containers +``` + +→ **[Full local development guide](https://armadaproject.io/docs/developer-guide)** — profiles, procfiles, service ports, authentication, and debugging. + +--- + +## Use cases + +Armada is used wherever batch jobs are too large, too many, or too complex for a single Kubernetes cluster: + +- **Quantitative finance & HPC** — millions of short-lived simulations per day with fair-share across research teams +- **ML and AI training** — distributed GPU training with gang scheduling across clusters +- **Platform engineering** — multi-tenant batch infrastructure with a single API surface +- **SLURM migration** — familiar scheduling semantics (queues, priorities, preemption) on Kubernetes-native infrastructure +- **CI/CD at scale** — priority control so critical merges always run first + +--- + +## In production + +Armada has been running in production at [G-Research](https://www.gresearch.co.uk/) since 2020. + +**Running Armada in production?** Open a PR to add yourself to [ADOPTERS.md](./ADOPTERS.md) 🙌 + +--- + +## Community + +Everyone is welcome — come and say hi! 👋 + +- 💬 **Slack** — [#armada on CNCF Slack](https://cloud-native.slack.com/archives/C03T9CBCEMC) — fastest way to get help and talk to maintainers +- 💡 **GitHub Discussions** — [longer-form questions and ideas](https://github.com/armadaproject/armada/discussions) +- 🐛 **GitHub Issues** — [bug reports and feature requests](https://github.com/armadaproject/armada/issues) +- 📅 **Community meetings** — bi-weekly, open to all. Join [#armada on Slack](https://cloud-native.slack.com/archives/C03T9CBCEMC) for the invite link +- ⭐ **Star the repo** — helps more people find Armada + +--- + +## Contributing + +We'd love your contributions — code, docs, bug reports, or ideas. All are welcome. + +- Read [CONTRIBUTING.md](./CONTRIBUTING.md) for guidelines +- Check [good first issues](https://github.com/armadaproject/armada/labels/good%20first%20issue) for a starting point +- All commits require a [DCO sign-off](https://developercertificate.org/): `git commit -s` +- Please review [CODE_OF_CONDUCT.md](./CODE_OF_CONDUCT.md) before contributing + +--- + +## Documentation + +| Resource | Link | +|---|---| +| Website & overview | [armadaproject.io](https://armadaproject.io) | +| Quickstart | [armadaproject.io/quickstart](https://armadaproject.io/quickstart) | +| Architecture | [armadaproject.io/docs/architecture](https://armadaproject.io/docs/architecture) | +| API reference | [armadaproject.io/docs/api](https://armadaproject.io/docs/api) | +| Developer guide | [armadaproject.io/docs/developer-guide](https://armadaproject.io/docs/developer-guide) | +| Release notes | [github.com/armadaproject/armada/releases](https://github.com/armadaproject/armada/releases) | + +--- + +## Talks and videos + +- [Armada — high-throughput batch scheduling](https://www.youtube.com/watch?v=FT8pXYciD9A) +- [Building Armada — Running Batch Jobs at Massive Scale on Kubernetes](https://www.youtube.com/watch?v=B3WPxw3OUl4) + +--- + +
+
+ CNCF logo +
+ Armada is a Cloud Native Computing Foundation Sandbox project 🚀 +
+ Apache 2.0 License +
+ + + diff --git a/website/content/docs/developer-guide.mdx b/website/content/docs/developer-guide.mdx index 39393d0cd96..376b83197ad 100644 --- a/website/content/docs/developer-guide.mdx +++ b/website/content/docs/developer-guide.mdx @@ -5,7 +5,7 @@ description: 'Set up your development environment and start contributing to Arma import { Callout } from 'fumadocs-ui/components/callout'; import { Step, Steps } from 'fumadocs-ui/components/steps'; -This guide walks you through setting up a local Armada development environment using Goreman our recommended approach for contributing to Armada. +This guide walks you through setting up a local Armada development environment using Goreman, our recommended approach for contributing to Armada. ## Prerequisites @@ -25,6 +25,9 @@ Before you begin, make sure you have the following installed: `tools.yaml`, including golangci-lint, sqlc, go-swagger, and others. +{/* divider */} +
+ ## Using Goreman [Goreman](https://github.com/mattn/goreman) is a Go-based clone of @@ -117,6 +120,9 @@ mage -l # list all available mage commands development. It's faster since components run as host processes. +{/* divider */} +
+ ## Running with authentication @@ -174,6 +180,9 @@ armadactl --config _local/.armadactl.yaml --context auth-oidc get queues Default Keycloak credentials — Admin: `admin` / `admin` · User: `user` / `password` +{/* divider */} +
+ ## Running without a Kubernetes cluster For testing Armada without a real Kubernetes cluster, use the fake executor @@ -192,6 +201,9 @@ The fake executor simulates: Useful for testing scheduling logic, development when Kubernetes is unavailable, and integration testing of job flows. +{/* divider */} +
+ ## Code structure Armada
├── cmd/ # Entry points for all components
@@ -213,6 +225,9 @@ Armada
├── magefiles/ # Build automation (mage targets)
└── testsuite/ # Integration test cases
+{/* divider */} +
+ ## Testing your setup Run the full test suite: @@ -230,6 +245,9 @@ export ARMADA_EXECUTOR_INGRESS_PORT=5001 go run cmd/testsuite/main.go test --tests "testsuite/testcases/basic/*" --junit junit.xml ``` +{/* divider */} +
+ ## Profiling with pprof Enable profiling in your component config: @@ -245,6 +263,9 @@ Then connect: go tool pprof http://localhost:6060/debug/pprof/profile ``` +{/* divider */} +
+ ## Debug port mappings | Component | Debug host | @@ -257,6 +278,9 @@ go tool pprof http://localhost:6060/debug/pprof/profile | `lookout` | `localhost:4005` | | `lookoutingester` | `localhost:4007` | +{/* divider */} +
+ ## Troubleshooting **Port 6443 already in use** @@ -279,9 +303,12 @@ See [Arm issue #2493](https://github.com/armadaproject/armada/issues/2493) and [Windows issue #2492](https://github.com/armadaproject/armada/issues/2492) for more details. -**Need help?** +{/* **Need help?** + +Ask in the [#armada channel on CNCF Slack](https://cloud-native.slack.com/archives/C03T9CBCEMC). */} -Ask in the [#armada channel on CNCF Slack](https://cloud-native.slack.com/archives/C03T9CBCEMC). +{/* divider */} +
{/* ── CTA ── */} From a45322983ccd38e04b046956ce04a953e6bb0b12 Mon Sep 17 00:00:00 2001 From: Maurice Yap Date: Tue, 14 Jul 2026 09:43:52 +0100 Subject: [PATCH 33/49] Add configurable client-side request mirroring to Lookout UI (#5014) This adds an optional `uiConfig.requestMirror` setting (`enabled` + `targetUrl`) so the Lookout UI can duplicate its Lookout API requests to a secondary backend (e.g. an experimental Lookout HC deployment), letting us observe its performance under real production query patterns without users changing their workflow. Mirrored requests are fire-and-forget, carry the same auth headers as the original, and are tagged with `X-Mirrored-Request: true`. Failures are silently ignored and never surface to users. --------- Signed-off-by: Maurice Yap Signed-off-by: sarhiri --- internal/lookout/configuration/doc.go | 6 ++ internal/lookout/configuration/types.go | 13 ++++ .../src/components/hooks/useJobsTableData.ts | 6 +- internal/lookoutui/src/config/config.ts | 1 + internal/lookoutui/src/config/types.ts | 11 ++++ .../src/lookoutApiRequestMirror/index.ts | 1 + .../mirrorLookoutApiRequest.ts | 60 +++++++++++++++++++ internal/lookoutui/src/oidcAuth/hooks.ts | 22 ++++++- .../lookout/useGetAllJobsMatchingFilters.ts | 8 +-- .../services/lookout/useGetBackendVersion.ts | 6 +- .../src/services/lookout/useGetJobError.ts | 6 +- .../lookout/useGetJobRunDebugMessage.ts | 12 ++-- .../src/services/lookout/useGetJobRunError.ts | 12 ++-- .../useGetJobRunSchedulerTerminationReason.ts | 8 +-- .../src/services/lookout/useGetJobSpec.ts | 6 +- .../src/services/lookout/useGetJobs.ts | 8 +-- .../src/services/lookout/useGroupJobs.ts | 8 +-- 17 files changed, 151 insertions(+), 43 deletions(-) create mode 100644 internal/lookout/configuration/doc.go create mode 100644 internal/lookoutui/src/lookoutApiRequestMirror/index.ts create mode 100644 internal/lookoutui/src/lookoutApiRequestMirror/mirrorLookoutApiRequest.ts diff --git a/internal/lookout/configuration/doc.go b/internal/lookout/configuration/doc.go new file mode 100644 index 00000000000..90699086094 --- /dev/null +++ b/internal/lookout/configuration/doc.go @@ -0,0 +1,6 @@ +// Package configuration defines the configuration types for the Lookout +// server, including database, TLS, pruner and UI configuration. UIConfig is +// serialised to JSON and served to the Lookout UI frontend, where it must be +// kept in sync with the LookoutUiConfig TypeScript interface defined in +// internal/lookoutui/src/config/types.ts. +package configuration diff --git a/internal/lookout/configuration/types.go b/internal/lookout/configuration/types.go index 77ba9a13baf..16e6ef08e66 100644 --- a/internal/lookout/configuration/types.go +++ b/internal/lookout/configuration/types.go @@ -196,6 +196,16 @@ type OidcConfig struct { DisplayNameClaim *string `json:"displayNameClaim,omitempty"` } +// RequestMirrorConfig configures client-side mirroring of Lookout UI API +// requests to a secondary backend, e.g. for comparing performance against an +// experimental deployment under real traffic patterns. TargetUrl must be an +// HTTPS origin: mirrored requests carry the user's Authorization header, so it +// must never be sent to a plaintext or untrusted origin. +type RequestMirrorConfig struct { + Enabled bool `json:"enabled"` + TargetUrl string `json:"targetUrl,omitempty"` +} + // UIConfig must match the LookoutUiConfig TypeScript interface defined in internal/lookoutui/src/config/types.ts type UIConfig struct { CustomTitle string `json:"customTitle"` @@ -227,4 +237,7 @@ type UIConfig struct { // Analytics is an optional analytics configuration Analytics *AnalyticsConfig `json:"analytics,omitempty"` + + // RequestMirror is an optional configuration for mirroring API requests to a secondary backend + RequestMirror *RequestMirrorConfig `json:"requestMirror,omitempty"` } diff --git a/internal/lookoutui/src/components/hooks/useJobsTableData.ts b/internal/lookoutui/src/components/hooks/useJobsTableData.ts index c1a92a50371..2e56876099e 100644 --- a/internal/lookoutui/src/components/hooks/useJobsTableData.ts +++ b/internal/lookoutui/src/components/hooks/useJobsTableData.ts @@ -25,7 +25,7 @@ import { getErrorMessage } from "../../common/utils" import { getConfig } from "../../config" import { JobGroupRow, JobRow, JobTableRow } from "../../models/jobsTableModels" import { AggregateType, Job, JobFilter, JobGroup, JobId, JobOrder, Match } from "../../models/lookoutModels" -import { useAuthenticatedFetch } from "../../oidcAuth" +import { useMirroredLookoutApiFetch } from "../../oidcAuth" import { GetJobsResponse } from "../../services/lookout/useGetJobs" import { GroupedField } from "../../services/lookout/useGroupJobs" @@ -153,7 +153,7 @@ export const useFetchJobsTableData = ({ const [jobInfoMap, setJobInfoMap] = useState>(new Map()) const [pendingData, setPendingData] = useState([]) - const authenticatedFetch = useAuthenticatedFetch() + const lookoutApiFetch = useMirroredLookoutApiFetch() useEffect(() => { const abortController = new AbortController() @@ -184,7 +184,7 @@ export const useFetchJobsTableData = ({ let newData try { if (isJobFetch) { - const { jobs } = await fetchJobs(authenticatedFetch, rowRequest, abortController.signal) + const { jobs } = await fetchJobs(lookoutApiFetch, rowRequest, abortController.signal) newData = jobsToRows(jobs) setJobInfoMap(new Map([...jobInfoMap.entries(), ...jobs.map((j): [JobId, Job] => [j.jobId, j])])) diff --git a/internal/lookoutui/src/config/config.ts b/internal/lookoutui/src/config/config.ts index 9403ee398f1..cb781ef5210 100644 --- a/internal/lookoutui/src/config/config.ts +++ b/internal/lookoutui/src/config/config.ts @@ -20,6 +20,7 @@ export const DEFAULT_LOOKOUT_UI_CONFIG: LookoutUiConfig = { }, customThemeConfigs: undefined, analytics: undefined, + requestMirror: undefined, } export const getConfig = (): Config => { diff --git a/internal/lookoutui/src/config/types.ts b/internal/lookoutui/src/config/types.ts index 62d75339af4..e9eaabdba2e 100644 --- a/internal/lookoutui/src/config/types.ts +++ b/internal/lookoutui/src/config/types.ts @@ -59,6 +59,16 @@ export interface AnalyticsConfig { dataWrapper?: string } +// RequestMirrorConfig configures client-side mirroring of Lookout UI API +// requests to a secondary backend, e.g. for comparing performance against an +// experimental deployment under real traffic patterns. targetUrl must be an +// HTTPS origin: mirrored requests carry the user's Authorization header, so it +// must never be sent to a plaintext or untrusted origin. +export interface RequestMirrorConfig { + enabled: boolean + targetUrl: string +} + // This must match the UIConfig Go struct defined in internal/lookout/configuration/types.go export interface LookoutUiConfig { armadaApiBaseUrl: string @@ -76,6 +86,7 @@ export interface LookoutUiConfig { errorMonitoring: ErrorMonitoringConfig customThemeConfigs: CustomThemeConfigs | undefined analytics: AnalyticsConfig | undefined + requestMirror: RequestMirrorConfig | undefined } export interface Config extends LookoutUiConfig { diff --git a/internal/lookoutui/src/lookoutApiRequestMirror/index.ts b/internal/lookoutui/src/lookoutApiRequestMirror/index.ts new file mode 100644 index 00000000000..ec45335a8aa --- /dev/null +++ b/internal/lookoutui/src/lookoutApiRequestMirror/index.ts @@ -0,0 +1 @@ +export * from "./mirrorLookoutApiRequest" diff --git a/internal/lookoutui/src/lookoutApiRequestMirror/mirrorLookoutApiRequest.ts b/internal/lookoutui/src/lookoutApiRequestMirror/mirrorLookoutApiRequest.ts new file mode 100644 index 00000000000..37fdd224780 --- /dev/null +++ b/internal/lookoutui/src/lookoutApiRequestMirror/mirrorLookoutApiRequest.ts @@ -0,0 +1,60 @@ +import { getConfig } from "../config" + +// resolveMirrorOrigin returns the origin to mirror Lookout API requests to, or +// undefined if mirroring is disabled or misconfigured. The target must be +// HTTPS: mirrored requests carry the user's Authorization header, which must +// never be sent to an untrusted or plaintext origin. +const resolveMirrorOrigin = (): string | undefined => { + const requestMirror = getConfig().requestMirror + if (!requestMirror?.enabled || !requestMirror.targetUrl) { + return undefined + } + try { + const url = new URL(requestMirror.targetUrl) + return url.protocol === "https:" ? url.origin : undefined + } catch { + return undefined + } +} + +const mirrorOrigin = resolveMirrorOrigin() + +// isLookoutApiRequest reports whether a request targets the Lookout server's +// own REST API. The Lookout API is served same-origin under /api/; requests to +// the Armada server (config.armadaApiBaseUrl) and Binoculars +// (config.binocularsBaseUrlPattern) use absolute URLs to other origins and are +// deliberately excluded, since only Lookout API load is being evaluated. +const isLookoutApiRequest = (url: URL): boolean => + url.origin === window.location.origin && url.pathname.startsWith("/api/") + +// mirrorLookoutApiRequest duplicates an outgoing Lookout API request to the +// configured mirror backend (e.g. an experimental deployment under performance +// evaluation), so it can observe real production query patterns without +// affecting the user. Requests that are not Lookout API requests are ignored. +// It is fire-and-forget: the mirrored response is discarded and failures are +// silently ignored. +export const mirrorLookoutApiRequest = (input: RequestInfo | URL, init: RequestInit | undefined): void => { + if (!mirrorOrigin) { + return + } + + try { + const requestUrl = new URL(input instanceof Request ? input.url : input.toString(), window.location.href) + if (!isLookoutApiRequest(requestUrl)) { + return + } + + const targetUrl = `${mirrorOrigin}${requestUrl.pathname}${requestUrl.search}` + + const headers = new Headers(init?.headers) + headers.set("X-Mirrored-Request", "true") + + fetch(targetUrl, { + ...init, + headers, + credentials: "include", + }).catch(() => undefined) + } catch { + // Silently ignore mirror failures + } +} diff --git a/internal/lookoutui/src/oidcAuth/hooks.ts b/internal/lookoutui/src/oidcAuth/hooks.ts index 1a64635b770..b11297ab818 100644 --- a/internal/lookoutui/src/oidcAuth/hooks.ts +++ b/internal/lookoutui/src/oidcAuth/hooks.ts @@ -3,6 +3,7 @@ import { useCallback, useContext, useEffect, useState } from "react" import { UserManager } from "oidc-client-ts" import { getConfig } from "../config" +import { mirrorLookoutApiRequest } from "../lookoutApiRequestMirror" import { OidcAuthContext } from "./OidcAuthContext" import { appendAuthorizationHeaders } from "./utils" @@ -61,7 +62,11 @@ export const useGetAccessToken = () => { }, [userManager]) } -export const useAuthenticatedFetch = () => { +// useAuthenticatedFetchInternal returns a fetch function that attaches the +// current access token, and additionally invokes onRequest (if given) with the +// final input and token-bearing init just before the request is sent. This is +// the shared core of useAuthenticatedFetch and useMirroredLookoutApiFetch. +const useAuthenticatedFetchInternal = (onRequest?: (input: RequestInfo | URL, init: RequestInit) => void) => { const getAccessToken = useGetAccessToken() return useCallback( (input, init) => @@ -70,8 +75,19 @@ export const useAuthenticatedFetch = () => { if (accessToken) { appendAuthorizationHeaders(headers, accessToken) } - return fetch(input, { ...init, headers }) + const authenticatedInit = { ...init, headers } + onRequest?.(input, authenticatedInit) + return fetch(input, authenticatedInit) }), - [getAccessToken], + [getAccessToken, onRequest], ) } + +export const useAuthenticatedFetch = () => useAuthenticatedFetchInternal() + +// useMirroredLookoutApiFetch behaves like useAuthenticatedFetch but also mirrors +// each Lookout API request to the configured mirror backend. Use it in place of +// useAuthenticatedFetch for calls to the Lookout server's own REST API, so that +// only Lookout API load is mirrored; requests to the Armada server or +// Binoculars should continue to use useAuthenticatedFetch directly. +export const useMirroredLookoutApiFetch = () => useAuthenticatedFetchInternal(mirrorLookoutApiRequest) diff --git a/internal/lookoutui/src/services/lookout/useGetAllJobsMatchingFilters.ts b/internal/lookoutui/src/services/lookout/useGetAllJobsMatchingFilters.ts index 1530d886704..db7d3bb22df 100644 --- a/internal/lookoutui/src/services/lookout/useGetAllJobsMatchingFilters.ts +++ b/internal/lookoutui/src/services/lookout/useGetAllJobsMatchingFilters.ts @@ -4,7 +4,7 @@ import _ from "lodash" import { getConfig } from "../../config" import { Job, JobFilter, JobFiltersWithExcludes } from "../../models/lookoutModels" -import { useAuthenticatedFetch } from "../../oidcAuth" +import { useMirroredLookoutApiFetch } from "../../oidcAuth" const MAX_JOBS_PER_REQUEST = 10_000 @@ -37,7 +37,7 @@ export const useGetAllJobsMatchingFilters = ({ const [error, setError] = useState(null) const [refetchCounter, setRefetchCounter] = useState(0) - const authenticatedFetch = useAuthenticatedFetch() + const lookoutApiFetch = useMirroredLookoutApiFetch() const config = getConfig() // Create a stable key for filtersGroups to avoid unnecessary re-renders @@ -50,7 +50,7 @@ export const useGetAllJobsMatchingFilters = ({ path += "?" + new URLSearchParams({ backend: config.backend }) } - const response = await authenticatedFetch(path, { + const response = await lookoutApiFetch(path, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ @@ -67,7 +67,7 @@ export const useGetAllJobsMatchingFilters = ({ jobs: json.jobs ?? [], } }, - [authenticatedFetch, config.backend], + [lookoutApiFetch, config.backend], ) const fetchJobsWithPagination = useCallback( diff --git a/internal/lookoutui/src/services/lookout/useGetBackendVersion.ts b/internal/lookoutui/src/services/lookout/useGetBackendVersion.ts index b86c0d145b0..539d25ededd 100644 --- a/internal/lookoutui/src/services/lookout/useGetBackendVersion.ts +++ b/internal/lookoutui/src/services/lookout/useGetBackendVersion.ts @@ -1,7 +1,7 @@ import { useQuery } from "@tanstack/react-query" import { getErrorMessage } from "../../common/utils" -import { useAuthenticatedFetch } from "../../oidcAuth" +import { useMirroredLookoutApiFetch } from "../../oidcAuth" export interface VersionInfo { version: string @@ -16,13 +16,13 @@ export const UNKNOWN_VERSION_INFO: VersionInfo = { } export const useGetBackendVersion = () => { - const authenticatedFetch = useAuthenticatedFetch() + const lookoutApiFetch = useMirroredLookoutApiFetch() return useQuery({ queryKey: ["getVersion"], queryFn: async ({ signal }) => { try { - const response = await authenticatedFetch("/api/v1/version", { signal }) + const response = await lookoutApiFetch("/api/v1/version", { signal }) if (!response.ok) { throw new Error(`Request for version failed with status ${response.status}`) } diff --git a/internal/lookoutui/src/services/lookout/useGetJobError.ts b/internal/lookoutui/src/services/lookout/useGetJobError.ts index 65aa6827b80..2feaac301c7 100644 --- a/internal/lookoutui/src/services/lookout/useGetJobError.ts +++ b/internal/lookoutui/src/services/lookout/useGetJobError.ts @@ -2,14 +2,14 @@ import { useQuery } from "@tanstack/react-query" import { getErrorMessage } from "../../common/utils" import { getConfig } from "../../config" -import { useAuthenticatedFetch } from "../../oidcAuth" +import { useMirroredLookoutApiFetch } from "../../oidcAuth" import { fakeJobError } from "./mocks/fakeData" export const useGetJobError = (jobId: string, enabled = true) => { const config = getConfig() - const authenticatedFetch = useAuthenticatedFetch() + const lookoutApiFetch = useMirroredLookoutApiFetch() return useQuery({ queryKey: ["getJobError", jobId], @@ -19,7 +19,7 @@ export const useGetJobError = (jobId: string, enabled = true) => { return fakeJobError } - const response = await authenticatedFetch("/api/v1/jobError", { + const response = await lookoutApiFetch("/api/v1/jobError", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ jobId }), diff --git a/internal/lookoutui/src/services/lookout/useGetJobRunDebugMessage.ts b/internal/lookoutui/src/services/lookout/useGetJobRunDebugMessage.ts index a7c2b076c56..ece7bb13e74 100644 --- a/internal/lookoutui/src/services/lookout/useGetJobRunDebugMessage.ts +++ b/internal/lookoutui/src/services/lookout/useGetJobRunDebugMessage.ts @@ -4,7 +4,7 @@ import { QueryFunction, QueryKey, useQueries, useQuery } from "@tanstack/react-q import { getErrorMessage } from "../../common/utils" import { getConfig } from "../../config" -import { useAuthenticatedFetch } from "../../oidcAuth" +import { useMirroredLookoutApiFetch } from "../../oidcAuth" import { fakeRunDebugMessage } from "./mocks/fakeData" @@ -33,11 +33,11 @@ const getQueryFn = export const useGetJobRunDebugMessage = (runId: string, enabled = true) => { const config = getConfig() - const authenticatedFetch = useAuthenticatedFetch() + const lookoutApiFetch = useMirroredLookoutApiFetch() const queryFn = useMemo( - () => getQueryFn(runId, authenticatedFetch, config.fakeDataEnabled), - [runId, authenticatedFetch, config.fakeDataEnabled], + () => getQueryFn(runId, lookoutApiFetch, config.fakeDataEnabled), + [runId, lookoutApiFetch, config.fakeDataEnabled], ) return useQuery({ @@ -51,12 +51,12 @@ export const useGetJobRunDebugMessage = (runId: string, enabled = true) => { export const useBatchGetJobRunDebugMessages = (runIds: string[], enabled = true) => { const config = getConfig() - const authenticatedFetch = useAuthenticatedFetch() + const lookoutApiFetch = useMirroredLookoutApiFetch() return useQueries({ queries: runIds.map((runId) => ({ queryKey: ["getJobRunDebugMessage", runId], - queryFn: getQueryFn(runId, authenticatedFetch, config.fakeDataEnabled), + queryFn: getQueryFn(runId, lookoutApiFetch, config.fakeDataEnabled), enabled, refetchOnMount: false, staleTime: 30_000, diff --git a/internal/lookoutui/src/services/lookout/useGetJobRunError.ts b/internal/lookoutui/src/services/lookout/useGetJobRunError.ts index 6c8f84e9b6a..f3c076ab341 100644 --- a/internal/lookoutui/src/services/lookout/useGetJobRunError.ts +++ b/internal/lookoutui/src/services/lookout/useGetJobRunError.ts @@ -4,7 +4,7 @@ import { QueryFunction, QueryKey, useQueries, useQuery } from "@tanstack/react-q import { getErrorMessage } from "../../common/utils" import { getConfig } from "../../config" -import { useAuthenticatedFetch } from "../../oidcAuth" +import { useMirroredLookoutApiFetch } from "../../oidcAuth" import { fakeRunError } from "./mocks/fakeData" @@ -32,11 +32,11 @@ const getQueryFn = export const useGetJobRunError = (runId: string, enabled = true) => { const config = getConfig() - const authenticatedFetch = useAuthenticatedFetch() + const lookoutApiFetch = useMirroredLookoutApiFetch() const queryFn = useMemo( - () => getQueryFn(runId, authenticatedFetch, config.fakeDataEnabled), - [runId, authenticatedFetch, config.fakeDataEnabled], + () => getQueryFn(runId, lookoutApiFetch, config.fakeDataEnabled), + [runId, lookoutApiFetch, config.fakeDataEnabled], ) return useQuery({ @@ -50,12 +50,12 @@ export const useGetJobRunError = (runId: string, enabled = true) => { export const useBatchGetJobRunErrors = (runIds: string[], enabled = true) => { const config = getConfig() - const authenticatedFetch = useAuthenticatedFetch() + const lookoutApiFetch = useMirroredLookoutApiFetch() return useQueries({ queries: runIds.map((runId) => ({ queryKey: ["getJobRunError", runId], - queryFn: getQueryFn(runId, authenticatedFetch, config.fakeDataEnabled), + queryFn: getQueryFn(runId, lookoutApiFetch, config.fakeDataEnabled), enabled, refetchOnMount: false, staleTime: 30_000, diff --git a/internal/lookoutui/src/services/lookout/useGetJobRunSchedulerTerminationReason.ts b/internal/lookoutui/src/services/lookout/useGetJobRunSchedulerTerminationReason.ts index e9be3308ffb..aeeda02a9a0 100644 --- a/internal/lookoutui/src/services/lookout/useGetJobRunSchedulerTerminationReason.ts +++ b/internal/lookoutui/src/services/lookout/useGetJobRunSchedulerTerminationReason.ts @@ -4,7 +4,7 @@ import { QueryFunction, QueryKey, useQuery } from "@tanstack/react-query" import { getErrorMessage } from "../../common/utils" import { getConfig } from "../../config" -import { useAuthenticatedFetch } from "../../oidcAuth" +import { useMirroredLookoutApiFetch } from "../../oidcAuth" const getQueryFn = (runId: string, fetchFunc: GlobalFetch["fetch"], fakeDataEnabled: boolean): QueryFunction => @@ -30,11 +30,11 @@ const getQueryFn = export const useGetJobRunSchedulerTerminationReason = (runId: string, enabled = true) => { const config = getConfig() - const authenticatedFetch = useAuthenticatedFetch() + const lookoutApiFetch = useMirroredLookoutApiFetch() const queryFn = useMemo( - () => getQueryFn(runId, authenticatedFetch, config.fakeDataEnabled), - [runId, authenticatedFetch, config.fakeDataEnabled], + () => getQueryFn(runId, lookoutApiFetch, config.fakeDataEnabled), + [runId, lookoutApiFetch, config.fakeDataEnabled], ) return useQuery({ diff --git a/internal/lookoutui/src/services/lookout/useGetJobSpec.ts b/internal/lookoutui/src/services/lookout/useGetJobSpec.ts index 2a433c631b7..08cbb5eb214 100644 --- a/internal/lookoutui/src/services/lookout/useGetJobSpec.ts +++ b/internal/lookoutui/src/services/lookout/useGetJobSpec.ts @@ -3,10 +3,10 @@ import { useQuery } from "@tanstack/react-query" import { makeFakeJobSpec } from "../../common/fakeJobsUtils" import { getErrorMessage } from "../../common/utils" import { getConfig } from "../../config" -import { useAuthenticatedFetch } from "../../oidcAuth" +import { useMirroredLookoutApiFetch } from "../../oidcAuth" export const useGetJobSpec = (jobId: string, enabled = true) => { - const authenticatedFetch = useAuthenticatedFetch() + const lookoutApiFetch = useMirroredLookoutApiFetch() const config = getConfig() return useQuery, string>({ @@ -17,7 +17,7 @@ export const useGetJobSpec = (jobId: string, enabled = true) => { return makeFakeJobSpec(jobId) } - const response = await authenticatedFetch("/api/v1/jobSpec", { + const response = await lookoutApiFetch("/api/v1/jobSpec", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ jobId }), diff --git a/internal/lookoutui/src/services/lookout/useGetJobs.ts b/internal/lookoutui/src/services/lookout/useGetJobs.ts index 7f8477cd3ea..36b0df48ac8 100644 --- a/internal/lookoutui/src/services/lookout/useGetJobs.ts +++ b/internal/lookoutui/src/services/lookout/useGetJobs.ts @@ -6,7 +6,7 @@ import { compareValues, makeRandomJobs, mergeFilters } from "../../common/fakeJo import { getErrorMessage } from "../../common/utils" import { getConfig } from "../../config" import { Job, JobFilter, JobKey, JobOrder } from "../../models/lookoutModels" -import { useAuthenticatedFetch } from "../../oidcAuth" +import { useMirroredLookoutApiFetch } from "../../oidcAuth" export interface GetJobsParams { filters: JobFilter[] @@ -79,11 +79,11 @@ const getQueryFn = export const useGetJobs = (params: GetJobsParams, enabled = true) => { const config = getConfig() - const authenticatedFetch = useAuthenticatedFetch() + const lookoutApiFetch = useMirroredLookoutApiFetch() const queryFn = useMemo( - () => getQueryFn(params, authenticatedFetch, config.backend, config.fakeDataEnabled), - [params, authenticatedFetch, config.backend, config.fakeDataEnabled], + () => getQueryFn(params, lookoutApiFetch, config.backend, config.fakeDataEnabled), + [params, lookoutApiFetch, config.backend, config.fakeDataEnabled], ) return useQuery({ diff --git a/internal/lookoutui/src/services/lookout/useGroupJobs.ts b/internal/lookoutui/src/services/lookout/useGroupJobs.ts index d1472fc2740..7c8e1ad6493 100644 --- a/internal/lookoutui/src/services/lookout/useGroupJobs.ts +++ b/internal/lookoutui/src/services/lookout/useGroupJobs.ts @@ -4,7 +4,7 @@ import { makeRandomJobs, mergeFilters } from "../../common/fakeJobsUtils" import { getErrorMessage } from "../../common/utils" import { getConfig } from "../../config" import { AggregateType, Job, JobFilter, JobGroup, JobOrder, JobState } from "../../models/lookoutModels" -import { useAuthenticatedFetch } from "../../oidcAuth" +import { useMirroredLookoutApiFetch } from "../../oidcAuth" export type GroupedField = { field: string @@ -64,7 +64,7 @@ function groupFakeJobs( } export const useGroupJobs = () => { - const authenticatedFetch = useAuthenticatedFetch() + const lookoutApiFetch = useMirroredLookoutApiFetch() const config = getConfig() const fakeJobs = useMemo(() => (config.fakeDataEnabled ? getFakeJobs() : []), [config.fakeDataEnabled]) @@ -89,7 +89,7 @@ export const useGroupJobs = () => { path += "?" + new URLSearchParams({ backend: config.backend }) } - const response = await authenticatedFetch(path, { + const response = await lookoutApiFetch(path, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ @@ -112,6 +112,6 @@ export const useGroupJobs = () => { throw await getErrorMessage(e) } }, - [authenticatedFetch, config.backend, config.fakeDataEnabled, fakeJobs], + [lookoutApiFetch, config.backend, config.fakeDataEnabled, fakeJobs], ) } From 7dceb0cc657a0a82415577ce7ea243892d155491 Mon Sep 17 00:00:00 2001 From: Maurice Yap Date: Tue, 14 Jul 2026 11:47:09 +0100 Subject: [PATCH 34/49] Remove X-Mirrored-Request header from Lookout requests (#5017) This header is not accepted by the Lookout API server Signed-off-by: sarhiri --- .../src/lookoutApiRequestMirror/mirrorLookoutApiRequest.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/internal/lookoutui/src/lookoutApiRequestMirror/mirrorLookoutApiRequest.ts b/internal/lookoutui/src/lookoutApiRequestMirror/mirrorLookoutApiRequest.ts index 37fdd224780..65bdd40282d 100644 --- a/internal/lookoutui/src/lookoutApiRequestMirror/mirrorLookoutApiRequest.ts +++ b/internal/lookoutui/src/lookoutApiRequestMirror/mirrorLookoutApiRequest.ts @@ -46,12 +46,8 @@ export const mirrorLookoutApiRequest = (input: RequestInfo | URL, init: RequestI const targetUrl = `${mirrorOrigin}${requestUrl.pathname}${requestUrl.search}` - const headers = new Headers(init?.headers) - headers.set("X-Mirrored-Request", "true") - fetch(targetUrl, { ...init, - headers, credentials: "include", }).catch(() => undefined) } catch { From a07cf3f5918bd4b0ea62811bfda89a73f0f38dec Mon Sep 17 00:00:00 2001 From: Ian Hockett <32877705+ianhockett@users.noreply.github.com> Date: Wed, 15 Jul 2026 09:33:42 -0500 Subject: [PATCH 35/49] Fix mage testsuite failures behind a TLS-intercepting proxy (#5019) fix: make mage testsuite pass behind a TLS-intercepting proxy `mage testsuite` relies on kind having its test images preloaded from the host, but three cases had drifted onto tags the preload list didn't cover, so kind pulled them directly from `docker.io` inside the cluster. Behind a TLS-intercepting corporate proxy that fails with `x509: certificate signed by unknown authority`, since the kind node's containerd doesn't trust the proxy's root CA. Rather than grow the preload list to chase the drifted tags, pin the tests to the versions already in use. The spread was accidental: the preempt tests inherited `alpine:latest` from an example-job template, and `submit_fileshare_1x1`'s reader was left on `alpine:3.16` when its writer (and every other test) moved to `3.20.0`. None of these tests depend on a specific version. - Pin `preempt_by_id_1x1`, `preempt_by_ids_1x5`, and the `submit_fileshare_1x1` reader to `alpine:3.20.0`, and add the missing `imagePullPolicy: IfNotPresent` on the preempt pods (without it, k8s defaults to `Always` for any tag and re-pulls regardless of the preload). - Bump the preloaded kubectl image to `bitnamilegacy/kubectl:1.33.4` to match what the gang/service_headless tests already reference. - Add a `docs/developer_guide.md` troubleshooting entry for the `docker buildx is not set to default context` error hit during `mage dev:full` on Docker runtimes other than Docker Desktop. Signed-off-by: Ian Hockett Signed-off-by: sarhiri --- docs/developer_guide.md | 24 +++++++++++++++++++ magefiles/kind.go | 2 +- .../testcases/basic/preempt_by_id_1x1.yaml | 3 ++- .../testcases/basic/preempt_by_ids_1x5.yaml | 3 ++- .../testcases/basic/submit_fileshare_1x1.yaml | 2 +- 5 files changed, 30 insertions(+), 4 deletions(-) diff --git a/docs/developer_guide.md b/docs/developer_guide.md index cd47a26b175..e2547686b44 100644 --- a/docs/developer_guide.md +++ b/docs/developer_guide.md @@ -16,6 +16,7 @@ - [Running the UI](#running-the-ui) - [Debugging error: port 6443 is already in use after running `mage dev:full`](#debugging-error-port-6443-is-already-in-use-after-running-mage-devfull) - [Identifying the conflict](#identifying-the-conflict) + - [Debugging error: docker buildx is not set to default context after running `mage dev:full`](#debugging-error-docker-buildx-is-not-set-to-default-context-after-running-mage-devfull) - [Debugging](#debugging) - [GoLand run configurations](#goland-run-configurations) - [VS Code Run and Debug configurations](#vs-code-run-and-debug-configurations) @@ -220,6 +221,29 @@ Before making any changes, identify which port is causing the conflict. Port 644 You are not limited to using port 6444. You can choose any available port that doesn't conflict with other services on your system. Select a port that suits your system configuration. +## Debugging error: docker buildx is not set to default context after running `mage dev:full` + +If `mage dev:full` fails during the image build step with an error like: + +``` +⨯ release failed after 7m49s + error= + │ docker build failed: docker buildx is not set to default context - please switch with 'docker context use default' + │ Learn more at https://goreleaser.com/errors/docker-build +``` + +This is a goreleaser/buildx requirement, not a sign that Docker itself is broken. `mage dev:full` builds images via goreleaser, whose docker builder requires the context named `default` to be the active one. If you use a Docker runtime other than Docker Desktop (Rancher Desktop, Colima, OrbStack, Lima, Podman, etc.), that runtime typically registers its own context name instead of `default`, so this check fails even though Docker itself is running fine. + +Find your runtime's socket and point `default` at it via `DOCKER_HOST`, then switch to it: + +```bash +docker context ls # find your runtime's context and its DOCKER ENDPOINT socket path +export DOCKER_HOST="unix:///path/to/that/socket" +docker context use default +``` + +`default` is a reserved context that always reflects `DOCKER_HOST` (falling back to `/var/run/docker.sock` if unset), so this doesn't persist across shells — export `DOCKER_HOST` in whichever shell runs `mage dev:full`. + ## Debugging The goreman-based flow (`dev:up`) builds each component with debug flags (`-gcflags="all=-N -l"`) diff --git a/magefiles/kind.go b/magefiles/kind.go index 069bf6558c8..61104124070 100644 --- a/magefiles/kind.go +++ b/magefiles/kind.go @@ -25,7 +25,7 @@ func getImagesUsedInTestsOrControllers() []string { return []string{ "nginx:1.27.0", // Used by ingress-controller "alpine:3.20.0", - "bitnamilegacy/kubectl:1.30", + "bitnamilegacy/kubectl:1.33.4", } } diff --git a/testsuite/testcases/basic/preempt_by_id_1x1.yaml b/testsuite/testcases/basic/preempt_by_id_1x1.yaml index 2adbec604f6..dec7024a129 100644 --- a/testsuite/testcases/basic/preempt_by_id_1x1.yaml +++ b/testsuite/testcases/basic/preempt_by_id_1x1.yaml @@ -11,7 +11,8 @@ jobs: priorityClassName: armada-preemptible containers: - name: sleeper - image: alpine:latest + imagePullPolicy: IfNotPresent + image: alpine:3.20.0 command: - sh args: diff --git a/testsuite/testcases/basic/preempt_by_ids_1x5.yaml b/testsuite/testcases/basic/preempt_by_ids_1x5.yaml index a743320ea35..011f633561c 100644 --- a/testsuite/testcases/basic/preempt_by_ids_1x5.yaml +++ b/testsuite/testcases/basic/preempt_by_ids_1x5.yaml @@ -11,7 +11,8 @@ jobs: priorityClassName: armada-preemptible containers: - name: sleeper - image: alpine:latest + imagePullPolicy: IfNotPresent + image: alpine:3.20.0 command: - sh args: diff --git a/testsuite/testcases/basic/submit_fileshare_1x1.yaml b/testsuite/testcases/basic/submit_fileshare_1x1.yaml index 5cca3e0ddc5..1462883841d 100644 --- a/testsuite/testcases/basic/submit_fileshare_1x1.yaml +++ b/testsuite/testcases/basic/submit_fileshare_1x1.yaml @@ -32,7 +32,7 @@ jobs: containers: - name: reader imagePullPolicy: IfNotPresent - image: alpine:3.16 + image: alpine:3.20.0 command: - sh - -c From f17dc2f6124e90ca3c1eb96eb275d5faabd7bd55 Mon Sep 17 00:00:00 2001 From: William Vega Date: Fri, 17 Jul 2026 10:10:51 -0500 Subject: [PATCH 36/49] limit reason length on submit api (#5005) - CancelJobs, PreemptJobs, and CancelJobSet now validate that the reason field does not exceed MaxReasonBytes (50) - Oversized reasons are rejected early with a gRPC InvalidArgument error before reaching the scheduler Signed-off-by: William Vega Signed-off-by: sarhiri --- internal/server/submit/submit.go | 15 +++++ internal/server/submit/submit_test.go | 58 +++++++++++++++++++ internal/server/submit/validation/job_set.go | 19 ++++++ .../server/submit/validation/job_set_test.go | 31 ++++++++++ 4 files changed, 123 insertions(+) diff --git a/internal/server/submit/submit.go b/internal/server/submit/submit.go index 80dc618da9b..95e5231e06a 100644 --- a/internal/server/submit/submit.go +++ b/internal/server/submit/submit.go @@ -154,6 +154,11 @@ func (s *Server) SubmitJobs(grpcCtx context.Context, req *api.JobSubmitRequest) func (s *Server) CancelJobs(grpcCtx context.Context, req *api.JobCancelRequest) (*api.CancellationResult, error) { ctx := armadacontext.FromGrpcCtx(grpcCtx) + + if err := validation.ValidateReason(req); err != nil { + return nil, err + } + jobIds := []string{} jobIds = append(jobIds, req.JobIds...) if req.JobId != "" { @@ -201,6 +206,11 @@ func (s *Server) CancelJobs(grpcCtx context.Context, req *api.JobCancelRequest) func (s *Server) PreemptJobs(grpcCtx context.Context, req *api.JobPreemptRequest) (*api.PreemptionResult, error) { ctx := armadacontext.FromGrpcCtx(grpcCtx) + + if err := validation.ValidateReason(req); err != nil { + return nil, err + } + err := validation.ValidateQueueAndJobSet(req) if err != nil { return nil, err @@ -324,6 +334,11 @@ func (s *Server) ReprioritizeJobs(grpcCtx context.Context, req *api.JobRepriorit func (s *Server) CancelJobSet(grpcCtx context.Context, req *api.JobSetCancelRequest) (*types.Empty, error) { ctx := armadacontext.FromGrpcCtx(grpcCtx) + + if err := validation.ValidateReason(req); err != nil { + return nil, err + } + err := validation.ValidateQueueAndJobSet(req) if err != nil { return nil, err diff --git a/internal/server/submit/submit_test.go b/internal/server/submit/submit_test.go index 17d3df49524..d5fa7e39ac1 100644 --- a/internal/server/submit/submit_test.go +++ b/internal/server/submit/submit_test.go @@ -1,23 +1,27 @@ package submit import ( + "strings" "testing" "time" "github.com/stretchr/testify/assert" "go.uber.org/mock/gomock" + "google.golang.org/grpc/codes" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" clock "k8s.io/utils/clock/testing" "k8s.io/utils/pointer" "github.com/armadaproject/armada/internal/common/armadacontext" + "github.com/armadaproject/armada/internal/common/armadaerrors" "github.com/armadaproject/armada/internal/common/auth/permission" commonMocks "github.com/armadaproject/armada/internal/common/mocks" "github.com/armadaproject/armada/internal/common/util" "github.com/armadaproject/armada/internal/server/mocks" "github.com/armadaproject/armada/internal/server/permissions" "github.com/armadaproject/armada/internal/server/submit/testfixtures" + "github.com/armadaproject/armada/internal/server/submit/validation" "github.com/armadaproject/armada/pkg/api" "github.com/armadaproject/armada/pkg/armadaevents" "github.com/armadaproject/armada/pkg/client/queue" @@ -381,6 +385,60 @@ func TestPreemptJobs_FailedValidation(t *testing.T) { } } +// tooLongReason is one byte over the server-enforced limit. +var tooLongReason = strings.Repeat("a", validation.MaxReasonBytes+1) + +// A reason longer than MaxReasonBytes must be rejected at the API and map to a +// gRPC InvalidArgument code end-to-end (the code the error interceptor returns to clients). +func TestCancelJobs_ReasonTooLong(t *testing.T) { + jobId1 := util.ULID().String() + ctx, cancel := armadacontext.WithTimeout(armadacontext.Background(), 5*time.Second) + defer cancel() + server, _ := createTestServer(t) + + resp, err := server.CancelJobs(ctx, &api.JobCancelRequest{ + JobId: jobId1, + Queue: testfixtures.DefaultQueue.Name, + JobSetId: testfixtures.DefaultJobset, + Reason: tooLongReason, + }) + assert.Error(t, err) + assert.Nil(t, resp) + assert.Equal(t, codes.InvalidArgument, armadaerrors.CodeFromError(err)) +} + +func TestPreemptJobs_ReasonTooLong(t *testing.T) { + jobId1 := util.ULID().String() + ctx, cancel := armadacontext.WithTimeout(armadacontext.Background(), 5*time.Second) + defer cancel() + server, _ := createTestServer(t) + + resp, err := server.PreemptJobs(ctx, &api.JobPreemptRequest{ + JobIds: []string{jobId1}, + Queue: testfixtures.DefaultQueue.Name, + JobSetId: testfixtures.DefaultJobset, + Reason: tooLongReason, + }) + assert.Error(t, err) + assert.Nil(t, resp) + assert.Equal(t, codes.InvalidArgument, armadaerrors.CodeFromError(err)) +} + +func TestCancelJobSet_ReasonTooLong(t *testing.T) { + ctx, cancel := armadacontext.WithTimeout(armadacontext.Background(), 5*time.Second) + defer cancel() + server, _ := createTestServer(t) + + resp, err := server.CancelJobSet(ctx, &api.JobSetCancelRequest{ + Queue: testfixtures.DefaultQueue.Name, + JobSetId: testfixtures.DefaultJobset, + Reason: tooLongReason, + }) + assert.Error(t, err) + assert.Nil(t, resp) + assert.Equal(t, codes.InvalidArgument, armadaerrors.CodeFromError(err)) +} + func TestReprioritizeJobs(t *testing.T) { jobId1 := util.ULID().String() jobId2 := util.ULID().String() diff --git a/internal/server/submit/validation/job_set.go b/internal/server/submit/validation/job_set.go index 9c78e976a83..1f8263b595c 100644 --- a/internal/server/submit/validation/job_set.go +++ b/internal/server/submit/validation/job_set.go @@ -34,6 +34,25 @@ func ValidateJobSetFilter(filter *api.JobSetFilter) error { return nil } +const MaxReasonBytes = 50 + +type ReasonRequest interface { + GetReason() string +} + +// ValidateReason rejects reasons longer than MaxReasonBytes. Length is measured in +// bytes, consistent with util.Truncate which slices reasons by byte. +func ValidateReason(req ReasonRequest) error { + if len(req.GetReason()) > MaxReasonBytes { + return &armadaerrors.ErrInvalidArgument{ + Name: "Reason", + Value: req.GetReason()[:MaxReasonBytes] + "...", + Message: fmt.Sprintf("reason cannot be longer than %d bytes", MaxReasonBytes), + } + } + return nil +} + type JobSetRequest interface { GetJobSetId() string GetQueue() string diff --git a/internal/server/submit/validation/job_set_test.go b/internal/server/submit/validation/job_set_test.go index 2ffcbec8d94..97d195679d4 100644 --- a/internal/server/submit/validation/job_set_test.go +++ b/internal/server/submit/validation/job_set_test.go @@ -1,6 +1,7 @@ package validation import ( + "strings" "testing" "github.com/stretchr/testify/assert" @@ -31,3 +32,33 @@ func TestValidateJobSetFilter_EnforcesPendingAndRunningOccurTogether(t *testing. result = ValidateJobSetFilter(&api.JobSetFilter{States: []api.JobState{api.JobState_PENDING, api.JobState_RUNNING}}) assert.NoError(t, result) } + +func TestValidateReason(t *testing.T) { + tests := map[string]struct { + reason string + expectError bool + }{ + "empty reason": { + reason: "", + expectError: false, + }, + "reason at max length": { + reason: strings.Repeat("a", MaxReasonBytes), + expectError: false, + }, + "reason over max length": { + reason: strings.Repeat("a", MaxReasonBytes+1), + expectError: true, + }, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + result := ValidateReason(&api.JobCancelRequest{Reason: tc.reason}) + if tc.expectError { + assert.Error(t, result) + } else { + assert.NoError(t, result) + } + }) + } +} From 4d2263b802da36e3b43d95d412822ac8696457df Mon Sep 17 00:00:00 2001 From: sarhiri Date: Fri, 17 Jul 2026 14:11:44 -0500 Subject: [PATCH 37/49] typos Signed-off-by: sarhiri --- website/content/docs/developer-guide.mdx | 2 +- website/content/index.mdx | 85 +----------------------- 2 files changed, 2 insertions(+), 85 deletions(-) diff --git a/website/content/docs/developer-guide.mdx b/website/content/docs/developer-guide.mdx index 376b83197ad..935fbf3cd3c 100644 --- a/website/content/docs/developer-guide.mdx +++ b/website/content/docs/developer-guide.mdx @@ -1,5 +1,5 @@ --- -title: 'Local Development Setup' +title: 'Local Development' description: 'Set up your development environment and start contributing to Armada' --- import { Callout } from 'fumadocs-ui/components/callout'; diff --git a/website/content/index.mdx b/website/content/index.mdx index 62d33121db5..d37bdb074ca 100644 --- a/website/content/index.mdx +++ b/website/content/index.mdx @@ -115,16 +115,6 @@ import { Boxes, Scale, Zap, MoveUpRight, Activity, LayoutGrid } from 'lucide-rea
-{/* ## What is Armada? - OLD */} - -{/* Armada is a multi-Kubernetes cluster batch job meta-scheduler designed to handle massive-scale workloads. Built on top of Kubernetes, Armada enables organizations to distribute millions of batch jobs per day across tens of thousands of nodes spanning multiple clusters, making it an ideal solution for high-throughput computational workloads. - -Armada serves as middleware that transforms Kubernetes into a powerful batch processing platform while maintaining compatibility with service workloads. It addresses the fundamental limitations of running batch workloads at scale on Kubernetes by providing: - -- **Multi-cluster orchestration**: Schedule jobs across many Kubernetes clusters seamlessly -- **High-throughput queueing**: Handle millions of queued jobs -- **Advanced batch scheduling**: Fair queuing, gang scheduling, preemption, and resource limits -- **Enterprise-grade reliability**: Secure, highly available components designed for production use */}
As a [CNCF Sandbox project](https://www.cncf.io/), Armada is actively maintained and used in production environments, including at [G-Research](https://www.gresearch.com/) where it processes millions of jobs daily. @@ -178,79 +168,6 @@ Armada serves as middleware that transforms Kubernetes into a powerful batch pro
-{/* ## Why use Armada? - OLD */} - -{/* ### Kubernetes Limitations for Batch Workloads - -Traditional Kubernetes faces several challenges when running batch workloads at scale: - -1. **Single Cluster Scaling Limits**: Scaling a single Kubernetes cluster beyond a certain size is [challenging](https://openai.com/blog/scaling-kubernetes-to-7500-nodes/), typically maxing out around 5,000-15,000 nodes depending on configuration. - -2. **Storage Backend Constraints**: Etcd, Kubernetes' in-cluster storage backend, has [performance limitations](https://etcd.io/docs/v3.5/op-guide/performance/) that make achieving very high throughput difficult and can become a bottleneck for job queuing. - -3. **Inadequate Batch Scheduling**: The default [kube-scheduler](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-scheduler/) lacks essential batch scheduling features like fair queuing, gang scheduling, and intelligent preemption. - -### Armada's Solution - -Armada overcomes these limitations by: - -- **Distributing across multiple clusters**: Manage thousands of nodes across many Kubernetes clusters -- **Partial Out-of-cluster scheduling**: Leverage external storage backends (e.g., PostgreSQL and Redis) for high-throughput batch job queueing and scheduling -- **Purpose-built batch scheduler**: Include advanced scheduling features designed specifically for batch workloads */} - -{/* ## Key Features and Benefits - -### Core Scheduling Features - -**Fair-Use Scheduling** - -- Maintains fair resource share over time across users and teams -- Based on dominant resource fairness principles -- Includes priority factors for different queues -- Inspired by HTCondor priority systems - -**High Throughput Processing** - -- Handle millions of queued jobs simultaneously -- Efficient job submission and status tracking - -**Gang Scheduling** - -- Atomically schedule sets of related jobs -- Ensures all jobs in a group start together or not at all -- Critical for distributed computing frameworks like MPI - -**Intelligent Preemption** - -- Run urgent jobs in a timely fashion -- Balance resource allocation between users -- Configurable preemption policies - -### Enterprise-Grade Operations - -**Massive Scale Support** - -- Utilize multiple Kubernetes clusters simultaneously -- Scale beyond single cluster limitations -- Add and remove clusters without service disruption - -**Advanced Resource Management** - -- Resource and job scheduling rate limits -- Detailed resource allocation controls - -**Comprehensive Monitoring** - -- Detailed analytics via [Prometheus](https://prometheus.io/) integration -- Resource allocation and system behavior insights -- Automatic failure detection and node removal - -**Production-Ready Features** - -- Secure authentication and authorization -- High availability architecture -- Automatic node failure handling */} - {/* ## Use cases */}
@@ -277,7 +194,7 @@ Armada overcomes these limitations by:
Machine learning training at scale - Your workers need to start together or not at all. Armada's gang scheduling makes sure they do — across however many clusters have the GPU capacity you need. + Your workers need to start together or not at all. Armada's gang scheduling makes sure they do, across however many clusters have the GPU capacity you need.
From 37caf898b400748e11299ac694da095d33f0284f Mon Sep 17 00:00:00 2001 From: sarhiri Date: Fri, 17 Jul 2026 15:01:29 -0500 Subject: [PATCH 38/49] fix: resolve broken links flagged by content check. Signed-off-by: sarhiri --- website/content/contribute/community.mdx | 2 +- website/content/contribute/contributor-guide.mdx | 2 +- website/content/operator-guide.mdx | 6 +++--- website/content/understanding-armada/index.mdx | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/website/content/contribute/community.mdx b/website/content/contribute/community.mdx index 91bb3ac3795..086f54336e4 100644 --- a/website/content/contribute/community.mdx +++ b/website/content/contribute/community.mdx @@ -28,7 +28,7 @@ Found a bug or have a feature request? Open an issue on our [GitHub repository]( Have a fix or enhancement ready? We'd love to see your contribution! Submit a pull request on our [GitHub repository](https://github.com/armadaproject/armada/pulls). Whether it's code, documentation, or improvements, every contribution helps make Armada better for everyone. -Interested in contributing but not sure where to start? Check out our [Contributor Guide](./contribute/contributor-guide.mdx) and browse open issues on GitHub. There's always something you can help with! +Interested in contributing but not sure where to start? Check out our [Contributor Guide](./contributor-guide.mdx) and browse open issues on GitHub. There's always something you can help with! ## Community Meetings diff --git a/website/content/contribute/contributor-guide.mdx b/website/content/contribute/contributor-guide.mdx index 8b05fe4dec3..d60d39893f1 100644 --- a/website/content/contribute/contributor-guide.mdx +++ b/website/content/contribute/contributor-guide.mdx @@ -7,7 +7,7 @@ description: 'Guidelines and tips for contributing to the Armada project.' To setup your development environment, follow the instructions based on the area you want to contribute to: -- **For Armada core development**: See the [Developer Guide](../developer-guide.mdx) for local setup, code structure, and development workflows +- **For Armada core development**: See the [Developer Guide](../docs/developer-guide.mdx) for local setup, code structure, and development workflows - **For Armada Operator development**: See the [Armada Operator repository](https://github.com/armadaproject/armada-operator) for setup instructions and development guidelines ## Reporting Issues diff --git a/website/content/operator-guide.mdx b/website/content/operator-guide.mdx index ff005317886..4e4a6495a71 100644 --- a/website/content/operator-guide.mdx +++ b/website/content/operator-guide.mdx @@ -26,7 +26,7 @@ Armada consists of several components that work together: - **Lookout**: Provides job monitoring and web UI - **Supporting services**: Pulsar (message broker), PostgreSQL, and Redis -For a detailed explanation of how these components interact, see the [Architecture documentation](./understanding-armada/architecture.mdx). +For a detailed explanation of how these components interact, see the [Architecture documentation](docs/architecture.mdx). ## Local Installation @@ -449,11 +449,11 @@ If you encounter issues not covered here: - **GitHub Issues**: Report bugs and request features at [github.com/armadaproject/armada/issues](https://github.com/armadaproject/armada/issues) - **Community Slack**: Join discussions on [CNCF Slack](https://cloud-native.slack.com/?redir=%2Farchives%2FC03T9CBCEMC) -- **Documentation**: Check the [Architecture documentation](./understanding-armada/architecture.mdx) for system design details +- **Documentation**: Check the [Architecture documentation](docs/architecture.mdx) for system design details ## Additional Resources -- [Architecture Overview](./understanding-armada/architecture.mdx) - Understand how Armada components work +- [Architecture Overview](docs/architecture.mdx) - Understand how Armada components work - [User Guide](./user-guide) - Learn how to submit and manage jobs - [Armada Operator](https://github.com/armadaproject/armada-operator) - Kubernetes-native deployment option - [Helm Charts Documentation](https://github.com/armadaproject/armada/tree/master/deployment) - Detailed Helm configuration reference diff --git a/website/content/understanding-armada/index.mdx b/website/content/understanding-armada/index.mdx index fcba84fced1..d953d6c3c68 100644 --- a/website/content/understanding-armada/index.mdx +++ b/website/content/understanding-armada/index.mdx @@ -10,7 +10,7 @@ This section provides a comprehensive overview of how Armada works, from its cor Learn about Armada's system architecture, including its components, event-sourcing design, and how jobs flow through the system from submission to completion. -[Learn more about Architecture →](./architecture.mdx) +[Learn more about Architecture →](../docs/architecture.mdx) ## Core Concepts @@ -18,4 +18,4 @@ Learn about Armada's system architecture, including its components, event-sourci Explore fundamental concepts like jobs, queues, job sets, priorities, and fair-use scheduling algorithms that make Armada a powerful batch scheduler. -[Learn more about Core Concepts →](./core-concepts.mdx) +[Learn more about Core Concepts →](../docs/core-concepts.mdx) From db2123ef1425677240b5403c60f2c977e9ae3808 Mon Sep 17 00:00:00 2001 From: sarhiri Date: Fri, 17 Jul 2026 15:08:32 -0500 Subject: [PATCH 39/49] fix: update remaining stale links not passing checks Signed-off-by: sarhiri --- website/content/docs/user-guide.mdx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/website/content/docs/user-guide.mdx b/website/content/docs/user-guide.mdx index 351ae936516..3326a4cb352 100644 --- a/website/content/docs/user-guide.mdx +++ b/website/content/docs/user-guide.mdx @@ -30,22 +30,22 @@ import { Cards, Card } from 'fumadocs-ui/components/card'; From 930cab6e86ca947a94a569f2c75c670755fd2f25 Mon Sep 17 00:00:00 2001 From: sarhiri Date: Fri, 17 Jul 2026 15:14:05 -0500 Subject: [PATCH 40/49] typo Signed-off-by: sarhiri --- website/content/docs/user-guide.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/content/docs/user-guide.mdx b/website/content/docs/user-guide.mdx index 3326a4cb352..40fcdd6fe30 100644 --- a/website/content/docs/user-guide.mdx +++ b/website/content/docs/user-guide.mdx @@ -45,7 +45,7 @@ import { Cards, Card } from 'fumadocs-ui/components/card'; description='Use client libraries for Go, Java, Scala, Python, and .NET in your applications' /> From 9dd5ec2dec943f5a7e2b651a0f4d3b8d3e5b066f Mon Sep 17 00:00:00 2001 From: sarhiri Date: Mon, 27 Jul 2026 14:29:04 -0500 Subject: [PATCH 41/49] docs: archive deprecated pages, fix internal links for new nav structure Signed-off-by: sarhiri --- website/.remarkignore | 3 + .../{content => _archive}/architecture.mdx | 0 website/{content => _archive}/armada-api.mdx | 0 .../armada_airflow_operator.mdx | 0 .../armada_client_package.mdx | 0 website/{content => _archive}/armada_demo.mdx | 0 .../armada_helm_charts.mdx | 0 .../client_libraries.mdx | 0 website/{content => _archive}/community.mdx | 0 .../creating_and_submitting_jobs.mdx | 0 .../design/architecture.mdx | 0 .../design/database_interfaces.mdx | 0 .../{content => _archive}/design/index.mdx | 0 .../design/jobservice/job-service.mdx | 0 .../{content => _archive}/design/priority.mdx | 0 .../design/relationships_diagram.mdx | 0 .../design/scheduler.mdx | 0 .../{content => _archive}/developer/api.mdx | 0 .../developer/aws-ec2.mdx | 0 .../developer/manual-localdev.mdx | 0 .../{content => _archive}/developer/oidc.mdx | 0 .../{content => _archive}/developer/pprof.mdx | 0 .../developer/ubuntu-setup.mdx | 0 .../{content => _archive}/developer/ui.mdx | 0 .../developer/usage_metrics.mdx | 0 .../developer/website.mdx | 0 .../{content => _archive}/developer_guide.mdx | 0 .../development_guide.mdx | 0 .../floating_resources.mdx | 0 .../kubernetes_native_auth.mdx | 0 .../maintaining_consistency_across_views.mdx | 0 .../{content => _archive}/operator-guide.mdx | 0 .../priority_algorithm.mdx | 0 .../production-install.mdx | 0 website/{content => _archive}/quickstart.mdx | 0 .../scheduling_and_preempting_jobs.mdx | 0 .../{content => _archive}/system_overview.mdx | 0 .../understanding-armada/index.mdx | 0 .../understanding-armada/meta.json | 0 .../user-guide/index.mdx | 0 .../user-guide/meta.json | 0 .../content/contribute/contributor-guide.mdx | 2 +- website/content/contribute/index.mdx | 2 +- website/content/docs/developer-guide.mdx | 358 +----------------- website/content/getting-started.mdx | 2 +- website/content/index.mdx | 4 +- 46 files changed, 10 insertions(+), 361 deletions(-) rename website/{content => _archive}/architecture.mdx (100%) rename website/{content => _archive}/armada-api.mdx (100%) rename website/{content => _archive}/armada_airflow_operator.mdx (100%) rename website/{content => _archive}/armada_client_package.mdx (100%) rename website/{content => _archive}/armada_demo.mdx (100%) rename website/{content => _archive}/armada_helm_charts.mdx (100%) rename website/{content => _archive}/client_libraries.mdx (100%) rename website/{content => _archive}/community.mdx (100%) rename website/{content => _archive}/creating_and_submitting_jobs.mdx (100%) rename website/{content => _archive}/design/architecture.mdx (100%) rename website/{content => _archive}/design/database_interfaces.mdx (100%) rename website/{content => _archive}/design/index.mdx (100%) rename website/{content => _archive}/design/jobservice/job-service.mdx (100%) rename website/{content => _archive}/design/priority.mdx (100%) rename website/{content => _archive}/design/relationships_diagram.mdx (100%) rename website/{content => _archive}/design/scheduler.mdx (100%) rename website/{content => _archive}/developer/api.mdx (100%) rename website/{content => _archive}/developer/aws-ec2.mdx (100%) rename website/{content => _archive}/developer/manual-localdev.mdx (100%) rename website/{content => _archive}/developer/oidc.mdx (100%) rename website/{content => _archive}/developer/pprof.mdx (100%) rename website/{content => _archive}/developer/ubuntu-setup.mdx (100%) rename website/{content => _archive}/developer/ui.mdx (100%) rename website/{content => _archive}/developer/usage_metrics.mdx (100%) rename website/{content => _archive}/developer/website.mdx (100%) rename website/{content => _archive}/developer_guide.mdx (100%) rename website/{content => _archive}/development_guide.mdx (100%) rename website/{content => _archive}/floating_resources.mdx (100%) rename website/{content => _archive}/kubernetes_native_auth.mdx (100%) rename website/{content => _archive}/maintaining_consistency_across_views.mdx (100%) rename website/{content => _archive}/operator-guide.mdx (100%) rename website/{content => _archive}/priority_algorithm.mdx (100%) rename website/{content => _archive}/production-install.mdx (100%) rename website/{content => _archive}/quickstart.mdx (100%) rename website/{content => _archive}/scheduling_and_preempting_jobs.mdx (100%) rename website/{content => _archive}/system_overview.mdx (100%) rename website/{content => _archive}/understanding-armada/index.mdx (100%) rename website/{content => _archive}/understanding-armada/meta.json (100%) rename website/{content => _archive}/user-guide/index.mdx (100%) rename website/{content => _archive}/user-guide/meta.json (100%) diff --git a/website/.remarkignore b/website/.remarkignore index 0c28ce9e136..4ff1e0346fa 100644 --- a/website/.remarkignore +++ b/website/.remarkignore @@ -9,3 +9,6 @@ tmp.* # Auto-generated files python-airflow-operator.md python-armada-client.md + +# Archived files +_archive/ \ No newline at end of file diff --git a/website/content/architecture.mdx b/website/_archive/architecture.mdx similarity index 100% rename from website/content/architecture.mdx rename to website/_archive/architecture.mdx diff --git a/website/content/armada-api.mdx b/website/_archive/armada-api.mdx similarity index 100% rename from website/content/armada-api.mdx rename to website/_archive/armada-api.mdx diff --git a/website/content/armada_airflow_operator.mdx b/website/_archive/armada_airflow_operator.mdx similarity index 100% rename from website/content/armada_airflow_operator.mdx rename to website/_archive/armada_airflow_operator.mdx diff --git a/website/content/armada_client_package.mdx b/website/_archive/armada_client_package.mdx similarity index 100% rename from website/content/armada_client_package.mdx rename to website/_archive/armada_client_package.mdx diff --git a/website/content/armada_demo.mdx b/website/_archive/armada_demo.mdx similarity index 100% rename from website/content/armada_demo.mdx rename to website/_archive/armada_demo.mdx diff --git a/website/content/armada_helm_charts.mdx b/website/_archive/armada_helm_charts.mdx similarity index 100% rename from website/content/armada_helm_charts.mdx rename to website/_archive/armada_helm_charts.mdx diff --git a/website/content/client_libraries.mdx b/website/_archive/client_libraries.mdx similarity index 100% rename from website/content/client_libraries.mdx rename to website/_archive/client_libraries.mdx diff --git a/website/content/community.mdx b/website/_archive/community.mdx similarity index 100% rename from website/content/community.mdx rename to website/_archive/community.mdx diff --git a/website/content/creating_and_submitting_jobs.mdx b/website/_archive/creating_and_submitting_jobs.mdx similarity index 100% rename from website/content/creating_and_submitting_jobs.mdx rename to website/_archive/creating_and_submitting_jobs.mdx diff --git a/website/content/design/architecture.mdx b/website/_archive/design/architecture.mdx similarity index 100% rename from website/content/design/architecture.mdx rename to website/_archive/design/architecture.mdx diff --git a/website/content/design/database_interfaces.mdx b/website/_archive/design/database_interfaces.mdx similarity index 100% rename from website/content/design/database_interfaces.mdx rename to website/_archive/design/database_interfaces.mdx diff --git a/website/content/design/index.mdx b/website/_archive/design/index.mdx similarity index 100% rename from website/content/design/index.mdx rename to website/_archive/design/index.mdx diff --git a/website/content/design/jobservice/job-service.mdx b/website/_archive/design/jobservice/job-service.mdx similarity index 100% rename from website/content/design/jobservice/job-service.mdx rename to website/_archive/design/jobservice/job-service.mdx diff --git a/website/content/design/priority.mdx b/website/_archive/design/priority.mdx similarity index 100% rename from website/content/design/priority.mdx rename to website/_archive/design/priority.mdx diff --git a/website/content/design/relationships_diagram.mdx b/website/_archive/design/relationships_diagram.mdx similarity index 100% rename from website/content/design/relationships_diagram.mdx rename to website/_archive/design/relationships_diagram.mdx diff --git a/website/content/design/scheduler.mdx b/website/_archive/design/scheduler.mdx similarity index 100% rename from website/content/design/scheduler.mdx rename to website/_archive/design/scheduler.mdx diff --git a/website/content/developer/api.mdx b/website/_archive/developer/api.mdx similarity index 100% rename from website/content/developer/api.mdx rename to website/_archive/developer/api.mdx diff --git a/website/content/developer/aws-ec2.mdx b/website/_archive/developer/aws-ec2.mdx similarity index 100% rename from website/content/developer/aws-ec2.mdx rename to website/_archive/developer/aws-ec2.mdx diff --git a/website/content/developer/manual-localdev.mdx b/website/_archive/developer/manual-localdev.mdx similarity index 100% rename from website/content/developer/manual-localdev.mdx rename to website/_archive/developer/manual-localdev.mdx diff --git a/website/content/developer/oidc.mdx b/website/_archive/developer/oidc.mdx similarity index 100% rename from website/content/developer/oidc.mdx rename to website/_archive/developer/oidc.mdx diff --git a/website/content/developer/pprof.mdx b/website/_archive/developer/pprof.mdx similarity index 100% rename from website/content/developer/pprof.mdx rename to website/_archive/developer/pprof.mdx diff --git a/website/content/developer/ubuntu-setup.mdx b/website/_archive/developer/ubuntu-setup.mdx similarity index 100% rename from website/content/developer/ubuntu-setup.mdx rename to website/_archive/developer/ubuntu-setup.mdx diff --git a/website/content/developer/ui.mdx b/website/_archive/developer/ui.mdx similarity index 100% rename from website/content/developer/ui.mdx rename to website/_archive/developer/ui.mdx diff --git a/website/content/developer/usage_metrics.mdx b/website/_archive/developer/usage_metrics.mdx similarity index 100% rename from website/content/developer/usage_metrics.mdx rename to website/_archive/developer/usage_metrics.mdx diff --git a/website/content/developer/website.mdx b/website/_archive/developer/website.mdx similarity index 100% rename from website/content/developer/website.mdx rename to website/_archive/developer/website.mdx diff --git a/website/content/developer_guide.mdx b/website/_archive/developer_guide.mdx similarity index 100% rename from website/content/developer_guide.mdx rename to website/_archive/developer_guide.mdx diff --git a/website/content/development_guide.mdx b/website/_archive/development_guide.mdx similarity index 100% rename from website/content/development_guide.mdx rename to website/_archive/development_guide.mdx diff --git a/website/content/floating_resources.mdx b/website/_archive/floating_resources.mdx similarity index 100% rename from website/content/floating_resources.mdx rename to website/_archive/floating_resources.mdx diff --git a/website/content/kubernetes_native_auth.mdx b/website/_archive/kubernetes_native_auth.mdx similarity index 100% rename from website/content/kubernetes_native_auth.mdx rename to website/_archive/kubernetes_native_auth.mdx diff --git a/website/content/maintaining_consistency_across_views.mdx b/website/_archive/maintaining_consistency_across_views.mdx similarity index 100% rename from website/content/maintaining_consistency_across_views.mdx rename to website/_archive/maintaining_consistency_across_views.mdx diff --git a/website/content/operator-guide.mdx b/website/_archive/operator-guide.mdx similarity index 100% rename from website/content/operator-guide.mdx rename to website/_archive/operator-guide.mdx diff --git a/website/content/priority_algorithm.mdx b/website/_archive/priority_algorithm.mdx similarity index 100% rename from website/content/priority_algorithm.mdx rename to website/_archive/priority_algorithm.mdx diff --git a/website/content/production-install.mdx b/website/_archive/production-install.mdx similarity index 100% rename from website/content/production-install.mdx rename to website/_archive/production-install.mdx diff --git a/website/content/quickstart.mdx b/website/_archive/quickstart.mdx similarity index 100% rename from website/content/quickstart.mdx rename to website/_archive/quickstart.mdx diff --git a/website/content/scheduling_and_preempting_jobs.mdx b/website/_archive/scheduling_and_preempting_jobs.mdx similarity index 100% rename from website/content/scheduling_and_preempting_jobs.mdx rename to website/_archive/scheduling_and_preempting_jobs.mdx diff --git a/website/content/system_overview.mdx b/website/_archive/system_overview.mdx similarity index 100% rename from website/content/system_overview.mdx rename to website/_archive/system_overview.mdx diff --git a/website/content/understanding-armada/index.mdx b/website/_archive/understanding-armada/index.mdx similarity index 100% rename from website/content/understanding-armada/index.mdx rename to website/_archive/understanding-armada/index.mdx diff --git a/website/content/understanding-armada/meta.json b/website/_archive/understanding-armada/meta.json similarity index 100% rename from website/content/understanding-armada/meta.json rename to website/_archive/understanding-armada/meta.json diff --git a/website/content/user-guide/index.mdx b/website/_archive/user-guide/index.mdx similarity index 100% rename from website/content/user-guide/index.mdx rename to website/_archive/user-guide/index.mdx diff --git a/website/content/user-guide/meta.json b/website/_archive/user-guide/meta.json similarity index 100% rename from website/content/user-guide/meta.json rename to website/_archive/user-guide/meta.json diff --git a/website/content/contribute/contributor-guide.mdx b/website/content/contribute/contributor-guide.mdx index d60d39893f1..e49933c68cf 100644 --- a/website/content/contribute/contributor-guide.mdx +++ b/website/content/contribute/contributor-guide.mdx @@ -64,7 +64,7 @@ For more details, see [DCO](https://github.com/apps/dco). ## Communication -For real-time discussions, Slack channels, GitHub Discussions, and community meetings, see our [Community page](../community.mdx). +For real-time discussions, Slack channels, GitHub Discussions, and community meetings, see our [Community page](./community.mdx). ## Security diff --git a/website/content/contribute/index.mdx b/website/content/contribute/index.mdx index f5de302d0db..beb3deaa95a 100644 --- a/website/content/contribute/index.mdx +++ b/website/content/contribute/index.mdx @@ -20,7 +20,7 @@ import { Cards, Card } from 'fumadocs-ui/components/card'; description='Our community standards and guidelines for participation' /> diff --git a/website/content/docs/developer-guide.mdx b/website/content/docs/developer-guide.mdx index 935fbf3cd3c..1a3860294fd 100644 --- a/website/content/docs/developer-guide.mdx +++ b/website/content/docs/developer-guide.mdx @@ -204,7 +204,7 @@ and integration testing of job flows. {/* divider */}
-## Code structure +{/* ## Code structure Armada
├── cmd/ # Entry points for all components
│ ├── server/ # Armada server (API server)
@@ -223,7 +223,7 @@ Armada
│ └── client/ # Client libraries
├── config/ # Configuration files for components
├── magefiles/ # Build automation (mage targets)
-└── testsuite/ # Integration test cases
+└── testsuite/ # Integration test cases
*/} {/* divider */}
@@ -348,357 +348,3 @@ Ask in the [#armada channel on CNCF Slack](https://cloud-native.slack.com/archiv
-{/* -This guide helps you set up a development environment for contributing to Armada or customizing it with new features. For contribution guidelines, see the [Contributor Guide](./contribute/contributor-guide.mdx). - -## Prerequisites - -Install the following tools before you begin. These are verified requirements from the Armada source code: - -- **[Go](https://go.dev/doc/install)** (version 1.26 or later) - Required for building Armada -- **gcc** (for Windows, see [tdm-gcc](https://jmeubank.github.io/tdm-gcc/)) - Required for CGO compilation -- **[mage](https://magefile.org/)** - Build tool used throughout the Armada project (similar to Make, written in Go) -- **[Docker](https://docs.docker.com/get-docker/)** - Container runtime for running dependencies -- **[kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl)** - Kubernetes command-line tool -- **[protobuf](https://github.com/protocolbuffers/protobuf/releases)** (version 3.17.3 or later) - Protocol buffer compiler (required if you modify `.proto` files) -- **[kind](https://kind.sigs.k8s.io/)** - Kubernetes in Docker (bootstrapped via `mage BootstrapTools`) - -**Note:** Additional tools are automatically installed via `mage BootstrapTools` from [`tools.yaml`](https://github.com/armadaproject/armada/blob/master/tools.yaml), including golangci-lint, sqlc, go-swagger, and others. - -## Development Environment Setup - -Armada provides two main ways to run components locally for development. Choose the method that best fits your workflow. - -### Using `mage dev` (Recommended) - -The `mage dev` targets automate the setup process and are the recommended way to get started: - -- Bootstraps required tools from [`tools.yaml`](https://github.com/armadaproject/armada/blob/master/tools.yaml) -- Starts dependencies (Pulsar, Redis, PostgreSQL) in containers -- Builds and starts Armada components (`mage dev:up` runs them via goreman; `mage dev:full` runs them in containers against a [kind](https://kind.sigs.k8s.io/) cluster) - -**Note:** If you edit a proto file, run `mage proto` to regenerate the Go code. - -The `mage dev` targets: - -```bash -# Run dependencies in containers and Armada components via goreman (fast iteration) -mage dev:up - -# Run the entire stack in containers against a Kind cluster (what CI uses) -mage dev:full -``` - -We use `mage dev:full` to test the CI pipeline. Use it to test changes to core components. - -To stop the local development environment: - -```bash -mage dev:down # stop the dependency containers (after `mage dev:up`) -mage dev:fullDown # stop the containerized stack and tear down Kind (after `mage dev:full`) -``` - -### Using Goreman - -[Goreman](https://github.com/mattn/goreman) is a Go-based clone of [Foreman](https://github.com/ddollar/foreman) that manages Procfile-based applications, allowing you to run multiple processes with a single command. Goreman will build the components from source and run them locally, making it easy to test changes quickly. - -1. Install `goreman`: - - ```bash - go install github.com/mattn/goreman@latest - ``` - -2. Start dependencies: - - ```bash - docker compose -f _local/compose/stack.yaml up -d - ``` - - **Note:** Images can be overridden using environment variables: `REDIS_IMAGE`, `POSTGRES_IMAGE`, `PULSAR_IMAGE`, `KEYCLOAK_IMAGE`, `OTEL_IMAGE`, `JAEGER_IMAGE`, `GRAFANA_IMAGE` - -3. Initialize databases and Kubernetes resources: - - ```bash - _local/scripts/init.sh - ``` - -4. Start Armada components: - ```bash - goreman -f _local/procfiles/no-auth.Procfile start - ``` - -#### Local Development with Authentication - -To run Armada with OIDC authentication enabled using Keycloak: - -1. Start dependencies with the auth profile: - - ```bash - docker compose -f _local/compose/stack.yaml --profile auth up -d - ``` - - This starts Redis, PostgreSQL, Pulsar, and Keycloak with a pre-configured realm. - -2. Initialize databases and Kubernetes resources: - - ```bash - _local/scripts/init.sh - ``` - -3. Start Armada components with auth configuration: - - ```bash - goreman -f _local/procfiles/auth.Procfile start - ``` - -4. Use armadactl with OIDC authentication: - ```bash - armadactl --config _local/.armadactl.yaml --context auth-oidc get queues - ``` - -#### Local Development with Fake Executor - -For testing Armada without a real Kubernetes cluster, you can use the fake executor that simulates a Kubernetes environment: - -```bash -goreman -f _local/procfiles/fake-executor.Procfile start -``` - -The fake executor simulates: - -- 2 virtual nodes with 8 CPUs and 32Gi memory each -- Pod lifecycle management without actual container execution -- Resource allocation and job state transitions - -This is useful for: - -- Testing Armada's scheduling logic -- Development when Kubernetes is not available -- Integration testing of job flows - -### Configuration Options - -You can set the `ARMADA_COMPONENTS` environment variable to choose which components to run: - -```bash -export ARMADA_COMPONENTS="server,executor" -``` - -### Testing Your Setup - -Verify that your development environment is working: - -```bash -# Run the test suite -mage testsuite -``` - -Or manually: - -```bash -go run cmd/armadactl/main.go create queue e2e-test-queue -export ARMADA_EXECUTOR_INGRESS_URL="http://localhost" -export ARMADA_EXECUTOR_INGRESS_PORT=5001 -go run cmd/testsuite/main.go test --tests "testsuite/testcases/basic/*" --junit junit.xml -``` - -## Code Structure - -Understanding Armada's codebase structure will help you navigate and contribute effectively. - -### Directory Layout - -``` -armada/ -├── cmd/ # Main entry points for all components -│ ├── server/ # Armada server (API server) -│ ├── executor/ # Executor (runs in each K8s cluster) -│ ├── scheduler/ # Scheduler (job scheduling logic) -│ ├── lookout/ # Lookout (job monitoring/UI backend) -│ └── armadactl/ # Command-line interface -├── internal/ # Internal packages (not for external use) -│ ├── server/ # Server implementation -│ ├── executor/ # Executor implementation -│ ├── scheduler/ # Scheduler implementation -│ ├── lookout/ # Lookout implementation -│ └── common/ # Shared utilities -├── pkg/ # Public packages (for external use) -│ ├── api/ # gRPC API definitions -│ └── client/ # Client libraries -├── config/ # Configuration files for components -├── deployment/ # Helm charts and deployment configs -├── magefiles/ # Build automation (mage targets) -└── testsuite/ # Integration test cases -``` - -### Key Components - -- **Server** (`cmd/server/`, `internal/server/`): The main API server that accepts job submissions and manages queues -- **Executor** (`cmd/executor/`, `internal/executor/`): Runs in each Kubernetes cluster and executes jobs -- **Scheduler** (`cmd/scheduler/`, `internal/scheduler/`): Determines when and where jobs should run -- **Lookout** (`cmd/lookout/`, `internal/lookout/`): Provides job monitoring and UI backend -- **armadactl** (`cmd/armadactl/`): Command-line interface for interacting with Armada - -### Using `mage` - -`mage` is the build tool used throughout the Armada project. To see all available commands: - -```bash -mage -l -``` - -Common mage targets: - -- `mage dev:up` - Run dependencies in containers and Armada components via goreman -- `mage dev:full` - Run the entire stack in containers against a Kind cluster (what CI uses) -- `mage buildDockers` - Build Docker images -- `mage proto` - Generate Go code from proto files -- `mage testsuite` - Run the test suite -- `mage ui` - Build and run the Lookout UI - -## Debugging and Profiling - -### Profiling with pprof - -Go provides a profiling tool called [pprof](https://pkg.go.dev/net/http/pprof). To use pprof with Armada, enable the profiling socket in your config. - -```yaml -profiling: - port: 6060 - hostnames: - - 'armada-scheduler-profiling.armada.my-k8s-cluster.com' - clusterIssuer: 'k8s-cluster-issuer' - auth: - anonymousAuth: true - permissionGroupMapping: - pprof: ['everyone'] -``` - -### Debugging components - -`mage dev:up` builds each component with debug flags (`-gcflags="all=-N -l"`) and runs them as host -processes, so you can attach a debugger (Delve, VS Code, or GoLand) to any running process directly. -Each component reads `_local//config.yaml`. - -For VS Code, use the launch configurations in `.vscode/launch.json`. See the -[VS Code Debugging Guide](https://code.visualstudio.com/docs/editor/debugging) for details. - -### Debug Port Mappings - -| Armada service | Debug host | -| ----------------- | ---------------- | -| `server` | `localhost:4000` | -| `executor` | `localhost:4001` | -| `binoculars` | `localhost:4002` | -| `eventingester` | `localhost:4003` | -| `lookoutui` | `localhost:4004` | -| `lookout` | `localhost:4005` | -| `lookoutingester` | `localhost:4007` | - -### GoLand Run Configurations - -Run configurations are available in the `.run` directory. When opening the project in GoLand, you can run Armada in both standard and debug mode. - -**Note:** The executor requires a Kubernetes config in `$PROJECT_DIR$/.kube/internal/config`. - -### Other Debugging Methods - -Run `mage dev:deps` to only spin up the dependencies (redis, postgres, pulsar), then run individual components yourself. Each component reads its config from `_local//config.yaml`. - -## Extending Armada - -Armada can be extended and customized in several ways: - -### Custom Schedulers - -The scheduler is designed to be extensible. You can implement custom scheduling algorithms by modifying the scheduler logic in `internal/scheduler/`. The scheduler handles: - -- Job queuing and prioritization -- Resource allocation -- Gang scheduling -- Preemption logic - -The scheduler code is located in `internal/scheduler/` and can be customized to implement different scheduling strategies. - -### Custom Executors - -While the standard executor works with Kubernetes, you could create custom executors for other platforms. The executor interface is defined in `pkg/executorapi/` and communicates with the scheduler via gRPC. - -### Client Libraries - -Armada provides client libraries for multiple languages that you can extend or use as reference: - -- **Python**: [`client/python/`](https://github.com/armadaproject/armada/tree/master/client/python) -- **Java**: [`client/java/`](https://github.com/armadaproject/armada/tree/master/client/java) -- **Scala**: [`client/scala/`](https://github.com/armadaproject/armada/tree/master/client/scala) -- **.NET**: [`client/DotNet/`](https://github.com/armadaproject/armada/tree/master/client/DotNet) - -These libraries provide programmatic access to Armada's APIs and can be used as reference for building custom clients. - -### Integration Examples - -Armada has been integrated with various systems. These can serve as examples for creating your own integrations: - -- **Airflow**: [`third_party/airflow/`](https://github.com/armadaproject/armada/tree/master/third_party/airflow) - Airflow operator for Armada -- **Metaflow**: [armada-metaflow repository](https://github.com/armadaproject/armada-metaflow) - Metaflow decorator for Armada -- **Jenkins**: [jenkins-plugin repository](https://github.com/armadaproject/jenkins-plugin) - Jenkins plugin for Armada -- **Spark**: [armada-spark repository](https://github.com/armadaproject/armada-spark) - Spark cluster manager for Armada - -### UI Development - -To develop the Lookout UI locally, the UI code is located in `internal/lookoutui/`. The UI is built with React and TypeScript. - -When using Goreman, the UI runs automatically on http://localhost:3000 (frontend dev server). - -To run the UI separately for development: - -```bash -cd internal/lookoutui -yarn -yarn openapi -PROXY_TARGET=http://localhost:8089 yarn dev -``` - -This starts a development server on http://localhost:3000 that proxies API requests to the backend. - -Alternatively, build a production version with: - -```bash -mage ui -``` - -This builds the UI and makes it available at http://localhost:8089. - -## Troubleshooting - -### Port 6443 Already in Use - -If port 6443 is already in use, modify `_local/kind/cluster.yaml` to use a different port: - -```yaml -- containerPort: 6443 - hostPort: 6444 # Change to an available port - protocol: TCP -``` - -### Arm/M1 Mac Issues - -On Arm/M1 Macs, you may need to set: - -```bash -export PULSAR_IMAGE=richgross/pulsar:2.11.0 -``` - -For more information on known issues: - -- [Arm issue](https://github.com/armadaproject/armada/issues/2493) -- [Windows issue](https://github.com/armadaproject/armada/issues/2492) - -## Additional Resources - -### Website Resources - -- [API Reference](./user-guide/api.mdx) - API reference for REST and gRPC -- [Contributor Guide](./contribute/contributor-guide.mdx) - Contribution guidelines and PR process -- [Understanding Armada](./understanding-armada) - Core concepts and architecture -- [Community](./community.mdx) - Get help, connect with the community, and find support resources */} diff --git a/website/content/getting-started.mdx b/website/content/getting-started.mdx index 308458a5926..0142dd4c688 100644 --- a/website/content/getting-started.mdx +++ b/website/content/getting-started.mdx @@ -156,7 +156,7 @@ import { Cards, Card } from 'fumadocs-ui/components/card'; diff --git a/website/content/index.mdx b/website/content/index.mdx index d37bdb074ca..f4614a579b1 100644 --- a/website/content/index.mdx +++ b/website/content/index.mdx @@ -333,7 +333,7 @@ G-Research, a leading quantitative research company, uses Armada in production t
Read the quickstart → @@ -367,7 +367,7 @@ Ready to explore Armada? Here are your next steps: description='Try Armada locally' /> From 8eebd1cbf32bea34dd89c68d4dc079034ab720fb Mon Sep 17 00:00:00 2001 From: sarhiri Date: Mon, 27 Jul 2026 15:12:10 -0500 Subject: [PATCH 42/49] docs: archive deprecated pages, fix internal links, formatting, and spelling Signed-off-by: sarhiri --- website/README.md | 27 ++- .../_archive/understanding-armada/meta.json | 5 +- website/content/docs/developer-guide.mdx | 127 +++++++------- website/content/docs/meta.json | 9 +- website/content/index.mdx | 159 ++++++------------ website/content/meta.json | 7 +- website/dict/armada.txt | 6 + website/src/app/(docs)/[[...slug]]/page.tsx | 2 +- website/src/components/logo.tsx | 132 ++++++++++++--- 9 files changed, 244 insertions(+), 230 deletions(-) diff --git a/website/README.md b/website/README.md index 89ae0f56bbc..b118b144958 100644 --- a/website/README.md +++ b/website/README.md @@ -1,6 +1,6 @@ # Armada Website Readme -The Armada documentation site is built with [Next.js](https://nextjs.org) and [Fumadocs](https://fumadocs.dev). All content is written in MDX, Markdown that can include React components. +The Armada documentation site is built with [Next.js](https://nextjs.org) and [Fumadocs](https://fumadocs.dev). All content is written in MDX, Markdown that can include React components. ## Requirements @@ -17,7 +17,6 @@ All commands should be run from inside the `website/` folder. If you are at the cd website ``` - ### Install dependencies ```bash @@ -39,6 +38,7 @@ The live site is deployed to GitHub Pages under a base path. If your changes inv ```bash cp .env.example .env.local ``` + Then open `.env.local` and follow the instructions inside. You do not need this for most content changes: only if you are working on routing, assets, or the Next.js config itself. ## How the site works @@ -48,15 +48,15 @@ Every page on the site is an `.mdx` file under `content/`. MDX is Markdown that ### Content structure content/ -├── index.mdx # Homepage / landing page +├── index.mdx # Homepage / landing page ├── getting-started.mdx # Quickstart guide -├── meta.json # Root nav configuration -├── docs/ # All documentation pages -│ ├── meta.json -│ ├── core-concepts.mdx -│ ├── developer-guide.mdx -│ └── ... -└── contribute/ # Contributing section +├── meta.json # Root nav configuration +├── docs/ # All documentation pages +│ ├── meta.json +│ ├── core-concepts.mdx +│ ├── developer-guide.mdx +│ └── ... +└── contribute/ # Contributing section ├── meta.json └── ... @@ -66,22 +66,21 @@ The left sidebar is driven entirely by `meta.json` files — Fumadocs reads them **Root nav — `content/meta.json`** - **Section nav — `content/docs/meta.json`** A `meta.json` inside a folder controls that section's title, page order, and which pages appear. **To add a page to the nav:** + 1. Create the `.mdx` file in the right folder 2. Add the filename (without `.mdx`) to the relevant `meta.json` pages array **To remove a page from the nav:** -Remove it from `meta.json`. +Remove it from `meta.json`. **To reorder pages:** Change the order in the `meta.json` pages array. Top to bottom = top to bottom in the sidebar. - **Important:** A folder with no `meta.json` is completely invisible to the nav. The pages exist and are routable URLs but won't appear in the sidebar. If a page you created isn't showing up, check whether its folder has a `meta.json` and whether that file is listed in it. ### Right TOC @@ -111,10 +110,10 @@ Markdown link syntax (`[text](url)`) does not render inside JSX elements. Use an [CNCF](https://cncf.io) {/* Use this instead */} + CNCF ``` - ## Format, Lint Content, Lint Code and Spell Check Please make sure to format and lint your code before committing: diff --git a/website/_archive/understanding-armada/meta.json b/website/_archive/understanding-armada/meta.json index 40688565eeb..efbe7311acc 100644 --- a/website/_archive/understanding-armada/meta.json +++ b/website/_archive/understanding-armada/meta.json @@ -1 +1,4 @@ -{"title": "Understanding Armada", "pages": ["index", "architecture", "core-concepts"]} +{ + "title": "Understanding Armada", + "pages": ["index", "architecture", "core-concepts"] +} diff --git a/website/content/docs/developer-guide.mdx b/website/content/docs/developer-guide.mdx index 1a3860294fd..8cb2ce592b3 100644 --- a/website/content/docs/developer-guide.mdx +++ b/website/content/docs/developer-guide.mdx @@ -2,6 +2,7 @@ title: 'Local Development' description: 'Set up your development environment and start contributing to Armada' --- + import { Callout } from 'fumadocs-ui/components/callout'; import { Step, Steps } from 'fumadocs-ui/components/steps'; @@ -26,7 +27,8 @@ Before you begin, make sure you have the following installed: {/* divider */} -
+ +
## Using Goreman @@ -92,7 +94,7 @@ Running processes are prefixed with `*`: *lookout
*lookoutingester
*binoculars
-*lookoutui +\*lookoutui @@ -104,6 +106,7 @@ Running processes are prefixed with `*`: ```bash goreman restart server ``` + **Useful mage commands** @@ -115,13 +118,16 @@ mage dev:down # stop dependency containers (after mage dev:up) mage dev:fullDown # stop containerised stack and tear down Kind (after mage dev:full) mage -l # list all available mage commands ``` + - Use `mage dev:full` to replicate what CI runs. Use `mage dev:up` for day-to-day - development. It's faster since components run as host processes. + Use `mage dev:full` to replicate what CI runs. Use `mage dev:up` for + day-to-day development. It's faster since components run as host + processes. {/* divider */} -
+ +
## Running with authentication @@ -177,11 +183,13 @@ armadactl --config _local/.armadactl.yaml --context auth-oidc get queues - Default Keycloak credentials — Admin: `admin` / `admin` · User: `user` / `password` + Default Keycloak credentials — Admin: `admin` / `admin` · User: `user` / + `password` {/* divider */} -
+ +
## Running without a Kubernetes cluster @@ -202,31 +210,8 @@ Useful for testing scheduling logic, development when Kubernetes is unavailable, and integration testing of job flows. {/* divider */} -
- -{/* ## Code structure -Armada
-├── cmd/ # Entry points for all components
-│ ├── server/ # Armada server (API server)
-│ ├── executor/ # Executor (runs in each K8s cluster)
-│ ├── scheduler/ # Scheduler (job scheduling logic)
-│ ├── lookout/ # Lookout (job monitoring/UI backend)
-│ └── armadactl/ # Command-line interface
-├── internal/ # Internal packages (not for external use)
-│ ├── server/ # Server implementation
-│ ├── executor/ # Executor implementation
-│ ├── scheduler/ # Scheduler implementation
-│ ├── lookout/ # Lookout implementation
-│ └── common/ # Shared utilities
-├── pkg/ # Public packages (for external use)
-│ ├── api/ # gRPC API definitions
-│ └── client/ # Client libraries
-├── config/ # Configuration files for components
-├── magefiles/ # Build automation (mage targets)
-└── testsuite/ # Integration test cases
*/} -{/* divider */} -
+
## Testing your setup @@ -246,7 +231,8 @@ go run cmd/testsuite/main.go test --tests "testsuite/testcases/basic/*" --junit ``` {/* divider */} -
+ +
## Profiling with pprof @@ -264,7 +250,8 @@ go tool pprof http://localhost:6060/debug/pprof/profile ``` {/* divider */} -
+ +
## Debug port mappings @@ -279,7 +266,8 @@ go tool pprof http://localhost:6060/debug/pprof/profile | `lookoutingester` | `localhost:4007` | {/* divider */} -
+ +
## Troubleshooting @@ -303,48 +291,53 @@ See [Arm issue #2493](https://github.com/armadaproject/armada/issues/2493) and [Windows issue #2492](https://github.com/armadaproject/armada/issues/2492) for more details. -{/* **Need help?** - -Ask in the [#armada channel on CNCF Slack](https://cloud-native.slack.com/archives/C03T9CBCEMC). */} - {/* divider */} -
+
{/* ── CTA ── */} +
-
+
-
- Need help? -
+
Need help?
-
- Ask in the Armada Slack channel or find us on Github! -
+
+ Ask in the Armada Slack channel or find us on Github! +
- + + + GitHub Issues + +
diff --git a/website/content/docs/meta.json b/website/content/docs/meta.json index 8a4e1c6fbaf..b16d6663eff 100644 --- a/website/content/docs/meta.json +++ b/website/content/docs/meta.json @@ -1,9 +1,4 @@ { "title": "Docs", - "pages": [ - "core-concepts", - "developer-guide", - "architecture", - "..." - ] -} \ No newline at end of file + "pages": ["core-concepts", "developer-guide", "architecture", "..."] +} diff --git a/website/content/index.mdx b/website/content/index.mdx index f4614a579b1..5eaa9f69370 100644 --- a/website/content/index.mdx +++ b/website/content/index.mdx @@ -2,11 +2,17 @@ title: Armada --- -import { ArmadaIcon, ArmadaText } from '@/components/logo'; import Link from 'next/link'; +import { + Boxes, + Scale, + Zap, + MoveUpRight, + Activity, + LayoutGrid, +} from 'lucide-react'; import { Cards, Card } from 'fumadocs-ui/components/card'; -import { Boxes, Scale, Zap, MoveUpRight, Activity, LayoutGrid } from 'lucide-react'; - +import { ArmadaIcon, ArmadaText } from '@/components/logo';
@@ -73,7 +79,7 @@ import { Boxes, Scale, Zap, MoveUpRight, Activity, LayoutGrid } from 'lucide-rea
- {/* Hero Section/What is Armada? */} +{/* Hero Section/What is Armada? */}
@@ -84,7 +90,7 @@ import { Boxes, Scale, Zap, MoveUpRight, Activity, LayoutGrid } from 'lucide-rea One API.
Any number of clusters.
- Millions of jobs. + Millions of jobs. {/* Armada description */} @@ -111,38 +117,45 @@ import { Boxes, Scale, Zap, MoveUpRight, Activity, LayoutGrid } from 'lucide-rea {/* divider */}
-
- -
- -
- As a [CNCF Sandbox project](https://www.cncf.io/), Armada is actively maintained and used in production environments, including at [G-Research](https://www.gresearch.com/) where it processes millions of jobs daily.
+
+
+ As a [CNCF Sandbox project](https://www.cncf.io/), Armada is actively + maintained and used in production environments, including at + [G-Research](https://www.gresearch.com/) where it processes millions of jobs + daily. +
{/* divider */} -
+
{/* ## Why use Armada? */} +
## What is Armada? - {/* heading */} - - The batch scheduler Kubernetes was missing. - +{/* heading */} - {/* description */} - - Armada sits above your Kubernetes clusters as a control plane. It doesn't replace Kubernetes, it allows Kubernetes to handle millions of jobs a day across tens of thousands of nodes. - + + The batch scheduler Kubernetes was missing. + + +{/* description */} + + + Armada sits above your Kubernetes clusters as a control plane. It does not + replace Kubernetes, but allows K8 to handle millions of jobs a day across tens + of thousands of nodes. + + +{/* cards — left-accent, content stays left-aligned */} - {/* cards — left-accent, content stays left-aligned */} } title='Multi-cluster native' className='border-0 border-l-2 border-fd-primary rounded-sm shadow-md shadow-black/20 dark:shadow-white/10 transition-colors hover:bg-fd-muted'> Run jobs across many clusters through one API, and add or remove capacity without disrupting what's already running. @@ -167,26 +180,24 @@ import { Boxes, Scale, Zap, MoveUpRight, Activity, LayoutGrid } from 'lucide-rea {/* divider */}
- - {/* ## Use cases */} +
## Use Cases - {/* heading */} - - Is Armada right for you? - +{/* heading */} - {/* description */} - - + + Is Armada right for you? + + +{/* description */} + + - {/* numbered list — divide-y draws the lines between rows; - font-mono primary number on the left, content on the right */} -
+
{/* 01 */}
@@ -216,7 +227,7 @@ import { Boxes, Scale, Zap, MoveUpRight, Activity, LayoutGrid } from 'lucide-rea
High-performance computing - MPI workloads in containers, scheduled across clusters, with hardware-aware placement. Cloud-native tooling without giving up the reproducibility HPC teams depend on. + MPI workloads in containers, scheduled across clusters, with hardware-aware placement. Cloud-native tooling without giving up the reproducibility HPC teams depend on.
@@ -227,7 +238,7 @@ import { Boxes, Scale, Zap, MoveUpRight, Activity, LayoutGrid } from 'lucide-rea
Multi-tenant compute environments - Multiple teams, one infrastructure. Fair-share scheduling means no single team can quietly consume everything while others wait. + Multiple teams, one infrastructure. Fair-share scheduling means no single team can quietly consume everything while others wait.
@@ -238,12 +249,13 @@ import { Boxes, Scale, Zap, MoveUpRight, Activity, LayoutGrid } from 'lucide-rea
CI/CD build and test - Critical merges go first. Large test suites don't block urgent builds. Priority and fairness built in, no manual queue management. + Critical merges go first. Large test suites don't block urgent builds. Priority and fairness built in, no manual queue management.
+ Armada's niche is multi-cluster. If you're not there yet, another project may be a better fit. @@ -259,59 +271,13 @@ import { Boxes, Scale, Zap, MoveUpRight, Activity, LayoutGrid } from 'lucide-rea
-{/* divider */} -
- - -{/* ## Use Cases and Success Stories - -### High-Performance Computing (HPC) - -- **Machine Learning Training**: Distribute large-scale ML training jobs across multiple clusters -- **Scientific Computing**: Run complex simulations and data analysis workloads -- **Financial Modeling**: Execute risk calculations and quantitative analysis at scale - -### Data Processing Pipelines - -- **ETL Workloads**: Process large datasets with parallel batch jobs -- **Data Analytics**: Run distributed analytics jobs across multiple clusters -- **Backup and Archival**: Coordinate large-scale data movement operations - -### CI/CD and Development - -- **Build Systems**: Distribute compilation and testing jobs -- **Integration Testing**: Run comprehensive test suites across multiple environments -- **Deployment Automation**: Coordinate complex deployment workflows - -### Production Deployment at G-Research - -G-Research, a leading quantitative research company, uses Armada in production to: - -- Process millions of jobs per day -- Manage tens of thousands of nodes -- Support diverse computational workloads -- Maintain high availability and performance - -## Comparison with Other Schedulers - -### vs. Native Kubernetes Scheduler - -- **Scale**: Armada spans multiple clusters vs. single cluster limitation -- **Throughput**: Millions of jobs vs. thousands with native scheduler -- **Batch Features**: Purpose-built for batch vs. service-oriented design -- **Fair Scheduling**: Advanced fair-use policies vs. basic priority classes - -### vs. Traditional HPC Schedulers (SLURM, PBS) */} - -{/* - **Container Native**: Built for containerized workloads vs. traditional HPC -- **Kubernetes Integration**: Leverages Kubernetes ecosystem vs. isolated systems -- **Cloud Ready**: Designed for cloud and hybrid environments -- **Modern APIs**: REST/gRPC APIs vs. command-line interfaces -- **Rich Client Support**: Client libraries available for multiple languages (Go, Java, Scala, Python and .NET) */} +{/* divider */} +
{/* ── CTA ── */} +
@@ -345,30 +311,9 @@ G-Research, a leading quantitative research company, uses Armada in production t > Join #armada on Slack +
{/* divider */}
- - - - - -{/* -## Next Steps - -Ready to explore Armada? Here are your next steps: - - - - - */} diff --git a/website/content/meta.json b/website/content/meta.json index 10e3935a806..c9d8a5a73fa 100644 --- a/website/content/meta.json +++ b/website/content/meta.json @@ -1,9 +1,4 @@ { "root": true, - "pages": [ - "index", - "getting-started", - "docs", - "contribute" - ] + "pages": ["index", "getting-started", "docs", "contribute"] } diff --git a/website/dict/armada.txt b/website/dict/armada.txt index 1dbf449c054..6a9fd88bac2 100644 --- a/website/dict/armada.txt +++ b/website/dict/armada.txt @@ -38,3 +38,9 @@ preemptibility krew uncordon gcflags +magefile +ingesters +scheduleringester +containerised +initialise +organisations \ No newline at end of file diff --git a/website/src/app/(docs)/[[...slug]]/page.tsx b/website/src/app/(docs)/[[...slug]]/page.tsx index aaa3dd1d64f..11ac8707aa6 100644 --- a/website/src/app/(docs)/[[...slug]]/page.tsx +++ b/website/src/app/(docs)/[[...slug]]/page.tsx @@ -17,7 +17,7 @@ export default async function Page(props: { const params = await props.params; const page = source.getPage(params.slug); const isHome = !params.slug || params.slug.length === 0; - + if (!page) notFound(); const MDXContent = page.data.body; diff --git a/website/src/components/logo.tsx b/website/src/components/logo.tsx index 5e1cc313ae5..d6dc6c5877c 100644 --- a/website/src/components/logo.tsx +++ b/website/src/components/logo.tsx @@ -54,34 +54,112 @@ export const CncfLogo = (props: SVGProps) => ( {...props} > Cloud Native Computing Foundation - + - - - - - - + + + + + + - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + - + -); \ No newline at end of file +); From 66adcffc3840e5da20b63c188d78dcda5f818554 Mon Sep 17 00:00:00 2001 From: sarhiri Date: Mon, 27 Jul 2026 15:49:13 -0500 Subject: [PATCH 43/49] fix: resolve community card to absolute /contribute/community path Signed-off-by: sarhiri --- website/content/contribute/index.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/content/contribute/index.mdx b/website/content/contribute/index.mdx index beb3deaa95a..00aabf0179a 100644 --- a/website/content/contribute/index.mdx +++ b/website/content/contribute/index.mdx @@ -20,7 +20,7 @@ import { Cards, Card } from 'fumadocs-ui/components/card'; description='Our community standards and guidelines for participation' /> From f9b96c54633e5d50fbed9fb3fe01428d5383117e Mon Sep 17 00:00:00 2001 From: sarhiri Date: Wed, 29 Jul 2026 15:53:48 -0500 Subject: [PATCH 44/49] fix /contribute/community path Signed-off-by: sarhiri --- website/content/contribute/index.mdx | 2 +- website/content/getting-started.mdx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/website/content/contribute/index.mdx b/website/content/contribute/index.mdx index 00aabf0179a..3eb7d2974dd 100644 --- a/website/content/contribute/index.mdx +++ b/website/content/contribute/index.mdx @@ -20,7 +20,7 @@ import { Cards, Card } from 'fumadocs-ui/components/card'; description='Our community standards and guidelines for participation' /> diff --git a/website/content/getting-started.mdx b/website/content/getting-started.mdx index 0142dd4c688..1accd6a9de4 100644 --- a/website/content/getting-started.mdx +++ b/website/content/getting-started.mdx @@ -156,7 +156,7 @@ import { Cards, Card } from 'fumadocs-ui/components/card'; From f6a51b18f1cfc3996dfd28f727cf71ca30206322 Mon Sep 17 00:00:00 2001 From: sarhiri Date: Thu, 30 Jul 2026 11:15:37 -0500 Subject: [PATCH 45/49] reroute for new nav Signed-off-by: sarhiri --- website/content/contribute/community.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/content/contribute/community.mdx b/website/content/contribute/community.mdx index 086f54336e4..af8438da157 100644 --- a/website/content/contribute/community.mdx +++ b/website/content/contribute/community.mdx @@ -28,7 +28,7 @@ Found a bug or have a feature request? Open an issue on our [GitHub repository]( Have a fix or enhancement ready? We'd love to see your contribution! Submit a pull request on our [GitHub repository](https://github.com/armadaproject/armada/pulls). Whether it's code, documentation, or improvements, every contribution helps make Armada better for everyone. -Interested in contributing but not sure where to start? Check out our [Contributor Guide](./contributor-guide.mdx) and browse open issues on GitHub. There's always something you can help with! +Interested in contributing but not sure where to start? Check out our [Contributor Guide](/contribute/contributor-guide.mdx) and browse open issues on GitHub. There's always something you can help with! ## Community Meetings From d9ec018e2290e425b634e63bba8f1a4e33e3b545 Mon Sep 17 00:00:00 2001 From: sarhiri Date: Thu, 30 Jul 2026 11:39:25 -0500 Subject: [PATCH 46/49] updated broken links for new routing in community.mdx & contributor-guide.mdx Signed-off-by: sarhiri --- website/content/contribute/community.mdx | 2 +- website/content/contribute/contributor-guide.mdx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/website/content/contribute/community.mdx b/website/content/contribute/community.mdx index af8438da157..9d6144bd00d 100644 --- a/website/content/contribute/community.mdx +++ b/website/content/contribute/community.mdx @@ -28,7 +28,7 @@ Found a bug or have a feature request? Open an issue on our [GitHub repository]( Have a fix or enhancement ready? We'd love to see your contribution! Submit a pull request on our [GitHub repository](https://github.com/armadaproject/armada/pulls). Whether it's code, documentation, or improvements, every contribution helps make Armada better for everyone. -Interested in contributing but not sure where to start? Check out our [Contributor Guide](/contribute/contributor-guide.mdx) and browse open issues on GitHub. There's always something you can help with! +Interested in contributing but not sure where to start? Check out our [Contributor Guide](contributor-guide.mdx) and browse open issues on GitHub. There's always something you can help with! ## Community Meetings diff --git a/website/content/contribute/contributor-guide.mdx b/website/content/contribute/contributor-guide.mdx index e49933c68cf..3468424e9a1 100644 --- a/website/content/contribute/contributor-guide.mdx +++ b/website/content/contribute/contributor-guide.mdx @@ -64,7 +64,7 @@ For more details, see [DCO](https://github.com/apps/dco). ## Communication -For real-time discussions, Slack channels, GitHub Discussions, and community meetings, see our [Community page](./community.mdx). +For real-time discussions, Slack channels, GitHub Discussions, and community meetings, see our [Community page](community.mdx). ## Security From 32906ec764811ed88616054804167e90f5789020 Mon Sep 17 00:00:00 2001 From: sarhiri Date: Thu, 30 Jul 2026 12:45:54 -0500 Subject: [PATCH 47/49] greptile routing fixes Signed-off-by: sarhiri --- website/content/contribute/community.mdx | 2 +- website/content/contribute/contributor-guide.mdx | 4 ++-- website/content/docs/api.mdx | 2 +- website/content/docs/meta.json | 12 +++++++++++- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/website/content/contribute/community.mdx b/website/content/contribute/community.mdx index 9d6144bd00d..0e26e051686 100644 --- a/website/content/contribute/community.mdx +++ b/website/content/contribute/community.mdx @@ -28,7 +28,7 @@ Found a bug or have a feature request? Open an issue on our [GitHub repository]( Have a fix or enhancement ready? We'd love to see your contribution! Submit a pull request on our [GitHub repository](https://github.com/armadaproject/armada/pulls). Whether it's code, documentation, or improvements, every contribution helps make Armada better for everyone. -Interested in contributing but not sure where to start? Check out our [Contributor Guide](contributor-guide.mdx) and browse open issues on GitHub. There's always something you can help with! +Interested in contributing but not sure where to start? Check out our [Contributor Guide](/contribute/contributor-guide) and browse open issues on GitHub. There's always something you can help with! ## Community Meetings diff --git a/website/content/contribute/contributor-guide.mdx b/website/content/contribute/contributor-guide.mdx index 3468424e9a1..c1942b8206f 100644 --- a/website/content/contribute/contributor-guide.mdx +++ b/website/content/contribute/contributor-guide.mdx @@ -7,7 +7,7 @@ description: 'Guidelines and tips for contributing to the Armada project.' To setup your development environment, follow the instructions based on the area you want to contribute to: -- **For Armada core development**: See the [Developer Guide](../docs/developer-guide.mdx) for local setup, code structure, and development workflows +- **For Armada core development**: See the [Developer Guide](/docs/developer-guide) for local setup, code structure, and development workflows - **For Armada Operator development**: See the [Armada Operator repository](https://github.com/armadaproject/armada-operator) for setup instructions and development guidelines ## Reporting Issues @@ -64,7 +64,7 @@ For more details, see [DCO](https://github.com/apps/dco). ## Communication -For real-time discussions, Slack channels, GitHub Discussions, and community meetings, see our [Community page](community.mdx). +For real-time discussions, Slack channels, GitHub Discussions, and community meetings, see our [Community page](/contribute/community). ## Security diff --git a/website/content/docs/api.mdx b/website/content/docs/api.mdx index c84ffd98951..9ef67f71799 100644 --- a/website/content/docs/api.mdx +++ b/website/content/docs/api.mdx @@ -78,4 +78,4 @@ For most use cases, we recommend using: - [OpenAPI Specification](https://github.com/armadaproject/armada/blob/master/pkg/api/api.swagger.json) - Full REST API documentation - [gRPC API Definitions](https://github.com/armadaproject/armada/tree/master/pkg/api) - Protocol buffer definitions -- [Client Libraries](./clients.mdx) - Pre-built libraries for popular languages +- [Client Libraries](/docs/clients) - Pre-built libraries for popular languages diff --git a/website/content/docs/meta.json b/website/content/docs/meta.json index b16d6663eff..1ee45d16627 100644 --- a/website/content/docs/meta.json +++ b/website/content/docs/meta.json @@ -1,4 +1,14 @@ { "title": "Docs", - "pages": ["core-concepts", "developer-guide", "architecture", "..."] + "pages": [ + "core-concepts", + "developer-guide", + "architecture", + "cli", + "clients", + "integrations", + "user-guide", + "api", + "..." + ] } From ac1fd387a616c6838c6ed47ec338c3accad2f2ab Mon Sep 17 00:00:00 2001 From: sarhiri Date: Thu, 30 Jul 2026 15:32:32 -0500 Subject: [PATCH 48/49] chore(website): skip route-shaped links in remark link validation. yarn and greptile were arguing about static vs absolute routing paths, and by adding remark-validate-links to stop checking routes at the base level, but they still run during the build script Signed-off-by: sarhiri --- website/.remarkrc-md.mjs | 14 +++++++++++++- website/.remarkrc-mdx.mjs | 14 +++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/website/.remarkrc-md.mjs b/website/.remarkrc-md.mjs index 8fbff6e5d46..bff8dd9cd82 100644 --- a/website/.remarkrc-md.mjs +++ b/website/.remarkrc-md.mjs @@ -2,7 +2,19 @@ const remarkConfig = { plugins: [ 'remark-preset-lint-consistent', 'remark-frontmatter', - 'remark-validate-links', + [ + 'remark-validate-links', + { + // Internal links use absolute Next.js routes (e.g. /docs/clients), + // not file paths. remark-validate-links resolves paths from the git + // root and has no extension resolution, so it can't check these. + // Skip route-shaped links; relative links and heading anchors are + // still validated. + // TODO: add a post-build link check against the static export to + // properly validate routes (see check-links.sh). + skipPathPatterns: [/\/[^/.]+\/?(#[^/]*)?$/], + }, + ], [ 'remark-lint-no-dead-urls', { diff --git a/website/.remarkrc-mdx.mjs b/website/.remarkrc-mdx.mjs index 2c1b6bf8679..345d1d07634 100644 --- a/website/.remarkrc-mdx.mjs +++ b/website/.remarkrc-mdx.mjs @@ -2,7 +2,19 @@ const remarkConfig = { plugins: [ 'remark-mdx', 'remark-mdx-frontmatter', - 'remark-validate-links', + [ + 'remark-validate-links', + { + // Internal links use absolute Next.js routes (e.g. /docs/clients), + // not file paths. remark-validate-links resolves paths from the git + // root and has no extension resolution, so it can't check these. + // Skip route-shaped links; relative links and heading anchors are + // still validated. + // TODO: add a post-build link check against the static export to + // properly validate routes (see check-links.sh). + skipPathPatterns: [/\/[^/.]+\/?(#[^/]*)?$/], + }, + ], [ 'remark-lint-no-dead-urls', { From 628d4277d295b83be81b26531fb1947c67d2c4bd Mon Sep 17 00:00:00 2001 From: sarhiri Date: Tue, 11 Aug 2026 17:17:33 -0500 Subject: [PATCH 49/49] docs: address PR review comments on README - Reframe gang scheduling as Armada fine grained capability & Simplify armadactl section to note make kind-all handles installation Signed-off-by: sarhiri --- README.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 1bfce653d3c..3bddb706f35 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@

CircleCI - Go Report Card + Artifact Hub LFX Health Score OpenSSF Best Practices @@ -32,7 +32,7 @@ When your job volume exceeds what a single cluster can handle, you need a contro - **No job queue** — Kubernetes has no concept of ordering. Jobs compete for resources with no fairness guarantees. Armada adds a proper queue with priority, fair-share, and rate limiting. - **No multi-cluster coordination** — Each Kubernetes cluster is an island. Armada routes jobs across as many clusters as you need from a single API. -- **No gang scheduling** — Distributed jobs that need all workers to start simultaneously (MPI, PyTorch, Spark) have no atomic startup guarantee in vanilla Kubernetes. Armada either starts the whole group or holds it. +- **Fine grained gang-scheduling** — Distributed jobs that need all workers to start together (MPI, PyTorch, Spark) are either fully scheduled or held in queue. Armada's implementation is battle-tested at scale with deep fairness and preemption integration. - **No fairness across teams** — One team can starve everyone else. Armada enforces fair-share scheduling so heavy users don't permanently dominate shared infrastructure. Armada is used in production at [G-Research](https://www.gresearch.co.uk/) since 2020, processing **millions of batch jobs per day** across tens of thousands of nodes. @@ -63,17 +63,19 @@ cd armada-operator make kind-all ``` -→ **[Full quickstart guide](https://armadaproject.io/quickstart)** — get up and running in under 15 minutes. +→ **[Full quickstart guide](https://armadaproject.io/quickstart)** — get up and running in an instant! + + ### armadactl -Armada's CLI for interacting with the system: +`armadactl` is installed automatically when you run `make kind-all`. To install it standalone or on a machine without the full Armada setup: ```bash # download via script scripts/get-armadactl.sh -# or grab the binary from the releases page +# or grab the binary from the release page https://github.com/armadaproject/armada/releases/latest ```