WIP: add RBAC testing to e2e suite

_local/.armadactl.yaml

@@ -30,14 +30,50 @@ contexts:
 
   # Context with OIDC Client Credentials authentication
   # For service accounts (requires confidential client with secret)
-  auth-client-creds:
+  auth-client-creds-executor:
     armadaUrl: localhost:50051
     openIdClientCredentialsAuth:
       providerUrl: "http://localhost:8180/realms/armada"
       clientId: "armada-executor"
       clientSecret: "executor-secret"
       scopes: ["openid", "profile", "email"]
 
+  auth-client-creds-user:
+    armadaUrl: localhost:50051
+    openIdClientCredentialsAuth:
+      providerUrl: "http://localhost:8180/realms/armada"
+      clientId: "armada-user"
+      clientSecret: "user-secret"
+      scopes: ["openid", "profile", "email"]
+
+  auth-client-creds-admin:
+    armadaUrl: localhost:50051
+    openIdClientCredentialsAuth:
+      providerUrl: "http://localhost:8180/realms/armada"
+      clientId: "armada-admin"
+      clientSecret: "admin-secret"
+      scopes: ["openid", "profile", "email"]
+
+  # Contexts for the rbac integration test suite, against the containerized full.yaml stack
+  # (server-auth on :50061, keycloak reachable from the host via the /etc/hosts alias that
+  # `mage dev:full` adds -- see magefiles/dev.go's ensureEtcHostsEntry). Same Keycloak realm/
+  # clients as auth-client-creds-admin/-user above; only the network path differs.
+  rbac-admin:
+    armadaUrl: localhost:50061
+    openIdClientCredentialsAuth:
+      providerUrl: "http://keycloak:8180/realms/armada"
+      clientId: "armada-admin"
+      clientSecret: "admin-secret"
+      scopes: ["openid", "profile", "email"]
+
+  rbac-user:
+    armadaUrl: localhost:50061
+    openIdClientCredentialsAuth:
+      providerUrl: "http://keycloak:8180/realms/armada"
+      clientId: "armada-user"
+      clientSecret: "user-secret"
+      scopes: ["openid", "profile", "email"]
+
   # Context with OIDC Password Flow authentication
   auth-password:
     armadaUrl: localhost:50051

_local/compose/full.yaml

@@ -138,6 +138,47 @@ services:
       ARMADA_POSTGRES_CONNECTION_HOST: postgres
     command: ./lookout --migrateDatabase --config /config/config.yaml
 
+  # =========================================================
+  # OIDC provider, for the rbac test suite only
+  # =========================================================
+  keycloak:
+    container_name: keycloak
+    image: ${KEYCLOAK_IMAGE:-quay.io/keycloak/keycloak:25.0}
+    environment:
+      KEYCLOAK_ADMIN: admin
+      KEYCLOAK_ADMIN_PASSWORD: admin
+      KC_HTTP_PORT: 8180
+      KC_HEALTH_ENABLED: "true"
+      # Pins the OIDC issuer to one literal hostname regardless of whether the caller is another
+      # container (reaching it via the "keycloak" service name) or the host (reaching it via the
+      # "keycloak" /etc/hosts alias that `mage dev:full` adds, pointing at 127.0.0.1). go-oidc
+      # requires the discovery document's issuer to exactly match the URL used to fetch it, and
+      # token verification then requires the token's iss claim to match that same issuer, so both
+      # paths must agree on one hostname string.
+      KC_HOSTNAME: keycloak
+    ports:
+      - "8180:8180"
+    command: start-dev --import-realm
+    volumes:
+      - ../keycloak/import:/opt/keycloak/data/import:ro
+    networks: [kind]
+    restart: unless-stopped
+    # server-auth dials Keycloak's discovery endpoint at its own startup and, unlike the
+    # dependency services above, doesn't retry/restart on failure -- so server-auth's
+    # depends_on below needs service_healthy, not just service_started, to not lose the race
+    # against Keycloak's realm import. No curl/wget in this image, so the check uses bash's
+    # /dev/tcp against the KC_HEALTH_ENABLED management endpoint (port 9000) instead.
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "bash -c 'exec 3<>/dev/tcp/127.0.0.1/9000 && printf \"GET /health/ready HTTP/1.1\\r\\nHost: localhost\\r\\nConnection: close\\r\\n\\r\\n\" >&3 && cat <&3' | grep -q '\"status\": \"UP\"'",
+        ]
+      interval: 5s
+      timeout: 5s
+      retries: 30
+      start_period: 20s
+
   # =========================================================
   # Armada components
   # =========================================================
@@ -198,6 +239,33 @@ services:
       ARMADA_SCHEDULERAPICONNECTION_ARMADAURL: scheduler:50052
     command: ./server --config /config/config.yaml
 
+  # Second API server instance, backed by the same scheduler/pulsar/redis/postgres as `server`,
+  # but with OIDC auth enabled (config-auth.yaml). Used only by the rbac test suite, so every
+  # other suite's unauthenticated connection to `server` is unaffected.
+  server-auth:
+    <<: *component
+    container_name: server-auth
+    image: ${ARMADA_IMAGE:-gresearch/armada-bundle}:${ARMADA_IMAGE_TAG:-latest}
+    ports:
+      - "50061:50051"
+      - "8091:8081"
+    depends_on:
+      lookout-migration: {condition: service_completed_successfully}
+      pulsar: {condition: service_healthy}
+      redis: {condition: service_healthy}
+      keycloak: {condition: service_healthy}
+    volumes:
+      - ../server/config-auth.yaml:/config/config.yaml:ro
+    environment:
+      ARMADA_EVENTSAPIREDIS_ADDRS: redis:6379
+      ARMADA_REDIS_ADDRS: redis:6379
+      ARMADA_POSTGRES_CONNECTION_HOST: postgres
+      ARMADA_QUERYAPI_POSTGRES_CONNECTION_HOST: postgres
+      ARMADA_PULSAR_URL: pulsar://pulsar:6650
+      ARMADA_SCHEDULERAPICONNECTION_ARMADAURL: scheduler:50052
+      ARMADA_AUTH_OPENIDAUTH_PROVIDERURL: http://keycloak:8180/realms/armada
+    command: ./server --config /config/config.yaml
+
   executor:
     <<: *component
     container_name: executor

_local/keycloak/import/armada-realm.json

@@ -102,6 +102,72 @@
         "offline_access",
         "microprofile-jwt"
       ]
+    },
+    {
+      "clientId": "armada-user",
+      "name": "Armada User Service",
+      "description": "Service account for Armada Users",
+      "enabled": true,
+      "publicClient": false,
+      "standardFlowEnabled": false,
+      "implicitFlowEnabled": false,
+      "directAccessGrantsEnabled": false,
+      "serviceAccountsEnabled": true,
+      "authorizationServicesEnabled": false,
+      "protocol": "openid-connect",
+      "bearerOnly": false,
+      "consentRequired": false,
+      "clientAuthenticatorType": "client-secret",
+      "secret": "user-secret",
+      "redirectUris": [],
+      "webOrigins": [],
+      "notBefore": 0,
+      "attributes": {
+        "use.refresh.tokens": "false",
+        "client_credentials.use_refresh_token": "false"
+      },
+      "fullScopeAllowed": true,
+      "nodeReRegistrationTimeout": -1,
+      "defaultClientScopes": ["web-origins", "profile", "roles", "email"],
+      "optionalClientScopes": [
+        "address",
+        "phone",
+        "offline_access",
+        "microprofile-jwt"
+      ]
+    },
+    {
+      "clientId": "armada-admin",
+      "name": "Armada Admin Service",
+      "description": "Service account for Armada Admins",
+      "enabled": true,
+      "publicClient": false,
+      "standardFlowEnabled": false,
+      "implicitFlowEnabled": false,
+      "directAccessGrantsEnabled": false,
+      "serviceAccountsEnabled": true,
+      "authorizationServicesEnabled": false,
+      "protocol": "openid-connect",
+      "bearerOnly": false,
+      "consentRequired": false,
+      "clientAuthenticatorType": "client-secret",
+      "secret": "admin-secret",
+      "redirectUris": [],
+      "webOrigins": [],
+      "notBefore": 0,
+      "attributes": {
+        "use.refresh.tokens": "false",
+        "client_credentials.use_refresh_token": "false"
+      },
+      "fullScopeAllowed": true,
+      "nodeReRegistrationTimeout": -1,
+      "defaultClientScopes": ["web-origins", "profile", "roles", "email"],
+      "optionalClientScopes": [
+        "address",
+        "phone",
+        "offline_access",
+        "microprofile-jwt"
+      ]
     }
   ],
   "clientScopes": [
@@ -239,6 +305,40 @@
       "realmRoles": ["default-roles-armada"],
       "groups": ["/admins"]
     },
+    {
+      "username": "executor",
+      "enabled": true,
+      "email": "executor@example.com",
+      "firstName": "Executor",
+      "lastName": "User",
+      "emailVerified": true,
+      "credentials": [
+        {
+          "type": "password",
+          "value": "executor",
+          "temporary": false
+        }
+      ],
+      "realmRoles": ["default-roles-armada"],
+      "groups": ["/executors"]
+    },
+    {
+      "username": "user",
+      "enabled": true,
+      "email": "user@example.com",
+      "firstName": "User",
+      "lastName": "User",
+      "emailVerified": true,
+      "credentials": [
+        {
+          "type": "password",
+          "value": "user",
+          "temporary": false
+        }
+      ],
+      "realmRoles": ["default-roles-armada"],
+      "groups": ["/users"]
+    },
     {
       "username": "service-account-armada-executor",
       "enabled": true,
@@ -247,6 +347,26 @@
       "lastName": "Executor",
       "emailVerified": true,
       "serviceAccountClientId": "armada-executor",
+      "groups": ["/executors"]
+    },
+    {
+      "username": "service-account-armada-user",
+      "enabled": true,
+      "email": "user@armada.local",
+      "firstName": "Service Account",
+      "lastName": "User",
+      "emailVerified": true,
+      "serviceAccountClientId": "armada-user",
+      "groups": ["/users"]
+    },
+    {
+      "username": "service-account-armada-admin",
+      "enabled": true,
+      "email": "admin-service@armada.local",
+      "firstName": "Service Account",
+      "lastName": "Admin",
+      "emailVerified": true,
+      "serviceAccountClientId": "armada-admin",
       "groups": ["/admins"]
     }
   ],

_local/queues/e2e-test-queue.yaml

@@ -0,0 +1,4 @@
+apiVersion: armadaproject.io/v1beta1
+kind: Queue
+name: e2e-test-queue
+priorityFactor: 1.0

_local/rbac-queues/rbac-fixture-plain.yaml

@@ -0,0 +1,4 @@
+apiVersion: armadaproject.io/v1beta1
+kind: Queue
+name: rbac-fixture-plain
+priorityFactor: 1.0

_local/rbac-queues/rbac-fixture-restricted.yaml

@@ -0,0 +1,10 @@
+apiVersion: armadaproject.io/v1beta1
+kind: Queue
+name: rbac-fixture-restricted
+priorityFactor: 1.0
+permissions:
+  - subjects:
+      - name: users
+        kind: Group
+    verbs:
+      - submit

_local/server/config-auth.yaml

@@ -34,9 +34,13 @@ auth:
     delete_retry_policy: ["admins"]
     delete_queue: ["admins"]
     cancel_any_jobs: ["admins"]
+    preempt_any_jobs: ["admins"]
     reprioritize_any_jobs: ["admins"]
     watch_all_events: ["admins"]
     execute_jobs: ["admins", "executors"]
+    cordon_queue: ["admins"]
+    cordon_nodes: ["admins"]
+    update_executor_settings: ["admins"]
 eventsApiRedis:
   addrs:
     - localhost:6379

internal/testsuite/app.go

@@ -317,9 +317,23 @@ func (a *App) RunTests(ctx context.Context, testSpecs []*api.TestSpec) (*TestSui
 	wg.Add(len(testSpecs))
 	for i, testSpec := range testSpecs {
 		i := i
+		apiConnectionDetails := a.Params.ApiConnectionDetails
+		if testSpec.AuthContext != "" {
+			resolved, err := client.ResolveNamedContext(testSpec.AuthContext)
+			if err != nil {
+				report := NewTestCaseReport(testSpec)
+				report.Finish = report.Start
+				report.FailureReason = fmt.Sprintf("failed to resolve authContext %q: %s", testSpec.AuthContext, err)
+				report.Out = &bytes.Buffer{}
+				rv.TestCaseReports[i] = report
+				wg.Done()
+				continue
+			}
+			apiConnectionDetails = resolved
+		}
 		testRunner := TestRunner{
 			Out:                  a.Out,
-			apiConnectionDetails: a.Params.ApiConnectionDetails,
+			apiConnectionDetails: apiConnectionDetails,
 			testSpec:             testSpec,
 			eventLogger:          eventLogger,
 		}

internal/testsuite/eventwatcher/eventwatcher.go

@@ -88,7 +88,9 @@ func (srv *EventWatcher) Run(ctx context.Context) error {
 		if err == nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
 			return err
 		}
-		if status.Code(err) == codes.Canceled {
+		if status.Code(err) == codes.Canceled || status.Code(err) == codes.PermissionDenied {
+			// Permission denials are permanent for the lifetime of a test (the caller's
+			// identity doesn't change mid-run), so retrying would only waste the backoff budget.
 			return err
 		}