Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 14 additions & 4 deletions internal/scheduler/metrics/cycle_metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ var (
poolAndOutcomeLabels = []string{poolLabel, outcomeLabel, terminationReasonLabel}
nodeLabels = []string{poolLabel, nodeLabel, clusterLabel, nodeTypeLabel, resourceLabel, reservationLabel, schedulableLabel, overAllocatedLabel, physicalPoolLabel, capacityClassLabel, scalableUnitLabel}
defaultType = "unknown"
homePlacementType = "home"
awayPlacementType = "away"
reconcilerFailureType = "reconciler"
)

Expand Down Expand Up @@ -398,7 +400,7 @@ func newCycleMetrics(publisher pulsarutils.Publisher[*metricevents.Event], scala
Name: ArmadaSchedulerMetricsPrefix + "scheduled_jobs",
Help: "Number of events scheduled",
},
poolAndQueueAndPriorityClassTypeLabels,
append(poolAndQueueAndPriorityClassTypeLabels, "placement_type"),
)

preemptedJobs := prometheus.NewCounterVec(
Expand Down Expand Up @@ -695,10 +697,18 @@ func (m *cycleMetrics) ReportSchedulerResult(ctx *armadacontext.Context, result

for _, jobCtx := range schedulingResult.ScheduledJobs {
schedulingType := defaultType
if jobCtx.PodSchedulingContext != nil && jobCtx.PodSchedulingContext.SchedulingMethod != "" {
schedulingType = string(jobCtx.PodSchedulingContext.SchedulingMethod)
placementType := defaultType
if jobCtx.PodSchedulingContext != nil {
if jobCtx.PodSchedulingContext.SchedulingMethod != "" {
schedulingType = string(jobCtx.PodSchedulingContext.SchedulingMethod)
}
placementType = homePlacementType
if jobCtx.PodSchedulingContext.ScheduledAway {
placementType = awayPlacementType
}
}
m.scheduledJobs.WithLabelValues(pool, jobCtx.Job.Queue(), jobCtx.Job.PriorityClassName(), schedulingType).Inc()

m.scheduledJobs.WithLabelValues(pool, jobCtx.Job.Queue(), jobCtx.Job.PriorityClassName(), schedulingType, placementType).Inc()
}

for _, jobCtx := range schedulingResult.PreemptedJobs {
Expand Down
4 changes: 2 additions & 2 deletions internal/scheduler/metrics/cycle_metrics_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@ func TestResetLeaderMetrics_Counters(t *testing.T) {
assert.Equal(t, 0.0, counterVal)
}

testResetCounter(m.scheduledJobs, poolAndQueueAndPriorityClassTypeLabels)
testResetCounter(m.scheduledJobs, append(poolAndQueueAndPriorityClassTypeLabels, homePlacementType))
testResetCounter(m.preemptedJobs, poolAndQueueAndPriorityClassTypeLabels)
testResetCounter(m.failedJobs, poolAndQueueAndPriorityClassTypeLabels)
}
Expand Down Expand Up @@ -234,7 +234,7 @@ func TestDisableLeaderMetrics(t *testing.T) {
poolAndQueueAndPriorityClassTypeLabels := []string{"pool1", "queue1", "priorityClass1", "type1"}

collect := func(m *cycleMetrics) []prometheus.Metric {
m.scheduledJobs.WithLabelValues(poolAndQueueAndPriorityClassTypeLabels...).Inc()
m.scheduledJobs.WithLabelValues(append(poolAndQueueAndPriorityClassTypeLabels, homePlacementType)...).Inc()
m.preemptedJobs.WithLabelValues(poolAndQueueAndPriorityClassTypeLabels...).Inc()
m.latestCycleMetrics.Load().consideredJobs.WithLabelValues(poolQueueLabelValues...).Inc()
m.latestCycleMetrics.Load().fairShare.WithLabelValues(poolQueueLabelValues...).Inc()
Expand Down
1 change: 0 additions & 1 deletion internal/scheduler/nodedb/nodedb.go
Original file line number Diff line number Diff line change
Expand Up @@ -619,7 +619,6 @@ func (nodeDb *NodeDb) SelectNodeForJobWithTxn(txn *memdb.Txn, jctx *context.JobS
}
if node != nil {
pctx.WellKnownNodeTypeName = awayNodeType.WellKnownNodeTypeName
pctx.SchedulingMethod = context.ScheduledAsAwayJob
pctx.ScheduledAway = true
return node, preemptedJobs, nil
}
Expand Down
11 changes: 7 additions & 4 deletions internal/scheduler/nodedb/nodedb_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -909,11 +909,12 @@ func TestHomeNodeScheduling(t *testing.T) {
if tc.defaultToleration != nil {
assert.Equal(t, *tc.defaultToleration, jctx.AdditionalTolerations[0])
}
assert.Equal(t, context.ScheduledWithoutPreemption, jctx.PodSchedulingContext.SchedulingMethod)
if tc.expectScheduledAway {
assert.Equal(t, context.ScheduledAsAwayJob, jctx.PodSchedulingContext.SchedulingMethod)
assert.True(t, jctx.PodSchedulingContext.ScheduledAway)
assert.Equal(t, int32(29000), jctx.PodSchedulingContext.ScheduledAtPriority)
} else {
assert.Equal(t, context.ScheduledWithoutPreemption, jctx.PodSchedulingContext.SchedulingMethod)
assert.False(t, jctx.PodSchedulingContext.ScheduledAway)
assert.Equal(t, int32(30000), jctx.PodSchedulingContext.ScheduledAtPriority)
}
} else {
Expand Down Expand Up @@ -1282,7 +1283,8 @@ func TestConditionalAwayNodeScheduling(t *testing.T) {
if tc.expectScheduled {
assert.NotNil(t, node)
assert.Equal(t, node.GetId(), jctx.PodSchedulingContext.NodeId)
assert.Equal(t, context.ScheduledAsAwayJob, jctx.PodSchedulingContext.SchedulingMethod)
assert.True(t, jctx.PodSchedulingContext.ScheduledAway)
assert.Equal(t, context.ScheduledWithoutPreemption, jctx.PodSchedulingContext.SchedulingMethod)
} else {
assert.Nil(t, node)
}
Expand Down Expand Up @@ -1405,7 +1407,8 @@ func TestAwayNodeScheduling(t *testing.T) {
assert.NotNil(t, jctx.PodSchedulingContext)
assert.True(t, jctx.PodSchedulingContext.IsSuccessful())
assert.Equal(t, node.GetId(), jctx.PodSchedulingContext.NodeId)
assert.Equal(t, context.ScheduledAsAwayJob, jctx.PodSchedulingContext.SchedulingMethod)
assert.True(t, jctx.PodSchedulingContext.ScheduledAway)
assert.Equal(t, context.ScheduledWithoutPreemption, jctx.PodSchedulingContext.SchedulingMethod)
assert.Equal(t, int32(29000), jctx.PodSchedulingContext.ScheduledAtPriority)
if tc.wellKnownNodeTypeTaint.Value == schedulerconfig.WildCardWellKnownNodeTypeValue {
require.Equal(
Expand Down
1 change: 0 additions & 1 deletion internal/scheduler/scheduling/context/pod.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ const (
ScheduledWithoutPreemption SchedulingType = "no-preemption"
ScheduledWithFairSharePreemption SchedulingType = "fairshare"
ScheduledWithUrgencyBasedPreemption SchedulingType = "urgency"
ScheduledAsAwayJob SchedulingType = "away"
ScheduledWithFairnessOptimiser SchedulingType = "optimiser"
)

Expand Down
Loading