From 842a40566401bd9bc74f465797c10e893634e1e9 Mon Sep 17 00:00:00 2001 From: Thibault NORMAND Date: Fri, 5 Jun 2026 09:24:33 +0200 Subject: [PATCH] feat(disk): add IOPS limiters. --- api/v1beta1/disk_pressure.go | 42 ++++- api/v1beta1/disk_pressure_test.go | 154 ++++++++++++++++++ api/v1beta1/zz_generated.deepcopy.go | 10 ++ .../chaos.datadoghq.com_disruptioncrons.yaml | 8 + ...haos.datadoghq.com_disruptionrollouts.yaml | 8 + .../chaos.datadoghq.com_disruptions.yaml | 8 + cli/chaosli/chaosli.DOCKERFILE | 5 +- cli/chaosli/cmd/create.go | 10 ++ cli/injector/disk_pressure.go | 16 ++ docs/disk_pressure.md | 12 +- docs/disruption_catalogue.md | 4 +- examples/complete.yaml | 2 + examples/disk_pressure_read_iops.yaml | 19 +++ injector/disk_pressure.go | 78 +++++++-- injector/disk_pressure_test.go | 12 ++ o11y/tags/tags.go | 1 + 16 files changed, 362 insertions(+), 27 deletions(-) create mode 100644 api/v1beta1/disk_pressure_test.go create mode 100644 examples/disk_pressure_read_iops.yaml diff --git a/api/v1beta1/disk_pressure.go b/api/v1beta1/disk_pressure.go index bac3612047..9e43436e0d 100644 --- a/api/v1beta1/disk_pressure.go +++ b/api/v1beta1/disk_pressure.go @@ -20,12 +20,34 @@ type DiskPressureSpec struct { // DiskPressureThrottlingSpec represents a throttle on read and write disk operations type DiskPressureThrottlingSpec struct { - ReadBytesPerSec *int `json:"readBytesPerSec,omitempty"` + // +kubebuilder:validation:Minimum=0 + ReadBytesPerSec *int `json:"readBytesPerSec,omitempty"` + // +kubebuilder:validation:Minimum=0 WriteBytesPerSec *int `json:"writeBytesPerSec,omitempty"` + // +kubebuilder:validation:Minimum=0 + ReadIOPSPerSec *int `json:"readIOPSPerSec,omitempty"` + // +kubebuilder:validation:Minimum=0 + WriteIOPSPerSec *int `json:"writeIOPSPerSec,omitempty"` } // Validate validates args for the given disruption func (s *DiskPressureSpec) Validate() error { + // a negative throttle value makes no sense and is rejected by the cgroup write. + // zero is allowed and means "no throttle" (it removes the limit on cgroups v1 and + // maps to "max" on cgroups v2). Reject negatives at the API level to fail fast. + throttles := map[string]*int{ + "readBytesPerSec": s.Throttling.ReadBytesPerSec, + "writeBytesPerSec": s.Throttling.WriteBytesPerSec, + "readIOPSPerSec": s.Throttling.ReadIOPSPerSec, + "writeIOPSPerSec": s.Throttling.WriteIOPSPerSec, + } + + for name, value := range throttles { + if value != nil && *value < 0 { + return fmt.Errorf("disk pressure throttling %s must be greater than or equal to 0, got %d", name, *value) + } + } + return nil } @@ -47,6 +69,16 @@ func (s *DiskPressureSpec) GenerateArgs() []string { args = append(args, []string{"--write-bytes-per-sec", strconv.Itoa(*s.Throttling.WriteBytesPerSec)}...) } + // add read iops throttling flag if specified + if s.Throttling.ReadIOPSPerSec != nil { + args = append(args, []string{"--read-iops-per-sec", strconv.Itoa(*s.Throttling.ReadIOPSPerSec)}...) + } + + // add write iops throttling flag if specified + if s.Throttling.WriteIOPSPerSec != nil { + args = append(args, []string{"--write-iops-per-sec", strconv.Itoa(*s.Throttling.WriteIOPSPerSec)}...) + } + return args } @@ -61,5 +93,13 @@ func (s *DiskPressureSpec) Explain() []string { explanation += fmt.Sprintf("%d write bytes per second.", *s.Throttling.WriteBytesPerSec) } + if s.Throttling.ReadIOPSPerSec != nil { + explanation += fmt.Sprintf("%d read io per second ", *s.Throttling.ReadIOPSPerSec) + } + + if s.Throttling.WriteIOPSPerSec != nil { + explanation += fmt.Sprintf("%d write io per second.", *s.Throttling.WriteIOPSPerSec) + } + return []string{"", explanation} } diff --git a/api/v1beta1/disk_pressure_test.go b/api/v1beta1/disk_pressure_test.go new file mode 100644 index 0000000000..41b87c4248 --- /dev/null +++ b/api/v1beta1/disk_pressure_test.go @@ -0,0 +1,154 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package v1beta1_test + +import ( + . "github.com/DataDog/chaos-controller/api/v1beta1" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("DiskPressureSpec", func() { + intPtr := func(i int) *int { return &i } + + When("Call the 'GenerateArgs' method", func() { + DescribeTable("argument generation", + func(spec DiskPressureSpec, expected []string) { + Expect(spec.GenerateArgs()).To(Equal(expected)) + }, + Entry("with only bandwidth throttling (back-compat)", + DiskPressureSpec{ + Path: "/mnt/data", + Throttling: DiskPressureThrottlingSpec{ + ReadBytesPerSec: intPtr(1024), + WriteBytesPerSec: intPtr(4096), + }, + }, + []string{ + "disk-pressure", "--path", "/mnt/data", + "--read-bytes-per-sec", "1024", + "--write-bytes-per-sec", "4096", + }, + ), + Entry("with only read iops throttling", + DiskPressureSpec{ + Path: "/mnt/data", + Throttling: DiskPressureThrottlingSpec{ + ReadIOPSPerSec: intPtr(50), + }, + }, + []string{ + "disk-pressure", "--path", "/mnt/data", + "--read-iops-per-sec", "50", + }, + ), + Entry("with only write iops throttling", + DiskPressureSpec{ + Path: "/mnt/data", + Throttling: DiskPressureThrottlingSpec{ + WriteIOPSPerSec: intPtr(75), + }, + }, + []string{ + "disk-pressure", "--path", "/mnt/data", + "--write-iops-per-sec", "75", + }, + ), + Entry("with bandwidth and iops throttling combined", + DiskPressureSpec{ + Path: "/mnt/data", + Throttling: DiskPressureThrottlingSpec{ + ReadBytesPerSec: intPtr(1024), + WriteBytesPerSec: intPtr(4096), + ReadIOPSPerSec: intPtr(50), + WriteIOPSPerSec: intPtr(75), + }, + }, + []string{ + "disk-pressure", "--path", "/mnt/data", + "--read-bytes-per-sec", "1024", + "--write-bytes-per-sec", "4096", + "--read-iops-per-sec", "50", + "--write-iops-per-sec", "75", + }, + ), + Entry("with no throttling set", + DiskPressureSpec{Path: "/mnt/data"}, + []string{"disk-pressure", "--path", "/mnt/data"}, + ), + ) + }) + + When("Call the 'Validate' method", func() { + It("accepts a spec with no throttling set", func() { + spec := DiskPressureSpec{Path: "/mnt/data"} + Expect(spec.Validate()).To(Succeed()) + }) + + It("accepts positive throttle values", func() { + spec := DiskPressureSpec{ + Path: "/mnt/data", + Throttling: DiskPressureThrottlingSpec{ + ReadBytesPerSec: intPtr(1024), + ReadIOPSPerSec: intPtr(50), + }, + } + Expect(spec.Validate()).To(Succeed()) + }) + + It("accepts a zero throttle value (no-op, removes the limit)", func() { + spec := DiskPressureSpec{ + Path: "/mnt/data", + Throttling: DiskPressureThrottlingSpec{ + ReadBytesPerSec: intPtr(0), + WriteIOPSPerSec: intPtr(0), + }, + } + Expect(spec.Validate()).To(Succeed()) + }) + + DescribeTable("rejects negative throttle values", + func(spec DiskPressureSpec, field string) { + err := spec.Validate() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(field)) + }, + Entry("negative read bytes", + DiskPressureSpec{Path: "/mnt/data", Throttling: DiskPressureThrottlingSpec{ReadBytesPerSec: intPtr(-1)}}, + "readBytesPerSec", + ), + Entry("negative write bytes", + DiskPressureSpec{Path: "/mnt/data", Throttling: DiskPressureThrottlingSpec{WriteBytesPerSec: intPtr(-1)}}, + "writeBytesPerSec", + ), + Entry("negative read iops", + DiskPressureSpec{Path: "/mnt/data", Throttling: DiskPressureThrottlingSpec{ReadIOPSPerSec: intPtr(-1)}}, + "readIOPSPerSec", + ), + Entry("negative write iops", + DiskPressureSpec{Path: "/mnt/data", Throttling: DiskPressureThrottlingSpec{WriteIOPSPerSec: intPtr(-5)}}, + "writeIOPSPerSec", + ), + ) + }) + + When("Call the 'Explain' method", func() { + It("mentions iops throttling when set", func() { + spec := DiskPressureSpec{ + Path: "/mnt/data", + Throttling: DiskPressureThrottlingSpec{ + ReadIOPSPerSec: intPtr(50), + WriteIOPSPerSec: intPtr(75), + }, + } + + explanation := spec.Explain() + + Expect(explanation).To(ContainElement(ContainSubstring("50 read io per second"))) + Expect(explanation).To(ContainElement(ContainSubstring("75 write io per second"))) + }) + }) +}) diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index c9271bd209..01ecbfc02f 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -212,6 +212,16 @@ func (in *DiskPressureThrottlingSpec) DeepCopyInto(out *DiskPressureThrottlingSp *out = new(int) **out = **in } + if in.ReadIOPSPerSec != nil { + in, out := &in.ReadIOPSPerSec, &out.ReadIOPSPerSec + *out = new(int) + **out = **in + } + if in.WriteIOPSPerSec != nil { + in, out := &in.WriteIOPSPerSec, &out.WriteIOPSPerSec + *out = new(int) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DiskPressureThrottlingSpec. diff --git a/chart/templates/generated/chaos.datadoghq.com_disruptioncrons.yaml b/chart/templates/generated/chaos.datadoghq.com_disruptioncrons.yaml index 14af5850d5..90de858825 100644 --- a/chart/templates/generated/chaos.datadoghq.com_disruptioncrons.yaml +++ b/chart/templates/generated/chaos.datadoghq.com_disruptioncrons.yaml @@ -172,8 +172,16 @@ spec: description: DiskPressureThrottlingSpec represents a throttle on read and write disk operations properties: readBytesPerSec: + minimum: 0 + type: integer + readIOPSPerSec: + minimum: 0 type: integer writeBytesPerSec: + minimum: 0 + type: integer + writeIOPSPerSec: + minimum: 0 type: integer type: object required: diff --git a/chart/templates/generated/chaos.datadoghq.com_disruptionrollouts.yaml b/chart/templates/generated/chaos.datadoghq.com_disruptionrollouts.yaml index e9f636d1f6..1227a86be9 100644 --- a/chart/templates/generated/chaos.datadoghq.com_disruptionrollouts.yaml +++ b/chart/templates/generated/chaos.datadoghq.com_disruptionrollouts.yaml @@ -173,8 +173,16 @@ spec: description: DiskPressureThrottlingSpec represents a throttle on read and write disk operations properties: readBytesPerSec: + minimum: 0 + type: integer + readIOPSPerSec: + minimum: 0 type: integer writeBytesPerSec: + minimum: 0 + type: integer + writeIOPSPerSec: + minimum: 0 type: integer type: object required: diff --git a/chart/templates/generated/chaos.datadoghq.com_disruptions.yaml b/chart/templates/generated/chaos.datadoghq.com_disruptions.yaml index fa6b4be1e3..182c33ca25 100644 --- a/chart/templates/generated/chaos.datadoghq.com_disruptions.yaml +++ b/chart/templates/generated/chaos.datadoghq.com_disruptions.yaml @@ -163,8 +163,16 @@ spec: description: DiskPressureThrottlingSpec represents a throttle on read and write disk operations properties: readBytesPerSec: + minimum: 0 + type: integer + readIOPSPerSec: + minimum: 0 type: integer writeBytesPerSec: + minimum: 0 + type: integer + writeIOPSPerSec: + minimum: 0 type: integer type: object required: diff --git a/cli/chaosli/chaosli.DOCKERFILE b/cli/chaosli/chaosli.DOCKERFILE index 9e2e60bccd..9176b50d5b 100644 --- a/cli/chaosli/chaosli.DOCKERFILE +++ b/cli/chaosli/chaosli.DOCKERFILE @@ -1,7 +1,6 @@ # --------------------------------------- -FROM amd64/golang:1.20-alpine as build +FROM golang:1.25-alpine as build -ENV GOARCH=amd64 ENV CGO_ENABLED=0 WORKDIR /app @@ -16,7 +15,7 @@ RUN go build \ ./cli/chaosli # --------------------------------------- -FROM amd64/golang:1.20-alpine as bin +FROM golang:1.25-alpine as bin RUN addgroup -S appgroup && adduser -S appuser -G appgroup USER appuser diff --git a/cli/chaosli/cmd/create.go b/cli/chaosli/cmd/create.go index 79937c09a9..2083564156 100644 --- a/cli/chaosli/cmd/create.go +++ b/cli/chaosli/cmd/create.go @@ -372,6 +372,16 @@ func getDiskPressure() *v1beta1.DiskPressureSpec { spec.Throttling.WriteBytesPerSec = &writeBPS } + if confirmOption("Would you like to apply read iops throttling?", "This applies read-based IO operations throttling (check the docs)") { + readIOPS, _ := strconv.Atoi(getInput("Specify the target amount of throttling, in io operations per second.", "check the docs", survey.WithValidator(integerValidator))) + spec.Throttling.ReadIOPSPerSec = &readIOPS + } + + if confirmOption("Would you like to apply write iops throttling?", "This applies write-based IO operations throttling (check the docs)") { + writeIOPS, _ := strconv.Atoi(getInput("Specify the target amount of throttling, in io operations per second.", "check the docs", survey.WithValidator(integerValidator))) + spec.Throttling.WriteIOPSPerSec = &writeIOPS + } + return spec } diff --git a/cli/injector/disk_pressure.go b/cli/injector/disk_pressure.go index 3fbc4e0a57..418b5e9b90 100644 --- a/cli/injector/disk_pressure.go +++ b/cli/injector/disk_pressure.go @@ -25,6 +25,8 @@ var diskPressureCmd = &cobra.Command{ path, _ := cmd.Flags().GetString("path") writeBytesPerSec, _ := cmd.Flags().GetInt("write-bytes-per-sec") readBytesPerSec, _ := cmd.Flags().GetInt("read-bytes-per-sec") + writeIOPSPerSec, _ := cmd.Flags().GetInt("write-iops-per-sec") + readIOPSPerSec, _ := cmd.Flags().GetInt("read-iops-per-sec") // prepare spec var writeBytesPerSecP *int @@ -37,11 +39,23 @@ var diskPressureCmd = &cobra.Command{ readBytesPerSecP = &readBytesPerSec } + var writeIOPSPerSecP *int + if writeIOPSPerSec != 0 { + writeIOPSPerSecP = &writeIOPSPerSec + } + + var readIOPSPerSecP *int + if readIOPSPerSec != 0 { + readIOPSPerSecP = &readIOPSPerSec + } + spec := v1beta1.DiskPressureSpec{ Path: path, Throttling: v1beta1.DiskPressureThrottlingSpec{ ReadBytesPerSec: readBytesPerSecP, WriteBytesPerSec: writeBytesPerSecP, + ReadIOPSPerSec: readIOPSPerSecP, + WriteIOPSPerSec: writeIOPSPerSecP, }, } @@ -72,6 +86,8 @@ func init() { diskPressureCmd.Flags().String("path", "", "Path to apply/clean disk pressure to/from (will be applied to the whole disk)") diskPressureCmd.Flags().Int("write-bytes-per-sec", 0, "Bytes per second throttling limit") diskPressureCmd.Flags().Int("read-bytes-per-sec", 0, "Bytes per second throttling limit") + diskPressureCmd.Flags().Int("write-iops-per-sec", 0, "IO operations per second throttling limit") + diskPressureCmd.Flags().Int("read-iops-per-sec", 0, "IO operations per second throttling limit") _ = cobra.MarkFlagRequired(diskPressureCmd.PersistentFlags(), "path") } diff --git a/docs/disk_pressure.md b/docs/disk_pressure.md index 91698013c0..0153724c10 100644 --- a/docs/disk_pressure.md +++ b/docs/disk_pressure.md @@ -4,11 +4,13 @@ The `diskPressure` field offers a way to apply IO throttling on a specific mount ## Throttling -Unlike the CPU pressure, this kind of disruption is not done by stressing the disk but by throttling its capacities. A throttle can be applied on read or write operations, or both. +Unlike the CPU pressure, this kind of disruption is not done by stressing the disk but by throttling its capacities. A throttle can be applied on read or write operations, or both. Two units are supported and can be combined: +* **bandwidth** — bytes per second (`readBytesPerSec` / `writeBytesPerSec`) +* **IOPS** — IO operations per second (`readIOPSPerSec` / `writeIOPSPerSec`) The throttling is done by using the [blkio cgroup controller](https://www.kernel.org/doc/Documentation/cgroup-v1/blkio-controller.txt), and more specifically: -* by the `blkio.throttle.read_bps_device` and `blkio.throttle.write_bps_device` files for cgroup v1 -* by the `io.max` file for cgroup v2 ([more to read here](https://docs.kernel.org/admin-guide/cgroup-v2.html#io-interface-files)) +* for cgroup v1, by the `blkio.throttle.read_bps_device` / `blkio.throttle.write_bps_device` files (bandwidth) and the `blkio.throttle.read_iops_device` / `blkio.throttle.write_iops_device` files (IOPS) +* by the `io.max` file for cgroup v2 — keys `rbps` / `wbps` (bandwidth) and `riops` / `wiops` (IOPS) ([more to read here](https://docs.kernel.org/admin-guide/cgroup-v2.html#io-interface-files)) To apply the throttle, the injector will: @@ -79,6 +81,8 @@ kubectl get -ojson pod demo-curl-547bb9c686-57484 | jq '.status.containerStatuse # echo "8:0 0" > /sys/fs/cgroup/blkio/kubepods/burstable/poda37541dc-4905-4a7f-98c0-7d13f58df0eb/cb33d4ce77f7396851196043a56e625f38429720cd5d3153cb061feae6038460/blkio.throttle.write_bps_device ``` +*If IOPS throttling was applied, reset `blkio.throttle.read_iops_device` and `blkio.throttle.write_iops_device` the same way (`echo "8:0 0" > ...`).* + * Ensure that the values are reset ``` @@ -95,7 +99,7 @@ kubectl get -ojson pod demo-curl-547bb9c686-57484 | jq '.status.containerStatuse * Reset throttle values for the found device ``` -# echo "8:0 rbps=max wbps=max" > /sys/fs/cgroup/kubepods/burstable/poda37541dc-4905-4a7f-98c0-7d13f58df0eb/cb33d4ce77f7396851196043a56e625f38429720cd5d3153cb061feae6038460/io.max +# echo "8:0 rbps=max wbps=max riops=max wiops=max" > /sys/fs/cgroup/kubepods/burstable/poda37541dc-4905-4a7f-98c0-7d13f58df0eb/cb33d4ce77f7396851196043a56e625f38429720cd5d3153cb061feae6038460/io.max ``` * Ensure that the values are reset diff --git a/docs/disruption_catalogue.md b/docs/disruption_catalogue.md index 547fed4c80..c1c2e11282 100644 --- a/docs/disruption_catalogue.md +++ b/docs/disruption_catalogue.md @@ -684,8 +684,10 @@ Throttles I/O throughput on the block device backing a given path using cgroup b | `path` | string | Mount point inside the container (required) | | `throttling.readBytesPerSec` | int | Read throughput limit in bytes/sec | | `throttling.writeBytesPerSec` | int | Write throughput limit in bytes/sec | +| `throttling.readIOPSPerSec` | int | Read limit in IO operations/sec | +| `throttling.writeIOPSPerSec` | int | Write limit in IO operations/sec | -At least one of `readBytesPerSec` or `writeBytesPerSec` is required. +At least one of `readBytesPerSec`, `writeBytesPerSec`, `readIOPSPerSec` or `writeIOPSPerSec` is required. ### Constraints and Limitations diff --git a/examples/complete.yaml b/examples/complete.yaml index b51c3295df..885f4f41c4 100644 --- a/examples/complete.yaml +++ b/examples/complete.yaml @@ -111,6 +111,8 @@ spec: throttling: readBytesPerSec: 1024 # optional, read throttling in bytes per sec writeBytesPerSec: 2048 # optional, write throttling in bytes per sec + readIOPSPerSec: 50 # optional, read throttling in io operations per sec + writeIOPSPerSec: 75 # optional, write throttling in io operations per sec grpc: # disrupt gRPC responses by faking results port: 50051 # port that target grpc server is listening on endpoints: diff --git a/examples/disk_pressure_read_iops.yaml b/examples/disk_pressure_read_iops.yaml new file mode 100644 index 0000000000..b06b068e48 --- /dev/null +++ b/examples/disk_pressure_read_iops.yaml @@ -0,0 +1,19 @@ +# Unless explicitly stated otherwise all files in this repository are licensed +# under the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2026 Datadog, Inc. + +apiVersion: chaos.datadoghq.com/v1beta1 +kind: Disruption +metadata: + name: disk-pressure-read-iops + namespace: chaos-demo +spec: + level: pod + selector: + service: demo-curl + count: 1 + diskPressure: + path: /mnt/data # mount point (in the pod) to apply throttle on + throttling: + readIOPSPerSec: 50 # read throttling in io operations per sec diff --git a/injector/disk_pressure.go b/injector/disk_pressure.go index aeefe618d7..4746e4d586 100644 --- a/injector/disk_pressure.go +++ b/injector/disk_pressure.go @@ -33,8 +33,10 @@ type DiskPressureInjectorConfig struct { type diskPressureThrottleMode int const ( - diskPressureThrottleModeRead diskPressureThrottleMode = iota - diskPressureThrottleModeWrite + diskPressureThrottleModeReadBps diskPressureThrottleMode = iota + diskPressureThrottleModeWriteBps + diskPressureThrottleModeReadIops + diskPressureThrottleModeWriteIops ) const diskPressureBlkioControllerName = "blkio" @@ -89,24 +91,42 @@ func (i *diskPressureInjector) GetDisruptionKind() types.DisruptionKindName { } func (i *diskPressureInjector) Inject() error { - // add read throttle + // add read bytes-per-second throttle if i.spec.Throttling.ReadBytesPerSec != nil { - if err := i.config.Cgroup.Write(diskPressureBlkioControllerName, i.getThrottleFilename(diskPressureThrottleModeRead), i.formatThrottle(*i.spec.Throttling.ReadBytesPerSec, diskPressureThrottleModeRead)); err != nil { + if err := i.config.Cgroup.Write(diskPressureBlkioControllerName, i.getThrottleFilename(diskPressureThrottleModeReadBps), i.formatThrottle(*i.spec.Throttling.ReadBytesPerSec, diskPressureThrottleModeReadBps)); err != nil { return fmt.Errorf("error throttling disk read: %w", err) } i.config.Log.Infow("read throttling injected", tags.DeviceKey, i.config.Informer.Source(), tags.BpsKey, *i.spec.Throttling.ReadBytesPerSec) } - // add write throttle + // add write bytes-per-second throttle if i.spec.Throttling.WriteBytesPerSec != nil { - if err := i.config.Cgroup.Write(diskPressureBlkioControllerName, i.getThrottleFilename(diskPressureThrottleModeWrite), i.formatThrottle(*i.spec.Throttling.WriteBytesPerSec, diskPressureThrottleModeWrite)); err != nil { + if err := i.config.Cgroup.Write(diskPressureBlkioControllerName, i.getThrottleFilename(diskPressureThrottleModeWriteBps), i.formatThrottle(*i.spec.Throttling.WriteBytesPerSec, diskPressureThrottleModeWriteBps)); err != nil { return fmt.Errorf("error throttling disk write: %w", err) } i.config.Log.Infow("write throttling injected", tags.DeviceKey, i.config.Informer.Source(), tags.BpsKey, *i.spec.Throttling.WriteBytesPerSec) } + // add read iops throttle + if i.spec.Throttling.ReadIOPSPerSec != nil { + if err := i.config.Cgroup.Write(diskPressureBlkioControllerName, i.getThrottleFilename(diskPressureThrottleModeReadIops), i.formatThrottle(*i.spec.Throttling.ReadIOPSPerSec, diskPressureThrottleModeReadIops)); err != nil { + return fmt.Errorf("error throttling disk read iops: %w", err) + } + + i.config.Log.Infow("read iops throttling injected", tags.DeviceKey, i.config.Informer.Source(), tags.IopsKey, *i.spec.Throttling.ReadIOPSPerSec) + } + + // add write iops throttle + if i.spec.Throttling.WriteIOPSPerSec != nil { + if err := i.config.Cgroup.Write(diskPressureBlkioControllerName, i.getThrottleFilename(diskPressureThrottleModeWriteIops), i.formatThrottle(*i.spec.Throttling.WriteIOPSPerSec, diskPressureThrottleModeWriteIops)); err != nil { + return fmt.Errorf("error throttling disk write iops: %w", err) + } + + i.config.Log.Infow("write iops throttling injected", tags.DeviceKey, i.config.Informer.Source(), tags.IopsKey, *i.spec.Throttling.WriteIOPSPerSec) + } + return nil } @@ -115,20 +135,34 @@ func (i *diskPressureInjector) UpdateConfig(config Config) { } func (i *diskPressureInjector) Clean() error { - // clean read throttle + // clean read bytes-per-second throttle i.config.Log.Infow("cleaning disk read throttle", tags.DeviceKey, i.config.Informer.Source()) - if err := i.config.Cgroup.Write(diskPressureBlkioControllerName, i.getThrottleFilename(diskPressureThrottleModeRead), i.formatThrottle(0, diskPressureThrottleModeRead)); err != nil { + if err := i.config.Cgroup.Write(diskPressureBlkioControllerName, i.getThrottleFilename(diskPressureThrottleModeReadBps), i.formatThrottle(0, diskPressureThrottleModeReadBps)); err != nil { return fmt.Errorf("error cleaning read disk throttle: %w", err) } - // clean write throttle + // clean write bytes-per-second throttle i.config.Log.Infow("cleaning disk write throttle", tags.DeviceKey, i.config.Informer.Source()) - if err := i.config.Cgroup.Write(diskPressureBlkioControllerName, i.getThrottleFilename(diskPressureThrottleModeWrite), i.formatThrottle(0, diskPressureThrottleModeWrite)); err != nil { + if err := i.config.Cgroup.Write(diskPressureBlkioControllerName, i.getThrottleFilename(diskPressureThrottleModeWriteBps), i.formatThrottle(0, diskPressureThrottleModeWriteBps)); err != nil { return fmt.Errorf("error cleaning write disk throttle: %w", err) } + // clean read iops throttle + i.config.Log.Infow("cleaning disk read iops throttle", tags.DeviceKey, i.config.Informer.Source()) + + if err := i.config.Cgroup.Write(diskPressureBlkioControllerName, i.getThrottleFilename(diskPressureThrottleModeReadIops), i.formatThrottle(0, diskPressureThrottleModeReadIops)); err != nil { + return fmt.Errorf("error cleaning read disk iops throttle: %w", err) + } + + // clean write iops throttle + i.config.Log.Infow("cleaning disk write iops throttle", tags.DeviceKey, i.config.Informer.Source()) + + if err := i.config.Cgroup.Write(diskPressureBlkioControllerName, i.getThrottleFilename(diskPressureThrottleModeWriteIops), i.formatThrottle(0, diskPressureThrottleModeWriteIops)); err != nil { + return fmt.Errorf("error cleaning write disk iops throttle: %w", err) + } + return nil } @@ -148,18 +182,22 @@ func (i *diskPressureInjector) formatThrottle(throttle int, mode diskPressureThr } // the file can be used to configure both read and write throttling (both iops and bps too) - // to set that value, it is now a key/value pair (rbps for read throttling, wbps for write throttling) + // to set that value, it is now a key/value pair (rbps/wbps for bandwidth, riops/wiops for iops) switch mode { - case diskPressureThrottleModeRead: + case diskPressureThrottleModeReadBps: return fmt.Sprintf("%d:0 rbps=%s", i.config.Informer.Major(), sThrottle) - case diskPressureThrottleModeWrite: + case diskPressureThrottleModeWriteBps: return fmt.Sprintf("%d:0 wbps=%s", i.config.Informer.Major(), sThrottle) + case diskPressureThrottleModeReadIops: + return fmt.Sprintf("%d:0 riops=%s", i.config.Informer.Major(), sThrottle) + case diskPressureThrottleModeWriteIops: + return fmt.Sprintf("%d:0 wiops=%s", i.config.Informer.Major(), sThrottle) default: return "" // should never be used } } - // cgroups v1 throttling format is much simple and only takes the bps value + // cgroups v1 throttling format is much simple and only takes the value // example: 252:0 1024 return fmt.Sprintf("%d:0 %d", i.config.Informer.Major(), throttle) } @@ -176,13 +214,17 @@ func (i *diskPressureInjector) getThrottleFilename(mode diskPressureThrottleMode // cgroups v1 uses separate files for both the mode and the unit // - read and bps // - write and bps - // - read and iops (unused here) - // - write and iops (unused here) + // - read and iops + // - write and iops switch mode { - case diskPressureThrottleModeRead: + case diskPressureThrottleModeReadBps: return "blkio.throttle.read_bps_device" - case diskPressureThrottleModeWrite: + case diskPressureThrottleModeWriteBps: return "blkio.throttle.write_bps_device" + case diskPressureThrottleModeReadIops: + return "blkio.throttle.read_iops_device" + case diskPressureThrottleModeWriteIops: + return "blkio.throttle.write_iops_device" } return "" // should never be used diff --git a/injector/disk_pressure_test.go b/injector/disk_pressure_test.go index 1fac3a896a..933319519e 100644 --- a/injector/disk_pressure_test.go +++ b/injector/disk_pressure_test.go @@ -60,10 +60,14 @@ var _ = Describe("Failure", func() { // spec read := 1024 write := 4096 + readIOPS := 50 + writeIOPS := 75 spec = v1beta1.DiskPressureSpec{ Throttling: v1beta1.DiskPressureThrottlingSpec{ ReadBytesPerSec: &read, WriteBytesPerSec: &write, + ReadIOPSPerSec: &readIOPS, + WriteIOPSPerSec: &writeIOPS, }, } }) @@ -91,6 +95,8 @@ var _ = Describe("Failure", func() { It("should throttle disk from cgroup", func() { cgroupManager.AssertCalled(GinkgoT(), "Write", "blkio", "blkio.throttle.read_bps_device", "8:0 1024") cgroupManager.AssertCalled(GinkgoT(), "Write", "blkio", "blkio.throttle.write_bps_device", "8:0 4096") + cgroupManager.AssertCalled(GinkgoT(), "Write", "blkio", "blkio.throttle.read_iops_device", "8:0 50") + cgroupManager.AssertCalled(GinkgoT(), "Write", "blkio", "blkio.throttle.write_iops_device", "8:0 75") }) }) @@ -102,6 +108,8 @@ var _ = Describe("Failure", func() { It("should throttle disk from cgroup", func() { cgroupManager.AssertCalled(GinkgoT(), "Write", "blkio", "io.max", "8:0 rbps=1024") cgroupManager.AssertCalled(GinkgoT(), "Write", "blkio", "io.max", "8:0 wbps=4096") + cgroupManager.AssertCalled(GinkgoT(), "Write", "blkio", "io.max", "8:0 riops=50") + cgroupManager.AssertCalled(GinkgoT(), "Write", "blkio", "io.max", "8:0 wiops=75") }) }) }) @@ -119,6 +127,8 @@ var _ = Describe("Failure", func() { It("should remove throttle from cgroup", func() { cgroupManager.AssertCalled(GinkgoT(), "Write", "blkio", "blkio.throttle.read_bps_device", "8:0 0") cgroupManager.AssertCalled(GinkgoT(), "Write", "blkio", "blkio.throttle.write_bps_device", "8:0 0") + cgroupManager.AssertCalled(GinkgoT(), "Write", "blkio", "blkio.throttle.read_iops_device", "8:0 0") + cgroupManager.AssertCalled(GinkgoT(), "Write", "blkio", "blkio.throttle.write_iops_device", "8:0 0") }) }) @@ -130,6 +140,8 @@ var _ = Describe("Failure", func() { It("should throttle disk from cgroup", func() { cgroupManager.AssertCalled(GinkgoT(), "Write", "blkio", "io.max", "8:0 rbps=max") cgroupManager.AssertCalled(GinkgoT(), "Write", "blkio", "io.max", "8:0 wbps=max") + cgroupManager.AssertCalled(GinkgoT(), "Write", "blkio", "io.max", "8:0 riops=max") + cgroupManager.AssertCalled(GinkgoT(), "Write", "blkio", "io.max", "8:0 wiops=max") }) }) }) diff --git a/o11y/tags/tags.go b/o11y/tags/tags.go index d3437733c8..1203a2184f 100644 --- a/o11y/tags/tags.go +++ b/o11y/tags/tags.go @@ -285,6 +285,7 @@ const ( // Metrics and counting AssignedCpusKey = "assigned_cpus" BpsKey = "bps" + IopsKey = "iops" CalculatedPercentOfTotalKey = "calculated_percent_of_total" ClusterThresholdKey = "cluster_threshold" CountKey = "count"