Skip to content

Commit 0b3ea3b

Browse files
authored
Merge pull request #500 from slashdevops/fix/retrier-lib
fix: replace httpretrier with httpx and apply Go 1.26 improvements
2 parents 8125aba + 44918ef commit 0b3ea3b

14 files changed

Lines changed: 217 additions & 245 deletions

File tree

README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,7 @@ sam deploy --parameter-overrides SyncUserFields=phoneNumbers,addresses,enterpris
191191
## ⚠️ Limitations
192192

193193
* **Group Limit**: The AWS SSO SCIM API has a limit of 50 groups per request. Please support the feature request on the [AWS Support site](https://repost.aws/questions/QUqqnVkIo_SYyF_SlX5LcUjg/aws-sso-scim-api-pagination-for-methods) to help get this limit increased.
194-
* **Throttling**: With a large number of users and groups, you may encounter a `ThrottlingException` from the AWS SSO SCIM API. This project uses a [retryable HTTP client](https://github.com/p2p-b2b/httpretrier) to mitigate this, but it's still a possibility.
194+
* **Throttling**: With a large number of users and groups, you may encounter a `ThrottlingException` from the AWS SSO SCIM API. This project uses the [httpx](https://github.com/slashdevops/httpx) library with automatic retry and jitter backoff to mitigate this, but it's still a possibility.
195195
* **User Status**: The Google Workspace API doesn't differentiate between normal and guest users except for their status. This project only syncs `ACTIVE` users.
196196

197197
## For `ssosync` Users

cmd/idpscim/cmd/root.go

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ import (
99
"time"
1010

1111
"github.com/aws/aws-lambda-go/lambda"
12-
"github.com/pkg/errors"
1312
"github.com/slashdevops/idp-scim-sync/internal/config"
1413
"github.com/slashdevops/idp-scim-sync/internal/setup"
1514
"github.com/slashdevops/idp-scim-sync/internal/version"
@@ -92,13 +91,13 @@ func run(ctx context.Context) error {
9291

9392
ss, err := setup.SyncService(ctx, &cfg)
9493
if err != nil {
95-
return errors.Wrap(err, "cannot create sync service")
94+
return fmt.Errorf("cannot create sync service: %w", err)
9695
}
9796

9897
slog.Debug("app config", "config", cfg)
9998

10099
if err := ss.SyncGroupsAndTheirMembers(ctx); err != nil {
101-
return errors.Wrap(err, "cannot sync groups and their members")
100+
return fmt.Errorf("cannot sync groups and their members: %w", err)
102101
}
103102

104103
slog.Info("sync groups completed", "duration", time.Since(timeStart).String())

cmd/idpscimcli/cmd/aws.go

Lines changed: 23 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,9 @@ package cmd
33
import (
44
"context"
55
"log/slog"
6-
"net/http"
76
"time"
87

9-
"github.com/p2p-b2b/httpretrier"
8+
"github.com/slashdevops/httpx"
109
"github.com/slashdevops/idp-scim-sync/internal/version"
1110
"github.com/slashdevops/idp-scim-sync/pkg/aws"
1211
"github.com/spf13/cobra"
@@ -92,11 +91,12 @@ func runAWSServiceConfig(_ *cobra.Command, _ []string) error {
9291
ctx, cancel := context.WithTimeout(context.Background(), reqTimeout)
9392
defer cancel()
9493

95-
httpRetryClient := httpretrier.NewClient(
96-
10, // Max Retries
97-
httpretrier.ExponentialBackoff(10*time.Millisecond, 500*time.Millisecond),
98-
nil, // Use http.DefaultTransport
99-
)
94+
httpRetryClient := httpx.NewClientBuilder().
95+
WithMaxRetries(10).
96+
WithRetryStrategy(httpx.JitterBackoffStrategy).
97+
WithRetryBaseDelay(500 * time.Millisecond).
98+
WithRetryMaxDelay(10 * time.Second).
99+
Build()
100100

101101
awsSCIMService, err := aws.NewSCIMService(httpRetryClient, cfg.AWSSCIMEndpoint, cfg.AWSSCIMAccessToken)
102102
if err != nil {
@@ -120,16 +120,14 @@ func runAWSGroupsList(_ *cobra.Command, _ []string) error {
120120
ctx, cancel := context.WithTimeout(context.Background(), reqTimeout)
121121
defer cancel()
122122

123-
httpTransport := http.DefaultTransport.(*http.Transport).Clone()
124-
httpTransport.MaxIdleConns = 100
125-
httpTransport.MaxConnsPerHost = 100
126-
httpTransport.MaxIdleConnsPerHost = 100
127-
128-
httpClient := httpretrier.NewClient(
129-
10, // Max Retries
130-
httpretrier.ExponentialBackoff(10*time.Millisecond, 500*time.Millisecond),
131-
httpTransport,
132-
)
123+
httpClient := httpx.NewClientBuilder().
124+
WithMaxRetries(10).
125+
WithRetryStrategy(httpx.JitterBackoffStrategy).
126+
WithRetryBaseDelay(500 * time.Millisecond).
127+
WithRetryMaxDelay(10 * time.Second).
128+
WithMaxIdleConns(100).
129+
WithMaxIdleConnsPerHost(100).
130+
Build()
133131

134132
awsSCIMService, err := aws.NewSCIMService(httpClient, cfg.AWSSCIMEndpoint, cfg.AWSSCIMAccessToken)
135133
if err != nil {
@@ -154,16 +152,14 @@ func runAWSUsersList(_ *cobra.Command, _ []string) error {
154152
ctx, cancel := context.WithTimeout(context.Background(), reqTimeout)
155153
defer cancel()
156154

157-
httpTransport := http.DefaultTransport.(*http.Transport).Clone()
158-
httpTransport.MaxIdleConns = 100
159-
httpTransport.MaxConnsPerHost = 100
160-
httpTransport.MaxIdleConnsPerHost = 100
161-
162-
httpClient := httpretrier.NewClient(
163-
10, // Max Retries
164-
httpretrier.ExponentialBackoff(10*time.Millisecond, 500*time.Millisecond),
165-
httpTransport,
166-
)
155+
httpClient := httpx.NewClientBuilder().
156+
WithMaxRetries(10).
157+
WithRetryStrategy(httpx.JitterBackoffStrategy).
158+
WithRetryBaseDelay(500 * time.Millisecond).
159+
WithRetryMaxDelay(10 * time.Second).
160+
WithMaxIdleConns(100).
161+
WithMaxIdleConnsPerHost(100).
162+
Build()
167163

168164
awsSCIMService, err := aws.NewSCIMService(httpClient, cfg.AWSSCIMEndpoint, cfg.AWSSCIMAccessToken)
169165
if err != nil {

cmd/idpscimcli/cmd/gws.go

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import (
77
"os"
88
"time"
99

10-
"github.com/p2p-b2b/httpretrier"
10+
"github.com/slashdevops/httpx"
1111
"github.com/slashdevops/idp-scim-sync/internal/config"
1212
"github.com/slashdevops/idp-scim-sync/internal/version"
1313
"github.com/slashdevops/idp-scim-sync/pkg/google"
@@ -126,11 +126,12 @@ func getGWSDirectoryService(ctx context.Context) *google.DirectoryService {
126126
"https://www.googleapis.com/auth/admin.directory.user.readonly",
127127
}
128128

129-
httpRetryClient := httpretrier.NewClient(
130-
3, // Max Retries
131-
httpretrier.ExponentialBackoff(10*time.Millisecond, 100*time.Millisecond),
132-
nil, // Use http.DefaultTransport
133-
)
129+
httpRetryClient := httpx.NewClientBuilder().
130+
WithMaxRetries(3).
131+
WithRetryStrategy(httpx.ExponentialBackoffStrategy).
132+
WithRetryBaseDelay(500 * time.Millisecond).
133+
WithRetryMaxDelay(10 * time.Second).
134+
Build()
134135

135136
gServiceConfig := google.DirectoryServiceConfig{
136137
UserEmail: cfg.GWSUserEmail,

docs/Whats-New.md

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,42 @@
22

33
This document tracks notable changes, new features, and bug fixes across releases.
44

5+
## v0.40.1
6+
7+
### Improved HTTP Retry Library
8+
9+
Replaced the `httpretrier` library with [httpx](https://github.com/slashdevops/httpx), a zero-dependency HTTP client with built-in retry support.
10+
11+
**Why:** The previous library did not properly handle HTTP `429 Too Many Requests` responses, which caused issues with AWS SSO SCIM API throttling under high load.
12+
13+
**What changed:**
14+
15+
* The `httpx` library automatically retries on `429` and `5xx` responses with configurable backoff strategies.
16+
* AWS SCIM API calls now use **jitter backoff** instead of simple exponential backoff, reducing the chance of thundering herd effects during rate limiting.
17+
* Google Workspace API calls use **exponential backoff** for reliable retries.
18+
* The `httpx` library has zero external dependencies and integrates with Go's `slog` logging.
19+
20+
### AWS SCIM Client Improvements (`pkg/aws`)
21+
22+
Several code quality improvements and bug fixes in the AWS SCIM client:
23+
24+
* **Bug fix:** `CreateOrGetUser` used `reflect.DeepEqual` to compare a `*CreateUserRequest` with a `*GetUserResponse` — different types, so the comparison always returned `false`, causing unnecessary PUT updates on every 409 conflict. Replaced with a typed `usersEqual` function that compares only sync-relevant attributes.
25+
* **Removed `pkg/errors` dependency:** Replaced with stdlib `errors` and `fmt` packages. Sentinel errors now use `errors.New` instead of `errors.Errorf`.
26+
* **Go 1.26 `errors.AsType`:** Migrated all `errors.As` calls to the generic `errors.AsType[T]` for compile-time type safety and better performance.
27+
* **Fixed `String()` methods:** `User.String()` and `Group.String()` no longer call `os.Exit(1)` on marshal failure. They return a safe fallback string instead.
28+
* **Eliminated double JSON decode:** `GetUserByUserName` and `GetGroupByDisplayName` no longer marshal a resource to JSON and re-parse it. They use direct type conversion instead.
29+
* **Fixed decode error fallback:** `CreateGroup` and `CreateOrGetGroup` no longer attempt to read an already-consumed response body on decode failure.
30+
* **Removed redundant context set:** `do()` no longer calls `req.WithContext(ctx)` since the request is already created with `http.NewRequestWithContext`.
31+
* **Simplified type conversions:** `CreateOrGetUser` and `CreateOrGetGroup` use type conversions instead of manual field-by-field struct copies.
32+
33+
### Go 1.26 Modernization
34+
35+
Applied Go 1.26 best practices across the codebase:
36+
37+
* **Removed `github.com/pkg/errors` dependency:** Replaced all `errors.Wrap` and `errors.Errorf` with stdlib `fmt.Errorf` (with `%w`) and `errors.New` in `internal/setup`, `internal/repository`, and `pkg/aws`.
38+
* **`errors.AsType[T]`:** Migrated `errors.As` calls to the generic `errors.AsType[T]` in `internal/core/sync.go` for type safety and performance.
39+
* **Fixed `os.Exit` in `Hash()`:** `internal/model.Hash()` no longer calls `os.Exit(1)` on nil input or encoding failure. It panics instead (appropriate for programming errors, recoverable, produces stack trace).
40+
541
## v0.44.0
642

743
### Configurable User Fields

go.mod

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,18 +7,17 @@ require (
77
github.com/aws/aws-sdk-go-v2 v1.41.5
88
github.com/aws/aws-sdk-go-v2/config v1.32.13
99
github.com/aws/aws-sdk-go-v2/credentials v1.19.13
10-
github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3
10+
github.com/aws/aws-sdk-go-v2/service/s3 v1.98.0
1111
github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.41.5
1212
github.com/google/go-cmp v0.7.0
13-
github.com/p2p-b2b/httpretrier v0.0.4
14-
github.com/pkg/errors v0.9.1
13+
github.com/slashdevops/httpx v0.0.4
1514
github.com/spf13/cobra v1.10.2
1615
github.com/spf13/viper v1.21.0
1716
github.com/stretchr/testify v1.11.1
1817
go.uber.org/mock v0.6.0
1918
golang.org/x/oauth2 v0.36.0
2019
golang.org/x/sync v0.20.0
21-
google.golang.org/api v0.273.0
20+
google.golang.org/api v0.273.1
2221
gopkg.in/yaml.v3 v3.0.1
2322
)
2423

go.sum

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3x
3232
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA=
3333
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM=
3434
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ=
35-
github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 h1:HwxWTbTrIHm5qY+CAEur0s/figc3qwvLWsNkF4RPToo=
36-
github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM=
35+
github.com/aws/aws-sdk-go-v2/service/s3 v1.98.0 h1:foqo/ocQ7WqKwy3FojGtZQJo0FR4vto9qnz9VaumbCo=
36+
github.com/aws/aws-sdk-go-v2/service/s3 v1.98.0/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM=
3737
github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.41.5 h1:z2ayoK3pOvf8ODj/vPR0FgAS5ONruBq0F94SRoW/BIU=
3838
github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.41.5/go.mod h1:mpZB5HAl4ZIISod9qCi12xZ170TbHX9CCJV5y7nb7QU=
3939
github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg=
@@ -82,19 +82,17 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
8282
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
8383
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
8484
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
85-
github.com/p2p-b2b/httpretrier v0.0.4 h1:TREDrCthpm0jMTLq2T9+rf8hhH9h36W8Vf8l3L5+WiI=
86-
github.com/p2p-b2b/httpretrier v0.0.4/go.mod h1:0yKtg4EekYBRzwLIiRVBqh4A/I3VQRiSdBnpu5InyOw=
8785
github.com/pelletier/go-toml/v2 v2.3.0 h1:k59bC/lIZREW0/iVaQR8nDHxVq8OVlIzYCOJf421CaM=
8886
github.com/pelletier/go-toml/v2 v2.3.0/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
89-
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
90-
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
9187
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
9288
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
9389
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
9490
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
9591
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
9692
github.com/sagikazarmark/locafero v0.12.0 h1:/NQhBAkUb4+fH1jivKHWusDYFjMOOKU88eegjfxfHb4=
9793
github.com/sagikazarmark/locafero v0.12.0/go.mod h1:sZh36u/YSZ918v0Io+U9ogLYQJ9tLLBmM4eneO6WwsI=
94+
github.com/slashdevops/httpx v0.0.4 h1:XhwMl9asfiPSqBaTv4wNjwFus6RFfgsEffl33BZazK4=
95+
github.com/slashdevops/httpx v0.0.4/go.mod h1:RIXp0/ylq/BqDlPo8glvay29Nr6zHw6jbExJneLYXKQ=
9896
github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
9997
github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
10098
github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
@@ -146,8 +144,8 @@ golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
146144
golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
147145
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
148146
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
149-
google.golang.org/api v0.273.0 h1:r/Bcv36Xa/te1ugaN1kdJ5LoA5Wj/cL+a4gj6FiPBjQ=
150-
google.golang.org/api v0.273.0/go.mod h1:JbAt7mF+XVmWu6xNP8/+CTiGH30ofmCmk9nM8d8fHew=
147+
google.golang.org/api v0.273.1 h1:L7G/TmpAMz0nKx/ciAVssVmWQiOF6+pOuXeKrWVsquY=
148+
google.golang.org/api v0.273.1/go.mod h1:JbAt7mF+XVmWu6xNP8/+CTiGH30ofmCmk9nM8d8fHew=
151149
google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5 h1:JNfk58HZ8lfmXbYK2vx/UvsqIL59TzByCxPIX4TDmsE=
152150
google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:x5julN69+ED4PcFk/XWayw35O0lf/nGa4aNgODCmNmw=
153151
google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5 h1:CogIeEXn4qWYzzQU0QqvYBM8yDF9cFYzDq9ojSpv0Js=

internal/core/sync.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -101,10 +101,10 @@ func (ss *SyncService) SyncGroupsAndTheirMembers(ctx context.Context) error {
101101
slog.Info("getting state data")
102102
state, err := ss.repo.GetState(ctx)
103103
if err != nil {
104-
var nsk *types.NoSuchKey
105-
var StateFileEmpty *repository.ErrStateFileEmpty
106-
107-
if errors.As(err, &nsk) || errors.As(err, &StateFileEmpty) {
104+
if _, ok := errors.AsType[*types.NoSuchKey](err); ok {
105+
slog.Warn("no state file found in the state repository, creating a new one")
106+
state = model.StateBuilder().Build()
107+
} else if _, ok := errors.AsType[*repository.ErrStateFileEmpty](err); ok {
108108
slog.Warn("no state file found in the state repository, creating a new one")
109109
state = model.StateBuilder().Build()
110110
} else {

internal/model/hash.go

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,22 +5,20 @@ import (
55
"crypto/sha256"
66
"encoding/gob"
77
"fmt"
8-
"log/slog"
9-
"os"
108
)
119

12-
// Hash returns a sha256 hash of value pass as argument
10+
// Hash returns a sha256 hash of value pass as argument.
11+
// It panics if value is nil or cannot be gob-encoded, since these
12+
// conditions indicate a programming error in the caller.
1313
func Hash(value any) string {
1414
if value == nil {
15-
slog.Error("value is nil")
16-
os.Exit(1)
15+
panic("model: Hash called with nil value")
1716
}
1817

1918
buf := new(bytes.Buffer)
2019
enc := gob.NewEncoder(buf)
2120
if err := enc.Encode(value); err != nil {
22-
slog.Error("error encoding value")
23-
os.Exit(1)
21+
panic(fmt.Sprintf("model: Hash encoding error: %v", err))
2422
}
2523

2624
return fmt.Sprintf("%x", sha256.Sum256(buf.Bytes()))

internal/repository/s3.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,11 @@ import (
44
"bytes"
55
"context"
66
"encoding/json"
7+
"errors"
78
"fmt"
89

910
"github.com/aws/aws-sdk-go-v2/aws"
1011
"github.com/aws/aws-sdk-go-v2/service/s3"
11-
"github.com/pkg/errors"
1212
"github.com/slashdevops/idp-scim-sync/internal/model"
1313
)
1414

0 commit comments

Comments
 (0)