Skip to content

feat: add modern rate limiter option using go-github-ratelimit #3236

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mkushakov wants to merge 3 commits into
base: main
Choose a base branch
from
+49 −0
Open

feat: add modern rate limiter option using go-github-ratelimit #3236

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ec32ac7
feat: add modern rate limiter option using go-github-ratelimit
mkushakov Feb 27, 2026
b0ce7b4
docs: add rate_limiter provider attribute to documentation
mkushakov Feb 27, 2026
9c7e231
feat: enhance ModernRateLimitedHTTPClient with logging and preview he…
mkushakov Mar 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions github/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"strings"
"time"

github_ratelimit "github.com/gofri/go-github-ratelimit/v2/github_ratelimit"
"github.com/google/go-github/v83/github"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/logging"
"github.com/shurcooL/githubv4"
Expand All @@ -27,6 +28,7 @@ type Config struct {
RetryableErrors map[int]bool
MaxRetries int
ParallelRequests bool
RateLimiter string // "modern" or "legacy"
}

type Owner struct {
Expand Down Expand Up @@ -72,13 +74,36 @@ func RateLimitedHTTPClient(client *http.Client, writeDelay, readDelay, retryDela
return client
}

// ModernRateLimitedHTTPClient creates an HTTP client that uses go-github-ratelimit
// for automatic GitHub API rate limit handling. When using this client, the
// read_delay_ms, write_delay_ms, and parallel_requests settings are ignored
// because rate limiting is handled automatically by the library.
func ModernRateLimitedHTTPClient(client *http.Client, retryDelay time.Duration, retryableErrors map[int]bool, maxRetries int) *http.Client {
client.Transport = NewEtagTransport(client.Transport)
Copy link
Collaborator

@deiga deiga Feb 27, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
client.Transport = NewEtagTransport(client.Transport)
client.Transport = NewEtagTransport(client.Transport)
client.Transport = logging.NewLoggingHTTPTransport(client.Transport)

Copy link
Author

@mkushakov mkushakov Mar 2, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added, also i have missed Stone Crop preview header which is added as well.

client.Transport = logging.NewLoggingHTTPTransport(client.Transport)
client.Transport = newPreviewHeaderInjectorTransport(map[string]string{
// TODO: remove when Stone Crop preview is moved to general availability in the GraphQL API
"Accept": "application/vnd.github.stone-crop-preview+json",
}, client.Transport)
rateLimitClient := github_ratelimit.NewClient(client.Transport)

if maxRetries > 0 {
rateLimitClient.Transport = NewRetryTransport(rateLimitClient.Transport, WithRetryDelay(retryDelay), WithRetryableErrors(retryableErrors), WithMaxRetries(maxRetries))
}

return rateLimitClient
}

func (c *Config) AuthenticatedHTTPClient() *http.Client {
ctx := context.Background()
ts := oauth2.StaticTokenSource(
&oauth2.Token{AccessToken: c.Token},
)
client := oauth2.NewClient(ctx, ts)

if c.RateLimiter == "modern" {
return ModernRateLimitedHTTPClient(client, c.RetryDelay, c.RetryableErrors, c.MaxRetries)
}
return RateLimitedHTTPClient(client, c.WriteDelay, c.ReadDelay, c.RetryDelay, c.ParallelRequests, c.RetryableErrors, c.MaxRetries)
}

Expand All @@ -88,6 +113,9 @@ func (c *Config) Anonymous() bool {

func (c *Config) AnonymousHTTPClient() *http.Client {
client := &http.Client{Transport: &http.Transport{}}
if c.RateLimiter == "modern" {
return ModernRateLimitedHTTPClient(client, c.RetryDelay, c.RetryableErrors, c.MaxRetries)
}
return RateLimitedHTTPClient(client, c.WriteDelay, c.ReadDelay, c.RetryDelay, c.ParallelRequests, c.RetryableErrors, c.MaxRetries)
}

Expand Down
16 changes: 16 additions & 0 deletions github/provider.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (

"github.com/hashicorp/terraform-plugin-sdk/v2/diag"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation"
)

func Provider() *schema.Provider {
Expand Down Expand Up @@ -93,6 +94,13 @@ func Provider() *schema.Provider {
Default: false,
Description: descriptions["parallel_requests"],
},
"rate_limiter": {
Type: schema.TypeString,
Optional: true,
Default: "legacy",
Description: descriptions["rate_limiter"],
ValidateDiagFunc: validation.ToDiagFunc(validation.StringInSlice([]string{"legacy", "modern"}, false)),
},
"app_auth": {
Type: schema.TypeList,
Optional: true,
Expand Down Expand Up @@ -342,6 +350,10 @@ func init() {
"Defaults to 3",
"max_per_page": "Number of items per page for pagination" +
"Defaults to 100",
"rate_limiter": "The rate limiting strategy to use. 'modern' uses go-github-ratelimit for automatic GitHub API rate limit handling. " +
"'legacy' uses the provider's built-in rate limiting with configurable delays. " +
"When using 'modern', the read_delay_ms, write_delay_ms, and parallel_requests settings are ignored. " +
"Defaults to 'legacy'.",
}
}

Expand Down Expand Up @@ -474,6 +486,9 @@ func providerConfigure(p *schema.Provider) schema.ConfigureContextFunc {

log.Printf("[DEBUG] Setting parallel_requests to %t", parallelRequests)

rateLimiter := d.Get("rate_limiter").(string)
log.Printf("[DEBUG] Setting rate_limiter to %s", rateLimiter)

config := Config{
Token: token,
BaseURL: baseURL,
Expand All @@ -486,6 +501,7 @@ func providerConfigure(p *schema.Provider) schema.ConfigureContextFunc {
MaxRetries: maxRetries,
ParallelRequests: parallelRequests,
IsGHES: isGHES,
RateLimiter: rateLimiter,
}

meta, err := config.Meta()
Expand Down
1 change: 1 addition & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ require (
github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
github.com/cloudflare/circl v1.6.1 // indirect
github.com/fatih/color v1.18.0 // indirect
github.com/gofri/go-github-ratelimit/v2 v2.0.2 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/go-cmp v0.7.0 // indirect
github.com/google/go-querystring v1.2.0 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68=
github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
github.com/gofri/go-github-ratelimit/v2 v2.0.2 h1:gS8wAS1jTmlWGdTjAM7KIpsLjwY1S0S/gKK5hthfSXM=
github.com/gofri/go-github-ratelimit/v2 v2.0.2/go.mod h1:YBQt4gTbdcbMjJFT05YFEaECwH78P5b0IwrnbLiHGdE=
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ=
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw=
github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
Expand Down
2 changes: 2 additions & 0 deletions website/docs/index.html.markdown
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,8 @@ The following arguments are supported in the `provider` block:

* `max_retries` - (Optional) Number of times to retry a request after receiving an error status code. Defaults to 3

* `rate_limiter` - (Optional) The rate limiting strategy to use. Can be one of `legacy` or `modern`. When set to `modern`, uses [go-github-ratelimit](https://github.com/gofri/go-github-ratelimit) for automatic GitHub API rate limit handling, including primary and secondary rate limits. When using `modern`, the `read_delay_ms`, `write_delay_ms`, and `parallel_requests` settings are ignored. Defaults to `legacy`.

Note: If you have a PEM file on disk, you can pass it in via `pem_file = file("path/to/file.pem")`.

For backwards compatibility, if more than one of `owner`, `organization`,
Expand Down