From 3e3a068d425255c0eefc29214269efef58072388 Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Mon, 20 Dec 2021 11:45:11 -0800 Subject: [PATCH 1/3] WIP: use fork's hydra-api.yml for now --- Makefile | 2 +- hydra/api/api.gen.go | 142 ++++++++++++++++++++++++++++++++++++++- hydra/api/hydra-api.yaml | 52 +++++++++++++- 3 files changed, 191 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 01a8dd21..860ed571 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ hydra/api/api.gen.go: hydra/api/hydra-api.yaml .PHONY: hydra/api/hydra-api.yaml hydra/api/hydra-api.yaml: mkdir -p $(shell dirname $@) - curl https://raw.githubusercontent.com/NixOS/hydra/master/hydra-api.yaml > $@ + curl https://raw.githubusercontent.com/DeterminateSystems/hydra/runcommand/dynamic/hydra-api.yaml > $@ # TODO # test: diff --git a/hydra/api/api.gen.go b/hydra/api/api.gen.go index c64bc961..e7d4672d 100644 --- a/hydra/api/api.gen.go +++ b/hydra/api/api.gen.go @@ -189,6 +189,9 @@ type Jobset struct { // email address to send notices to instead of the package maintainer (can be a comma separated list) Emailoverride *string `json:"emailoverride,omitempty"` + // when true the jobset supports executing dynamically defined RunCommand hooks. Requires the server and project's configuration to also enable dynamic RunCommand. + EnableDynamicRunCommand *bool `json:"enable_dynamic_run_command,omitempty"` + // 0 is disabled, 1 is enabled, 2 is one-shot, and 3 is one-at-a-time Enabled *int `json:"enabled,omitempty"` @@ -252,13 +255,25 @@ type Jobset_Inputs struct { // JobsetEval defines model for JobsetEval. type JobsetEval struct { - // List of builds generated for this jobset evaluation + // List of builds generated for this jobset evaluation. Builds *[]int `json:"builds,omitempty"` - // is true if the number of JobsetEval members is different from the prior evaluation. (will always be true on the first evaluation) + // How long it took (in seconds) to fetch the jobset inputs. + Checkouttime *int `json:"checkouttime,omitempty"` + + // How long it took (in seconds) to evaluate the jobset. + Evaltime *int `json:"evaltime,omitempty"` + + // For flake jobsets, the immutable flake reference allowing you to reproduce this evaluation. Null otherwise. + Flake *string `json:"flake"` + + // Whether the number of JobsetEval members is different from the prior evaluation. This is always true on the first evaluation. Hasnewbuilds *bool `json:"hasnewbuilds,omitempty"` Id *int `json:"id,omitempty"` Jobsetevalinputs *JobsetEval_Jobsetevalinputs `json:"jobsetevalinputs,omitempty"` + + // Time in seconds since the Unix epoch when this evaluation was created. + Timestamp *int `json:"timestamp,omitempty"` } // JobsetEval_Jobsetevalinputs defines model for JobsetEval.Jobsetevalinputs. @@ -351,6 +366,9 @@ type Project struct { // name to be displayed in the web interface Displayname *string `json:"displayname,omitempty"` + // when true the project's jobsets support executing dynamically defined RunCommand hooks. Requires the server and project's configuration to also enable dynamic RunCommand. + EnableDynamicRunCommand *bool `json:"enable_dynamic_run_command,omitempty"` + // when set to true the project gets scheduled for evaluation Enabled *bool `json:"enabled,omitempty"` @@ -423,6 +441,9 @@ type PutProjectIdJSONBody struct { // display name of the project Displayname *string `json:"displayname,omitempty"` + // when true the project's jobsets support executing dynamically defined RunCommand hooks. Requires the server and project's configuration to also enable dynamic RunCommand. + EnableDynamicRunCommand *bool `json:"enable_dynamic_run_command,omitempty"` + // when set to true the project gets scheduled for evaluation Enabled *bool `json:"enabled,omitempty"` @@ -752,6 +773,9 @@ type ClientInterface interface { // GetBuildBuildId request GetBuildBuildId(ctx context.Context, buildId int, reqEditors ...RequestEditorFn) (*http.Response, error) + // GetBuildBuildIdConstituents request + GetBuildBuildIdConstituents(ctx context.Context, buildId int, reqEditors ...RequestEditorFn) (*http.Response, error) + // GetEvalBuildId request GetEvalBuildId(ctx context.Context, buildId int, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -840,6 +864,18 @@ func (c *Client) GetBuildBuildId(ctx context.Context, buildId int, reqEditors .. return c.Client.Do(req) } +func (c *Client) GetBuildBuildIdConstituents(ctx context.Context, buildId int, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewGetBuildBuildIdConstituentsRequest(c.Server, buildId) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + func (c *Client) GetEvalBuildId(ctx context.Context, buildId int, reqEditors ...RequestEditorFn) (*http.Response, error) { req, err := NewGetEvalBuildIdRequest(c.Server, buildId) if err != nil { @@ -1163,6 +1199,40 @@ func NewGetBuildBuildIdRequest(server string, buildId int) (*http.Request, error return req, nil } +// NewGetBuildBuildIdConstituentsRequest generates requests for GetBuildBuildIdConstituents +func NewGetBuildBuildIdConstituentsRequest(server string, buildId int) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithLocation("simple", false, "build-id", runtime.ParamLocationPath, buildId) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/build/%s/constituents", pathParam0) + if operationPath[0] == '/' { + operationPath = operationPath[1:] + } + operationURL := url.URL{ + Path: operationPath, + } + + queryURL := serverURL.ResolveReference(&operationURL) + + req, err := http.NewRequest("GET", queryURL.String(), nil) + if err != nil { + return nil, err + } + + return req, nil +} + // NewGetEvalBuildIdRequest generates requests for GetEvalBuildId func NewGetEvalBuildIdRequest(server string, buildId int) (*http.Request, error) { var err error @@ -1679,6 +1749,9 @@ type ClientWithResponsesInterface interface { // GetBuildBuildId request GetBuildBuildIdWithResponse(ctx context.Context, buildId int, reqEditors ...RequestEditorFn) (*GetBuildBuildIdResponse, error) + // GetBuildBuildIdConstituents request + GetBuildBuildIdConstituentsWithResponse(ctx context.Context, buildId int, reqEditors ...RequestEditorFn) (*GetBuildBuildIdConstituentsResponse, error) + // GetEvalBuildId request GetEvalBuildIdWithResponse(ctx context.Context, buildId int, reqEditors ...RequestEditorFn) (*GetEvalBuildIdResponse, error) @@ -1811,6 +1884,29 @@ func (r GetBuildBuildIdResponse) StatusCode() int { return 0 } +type GetBuildBuildIdConstituentsResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *[]Build + JSON404 *Error +} + +// Status returns HTTPResponse.Status +func (r GetBuildBuildIdConstituentsResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r GetBuildBuildIdConstituentsResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + type GetEvalBuildIdResponse struct { Body []byte HTTPResponse *http.Response @@ -2166,6 +2262,15 @@ func (c *ClientWithResponses) GetBuildBuildIdWithResponse(ctx context.Context, b return ParseGetBuildBuildIdResponse(rsp) } +// GetBuildBuildIdConstituentsWithResponse request returning *GetBuildBuildIdConstituentsResponse +func (c *ClientWithResponses) GetBuildBuildIdConstituentsWithResponse(ctx context.Context, buildId int, reqEditors ...RequestEditorFn) (*GetBuildBuildIdConstituentsResponse, error) { + rsp, err := c.GetBuildBuildIdConstituents(ctx, buildId, reqEditors...) + if err != nil { + return nil, err + } + return ParseGetBuildBuildIdConstituentsResponse(rsp) +} + // GetEvalBuildIdWithResponse request returning *GetEvalBuildIdResponse func (c *ClientWithResponses) GetEvalBuildIdWithResponse(ctx context.Context, buildId int, reqEditors ...RequestEditorFn) (*GetEvalBuildIdResponse, error) { rsp, err := c.GetEvalBuildId(ctx, buildId, reqEditors...) @@ -2409,6 +2514,39 @@ func ParseGetBuildBuildIdResponse(rsp *http.Response) (*GetBuildBuildIdResponse, return response, nil } +// ParseGetBuildBuildIdConstituentsResponse parses an HTTP response from a GetBuildBuildIdConstituentsWithResponse call +func ParseGetBuildBuildIdConstituentsResponse(rsp *http.Response) (*GetBuildBuildIdConstituentsResponse, error) { + bodyBytes, err := ioutil.ReadAll(rsp.Body) + defer rsp.Body.Close() + if err != nil { + return nil, err + } + + response := &GetBuildBuildIdConstituentsResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest []Build + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 404: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON404 = &dest + + } + + return response, nil +} + // ParseGetEvalBuildIdResponse parses an HTTP response from a GetEvalBuildIdWithResponse call func ParseGetEvalBuildIdResponse(rsp *http.Response) (*GetEvalBuildIdResponse, error) { bodyBytes, err := ioutil.ReadAll(rsp.Body) diff --git a/hydra/api/hydra-api.yaml b/hydra/api/hydra-api.yaml index 2a454942..ce7e0f9a 100644 --- a/hydra/api/hydra-api.yaml +++ b/hydra/api/hydra-api.yaml @@ -178,6 +178,9 @@ paths: enabled: description: when set to true the project gets scheduled for evaluation type: boolean + enable_dynamic_run_command: + description: when true the project's jobsets support executing dynamically defined RunCommand hooks. Requires the server and project's configuration to also enable dynamic RunCommand. + type: boolean visible: description: when set to true the project is displayed in the web interface type: boolean @@ -504,6 +507,32 @@ paths: schema: $ref: '#/components/schemas/Error' + /build/{build-id}/constituents: + get: + summary: Retrieves a build's constituent jobs + parameters: + - name: build-id + in: path + description: build identifier + required: true + schema: + type: integer + responses: + '200': + description: build + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/Build' + '404': + description: build couldn't be found + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + /eval/{build-id}: get: summary: Retrieves evaluations identified by build id @@ -581,6 +610,9 @@ components: enabled: description: when set to true the project gets scheduled for evaluation type: boolean + enable_dynamic_run_command: + description: when true the project's jobsets support executing dynamically defined RunCommand hooks. Requires the server and project's configuration to also enable dynamic RunCommand. + type: boolean declarative: description: declarative input configured for this project type: object @@ -663,6 +695,9 @@ components: enableemail: description: when true the jobset sends emails when previously-successful builds fail type: boolean + enable_dynamic_run_command: + description: when true the jobset supports executing dynamically defined RunCommand hooks. Requires the server and project's configuration to also enable dynamic RunCommand. + type: boolean visible: description: when true the jobset is visible in the web frontend type: boolean @@ -735,11 +770,24 @@ components: properties: id: type: integer + timestamp: + description: Time in seconds since the Unix epoch when this evaluation was created. + type: integer + checkouttime: + description: How long it took (in seconds) to fetch the jobset inputs. + type: integer + evaltime: + description: How long it took (in seconds) to evaluate the jobset. + type: integer hasnewbuilds: - description: is true if the number of JobsetEval members is different from the prior evaluation. (will always be true on the first evaluation) + description: Whether the number of JobsetEval members is different from the prior evaluation. This is always true on the first evaluation. type: boolean + flake: + description: For flake jobsets, the immutable flake reference allowing you to reproduce this evaluation. Null otherwise. + nullable: true + type: string builds: - description: List of builds generated for this jobset evaluation + description: List of builds generated for this jobset evaluation. type: array items: type: integer From e72fb05d4cf1a75d6afc9709f4eb4c3ca71cb758 Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Mon, 20 Dec 2021 11:46:22 -0800 Subject: [PATCH 2/3] resource_hydra_project: support dynamic runcommands --- hydra/resource_hydra_project.go | 11 +++++++++ hydra/resource_hydra_project_test.go | 37 ++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/hydra/resource_hydra_project.go b/hydra/resource_hydra_project.go index a2fcd06f..acd962b1 100644 --- a/hydra/resource_hydra_project.go +++ b/hydra/resource_hydra_project.go @@ -92,6 +92,12 @@ func resourceHydraProject() *schema.Resource { MaxItems: 1, Elem: declInputSchema(), }, + "enable_dynamic_run_command": { + Description: "Whether or not to enable dynamic RunCommands for the project. Must be enabled by the server.", + Type: schema.TypeBool, + Optional: true, + Default: false, + }, }, } } @@ -138,6 +144,11 @@ func createProjectPutBody(project string, d *schema.ResourceData) *api.PutProjec } } + enableDynamicRunCommand := d.Get("enable_dynamic_run_command").(bool) + if enableDynamicRunCommand { + body.EnableDynamicRunCommand = &enableDynamicRunCommand + } + return &body } diff --git a/hydra/resource_hydra_project_test.go b/hydra/resource_hydra_project_test.go index 9a3e9a46..f4903249 100644 --- a/hydra/resource_hydra_project_test.go +++ b/hydra/resource_hydra_project_test.go @@ -116,6 +116,28 @@ func TestAccHydraProject_declarative(t *testing.T) { }) } +// Requires enabling the dynamicruncommand feature on the server +func TestAccHydraProject_dynamicRunCommand(t *testing.T) { + // identifier must start with a letter + name := fmt.Sprintf("p%s", acctest.RandString(7)) + resourceName := "hydra_project.test" + + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckHydraProjectDestroy, + Steps: []resource.TestStep{ + // Test creation of project with dynamic runcommands enabled + { + Config: testAccHydraProjectConfigDynamicRunCommand(name), + Check: resource.ComposeTestCheckFunc( + testAccCheckProjectExists(resourceName), + ), + }, + }, + }) +} + // testAccCheckExampleResourceDestroy verifies the Project has been destroyed func testAccCheckHydraProjectDestroy(s *terraform.State) error { client := testAccProvider.Meta().(*api.ClientWithResponses) @@ -313,3 +335,18 @@ resource "hydra_project" "test" { } `, name, os.Getenv("HYDRA_USERNAME")) } + +func testAccHydraProjectConfigDynamicRunCommand(name string) string { + return fmt.Sprintf(` +resource "hydra_project" "test" { + name = "%s" + display_name = "Nixpkgs" + description = "Nix Packages collection" + homepage = "http://nixos.org/nixpkgs" + owner = "%s" + enabled = true + visible = true + enable_dynamic_run_command = true +} +`, name, os.Getenv("HYDRA_USERNAME")) +} From 1a24580ae5b74b4070f2db32bdd86ddb864c4e6d Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Mon, 20 Dec 2021 11:47:03 -0800 Subject: [PATCH 3/3] resource_hydra_jobset: support dynamic runcommands --- hydra/resource_hydra_jobset.go | 11 +++++ hydra/resource_hydra_jobset_test.go | 70 +++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) diff --git a/hydra/resource_hydra_jobset.go b/hydra/resource_hydra_jobset.go index 285108e5..ed5cadb6 100644 --- a/hydra/resource_hydra_jobset.go +++ b/hydra/resource_hydra_jobset.go @@ -164,6 +164,12 @@ func resourceHydraJobset() *schema.Resource { MinItems: 1, Elem: inputSchema(), }, + "enable_dynamic_run_command": { + Description: "Whether or not to enable dynamic RunCommands for the jobset. Must be enabled by the server and parent project.", + Type: schema.TypeBool, + Optional: true, + Default: false, + }, }, } } @@ -360,6 +366,11 @@ func createJobsetPutBody(project string, jobset string, d *schema.ResourceData) } } + enableDynamicRunCommand := d.Get("enable_dynamic_run_command").(bool) + if enableDynamicRunCommand { + body.EnableDynamicRunCommand = &enableDynamicRunCommand + } + return &body, diags } diff --git a/hydra/resource_hydra_jobset_test.go b/hydra/resource_hydra_jobset_test.go index 54f69638..49ca4b40 100644 --- a/hydra/resource_hydra_jobset_test.go +++ b/hydra/resource_hydra_jobset_test.go @@ -231,6 +231,27 @@ func TestAccHydraJobset_legacyToFlakeAndBack(t *testing.T) { }) } +func TestAccHydraJobset_dynamicRunCommand(t *testing.T) { + // identifier must start with a letter + name := fmt.Sprintf("j%s", acctest.RandString(7)) + resourceName := "hydra_jobset.test" + + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckHydraJobsetDestroy, + Steps: []resource.TestStep{ + // Test creation of jobset with dynamic runcommands enabled + { + Config: testAccHydraJobsetConfigDynamicRunCommand(name, name), + Check: resource.ComposeTestCheckFunc( + testAccCheckJobsetExists(resourceName), + ), + }, + }, + }) +} + // testAccCheckExampleResourceDestroy verifies the Jobset has been destroyed func testAccCheckHydraJobsetDestroy(s *terraform.State) error { client := testAccProvider.Meta().(*api.ClientWithResponses) @@ -822,3 +843,52 @@ resource "hydra_jobset" "test" { } }`, project, os.Getenv("HYDRA_USERNAME"), jobset) } + +func testAccHydraJobsetConfigDynamicRunCommand(project string, jobset string) string { + return fmt.Sprintf(` +resource "hydra_project" "test" { + name = "%s" + display_name = "Ofborg" + description = "ofborg automation" + homepage = "https://github.com/nixos/ofborg" + owner = "%s" + enabled = true + visible = true + enable_dynamic_run_command = true +} + +resource "hydra_jobset" "test" { + project = hydra_project.test.name + state = "enabled" + visible = true + name = "%s" + type = "legacy" + description = "" + enable_dynamic_run_command = true + + nix_expression { + file = "release.nix" + input = "ofborg" + } + + check_interval = 0 + scheduling_shares = 3000 + + email_notifications = false + keep_evaluations = 3 + + input { + name = "nixpkgs" + type = "git" + value = "https://github.com/NixOS/nixpkgs.git nixpkgs-unstable" + notify_committers = false + } + + input { + name = "ofborg" + type = "git" + value = "https://github.com/nixos/ofborg.git released" + notify_committers = false + } +}`, project, os.Getenv("HYDRA_USERNAME"), jobset) +}