From 8c6fd63520f7218f736deeb5d3cfb35739aa056a Mon Sep 17 00:00:00 2001 From: McKayla Washburn Date: Tue, 13 May 2025 20:36:26 +0000 Subject: [PATCH] feat: load terraform modules when using dynamic parameters --- .gitignore | 1 + coderd/files/overlay.go | 86 +++++++++++++++++ coderd/files/overlay_test.go | 44 +++++++++ coderd/parameters.go | 27 +++++- coderd/parameters_test.go | 49 ++++++++++ .../modules/jetbrains_gateway/main.tf | 94 +++++++++++++++++++ .../modules/.terraform/modules/modules.json | 1 + coderd/testdata/parameters/modules/main.tf | 5 + provisioner/terraform/executor.go | 2 +- provisioner/terraform/modules.go | 82 +++++++++++----- .../terraform/modules_internal_test.go | 13 ++- .../.terraform/modules/modules.json | 2 +- 12 files changed, 374 insertions(+), 32 deletions(-) create mode 100644 coderd/files/overlay.go create mode 100644 coderd/files/overlay_test.go create mode 100644 coderd/testdata/parameters/modules/.terraform/modules/jetbrains_gateway/main.tf create mode 100644 coderd/testdata/parameters/modules/.terraform/modules/modules.json create mode 100644 coderd/testdata/parameters/modules/main.tf diff --git a/.gitignore b/.gitignore index 66f36c49bcb07..5aa08b2512527 100644 --- a/.gitignore +++ b/.gitignore @@ -50,6 +50,7 @@ site/stats/ *.tfplan *.lock.hcl .terraform/ +!coderd/testdata/parameters/modules/.terraform/ !provisioner/terraform/testdata/modules-source-caching/.terraform/ **/.coderv2/* diff --git a/coderd/files/overlay.go b/coderd/files/overlay.go new file mode 100644 index 0000000000000..d7e2adf8db4e8 --- /dev/null +++ b/coderd/files/overlay.go @@ -0,0 +1,86 @@ +package files + +import ( + "io/fs" + "path" + "strings" + + "golang.org/x/xerrors" +) + +// overlayFS allows you to "join" together the template files tar file fs.FS +// with the Terraform modules tar file fs.FS. We could potentially turn this +// into something more parameterized/configurable, but the requirements here are +// a _bit_ odd, because every file in the modulesFS includes the +// .terraform/modules/ folder at the beginning of it's path. +type overlayFS struct { + baseFS fs.FS + overlays []Overlay +} + +type Overlay struct { + Path string + fs.FS +} + +func NewOverlayFS(baseFS fs.FS, overlays []Overlay) (fs.FS, error) { + if err := valid(baseFS); err != nil { + return nil, xerrors.Errorf("baseFS: %w", err) + } + + for _, overlay := range overlays { + if err := valid(overlay.FS); err != nil { + return nil, xerrors.Errorf("overlayFS: %w", err) + } + } + + return overlayFS{ + baseFS: baseFS, + overlays: overlays, + }, nil +} + +func (f overlayFS) Open(p string) (fs.File, error) { + for _, overlay := range f.overlays { + if strings.HasPrefix(path.Clean(p), overlay.Path) { + return overlay.FS.Open(p) + } + } + return f.baseFS.Open(p) +} + +func (f overlayFS) ReadDir(p string) ([]fs.DirEntry, error) { + for _, overlay := range f.overlays { + if strings.HasPrefix(path.Clean(p), overlay.Path) { + //nolint:forcetypeassert + return overlay.FS.(fs.ReadDirFS).ReadDir(p) + } + } + //nolint:forcetypeassert + return f.baseFS.(fs.ReadDirFS).ReadDir(p) +} + +func (f overlayFS) ReadFile(p string) ([]byte, error) { + for _, overlay := range f.overlays { + if strings.HasPrefix(path.Clean(p), overlay.Path) { + //nolint:forcetypeassert + return overlay.FS.(fs.ReadFileFS).ReadFile(p) + } + } + //nolint:forcetypeassert + return f.baseFS.(fs.ReadFileFS).ReadFile(p) +} + +// valid checks that the fs.FS implements the required interfaces. +// The fs.FS interface is not sufficient. +func valid(fsys fs.FS) error { + _, ok := fsys.(fs.ReadDirFS) + if !ok { + return xerrors.New("overlayFS does not implement ReadDirFS") + } + _, ok = fsys.(fs.ReadFileFS) + if !ok { + return xerrors.New("overlayFS does not implement ReadFileFS") + } + return nil +} diff --git a/coderd/files/overlay_test.go b/coderd/files/overlay_test.go new file mode 100644 index 0000000000000..8d30f6e0a5a1f --- /dev/null +++ b/coderd/files/overlay_test.go @@ -0,0 +1,44 @@ +package files_test + +import ( + "io/fs" + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/require" + + "github.com/coder/coder/v2/coderd/files" +) + +func TestOverlayFS(t *testing.T) { + t.Parallel() + + a := afero.NewMemMapFs() + afero.WriteFile(a, "main.tf", []byte("terraform {}"), 0o644) + afero.WriteFile(a, ".terraform/modules/example_module/main.tf", []byte("inaccessible"), 0o644) + afero.WriteFile(a, ".terraform/modules/other_module/main.tf", []byte("inaccessible"), 0o644) + b := afero.NewMemMapFs() + afero.WriteFile(b, ".terraform/modules/modules.json", []byte("{}"), 0o644) + afero.WriteFile(b, ".terraform/modules/example_module/main.tf", []byte("terraform {}"), 0o644) + + it, err := files.NewOverlayFS(afero.NewIOFS(a), []files.Overlay{{ + Path: ".terraform/modules", + FS: afero.NewIOFS(b), + }}) + require.NoError(t, err) + + content, err := fs.ReadFile(it, "main.tf") + require.NoError(t, err) + require.Equal(t, "terraform {}", string(content)) + + _, err = fs.ReadFile(it, ".terraform/modules/other_module/main.tf") + require.Error(t, err) + + content, err = fs.ReadFile(it, ".terraform/modules/modules.json") + require.NoError(t, err) + require.Equal(t, "{}", string(content)) + + content, err = fs.ReadFile(it, ".terraform/modules/example_module/main.tf") + require.NoError(t, err) + require.Equal(t, "terraform {}", string(content)) +} diff --git a/coderd/parameters.go b/coderd/parameters.go index a4d6a3c18b129..6b6f4db531533 100644 --- a/coderd/parameters.go +++ b/coderd/parameters.go @@ -13,6 +13,7 @@ import ( "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/database/dbauthz" + "github.com/coder/coder/v2/coderd/files" "github.com/coder/coder/v2/coderd/httpapi" "github.com/coder/coder/v2/coderd/httpmw" "github.com/coder/coder/v2/codersdk" @@ -68,7 +69,7 @@ func (api *API) templateVersionDynamicParameters(rw http.ResponseWriter, r *http return } - fs, err := api.FileCache.Acquire(fileCtx, fileID) + templateFS, err := api.FileCache.Acquire(fileCtx, fileID) if err != nil { httpapi.Write(ctx, rw, http.StatusNotFound, codersdk.Response{ Message: "Internal error fetching template version Terraform.", @@ -85,6 +86,26 @@ func (api *API) templateVersionDynamicParameters(rw http.ResponseWriter, r *http tf, err := api.Database.GetTemplateVersionTerraformValues(ctx, templateVersion.ID) if err == nil { plan = tf.CachedPlan + + if tf.CachedModuleFiles.Valid { + moduleFilesFS, err := api.FileCache.Acquire(fileCtx, tf.CachedModuleFiles.UUID) + if err != nil { + httpapi.Write(ctx, rw, http.StatusNotFound, codersdk.Response{ + Message: "Internal error fetching Terraform modules.", + Detail: err.Error(), + }) + return + } + defer api.FileCache.Release(tf.CachedModuleFiles.UUID) + templateFS, err = files.NewOverlayFS(templateFS, []files.Overlay{{Path: ".terraform/modules", FS: moduleFilesFS}}) + if err != nil { + httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ + Message: "Internal error creating overlay filesystem.", + Detail: err.Error(), + }) + return + } + } } else if !xerrors.Is(err, sql.ErrNoRows) { httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ Message: "Failed to retrieve Terraform values for template version", @@ -124,7 +145,7 @@ func (api *API) templateVersionDynamicParameters(rw http.ResponseWriter, r *http ) // Send an initial form state, computed without any user input. - result, diagnostics := preview.Preview(ctx, input, fs) + result, diagnostics := preview.Preview(ctx, input, templateFS) response := codersdk.DynamicParametersResponse{ ID: -1, Diagnostics: previewtypes.Diagnostics(diagnostics), @@ -152,7 +173,7 @@ func (api *API) templateVersionDynamicParameters(rw http.ResponseWriter, r *http return } input.ParameterValues = update.Inputs - result, diagnostics := preview.Preview(ctx, input, fs) + result, diagnostics := preview.Preview(ctx, input, templateFS) response := codersdk.DynamicParametersResponse{ ID: update.ID, Diagnostics: previewtypes.Diagnostics(diagnostics), diff --git a/coderd/parameters_test.go b/coderd/parameters_test.go index 60189e9aeaa33..f335f60f2b8cf 100644 --- a/coderd/parameters_test.go +++ b/coderd/parameters_test.go @@ -10,6 +10,7 @@ import ( "github.com/coder/coder/v2/coderd/rbac" "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/provisioner/echo" + "github.com/coder/coder/v2/provisioner/terraform" "github.com/coder/coder/v2/provisionersdk/proto" "github.com/coder/coder/v2/testutil" "github.com/coder/websocket" @@ -132,3 +133,51 @@ func TestDynamicParametersOwnerSSHPublicKey(t *testing.T) { require.True(t, preview.Parameters[0].Value.Valid()) require.Equal(t, sshKey.PublicKey, preview.Parameters[0].Value.Value.AsString()) } + +func TestDynamicParametersWithTerraformModules(t *testing.T) { + t.Parallel() + + cfg := coderdtest.DeploymentValues(t) + cfg.Experiments = []string{string(codersdk.ExperimentDynamicParameters)} + ownerClient := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerDaemon: true, DeploymentValues: cfg}) + owner := coderdtest.CreateFirstUser(t, ownerClient) + templateAdmin, templateAdminUser := coderdtest.CreateAnotherUser(t, ownerClient, owner.OrganizationID, rbac.RoleTemplateAdmin()) + + dynamicParametersTerraformSource, err := os.ReadFile("testdata/parameters/modules/main.tf") + require.NoError(t, err) + modulesArchive, err := terraform.GetModulesArchive(os.DirFS("testdata/parameters/modules")) + require.NoError(t, err) + + files := echo.WithExtraFiles(map[string][]byte{ + "main.tf": dynamicParametersTerraformSource, + }) + files.ProvisionPlan = []*proto.Response{{ + Type: &proto.Response_Plan{ + Plan: &proto.PlanComplete{ + Plan: []byte("{}"), + ModuleFiles: modulesArchive, + }, + }, + }} + + version := coderdtest.CreateTemplateVersion(t, templateAdmin, owner.OrganizationID, files) + coderdtest.AwaitTemplateVersionJobCompleted(t, templateAdmin, version.ID) + _ = coderdtest.CreateTemplate(t, templateAdmin, owner.OrganizationID, version.ID) + + ctx := testutil.Context(t, testutil.WaitShort) + stream, err := templateAdmin.TemplateVersionDynamicParameters(ctx, templateAdminUser.ID, version.ID) + require.NoError(t, err) + defer stream.Close(websocket.StatusGoingAway) + + previews := stream.Chan() + + // Should see the output of the module represented + preview := testutil.RequireReceive(ctx, t, previews) + require.Equal(t, -1, preview.ID) + require.Empty(t, preview.Diagnostics) + + require.Len(t, preview.Parameters, 1) + require.Equal(t, "jetbrains_ide", preview.Parameters[0].Name) + require.True(t, preview.Parameters[0].Value.Valid()) + require.Equal(t, "CL", preview.Parameters[0].Value.AsString()) +} diff --git a/coderd/testdata/parameters/modules/.terraform/modules/jetbrains_gateway/main.tf b/coderd/testdata/parameters/modules/.terraform/modules/jetbrains_gateway/main.tf new file mode 100644 index 0000000000000..54c03f0a79560 --- /dev/null +++ b/coderd/testdata/parameters/modules/.terraform/modules/jetbrains_gateway/main.tf @@ -0,0 +1,94 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + coder = { + source = "coder/coder" + version = ">= 0.17" + } + } +} + +locals { + jetbrains_ides = { + "GO" = { + icon = "/icon/goland.svg", + name = "GoLand", + identifier = "GO", + }, + "WS" = { + icon = "/icon/webstorm.svg", + name = "WebStorm", + identifier = "WS", + }, + "IU" = { + icon = "/icon/intellij.svg", + name = "IntelliJ IDEA Ultimate", + identifier = "IU", + }, + "PY" = { + icon = "/icon/pycharm.svg", + name = "PyCharm Professional", + identifier = "PY", + }, + "CL" = { + icon = "/icon/clion.svg", + name = "CLion", + identifier = "CL", + }, + "PS" = { + icon = "/icon/phpstorm.svg", + name = "PhpStorm", + identifier = "PS", + }, + "RM" = { + icon = "/icon/rubymine.svg", + name = "RubyMine", + identifier = "RM", + }, + "RD" = { + icon = "/icon/rider.svg", + name = "Rider", + identifier = "RD", + }, + "RR" = { + icon = "/icon/rustrover.svg", + name = "RustRover", + identifier = "RR" + } + } + + icon = local.jetbrains_ides[data.coder_parameter.jetbrains_ide.value].icon + display_name = local.jetbrains_ides[data.coder_parameter.jetbrains_ide.value].name + identifier = data.coder_parameter.jetbrains_ide.value +} + +data "coder_parameter" "jetbrains_ide" { + type = "string" + name = "jetbrains_ide" + display_name = "JetBrains IDE" + icon = "/icon/gateway.svg" + mutable = true + default = sort(keys(local.jetbrains_ides))[0] + + dynamic "option" { + for_each = local.jetbrains_ides + content { + icon = option.value.icon + name = option.value.name + value = option.key + } + } +} + +output "identifier" { + value = local.identifier +} + +output "display_name" { + value = local.display_name +} + +output "icon" { + value = local.icon +} diff --git a/coderd/testdata/parameters/modules/.terraform/modules/modules.json b/coderd/testdata/parameters/modules/.terraform/modules/modules.json new file mode 100644 index 0000000000000..bfbd1ffc2c750 --- /dev/null +++ b/coderd/testdata/parameters/modules/.terraform/modules/modules.json @@ -0,0 +1 @@ +{"Modules":[{"Key":"","Source":"","Dir":"."},{"Key":"jetbrains_gateway","Source":"jetbrains_gateway","Dir":".terraform/modules/jetbrains_gateway"}]} diff --git a/coderd/testdata/parameters/modules/main.tf b/coderd/testdata/parameters/modules/main.tf new file mode 100644 index 0000000000000..18f14ece154f2 --- /dev/null +++ b/coderd/testdata/parameters/modules/main.tf @@ -0,0 +1,5 @@ +terraform {} + +module "jetbrains_gateway" { + source = "jetbrains_gateway" +} diff --git a/provisioner/terraform/executor.go b/provisioner/terraform/executor.go index 7d6ec689a40b1..ca353123cf3c8 100644 --- a/provisioner/terraform/executor.go +++ b/provisioner/terraform/executor.go @@ -309,7 +309,7 @@ func (e *executor) plan(ctx, killCtx context.Context, env, vars []string, logr l graphTimings.ingest(createGraphTimingsEvent(timingGraphComplete)) - moduleFiles, err := getModulesArchive(os.DirFS(e.workdir)) + moduleFiles, err := GetModulesArchive(os.DirFS(e.workdir)) if err != nil { // TODO: we probably want to persist this error or make it louder eventually e.logger.Warn(ctx, "failed to archive terraform modules", slog.Error(err)) diff --git a/provisioner/terraform/modules.go b/provisioner/terraform/modules.go index 6a3846ebbdec2..363afe3f40fc0 100644 --- a/provisioner/terraform/modules.go +++ b/provisioner/terraform/modules.go @@ -4,10 +4,12 @@ import ( "archive/tar" "bytes" "encoding/json" + "io" "io/fs" "os" "path/filepath" "strings" + "time" "golang.org/x/xerrors" @@ -68,7 +70,7 @@ func getModules(workdir string) ([]*proto.Module, error) { return filteredModules, nil } -func getModulesArchive(root fs.FS) ([]byte, error) { +func GetModulesArchive(root fs.FS) ([]byte, error) { modulesFileContent, err := fs.ReadFile(root, ".terraform/modules/modules.json") if err != nil { if xerrors.Is(err, fs.ErrNotExist) { @@ -93,31 +95,39 @@ func getModulesArchive(root fs.FS) ([]byte, error) { continue } - err := fs.WalkDir(root, it.Dir, func(filePath string, info fs.DirEntry, err error) error { + err := fs.WalkDir(root, it.Dir, func(filePath string, d fs.DirEntry, err error) error { if err != nil { return xerrors.Errorf("failed to create modules archive: %w", err) } - if info.IsDir() { + fileMode := d.Type() + if !fileMode.IsRegular() && !fileMode.IsDir() { return nil } - - content, err := fs.ReadFile(root, filePath) + fileInfo, err := d.Info() + if err != nil { + return xerrors.Errorf("failed to archive module file %q: %w", filePath, err) + } + header, err := fileHeader(filePath, fileMode, fileInfo) if err != nil { - return xerrors.Errorf("failed to read module file while archiving: %w", err) + return xerrors.Errorf("failed to archive module file %q: %w", filePath, err) + } + err = w.WriteHeader(header) + if err != nil { + return xerrors.Errorf("failed to add module file %q to archive: %w", filePath, err) + } + + if !fileMode.IsRegular() { + return nil } empty = false - err = w.WriteHeader(&tar.Header{ - Name: filePath, - Size: int64(len(content)), - Mode: 0o644, - Uid: 1000, - Gid: 1000, - }) + file, err := root.Open(filePath) if err != nil { - return xerrors.Errorf("failed to add module file to archive: %w", err) + return xerrors.Errorf("failed to open module file %q while archiving: %w", filePath, err) } - if _, err = w.Write(content); err != nil { - return xerrors.Errorf("failed to write module file to archive: %w", err) + defer file.Close() + _, err = io.Copy(w, file) + if err != nil { + return xerrors.Errorf("failed to copy module file %q while archiving: %w", filePath, err) } return nil }) @@ -126,13 +136,7 @@ func getModulesArchive(root fs.FS) ([]byte, error) { } } - err = w.WriteHeader(&tar.Header{ - Name: ".terraform/modules/modules.json", - Size: int64(len(modulesFileContent)), - Mode: 0o644, - Uid: 1000, - Gid: 1000, - }) + err = w.WriteHeader(defaultFileHeader(".terraform/modules/modules.json", len(modulesFileContent))) if err != nil { return nil, xerrors.Errorf("failed to write modules.json to archive: %w", err) } @@ -149,3 +153,35 @@ func getModulesArchive(root fs.FS) ([]byte, error) { } return b.Bytes(), nil } + +func fileHeader(filePath string, fileMode fs.FileMode, fileInfo fs.FileInfo) (*tar.Header, error) { + header, err := tar.FileInfoHeader(fileInfo, "") + if err != nil { + return nil, xerrors.Errorf("failed to archive module file %q: %w", filePath, err) + } + header.Name = filePath + if fileMode.IsDir() { + header.Name += "/" + } + // Erase a bunch of metadata that we don't need so that we get more consistent + // hashes from the resulting archive. + header.AccessTime = time.Time{} + header.ChangeTime = time.Time{} + header.ModTime = time.Time{} + header.Uid = 1000 + header.Uname = "" + header.Gid = 1000 + header.Gname = "" + + return header, nil +} + +func defaultFileHeader(filePath string, length int) *tar.Header { + return &tar.Header{ + Name: filePath, + Size: int64(length), + Mode: 0o644, + Uid: 1000, + Gid: 1000, + } +} diff --git a/provisioner/terraform/modules_internal_test.go b/provisioner/terraform/modules_internal_test.go index b971e0d7090dc..9deff602fe0aa 100644 --- a/provisioner/terraform/modules_internal_test.go +++ b/provisioner/terraform/modules_internal_test.go @@ -26,7 +26,7 @@ func TestGetModulesArchive(t *testing.T) { t.Run("Success", func(t *testing.T) { t.Parallel() - archive, err := getModulesArchive(os.DirFS(filepath.Join("testdata", "modules-source-caching"))) + archive, err := GetModulesArchive(os.DirFS(filepath.Join("testdata", "modules-source-caching"))) require.NoError(t, err) // Check that all of the files it should contain are correct @@ -37,6 +37,11 @@ func TestGetModulesArchive(t *testing.T) { require.NoError(t, err) require.True(t, strings.HasPrefix(string(content), `{"Modules":[{"Key":"","Source":"","Dir":"."},`)) + dirFiles, err := fs.ReadDir(tarfs, ".terraform/modules/example_module") + require.NoError(t, err) + require.Len(t, dirFiles, 1) + require.Equal(t, "main.tf", dirFiles[0].Name()) + content, err = fs.ReadFile(tarfs, ".terraform/modules/example_module/main.tf") require.NoError(t, err) require.True(t, strings.HasPrefix(string(content), "terraform {")) @@ -53,9 +58,9 @@ func TestGetModulesArchive(t *testing.T) { hashBytes := sha256.Sum256(archive) hash := hex.EncodeToString(hashBytes[:]) if runtime.GOOS != "windows" { - require.Equal(t, "05d2994c1a50ce573fe2c2b29507e5131ba004d15812d8bb0a46dc732f3211f5", hash) + require.Equal(t, "edcccdd4db68869552542e66bad87a51e2e455a358964912805a32b06123cb5c", hash) } else { - require.Equal(t, "c219943913051e4637527cd03ae2b7303f6945005a262cdd420f9c2af490d572", hash) + require.Equal(t, "67027a27452d60ce2799fcfd70329c185f9aee7115b0944e3aa00b4776be9d92", hash) } }) @@ -65,7 +70,7 @@ func TestGetModulesArchive(t *testing.T) { root := afero.NewMemMapFs() afero.WriteFile(root, ".terraform/modules/modules.json", []byte(`{"Modules":[{"Key":"","Source":"","Dir":"."}]}`), 0o644) - archive, err := getModulesArchive(afero.NewIOFS(root)) + archive, err := GetModulesArchive(afero.NewIOFS(root)) require.NoError(t, err) require.Equal(t, []byte{}, archive) }) diff --git a/provisioner/terraform/testdata/modules-source-caching/.terraform/modules/modules.json b/provisioner/terraform/testdata/modules-source-caching/.terraform/modules/modules.json index 8438527ba209d..710ebb1e241c3 100644 --- a/provisioner/terraform/testdata/modules-source-caching/.terraform/modules/modules.json +++ b/provisioner/terraform/testdata/modules-source-caching/.terraform/modules/modules.json @@ -1 +1 @@ -{"Modules":[{"Key":"","Source":"","Dir":"."},{"Key":"example_module","Source":"example_module","Dir":".terraform/modules/example_module"}]} \ No newline at end of file +{"Modules":[{"Key":"","Source":"","Dir":"."},{"Key":"example_module","Source":"example_module","Dir":".terraform/modules/example_module"}]} pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy