Skip to content

Commit 144b1ed

Browse files
committed
fix: reduce size of terraform modules archive
1 parent 5c77cdf commit 144b1ed

File tree

6 files changed

+340
-286
lines changed

6 files changed

+340
-286
lines changed

provisioner/terraform/modules.go

Lines changed: 67 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -24,12 +24,8 @@ type modulesFile struct {
2424
Modules []*module `json:"Modules"`
2525
}
2626

27-
func getModulesDirectory(workdir string) string {
28-
return filepath.Join(workdir, ".terraform", "modules")
29-
}
30-
3127
func getModulesFilePath(workdir string) string {
32-
return filepath.Join(getModulesDirectory(workdir), "modules.json")
28+
return filepath.Join(workdir, ".terraform", "modules", "modules.json")
3329
}
3430

3531
func parseModulesFile(filePath string) ([]*proto.Module, error) {
@@ -72,52 +68,83 @@ func getModules(workdir string) ([]*proto.Module, error) {
7268
}
7369

7470
func getModulesArchive(workdir string) ([]byte, error) {
75-
modulesDir := getModulesDirectory(workdir)
76-
if _, err := os.ReadDir(modulesDir); err != nil {
71+
modulesFile, err := os.ReadFile(getModulesFilePath(workdir))
72+
if err != nil {
7773
if os.IsNotExist(err) {
7874
return []byte{}, nil
7975
}
80-
81-
return nil, err
76+
return nil, xerrors.Errorf("failed to read modules.json: %w", err)
8277
}
78+
var m struct{ Modules []*proto.Module }
79+
err = json.Unmarshal(modulesFile, &m)
80+
if err != nil {
81+
return nil, xerrors.Errorf("failed to parse modules.json: %w", err)
82+
}
83+
8384
empty := true
8485
var b bytes.Buffer
8586
w := tar.NewWriter(&b)
86-
err := filepath.WalkDir(modulesDir, func(filePath string, info fs.DirEntry, err error) error {
87-
if err != nil {
88-
return xerrors.Errorf("failed to create modules archive: %w", err)
89-
}
90-
if info.IsDir() {
91-
return nil
92-
}
93-
archivePath, found := strings.CutPrefix(filePath, workdir+string(os.PathSeparator))
94-
if !found {
95-
return xerrors.Errorf("walked invalid file path: %q", filePath)
96-
}
9787

98-
content, err := os.ReadFile(filePath)
99-
if err != nil {
100-
return xerrors.Errorf("failed to read module file while archiving: %w", err)
88+
for _, module := range m.Modules {
89+
// Check to make sure that the module is a remote module fetched by
90+
// Terraform. Any module that doesn't start with this path is already local,
91+
// and should be part of the template files already.
92+
if !strings.HasPrefix(module.Dir, ".terraform/modules/") {
93+
continue
10194
}
102-
empty = false
103-
err = w.WriteHeader(&tar.Header{
104-
Name: archivePath,
105-
Size: int64(len(content)),
106-
Mode: 0o644,
107-
Uid: 1000,
108-
Gid: 1000,
95+
96+
err := filepath.WalkDir(filepath.Join(workdir, module.Dir), func(filePath string, info fs.DirEntry, err error) error {
97+
if err != nil {
98+
return xerrors.Errorf("failed to create modules archive: %w", err)
99+
}
100+
if info.IsDir() {
101+
return nil
102+
}
103+
archivePath, found := strings.CutPrefix(filePath, workdir+string(os.PathSeparator))
104+
if !found {
105+
return xerrors.Errorf("walked invalid file path: %q", filePath)
106+
}
107+
108+
content, err := os.ReadFile(filePath)
109+
if err != nil {
110+
return xerrors.Errorf("failed to read module file while archiving: %w", err)
111+
}
112+
empty = false
113+
err = w.WriteHeader(&tar.Header{
114+
Name: archivePath,
115+
Size: int64(len(content)),
116+
Mode: 0o644,
117+
Uid: 1000,
118+
Gid: 1000,
119+
})
120+
if err != nil {
121+
return xerrors.Errorf("failed to add module file to archive: %w", err)
122+
}
123+
if _, err = w.Write(content); err != nil {
124+
return xerrors.Errorf("failed to write module file to archive: %w", err)
125+
}
126+
return nil
109127
})
110128
if err != nil {
111-
return xerrors.Errorf("failed to add module file to archive: %w", err)
112-
}
113-
if _, err = w.Write(content); err != nil {
114-
return xerrors.Errorf("failed to write module file to archive: %w", err)
129+
return nil, err
115130
}
116-
return nil
131+
}
132+
133+
err = w.WriteHeader(&tar.Header{
134+
Name: ".terraform/modules/modules.json",
135+
Size: int64(len(modulesFile)),
136+
Mode: 0o644,
137+
Uid: 1000,
138+
Gid: 1000,
117139
})
118140
if err != nil {
119-
return nil, err
141+
return nil, xerrors.Errorf("failed to write modules.json to archive: %w", err)
142+
}
143+
_, err = w.Write(modulesFile)
144+
if err != nil {
145+
return nil, xerrors.Errorf("failed to write modules.json to archive: %w", err)
120146
}
147+
121148
err = w.Close()
122149
if err != nil {
123150
return nil, xerrors.Errorf("failed to close module files archive: %w", err)
@@ -126,5 +153,9 @@ func getModulesArchive(workdir string) ([]byte, error) {
126153
if empty {
127154
return []byte{}, nil
128155
}
156+
157+
if b.Len() > (10 << 20) {
158+
return nil, xerrors.New("modules archive exceeds 10MB, modules will not be persisted")
159+
}
129160
return b.Bytes(), nil
130161
}

provisioner/terraform/modules_internal_test.go

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,14 @@ func TestGetModulesArchive(t *testing.T) {
2525
require.NoError(t, err)
2626

2727
// Check that all of the files it should contain are correct
28-
r := bytes.NewBuffer(archive)
29-
tarfs := archivefs.FromTarReader(r)
30-
content, err := fs.ReadFile(tarfs, ".terraform/modules/example_module/main.tf")
28+
b := bytes.NewBuffer(archive)
29+
tarfs := archivefs.FromTarReader(b)
30+
31+
content, err := fs.ReadFile(tarfs, ".terraform/modules/modules.json")
32+
require.NoError(t, err)
33+
require.True(t, strings.HasPrefix(string(content), `{"Modules":[{"Key":"","Source":"","Dir":"."},`))
34+
35+
content, err = fs.ReadFile(tarfs, ".terraform/modules/example_module/main.tf")
3136
require.NoError(t, err)
3237
require.True(t, strings.HasPrefix(string(content), "terraform {"))
3338
if runtime.GOOS != "windows" {
@@ -36,6 +41,9 @@ func TestGetModulesArchive(t *testing.T) {
3641
require.Len(t, content, 3812)
3742
}
3843

44+
_, err = fs.ReadFile(tarfs, ".terraform/modules/stuff_that_should_not_be_included/nothing.txt")
45+
require.Error(t, err)
46+
3947
// It should always be byte-identical to optimize storage
4048
hashBytes := sha256.Sum256(archive)
4149
hash := hex.EncodeToString(hashBytes[:])
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
ここには何もありません

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy