From 7ab24a339c2644ff522af2e3feef7aea848b222f Mon Sep 17 00:00:00 2001
From: Vikrant Kumar Sinha <96419531+VikrantKS@users.noreply.github.com>
Date: Wed, 3 Aug 2022 18:52:32 +0530
Subject: [PATCH] feature/tas yml segregation (#223)
* added tasconfigdownloader
* refactor tasconfigdownloader
* added YMLParsingMessage struct
* modify YMLParsingMessage struct
* refactor code
* added message type for yml parsing
* fix issue with github download
* fix bitbucket download url
* fix gitlab download url
* resolve go lint error
* resolve PR comments
* fix panic in test case
---
cmd/synapse/bin.go | 7 +--
pkg/core/interfaces.go | 7 ++-
pkg/core/lifecycle.go | 9 ++--
pkg/core/models.go | 28 ++++++++++-
pkg/core/secrets.go | 2 +
pkg/core/wsproto.go | 18 ++++---
pkg/driver/builder.go | 4 +-
pkg/driver/builder_test.go | 2 +-
pkg/driver/driver_v1.go | 32 ++++++++++--
pkg/driver/driver_v2.go | 31 ++++++++++--
pkg/gitmanager/setup.go | 78 ++++++++++++++++++++++--------
pkg/gitmanager/setup_test.go | 12 ++++-
pkg/secrets/secrets.go | 7 +++
pkg/synapse/synapse.go | 44 +++++++++++++++++
pkg/synapse/utils.go | 13 +++++
pkg/tasconfigdownloader/setup.go | 51 +++++++++++++++++++
pkg/tasconfigmanager/setup.go | 53 +++++---------------
pkg/tasconfigmanager/setup_test.go | 6 ---
pkg/urlmanager/urlmanager.go | 20 ++++++++
pkg/utils/utils.go | 40 +++++++++++++++
20 files changed, 367 insertions(+), 97 deletions(-)
create mode 100644 pkg/tasconfigdownloader/setup.go
diff --git a/cmd/synapse/bin.go b/cmd/synapse/bin.go
index 20d8f193..0fba61ca 100644
--- a/cmd/synapse/bin.go
+++ b/cmd/synapse/bin.go
@@ -20,7 +20,8 @@ import (
"github.com/LambdaTest/test-at-scale/pkg/proxyserver"
"github.com/LambdaTest/test-at-scale/pkg/runner/docker"
"github.com/LambdaTest/test-at-scale/pkg/secrets"
- "github.com/LambdaTest/test-at-scale/pkg/synapse"
+ synapsepkg "github.com/LambdaTest/test-at-scale/pkg/synapse"
+ "github.com/LambdaTest/test-at-scale/pkg/tasconfigdownloader"
"github.com/LambdaTest/test-at-scale/pkg/utils"
"github.com/joho/godotenv"
"github.com/spf13/cobra"
@@ -88,8 +89,8 @@ func run(cmd *cobra.Command, args []string) {
if err != nil {
logger.Fatalf("could not instantiate k8s runner %v", err)
}
-
- synapse := synapse.New(runner, logger, secretsManager)
+ tasConfigDownloader := tasconfigdownloader.New(logger)
+ synapse := synapsepkg.New(runner, logger, secretsManager, tasConfigDownloader)
proxyHandler, err := proxyserver.NewProxyHandler(logger)
if err != nil {
diff --git a/pkg/core/interfaces.go b/pkg/core/interfaces.go
index 8b3f9024..acef7bd2 100644
--- a/pkg/core/interfaces.go
+++ b/pkg/core/interfaces.go
@@ -20,12 +20,17 @@ type TASConfigManager interface {
// GetVersion returns TAS yml version
GetVersion(path string) (int, error)
+
+ // GetTasConfigFilePath returns file path of tas config
+ GetTasConfigFilePath(payload *Payload) (string, error)
}
// GitManager manages the cloning of git repositories
type GitManager interface {
// Clone repository from TAS config
Clone(ctx context.Context, payload *Payload, oauth *Oauth) error
+ // DownloadFileByCommit download file from repo for given commit
+ DownloadFileByCommit(ctx context.Context, gitProvider, repoSlug, commitID, filePath string, oauth *Oauth) (string, error)
}
// DiffManager manages the diff findings for the given payload
@@ -173,5 +178,5 @@ type LogWriterStrategy interface {
// Builder builds the driver for given tas yml version
type Builder interface {
// GetDriver returns driver for use
- GetDriver(version int) (Driver, error)
+ GetDriver(version int, ymlFilePath string) (Driver, error)
}
diff --git a/pkg/core/lifecycle.go b/pkg/core/lifecycle.go
index fcbb95b3..0a08e3c3 100644
--- a/pkg/core/lifecycle.go
+++ b/pkg/core/lifecycle.go
@@ -136,8 +136,11 @@ func (pl *Pipeline) Start(ctx context.Context) (err error) {
return err
}
}
- // load tas yaml file
- version, err := pl.TASConfigManager.GetVersion(payload.TasFileName)
+ filePath, err := pl.TASConfigManager.GetTasConfigFilePath(pl.Payload)
+ if err != nil {
+ return err
+ }
+ version, err := pl.TASConfigManager.GetVersion(filePath)
if err != nil {
pl.Logger.Errorf("Unable to load tas yaml file, error: %v", err)
err = &errs.StatusFailed{Remark: err.Error()}
@@ -145,7 +148,7 @@ func (pl *Pipeline) Start(ctx context.Context) (err error) {
}
pl.Logger.Infof("TAS Version %f", version)
pl.setEnv(payload, coverageDir)
- newDriver, err := pl.Builder.GetDriver(version)
+ newDriver, err := pl.Builder.GetDriver(version, filePath)
if err != nil {
pl.Logger.Errorf("error crearing driver, error %v", err)
return err
diff --git a/pkg/core/models.go b/pkg/core/models.go
index 69a8a458..e5ee9570 100644
--- a/pkg/core/models.go
+++ b/pkg/core/models.go
@@ -152,9 +152,7 @@ type DiscoveryResult struct {
TaskID string `json:"taskID"`
OrgID string `json:"orgID"`
Branch string `json:"branch"`
- Tier Tier `json:"tier"`
SubModule string `json:"subModule"`
- ContainerImage string `json:"containerImage"`
}
// ExecutionResult represents the request body for test and test suite execution
@@ -456,3 +454,29 @@ type TestExecutionArgs struct {
FrameWorkVersion int
CWD string
}
+
+// YMLParsingRequestMessage defines yml parsing request received from TAS server
+type YMLParsingRequestMessage struct {
+ GitProvider string `json:"gitProvider"`
+ CommitID string `json:"commitID"`
+ Event EventType `json:"eventType"`
+ RepoSlug string `json:"repoSlug"`
+ TasFileName string `json:"tasFilePath"`
+ LicenseTier Tier `json:"license_tier"`
+ OrgID string `json:"orgID"`
+ BuildID string `json:"buildID"`
+}
+
+// TASConfigDownloaderOutput repersent output return by tasconfig downloader
+type TASConfigDownloaderOutput struct {
+ Version int `json:"version"`
+ TASConfig interface{} `json:"tasConfig"`
+}
+
+// YMLParsingResultMessage repersent message sent to TAS server in response of yml parsing request
+type YMLParsingResultMessage struct {
+ ErrorMsg string `json:"ErrorMsg"`
+ OrgID string `json:"orgID"`
+ BuildID string `json:"buildID"`
+ YMLOutput TASConfigDownloaderOutput `json:"ymlOutput"`
+}
diff --git a/pkg/core/secrets.go b/pkg/core/secrets.go
index c79bb87d..7ccb1121 100644
--- a/pkg/core/secrets.go
+++ b/pkg/core/secrets.go
@@ -20,6 +20,8 @@ type SecretsManager interface {
// GetSynapseName returns synapse name mentioned in config
GetSynapseName() string
+ // GetOauthToken returns oauth token
+ GetOauthToken() *Oauth
// GetGitSecretBytes get git secrets in bytes
GetGitSecretBytes() ([]byte, error)
diff --git a/pkg/core/wsproto.go b/pkg/core/wsproto.go
index dad0cfa8..122b84e9 100644
--- a/pkg/core/wsproto.go
+++ b/pkg/core/wsproto.go
@@ -11,14 +11,16 @@ type StatType string
// types of messages
const (
- MsgLogin MessageType = "login"
- MsgLogout MessageType = "logout"
- MsgTask MessageType = "task"
- MsgInfo MessageType = "info"
- MsgError MessageType = "error"
- MsgResourceStats MessageType = "resourcestats"
- MsgJobInfo MessageType = "jobinfo"
- MsgBuildAbort MessageType = "build_abort"
+ MsgLogin MessageType = "login"
+ MsgLogout MessageType = "logout"
+ MsgTask MessageType = "task"
+ MsgInfo MessageType = "info"
+ MsgError MessageType = "error"
+ MsgResourceStats MessageType = "resourcestats"
+ MsgJobInfo MessageType = "jobinfo"
+ MsgBuildAbort MessageType = "build_abort"
+ MsgYMLParsingRequest MessageType = "yml_parsing_request"
+ MsgYMLParsingResult MessageType = "yml_parsing_result"
)
// JobInfo types
diff --git a/pkg/driver/builder.go b/pkg/driver/builder.go
index b6dca644..64c53154 100644
--- a/pkg/driver/builder.go
+++ b/pkg/driver/builder.go
@@ -34,7 +34,7 @@ type (
}
)
-func (b *Builder) GetDriver(version int) (core.Driver, error) {
+func (b *Builder) GetDriver(version int, filePath string) (core.Driver, error) {
switch version {
case firstVersion:
return &driverV1{
@@ -49,6 +49,7 @@ func (b *Builder) GetDriver(version int) (core.Driver, error) {
DiffManager: b.DiffManager,
ListSubModuleService: b.ListSubModuleService,
TASVersion: firstVersion,
+ TASFilePath: filePath,
nodeInstaller: NodeInstaller{
logger: b.Logger,
ExecutionManager: b.ExecutionManager,
@@ -67,6 +68,7 @@ func (b *Builder) GetDriver(version int) (core.Driver, error) {
DiffManager: b.DiffManager,
ListSubModuleService: b.ListSubModuleService,
TASVersion: secondVersion,
+ TASFilePath: filePath,
nodeInstaller: NodeInstaller{
logger: b.Logger,
ExecutionManager: b.ExecutionManager,
diff --git a/pkg/driver/builder_test.go b/pkg/driver/builder_test.go
index 766d3c3f..9722bcb0 100644
--- a/pkg/driver/builder_test.go
+++ b/pkg/driver/builder_test.go
@@ -8,7 +8,7 @@ import (
func Test_driver(t *testing.T) {
b := Builder{}
invalidVersion := 4
- _, err := b.GetDriver(invalidVersion)
+ _, err := b.GetDriver(invalidVersion, "")
wantErr := fmt.Sprintf("invalid version ( %d ) mentioned in yml file", invalidVersion)
if err.Error() != wantErr {
t.Errorf("want %s , got %s", err.Error(), wantErr)
diff --git a/pkg/driver/driver_v1.go b/pkg/driver/driver_v1.go
index cf2dcda2..c508e92c 100644
--- a/pkg/driver/driver_v1.go
+++ b/pkg/driver/driver_v1.go
@@ -6,6 +6,7 @@ package driver
import (
"context"
"errors"
+ "fmt"
"os"
"github.com/LambdaTest/test-at-scale/pkg/core"
@@ -13,6 +14,7 @@ import (
"github.com/LambdaTest/test-at-scale/pkg/global"
"github.com/LambdaTest/test-at-scale/pkg/logwriter"
"github.com/LambdaTest/test-at-scale/pkg/lumber"
+ "github.com/LambdaTest/test-at-scale/pkg/utils"
"golang.org/x/sync/errgroup"
)
@@ -32,6 +34,7 @@ type (
DiffManager core.DiffManager
ListSubModuleService core.ListSubModuleService
TASVersion int
+ TASFilePath string
}
setUpResultV1 struct {
@@ -43,7 +46,7 @@ type (
func (d *driverV1) RunDiscovery(ctx context.Context, payload *core.Payload,
taskPayload *core.TaskPayload, oauth *core.Oauth, coverageDir string, secretMap map[string]string) error {
- tas, err := d.TASConfigManager.LoadAndValidate(ctx, d.TASVersion, payload.TasFileName, payload.EventType, payload.LicenseTier)
+ tas, err := d.TASConfigManager.LoadAndValidate(ctx, d.TASVersion, d.TASFilePath, payload.EventType, payload.LicenseTier)
if err != nil {
d.logger.Errorf("Unable to load tas yaml file, error: %v", err)
err = &errs.StatusFailed{Remark: err.Error()}
@@ -117,13 +120,16 @@ func (d *driverV1) RunDiscovery(ctx context.Context, payload *core.Payload,
func (d *driverV1) RunExecution(ctx context.Context, payload *core.Payload,
taskPayload *core.TaskPayload, oauth *core.Oauth, coverageDir string, secretMap map[string]string) error {
- tas, err := d.TASConfigManager.LoadAndValidate(ctx, 1, payload.TasFileName, payload.EventType, payload.LicenseTier)
+ tas, err := d.TASConfigManager.LoadAndValidate(ctx, 1, d.TASFilePath, payload.EventType, payload.LicenseTier)
if err != nil {
d.logger.Errorf("Unable to load tas yaml file, error: %v", err)
err = &errs.StatusFailed{Remark: err.Error()}
return err
}
tasConfig := tas.(*core.TASConfig)
+ if cachErr := d.setCache(tasConfig); cachErr != nil {
+ return cachErr
+ }
if errG := d.BlockTestService.GetBlockTests(ctx, tasConfig.Blocklist, payload.BranchName); errG != nil {
d.logger.Errorf("Unable to fetch blocklisted tests: %v", errG)
errG = errs.New(errs.GenericErrRemark.Error())
@@ -164,7 +170,9 @@ func (d *driverV1) RunExecution(ctx context.Context, payload *core.Payload,
func (d *driverV1) setUp(ctx context.Context, payload *core.Payload,
tasConfig *core.TASConfig, oauth *core.Oauth, language string) (*setUpResultV1, error) {
d.logger.Infof("Tas yaml: %+v", tasConfig)
-
+ if err := d.setCache(tasConfig); err != nil {
+ return nil, err
+ }
cacheKey := ""
if language == languageJs {
cacheKey = tasConfig.Cache.Key
@@ -273,6 +281,20 @@ func (d *driverV1) getEnvAndPattern(payload *core.Payload, tasConfig *core.TASCo
func populateDiscovery(testDiscoveryResult *core.DiscoveryResult, tasConfig *core.TASConfig) {
testDiscoveryResult.Parallelism = tasConfig.Parallelism
testDiscoveryResult.SplitMode = tasConfig.SplitMode
- testDiscoveryResult.ContainerImage = tasConfig.ContainerImage
- testDiscoveryResult.Tier = tasConfig.Tier
+}
+
+func (d *driverV1) setCache(tasConfig *core.TASConfig) error {
+ language := global.FrameworkLanguageMap[tasConfig.Framework]
+ if tasConfig.Cache == nil && language == "javascript" {
+ checksum, err := utils.ComputeChecksum(fmt.Sprintf("%s/%s", global.RepoDir, global.PackageJSON))
+ if err != nil {
+ d.logger.Errorf("Error while computing checksum, error %v", err)
+ return err
+ }
+ tasConfig.Cache = &core.Cache{
+ Key: checksum,
+ Paths: []string{},
+ }
+ }
+ return nil
}
diff --git a/pkg/driver/driver_v2.go b/pkg/driver/driver_v2.go
index bb716abb..34bb8be0 100644
--- a/pkg/driver/driver_v2.go
+++ b/pkg/driver/driver_v2.go
@@ -7,6 +7,7 @@ import (
"bytes"
"context"
"errors"
+ "fmt"
"os"
"path"
"strings"
@@ -17,6 +18,7 @@ import (
"github.com/LambdaTest/test-at-scale/pkg/global"
"github.com/LambdaTest/test-at-scale/pkg/logwriter"
"github.com/LambdaTest/test-at-scale/pkg/lumber"
+ "github.com/LambdaTest/test-at-scale/pkg/utils"
"golang.org/x/sync/errgroup"
)
@@ -36,6 +38,7 @@ type (
nodeInstaller NodeInstaller
TestDiscoveryService core.TestDiscoveryService
TASVersion int
+ TASFilePath string
}
setUpResultV2 struct {
@@ -49,7 +52,7 @@ func (d *driverV2) RunDiscovery(ctx context.Context, payload *core.Payload,
taskPayload *core.TaskPayload, oauth *core.Oauth, coverageDir string, secretMap map[string]string) error {
// do something
d.logger.Debugf("Running in %d version", d.TASVersion)
- tas, err := d.TASConfigManager.LoadAndValidate(ctx, d.TASVersion, payload.TasFileName, payload.EventType, payload.LicenseTier)
+ tas, err := d.TASConfigManager.LoadAndValidate(ctx, d.TASVersion, d.TASFilePath, payload.EventType, payload.LicenseTier)
if err != nil {
d.logger.Errorf("Unable to load tas yaml file, error: %v", err)
err = &errs.StatusFailed{Remark: err.Error()}
@@ -94,14 +97,18 @@ func (d *driverV2) RunDiscovery(ctx context.Context, payload *core.Payload,
func (d *driverV2) RunExecution(ctx context.Context, payload *core.Payload,
taskPayload *core.TaskPayload, oauth *core.Oauth, coverageDir string, secretMap map[string]string) error {
- tas, err := d.TASConfigManager.LoadAndValidate(ctx, d.TASVersion, payload.TasFileName, payload.EventType, payload.LicenseTier)
+ tas, err := d.TASConfigManager.LoadAndValidate(ctx, d.TASVersion, d.TASFilePath, payload.EventType, payload.LicenseTier)
if err != nil {
d.logger.Errorf("Unable to load tas yaml file, error: %v", err)
err = &errs.StatusFailed{Remark: err.Error()}
return err
}
+
subModuleName := os.Getenv(global.SubModuleName)
tasConfig := tas.(*core.TASConfigV2)
+ if cachErr := d.setCache(tasConfig); cachErr != nil {
+ return cachErr
+ }
subModule, err := d.findSubmodule(tasConfig, payload, subModuleName)
if err != nil {
d.logger.Errorf("Error finding sub module %s in tas config file", subModuleName)
@@ -355,6 +362,9 @@ func (d *driverV2) setUpDiscovery(ctx context.Context,
payload *core.Payload,
tasConfig *core.TASConfigV2,
oauth *core.Oauth) (*setUpResultV2, error) {
+ if err := d.setCache(tasConfig); err != nil {
+ return nil, err
+ }
cacheKey := tasConfig.Cache.Key
g, errCtx := errgroup.WithContext(ctx)
@@ -445,8 +455,6 @@ func populateTestDiscoveryV2(testDiscoveryResult *core.DiscoveryResult, subModul
testDiscoveryResult.Parallelism = subModule.Parallelism
testDiscoveryResult.SplitMode = tasConfig.SplitMode
testDiscoveryResult.SubModule = subModule.Name
- testDiscoveryResult.Tier = tasConfig.Tier
- testDiscoveryResult.ContainerImage = tasConfig.ContainerImage
}
func (d *driverV2) findSubmodule(tasConfig *core.TASConfigV2, payload *core.Payload, subModuleName string) (*core.SubModule, error) {
@@ -503,3 +511,18 @@ func GetSubmoduleBasedDiff(diff map[string]int, subModulePath string) map[string
}
return newDiff
}
+
+func (d *driverV2) setCache(tasConfig *core.TASConfigV2) error {
+ if tasConfig.Cache == nil {
+ checksum, err := utils.ComputeChecksum(fmt.Sprintf("%s/%s", global.RepoDir, global.PackageJSON))
+ if err != nil {
+ d.logger.Errorf("Error while computing checksum, error %v", err)
+ return err
+ }
+ tasConfig.Cache = &core.Cache{
+ Key: checksum,
+ Paths: []string{},
+ }
+ }
+ return nil
+}
diff --git a/pkg/gitmanager/setup.go b/pkg/gitmanager/setup.go
index e5a599dc..0450b18c 100644
--- a/pkg/gitmanager/setup.go
+++ b/pkg/gitmanager/setup.go
@@ -5,7 +5,6 @@ import (
"context"
"encoding/base64"
"fmt"
- "io"
"net/http"
"net/url"
"os"
@@ -13,19 +12,30 @@ import (
"strings"
"github.com/LambdaTest/test-at-scale/pkg/core"
- "github.com/LambdaTest/test-at-scale/pkg/errs"
"github.com/LambdaTest/test-at-scale/pkg/global"
"github.com/LambdaTest/test-at-scale/pkg/lumber"
+ "github.com/LambdaTest/test-at-scale/pkg/requestutils"
"github.com/LambdaTest/test-at-scale/pkg/urlmanager"
+ "github.com/LambdaTest/test-at-scale/pkg/utils"
+ "github.com/cenkalti/backoff/v4"
"github.com/mholt/archiver/v3"
)
+type GitLabSingleFileResponse struct {
+ Content string `json:"content"`
+}
+
type gitManager struct {
logger lumber.Logger
httpClient http.Client
execManager core.ExecutionManager
+ request core.Requests
}
+const (
+ authorization = "Authorization"
+)
+
// NewGitManager returns a new GitManager
func NewGitManager(logger lumber.Logger, execManager core.ExecutionManager) core.GitManager {
return &gitManager{
@@ -34,6 +44,7 @@ func NewGitManager(logger lumber.Logger, execManager core.ExecutionManager) core
Timeout: global.DefaultGitCloneTimeout,
},
execManager: execManager,
+ request: requestutils.New(logger, global.DefaultAPITimeout, &backoff.StopBackOff{}),
}
}
@@ -66,25 +77,16 @@ func (gm *gitManager) Clone(ctx context.Context, payload *core.Payload, oauth *c
// downloadFile clones the archive from github and extracts the file if it is a zip file.
func (gm *gitManager) downloadFile(ctx context.Context, archiveURL, fileName string, oauth *core.Oauth) error {
- req, err := http.NewRequestWithContext(ctx, http.MethodGet, archiveURL, nil)
+ header := getHeaderMap(oauth)
+ respBody, stausCode, err := gm.request.MakeAPIRequest(ctx, http.MethodGet, archiveURL, nil, nil, header)
if err != nil {
return err
}
- if oauth.AccessToken != "" {
- req.Header.Add("Authorization", fmt.Sprintf("%s %s", oauth.Type, oauth.AccessToken))
+ if stausCode >= http.StatusMultipleChoices {
+ return fmt.Errorf("received non 200 status code [%d]", stausCode)
}
- resp, err := gm.httpClient.Do(req)
- if err != nil {
- gm.logger.Errorf("error while making http request %v", err)
- return err
- }
- defer resp.Body.Close()
- if resp.StatusCode != http.StatusOK {
- gm.logger.Errorf("non 200 status while cloning from endpoint %s, status %d ", archiveURL, resp.StatusCode)
- return errs.ErrAPIStatus
- }
- err = gm.copyAndExtractFile(ctx, resp, fileName)
+ err = gm.copyAndExtractFile(ctx, respBody, fileName)
if err != nil {
gm.logger.Errorf("failed to copy file %v", err)
return err
@@ -94,15 +96,15 @@ func (gm *gitManager) downloadFile(ctx context.Context, archiveURL, fileName str
// copyAndExtractFile copies the content of http response directly to the local storage
// and extracts the file if it is a zip file.
-func (gm *gitManager) copyAndExtractFile(ctx context.Context, resp *http.Response, path string) error {
+func (gm *gitManager) copyAndExtractFile(ctx context.Context, respBody []byte, path string) error {
out, err := os.Create(path)
if err != nil {
gm.logger.Errorf("failed to create file err %v", err)
return err
}
- _, err = io.Copy(out, resp.Body)
+ _, err = out.Write(respBody)
if err != nil {
- gm.logger.Errorf("failed to copy file %v", err)
+ gm.logger.Errorf("failed to write to file %v", err)
out.Close()
return err
}
@@ -115,7 +117,6 @@ func (gm *gitManager) copyAndExtractFile(ctx context.Context, resp *http.Respons
if err = zip.Unarchive(path, fmt.Sprintf("%s/clonedir", filepath.Dir(path))); err != nil {
gm.logger.Errorf("failed to unarchive file %v", err)
return err
-
}
}
@@ -174,3 +175,40 @@ func (gm *gitManager) initGit(ctx context.Context, payload *core.Payload, oauth
}
return nil
}
+
+func (gm *gitManager) DownloadFileByCommit(ctx context.Context, gitProvider, repoSlug,
+ commitID, filePath string, oauth *core.Oauth) (string, error) {
+ downloadURL, err := urlmanager.GetFileDownloadURL(gitProvider, commitID, repoSlug, filePath)
+ if err != nil {
+ return "", err
+ }
+ header := getHeaderMap(oauth)
+ respBody, stausCode, err := gm.request.MakeAPIRequest(ctx, http.MethodGet, downloadURL, nil, nil, header)
+ if err != nil {
+ return "", err
+ }
+ if stausCode >= http.StatusMultipleChoices {
+ return "", fmt.Errorf("received non 200 status code [%d]", stausCode)
+ }
+ path := utils.GenerateUUID() + ".yml"
+ out, err := os.Create(path)
+ if err != nil {
+ gm.logger.Errorf("failed to create file err %v", err)
+ return "", err
+ }
+ _, err = out.Write(respBody)
+ if err != nil {
+ gm.logger.Errorf("failed to copy file %v", err)
+ out.Close()
+ return "", err
+ }
+ out.Close()
+ return path, nil
+}
+
+func getHeaderMap(oauth *core.Oauth) map[string]string {
+ header := map[string]string{}
+
+ header[authorization] = fmt.Sprintf("%s %s", oauth.Type, oauth.AccessToken)
+ return header
+}
diff --git a/pkg/gitmanager/setup_test.go b/pkg/gitmanager/setup_test.go
index dc0e4f10..0cd9744b 100644
--- a/pkg/gitmanager/setup_test.go
+++ b/pkg/gitmanager/setup_test.go
@@ -17,7 +17,9 @@ import (
"github.com/LambdaTest/test-at-scale/pkg/core"
"github.com/LambdaTest/test-at-scale/pkg/global"
"github.com/LambdaTest/test-at-scale/pkg/lumber"
+ "github.com/LambdaTest/test-at-scale/pkg/requestutils"
"github.com/LambdaTest/test-at-scale/testutils"
+ "github.com/cenkalti/backoff/v4"
"github.com/stretchr/testify/mock"
)
@@ -60,6 +62,7 @@ func Test_downloadFile(t *testing.T) {
logger: logger,
httpClient: httpClient,
execManager: execManager,
+ request: requestutils.New(logger, global.DefaultAPITimeout, &backoff.StopBackOff{}),
}
archiveURL := server.URL + "/archive/zipfile.zip"
fileName := "copyAndExtracted"
@@ -93,6 +96,7 @@ func Test_copyAndExtractFile(t *testing.T) {
logger: logger,
httpClient: httpClient,
execManager: execManager,
+ request: requestutils.New(logger, global.DefaultAPITimeout, &backoff.StopBackOff{}),
}
fileBody := "Hello World!"
resp := http.Response{
@@ -100,7 +104,13 @@ func Test_copyAndExtractFile(t *testing.T) {
}
path := "newFile"
defer removeFile(path)
- err2 := gm.copyAndExtractFile(context.TODO(), &resp, path)
+ respBodyBuffer := bytes.Buffer{}
+ _, err = io.Copy(&respBodyBuffer, resp.Body)
+ if err != nil {
+ t.Errorf("Error: %v", err)
+ return
+ }
+ err2 := gm.copyAndExtractFile(context.TODO(), respBodyBuffer.Bytes(), path)
if err2 != nil {
t.Errorf("Error: %v", err2)
return
diff --git a/pkg/secrets/secrets.go b/pkg/secrets/secrets.go
index ba906c5a..7591eb11 100644
--- a/pkg/secrets/secrets.go
+++ b/pkg/secrets/secrets.go
@@ -89,3 +89,10 @@ func (s *secertManager) GetDockerSecrets(r *core.RunnerOptions) (core.ContainerI
containerImageConfig.AuthRegistry = base64.StdEncoding.EncodeToString(jsonBytes)
return containerImageConfig, nil
}
+
+func (s *secertManager) GetOauthToken() *core.Oauth {
+ return &core.Oauth{
+ AccessToken: s.cfg.Git.Token,
+ Type: core.TokenType(s.cfg.Git.TokenType),
+ }
+}
diff --git a/pkg/synapse/synapse.go b/pkg/synapse/synapse.go
index a7410283..10174f9b 100644
--- a/pkg/synapse/synapse.go
+++ b/pkg/synapse/synapse.go
@@ -10,6 +10,7 @@ import (
"github.com/LambdaTest/test-at-scale/pkg/core"
"github.com/LambdaTest/test-at-scale/pkg/global"
"github.com/LambdaTest/test-at-scale/pkg/lumber"
+ "github.com/LambdaTest/test-at-scale/pkg/tasconfigdownloader"
"github.com/cenkalti/backoff/v4"
"github.com/denisbrodbeck/machineid"
"github.com/gorilla/websocket"
@@ -40,6 +41,7 @@ type synapse struct {
ConnectionAborted chan struct{}
InvalidConnectionRequest chan struct{}
LogoutRequired bool
+ tasConfigDownloader *tasconfigdownloader.TASConfigDownloader
}
// New returns new instance of synapse
@@ -47,6 +49,7 @@ func New(
runner core.DockerRunner,
logger lumber.Logger,
secretsManager core.SecretsManager,
+ tasConfigDownloader *tasconfigdownloader.TASConfigDownloader,
) core.SynapseManager {
return &synapse{
runner: runner,
@@ -57,6 +60,7 @@ func New(
MsgChan: make(chan []byte, 1024),
ConnectionAborted: make(chan struct{}, 10),
LogoutRequired: true,
+ tasConfigDownloader: tasConfigDownloader,
}
}
@@ -216,6 +220,9 @@ func (s *synapse) processMessage(msg []byte, duplicateConnectionChan chan struct
case core.MsgTask:
s.logger.Debugf("task message received from server")
go s.processTask(message)
+ case core.MsgYMLParsingRequest:
+ s.logger.Debugf("yml parsing request received from server")
+ go s.processYMLParsingRequest(message)
case core.MsgBuildAbort:
s.logger.Debugf("abort-build message received from server")
go s.processAbortBuild(message)
@@ -376,3 +383,40 @@ func (s *synapse) messageWriter(conn *websocket.Conn) {
}
}
}
+
+func (s *synapse) processYMLParsingRequest(message core.Message) {
+ var parsingReqMsg *core.YMLParsingRequestMessage
+ var writeMsg core.Message
+ defer s.writeMessageToBuffer(&writeMsg)
+ if err := json.Unmarshal(message.Content, &parsingReqMsg); err != nil {
+ s.logger.Errorf("error in unmarshaling message for yml parsing request, error %v ", err)
+
+ writeMsg = createYMlParsingResultMessage(core.YMLParsingResultMessage{
+ OrgID: parsingReqMsg.OrgID,
+ BuildID: parsingReqMsg.BuildID,
+ ErrorMsg: err.Error(),
+ })
+ return
+ }
+ oauth := s.secretsManager.GetOauthToken()
+
+ tasOutput, err := s.tasConfigDownloader.GetTASConfig(context.TODO(), parsingReqMsg.GitProvider,
+ parsingReqMsg.CommitID,
+ parsingReqMsg.RepoSlug, parsingReqMsg.TasFileName, oauth,
+ parsingReqMsg.Event, parsingReqMsg.LicenseTier)
+ if err != nil {
+ s.logger.Errorf("error occurred while fetching tas config file for buildID %s orgID %s, error %v",
+ parsingReqMsg.BuildID, parsingReqMsg.OrgID, err)
+ writeMsg = createYMlParsingResultMessage(core.YMLParsingResultMessage{
+ OrgID: parsingReqMsg.OrgID,
+ BuildID: parsingReqMsg.BuildID,
+ ErrorMsg: err.Error(),
+ })
+ return
+ }
+ writeMsg = createYMlParsingResultMessage(core.YMLParsingResultMessage{
+ OrgID: parsingReqMsg.OrgID,
+ BuildID: parsingReqMsg.BuildID,
+ YMLOutput: *tasOutput,
+ })
+}
diff --git a/pkg/synapse/utils.go b/pkg/synapse/utils.go
index a0ce92b3..c68620fb 100644
--- a/pkg/synapse/utils.go
+++ b/pkg/synapse/utils.go
@@ -75,3 +75,16 @@ func GetResources(tierOpts core.Tier) core.Specs {
}
return core.Specs{CPU: 0, RAM: 0}
}
+
+// createYMlParsingResultMessage creates message for YML parsing result
+func createYMlParsingResultMessage(ymlParsingOutput core.YMLParsingResultMessage) core.Message {
+ ymlParsingOutputJSON, err := json.Marshal(ymlParsingOutput)
+ if err != nil {
+ return core.Message{}
+ }
+ return core.Message{
+ Type: core.MsgYMLParsingResult,
+ Content: ymlParsingOutputJSON,
+ Success: true,
+ }
+}
diff --git a/pkg/tasconfigdownloader/setup.go b/pkg/tasconfigdownloader/setup.go
new file mode 100644
index 00000000..c666734d
--- /dev/null
+++ b/pkg/tasconfigdownloader/setup.go
@@ -0,0 +1,51 @@
+package tasconfigdownloader
+
+import (
+ "context"
+ "os"
+
+ "github.com/LambdaTest/test-at-scale/pkg/core"
+ "github.com/LambdaTest/test-at-scale/pkg/gitmanager"
+ "github.com/LambdaTest/test-at-scale/pkg/lumber"
+ "github.com/LambdaTest/test-at-scale/pkg/tasconfigmanager"
+)
+
+type TASConfigDownloader struct {
+ logger lumber.Logger
+ gitmanager core.GitManager
+ tasconfigmanager core.TASConfigManager
+}
+
+func New(logger lumber.Logger) *TASConfigDownloader {
+ return &TASConfigDownloader{
+ logger: logger,
+ gitmanager: gitmanager.NewGitManager(logger, nil),
+ tasconfigmanager: tasconfigmanager.NewTASConfigManager(logger),
+ }
+}
+
+func (t *TASConfigDownloader) GetTASConfig(ctx context.Context, gitProvider, commitID, repoSlug,
+ filePath string, oauth *core.Oauth, eventType core.EventType, licenseTier core.Tier) (*core.TASConfigDownloaderOutput, error) {
+ ymlPath, err := t.gitmanager.DownloadFileByCommit(ctx, gitProvider, repoSlug, commitID, filePath, oauth)
+ if err != nil {
+ t.logger.Errorf("error occurred while downloading file %s from %s for commitID %s, error %v", filePath, repoSlug, commitID, err)
+ return nil, err
+ }
+
+ version, err := t.tasconfigmanager.GetVersion(ymlPath)
+ if err != nil {
+ t.logger.Errorf("error reading version for tas config file %s, error %v", ymlPath, err)
+ return nil, err
+ }
+
+ tasConfig, err := t.tasconfigmanager.LoadAndValidate(ctx, version, ymlPath, eventType, licenseTier)
+ if err != nil {
+ t.logger.Errorf("error while parsing yml for commitID %s error %v", commitID, err)
+ return nil, err
+ }
+ if err := os.Remove(ymlPath); err != nil {
+ t.logger.Errorf("failed to delete file %s , error %v", ymlPath, err)
+ return nil, err
+ }
+ return &core.TASConfigDownloaderOutput{Version: version, TASConfig: tasConfig}, nil
+}
diff --git a/pkg/tasconfigmanager/setup.go b/pkg/tasconfigmanager/setup.go
index fe309842..025ad973 100644
--- a/pkg/tasconfigmanager/setup.go
+++ b/pkg/tasconfigmanager/setup.go
@@ -24,7 +24,6 @@ var tierEnumMapping = map[core.Tier]int{
core.Large: 4,
core.XLarge: 5,
}
-var getTasFilePathFn = getTasFilePath
// tasConfigManager represents an instance of TASConfigManager instance
type tasConfigManager struct {
@@ -48,17 +47,13 @@ func (tc *tasConfigManager) LoadAndValidate(ctx context.Context,
}
func (tc *tasConfigManager) loadAndValidateV1(ctx context.Context,
- path string,
+ filePath string,
eventType core.EventType,
licenseTier core.Tier) (*core.TASConfig, error) {
- filePath, err := getTasFilePathFn(path)
- if err != nil {
- return nil, err
- }
yamlFile, err := os.ReadFile(filePath)
if err != nil {
tc.logger.Errorf("Error while reading file %s, error %v", filePath, err)
- return nil, errs.New(fmt.Sprintf("Error while reading configuration file at path: %s", path))
+ return nil, errs.New(fmt.Sprintf("Error while reading configuration file at path: %s", filePath))
}
return tc.validateYMLV1(ctx, yamlFile, eventType, licenseTier, filePath)
}
@@ -91,19 +86,6 @@ func (tc *tasConfigManager) validateYMLV1(ctx context.Context,
tc.logger.Errorf("LicenseTier validation failed. error: %v", err)
return nil, err
}
-
- language := global.FrameworkLanguageMap[tasConfig.Framework]
- if tasConfig.Cache == nil && language == "javascript" {
- checksum, err := utils.ComputeChecksum(fmt.Sprintf("%s/%s", global.RepoDir, global.PackageJSON))
- if err != nil {
- tc.logger.Errorf("Error while computing checksum, error %v", err)
- return nil, err
- }
- tasConfig.Cache = &core.Cache{
- Key: checksum,
- Paths: []string{},
- }
- }
return tasConfig, nil
}
@@ -167,22 +149,12 @@ func (tc *tasConfigManager) validateYMLV2(ctx context.Context,
tc.logger.Errorf("LicenseTier validation failed. error: %v", err)
return nil, err
}
- if tasConfig.Cache == nil {
- checksum, err := utils.ComputeChecksum(fmt.Sprintf("%s/%s", global.RepoDir, packageJSON))
- if err != nil {
- tc.logger.Errorf("Error while computing checksum, error %v", err)
- return nil, err
- }
- tasConfig.Cache = &core.Cache{
- Key: checksum,
- Paths: []string{},
- }
- }
+
return tasConfig, nil
}
func (tc *tasConfigManager) GetVersion(path string) (int, error) {
- yamlFile, err := os.ReadFile(fmt.Sprintf("%s/%s", global.RepoDir, path))
+ yamlFile, err := os.ReadFile(path)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return 0, errs.New(fmt.Sprintf("Configuration file not found at path: %s", path))
@@ -199,29 +171,26 @@ func (tc *tasConfigManager) GetVersion(path string) (int, error) {
}
func (tc *tasConfigManager) loadAndValidateV2(ctx context.Context,
- path string,
+ yamlFilePath string,
eventType core.EventType,
licenseTier core.Tier) (*core.TASConfigV2, error) {
- yamlFilePath, err := getTasFilePathFn(path)
- if err != nil {
- return nil, err
- }
yamlFile, err := os.ReadFile(yamlFilePath)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
- return nil, errs.New(fmt.Sprintf("Configuration file not found at path: %s", path))
+ return nil, errs.New(fmt.Sprintf("Configuration file not found at path: %s", yamlFilePath))
}
tc.logger.Errorf("Error while reading file, error %v", err)
- return nil, errs.New(fmt.Sprintf("Error while reading configuration file at path: %s", path))
+ return nil, errs.New(fmt.Sprintf("Error while reading configuration file at path: %s", yamlFilePath))
}
return tc.validateYMLV2(ctx, yamlFile, eventType, licenseTier, yamlFilePath)
}
-func getTasFilePath(path string) (string, error) {
- path, err := utils.GetConfigFileName(path)
+func (tc *tasConfigManager) GetTasConfigFilePath(payload *core.Payload) (string, error) {
+ // load tas yaml file
+ filePath, err := utils.GetTASFilePath(payload.TasFileName)
if err != nil {
+ tc.logger.Errorf("Unable to load tas yaml file, error: %v", err)
return "", err
}
- filePath := fmt.Sprintf("%s/%s", global.RepoDir, path)
return filePath, nil
}
diff --git a/pkg/tasconfigmanager/setup_test.go b/pkg/tasconfigmanager/setup_test.go
index 10419a66..1cf13b5f 100644
--- a/pkg/tasconfigmanager/setup_test.go
+++ b/pkg/tasconfigmanager/setup_test.go
@@ -86,9 +86,6 @@ func TestLoadAndValidateV1(t *testing.T) {
if err != nil {
t.Errorf("Couldn't initialize logger, error: %v", err)
}
- getTasFilePathFn = func(path string) (string, error) {
- return path, nil
- }
tasConfigManager := NewTASConfigManager(logger)
ctx := context.TODO()
@@ -180,9 +177,6 @@ func TestLoadAndValidateV2(t *testing.T) {
if err != nil {
t.Errorf("Couldn't initialize logger, error: %v", err)
}
- getTasFilePathFn = func(path string) (string, error) {
- return path, nil
- }
tasConfigManager := NewTASConfigManager(logger)
ctx := context.TODO()
diff --git a/pkg/urlmanager/urlmanager.go b/pkg/urlmanager/urlmanager.go
index 918ba54a..9f4693c6 100644
--- a/pkg/urlmanager/urlmanager.go
+++ b/pkg/urlmanager/urlmanager.go
@@ -80,3 +80,23 @@ func GetPullRequestDiffURL(gitprovider, path string, prNumber int) (string, erro
return "", errs.ErrUnsupportedGitProvider
}
}
+
+// GetFileDownloadURL returns download URL for file in repo
+func GetFileDownloadURL(gitprovider, commitID, repoSlug, filePath string) (string, error) {
+ if global.TestEnv {
+ return global.TestServer, nil
+ }
+ switch gitprovider {
+ case core.GitHub:
+ return fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s", repoSlug, commitID, filePath), nil
+ case core.GitLab:
+ repoSlug = url.PathEscape(repoSlug)
+ filePath = url.PathEscape(filePath)
+ return fmt.Sprintf("%s/%s/repository/files/%s/raw?ref=%s", global.APIHostURLMap[gitprovider], repoSlug, filePath, commitID), nil
+ case core.Bitbucket:
+ // TODO: check for fork PR
+ return fmt.Sprintf("%s/repositories/%s/src/%s/%s", global.APIHostURLMap[gitprovider], repoSlug, commitID, filePath), nil
+ default:
+ return "", nil
+ }
+}
diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go
index fa13e390..36f6b6e7 100644
--- a/pkg/utils/utils.go
+++ b/pkg/utils/utils.go
@@ -19,6 +19,7 @@ import (
ut "github.com/go-playground/universal-translator"
"github.com/go-playground/validator/v10"
en_translations "github.com/go-playground/validator/v10/translations/en"
+ "github.com/google/uuid"
"gopkg.in/yaml.v3"
)
@@ -27,6 +28,8 @@ const (
emptyTagName = "-"
yamlTagName = "yaml"
requiredTagName = "required"
+ v1 = 1
+ v2 = 2
)
// Min returns the smaller of x or y.
@@ -269,3 +272,40 @@ func GetArgs(command string, frameWork string, frameworkVersion int,
return args
}
+
+// GetTASFilePath returns tas file path
+func GetTASFilePath(path string) (string, error) {
+ path, err := GetConfigFileName(path)
+ if err != nil {
+ return "", err
+ }
+ filePath := fmt.Sprintf("%s/%s", global.RepoDir, path)
+ return filePath, nil
+}
+
+// GenerateUUID generates uuid v4
+func GenerateUUID() string {
+ uuidV4 := uuid.New() // panics on error
+ return strings.Map(func(r rune) rune {
+ if r == '-' {
+ return -1
+ }
+ return r
+ }, uuidV4.String())
+}
+
+// ValidateStructTASYml validates the TAS config for all supported version
+func ValidateStructTASYml(ctx context.Context, ymlContent []byte, ymlFilename string) (interface{}, error) {
+ version, err := GetVersion(ymlContent)
+ if err != nil {
+ return nil, err
+ }
+ switch version {
+ case v1:
+ return ValidateStructTASYmlV1(ctx, ymlContent, ymlFilename)
+ case v2:
+ return ValidateStructTASYmlV2(ctx, ymlContent, ymlFilename)
+ default:
+ return nil, fmt.Errorf("")
+ }
+}
pFad - Phonifier reborn
Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.
Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.
Alternative Proxies:
Alternative Proxy
pFad Proxy
pFad v3 Proxy
pFad v4 Proxy