adding daisy chain transfers and configurable retries for terminating…

… messages (#4)
Minehut · Feb 27, 2023 · 04c5e8c · 04c5e8c
1 parent 2f3b9f4
commit 04c5e8c
Show file tree

Hide file tree

Showing 14 changed files with 127 additions and 47 deletions.
diff --git a/CONFIGURE.md b/CONFIGURE.md
@@ -29,24 +29,30 @@ apiVersion: v1
 
 logLevel: info
 shutdownWait: 30s
+skipEventBucketValidation: true
 skipLifecycleExpired: true
+maxRetries: 5
 msgTimeout: 15m
 excludePaths:
   copyObject:
     - ^\w{3}/(\w+)/\1\.tar\.zst$
   removeObject:
     - ^\w{3}/(\w+)/\1\.tar\.zst$
+waitForMatchingETag: false
 ```
 
 | Setting                     | Description                                                                                                             |
 |-----------------------------|-------------------------------------------------------------------------------------------------------------------------|
 | `apiVersion`                | config file api version (required)                                                                                      |
 | `logLevel`                  | set the log level (default: info)                                                                                       |
 | `shutdownWait`              | time to wait for running transfers to complete before exiting                                                           |
+| `skipEventBucketValidation` | don't check if the event's bucket name and source bucket name match                                                     |
 | `skipLifecycleExpired`      | don't propagate deletes initiated by the minio lifecycle expiration                                                     |
+| `maxRetries`                | the max retries for to retry when either the copy's source object or the object to be deleted are missing               |
 | `msgTimeout`                | the max duration for a transfer includes the jetstream stream message ack timeout and internal transfer context timeout |
 | `excludePaths.copyObject`   | list of paths as regex patterns to exclude from copy operations   (pcre support)                                        |
 | `excludePaths.removeObject` | list of paths as regex patterns to exclude from delete operations (pcre support)                                        |
+| `waitForMatchingETag`       | when copying files wait for the matching etag                                                                           |
 
 
 ### JetStream Options

diff --git a/archie/archiver.go b/archie/archiver.go
@@ -7,21 +7,24 @@ import (
 )
 
 type Archiver struct {
-	DestBucket           string
-	DestClient           client.Client
-	DestName             string
-	DestPartSize         uint64
-	DestThreads          uint
-	FetchDone            chan string
-	HealthCheckDisabled  bool
-	IsOffline            bool
-	MsgTimeout           string
-	SkipLifecycleExpired bool
-	SrcBucket            string
-	SrcClient            client.Client
-	SrcName              string
-	WaitGroup            *sync.WaitGroup
-	ExcludePaths         struct {
+	DestBucket                string
+	DestClient                client.Client
+	DestName                  string
+	DestPartSize              uint64
+	DestThreads               uint
+	FetchDone                 chan string
+	HealthCheckDisabled       bool
+	IsOffline                 bool
+	MaxRetries                uint64
+	MsgTimeout                string
+	SkipEventBucketValidation bool
+	SkipLifecycleExpired      bool
+	SrcBucket                 string
+	SrcClient                 client.Client
+	SrcName                   string
+	WaitForMatchingETag       bool
+	WaitGroup                 *sync.WaitGroup
+	ExcludePaths              struct {
 		CopyObject   []*pcre.Regexp
 		RemoveObject []*pcre.Regexp
 	}
@@ -34,7 +37,7 @@ const (
 	Nak
 	SkipAck
 	Term
-	FiveNakThenTerm
+	NakThenTerm
 	None
 )
 
@@ -46,8 +49,8 @@ func (s AckType) String() string {
 		return "nak"
 	case Term:
 		return "term"
-	case FiveNakThenTerm:
-		return "5nak_then_term"
+	case NakThenTerm:
+		return "nak_then_term"
 	case None:
 		return "none"
 	}

diff --git a/archie/copyObject.go b/archie/copyObject.go
@@ -2,13 +2,15 @@ package archie
 
 import (
 	"archie/client"
+	"archie/event"
 	"context"
+	"fmt"
 	"github.com/nats-io/nats.go"
 	"github.com/rs/zerolog"
 	"time"
 )
 
-func (a *Archiver) copyObject(ctx context.Context, mLog zerolog.Logger, eventObjKey string, msg *nats.Msg) (error, string, AckType) {
+func (a *Archiver) copyObject(ctx context.Context, mLog zerolog.Logger, eventObjKey string, msg *nats.Msg, record event.Record) (error, string, AckType) {
 	metadata, _ := msg.Metadata()
 
 	for _, excludedPathRegexp := range a.ExcludePaths.CopyObject {
@@ -38,15 +40,23 @@ func (a *Archiver) copyObject(ctx context.Context, mLog zerolog.Logger, eventObj
 	if err != nil {
 		if err.Error() == "The specified key does not exist." {
 			// minio error
-			return err, "Failed to Stat the source object", FiveNakThenTerm
+			return err, "Failed to Stat the source object", NakThenTerm
 		} else if err.Error() == "storage: object doesn't exist" {
 			// gcs error
-			return err, "Failed to Stat the source object", FiveNakThenTerm
+			return err, "Failed to Stat the source object", NakThenTerm
 		} else {
 			return err, "Failed to Stat the source object", Nak
 		}
 	}
 
+	if a.WaitForMatchingETag {
+		if srcStat.ETag != record.S3.Object.ETag {
+			return fmt.Errorf(
+				"mismatch of ETags from the event (%s) and source (%s)", record.S3.Object.ETag, srcStat.ETag,
+			), "ETag mismatch", Nak
+		}
+	}
+
 	mLog.Info().
 		Int64("size", srcStat.Size).
 		Str("hSize", size(srcStat.Size)).
@@ -60,6 +70,10 @@ func (a *Archiver) copyObject(ctx context.Context, mLog zerolog.Logger, eventObj
 		PartSize:    destPartSizeBytes,
 	}
 
+	if record.S3.Object.ETag != "" {
+		putOpts.ETag = record.S3.Object.ETag
+	}
+
 	start = time.Now()
 	_, err = a.DestClient.PutObject(ctx, a.DestBucket, eventObjKey, srcObject.GetReader(), srcStat.Size, putOpts)
 	if err != nil {

diff --git a/archie/message.go b/archie/message.go
@@ -100,7 +100,7 @@ func (a *Archiver) message(ctx context.Context, msg *nats.Msg) {
 		// message type router
 		switch eventType {
 		case "s3:ObjectCreated":
-			err, execContext, ack = a.copyObject(ctx, mLog, eventObjKey, msg)
+			err, execContext, ack = a.copyObject(ctx, mLog, eventObjKey, msg, eventRecord)
 			if err != nil {
 				s3ErrMsg, s3ErrCode = logS3Error(err, execContext, &mLog)
 			}
@@ -133,8 +133,8 @@ func (a *Archiver) message(ctx context.Context, msg *nats.Msg) {
 		case Nak:
 			sendNakSignal(msg, &mLog)
 			a.cleanupAndCountMessagesProcessedMetric("failed", s3ErrMsg, s3ErrCode, event.EventName, eventType)
-		case FiveNakThenTerm:
-			maxDelivered := uint64(5)
+		case NakThenTerm:
+			maxDelivered := a.MaxRetries - 1
 			if metadata.NumDelivered > maxDelivered {
 				mLog.Error().Uint64("numDelivered", metadata.NumDelivered).Msg("Reached max delivered")
 				termErr := sendTermSignal(msg, &mLog)

diff --git a/archie/messageValidation.go b/archie/messageValidation.go
@@ -18,8 +18,10 @@ func (a *Archiver) validateEventName(event event.Minio) error {
 
 // validate src bucket name in config matches event bucket name
 func (a *Archiver) validateEventBucket(eventBucket string) error {
-	if a.SrcBucket != eventBucket {
-		return fmt.Errorf("event bucket (%s) doesn't match configured source bucket (%s)", eventBucket, a.SrcBucket)
+	if !a.SkipEventBucketValidation {
+		if a.SrcBucket != eventBucket {
+			return fmt.Errorf("event bucket (%s) doesn't match configured source bucket (%s)", eventBucket, a.SrcBucket)
+		}
 	}
 	return nil
 }
diff --git a/archie/removeObject.go b/archie/removeObject.go
@@ -44,10 +44,10 @@ func (a *Archiver) removeObject(ctx context.Context, mLog zerolog.Logger, eventO
 	if err != nil {
 		if err.Error() == "The specified key does not exist." {
 			// minio error
-			return err, "Failed to RemoveObject from destination bucket", FiveNakThenTerm
+			return err, "Failed to RemoveObject from destination bucket", NakThenTerm
 		} else if err.Error() == "storage: object doesn't exist" {
 			// gcs error
-			return err, "Failed to RemoveObject from destination bucket", FiveNakThenTerm
+			return err, "Failed to RemoveObject from destination bucket", NakThenTerm
 		} else {
 			return err, "Failed to RemoveObject from destination bucket", Nak
 		}

diff --git a/client/client.go b/client/client.go
@@ -22,12 +22,14 @@ type Object interface {
 
 type PutOptions struct {
 	ContentType string
+	ETag        string
 	NumThreads  uint
 	PartSize    uint64
 }
 
 type ObjectInfo struct {
 	ContentType string
+	ETag        string
 	Size        int64
 }
 

diff --git a/client/gcs.go b/client/gcs.go
@@ -121,7 +121,7 @@ func (o *GCSObject) Stat(ctx context.Context) (*ObjectInfo, error) {
 	if err != nil {
 		return nil, err
 	}
-	return &ObjectInfo{Size: obj.Size, ContentType: obj.ContentType}, nil
+	return &ObjectInfo{Size: obj.Size, ContentType: obj.ContentType, ETag: obj.Etag}, nil
 }
 
 func (o *GCSObject) GetReader() io.Reader {

diff --git a/client/minio.go b/client/minio.go
@@ -79,6 +79,11 @@ func (m *Minio) PutObject(ctx context.Context, bucket string, key string, reader
 		PartSize:    opts.PartSize,
 	}
 
+	if opts.ETag != "" {
+		putOpts.UserMetadata = map[string]string{
+			"eTag": opts.ETag,
+		}
+	}
 	_, err := m.client.PutObject(ctx, bucket, key, reader, objectSize, putOpts)
 	if err != nil {
 		return UploadInfo{}, err
@@ -107,7 +112,7 @@ func (o *MinioObject) Stat(ctx context.Context) (*ObjectInfo, error) {
 	if err != nil {
 		return nil, err
 	}
-	return &ObjectInfo{Size: srcStat.Size, ContentType: srcStat.ContentType}, nil
+	return &ObjectInfo{Size: srcStat.Size, ContentType: srcStat.ContentType, ETag: srcStat.ETag}, nil
 }
 
 func (o *MinioObject) GetReader() io.Reader {

diff --git a/config.go b/config.go
@@ -3,10 +3,13 @@ package main
 type Config struct {
 	ApiVersion string `fig:"apiVersion" validate:"required"`
 
-	LogLevel             string `fig:"logLevel" default:"info"`
-	MsgTimeout           string `fig:"msgTimeout" default:"30m"`
-	ShutdownWait         string `fig:"shutdownWait" default:"0s"`
-	SkipLifecycleExpired bool   `fig:"skipLifecycleExpired"`
+	LogLevel                  string `fig:"logLevel" default:"info"`
+	MaxRetries                uint64 `fig:"maxRetries" default:"5"`
+	MsgTimeout                string `fig:"msgTimeout" default:"30m"`
+	ShutdownWait              string `fig:"shutdownWait" default:"0s"`
+	SkipEventBucketValidation bool   `fig:"skipEventBucketValidation"`
+	SkipLifecycleExpired      bool   `fig:"skipLifecycleExpired"`
+	WaitForMatchingETag       bool   `fig:"waitForMatchingETag"`
 
 	Src struct {
 		AccessKey         string `fig:"accessKey"`

diff --git a/helm/archie/Chart.yaml b/helm/archie/Chart.yaml
@@ -7,10 +7,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.3.3
+version: 0.4.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "0.3.0"
+appVersion: "0.4.0"
diff --git a/helm/archie/templates/secret.yaml b/helm/archie/templates/secret.yaml
@@ -13,6 +13,12 @@ stringData:
     logLevel: {{ .Values.archie.logLevel }}
     shutdownWait: {{ $shutdownWaitDuration }}
     skipLifecycleExpired: {{ .Values.archie.skipLifecycleExpired }}
+    skipEventBucketValidation: {{ .Values.archie.skipEventBucketValidation }}
+    waitForMatchingETag: {{ .Values.archie.waitForMatchingETag }}
+
+    {{- if .Values.archie.maxRetries }}
+    maxRetries: {{ .Values.archie.maxRetries }}
+    {{- end }}
 
     {{- if .Values.archie.msgTimeout }}
     msgTimeout: {{ .Values.archie.msgTimeout }}

diff --git a/helm/archie/values.yaml b/helm/archie/values.yaml
@@ -9,8 +9,10 @@ image:
 
 archie:
   logLevel: info # or "debug" or "trace"
+  maxRetries: 5  # for all terminating actions
   msgTimeout: 30m
   shutdownWait: 30 # seconds
+  skipEventBucketValidation: false
   skipLifecycleExpired: false
   # pcre regex matching
   #excludePaths:
@@ -48,6 +50,7 @@ archie:
       natsMessagesDeliveredLowThreshold: 30
       natsMessagesPendingThreshold: 20000
       natsMessagesRedeliveredPercentageThreshold: 2
+  waitForMatchingETag: false
 
 jetstream:
   url: nats://localhost:4222

diff --git a/main.go b/main.go
@@ -4,6 +4,7 @@ import (
 	"archie/archie"
 	"archie/client"
 	"context"
+	"encoding/json"
 	"flag"
 	"github.com/kkyr/fig"
 	"github.com/rs/zerolog"
@@ -88,19 +89,54 @@ func main() {
 		msgCancel()
 	}()
 
+	// log all config settings
+	redactedCfg := cfg
+
+	if redactedCfg.Src.AccessKey != "" {
+		redactedCfg.Src.AccessKey = "REDACTED"
+	}
+	if redactedCfg.Src.SecretKey != "" {
+		redactedCfg.Src.SecretKey = "REDACTED"
+	}
+	if redactedCfg.Src.GoogleCredentials != "" {
+		redactedCfg.Src.GoogleCredentials = "REDACTED"
+	}
+	if redactedCfg.Dest.AccessKey != "" {
+		redactedCfg.Dest.AccessKey = "REDACTED"
+	}
+	if redactedCfg.Dest.SecretKey != "" {
+		redactedCfg.Dest.SecretKey = "REDACTED"
+	}
+	if redactedCfg.Dest.GoogleCredentials != "" {
+		redactedCfg.Dest.GoogleCredentials = "REDACTED"
+	}
+	if redactedCfg.Jetstream.Password != "" {
+		redactedCfg.Jetstream.Password = "REDACTED"
+	}
+
+	redactedCfgJSON, err := json.Marshal(redactedCfg)
+	if err != nil {
+		log.Fatal().Err(err).Msg("Failed to marshal config to json")
+	}
+
+	log.Info().RawJSON("cfg", redactedCfgJSON).Msg("Startup configuration")
+
 	// archiver
 	a := archie.Archiver{
-		DestBucket:           cfg.Dest.Bucket,
-		DestName:             cfg.Dest.Name,
-		DestPartSize:         cfg.Dest.PartSize,
-		DestThreads:          cfg.Dest.Threads,
-		FetchDone:            make(chan string, 1),
-		HealthCheckDisabled:  cfg.HealthCheck.Disabled,
-		MsgTimeout:           cfg.MsgTimeout,
-		SkipLifecycleExpired: cfg.SkipLifecycleExpired,
-		SrcBucket:            cfg.Src.Bucket,
-		SrcName:              cfg.Src.Name,
-		WaitGroup:            &sync.WaitGroup{},
+		DestBucket:                cfg.Dest.Bucket,
+		DestName:                  cfg.Dest.Name,
+		DestPartSize:              cfg.Dest.PartSize,
+		DestThreads:               cfg.Dest.Threads,
+		FetchDone:                 make(chan string, 1),
+		HealthCheckDisabled:       cfg.HealthCheck.Disabled,
+		MaxRetries:                cfg.MaxRetries,
+		MsgTimeout:                cfg.MsgTimeout,
+		SkipEventBucketValidation: cfg.SkipEventBucketValidation,
+		SkipLifecycleExpired:      cfg.SkipLifecycleExpired,
+		SrcBucket:                 cfg.Src.Bucket,
+		SrcName:                   cfg.Src.Name,
+		WaitForMatchingETag:       cfg.WaitForMatchingETag,
+		WaitGroup:                 &sync.WaitGroup{},
 		ExcludePaths: struct {
 			CopyObject   []*pcre.Regexp
 			RemoveObject []*pcre.Regexp