From 565c30eac9e00b2ebcbdbb8e05b5e8238a15fefb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Mon, 16 Dec 2024 08:34:17 +0100 Subject: [PATCH] js: Fix js.Batch for multihost setups Note that this is an unreleased feature. Fixes #13151 --- deps/deps.go | 7 ++ hugolib/paths/paths.go | 2 +- hugolib/site.go | 8 +- internal/js/api.go | 51 +++++++++ internal/js/esbuild/batch.go | 105 ++++++++++-------- internal/js/esbuild/batch_integration_test.go | 63 +++++++++++ resources/resource.go | 7 -- tpl/js/js.go | 18 +-- 8 files changed, 190 insertions(+), 71 deletions(-) create mode 100644 internal/js/api.go diff --git a/deps/deps.go b/deps/deps.go index 8e9ec42d8b6..56a3d36446a 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -24,6 +24,7 @@ import ( "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/identity" + "github.com/gohugoio/hugo/internal/js" "github.com/gohugoio/hugo/internal/warpc" "github.com/gohugoio/hugo/media" "github.com/gohugoio/hugo/resources/page" @@ -105,6 +106,12 @@ type Deps struct { // TODO(bep) rethink this re. a plugin setup, but this will have to do for now. WasmDispatchers *warpc.Dispatchers + // The JS batcher client. + JSBatcherClient js.BatcherClient + + // The JS batcher client. + // JSBatcherClient *esbuild.BatcherClient + isClosed bool *globalErrHandler diff --git a/hugolib/paths/paths.go b/hugolib/paths/paths.go index 397dba3f809..60ec873f97b 100644 --- a/hugolib/paths/paths.go +++ b/hugolib/paths/paths.go @@ -67,7 +67,7 @@ func New(fs *hugofs.Fs, cfg config.AllProvider) (*Paths, error) { var multihostTargetBasePaths []string if cfg.IsMultihost() && len(cfg.Languages()) > 1 { for _, l := range cfg.Languages() { - multihostTargetBasePaths = append(multihostTargetBasePaths, l.Lang) + multihostTargetBasePaths = append(multihostTargetBasePaths, hpaths.ToSlashPreserveLeading(l.Lang)) } } diff --git a/hugolib/site.go b/hugolib/site.go index f73bd2517e1..4e2497ee1f1 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -42,6 +42,7 @@ import ( "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/hugolib/doctree" "github.com/gohugoio/hugo/hugolib/pagesfromdata" + "github.com/gohugoio/hugo/internal/js/esbuild" "github.com/gohugoio/hugo/internal/warpc" "github.com/gohugoio/hugo/langs/i18n" "github.com/gohugoio/hugo/modules" @@ -205,6 +206,12 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) { return nil, err } + batcherClient, err := esbuild.NewBatcherClient(firstSiteDeps) + if err != nil { + return nil, err + } + firstSiteDeps.JSBatcherClient = batcherClient + confm := cfg.Configs if err := confm.Validate(logger); err != nil { return nil, err @@ -313,7 +320,6 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) { return li.Lang < lj.Lang }) - var err error h, err = newHugoSites(cfg, firstSiteDeps, pageTrees, sites) if err == nil && h == nil { panic("hugo: newHugoSitesNew returned nil error and nil HugoSites") diff --git a/internal/js/api.go b/internal/js/api.go new file mode 100644 index 00000000000..30180dece07 --- /dev/null +++ b/internal/js/api.go @@ -0,0 +1,51 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package js + +import ( + "context" + + "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/resources/resource" +) + +// BatcherClient is used to do JS batch operations. +type BatcherClient interface { + New(id string) (Batcher, error) + Store() *maps.Cache[string, Batcher] +} + +// BatchPackage holds a group of JavaScript resources. +type BatchPackage interface { + Groups() map[string]resource.Resources +} + +// Batcher is used to build JavaScript packages. +type Batcher interface { + Build(context.Context) (BatchPackage, error) + Config(ctx context.Context) OptionsSetter + Group(ctx context.Context, id string) BatcherGroup +} + +// BatcherGroup is a group of scripts and instances. +type BatcherGroup interface { + Instance(sid, iid string) OptionsSetter + Runner(id string) OptionsSetter + Script(id string) OptionsSetter +} + +// OptionsSetter is used to set options for a batch, script or instance. +type OptionsSetter interface { + SetOptions(map[string]any) string +} diff --git a/internal/js/esbuild/batch.go b/internal/js/esbuild/batch.go index d0b6dba3337..c5394ac0a9c 100644 --- a/internal/js/esbuild/batch.go +++ b/internal/js/esbuild/batch.go @@ -20,6 +20,7 @@ import ( _ "embed" "encoding/json" "fmt" + "io" "path" "path/filepath" "reflect" @@ -34,7 +35,9 @@ import ( "github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/common/paths" "github.com/gohugoio/hugo/deps" + "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/identity" + "github.com/gohugoio/hugo/internal/js" "github.com/gohugoio/hugo/lazy" "github.com/gohugoio/hugo/media" "github.com/gohugoio/hugo/resources" @@ -42,11 +45,10 @@ import ( "github.com/gohugoio/hugo/resources/resource_factories/create" "github.com/gohugoio/hugo/tpl" "github.com/mitchellh/mapstructure" - "github.com/spf13/afero" "github.com/spf13/cast" ) -var _ Batcher = (*batcher)(nil) +var _ js.Batcher = (*batcher)(nil) const ( NsBatch = "_hugo-js-batch" @@ -58,7 +60,7 @@ const ( //go:embed batch-esm-runner.gotmpl var runnerTemplateStr string -var _ BatchPackage = (*Package)(nil) +var _ js.BatchPackage = (*Package)(nil) var _ buildToucher = (*optsHolder[scriptOptions])(nil) @@ -67,16 +69,17 @@ var ( _ isBuiltOrTouchedProvider = (*scriptGroup)(nil) ) -func NewBatcherClient(deps *deps.Deps) (*BatcherClient, error) { +func NewBatcherClient(deps *deps.Deps) (js.BatcherClient, error) { c := &BatcherClient{ d: deps, buildClient: NewBuildClient(deps.BaseFs.Assets, deps.ResourceSpec), createClient: create.New(deps.ResourceSpec), - bundlesCache: maps.NewCache[string, BatchPackage](), + batcherStore: maps.NewCache[string, js.Batcher](), + bundlesStore: maps.NewCache[string, js.BatchPackage](), } deps.BuildEndListeners.Add(func(...any) bool { - c.bundlesCache.Reset() + c.bundlesStore.Reset() return false }) @@ -125,7 +128,7 @@ func (o *opts[K, C]) Reset() { o.h.resetCounter++ } -func (o *opts[K, C]) Get(id uint32) OptionsSetter { +func (o *opts[K, C]) Get(id uint32) js.OptionsSetter { var b *optsHolder[C] o.once.Do(func() { b = o.h @@ -184,18 +187,6 @@ func newOpts[K any, C optionsCompiler[C]](key K, optionsID string, defaults defa } } -// BatchPackage holds a group of JavaScript resources. -type BatchPackage interface { - Groups() map[string]resource.Resources -} - -// Batcher is used to build JavaScript packages. -type Batcher interface { - Build(context.Context) (BatchPackage, error) - Config(ctx context.Context) OptionsSetter - Group(ctx context.Context, id string) BatcherGroup -} - // BatcherClient is a client for building JavaScript packages. type BatcherClient struct { d *deps.Deps @@ -206,12 +197,13 @@ type BatcherClient struct { createClient *create.Client buildClient *BuildClient - bundlesCache *maps.Cache[string, BatchPackage] + batcherStore *maps.Cache[string, js.Batcher] + bundlesStore *maps.Cache[string, js.BatchPackage] } // New creates a new Batcher with the given ID. // This will be typically created once and reused across rebuilds. -func (c *BatcherClient) New(id string) (Batcher, error) { +func (c *BatcherClient) New(id string) (js.Batcher, error) { var initErr error c.once.Do(func() { // We should fix the initialization order here (or use the Go template package directly), but we need to wait @@ -288,6 +280,10 @@ func (c *BatcherClient) New(id string) (Batcher, error) { return b, nil } +func (c *BatcherClient) Store() *maps.Cache[string, js.Batcher] { + return c.batcherStore +} + func (c *BatcherClient) buildBatchGroup(ctx context.Context, t *batchGroupTemplateContext) (resource.Resource, string, error) { var buf bytes.Buffer @@ -304,18 +300,6 @@ func (c *BatcherClient) buildBatchGroup(ctx context.Context, t *batchGroupTempla return r, s, nil } -// BatcherGroup is a group of scripts and instances. -type BatcherGroup interface { - Instance(sid, iid string) OptionsSetter - Runner(id string) OptionsSetter - Script(id string) OptionsSetter -} - -// OptionsSetter is used to set options for a batch, script or instance. -type OptionsSetter interface { - SetOptions(map[string]any) string -} - // Package holds a group of JavaScript resources. type Package struct { id string @@ -353,9 +337,9 @@ type batcher struct { } // Build builds the batch if not already built or if it's stale. -func (b *batcher) Build(ctx context.Context) (BatchPackage, error) { +func (b *batcher) Build(ctx context.Context) (js.BatchPackage, error) { key := dynacache.CleanKey(b.id + ".js") - p, err := b.client.bundlesCache.GetOrCreate(key, func() (BatchPackage, error) { + p, err := b.client.bundlesStore.GetOrCreate(key, func() (js.BatchPackage, error) { return b.build(ctx) }) if err != nil { @@ -364,11 +348,11 @@ func (b *batcher) Build(ctx context.Context) (BatchPackage, error) { return p, nil } -func (b *batcher) Config(ctx context.Context) OptionsSetter { +func (b *batcher) Config(ctx context.Context) js.OptionsSetter { return b.configOptions.Get(b.buildCount) } -func (b *batcher) Group(ctx context.Context, id string) BatcherGroup { +func (b *batcher) Group(ctx context.Context, id string) js.BatcherGroup { if err := ValidateBatchID(id, false); err != nil { panic(err) } @@ -419,7 +403,7 @@ func (b *batcher) isStale() bool { return false } -func (b *batcher) build(ctx context.Context) (BatchPackage, error) { +func (b *batcher) build(ctx context.Context) (js.BatchPackage, error) { b.mu.Lock() defer b.mu.Unlock() defer func() { @@ -463,6 +447,8 @@ func (b *batcher) doBuild(ctx context.Context) (*Package, error) { pathGroup: maps.NewCache[string, string](), } + multihostBasePaths := b.client.d.ResourceSpec.MultihostTargetBasePaths + // Entry points passed to ESBuid. var entryPoints []string addResource := func(group, pth string, r resource.Resource, isResult bool) { @@ -701,15 +687,36 @@ func (b *batcher) doBuild(ctx context.Context) (*Package, error) { if !handled { // Copy to destination. - p := strings.TrimPrefix(o.Path, outDir) - targetFilename := filepath.Join(b.id, p) - fs := b.client.d.BaseFs.PublishFs - if err := fs.MkdirAll(filepath.Dir(targetFilename), 0o777); err != nil { - return nil, fmt.Errorf("failed to create dir %q: %w", targetFilename, err) + // In a multihost setup, we will have multiple targets. + var targetFilenames []string + if len(multihostBasePaths) > 0 { + for _, base := range multihostBasePaths { + p := strings.TrimPrefix(o.Path, outDir) + targetFilename := filepath.Join(base, b.id, p) + targetFilenames = append(targetFilenames, targetFilename) + } + } else { + p := strings.TrimPrefix(o.Path, outDir) + targetFilename := filepath.Join(b.id, p) + targetFilenames = append(targetFilenames, targetFilename) } - if err := afero.WriteFile(fs, targetFilename, o.Contents, 0o666); err != nil { - return nil, fmt.Errorf("failed to write to %q: %w", targetFilename, err) + fs := b.client.d.BaseFs.PublishFs + + if err := func() error { + fw, err := helpers.OpenFilesForWriting(fs, targetFilenames...) + if err != nil { + return err + } + defer fw.Close() + + fr := bytes.NewReader(o.Contents) + + _, err = io.Copy(fw, fr) + + return err + }(); err != nil { + return nil, fmt.Errorf("failed to copy to %q: %w", targetFilenames, err) } } } @@ -845,7 +852,7 @@ type optionsGetSetter[K, C any] interface { Key() K Reset() - Get(uint32) OptionsSetter + Get(uint32) js.OptionsSetter isStale() bool currPrev() (map[string]any, map[string]any) } @@ -975,7 +982,7 @@ func (b *scriptGroup) IdentifierBase() string { return b.id } -func (s *scriptGroup) Instance(sid, id string) OptionsSetter { +func (s *scriptGroup) Instance(sid, id string) js.OptionsSetter { if err := ValidateBatchID(sid, false); err != nil { panic(err) } @@ -1014,7 +1021,7 @@ func (g *scriptGroup) Reset() { } } -func (s *scriptGroup) Runner(id string) OptionsSetter { +func (s *scriptGroup) Runner(id string) js.OptionsSetter { if err := ValidateBatchID(id, false); err != nil { panic(err) } @@ -1043,7 +1050,7 @@ func (s *scriptGroup) Runner(id string) OptionsSetter { return s.runnersOptions[sid].Get(s.b.buildCount) } -func (s *scriptGroup) Script(id string) OptionsSetter { +func (s *scriptGroup) Script(id string) js.OptionsSetter { if err := ValidateBatchID(id, false); err != nil { panic(err) } diff --git a/internal/js/esbuild/batch_integration_test.go b/internal/js/esbuild/batch_integration_test.go index 3501f820a86..55528bdf042 100644 --- a/internal/js/esbuild/batch_integration_test.go +++ b/internal/js/esbuild/batch_integration_test.go @@ -184,6 +184,69 @@ func TestBatchEditScriptParam(t *testing.T) { b.AssertFileContent("public/mybatch/mygroup.js", "param-p1-main-edited") } +func TestBatchMultiHost(t *testing.T) { + files := ` +-- hugo.toml -- +disableKinds = ["taxonomy", "term", "section"] +[languages] +[languages.en] +weight = 1 +baseURL = "https://example.com/en" +[languages.fr] +weight = 2 +baseURL = "https://example.com/fr" +disableLiveReload = true +-- assets/js/styles.css -- +body { + background-color: red; +} +-- assets/js/main.js -- +import * as foo from 'mylib'; +console.log("Hello, Main!"); +-- assets/js/runner.js -- +console.log("Hello, Runner!"); +-- node_modules/mylib/index.js -- +console.log("Hello, My Lib!"); +-- layouts/index.html -- +Home. +{{ $batch := (js.Batch "mybatch") }} + {{ with $batch.Config }} + {{ .SetOptions (dict + "params" (dict "id" "config") + "sourceMap" "" + ) + }} +{{ end }} +{{ with (templates.Defer (dict "key" "global")) }} +Defer: +{{ $batch := (js.Batch "mybatch") }} +{{ range $k, $v := $batch.Build.Groups }} + {{ range $kk, $vv := . -}} + {{ $k }}: {{ .RelPermalink }} + {{ end }} +{{ end -}} +{{ end }} +{{ $batch := (js.Batch "mybatch") }} +{{ with $batch.Group "mygroup" }} + {{ with .Runner "run" }} + {{ .SetOptions (dict "resource" (resources.Get "js/runner.js")) }} + {{ end }} + {{ with .Script "main" }} + {{ .SetOptions (dict "resource" (resources.Get "js/main.js") "params" (dict "p1" "param-p1-main" )) }} + {{ end }} + {{ with .Instance "main" "i1" }} + {{ .SetOptions (dict "params" (dict "title" "Instance 1")) }} + {{ end }} +{{ end }} + + +` + b := hugolib.Test(t, files, hugolib.TestOptWithOSFs()) + b.AssertPublishDir( + "en/mybatch/chunk-TOZKWCDE.js", "en/mybatch/mygroup.js ", + "fr/mybatch/mygroup.js", "fr/mybatch/chunk-TOZKWCDE.js") +} + func TestBatchRenameBundledScript(t *testing.T) { files := jsBatchFilesTemplate b := hugolib.TestRunning(t, files, hugolib.TestOptWithOSFs()) diff --git a/resources/resource.go b/resources/resource.go index 6025cbf4c3e..4b81a478a42 100644 --- a/resources/resource.go +++ b/resources/resource.go @@ -141,13 +141,6 @@ func (fd *ResourceSourceDescriptor) init(r *Spec) error { } fd.TargetPath = paths.ToSlashPreserveLeading(fd.TargetPath) - for i, base := range fd.TargetBasePaths { - dir := paths.ToSlashPreserveLeading(base) - if dir == "/" { - dir = "" - } - fd.TargetBasePaths[i] = dir - } if fd.NameNormalized == "" { fd.NameNormalized = fd.TargetPath diff --git a/tpl/js/js.go b/tpl/js/js.go index b686b76a7bc..dfd0a358118 100644 --- a/tpl/js/js.go +++ b/tpl/js/js.go @@ -17,8 +17,8 @@ package js import ( "errors" - "github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/deps" + "github.com/gohugoio/hugo/internal/js" "github.com/gohugoio/hugo/internal/js/esbuild" "github.com/gohugoio/hugo/resources" "github.com/gohugoio/hugo/resources/resource" @@ -34,16 +34,9 @@ func New(d *deps.Deps) (*Namespace, error) { return &Namespace{}, nil } - batcherClient, err := esbuild.NewBatcherClient(d) - if err != nil { - return nil, err - } - return &Namespace{ d: d, jsTransformClient: jstransform.New(d.BaseFs.Assets, d.ResourceSpec), - jsBatcherClient: batcherClient, - jsBatcherStore: maps.NewCache[string, esbuild.Batcher](), createClient: create.New(d.ResourceSpec), babelClient: babel.New(d.ResourceSpec), }, nil @@ -56,8 +49,6 @@ type Namespace struct { jsTransformClient *jstransform.Client createClient *create.Client babelClient *babel.Client - jsBatcherClient *esbuild.BatcherClient - jsBatcherStore *maps.Cache[string, esbuild.Batcher] } // Build processes the given Resource with ESBuild. @@ -90,12 +81,13 @@ func (ns *Namespace) Build(args ...any) (resource.Resource, error) { // Repeated calls with the same ID will return the same Batcher. // The ID will be used to name the root directory of the batch. // Forward slashes in the ID is allowed. -func (ns *Namespace) Batch(id string) (esbuild.Batcher, error) { +func (ns *Namespace) Batch(id string) (js.Batcher, error) { if err := esbuild.ValidateBatchID(id, true); err != nil { return nil, err } - b, err := ns.jsBatcherStore.GetOrCreate(id, func() (esbuild.Batcher, error) { - return ns.jsBatcherClient.New(id) + + b, err := ns.d.JSBatcherClient.Store().GetOrCreate(id, func() (js.Batcher, error) { + return ns.d.JSBatcherClient.New(id) }) if err != nil { return nil, err pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy