Make js.Build fully support modules

Fixes #7816
Fixes #7777
Fixes #7916

4 files changed, 109 insertions, 336 deletions

diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go
index 1a6a07b03..25ae3dd19 100644
--- a/hugolib/hugo_sites.go
+++ b/hugolib/hugo_sites.go
@@ -22,6 +22,8 @@ import (
 	"sync"
 	"sync/atomic"
 
+	"github.com/fsnotify/fsnotify"
+
 	"github.com/gohugoio/hugo/identity"
 
 	radix "github.com/armon/go-radix"
@@ -85,6 +87,10 @@ type HugoSites struct {
 	// Keeps track of bundle directories and symlinks to enable partial rebuilding.
 	ContentChanges *contentChangeMap
 
+	// File change events with filename stored in this map will be skipped.
+	skipRebuildForFilenamesMu sync.Mutex
+	skipRebuildForFilenames   map[string]bool
+
 	init *hugoSitesInit
 
 	workers    *para.Workers
@@ -94,6 +100,14 @@ type HugoSites struct {
 	*testCounters
 }
 
+// ShouldSkipFileChangeEvent allows skipping filesystem event early before
+// the build is started.
+func (h *HugoSites) ShouldSkipFileChangeEvent(ev fsnotify.Event) bool {
+	h.skipRebuildForFilenamesMu.Lock()
+	defer h.skipRebuildForFilenamesMu.Unlock()
+	return h.skipRebuildForFilenames[ev.Name]
+}
+
 func (h *HugoSites) getContentMaps() *pageMaps {
 	h.contentInit.Do(func() {
 		h.content = newPageMaps(h)
@@ -304,12 +318,13 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) {
 	}
 
 	h := &HugoSites{
-		running:      cfg.Running,
-		multilingual: langConfig,
-		multihost:    cfg.Cfg.GetBool("multihost"),
-		Sites:        sites,
-		workers:      workers,
-		numWorkers:   numWorkers,
+		running:                 cfg.Running,
+		multilingual:            langConfig,
+		multihost:               cfg.Cfg.GetBool("multihost"),
+		Sites:                   sites,
+		workers:                 workers,
+		numWorkers:              numWorkers,
+		skipRebuildForFilenames: make(map[string]bool),
 		init: &hugoSitesInit{
 			data:         lazy.New(),
 			layouts:      lazy.New(),
diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go
index 3c0440a97..603772afd 100644
--- a/hugolib/hugo_sites_build.go
+++ b/hugolib/hugo_sites_build.go
@@ -33,8 +33,6 @@ import (
 
 	"github.com/spf13/afero"
 
-	"github.com/gohugoio/hugo/resources/resource"
-
 	"github.com/gohugoio/hugo/output"
 
 	"github.com/pkg/errors"
@@ -351,14 +349,45 @@ func (h *HugoSites) postProcess() error {
 		return err
 	}
 
-	var toPostProcess []resource.OriginProvider
-	for _, s := range h.Sites {
-		for _, v := range s.ResourceSpec.PostProcessResources {
-			toPostProcess = append(toPostProcess, v)
+	// This will only be set when js.Build have been triggered with
+	// imports that resolves to the project or a module.
+	// Write a jsconfig.json file to the project's /asset directory
+	// to help JS intellisense in VS Code etc.
+	if !h.ResourceSpec.BuildConfig.NoJSConfigInAssets && h.BaseFs.Assets.Dirs != nil {
+		m := h.BaseFs.Assets.Dirs[0].Meta()
+		assetsDir := m.Filename()
+		if strings.HasPrefix(assetsDir, h.ResourceSpec.WorkingDir) {
+			if jsConfig := h.ResourceSpec.JSConfigBuilder.Build(assetsDir); jsConfig != nil {
+
+				b, err := json.MarshalIndent(jsConfig, "", " ")
+				if err != nil {
+					h.Log.Warnf("Failed to create jsconfig.json: %s", err)
+
+				} else {
+					filename := filepath.Join(assetsDir, "jsconfig.json")
+					if h.running {
+						h.skipRebuildForFilenamesMu.Lock()
+						h.skipRebuildForFilenames[filename] = true
+						h.skipRebuildForFilenamesMu.Unlock()
+					}
+					// Make sure it's  written to the OS fs as this is used by
+					// editors.
+					if err := afero.WriteFile(hugofs.Os, filename, b, 0666); err != nil {
+						h.Log.Warnf("Failed to write jsconfig.json: %s", err)
+					}
+				}
+			}
+
 		}
 	}
 
+	var toPostProcess []postpub.PostPublishedResource
+	for _, r := range h.ResourceSpec.PostProcessResources {
+		toPostProcess = append(toPostProcess, r)
+	}
+
 	if len(toPostProcess) == 0 {
+		// Nothing more to do.
 		return nil
 	}
 
diff --git a/hugolib/js_test.go b/hugolib/js_test.go
index e34ce3867..6c27219f3 100644
--- a/hugolib/js_test.go
+++ b/hugolib/js_test.go
@@ -14,6 +14,7 @@
 package hugolib
 
 import (
+	"fmt"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -22,7 +23,6 @@ import (
 
 	"github.com/gohugoio/hugo/htesting"
 
-	"github.com/spf13/afero"
 	"github.com/spf13/viper"
 
 	qt "github.com/frankban/quicktest"
@@ -82,9 +82,7 @@ document.body.textContent = greeter(user);`
   "scripts": {},
 
   "dependencies": {
-		"to-camel-case": "1.0.0",
-		"react": "^16",
-		"react-dom": "^16"
+    "to-camel-case": "1.0.0"
   }
 }
 `
@@ -153,333 +151,46 @@ func TestJSBuild(t *testing.T) {
 
 	c := qt.New(t)
 
-	mainJS := `
-	import "./included";
-	
-	console.log("main");
-
-`
-	includedJS := `
-	console.log("included");
-	
-	`
-
-	workDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-test-js")
-	c.Assert(err, qt.IsNil)
-	defer clean()
-
-	v := viper.New()
-	v.Set("workingDir", workDir)
-	v.Set("disableKinds", []string{"taxonomy", "term", "page"})
-	b := newTestSitesBuilder(t).WithLogger(loggers.NewWarningLogger())
-
-	b.Fs = hugofs.NewDefault(v)
-	b.WithWorkingDir(workDir)
-	b.WithViper(v)
-	b.WithContent("p1.md", "")
-
-	b.WithTemplates("index.html", `
-{{ $js := resources.Get "js/main.js" | js.Build }}
-JS:  {{ template "print" $js }}
-
-
-{{ define "print" }}RelPermalink: {{.RelPermalink}}|MIME: {{ .MediaType }}|Content: {{ .Content | safeJS }}{{ end }}
-
-`)
-
-	jsDir := filepath.Join(workDir, "assets", "js")
-	b.Assert(os.MkdirAll(jsDir, 0777), qt.IsNil)
-	b.Assert(os.Chdir(workDir), qt.IsNil)
-	b.WithSourceFile("assets/js/main.js", mainJS)
-	b.WithSourceFile("assets/js/included.js", includedJS)
-
-	b.Build(BuildCfg{})
-
-	b.AssertFileContent("public/index.html", `
-console.log("included");
-
-`)
-
-}
-
-func TestJSBuildGlobals(t *testing.T) {
-	if !isCI() {
-		t.Skip("skip (relative) long running modules test when running locally")
-	}
-
-	wd, _ := os.Getwd()
-	defer func() {
-		os.Chdir(wd)
-	}()
-
-	c := qt.New(t)
-
-	workDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-test-js")
-	c.Assert(err, qt.IsNil)
-	defer clean()
-
-	v := viper.New()
-	v.Set("workingDir", workDir)
-	v.Set("disableKinds", []string{"taxonomy", "term", "page"})
-	b := newTestSitesBuilder(t).WithLogger(loggers.NewWarningLogger())
-
-	b.Fs = hugofs.NewDefault(v)
-	b.WithWorkingDir(workDir)
-	b.WithViper(v)
-	b.WithContent("p1.md", "")
-
-	jsDir := filepath.Join(workDir, "assets", "js")
-	b.Assert(os.MkdirAll(jsDir, 0777), qt.IsNil)
-	b.Assert(os.Chdir(workDir), qt.IsNil)
-
-	b.WithTemplates("index.html", `
-{{- $js := resources.Get "js/main-project.js" | js.Build -}}
-{{ template "print" (dict "js" $js "name" "root") }}
-
-{{- define "print" -}}
-{{ printf "rellink-%s-%s" .name .js.RelPermalink | safeHTML }}
-{{ printf "mime-%s-%s" .name .js.MediaType | safeHTML }}
-{{ printf "content-%s-%s" .name .js.Content | safeHTML }}
-{{- end -}}
-`)
-
-	b.WithSourceFile("assets/js/normal.js", `
-const name = "root-normal";
-export default name;
-`)
-	b.WithSourceFile("assets/js/main-project.js", `
-import normal from "@js/normal";
-window.normal = normal; // make sure not to tree-shake
-`)
-
-	b.Build(BuildCfg{})
-
-	b.AssertFileContent("public/index.html", `
-const name = "root-normal";
-`)
-}
-
-func TestJSBuildOverride(t *testing.T) {
-	if !isCI() {
-		t.Skip("skip (relative) long running modules test when running locally")
-	}
-
-	wd, _ := os.Getwd()
-	defer func() {
-		os.Chdir(wd)
-	}()
-
-	c := qt.New(t)
-
-	workDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-test-js2")
+	workDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-test-js-mod")
 	c.Assert(err, qt.IsNil)
 	defer clean()
-	// workDir := "/tmp/hugo-test-js2"
-	c.Assert(os.Chdir(workDir), qt.IsNil)
-
-	cfg := viper.New()
-	cfg.Set("workingDir", workDir)
-	fs := hugofs.NewFrom(afero.NewOsFs(), cfg)
 
-	b := newTestSitesBuilder(t)
-	b.Fs = fs
-	b.WithLogger(loggers.NewWarningLogger())
-
-	realWrite := func(name string, content string) {
-		realLocation := filepath.Join(workDir, name)
-		realDir := filepath.Dir(realLocation)
-		if _, err := os.Stat(realDir); err != nil {
-			os.MkdirAll(realDir, 0777)
-		}
-		bytesContent := []byte(content)
-		// c.Assert(ioutil.WriteFile(realLocation, bytesContent, 0777), qt.IsNil)
-		c.Assert(afero.WriteFile(b.Fs.Source, realLocation, bytesContent, 0777), qt.IsNil)
-	}
+	config := fmt.Sprintf(`
+baseURL = "https://example.org"
+workingDir = %q
 
-	realWrite("config.toml", `
-baseURL="https://example.org"
+disableKinds = ["page", "section", "term", "taxonomy"]
 
 [module]
 [[module.imports]]
-path="mod2"
-[[module.imports.mounts]]
-source="assets"
-target="assets"
-[[module.imports.mounts]]
-source="layouts"
-target="layouts"
-[[module.imports]]
-path="mod1"
-[[module.imports.mounts]]
-source="assets"
-target="assets"
-[[module.imports.mounts]]
-source="layouts"
-target="layouts"
-`)
+path="github.com/gohugoio/hugoTestProjectJSModImports"
 
-	realWrite("content/p1.md", `---
-layout: sample
----
-`)
-	realWrite("themes/mod1/layouts/_default/sample.html", `
-{{- $js := resources.Get "js/main-project.js" | js.Build -}}
-{{ template "print" (dict "js" $js "name" "root") }}
 
-{{- $js = resources.Get "js/main-mod1.js" | js.Build -}}
-{{ template "print" (dict "js" $js "name" "mod1") }}
 
-{{- $js = resources.Get "js/main-mod2.js" | js.Build (dict "data" .Site.Params) -}}
-{{ template "print" (dict "js" $js "name" "mod2") }}
+`, workDir)
 
-{{- $js = resources.Get "js/main-mod2.js" | js.Build (dict "data" .Site.Params "sourceMap" "inline" "targetPath" "js/main-mod2-inline.js") -}}
-{{ template "print" (dict "js" $js "name" "mod2") }}
-
-{{- $js = resources.Get "js/main-mod2.js" | js.Build (dict "data" .Site.Params "sourceMap" "external" "targetPath" "js/main-mod2-external.js") -}}
-{{ template "print" (dict "js" $js "name" "mod2") }}
+	b := newTestSitesBuilder(t)
+	b.Fs = hugofs.NewDefault(viper.New())
+	b.WithWorkingDir(workDir).WithConfigFile("toml", config).WithLogger(loggers.NewInfoLogger())
+	b.WithSourceFile("go.mod", `module github.com/gohugoio/tests/testHugoModules
+        
+go 1.15
+        
+require github.com/gohugoio/hugoTestProjectJSModImports v0.3.0 // indirect
 
-{{- define "print" -}}
-{{ printf "rellink-%s-%s" .name .js.RelPermalink | safeHTML }}
-{{ printf "mime-%s-%s" .name .js.MediaType | safeHTML }}
-{{ printf "content-%s-%s" .name .js.Content | safeHTML }}
-{{- end -}}
 `)
 
-	// Override project included file
-	// This file will override the one in mod1 and mod2
-	realWrite("assets/js/override.js", `
-const name = "root-override";
-export default name;
-`)
-
-	// Add empty theme mod config files
-	realWrite("themes/mod1/config.yml", ``)
-	realWrite("themes/mod2/config.yml", ``)
-
-	// This is the main project js file.
-	// try to include @js/override which is overridden inside of project
-	// try to include @js/override-mod which is overridden in mod2
-	realWrite("assets/js/main-project.js", `
-import override from "@js/override";
-import overrideMod from "@js/override-mod";
-window.override = override; // make sure to prevent tree-shake
-window.overrideMod  = overrideMod; // make sure to prevent tree-shake
-`)
-	// This is the mod1 js file
-	// try to include @js/override which is overridden inside of the project
-	// try to include @js/override-mod which is overridden in mod2
-	realWrite("themes/mod1/assets/js/main-mod1.js", `
-import override from "@js/override";
-import overrideMod from "@js/override-mod";
-window.mod = "mod1";
-window.override = override; // make sure to prevent tree-shake
-window.overrideMod  = overrideMod; // make sure to prevent tree-shake
-`)
-	// This is the mod1 js file (overridden in mod2)
-	// try to include @js/override which is overridden inside of the project
-	// try to include @js/override-mod which is overridden in mod2
-	realWrite("themes/mod2/assets/js/main-mod1.js", `
-import override from "@js/override";
-import overrideMod from "@js/override-mod";
-window.mod = "mod2";
-window.override = override; // make sure to prevent tree-shake
-window.overrideMod  = overrideMod; // make sure to prevent tree-shake
-`)
-	// This is mod2 js file
-	// try to include @js/override which is overridden inside of the project
-	// try to include @js/override-mod which is overridden in mod2
-	// try to include @config which is declared in a local jsconfig.json file
-	// try to include @data which was passed as "data" into js.Build
-	realWrite("themes/mod2/assets/js/main-mod2.js", `
-import override from "@js/override";
-import overrideMod from "@js/override-mod";
-import config from "@config";
-import data from "@data";
-window.data = data;
-window.override = override; // make sure to prevent tree-shake
-window.overrideMod  = overrideMod; // make sure to prevent tree-shake
-window.config = config;
-`)
-	realWrite("themes/mod2/assets/js/jsconfig.json", `
-{
-	"compilerOptions": {
-		"baseUrl": ".",
-		"paths": {
-			"@config": ["./config.json"]
-		}
-	}
-}
-`)
-	realWrite("themes/mod2/assets/js/config.json", `
-{
-	"data": {
-		"sample": "sample"
-	}
-}
-`)
-	realWrite("themes/mod1/assets/js/override.js", `
-const name = "mod1-override";
-export default name;
-`)
-	realWrite("themes/mod2/assets/js/override.js", `
-const name = "mod2-override";
-export default name;
-`)
-	realWrite("themes/mod1/assets/js/override-mod.js", `
-const nameMod = "mod1-override";
-export default nameMod;
-`)
-	realWrite("themes/mod2/assets/js/override-mod.js", `
-const nameMod = "mod2-override";
-export default nameMod;
-`)
-	b.WithConfigFile("toml", `
-baseURL="https://example.org"
-themesDir="./themes"
-[module]
-[[module.imports]]
-path="mod2"
-[[module.imports.mounts]]
-source="assets"
-target="assets"
-[[module.imports.mounts]]
-source="layouts"
-target="layouts"
-[[module.imports]]
-path="mod1"
-[[module.imports.mounts]]
-source="assets"
-target="assets"
-[[module.imports.mounts]]
-source="layouts"
-target="layouts"
-`)
-
-	b.WithWorkingDir(workDir)
-	b.LoadConfig()
+	b.WithContent("p1.md", "").WithNothingAdded()
 
 	b.Build(BuildCfg{})
 
-	b.AssertFileContent("public/js/main-mod1.js", `
-name = "root-override";
-nameMod = "mod2-override";
-window.mod = "mod2";
-`)
-	b.AssertFileContent("public/js/main-mod2.js", `
-name = "root-override";
-nameMod = "mod2-override";
-sample: "sample"
-"sect"
-`)
-	b.AssertFileContent("public/js/main-project.js", `
-name = "root-override";
-nameMod = "mod2-override";
-`)
-	b.AssertFileContent("public/js/main-mod2-external.js.map", `
-const nameMod = \"mod2-override\";\nexport default nameMod;\n
-"\nimport override from \"@js/override\";\nimport overrideMod from \"@js/override-mod\";\nimport config from \"@config\";\nimport data from \"@data\";\nwindow.data = data;\nwindow.override = override; // make sure to prevent tree-shake\nwindow.overrideMod  = overrideMod; // make sure to prevent tree-shake\nwindow.config = config;\n"
-`)
-	b.AssertFileContent("public/js/main-mod2-inline.js", `
-	sourceMappingURL=data:application/json;base64,ewogICJ2ZXJzaW9uIjogMywKICAic291cmNlcyI6IFsiYXNzZXRzL2pzL292ZXJyaWRlLmpzIiwgInRoZW
-`)
+	b.AssertFileContent("public/js/main.js", `
+greeting: "greeting configured in mod2"
+Hello1 from mod1: $
+return "Hello2 from mod1";
+var Hugo = "Rocks!";
+return "Hello3 from mod2";
+return "Hello from lib in the main project";
+var myparam = "Hugo Rocks!";`)
+
 }
diff --git a/hugolib/site.go b/hugolib/site.go
index ec2939530..3679e354c 100644
--- a/hugolib/site.go
+++ b/hugolib/site.go
@@ -997,6 +997,16 @@ func (s *Site) translateFileEvents(events []fsnotify.Event) []fsnotify.Event {
 	return filtered
 }
 
+var (
+	// These are only used for cache busting, so false positives are fine.
+	// We also deliberately do not match for file suffixes to also catch
+	// directory names.
+	// TODO(bep) consider this when completing the relevant PR rewrite on this.
+	cssFileRe   = regexp.MustCompile("(css|sass|scss)")
+	cssConfigRe = regexp.MustCompile(`(postcss|tailwind)\.config\.js`)
+	jsFileRe    = regexp.MustCompile("(js|ts|jsx|tsx)")
+)
+
 // reBuild partially rebuilds a site given the filesystem events.
 // It returns whetever the content source was changed.
 // TODO(bep) clean up/rewrite this method.
@@ -1028,19 +1038,24 @@ func (s *Site) processPartial(config *BuildCfg, init func(config *BuildCfg) erro
 		logger = helpers.NewDistinctFeedbackLogger()
 	)
 
-	var isCSSConfigRe = regexp.MustCompile(`(postcss|tailwind)\.config\.js`)
-	var isCSSFileRe = regexp.MustCompile(`\.(css|scss|sass)`)
-
 	var cachePartitions []string
 	// Special case
 	// TODO(bep) I have a ongoing branch where I have redone the cache. Consider this there.
-	var isCSSChange bool
+	var (
+		evictCSSRe *regexp.Regexp
+		evictJSRe  *regexp.Regexp
+	)
 
 	for _, ev := range events {
 		if assetsFilename := s.BaseFs.Assets.MakePathRelative(ev.Name); assetsFilename != "" {
 			cachePartitions = append(cachePartitions, resources.ResourceKeyPartitions(assetsFilename)...)
-			if !isCSSChange {
-				isCSSChange = isCSSFileRe.MatchString(assetsFilename) || isCSSConfigRe.MatchString(assetsFilename)
+			if evictCSSRe == nil {
+				if cssFileRe.MatchString(assetsFilename) || cssConfigRe.MatchString(assetsFilename) {
+					evictCSSRe = cssFileRe
+				}
+			}
+			if evictJSRe == nil && jsFileRe.MatchString(assetsFilename) {
+				evictJSRe = jsFileRe
 			}
 		}
 
@@ -1088,8 +1103,11 @@ func (s *Site) processPartial(config *BuildCfg, init func(config *BuildCfg) erro
 	// These in memory resource caches will be rebuilt on demand.
 	for _, s := range s.h.Sites {
 		s.ResourceSpec.ResourceCache.DeletePartitions(cachePartitions...)
-		if isCSSChange {
-			s.ResourceSpec.ResourceCache.DeleteContains("css", "scss", "sass")
+		if evictCSSRe != nil {
+			s.ResourceSpec.ResourceCache.DeleteMatches(evictCSSRe)
+		}
+		if evictJSRe != nil {
+			s.ResourceSpec.ResourceCache.DeleteMatches(evictJSRe)
 		}
 	}