Localization support in HTML docs, simplification of checklinks.sh (#5342)

* Add localization support to HTML docs site and simplify checklinks.sh
* Support localization in the html build process
* Show language selection menu
* Get the Chinese content additional pages into the localization project
* Have PRs in `netdata/localization` automatically trigger the `netdata/netdata` Netlify builds. The checks must pass before a PR is merged
* Make link to edit lead to the root directory of localization, which contains instructions

7 files changed, 199 insertions, 101 deletions

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index ec2ecf1fb5..6183f05ef4 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -21,6 +21,10 @@ Community growth allows the project to attract new talent willing to contribute.
 
 Is there anything that bothers you about netdata? Did you experience an issue while installing it or using it? Would you like to see it evolve to you need? Let us know. [Open a github issue](https://github.com/netdata/netdata/issues) to discuss it. Feedback is very important for open-source projects. We can't commit we will do everything, but your feedback influences our road-map significantly. **We rely on your feedback to make Netdata better**.
 
+### Translate some documentation
+
+The [netdata localization project](https://github.com/netdata/localization) contains instructions on how to provide translations for parts of our documentation. Translating the entire documentation is a daunting task, but you can contribute as much as you like, even a single file. The Chinese translation effort has already begun and we are looking forward to more contributions.
+
 ### Sponsor a part of Netdata
 
 Netdata is a complex system, with many integrations for the various  collectors, backends and notification endpoints. As a result, we rely on help from "sponsors", a concept similar to "power users" or "product owners". To become a sponsor, just let us know in any Github issue and we will record your GitHub username in a "CONTRIBUTORS.md" in the appropriate directory.
diff --git a/docs/generator/buildhtml.sh b/docs/generator/buildhtml.sh
index 3f73420815..0431122420 100755
--- a/docs/generator/buildhtml.sh
+++ b/docs/generator/buildhtml.sh
@@ -13,27 +13,27 @@ if [ "$currentdir" = "generator" ]; then
 	cd ../..
 fi
 GENERATOR_DIR="docs/generator"
-
+SRC_DIR="${GENERATOR_DIR}/src"
 # Fetch go.d.plugin docs
-rm -rf ./collectors/go.d.plugin
-git clone https://github.com/netdata/go.d.plugin.git ./collectors/go.d.plugin
+GO_D_DIR="collectors/go.d.plugin"
+rm -rf ${GO_D_DIR}
+git clone https://github.com/netdata/go.d.plugin.git ${GO_D_DIR}
 
 # Copy all netdata .md files to docs/generator/src. Exclude htmldoc itself and also the directory node_modules generatord by Netlify
 echo "Copying files"
-rm -rf ${GENERATOR_DIR}/src
-find . -type d \( -path ./${GENERATOR_DIR} -o -path ./node_modules \) -prune -o -name "*.md" -print | cpio -pd ${GENERATOR_DIR}/src
+rm -rf ${SRC_DIR}
+find . -type d \( -path ./${GENERATOR_DIR} -o -path ./node_modules \) -prune -o -name "*.md" -print | cpio -pd ${SRC_DIR}
 
 # Copy netdata html resources
-cp -a ./${GENERATOR_DIR}/custom ./${GENERATOR_DIR}/src/
-
+cp -a ./${GENERATOR_DIR}/custom ./${SRC_DIR}/
 
 
 # Modify the first line of the main README.md, to enable proper static html generation
 echo "Modifying README header"
-sed -i -e '0,/# netdata /s//# Introduction\n\n/' ${GENERATOR_DIR}/src/README.md
+sed -i -e '0,/# netdata /s//# Introduction\n\n/' ${SRC_DIR}/README.md
 
 # Remove all GA tracking code
-find ${GENERATOR_DIR}/src -name "*.md" -print0 | xargs -0 sed -i -e 's/\[!\[analytics.*UA-64295674-3)\]()//g'
+find ${SRC_DIR} -name "*.md" -print0 | xargs -0 sed -i -e 's/\[!\[analytics.*UA-64295674-3)\]()//g'
 
 # Remove specific files that don't belong in the documentation
 declare -a EXCLUDE_LIST=(
@@ -43,28 +43,62 @@ declare -a EXCLUDE_LIST=(
 )
 
 for f in "${EXCLUDE_LIST[@]}"; do
-	rm "${GENERATOR_DIR}/src/$f"
+	rm "${SRC_DIR}/$f"
 done
 
-echo "Creating mkdocs.yaml"
-
-# Generate mkdocs.yaml
-${GENERATOR_DIR}/buildyaml.sh >${GENERATOR_DIR}/mkdocs.yml
-
-echo "Fixing links"
-
-# Fix links (recursively, all types, executing replacements)
-${GENERATOR_DIR}/checklinks.sh -rax
-
-echo "Calling mkdocs"
-
-# Build html docs
-mkdocs build --config-file=${GENERATOR_DIR}/mkdocs.yml
-
-# Fix edit buttons for the markdowns that are not on the main netdata repo
-find ${GENERATOR_DIR}/build/collectors/go.d.plugin -name "*.html" -print0 | xargs -0 sed -i -e 's/https:\/\/github.com\/netdata\/netdata\/blob\/master\/collectors\/go.d.plugin/https:\/\/github.com\/netdata\/go.d.plugin\/blob\/master/g'
+echo "Fetching localization project"
+LOC_DIR=${GENERATOR_DIR}/localization
+rm -rf ${LOC_DIR}
+git clone https://github.com/netdata/localization.git ${LOC_DIR}
+
+echo "Preparing directories"
+MKDOCS_CONFIG_FILE="${GENERATOR_DIR}/mkdocs.yml"
+MKDOCS_DIR="doc"
+DOCS_DIR=${GENERATOR_DIR}/${MKDOCS_DIR}
+rm -rf ${DOCS_DIR}
+mkdir ${DOCS_DIR}
+
+prep_html() {
+	lang="${1}"
+	echo "Creating ${lang} mkdocs.yaml"
+
+	if [ "${lang}" = "en" ] ; then
+		SITE_DIR="build"
+	else
+		SITE_DIR="build/${lang}"
+	fi
+
+	# Generate mkdocs.yaml
+	${GENERATOR_DIR}/buildyaml.sh ${MKDOCS_DIR} ${SITE_DIR} ${lang}>${MKDOCS_CONFIG_FILE}
+
+	echo "Fixing links"
+
+	# Fix links (recursively, all types, executing replacements)
+	${GENERATOR_DIR}/checklinks.sh -rax
+
+	echo "Calling mkdocs"
+
+	# Build html docs
+	mkdocs build --config-file="${MKDOCS_CONFIG_FILE}"
+
+	# Fix edit buttons for the markdowns that are not on the main netdata repo
+	find "${GENERATOR_DIR}/${SITE_DIR}/${GO_D_DIR}" -name "*.html" -print0 | xargs -0 sed -i -e 's/https:\/\/github.com\/netdata\/netdata\/blob\/master\/collectors\/go.d.plugin/https:\/\/github.com\/netdata\/go.d.plugin\/blob\/master/g'
+	if [ "${lang}" != "en" ] ; then
+		find "${GENERATOR_DIR}/${SITE_DIR}" -name "*.html" -print0 | xargs -0 sed -i -e 's/https:\/\/github.com\/netdata\/netdata\/blob\/master\/\S*md/https:\/\/github.com\/netdata\/localization\//g'
+	fi
+}
+
+for d in "en" $(find ${LOC_DIR} -mindepth 1 -maxdepth 1 -name .git -prune -o -type d -printf '%f ') ; do
+	echo "Preparing source for $d"
+	cp -a ${SRC_DIR}/* ${DOCS_DIR}/
+	if [ "${d}" != "en" ] ; then
+		cp -a ${LOC_DIR}/${d}/* ${DOCS_DIR}/
+	fi
+	prep_html $d
+	rm -rf ${DOCS_DIR}/*
+done
 
-# Remove the cloned go.d.plugin project
-rm -rf ./collectors/go.d.plugin
+# Remove cloned projects and temp directories
+rm -rf ${GO_D_DIR} ${LOC_DIR} ${DOCS_DIR} ${SRC_DIR}
 
 echo "Finished"
diff --git a/docs/generator/buildyaml.sh b/docs/generator/buildyaml.sh
index bafec7fc4a..553f60c82f 100755
--- a/docs/generator/buildyaml.sh
+++ b/docs/generator/buildyaml.sh
@@ -1,7 +1,12 @@
 #!/bin/bash
 
 GENERATOR_DIR="docs/generator"
-cd ${GENERATOR_DIR}/src
+
+docs_dir="${1}"
+site_dir="${2}"
+language="${3}"
+
+cd ${GENERATOR_DIR}/${docs_dir}
 
 # create yaml nav subtree with all the files directly under a specific directory
 # arguments:
@@ -48,8 +53,8 @@ repo_name: GitHub
 edit_uri: blob/master
 site_description: Netdata Documentation
 copyright: Netdata, 2018
-docs_dir: src
-site_dir: build
+docs_dir: '${docs_dir}'
+site_dir: '${site_dir}'
 #use_directory_urls: false
 strict: true
 extra:
@@ -64,6 +69,7 @@ theme:
     name: "material"
     custom_dir: custom/themes/material
     favicon: custom/img/favicon.ico
+    language: '${language}'
 extra_css:
   - "https://cdnjs.cloudflare.com/ajax/libs/cookieconsent2/3.1.0/cookieconsent.min.css"
   - "custom/css/netdata.css"
diff --git a/docs/generator/checklinks.sh b/docs/generator/checklinks.sh
index d0c3b165ce..6538d39b74 100755
--- a/docs/generator/checklinks.sh
+++ b/docs/generator/checklinks.sh
@@ -36,72 +36,6 @@ fix () {
 	fi
 }
 
-ck_netdata_absolute () {
-	f=$1
-	alnk=$2
-	lnkinfile=$3
-	testURL "$alnk"
-
-	if [[ $f =~ ^(.*)/([^/]*)$ ]] ; then
-		fpath="${BASH_REMATCH[1]}"
-		dbg "-- Current file is at $fpath"
-	fi
-
-	if [ $? -eq 0 ] ; then 
-		rlnk=$(echo "$alnk" | sed 's/https:\/\/github.com\/netdata\/netdata\/....\/master\///g')
-		case $rlnk in
-			\#* ) dbg "-- (#somelink)" ;;
-			*/ ) dbg "-- # (path/)" ;;
-			*/#* ) dbg "-- # (path/#somelink)" ;;
-			*/*.md ) dbg "-- # (path/filename.md)" ;;
-			*/*.md#* ) dbg "-- # (path/filename.md#somelink)" ;;
-			*#* ) 
-				dbg "-- # (path#somelink) -> (path/#somelink)" 
-				if [[ $rlnk =~ ^(.*)#(.*)$ ]] ; then
-					dbg "-- $rlnk -> ${BASH_REMATCH[1]}/#${BASH_REMATCH[2]}"
-					rlnk="${BASH_REMATCH[1]}/#${BASH_REMATCH[2]}"
-				fi
-				;;		
-			* )
-				if [ -f "$rlnk" ] ; then
-					dbg "-- # (path/someotherfile) $rlnk"
-				else
-					if [ -d "$rlnk" ] ; then
-						dbg "-- # (path) -> (path/)"
-						rlnk="$rlnk/"
-					else
-						echo "-- ERROR: $f - $alnk is neither a file nor a directory. Giving up!"
-						EXITCODE=1
-						return
-					fi
-				fi
-				;;
-		esac
-
-		if [[ $rlnk =~ ^(.*)/([^/]*)$ ]] ; then
-			abspath="${BASH_REMATCH[1]}"
-			rest="${BASH_REMATCH[2]}"
-			dbg "-- Target file is at $abspath"
-		fi
-		relativelink=$(realpath --relative-to="$fpath" "$abspath")
-		if [ $? -eq 0 ] ; then
-			srch=$(echo "$lnkinfile" | sed 's/\//\\\//g')
-			if [ "$relativelink" = "." ] ; then
-				rplc=$(echo "$rest" | sed 's/\//\\\//g')
-			else
-				rplc=$(echo "$relativelink/$rest" | sed 's/\//\\\//g')
-			fi
-			fix "sed -i 's/($srch)/($rplc)/g' $f"
-		else
-			echo "-- ERROR: $f - Can't determine relative path of $alnk" 
-		fi
-	else
-		echo "-- ERROR: $f - $alnk is a broken link"
-		EXITCODE=1
-		return
-	fi
-}
-
 testURL () {
 	if [ "$TESTURLS" -eq 0 ] ; then return 0 ; fi
 	dbg "-- Testing URL $1"
@@ -278,7 +212,7 @@ ck_netdata_relative () {
 		if [[ ! -z $s ]] ; then
 			srch=$(echo "$rlnk" | sed 's/\//\\\//g')
 			rplc=$(echo "$s" | sed 's/\//\\\//g')
-			fix "sed -i 's/($srch)/($rplc)/g' $GENERATOR_DIR/src/$f"
+			fix "sed -i 's/($srch)/($rplc)/g' $GENERATOR_DIR/doc/$f"
 		fi
 }
 
@@ -299,8 +233,8 @@ checklinks () {
 						if [ "$CHKWIKI" -eq 1 ] ; then echo "-- WARNING: $f - $lnk points to the wiki. Please replace it manually" ; fi
 					;;
 					https://github.com/netdata/netdata/????/master* )
-						dbg "-- Absolute link $lnk"
-						if [ "$CHKABSOLUTE" -eq 1 ] ; then ck_netdata_absolute "$f" "$lnk" "$lnk" ; fi
+						echo "-- ERROR: $f - $lnk is an absolute link to a netdata file. Please convert to relative."
+						EXITCODE=1
 					;;
 					http* ) 
 						dbg "-- External link $lnk"
diff --git a/docs/generator/custom/img/geography-16.png b/docs/generator/custom/img/geography-16.png
new file mode 100644
index 0000000000..48391f958c
--- /dev/null
+++ b/docs/generator/custom/img/geography-16.png
diff --git a/docs/generator/custom/themes/material/partials/footer.html b/docs/generator/custom/themes/material/partials/footer.html
index fe232b6d50..0631a30426 100644
--- a/docs/generator/custom/themes/material/partials/footer.html
+++ b/docs/generator/custom/themes/material/partials/footer.html
@@ -52,3 +52,16 @@
   </div>
 </footer>
 <script>!function(e,a,t,n,o,c,i){e.GoogleAnalyticsObject=o,e.ga=e.ga||function(){(e.ga.q=e.ga.q||[]).push(arguments)},e.ga.l=1*new Date,c=a.createElement(t),i=a.getElementsByTagName(t)[0],c.async=1,c.src="https://www.google-analytics.com/analytics.js",i.parentNode.insertBefore(c,i)}(window,document,"script",0,"ga"),ga("create","UA-64295674-3",""),ga("set","anonymizeIp",!0),ga("send","pageview","/doc"+window.location.pathname);var links=document.getElementsByTagName("a");if(Array.prototype.map.call(links,function(a){a.host!=document.location.host&&a.addEventListener("click",function(){var e=a.getAttribute("data-md-action")||"follow";ga("send","event","outbound",e,a.href)})}),document.forms.search){var query=document.forms.search.query;query.addEventListener("blur",function(){if(this.value){var e=document.location.pathname;ga("send","pageview",e+"?q="+this.value)}})}</script>
+<script>
+  let currentLang = getLanguage();
+
+  let sel = document.getElementById('sel');
+  let opts = sel.options;
+  for (let opt, j = 0; opt = opts[j]; j++) {
+    if (opt.value == currentLang) {
+      sel.selectedIndex = j;
+      break;
+    }
+  }
+
+</script>
diff --git a/docs/generator/custom/themes/material/partials/header.html b/docs/generator/custom/themes/material/partials/header.html
new file mode 100644
index 0000000000..54086ecf5c
--- /dev/null
+++ b/docs/generator/custom/themes/material/partials/header.html
@@ -0,0 +1,107 @@
+<header class="md-header" data-md-component="header">
+  <nav class="md-header-nav md-grid">
+    <div class="md-flex">
+      <div class="md-flex__cell md-flex__cell--shrink">
+        <a href="{{ config.site_url | default(nav.homepage.url, true) | url }}" title="{{ config.site_name }}" class="md-header-nav__button md-logo">
+          {% if config.theme.logo.icon %}
+            <i class="md-icon">{{ config.theme.logo.icon }}</i>
+          {% else %}
+            <img src="{{ config.theme.logo | url }}" width="24" height="24">
+          {% endif %}
+        </a>
+      </div>
+      <div class="md-flex__cell md-flex__cell--shrink">
+        <label class="md-icon md-icon--menu md-header-nav__button" for="__drawer"></label>
+      </div>
+      <div class="md-flex__cell md-flex__cell--stretch">
+        <div class="md-flex__ellipsis md-header-nav__title" data-md-component="title">
+          {% block site_name %}
+            {% if config.site_name == page.title %}
+              {{ config.site_name }}
+            {% else %}
+              <span class="md-header-nav__topic">
+                {{ config.site_name }}
+              </span>
+              <span class="md-header-nav__topic">
+                {{ page.title }}
+              </span>
+            {% endif %}
+          {% endblock %}
+        </div>
+      </div>
+      <div class="md-flex__cell md-flex__cell--shrink">
+        {% block search_box %}
+          {% if "search" in config["plugins"] %}
+            <label class="md-icon md-icon--search md-header-nav__button" for="__search"></label>
+            {% include "partials/search.html" %}
+          {% endif %}
+        {% endblock %}
+      </div>
+
+      <!-- netdata -->
+      <style>
+        .language-selector li {
+          list-style: none;
+        }
+
+        .language-option.selected {
+          background-color: #ccc;
+        }
+      </style>
+      <script>
+        function getLanguage() {
+          const lang = window.location.pathname.split("/")[1];
+
+          if (lang.length == 0 || lang.length > 2) {
+            return "en";
+          }
+
+          return lang;
+        }
+
+        function languagePrefix(lang) {
+          if (lang === "en") {
+            return "";
+          }
+
+          return `/${lang}`;
+        }
+
+        function updatePathname(pathname, lang) {
+          if (currentLang !== "en") {
+            const parts = pathname.split("/");
+            parts.shift();
+            parts.shift();
+            pathname = `/${parts.join("/")}`;
+          }
+
+          return `${languagePrefix(lang)}${pathname}`;
+        }
+
+        function setLanguage(sel) {
+          if (sel.value === currentLang) {
+            return;
+          }
+
+          window.location.pathname = updatePathname(window.location.pathname, sel.value);
+        }
+      </script>
+
+      <div style="vertical-align: middle; white-space: nowrap; padding-left: 20px;" class="md-flex__cell md-flex__cell--shrink">
+        <img src="/custom/img/geography-16.png" style="vertical-align: middle;"/>
+        <select id="sel" onchange="setLanguage(this);" style="vertical-align: middle; background-color: #3f51b5; color: white; border: none;">
+          <option href="#" value='en'>English</option>
+          <option href="#" value='zh'>中文</option>
+        </select>
+      </div>
+
+      {% if config.repo_url %}
+        <div class="md-flex__cell md-flex__cell--shrink">
+          <div class="md-header-nav__source">
+            {% include "partials/source.html" %}
+          </div>
+        </div>
+      {% endif %}
+    </div>
+  </nav>
+</header>