Hugo import from jekyll

usage: hugo import jekyll jekyll_root_path target_path

Implemented:
 * Create new hugo site
 * Create config.yaml
 * Convert all markdown contents.
 * Copy all other files and folders to static

Fixes #101

1 files changed, 483 insertions, 0 deletions

diff --git a/commands/import.go b/commands/import.go
new file mode 100644
index 000000000..f6ca75a7a
--- /dev/null
+++ b/commands/import.go
@@ -0,0 +1,483 @@
+// Copyright © 2015 Steve Francia <spf@spf13.com>.
+//
+// Licensed under the Simple Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://opensource.org/licenses/Simple-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package commands
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/spf13/cast"
+	"github.com/spf13/cobra"
+	"github.com/spf13/hugo/helpers"
+	"github.com/spf13/hugo/hugofs"
+	"github.com/spf13/hugo/hugolib"
+	"github.com/spf13/hugo/parser"
+	jww "github.com/spf13/jwalterweatherman"
+)
+
+func init() {
+	importCmd.AddCommand(importJekyllCmd)
+}
+
+var importCmd = &cobra.Command{
+	Use:   "import",
+	Short: "import from others",
+	Long: `import from others like jekyll.
+
+Import requires a subcommand, e.g. ` + "`hugo import jekyll jekyll_root_path target_path`.",
+	Run: nil,
+}
+
+var importJekyllCmd = &cobra.Command{
+	Use:   "jekyll",
+	Short: "hugo import from jekyll",
+	Long: `hugo import from jekyll.
+
+Import jekyll requires two path, e.g. ` + "`hugo import jekyll jekyll_root_path target_path`.",
+	Run: importFromJekyll,
+}
+
+func importFromJekyll(cmd *cobra.Command, args []string) {
+	jww.SetLogThreshold(jww.LevelTrace)
+	jww.SetStdoutThreshold(jww.LevelWarn)
+
+	if len(args) < 2 {
+		jww.ERROR.Println(`Import jekyll requires two path, e.g. ` + "`hugo import jekyll jekyll_root_path target_path`.")
+		return
+	}
+
+	jekyllRoot, err := filepath.Abs(filepath.Clean(args[0]))
+	if err != nil {
+		jww.ERROR.Println("Path error:", args[0])
+		return
+	}
+
+	targetDir, err := filepath.Abs(filepath.Clean(args[1]))
+	if err != nil {
+		jww.ERROR.Println("Path error:", args[1])
+		return
+	}
+
+	createSiteFromJekyll(jekyllRoot, targetDir)
+
+	jww.INFO.Println("Import jekyll from:", jekyllRoot, "to:", targetDir)
+	fmt.Println("Importing...")
+
+	fileCount := 0
+	callback := func(path string, fi os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if fi.IsDir() {
+			return nil
+		}
+
+		relPath, err := filepath.Rel(jekyllRoot, path)
+		if err != nil {
+			jww.ERROR.Println("Get rel path error:", path)
+			return err
+		}
+
+		relPath = filepath.ToSlash(relPath)
+		var draft bool = false
+
+		switch {
+		case strings.HasPrefix(relPath, "_posts/"):
+			relPath = "content/post" + relPath[len("_posts"):]
+		case strings.HasPrefix(relPath, "_drafts/"):
+			relPath = "content/draft" + relPath[len("_drafts"):]
+			draft = true
+		default:
+			return nil
+		}
+
+		fileCount++
+		return convertJekyllPost(path, relPath, targetDir, draft)
+	}
+
+	err = filepath.Walk(jekyllRoot, callback)
+
+	if err != nil {
+		fmt.Println(err)
+	} else {
+		fmt.Println("Congratulations!", fileCount, "posts imported!")
+		fmt.Println("Now, start hugo by yourself: \n" +
+			"$ git clone https://github.com/spf13/herring-cove.git " + args[1] + "/themes/herring-cove")
+		fmt.Println("$ cd " + args[1] + "\n$ hugo server -w --theme=herring-cove")
+	}
+}
+
+func createSiteFromJekyll(jekyllRoot, targetDir string) {
+	mkdir(targetDir, "layouts")
+	mkdir(targetDir, "content")
+	mkdir(targetDir, "archetypes")
+	mkdir(targetDir, "static")
+	mkdir(targetDir, "data")
+	mkdir(targetDir, "themes")
+
+	jekyllConfig := loadJekyllConfig(jekyllRoot)
+	createConfigFromJekyll(targetDir, "yaml", jekyllConfig)
+
+	copyJekyllFilesAndFolders(jekyllRoot, filepath.Join(targetDir, "static"))
+}
+
+func loadJekyllConfig(jekyllRoot string) map[string]interface{} {
+	fs := hugofs.SourceFs
+	path := filepath.Join(jekyllRoot, "_config.yml")
+
+	exists, err := helpers.Exists(path, fs)
+
+	if err != nil || !exists {
+		return nil
+	}
+
+	f, err := fs.Open(path)
+	if err != nil {
+		return nil
+	}
+
+	defer f.Close()
+
+	b, err := ioutil.ReadAll(f)
+
+	if err != nil {
+		return nil
+	}
+
+	c, err := parser.HandleYAMLMetaData(b)
+
+	if err != nil {
+		return nil
+	}
+
+	return c.(map[string]interface{})
+}
+
+func createConfigFromJekyll(inpath string, kind string, jekyllConfig map[string]interface{}) (err error) {
+	title := "My New Hugo Site"
+	baseurl := "http://replace-this-with-your-hugo-site.com/"
+
+	for key, value := range jekyllConfig {
+		lowerKey := strings.ToLower(key)
+
+		switch lowerKey {
+		case "title":
+			if str, ok := value.(string); ok {
+				title = str
+			}
+
+		case "url":
+			if str, ok := value.(string); ok {
+				baseurl = str
+			}
+		}
+	}
+
+	in := map[string]interface{}{
+		"baseurl":            baseurl,
+		"title":              title,
+		"languageCode":       "en-us",
+		"disablePathToLower": true,
+	}
+	kind = parser.FormatSanitize(kind)
+
+	by, err := parser.InterfaceToConfig(in, parser.FormatToLeadRune(kind))
+	if err != nil {
+		return err
+	}
+
+	err = helpers.WriteToDisk(filepath.Join(inpath, "config."+kind), bytes.NewReader(by), hugofs.SourceFs)
+	if err != nil {
+		return
+	}
+
+	return nil
+}
+
+func copyFile(source string, dest string) (err error) {
+	sf, err := os.Open(source)
+	if err != nil {
+		return err
+	}
+	defer sf.Close()
+	df, err := os.Create(dest)
+	if err != nil {
+		return err
+	}
+	defer df.Close()
+	_, err = io.Copy(df, sf)
+	if err == nil {
+		si, err := os.Stat(source)
+		if err != nil {
+			err = os.Chmod(dest, si.Mode())
+		}
+
+	}
+	return
+}
+
+func copyDir(source string, dest string) (err error) {
+	fi, err := os.Stat(source)
+	if err != nil {
+		return err
+	}
+	if !fi.IsDir() {
+		return errors.New(source + " is not a directory")
+	}
+	err = os.MkdirAll(dest, fi.Mode())
+	if err != nil {
+		return err
+	}
+	entries, err := ioutil.ReadDir(source)
+	for _, entry := range entries {
+		sfp := filepath.Join(source, entry.Name())
+		dfp := filepath.Join(dest, entry.Name())
+		if entry.IsDir() {
+			err = copyDir(sfp, dfp)
+			if err != nil {
+				jww.ERROR.Println(err)
+			}
+		} else {
+			err = copyFile(sfp, dfp)
+			if err != nil {
+				jww.ERROR.Println(err)
+			}
+		}
+
+	}
+	return nil
+}
+
+func copyJekyllFilesAndFolders(jekyllRoot string, dest string) (err error) {
+	fi, err := os.Stat(jekyllRoot)
+	if err != nil {
+		return err
+	}
+	if !fi.IsDir() {
+		return errors.New(jekyllRoot + " is not a directory")
+	}
+	err = os.MkdirAll(dest, fi.Mode())
+	if err != nil {
+		return err
+	}
+	entries, err := ioutil.ReadDir(jekyllRoot)
+	for _, entry := range entries {
+		sfp := filepath.Join(jekyllRoot, entry.Name())
+		dfp := filepath.Join(dest, entry.Name())
+		if entry.IsDir() {
+			if entry.Name()[0] != '_' && entry.Name()[0] != '.' {
+				err = copyDir(sfp, dfp)
+				if err != nil {
+					jww.ERROR.Println(err)
+				}
+			}
+		} else {
+			lowerEntryName := strings.ToLower(entry.Name())
+			exceptSuffix := []string{".md", ".markdown", ".html", ".htm",
+				".xml", ".textile", "rakefile", "gemfile", ".lock"}
+			isExcept := false
+			for _, suffix := range exceptSuffix {
+				if strings.HasSuffix(lowerEntryName, suffix) {
+					isExcept = true
+					break
+				}
+			}
+
+			if !isExcept && entry.Name()[0] != '.' && entry.Name()[0] != '_' {
+				err = copyFile(sfp, dfp)
+				if err != nil {
+					jww.ERROR.Println(err)
+				}
+			}
+		}
+
+	}
+	return nil
+}
+
+func parseJekyllFilename(filename string) (time.Time, string, error) {
+	re := regexp.MustCompile(`(\d+-\d+-\d+)-(.+)\..*`)
+	r := re.FindAllStringSubmatch(filename, -1)
+	if len(r) == 0 {
+		return time.Now(), "", errors.New("filename not match")
+	}
+
+	postDate, err := time.Parse("2006-01-02", r[0][1])
+	if err != nil {
+		return time.Now(), "", err
+	}
+
+	postName := r[0][2]
+
+	return postDate, postName, nil
+}
+
+func convertJekyllPost(path, relPath, targetDir string, draft bool) error {
+	jww.TRACE.Println("Converting", path)
+
+	filename := filepath.Base(path)
+	postDate, postName, err := parseJekyllFilename(filename)
+	if err != nil {
+		jww.ERROR.Println("Parse filename error:", filename)
+		return err
+	}
+
+	jww.TRACE.Println(filename, postDate, postName)
+
+	targetFile := filepath.Join(targetDir, relPath)
+	targetParentDir := filepath.Dir(targetFile)
+	os.MkdirAll(targetParentDir, 0777)
+
+	contentBytes, err := ioutil.ReadFile(path)
+	if err != nil {
+		jww.ERROR.Println("Read file error:", path)
+		return err
+	}
+
+	psr, err := parser.ReadFrom(bytes.NewReader(contentBytes))
+	if err != nil {
+		jww.ERROR.Println("Parse file error:", path)
+		return err
+	}
+
+	metadata, err := psr.Metadata()
+	if err != nil {
+		jww.ERROR.Println("Processing file error:", path)
+		return err
+	}
+
+	newmetadata, err := convertJekyllMetaData(metadata, postName, postDate, draft)
+	if err != nil {
+		jww.ERROR.Println("Convert metadata error:", path)
+		return err
+	}
+
+	jww.TRACE.Println(newmetadata)
+	content := convertJekyllContent(newmetadata, string(psr.Content()))
+
+	page, err := hugolib.NewPage(filename)
+	if err != nil {
+		jww.ERROR.Println("New page error", filename)
+		return err
+	}
+
+	page.SetDir(targetParentDir)
+	page.SetSourceContent([]byte(content))
+	page.SetSourceMetaData(newmetadata, parser.FormatToLeadRune("yaml"))
+	page.SaveSourceAs(targetFile)
+
+	jww.TRACE.Println("Target file:", targetFile)
+
+	return nil
+}
+
+func convertJekyllMetaData(m interface{}, postName string, postDate time.Time, draft bool) (interface{}, error) {
+	url := postDate.Format("/2006/01/02/") + postName + "/"
+
+	metadata, err := cast.ToStringMapE(m)
+	if err != nil {
+		return nil, err
+	}
+
+	if draft {
+		metadata["draft"] = true
+	}
+
+	for key, value := range metadata {
+		lowerKey := strings.ToLower(key)
+
+		switch lowerKey {
+		case "layout":
+			delete(metadata, key)
+		case "permalink":
+			if str, ok := value.(string); ok {
+				url = str
+			}
+			delete(metadata, key)
+		case "category":
+			if str, ok := value.(string); ok {
+				metadata["categories"] = []string{str}
+			}
+			delete(metadata, key)
+		case "excerpt_separator":
+			if key != lowerKey {
+				delete(metadata, key)
+				metadata[lowerKey] = value
+			}
+		case "date":
+			if str, ok := value.(string); ok {
+				re := regexp.MustCompile(`(\d+):(\d+):(\d+)`)
+				r := re.FindAllStringSubmatch(str, -1)
+				if len(r) > 0 {
+					hour, _ := strconv.Atoi(r[0][1])
+					minute, _ := strconv.Atoi(r[0][2])
+					second, _ := strconv.Atoi(r[0][3])
+					postDate = time.Date(postDate.Year(), postDate.Month(), postDate.Day(), hour, minute, second, 0, time.UTC)
+				}
+			}
+			delete(metadata, key)
+		}
+
+	}
+
+	metadata["url"] = url
+	metadata["date"] = postDate.Format(time.RFC3339)
+
+	return metadata, nil
+}
+
+func convertJekyllContent(m interface{}, content string) string {
+	metadata, _ := cast.ToStringMapE(m)
+
+	lines := strings.Split(content, "\n")
+	var resultLines []string
+	for _, line := range lines {
+		resultLines = append(resultLines, strings.Trim(line, "\r\n"))
+	}
+
+	content = strings.Join(resultLines, "\n")
+
+	excerptSep := "<!--more-->"
+	if value, ok := metadata["excerpt_separator"]; ok {
+		if str, strOk := value.(string); strOk {
+			content = strings.Replace(content, strings.TrimSpace(str), excerptSep, -1)
+		}
+	}
+
+	replaceList := []struct {
+		re      *regexp.Regexp
+		replace string
+	}{
+		{regexp.MustCompile("<!-- more -->"), "<!--more-->"},
+		{regexp.MustCompile(`\{%\s*raw\s*%\}\s*(.*?)\s*\{%\s*endraw\s*%\}`), "$1"},
+		{regexp.MustCompile(`{%\s*highlight\s*(.*?)\s*%}`), "{{< highlight $1 >}}"},
+		{regexp.MustCompile(`{%\s*endhighlight\s*%}`), "{{< / highlight >}}"},
+	}
+
+	for _, replace := range replaceList {
+		content = replace.re.ReplaceAllString(content, replace.replace)
+	}
+
+	return content
+}