From 0f9e84d2a4754c51cba79615820bcd53b0d9dd48 Mon Sep 17 00:00:00 2001 From: Sergey Grebenshchikov Date: Fri, 30 Mar 2018 21:10:19 +0200 Subject: Add heatmap plots --- Makefile | 2 +- README.md | 52 ++++++++++++++++++++++++++++++----- README.template.md | 40 ++++++++++++++++++++++++++- cmd/jp/canvas_other.go | 1 + cmd/jp/canvas_windows.go | 1 + cmd/jp/heatmap.go | 55 +++++++++++++++++++++++++++++++++++++ cmd/jp/hist.go | 2 +- cmd/jp/main.go | 13 +++++++-- pkg/data/heatmap.go | 68 ++++++++++++++++++++++++++++++++++++++++++++++ pkg/data/histogram.go | 52 +++++++++++++++++++++++++++++++---- pkg/draw/heatmap.go | 23 ++++++++++++++++ pkg/jsonpath/jsonpath.go | 2 ++ pkg/plot/heatmap.go | 71 ++++++++++++++++++++++++++++++++++++++++++++++++ 13 files changed, 364 insertions(+), 18 deletions(-) create mode 100644 cmd/jp/heatmap.go create mode 100644 pkg/data/heatmap.go create mode 100644 pkg/draw/heatmap.go create mode 100644 pkg/plot/heatmap.go diff --git a/Makefile b/Makefile index 7330c94..afc7af4 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -VERSION = 1.1.7 +VERSION = 1.1.8 APP := jp PACKAGES := $(shell go list -f {{.Dir}} ./...) diff --git a/README.md b/README.md index 588b983..eafb5dc 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ Dead simple terminal plots from JSON (or CSV) data. Bar charts, line charts, and - [Array data, XY pairs](#array-data-xy-pairs) - [Y values only (X=index)](#y-values-only-xindex-1) - [Scatter plot](#scatter-plot) + - [Heatmap](#heatmap) - [Histogram](#histogram) - [Auto bin number](#auto-bin-number) - [Fixed bin number](#fixed-bin-number) @@ -38,16 +39,16 @@ Or [download the binary](https://github.com/sgreben/jp/releases/latest) from the ```bash # Linux -curl -LO https://github.com/sgreben/jp/releases/download/1.1.7/jp_1.1.7_linux_x86_64.zip -unzip jp_1.1.7_linux_x86_64.zip +curl -LO https://github.com/sgreben/jp/releases/download/1.1.8/jp_1.1.8_linux_x86_64.zip +unzip jp_1.1.8_linux_x86_64.zip # OS X -curl -LO https://github.com/sgreben/jp/releases/download/1.1.7/jp_1.1.7_osx_x86_64.zip -unzip jp_1.1.7_osx_x86_64.zip +curl -LO https://github.com/sgreben/jp/releases/download/1.1.8/jp_1.1.8_osx_x86_64.zip +unzip jp_1.1.8_osx_x86_64.zip # Windows -curl -LO https://github.com/sgreben/jp/releases/download/1.1.7/jp_1.1.7_windows_x86_64.zip -unzip jp_1.1.7_windows_x86_64.zip +curl -LO https://github.com/sgreben/jp/releases/download/1.1.8/jp_1.1.8_windows_x86_64.zip +unzip jp_1.1.8_windows_x86_64.zip ``` ## Use it @@ -57,7 +58,7 @@ unzip jp_1.1.7_windows_x86_64.zip ```text Usage of jp: -type value - Plot type. One of [line bar scatter hist] (default line) + Plot type. One of [line bar scatter hist hist2d] (default line) -x string x values (JSONPath expression) -y string @@ -282,6 +283,43 @@ $ cat examples/mvrnorm.json | jp -xy '..[x,y]' -type scatter -4.08815 3.79083 ``` +### Heatmap + +``` +$ cat examples/mvrnorm.json | jp -xy '..[x,y]' -type hist2d + + 3.3608│ ···· ········ ···· + │ ···· ········ ···· + │ ···· ········ ···· + │ ················ ···· + │ ················ ···· + │ ································ + │ ································ + │ ················░░░░░░░░░░░░················ + │ ················░░░░░░░░░░░░················ + │ ············▒▒▒▒▓▓▓▓▒▒▒▒▒▒▒▒░░░░············ + │ ············▒▒▒▒▓▓▓▓▒▒▒▒▒▒▒▒░░░░············ + │···············▒▒▒▒▒▒▒▒▓▓▓▓▓▓▓▓▓▓▓▓▒▒▒▒············ + │···············▒▒▒▒▒▒▒▒▓▓▓▓▓▓▓▓▓▓▓▓▒▒▒▒············ + │ ········░░░░▓▓▓▓▓▓▓▓████▓▓▓▓▒▒▒▒············ + │ ········░░░░▓▓▓▓▓▓▓▓████▓▓▓▓▒▒▒▒············ + │ ········░░░░▓▓▓▓▓▓▓▓████▓▓▓▓▒▒▒▒············ + │ ············▒▒▒▒▒▒▒▒▓▓▓▓▓▓▓▓▒▒▒▒▒▒▒▒············ + │ ············▒▒▒▒▒▒▒▒▓▓▓▓▓▓▓▓▒▒▒▒▒▒▒▒············ + │ ············▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒················ + │ ············▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒················ + │ ···································· + │ ···································· + │ ···························· + │ ···························· + │ ···· ···· ···· + │ ···· ···· ···· + │ ···· + │ ···· + -4.0045└─────────────────────────────────────────────────── + -3.8421 3.5909 +``` + ### Histogram #### Auto bin number diff --git a/README.template.md b/README.template.md index ea0db50..6d83d29 100644 --- a/README.template.md +++ b/README.template.md @@ -19,6 +19,7 @@ Dead simple terminal plots from JSON (or CSV) data. Bar charts, line charts, and - [Array data, XY pairs](#array-data-xy-pairs) - [Y values only (X=index)](#y-values-only-xindex-1) - [Scatter plot](#scatter-plot) + - [Heatmap](#heatmap) - [Histogram](#histogram) - [Auto bin number](#auto-bin-number) - [Fixed bin number](#fixed-bin-number) @@ -57,7 +58,7 @@ unzip jp_${VERSION}_windows_x86_64.zip ```text Usage of jp: -type value - Plot type. One of [line bar scatter hist] (default line) + Plot type. One of [line bar scatter hist hist2d] (default line) -x string x values (JSONPath expression) -y string @@ -282,6 +283,43 @@ $ cat examples/mvrnorm.json | jp -xy '..[x,y]' -type scatter -4.08815 3.79083 ``` +### Heatmap + +``` +$ cat examples/mvrnorm.json | jp -xy '..[x,y]' -type hist2d + + 3.3608│ ···· ········ ···· + │ ···· ········ ···· + │ ···· ········ ···· + │ ················ ···· + │ ················ ···· + │ ································ + │ ································ + │ ················░░░░░░░░░░░░················ + │ ················░░░░░░░░░░░░················ + │ ············▒▒▒▒▓▓▓▓▒▒▒▒▒▒▒▒░░░░············ + │ ············▒▒▒▒▓▓▓▓▒▒▒▒▒▒▒▒░░░░············ + │···············▒▒▒▒▒▒▒▒▓▓▓▓▓▓▓▓▓▓▓▓▒▒▒▒············ + │···············▒▒▒▒▒▒▒▒▓▓▓▓▓▓▓▓▓▓▓▓▒▒▒▒············ + │ ········░░░░▓▓▓▓▓▓▓▓████▓▓▓▓▒▒▒▒············ + │ ········░░░░▓▓▓▓▓▓▓▓████▓▓▓▓▒▒▒▒············ + │ ········░░░░▓▓▓▓▓▓▓▓████▓▓▓▓▒▒▒▒············ + │ ············▒▒▒▒▒▒▒▒▓▓▓▓▓▓▓▓▒▒▒▒▒▒▒▒············ + │ ············▒▒▒▒▒▒▒▒▓▓▓▓▓▓▓▓▒▒▒▒▒▒▒▒············ + │ ············▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒················ + │ ············▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒················ + │ ···································· + │ ···································· + │ ···························· + │ ···························· + │ ···· ···· ···· + │ ···· ···· ···· + │ ···· + │ ···· + -4.0045└─────────────────────────────────────────────────── + -3.8421 3.5909 +``` + ### Histogram #### Auto bin number diff --git a/cmd/jp/canvas_other.go b/cmd/jp/canvas_other.go index 053cef7..db0b3d4 100644 --- a/cmd/jp/canvas_other.go +++ b/cmd/jp/canvas_other.go @@ -7,4 +7,5 @@ var autoCanvas = map[string]string{ plotTypeLine: canvasTypeQuarter, plotTypeScatter: canvasTypeBraille, plotTypeHist: canvasTypeQuarter, + plotTypeHeatmap: canvasTypeFull, } diff --git a/cmd/jp/canvas_windows.go b/cmd/jp/canvas_windows.go index de22e5b..56cc212 100644 --- a/cmd/jp/canvas_windows.go +++ b/cmd/jp/canvas_windows.go @@ -7,4 +7,5 @@ var autoCanvas = map[string]string{ plotTypeLine: canvasTypeFull, plotTypeScatter: canvasTypeFull, plotTypeHist: canvasTypeFull, + plotTypeHeatmap: canvasTypeFull, } diff --git a/cmd/jp/heatmap.go b/cmd/jp/heatmap.go new file mode 100644 index 0000000..9ebd2fe --- /dev/null +++ b/cmd/jp/heatmap.go @@ -0,0 +1,55 @@ +package main + +import ( + "log" + "reflect" + + "github.com/sgreben/jp/pkg/data" + "github.com/sgreben/jp/pkg/draw" + "github.com/sgreben/jp/pkg/plot" +) + +func heatmapData(xv []reflect.Value, yv []reflect.Value, nbins uint) (heatmap *data.Heatmap) { + var x, y []float64 + for i := range xv { + if xv[i].IsValid() && xv[i].CanInterface() { + xvi, ok := xv[i].Interface().(float64) + if ok { + x = append(x, xvi) + } + } + } + for i := range yv { + if yv[i].IsValid() && yv[i].CanInterface() { + yvi, ok := yv[i].Interface().(float64) + if ok { + y = append(y, yvi) + } + } + } + if len(x) != len(y) { + log.Fatal(len(x), " = len(x) != len(y) = ", len(y)) + } + points := make([][2]float64, len(x)) + for i := 0; i < len(x); i++ { + points[i] = [2]float64{x[i], y[i]} + } + if len(x) == 0 { + log.Fatal("no valid x values given") + } + bins := data.NewBins2D(points) + bins.X.Number = int(nbins) + bins.Y.Number = int(nbins) + if nbins == 0 { + bins.X.Number = data.BinsSturges(len(points)) + bins.Y.Number = data.BinsSturges(len(points)) + } + heatmap = data.NewHeatmap(data.Histogram2D(points, bins)) + return +} + +func heatmap(xv, yv []reflect.Value, c draw.Canvas, nbins uint) string { + heatmap := heatmapData(xv, yv, nbins) + chart := plot.NewHeatMap(c.GetBuffer()) + return chart.Draw(heatmap) +} diff --git a/cmd/jp/hist.go b/cmd/jp/hist.go index 77c04e6..c49f387 100644 --- a/cmd/jp/hist.go +++ b/cmd/jp/hist.go @@ -25,7 +25,7 @@ func histogramData(xv []reflect.Value, nbins uint) (groups []string, counts []fl bins := data.NewBins(x) bins.Number = int(nbins) if nbins == 0 { - bins.ChooseSturges() + bins.Number = data.BinsSturges(len(x)) } hist := data.Histogram(x, bins) groups = make([]string, len(hist)) diff --git a/cmd/jp/main.go b/cmd/jp/main.go index 173ac27..ac5b099 100644 --- a/cmd/jp/main.go +++ b/cmd/jp/main.go @@ -31,6 +31,7 @@ const ( plotTypeBar = "bar" plotTypeScatter = "scatter" plotTypeHist = "hist" + plotTypeHeatmap = "hist2d" ) const ( @@ -53,6 +54,7 @@ var config = configuration{ plotTypeBar, plotTypeScatter, plotTypeHist, + plotTypeHeatmap, }, }, CanvasType: enumVar{ @@ -123,6 +125,11 @@ func init() { } func match(in interface{}, p *jsonpath.JSONPath) [][]reflect.Value { + defer func() { + if r := recover(); r != nil { + log.Println("error evaluating JSONPath", p.String+":", r) + } + }() out, err := p.FindResults(in) if err != nil { log.Println(err) @@ -137,13 +144,13 @@ func main() { dec := json.NewDecoder(os.Stdin) err := dec.Decode(&in) if err != nil { - log.Println(err) + log.Println(inputTypeJSON, "input:", err) } case inputTypeCSV: r := csv.NewReader(os.Stdin) rows, err := r.ReadAll() if err != nil { - log.Println(err) + log.Println(inputTypeCSV, "input:", err) } in = parseRows(rows) } @@ -175,5 +182,7 @@ func main() { fmt.Println(barPlot(x, y, c)) case plotTypeHist: fmt.Println(histogram(x, c, config.HistBins)) + case plotTypeHeatmap: + fmt.Println(heatmap(x, y, c, config.HistBins)) } } diff --git a/pkg/data/heatmap.go b/pkg/data/heatmap.go new file mode 100644 index 0000000..7a0aa6f --- /dev/null +++ b/pkg/data/heatmap.go @@ -0,0 +1,68 @@ +package data + +import "math" + +type Heatmap struct { + X, Y []Bin + Z [][]float64 + MinX, MaxX uint64 + MinY, MaxY uint64 + MinZ, MaxZ uint64 +} + +func NewHeatmap(x, y []Bin, z [][]uint64) *Heatmap { + h := new(Heatmap) + h.X, h.Y = x, y + h.Z = make([][]float64, len(z)) + h.MinX, h.MinY, h.MinZ = math.MaxUint64, math.MaxUint64, math.MaxUint64 + h.MaxX, h.MaxY, h.MaxZ = 0, 0, 0 + for _, b := range x { + if b.Count > h.MaxX { + h.MaxX = b.Count + } + if b.Count < h.MinX { + h.MinX = b.Count + } + } + for _, b := range x { + b.CountNorm = float64(b.Count-h.MinX) / float64(h.MaxX-h.MinX) + } + for _, b := range y { + if b.Count > h.MaxY { + h.MaxY = b.Count + } + if b.Count < h.MinY { + h.MinY = b.Count + } + } + for _, b := range y { + b.CountNorm = float64(b.Count-h.MinY) / float64(h.MaxY-h.MinY) + } + for i := range z { + h.Z[i] = make([]float64, len(z[i])) + for _, b := range z[i] { + if b > h.MaxZ { + h.MaxZ = b + } + if b < h.MinZ { + h.MinZ = b + } + } + } + for i := range z { + for j := range z[i] { + h.Z[i][j] = float64(z[i][j]-h.MinZ) / float64(h.MaxZ-h.MinZ) + } + } + if h.MaxX == 0 { + h.MaxX = 1 + } + if h.MaxY == 0 { + h.MaxY = 1 + } + if h.MaxZ == 0 { + h.MaxZ = 1 + } + + return h +} diff --git a/pkg/data/histogram.go b/pkg/data/histogram.go index e16d000..be72142 100644 --- a/pkg/data/histogram.go +++ b/pkg/data/histogram.go @@ -23,6 +23,7 @@ type Bin struct { Right float64 RightInclusive bool Count uint64 + CountNorm float64 } func (b *Bin) String() string { @@ -38,16 +39,16 @@ type Bins struct { numPoints int } -func (b *Bins) ChooseSqrt() { - b.Number = int(math.Sqrt(float64(b.numPoints))) +func BinsSqrt(numPoints int) int { + return int(math.Sqrt(float64(numPoints))) } -func (b *Bins) ChooseSturges() { - b.Number = int(math.Ceil(math.Log2(float64(b.numPoints))) + 1) +func BinsSturges(numPoints int) int { + return int(math.Ceil(math.Log2(float64(numPoints))) + 1) } -func (b *Bins) ChooseRice() { - b.Number = int(math.Ceil(2 * math.Pow(float64(b.numPoints), 1.0/3.0))) +func BinsRice(numPoints int) int { + return int(math.Ceil(2 * math.Pow(float64(numPoints), 1.0/3.0))) } func NewBins(points []float64) *Bins { @@ -106,3 +107,42 @@ func Histogram(points []float64, bins *Bins) (out []Bin) { } return } + +type Bins2D struct { + X *Bins + Y *Bins +} + +func NewBins2D(points [][2]float64) *Bins2D { + bins := new(Bins2D) + xs := make([]float64, len(points)) + ys := make([]float64, len(points)) + for i := range points { + xs[i] = points[i][0] + ys[i] = points[i][1] + } + bins.X = NewBins(xs) + bins.Y = NewBins(ys) + return bins +} + +func Histogram2D(points [][2]float64, bins *Bins2D) (x, y []Bin, z [][]uint64) { + x = bins.X.All() + y = bins.Y.All() + z = make([][]uint64, len(y)) + for _, b := range x { + b.Count = 0 + } + for i, b := range y { + z[i] = make([]uint64, len(x)) + b.Count = 0 + } + for _, p := range points { + i := bins.X.Point(p[0]) + j := bins.Y.Point(p[1]) + x[i].Count++ + y[j].Count++ + z[i][j]++ + } + return +} diff --git a/pkg/draw/heatmap.go b/pkg/draw/heatmap.go new file mode 100644 index 0000000..7f913fd --- /dev/null +++ b/pkg/draw/heatmap.go @@ -0,0 +1,23 @@ +package draw + +type Heatmap struct{ *Buffer } + +func (b *Heatmap) Size() Box { return b.Box } + +var shades = []rune(" ·░▒▒▒▒▓▓▓▓█") +var nonZeroShade = 1.0 / float64(len(shades)-1) + +func (b *Heatmap) Set(y, x int, fill float64) { + if fill < 0.0 { + fill = 0.0 + } + if fill > 0.0 && fill < nonZeroShade { + fill = nonZeroShade + } + if fill > 1.0 { + fill = 1.0 + } + b.Buffer.Set(y, x, shades[int(fill*float64(len(shades)-1))]) +} + +func (b *Heatmap) Clear() { b.Fill(shades[0]) } diff --git a/pkg/jsonpath/jsonpath.go b/pkg/jsonpath/jsonpath.go index 7ed9b16..6b11660 100644 --- a/pkg/jsonpath/jsonpath.go +++ b/pkg/jsonpath/jsonpath.go @@ -26,6 +26,7 @@ import ( type JSONPath struct { name string + String string parser *Parser stack [][]reflect.Value // push and pop values in different scopes cur []reflect.Value // current scope values @@ -56,6 +57,7 @@ func (j *JSONPath) AllowMissingKeys(allow bool) *JSONPath { // Parse parses the given template and returns an error. func (j *JSONPath) Parse(text string) error { var err error + j.String = text j.parser, err = Parse(j.name, text) return err } diff --git a/pkg/plot/heatmap.go b/pkg/plot/heatmap.go new file mode 100644 index 0000000..2667b40 --- /dev/null +++ b/pkg/plot/heatmap.go @@ -0,0 +1,71 @@ +package plot + +import ( + "bytes" + + "github.com/sgreben/jp/pkg/data" + "github.com/sgreben/jp/pkg/draw" +) + +// HeatMap is a heatmap +type HeatMap struct{ draw.Heatmap } + +// NewHeatMap returns a new line chart +func NewHeatMap(buffer *draw.Buffer) *HeatMap { return &HeatMap{draw.Heatmap{buffer}} } + +func (c *HeatMap) drawAxes(paddingX, paddingY int, minX, maxX, minY, maxY float64) { + buffer := c.GetBuffer() + // X axis + buffer.SetRow(1, paddingX, buffer.Width, draw.HorizontalLine) + // Y axis + buffer.SetColumn(1, buffer.Height, paddingX, draw.VerticalLine) + // Corner + buffer.Set(1, paddingX, draw.CornerBottomLeft) + // Labels + buffer.WriteRight(1, 1, Ff(minY)) + buffer.WriteLeft(buffer.Height-1, paddingX, Ff(maxY)) + buffer.WriteRight(0, paddingX, Ff(minX)) + buffer.WriteLeft(0, buffer.Width, Ff(maxX)) +} + +// Draw implements Chart +func (c *HeatMap) Draw(heatmap *data.Heatmap) string { + var scaleY, scaleX float64 + + minX := heatmap.X[0].LeftInclusive + maxX := heatmap.X[len(heatmap.X)-1].Right + minY := heatmap.Y[0].LeftInclusive + maxY := heatmap.Y[len(heatmap.Y)-1].Right + minLabelWidth := len(Ff(minY)) + maxLabelWidth := len(Ff(maxY)) + + paddingX := minLabelWidth + 1 + paddingY := 2 + if minLabelWidth < maxLabelWidth { + paddingX = maxLabelWidth + 1 + } + chartWidth := c.Size().Width - (paddingX + 1) + chartHeight := c.Size().Height - paddingY + scaleX = float64(chartWidth) / (maxX - minX) + scaleY = float64(chartHeight) / (maxY - minY) + + for i := range heatmap.Z { + for j := range heatmap.Z[i] { + x0 := int((heatmap.X[j].LeftInclusive-minX)*scaleX + float64(paddingX+1)) + y0 := int((heatmap.Y[i].LeftInclusive-minY)*scaleY + float64(paddingY)) + x1 := int((heatmap.X[j].Right-minX)*scaleX + float64(paddingX+1)) + y1 := int((heatmap.Y[i].Right-minY)*scaleY + float64(paddingY)) + z := heatmap.Z[i][j] + for x := x0; x < x1; x++ { + for y := y0; y < y1; y++ { + c.Set(y, x, z) + } + } + } + } + c.drawAxes(paddingX, paddingY, minX, maxX, minY, maxY) + + b := bytes.NewBuffer(nil) + c.GetBuffer().Render(b) + return b.String() +} -- cgit v1.2.3