Merge branch 'master' of github.com:go-gitea/gitea

vendor/gitea.com/jolheiser/gitea-vet/.gitignore

@@ -0,0 +1,5 @@
+# GoLand
+.idea/
+
+# Binaries
+/gitea-vet*

vendor/gitea.com/jolheiser/gitea-vet/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2020 The Gitea Authors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

vendor/gitea.com/jolheiser/gitea-vet/Makefile

@@ -0,0 +1,7 @@
+.PHONY: build
+build:
+	go build
+
+.PHONY: fmt
+fmt:
+	go fmt ./...

vendor/gitea.com/jolheiser/gitea-vet/README.md

@@ -0,0 +1,7 @@
+# gitea-vet
+`go vet` tool for Gitea
+
+| Analyzer | Description                                                         |
+|----------|---------------------------------------------------------------------|
+| Imports  | Checks for import sorting. stdlib->code.gitea.io->other             |
+| License  | Checks file headers for some form of `Copyright...YYYY...Gitea/Gogs`|

vendor/gitea.com/jolheiser/gitea-vet/checks/imports.go

@@ -0,0 +1,54 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package checks
+
+import (
+	"strings"
+
+	"golang.org/x/tools/go/analysis"
+)
+
+var Imports = &analysis.Analyzer{
+	Name: "imports",
+	Doc:  "check for import order.",
+	Run:  runImports,
+}
+
+func runImports(pass *analysis.Pass) (interface{}, error) {
+	for _, file := range pass.Files {
+		level := 0
+		for _, im := range file.Imports {
+			var lvl int
+			val := im.Path.Value
+			if importHasPrefix(val, "code.gitea.io") {
+				lvl = 2
+			} else if strings.Contains(val, ".") {
+				lvl = 3
+			} else {
+				lvl = 1
+			}
+
+			if lvl < level {
+				pass.Reportf(file.Pos(), "Imports are sorted wrong")
+				break
+			}
+			level = lvl
+		}
+	}
+	return nil, nil
+}
+
+func importHasPrefix(s, p string) bool {
+	return strings.HasPrefix(s, "\""+p)
+}
+
+func sliceHasPrefix(s string, prefixes ...string) bool {
+	for _, p := range prefixes {
+		if importHasPrefix(s, p) {
+			return true
+		}
+	}
+	return false
+}

vendor/gitea.com/jolheiser/gitea-vet/checks/license.go

@@ -0,0 +1,73 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package checks
+
+import (
+	"regexp"
+	"strings"
+
+	"golang.org/x/tools/go/analysis"
+)
+
+var (
+	header     = regexp.MustCompile(`.*Copyright.*\d{4}.*(Gitea|Gogs)`)
+	goGenerate = "//go:generate"
+	buildTag   = "// +build"
+)
+
+var License = &analysis.Analyzer{
+	Name: "license",
+	Doc:  "check for a copyright header.",
+	Run:  runLicense,
+}
+
+func runLicense(pass *analysis.Pass) (interface{}, error) {
+	for _, file := range pass.Files {
+		if len(file.Comments) == 0 {
+			pass.Reportf(file.Pos(), "Copyright not found")
+			continue
+		}
+
+		if len(file.Comments[0].List) == 0 {
+			pass.Reportf(file.Pos(), "Copyright not found or wrong")
+			continue
+		}
+
+		commentGroup := 0
+		if strings.HasPrefix(file.Comments[0].List[0].Text, goGenerate) {
+			if len(file.Comments[0].List) > 1 {
+				pass.Reportf(file.Pos(), "Must be an empty line between the go:generate and the Copyright")
+				continue
+			}
+			commentGroup++
+		}
+
+		if strings.HasPrefix(file.Comments[0].List[0].Text, buildTag) {
+			commentGroup++
+		}
+
+		if len(file.Comments) < commentGroup+1 {
+			pass.Reportf(file.Pos(), "Copyright not found")
+			continue
+		}
+
+		if len(file.Comments[commentGroup].List) < 1 {
+			pass.Reportf(file.Pos(), "Copyright not found or wrong")
+			continue
+		}
+
+		var check bool
+		for _, comment := range file.Comments[commentGroup].List {
+			if header.MatchString(comment.Text) {
+				check = true
+			}
+		}
+
+		if !check {
+			pass.Reportf(file.Pos(), "Copyright did not match check")
+		}
+	}
+	return nil, nil
+}

vendor/gitea.com/jolheiser/gitea-vet/go.mod

@@ -0,0 +1,5 @@
+module gitea.com/jolheiser/gitea-vet
+
+go 1.14
+
+require golang.org/x/tools v0.0.0-20200325010219-a49f79bcc224

vendor/gitea.com/jolheiser/gitea-vet/go.sum

@@ -0,0 +1,20 @@
+github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200325010219-a49f79bcc224 h1:azwY/v0y0K4mFHVsg5+UrTgchqALYWpqVo6vL5OmkmI=
+golang.org/x/tools v0.0.0-20200325010219-a49f79bcc224/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

vendor/gitea.com/jolheiser/gitea-vet/main.go

@@ -0,0 +1,17 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"gitea.com/jolheiser/gitea-vet/checks"
+	"golang.org/x/tools/go/analysis/unitchecker"
+)
+
+func main() {
+	unitchecker.Main(
+		checks.Imports,
+		checks.License,
+	)
+}

vendor/gitea.com/lunny/levelqueue/.drone.yml

@@ -0,0 +1,23 @@
+---
+kind: pipeline
+name: test
+
+workspace:
+  base: /go
+  path: src/gitea.com/lunny/levelqueue
+
+steps:
+- name: test
+  pull: default
+  image: golang:1.13
+  commands:
+  - go get -t -d -v ./...
+  - go build -v
+  - go test -v -race -coverprofile=coverage.txt -covermode=atomic
+  environment:
+    GO111MODULE: "on"
+    GOPROXY: https://goproxy.cn
+  when:
+    event:
+    - push
+    - pull_request

vendor/gitea.com/lunny/levelqueue/.gitignore

@@ -0,0 +1,7 @@
+queue/
+queue_pop/
+queue_push/
+uniquequeue/
+uniquequeue_pop/
+uniquequeue_push/
+set/

vendor/gitea.com/lunny/levelqueue/README.md

@@ -0,0 +1,60 @@
+# levelqueue
+
+Level queue is a simple queue golang library base on go-leveldb.
+
+[![Build Status](https://drone.gitea.com/api/badges/lunny/levelqueue/status.svg)](https://drone.gitea.com/lunny/levelqueue)  [![](http://gocover.io/_badge/gitea.com/lunny/levelqueue)](http://gocover.io/gitea.com/lunny/levelqueue)
+[![](https://goreportcard.com/badge/gitea.com/lunny/levelqueue)](https://goreportcard.com/report/gitea.com/lunny/levelqueue)
+
+## Installation
+
+```
+go get gitea.com/lunny/levelqueue
+```
+
+## Usage
+
+```Go
+queue, err := levelqueue.Open("./queue")
+
+err = queue.RPush([]byte("test"))
+
+// pop an element from left of the queue
+data, err = queue.LPop()
+
+// if handle success, element will be pop, otherwise it will be keep
+queue.LHandle(func(dt []byte) error{
+    return nil
+})
+```
+
+You can now create a Set from a leveldb:
+
+```Go
+set, err := levelqueue.OpenSet("./set")
+
+added, err:= set.Add([]byte("member1"))
+
+has, err := set.Has([]byte("member1"))
+
+members, err := set.Members()
+
+removed, err := set.Remove([]byte("member1"))
+```
+
+And you can create a UniqueQueue from a leveldb:
+
+```Go
+queue, err := levelqueue.OpenUnique("./queue")
+
+err := queue.RPush([]byte("member1"))
+
+err = queue.LPush([]byte("member1"))
+// Will return ErrAlreadyInQueue
+
+// and so on.
+```
+
+## Creating Queues, UniqueQueues and Sets from already open DB
+
+If you have an already open DB you can create these from this using the
+`NewQueue`, `NewUniqueQueue` and `NewSet` functions.

vendor/gitea.com/lunny/levelqueue/error.go

@@ -7,6 +7,8 @@ package levelqueue
 import "errors"
 
 var (
-	// ErrNotFound means no element in queue
+	// ErrNotFound means no elements in queue
 	ErrNotFound = errors.New("no key found")
+
+	ErrAlreadyInQueue = errors.New("value already in queue")
 )

vendor/gitea.com/lunny/levelqueue/go.mod

@@ -0,0 +1,9 @@
+module gitea.com/lunny/levelqueue
+
+require (
+	github.com/golang/snappy v0.0.1 // indirect
+	github.com/stretchr/testify v1.3.0
+	github.com/syndtr/goleveldb v1.0.0
+)
+
+go 1.13

vendor/gitea.com/lunny/levelqueue/go.sum

@@ -0,0 +1,42 @@
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w=
+github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
+github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
+github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/syndtr/goleveldb v0.0.0-20190203031304-2f17a3356c66 h1:AwmkkZT+TucFotNCL+aNJ/0KCMsRtlXN9fs8uoOMSRk=
+github.com/syndtr/goleveldb v0.0.0-20190203031304-2f17a3356c66/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
+github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
+github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e h1:o3PsSEY8E4eXWkXrIP9YJALUkVZqzHJT5DOasTyn8Vs=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

vendor/gitea.com/lunny/levelqueue/queue.go

@@ -0,0 +1,305 @@
+// Copyright 2019 Lunny Xiao. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package levelqueue
+
+import (
+	"bytes"
+	"encoding/binary"
+	"sync"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/errors"
+)
+
+const (
+	lowKeyStr  = "low"
+	highKeyStr = "high"
+)
+
+// Queue defines a queue struct
+type Queue struct {
+	db                *leveldb.DB
+	highLock          sync.Mutex
+	lowLock           sync.Mutex
+	low               int64
+	high              int64
+	lowKey            []byte
+	highKey           []byte
+	prefix            []byte
+	closeUnderlyingDB bool
+}
+
+// Open opens a queue from the db path or creates a
+// queue if it doesn't exist.
+// The keys will not be prefixed by default
+func Open(dataDir string) (*Queue, error) {
+	db, err := leveldb.OpenFile(dataDir, nil)
+	if err != nil {
+		if !errors.IsCorrupted(err) {
+			return nil, err
+		}
+		db, err = leveldb.RecoverFile(dataDir, nil)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return NewQueue(db, []byte{}, true)
+}
+
+// NewQueue creates a queue from a db. The keys will be prefixed with prefix
+// and at close the db will be closed as per closeUnderlyingDB
+func NewQueue(db *leveldb.DB, prefix []byte, closeUnderlyingDB bool) (*Queue, error) {
+	var err error
+
+	var queue = &Queue{
+		db:                db,
+		closeUnderlyingDB: closeUnderlyingDB,
+	}
+
+	queue.prefix = make([]byte, len(prefix))
+	copy(queue.prefix, prefix)
+	queue.lowKey = withPrefix(prefix, []byte(lowKeyStr))
+	queue.highKey = withPrefix(prefix, []byte(highKeyStr))
+
+	queue.low, err = queue.readID(queue.lowKey)
+	if err == leveldb.ErrNotFound {
+		queue.low = 1
+		err = db.Put(queue.lowKey, id2bytes(1), nil)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	queue.high, err = queue.readID(queue.highKey)
+	if err == leveldb.ErrNotFound {
+		err = db.Put(queue.highKey, id2bytes(0), nil)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	return queue, nil
+}
+
+func (queue *Queue) readID(key []byte) (int64, error) {
+	bs, err := queue.db.Get(key, nil)
+	if err != nil {
+		return 0, err
+	}
+	return bytes2id(bs)
+}
+
+func (queue *Queue) highincrement() (int64, error) {
+	id := queue.high + 1
+	queue.high = id
+	err := queue.db.Put(queue.highKey, id2bytes(queue.high), nil)
+	if err != nil {
+		queue.high = queue.high - 1
+		return 0, err
+	}
+	return id, nil
+}
+
+func (queue *Queue) highdecrement() (int64, error) {
+	queue.high = queue.high - 1
+	err := queue.db.Put(queue.highKey, id2bytes(queue.high), nil)
+	if err != nil {
+		queue.high = queue.high + 1
+		return 0, err
+	}
+	return queue.high, nil
+}
+
+func (queue *Queue) lowincrement() (int64, error) {
+	queue.low = queue.low + 1
+	err := queue.db.Put(queue.lowKey, id2bytes(queue.low), nil)
+	if err != nil {
+		queue.low = queue.low - 1
+		return 0, err
+	}
+	return queue.low, nil
+}
+
+func (queue *Queue) lowdecrement() (int64, error) {
+	queue.low = queue.low - 1
+	err := queue.db.Put(queue.lowKey, id2bytes(queue.low), nil)
+	if err != nil {
+		queue.low = queue.low + 1
+		return 0, err
+	}
+	return queue.low, nil
+}
+
+// Len returns the length of the queue
+func (queue *Queue) Len() int64 {
+	queue.lowLock.Lock()
+	queue.highLock.Lock()
+	l := queue.high - queue.low + 1
+	queue.highLock.Unlock()
+	queue.lowLock.Unlock()
+	return l
+}
+
+func id2bytes(id int64) []byte {
+	var buf = make([]byte, 8)
+	binary.PutVarint(buf, id)
+	return buf
+}
+
+func bytes2id(b []byte) (int64, error) {
+	return binary.ReadVarint(bytes.NewReader(b))
+}
+
+func withPrefix(prefix []byte, value []byte) []byte {
+	if len(prefix) == 0 {
+		return value
+	}
+	prefixed := make([]byte, len(prefix)+1+len(value))
+	copy(prefixed[0:len(prefix)], prefix)
+	prefixed[len(prefix)] = '-'
+	copy(prefixed[len(prefix)+1:], value)
+	return prefixed
+}
+
+// RPush pushes a data from right of queue
+func (queue *Queue) RPush(data []byte) error {
+	queue.highLock.Lock()
+	id, err := queue.highincrement()
+	if err != nil {
+		queue.highLock.Unlock()
+		return err
+	}
+	err = queue.db.Put(withPrefix(queue.prefix, id2bytes(id)), data, nil)
+	queue.highLock.Unlock()
+	return err
+}
+
+// LPush pushes a data from left of queue
+func (queue *Queue) LPush(data []byte) error {
+	queue.lowLock.Lock()
+	id, err := queue.lowdecrement()
+	if err != nil {
+		queue.lowLock.Unlock()
+		return err
+	}
+	err = queue.db.Put(withPrefix(queue.prefix, id2bytes(id)), data, nil)
+	queue.lowLock.Unlock()
+	return err
+}
+
+// RPop pop a data from right of queue
+func (queue *Queue) RPop() ([]byte, error) {
+	queue.highLock.Lock()
+	defer queue.highLock.Unlock()
+	currentID := queue.high
+
+	res, err := queue.db.Get(withPrefix(queue.prefix, id2bytes(currentID)), nil)
+	if err != nil {
+		if err == leveldb.ErrNotFound {
+			return nil, ErrNotFound
+		}
+		return nil, err
+	}
+
+	_, err = queue.highdecrement()
+	if err != nil {
+		return nil, err
+	}
+
+	err = queue.db.Delete(withPrefix(queue.prefix, id2bytes(currentID)), nil)
+	if err != nil {
+		return nil, err
+	}
+	return res, nil
+}
+
+// RHandle receives a user callback function to handle the right element of the queue, if function return nil, then delete the element, otherwise keep the element.
+func (queue *Queue) RHandle(h func([]byte) error) error {
+	queue.highLock.Lock()
+	defer queue.highLock.Unlock()
+	currentID := queue.high
+
+	res, err := queue.db.Get(withPrefix(queue.prefix, id2bytes(currentID)), nil)
+	if err != nil {
+		if err == leveldb.ErrNotFound {
+			return ErrNotFound
+		}
+		return err
+	}
+
+	if err = h(res); err != nil {
+		return err
+	}
+
+	_, err = queue.highdecrement()
+	if err != nil {
+		return err
+	}
+
+	return queue.db.Delete(withPrefix(queue.prefix, id2bytes(currentID)), nil)
+}
+
+// LPop pop a data from left of queue
+func (queue *Queue) LPop() ([]byte, error) {
+	queue.lowLock.Lock()
+	defer queue.lowLock.Unlock()
+	currentID := queue.low
+
+	res, err := queue.db.Get(withPrefix(queue.prefix, id2bytes(currentID)), nil)
+	if err != nil {
+		if err == leveldb.ErrNotFound {
+			return nil, ErrNotFound
+		}
+		return nil, err
+	}
+
+	_, err = queue.lowincrement()
+	if err != nil {
+		return nil, err
+	}
+
+	err = queue.db.Delete(withPrefix(queue.prefix, id2bytes(currentID)), nil)
+	if err != nil {
+		return nil, err
+	}
+	return res, nil
+}
+
+// LHandle receives a user callback function to handle the left element of the queue, if function return nil, then delete the element, otherwise keep the element.
+func (queue *Queue) LHandle(h func([]byte) error) error {
+	queue.lowLock.Lock()
+	defer queue.lowLock.Unlock()
+	currentID := queue.low
+
+	res, err := queue.db.Get(withPrefix(queue.prefix, id2bytes(currentID)), nil)
+	if err != nil {
+		if err == leveldb.ErrNotFound {
+			return ErrNotFound
+		}
+		return err
+	}
+
+	if err = h(res); err != nil {
+		return err
+	}
+
+	_, err = queue.lowincrement()
+	if err != nil {
+		return err
+	}
+
+	return queue.db.Delete(withPrefix(queue.prefix, id2bytes(currentID)), nil)
+}
+
+// Close closes the queue (and the underlying db is set to closeUnderlyingDB)
+func (queue *Queue) Close() error {
+	if !queue.closeUnderlyingDB {
+		queue.db = nil
+		return nil
+	}
+	err := queue.db.Close()
+	queue.db = nil
+	return err
+}

vendor/gitea.com/lunny/levelqueue/set.go

@@ -0,0 +1,117 @@
+// Copyright 2020 Andrew Thornton. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package levelqueue
+
+import (
+	"sync"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/errors"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+const (
+	setPrefixStr = "set"
+)
+
+// Set defines a set struct
+type Set struct {
+	db                *leveldb.DB
+	closeUnderlyingDB bool
+	lock              sync.Mutex
+	prefix            []byte
+}
+
+// OpenSet opens a set from the db path or creates a set if it doesn't exist.
+// The keys will be prefixed with "set-" by default
+func OpenSet(dataDir string) (*Set, error) {
+	db, err := leveldb.OpenFile(dataDir, nil)
+	if err != nil {
+		if !errors.IsCorrupted(err) {
+			return nil, err
+		}
+		db, err = leveldb.RecoverFile(dataDir, nil)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return NewSet(db, []byte(setPrefixStr), true)
+}
+
+// NewSet creates a set from a db. The keys will be prefixed with prefix
+// and at close the db will be closed as per closeUnderlyingDB
+func NewSet(db *leveldb.DB, prefix []byte, closeUnderlyingDB bool) (*Set, error) {
+	set := &Set{
+		db:                db,
+		closeUnderlyingDB: closeUnderlyingDB,
+	}
+	set.prefix = make([]byte, len(prefix))
+	copy(set.prefix, prefix)
+
+	return set, nil
+}
+
+// Add adds a member string to a key set, returns true if the member was not already present
+func (set *Set) Add(value []byte) (bool, error) {
+	set.lock.Lock()
+	defer set.lock.Unlock()
+	setKey := withPrefix(set.prefix, value)
+	has, err := set.db.Has(setKey, nil)
+	if err != nil || has {
+		return !has, err
+	}
+	return !has, set.db.Put(setKey, []byte(""), nil)
+}
+
+// Members returns the current members of the set
+func (set *Set) Members() ([][]byte, error) {
+	set.lock.Lock()
+	defer set.lock.Unlock()
+	var members [][]byte
+	prefix := withPrefix(set.prefix, []byte{})
+	iter := set.db.NewIterator(util.BytesPrefix(prefix), nil)
+	for iter.Next() {
+		slice := iter.Key()[len(prefix):]
+		value := make([]byte, len(slice))
+		copy(value, slice)
+		members = append(members, value)
+	}
+	iter.Release()
+	return members, iter.Error()
+}
+
+// Has returns if the member is in the set
+func (set *Set) Has(value []byte) (bool, error) {
+	set.lock.Lock()
+	defer set.lock.Unlock()
+	setKey := withPrefix(set.prefix, value)
+
+	return set.db.Has(setKey, nil)
+}
+
+// Remove removes a member from the set, returns true if the member was present
+func (set *Set) Remove(value []byte) (bool, error) {
+	set.lock.Lock()
+	defer set.lock.Unlock()
+	setKey := withPrefix(set.prefix, value)
+
+	has, err := set.db.Has(setKey, nil)
+	if err != nil || !has {
+		return has, err
+	}
+
+	return has, set.db.Delete(setKey, nil)
+}
+
+// Close closes the set (and the underlying db if set to closeUnderlyingDB)
+func (set *Set) Close() error {
+	if !set.closeUnderlyingDB {
+		set.db = nil
+		return nil
+	}
+	err := set.db.Close()
+	set.db = nil
+	return err
+}

vendor/gitea.com/lunny/levelqueue/uniquequeue.go

@@ -0,0 +1,191 @@
+// Copyright 2020 Andrew Thornton. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package levelqueue
+
+import (
+	"fmt"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/errors"
+)
+
+const (
+	uniqueQueuePrefixStr = "unique"
+)
+
+// UniqueQueue defines an unique queue struct
+type UniqueQueue struct {
+	q                 *Queue
+	set               *Set
+	db                *leveldb.DB
+	closeUnderlyingDB bool
+}
+
+// OpenUnique opens an unique queue from the db path or creates a set if it doesn't exist.
+// The keys in the queue portion will not be prefixed, and the set keys will be prefixed with "set-"
+func OpenUnique(dataDir string) (*UniqueQueue, error) {
+	db, err := leveldb.OpenFile(dataDir, nil)
+	if err != nil {
+		if !errors.IsCorrupted(err) {
+			return nil, err
+		}
+		db, err = leveldb.RecoverFile(dataDir, nil)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return NewUniqueQueue(db, []byte{}, []byte(uniqueQueuePrefixStr), true)
+}
+
+// NewUniqueQueue creates a new unique queue from a db.
+// The queue keys will be prefixed with queuePrefix and the set keys with setPrefix
+// and at close the db will be closed as per closeUnderlyingDB
+func NewUniqueQueue(db *leveldb.DB, queuePrefix []byte, setPrefix []byte, closeUnderlyingDB bool) (*UniqueQueue, error) {
+	internal, err := NewQueue(db, queuePrefix, false)
+	if err != nil {
+		return nil, err
+	}
+	set, err := NewSet(db, setPrefix, false)
+	if err != nil {
+		return nil, err
+	}
+	queue := &UniqueQueue{
+		q:                 internal,
+		set:               set,
+		db:                db,
+		closeUnderlyingDB: closeUnderlyingDB,
+	}
+
+	return queue, err
+}
+
+// LPush pushes data to the left of the queue
+func (queue *UniqueQueue) LPush(data []byte) error {
+	return queue.LPushFunc(data, nil)
+}
+
+// LPushFunc pushes data to the left of the queue and calls the callback if it is added
+func (queue *UniqueQueue) LPushFunc(data []byte, fn func() error) error {
+	added, err := queue.set.Add(data)
+	if err != nil {
+		return err
+	}
+	if !added {
+		return ErrAlreadyInQueue
+	}
+
+	if fn != nil {
+		err = fn()
+		if err != nil {
+			_, remErr := queue.set.Remove(data)
+			if remErr != nil {
+				return fmt.Errorf("%v & %v", err, remErr)
+			}
+			return err
+		}
+	}
+
+	return queue.q.LPush(data)
+}
+
+// RPush pushes data to the right of the queue
+func (queue *UniqueQueue) RPush(data []byte) error {
+	return queue.RPushFunc(data, nil)
+}
+
+// RPushFunc pushes data to the right of the queue and calls the callback if is added
+func (queue *UniqueQueue) RPushFunc(data []byte, fn func() error) error {
+	added, err := queue.set.Add(data)
+	if err != nil {
+		return err
+	}
+	if !added {
+		return ErrAlreadyInQueue
+	}
+
+	if fn != nil {
+		err = fn()
+		if err != nil {
+			_, remErr := queue.set.Remove(data)
+			if remErr != nil {
+				return fmt.Errorf("%v & %v", err, remErr)
+			}
+			return err
+		}
+	}
+
+	return queue.q.RPush(data)
+}
+
+// RPop pop data from the right of the queue
+func (queue *UniqueQueue) RPop() ([]byte, error) {
+	popped, err := queue.q.RPop()
+	if err != nil {
+		return popped, err
+	}
+	_, err = queue.set.Remove(popped)
+
+	return popped, err
+}
+
+// RHandle receives a user callback function to handle the right element of the queue, if the function returns nil, then delete the element, otherwise keep the element.
+func (queue *UniqueQueue) RHandle(h func([]byte) error) error {
+	return queue.q.RHandle(func(data []byte) error {
+		err := h(data)
+		if err != nil {
+			return err
+		}
+		_, err = queue.set.Remove(data)
+		return err
+	})
+}
+
+// LPop pops data from left of the queue
+func (queue *UniqueQueue) LPop() ([]byte, error) {
+	popped, err := queue.q.LPop()
+	if err != nil {
+		return popped, err
+	}
+	_, err = queue.set.Remove(popped)
+
+	return popped, err
+}
+
+// LHandle receives a user callback function to handle the left element of the queue, if the function returns nil, then delete the element, otherwise keep the element.
+func (queue *UniqueQueue) LHandle(h func([]byte) error) error {
+	return queue.q.LHandle(func(data []byte) error {
+		err := h(data)
+		if err != nil {
+			return err
+		}
+		_, err = queue.set.Remove(data)
+		return err
+	})
+}
+
+// Has checks whether the data is already in the queue
+func (queue *UniqueQueue) Has(data []byte) (bool, error) {
+	return queue.set.Has(data)
+}
+
+// Len returns the length of the queue
+func (queue *UniqueQueue) Len() int64 {
+	queue.set.lock.Lock()
+	defer queue.set.lock.Unlock()
+	return queue.q.Len()
+}
+
+// Close closes the queue (and the underlying DB if set to closeUnderlyingDB)
+func (queue *UniqueQueue) Close() error {
+	_ = queue.q.Close()
+	_ = queue.set.Close()
+	if !queue.closeUnderlyingDB {
+		queue.db = nil
+		return nil
+	}
+	err := queue.db.Close()
+	queue.db = nil
+	return err
+}

vendor/gitea.com/macaron/cors/cors.go

@@ -151,7 +151,8 @@ func CORS(options ...Options) macaron.Handler {
 			}
 		})
 		if reqOptions {
-			ctx.Status(200) // return response
+			ctx.Resp.WriteHeader(200) // return response
+			return
 		}
 	}
 }

vendor/gitea.com/macaron/gzip/.drone.yml

@@ -0,0 +1,24 @@
+kind: pipeline
+name: go1-1-2
+
+steps:
+- name: test
+  image: golang:1.12
+  environment:
+    GOPROXY: https://goproxy.cn
+  commands:
+  - go build -v
+  - go test -v -race -coverprofile=coverage.txt -covermode=atomic
+
+---
+kind: pipeline
+name: go1-1-3
+
+steps:
+- name: test
+  image: golang:1.13
+  environment:
+    GOPROXY: https://goproxy.cn
+  commands:
+  - go build -v
+  - go test -v -race -coverprofile=coverage.txt -covermode=atomic

vendor/gitea.com/macaron/gzip/README.md

@@ -0,0 +1,19 @@
+# gzip
+
+Middleware gzip provides gzip comparess middleware for [Macaron](https://gitea.com/macaron/macaron).
+
+### Installation
+
+	go get gitea.com/macaron/gzip
+
+## Getting Help
+
+- [API Reference](https://godoc.org/gitea.com/macaron/gzip)
+
+## Credits
+
+This package is a modified version of [go-macaron gzip](github.com/go-macaron/gzip).
+
+## License
+
+This project is under the Apache License, Version 2.0. See the [LICENSE](LICENSE) file for the full license text.

vendor/gitea.com/macaron/gzip/go.mod

@@ -0,0 +1,9 @@
+module gitea.com/macaron/gzip
+
+go 1.12
+
+require (
+	gitea.com/macaron/macaron v1.3.3-0.20190821202302-9646c0587edb
+	github.com/klauspost/compress v1.9.2
+	github.com/stretchr/testify v1.4.0
+)

vendor/gitea.com/macaron/gzip/go.sum

@@ -0,0 +1,42 @@
+gitea.com/macaron/inject v0.0.0-20190803172902-8375ba841591 h1:UbCTjPcLrNxR9LzKDjQBMT2zoxZuEnca1pZCpgeMuhQ=
+gitea.com/macaron/inject v0.0.0-20190803172902-8375ba841591/go.mod h1:h6E4kLao1Yko6DOU6QDnQPcuoNzvbZqzj2mtPcEn1aM=
+gitea.com/macaron/macaron v1.3.3-0.20190821202302-9646c0587edb h1:amL0md6orTj1tXY16ANzVU9FmzQB+W7aJwp8pVDbrmA=
+gitea.com/macaron/macaron v1.3.3-0.20190821202302-9646c0587edb/go.mod h1:0coI+mSPSwbsyAbOuFllVS38awuk9mevhLD52l50Gjs=
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg=
+github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/klauspost/compress v1.9.2 h1:LfVyl+ZlLlLDeQ/d2AqfGIIH4qEDu0Ed2S5GyhCWIWY=
+github.com/klauspost/compress v1.9.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304 h1:Jpy1PXuP99tXNrhbq2BaPz9B+jNAvH1JPQQpG/9GCXY=
+github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
+github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8=
+github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/unknwon/com v0.0.0-20190804042917-757f69c95f3e h1:GSGeB9EAKY2spCABz6xOX5DbxZEXolK+nBSvmsQwRjM=
+github.com/unknwon/com v0.0.0-20190804042917-757f69c95f3e/go.mod h1:tOOxU81rwgoCLoOVVPHb6T/wt8HZygqH5id+GNnlCXM=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc=
+golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/ini.v1 v1.44.0 h1:YRJzTUp0kSYWUVFF5XAbDFfyiqwsl0Vb9R8TVP5eRi0=
+gopkg.in/ini.v1 v1.44.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

vendor/gitea.com/macaron/gzip/gzip.go

@@ -0,0 +1,358 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package gzip
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"regexp"
+	"strconv"
+	"strings"
+	"sync"
+
+	"gitea.com/macaron/macaron"
+	"github.com/klauspost/compress/gzip"
+)
+
+const (
+	acceptEncodingHeader  = "Accept-Encoding"
+	contentEncodingHeader = "Content-Encoding"
+	contentLengthHeader   = "Content-Length"
+	contentTypeHeader     = "Content-Type"
+	rangeHeader           = "Range"
+	varyHeader            = "Vary"
+)
+
+const (
+	// MinSize is the minimum size of content we will compress
+	MinSize = 1400
+)
+
+// noopClosers are io.Writers with a shim to prevent early closure
+type noopCloser struct {
+	io.Writer
+}
+
+func (noopCloser) Close() error { return nil }
+
+// WriterPool is a gzip writer pool to reduce workload on creation of
+// gzip writers
+type WriterPool struct {
+	pool             sync.Pool
+	compressionLevel int
+}
+
+// NewWriterPool creates a new pool
+func NewWriterPool(compressionLevel int) *WriterPool {
+	return &WriterPool{pool: sync.Pool{
+		// New will return nil, we'll manage the creation of new
+		// writers in the middleware
+		New: func() interface{} { return nil },
+	},
+		compressionLevel: compressionLevel}
+}
+
+// Get a writer from the pool - or create one if not available
+func (wp *WriterPool) Get(rw macaron.ResponseWriter) *gzip.Writer {
+	ret := wp.pool.Get()
+	if ret == nil {
+		ret, _ = gzip.NewWriterLevel(rw, wp.compressionLevel)
+	} else {
+		ret.(*gzip.Writer).Reset(rw)
+	}
+	return ret.(*gzip.Writer)
+}
+
+// Put returns a writer to the pool
+func (wp *WriterPool) Put(w *gzip.Writer) {
+	wp.pool.Put(w)
+}
+
+var writerPool WriterPool
+
+// Options represents the configuration for the gzip middleware
+type Options struct {
+	CompressionLevel int
+}
+
+func validateCompressionLevel(level int) bool {
+	return level == gzip.DefaultCompression ||
+		level == gzip.ConstantCompression ||
+		(level >= gzip.BestSpeed && level <= gzip.BestCompression)
+}
+
+func validate(options []Options) Options {
+	// Default to level 4 compression (Best results seem to be between 4 and 6)
+	opt := Options{CompressionLevel: 4}
+	if len(options) > 0 {
+		opt = options[0]
+	}
+	if !validateCompressionLevel(opt.CompressionLevel) {
+		opt.CompressionLevel = 4
+	}
+	return opt
+}
+
+// Middleware creates a macaron.Handler to proxy the response
+func Middleware(options ...Options) macaron.Handler {
+	opt := validate(options)
+	writerPool = *NewWriterPool(opt.CompressionLevel)
+	regex := regexp.MustCompile(`bytes=(\d+)\-.*`)
+
+	return func(ctx *macaron.Context) {
+		// If the client won't accept gzip or x-gzip don't compress
+		if !strings.Contains(ctx.Req.Header.Get(acceptEncodingHeader), "gzip") &&
+			!strings.Contains(ctx.Req.Header.Get(acceptEncodingHeader), "x-gzip") {
+			return
+		}
+
+		// If the client is asking for a specific range of bytes - don't compress
+		if rangeHdr := ctx.Req.Header.Get(rangeHeader); rangeHdr != "" {
+
+			match := regex.FindStringSubmatch(rangeHdr)
+			if len(match) > 1 {
+				return
+			}
+		}
+
+		// OK we should proxy the response writer
+		// We are still not necessarily going to compress...
+		proxyWriter := &ProxyResponseWriter{
+			internal: ctx.Resp,
+		}
+		defer proxyWriter.Close()
+
+		ctx.Resp = proxyWriter
+		ctx.MapTo(proxyWriter, (*http.ResponseWriter)(nil))
+
+		// Check if render middleware has been registered,
+		// if yes, we need to modify ResponseWriter for it as well.
+		if _, ok := ctx.Render.(*macaron.DummyRender); !ok {
+			ctx.Render.SetResponseWriter(proxyWriter)
+		}
+
+		ctx.Next()
+		ctx.Resp = proxyWriter.internal
+	}
+}
+
+// ProxyResponseWriter is a wrapped macaron ResponseWriter that may compress its contents
+type ProxyResponseWriter struct {
+	writer   io.WriteCloser
+	internal macaron.ResponseWriter
+	stopped  bool
+
+	code int
+	buf  []byte
+}
+
+// Header returns the header map
+func (proxy *ProxyResponseWriter) Header() http.Header {
+	return proxy.internal.Header()
+}
+
+// Status returns the status code of the response or 0 if the response has not been written.
+func (proxy *ProxyResponseWriter) Status() int {
+	if proxy.code != 0 {
+		return proxy.code
+	}
+	return proxy.internal.Status()
+}
+
+// Written returns whether or not the ResponseWriter has been written.
+func (proxy *ProxyResponseWriter) Written() bool {
+	if proxy.code != 0 {
+		return true
+	}
+	return proxy.internal.Written()
+}
+
+// Size returns the size of the response body.
+func (proxy *ProxyResponseWriter) Size() int {
+	return proxy.internal.Size()
+}
+
+// Before allows for a function to be called before the ResponseWriter has been written to. This is
+// useful for setting headers or any other operations that must happen before a response has been written.
+func (proxy *ProxyResponseWriter) Before(before macaron.BeforeFunc) {
+	proxy.internal.Before(before)
+}
+
+// Write appends data to the proxied gzip writer.
+func (proxy *ProxyResponseWriter) Write(b []byte) (int, error) {
+	// if writer is initialized, use the writer
+	if proxy.writer != nil {
+		return proxy.writer.Write(b)
+	}
+
+	proxy.buf = append(proxy.buf, b...)
+
+	var (
+		contentLength, _ = strconv.Atoi(proxy.Header().Get(contentLengthHeader))
+		contentType      = proxy.Header().Get(contentTypeHeader)
+		contentEncoding  = proxy.Header().Get(contentEncodingHeader)
+	)
+
+	// OK if an encoding hasn't been chosen, and content length > 1400
+	// and content type isn't a compressed type
+	if contentEncoding == "" &&
+		(contentLength == 0 || contentLength >= MinSize) &&
+		(contentType == "" || !compressedContentType(contentType)) {
+		// If current buffer is less than the min size and a Content-Length isn't set, then wait
+		if len(proxy.buf) < MinSize && contentLength == 0 {
+			return len(b), nil
+		}
+
+		// If the Content-Length is larger than minSize or the current buffer is larger than minSize, then continue.
+		if contentLength >= MinSize || len(proxy.buf) >= MinSize {
+			// if we don't know the content type, infer it
+			if contentType == "" {
+				contentType = http.DetectContentType(proxy.buf)
+				proxy.Header().Set(contentTypeHeader, contentType)
+			}
+			// If the Content-Type is not compressed - Compress!
+			if !compressedContentType(contentType) {
+				if err := proxy.startGzip(); err != nil {
+					return 0, err
+				}
+				return len(b), nil
+			}
+		}
+	}
+	// If we got here, we should not GZIP this response.
+	if err := proxy.startPlain(); err != nil {
+		return 0, err
+	}
+	return len(b), nil
+}
+
+func (proxy *ProxyResponseWriter) startGzip() error {
+	// Set the content-encoding and vary headers.
+	proxy.Header().Set(contentEncodingHeader, "gzip")
+	proxy.Header().Set(varyHeader, acceptEncodingHeader)
+
+	// if the Content-Length is already set, then calls to Write on gzip
+	// will fail to set the Content-Length header since its already set
+	// See: https://github.com/golang/go/issues/14975.
+	proxy.Header().Del(contentLengthHeader)
+
+	// Write the header to gzip response.
+	if proxy.code != 0 {
+		proxy.internal.WriteHeader(proxy.code)
+		// Ensure that no other WriteHeader's happen
+		proxy.code = 0
+	}
+
+	// Initialize and flush the buffer into the gzip response if there are any bytes.
+	// If there aren't any, we shouldn't initialize it yet because on Close it will
+	// write the gzip header even if nothing was ever written.
+	if len(proxy.buf) > 0 {
+		// Initialize the GZIP response.
+		proxy.writer = writerPool.Get(proxy.internal)
+
+		return proxy.writeBuf()
+	}
+	return nil
+}
+
+func (proxy *ProxyResponseWriter) startPlain() error {
+	if proxy.code != 0 {
+		proxy.internal.WriteHeader(proxy.code)
+		proxy.code = 0
+	}
+	proxy.stopped = true
+	proxy.writer = noopCloser{proxy.internal}
+	return proxy.writeBuf()
+}
+
+func (proxy *ProxyResponseWriter) writeBuf() error {
+	if proxy.buf == nil {
+		return nil
+	}
+
+	n, err := proxy.writer.Write(proxy.buf)
+
+	// This should never happen (per io.Writer docs), but if the write didn't
+	// accept the entire buffer but returned no specific error, we have no clue
+	// what's going on, so abort just to be safe.
+	if err == nil && n < len(proxy.buf) {
+		err = io.ErrShortWrite
+	}
+	proxy.buf = nil
+	return err
+}
+
+// WriteHeader will ensure that we have setup the writer before we write the header
+func (proxy *ProxyResponseWriter) WriteHeader(code int) {
+	if proxy.code == 0 {
+		proxy.code = code
+	}
+}
+
+// Close the writer
+func (proxy *ProxyResponseWriter) Close() error {
+	if proxy.stopped {
+		return nil
+	}
+
+	if proxy.writer == nil {
+		err := proxy.startPlain()
+		if err != nil {
+			return fmt.Errorf("GzipMiddleware: write to regular responseWriter at close gets error: %q", err.Error())
+		}
+	}
+
+	err := proxy.writer.Close()
+
+	if poolWriter, ok := proxy.writer.(*gzip.Writer); ok {
+		writerPool.Put(poolWriter)
+	}
+
+	proxy.writer = nil
+	proxy.stopped = true
+	return err
+}
+
+// Flush the writer
+func (proxy *ProxyResponseWriter) Flush() {
+	if proxy.writer == nil {
+		return
+	}
+
+	if gw, ok := proxy.writer.(*gzip.Writer); ok {
+		gw.Flush()
+	}
+
+	proxy.internal.Flush()
+}
+
+// Hijack implements http.Hijacker. If the underlying ResponseWriter is a
+// Hijacker, its Hijack method is returned. Otherwise an error is returned.
+func (proxy *ProxyResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
+	hijacker, ok := proxy.internal.(http.Hijacker)
+	if !ok {
+		return nil, nil, fmt.Errorf("the ResponseWriter doesn't support the Hijacker interface")
+	}
+	return hijacker.Hijack()
+}
+
+// verify Hijacker interface implementation
+var _ http.Hijacker = &ProxyResponseWriter{}
+
+func compressedContentType(contentType string) bool {
+	switch contentType {
+	case "application/zip":
+		return true
+	case "application/x-gzip":
+		return true
+	case "application/gzip":
+		return true
+	default:
+		return false
+	}
+}

vendor/gitea.com/macaron/macaron/go.mod

@@ -3,9 +3,10 @@ module gitea.com/macaron/macaron
 go 1.11
 
 require (
-	gitea.com/macaron/inject v0.0.0-20190803172902-8375ba841591
+	gitea.com/macaron/inject v0.0.0-20190805023432-d4c86e31027a
+	github.com/smartystreets/assertions v1.0.1 // indirect
 	github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337
-	github.com/unknwon/com v0.0.0-20190804042917-757f69c95f3e
+	github.com/unknwon/com v1.0.1
 	golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4
 	gopkg.in/ini.v1 v1.44.0
 )

vendor/gitea.com/macaron/macaron/go.sum

@@ -1,19 +1,23 @@
-gitea.com/macaron/inject v0.0.0-20190803172902-8375ba841591 h1:UbCTjPcLrNxR9LzKDjQBMT2zoxZuEnca1pZCpgeMuhQ=
-gitea.com/macaron/inject v0.0.0-20190803172902-8375ba841591/go.mod h1:h6E4kLao1Yko6DOU6QDnQPcuoNzvbZqzj2mtPcEn1aM=
+gitea.com/macaron/inject v0.0.0-20190805023432-d4c86e31027a h1:aOKEXkDTnh4euoH0so/THLXeHtQuqHmDPb1xEk6Ehok=
+gitea.com/macaron/inject v0.0.0-20190805023432-d4c86e31027a/go.mod h1:h6E4kLao1Yko6DOU6QDnQPcuoNzvbZqzj2mtPcEn1aM=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg=
 github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE=
 github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304 h1:Jpy1PXuP99tXNrhbq2BaPz9B+jNAvH1JPQQpG/9GCXY=
 github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w=
+github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
+github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c h1:Ho+uVpkel/udgjbwB5Lktg9BtvJSh2DT0Hi6LPSyI2w=
 github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
 github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8=
 github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
-github.com/unknwon/com v0.0.0-20190804042917-757f69c95f3e h1:GSGeB9EAKY2spCABz6xOX5DbxZEXolK+nBSvmsQwRjM=
-github.com/unknwon/com v0.0.0-20190804042917-757f69c95f3e/go.mod h1:tOOxU81rwgoCLoOVVPHb6T/wt8HZygqH5id+GNnlCXM=
+github.com/unknwon/com v1.0.1 h1:3d1LTxD+Lnf3soQiD4Cp/0BRB+Rsa/+RTvz8GMMzIXs=
+github.com/unknwon/com v1.0.1/go.mod h1:tOOxU81rwgoCLoOVVPHb6T/wt8HZygqH5id+GNnlCXM=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc=
 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=

vendor/gitea.com/macaron/macaron/macaron.go

@@ -1,5 +1,3 @@
-// +build go1.3
-
 // Copyright 2014 The Macaron Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License"): you may
@@ -283,6 +281,7 @@ var (
 	envLock sync.Mutex
 
 	// Path of work directory.
+	// You must set this value yourself
 	Root string
 
 	// Flash applies to current request.
@@ -310,12 +309,6 @@ func safeEnv() string {
 
 func init() {
 	setENV(os.Getenv("MACARON_ENV"))
-
-	var err error
-	Root, err = os.Getwd()
-	if err != nil {
-		panic("error getting work directory: " + err.Error())
-	}
 }
 
 // SetConfig sets data sources for configuration.

vendor/gitea.com/macaron/session/go.mod

@@ -6,7 +6,7 @@ require (
 	gitea.com/macaron/macaron v1.3.3-0.20190821202302-9646c0587edb
 	github.com/bradfitz/gomemcache v0.0.0-20190329173943-551aad21a668
 	github.com/couchbase/gomemcached v0.0.0-20190515232915-c4b4ca0eb21d // indirect
-	github.com/couchbase/goutils v0.0.0-20190315194238-f9d42b11473b // indirect
+	github.com/couchbase/goutils v0.0.0-20191018232750-b49639060d85 // indirect
 	github.com/couchbaselabs/go-couchbase v0.0.0-20190708161019-23e7ca2ce2b7
 	github.com/cupcake/rdb v0.0.0-20161107195141-43ba34106c76 // indirect
 	github.com/edsrzf/mmap-go v1.0.0 // indirect

vendor/gitea.com/macaron/session/go.sum

@@ -8,8 +8,8 @@ github.com/bradfitz/gomemcache v0.0.0-20190329173943-551aad21a668 h1:U/lr3Dgy4WK
 github.com/bradfitz/gomemcache v0.0.0-20190329173943-551aad21a668/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA=
 github.com/couchbase/gomemcached v0.0.0-20190515232915-c4b4ca0eb21d h1:XMf4E1U+b9E3ElF0mjvfXZdflBRZz4gLp16nQ/QSHQM=
 github.com/couchbase/gomemcached v0.0.0-20190515232915-c4b4ca0eb21d/go.mod h1:srVSlQLB8iXBVXHgnqemxUXqN6FCvClgCMPCsjBDR7c=
-github.com/couchbase/goutils v0.0.0-20190315194238-f9d42b11473b h1:bZ9rKU2/V8sY+NulSfxDOnXTWcs1rySqdF1sVepihvo=
-github.com/couchbase/goutils v0.0.0-20190315194238-f9d42b11473b/go.mod h1:BQwMFlJzDjFDG3DJUdU0KORxn88UlsOULuxLExMh3Hs=
+github.com/couchbase/goutils v0.0.0-20191018232750-b49639060d85 h1:0WMIDtuXCKEm4wtAJgAAXa/qtM5O9MariLwgHaRlYmk=
+github.com/couchbase/goutils v0.0.0-20191018232750-b49639060d85/go.mod h1:BQwMFlJzDjFDG3DJUdU0KORxn88UlsOULuxLExMh3Hs=
 github.com/couchbaselabs/go-couchbase v0.0.0-20190708161019-23e7ca2ce2b7 h1:1XjEY/gnjQ+AfXef2U6dxCquhiRzkEpxZuWqs+QxTL8=
 github.com/couchbaselabs/go-couchbase v0.0.0-20190708161019-23e7ca2ce2b7/go.mod h1:mby/05p8HE5yHEAKiIH/555NoblMs7PtW6NrYshDruc=
 github.com/cupcake/rdb v0.0.0-20161107195141-43ba34106c76 h1:Lgdd/Qp96Qj8jqLpq2cI1I1X7BJnu06efS+XkhRoLUQ=

vendor/github.com/RoaringBitmap/roaring/.drone.yml

@@ -0,0 +1,20 @@
+kind: pipeline
+name: default
+
+workspace:
+  base: /go
+  path: src/github.com/RoaringBitmap/roaring
+
+steps:
+- name: test
+  image: golang
+  commands:
+  - go get -t 
+  - go test 
+  - go test -race -run TestConcurrent*
+  - go build -tags appengine 
+  - go test -tags appengine 
+  - GOARCH=386 go build
+  - GOARCH=386 go test
+  - GOARCH=arm go build
+  - GOARCH=arm64 go build

vendor/github.com/RoaringBitmap/roaring/.travis.yml

@@ -8,10 +8,13 @@ install:
 notifications:
   email: false
 go:
-- 1.7.x
-- 1.8.x
-- 1.9.x
-- 1.10.x
+- "1.7.x"
+- "1.8.x"
+- "1.9.x"
+- "1.10.x"
+- "1.11.x"
+- "1.12.x"
+- "1.13.x"
 - tip
 
 # whitelist
@@ -21,10 +24,14 @@ branches:
 script:
 - goveralls -v -service travis-ci -ignore arraycontainer_gen.go,bitmapcontainer_gen.go,rle16_gen.go,rle_gen.go,roaringarray_gen.go,rle.go || go test
 - go test -race -run TestConcurrent*
+- go build -tags appengine
+- go test -tags appengine
 - GOARCH=arm64 go build
 - GOARCH=386 go build
 - GOARCH=386 go test
 - GOARCH=arm go build
+- GOARCH=arm64 go build
+
 matrix:
   allow_failures:
   - go: tip

vendor/github.com/RoaringBitmap/roaring/AUTHORS

@@ -7,4 +7,5 @@ Bob Potter (@bpot),
 Tyson Maly (@tvmaly),
 Will Glynn (@willglynn),
 Brent Pedersen (@brentp)
-Maciej Biłas (@maciej)
+Maciej Biłas (@maciej),
+Joe Nall (@joenall)

vendor/github.com/RoaringBitmap/roaring/CONTRIBUTORS

@@ -9,4 +9,8 @@ Will Glynn (@willglynn),
 Brent Pedersen (@brentp),
 Jason E. Aten (@glycerine),
 Vali Malinoiu (@0x4139),
-Forud Ghafouri (@fzerorubigd)
+Forud Ghafouri (@fzerorubigd),
+Joe Nall (@joenall),
+(@fredim),
+Edd Robinson (@e-dard),
+Alexander Petrov (@alldroll)

vendor/github.com/RoaringBitmap/roaring/LICENSE

@@ -200,3 +200,36 @@
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
+
+================================================================================
+
+Portions of runcontainer.go are from the Go standard library, which is licensed
+under:
+
+Copyright (c) 2009 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+     copyright notice, this list of conditions and the following disclaimer
+     in the documentation and/or other materials provided with the
+     distribution.
+   * Neither the name of Google Inc. nor the names of its
+     contributors may be used to endorse or promote products derived from
+     this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

vendor/github.com/RoaringBitmap/roaring/Makefile

@@ -1,4 +1,4 @@
-.PHONY: help all test format fmtcheck vet lint     qa deps clean nuke rle backrle ser fetch-real-roaring-datasets
+.PHONY: help all test format fmtcheck vet lint     qa deps clean nuke ser fetch-real-roaring-datasets
 
 
 
@@ -63,7 +63,7 @@ qa: fmtcheck test vet lint
 
 # Get the dependencies
 deps:
-	GOPATH=$(GOPATH) go get github.com/smartystreets/goconvey/convey
+	GOPATH=$(GOPATH) go get github.com/stretchr/testify
 	GOPATH=$(GOPATH) go get github.com/willf/bitset
 	GOPATH=$(GOPATH) go get github.com/golang/lint/golint
 	GOPATH=$(GOPATH) go get github.com/mschoch/smat
@@ -97,18 +97,8 @@ nuke:
 	rm -rf ./target
 	GOPATH=$(GOPATH) go clean -i ./...
 
-rle:
-	cp rle.go rle16.go
-	perl -pi -e 's/32/16/g' rle16.go
-	cp rle_test.go rle16_test.go
-	perl -pi -e 's/32/16/g' rle16_test.go
 
-backrle:
-	cp rle16.go rle.go
-	perl -pi -e 's/16/32/g' rle.go
-	perl -pi -e 's/2032/2016/g' rle.go
-
-ser: rle
+ser: 
 	go generate
 
 cover:

vendor/github.com/RoaringBitmap/roaring/README.md

@@ -1,4 +1,5 @@
 roaring [![Build Status](https://travis-ci.org/RoaringBitmap/roaring.png)](https://travis-ci.org/RoaringBitmap/roaring) [![Coverage Status](https://coveralls.io/repos/github/RoaringBitmap/roaring/badge.svg?branch=master)](https://coveralls.io/github/RoaringBitmap/roaring?branch=master) [![GoDoc](https://godoc.org/github.com/RoaringBitmap/roaring?status.svg)](https://godoc.org/github.com/RoaringBitmap/roaring) [![Go Report Card](https://goreportcard.com/badge/RoaringBitmap/roaring)](https://goreportcard.com/report/github.com/RoaringBitmap/roaring)
+[![Build Status](https://cloud.drone.io/api/badges/RoaringBitmap/roaring/status.svg)](https://cloud.drone.io/RoaringBitmap/roaring)
 =============
 
 This is a go version of the Roaring bitmap data structure. 
@@ -6,12 +7,12 @@ This is a go version of the Roaring bitmap data structure.
 
 
 Roaring bitmaps are used by several major systems such as [Apache Lucene][lucene] and derivative systems such as [Solr][solr] and
-[Elasticsearch][elasticsearch], [Metamarkets' Druid][druid], [LinkedIn Pinot][pinot], [Netflix Atlas][atlas],  [Apache Spark][spark], [OpenSearchServer][opensearchserver], [Cloud Torrent][cloudtorrent], [Whoosh][whoosh],  [Pilosa][pilosa],  [Microsoft Visual Studio Team Services (VSTS)][vsts], and eBay's [Apache Kylin][kylin].
+[Elasticsearch][elasticsearch], [Apache Druid (Incubating)][druid], [LinkedIn Pinot][pinot], [Netflix Atlas][atlas],  [Apache Spark][spark], [OpenSearchServer][opensearchserver], [Cloud Torrent][cloudtorrent], [Whoosh][whoosh],  [Pilosa][pilosa],  [Microsoft Visual Studio Team Services (VSTS)][vsts], and eBay's [Apache Kylin][kylin].
 
 [lucene]: https://lucene.apache.org/
 [solr]: https://lucene.apache.org/solr/
 [elasticsearch]: https://www.elastic.co/products/elasticsearch
-[druid]: http://druid.io/
+[druid]: https://druid.apache.org/
 [spark]: https://spark.apache.org/
 [opensearchserver]: http://www.opensearchserver.com
 [cloudtorrent]: https://github.com/jpillora/cloud-torrent
@@ -28,11 +29,17 @@ Roaring bitmaps are found to work well in many important applications:
 
 
 The ``roaring`` Go library is used by
-* [Cloud Torrent](https://github.com/jpillora/cloud-torrent): a self-hosted remote torrent client
-* [runv](https://github.com/hyperhq/runv): an Hypervisor-based runtime for the Open Containers Initiative
+* [Cloud Torrent](https://github.com/jpillora/cloud-torrent)
+* [runv](https://github.com/hyperhq/runv)
 * [InfluxDB](https://www.influxdata.com)
 * [Pilosa](https://www.pilosa.com/)
 * [Bleve](http://www.blevesearch.com)
+* [lindb](https://github.com/lindb/lindb)
+* [Elasticell](https://github.com/deepfabric/elasticell)
+* [SourceGraph](https://github.com/sourcegraph/sourcegraph)
+* [M3](https://github.com/m3db/m3)
+* [trident](https://github.com/NetApp/trident)
+
 
 This library is used in production in several systems, it is part of the [Awesome Go collection](https://awesome-go.com).
 
@@ -61,7 +68,6 @@ http://arxiv.org/abs/1402.6407 This paper used data from http://lemire.me/data/r
 Dependencies are fetched automatically by giving the `-t` flag to `go get`.
 
 they include
-  - github.com/smartystreets/goconvey/convey
   - github.com/willf/bitset
   - github.com/mschoch/smat
   - github.com/glycerine/go-unsnap-stream
@@ -133,6 +139,7 @@ func main() {
     if rb1.Equals(newrb) {
     	fmt.Println("I wrote the content to a byte stream and read it back.")
     }
+    // you can iterate over bitmaps using ReverseIterator(), Iterator, ManyIterator()
 }
 ```
 
@@ -206,7 +213,7 @@ You can use roaring with gore:
 
 - go get -u github.com/motemen/gore
 - Make sure that ``$GOPATH/bin`` is in your ``$PATH``.
-- go get github/RoaringBitmap/roaring
+- go get github.com/RoaringBitmap/roaring
 
 ```go
 $ gore

vendor/github.com/RoaringBitmap/roaring/arraycontainer.go

@@ -24,12 +24,28 @@ func (ac *arrayContainer) fillLeastSignificant16bits(x []uint32, i int, mask uin
 	}
 }
 
-func (ac *arrayContainer) getShortIterator() shortIterable {
+func (ac *arrayContainer) iterate(cb func(x uint16) bool) bool {
+	iterator := shortIterator{ac.content, 0}
+
+	for iterator.hasNext() {
+		if !cb(iterator.next()) {
+			return false
+		}
+	}
+
+	return true
+}
+
+func (ac *arrayContainer) getShortIterator() shortPeekable {
 	return &shortIterator{ac.content, 0}
 }
 
+func (ac *arrayContainer) getReverseIterator() shortIterable {
+	return &reverseIterator{ac.content, len(ac.content) - 1}
+}
+
 func (ac *arrayContainer) getManyIterator() manyIterable {
-	return &manyIterator{ac.content, 0}
+	return &shortIterator{ac.content, 0}
 }
 
 func (ac *arrayContainer) minimum() uint16 {
@@ -115,7 +131,6 @@ func (ac *arrayContainer) iremoveRange(firstOfRange, endx int) container {
 // flip the values in the range [firstOfRange,endx)
 func (ac *arrayContainer) not(firstOfRange, endx int) container {
 	if firstOfRange >= endx {
-		//p("arrayContainer.not(): exiting early with ac.clone()")
 		return ac.clone()
 	}
 	return ac.notClose(firstOfRange, endx-1) // remove everything in [firstOfRange,endx-1]
@@ -124,18 +139,15 @@ func (ac *arrayContainer) not(firstOfRange, endx int) container {
 // flip the values in the range [firstOfRange,lastOfRange]
 func (ac *arrayContainer) notClose(firstOfRange, lastOfRange int) container {
 	if firstOfRange > lastOfRange { // unlike add and remove, not uses an inclusive range [firstOfRange,lastOfRange]
-		//p("arrayContainer.notClose(): exiting early with ac.clone()")
 		return ac.clone()
 	}
 
 	// determine the span of array indices to be affected^M
 	startIndex := binarySearch(ac.content, uint16(firstOfRange))
-	//p("startIndex=%v", startIndex)
 	if startIndex < 0 {
 		startIndex = -startIndex - 1
 	}
 	lastIndex := binarySearch(ac.content, uint16(lastOfRange))
-	//p("lastIndex=%v", lastIndex)
 	if lastIndex < 0 {
 		lastIndex = -lastIndex - 2
 	}
@@ -144,9 +156,7 @@ func (ac *arrayContainer) notClose(firstOfRange, lastOfRange int) container {
 	newValuesInRange := spanToBeFlipped - currentValuesInRange
 	cardinalityChange := newValuesInRange - currentValuesInRange
 	newCardinality := len(ac.content) + cardinalityChange
-	//p("new card is %v", newCardinality)
 	if newCardinality > arrayDefaultMaxSize {
-		//p("new card over arrayDefaultMaxSize, so returning bitmap")
 		return ac.toBitmapContainer().not(firstOfRange, lastOfRange+1)
 	}
 	answer := newArrayContainer()
@@ -503,7 +513,6 @@ func (ac *arrayContainer) lazyorArray(value2 *arrayContainer) container {
 }
 
 func (ac *arrayContainer) and(a container) container {
-	//p("ac.and() called")
 	switch x := a.(type) {
 	case *arrayContainer:
 		return ac.andArray(x)
@@ -550,7 +559,7 @@ func (ac *arrayContainer) iand(a container) container {
 		return ac.iandBitmap(x)
 	case *runContainer16:
 		if x.isFull() {
-			return ac.clone()
+			return ac
 		}
 		return x.andArray(ac)
 	}
@@ -722,7 +731,6 @@ func (ac *arrayContainer) inot(firstOfRange, endx int) container {
 
 // flip the values in the range [firstOfRange,lastOfRange]
 func (ac *arrayContainer) inotClose(firstOfRange, lastOfRange int) container {
-	//p("ac.inotClose() starting")
 	if firstOfRange > lastOfRange { // unlike add and remove, not uses an inclusive range [firstOfRange,lastOfRange]
 		return ac
 	}
@@ -745,7 +753,6 @@ func (ac *arrayContainer) inotClose(firstOfRange, lastOfRange int) container {
 	if cardinalityChange > 0 {
 		if newCardinality > len(ac.content) {
 			if newCardinality > arrayDefaultMaxSize {
-				//p("ac.inotClose() converting to bitmap and doing inot there")
 				bcRet := ac.toBitmapContainer()
 				bcRet.inot(firstOfRange, lastOfRange+1)
 				*ac = *bcRet.toArrayContainer()
@@ -766,7 +773,6 @@ func (ac *arrayContainer) inotClose(firstOfRange, lastOfRange int) container {
 		}
 	}
 	ac.content = ac.content[:newCardinality]
-	//p("bottom of ac.inotClose(): returning ac")
 	return ac
 }
 
@@ -958,3 +964,17 @@ func (ac *arrayContainer) toEfficientContainer() container {
 func (ac *arrayContainer) containerType() contype {
 	return arrayContype
 }
+
+func (ac *arrayContainer) addOffset(x uint16) []container {
+	low := &arrayContainer{}
+	high := &arrayContainer{}
+	for _, val := range ac.content {
+		y := uint32(val) + uint32(x)
+		if highbits(y) > 0 {
+			high.content = append(high.content, lowbits(y))
+		} else {
+			low.content = append(low.content, lowbits(y))
+		}
+	}
+	return []container{low, high}
+}

vendor/github.com/RoaringBitmap/roaring/arraycontainer_gen.go

@@ -6,7 +6,7 @@ package roaring
 
 import "github.com/tinylib/msgp/msgp"
 
-// DecodeMsg implements msgp.Decodable
+// Deprecated: DecodeMsg implements msgp.Decodable
 func (z *arrayContainer) DecodeMsg(dc *msgp.Reader) (err error) {
 	var field []byte
 	_ = field
@@ -49,7 +49,7 @@ func (z *arrayContainer) DecodeMsg(dc *msgp.Reader) (err error) {
 	return
 }
 
-// EncodeMsg implements msgp.Encodable
+// Deprecated: EncodeMsg implements msgp.Encodable
 func (z *arrayContainer) EncodeMsg(en *msgp.Writer) (err error) {
 	// map header, size 1
 	// write "content"
@@ -70,7 +70,7 @@ func (z *arrayContainer) EncodeMsg(en *msgp.Writer) (err error) {
 	return
 }
 
-// MarshalMsg implements msgp.Marshaler
+// Deprecated: MarshalMsg implements msgp.Marshaler
 func (z *arrayContainer) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
 	// map header, size 1
@@ -83,7 +83,7 @@ func (z *arrayContainer) MarshalMsg(b []byte) (o []byte, err error) {
 	return
 }
 
-// UnmarshalMsg implements msgp.Unmarshaler
+// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
 func (z *arrayContainer) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	var field []byte
 	_ = field
@@ -127,7 +127,7 @@ func (z *arrayContainer) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	return
 }
 
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *arrayContainer) Msgsize() (s int) {
 	s = 1 + 8 + msgp.ArrayHeaderSize + (len(z.content) * (msgp.Uint16Size))
 	return

vendor/github.com/RoaringBitmap/roaring/bitmapcontainer.go

@@ -96,6 +96,18 @@ func (bc *bitmapContainer) maximum() uint16 {
 	return uint16(0)
 }
 
+func (bc *bitmapContainer) iterate(cb func(x uint16) bool) bool {
+	iterator := bitmapContainerShortIterator{bc, bc.NextSetBit(0)}
+
+	for iterator.hasNext() {
+		if !cb(iterator.next()) {
+			return false
+		}
+	}
+
+	return true
+}
+
 type bitmapContainerShortIterator struct {
 	ptr *bitmapContainer
 	i   int
@@ -110,14 +122,54 @@ func (bcsi *bitmapContainerShortIterator) hasNext() bool {
 	return bcsi.i >= 0
 }
 
+func (bcsi *bitmapContainerShortIterator) peekNext() uint16 {
+	return uint16(bcsi.i)
+}
+
+func (bcsi *bitmapContainerShortIterator) advanceIfNeeded(minval uint16) {
+	if bcsi.hasNext() && bcsi.peekNext() < minval {
+		bcsi.i = bcsi.ptr.NextSetBit(int(minval))
+	}
+}
+
 func newBitmapContainerShortIterator(a *bitmapContainer) *bitmapContainerShortIterator {
 	return &bitmapContainerShortIterator{a, a.NextSetBit(0)}
 }
 
-func (bc *bitmapContainer) getShortIterator() shortIterable {
+func (bc *bitmapContainer) getShortIterator() shortPeekable {
 	return newBitmapContainerShortIterator(bc)
 }
 
+type reverseBitmapContainerShortIterator struct {
+	ptr *bitmapContainer
+	i   int
+}
+
+func (bcsi *reverseBitmapContainerShortIterator) next() uint16 {
+	if bcsi.i == -1 {
+		panic("reverseBitmapContainerShortIterator.next() going beyond what is available")
+	}
+
+	j := bcsi.i
+	bcsi.i = bcsi.ptr.PrevSetBit(bcsi.i - 1)
+	return uint16(j)
+}
+
+func (bcsi *reverseBitmapContainerShortIterator) hasNext() bool {
+	return bcsi.i >= 0
+}
+
+func newReverseBitmapContainerShortIterator(a *bitmapContainer) *reverseBitmapContainerShortIterator {
+	if a.cardinality == 0 {
+		return &reverseBitmapContainerShortIterator{a, -1}
+	}
+	return &reverseBitmapContainerShortIterator{a, int(a.maximum())}
+}
+
+func (bc *bitmapContainer) getReverseIterator() shortIterable {
+	return newReverseBitmapContainerShortIterator(bc)
+}
+
 type bitmapContainerManyIterator struct {
 	ptr    *bitmapContainer
 	base   int
@@ -131,7 +183,7 @@ func (bcmi *bitmapContainerManyIterator) nextMany(hs uint32, buf []uint32) int {
 
 	for n < len(buf) {
 		if bitset == 0 {
-			base += 1
+			base++
 			if base >= len(bcmi.ptr.bitmap) {
 				bcmi.base = base
 				bcmi.bitset = bitset
@@ -177,16 +229,13 @@ func bitmapContainerSizeInBytes() int {
 
 func bitmapEquals(a, b []uint64) bool {
 	if len(a) != len(b) {
-		//p("bitmaps differ on length. len(a)=%v; len(b)=%v", len(a), len(b))
 		return false
 	}
 	for i, v := range a {
 		if v != b[i] {
-			//p("bitmaps differ on element i=%v", i)
 			return false
 		}
 	}
-	//p("bitmapEquals returning true")
 	return true
 }
 
@@ -209,9 +258,7 @@ func (bc *bitmapContainer) fillLeastSignificant16bits(x []uint32, i int, mask ui
 func (bc *bitmapContainer) equals(o container) bool {
 	srb, ok := o.(*bitmapContainer)
 	if ok {
-		//p("bitmapContainers.equals: both are bitmapContainers")
 		if srb.cardinality != bc.cardinality {
-			//p("bitmapContainers.equals: card differs: %v vs %v", srb.cardinality, bc.cardinality)
 			return false
 		}
 		return bitmapEquals(bc.bitmap, srb.bitmap)
@@ -261,12 +308,6 @@ func (bc *bitmapContainer) iremoveReturnMinimized(i uint16) container {
 
 // iremove returns true if i was found.
 func (bc *bitmapContainer) iremove(i uint16) bool {
-	/* branchless code
-	  w := bc.bitmap[i>>6]
-		mask := uint64(1) << (i % 64)
-		neww := w &^ mask
-		bc.cardinality -= int((w ^ neww) >> (i % 64))
-		bc.bitmap[i>>6] = neww */
 	if bc.contains(i) {
 		bc.cardinality--
 		bc.bitmap[i/64] &^= (uint64(1) << (i % 64))
@@ -306,14 +347,10 @@ func (bc *bitmapContainer) iremoveRange(firstOfRange, lastOfRange int) container
 
 // flip all values in range [firstOfRange,endx)
 func (bc *bitmapContainer) inot(firstOfRange, endx int) container {
-	p("bc.inot() called with [%v, %v)", firstOfRange, endx)
 	if endx-firstOfRange == maxCapacity {
-		//p("endx-firstOfRange == maxCapacity")
 		flipBitmapRange(bc.bitmap, firstOfRange, endx)
 		bc.cardinality = maxCapacity - bc.cardinality
-		//p("bc.cardinality is now %v", bc.cardinality)
 	} else if endx-firstOfRange > maxCapacity/2 {
-		//p("endx-firstOfRange > maxCapacity/2")
 		flipBitmapRange(bc.bitmap, firstOfRange, endx)
 		bc.computeCardinality()
 	} else {
@@ -517,11 +554,31 @@ func (bc *bitmapContainer) iorBitmap(value2 *bitmapContainer) container {
 func (bc *bitmapContainer) lazyIORArray(value2 *arrayContainer) container {
 	answer := bc
 	c := value2.getCardinality()
-	for k := 0; k < c; k++ {
+	for k := 0; k+3 < c; k += 4 {
+		content := (*[4]uint16)(unsafe.Pointer(&value2.content[k]))
+		vc0 := content[0]
+		i0 := uint(vc0) >> 6
+		answer.bitmap[i0] = answer.bitmap[i0] | (uint64(1) << (vc0 % 64))
+
+		vc1 := content[1]
+		i1 := uint(vc1) >> 6
+		answer.bitmap[i1] = answer.bitmap[i1] | (uint64(1) << (vc1 % 64))
+
+		vc2 := content[2]
+		i2 := uint(vc2) >> 6
+		answer.bitmap[i2] = answer.bitmap[i2] | (uint64(1) << (vc2 % 64))
+
+		vc3 := content[3]
+		i3 := uint(vc3) >> 6
+		answer.bitmap[i3] = answer.bitmap[i3] | (uint64(1) << (vc3 % 64))
+	}
+
+	for k := c &^ 3; k < c; k++ {
 		vc := value2.content[k]
 		i := uint(vc) >> 6
 		answer.bitmap[i] = answer.bitmap[i] | (uint64(1) << (vc % 64))
 	}
+
 	answer.cardinality = invalidCardinality
 	return answer
 }
@@ -789,8 +846,6 @@ func (bc *bitmapContainer) andNotRun16(rc *runContainer16) container {
 }
 
 func (bc *bitmapContainer) iandNot(a container) container {
-	//p("bitmapContainer.iandNot() starting")
-
 	switch x := a.(type) {
 	case *arrayContainer:
 		return bc.iandNotArray(x)
@@ -844,12 +899,15 @@ func (bc *bitmapContainer) andNotBitmap(value2 *bitmapContainer) container {
 	return ac
 }
 
-func (bc *bitmapContainer) iandNotBitmapSurely(value2 *bitmapContainer) *bitmapContainer {
+func (bc *bitmapContainer) iandNotBitmapSurely(value2 *bitmapContainer) container {
 	newCardinality := int(popcntMaskSlice(bc.bitmap, value2.bitmap))
 	for k := 0; k < len(bc.bitmap); k++ {
 		bc.bitmap[k] = bc.bitmap[k] &^ value2.bitmap[k]
 	}
 	bc.cardinality = newCardinality
+	if bc.getCardinality() <= arrayDefaultMaxSize {
+		return bc.toArrayContainer()
+	}
 	return bc
 }
 
@@ -917,6 +975,32 @@ func (bc *bitmapContainer) NextSetBit(i int) int {
 	return -1
 }
 
+func (bc *bitmapContainer) PrevSetBit(i int) int {
+	if i < 0 {
+		return -1
+	}
+	x := i / 64
+	if x >= len(bc.bitmap) {
+		return -1
+	}
+
+	w := bc.bitmap[x]
+
+	b := i % 64
+
+	w = w << uint(63-b)
+	if w != 0 {
+		return i - countLeadingZeros(w)
+	}
+	x--
+	for ; x >= 0; x-- {
+		if bc.bitmap[x] != 0 {
+			return (x * 64) + 63 - countLeadingZeros(bc.bitmap[x])
+		}
+	}
+	return -1
+}
+
 // reference the java implementation
 // https://github.com/RoaringBitmap/RoaringBitmap/blob/master/src/main/java/org/roaringbitmap/BitmapContainer.java#L875-L892
 //
@@ -980,3 +1064,35 @@ func newBitmapContainerFromRun(rc *runContainer16) *bitmapContainer {
 func (bc *bitmapContainer) containerType() contype {
 	return bitmapContype
 }
+
+func (bc *bitmapContainer) addOffset(x uint16) []container {
+	low := newBitmapContainer()
+	high := newBitmapContainer()
+	b := uint32(x) >> 6
+	i := uint32(x) % 64
+	end := uint32(1024) - b
+	if i == 0 {
+		copy(low.bitmap[b:], bc.bitmap[:end])
+		copy(high.bitmap[:b], bc.bitmap[end:])
+	} else {
+		low.bitmap[b] = bc.bitmap[0] << i
+		for k := uint32(1); k < end; k++ {
+			newval := bc.bitmap[k] << i
+			if newval == 0 {
+				newval = bc.bitmap[k-1] >> (64 - i)
+			}
+			low.bitmap[b+k] = newval
+		}
+		for k := end; k < 1024; k++ {
+			newval := bc.bitmap[k] << i
+			if newval == 0 {
+				newval = bc.bitmap[k-1] >> (64 - i)
+			}
+			high.bitmap[k-end] = newval
+		}
+		high.bitmap[b] = bc.bitmap[1023] >> (64 - i)
+	}
+	low.computeCardinality()
+	high.computeCardinality()
+	return []container{low, high}
+}

vendor/github.com/RoaringBitmap/roaring/bitmapcontainer_gen.go

@@ -6,7 +6,7 @@ package roaring
 
 import "github.com/tinylib/msgp/msgp"
 
-// DecodeMsg implements msgp.Decodable
+// Deprecated: DecodeMsg implements msgp.Decodable
 func (z *bitmapContainer) DecodeMsg(dc *msgp.Reader) (err error) {
 	var field []byte
 	_ = field
@@ -54,7 +54,7 @@ func (z *bitmapContainer) DecodeMsg(dc *msgp.Reader) (err error) {
 	return
 }
 
-// EncodeMsg implements msgp.Encodable
+// Deprecated: EncodeMsg implements msgp.Encodable
 func (z *bitmapContainer) EncodeMsg(en *msgp.Writer) (err error) {
 	// map header, size 2
 	// write "cardinality"
@@ -84,7 +84,7 @@ func (z *bitmapContainer) EncodeMsg(en *msgp.Writer) (err error) {
 	return
 }
 
-// MarshalMsg implements msgp.Marshaler
+// Deprecated: MarshalMsg implements msgp.Marshaler
 func (z *bitmapContainer) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
 	// map header, size 2
@@ -100,7 +100,7 @@ func (z *bitmapContainer) MarshalMsg(b []byte) (o []byte, err error) {
 	return
 }
 
-// UnmarshalMsg implements msgp.Unmarshaler
+// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
 func (z *bitmapContainer) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	var field []byte
 	_ = field
@@ -149,13 +149,13 @@ func (z *bitmapContainer) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	return
 }
 
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *bitmapContainer) Msgsize() (s int) {
 	s = 1 + 12 + msgp.IntSize + 7 + msgp.ArrayHeaderSize + (len(z.bitmap) * (msgp.Uint64Size))
 	return
 }
 
-// DecodeMsg implements msgp.Decodable
+// Deprecated: DecodeMsg implements msgp.Decodable
 func (z *bitmapContainerShortIterator) DecodeMsg(dc *msgp.Reader) (err error) {
 	var field []byte
 	_ = field
@@ -239,7 +239,7 @@ func (z *bitmapContainerShortIterator) DecodeMsg(dc *msgp.Reader) (err error) {
 	return
 }
 
-// EncodeMsg implements msgp.Encodable
+// Deprecated: EncodeMsg implements msgp.Encodable
 func (z *bitmapContainerShortIterator) EncodeMsg(en *msgp.Writer) (err error) {
 	// map header, size 2
 	// write "ptr"
@@ -291,7 +291,7 @@ func (z *bitmapContainerShortIterator) EncodeMsg(en *msgp.Writer) (err error) {
 	return
 }
 
-// MarshalMsg implements msgp.Marshaler
+// Deprecated: MarshalMsg implements msgp.Marshaler
 func (z *bitmapContainerShortIterator) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
 	// map header, size 2
@@ -317,7 +317,7 @@ func (z *bitmapContainerShortIterator) MarshalMsg(b []byte) (o []byte, err error
 	return
 }
 
-// UnmarshalMsg implements msgp.Unmarshaler
+// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
 func (z *bitmapContainerShortIterator) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	var field []byte
 	_ = field
@@ -402,7 +402,7 @@ func (z *bitmapContainerShortIterator) UnmarshalMsg(bts []byte) (o []byte, err e
 	return
 }
 
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *bitmapContainerShortIterator) Msgsize() (s int) {
 	s = 1 + 4
 	if z.ptr == nil {

vendor/github.com/RoaringBitmap/roaring/byte_input.go

@@ -0,0 +1,161 @@
+package roaring
+
+import (
+	"encoding/binary"
+	"io"
+)
+
+type byteInput interface {
+	// next returns a slice containing the next n bytes from the buffer,
+	// advancing the buffer as if the bytes had been returned by Read.
+	next(n int) ([]byte, error)
+	// readUInt32 reads uint32 with LittleEndian order
+	readUInt32() (uint32, error)
+	// readUInt16 reads uint16 with LittleEndian order
+	readUInt16() (uint16, error)
+	// getReadBytes returns read bytes
+	getReadBytes() int64
+	// skipBytes skips exactly n bytes
+	skipBytes(n int) error
+}
+
+func newByteInputFromReader(reader io.Reader) byteInput {
+	return &byteInputAdapter{
+		r:         reader,
+		readBytes: 0,
+	}
+}
+
+func newByteInput(buf []byte) byteInput {
+	return &byteBuffer{
+		buf: buf,
+		off: 0,
+	}
+}
+
+type byteBuffer struct {
+	buf []byte
+	off int
+}
+
+// next returns a slice containing the next n bytes from the reader
+// If there are fewer bytes than the given n, io.ErrUnexpectedEOF will be returned
+func (b *byteBuffer) next(n int) ([]byte, error) {
+	m := len(b.buf) - b.off
+
+	if n > m {
+		return nil, io.ErrUnexpectedEOF
+	}
+
+	data := b.buf[b.off : b.off+n]
+	b.off += n
+
+	return data, nil
+}
+
+// readUInt32 reads uint32 with LittleEndian order
+func (b *byteBuffer) readUInt32() (uint32, error) {
+	if len(b.buf)-b.off < 4 {
+		return 0, io.ErrUnexpectedEOF
+	}
+
+	v := binary.LittleEndian.Uint32(b.buf[b.off:])
+	b.off += 4
+
+	return v, nil
+}
+
+// readUInt16 reads uint16 with LittleEndian order
+func (b *byteBuffer) readUInt16() (uint16, error) {
+	if len(b.buf)-b.off < 2 {
+		return 0, io.ErrUnexpectedEOF
+	}
+
+	v := binary.LittleEndian.Uint16(b.buf[b.off:])
+	b.off += 2
+
+	return v, nil
+}
+
+// getReadBytes returns read bytes
+func (b *byteBuffer) getReadBytes() int64 {
+	return int64(b.off)
+}
+
+// skipBytes skips exactly n bytes
+func (b *byteBuffer) skipBytes(n int) error {
+	m := len(b.buf) - b.off
+
+	if n > m {
+		return io.ErrUnexpectedEOF
+	}
+
+	b.off += n
+
+	return nil
+}
+
+// reset resets the given buffer with a new byte slice
+func (b *byteBuffer) reset(buf []byte) {
+	b.buf = buf
+	b.off = 0
+}
+
+type byteInputAdapter struct {
+	r         io.Reader
+	readBytes int
+}
+
+// next returns a slice containing the next n bytes from the buffer,
+// advancing the buffer as if the bytes had been returned by Read.
+func (b *byteInputAdapter) next(n int) ([]byte, error) {
+	buf := make([]byte, n)
+	m, err := io.ReadAtLeast(b.r, buf, n)
+	b.readBytes += m
+
+	if err != nil {
+		return nil, err
+	}
+
+	return buf, nil
+}
+
+// readUInt32 reads uint32 with LittleEndian order
+func (b *byteInputAdapter) readUInt32() (uint32, error) {
+	buf, err := b.next(4)
+
+	if err != nil {
+		return 0, err
+	}
+
+	return binary.LittleEndian.Uint32(buf), nil
+}
+
+// readUInt16 reads uint16 with LittleEndian order
+func (b *byteInputAdapter) readUInt16() (uint16, error) {
+	buf, err := b.next(2)
+
+	if err != nil {
+		return 0, err
+	}
+
+	return binary.LittleEndian.Uint16(buf), nil
+}
+
+// getReadBytes returns read bytes
+func (b *byteInputAdapter) getReadBytes() int64 {
+	return int64(b.readBytes)
+}
+
+// skipBytes skips exactly n bytes
+func (b *byteInputAdapter) skipBytes(n int) error {
+	_, err := b.next(n)
+
+	return err
+}
+
+// reset resets the given buffer with a new stream
+func (b *byteInputAdapter) reset(stream io.Reader) {
+	b.r = stream
+	b.readBytes = 0
+}

vendor/github.com/RoaringBitmap/roaring/clz.go

@@ -0,0 +1,11 @@
+// +build go1.9
+// "go1.9", from Go version 1.9 onward
+// See https://golang.org/pkg/go/build/#hdr-Build_Constraints
+
+package roaring
+
+import "math/bits"
+
+func countLeadingZeros(x uint64) int {
+	return bits.LeadingZeros64(x)
+}

vendor/github.com/RoaringBitmap/roaring/clz_compat.go

@@ -0,0 +1,36 @@
+// +build !go1.9
+
+package roaring
+
+// LeadingZeroBits returns the number of consecutive most significant zero
+// bits of x.
+func countLeadingZeros(i uint64) int {
+	if i == 0 {
+		return 64
+	}
+	n := 1
+	x := uint32(i >> 32)
+	if x == 0 {
+		n += 32
+		x = uint32(i)
+	}
+	if (x >> 16) == 0 {
+		n += 16
+		x <<= 16
+	}
+	if (x >> 24) == 0 {
+		n += 8
+		x <<= 8
+	}
+	if x>>28 == 0 {
+		n += 4
+		x <<= 4
+	}
+	if x>>30 == 0 {
+		n += 2
+		x <<= 2
+
+	}
+	n -= int(x >> 31)
+	return n
+}

vendor/github.com/RoaringBitmap/roaring/go.mod

@@ -0,0 +1,16 @@
+module github.com/RoaringBitmap/roaring
+
+go 1.12
+
+require (
+	github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2
+	github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 // indirect
+	github.com/golang/snappy v0.0.1 // indirect
+	github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99 // indirect
+	github.com/jtolds/gls v4.20.0+incompatible // indirect
+	github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae
+	github.com/philhofer/fwd v1.0.0 // indirect
+	github.com/stretchr/testify v1.4.0
+	github.com/tinylib/msgp v1.1.0
+	github.com/willf/bitset v1.1.10
+)

vendor/github.com/RoaringBitmap/roaring/go.sum

@@ -0,0 +1,30 @@
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2 h1:Ujru1hufTHVb++eG6OuNDKMxZnGIvF6o/u8q/8h2+I4=
+github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
+github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 h1:gclg6gY70GLy3PbkQ1AERPfmLMMagS60DKF78eWwLn8=
+github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99 h1:twflg0XRTjwKpxb/jFExr4HGq6on2dEOmnL6FV+fgPw=
+github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae h1:VeRdUYdCw49yizlSbMEn2SZ+gT+3IUKx8BqxyQdz+BY=
+github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg=
+github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ=
+github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU=
+github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
+github.com/willf/bitset v1.1.10 h1:NotGKqX0KwQ72NUzqrjZq5ipPNDQex9lo3WpaS8L2sc=
+github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

vendor/github.com/RoaringBitmap/roaring/manyiterator.go

@@ -4,12 +4,7 @@ type manyIterable interface {
 	nextMany(hs uint32, buf []uint32) int
 }
 
-type manyIterator struct {
-	slice []uint16
-	loc   int
-}
-
-func (si *manyIterator) nextMany(hs uint32, buf []uint32) int {
+func (si *shortIterator) nextMany(hs uint32, buf []uint32) int {
 	n := 0
 	l := si.loc
 	s := si.slice

vendor/github.com/RoaringBitmap/roaring/parallel.go

@@ -143,8 +143,8 @@ func toBitmapContainer(c container) container {
 func appenderRoutine(bitmapChan chan<- *Bitmap, resultChan <-chan keyedContainer, expectedKeysChan <-chan int) {
 	expectedKeys := -1
 	appendedKeys := 0
-	keys := make([]uint16, 0)
-	containers := make([]container, 0)
+	var keys []uint16
+	var containers []container
 	for appendedKeys != expectedKeys {
 		select {
 		case item := <-resultChan:
@@ -337,7 +337,7 @@ func ParAnd(parallelism int, bitmaps ...*Bitmap) *Bitmap {
 // (if it is set to 0, a default number of workers is chosen)
 func ParOr(parallelism int, bitmaps ...*Bitmap) *Bitmap {
 	var lKey uint16 = MaxUint16
-	var hKey uint16 = 0
+	var hKey uint16
 
 	bitmapsFiltered := bitmaps[:0]
 	for _, b := range bitmaps {

vendor/github.com/RoaringBitmap/roaring/rlecommon.go

@@ -1,163 +0,0 @@
-package roaring
-
-import (
-	"fmt"
-)
-
-// common to rle32.go and rle16.go
-
-// rleVerbose controls whether p() prints show up.
-// The testing package sets this based on
-// testing.Verbose().
-var rleVerbose bool
-
-// p is a shorthand for fmt.Printf with beginning and
-// trailing newlines. p() makes it easy
-// to add diagnostic print statements.
-func p(format string, args ...interface{}) {
-	if rleVerbose {
-		fmt.Printf("\n"+format+"\n", args...)
-	}
-}
-
-// MaxUint32 is the largest uint32 value.
-const MaxUint32 = 4294967295
-
-// MaxUint16 is the largest 16 bit unsigned int.
-// This is the largest value an interval16 can store.
-const MaxUint16 = 65535
-
-// searchOptions allows us to accelerate runContainer32.search with
-// prior knowledge of (mostly lower) bounds. This is used by Union
-// and Intersect.
-type searchOptions struct {
-	// start here instead of at 0
-	startIndex int64
-
-	// upper bound instead of len(rc.iv);
-	// endxIndex == 0 means ignore the bound and use
-	// endxIndex == n ==len(rc.iv) which is also
-	// naturally the default for search()
-	// when opt = nil.
-	endxIndex int64
-}
-
-// And finds the intersection of rc and b.
-func (rc *runContainer32) And(b *Bitmap) *Bitmap {
-	out := NewBitmap()
-	for _, p := range rc.iv {
-		for i := p.start; i <= p.last; i++ {
-			if b.Contains(i) {
-				out.Add(i)
-			}
-		}
-	}
-	return out
-}
-
-// Xor returns the exclusive-or of rc and b.
-func (rc *runContainer32) Xor(b *Bitmap) *Bitmap {
-	out := b.Clone()
-	for _, p := range rc.iv {
-		for v := p.start; v <= p.last; v++ {
-			if out.Contains(v) {
-				out.RemoveRange(uint64(v), uint64(v+1))
-			} else {
-				out.Add(v)
-			}
-		}
-	}
-	return out
-}
-
-// Or returns the union of rc and b.
-func (rc *runContainer32) Or(b *Bitmap) *Bitmap {
-	out := b.Clone()
-	for _, p := range rc.iv {
-		for v := p.start; v <= p.last; v++ {
-			out.Add(v)
-		}
-	}
-	return out
-}
-
-// trial is used in the randomized testing of runContainers
-type trial struct {
-	n           int
-	percentFill float64
-	ntrial      int
-
-	// only in the union test
-	// only subtract test
-	percentDelete float64
-
-	// only in 067 randomized operations
-	// we do this + 1 passes
-	numRandomOpsPass int
-
-	// allow sampling range control
-	// only recent tests respect this.
-	srang *interval16
-}
-
-// And finds the intersection of rc and b.
-func (rc *runContainer16) And(b *Bitmap) *Bitmap {
-	out := NewBitmap()
-	for _, p := range rc.iv {
-		plast := p.last()
-		for i := p.start; i <= plast; i++ {
-			if b.Contains(uint32(i)) {
-				out.Add(uint32(i))
-			}
-		}
-	}
-	return out
-}
-
-// Xor returns the exclusive-or of rc and b.
-func (rc *runContainer16) Xor(b *Bitmap) *Bitmap {
-	out := b.Clone()
-	for _, p := range rc.iv {
-		plast := p.last()
-		for v := p.start; v <= plast; v++ {
-			w := uint32(v)
-			if out.Contains(w) {
-				out.RemoveRange(uint64(w), uint64(w+1))
-			} else {
-				out.Add(w)
-			}
-		}
-	}
-	return out
-}
-
-// Or returns the union of rc and b.
-func (rc *runContainer16) Or(b *Bitmap) *Bitmap {
-	out := b.Clone()
-	for _, p := range rc.iv {
-		plast := p.last()
-		for v := p.start; v <= plast; v++ {
-			out.Add(uint32(v))
-		}
-	}
-	return out
-}
-
-//func (rc *runContainer32) and(container) container {
-//	panic("TODO. not yet implemented")
-//}
-
-// serializedSizeInBytes returns the number of bytes of memory
-// required by this runContainer16. This is for the
-// Roaring format, as specified https://github.com/RoaringBitmap/RoaringFormatSpec/
-func (rc *runContainer16) serializedSizeInBytes() int {
-	// number of runs in one uint16, then each run
-	// needs two more uint16
-	return 2 + len(rc.iv)*4
-}
-
-// serializedSizeInBytes returns the number of bytes of memory
-// required by this runContainer32.
-func (rc *runContainer32) serializedSizeInBytes() int {
-	return 4 + len(rc.iv)*8
-}

vendor/github.com/RoaringBitmap/roaring/rlei.go

@@ -1,695 +0,0 @@
-package roaring
-
-///////////////////////////////////////////////////
-//
-// container interface methods for runContainer16
-//
-///////////////////////////////////////////////////
-
-import (
-	"fmt"
-)
-
-// compile time verify we meet interface requirements
-var _ container = &runContainer16{}
-
-func (rc *runContainer16) clone() container {
-	return newRunContainer16CopyIv(rc.iv)
-}
-
-func (rc *runContainer16) minimum() uint16 {
-	return rc.iv[0].start // assume not empty
-}
-
-func (rc *runContainer16) maximum() uint16 {
-	return rc.iv[len(rc.iv)-1].last() // assume not empty
-}
-
-func (rc *runContainer16) isFull() bool {
-	return (len(rc.iv) == 1) && ((rc.iv[0].start == 0) && (rc.iv[0].last() == MaxUint16))
-}
-
-func (rc *runContainer16) and(a container) container {
-	if rc.isFull() {
-		return a.clone()
-	}
-	switch c := a.(type) {
-	case *runContainer16:
-		return rc.intersect(c)
-	case *arrayContainer:
-		return rc.andArray(c)
-	case *bitmapContainer:
-		return rc.andBitmapContainer(c)
-	}
-	panic("unsupported container type")
-}
-
-func (rc *runContainer16) andCardinality(a container) int {
-	switch c := a.(type) {
-	case *runContainer16:
-		return int(rc.intersectCardinality(c))
-	case *arrayContainer:
-		return rc.andArrayCardinality(c)
-	case *bitmapContainer:
-		return rc.andBitmapContainerCardinality(c)
-	}
-	panic("unsupported container type")
-}
-
-// andBitmapContainer finds the intersection of rc and b.
-func (rc *runContainer16) andBitmapContainer(bc *bitmapContainer) container {
-	bc2 := newBitmapContainerFromRun(rc)
-	return bc2.andBitmap(bc)
-}
-
-func (rc *runContainer16) andArrayCardinality(ac *arrayContainer) int {
-	pos := 0
-	answer := 0
-	maxpos := ac.getCardinality()
-	if maxpos == 0 {
-		return 0 // won't happen in actual code
-	}
-	v := ac.content[pos]
-mainloop:
-	for _, p := range rc.iv {
-		for v < p.start {
-			pos++
-			if pos == maxpos {
-				break mainloop
-			}
-			v = ac.content[pos]
-		}
-		for v <= p.last() {
-			answer++
-			pos++
-			if pos == maxpos {
-				break mainloop
-			}
-			v = ac.content[pos]
-		}
-	}
-	return answer
-}
-
-func (rc *runContainer16) iand(a container) container {
-	if rc.isFull() {
-		return a.clone()
-	}
-	switch c := a.(type) {
-	case *runContainer16:
-		return rc.inplaceIntersect(c)
-	case *arrayContainer:
-		return rc.andArray(c)
-	case *bitmapContainer:
-		return rc.iandBitmapContainer(c)
-	}
-	panic("unsupported container type")
-}
-
-func (rc *runContainer16) inplaceIntersect(rc2 *runContainer16) container {
-	// TODO: optimize by doing less allocation, possibly?
-
-	// sect will be new
-	sect := rc.intersect(rc2)
-	*rc = *sect
-	return rc
-}
-
-func (rc *runContainer16) iandBitmapContainer(bc *bitmapContainer) container {
-	isect := rc.andBitmapContainer(bc)
-	*rc = *newRunContainer16FromContainer(isect)
-	return rc
-}
-
-func (rc *runContainer16) andArray(ac *arrayContainer) container {
-	if len(rc.iv) == 0 {
-		return newArrayContainer()
-	}
-
-	acCardinality := ac.getCardinality()
-	c := newArrayContainerCapacity(acCardinality)
-
-	for rlePos, arrayPos := 0, 0; arrayPos < acCardinality; {
-		iv := rc.iv[rlePos]
-		arrayVal := ac.content[arrayPos]
-
-		for iv.last() < arrayVal {
-			rlePos++
-			if rlePos == len(rc.iv) {
-				return c
-			}
-			iv = rc.iv[rlePos]
-		}
-
-		if iv.start > arrayVal {
-			arrayPos = advanceUntil(ac.content, arrayPos, len(ac.content), iv.start)
-		} else {
-			c.content = append(c.content, arrayVal)
-			arrayPos++
-		}
-	}
-	return c
-}
-
-func (rc *runContainer16) andNot(a container) container {
-	switch c := a.(type) {
-	case *arrayContainer:
-		return rc.andNotArray(c)
-	case *bitmapContainer:
-		return rc.andNotBitmap(c)
-	case *runContainer16:
-		return rc.andNotRunContainer16(c)
-	}
-	panic("unsupported container type")
-}
-
-func (rc *runContainer16) fillLeastSignificant16bits(x []uint32, i int, mask uint32) {
-	k := 0
-	var val int64
-	for _, p := range rc.iv {
-		n := p.runlen()
-		for j := int64(0); j < n; j++ {
-			val = int64(p.start) + j
-			x[k+i] = uint32(val) | mask
-			k++
-		}
-	}
-}
-
-func (rc *runContainer16) getShortIterator() shortIterable {
-	return rc.newRunIterator16()
-}
-
-func (rc *runContainer16) getManyIterator() manyIterable {
-	return rc.newManyRunIterator16()
-}
-
-// add the values in the range [firstOfRange, endx). endx
-// is still abe to express 2^16 because it is an int not an uint16.
-func (rc *runContainer16) iaddRange(firstOfRange, endx int) container {
-
-	if firstOfRange >= endx {
-		panic(fmt.Sprintf("invalid %v = endx >= firstOfRange", endx))
-	}
-	addme := newRunContainer16TakeOwnership([]interval16{
-		{
-			start:  uint16(firstOfRange),
-			length: uint16(endx - 1 - firstOfRange),
-		},
-	})
-	*rc = *rc.union(addme)
-	return rc
-}
-
-// remove the values in the range [firstOfRange,endx)
-func (rc *runContainer16) iremoveRange(firstOfRange, endx int) container {
-	if firstOfRange >= endx {
-		panic(fmt.Sprintf("request to iremove empty set [%v, %v),"+
-			" nothing to do.", firstOfRange, endx))
-		//return rc
-	}
-	x := newInterval16Range(uint16(firstOfRange), uint16(endx-1))
-	rc.isubtract(x)
-	return rc
-}
-
-// not flip the values in the range [firstOfRange,endx)
-func (rc *runContainer16) not(firstOfRange, endx int) container {
-	if firstOfRange >= endx {
-		panic(fmt.Sprintf("invalid %v = endx >= firstOfRange = %v", endx, firstOfRange))
-	}
-
-	return rc.Not(firstOfRange, endx)
-}
-
-// Not flips the values in the range [firstOfRange,endx).
-// This is not inplace. Only the returned value has the flipped bits.
-//
-// Currently implemented as (!A intersect B) union (A minus B),
-// where A is rc, and B is the supplied [firstOfRange, endx) interval.
-//
-// TODO(time optimization): convert this to a single pass
-// algorithm by copying AndNotRunContainer16() and modifying it.
-// Current routine is correct but
-// makes 2 more passes through the arrays than should be
-// strictly necessary. Measure both ways though--this may not matter.
-//
-func (rc *runContainer16) Not(firstOfRange, endx int) *runContainer16 {
-
-	if firstOfRange >= endx {
-		panic(fmt.Sprintf("invalid %v = endx >= firstOfRange == %v", endx, firstOfRange))
-	}
-
-	if firstOfRange >= endx {
-		return rc.Clone()
-	}
-
-	a := rc
-	// algo:
-	// (!A intersect B) union (A minus B)
-
-	nota := a.invert()
-
-	bs := []interval16{newInterval16Range(uint16(firstOfRange), uint16(endx-1))}
-	b := newRunContainer16TakeOwnership(bs)
-
-	notAintersectB := nota.intersect(b)
-
-	aMinusB := a.AndNotRunContainer16(b)
-
-	rc2 := notAintersectB.union(aMinusB)
-	return rc2
-}
-
-// equals is now logical equals; it does not require the
-// same underlying container type.
-func (rc *runContainer16) equals(o container) bool {
-	srb, ok := o.(*runContainer16)
-
-	if !ok {
-		// maybe value instead of pointer
-		val, valok := o.(*runContainer16)
-		if valok {
-			srb = val
-			ok = true
-		}
-	}
-	if ok {
-		// Check if the containers are the same object.
-		if rc == srb {
-			return true
-		}
-
-		if len(srb.iv) != len(rc.iv) {
-			return false
-		}
-
-		for i, v := range rc.iv {
-			if v != srb.iv[i] {
-				return false
-			}
-		}
-		return true
-	}
-
-	// use generic comparison
-	if o.getCardinality() != rc.getCardinality() {
-		return false
-	}
-	rit := rc.getShortIterator()
-	bit := o.getShortIterator()
-
-	//k := 0
-	for rit.hasNext() {
-		if bit.next() != rit.next() {
-			return false
-		}
-		//k++
-	}
-	return true
-}
-
-func (rc *runContainer16) iaddReturnMinimized(x uint16) container {
-	rc.Add(x)
-	return rc
-}
-
-func (rc *runContainer16) iadd(x uint16) (wasNew bool) {
-	return rc.Add(x)
-}
-
-func (rc *runContainer16) iremoveReturnMinimized(x uint16) container {
-	rc.removeKey(x)
-	return rc
-}
-
-func (rc *runContainer16) iremove(x uint16) bool {
-	return rc.removeKey(x)
-}
-
-func (rc *runContainer16) or(a container) container {
-	if rc.isFull() {
-		return rc.clone()
-	}
-	switch c := a.(type) {
-	case *runContainer16:
-		return rc.union(c)
-	case *arrayContainer:
-		return rc.orArray(c)
-	case *bitmapContainer:
-		return rc.orBitmapContainer(c)
-	}
-	panic("unsupported container type")
-}
-
-func (rc *runContainer16) orCardinality(a container) int {
-	switch c := a.(type) {
-	case *runContainer16:
-		return int(rc.unionCardinality(c))
-	case *arrayContainer:
-		return rc.orArrayCardinality(c)
-	case *bitmapContainer:
-		return rc.orBitmapContainerCardinality(c)
-	}
-	panic("unsupported container type")
-}
-
-// orBitmapContainer finds the union of rc and bc.
-func (rc *runContainer16) orBitmapContainer(bc *bitmapContainer) container {
-	bc2 := newBitmapContainerFromRun(rc)
-	return bc2.iorBitmap(bc)
-}
-
-func (rc *runContainer16) andBitmapContainerCardinality(bc *bitmapContainer) int {
-	answer := 0
-	for i := range rc.iv {
-		answer += bc.getCardinalityInRange(uint(rc.iv[i].start), uint(rc.iv[i].last())+1)
-	}
-	//bc.computeCardinality()
-	return answer
-}
-
-func (rc *runContainer16) orBitmapContainerCardinality(bc *bitmapContainer) int {
-	return rc.getCardinality() + bc.getCardinality() - rc.andBitmapContainerCardinality(bc)
-}
-
-// orArray finds the union of rc and ac.
-func (rc *runContainer16) orArray(ac *arrayContainer) container {
-	bc1 := newBitmapContainerFromRun(rc)
-	bc2 := ac.toBitmapContainer()
-	return bc1.orBitmap(bc2)
-}
-
-// orArray finds the union of rc and ac.
-func (rc *runContainer16) orArrayCardinality(ac *arrayContainer) int {
-	return ac.getCardinality() + rc.getCardinality() - rc.andArrayCardinality(ac)
-}
-
-func (rc *runContainer16) ior(a container) container {
-	if rc.isFull() {
-		return rc
-	}
-	switch c := a.(type) {
-	case *runContainer16:
-		return rc.inplaceUnion(c)
-	case *arrayContainer:
-		return rc.iorArray(c)
-	case *bitmapContainer:
-		return rc.iorBitmapContainer(c)
-	}
-	panic("unsupported container type")
-}
-
-func (rc *runContainer16) inplaceUnion(rc2 *runContainer16) container {
-	p("rc.inplaceUnion with len(rc2.iv)=%v", len(rc2.iv))
-	for _, p := range rc2.iv {
-		last := int64(p.last())
-		for i := int64(p.start); i <= last; i++ {
-			rc.Add(uint16(i))
-		}
-	}
-	return rc
-}
-
-func (rc *runContainer16) iorBitmapContainer(bc *bitmapContainer) container {
-
-	it := bc.getShortIterator()
-	for it.hasNext() {
-		rc.Add(it.next())
-	}
-	return rc
-}
-
-func (rc *runContainer16) iorArray(ac *arrayContainer) container {
-	it := ac.getShortIterator()
-	for it.hasNext() {
-		rc.Add(it.next())
-	}
-	return rc
-}
-
-// lazyIOR is described (not yet implemented) in
-// this nice note from @lemire on
-// https://github.com/RoaringBitmap/roaring/pull/70#issuecomment-263613737
-//
-// Description of lazyOR and lazyIOR from @lemire:
-//
-// Lazy functions are optional and can be simply
-// wrapper around non-lazy functions.
-//
-// The idea of "laziness" is as follows. It is
-// inspired by the concept of lazy evaluation
-// you might be familiar with (functional programming
-// and all that). So a roaring bitmap is
-// such that all its containers are, in some
-// sense, chosen to use as little memory as
-// possible. This is nice. Also, all bitsets
-// are "cardinality aware" so that you can do
-// fast rank/select queries, or query the
-// cardinality of the whole bitmap... very fast,
-// without latency.
-//
-// However, imagine that you are aggregating 100
-// bitmaps together. So you OR the first two, then OR
-// that with the third one and so forth. Clearly,
-// intermediate bitmaps don't need to be as
-// compressed as possible, right? They can be
-// in a "dirty state". You only need the end
-// result to be in a nice state... which you
-// can achieve by calling repairAfterLazy at the end.
-//
-// The Java/C code does something special for
-// the in-place lazy OR runs. The idea is that
-// instead of taking two run containers and
-// generating a new one, we actually try to
-// do the computation in-place through a
-// technique invented by @gssiyankai (pinging him!).
-// What you do is you check whether the host
-// run container has lots of extra capacity.
-// If it does, you move its data at the end of
-// the backing array, and then you write
-// the answer at the beginning. What this
-// trick does is minimize memory allocations.
-//
-func (rc *runContainer16) lazyIOR(a container) container {
-	// not lazy at the moment
-	// TODO: make it lazy
-	return rc.ior(a)
-
-	/*
-		switch c := a.(type) {
-		case *arrayContainer:
-			return rc.lazyIorArray(c)
-		case *bitmapContainer:
-			return rc.lazyIorBitmap(c)
-		case *runContainer16:
-			return rc.lazyIorRun16(c)
-		}
-		panic("unsupported container type")
-	*/
-}
-
-// lazyOR is described above in lazyIOR.
-func (rc *runContainer16) lazyOR(a container) container {
-
-	// not lazy at the moment
-	// TODO: make it lazy
-	return rc.or(a)
-
-	/*
-		switch c := a.(type) {
-		case *arrayContainer:
-			return rc.lazyOrArray(c)
-		case *bitmapContainer:
-			return rc.lazyOrBitmap(c)
-		case *runContainer16:
-			return rc.lazyOrRunContainer16(c)
-		}
-		panic("unsupported container type")
-	*/
-}
-
-func (rc *runContainer16) intersects(a container) bool {
-	// TODO: optimize by doing inplace/less allocation, possibly?
-	isect := rc.and(a)
-	return isect.getCardinality() > 0
-}
-
-func (rc *runContainer16) xor(a container) container {
-	switch c := a.(type) {
-	case *arrayContainer:
-		return rc.xorArray(c)
-	case *bitmapContainer:
-		return rc.xorBitmap(c)
-	case *runContainer16:
-		return rc.xorRunContainer16(c)
-	}
-	panic("unsupported container type")
-}
-
-func (rc *runContainer16) iandNot(a container) container {
-	switch c := a.(type) {
-	case *arrayContainer:
-		return rc.iandNotArray(c)
-	case *bitmapContainer:
-		return rc.iandNotBitmap(c)
-	case *runContainer16:
-		return rc.iandNotRunContainer16(c)
-	}
-	panic("unsupported container type")
-}
-
-// flip the values in the range [firstOfRange,endx)
-func (rc *runContainer16) inot(firstOfRange, endx int) container {
-	if firstOfRange >= endx {
-		panic(fmt.Sprintf("invalid %v = endx >= firstOfRange = %v", endx, firstOfRange))
-	}
-	// TODO: minimize copies, do it all inplace; not() makes a copy.
-	rc = rc.Not(firstOfRange, endx)
-	return rc
-}
-
-func (rc *runContainer16) getCardinality() int {
-	return int(rc.cardinality())
-}
-
-func (rc *runContainer16) rank(x uint16) int {
-	n := int64(len(rc.iv))
-	xx := int64(x)
-	w, already, _ := rc.search(xx, nil)
-	if w < 0 {
-		return 0
-	}
-	if !already && w == n-1 {
-		return rc.getCardinality()
-	}
-	var rnk int64
-	if !already {
-		for i := int64(0); i <= w; i++ {
-			rnk += rc.iv[i].runlen()
-		}
-		return int(rnk)
-	}
-	for i := int64(0); i < w; i++ {
-		rnk += rc.iv[i].runlen()
-	}
-	rnk += int64(x-rc.iv[w].start) + 1
-	return int(rnk)
-}
-
-func (rc *runContainer16) selectInt(x uint16) int {
-	return rc.selectInt16(x)
-}
-
-func (rc *runContainer16) andNotRunContainer16(b *runContainer16) container {
-	return rc.AndNotRunContainer16(b)
-}
-
-func (rc *runContainer16) andNotArray(ac *arrayContainer) container {
-	rcb := rc.toBitmapContainer()
-	acb := ac.toBitmapContainer()
-	return rcb.andNotBitmap(acb)
-}
-
-func (rc *runContainer16) andNotBitmap(bc *bitmapContainer) container {
-	rcb := rc.toBitmapContainer()
-	return rcb.andNotBitmap(bc)
-}
-
-func (rc *runContainer16) toBitmapContainer() *bitmapContainer {
-	p("run16 toBitmap starting; rc has %v ranges", len(rc.iv))
-	bc := newBitmapContainer()
-	for i := range rc.iv {
-		bc.iaddRange(int(rc.iv[i].start), int(rc.iv[i].last())+1)
-	}
-	bc.computeCardinality()
-	return bc
-}
-
-func (rc *runContainer16) iandNotRunContainer16(x2 *runContainer16) container {
-	rcb := rc.toBitmapContainer()
-	x2b := x2.toBitmapContainer()
-	rcb.iandNotBitmapSurely(x2b)
-	// TODO: check size and optimize the return value
-	// TODO: is inplace modification really required? If not, elide the copy.
-	rc2 := newRunContainer16FromBitmapContainer(rcb)
-	*rc = *rc2
-	return rc
-}
-
-func (rc *runContainer16) iandNotArray(ac *arrayContainer) container {
-	rcb := rc.toBitmapContainer()
-	acb := ac.toBitmapContainer()
-	rcb.iandNotBitmapSurely(acb)
-	// TODO: check size and optimize the return value
-	// TODO: is inplace modification really required? If not, elide the copy.
-	rc2 := newRunContainer16FromBitmapContainer(rcb)
-	*rc = *rc2
-	return rc
-}
-
-func (rc *runContainer16) iandNotBitmap(bc *bitmapContainer) container {
-	rcb := rc.toBitmapContainer()
-	rcb.iandNotBitmapSurely(bc)
-	// TODO: check size and optimize the return value
-	// TODO: is inplace modification really required? If not, elide the copy.
-	rc2 := newRunContainer16FromBitmapContainer(rcb)
-	*rc = *rc2
-	return rc
-}
-
-func (rc *runContainer16) xorRunContainer16(x2 *runContainer16) container {
-	rcb := rc.toBitmapContainer()
-	x2b := x2.toBitmapContainer()
-	return rcb.xorBitmap(x2b)
-}
-
-func (rc *runContainer16) xorArray(ac *arrayContainer) container {
-	rcb := rc.toBitmapContainer()
-	acb := ac.toBitmapContainer()
-	return rcb.xorBitmap(acb)
-}
-
-func (rc *runContainer16) xorBitmap(bc *bitmapContainer) container {
-	rcb := rc.toBitmapContainer()
-	return rcb.xorBitmap(bc)
-}
-
-// convert to bitmap or array *if needed*
-func (rc *runContainer16) toEfficientContainer() container {
-
-	// runContainer16SerializedSizeInBytes(numRuns)
-	sizeAsRunContainer := rc.getSizeInBytes()
-	sizeAsBitmapContainer := bitmapContainerSizeInBytes()
-	card := int(rc.cardinality())
-	sizeAsArrayContainer := arrayContainerSizeInBytes(card)
-	if sizeAsRunContainer <= minOfInt(sizeAsBitmapContainer, sizeAsArrayContainer) {
-		return rc
-	}
-	if card <= arrayDefaultMaxSize {
-		return rc.toArrayContainer()
-	}
-	bc := newBitmapContainerFromRun(rc)
-	return bc
-}
-
-func (rc *runContainer16) toArrayContainer() *arrayContainer {
-	ac := newArrayContainer()
-	for i := range rc.iv {
-		ac.iaddRange(int(rc.iv[i].start), int(rc.iv[i].last())+1)
-	}
-	return ac
-}
-
-func newRunContainer16FromContainer(c container) *runContainer16 {
-
-	switch x := c.(type) {
-	case *runContainer16:
-		return x.Clone()
-	case *arrayContainer:
-		return newRunContainer16FromArray(x)
-	case *bitmapContainer:
-		return newRunContainer16FromBitmapContainer(x)
-	}
-	panic("unsupported container type")
-}

vendor/github.com/RoaringBitmap/roaring/roaring.go

@@ -6,12 +6,12 @@
 package roaring
 
 import (
-	"bufio"
 	"bytes"
 	"encoding/base64"
 	"fmt"
 	"io"
 	"strconv"
+	"sync"
 )
 
 // Bitmap represents a compressed bitmap where you can add integers.
@@ -52,7 +52,7 @@ func (rb *Bitmap) ToBytes() ([]byte, error) {
 	return rb.highlowcontainer.toBytes()
 }
 
-// WriteToMsgpack writes a msgpack2/snappy-streaming compressed serialized
+// Deprecated: WriteToMsgpack writes a msgpack2/snappy-streaming compressed serialized
 // version of this bitmap to stream. The format is not
 // compatible with the WriteTo() format, and is
 // experimental: it may produce smaller on disk
@@ -67,8 +67,14 @@ func (rb *Bitmap) WriteToMsgpack(stream io.Writer) (int64, error) {
 // The format is compatible with other RoaringBitmap
 // implementations (Java, C) and is documented here:
 // https://github.com/RoaringBitmap/RoaringFormatSpec
-func (rb *Bitmap) ReadFrom(stream io.Reader) (int64, error) {
-	return rb.highlowcontainer.readFrom(stream)
+func (rb *Bitmap) ReadFrom(reader io.Reader) (p int64, err error) {
+	stream := byteInputAdapterPool.Get().(*byteInputAdapter)
+	stream.reset(reader)
+
+	p, err = rb.highlowcontainer.readFrom(stream)
+	byteInputAdapterPool.Put(stream)
+
+	return
 }
 
 // FromBuffer creates a bitmap from its serialized version stored in buffer
@@ -87,10 +93,36 @@ func (rb *Bitmap) ReadFrom(stream io.Reader) (int64, error) {
 // You should *not* change the copy-on-write status of the resulting
 // bitmaps (SetCopyOnWrite).
 //
-func (rb *Bitmap) FromBuffer(buf []byte) (int64, error) {
-	return rb.highlowcontainer.fromBuffer(buf)
+// If buf becomes unavailable, then a bitmap created with
+// FromBuffer would be effectively broken. Furthermore, any
+// bitmap derived from this bitmap (e.g., via Or, And) might
+// also be broken. Thus, before making buf unavailable, you should
+// call CloneCopyOnWriteContainers on all such bitmaps.
+//
+func (rb *Bitmap) FromBuffer(buf []byte) (p int64, err error) {
+	stream := byteBufferPool.Get().(*byteBuffer)
+	stream.reset(buf)
+
+	p, err = rb.highlowcontainer.readFrom(stream)
+	byteBufferPool.Put(stream)
+
+	return
 }
 
+var (
+	byteBufferPool = sync.Pool{
+		New: func() interface{} {
+			return &byteBuffer{}
+		},
+	}
+
+	byteInputAdapterPool = sync.Pool{
+		New: func() interface{} {
+			return &byteInputAdapter{}
+		},
+	}
+)
+
 // RunOptimize attempts to further compress the runs of consecutive values found in the bitmap
 func (rb *Bitmap) RunOptimize() {
 	rb.highlowcontainer.runOptimize()
@@ -101,7 +133,7 @@ func (rb *Bitmap) HasRunCompression() bool {
 	return rb.highlowcontainer.hasRunCompression()
 }
 
-// ReadFromMsgpack reads a msgpack2/snappy-streaming serialized
+// Deprecated: ReadFromMsgpack reads a msgpack2/snappy-streaming serialized
 // version of this bitmap from stream. The format is
 // expected is that written by the WriteToMsgpack()
 // call; see additional notes there.
@@ -110,29 +142,15 @@ func (rb *Bitmap) ReadFromMsgpack(stream io.Reader) (int64, error) {
 }
 
 // MarshalBinary implements the encoding.BinaryMarshaler interface for the bitmap
+// (same as ToBytes)
 func (rb *Bitmap) MarshalBinary() ([]byte, error) {
-	var buf bytes.Buffer
-	writer := bufio.NewWriter(&buf)
-	_, err := rb.WriteTo(writer)
-	if err != nil {
-		return nil, err
-	}
-	err = writer.Flush()
-	if err != nil {
-		return nil, err
-	}
-	return buf.Bytes(), nil
+	return rb.ToBytes()
 }
 
 // UnmarshalBinary implements the encoding.BinaryUnmarshaler interface for the bitmap
 func (rb *Bitmap) UnmarshalBinary(data []byte) error {
-	var buf bytes.Buffer
-	_, err := buf.Write(data)
-	if err != nil {
-		return err
-	}
-	reader := bufio.NewReader(&buf)
-	_, err = rb.ReadFrom(reader)
+	r := bytes.NewReader(data)
+	_, err := rb.ReadFrom(r)
 	return err
 }
 
@@ -215,10 +233,20 @@ type IntIterable interface {
 	Next() uint32
 }
 
+// IntPeekable allows you to look at the next value without advancing and
+// advance as long as the next value is smaller than minval
+type IntPeekable interface {
+	IntIterable
+	// PeekNext peeks the next value without advancing the iterator
+	PeekNext() uint32
+	// AdvanceIfNeeded advances as long as the next value is smaller than minval
+	AdvanceIfNeeded(minval uint32)
+}
+
 type intIterator struct {
 	pos              int
 	hs               uint32
-	iter             shortIterable
+	iter             shortPeekable
 	highlowcontainer *roaringArray
 }
 
@@ -244,6 +272,30 @@ func (ii *intIterator) Next() uint32 {
 	return x
 }
 
+// PeekNext peeks the next value without advancing the iterator
+func (ii *intIterator) PeekNext() uint32 {
+	return uint32(ii.iter.peekNext()&maxLowBit) | ii.hs
+}
+
+// AdvanceIfNeeded advances as long as the next value is smaller than minval
+func (ii *intIterator) AdvanceIfNeeded(minval uint32) {
+	to := minval >> 16
+
+	for ii.HasNext() && (ii.hs>>16) < to {
+		ii.pos++
+		ii.init()
+	}
+
+	if ii.HasNext() && (ii.hs>>16) == to {
+		ii.iter.advanceIfNeeded(lowbits(minval))
+
+		if !ii.iter.hasNext() {
+			ii.pos++
+			ii.init()
+		}
+	}
+}
+
 func newIntIterator(a *Bitmap) *intIterator {
 	p := new(intIterator)
 	p.pos = 0
@@ -252,6 +304,45 @@ func newIntIterator(a *Bitmap) *intIterator {
 	return p
 }
 
+type intReverseIterator struct {
+	pos              int
+	hs               uint32
+	iter             shortIterable
+	highlowcontainer *roaringArray
+}
+
+// HasNext returns true if there are more integers to iterate over
+func (ii *intReverseIterator) HasNext() bool {
+	return ii.pos >= 0
+}
+
+func (ii *intReverseIterator) init() {
+	if ii.pos >= 0 {
+		ii.iter = ii.highlowcontainer.getContainerAtIndex(ii.pos).getReverseIterator()
+		ii.hs = uint32(ii.highlowcontainer.getKeyAtIndex(ii.pos)) << 16
+	} else {
+		ii.iter = nil
+	}
+}
+
+// Next returns the next integer
+func (ii *intReverseIterator) Next() uint32 {
+	x := uint32(ii.iter.next()) | ii.hs
+	if !ii.iter.hasNext() {
+		ii.pos = ii.pos - 1
+		ii.init()
+	}
+	return x
+}
+
+func newIntReverseIterator(a *Bitmap) *intReverseIterator {
+	p := new(intReverseIterator)
+	p.highlowcontainer = &a.highlowcontainer
+	p.pos = a.highlowcontainer.size() - 1
+	p.init()
+	return p
+}
+
 // ManyIntIterable allows you to iterate over the values in a Bitmap
 type ManyIntIterable interface {
 	// pass in a buffer to fill up with values, returns how many values were returned
@@ -325,12 +416,52 @@ func (rb *Bitmap) String() string {
 	return buffer.String()
 }
 
-// Iterator creates a new IntIterable to iterate over the integers contained in the bitmap, in sorted order
-func (rb *Bitmap) Iterator() IntIterable {
+// Iterate iterates over the bitmap, calling the given callback with each value in the bitmap.  If the callback returns
+// false, the iteration is halted.
+// The iteration results are undefined if the bitmap is modified (e.g., with Add or Remove).
+// There is no guarantee as to what order the values will be iterated
+func (rb *Bitmap) Iterate(cb func(x uint32) bool) {
+	for i := 0; i < rb.highlowcontainer.size(); i++ {
+		hs := uint32(rb.highlowcontainer.getKeyAtIndex(i)) << 16
+		c := rb.highlowcontainer.getContainerAtIndex(i)
+
+		var shouldContinue bool
+		// This is hacky but it avoids allocations from invoking an interface method with a closure
+		switch t := c.(type) {
+		case *arrayContainer:
+			shouldContinue = t.iterate(func(x uint16) bool {
+				return cb(uint32(x) | hs)
+			})
+		case *runContainer16:
+			shouldContinue = t.iterate(func(x uint16) bool {
+				return cb(uint32(x) | hs)
+			})
+		case *bitmapContainer:
+			shouldContinue = t.iterate(func(x uint16) bool {
+				return cb(uint32(x) | hs)
+			})
+		}
+
+		if !shouldContinue {
+			break
+		}
+	}
+}
+
+// Iterator creates a new IntPeekable to iterate over the integers contained in the bitmap, in sorted order;
+// the iterator becomes invalid if the bitmap is modified (e.g., with Add or Remove).
+func (rb *Bitmap) Iterator() IntPeekable {
 	return newIntIterator(rb)
 }
 
-// Iterator creates a new ManyIntIterable to iterate over the integers contained in the bitmap, in sorted order
+// ReverseIterator creates a new IntIterable to iterate over the integers contained in the bitmap, in sorted order;
+// the iterator becomes invalid if the bitmap is modified (e.g., with Add or Remove).
+func (rb *Bitmap) ReverseIterator() IntIterable {
+	return newIntReverseIterator(rb)
+}
+
+// ManyIterator creates a new ManyIntIterable to iterate over the integers contained in the bitmap, in sorted order;
+// the iterator becomes invalid if the bitmap is modified (e.g., with Add or Remove).
 func (rb *Bitmap) ManyIterator() ManyIntIterable {
 	return newManyIntIterator(rb)
 }
@@ -374,6 +505,77 @@ func (rb *Bitmap) Equals(o interface{}) bool {
 	return false
 }
 
+// AddOffset adds the value 'offset' to each and every value in a bitmap, generating a new bitmap in the process
+func AddOffset(x *Bitmap, offset uint32) (answer *Bitmap) {
+	return AddOffset64(x, int64(offset))
+}
+
+// AddOffset64 adds the value 'offset' to each and every value in a bitmap, generating a new bitmap in the process
+// If offset + element is outside of the range [0,2^32), that the element will be dropped
+func AddOffset64(x *Bitmap, offset int64) (answer *Bitmap) {
+	// we need "offset" to be a long because we want to support values
+	// between -0xFFFFFFFF up to +-0xFFFFFFFF
+	var containerOffset64 int64
+
+	if offset < 0 {
+		containerOffset64 = (offset - (1 << 16) + 1) / (1 << 16)
+	} else {
+		containerOffset64 = offset >> 16
+	}
+
+	if containerOffset64 >= (1<<16) || containerOffset64 <= -(1<<16) {
+		return New()
+	}
+
+	containerOffset := int32(containerOffset64)
+	inOffset := (uint16)(offset - containerOffset64*(1<<16))
+
+	if inOffset == 0 {
+		answer = x.Clone()
+		for pos := 0; pos < answer.highlowcontainer.size(); pos++ {
+			key := int32(answer.highlowcontainer.getKeyAtIndex(pos))
+			key += containerOffset
+
+			if key >= 0 && key <= MaxUint16 {
+				answer.highlowcontainer.keys[pos] = uint16(key)
+			}
+		}
+	} else {
+		answer = New()
+
+		for pos := 0; pos < x.highlowcontainer.size(); pos++ {
+			key := int32(x.highlowcontainer.getKeyAtIndex(pos))
+			key += containerOffset
+
+			c := x.highlowcontainer.getContainerAtIndex(pos)
+			offsetted := c.addOffset(inOffset)
+
+			if offsetted[0].getCardinality() > 0 && (key >= 0 && key <= MaxUint16) {
+				curSize := answer.highlowcontainer.size()
+				lastkey := int32(0)
+
+				if curSize > 0 {
+					lastkey = int32(answer.highlowcontainer.getKeyAtIndex(curSize - 1))
+				}
+
+				if curSize > 0 && lastkey == key {
+					prev := answer.highlowcontainer.getContainerAtIndex(curSize - 1)
+					orrseult := prev.ior(offsetted[0])
+					answer.highlowcontainer.setContainerAtIndex(curSize-1, orrseult)
+				} else {
+					answer.highlowcontainer.appendContainer(uint16(key), offsetted[0], false)
+				}
+			}
+
+			if offsetted[1].getCardinality() > 0 && ((key+1) >= 0 && (key+1) <= MaxUint16) {
+				answer.highlowcontainer.appendContainer(uint16(key+1), offsetted[1], false)
+			}
+		}
+	}
+
+	return answer
+}
+
 // Add the integer x to the bitmap
 func (rb *Bitmap) Add(x uint32) {
 	hb := highbits(x)
@@ -794,11 +996,6 @@ main:
 	}
 }
 
-/*func (rb *Bitmap) Or(x2 *Bitmap) {
-	results := Or(rb, x2) // Todo: could be computed in-place for reduced memory usage
-	rb.highlowcontainer = results.highlowcontainer
-}*/
-
 // AndNot computes the difference between two bitmaps and stores the result in the current bitmap
 func (rb *Bitmap) AndNot(x2 *Bitmap) {
 	pos1 := 0
@@ -1086,10 +1283,10 @@ func (rb *Bitmap) Flip(rangeStart, rangeEnd uint64) {
 		return
 	}
 
-	hbStart := highbits(uint32(rangeStart))
-	lbStart := lowbits(uint32(rangeStart))
-	hbLast := highbits(uint32(rangeEnd - 1))
-	lbLast := lowbits(uint32(rangeEnd - 1))
+	hbStart := uint32(highbits(uint32(rangeStart)))
+	lbStart := uint32(lowbits(uint32(rangeStart)))
+	hbLast := uint32(highbits(uint32(rangeEnd - 1)))
+	lbLast := uint32(lowbits(uint32(rangeEnd - 1)))
 
 	var max uint32 = maxLowBit
 	for hb := hbStart; hb <= hbLast; hb++ {
@@ -1102,7 +1299,7 @@ func (rb *Bitmap) Flip(rangeStart, rangeEnd uint64) {
 			containerLast = uint32(lbLast)
 		}
 
-		i := rb.highlowcontainer.getIndex(hb)
+		i := rb.highlowcontainer.getIndex(uint16(hb))
 
 		if i >= 0 {
 			c := rb.highlowcontainer.getWritableContainerAtIndex(i).inot(int(containerStart), int(containerLast)+1)
@@ -1113,7 +1310,7 @@ func (rb *Bitmap) Flip(rangeStart, rangeEnd uint64) {
 			}
 		} else { // *think* the range of ones must never be
 			// empty.
-			rb.highlowcontainer.insertNewKeyValueAt(-i-1, hb, rangeOfOnes(int(containerStart), int(containerLast)))
+			rb.highlowcontainer.insertNewKeyValueAt(-i-1, uint16(hb), rangeOfOnes(int(containerStart), int(containerLast)))
 		}
 	}
 }
@@ -1139,24 +1336,24 @@ func (rb *Bitmap) AddRange(rangeStart, rangeEnd uint64) {
 	lbLast := uint32(lowbits(uint32(rangeEnd - 1)))
 
 	var max uint32 = maxLowBit
-	for hb := uint16(hbStart); hb <= uint16(hbLast); hb++ {
+	for hb := hbStart; hb <= hbLast; hb++ {
 		containerStart := uint32(0)
-		if hb == uint16(hbStart) {
+		if hb == hbStart {
 			containerStart = lbStart
 		}
 		containerLast := max
-		if hb == uint16(hbLast) {
+		if hb == hbLast {
 			containerLast = lbLast
 		}
 
-		i := rb.highlowcontainer.getIndex(hb)
+		i := rb.highlowcontainer.getIndex(uint16(hb))
 
 		if i >= 0 {
 			c := rb.highlowcontainer.getWritableContainerAtIndex(i).iaddRange(int(containerStart), int(containerLast)+1)
 			rb.highlowcontainer.setContainerAtIndex(i, c)
 		} else { // *think* the range of ones must never be
 			// empty.
-			rb.highlowcontainer.insertNewKeyValueAt(-i-1, hb, rangeOfOnes(int(containerStart), int(containerLast)))
+			rb.highlowcontainer.insertNewKeyValueAt(-i-1, uint16(hb), rangeOfOnes(int(containerStart), int(containerLast)))
 		}
 	}
 }
@@ -1243,13 +1440,13 @@ func Flip(bm *Bitmap, rangeStart, rangeEnd uint64) *Bitmap {
 	}
 
 	answer := NewBitmap()
-	hbStart := highbits(uint32(rangeStart))
-	lbStart := lowbits(uint32(rangeStart))
-	hbLast := highbits(uint32(rangeEnd - 1))
-	lbLast := lowbits(uint32(rangeEnd - 1))
+	hbStart := uint32(highbits(uint32(rangeStart)))
+	lbStart := uint32(lowbits(uint32(rangeStart)))
+	hbLast := uint32(highbits(uint32(rangeEnd - 1)))
+	lbLast := uint32(lowbits(uint32(rangeEnd - 1)))
 
 	// copy the containers before the active area
-	answer.highlowcontainer.appendCopiesUntil(bm.highlowcontainer, hbStart)
+	answer.highlowcontainer.appendCopiesUntil(bm.highlowcontainer, uint16(hbStart))
 
 	var max uint32 = maxLowBit
 	for hb := hbStart; hb <= hbLast; hb++ {
@@ -1262,23 +1459,23 @@ func Flip(bm *Bitmap, rangeStart, rangeEnd uint64) *Bitmap {
 			containerLast = uint32(lbLast)
 		}
 
-		i := bm.highlowcontainer.getIndex(hb)
-		j := answer.highlowcontainer.getIndex(hb)
+		i := bm.highlowcontainer.getIndex(uint16(hb))
+		j := answer.highlowcontainer.getIndex(uint16(hb))
 
 		if i >= 0 {
 			c := bm.highlowcontainer.getContainerAtIndex(i).not(int(containerStart), int(containerLast)+1)
 			if c.getCardinality() > 0 {
-				answer.highlowcontainer.insertNewKeyValueAt(-j-1, hb, c)
+				answer.highlowcontainer.insertNewKeyValueAt(-j-1, uint16(hb), c)
 			}
 
 		} else { // *think* the range of ones must never be
 			// empty.
-			answer.highlowcontainer.insertNewKeyValueAt(-j-1, hb,
+			answer.highlowcontainer.insertNewKeyValueAt(-j-1, uint16(hb),
 				rangeOfOnes(int(containerStart), int(containerLast)))
 		}
 	}
 	// copy the containers after the active area.
-	answer.highlowcontainer.appendCopiesAfter(bm.highlowcontainer, hbLast)
+	answer.highlowcontainer.appendCopiesAfter(bm.highlowcontainer, uint16(hbLast))
 
 	return answer
 }
@@ -1296,6 +1493,21 @@ func (rb *Bitmap) GetCopyOnWrite() (val bool) {
 	return rb.highlowcontainer.copyOnWrite
 }
 
+// CloneCopyOnWriteContainers clones all containers which have
+// needCopyOnWrite set to true.
+// This can be used to make sure it is safe to munmap a []byte
+// that the roaring array may still have a reference to, after
+// calling FromBuffer.
+// More generally this function is useful if you call FromBuffer
+// to construct a bitmap with a backing array buf
+// and then later discard the buf array. Note that you should call
+// CloneCopyOnWriteContainers on all bitmaps that were derived
+// from the 'FromBuffer' bitmap since they map have dependencies
+// on the buf array as well.
+func (rb *Bitmap) CloneCopyOnWriteContainers() {
+	rb.highlowcontainer.cloneCopyOnWriteContainers()
+}
+
 // FlipInt calls Flip after casting the parameters (convenience method)
 func FlipInt(bm *Bitmap, rangeStart, rangeEnd int) *Bitmap {
 	return Flip(bm, uint64(rangeStart), uint64(rangeEnd))

vendor/github.com/RoaringBitmap/roaring/roaringarray.go

@@ -5,7 +5,6 @@ import (
 	"encoding/binary"
 	"fmt"
 	"io"
-	"io/ioutil"
 
 	snappy "github.com/glycerine/go-unsnap-stream"
 	"github.com/tinylib/msgp/msgp"
@@ -14,6 +13,8 @@ import (
 //go:generate msgp -unexported
 
 type container interface {
+	addOffset(uint16) []container
+
 	clone() container
 	and(container) container
 	andCardinality(container) int
@@ -37,7 +38,9 @@ type container interface {
 	not(start, final int) container        // range is [firstOfRange,lastOfRange)
 	inot(firstOfRange, endx int) container // i stands for inplace, range is [firstOfRange,endx)
 	xor(r container) container
-	getShortIterator() shortIterable
+	getShortIterator() shortPeekable
+	iterate(cb func(x uint16) bool) bool
+	getReverseIterator() shortIterable
 	getManyIterator() manyIterable
 	contains(i uint16) bool
 	maximum() uint16
@@ -61,7 +64,6 @@ type container interface {
 	iremoveRange(start, final int) container // i stands for inplace, range is [firstOfRange,lastOfRange)
 	selectInt(x uint16) int                  // selectInt returns the xth integer in the container
 	serializedSizeInBytes() int
-	readFrom(io.Reader) (int, error)
 	writeTo(io.Writer) (int, error)
 
 	numberOfRuns() int
@@ -280,6 +282,18 @@ func (ra *roaringArray) clone() *roaringArray {
 	return &sa
 }
 
+// clone all containers which have needCopyOnWrite set to true
+// This can be used to make sure it is safe to munmap a []byte
+// that the roaring array may still have a reference to.
+func (ra *roaringArray) cloneCopyOnWriteContainers() {
+	for i, needCopyOnWrite := range ra.needCopyOnWrite {
+		if needCopyOnWrite {
+			ra.containers[i] = ra.containers[i].clone()
+			ra.needCopyOnWrite[i] = false
+		}
+	}
+}
+
 // unused function:
 //func (ra *roaringArray) containsKey(x uint16) bool {
 //	return (ra.binarySearch(0, int64(len(ra.keys)), x) >= 0)
@@ -456,8 +470,7 @@ func (ra *roaringArray) serializedSizeInBytes() uint64 {
 //
 // spec: https://github.com/RoaringBitmap/RoaringFormatSpec
 //
-func (ra *roaringArray) toBytes() ([]byte, error) {
-	stream := &bytes.Buffer{}
+func (ra *roaringArray) writeTo(w io.Writer) (n int64, err error) {
 	hasRun := ra.hasRunCompression()
 	isRunSizeInBytes := 0
 	cookieSize := 8
@@ -477,20 +490,15 @@ func (ra *roaringArray) toBytes() ([]byte, error) {
 		nw += 2
 		binary.LittleEndian.PutUint16(buf[2:], uint16(len(ra.keys)-1))
 		nw += 2
-
-		// compute isRun bitmap
-		var ir []byte
-
-		isRun := newBitmapContainer()
+		// compute isRun bitmap without temporary allocation
+		var runbitmapslice = buf[nw:nw+isRunSizeInBytes]
 		for i, c := range ra.containers {
 			switch c.(type) {
 			case *runContainer16:
-				isRun.iadd(uint16(i))
+				runbitmapslice[i / 8] |= 1<<(uint(i)%8)
 			}
 		}
-		// convert to little endian
-		ir = isRun.asLittleEndianByteSlice()[:isRunSizeInBytes]
-		nw += copy(buf[nw:], ir)
+		nw += isRunSizeInBytes
 	} else {
 		binary.LittleEndian.PutUint32(buf[0:], uint32(serialCookieNoRunContainer))
 		nw += 4
@@ -522,79 +530,77 @@ func (ra *roaringArray) toBytes() ([]byte, error) {
 		}
 	}
 
-	_, err := stream.Write(buf[:nw])
+	written, err := w.Write(buf[:nw])
 	if err != nil {
-		return nil, err
+		return n, err
 	}
-	for i, c := range ra.containers {
-		_ = i
-		_, err := c.writeTo(stream)
+	n += int64(written)
+
+	for _, c := range ra.containers {
+		written, err := c.writeTo(w)
 		if err != nil {
-			return nil, err
+			return n, err
 		}
+		n += int64(written)
 	}
-	return stream.Bytes(), nil
+	return n, nil
 }
 
 //
 // spec: https://github.com/RoaringBitmap/RoaringFormatSpec
 //
-func (ra *roaringArray) writeTo(out io.Writer) (int64, error) {
-	by, err := ra.toBytes()
-	if err != nil {
-		return 0, err
-	}
-	n, err := out.Write(by)
-	if err == nil && n < len(by) {
-		err = io.ErrShortWrite
-	}
-	return int64(n), err
+func (ra *roaringArray) toBytes() ([]byte, error) {
+	var buf bytes.Buffer
+	_, err := ra.writeTo(&buf)
+	return buf.Bytes(), err
 }
 
-func (ra *roaringArray) fromBuffer(buf []byte) (int64, error) {
-	pos := 0
-	if len(buf) < 8 {
-		return 0, fmt.Errorf("buffer too small, expecting at least 8 bytes, was %d", len(buf))
+func (ra *roaringArray) readFrom(stream byteInput) (int64, error) {
+	cookie, err := stream.readUInt32()
+
+	if err != nil {
+		return stream.getReadBytes(), fmt.Errorf("error in roaringArray.readFrom: could not read initial cookie: %s", err)
 	}
 
-	cookie := binary.LittleEndian.Uint32(buf)
-	pos += 4
-	var size uint32 // number of containers
-	haveRunContainers := false
+	var size uint32
 	var isRunBitmap []byte
 
-	// cookie header
 	if cookie&0x0000FFFF == serialCookie {
-		haveRunContainers = true
-		size = uint32(uint16(cookie>>16) + 1) // number of containers
-
+		size = uint32(uint16(cookie>>16) + 1)
 		// create is-run-container bitmap
 		isRunBitmapSize := (int(size) + 7) / 8
-		if pos+isRunBitmapSize > len(buf) {
-			return 0, fmt.Errorf("malformed bitmap, is-run bitmap overruns buffer at %d", pos+isRunBitmapSize)
+		isRunBitmap, err = stream.next(isRunBitmapSize)
+
+		if err != nil {
+			return stream.getReadBytes(), fmt.Errorf("malformed bitmap, failed to read is-run bitmap, got: %s", err)
 		}
-
-		isRunBitmap = buf[pos : pos+isRunBitmapSize]
-		pos += isRunBitmapSize
 	} else if cookie == serialCookieNoRunContainer {
-		size = binary.LittleEndian.Uint32(buf[pos:])
-		pos += 4
-	} else {
-		return 0, fmt.Errorf("error in roaringArray.readFrom: did not find expected serialCookie in header")
-	}
-	if size > (1 << 16) {
-		return 0, fmt.Errorf("It is logically impossible to have more than (1<<16) containers.")
-	}
-	// descriptive header
-	// keycard - is {key, cardinality} tuple slice
-	if pos+2*2*int(size) > len(buf) {
-		return 0, fmt.Errorf("malfomred bitmap, key-cardinality slice overruns buffer at %d", pos+2*2*int(size))
-	}
-	keycard := byteSliceAsUint16Slice(buf[pos : pos+2*2*int(size)])
-	pos += 2 * 2 * int(size)
+		size, err = stream.readUInt32()
 
-	if !haveRunContainers || size >= noOffsetThreshold {
-		pos += 4 * int(size)
+		if err != nil {
+			return stream.getReadBytes(), fmt.Errorf("malformed bitmap, failed to read a bitmap size: %s", err)
+		}
+	} else {
+		return stream.getReadBytes(), fmt.Errorf("error in roaringArray.readFrom: did not find expected serialCookie in header")
+	}
+
+	if size > (1 << 16) {
+		return stream.getReadBytes(), fmt.Errorf("it is logically impossible to have more than (1<<16) containers")
+	}
+
+	// descriptive header
+	buf, err := stream.next(2 * 2 * int(size))
+
+	if err != nil {
+		return stream.getReadBytes(), fmt.Errorf("failed to read descriptive header: %s", err)
+	}
+
+	keycard := byteSliceAsUint16Slice(buf)
+
+	if isRunBitmap == nil || size >= noOffsetThreshold {
+		if err := stream.skipBytes(int(size) * 4); err != nil {
+			return stream.getReadBytes(), fmt.Errorf("failed to skip bytes: %s", err)
+		}
 	}
 
 	// Allocate slices upfront as number of containers is known
@@ -603,11 +609,13 @@ func (ra *roaringArray) fromBuffer(buf []byte) (int64, error) {
 	} else {
 		ra.containers = make([]container, size)
 	}
+
 	if cap(ra.keys) >= int(size) {
 		ra.keys = ra.keys[:size]
 	} else {
 		ra.keys = make([]uint16, size)
 	}
+
 	if cap(ra.needCopyOnWrite) >= int(size) {
 		ra.needCopyOnWrite = ra.needCopyOnWrite[:size]
 	} else {
@@ -615,129 +623,62 @@ func (ra *roaringArray) fromBuffer(buf []byte) (int64, error) {
 	}
 
 	for i := uint32(0); i < size; i++ {
-		key := uint16(keycard[2*i])
+		key := keycard[2*i]
 		card := int(keycard[2*i+1]) + 1
 		ra.keys[i] = key
 		ra.needCopyOnWrite[i] = true
 
-		if haveRunContainers && isRunBitmap[i/8]&(1<<(i%8)) != 0 {
+		if isRunBitmap != nil && isRunBitmap[i/8]&(1<<(i%8)) != 0 {
 			// run container
-			nr := binary.LittleEndian.Uint16(buf[pos:])
-			pos += 2
-			if pos+int(nr)*4 > len(buf) {
-				return 0, fmt.Errorf("malformed bitmap, a run container overruns buffer at %d:%d", pos, pos+int(nr)*4)
+			nr, err := stream.readUInt16()
+
+			if err != nil {
+				return 0, fmt.Errorf("failed to read runtime container size: %s", err)
 			}
+
+			buf, err := stream.next(int(nr) * 4)
+
+			if err != nil {
+				return stream.getReadBytes(), fmt.Errorf("failed to read runtime container content: %s", err)
+			}
+
 			nb := runContainer16{
-				iv:   byteSliceAsInterval16Slice(buf[pos : pos+int(nr)*4]),
+				iv:   byteSliceAsInterval16Slice(buf),
 				card: int64(card),
 			}
-			pos += int(nr) * 4
+
 			ra.containers[i] = &nb
 		} else if card > arrayDefaultMaxSize {
 			// bitmap container
+			buf, err := stream.next(arrayDefaultMaxSize * 2)
+
+			if err != nil {
+				return stream.getReadBytes(), fmt.Errorf("failed to read bitmap container: %s", err)
+			}
+
 			nb := bitmapContainer{
 				cardinality: card,
-				bitmap:      byteSliceAsUint64Slice(buf[pos : pos+arrayDefaultMaxSize*2]),
+				bitmap:      byteSliceAsUint64Slice(buf),
 			}
-			pos += arrayDefaultMaxSize * 2
+
 			ra.containers[i] = &nb
 		} else {
 			// array container
-			nb := arrayContainer{
-				byteSliceAsUint16Slice(buf[pos : pos+card*2]),
+			buf, err := stream.next(card * 2)
+
+			if err != nil {
+				return stream.getReadBytes(), fmt.Errorf("failed to read array container: %s", err)
 			}
-			pos += card * 2
+
+			nb := arrayContainer{
+				byteSliceAsUint16Slice(buf),
+			}
+
 			ra.containers[i] = &nb
 		}
 	}
 
-	return int64(pos), nil
-}
-
-func (ra *roaringArray) readFrom(stream io.Reader) (int64, error) {
-	pos := 0
-	var cookie uint32
-	err := binary.Read(stream, binary.LittleEndian, &cookie)
-	if err != nil {
-		return 0, fmt.Errorf("error in roaringArray.readFrom: could not read initial cookie: %s", err)
-	}
-	pos += 4
-	var size uint32
-	haveRunContainers := false
-	var isRun *bitmapContainer
-	if cookie&0x0000FFFF == serialCookie {
-		haveRunContainers = true
-		size = uint32(uint16(cookie>>16) + 1)
-		bytesToRead := (int(size) + 7) / 8
-		numwords := (bytesToRead + 7) / 8
-		by := make([]byte, bytesToRead, numwords*8)
-		nr, err := io.ReadFull(stream, by)
-		if err != nil {
-			return 8 + int64(nr), fmt.Errorf("error in readFrom: could not read the "+
-				"runContainer bit flags of length %v bytes: %v", bytesToRead, err)
-		}
-		pos += bytesToRead
-		by = by[:cap(by)]
-		isRun = newBitmapContainer()
-		for i := 0; i < numwords; i++ {
-			isRun.bitmap[i] = binary.LittleEndian.Uint64(by)
-			by = by[8:]
-		}
-	} else if cookie == serialCookieNoRunContainer {
-		err = binary.Read(stream, binary.LittleEndian, &size)
-		if err != nil {
-			return 0, fmt.Errorf("error in roaringArray.readFrom: when reading size, got: %s", err)
-		}
-		pos += 4
-	} else {
-		return 0, fmt.Errorf("error in roaringArray.readFrom: did not find expected serialCookie in header")
-	}
-	if size > (1 << 16) {
-		return 0, fmt.Errorf("It is logically impossible to have more than (1<<16) containers.")
-	}
-	// descriptive header
-	keycard := make([]uint16, 2*size, 2*size)
-	err = binary.Read(stream, binary.LittleEndian, keycard)
-	if err != nil {
-		return 0, err
-	}
-	pos += 2 * 2 * int(size)
-	// offset header
-	if !haveRunContainers || size >= noOffsetThreshold {
-		io.CopyN(ioutil.Discard, stream, 4*int64(size)) // we never skip ahead so this data can be ignored
-		pos += 4 * int(size)
-	}
-	for i := uint32(0); i < size; i++ {
-		key := int(keycard[2*i])
-		card := int(keycard[2*i+1]) + 1
-		if haveRunContainers && isRun.contains(uint16(i)) {
-			nb := newRunContainer16()
-			nr, err := nb.readFrom(stream)
-			if err != nil {
-				return 0, err
-			}
-			pos += nr
-			ra.appendContainer(uint16(key), nb, false)
-		} else if card > arrayDefaultMaxSize {
-			nb := newBitmapContainer()
-			nr, err := nb.readFrom(stream)
-			if err != nil {
-				return 0, err
-			}
-			nb.cardinality = card
-			pos += nr
-			ra.appendContainer(keycard[2*i], nb, false)
-		} else {
-			nb := newArrayContainerSize(card)
-			nr, err := nb.readFrom(stream)
-			if err != nil {
-				return 0, err
-			}
-			pos += nr
-			ra.appendContainer(keycard[2*i], nb, false)
-		}
-	}
-	return int64(pos), nil
+	return stream.getReadBytes(), nil
 }
 
 func (ra *roaringArray) hasRunCompression() bool {

vendor/github.com/RoaringBitmap/roaring/roaringarray_gen.go

@@ -8,7 +8,7 @@ import (
 	"github.com/tinylib/msgp/msgp"
 )
 
-// DecodeMsg implements msgp.Decodable
+// Deprecated: DecodeMsg implements msgp.Decodable
 func (z *containerSerz) DecodeMsg(dc *msgp.Reader) (err error) {
 	var field []byte
 	_ = field
@@ -48,7 +48,7 @@ func (z *containerSerz) DecodeMsg(dc *msgp.Reader) (err error) {
 	return
 }
 
-// EncodeMsg implements msgp.Encodable
+// Deprecated: EncodeMsg implements msgp.Encodable
 func (z *containerSerz) EncodeMsg(en *msgp.Writer) (err error) {
 	// map header, size 2
 	// write "t"
@@ -72,7 +72,7 @@ func (z *containerSerz) EncodeMsg(en *msgp.Writer) (err error) {
 	return
 }
 
-// MarshalMsg implements msgp.Marshaler
+// Deprecated: MarshalMsg implements msgp.Marshaler
 func (z *containerSerz) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
 	// map header, size 2
@@ -88,7 +88,7 @@ func (z *containerSerz) MarshalMsg(b []byte) (o []byte, err error) {
 	return
 }
 
-// UnmarshalMsg implements msgp.Unmarshaler
+// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
 func (z *containerSerz) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	var field []byte
 	_ = field
@@ -129,13 +129,13 @@ func (z *containerSerz) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	return
 }
 
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *containerSerz) Msgsize() (s int) {
 	s = 1 + 2 + msgp.Uint8Size + 2 + z.r.Msgsize()
 	return
 }
 
-// DecodeMsg implements msgp.Decodable
+// Deprecated: DecodeMsg implements msgp.Decodable
 func (z *contype) DecodeMsg(dc *msgp.Reader) (err error) {
 	{
 		var zajw uint8
@@ -148,7 +148,7 @@ func (z *contype) DecodeMsg(dc *msgp.Reader) (err error) {
 	return
 }
 
-// EncodeMsg implements msgp.Encodable
+// Deprecated: EncodeMsg implements msgp.Encodable
 func (z contype) EncodeMsg(en *msgp.Writer) (err error) {
 	err = en.WriteUint8(uint8(z))
 	if err != nil {
@@ -157,14 +157,14 @@ func (z contype) EncodeMsg(en *msgp.Writer) (err error) {
 	return
 }
 
-// MarshalMsg implements msgp.Marshaler
+// Deprecated: MarshalMsg implements msgp.Marshaler
 func (z contype) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
 	o = msgp.AppendUint8(o, uint8(z))
 	return
 }
 
-// UnmarshalMsg implements msgp.Unmarshaler
+// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
 func (z *contype) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	{
 		var zwht uint8
@@ -178,13 +178,13 @@ func (z *contype) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	return
 }
 
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z contype) Msgsize() (s int) {
 	s = msgp.Uint8Size
 	return
 }
 
-// DecodeMsg implements msgp.Decodable
+// Deprecated: DecodeMsg implements msgp.Decodable
 func (z *roaringArray) DecodeMsg(dc *msgp.Reader) (err error) {
 	var field []byte
 	_ = field
@@ -295,7 +295,7 @@ func (z *roaringArray) DecodeMsg(dc *msgp.Reader) (err error) {
 	return
 }
 
-// EncodeMsg implements msgp.Encodable
+// Deprecated: EncodeMsg implements msgp.Encodable
 func (z *roaringArray) EncodeMsg(en *msgp.Writer) (err error) {
 	// map header, size 4
 	// write "keys"
@@ -370,7 +370,7 @@ func (z *roaringArray) EncodeMsg(en *msgp.Writer) (err error) {
 	return
 }
 
-// MarshalMsg implements msgp.Marshaler
+// Deprecated: MarshalMsg implements msgp.Marshaler
 func (z *roaringArray) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
 	// map header, size 4
@@ -407,7 +407,7 @@ func (z *roaringArray) MarshalMsg(b []byte) (o []byte, err error) {
 	return
 }
 
-// UnmarshalMsg implements msgp.Unmarshaler
+// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
 func (z *roaringArray) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	var field []byte
 	_ = field
@@ -519,7 +519,7 @@ func (z *roaringArray) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	return
 }
 
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *roaringArray) Msgsize() (s int) {
 	s = 1 + 5 + msgp.ArrayHeaderSize + (len(z.keys) * (msgp.Uint16Size)) + 16 + msgp.ArrayHeaderSize + (len(z.needCopyOnWrite) * (msgp.BoolSize)) + 12 + msgp.BoolSize + 8 + msgp.ArrayHeaderSize
 	for zxhx := range z.conserz {

vendor/github.com/RoaringBitmap/roaring/runcontainer_gen.go

@@ -6,7 +6,7 @@ package roaring
 
 import "github.com/tinylib/msgp/msgp"
 
-// DecodeMsg implements msgp.Decodable
+// Deprecated: DecodeMsg implements msgp.Decodable
 func (z *addHelper16) DecodeMsg(dc *msgp.Reader) (err error) {
 	var field []byte
 	_ = field
@@ -169,7 +169,7 @@ func (z *addHelper16) DecodeMsg(dc *msgp.Reader) (err error) {
 	return
 }
 
-// EncodeMsg implements msgp.Encodable
+// Deprecated: EncodeMsg implements msgp.Encodable
 func (z *addHelper16) EncodeMsg(en *msgp.Writer) (err error) {
 	// map header, size 5
 	// write "runstart"
@@ -284,7 +284,7 @@ func (z *addHelper16) EncodeMsg(en *msgp.Writer) (err error) {
 	return
 }
 
-// MarshalMsg implements msgp.Marshaler
+// Deprecated: MarshalMsg implements msgp.Marshaler
 func (z *addHelper16) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
 	// map header, size 5
@@ -334,7 +334,7 @@ func (z *addHelper16) MarshalMsg(b []byte) (o []byte, err error) {
 	return
 }
 
-// UnmarshalMsg implements msgp.Unmarshaler
+// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
 func (z *addHelper16) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	var field []byte
 	_ = field
@@ -498,7 +498,7 @@ func (z *addHelper16) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	return
 }
 
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *addHelper16) Msgsize() (s int) {
 	s = 1 + 9 + msgp.Uint16Size + 7 + msgp.Uint16Size + 14 + msgp.Uint16Size + 2 + msgp.ArrayHeaderSize + (len(z.m) * (12 + msgp.Uint16Size + msgp.Uint16Size)) + 3
 	if z.rc == nil {
@@ -509,7 +509,7 @@ func (z *addHelper16) Msgsize() (s int) {
 	return
 }
 
-// DecodeMsg implements msgp.Decodable
+// Deprecated: DecodeMsg implements msgp.Decodable
 func (z *interval16) DecodeMsg(dc *msgp.Reader) (err error) {
 	var field []byte
 	_ = field
@@ -546,7 +546,7 @@ func (z *interval16) DecodeMsg(dc *msgp.Reader) (err error) {
 	return
 }
 
-// EncodeMsg implements msgp.Encodable
+// Deprecated: EncodeMsg implements msgp.Encodable
 func (z interval16) EncodeMsg(en *msgp.Writer) (err error) {
 	// map header, size 2
 	// write "start"
@@ -570,7 +570,7 @@ func (z interval16) EncodeMsg(en *msgp.Writer) (err error) {
 	return
 }
 
-// MarshalMsg implements msgp.Marshaler
+// Deprecated: MarshalMsg implements msgp.Marshaler
 func (z interval16) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
 	// map header, size 2
@@ -583,7 +583,7 @@ func (z interval16) MarshalMsg(b []byte) (o []byte, err error) {
 	return
 }
 
-// UnmarshalMsg implements msgp.Unmarshaler
+// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
 func (z *interval16) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	var field []byte
 	_ = field
@@ -621,13 +621,13 @@ func (z *interval16) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	return
 }
 
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z interval16) Msgsize() (s int) {
 	s = 1 + 6 + msgp.Uint16Size + 5 + msgp.Uint16Size
 	return
 }
 
-// DecodeMsg implements msgp.Decodable
+// Deprecated: DecodeMsg implements msgp.Decodable
 func (z *runContainer16) DecodeMsg(dc *msgp.Reader) (err error) {
 	var field []byte
 	_ = field
@@ -701,7 +701,7 @@ func (z *runContainer16) DecodeMsg(dc *msgp.Reader) (err error) {
 	return
 }
 
-// EncodeMsg implements msgp.Encodable
+// Deprecated: EncodeMsg implements msgp.Encodable
 func (z *runContainer16) EncodeMsg(en *msgp.Writer) (err error) {
 	// map header, size 2
 	// write "iv"
@@ -746,7 +746,7 @@ func (z *runContainer16) EncodeMsg(en *msgp.Writer) (err error) {
 	return
 }
 
-// MarshalMsg implements msgp.Marshaler
+// Deprecated: MarshalMsg implements msgp.Marshaler
 func (z *runContainer16) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
 	// map header, size 2
@@ -768,7 +768,7 @@ func (z *runContainer16) MarshalMsg(b []byte) (o []byte, err error) {
 	return
 }
 
-// UnmarshalMsg implements msgp.Unmarshaler
+// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
 func (z *runContainer16) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	var field []byte
 	_ = field
@@ -843,13 +843,13 @@ func (z *runContainer16) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	return
 }
 
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *runContainer16) Msgsize() (s int) {
 	s = 1 + 3 + msgp.ArrayHeaderSize + (len(z.iv) * (12 + msgp.Uint16Size + msgp.Uint16Size)) + 5 + msgp.Int64Size
 	return
 }
 
-// DecodeMsg implements msgp.Decodable
+// Deprecated: DecodeMsg implements msgp.Decodable
 func (z *runIterator16) DecodeMsg(dc *msgp.Reader) (err error) {
 	var field []byte
 	_ = field
@@ -891,11 +891,6 @@ func (z *runIterator16) DecodeMsg(dc *msgp.Reader) (err error) {
 			if err != nil {
 				return
 			}
-		case "curSeq":
-			z.curSeq, err = dc.ReadInt64()
-			if err != nil {
-				return
-			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -906,11 +901,11 @@ func (z *runIterator16) DecodeMsg(dc *msgp.Reader) (err error) {
 	return
 }
 
-// EncodeMsg implements msgp.Encodable
+// Deprecated: EncodeMsg implements msgp.Encodable
 func (z *runIterator16) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 4
+	// map header, size 3
 	// write "rc"
-	err = en.Append(0x84, 0xa2, 0x72, 0x63)
+	err = en.Append(0x83, 0xa2, 0x72, 0x63)
 	if err != nil {
 		return err
 	}
@@ -943,24 +938,15 @@ func (z *runIterator16) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	// write "curSeq"
-	err = en.Append(0xa6, 0x63, 0x75, 0x72, 0x53, 0x65, 0x71)
-	if err != nil {
-		return err
-	}
-	err = en.WriteInt64(z.curSeq)
-	if err != nil {
-		return
-	}
 	return
 }
 
-// MarshalMsg implements msgp.Marshaler
+// Deprecated: MarshalMsg implements msgp.Marshaler
 func (z *runIterator16) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 4
+	// map header, size 3
 	// string "rc"
-	o = append(o, 0x84, 0xa2, 0x72, 0x63)
+	o = append(o, 0x83, 0xa2, 0x72, 0x63)
 	if z.rc == nil {
 		o = msgp.AppendNil(o)
 	} else {
@@ -975,13 +961,10 @@ func (z *runIterator16) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "curPosInIndex"
 	o = append(o, 0xad, 0x63, 0x75, 0x72, 0x50, 0x6f, 0x73, 0x49, 0x6e, 0x49, 0x6e, 0x64, 0x65, 0x78)
 	o = msgp.AppendUint16(o, z.curPosInIndex)
-	// string "curSeq"
-	o = append(o, 0xa6, 0x63, 0x75, 0x72, 0x53, 0x65, 0x71)
-	o = msgp.AppendInt64(o, z.curSeq)
 	return
 }
 
-// UnmarshalMsg implements msgp.Unmarshaler
+// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
 func (z *runIterator16) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	var field []byte
 	_ = field
@@ -1023,11 +1006,6 @@ func (z *runIterator16) UnmarshalMsg(bts []byte) (o []byte, err error) {
 			if err != nil {
 				return
 			}
-		case "curSeq":
-			z.curSeq, bts, err = msgp.ReadInt64Bytes(bts)
-			if err != nil {
-				return
-			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -1039,7 +1017,7 @@ func (z *runIterator16) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	return
 }
 
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *runIterator16) Msgsize() (s int) {
 	s = 1 + 3
 	if z.rc == nil {
@@ -1047,11 +1025,11 @@ func (z *runIterator16) Msgsize() (s int) {
 	} else {
 		s += z.rc.Msgsize()
 	}
-	s += 9 + msgp.Int64Size + 14 + msgp.Uint16Size + 7 + msgp.Int64Size
+	s += 9 + msgp.Int64Size + 14 + msgp.Uint16Size
 	return
 }
 
-// DecodeMsg implements msgp.Decodable
+// Deprecated: DecodeMsg implements msgp.Decodable
 func (z *uint16Slice) DecodeMsg(dc *msgp.Reader) (err error) {
 	var zjpj uint32
 	zjpj, err = dc.ReadArrayHeader()
@@ -1072,7 +1050,7 @@ func (z *uint16Slice) DecodeMsg(dc *msgp.Reader) (err error) {
 	return
 }
 
-// EncodeMsg implements msgp.Encodable
+// Deprecated: EncodeMsg implements msgp.Encodable
 func (z uint16Slice) EncodeMsg(en *msgp.Writer) (err error) {
 	err = en.WriteArrayHeader(uint32(len(z)))
 	if err != nil {
@@ -1087,7 +1065,7 @@ func (z uint16Slice) EncodeMsg(en *msgp.Writer) (err error) {
 	return
 }
 
-// MarshalMsg implements msgp.Marshaler
+// Deprecated: MarshalMsg implements msgp.Marshaler
 func (z uint16Slice) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
 	o = msgp.AppendArrayHeader(o, uint32(len(z)))
@@ -1097,7 +1075,7 @@ func (z uint16Slice) MarshalMsg(b []byte) (o []byte, err error) {
 	return
 }
 
-// UnmarshalMsg implements msgp.Unmarshaler
+// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
 func (z *uint16Slice) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	var zgmo uint32
 	zgmo, bts, err = msgp.ReadArrayHeaderBytes(bts)
@@ -1119,7 +1097,7 @@ func (z *uint16Slice) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	return
 }
 
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z uint16Slice) Msgsize() (s int) {
 	s = msgp.ArrayHeaderSize + (len(z) * (msgp.Uint16Size))
 	return

vendor/github.com/RoaringBitmap/roaring/serialization.go

@@ -2,8 +2,6 @@ package roaring
 
 import (
 	"encoding/binary"
-	"errors"
-	"fmt"
 	"io"
 
 	"github.com/tinylib/msgp/msgp"
@@ -22,14 +20,6 @@ func (b *runContainer16) writeTo(stream io.Writer) (int, error) {
 	return stream.Write(buf)
 }
 
-func (b *runContainer32) writeToMsgpack(stream io.Writer) (int, error) {
-	bts, err := b.MarshalMsg(nil)
-	if err != nil {
-		return 0, err
-	}
-	return stream.Write(bts)
-}
-
 func (b *runContainer16) writeToMsgpack(stream io.Writer) (int, error) {
 	bts, err := b.MarshalMsg(nil)
 	if err != nil {
@@ -38,46 +28,7 @@ func (b *runContainer16) writeToMsgpack(stream io.Writer) (int, error) {
 	return stream.Write(bts)
 }
 
-func (b *runContainer32) readFromMsgpack(stream io.Reader) (int, error) {
-	err := msgp.Decode(stream, b)
-	return 0, err
-}
-
 func (b *runContainer16) readFromMsgpack(stream io.Reader) (int, error) {
 	err := msgp.Decode(stream, b)
 	return 0, err
 }
-
-var errCorruptedStream = errors.New("insufficient/odd number of stored bytes, corrupted stream detected")
-
-func (b *runContainer16) readFrom(stream io.Reader) (int, error) {
-	b.iv = b.iv[:0]
-	b.card = 0
-	var numRuns uint16
-	err := binary.Read(stream, binary.LittleEndian, &numRuns)
-	if err != nil {
-		return 0, err
-	}
-	nr := int(numRuns)
-	encRun := make([]uint16, 2*nr)
-	by := make([]byte, 4*nr)
-	err = binary.Read(stream, binary.LittleEndian, &by)
-	if err != nil {
-		return 0, err
-	}
-	for i := range encRun {
-		if len(by) < 2 {
-			return 0, errCorruptedStream
-		}
-		encRun[i] = binary.LittleEndian.Uint16(by)
-		by = by[2:]
-	}
-	for i := 0; i < nr; i++ {
-		if i > 0 && b.iv[i-1].last() >= encRun[i*2] {
-			return 0, fmt.Errorf("error: stored runContainer had runs that were not in sorted order!! (b.iv[i-1=%v].last = %v >= encRun[i=%v] = %v)", i-1, b.iv[i-1].last(), i, encRun[i*2])
-		}
-		b.iv = append(b.iv, interval16{start: encRun[i*2], length: encRun[i*2+1]})
-		b.card += int64(encRun[i*2+1]) + 1
-	}
-	return 0, err
-}

vendor/github.com/RoaringBitmap/roaring/serialization_generic.go

@@ -4,6 +4,7 @@ package roaring
 
 import (
 	"encoding/binary"
+	"errors"
 	"io"
 )
 
@@ -26,6 +27,10 @@ func (b *arrayContainer) readFrom(stream io.Reader) (int, error) {
 }
 
 func (b *bitmapContainer) writeTo(stream io.Writer) (int, error) {
+	if b.cardinality <= arrayDefaultMaxSize {
+		return 0, errors.New("refusing to write bitmap container with cardinality of array container")
+	}
+
 	// Write set
 	buf := make([]byte, 8*len(b.bitmap))
 	for i, v := range b.bitmap {
@@ -69,6 +74,16 @@ func uint64SliceAsByteSlice(slice []uint64) []byte {
 	return by
 }
 
+func uint16SliceAsByteSlice(slice []uint16) []byte {
+	by := make([]byte, len(slice)*2)
+
+	for i, v := range slice {
+		binary.LittleEndian.PutUint16(by[i*2:], v)
+	}
+
+	return by
+}
+
 func byteSliceAsUint16Slice(slice []byte) []uint16 {
 	if len(slice)%2 != 0 {
 		panic("Slice size should be divisible by 2")

vendor/github.com/RoaringBitmap/roaring/serialization_littleendian.go

@@ -3,8 +3,10 @@
 package roaring
 
 import (
+	"errors"
 	"io"
 	"reflect"
+	"runtime"
 	"unsafe"
 )
 
@@ -14,26 +16,13 @@ func (ac *arrayContainer) writeTo(stream io.Writer) (int, error) {
 }
 
 func (bc *bitmapContainer) writeTo(stream io.Writer) (int, error) {
+	if bc.cardinality <= arrayDefaultMaxSize {
+		return 0, errors.New("refusing to write bitmap container with cardinality of array container")
+	}
 	buf := uint64SliceAsByteSlice(bc.bitmap)
 	return stream.Write(buf)
 }
 
-// readFrom reads an arrayContainer from stream.
-// PRE-REQUISITE: you must size the arrayContainer correctly (allocate b.content)
-// *before* you call readFrom. We can't guess the size in the stream
-// by this point.
-func (ac *arrayContainer) readFrom(stream io.Reader) (int, error) {
-	buf := uint16SliceAsByteSlice(ac.content)
-	return io.ReadFull(stream, buf)
-}
-
-func (bc *bitmapContainer) readFrom(stream io.Reader) (int, error) {
-	buf := uint64SliceAsByteSlice(bc.bitmap)
-	n, err := io.ReadFull(stream, buf)
-	bc.computeCardinality()
-	return n, err
-}
-
 func uint64SliceAsByteSlice(slice []uint64) []byte {
 	// make a new slice header
 	header := *(*reflect.SliceHeader)(unsafe.Pointer(&slice))
@@ -42,8 +31,12 @@ func uint64SliceAsByteSlice(slice []uint64) []byte {
 	header.Len *= 8
 	header.Cap *= 8
 
+	// instantiate result and use KeepAlive so data isn't unmapped.
+	result := *(*[]byte)(unsafe.Pointer(&header))
+	runtime.KeepAlive(&slice)
+
 	// return it
-	return *(*[]byte)(unsafe.Pointer(&header))
+	return result
 }
 
 func uint16SliceAsByteSlice(slice []uint16) []byte {
@@ -54,8 +47,12 @@ func uint16SliceAsByteSlice(slice []uint16) []byte {
 	header.Len *= 2
 	header.Cap *= 2
 
+	// instantiate result and use KeepAlive so data isn't unmapped.
+	result := *(*[]byte)(unsafe.Pointer(&header))
+	runtime.KeepAlive(&slice)
+
 	// return it
-	return *(*[]byte)(unsafe.Pointer(&header))
+	return result
 }
 
 func (bc *bitmapContainer) asLittleEndianByteSlice() []byte {
@@ -64,50 +61,74 @@ func (bc *bitmapContainer) asLittleEndianByteSlice() []byte {
 
 // Deserialization code follows
 
-func byteSliceAsUint16Slice(slice []byte) []uint16 {
+////
+// These methods (byteSliceAsUint16Slice,...) do not make copies,
+// they are pointer-based (unsafe). The caller is responsible to
+// ensure that the input slice does not get garbage collected, deleted
+// or modified while you hold the returned slince.
+////
+func byteSliceAsUint16Slice(slice []byte) (result []uint16) { // here we create a new slice holder
 	if len(slice)%2 != 0 {
 		panic("Slice size should be divisible by 2")
 	}
+	// reference: https://go101.org/article/unsafe.html
 
 	// make a new slice header
-	header := *(*reflect.SliceHeader)(unsafe.Pointer(&slice))
+	bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice))
+	rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result))
 
-	// update its capacity and length
-	header.Len /= 2
-	header.Cap /= 2
+	// transfer the data from the given slice to a new variable (our result)
+	rHeader.Data = bHeader.Data
+	rHeader.Len = bHeader.Len / 2
+	rHeader.Cap = bHeader.Cap / 2
 
-	// return it
-	return *(*[]uint16)(unsafe.Pointer(&header))
+	// instantiate result and use KeepAlive so data isn't unmapped.
+	runtime.KeepAlive(&slice) // it is still crucial, GC can free it)
+
+	// return result
+	return
 }
 
-func byteSliceAsUint64Slice(slice []byte) []uint64 {
+func byteSliceAsUint64Slice(slice []byte) (result []uint64) {
 	if len(slice)%8 != 0 {
 		panic("Slice size should be divisible by 8")
 	}
+	// reference: https://go101.org/article/unsafe.html
 
 	// make a new slice header
-	header := *(*reflect.SliceHeader)(unsafe.Pointer(&slice))
+	bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice))
+	rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result))
 
-	// update its capacity and length
-	header.Len /= 8
-	header.Cap /= 8
+	// transfer the data from the given slice to a new variable (our result)
+	rHeader.Data = bHeader.Data
+	rHeader.Len = bHeader.Len / 8
+	rHeader.Cap = bHeader.Cap / 8
 
-	// return it
-	return *(*[]uint64)(unsafe.Pointer(&header))
+	// instantiate result and use KeepAlive so data isn't unmapped.
+	runtime.KeepAlive(&slice) // it is still crucial, GC can free it)
+
+	// return result
+	return
 }
 
-func byteSliceAsInterval16Slice(slice []byte) []interval16 {
+func byteSliceAsInterval16Slice(slice []byte) (result []interval16) {
 	if len(slice)%4 != 0 {
 		panic("Slice size should be divisible by 4")
 	}
+	// reference: https://go101.org/article/unsafe.html
 
 	// make a new slice header
-	header := *(*reflect.SliceHeader)(unsafe.Pointer(&slice))
+	bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice))
+	rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result))
 
-	// update its capacity and length
-	header.Len /= 4
-	header.Cap /= 4
+	// transfer the data from the given slice to a new variable (our result)
+	rHeader.Data = bHeader.Data
+	rHeader.Len = bHeader.Len / 4
+	rHeader.Cap = bHeader.Cap / 4
 
-	// return it
-	return *(*[]interval16)(unsafe.Pointer(&header))
+	// instantiate result and use KeepAlive so data isn't unmapped.
+	runtime.KeepAlive(&slice) // it is still crucial, GC can free it)
+
+	// return result
+	return
 }

vendor/github.com/RoaringBitmap/roaring/setutil.go

@@ -14,6 +14,7 @@ func equal(a, b []uint16) bool {
 
 func difference(set1 []uint16, set2 []uint16, buffer []uint16) int {
 	if 0 == len(set2) {
+		buffer = buffer[:len(set1)]
 		for k := 0; k < len(set1); k++ {
 			buffer[k] = set1[k]
 		}

vendor/github.com/RoaringBitmap/roaring/shortiterator.go

@@ -5,6 +5,12 @@ type shortIterable interface {
 	next() uint16
 }
 
+type shortPeekable interface {
+	shortIterable
+	peekNext() uint16
+	advanceIfNeeded(minval uint16)
+}
+
 type shortIterator struct {
 	slice []uint16
 	loc   int
@@ -19,3 +25,28 @@ func (si *shortIterator) next() uint16 {
 	si.loc++
 	return a
 }
+
+func (si *shortIterator) peekNext() uint16 {
+	return si.slice[si.loc]
+}
+
+func (si *shortIterator) advanceIfNeeded(minval uint16) {
+	if si.hasNext() && si.peekNext() < minval {
+		si.loc = advanceUntil(si.slice, si.loc, len(si.slice), minval)
+	}
+}
+
+type reverseIterator struct {
+	slice []uint16
+	loc   int
+}
+
+func (si *reverseIterator) hasNext() bool {
+	return si.loc >= 0
+}
+
+func (si *reverseIterator) next() uint16 {
+	a := si.slice[si.loc]
+	si.loc--
+	return a
+}

vendor/github.com/RoaringBitmap/roaring/util.go

@@ -14,6 +14,17 @@ const (
 	serialCookie               = 12347 // runs, arrays, and bitmaps
 	noOffsetThreshold          = 4
 
+	// MaxUint32 is the largest uint32 value.
+	MaxUint32 = 4294967295
+
+	// MaxRange is One more than the maximum allowed bitmap bit index. For use as an upper
+	// bound for ranges.
+	MaxRange uint64 = MaxUint32 + 1
+
+	// MaxUint16 is the largest 16 bit unsigned int.
+	// This is the largest value an interval16 can store.
+	MaxUint16 = 65535
+
 	// Compute wordSizeInBytes, the size of a word in bytes.
 	_m              = ^uint64(0)
 	_logS           = _m>>8&1 + _m>>16&1 + _m>>32&1
@@ -101,7 +112,7 @@ func highbits(x uint32) uint16 {
 	return uint16(x >> 16)
 }
 func lowbits(x uint32) uint16 {
-	return uint16(x & 0xFFFF)
+	return uint16(x & maxLowBit)
 }
 
 const maxLowBit = 0xFFFF
@@ -114,7 +125,6 @@ func flipBitmapRange(bitmap []uint64, start int, end int) {
 	endword := (end - 1) / 64
 	bitmap[firstword] ^= ^(^uint64(0) << uint(start%64))
 	for i := firstword; i < endword; i++ {
-		//p("flipBitmapRange on i=%v", i)
 		bitmap[i] = ^bitmap[i]
 	}
 	bitmap[endword] ^= ^uint64(0) >> (uint(-end) % 64)
@@ -292,24 +302,3 @@ func minOfUint16(a, b uint16) uint16 {
 	}
 	return b
 }
-
-func maxInt(a, b int) int {
-	if a > b {
-		return a
-	}
-	return b
-}
-
-func maxUint16(a, b uint16) uint16 {
-	if a > b {
-		return a
-	}
-	return b
-}
-
-func minUint16(a, b uint16) uint16 {
-	if a < b {
-		return a
-	}
-	return b
-}

vendor/github.com/andybalholm/brotli/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2009, 2010, 2013-2016 by the Brotli Authors.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

vendor/github.com/andybalholm/brotli/README.md

@@ -0,0 +1,5 @@
+This package is a brotli compressor and decompressor implemented in Go.
+It was translated from the reference implementation (https://github.com/google/brotli)
+with the `c2go` tool at https://github.com/andybalholm/c2go.
+
+I am using it in production with https://github.com/andybalholm/redwood.

vendor/github.com/andybalholm/brotli/backward_references.go

@@ -0,0 +1,177 @@
+package brotli
+
+/* Copyright 2013 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* Function to find backward reference copies. */
+
+func computeDistanceCode(distance uint, max_distance uint, dist_cache []int) uint {
+	if distance <= max_distance {
+		var distance_plus_3 uint = distance + 3
+		var offset0 uint = distance_plus_3 - uint(dist_cache[0])
+		var offset1 uint = distance_plus_3 - uint(dist_cache[1])
+		if distance == uint(dist_cache[0]) {
+			return 0
+		} else if distance == uint(dist_cache[1]) {
+			return 1
+		} else if offset0 < 7 {
+			return (0x9750468 >> (4 * offset0)) & 0xF
+		} else if offset1 < 7 {
+			return (0xFDB1ACE >> (4 * offset1)) & 0xF
+		} else if distance == uint(dist_cache[2]) {
+			return 2
+		} else if distance == uint(dist_cache[3]) {
+			return 3
+		}
+	}
+
+	return distance + numDistanceShortCodes - 1
+}
+
+/* "commands" points to the next output command to write to, "*num_commands" is
+   initially the total amount of commands output by previous
+   CreateBackwardReferences calls, and must be incremented by the amount written
+   by this call. */
+func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) {
+	var max_backward_limit uint = maxBackwardLimit(params.lgwin)
+	var orig_commands []command = commands
+	var insert_length uint = *last_insert_len
+	var pos_end uint = position + num_bytes
+	var store_end uint
+	if num_bytes >= hasher.StoreLookahead() {
+		store_end = position + num_bytes - hasher.StoreLookahead() + 1
+	} else {
+		store_end = position
+	}
+	var random_heuristics_window_size uint = literalSpreeLengthForSparseSearch(params)
+	var apply_random_heuristics uint = position + random_heuristics_window_size
+	var gap uint = 0
+	/* Set maximum distance, see section 9.1. of the spec. */
+
+	const kMinScore uint = scoreBase + 100
+
+	/* For speed up heuristics for random data. */
+
+	/* Minimum score to accept a backward reference. */
+	hasher.PrepareDistanceCache(dist_cache)
+	var sr2 hasherSearchResult
+	var sr hasherSearchResult
+
+	for position+hasher.HashTypeLength() < pos_end {
+		var max_length uint = pos_end - position
+		var max_distance uint = brotli_min_size_t(position, max_backward_limit)
+		sr.len = 0
+		sr.len_code_delta = 0
+		sr.distance = 0
+		sr.score = kMinScore
+		hasher.FindLongestMatch(&params.dictionary, ringbuffer, ringbuffer_mask, dist_cache, position, max_length, max_distance, gap, params.dist.max_distance, &sr)
+		if sr.score > kMinScore {
+			/* Found a match. Let's look for something even better ahead. */
+			var delayed_backward_references_in_row int = 0
+			max_length--
+			for ; ; max_length-- {
+				var cost_diff_lazy uint = 175
+				if params.quality < minQualityForExtensiveReferenceSearch {
+					sr2.len = brotli_min_size_t(sr.len-1, max_length)
+				} else {
+					sr2.len = 0
+				}
+				sr2.len_code_delta = 0
+				sr2.distance = 0
+				sr2.score = kMinScore
+				max_distance = brotli_min_size_t(position+1, max_backward_limit)
+				hasher.FindLongestMatch(&params.dictionary, ringbuffer, ringbuffer_mask, dist_cache, position+1, max_length, max_distance, gap, params.dist.max_distance, &sr2)
+				if sr2.score >= sr.score+cost_diff_lazy {
+					/* Ok, let's just write one byte for now and start a match from the
+					   next byte. */
+					position++
+
+					insert_length++
+					sr = sr2
+					delayed_backward_references_in_row++
+					if delayed_backward_references_in_row < 4 && position+hasher.HashTypeLength() < pos_end {
+						continue
+					}
+				}
+
+				break
+			}
+
+			apply_random_heuristics = position + 2*sr.len + random_heuristics_window_size
+			max_distance = brotli_min_size_t(position, max_backward_limit)
+			{
+				/* The first 16 codes are special short-codes,
+				   and the minimum offset is 1. */
+				var distance_code uint = computeDistanceCode(sr.distance, max_distance+gap, dist_cache)
+				if (sr.distance <= (max_distance + gap)) && distance_code > 0 {
+					dist_cache[3] = dist_cache[2]
+					dist_cache[2] = dist_cache[1]
+					dist_cache[1] = dist_cache[0]
+					dist_cache[0] = int(sr.distance)
+					hasher.PrepareDistanceCache(dist_cache)
+				}
+
+				initCommand(&commands[0], &params.dist, insert_length, sr.len, sr.len_code_delta, distance_code)
+				commands = commands[1:]
+			}
+
+			*num_literals += insert_length
+			insert_length = 0
+			/* Put the hash keys into the table, if there are enough bytes left.
+			   Depending on the hasher implementation, it can push all positions
+			   in the given range or only a subset of them.
+			   Avoid hash poisoning with RLE data. */
+			{
+				var range_start uint = position + 2
+				var range_end uint = brotli_min_size_t(position+sr.len, store_end)
+				if sr.distance < sr.len>>2 {
+					range_start = brotli_min_size_t(range_end, brotli_max_size_t(range_start, position+sr.len-(sr.distance<<2)))
+				}
+
+				hasher.StoreRange(ringbuffer, ringbuffer_mask, range_start, range_end)
+			}
+
+			position += sr.len
+		} else {
+			insert_length++
+			position++
+
+			/* If we have not seen matches for a long time, we can skip some
+			   match lookups. Unsuccessful match lookups are very very expensive
+			   and this kind of a heuristic speeds up compression quite
+			   a lot. */
+			if position > apply_random_heuristics {
+				/* Going through uncompressible data, jump. */
+				if position > apply_random_heuristics+4*random_heuristics_window_size {
+					var kMargin uint = brotli_max_size_t(hasher.StoreLookahead()-1, 4)
+					/* It is quite a long time since we saw a copy, so we assume
+					   that this data is not compressible, and store hashes less
+					   often. Hashes of non compressible data are less likely to
+					   turn out to be useful in the future, too, so we store less of
+					   them to not to flood out the hash table of good compressible
+					   data. */
+
+					var pos_jump uint = brotli_min_size_t(position+16, pos_end-kMargin)
+					for ; position < pos_jump; position += 4 {
+						hasher.Store(ringbuffer, ringbuffer_mask, position)
+						insert_length += 4
+					}
+				} else {
+					var kMargin uint = brotli_max_size_t(hasher.StoreLookahead()-1, 2)
+					var pos_jump uint = brotli_min_size_t(position+8, pos_end-kMargin)
+					for ; position < pos_jump; position += 2 {
+						hasher.Store(ringbuffer, ringbuffer_mask, position)
+						insert_length += 2
+					}
+				}
+			}
+		}
+	}
+
+	insert_length += pos_end - position
+	*last_insert_len = insert_length
+	*num_commands += uint(-cap(commands) + cap(orig_commands))
+}

vendor/github.com/andybalholm/brotli/backward_references_hq.go

@@ -0,0 +1,795 @@
+package brotli
+
+import "math"
+
+type zopfliNode struct {
+	length              uint32
+	distance            uint32
+	dcode_insert_length uint32
+	u                   struct {
+		cost     float32
+		next     uint32
+		shortcut uint32
+	}
+}
+
+const maxEffectiveDistanceAlphabetSize = 544
+
+const kInfinity float32 = 1.7e38 /* ~= 2 ^ 127 */
+
+var kDistanceCacheIndex = []uint32{0, 1, 2, 3, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1}
+
+var kDistanceCacheOffset = []int{0, 0, 0, 0, -1, 1, -2, 2, -3, 3, -1, 1, -2, 2, -3, 3}
+
+func initZopfliNodes(array []zopfliNode, length uint) {
+	var stub zopfliNode
+	var i uint
+	stub.length = 1
+	stub.distance = 0
+	stub.dcode_insert_length = 0
+	stub.u.cost = kInfinity
+	for i = 0; i < length; i++ {
+		array[i] = stub
+	}
+}
+
+func zopfliNodeCopyLength(self *zopfliNode) uint32 {
+	return self.length & 0x1FFFFFF
+}
+
+func zopfliNodeLengthCode(self *zopfliNode) uint32 {
+	var modifier uint32 = self.length >> 25
+	return zopfliNodeCopyLength(self) + 9 - modifier
+}
+
+func zopfliNodeCopyDistance(self *zopfliNode) uint32 {
+	return self.distance
+}
+
+func zopfliNodeDistanceCode(self *zopfliNode) uint32 {
+	var short_code uint32 = self.dcode_insert_length >> 27
+	if short_code == 0 {
+		return zopfliNodeCopyDistance(self) + numDistanceShortCodes - 1
+	} else {
+		return short_code - 1
+	}
+}
+
+func zopfliNodeCommandLength(self *zopfliNode) uint32 {
+	return zopfliNodeCopyLength(self) + (self.dcode_insert_length & 0x7FFFFFF)
+}
+
+/* Histogram based cost model for zopflification. */
+type zopfliCostModel struct {
+	cost_cmd_               [numCommandSymbols]float32
+	cost_dist_              []float32
+	distance_histogram_size uint32
+	literal_costs_          []float32
+	min_cost_cmd_           float32
+	num_bytes_              uint
+}
+
+func initZopfliCostModel(self *zopfliCostModel, dist *distanceParams, num_bytes uint) {
+	var distance_histogram_size uint32 = dist.alphabet_size
+	if distance_histogram_size > maxEffectiveDistanceAlphabetSize {
+		distance_histogram_size = maxEffectiveDistanceAlphabetSize
+	}
+
+	self.num_bytes_ = num_bytes
+	self.literal_costs_ = make([]float32, (num_bytes + 2))
+	self.cost_dist_ = make([]float32, (dist.alphabet_size))
+	self.distance_histogram_size = distance_histogram_size
+}
+
+func cleanupZopfliCostModel(self *zopfliCostModel) {
+	self.literal_costs_ = nil
+	self.cost_dist_ = nil
+}
+
+func setCost(histogram []uint32, histogram_size uint, literal_histogram bool, cost []float32) {
+	var sum uint = 0
+	var missing_symbol_sum uint
+	var log2sum float32
+	var missing_symbol_cost float32
+	var i uint
+	for i = 0; i < histogram_size; i++ {
+		sum += uint(histogram[i])
+	}
+
+	log2sum = float32(fastLog2(sum))
+	missing_symbol_sum = sum
+	if !literal_histogram {
+		for i = 0; i < histogram_size; i++ {
+			if histogram[i] == 0 {
+				missing_symbol_sum++
+			}
+		}
+	}
+
+	missing_symbol_cost = float32(fastLog2(missing_symbol_sum)) + 2
+	for i = 0; i < histogram_size; i++ {
+		if histogram[i] == 0 {
+			cost[i] = missing_symbol_cost
+			continue
+		}
+
+		/* Shannon bits for this symbol. */
+		cost[i] = log2sum - float32(fastLog2(uint(histogram[i])))
+
+		/* Cannot be coded with less than 1 bit */
+		if cost[i] < 1 {
+			cost[i] = 1
+		}
+	}
+}
+
+func zopfliCostModelSetFromCommands(self *zopfliCostModel, position uint, ringbuffer []byte, ringbuffer_mask uint, commands []command, num_commands uint, last_insert_len uint) {
+	var histogram_literal [numLiteralSymbols]uint32
+	var histogram_cmd [numCommandSymbols]uint32
+	var histogram_dist [maxEffectiveDistanceAlphabetSize]uint32
+	var cost_literal [numLiteralSymbols]float32
+	var pos uint = position - last_insert_len
+	var min_cost_cmd float32 = kInfinity
+	var i uint
+	var cost_cmd []float32 = self.cost_cmd_[:]
+	var literal_costs []float32
+
+	histogram_literal = [numLiteralSymbols]uint32{}
+	histogram_cmd = [numCommandSymbols]uint32{}
+	histogram_dist = [maxEffectiveDistanceAlphabetSize]uint32{}
+
+	for i = 0; i < num_commands; i++ {
+		var inslength uint = uint(commands[i].insert_len_)
+		var copylength uint = uint(commandCopyLen(&commands[i]))
+		var distcode uint = uint(commands[i].dist_prefix_) & 0x3FF
+		var cmdcode uint = uint(commands[i].cmd_prefix_)
+		var j uint
+
+		histogram_cmd[cmdcode]++
+		if cmdcode >= 128 {
+			histogram_dist[distcode]++
+		}
+
+		for j = 0; j < inslength; j++ {
+			histogram_literal[ringbuffer[(pos+j)&ringbuffer_mask]]++
+		}
+
+		pos += inslength + copylength
+	}
+
+	setCost(histogram_literal[:], numLiteralSymbols, true, cost_literal[:])
+	setCost(histogram_cmd[:], numCommandSymbols, false, cost_cmd)
+	setCost(histogram_dist[:], uint(self.distance_histogram_size), false, self.cost_dist_)
+
+	for i = 0; i < numCommandSymbols; i++ {
+		min_cost_cmd = brotli_min_float(min_cost_cmd, cost_cmd[i])
+	}
+
+	self.min_cost_cmd_ = min_cost_cmd
+	{
+		literal_costs = self.literal_costs_
+		var literal_carry float32 = 0.0
+		var num_bytes uint = self.num_bytes_
+		literal_costs[0] = 0.0
+		for i = 0; i < num_bytes; i++ {
+			literal_carry += cost_literal[ringbuffer[(position+i)&ringbuffer_mask]]
+			literal_costs[i+1] = literal_costs[i] + literal_carry
+			literal_carry -= literal_costs[i+1] - literal_costs[i]
+		}
+	}
+}
+
+func zopfliCostModelSetFromLiteralCosts(self *zopfliCostModel, position uint, ringbuffer []byte, ringbuffer_mask uint) {
+	var literal_costs []float32 = self.literal_costs_
+	var literal_carry float32 = 0.0
+	var cost_dist []float32 = self.cost_dist_
+	var cost_cmd []float32 = self.cost_cmd_[:]
+	var num_bytes uint = self.num_bytes_
+	var i uint
+	estimateBitCostsForLiterals(position, num_bytes, ringbuffer_mask, ringbuffer, literal_costs[1:])
+	literal_costs[0] = 0.0
+	for i = 0; i < num_bytes; i++ {
+		literal_carry += literal_costs[i+1]
+		literal_costs[i+1] = literal_costs[i] + literal_carry
+		literal_carry -= literal_costs[i+1] - literal_costs[i]
+	}
+
+	for i = 0; i < numCommandSymbols; i++ {
+		cost_cmd[i] = float32(fastLog2(uint(11 + uint32(i))))
+	}
+
+	for i = 0; uint32(i) < self.distance_histogram_size; i++ {
+		cost_dist[i] = float32(fastLog2(uint(20 + uint32(i))))
+	}
+
+	self.min_cost_cmd_ = float32(fastLog2(11))
+}
+
+func zopfliCostModelGetCommandCost(self *zopfliCostModel, cmdcode uint16) float32 {
+	return self.cost_cmd_[cmdcode]
+}
+
+func zopfliCostModelGetDistanceCost(self *zopfliCostModel, distcode uint) float32 {
+	return self.cost_dist_[distcode]
+}
+
+func zopfliCostModelGetLiteralCosts(self *zopfliCostModel, from uint, to uint) float32 {
+	return self.literal_costs_[to] - self.literal_costs_[from]
+}
+
+func zopfliCostModelGetMinCostCmd(self *zopfliCostModel) float32 {
+	return self.min_cost_cmd_
+}
+
+/* REQUIRES: len >= 2, start_pos <= pos */
+/* REQUIRES: cost < kInfinity, nodes[start_pos].cost < kInfinity */
+/* Maintains the "ZopfliNode array invariant". */
+func updateZopfliNode(nodes []zopfliNode, pos uint, start_pos uint, len uint, len_code uint, dist uint, short_code uint, cost float32) {
+	var next *zopfliNode = &nodes[pos+len]
+	next.length = uint32(len | (len+9-len_code)<<25)
+	next.distance = uint32(dist)
+	next.dcode_insert_length = uint32(short_code<<27 | (pos - start_pos))
+	next.u.cost = cost
+}
+
+type posData struct {
+	pos            uint
+	distance_cache [4]int
+	costdiff       float32
+	cost           float32
+}
+
+/* Maintains the smallest 8 cost difference together with their positions */
+type startPosQueue struct {
+	q_   [8]posData
+	idx_ uint
+}
+
+func initStartPosQueue(self *startPosQueue) {
+	self.idx_ = 0
+}
+
+func startPosQueueSize(self *startPosQueue) uint {
+	return brotli_min_size_t(self.idx_, 8)
+}
+
+func startPosQueuePush(self *startPosQueue, posdata *posData) {
+	var offset uint = ^(self.idx_) & 7
+	self.idx_++
+	var len uint = startPosQueueSize(self)
+	var i uint
+	var q []posData = self.q_[:]
+	q[offset] = *posdata
+
+	/* Restore the sorted order. In the list of |len| items at most |len - 1|
+	   adjacent element comparisons / swaps are required. */
+	for i = 1; i < len; i++ {
+		if q[offset&7].costdiff > q[(offset+1)&7].costdiff {
+			var tmp posData = q[offset&7]
+			q[offset&7] = q[(offset+1)&7]
+			q[(offset+1)&7] = tmp
+		}
+
+		offset++
+	}
+}
+
+func startPosQueueAt(self *startPosQueue, k uint) *posData {
+	return &self.q_[(k-self.idx_)&7]
+}
+
+/* Returns the minimum possible copy length that can improve the cost of any */
+/* future position. */
+func computeMinimumCopyLength(start_cost float32, nodes []zopfliNode, num_bytes uint, pos uint) uint {
+	var min_cost float32 = start_cost
+	var len uint = 2
+	var next_len_bucket uint = 4
+	/* Compute the minimum possible cost of reaching any future position. */
+
+	var next_len_offset uint = 10
+	for pos+len <= num_bytes && nodes[pos+len].u.cost <= min_cost {
+		/* We already reached (pos + len) with no more cost than the minimum
+		   possible cost of reaching anything from this pos, so there is no point in
+		   looking for lengths <= len. */
+		len++
+
+		if len == next_len_offset {
+			/* We reached the next copy length code bucket, so we add one more
+			   extra bit to the minimum cost. */
+			min_cost += 1.0
+
+			next_len_offset += next_len_bucket
+			next_len_bucket *= 2
+		}
+	}
+
+	return uint(len)
+}
+
+/* REQUIRES: nodes[pos].cost < kInfinity
+   REQUIRES: nodes[0..pos] satisfies that "ZopfliNode array invariant". */
+func computeDistanceShortcut(block_start uint, pos uint, max_backward_limit uint, gap uint, nodes []zopfliNode) uint32 {
+	var clen uint = uint(zopfliNodeCopyLength(&nodes[pos]))
+	var ilen uint = uint(nodes[pos].dcode_insert_length & 0x7FFFFFF)
+	var dist uint = uint(zopfliNodeCopyDistance(&nodes[pos]))
+
+	/* Since |block_start + pos| is the end position of the command, the copy part
+	   starts from |block_start + pos - clen|. Distances that are greater than
+	   this or greater than |max_backward_limit| + |gap| are static dictionary
+	   references, and do not update the last distances.
+	   Also distance code 0 (last distance) does not update the last distances. */
+	if pos == 0 {
+		return 0
+	} else if dist+clen <= block_start+pos+gap && dist <= max_backward_limit+gap && zopfliNodeDistanceCode(&nodes[pos]) > 0 {
+		return uint32(pos)
+	} else {
+		return nodes[pos-clen-ilen].u.shortcut
+	}
+}
+
+/* Fills in dist_cache[0..3] with the last four distances (as defined by
+   Section 4. of the Spec) that would be used at (block_start + pos) if we
+   used the shortest path of commands from block_start, computed from
+   nodes[0..pos]. The last four distances at block_start are in
+   starting_dist_cache[0..3].
+   REQUIRES: nodes[pos].cost < kInfinity
+   REQUIRES: nodes[0..pos] satisfies that "ZopfliNode array invariant". */
+func computeDistanceCache(pos uint, starting_dist_cache []int, nodes []zopfliNode, dist_cache []int) {
+	var idx int = 0
+	var p uint = uint(nodes[pos].u.shortcut)
+	for idx < 4 && p > 0 {
+		var ilen uint = uint(nodes[p].dcode_insert_length & 0x7FFFFFF)
+		var clen uint = uint(zopfliNodeCopyLength(&nodes[p]))
+		var dist uint = uint(zopfliNodeCopyDistance(&nodes[p]))
+		dist_cache[idx] = int(dist)
+		idx++
+
+		/* Because of prerequisite, p >= clen + ilen >= 2. */
+		p = uint(nodes[p-clen-ilen].u.shortcut)
+	}
+
+	for ; idx < 4; idx++ {
+		dist_cache[idx] = starting_dist_cache[0]
+		starting_dist_cache = starting_dist_cache[1:]
+	}
+}
+
+/* Maintains "ZopfliNode array invariant" and pushes node to the queue, if it
+   is eligible. */
+func evaluateNode(block_start uint, pos uint, max_backward_limit uint, gap uint, starting_dist_cache []int, model *zopfliCostModel, queue *startPosQueue, nodes []zopfliNode) {
+	/* Save cost, because ComputeDistanceCache invalidates it. */
+	var node_cost float32 = nodes[pos].u.cost
+	nodes[pos].u.shortcut = computeDistanceShortcut(block_start, pos, max_backward_limit, gap, nodes)
+	if node_cost <= zopfliCostModelGetLiteralCosts(model, 0, pos) {
+		var posdata posData
+		posdata.pos = pos
+		posdata.cost = node_cost
+		posdata.costdiff = node_cost - zopfliCostModelGetLiteralCosts(model, 0, pos)
+		computeDistanceCache(pos, starting_dist_cache, nodes, posdata.distance_cache[:])
+		startPosQueuePush(queue, &posdata)
+	}
+}
+
+/* Returns longest copy length. */
+func updateNodes(num_bytes uint, block_start uint, pos uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, max_backward_limit uint, starting_dist_cache []int, num_matches uint, matches []backwardMatch, model *zopfliCostModel, queue *startPosQueue, nodes []zopfliNode) uint {
+	var cur_ix uint = block_start + pos
+	var cur_ix_masked uint = cur_ix & ringbuffer_mask
+	var max_distance uint = brotli_min_size_t(cur_ix, max_backward_limit)
+	var max_len uint = num_bytes - pos
+	var max_zopfli_len uint = maxZopfliLen(params)
+	var max_iters uint = maxZopfliCandidates(params)
+	var min_len uint
+	var result uint = 0
+	var k uint
+	var gap uint = 0
+
+	evaluateNode(block_start, pos, max_backward_limit, gap, starting_dist_cache, model, queue, nodes)
+	{
+		var posdata *posData = startPosQueueAt(queue, 0)
+		var min_cost float32 = (posdata.cost + zopfliCostModelGetMinCostCmd(model) + zopfliCostModelGetLiteralCosts(model, posdata.pos, pos))
+		min_len = computeMinimumCopyLength(min_cost, nodes, num_bytes, pos)
+	}
+
+	/* Go over the command starting positions in order of increasing cost
+	   difference. */
+	for k = 0; k < max_iters && k < startPosQueueSize(queue); k++ {
+		var posdata *posData = startPosQueueAt(queue, k)
+		var start uint = posdata.pos
+		var inscode uint16 = getInsertLengthCode(pos - start)
+		var start_costdiff float32 = posdata.costdiff
+		var base_cost float32 = start_costdiff + float32(getInsertExtra(inscode)) + zopfliCostModelGetLiteralCosts(model, 0, pos)
+		var best_len uint = min_len - 1
+		var j uint = 0
+		/* Look for last distance matches using the distance cache from this
+		   starting position. */
+		for ; j < numDistanceShortCodes && best_len < max_len; j++ {
+			var idx uint = uint(kDistanceCacheIndex[j])
+			var backward uint = uint(posdata.distance_cache[idx] + kDistanceCacheOffset[j])
+			var prev_ix uint = cur_ix - backward
+			var len uint = 0
+			var continuation byte = ringbuffer[cur_ix_masked+best_len]
+			if cur_ix_masked+best_len > ringbuffer_mask {
+				break
+			}
+
+			if backward > max_distance+gap {
+				/* Word dictionary -> ignore. */
+				continue
+			}
+
+			if backward <= max_distance {
+				/* Regular backward reference. */
+				if prev_ix >= cur_ix {
+					continue
+				}
+
+				prev_ix &= ringbuffer_mask
+				if prev_ix+best_len > ringbuffer_mask || continuation != ringbuffer[prev_ix+best_len] {
+					continue
+				}
+
+				len = findMatchLengthWithLimit(ringbuffer[prev_ix:], ringbuffer[cur_ix_masked:], max_len)
+			} else {
+				continue
+			}
+			{
+				var dist_cost float32 = base_cost + zopfliCostModelGetDistanceCost(model, j)
+				var l uint
+				for l = best_len + 1; l <= len; l++ {
+					var copycode uint16 = getCopyLengthCode(l)
+					var cmdcode uint16 = combineLengthCodes(inscode, copycode, j == 0)
+					var tmp float32
+					if cmdcode < 128 {
+						tmp = base_cost
+					} else {
+						tmp = dist_cost
+					}
+					var cost float32 = tmp + float32(getCopyExtra(copycode)) + zopfliCostModelGetCommandCost(model, cmdcode)
+					if cost < nodes[pos+l].u.cost {
+						updateZopfliNode(nodes, pos, start, l, l, backward, j+1, cost)
+						result = brotli_max_size_t(result, l)
+					}
+
+					best_len = l
+				}
+			}
+		}
+
+		/* At higher iterations look only for new last distance matches, since
+		   looking only for new command start positions with the same distances
+		   does not help much. */
+		if k >= 2 {
+			continue
+		}
+		{
+			/* Loop through all possible copy lengths at this position. */
+			var len uint = min_len
+			for j = 0; j < num_matches; j++ {
+				var match backwardMatch = matches[j]
+				var dist uint = uint(match.distance)
+				var is_dictionary_match bool = (dist > max_distance+gap)
+				var dist_code uint = dist + numDistanceShortCodes - 1
+				var dist_symbol uint16
+				var distextra uint32
+				var distnumextra uint32
+				var dist_cost float32
+				var max_match_len uint
+				/* We already tried all possible last distance matches, so we can use
+				   normal distance code here. */
+				prefixEncodeCopyDistance(dist_code, uint(params.dist.num_direct_distance_codes), uint(params.dist.distance_postfix_bits), &dist_symbol, &distextra)
+
+				distnumextra = uint32(dist_symbol) >> 10
+				dist_cost = base_cost + float32(distnumextra) + zopfliCostModelGetDistanceCost(model, uint(dist_symbol)&0x3FF)
+
+				/* Try all copy lengths up until the maximum copy length corresponding
+				   to this distance. If the distance refers to the static dictionary, or
+				   the maximum length is long enough, try only one maximum length. */
+				max_match_len = backwardMatchLength(&match)
+
+				if len < max_match_len && (is_dictionary_match || max_match_len > max_zopfli_len) {
+					len = max_match_len
+				}
+
+				for ; len <= max_match_len; len++ {
+					var len_code uint
+					if is_dictionary_match {
+						len_code = backwardMatchLengthCode(&match)
+					} else {
+						len_code = len
+					}
+					var copycode uint16 = getCopyLengthCode(len_code)
+					var cmdcode uint16 = combineLengthCodes(inscode, copycode, false)
+					var cost float32 = dist_cost + float32(getCopyExtra(copycode)) + zopfliCostModelGetCommandCost(model, cmdcode)
+					if cost < nodes[pos+len].u.cost {
+						updateZopfliNode(nodes, pos, start, uint(len), len_code, dist, 0, cost)
+						result = brotli_max_size_t(result, uint(len))
+					}
+				}
+			}
+		}
+	}
+
+	return result
+}
+
+func computeShortestPathFromNodes(num_bytes uint, nodes []zopfliNode) uint {
+	var index uint = num_bytes
+	var num_commands uint = 0
+	for nodes[index].dcode_insert_length&0x7FFFFFF == 0 && nodes[index].length == 1 {
+		index--
+	}
+	nodes[index].u.next = math.MaxUint32
+	for index != 0 {
+		var len uint = uint(zopfliNodeCommandLength(&nodes[index]))
+		index -= uint(len)
+		nodes[index].u.next = uint32(len)
+		num_commands++
+	}
+
+	return num_commands
+}
+
+/* REQUIRES: nodes != NULL and len(nodes) >= num_bytes + 1 */
+func zopfliCreateCommands(num_bytes uint, block_start uint, nodes []zopfliNode, dist_cache []int, last_insert_len *uint, params *encoderParams, commands []command, num_literals *uint) {
+	var max_backward_limit uint = maxBackwardLimit(params.lgwin)
+	var pos uint = 0
+	var offset uint32 = nodes[0].u.next
+	var i uint
+	var gap uint = 0
+	for i = 0; offset != math.MaxUint32; i++ {
+		var next *zopfliNode = &nodes[uint32(pos)+offset]
+		var copy_length uint = uint(zopfliNodeCopyLength(next))
+		var insert_length uint = uint(next.dcode_insert_length & 0x7FFFFFF)
+		pos += insert_length
+		offset = next.u.next
+		if i == 0 {
+			insert_length += *last_insert_len
+			*last_insert_len = 0
+		}
+		{
+			var distance uint = uint(zopfliNodeCopyDistance(next))
+			var len_code uint = uint(zopfliNodeLengthCode(next))
+			var max_distance uint = brotli_min_size_t(block_start+pos, max_backward_limit)
+			var is_dictionary bool = (distance > max_distance+gap)
+			var dist_code uint = uint(zopfliNodeDistanceCode(next))
+			initCommand(&commands[i], &params.dist, insert_length, copy_length, int(len_code)-int(copy_length), dist_code)
+
+			if !is_dictionary && dist_code > 0 {
+				dist_cache[3] = dist_cache[2]
+				dist_cache[2] = dist_cache[1]
+				dist_cache[1] = dist_cache[0]
+				dist_cache[0] = int(distance)
+			}
+		}
+
+		*num_literals += insert_length
+		pos += copy_length
+	}
+
+	*last_insert_len += num_bytes - pos
+}
+
+func zopfliIterate(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, gap uint, dist_cache []int, model *zopfliCostModel, num_matches []uint32, matches []backwardMatch, nodes []zopfliNode) uint {
+	var max_backward_limit uint = maxBackwardLimit(params.lgwin)
+	var max_zopfli_len uint = maxZopfliLen(params)
+	var queue startPosQueue
+	var cur_match_pos uint = 0
+	var i uint
+	nodes[0].length = 0
+	nodes[0].u.cost = 0
+	initStartPosQueue(&queue)
+	for i = 0; i+3 < num_bytes; i++ {
+		var skip uint = updateNodes(num_bytes, position, i, ringbuffer, ringbuffer_mask, params, max_backward_limit, dist_cache, uint(num_matches[i]), matches[cur_match_pos:], model, &queue, nodes)
+		if skip < longCopyQuickStep {
+			skip = 0
+		}
+		cur_match_pos += uint(num_matches[i])
+		if num_matches[i] == 1 && backwardMatchLength(&matches[cur_match_pos-1]) > max_zopfli_len {
+			skip = brotli_max_size_t(backwardMatchLength(&matches[cur_match_pos-1]), skip)
+		}
+
+		if skip > 1 {
+			skip--
+			for skip != 0 {
+				i++
+				if i+3 >= num_bytes {
+					break
+				}
+				evaluateNode(position, i, max_backward_limit, gap, dist_cache, model, &queue, nodes)
+				cur_match_pos += uint(num_matches[i])
+				skip--
+			}
+		}
+	}
+
+	return computeShortestPathFromNodes(num_bytes, nodes)
+}
+
+/* Computes the shortest path of commands from position to at most
+   position + num_bytes.
+
+   On return, path->size() is the number of commands found and path[i] is the
+   length of the i-th command (copy length plus insert length).
+   Note that the sum of the lengths of all commands can be less than num_bytes.
+
+   On return, the nodes[0..num_bytes] array will have the following
+   "ZopfliNode array invariant":
+   For each i in [1..num_bytes], if nodes[i].cost < kInfinity, then
+     (1) nodes[i].copy_length() >= 2
+     (2) nodes[i].command_length() <= i and
+     (3) nodes[i - nodes[i].command_length()].cost < kInfinity
+
+ REQUIRES: nodes != nil and len(nodes) >= num_bytes + 1 */
+func zopfliComputeShortestPath(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, dist_cache []int, hasher *h10, nodes []zopfliNode) uint {
+	var max_backward_limit uint = maxBackwardLimit(params.lgwin)
+	var max_zopfli_len uint = maxZopfliLen(params)
+	var model zopfliCostModel
+	var queue startPosQueue
+	var matches [2 * (maxNumMatchesH10 + 64)]backwardMatch
+	var store_end uint
+	if num_bytes >= hasher.StoreLookahead() {
+		store_end = position + num_bytes - hasher.StoreLookahead() + 1
+	} else {
+		store_end = position
+	}
+	var i uint
+	var gap uint = 0
+	var lz_matches_offset uint = 0
+	nodes[0].length = 0
+	nodes[0].u.cost = 0
+	initZopfliCostModel(&model, &params.dist, num_bytes)
+	zopfliCostModelSetFromLiteralCosts(&model, position, ringbuffer, ringbuffer_mask)
+	initStartPosQueue(&queue)
+	for i = 0; i+hasher.HashTypeLength()-1 < num_bytes; i++ {
+		var pos uint = position + i
+		var max_distance uint = brotli_min_size_t(pos, max_backward_limit)
+		var skip uint
+		var num_matches uint
+		num_matches = findAllMatchesH10(hasher, &params.dictionary, ringbuffer, ringbuffer_mask, pos, num_bytes-i, max_distance, gap, params, matches[lz_matches_offset:])
+		if num_matches > 0 && backwardMatchLength(&matches[num_matches-1]) > max_zopfli_len {
+			matches[0] = matches[num_matches-1]
+			num_matches = 1
+		}
+
+		skip = updateNodes(num_bytes, position, i, ringbuffer, ringbuffer_mask, params, max_backward_limit, dist_cache, num_matches, matches[:], &model, &queue, nodes)
+		if skip < longCopyQuickStep {
+			skip = 0
+		}
+		if num_matches == 1 && backwardMatchLength(&matches[0]) > max_zopfli_len {
+			skip = brotli_max_size_t(backwardMatchLength(&matches[0]), skip)
+		}
+
+		if skip > 1 {
+			/* Add the tail of the copy to the hasher. */
+			hasher.StoreRange(ringbuffer, ringbuffer_mask, pos+1, brotli_min_size_t(pos+skip, store_end))
+
+			skip--
+			for skip != 0 {
+				i++
+				if i+hasher.HashTypeLength()-1 >= num_bytes {
+					break
+				}
+				evaluateNode(position, i, max_backward_limit, gap, dist_cache, &model, &queue, nodes)
+				skip--
+			}
+		}
+	}
+
+	cleanupZopfliCostModel(&model)
+	return computeShortestPathFromNodes(num_bytes, nodes)
+}
+
+func createZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher *h10, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) {
+	var nodes []zopfliNode
+	nodes = make([]zopfliNode, (num_bytes + 1))
+	initZopfliNodes(nodes, num_bytes+1)
+	*num_commands += zopfliComputeShortestPath(num_bytes, position, ringbuffer, ringbuffer_mask, params, dist_cache, hasher, nodes)
+	zopfliCreateCommands(num_bytes, position, nodes, dist_cache, last_insert_len, params, commands, num_literals)
+	nodes = nil
+}
+
+func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) {
+	var max_backward_limit uint = maxBackwardLimit(params.lgwin)
+	var num_matches []uint32 = make([]uint32, num_bytes)
+	var matches_size uint = 4 * num_bytes
+	var store_end uint
+	if num_bytes >= hasher.StoreLookahead() {
+		store_end = position + num_bytes - hasher.StoreLookahead() + 1
+	} else {
+		store_end = position
+	}
+	var cur_match_pos uint = 0
+	var i uint
+	var orig_num_literals uint
+	var orig_last_insert_len uint
+	var orig_dist_cache [4]int
+	var orig_num_commands uint
+	var model zopfliCostModel
+	var nodes []zopfliNode
+	var matches []backwardMatch = make([]backwardMatch, matches_size)
+	var gap uint = 0
+	var shadow_matches uint = 0
+	var new_array []backwardMatch
+	for i = 0; i+hasher.HashTypeLength()-1 < num_bytes; i++ {
+		var pos uint = position + i
+		var max_distance uint = brotli_min_size_t(pos, max_backward_limit)
+		var max_length uint = num_bytes - i
+		var num_found_matches uint
+		var cur_match_end uint
+		var j uint
+
+		/* Ensure that we have enough free slots. */
+		if matches_size < cur_match_pos+maxNumMatchesH10+shadow_matches {
+			var new_size uint = matches_size
+			if new_size == 0 {
+				new_size = cur_match_pos + maxNumMatchesH10 + shadow_matches
+			}
+
+			for new_size < cur_match_pos+maxNumMatchesH10+shadow_matches {
+				new_size *= 2
+			}
+
+			new_array = make([]backwardMatch, new_size)
+			if matches_size != 0 {
+				copy(new_array, matches[:matches_size])
+			}
+
+			matches = new_array
+			matches_size = new_size
+		}
+
+		num_found_matches = findAllMatchesH10(hasher.(*h10), &params.dictionary, ringbuffer, ringbuffer_mask, pos, max_length, max_distance, gap, params, matches[cur_match_pos+shadow_matches:])
+		cur_match_end = cur_match_pos + num_found_matches
+		for j = cur_match_pos; j+1 < cur_match_end; j++ {
+			assert(backwardMatchLength(&matches[j]) <= backwardMatchLength(&matches[j+1]))
+		}
+
+		num_matches[i] = uint32(num_found_matches)
+		if num_found_matches > 0 {
+			var match_len uint = backwardMatchLength(&matches[cur_match_end-1])
+			if match_len > maxZopfliLenQuality11 {
+				var skip uint = match_len - 1
+				matches[cur_match_pos] = matches[cur_match_end-1]
+				cur_match_pos++
+				num_matches[i] = 1
+
+				/* Add the tail of the copy to the hasher. */
+				hasher.StoreRange(ringbuffer, ringbuffer_mask, pos+1, brotli_min_size_t(pos+match_len, store_end))
+				var pos uint = i
+				for i := 0; i < int(skip); i++ {
+					num_matches[pos+1:][i] = 0
+				}
+				i += skip
+			} else {
+				cur_match_pos = cur_match_end
+			}
+		}
+	}
+
+	orig_num_literals = *num_literals
+	orig_last_insert_len = *last_insert_len
+	copy(orig_dist_cache[:], dist_cache[:4])
+	orig_num_commands = *num_commands
+	nodes = make([]zopfliNode, (num_bytes + 1))
+	initZopfliCostModel(&model, &params.dist, num_bytes)
+	for i = 0; i < 2; i++ {
+		initZopfliNodes(nodes, num_bytes+1)
+		if i == 0 {
+			zopfliCostModelSetFromLiteralCosts(&model, position, ringbuffer, ringbuffer_mask)
+		} else {
+			zopfliCostModelSetFromCommands(&model, position, ringbuffer, ringbuffer_mask, commands, *num_commands-orig_num_commands, orig_last_insert_len)
+		}
+
+		*num_commands = orig_num_commands
+		*num_literals = orig_num_literals
+		*last_insert_len = orig_last_insert_len
+		copy(dist_cache, orig_dist_cache[:4])
+		*num_commands += zopfliIterate(num_bytes, position, ringbuffer, ringbuffer_mask, params, gap, dist_cache, &model, num_matches, matches, nodes)
+		zopfliCreateCommands(num_bytes, position, nodes, dist_cache, last_insert_len, params, commands, num_literals)
+	}
+
+	cleanupZopfliCostModel(&model)
+	nodes = nil
+	matches = nil
+	num_matches = nil
+}

vendor/github.com/andybalholm/brotli/bit_cost.go

@@ -0,0 +1,436 @@
+package brotli
+
+/* Copyright 2013 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* Functions to estimate the bit cost of Huffman trees. */
+func shannonEntropy(population []uint32, size uint, total *uint) float64 {
+	var sum uint = 0
+	var retval float64 = 0
+	var population_end []uint32 = population[size:]
+	var p uint
+	for -cap(population) < -cap(population_end) {
+		p = uint(population[0])
+		population = population[1:]
+		sum += p
+		retval -= float64(p) * fastLog2(p)
+	}
+
+	if sum != 0 {
+		retval += float64(sum) * fastLog2(sum)
+	}
+	*total = sum
+	return retval
+}
+
+func bitsEntropy(population []uint32, size uint) float64 {
+	var sum uint
+	var retval float64 = shannonEntropy(population, size, &sum)
+	if retval < float64(sum) {
+		/* At least one bit per literal is needed. */
+		retval = float64(sum)
+	}
+
+	return retval
+}
+
+const kOneSymbolHistogramCost float64 = 12
+const kTwoSymbolHistogramCost float64 = 20
+const kThreeSymbolHistogramCost float64 = 28
+const kFourSymbolHistogramCost float64 = 37
+
+func populationCostLiteral(histogram *histogramLiteral) float64 {
+	var data_size uint = histogramDataSizeLiteral()
+	var count int = 0
+	var s [5]uint
+	var bits float64 = 0.0
+	var i uint
+	if histogram.total_count_ == 0 {
+		return kOneSymbolHistogramCost
+	}
+
+	for i = 0; i < data_size; i++ {
+		if histogram.data_[i] > 0 {
+			s[count] = i
+			count++
+			if count > 4 {
+				break
+			}
+		}
+	}
+
+	if count == 1 {
+		return kOneSymbolHistogramCost
+	}
+
+	if count == 2 {
+		return kTwoSymbolHistogramCost + float64(histogram.total_count_)
+	}
+
+	if count == 3 {
+		var histo0 uint32 = histogram.data_[s[0]]
+		var histo1 uint32 = histogram.data_[s[1]]
+		var histo2 uint32 = histogram.data_[s[2]]
+		var histomax uint32 = brotli_max_uint32_t(histo0, brotli_max_uint32_t(histo1, histo2))
+		return kThreeSymbolHistogramCost + 2*(float64(histo0)+float64(histo1)+float64(histo2)) - float64(histomax)
+	}
+
+	if count == 4 {
+		var histo [4]uint32
+		var h23 uint32
+		var histomax uint32
+		for i = 0; i < 4; i++ {
+			histo[i] = histogram.data_[s[i]]
+		}
+
+		/* Sort */
+		for i = 0; i < 4; i++ {
+			var j uint
+			for j = i + 1; j < 4; j++ {
+				if histo[j] > histo[i] {
+					var tmp uint32 = histo[j]
+					histo[j] = histo[i]
+					histo[i] = tmp
+				}
+			}
+		}
+
+		h23 = histo[2] + histo[3]
+		histomax = brotli_max_uint32_t(h23, histo[0])
+		return kFourSymbolHistogramCost + 3*float64(h23) + 2*(float64(histo[0])+float64(histo[1])) - float64(histomax)
+	}
+	{
+		var max_depth uint = 1
+		var depth_histo = [codeLengthCodes]uint32{0}
+		/* In this loop we compute the entropy of the histogram and simultaneously
+		   build a simplified histogram of the code length codes where we use the
+		   zero repeat code 17, but we don't use the non-zero repeat code 16. */
+
+		var log2total float64 = fastLog2(histogram.total_count_)
+		for i = 0; i < data_size; {
+			if histogram.data_[i] > 0 {
+				var log2p float64 = log2total - fastLog2(uint(histogram.data_[i]))
+				/* Compute -log2(P(symbol)) = -log2(count(symbol)/total_count) =
+				   = log2(total_count) - log2(count(symbol)) */
+
+				var depth uint = uint(log2p + 0.5)
+				/* Approximate the bit depth by round(-log2(P(symbol))) */
+				bits += float64(histogram.data_[i]) * log2p
+
+				if depth > 15 {
+					depth = 15
+				}
+
+				if depth > max_depth {
+					max_depth = depth
+				}
+
+				depth_histo[depth]++
+				i++
+			} else {
+				var reps uint32 = 1
+				/* Compute the run length of zeros and add the appropriate number of 0
+				   and 17 code length codes to the code length code histogram. */
+
+				var k uint
+				for k = i + 1; k < data_size && histogram.data_[k] == 0; k++ {
+					reps++
+				}
+
+				i += uint(reps)
+				if i == data_size {
+					/* Don't add any cost for the last zero run, since these are encoded
+					   only implicitly. */
+					break
+				}
+
+				if reps < 3 {
+					depth_histo[0] += reps
+				} else {
+					reps -= 2
+					for reps > 0 {
+						depth_histo[repeatZeroCodeLength]++
+
+						/* Add the 3 extra bits for the 17 code length code. */
+						bits += 3
+
+						reps >>= 3
+					}
+				}
+			}
+		}
+
+		/* Add the estimated encoding cost of the code length code histogram. */
+		bits += float64(18 + 2*max_depth)
+
+		/* Add the entropy of the code length code histogram. */
+		bits += bitsEntropy(depth_histo[:], codeLengthCodes)
+	}
+
+	return bits
+}
+
+func populationCostCommand(histogram *histogramCommand) float64 {
+	var data_size uint = histogramDataSizeCommand()
+	var count int = 0
+	var s [5]uint
+	var bits float64 = 0.0
+	var i uint
+	if histogram.total_count_ == 0 {
+		return kOneSymbolHistogramCost
+	}
+
+	for i = 0; i < data_size; i++ {
+		if histogram.data_[i] > 0 {
+			s[count] = i
+			count++
+			if count > 4 {
+				break
+			}
+		}
+	}
+
+	if count == 1 {
+		return kOneSymbolHistogramCost
+	}
+
+	if count == 2 {
+		return kTwoSymbolHistogramCost + float64(histogram.total_count_)
+	}
+
+	if count == 3 {
+		var histo0 uint32 = histogram.data_[s[0]]
+		var histo1 uint32 = histogram.data_[s[1]]
+		var histo2 uint32 = histogram.data_[s[2]]
+		var histomax uint32 = brotli_max_uint32_t(histo0, brotli_max_uint32_t(histo1, histo2))
+		return kThreeSymbolHistogramCost + 2*(float64(histo0)+float64(histo1)+float64(histo2)) - float64(histomax)
+	}
+
+	if count == 4 {
+		var histo [4]uint32
+		var h23 uint32
+		var histomax uint32
+		for i = 0; i < 4; i++ {
+			histo[i] = histogram.data_[s[i]]
+		}
+
+		/* Sort */
+		for i = 0; i < 4; i++ {
+			var j uint
+			for j = i + 1; j < 4; j++ {
+				if histo[j] > histo[i] {
+					var tmp uint32 = histo[j]
+					histo[j] = histo[i]
+					histo[i] = tmp
+				}
+			}
+		}
+
+		h23 = histo[2] + histo[3]
+		histomax = brotli_max_uint32_t(h23, histo[0])
+		return kFourSymbolHistogramCost + 3*float64(h23) + 2*(float64(histo[0])+float64(histo[1])) - float64(histomax)
+	}
+	{
+		var max_depth uint = 1
+		var depth_histo = [codeLengthCodes]uint32{0}
+		/* In this loop we compute the entropy of the histogram and simultaneously
+		   build a simplified histogram of the code length codes where we use the
+		   zero repeat code 17, but we don't use the non-zero repeat code 16. */
+
+		var log2total float64 = fastLog2(histogram.total_count_)
+		for i = 0; i < data_size; {
+			if histogram.data_[i] > 0 {
+				var log2p float64 = log2total - fastLog2(uint(histogram.data_[i]))
+				/* Compute -log2(P(symbol)) = -log2(count(symbol)/total_count) =
+				   = log2(total_count) - log2(count(symbol)) */
+
+				var depth uint = uint(log2p + 0.5)
+				/* Approximate the bit depth by round(-log2(P(symbol))) */
+				bits += float64(histogram.data_[i]) * log2p
+
+				if depth > 15 {
+					depth = 15
+				}
+
+				if depth > max_depth {
+					max_depth = depth
+				}
+
+				depth_histo[depth]++
+				i++
+			} else {
+				var reps uint32 = 1
+				/* Compute the run length of zeros and add the appropriate number of 0
+				   and 17 code length codes to the code length code histogram. */
+
+				var k uint
+				for k = i + 1; k < data_size && histogram.data_[k] == 0; k++ {
+					reps++
+				}
+
+				i += uint(reps)
+				if i == data_size {
+					/* Don't add any cost for the last zero run, since these are encoded
+					   only implicitly. */
+					break
+				}
+
+				if reps < 3 {
+					depth_histo[0] += reps
+				} else {
+					reps -= 2
+					for reps > 0 {
+						depth_histo[repeatZeroCodeLength]++
+
+						/* Add the 3 extra bits for the 17 code length code. */
+						bits += 3
+
+						reps >>= 3
+					}
+				}
+			}
+		}
+
+		/* Add the estimated encoding cost of the code length code histogram. */
+		bits += float64(18 + 2*max_depth)
+
+		/* Add the entropy of the code length code histogram. */
+		bits += bitsEntropy(depth_histo[:], codeLengthCodes)
+	}
+
+	return bits
+}
+
+func populationCostDistance(histogram *histogramDistance) float64 {
+	var data_size uint = histogramDataSizeDistance()
+	var count int = 0
+	var s [5]uint
+	var bits float64 = 0.0
+	var i uint
+	if histogram.total_count_ == 0 {
+		return kOneSymbolHistogramCost
+	}
+
+	for i = 0; i < data_size; i++ {
+		if histogram.data_[i] > 0 {
+			s[count] = i
+			count++
+			if count > 4 {
+				break
+			}
+		}
+	}
+
+	if count == 1 {
+		return kOneSymbolHistogramCost
+	}
+
+	if count == 2 {
+		return kTwoSymbolHistogramCost + float64(histogram.total_count_)
+	}
+
+	if count == 3 {
+		var histo0 uint32 = histogram.data_[s[0]]
+		var histo1 uint32 = histogram.data_[s[1]]
+		var histo2 uint32 = histogram.data_[s[2]]
+		var histomax uint32 = brotli_max_uint32_t(histo0, brotli_max_uint32_t(histo1, histo2))
+		return kThreeSymbolHistogramCost + 2*(float64(histo0)+float64(histo1)+float64(histo2)) - float64(histomax)
+	}
+
+	if count == 4 {
+		var histo [4]uint32
+		var h23 uint32
+		var histomax uint32
+		for i = 0; i < 4; i++ {
+			histo[i] = histogram.data_[s[i]]
+		}
+
+		/* Sort */
+		for i = 0; i < 4; i++ {
+			var j uint
+			for j = i + 1; j < 4; j++ {
+				if histo[j] > histo[i] {
+					var tmp uint32 = histo[j]
+					histo[j] = histo[i]
+					histo[i] = tmp
+				}
+			}
+		}
+
+		h23 = histo[2] + histo[3]
+		histomax = brotli_max_uint32_t(h23, histo[0])
+		return kFourSymbolHistogramCost + 3*float64(h23) + 2*(float64(histo[0])+float64(histo[1])) - float64(histomax)
+	}
+	{
+		var max_depth uint = 1
+		var depth_histo = [codeLengthCodes]uint32{0}
+		/* In this loop we compute the entropy of the histogram and simultaneously
+		   build a simplified histogram of the code length codes where we use the
+		   zero repeat code 17, but we don't use the non-zero repeat code 16. */
+
+		var log2total float64 = fastLog2(histogram.total_count_)
+		for i = 0; i < data_size; {
+			if histogram.data_[i] > 0 {
+				var log2p float64 = log2total - fastLog2(uint(histogram.data_[i]))
+				/* Compute -log2(P(symbol)) = -log2(count(symbol)/total_count) =
+				   = log2(total_count) - log2(count(symbol)) */
+
+				var depth uint = uint(log2p + 0.5)
+				/* Approximate the bit depth by round(-log2(P(symbol))) */
+				bits += float64(histogram.data_[i]) * log2p
+
+				if depth > 15 {
+					depth = 15
+				}
+
+				if depth > max_depth {
+					max_depth = depth
+				}
+
+				depth_histo[depth]++
+				i++
+			} else {
+				var reps uint32 = 1
+				/* Compute the run length of zeros and add the appropriate number of 0
+				   and 17 code length codes to the code length code histogram. */
+
+				var k uint
+				for k = i + 1; k < data_size && histogram.data_[k] == 0; k++ {
+					reps++
+				}
+
+				i += uint(reps)
+				if i == data_size {
+					/* Don't add any cost for the last zero run, since these are encoded
+					   only implicitly. */
+					break
+				}
+
+				if reps < 3 {
+					depth_histo[0] += reps
+				} else {
+					reps -= 2
+					for reps > 0 {
+						depth_histo[repeatZeroCodeLength]++
+
+						/* Add the 3 extra bits for the 17 code length code. */
+						bits += 3
+
+						reps >>= 3
+					}
+				}
+			}
+		}
+
+		/* Add the estimated encoding cost of the code length code histogram. */
+		bits += float64(18 + 2*max_depth)
+
+		/* Add the entropy of the code length code histogram. */
+		bits += bitsEntropy(depth_histo[:], codeLengthCodes)
+	}
+
+	return bits
+}

vendor/github.com/andybalholm/brotli/bit_reader.go

@@ -0,0 +1,266 @@
+package brotli
+
+import "encoding/binary"
+
+/* Copyright 2013 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* Bit reading helpers */
+
+const shortFillBitWindowRead = (8 >> 1)
+
+var kBitMask = [33]uint32{
+	0x00000000,
+	0x00000001,
+	0x00000003,
+	0x00000007,
+	0x0000000F,
+	0x0000001F,
+	0x0000003F,
+	0x0000007F,
+	0x000000FF,
+	0x000001FF,
+	0x000003FF,
+	0x000007FF,
+	0x00000FFF,
+	0x00001FFF,
+	0x00003FFF,
+	0x00007FFF,
+	0x0000FFFF,
+	0x0001FFFF,
+	0x0003FFFF,
+	0x0007FFFF,
+	0x000FFFFF,
+	0x001FFFFF,
+	0x003FFFFF,
+	0x007FFFFF,
+	0x00FFFFFF,
+	0x01FFFFFF,
+	0x03FFFFFF,
+	0x07FFFFFF,
+	0x0FFFFFFF,
+	0x1FFFFFFF,
+	0x3FFFFFFF,
+	0x7FFFFFFF,
+	0xFFFFFFFF,
+}
+
+func bitMask(n uint32) uint32 {
+	return kBitMask[n]
+}
+
+type bitReader struct {
+	val_      uint64
+	bit_pos_  uint32
+	input     []byte
+	input_len uint
+	byte_pos  uint
+}
+
+type bitReaderState struct {
+	val_      uint64
+	bit_pos_  uint32
+	input     []byte
+	input_len uint
+	byte_pos  uint
+}
+
+/* Initializes the BrotliBitReader fields. */
+
+/* Ensures that accumulator is not empty.
+   May consume up to sizeof(brotli_reg_t) - 1 bytes of input.
+   Returns false if data is required but there is no input available.
+   For BROTLI_ALIGNED_READ this function also prepares bit reader for aligned
+   reading. */
+func bitReaderSaveState(from *bitReader, to *bitReaderState) {
+	to.val_ = from.val_
+	to.bit_pos_ = from.bit_pos_
+	to.input = from.input
+	to.input_len = from.input_len
+	to.byte_pos = from.byte_pos
+}
+
+func bitReaderRestoreState(to *bitReader, from *bitReaderState) {
+	to.val_ = from.val_
+	to.bit_pos_ = from.bit_pos_
+	to.input = from.input
+	to.input_len = from.input_len
+	to.byte_pos = from.byte_pos
+}
+
+func getAvailableBits(br *bitReader) uint32 {
+	return 64 - br.bit_pos_
+}
+
+/* Returns amount of unread bytes the bit reader still has buffered from the
+   BrotliInput, including whole bytes in br->val_. */
+func getRemainingBytes(br *bitReader) uint {
+	return uint(uint32(br.input_len-br.byte_pos) + (getAvailableBits(br) >> 3))
+}
+
+/* Checks if there is at least |num| bytes left in the input ring-buffer
+   (excluding the bits remaining in br->val_). */
+func checkInputAmount(br *bitReader, num uint) bool {
+	return br.input_len-br.byte_pos >= num
+}
+
+/* Guarantees that there are at least |n_bits| + 1 bits in accumulator.
+   Precondition: accumulator contains at least 1 bit.
+   |n_bits| should be in the range [1..24] for regular build. For portable
+   non-64-bit little-endian build only 16 bits are safe to request. */
+func fillBitWindow(br *bitReader, n_bits uint32) {
+	if br.bit_pos_ >= 32 {
+		br.val_ >>= 32
+		br.bit_pos_ ^= 32 /* here same as -= 32 because of the if condition */
+		br.val_ |= (uint64(binary.LittleEndian.Uint32(br.input[br.byte_pos:]))) << 32
+		br.byte_pos += 4
+	}
+}
+
+/* Mostly like BrotliFillBitWindow, but guarantees only 16 bits and reads no
+   more than BROTLI_SHORT_FILL_BIT_WINDOW_READ bytes of input. */
+func fillBitWindow16(br *bitReader) {
+	fillBitWindow(br, 17)
+}
+
+/* Tries to pull one byte of input to accumulator.
+   Returns false if there is no input available. */
+func pullByte(br *bitReader) bool {
+	if br.byte_pos == br.input_len {
+		return false
+	}
+
+	br.val_ >>= 8
+	br.val_ |= (uint64(br.input[br.byte_pos])) << 56
+	br.bit_pos_ -= 8
+	br.byte_pos++
+	return true
+}
+
+/* Returns currently available bits.
+   The number of valid bits could be calculated by BrotliGetAvailableBits. */
+func getBitsUnmasked(br *bitReader) uint64 {
+	return br.val_ >> br.bit_pos_
+}
+
+/* Like BrotliGetBits, but does not mask the result.
+   The result contains at least 16 valid bits. */
+func get16BitsUnmasked(br *bitReader) uint32 {
+	fillBitWindow(br, 16)
+	return uint32(getBitsUnmasked(br))
+}
+
+/* Returns the specified number of bits from |br| without advancing bit
+   position. */
+func getBits(br *bitReader, n_bits uint32) uint32 {
+	fillBitWindow(br, n_bits)
+	return uint32(getBitsUnmasked(br)) & bitMask(n_bits)
+}
+
+/* Tries to peek the specified amount of bits. Returns false, if there
+   is not enough input. */
+func safeGetBits(br *bitReader, n_bits uint32, val *uint32) bool {
+	for getAvailableBits(br) < n_bits {
+		if !pullByte(br) {
+			return false
+		}
+	}
+
+	*val = uint32(getBitsUnmasked(br)) & bitMask(n_bits)
+	return true
+}
+
+/* Advances the bit pos by |n_bits|. */
+func dropBits(br *bitReader, n_bits uint32) {
+	br.bit_pos_ += n_bits
+}
+
+func bitReaderUnload(br *bitReader) {
+	var unused_bytes uint32 = getAvailableBits(br) >> 3
+	var unused_bits uint32 = unused_bytes << 3
+	br.byte_pos -= uint(unused_bytes)
+	if unused_bits == 64 {
+		br.val_ = 0
+	} else {
+		br.val_ <<= unused_bits
+	}
+
+	br.bit_pos_ += unused_bits
+}
+
+/* Reads the specified number of bits from |br| and advances the bit pos.
+   Precondition: accumulator MUST contain at least |n_bits|. */
+func takeBits(br *bitReader, n_bits uint32, val *uint32) {
+	*val = uint32(getBitsUnmasked(br)) & bitMask(n_bits)
+	dropBits(br, n_bits)
+}
+
+/* Reads the specified number of bits from |br| and advances the bit pos.
+   Assumes that there is enough input to perform BrotliFillBitWindow. */
+func readBits(br *bitReader, n_bits uint32) uint32 {
+	var val uint32
+	fillBitWindow(br, n_bits)
+	takeBits(br, n_bits, &val)
+	return val
+}
+
+/* Tries to read the specified amount of bits. Returns false, if there
+   is not enough input. |n_bits| MUST be positive. */
+func safeReadBits(br *bitReader, n_bits uint32, val *uint32) bool {
+	for getAvailableBits(br) < n_bits {
+		if !pullByte(br) {
+			return false
+		}
+	}
+
+	takeBits(br, n_bits, val)
+	return true
+}
+
+/* Advances the bit reader position to the next byte boundary and verifies
+   that any skipped bits are set to zero. */
+func bitReaderJumpToByteBoundary(br *bitReader) bool {
+	var pad_bits_count uint32 = getAvailableBits(br) & 0x7
+	var pad_bits uint32 = 0
+	if pad_bits_count != 0 {
+		takeBits(br, pad_bits_count, &pad_bits)
+	}
+
+	return pad_bits == 0
+}
+
+/* Copies remaining input bytes stored in the bit reader to the output. Value
+   |num| may not be larger than BrotliGetRemainingBytes. The bit reader must be
+   warmed up again after this. */
+func copyBytes(dest []byte, br *bitReader, num uint) {
+	for getAvailableBits(br) >= 8 && num > 0 {
+		dest[0] = byte(getBitsUnmasked(br))
+		dropBits(br, 8)
+		dest = dest[1:]
+		num--
+	}
+
+	copy(dest, br.input[br.byte_pos:][:num])
+	br.byte_pos += num
+}
+
+func initBitReader(br *bitReader) {
+	br.val_ = 0
+	br.bit_pos_ = 64
+}
+
+func warmupBitReader(br *bitReader) bool {
+	/* Fixing alignment after unaligned BrotliFillWindow would result accumulator
+	   overflow. If unalignment is caused by BrotliSafeReadBits, then there is
+	   enough space in accumulator to fix alignment. */
+	if getAvailableBits(br) == 0 {
+		if !pullByte(br) {
+			return false
+		}
+	}
+
+	return true
+}

vendor/github.com/andybalholm/brotli/block_splitter.go

@@ -0,0 +1,153 @@
+package brotli
+
+/* Copyright 2013 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* Block split point selection utilities. */
+
+type blockSplit struct {
+	num_types          uint
+	num_blocks         uint
+	types              []byte
+	lengths            []uint32
+	types_alloc_size   uint
+	lengths_alloc_size uint
+}
+
+const (
+	kMaxLiteralHistograms        uint    = 100
+	kMaxCommandHistograms        uint    = 50
+	kLiteralBlockSwitchCost      float64 = 28.1
+	kCommandBlockSwitchCost      float64 = 13.5
+	kDistanceBlockSwitchCost     float64 = 14.6
+	kLiteralStrideLength         uint    = 70
+	kCommandStrideLength         uint    = 40
+	kSymbolsPerLiteralHistogram  uint    = 544
+	kSymbolsPerCommandHistogram  uint    = 530
+	kSymbolsPerDistanceHistogram uint    = 544
+	kMinLengthForBlockSplitting  uint    = 128
+	kIterMulForRefining          uint    = 2
+	kMinItersForRefining         uint    = 100
+)
+
+func countLiterals(cmds []command, num_commands uint) uint {
+	var total_length uint = 0
+	/* Count how many we have. */
+
+	var i uint
+	for i = 0; i < num_commands; i++ {
+		total_length += uint(cmds[i].insert_len_)
+	}
+
+	return total_length
+}
+
+func copyLiteralsToByteArray(cmds []command, num_commands uint, data []byte, offset uint, mask uint, literals []byte) {
+	var pos uint = 0
+	var from_pos uint = offset & mask
+	var i uint
+	for i = 0; i < num_commands; i++ {
+		var insert_len uint = uint(cmds[i].insert_len_)
+		if from_pos+insert_len > mask {
+			var head_size uint = mask + 1 - from_pos
+			copy(literals[pos:], data[from_pos:][:head_size])
+			from_pos = 0
+			pos += head_size
+			insert_len -= head_size
+		}
+
+		if insert_len > 0 {
+			copy(literals[pos:], data[from_pos:][:insert_len])
+			pos += insert_len
+		}
+
+		from_pos = uint((uint32(from_pos+insert_len) + commandCopyLen(&cmds[i])) & uint32(mask))
+	}
+}
+
+func myRand(seed *uint32) uint32 {
+	/* Initial seed should be 7. In this case, loop length is (1 << 29). */
+	*seed *= 16807
+
+	return *seed
+}
+
+func bitCost(count uint) float64 {
+	if count == 0 {
+		return -2.0
+	} else {
+		return fastLog2(count)
+	}
+}
+
+const histogramsPerBatch = 64
+
+const clustersPerBatch = 16
+
+func initBlockSplit(self *blockSplit) {
+	self.num_types = 0
+	self.num_blocks = 0
+	self.types = nil
+	self.lengths = nil
+	self.types_alloc_size = 0
+	self.lengths_alloc_size = 0
+}
+
+func destroyBlockSplit(self *blockSplit) {
+	self.types = nil
+	self.lengths = nil
+}
+
+func splitBlock(cmds []command, num_commands uint, data []byte, pos uint, mask uint, params *encoderParams, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit) {
+	{
+		var literals_count uint = countLiterals(cmds, num_commands)
+		var literals []byte = make([]byte, literals_count)
+
+		/* Create a continuous array of literals. */
+		copyLiteralsToByteArray(cmds, num_commands, data, pos, mask, literals)
+
+		/* Create the block split on the array of literals.
+		   Literal histograms have alphabet size 256. */
+		splitByteVectorLiteral(literals, literals_count, kSymbolsPerLiteralHistogram, kMaxLiteralHistograms, kLiteralStrideLength, kLiteralBlockSwitchCost, params, literal_split)
+
+		literals = nil
+	}
+	{
+		var insert_and_copy_codes []uint16 = make([]uint16, num_commands)
+		/* Compute prefix codes for commands. */
+
+		var i uint
+		for i = 0; i < num_commands; i++ {
+			insert_and_copy_codes[i] = cmds[i].cmd_prefix_
+		}
+
+		/* Create the block split on the array of command prefixes. */
+		splitByteVectorCommand(insert_and_copy_codes, num_commands, kSymbolsPerCommandHistogram, kMaxCommandHistograms, kCommandStrideLength, kCommandBlockSwitchCost, params, insert_and_copy_split)
+
+		/* TODO: reuse for distances? */
+
+		insert_and_copy_codes = nil
+	}
+	{
+		var distance_prefixes []uint16 = make([]uint16, num_commands)
+		var j uint = 0
+		/* Create a continuous array of distance prefixes. */
+
+		var i uint
+		for i = 0; i < num_commands; i++ {
+			var cmd *command = &cmds[i]
+			if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 {
+				distance_prefixes[j] = cmd.dist_prefix_ & 0x3FF
+				j++
+			}
+		}
+
+		/* Create the block split on the array of distance prefixes. */
+		splitByteVectorDistance(distance_prefixes, j, kSymbolsPerDistanceHistogram, kMaxCommandHistograms, kCommandStrideLength, kDistanceBlockSwitchCost, params, dist_split)
+
+		distance_prefixes = nil
+	}
+}

vendor/github.com/andybalholm/brotli/block_splitter_command.go

@@ -0,0 +1,433 @@
+package brotli
+
+import "math"
+
+/* Copyright 2013 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+func initialEntropyCodesCommand(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramCommand) {
+	var seed uint32 = 7
+	var block_length uint = length / num_histograms
+	var i uint
+	clearHistogramsCommand(histograms, num_histograms)
+	for i = 0; i < num_histograms; i++ {
+		var pos uint = length * i / num_histograms
+		if i != 0 {
+			pos += uint(myRand(&seed) % uint32(block_length))
+		}
+
+		if pos+stride >= length {
+			pos = length - stride - 1
+		}
+
+		histogramAddVectorCommand(&histograms[i], data[pos:], stride)
+	}
+}
+
+func randomSampleCommand(seed *uint32, data []uint16, length uint, stride uint, sample *histogramCommand) {
+	var pos uint = 0
+	if stride >= length {
+		stride = length
+	} else {
+		pos = uint(myRand(seed) % uint32(length-stride+1))
+	}
+
+	histogramAddVectorCommand(sample, data[pos:], stride)
+}
+
+func refineEntropyCodesCommand(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramCommand) {
+	var iters uint = kIterMulForRefining*length/stride + kMinItersForRefining
+	var seed uint32 = 7
+	var iter uint
+	iters = ((iters + num_histograms - 1) / num_histograms) * num_histograms
+	for iter = 0; iter < iters; iter++ {
+		var sample histogramCommand
+		histogramClearCommand(&sample)
+		randomSampleCommand(&seed, data, length, stride, &sample)
+		histogramAddHistogramCommand(&histograms[iter%num_histograms], &sample)
+	}
+}
+
+/* Assigns a block id from the range [0, num_histograms) to each data element
+   in data[0..length) and fills in block_id[0..length) with the assigned values.
+   Returns the number of blocks, i.e. one plus the number of block switches. */
+func findBlocksCommand(data []uint16, length uint, block_switch_bitcost float64, num_histograms uint, histograms []histogramCommand, insert_cost []float64, cost []float64, switch_signal []byte, block_id []byte) uint {
+	var data_size uint = histogramDataSizeCommand()
+	var bitmaplen uint = (num_histograms + 7) >> 3
+	var num_blocks uint = 1
+	var i uint
+	var j uint
+	assert(num_histograms <= 256)
+	if num_histograms <= 1 {
+		for i = 0; i < length; i++ {
+			block_id[i] = 0
+		}
+
+		return 1
+	}
+
+	for i := 0; i < int(data_size*num_histograms); i++ {
+		insert_cost[i] = 0
+	}
+	for i = 0; i < num_histograms; i++ {
+		insert_cost[i] = fastLog2(uint(uint32(histograms[i].total_count_)))
+	}
+
+	for i = data_size; i != 0; {
+		i--
+		for j = 0; j < num_histograms; j++ {
+			insert_cost[i*num_histograms+j] = insert_cost[j] - bitCost(uint(histograms[j].data_[i]))
+		}
+	}
+
+	for i := 0; i < int(num_histograms); i++ {
+		cost[i] = 0
+	}
+	for i := 0; i < int(length*bitmaplen); i++ {
+		switch_signal[i] = 0
+	}
+
+	/* After each iteration of this loop, cost[k] will contain the difference
+	   between the minimum cost of arriving at the current byte position using
+	   entropy code k, and the minimum cost of arriving at the current byte
+	   position. This difference is capped at the block switch cost, and if it
+	   reaches block switch cost, it means that when we trace back from the last
+	   position, we need to switch here. */
+	for i = 0; i < length; i++ {
+		var byte_ix uint = i
+		var ix uint = byte_ix * bitmaplen
+		var insert_cost_ix uint = uint(data[byte_ix]) * num_histograms
+		var min_cost float64 = 1e99
+		var block_switch_cost float64 = block_switch_bitcost
+		var k uint
+		for k = 0; k < num_histograms; k++ {
+			/* We are coding the symbol in data[byte_ix] with entropy code k. */
+			cost[k] += insert_cost[insert_cost_ix+k]
+
+			if cost[k] < min_cost {
+				min_cost = cost[k]
+				block_id[byte_ix] = byte(k)
+			}
+		}
+
+		/* More blocks for the beginning. */
+		if byte_ix < 2000 {
+			block_switch_cost *= 0.77 + 0.07*float64(byte_ix)/2000
+		}
+
+		for k = 0; k < num_histograms; k++ {
+			cost[k] -= min_cost
+			if cost[k] >= block_switch_cost {
+				var mask byte = byte(1 << (k & 7))
+				cost[k] = block_switch_cost
+				assert(k>>3 < bitmaplen)
+				switch_signal[ix+(k>>3)] |= mask
+				/* Trace back from the last position and switch at the marked places. */
+			}
+		}
+	}
+	{
+		var byte_ix uint = length - 1
+		var ix uint = byte_ix * bitmaplen
+		var cur_id byte = block_id[byte_ix]
+		for byte_ix > 0 {
+			var mask byte = byte(1 << (cur_id & 7))
+			assert(uint(cur_id)>>3 < bitmaplen)
+			byte_ix--
+			ix -= bitmaplen
+			if switch_signal[ix+uint(cur_id>>3)]&mask != 0 {
+				if cur_id != block_id[byte_ix] {
+					cur_id = block_id[byte_ix]
+					num_blocks++
+				}
+			}
+
+			block_id[byte_ix] = cur_id
+		}
+	}
+
+	return num_blocks
+}
+
+var remapBlockIdsCommand_kInvalidId uint16 = 256
+
+func remapBlockIdsCommand(block_ids []byte, length uint, new_id []uint16, num_histograms uint) uint {
+	var next_id uint16 = 0
+	var i uint
+	for i = 0; i < num_histograms; i++ {
+		new_id[i] = remapBlockIdsCommand_kInvalidId
+	}
+
+	for i = 0; i < length; i++ {
+		assert(uint(block_ids[i]) < num_histograms)
+		if new_id[block_ids[i]] == remapBlockIdsCommand_kInvalidId {
+			new_id[block_ids[i]] = next_id
+			next_id++
+		}
+	}
+
+	for i = 0; i < length; i++ {
+		block_ids[i] = byte(new_id[block_ids[i]])
+		assert(uint(block_ids[i]) < num_histograms)
+	}
+
+	assert(uint(next_id) <= num_histograms)
+	return uint(next_id)
+}
+
+func buildBlockHistogramsCommand(data []uint16, length uint, block_ids []byte, num_histograms uint, histograms []histogramCommand) {
+	var i uint
+	clearHistogramsCommand(histograms, num_histograms)
+	for i = 0; i < length; i++ {
+		histogramAddCommand(&histograms[block_ids[i]], uint(data[i]))
+	}
+}
+
+var clusterBlocksCommand_kInvalidIndex uint32 = math.MaxUint32
+
+func clusterBlocksCommand(data []uint16, length uint, num_blocks uint, block_ids []byte, split *blockSplit) {
+	var histogram_symbols []uint32 = make([]uint32, num_blocks)
+	var block_lengths []uint32 = make([]uint32, num_blocks)
+	var expected_num_clusters uint = clustersPerBatch * (num_blocks + histogramsPerBatch - 1) / histogramsPerBatch
+	var all_histograms_size uint = 0
+	var all_histograms_capacity uint = expected_num_clusters
+	var all_histograms []histogramCommand = make([]histogramCommand, all_histograms_capacity)
+	var cluster_size_size uint = 0
+	var cluster_size_capacity uint = expected_num_clusters
+	var cluster_size []uint32 = make([]uint32, cluster_size_capacity)
+	var num_clusters uint = 0
+	var histograms []histogramCommand = make([]histogramCommand, brotli_min_size_t(num_blocks, histogramsPerBatch))
+	var max_num_pairs uint = histogramsPerBatch * histogramsPerBatch / 2
+	var pairs_capacity uint = max_num_pairs + 1
+	var pairs []histogramPair = make([]histogramPair, pairs_capacity)
+	var pos uint = 0
+	var clusters []uint32
+	var num_final_clusters uint
+	var new_index []uint32
+	var i uint
+	var sizes = [histogramsPerBatch]uint32{0}
+	var new_clusters = [histogramsPerBatch]uint32{0}
+	var symbols = [histogramsPerBatch]uint32{0}
+	var remap = [histogramsPerBatch]uint32{0}
+
+	for i := 0; i < int(num_blocks); i++ {
+		block_lengths[i] = 0
+	}
+	{
+		var block_idx uint = 0
+		for i = 0; i < length; i++ {
+			assert(block_idx < num_blocks)
+			block_lengths[block_idx]++
+			if i+1 == length || block_ids[i] != block_ids[i+1] {
+				block_idx++
+			}
+		}
+
+		assert(block_idx == num_blocks)
+	}
+
+	for i = 0; i < num_blocks; i += histogramsPerBatch {
+		var num_to_combine uint = brotli_min_size_t(num_blocks-i, histogramsPerBatch)
+		var num_new_clusters uint
+		var j uint
+		for j = 0; j < num_to_combine; j++ {
+			var k uint
+			histogramClearCommand(&histograms[j])
+			for k = 0; uint32(k) < block_lengths[i+j]; k++ {
+				histogramAddCommand(&histograms[j], uint(data[pos]))
+				pos++
+			}
+
+			histograms[j].bit_cost_ = populationCostCommand(&histograms[j])
+			new_clusters[j] = uint32(j)
+			symbols[j] = uint32(j)
+			sizes[j] = 1
+		}
+
+		num_new_clusters = histogramCombineCommand(histograms, sizes[:], symbols[:], new_clusters[:], []histogramPair(pairs), num_to_combine, num_to_combine, histogramsPerBatch, max_num_pairs)
+		if all_histograms_capacity < (all_histograms_size + num_new_clusters) {
+			var _new_size uint
+			if all_histograms_capacity == 0 {
+				_new_size = all_histograms_size + num_new_clusters
+			} else {
+				_new_size = all_histograms_capacity
+			}
+			var new_array []histogramCommand
+			for _new_size < (all_histograms_size + num_new_clusters) {
+				_new_size *= 2
+			}
+			new_array = make([]histogramCommand, _new_size)
+			if all_histograms_capacity != 0 {
+				copy(new_array, all_histograms[:all_histograms_capacity])
+			}
+
+			all_histograms = new_array
+			all_histograms_capacity = _new_size
+		}
+
+		brotli_ensure_capacity_uint32_t(&cluster_size, &cluster_size_capacity, cluster_size_size+num_new_clusters)
+		for j = 0; j < num_new_clusters; j++ {
+			all_histograms[all_histograms_size] = histograms[new_clusters[j]]
+			all_histograms_size++
+			cluster_size[cluster_size_size] = sizes[new_clusters[j]]
+			cluster_size_size++
+			remap[new_clusters[j]] = uint32(j)
+		}
+
+		for j = 0; j < num_to_combine; j++ {
+			histogram_symbols[i+j] = uint32(num_clusters) + remap[symbols[j]]
+		}
+
+		num_clusters += num_new_clusters
+		assert(num_clusters == cluster_size_size)
+		assert(num_clusters == all_histograms_size)
+	}
+
+	histograms = nil
+
+	max_num_pairs = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters)
+	if pairs_capacity < max_num_pairs+1 {
+		pairs = nil
+		pairs = make([]histogramPair, (max_num_pairs + 1))
+	}
+
+	clusters = make([]uint32, num_clusters)
+	for i = 0; i < num_clusters; i++ {
+		clusters[i] = uint32(i)
+	}
+
+	num_final_clusters = histogramCombineCommand(all_histograms, cluster_size, histogram_symbols, clusters, pairs, num_clusters, num_blocks, maxNumberOfBlockTypes, max_num_pairs)
+	pairs = nil
+	cluster_size = nil
+
+	new_index = make([]uint32, num_clusters)
+	for i = 0; i < num_clusters; i++ {
+		new_index[i] = clusterBlocksCommand_kInvalidIndex
+	}
+	pos = 0
+	{
+		var next_index uint32 = 0
+		for i = 0; i < num_blocks; i++ {
+			var histo histogramCommand
+			var j uint
+			var best_out uint32
+			var best_bits float64
+			histogramClearCommand(&histo)
+			for j = 0; uint32(j) < block_lengths[i]; j++ {
+				histogramAddCommand(&histo, uint(data[pos]))
+				pos++
+			}
+
+			if i == 0 {
+				best_out = histogram_symbols[0]
+			} else {
+				best_out = histogram_symbols[i-1]
+			}
+			best_bits = histogramBitCostDistanceCommand(&histo, &all_histograms[best_out])
+			for j = 0; j < num_final_clusters; j++ {
+				var cur_bits float64 = histogramBitCostDistanceCommand(&histo, &all_histograms[clusters[j]])
+				if cur_bits < best_bits {
+					best_bits = cur_bits
+					best_out = clusters[j]
+				}
+			}
+
+			histogram_symbols[i] = best_out
+			if new_index[best_out] == clusterBlocksCommand_kInvalidIndex {
+				new_index[best_out] = next_index
+				next_index++
+			}
+		}
+	}
+
+	clusters = nil
+	all_histograms = nil
+	brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, num_blocks)
+	brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, num_blocks)
+	{
+		var cur_length uint32 = 0
+		var block_idx uint = 0
+		var max_type byte = 0
+		for i = 0; i < num_blocks; i++ {
+			cur_length += block_lengths[i]
+			if i+1 == num_blocks || histogram_symbols[i] != histogram_symbols[i+1] {
+				var id byte = byte(new_index[histogram_symbols[i]])
+				split.types[block_idx] = id
+				split.lengths[block_idx] = cur_length
+				max_type = brotli_max_uint8_t(max_type, id)
+				cur_length = 0
+				block_idx++
+			}
+		}
+
+		split.num_blocks = block_idx
+		split.num_types = uint(max_type) + 1
+	}
+
+	new_index = nil
+	block_lengths = nil
+	histogram_symbols = nil
+}
+
+func splitByteVectorCommand(data []uint16, length uint, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) {
+	var data_size uint = histogramDataSizeCommand()
+	var num_histograms uint = length/literals_per_histogram + 1
+	var histograms []histogramCommand
+	if num_histograms > max_histograms {
+		num_histograms = max_histograms
+	}
+
+	if length == 0 {
+		split.num_types = 1
+		return
+	} else if length < kMinLengthForBlockSplitting {
+		brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, split.num_blocks+1)
+		brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, split.num_blocks+1)
+		split.num_types = 1
+		split.types[split.num_blocks] = 0
+		split.lengths[split.num_blocks] = uint32(length)
+		split.num_blocks++
+		return
+	}
+
+	histograms = make([]histogramCommand, num_histograms)
+
+	/* Find good entropy codes. */
+	initialEntropyCodesCommand(data, length, sampling_stride_length, num_histograms, histograms)
+
+	refineEntropyCodesCommand(data, length, sampling_stride_length, num_histograms, histograms)
+	{
+		var block_ids []byte = make([]byte, length)
+		var num_blocks uint = 0
+		var bitmaplen uint = (num_histograms + 7) >> 3
+		var insert_cost []float64 = make([]float64, (data_size * num_histograms))
+		var cost []float64 = make([]float64, num_histograms)
+		var switch_signal []byte = make([]byte, (length * bitmaplen))
+		var new_id []uint16 = make([]uint16, num_histograms)
+		var iters uint
+		if params.quality < hqZopflificationQuality {
+			iters = 3
+		} else {
+			iters = 10
+		}
+		/* Find a good path through literals with the good entropy codes. */
+
+		var i uint
+		for i = 0; i < iters; i++ {
+			num_blocks = findBlocksCommand(data, length, block_switch_cost, num_histograms, histograms, insert_cost, cost, switch_signal, block_ids)
+			num_histograms = remapBlockIdsCommand(block_ids, length, new_id, num_histograms)
+			buildBlockHistogramsCommand(data, length, block_ids, num_histograms, histograms)
+		}
+
+		insert_cost = nil
+		cost = nil
+		switch_signal = nil
+		new_id = nil
+		histograms = nil
+		clusterBlocksCommand(data, length, num_blocks, block_ids, split)
+		block_ids = nil
+	}
+}

vendor/github.com/andybalholm/brotli/block_splitter_distance.go

@@ -0,0 +1,433 @@
+package brotli
+
+import "math"
+
+/* Copyright 2013 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+func initialEntropyCodesDistance(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramDistance) {
+	var seed uint32 = 7
+	var block_length uint = length / num_histograms
+	var i uint
+	clearHistogramsDistance(histograms, num_histograms)
+	for i = 0; i < num_histograms; i++ {
+		var pos uint = length * i / num_histograms
+		if i != 0 {
+			pos += uint(myRand(&seed) % uint32(block_length))
+		}
+
+		if pos+stride >= length {
+			pos = length - stride - 1
+		}
+
+		histogramAddVectorDistance(&histograms[i], data[pos:], stride)
+	}
+}
+
+func randomSampleDistance(seed *uint32, data []uint16, length uint, stride uint, sample *histogramDistance) {
+	var pos uint = 0
+	if stride >= length {
+		stride = length
+	} else {
+		pos = uint(myRand(seed) % uint32(length-stride+1))
+	}
+
+	histogramAddVectorDistance(sample, data[pos:], stride)
+}
+
+func refineEntropyCodesDistance(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramDistance) {
+	var iters uint = kIterMulForRefining*length/stride + kMinItersForRefining
+	var seed uint32 = 7
+	var iter uint
+	iters = ((iters + num_histograms - 1) / num_histograms) * num_histograms
+	for iter = 0; iter < iters; iter++ {
+		var sample histogramDistance
+		histogramClearDistance(&sample)
+		randomSampleDistance(&seed, data, length, stride, &sample)
+		histogramAddHistogramDistance(&histograms[iter%num_histograms], &sample)
+	}
+}
+
+/* Assigns a block id from the range [0, num_histograms) to each data element
+   in data[0..length) and fills in block_id[0..length) with the assigned values.
+   Returns the number of blocks, i.e. one plus the number of block switches. */
+func findBlocksDistance(data []uint16, length uint, block_switch_bitcost float64, num_histograms uint, histograms []histogramDistance, insert_cost []float64, cost []float64, switch_signal []byte, block_id []byte) uint {
+	var data_size uint = histogramDataSizeDistance()
+	var bitmaplen uint = (num_histograms + 7) >> 3
+	var num_blocks uint = 1
+	var i uint
+	var j uint
+	assert(num_histograms <= 256)
+	if num_histograms <= 1 {
+		for i = 0; i < length; i++ {
+			block_id[i] = 0
+		}
+
+		return 1
+	}
+
+	for i := 0; i < int(data_size*num_histograms); i++ {
+		insert_cost[i] = 0
+	}
+	for i = 0; i < num_histograms; i++ {
+		insert_cost[i] = fastLog2(uint(uint32(histograms[i].total_count_)))
+	}
+
+	for i = data_size; i != 0; {
+		i--
+		for j = 0; j < num_histograms; j++ {
+			insert_cost[i*num_histograms+j] = insert_cost[j] - bitCost(uint(histograms[j].data_[i]))
+		}
+	}
+
+	for i := 0; i < int(num_histograms); i++ {
+		cost[i] = 0
+	}
+	for i := 0; i < int(length*bitmaplen); i++ {
+		switch_signal[i] = 0
+	}
+
+	/* After each iteration of this loop, cost[k] will contain the difference
+	   between the minimum cost of arriving at the current byte position using
+	   entropy code k, and the minimum cost of arriving at the current byte
+	   position. This difference is capped at the block switch cost, and if it
+	   reaches block switch cost, it means that when we trace back from the last
+	   position, we need to switch here. */
+	for i = 0; i < length; i++ {
+		var byte_ix uint = i
+		var ix uint = byte_ix * bitmaplen
+		var insert_cost_ix uint = uint(data[byte_ix]) * num_histograms
+		var min_cost float64 = 1e99
+		var block_switch_cost float64 = block_switch_bitcost
+		var k uint
+		for k = 0; k < num_histograms; k++ {
+			/* We are coding the symbol in data[byte_ix] with entropy code k. */
+			cost[k] += insert_cost[insert_cost_ix+k]
+
+			if cost[k] < min_cost {
+				min_cost = cost[k]
+				block_id[byte_ix] = byte(k)
+			}
+		}
+
+		/* More blocks for the beginning. */
+		if byte_ix < 2000 {
+			block_switch_cost *= 0.77 + 0.07*float64(byte_ix)/2000
+		}
+
+		for k = 0; k < num_histograms; k++ {
+			cost[k] -= min_cost
+			if cost[k] >= block_switch_cost {
+				var mask byte = byte(1 << (k & 7))
+				cost[k] = block_switch_cost
+				assert(k>>3 < bitmaplen)
+				switch_signal[ix+(k>>3)] |= mask
+				/* Trace back from the last position and switch at the marked places. */
+			}
+		}
+	}
+	{
+		var byte_ix uint = length - 1
+		var ix uint = byte_ix * bitmaplen
+		var cur_id byte = block_id[byte_ix]
+		for byte_ix > 0 {
+			var mask byte = byte(1 << (cur_id & 7))
+			assert(uint(cur_id)>>3 < bitmaplen)
+			byte_ix--
+			ix -= bitmaplen
+			if switch_signal[ix+uint(cur_id>>3)]&mask != 0 {
+				if cur_id != block_id[byte_ix] {
+					cur_id = block_id[byte_ix]
+					num_blocks++
+				}
+			}
+
+			block_id[byte_ix] = cur_id
+		}
+	}
+
+	return num_blocks
+}
+
+var remapBlockIdsDistance_kInvalidId uint16 = 256
+
+func remapBlockIdsDistance(block_ids []byte, length uint, new_id []uint16, num_histograms uint) uint {
+	var next_id uint16 = 0
+	var i uint
+	for i = 0; i < num_histograms; i++ {
+		new_id[i] = remapBlockIdsDistance_kInvalidId
+	}
+
+	for i = 0; i < length; i++ {
+		assert(uint(block_ids[i]) < num_histograms)
+		if new_id[block_ids[i]] == remapBlockIdsDistance_kInvalidId {
+			new_id[block_ids[i]] = next_id
+			next_id++
+		}
+	}
+
+	for i = 0; i < length; i++ {
+		block_ids[i] = byte(new_id[block_ids[i]])
+		assert(uint(block_ids[i]) < num_histograms)
+	}
+
+	assert(uint(next_id) <= num_histograms)
+	return uint(next_id)
+}
+
+func buildBlockHistogramsDistance(data []uint16, length uint, block_ids []byte, num_histograms uint, histograms []histogramDistance) {
+	var i uint
+	clearHistogramsDistance(histograms, num_histograms)
+	for i = 0; i < length; i++ {
+		histogramAddDistance(&histograms[block_ids[i]], uint(data[i]))
+	}
+}
+
+var clusterBlocksDistance_kInvalidIndex uint32 = math.MaxUint32
+
+func clusterBlocksDistance(data []uint16, length uint, num_blocks uint, block_ids []byte, split *blockSplit) {
+	var histogram_symbols []uint32 = make([]uint32, num_blocks)
+	var block_lengths []uint32 = make([]uint32, num_blocks)
+	var expected_num_clusters uint = clustersPerBatch * (num_blocks + histogramsPerBatch - 1) / histogramsPerBatch
+	var all_histograms_size uint = 0
+	var all_histograms_capacity uint = expected_num_clusters
+	var all_histograms []histogramDistance = make([]histogramDistance, all_histograms_capacity)
+	var cluster_size_size uint = 0
+	var cluster_size_capacity uint = expected_num_clusters
+	var cluster_size []uint32 = make([]uint32, cluster_size_capacity)
+	var num_clusters uint = 0
+	var histograms []histogramDistance = make([]histogramDistance, brotli_min_size_t(num_blocks, histogramsPerBatch))
+	var max_num_pairs uint = histogramsPerBatch * histogramsPerBatch / 2
+	var pairs_capacity uint = max_num_pairs + 1
+	var pairs []histogramPair = make([]histogramPair, pairs_capacity)
+	var pos uint = 0
+	var clusters []uint32
+	var num_final_clusters uint
+	var new_index []uint32
+	var i uint
+	var sizes = [histogramsPerBatch]uint32{0}
+	var new_clusters = [histogramsPerBatch]uint32{0}
+	var symbols = [histogramsPerBatch]uint32{0}
+	var remap = [histogramsPerBatch]uint32{0}
+
+	for i := 0; i < int(num_blocks); i++ {
+		block_lengths[i] = 0
+	}
+	{
+		var block_idx uint = 0
+		for i = 0; i < length; i++ {
+			assert(block_idx < num_blocks)
+			block_lengths[block_idx]++
+			if i+1 == length || block_ids[i] != block_ids[i+1] {
+				block_idx++
+			}
+		}
+
+		assert(block_idx == num_blocks)
+	}
+
+	for i = 0; i < num_blocks; i += histogramsPerBatch {
+		var num_to_combine uint = brotli_min_size_t(num_blocks-i, histogramsPerBatch)
+		var num_new_clusters uint
+		var j uint
+		for j = 0; j < num_to_combine; j++ {
+			var k uint
+			histogramClearDistance(&histograms[j])
+			for k = 0; uint32(k) < block_lengths[i+j]; k++ {
+				histogramAddDistance(&histograms[j], uint(data[pos]))
+				pos++
+			}
+
+			histograms[j].bit_cost_ = populationCostDistance(&histograms[j])
+			new_clusters[j] = uint32(j)
+			symbols[j] = uint32(j)
+			sizes[j] = 1
+		}
+
+		num_new_clusters = histogramCombineDistance(histograms, sizes[:], symbols[:], new_clusters[:], []histogramPair(pairs), num_to_combine, num_to_combine, histogramsPerBatch, max_num_pairs)
+		if all_histograms_capacity < (all_histograms_size + num_new_clusters) {
+			var _new_size uint
+			if all_histograms_capacity == 0 {
+				_new_size = all_histograms_size + num_new_clusters
+			} else {
+				_new_size = all_histograms_capacity
+			}
+			var new_array []histogramDistance
+			for _new_size < (all_histograms_size + num_new_clusters) {
+				_new_size *= 2
+			}
+			new_array = make([]histogramDistance, _new_size)
+			if all_histograms_capacity != 0 {
+				copy(new_array, all_histograms[:all_histograms_capacity])
+			}
+
+			all_histograms = new_array
+			all_histograms_capacity = _new_size
+		}
+
+		brotli_ensure_capacity_uint32_t(&cluster_size, &cluster_size_capacity, cluster_size_size+num_new_clusters)
+		for j = 0; j < num_new_clusters; j++ {
+			all_histograms[all_histograms_size] = histograms[new_clusters[j]]
+			all_histograms_size++
+			cluster_size[cluster_size_size] = sizes[new_clusters[j]]
+			cluster_size_size++
+			remap[new_clusters[j]] = uint32(j)
+		}
+
+		for j = 0; j < num_to_combine; j++ {
+			histogram_symbols[i+j] = uint32(num_clusters) + remap[symbols[j]]
+		}
+
+		num_clusters += num_new_clusters
+		assert(num_clusters == cluster_size_size)
+		assert(num_clusters == all_histograms_size)
+	}
+
+	histograms = nil
+
+	max_num_pairs = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters)
+	if pairs_capacity < max_num_pairs+1 {
+		pairs = nil
+		pairs = make([]histogramPair, (max_num_pairs + 1))
+	}
+
+	clusters = make([]uint32, num_clusters)
+	for i = 0; i < num_clusters; i++ {
+		clusters[i] = uint32(i)
+	}
+
+	num_final_clusters = histogramCombineDistance(all_histograms, cluster_size, histogram_symbols, clusters, pairs, num_clusters, num_blocks, maxNumberOfBlockTypes, max_num_pairs)
+	pairs = nil
+	cluster_size = nil
+
+	new_index = make([]uint32, num_clusters)
+	for i = 0; i < num_clusters; i++ {
+		new_index[i] = clusterBlocksDistance_kInvalidIndex
+	}
+	pos = 0
+	{
+		var next_index uint32 = 0
+		for i = 0; i < num_blocks; i++ {
+			var histo histogramDistance
+			var j uint
+			var best_out uint32
+			var best_bits float64
+			histogramClearDistance(&histo)
+			for j = 0; uint32(j) < block_lengths[i]; j++ {
+				histogramAddDistance(&histo, uint(data[pos]))
+				pos++
+			}
+
+			if i == 0 {
+				best_out = histogram_symbols[0]
+			} else {
+				best_out = histogram_symbols[i-1]
+			}
+			best_bits = histogramBitCostDistanceDistance(&histo, &all_histograms[best_out])
+			for j = 0; j < num_final_clusters; j++ {
+				var cur_bits float64 = histogramBitCostDistanceDistance(&histo, &all_histograms[clusters[j]])
+				if cur_bits < best_bits {
+					best_bits = cur_bits
+					best_out = clusters[j]
+				}
+			}
+
+			histogram_symbols[i] = best_out
+			if new_index[best_out] == clusterBlocksDistance_kInvalidIndex {
+				new_index[best_out] = next_index
+				next_index++
+			}
+		}
+	}
+
+	clusters = nil
+	all_histograms = nil
+	brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, num_blocks)
+	brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, num_blocks)
+	{
+		var cur_length uint32 = 0
+		var block_idx uint = 0
+		var max_type byte = 0
+		for i = 0; i < num_blocks; i++ {
+			cur_length += block_lengths[i]
+			if i+1 == num_blocks || histogram_symbols[i] != histogram_symbols[i+1] {
+				var id byte = byte(new_index[histogram_symbols[i]])
+				split.types[block_idx] = id
+				split.lengths[block_idx] = cur_length
+				max_type = brotli_max_uint8_t(max_type, id)
+				cur_length = 0
+				block_idx++
+			}
+		}
+
+		split.num_blocks = block_idx
+		split.num_types = uint(max_type) + 1
+	}
+
+	new_index = nil
+	block_lengths = nil
+	histogram_symbols = nil
+}
+
+func splitByteVectorDistance(data []uint16, length uint, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) {
+	var data_size uint = histogramDataSizeDistance()
+	var num_histograms uint = length/literals_per_histogram + 1
+	var histograms []histogramDistance
+	if num_histograms > max_histograms {
+		num_histograms = max_histograms
+	}
+
+	if length == 0 {
+		split.num_types = 1
+		return
+	} else if length < kMinLengthForBlockSplitting {
+		brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, split.num_blocks+1)
+		brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, split.num_blocks+1)
+		split.num_types = 1
+		split.types[split.num_blocks] = 0
+		split.lengths[split.num_blocks] = uint32(length)
+		split.num_blocks++
+		return
+	}
+
+	histograms = make([]histogramDistance, num_histograms)
+
+	/* Find good entropy codes. */
+	initialEntropyCodesDistance(data, length, sampling_stride_length, num_histograms, histograms)
+
+	refineEntropyCodesDistance(data, length, sampling_stride_length, num_histograms, histograms)
+	{
+		var block_ids []byte = make([]byte, length)
+		var num_blocks uint = 0
+		var bitmaplen uint = (num_histograms + 7) >> 3
+		var insert_cost []float64 = make([]float64, (data_size * num_histograms))
+		var cost []float64 = make([]float64, num_histograms)
+		var switch_signal []byte = make([]byte, (length * bitmaplen))
+		var new_id []uint16 = make([]uint16, num_histograms)
+		var iters uint
+		if params.quality < hqZopflificationQuality {
+			iters = 3
+		} else {
+			iters = 10
+		}
+		/* Find a good path through literals with the good entropy codes. */
+
+		var i uint
+		for i = 0; i < iters; i++ {
+			num_blocks = findBlocksDistance(data, length, block_switch_cost, num_histograms, histograms, insert_cost, cost, switch_signal, block_ids)
+			num_histograms = remapBlockIdsDistance(block_ids, length, new_id, num_histograms)
+			buildBlockHistogramsDistance(data, length, block_ids, num_histograms, histograms)
+		}
+
+		insert_cost = nil
+		cost = nil
+		switch_signal = nil
+		new_id = nil
+		histograms = nil
+		clusterBlocksDistance(data, length, num_blocks, block_ids, split)
+		block_ids = nil
+	}
+}

vendor/github.com/andybalholm/brotli/block_splitter_literal.go

@@ -0,0 +1,433 @@
+package brotli
+
+import "math"
+
+/* Copyright 2013 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+func initialEntropyCodesLiteral(data []byte, length uint, stride uint, num_histograms uint, histograms []histogramLiteral) {
+	var seed uint32 = 7
+	var block_length uint = length / num_histograms
+	var i uint
+	clearHistogramsLiteral(histograms, num_histograms)
+	for i = 0; i < num_histograms; i++ {
+		var pos uint = length * i / num_histograms
+		if i != 0 {
+			pos += uint(myRand(&seed) % uint32(block_length))
+		}
+
+		if pos+stride >= length {
+			pos = length - stride - 1
+		}
+
+		histogramAddVectorLiteral(&histograms[i], data[pos:], stride)
+	}
+}
+
+func randomSampleLiteral(seed *uint32, data []byte, length uint, stride uint, sample *histogramLiteral) {
+	var pos uint = 0
+	if stride >= length {
+		stride = length
+	} else {
+		pos = uint(myRand(seed) % uint32(length-stride+1))
+	}
+
+	histogramAddVectorLiteral(sample, data[pos:], stride)
+}
+
+func refineEntropyCodesLiteral(data []byte, length uint, stride uint, num_histograms uint, histograms []histogramLiteral) {
+	var iters uint = kIterMulForRefining*length/stride + kMinItersForRefining
+	var seed uint32 = 7
+	var iter uint
+	iters = ((iters + num_histograms - 1) / num_histograms) * num_histograms
+	for iter = 0; iter < iters; iter++ {
+		var sample histogramLiteral
+		histogramClearLiteral(&sample)
+		randomSampleLiteral(&seed, data, length, stride, &sample)
+		histogramAddHistogramLiteral(&histograms[iter%num_histograms], &sample)
+	}
+}
+
+/* Assigns a block id from the range [0, num_histograms) to each data element
+   in data[0..length) and fills in block_id[0..length) with the assigned values.
+   Returns the number of blocks, i.e. one plus the number of block switches. */
+func findBlocksLiteral(data []byte, length uint, block_switch_bitcost float64, num_histograms uint, histograms []histogramLiteral, insert_cost []float64, cost []float64, switch_signal []byte, block_id []byte) uint {
+	var data_size uint = histogramDataSizeLiteral()
+	var bitmaplen uint = (num_histograms + 7) >> 3
+	var num_blocks uint = 1
+	var i uint
+	var j uint
+	assert(num_histograms <= 256)
+	if num_histograms <= 1 {
+		for i = 0; i < length; i++ {
+			block_id[i] = 0
+		}
+
+		return 1
+	}
+
+	for i := 0; i < int(data_size*num_histograms); i++ {
+		insert_cost[i] = 0
+	}
+	for i = 0; i < num_histograms; i++ {
+		insert_cost[i] = fastLog2(uint(uint32(histograms[i].total_count_)))
+	}
+
+	for i = data_size; i != 0; {
+		i--
+		for j = 0; j < num_histograms; j++ {
+			insert_cost[i*num_histograms+j] = insert_cost[j] - bitCost(uint(histograms[j].data_[i]))
+		}
+	}
+
+	for i := 0; i < int(num_histograms); i++ {
+		cost[i] = 0
+	}
+	for i := 0; i < int(length*bitmaplen); i++ {
+		switch_signal[i] = 0
+	}
+
+	/* After each iteration of this loop, cost[k] will contain the difference
+	   between the minimum cost of arriving at the current byte position using
+	   entropy code k, and the minimum cost of arriving at the current byte
+	   position. This difference is capped at the block switch cost, and if it
+	   reaches block switch cost, it means that when we trace back from the last
+	   position, we need to switch here. */
+	for i = 0; i < length; i++ {
+		var byte_ix uint = i
+		var ix uint = byte_ix * bitmaplen
+		var insert_cost_ix uint = uint(data[byte_ix]) * num_histograms
+		var min_cost float64 = 1e99
+		var block_switch_cost float64 = block_switch_bitcost
+		var k uint
+		for k = 0; k < num_histograms; k++ {
+			/* We are coding the symbol in data[byte_ix] with entropy code k. */
+			cost[k] += insert_cost[insert_cost_ix+k]
+
+			if cost[k] < min_cost {
+				min_cost = cost[k]
+				block_id[byte_ix] = byte(k)
+			}
+		}
+
+		/* More blocks for the beginning. */
+		if byte_ix < 2000 {
+			block_switch_cost *= 0.77 + 0.07*float64(byte_ix)/2000
+		}
+
+		for k = 0; k < num_histograms; k++ {
+			cost[k] -= min_cost
+			if cost[k] >= block_switch_cost {
+				var mask byte = byte(1 << (k & 7))
+				cost[k] = block_switch_cost
+				assert(k>>3 < bitmaplen)
+				switch_signal[ix+(k>>3)] |= mask
+				/* Trace back from the last position and switch at the marked places. */
+			}
+		}
+	}
+	{
+		var byte_ix uint = length - 1
+		var ix uint = byte_ix * bitmaplen
+		var cur_id byte = block_id[byte_ix]
+		for byte_ix > 0 {
+			var mask byte = byte(1 << (cur_id & 7))
+			assert(uint(cur_id)>>3 < bitmaplen)
+			byte_ix--
+			ix -= bitmaplen
+			if switch_signal[ix+uint(cur_id>>3)]&mask != 0 {
+				if cur_id != block_id[byte_ix] {
+					cur_id = block_id[byte_ix]
+					num_blocks++
+				}
+			}
+
+			block_id[byte_ix] = cur_id
+		}
+	}
+
+	return num_blocks
+}
+
+var remapBlockIdsLiteral_kInvalidId uint16 = 256
+
+func remapBlockIdsLiteral(block_ids []byte, length uint, new_id []uint16, num_histograms uint) uint {
+	var next_id uint16 = 0
+	var i uint
+	for i = 0; i < num_histograms; i++ {
+		new_id[i] = remapBlockIdsLiteral_kInvalidId
+	}
+
+	for i = 0; i < length; i++ {
+		assert(uint(block_ids[i]) < num_histograms)
+		if new_id[block_ids[i]] == remapBlockIdsLiteral_kInvalidId {
+			new_id[block_ids[i]] = next_id
+			next_id++
+		}
+	}
+
+	for i = 0; i < length; i++ {
+		block_ids[i] = byte(new_id[block_ids[i]])
+		assert(uint(block_ids[i]) < num_histograms)
+	}
+
+	assert(uint(next_id) <= num_histograms)
+	return uint(next_id)
+}
+
+func buildBlockHistogramsLiteral(data []byte, length uint, block_ids []byte, num_histograms uint, histograms []histogramLiteral) {
+	var i uint
+	clearHistogramsLiteral(histograms, num_histograms)
+	for i = 0; i < length; i++ {
+		histogramAddLiteral(&histograms[block_ids[i]], uint(data[i]))
+	}
+}
+
+var clusterBlocksLiteral_kInvalidIndex uint32 = math.MaxUint32
+
+func clusterBlocksLiteral(data []byte, length uint, num_blocks uint, block_ids []byte, split *blockSplit) {
+	var histogram_symbols []uint32 = make([]uint32, num_blocks)
+	var block_lengths []uint32 = make([]uint32, num_blocks)
+	var expected_num_clusters uint = clustersPerBatch * (num_blocks + histogramsPerBatch - 1) / histogramsPerBatch
+	var all_histograms_size uint = 0
+	var all_histograms_capacity uint = expected_num_clusters
+	var all_histograms []histogramLiteral = make([]histogramLiteral, all_histograms_capacity)
+	var cluster_size_size uint = 0
+	var cluster_size_capacity uint = expected_num_clusters
+	var cluster_size []uint32 = make([]uint32, cluster_size_capacity)
+	var num_clusters uint = 0
+	var histograms []histogramLiteral = make([]histogramLiteral, brotli_min_size_t(num_blocks, histogramsPerBatch))
+	var max_num_pairs uint = histogramsPerBatch * histogramsPerBatch / 2
+	var pairs_capacity uint = max_num_pairs + 1
+	var pairs []histogramPair = make([]histogramPair, pairs_capacity)
+	var pos uint = 0
+	var clusters []uint32
+	var num_final_clusters uint
+	var new_index []uint32
+	var i uint
+	var sizes = [histogramsPerBatch]uint32{0}
+	var new_clusters = [histogramsPerBatch]uint32{0}
+	var symbols = [histogramsPerBatch]uint32{0}
+	var remap = [histogramsPerBatch]uint32{0}
+
+	for i := 0; i < int(num_blocks); i++ {
+		block_lengths[i] = 0
+	}
+	{
+		var block_idx uint = 0
+		for i = 0; i < length; i++ {
+			assert(block_idx < num_blocks)
+			block_lengths[block_idx]++
+			if i+1 == length || block_ids[i] != block_ids[i+1] {
+				block_idx++
+			}
+		}
+
+		assert(block_idx == num_blocks)
+	}
+
+	for i = 0; i < num_blocks; i += histogramsPerBatch {
+		var num_to_combine uint = brotli_min_size_t(num_blocks-i, histogramsPerBatch)
+		var num_new_clusters uint
+		var j uint
+		for j = 0; j < num_to_combine; j++ {
+			var k uint
+			histogramClearLiteral(&histograms[j])
+			for k = 0; uint32(k) < block_lengths[i+j]; k++ {
+				histogramAddLiteral(&histograms[j], uint(data[pos]))
+				pos++
+			}
+
+			histograms[j].bit_cost_ = populationCostLiteral(&histograms[j])
+			new_clusters[j] = uint32(j)
+			symbols[j] = uint32(j)
+			sizes[j] = 1
+		}
+
+		num_new_clusters = histogramCombineLiteral(histograms, sizes[:], symbols[:], new_clusters[:], []histogramPair(pairs), num_to_combine, num_to_combine, histogramsPerBatch, max_num_pairs)
+		if all_histograms_capacity < (all_histograms_size + num_new_clusters) {
+			var _new_size uint
+			if all_histograms_capacity == 0 {
+				_new_size = all_histograms_size + num_new_clusters
+			} else {
+				_new_size = all_histograms_capacity
+			}
+			var new_array []histogramLiteral
+			for _new_size < (all_histograms_size + num_new_clusters) {
+				_new_size *= 2
+			}
+			new_array = make([]histogramLiteral, _new_size)
+			if all_histograms_capacity != 0 {
+				copy(new_array, all_histograms[:all_histograms_capacity])
+			}
+
+			all_histograms = new_array
+			all_histograms_capacity = _new_size
+		}
+
+		brotli_ensure_capacity_uint32_t(&cluster_size, &cluster_size_capacity, cluster_size_size+num_new_clusters)
+		for j = 0; j < num_new_clusters; j++ {
+			all_histograms[all_histograms_size] = histograms[new_clusters[j]]
+			all_histograms_size++
+			cluster_size[cluster_size_size] = sizes[new_clusters[j]]
+			cluster_size_size++
+			remap[new_clusters[j]] = uint32(j)
+		}
+
+		for j = 0; j < num_to_combine; j++ {
+			histogram_symbols[i+j] = uint32(num_clusters) + remap[symbols[j]]
+		}
+
+		num_clusters += num_new_clusters
+		assert(num_clusters == cluster_size_size)
+		assert(num_clusters == all_histograms_size)
+	}
+
+	histograms = nil
+
+	max_num_pairs = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters)
+	if pairs_capacity < max_num_pairs+1 {
+		pairs = nil
+		pairs = make([]histogramPair, (max_num_pairs + 1))
+	}
+
+	clusters = make([]uint32, num_clusters)
+	for i = 0; i < num_clusters; i++ {
+		clusters[i] = uint32(i)
+	}
+
+	num_final_clusters = histogramCombineLiteral(all_histograms, cluster_size, histogram_symbols, clusters, pairs, num_clusters, num_blocks, maxNumberOfBlockTypes, max_num_pairs)
+	pairs = nil
+	cluster_size = nil
+
+	new_index = make([]uint32, num_clusters)
+	for i = 0; i < num_clusters; i++ {
+		new_index[i] = clusterBlocksLiteral_kInvalidIndex
+	}
+	pos = 0
+	{
+		var next_index uint32 = 0
+		for i = 0; i < num_blocks; i++ {
+			var histo histogramLiteral
+			var j uint
+			var best_out uint32
+			var best_bits float64
+			histogramClearLiteral(&histo)
+			for j = 0; uint32(j) < block_lengths[i]; j++ {
+				histogramAddLiteral(&histo, uint(data[pos]))
+				pos++
+			}
+
+			if i == 0 {
+				best_out = histogram_symbols[0]
+			} else {
+				best_out = histogram_symbols[i-1]
+			}
+			best_bits = histogramBitCostDistanceLiteral(&histo, &all_histograms[best_out])
+			for j = 0; j < num_final_clusters; j++ {
+				var cur_bits float64 = histogramBitCostDistanceLiteral(&histo, &all_histograms[clusters[j]])
+				if cur_bits < best_bits {
+					best_bits = cur_bits
+					best_out = clusters[j]
+				}
+			}
+
+			histogram_symbols[i] = best_out
+			if new_index[best_out] == clusterBlocksLiteral_kInvalidIndex {
+				new_index[best_out] = next_index
+				next_index++
+			}
+		}
+	}
+
+	clusters = nil
+	all_histograms = nil
+	brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, num_blocks)
+	brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, num_blocks)
+	{
+		var cur_length uint32 = 0
+		var block_idx uint = 0
+		var max_type byte = 0
+		for i = 0; i < num_blocks; i++ {
+			cur_length += block_lengths[i]
+			if i+1 == num_blocks || histogram_symbols[i] != histogram_symbols[i+1] {
+				var id byte = byte(new_index[histogram_symbols[i]])
+				split.types[block_idx] = id
+				split.lengths[block_idx] = cur_length
+				max_type = brotli_max_uint8_t(max_type, id)
+				cur_length = 0
+				block_idx++
+			}
+		}
+
+		split.num_blocks = block_idx
+		split.num_types = uint(max_type) + 1
+	}
+
+	new_index = nil
+	block_lengths = nil
+	histogram_symbols = nil
+}
+
+func splitByteVectorLiteral(data []byte, length uint, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) {
+	var data_size uint = histogramDataSizeLiteral()
+	var num_histograms uint = length/literals_per_histogram + 1
+	var histograms []histogramLiteral
+	if num_histograms > max_histograms {
+		num_histograms = max_histograms
+	}
+
+	if length == 0 {
+		split.num_types = 1
+		return
+	} else if length < kMinLengthForBlockSplitting {
+		brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, split.num_blocks+1)
+		brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, split.num_blocks+1)
+		split.num_types = 1
+		split.types[split.num_blocks] = 0
+		split.lengths[split.num_blocks] = uint32(length)
+		split.num_blocks++
+		return
+	}
+
+	histograms = make([]histogramLiteral, num_histograms)
+
+	/* Find good entropy codes. */
+	initialEntropyCodesLiteral(data, length, sampling_stride_length, num_histograms, histograms)
+
+	refineEntropyCodesLiteral(data, length, sampling_stride_length, num_histograms, histograms)
+	{
+		var block_ids []byte = make([]byte, length)
+		var num_blocks uint = 0
+		var bitmaplen uint = (num_histograms + 7) >> 3
+		var insert_cost []float64 = make([]float64, (data_size * num_histograms))
+		var cost []float64 = make([]float64, num_histograms)
+		var switch_signal []byte = make([]byte, (length * bitmaplen))
+		var new_id []uint16 = make([]uint16, num_histograms)
+		var iters uint
+		if params.quality < hqZopflificationQuality {
+			iters = 3
+		} else {
+			iters = 10
+		}
+		/* Find a good path through literals with the good entropy codes. */
+
+		var i uint
+		for i = 0; i < iters; i++ {
+			num_blocks = findBlocksLiteral(data, length, block_switch_cost, num_histograms, histograms, insert_cost, cost, switch_signal, block_ids)
+			num_histograms = remapBlockIdsLiteral(block_ids, length, new_id, num_histograms)
+			buildBlockHistogramsLiteral(data, length, block_ids, num_histograms, histograms)
+		}
+
+		insert_cost = nil
+		cost = nil
+		switch_signal = nil
+		new_id = nil
+		histograms = nil
+		clusterBlocksLiteral(data, length, num_blocks, block_ids, split)
+		block_ids = nil
+	}
+}

vendor/github.com/andybalholm/brotli/cluster.go

@@ -0,0 +1,30 @@
+package brotli
+
+/* Copyright 2013 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* Functions for clustering similar histograms together. */
+
+type histogramPair struct {
+	idx1       uint32
+	idx2       uint32
+	cost_combo float64
+	cost_diff  float64
+}
+
+func histogramPairIsLess(p1 *histogramPair, p2 *histogramPair) bool {
+	if p1.cost_diff != p2.cost_diff {
+		return p1.cost_diff > p2.cost_diff
+	}
+
+	return (p1.idx2 - p1.idx1) > (p2.idx2 - p2.idx1)
+}
+
+/* Returns entropy reduction of the context map when we combine two clusters. */
+func clusterCostDiff(size_a uint, size_b uint) float64 {
+	var size_c uint = size_a + size_b
+	return float64(size_a)*fastLog2(size_a) + float64(size_b)*fastLog2(size_b) - float64(size_c)*fastLog2(size_c)
+}

vendor/github.com/andybalholm/brotli/cluster_command.go

@@ -0,0 +1,326 @@
+package brotli
+
+import "math"
+
+/* Copyright 2013 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* Computes the bit cost reduction by combining out[idx1] and out[idx2] and if
+   it is below a threshold, stores the pair (idx1, idx2) in the *pairs queue. */
+func compareAndPushToQueueCommand(out []histogramCommand, cluster_size []uint32, idx1 uint32, idx2 uint32, max_num_pairs uint, pairs []histogramPair, num_pairs *uint) {
+	var is_good_pair bool = false
+	var p histogramPair
+	p.idx2 = 0
+	p.idx1 = p.idx2
+	p.cost_combo = 0
+	p.cost_diff = p.cost_combo
+	if idx1 == idx2 {
+		return
+	}
+
+	if idx2 < idx1 {
+		var t uint32 = idx2
+		idx2 = idx1
+		idx1 = t
+	}
+
+	p.idx1 = idx1
+	p.idx2 = idx2
+	p.cost_diff = 0.5 * clusterCostDiff(uint(cluster_size[idx1]), uint(cluster_size[idx2]))
+	p.cost_diff -= out[idx1].bit_cost_
+	p.cost_diff -= out[idx2].bit_cost_
+
+	if out[idx1].total_count_ == 0 {
+		p.cost_combo = out[idx2].bit_cost_
+		is_good_pair = true
+	} else if out[idx2].total_count_ == 0 {
+		p.cost_combo = out[idx1].bit_cost_
+		is_good_pair = true
+	} else {
+		var threshold float64
+		if *num_pairs == 0 {
+			threshold = 1e99
+		} else {
+			threshold = brotli_max_double(0.0, pairs[0].cost_diff)
+		}
+		var combo histogramCommand = out[idx1]
+		var cost_combo float64
+		histogramAddHistogramCommand(&combo, &out[idx2])
+		cost_combo = populationCostCommand(&combo)
+		if cost_combo < threshold-p.cost_diff {
+			p.cost_combo = cost_combo
+			is_good_pair = true
+		}
+	}
+
+	if is_good_pair {
+		p.cost_diff += p.cost_combo
+		if *num_pairs > 0 && histogramPairIsLess(&pairs[0], &p) {
+			/* Replace the top of the queue if needed. */
+			if *num_pairs < max_num_pairs {
+				pairs[*num_pairs] = pairs[0]
+				(*num_pairs)++
+			}
+
+			pairs[0] = p
+		} else if *num_pairs < max_num_pairs {
+			pairs[*num_pairs] = p
+			(*num_pairs)++
+		}
+	}
+}
+
+func histogramCombineCommand(out []histogramCommand, cluster_size []uint32, symbols []uint32, clusters []uint32, pairs []histogramPair, num_clusters uint, symbols_size uint, max_clusters uint, max_num_pairs uint) uint {
+	var cost_diff_threshold float64 = 0.0
+	var min_cluster_size uint = 1
+	var num_pairs uint = 0
+	{
+		/* We maintain a vector of histogram pairs, with the property that the pair
+		   with the maximum bit cost reduction is the first. */
+		var idx1 uint
+		for idx1 = 0; idx1 < num_clusters; idx1++ {
+			var idx2 uint
+			for idx2 = idx1 + 1; idx2 < num_clusters; idx2++ {
+				compareAndPushToQueueCommand(out, cluster_size, clusters[idx1], clusters[idx2], max_num_pairs, pairs[0:], &num_pairs)
+			}
+		}
+	}
+
+	for num_clusters > min_cluster_size {
+		var best_idx1 uint32
+		var best_idx2 uint32
+		var i uint
+		if pairs[0].cost_diff >= cost_diff_threshold {
+			cost_diff_threshold = 1e99
+			min_cluster_size = max_clusters
+			continue
+		}
+
+		/* Take the best pair from the top of heap. */
+		best_idx1 = pairs[0].idx1
+
+		best_idx2 = pairs[0].idx2
+		histogramAddHistogramCommand(&out[best_idx1], &out[best_idx2])
+		out[best_idx1].bit_cost_ = pairs[0].cost_combo
+		cluster_size[best_idx1] += cluster_size[best_idx2]
+		for i = 0; i < symbols_size; i++ {
+			if symbols[i] == best_idx2 {
+				symbols[i] = best_idx1
+			}
+		}
+
+		for i = 0; i < num_clusters; i++ {
+			if clusters[i] == best_idx2 {
+				copy(clusters[i:], clusters[i+1:][:num_clusters-i-1])
+				break
+			}
+		}
+
+		num_clusters--
+		{
+			/* Remove pairs intersecting the just combined best pair. */
+			var copy_to_idx uint = 0
+			for i = 0; i < num_pairs; i++ {
+				var p *histogramPair = &pairs[i]
+				if p.idx1 == best_idx1 || p.idx2 == best_idx1 || p.idx1 == best_idx2 || p.idx2 == best_idx2 {
+					/* Remove invalid pair from the queue. */
+					continue
+				}
+
+				if histogramPairIsLess(&pairs[0], p) {
+					/* Replace the top of the queue if needed. */
+					var front histogramPair = pairs[0]
+					pairs[0] = *p
+					pairs[copy_to_idx] = front
+				} else {
+					pairs[copy_to_idx] = *p
+				}
+
+				copy_to_idx++
+			}
+
+			num_pairs = copy_to_idx
+		}
+
+		/* Push new pairs formed with the combined histogram to the heap. */
+		for i = 0; i < num_clusters; i++ {
+			compareAndPushToQueueCommand(out, cluster_size, best_idx1, clusters[i], max_num_pairs, pairs[0:], &num_pairs)
+		}
+	}
+
+	return num_clusters
+}
+
+/* What is the bit cost of moving histogram from cur_symbol to candidate. */
+func histogramBitCostDistanceCommand(histogram *histogramCommand, candidate *histogramCommand) float64 {
+	if histogram.total_count_ == 0 {
+		return 0.0
+	} else {
+		var tmp histogramCommand = *histogram
+		histogramAddHistogramCommand(&tmp, candidate)
+		return populationCostCommand(&tmp) - candidate.bit_cost_
+	}
+}
+
+/* Find the best 'out' histogram for each of the 'in' histograms.
+   When called, clusters[0..num_clusters) contains the unique values from
+   symbols[0..in_size), but this property is not preserved in this function.
+   Note: we assume that out[]->bit_cost_ is already up-to-date. */
+func histogramRemapCommand(in []histogramCommand, in_size uint, clusters []uint32, num_clusters uint, out []histogramCommand, symbols []uint32) {
+	var i uint
+	for i = 0; i < in_size; i++ {
+		var best_out uint32
+		if i == 0 {
+			best_out = symbols[0]
+		} else {
+			best_out = symbols[i-1]
+		}
+		var best_bits float64 = histogramBitCostDistanceCommand(&in[i], &out[best_out])
+		var j uint
+		for j = 0; j < num_clusters; j++ {
+			var cur_bits float64 = histogramBitCostDistanceCommand(&in[i], &out[clusters[j]])
+			if cur_bits < best_bits {
+				best_bits = cur_bits
+				best_out = clusters[j]
+			}
+		}
+
+		symbols[i] = best_out
+	}
+
+	/* Recompute each out based on raw and symbols. */
+	for i = 0; i < num_clusters; i++ {
+		histogramClearCommand(&out[clusters[i]])
+	}
+
+	for i = 0; i < in_size; i++ {
+		histogramAddHistogramCommand(&out[symbols[i]], &in[i])
+	}
+}
+
+/* Reorders elements of the out[0..length) array and changes values in
+   symbols[0..length) array in the following way:
+     * when called, symbols[] contains indexes into out[], and has N unique
+       values (possibly N < length)
+     * on return, symbols'[i] = f(symbols[i]) and
+                  out'[symbols'[i]] = out[symbols[i]], for each 0 <= i < length,
+       where f is a bijection between the range of symbols[] and [0..N), and
+       the first occurrences of values in symbols'[i] come in consecutive
+       increasing order.
+   Returns N, the number of unique values in symbols[]. */
+
+var histogramReindexCommand_kInvalidIndex uint32 = math.MaxUint32
+
+func histogramReindexCommand(out []histogramCommand, symbols []uint32, length uint) uint {
+	var new_index []uint32 = make([]uint32, length)
+	var next_index uint32
+	var tmp []histogramCommand
+	var i uint
+	for i = 0; i < length; i++ {
+		new_index[i] = histogramReindexCommand_kInvalidIndex
+	}
+
+	next_index = 0
+	for i = 0; i < length; i++ {
+		if new_index[symbols[i]] == histogramReindexCommand_kInvalidIndex {
+			new_index[symbols[i]] = next_index
+			next_index++
+		}
+	}
+
+	/* TODO: by using idea of "cycle-sort" we can avoid allocation of
+	   tmp and reduce the number of copying by the factor of 2. */
+	tmp = make([]histogramCommand, next_index)
+
+	next_index = 0
+	for i = 0; i < length; i++ {
+		if new_index[symbols[i]] == next_index {
+			tmp[next_index] = out[symbols[i]]
+			next_index++
+		}
+
+		symbols[i] = new_index[symbols[i]]
+	}
+
+	new_index = nil
+	for i = 0; uint32(i) < next_index; i++ {
+		out[i] = tmp[i]
+	}
+
+	tmp = nil
+	return uint(next_index)
+}
+
+func clusterHistogramsCommand(in []histogramCommand, in_size uint, max_histograms uint, out []histogramCommand, out_size *uint, histogram_symbols []uint32) {
+	var cluster_size []uint32 = make([]uint32, in_size)
+	var clusters []uint32 = make([]uint32, in_size)
+	var num_clusters uint = 0
+	var max_input_histograms uint = 64
+	var pairs_capacity uint = max_input_histograms * max_input_histograms / 2
+	var pairs []histogramPair = make([]histogramPair, (pairs_capacity + 1))
+	var i uint
+
+	/* For the first pass of clustering, we allow all pairs. */
+	for i = 0; i < in_size; i++ {
+		cluster_size[i] = 1
+	}
+
+	for i = 0; i < in_size; i++ {
+		out[i] = in[i]
+		out[i].bit_cost_ = populationCostCommand(&in[i])
+		histogram_symbols[i] = uint32(i)
+	}
+
+	for i = 0; i < in_size; i += max_input_histograms {
+		var num_to_combine uint = brotli_min_size_t(in_size-i, max_input_histograms)
+		var num_new_clusters uint
+		var j uint
+		for j = 0; j < num_to_combine; j++ {
+			clusters[num_clusters+j] = uint32(i + j)
+		}
+
+		num_new_clusters = histogramCombineCommand(out, cluster_size, histogram_symbols[i:], clusters[num_clusters:], pairs, num_to_combine, num_to_combine, max_histograms, pairs_capacity)
+		num_clusters += num_new_clusters
+	}
+	{
+		/* For the second pass, we limit the total number of histogram pairs.
+		   After this limit is reached, we only keep searching for the best pair. */
+		var max_num_pairs uint = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters)
+		if pairs_capacity < (max_num_pairs + 1) {
+			var _new_size uint
+			if pairs_capacity == 0 {
+				_new_size = max_num_pairs + 1
+			} else {
+				_new_size = pairs_capacity
+			}
+			var new_array []histogramPair
+			for _new_size < (max_num_pairs + 1) {
+				_new_size *= 2
+			}
+			new_array = make([]histogramPair, _new_size)
+			if pairs_capacity != 0 {
+				copy(new_array, pairs[:pairs_capacity])
+			}
+
+			pairs = new_array
+			pairs_capacity = _new_size
+		}
+
+		/* Collapse similar histograms. */
+		num_clusters = histogramCombineCommand(out, cluster_size, histogram_symbols, clusters, pairs, num_clusters, in_size, max_histograms, max_num_pairs)
+	}
+
+	pairs = nil
+	cluster_size = nil
+
+	/* Find the optimal map from original histograms to the final ones. */
+	histogramRemapCommand(in, in_size, clusters, num_clusters, out, histogram_symbols)
+
+	clusters = nil
+
+	/* Convert the context map to a canonical form. */
+	*out_size = histogramReindexCommand(out, histogram_symbols, in_size)
+}

vendor/github.com/andybalholm/brotli/cluster_distance.go

@@ -0,0 +1,326 @@
+package brotli
+
+import "math"
+
+/* Copyright 2013 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* Computes the bit cost reduction by combining out[idx1] and out[idx2] and if
+   it is below a threshold, stores the pair (idx1, idx2) in the *pairs queue. */
+func compareAndPushToQueueDistance(out []histogramDistance, cluster_size []uint32, idx1 uint32, idx2 uint32, max_num_pairs uint, pairs []histogramPair, num_pairs *uint) {
+	var is_good_pair bool = false
+	var p histogramPair
+	p.idx2 = 0
+	p.idx1 = p.idx2
+	p.cost_combo = 0
+	p.cost_diff = p.cost_combo
+	if idx1 == idx2 {
+		return
+	}
+
+	if idx2 < idx1 {
+		var t uint32 = idx2
+		idx2 = idx1
+		idx1 = t
+	}
+
+	p.idx1 = idx1
+	p.idx2 = idx2
+	p.cost_diff = 0.5 * clusterCostDiff(uint(cluster_size[idx1]), uint(cluster_size[idx2]))
+	p.cost_diff -= out[idx1].bit_cost_
+	p.cost_diff -= out[idx2].bit_cost_
+
+	if out[idx1].total_count_ == 0 {
+		p.cost_combo = out[idx2].bit_cost_
+		is_good_pair = true
+	} else if out[idx2].total_count_ == 0 {
+		p.cost_combo = out[idx1].bit_cost_
+		is_good_pair = true
+	} else {
+		var threshold float64
+		if *num_pairs == 0 {
+			threshold = 1e99
+		} else {
+			threshold = brotli_max_double(0.0, pairs[0].cost_diff)
+		}
+		var combo histogramDistance = out[idx1]
+		var cost_combo float64
+		histogramAddHistogramDistance(&combo, &out[idx2])
+		cost_combo = populationCostDistance(&combo)
+		if cost_combo < threshold-p.cost_diff {
+			p.cost_combo = cost_combo
+			is_good_pair = true
+		}
+	}
+
+	if is_good_pair {
+		p.cost_diff += p.cost_combo
+		if *num_pairs > 0 && histogramPairIsLess(&pairs[0], &p) {
+			/* Replace the top of the queue if needed. */
+			if *num_pairs < max_num_pairs {
+				pairs[*num_pairs] = pairs[0]
+				(*num_pairs)++
+			}
+
+			pairs[0] = p
+		} else if *num_pairs < max_num_pairs {
+			pairs[*num_pairs] = p
+			(*num_pairs)++
+		}
+	}
+}
+
+func histogramCombineDistance(out []histogramDistance, cluster_size []uint32, symbols []uint32, clusters []uint32, pairs []histogramPair, num_clusters uint, symbols_size uint, max_clusters uint, max_num_pairs uint) uint {
+	var cost_diff_threshold float64 = 0.0
+	var min_cluster_size uint = 1
+	var num_pairs uint = 0
+	{
+		/* We maintain a vector of histogram pairs, with the property that the pair
+		   with the maximum bit cost reduction is the first. */
+		var idx1 uint
+		for idx1 = 0; idx1 < num_clusters; idx1++ {
+			var idx2 uint
+			for idx2 = idx1 + 1; idx2 < num_clusters; idx2++ {
+				compareAndPushToQueueDistance(out, cluster_size, clusters[idx1], clusters[idx2], max_num_pairs, pairs[0:], &num_pairs)
+			}
+		}
+	}
+
+	for num_clusters > min_cluster_size {
+		var best_idx1 uint32
+		var best_idx2 uint32
+		var i uint
+		if pairs[0].cost_diff >= cost_diff_threshold {
+			cost_diff_threshold = 1e99
+			min_cluster_size = max_clusters
+			continue
+		}
+
+		/* Take the best pair from the top of heap. */
+		best_idx1 = pairs[0].idx1
+
+		best_idx2 = pairs[0].idx2
+		histogramAddHistogramDistance(&out[best_idx1], &out[best_idx2])
+		out[best_idx1].bit_cost_ = pairs[0].cost_combo
+		cluster_size[best_idx1] += cluster_size[best_idx2]
+		for i = 0; i < symbols_size; i++ {
+			if symbols[i] == best_idx2 {
+				symbols[i] = best_idx1
+			}
+		}
+
+		for i = 0; i < num_clusters; i++ {
+			if clusters[i] == best_idx2 {
+				copy(clusters[i:], clusters[i+1:][:num_clusters-i-1])
+				break
+			}
+		}
+
+		num_clusters--
+		{
+			/* Remove pairs intersecting the just combined best pair. */
+			var copy_to_idx uint = 0
+			for i = 0; i < num_pairs; i++ {
+				var p *histogramPair = &pairs[i]
+				if p.idx1 == best_idx1 || p.idx2 == best_idx1 || p.idx1 == best_idx2 || p.idx2 == best_idx2 {
+					/* Remove invalid pair from the queue. */
+					continue
+				}
+
+				if histogramPairIsLess(&pairs[0], p) {
+					/* Replace the top of the queue if needed. */
+					var front histogramPair = pairs[0]
+					pairs[0] = *p
+					pairs[copy_to_idx] = front
+				} else {
+					pairs[copy_to_idx] = *p
+				}
+
+				copy_to_idx++
+			}
+
+			num_pairs = copy_to_idx
+		}
+
+		/* Push new pairs formed with the combined histogram to the heap. */
+		for i = 0; i < num_clusters; i++ {
+			compareAndPushToQueueDistance(out, cluster_size, best_idx1, clusters[i], max_num_pairs, pairs[0:], &num_pairs)
+		}
+	}
+
+	return num_clusters
+}
+
+/* What is the bit cost of moving histogram from cur_symbol to candidate. */
+func histogramBitCostDistanceDistance(histogram *histogramDistance, candidate *histogramDistance) float64 {
+	if histogram.total_count_ == 0 {
+		return 0.0
+	} else {
+		var tmp histogramDistance = *histogram
+		histogramAddHistogramDistance(&tmp, candidate)
+		return populationCostDistance(&tmp) - candidate.bit_cost_
+	}
+}
+
+/* Find the best 'out' histogram for each of the 'in' histograms.
+   When called, clusters[0..num_clusters) contains the unique values from
+   symbols[0..in_size), but this property is not preserved in this function.
+   Note: we assume that out[]->bit_cost_ is already up-to-date. */
+func histogramRemapDistance(in []histogramDistance, in_size uint, clusters []uint32, num_clusters uint, out []histogramDistance, symbols []uint32) {
+	var i uint
+	for i = 0; i < in_size; i++ {
+		var best_out uint32
+		if i == 0 {
+			best_out = symbols[0]
+		} else {
+			best_out = symbols[i-1]
+		}
+		var best_bits float64 = histogramBitCostDistanceDistance(&in[i], &out[best_out])
+		var j uint
+		for j = 0; j < num_clusters; j++ {
+			var cur_bits float64 = histogramBitCostDistanceDistance(&in[i], &out[clusters[j]])
+			if cur_bits < best_bits {
+				best_bits = cur_bits
+				best_out = clusters[j]
+			}
+		}
+
+		symbols[i] = best_out
+	}
+
+	/* Recompute each out based on raw and symbols. */
+	for i = 0; i < num_clusters; i++ {
+		histogramClearDistance(&out[clusters[i]])
+	}
+
+	for i = 0; i < in_size; i++ {
+		histogramAddHistogramDistance(&out[symbols[i]], &in[i])
+	}
+}
+
+/* Reorders elements of the out[0..length) array and changes values in
+   symbols[0..length) array in the following way:
+     * when called, symbols[] contains indexes into out[], and has N unique
+       values (possibly N < length)
+     * on return, symbols'[i] = f(symbols[i]) and
+                  out'[symbols'[i]] = out[symbols[i]], for each 0 <= i < length,
+       where f is a bijection between the range of symbols[] and [0..N), and
+       the first occurrences of values in symbols'[i] come in consecutive
+       increasing order.
+   Returns N, the number of unique values in symbols[]. */
+
+var histogramReindexDistance_kInvalidIndex uint32 = math.MaxUint32
+
+func histogramReindexDistance(out []histogramDistance, symbols []uint32, length uint) uint {
+	var new_index []uint32 = make([]uint32, length)
+	var next_index uint32
+	var tmp []histogramDistance
+	var i uint
+	for i = 0; i < length; i++ {
+		new_index[i] = histogramReindexDistance_kInvalidIndex
+	}
+
+	next_index = 0
+	for i = 0; i < length; i++ {
+		if new_index[symbols[i]] == histogramReindexDistance_kInvalidIndex {
+			new_index[symbols[i]] = next_index
+			next_index++
+		}
+	}
+
+	/* TODO: by using idea of "cycle-sort" we can avoid allocation of
+	   tmp and reduce the number of copying by the factor of 2. */
+	tmp = make([]histogramDistance, next_index)
+
+	next_index = 0
+	for i = 0; i < length; i++ {
+		if new_index[symbols[i]] == next_index {
+			tmp[next_index] = out[symbols[i]]
+			next_index++
+		}
+
+		symbols[i] = new_index[symbols[i]]
+	}
+
+	new_index = nil
+	for i = 0; uint32(i) < next_index; i++ {
+		out[i] = tmp[i]
+	}
+
+	tmp = nil
+	return uint(next_index)
+}
+
+func clusterHistogramsDistance(in []histogramDistance, in_size uint, max_histograms uint, out []histogramDistance, out_size *uint, histogram_symbols []uint32) {
+	var cluster_size []uint32 = make([]uint32, in_size)
+	var clusters []uint32 = make([]uint32, in_size)
+	var num_clusters uint = 0
+	var max_input_histograms uint = 64
+	var pairs_capacity uint = max_input_histograms * max_input_histograms / 2
+	var pairs []histogramPair = make([]histogramPair, (pairs_capacity + 1))
+	var i uint
+
+	/* For the first pass of clustering, we allow all pairs. */
+	for i = 0; i < in_size; i++ {
+		cluster_size[i] = 1
+	}
+
+	for i = 0; i < in_size; i++ {
+		out[i] = in[i]
+		out[i].bit_cost_ = populationCostDistance(&in[i])
+		histogram_symbols[i] = uint32(i)
+	}
+
+	for i = 0; i < in_size; i += max_input_histograms {
+		var num_to_combine uint = brotli_min_size_t(in_size-i, max_input_histograms)
+		var num_new_clusters uint
+		var j uint
+		for j = 0; j < num_to_combine; j++ {
+			clusters[num_clusters+j] = uint32(i + j)
+		}
+
+		num_new_clusters = histogramCombineDistance(out, cluster_size, histogram_symbols[i:], clusters[num_clusters:], pairs, num_to_combine, num_to_combine, max_histograms, pairs_capacity)
+		num_clusters += num_new_clusters
+	}
+	{
+		/* For the second pass, we limit the total number of histogram pairs.
+		   After this limit is reached, we only keep searching for the best pair. */
+		var max_num_pairs uint = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters)
+		if pairs_capacity < (max_num_pairs + 1) {
+			var _new_size uint
+			if pairs_capacity == 0 {
+				_new_size = max_num_pairs + 1
+			} else {
+				_new_size = pairs_capacity
+			}
+			var new_array []histogramPair
+			for _new_size < (max_num_pairs + 1) {
+				_new_size *= 2
+			}
+			new_array = make([]histogramPair, _new_size)
+			if pairs_capacity != 0 {
+				copy(new_array, pairs[:pairs_capacity])
+			}
+
+			pairs = new_array
+			pairs_capacity = _new_size
+		}
+
+		/* Collapse similar histograms. */
+		num_clusters = histogramCombineDistance(out, cluster_size, histogram_symbols, clusters, pairs, num_clusters, in_size, max_histograms, max_num_pairs)
+	}
+
+	pairs = nil
+	cluster_size = nil
+
+	/* Find the optimal map from original histograms to the final ones. */
+	histogramRemapDistance(in, in_size, clusters, num_clusters, out, histogram_symbols)
+
+	clusters = nil
+
+	/* Convert the context map to a canonical form. */
+	*out_size = histogramReindexDistance(out, histogram_symbols, in_size)
+}

vendor/github.com/andybalholm/brotli/cluster_literal.go

@@ -0,0 +1,326 @@
+package brotli
+
+import "math"
+
+/* Copyright 2013 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* Computes the bit cost reduction by combining out[idx1] and out[idx2] and if
+   it is below a threshold, stores the pair (idx1, idx2) in the *pairs queue. */
+func compareAndPushToQueueLiteral(out []histogramLiteral, cluster_size []uint32, idx1 uint32, idx2 uint32, max_num_pairs uint, pairs []histogramPair, num_pairs *uint) {
+	var is_good_pair bool = false
+	var p histogramPair
+	p.idx2 = 0
+	p.idx1 = p.idx2
+	p.cost_combo = 0
+	p.cost_diff = p.cost_combo
+	if idx1 == idx2 {
+		return
+	}
+
+	if idx2 < idx1 {
+		var t uint32 = idx2
+		idx2 = idx1
+		idx1 = t
+	}
+
+	p.idx1 = idx1
+	p.idx2 = idx2
+	p.cost_diff = 0.5 * clusterCostDiff(uint(cluster_size[idx1]), uint(cluster_size[idx2]))
+	p.cost_diff -= out[idx1].bit_cost_
+	p.cost_diff -= out[idx2].bit_cost_
+
+	if out[idx1].total_count_ == 0 {
+		p.cost_combo = out[idx2].bit_cost_
+		is_good_pair = true
+	} else if out[idx2].total_count_ == 0 {
+		p.cost_combo = out[idx1].bit_cost_
+		is_good_pair = true
+	} else {
+		var threshold float64
+		if *num_pairs == 0 {
+			threshold = 1e99
+		} else {
+			threshold = brotli_max_double(0.0, pairs[0].cost_diff)
+		}
+		var combo histogramLiteral = out[idx1]
+		var cost_combo float64
+		histogramAddHistogramLiteral(&combo, &out[idx2])
+		cost_combo = populationCostLiteral(&combo)
+		if cost_combo < threshold-p.cost_diff {
+			p.cost_combo = cost_combo
+			is_good_pair = true
+		}
+	}
+
+	if is_good_pair {
+		p.cost_diff += p.cost_combo
+		if *num_pairs > 0 && histogramPairIsLess(&pairs[0], &p) {
+			/* Replace the top of the queue if needed. */
+			if *num_pairs < max_num_pairs {
+				pairs[*num_pairs] = pairs[0]
+				(*num_pairs)++
+			}
+
+			pairs[0] = p
+		} else if *num_pairs < max_num_pairs {
+			pairs[*num_pairs] = p
+			(*num_pairs)++
+		}
+	}
+}
+
+func histogramCombineLiteral(out []histogramLiteral, cluster_size []uint32, symbols []uint32, clusters []uint32, pairs []histogramPair, num_clusters uint, symbols_size uint, max_clusters uint, max_num_pairs uint) uint {
+	var cost_diff_threshold float64 = 0.0
+	var min_cluster_size uint = 1
+	var num_pairs uint = 0
+	{
+		/* We maintain a vector of histogram pairs, with the property that the pair
+		   with the maximum bit cost reduction is the first. */
+		var idx1 uint
+		for idx1 = 0; idx1 < num_clusters; idx1++ {
+			var idx2 uint
+			for idx2 = idx1 + 1; idx2 < num_clusters; idx2++ {
+				compareAndPushToQueueLiteral(out, cluster_size, clusters[idx1], clusters[idx2], max_num_pairs, pairs[0:], &num_pairs)
+			}
+		}
+	}
+
+	for num_clusters > min_cluster_size {
+		var best_idx1 uint32
+		var best_idx2 uint32
+		var i uint
+		if pairs[0].cost_diff >= cost_diff_threshold {
+			cost_diff_threshold = 1e99
+			min_cluster_size = max_clusters
+			continue
+		}
+
+		/* Take the best pair from the top of heap. */
+		best_idx1 = pairs[0].idx1
+
+		best_idx2 = pairs[0].idx2
+		histogramAddHistogramLiteral(&out[best_idx1], &out[best_idx2])
+		out[best_idx1].bit_cost_ = pairs[0].cost_combo
+		cluster_size[best_idx1] += cluster_size[best_idx2]
+		for i = 0; i < symbols_size; i++ {
+			if symbols[i] == best_idx2 {
+				symbols[i] = best_idx1
+			}
+		}
+
+		for i = 0; i < num_clusters; i++ {
+			if clusters[i] == best_idx2 {
+				copy(clusters[i:], clusters[i+1:][:num_clusters-i-1])
+				break
+			}
+		}
+
+		num_clusters--
+		{
+			/* Remove pairs intersecting the just combined best pair. */
+			var copy_to_idx uint = 0
+			for i = 0; i < num_pairs; i++ {
+				var p *histogramPair = &pairs[i]
+				if p.idx1 == best_idx1 || p.idx2 == best_idx1 || p.idx1 == best_idx2 || p.idx2 == best_idx2 {
+					/* Remove invalid pair from the queue. */
+					continue
+				}
+
+				if histogramPairIsLess(&pairs[0], p) {
+					/* Replace the top of the queue if needed. */
+					var front histogramPair = pairs[0]
+					pairs[0] = *p
+					pairs[copy_to_idx] = front
+				} else {
+					pairs[copy_to_idx] = *p
+				}
+
+				copy_to_idx++
+			}
+
+			num_pairs = copy_to_idx
+		}
+
+		/* Push new pairs formed with the combined histogram to the heap. */
+		for i = 0; i < num_clusters; i++ {
+			compareAndPushToQueueLiteral(out, cluster_size, best_idx1, clusters[i], max_num_pairs, pairs[0:], &num_pairs)
+		}
+	}
+
+	return num_clusters
+}
+
+/* What is the bit cost of moving histogram from cur_symbol to candidate. */
+func histogramBitCostDistanceLiteral(histogram *histogramLiteral, candidate *histogramLiteral) float64 {
+	if histogram.total_count_ == 0 {
+		return 0.0
+	} else {
+		var tmp histogramLiteral = *histogram
+		histogramAddHistogramLiteral(&tmp, candidate)
+		return populationCostLiteral(&tmp) - candidate.bit_cost_
+	}
+}
+
+/* Find the best 'out' histogram for each of the 'in' histograms.
+   When called, clusters[0..num_clusters) contains the unique values from
+   symbols[0..in_size), but this property is not preserved in this function.
+   Note: we assume that out[]->bit_cost_ is already up-to-date. */
+func histogramRemapLiteral(in []histogramLiteral, in_size uint, clusters []uint32, num_clusters uint, out []histogramLiteral, symbols []uint32) {
+	var i uint
+	for i = 0; i < in_size; i++ {
+		var best_out uint32
+		if i == 0 {
+			best_out = symbols[0]
+		} else {
+			best_out = symbols[i-1]
+		}
+		var best_bits float64 = histogramBitCostDistanceLiteral(&in[i], &out[best_out])
+		var j uint
+		for j = 0; j < num_clusters; j++ {
+			var cur_bits float64 = histogramBitCostDistanceLiteral(&in[i], &out[clusters[j]])
+			if cur_bits < best_bits {
+				best_bits = cur_bits
+				best_out = clusters[j]
+			}
+		}
+
+		symbols[i] = best_out
+	}
+
+	/* Recompute each out based on raw and symbols. */
+	for i = 0; i < num_clusters; i++ {
+		histogramClearLiteral(&out[clusters[i]])
+	}
+
+	for i = 0; i < in_size; i++ {
+		histogramAddHistogramLiteral(&out[symbols[i]], &in[i])
+	}
+}
+
+/* Reorders elements of the out[0..length) array and changes values in
+   symbols[0..length) array in the following way:
+     * when called, symbols[] contains indexes into out[], and has N unique
+       values (possibly N < length)
+     * on return, symbols'[i] = f(symbols[i]) and
+                  out'[symbols'[i]] = out[symbols[i]], for each 0 <= i < length,
+       where f is a bijection between the range of symbols[] and [0..N), and
+       the first occurrences of values in symbols'[i] come in consecutive
+       increasing order.
+   Returns N, the number of unique values in symbols[]. */
+
+var histogramReindexLiteral_kInvalidIndex uint32 = math.MaxUint32
+
+func histogramReindexLiteral(out []histogramLiteral, symbols []uint32, length uint) uint {
+	var new_index []uint32 = make([]uint32, length)
+	var next_index uint32
+	var tmp []histogramLiteral
+	var i uint
+	for i = 0; i < length; i++ {
+		new_index[i] = histogramReindexLiteral_kInvalidIndex
+	}
+
+	next_index = 0
+	for i = 0; i < length; i++ {
+		if new_index[symbols[i]] == histogramReindexLiteral_kInvalidIndex {
+			new_index[symbols[i]] = next_index
+			next_index++
+		}
+	}
+
+	/* TODO: by using idea of "cycle-sort" we can avoid allocation of
+	   tmp and reduce the number of copying by the factor of 2. */
+	tmp = make([]histogramLiteral, next_index)
+
+	next_index = 0
+	for i = 0; i < length; i++ {
+		if new_index[symbols[i]] == next_index {
+			tmp[next_index] = out[symbols[i]]
+			next_index++
+		}
+
+		symbols[i] = new_index[symbols[i]]
+	}
+
+	new_index = nil
+	for i = 0; uint32(i) < next_index; i++ {
+		out[i] = tmp[i]
+	}
+
+	tmp = nil
+	return uint(next_index)
+}
+
+func clusterHistogramsLiteral(in []histogramLiteral, in_size uint, max_histograms uint, out []histogramLiteral, out_size *uint, histogram_symbols []uint32) {
+	var cluster_size []uint32 = make([]uint32, in_size)
+	var clusters []uint32 = make([]uint32, in_size)
+	var num_clusters uint = 0
+	var max_input_histograms uint = 64
+	var pairs_capacity uint = max_input_histograms * max_input_histograms / 2
+	var pairs []histogramPair = make([]histogramPair, (pairs_capacity + 1))
+	var i uint
+
+	/* For the first pass of clustering, we allow all pairs. */
+	for i = 0; i < in_size; i++ {
+		cluster_size[i] = 1
+	}
+
+	for i = 0; i < in_size; i++ {
+		out[i] = in[i]
+		out[i].bit_cost_ = populationCostLiteral(&in[i])
+		histogram_symbols[i] = uint32(i)
+	}
+
+	for i = 0; i < in_size; i += max_input_histograms {
+		var num_to_combine uint = brotli_min_size_t(in_size-i, max_input_histograms)
+		var num_new_clusters uint
+		var j uint
+		for j = 0; j < num_to_combine; j++ {
+			clusters[num_clusters+j] = uint32(i + j)
+		}
+
+		num_new_clusters = histogramCombineLiteral(out, cluster_size, histogram_symbols[i:], clusters[num_clusters:], pairs, num_to_combine, num_to_combine, max_histograms, pairs_capacity)
+		num_clusters += num_new_clusters
+	}
+	{
+		/* For the second pass, we limit the total number of histogram pairs.
+		   After this limit is reached, we only keep searching for the best pair. */
+		var max_num_pairs uint = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters)
+		if pairs_capacity < (max_num_pairs + 1) {
+			var _new_size uint
+			if pairs_capacity == 0 {
+				_new_size = max_num_pairs + 1
+			} else {
+				_new_size = pairs_capacity
+			}
+			var new_array []histogramPair
+			for _new_size < (max_num_pairs + 1) {
+				_new_size *= 2
+			}
+			new_array = make([]histogramPair, _new_size)
+			if pairs_capacity != 0 {
+				copy(new_array, pairs[:pairs_capacity])
+			}
+
+			pairs = new_array
+			pairs_capacity = _new_size
+		}
+
+		/* Collapse similar histograms. */
+		num_clusters = histogramCombineLiteral(out, cluster_size, histogram_symbols, clusters, pairs, num_clusters, in_size, max_histograms, max_num_pairs)
+	}
+
+	pairs = nil
+	cluster_size = nil
+
+	/* Find the optimal map from original histograms to the final ones. */
+	histogramRemapLiteral(in, in_size, clusters, num_clusters, out, histogram_symbols)
+
+	clusters = nil
+
+	/* Convert the context map to a canonical form. */
+	*out_size = histogramReindexLiteral(out, histogram_symbols, in_size)
+}

vendor/github.com/andybalholm/brotli/command.go

@@ -0,0 +1,252 @@
+package brotli
+
+var kInsBase = []uint32{
+	0,
+	1,
+	2,
+	3,
+	4,
+	5,
+	6,
+	8,
+	10,
+	14,
+	18,
+	26,
+	34,
+	50,
+	66,
+	98,
+	130,
+	194,
+	322,
+	578,
+	1090,
+	2114,
+	6210,
+	22594,
+}
+
+var kInsExtra = []uint32{
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	1,
+	1,
+	2,
+	2,
+	3,
+	3,
+	4,
+	4,
+	5,
+	5,
+	6,
+	7,
+	8,
+	9,
+	10,
+	12,
+	14,
+	24,
+}
+
+var kCopyBase = []uint32{
+	2,
+	3,
+	4,
+	5,
+	6,
+	7,
+	8,
+	9,
+	10,
+	12,
+	14,
+	18,
+	22,
+	30,
+	38,
+	54,
+	70,
+	102,
+	134,
+	198,
+	326,
+	582,
+	1094,
+	2118,
+}
+
+var kCopyExtra = []uint32{
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	1,
+	1,
+	2,
+	2,
+	3,
+	3,
+	4,
+	4,
+	5,
+	5,
+	6,
+	7,
+	8,
+	9,
+	10,
+	24,
+}
+
+func getInsertLengthCode(insertlen uint) uint16 {
+	if insertlen < 6 {
+		return uint16(insertlen)
+	} else if insertlen < 130 {
+		var nbits uint32 = log2FloorNonZero(insertlen-2) - 1
+		return uint16((nbits << 1) + uint32((insertlen-2)>>nbits) + 2)
+	} else if insertlen < 2114 {
+		return uint16(log2FloorNonZero(insertlen-66) + 10)
+	} else if insertlen < 6210 {
+		return 21
+	} else if insertlen < 22594 {
+		return 22
+	} else {
+		return 23
+	}
+}
+
+func getCopyLengthCode(copylen uint) uint16 {
+	if copylen < 10 {
+		return uint16(copylen - 2)
+	} else if copylen < 134 {
+		var nbits uint32 = log2FloorNonZero(copylen-6) - 1
+		return uint16((nbits << 1) + uint32((copylen-6)>>nbits) + 4)
+	} else if copylen < 2118 {
+		return uint16(log2FloorNonZero(copylen-70) + 12)
+	} else {
+		return 23
+	}
+}
+
+func combineLengthCodes(inscode uint16, copycode uint16, use_last_distance bool) uint16 {
+	var bits64 uint16 = uint16(copycode&0x7 | (inscode&0x7)<<3)
+	if use_last_distance && inscode < 8 && copycode < 16 {
+		if copycode < 8 {
+			return bits64
+		} else {
+			return bits64 | 64
+		}
+	} else {
+		/* Specification: 5 Encoding of ... (last table) */
+		/* offset = 2 * index, where index is in range [0..8] */
+		var offset uint32 = 2 * ((uint32(copycode) >> 3) + 3*(uint32(inscode)>>3))
+
+		/* All values in specification are K * 64,
+		   where   K = [2, 3, 6, 4, 5, 8, 7, 9, 10],
+		       i + 1 = [1, 2, 3, 4, 5, 6, 7, 8,  9],
+		   K - i - 1 = [1, 1, 3, 0, 0, 2, 0, 1,  2] = D.
+		   All values in D require only 2 bits to encode.
+		   Magic constant is shifted 6 bits left, to avoid final multiplication. */
+		offset = (offset << 5) + 0x40 + ((0x520D40 >> offset) & 0xC0)
+
+		return uint16(offset | uint32(bits64))
+	}
+}
+
+func getLengthCode(insertlen uint, copylen uint, use_last_distance bool, code *uint16) {
+	var inscode uint16 = getInsertLengthCode(insertlen)
+	var copycode uint16 = getCopyLengthCode(copylen)
+	*code = combineLengthCodes(inscode, copycode, use_last_distance)
+}
+
+func getInsertBase(inscode uint16) uint32 {
+	return kInsBase[inscode]
+}
+
+func getInsertExtra(inscode uint16) uint32 {
+	return kInsExtra[inscode]
+}
+
+func getCopyBase(copycode uint16) uint32 {
+	return kCopyBase[copycode]
+}
+
+func getCopyExtra(copycode uint16) uint32 {
+	return kCopyExtra[copycode]
+}
+
+type command struct {
+	insert_len_  uint32
+	copy_len_    uint32
+	dist_extra_  uint32
+	cmd_prefix_  uint16
+	dist_prefix_ uint16
+}
+
+/* distance_code is e.g. 0 for same-as-last short code, or 16 for offset 1. */
+func initCommand(self *command, dist *distanceParams, insertlen uint, copylen uint, copylen_code_delta int, distance_code uint) {
+	/* Don't rely on signed int representation, use honest casts. */
+	var delta uint32 = uint32(byte(int8(copylen_code_delta)))
+	self.insert_len_ = uint32(insertlen)
+	self.copy_len_ = uint32(uint32(copylen) | delta<<25)
+
+	/* The distance prefix and extra bits are stored in this Command as if
+	   npostfix and ndirect were 0, they are only recomputed later after the
+	   clustering if needed. */
+	prefixEncodeCopyDistance(distance_code, uint(dist.num_direct_distance_codes), uint(dist.distance_postfix_bits), &self.dist_prefix_, &self.dist_extra_)
+
+	getLengthCode(insertlen, uint(int(copylen)+copylen_code_delta), (self.dist_prefix_&0x3FF == 0), &self.cmd_prefix_)
+}
+
+func initInsertCommand(self *command, insertlen uint) {
+	self.insert_len_ = uint32(insertlen)
+	self.copy_len_ = 4 << 25
+	self.dist_extra_ = 0
+	self.dist_prefix_ = numDistanceShortCodes
+	getLengthCode(insertlen, 4, false, &self.cmd_prefix_)
+}
+
+func commandRestoreDistanceCode(self *command, dist *distanceParams) uint32 {
+	if uint32(self.dist_prefix_&0x3FF) < numDistanceShortCodes+dist.num_direct_distance_codes {
+		return uint32(self.dist_prefix_) & 0x3FF
+	} else {
+		var dcode uint32 = uint32(self.dist_prefix_) & 0x3FF
+		var nbits uint32 = uint32(self.dist_prefix_) >> 10
+		var extra uint32 = self.dist_extra_
+		var postfix_mask uint32 = (1 << dist.distance_postfix_bits) - 1
+		var hcode uint32 = (dcode - dist.num_direct_distance_codes - numDistanceShortCodes) >> dist.distance_postfix_bits
+		var lcode uint32 = (dcode - dist.num_direct_distance_codes - numDistanceShortCodes) & postfix_mask
+		var offset uint32 = ((2 + (hcode & 1)) << nbits) - 4
+		return ((offset + extra) << dist.distance_postfix_bits) + lcode + dist.num_direct_distance_codes + numDistanceShortCodes
+	}
+}
+
+func commandDistanceContext(self *command) uint32 {
+	var r uint32 = uint32(self.cmd_prefix_) >> 6
+	var c uint32 = uint32(self.cmd_prefix_) & 7
+	if (r == 0 || r == 2 || r == 4 || r == 7) && (c <= 2) {
+		return c
+	}
+
+	return 3
+}
+
+func commandCopyLen(self *command) uint32 {
+	return self.copy_len_ & 0x1FFFFFF
+}
+
+func commandCopyLenCode(self *command) uint32 {
+	var modifier uint32 = self.copy_len_ >> 25
+	var delta int32 = int32(int8(byte(modifier | (modifier&0x40)<<1)))
+	return uint32(int32(self.copy_len_&0x1FFFFFF) + delta)
+}

vendor/github.com/andybalholm/brotli/compress_fragment.go

@@ -0,0 +1,840 @@
+package brotli
+
+import "encoding/binary"
+
+/* Copyright 2015 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* Function for fast encoding of an input fragment, independently from the input
+   history. This function uses one-pass processing: when we find a backward
+   match, we immediately emit the corresponding command and literal codes to
+   the bit stream.
+
+   Adapted from the CompressFragment() function in
+   https://github.com/google/snappy/blob/master/snappy.cc */
+
+const maxDistance_compress_fragment = 262128
+
+func hash5(p []byte, shift uint) uint32 {
+	var h uint64 = (binary.LittleEndian.Uint64(p) << 24) * uint64(kHashMul32)
+	return uint32(h >> shift)
+}
+
+func hashBytesAtOffset5(v uint64, offset int, shift uint) uint32 {
+	assert(offset >= 0)
+	assert(offset <= 3)
+	{
+		var h uint64 = ((v >> uint(8*offset)) << 24) * uint64(kHashMul32)
+		return uint32(h >> shift)
+	}
+}
+
+func isMatch5(p1 []byte, p2 []byte) bool {
+	var i int
+	for i = 0; i < 5; i++ {
+		if p1[i] != p2[i] {
+			return false
+		}
+	}
+
+	return true
+}
+
+/* Builds a literal prefix code into "depths" and "bits" based on the statistics
+   of the "input" string and stores it into the bit stream.
+   Note that the prefix code here is built from the pre-LZ77 input, therefore
+   we can only approximate the statistics of the actual literal stream.
+   Moreover, for long inputs we build a histogram from a sample of the input
+   and thus have to assign a non-zero depth for each literal.
+   Returns estimated compression ratio millibytes/char for encoding given input
+   with generated code. */
+func buildAndStoreLiteralPrefixCode(input []byte, input_size uint, depths []byte, bits []uint16, storage_ix *uint, storage []byte) uint {
+	var histogram = [256]uint32{0}
+	var histogram_total uint
+	var i uint
+	if input_size < 1<<15 {
+		for i = 0; i < input_size; i++ {
+			histogram[input[i]]++
+		}
+
+		histogram_total = input_size
+		for i = 0; i < 256; i++ {
+			/* We weigh the first 11 samples with weight 3 to account for the
+			   balancing effect of the LZ77 phase on the histogram. */
+			var adjust uint32 = 2 * brotli_min_uint32_t(histogram[i], 11)
+			histogram[i] += adjust
+			histogram_total += uint(adjust)
+		}
+	} else {
+		const kSampleRate uint = 29
+		for i = 0; i < input_size; i += kSampleRate {
+			histogram[input[i]]++
+		}
+
+		histogram_total = (input_size + kSampleRate - 1) / kSampleRate
+		for i = 0; i < 256; i++ {
+			/* We add 1 to each population count to avoid 0 bit depths (since this is
+			   only a sample and we don't know if the symbol appears or not), and we
+			   weigh the first 11 samples with weight 3 to account for the balancing
+			   effect of the LZ77 phase on the histogram (more frequent symbols are
+			   more likely to be in backward references instead as literals). */
+			var adjust uint32 = 1 + 2*brotli_min_uint32_t(histogram[i], 11)
+			histogram[i] += adjust
+			histogram_total += uint(adjust)
+		}
+	}
+
+	buildAndStoreHuffmanTreeFast(histogram[:], histogram_total, /* max_bits = */
+		8, depths, bits, storage_ix, storage)
+	{
+		var literal_ratio uint = 0
+		for i = 0; i < 256; i++ {
+			if histogram[i] != 0 {
+				literal_ratio += uint(histogram[i] * uint32(depths[i]))
+			}
+		}
+
+		/* Estimated encoding ratio, millibytes per symbol. */
+		return (literal_ratio * 125) / histogram_total
+	}
+}
+
+/* Builds a command and distance prefix code (each 64 symbols) into "depth" and
+   "bits" based on "histogram" and stores it into the bit stream. */
+func buildAndStoreCommandPrefixCode1(histogram []uint32, depth []byte, bits []uint16, storage_ix *uint, storage []byte) {
+	var tree [129]huffmanTree
+	var cmd_depth = [numCommandSymbols]byte{0}
+	/* Tree size for building a tree over 64 symbols is 2 * 64 + 1. */
+
+	var cmd_bits [64]uint16
+
+	createHuffmanTree(histogram, 64, 15, tree[:], depth)
+	createHuffmanTree(histogram[64:], 64, 14, tree[:], depth[64:])
+
+	/* We have to jump through a few hoops here in order to compute
+	   the command bits because the symbols are in a different order than in
+	   the full alphabet. This looks complicated, but having the symbols
+	   in this order in the command bits saves a few branches in the Emit*
+	   functions. */
+	copy(cmd_depth[:], depth[:24])
+
+	copy(cmd_depth[24:][:], depth[40:][:8])
+	copy(cmd_depth[32:][:], depth[24:][:8])
+	copy(cmd_depth[40:][:], depth[48:][:8])
+	copy(cmd_depth[48:][:], depth[32:][:8])
+	copy(cmd_depth[56:][:], depth[56:][:8])
+	convertBitDepthsToSymbols(cmd_depth[:], 64, cmd_bits[:])
+	copy(bits, cmd_bits[:24])
+	copy(bits[24:], cmd_bits[32:][:8])
+	copy(bits[32:], cmd_bits[48:][:8])
+	copy(bits[40:], cmd_bits[24:][:8])
+	copy(bits[48:], cmd_bits[40:][:8])
+	copy(bits[56:], cmd_bits[56:][:8])
+	convertBitDepthsToSymbols(depth[64:], 64, bits[64:])
+	{
+		/* Create the bit length array for the full command alphabet. */
+		var i uint
+		for i := 0; i < int(64); i++ {
+			cmd_depth[i] = 0
+		} /* only 64 first values were used */
+		copy(cmd_depth[:], depth[:8])
+		copy(cmd_depth[64:][:], depth[8:][:8])
+		copy(cmd_depth[128:][:], depth[16:][:8])
+		copy(cmd_depth[192:][:], depth[24:][:8])
+		copy(cmd_depth[384:][:], depth[32:][:8])
+		for i = 0; i < 8; i++ {
+			cmd_depth[128+8*i] = depth[40+i]
+			cmd_depth[256+8*i] = depth[48+i]
+			cmd_depth[448+8*i] = depth[56+i]
+		}
+
+		storeHuffmanTree(cmd_depth[:], numCommandSymbols, tree[:], storage_ix, storage)
+	}
+
+	storeHuffmanTree(depth[64:], 64, tree[:], storage_ix, storage)
+}
+
+/* REQUIRES: insertlen < 6210 */
+func emitInsertLen1(insertlen uint, depth []byte, bits []uint16, histo []uint32, storage_ix *uint, storage []byte) {
+	if insertlen < 6 {
+		var code uint = insertlen + 40
+		writeBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage)
+		histo[code]++
+	} else if insertlen < 130 {
+		var tail uint = insertlen - 2
+		var nbits uint32 = log2FloorNonZero(tail) - 1
+		var prefix uint = tail >> nbits
+		var inscode uint = uint((nbits << 1) + uint32(prefix) + 42)
+		writeBits(uint(depth[inscode]), uint64(bits[inscode]), storage_ix, storage)
+		writeBits(uint(nbits), uint64(tail)-(uint64(prefix)<<nbits), storage_ix, storage)
+		histo[inscode]++
+	} else if insertlen < 2114 {
+		var tail uint = insertlen - 66
+		var nbits uint32 = log2FloorNonZero(tail)
+		var code uint = uint(nbits + 50)
+		writeBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage)
+		writeBits(uint(nbits), uint64(tail)-(uint64(uint(1))<<nbits), storage_ix, storage)
+		histo[code]++
+	} else {
+		writeBits(uint(depth[61]), uint64(bits[61]), storage_ix, storage)
+		writeBits(12, uint64(insertlen)-2114, storage_ix, storage)
+		histo[61]++
+	}
+}
+
+func emitLongInsertLen(insertlen uint, depth []byte, bits []uint16, histo []uint32, storage_ix *uint, storage []byte) {
+	if insertlen < 22594 {
+		writeBits(uint(depth[62]), uint64(bits[62]), storage_ix, storage)
+		writeBits(14, uint64(insertlen)-6210, storage_ix, storage)
+		histo[62]++
+	} else {
+		writeBits(uint(depth[63]), uint64(bits[63]), storage_ix, storage)
+		writeBits(24, uint64(insertlen)-22594, storage_ix, storage)
+		histo[63]++
+	}
+}
+
+func emitCopyLen1(copylen uint, depth []byte, bits []uint16, histo []uint32, storage_ix *uint, storage []byte) {
+	if copylen < 10 {
+		writeBits(uint(depth[copylen+14]), uint64(bits[copylen+14]), storage_ix, storage)
+		histo[copylen+14]++
+	} else if copylen < 134 {
+		var tail uint = copylen - 6
+		var nbits uint32 = log2FloorNonZero(tail) - 1
+		var prefix uint = tail >> nbits
+		var code uint = uint((nbits << 1) + uint32(prefix) + 20)
+		writeBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage)
+		writeBits(uint(nbits), uint64(tail)-(uint64(prefix)<<nbits), storage_ix, storage)
+		histo[code]++
+	} else if copylen < 2118 {
+		var tail uint = copylen - 70
+		var nbits uint32 = log2FloorNonZero(tail)
+		var code uint = uint(nbits + 28)
+		writeBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage)
+		writeBits(uint(nbits), uint64(tail)-(uint64(uint(1))<<nbits), storage_ix, storage)
+		histo[code]++
+	} else {
+		writeBits(uint(depth[39]), uint64(bits[39]), storage_ix, storage)
+		writeBits(24, uint64(copylen)-2118, storage_ix, storage)
+		histo[39]++
+	}
+}
+
+func emitCopyLenLastDistance1(copylen uint, depth []byte, bits []uint16, histo []uint32, storage_ix *uint, storage []byte) {
+	if copylen < 12 {
+		writeBits(uint(depth[copylen-4]), uint64(bits[copylen-4]), storage_ix, storage)
+		histo[copylen-4]++
+	} else if copylen < 72 {
+		var tail uint = copylen - 8
+		var nbits uint32 = log2FloorNonZero(tail) - 1
+		var prefix uint = tail >> nbits
+		var code uint = uint((nbits << 1) + uint32(prefix) + 4)
+		writeBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage)
+		writeBits(uint(nbits), uint64(tail)-(uint64(prefix)<<nbits), storage_ix, storage)
+		histo[code]++
+	} else if copylen < 136 {
+		var tail uint = copylen - 8
+		var code uint = (tail >> 5) + 30
+		writeBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage)
+		writeBits(5, uint64(tail)&31, storage_ix, storage)
+		writeBits(uint(depth[64]), uint64(bits[64]), storage_ix, storage)
+		histo[code]++
+		histo[64]++
+	} else if copylen < 2120 {
+		var tail uint = copylen - 72
+		var nbits uint32 = log2FloorNonZero(tail)
+		var code uint = uint(nbits + 28)
+		writeBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage)
+		writeBits(uint(nbits), uint64(tail)-(uint64(uint(1))<<nbits), storage_ix, storage)
+		writeBits(uint(depth[64]), uint64(bits[64]), storage_ix, storage)
+		histo[code]++
+		histo[64]++
+	} else {
+		writeBits(uint(depth[39]), uint64(bits[39]), storage_ix, storage)
+		writeBits(24, uint64(copylen)-2120, storage_ix, storage)
+		writeBits(uint(depth[64]), uint64(bits[64]), storage_ix, storage)
+		histo[39]++
+		histo[64]++
+	}
+}
+
+func emitDistance1(distance uint, depth []byte, bits []uint16, histo []uint32, storage_ix *uint, storage []byte) {
+	var d uint = distance + 3
+	var nbits uint32 = log2FloorNonZero(d) - 1
+	var prefix uint = (d >> nbits) & 1
+	var offset uint = (2 + prefix) << nbits
+	var distcode uint = uint(2*(nbits-1) + uint32(prefix) + 80)
+	writeBits(uint(depth[distcode]), uint64(bits[distcode]), storage_ix, storage)
+	writeBits(uint(nbits), uint64(d)-uint64(offset), storage_ix, storage)
+	histo[distcode]++
+}
+
+func emitLiterals(input []byte, len uint, depth []byte, bits []uint16, storage_ix *uint, storage []byte) {
+	var j uint
+	for j = 0; j < len; j++ {
+		var lit byte = input[j]
+		writeBits(uint(depth[lit]), uint64(bits[lit]), storage_ix, storage)
+	}
+}
+
+/* REQUIRES: len <= 1 << 24. */
+func storeMetaBlockHeader1(len uint, is_uncompressed bool, storage_ix *uint, storage []byte) {
+	var nibbles uint = 6
+
+	/* ISLAST */
+	writeBits(1, 0, storage_ix, storage)
+
+	if len <= 1<<16 {
+		nibbles = 4
+	} else if len <= 1<<20 {
+		nibbles = 5
+	}
+
+	writeBits(2, uint64(nibbles)-4, storage_ix, storage)
+	writeBits(nibbles*4, uint64(len)-1, storage_ix, storage)
+
+	/* ISUNCOMPRESSED */
+	writeSingleBit(is_uncompressed, storage_ix, storage)
+}
+
+func updateBits(n_bits uint, bits uint32, pos uint, array []byte) {
+	for n_bits > 0 {
+		var byte_pos uint = pos >> 3
+		var n_unchanged_bits uint = pos & 7
+		var n_changed_bits uint = brotli_min_size_t(n_bits, 8-n_unchanged_bits)
+		var total_bits uint = n_unchanged_bits + n_changed_bits
+		var mask uint32 = (^((1 << total_bits) - 1)) | ((1 << n_unchanged_bits) - 1)
+		var unchanged_bits uint32 = uint32(array[byte_pos]) & mask
+		var changed_bits uint32 = bits & ((1 << n_changed_bits) - 1)
+		array[byte_pos] = byte(changed_bits<<n_unchanged_bits | unchanged_bits)
+		n_bits -= n_changed_bits
+		bits >>= n_changed_bits
+		pos += n_changed_bits
+	}
+}
+
+func rewindBitPosition1(new_storage_ix uint, storage_ix *uint, storage []byte) {
+	var bitpos uint = new_storage_ix & 7
+	var mask uint = (1 << bitpos) - 1
+	storage[new_storage_ix>>3] &= byte(mask)
+	*storage_ix = new_storage_ix
+}
+
+var shouldMergeBlock_kSampleRate uint = 43
+
+func shouldMergeBlock(data []byte, len uint, depths []byte) bool {
+	var histo = [256]uint{0}
+	var i uint
+	for i = 0; i < len; i += shouldMergeBlock_kSampleRate {
+		histo[data[i]]++
+	}
+	{
+		var total uint = (len + shouldMergeBlock_kSampleRate - 1) / shouldMergeBlock_kSampleRate
+		var r float64 = (fastLog2(total)+0.5)*float64(total) + 200
+		for i = 0; i < 256; i++ {
+			r -= float64(histo[i]) * (float64(depths[i]) + fastLog2(histo[i]))
+		}
+
+		return r >= 0.0
+	}
+}
+
+func shouldUseUncompressedMode(metablock_start []byte, next_emit []byte, insertlen uint, literal_ratio uint) bool {
+	var compressed uint = uint(-cap(next_emit) + cap(metablock_start))
+	if compressed*50 > insertlen {
+		return false
+	} else {
+		return literal_ratio > 980
+	}
+}
+
+func emitUncompressedMetaBlock1(begin []byte, end []byte, storage_ix_start uint, storage_ix *uint, storage []byte) {
+	var len uint = uint(-cap(end) + cap(begin))
+	rewindBitPosition1(storage_ix_start, storage_ix, storage)
+	storeMetaBlockHeader1(uint(len), true, storage_ix, storage)
+	*storage_ix = (*storage_ix + 7) &^ 7
+	copy(storage[*storage_ix>>3:], begin[:len])
+	*storage_ix += uint(len << 3)
+	storage[*storage_ix>>3] = 0
+}
+
+var kCmdHistoSeed = [128]uint32{
+	0,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	0,
+	0,
+	0,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	0,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	0,
+	0,
+	0,
+	0,
+}
+
+var compressFragmentFastImpl_kFirstBlockSize uint = 3 << 15
+var compressFragmentFastImpl_kMergeBlockSize uint = 1 << 16
+
+func compressFragmentFastImpl(in []byte, input_size uint, is_last bool, table []int, table_bits uint, cmd_depth []byte, cmd_bits []uint16, cmd_code_numbits *uint, cmd_code []byte, storage_ix *uint, storage []byte) {
+	var cmd_histo [128]uint32
+	var ip_end int
+	var next_emit int = 0
+	var base_ip int = 0
+	var input int = 0
+	const kInputMarginBytes uint = windowGap
+	const kMinMatchLen uint = 5
+	var metablock_start int = input
+	var block_size uint = brotli_min_size_t(input_size, compressFragmentFastImpl_kFirstBlockSize)
+	var total_block_size uint = block_size
+	var mlen_storage_ix uint = *storage_ix + 3
+	var lit_depth [256]byte
+	var lit_bits [256]uint16
+	var literal_ratio uint
+	var ip int
+	var last_distance int
+	var shift uint = 64 - table_bits
+
+	/* "next_emit" is a pointer to the first byte that is not covered by a
+	   previous copy. Bytes between "next_emit" and the start of the next copy or
+	   the end of the input will be emitted as literal bytes. */
+
+	/* Save the start of the first block for position and distance computations.
+	 */
+
+	/* Save the bit position of the MLEN field of the meta-block header, so that
+	   we can update it later if we decide to extend this meta-block. */
+	storeMetaBlockHeader1(block_size, false, storage_ix, storage)
+
+	/* No block splits, no contexts. */
+	writeBits(13, 0, storage_ix, storage)
+
+	literal_ratio = buildAndStoreLiteralPrefixCode(in[input:], block_size, lit_depth[:], lit_bits[:], storage_ix, storage)
+	{
+		/* Store the pre-compressed command and distance prefix codes. */
+		var i uint
+		for i = 0; i+7 < *cmd_code_numbits; i += 8 {
+			writeBits(8, uint64(cmd_code[i>>3]), storage_ix, storage)
+		}
+	}
+
+	writeBits(*cmd_code_numbits&7, uint64(cmd_code[*cmd_code_numbits>>3]), storage_ix, storage)
+
+	/* Initialize the command and distance histograms. We will gather
+	   statistics of command and distance codes during the processing
+	   of this block and use it to update the command and distance
+	   prefix codes for the next block. */
+emit_commands:
+	copy(cmd_histo[:], kCmdHistoSeed[:])
+
+	/* "ip" is the input pointer. */
+	ip = input
+
+	last_distance = -1
+	ip_end = int(uint(input) + block_size)
+
+	if block_size >= kInputMarginBytes {
+		var len_limit uint = brotli_min_size_t(block_size-kMinMatchLen, input_size-kInputMarginBytes)
+		var ip_limit int = int(uint(input) + len_limit)
+		/* For the last block, we need to keep a 16 bytes margin so that we can be
+		   sure that all distances are at most window size - 16.
+		   For all other blocks, we only need to keep a margin of 5 bytes so that
+		   we don't go over the block size with a copy. */
+
+		var next_hash uint32
+		ip++
+		for next_hash = hash5(in[ip:], shift); ; {
+			var skip uint32 = 32
+			var next_ip int = ip
+			/* Step 1: Scan forward in the input looking for a 5-byte-long match.
+			   If we get close to exhausting the input then goto emit_remainder.
+
+			   Heuristic match skipping: If 32 bytes are scanned with no matches
+			   found, start looking only at every other byte. If 32 more bytes are
+			   scanned, look at every third byte, etc.. When a match is found,
+			   immediately go back to looking at every byte. This is a small loss
+			   (~5% performance, ~0.1% density) for compressible data due to more
+			   bookkeeping, but for non-compressible data (such as JPEG) it's a huge
+			   win since the compressor quickly "realizes" the data is incompressible
+			   and doesn't bother looking for matches everywhere.
+
+			   The "skip" variable keeps track of how many bytes there are since the
+			   last match; dividing it by 32 (i.e. right-shifting by five) gives the
+			   number of bytes to move ahead for each iteration. */
+
+			var candidate int
+			assert(next_emit < ip)
+
+		trawl:
+			for {
+				var hash uint32 = next_hash
+				var bytes_between_hash_lookups uint32 = skip >> 5
+				skip++
+				assert(hash == hash5(in[next_ip:], shift))
+				ip = next_ip
+				next_ip = int(uint32(ip) + bytes_between_hash_lookups)
+				if next_ip > ip_limit {
+					goto emit_remainder
+				}
+
+				next_hash = hash5(in[next_ip:], shift)
+				candidate = ip - last_distance
+				if isMatch5(in[ip:], in[candidate:]) {
+					if candidate < ip {
+						table[hash] = int(ip - base_ip)
+						break
+					}
+				}
+
+				candidate = base_ip + table[hash]
+				assert(candidate >= base_ip)
+				assert(candidate < ip)
+
+				table[hash] = int(ip - base_ip)
+				if !(!isMatch5(in[ip:], in[candidate:])) {
+					break
+				}
+			}
+
+			/* Check copy distance. If candidate is not feasible, continue search.
+			   Checking is done outside of hot loop to reduce overhead. */
+			if ip-candidate > maxDistance_compress_fragment {
+				goto trawl
+			}
+
+			/* Step 2: Emit the found match together with the literal bytes from
+			   "next_emit" to the bit stream, and then see if we can find a next match
+			   immediately afterwards. Repeat until we find no match for the input
+			   without emitting some literal bytes. */
+			{
+				var base int = ip
+				/* > 0 */
+				var matched uint = 5 + findMatchLengthWithLimit(in[candidate+5:], in[ip+5:], uint(ip_end-ip)-5)
+				var distance int = int(base - candidate)
+				/* We have a 5-byte match at ip, and we need to emit bytes in
+				   [next_emit, ip). */
+
+				var insert uint = uint(base - next_emit)
+				ip += int(matched)
+				if insert < 6210 {
+					emitInsertLen1(insert, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage)
+				} else if shouldUseUncompressedMode(in[metablock_start:], in[next_emit:], insert, literal_ratio) {
+					emitUncompressedMetaBlock1(in[metablock_start:], in[base:], mlen_storage_ix-3, storage_ix, storage)
+					input_size -= uint(base - input)
+					input = base
+					next_emit = input
+					goto next_block
+				} else {
+					emitLongInsertLen(insert, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage)
+				}
+
+				emitLiterals(in[next_emit:], insert, lit_depth[:], lit_bits[:], storage_ix, storage)
+				if distance == last_distance {
+					writeBits(uint(cmd_depth[64]), uint64(cmd_bits[64]), storage_ix, storage)
+					cmd_histo[64]++
+				} else {
+					emitDistance1(uint(distance), cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage)
+					last_distance = distance
+				}
+
+				emitCopyLenLastDistance1(matched, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage)
+
+				next_emit = ip
+				if ip >= ip_limit {
+					goto emit_remainder
+				}
+
+				/* We could immediately start working at ip now, but to improve
+				   compression we first update "table" with the hashes of some positions
+				   within the last copy. */
+				{
+					var input_bytes uint64 = binary.LittleEndian.Uint64(in[ip-3:])
+					var prev_hash uint32 = hashBytesAtOffset5(input_bytes, 0, shift)
+					var cur_hash uint32 = hashBytesAtOffset5(input_bytes, 3, shift)
+					table[prev_hash] = int(ip - base_ip - 3)
+					prev_hash = hashBytesAtOffset5(input_bytes, 1, shift)
+					table[prev_hash] = int(ip - base_ip - 2)
+					prev_hash = hashBytesAtOffset5(input_bytes, 2, shift)
+					table[prev_hash] = int(ip - base_ip - 1)
+
+					candidate = base_ip + table[cur_hash]
+					table[cur_hash] = int(ip - base_ip)
+				}
+			}
+
+			for isMatch5(in[ip:], in[candidate:]) {
+				var base int = ip
+				/* We have a 5-byte match at ip, and no need to emit any literal bytes
+				   prior to ip. */
+
+				var matched uint = 5 + findMatchLengthWithLimit(in[candidate+5:], in[ip+5:], uint(ip_end-ip)-5)
+				if ip-candidate > maxDistance_compress_fragment {
+					break
+				}
+				ip += int(matched)
+				last_distance = int(base - candidate) /* > 0 */
+				emitCopyLen1(matched, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage)
+				emitDistance1(uint(last_distance), cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage)
+
+				next_emit = ip
+				if ip >= ip_limit {
+					goto emit_remainder
+				}
+
+				/* We could immediately start working at ip now, but to improve
+				   compression we first update "table" with the hashes of some positions
+				   within the last copy. */
+				{
+					var input_bytes uint64 = binary.LittleEndian.Uint64(in[ip-3:])
+					var prev_hash uint32 = hashBytesAtOffset5(input_bytes, 0, shift)
+					var cur_hash uint32 = hashBytesAtOffset5(input_bytes, 3, shift)
+					table[prev_hash] = int(ip - base_ip - 3)
+					prev_hash = hashBytesAtOffset5(input_bytes, 1, shift)
+					table[prev_hash] = int(ip - base_ip - 2)
+					prev_hash = hashBytesAtOffset5(input_bytes, 2, shift)
+					table[prev_hash] = int(ip - base_ip - 1)
+
+					candidate = base_ip + table[cur_hash]
+					table[cur_hash] = int(ip - base_ip)
+				}
+			}
+
+			ip++
+			next_hash = hash5(in[ip:], shift)
+		}
+	}
+
+emit_remainder:
+	assert(next_emit <= ip_end)
+	input += int(block_size)
+	input_size -= block_size
+	block_size = brotli_min_size_t(input_size, compressFragmentFastImpl_kMergeBlockSize)
+
+	/* Decide if we want to continue this meta-block instead of emitting the
+	   last insert-only command. */
+	if input_size > 0 && total_block_size+block_size <= 1<<20 && shouldMergeBlock(in[input:], block_size, lit_depth[:]) {
+		assert(total_block_size > 1<<16)
+
+		/* Update the size of the current meta-block and continue emitting commands.
+		   We can do this because the current size and the new size both have 5
+		   nibbles. */
+		total_block_size += block_size
+
+		updateBits(20, uint32(total_block_size-1), mlen_storage_ix, storage)
+		goto emit_commands
+	}
+
+	/* Emit the remaining bytes as literals. */
+	if next_emit < ip_end {
+		var insert uint = uint(ip_end - next_emit)
+		if insert < 6210 {
+			emitInsertLen1(insert, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage)
+			emitLiterals(in[next_emit:], insert, lit_depth[:], lit_bits[:], storage_ix, storage)
+		} else if shouldUseUncompressedMode(in[metablock_start:], in[next_emit:], insert, literal_ratio) {
+			emitUncompressedMetaBlock1(in[metablock_start:], in[ip_end:], mlen_storage_ix-3, storage_ix, storage)
+		} else {
+			emitLongInsertLen(insert, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage)
+			emitLiterals(in[next_emit:], insert, lit_depth[:], lit_bits[:], storage_ix, storage)
+		}
+	}
+
+	next_emit = ip_end
+
+	/* If we have more data, write a new meta-block header and prefix codes and
+	   then continue emitting commands. */
+next_block:
+	if input_size > 0 {
+		metablock_start = input
+		block_size = brotli_min_size_t(input_size, compressFragmentFastImpl_kFirstBlockSize)
+		total_block_size = block_size
+
+		/* Save the bit position of the MLEN field of the meta-block header, so that
+		   we can update it later if we decide to extend this meta-block. */
+		mlen_storage_ix = *storage_ix + 3
+
+		storeMetaBlockHeader1(block_size, false, storage_ix, storage)
+
+		/* No block splits, no contexts. */
+		writeBits(13, 0, storage_ix, storage)
+
+		literal_ratio = buildAndStoreLiteralPrefixCode(in[input:], block_size, lit_depth[:], lit_bits[:], storage_ix, storage)
+		buildAndStoreCommandPrefixCode1(cmd_histo[:], cmd_depth, cmd_bits, storage_ix, storage)
+		goto emit_commands
+	}
+
+	if !is_last {
+		/* If this is not the last block, update the command and distance prefix
+		   codes for the next block and store the compressed forms. */
+		cmd_code[0] = 0
+
+		*cmd_code_numbits = 0
+		buildAndStoreCommandPrefixCode1(cmd_histo[:], cmd_depth, cmd_bits, cmd_code_numbits, cmd_code)
+	}
+}
+
+/* Compresses "input" string to the "*storage" buffer as one or more complete
+   meta-blocks, and updates the "*storage_ix" bit position.
+
+   If "is_last" is 1, emits an additional empty last meta-block.
+
+   "cmd_depth" and "cmd_bits" contain the command and distance prefix codes
+   (see comment in encode.h) used for the encoding of this input fragment.
+   If "is_last" is 0, they are updated to reflect the statistics
+   of this input fragment, to be used for the encoding of the next fragment.
+
+   "*cmd_code_numbits" is the number of bits of the compressed representation
+   of the command and distance prefix codes, and "cmd_code" is an array of
+   at least "(*cmd_code_numbits + 7) >> 3" size that contains the compressed
+   command and distance prefix codes. If "is_last" is 0, these are also
+   updated to represent the updated "cmd_depth" and "cmd_bits".
+
+   REQUIRES: "input_size" is greater than zero, or "is_last" is 1.
+   REQUIRES: "input_size" is less or equal to maximal metablock size (1 << 24).
+   REQUIRES: All elements in "table[0..table_size-1]" are initialized to zero.
+   REQUIRES: "table_size" is an odd (9, 11, 13, 15) power of two
+   OUTPUT: maximal copy distance <= |input_size|
+   OUTPUT: maximal copy distance <= BROTLI_MAX_BACKWARD_LIMIT(18) */
+func compressFragmentFast(input []byte, input_size uint, is_last bool, table []int, table_size uint, cmd_depth []byte, cmd_bits []uint16, cmd_code_numbits *uint, cmd_code []byte, storage_ix *uint, storage []byte) {
+	var initial_storage_ix uint = *storage_ix
+	var table_bits uint = uint(log2FloorNonZero(table_size))
+
+	if input_size == 0 {
+		assert(is_last)
+		writeBits(1, 1, storage_ix, storage) /* islast */
+		writeBits(1, 1, storage_ix, storage) /* isempty */
+		*storage_ix = (*storage_ix + 7) &^ 7
+		return
+	}
+
+	compressFragmentFastImpl(input, input_size, is_last, table, table_bits, cmd_depth, cmd_bits, cmd_code_numbits, cmd_code, storage_ix, storage)
+
+	/* If output is larger than single uncompressed block, rewrite it. */
+	if *storage_ix-initial_storage_ix > 31+(input_size<<3) {
+		emitUncompressedMetaBlock1(input, input[input_size:], initial_storage_ix, storage_ix, storage)
+	}
+
+	if is_last {
+		writeBits(1, 1, storage_ix, storage) /* islast */
+		writeBits(1, 1, storage_ix, storage) /* isempty */
+		*storage_ix = (*storage_ix + 7) &^ 7
+	}
+}

vendor/github.com/andybalholm/brotli/compress_fragment_two_pass.go

@@ -0,0 +1,749 @@
+package brotli
+
+import "encoding/binary"
+
+/* Copyright 2015 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* Function for fast encoding of an input fragment, independently from the input
+   history. This function uses two-pass processing: in the first pass we save
+   the found backward matches and literal bytes into a buffer, and in the
+   second pass we emit them into the bit stream using prefix codes built based
+   on the actual command and literal byte histograms. */
+
+const kCompressFragmentTwoPassBlockSize uint = 1 << 17
+
+func hash1(p []byte, shift uint, length uint) uint32 {
+	var h uint64 = (binary.LittleEndian.Uint64(p) << ((8 - length) * 8)) * uint64(kHashMul32)
+	return uint32(h >> shift)
+}
+
+func hashBytesAtOffset(v uint64, offset uint, shift uint, length uint) uint32 {
+	assert(offset <= 8-length)
+	{
+		var h uint64 = ((v >> (8 * offset)) << ((8 - length) * 8)) * uint64(kHashMul32)
+		return uint32(h >> shift)
+	}
+}
+
+func isMatch1(p1 []byte, p2 []byte, length uint) bool {
+	var i uint
+	for i = 0; i < length && i < 6; i++ {
+		if p1[i] != p2[i] {
+			return false
+		}
+	}
+
+	return true
+}
+
+/* Builds a command and distance prefix code (each 64 symbols) into "depth" and
+   "bits" based on "histogram" and stores it into the bit stream. */
+func buildAndStoreCommandPrefixCode(histogram []uint32, depth []byte, bits []uint16, storage_ix *uint, storage []byte) {
+	var tree [129]huffmanTree
+	var cmd_depth = [numCommandSymbols]byte{0}
+	/* Tree size for building a tree over 64 symbols is 2 * 64 + 1. */
+
+	var cmd_bits [64]uint16
+	createHuffmanTree(histogram, 64, 15, tree[:], depth)
+	createHuffmanTree(histogram[64:], 64, 14, tree[:], depth[64:])
+
+	/* We have to jump through a few hoops here in order to compute
+	   the command bits because the symbols are in a different order than in
+	   the full alphabet. This looks complicated, but having the symbols
+	   in this order in the command bits saves a few branches in the Emit*
+	   functions. */
+	copy(cmd_depth[:], depth[24:][:24])
+
+	copy(cmd_depth[24:][:], depth[:8])
+	copy(cmd_depth[32:][:], depth[48:][:8])
+	copy(cmd_depth[40:][:], depth[8:][:8])
+	copy(cmd_depth[48:][:], depth[56:][:8])
+	copy(cmd_depth[56:][:], depth[16:][:8])
+	convertBitDepthsToSymbols(cmd_depth[:], 64, cmd_bits[:])
+	copy(bits, cmd_bits[24:][:8])
+	copy(bits[8:], cmd_bits[40:][:8])
+	copy(bits[16:], cmd_bits[56:][:8])
+	copy(bits[24:], cmd_bits[:24])
+	copy(bits[48:], cmd_bits[32:][:8])
+	copy(bits[56:], cmd_bits[48:][:8])
+	convertBitDepthsToSymbols(depth[64:], 64, bits[64:])
+	{
+		/* Create the bit length array for the full command alphabet. */
+		var i uint
+		for i := 0; i < int(64); i++ {
+			cmd_depth[i] = 0
+		} /* only 64 first values were used */
+		copy(cmd_depth[:], depth[24:][:8])
+		copy(cmd_depth[64:][:], depth[32:][:8])
+		copy(cmd_depth[128:][:], depth[40:][:8])
+		copy(cmd_depth[192:][:], depth[48:][:8])
+		copy(cmd_depth[384:][:], depth[56:][:8])
+		for i = 0; i < 8; i++ {
+			cmd_depth[128+8*i] = depth[i]
+			cmd_depth[256+8*i] = depth[8+i]
+			cmd_depth[448+8*i] = depth[16+i]
+		}
+
+		storeHuffmanTree(cmd_depth[:], numCommandSymbols, tree[:], storage_ix, storage)
+	}
+
+	storeHuffmanTree(depth[64:], 64, tree[:], storage_ix, storage)
+}
+
+func emitInsertLen(insertlen uint32, commands *[]uint32) {
+	if insertlen < 6 {
+		(*commands)[0] = insertlen
+	} else if insertlen < 130 {
+		var tail uint32 = insertlen - 2
+		var nbits uint32 = log2FloorNonZero(uint(tail)) - 1
+		var prefix uint32 = tail >> nbits
+		var inscode uint32 = (nbits << 1) + prefix + 2
+		var extra uint32 = tail - (prefix << nbits)
+		(*commands)[0] = inscode | extra<<8
+	} else if insertlen < 2114 {
+		var tail uint32 = insertlen - 66
+		var nbits uint32 = log2FloorNonZero(uint(tail))
+		var code uint32 = nbits + 10
+		var extra uint32 = tail - (1 << nbits)
+		(*commands)[0] = code | extra<<8
+	} else if insertlen < 6210 {
+		var extra uint32 = insertlen - 2114
+		(*commands)[0] = 21 | extra<<8
+	} else if insertlen < 22594 {
+		var extra uint32 = insertlen - 6210
+		(*commands)[0] = 22 | extra<<8
+	} else {
+		var extra uint32 = insertlen - 22594
+		(*commands)[0] = 23 | extra<<8
+	}
+
+	*commands = (*commands)[1:]
+}
+
+func emitCopyLen(copylen uint, commands *[]uint32) {
+	if copylen < 10 {
+		(*commands)[0] = uint32(copylen + 38)
+	} else if copylen < 134 {
+		var tail uint = copylen - 6
+		var nbits uint = uint(log2FloorNonZero(tail) - 1)
+		var prefix uint = tail >> nbits
+		var code uint = (nbits << 1) + prefix + 44
+		var extra uint = tail - (prefix << nbits)
+		(*commands)[0] = uint32(code | extra<<8)
+	} else if copylen < 2118 {
+		var tail uint = copylen - 70
+		var nbits uint = uint(log2FloorNonZero(tail))
+		var code uint = nbits + 52
+		var extra uint = tail - (uint(1) << nbits)
+		(*commands)[0] = uint32(code | extra<<8)
+	} else {
+		var extra uint = copylen - 2118
+		(*commands)[0] = uint32(63 | extra<<8)
+	}
+
+	*commands = (*commands)[1:]
+}
+
+func emitCopyLenLastDistance(copylen uint, commands *[]uint32) {
+	if copylen < 12 {
+		(*commands)[0] = uint32(copylen + 20)
+		*commands = (*commands)[1:]
+	} else if copylen < 72 {
+		var tail uint = copylen - 8
+		var nbits uint = uint(log2FloorNonZero(tail) - 1)
+		var prefix uint = tail >> nbits
+		var code uint = (nbits << 1) + prefix + 28
+		var extra uint = tail - (prefix << nbits)
+		(*commands)[0] = uint32(code | extra<<8)
+		*commands = (*commands)[1:]
+	} else if copylen < 136 {
+		var tail uint = copylen - 8
+		var code uint = (tail >> 5) + 54
+		var extra uint = tail & 31
+		(*commands)[0] = uint32(code | extra<<8)
+		*commands = (*commands)[1:]
+		(*commands)[0] = 64
+		*commands = (*commands)[1:]
+	} else if copylen < 2120 {
+		var tail uint = copylen - 72
+		var nbits uint = uint(log2FloorNonZero(tail))
+		var code uint = nbits + 52
+		var extra uint = tail - (uint(1) << nbits)
+		(*commands)[0] = uint32(code | extra<<8)
+		*commands = (*commands)[1:]
+		(*commands)[0] = 64
+		*commands = (*commands)[1:]
+	} else {
+		var extra uint = copylen - 2120
+		(*commands)[0] = uint32(63 | extra<<8)
+		*commands = (*commands)[1:]
+		(*commands)[0] = 64
+		*commands = (*commands)[1:]
+	}
+}
+
+func emitDistance(distance uint32, commands *[]uint32) {
+	var d uint32 = distance + 3
+	var nbits uint32 = log2FloorNonZero(uint(d)) - 1
+	var prefix uint32 = (d >> nbits) & 1
+	var offset uint32 = (2 + prefix) << nbits
+	var distcode uint32 = 2*(nbits-1) + prefix + 80
+	var extra uint32 = d - offset
+	(*commands)[0] = distcode | extra<<8
+	*commands = (*commands)[1:]
+}
+
+/* REQUIRES: len <= 1 << 24. */
+func storeMetaBlockHeader(len uint, is_uncompressed bool, storage_ix *uint, storage []byte) {
+	var nibbles uint = 6
+
+	/* ISLAST */
+	writeBits(1, 0, storage_ix, storage)
+
+	if len <= 1<<16 {
+		nibbles = 4
+	} else if len <= 1<<20 {
+		nibbles = 5
+	}
+
+	writeBits(2, uint64(nibbles)-4, storage_ix, storage)
+	writeBits(nibbles*4, uint64(len)-1, storage_ix, storage)
+
+	/* ISUNCOMPRESSED */
+	writeSingleBit(is_uncompressed, storage_ix, storage)
+}
+
+func createCommands(input []byte, block_size uint, input_size uint, base_ip_ptr []byte, table []int, table_bits uint, min_match uint, literals *[]byte, commands *[]uint32) {
+	var ip int = 0
+	var shift uint = 64 - table_bits
+	var ip_end int = int(block_size)
+	var base_ip int = -cap(base_ip_ptr) + cap(input)
+	var next_emit int = 0
+	var last_distance int = -1
+	/* "ip" is the input pointer. */
+
+	const kInputMarginBytes uint = windowGap
+
+	/* "next_emit" is a pointer to the first byte that is not covered by a
+	   previous copy. Bytes between "next_emit" and the start of the next copy or
+	   the end of the input will be emitted as literal bytes. */
+	if block_size >= kInputMarginBytes {
+		var len_limit uint = brotli_min_size_t(block_size-min_match, input_size-kInputMarginBytes)
+		var ip_limit int = int(len_limit)
+		/* For the last block, we need to keep a 16 bytes margin so that we can be
+		   sure that all distances are at most window size - 16.
+		   For all other blocks, we only need to keep a margin of 5 bytes so that
+		   we don't go over the block size with a copy. */
+
+		var next_hash uint32
+		ip++
+		for next_hash = hash1(input[ip:], shift, min_match); ; {
+			var skip uint32 = 32
+			var next_ip int = ip
+			/* Step 1: Scan forward in the input looking for a 6-byte-long match.
+			   If we get close to exhausting the input then goto emit_remainder.
+
+			   Heuristic match skipping: If 32 bytes are scanned with no matches
+			   found, start looking only at every other byte. If 32 more bytes are
+			   scanned, look at every third byte, etc.. When a match is found,
+			   immediately go back to looking at every byte. This is a small loss
+			   (~5% performance, ~0.1% density) for compressible data due to more
+			   bookkeeping, but for non-compressible data (such as JPEG) it's a huge
+			   win since the compressor quickly "realizes" the data is incompressible
+			   and doesn't bother looking for matches everywhere.
+
+			   The "skip" variable keeps track of how many bytes there are since the
+			   last match; dividing it by 32 (ie. right-shifting by five) gives the
+			   number of bytes to move ahead for each iteration. */
+
+			var candidate int
+
+			assert(next_emit < ip)
+
+		trawl:
+			for {
+				var hash uint32 = next_hash
+				var bytes_between_hash_lookups uint32 = skip >> 5
+				skip++
+				ip = next_ip
+				assert(hash == hash1(input[ip:], shift, min_match))
+				next_ip = int(uint32(ip) + bytes_between_hash_lookups)
+				if next_ip > ip_limit {
+					goto emit_remainder
+				}
+
+				next_hash = hash1(input[next_ip:], shift, min_match)
+				candidate = ip - last_distance
+				if isMatch1(input[ip:], base_ip_ptr[candidate-base_ip:], min_match) {
+					if candidate < ip {
+						table[hash] = int(ip - base_ip)
+						break
+					}
+				}
+
+				candidate = base_ip + table[hash]
+				assert(candidate >= base_ip)
+				assert(candidate < ip)
+
+				table[hash] = int(ip - base_ip)
+				if isMatch1(input[ip:], base_ip_ptr[candidate-base_ip:], min_match) {
+					break
+				}
+			}
+
+			/* Check copy distance. If candidate is not feasible, continue search.
+			   Checking is done outside of hot loop to reduce overhead. */
+			if ip-candidate > maxDistance_compress_fragment {
+				goto trawl
+			}
+
+			/* Step 2: Emit the found match together with the literal bytes from
+			   "next_emit", and then see if we can find a next match immediately
+			   afterwards. Repeat until we find no match for the input
+			   without emitting some literal bytes. */
+			{
+				var base int = ip
+				/* > 0 */
+				var matched uint = min_match + findMatchLengthWithLimit(base_ip_ptr[uint(candidate-base_ip)+min_match:], input[uint(ip)+min_match:], uint(ip_end-ip)-min_match)
+				var distance int = int(base - candidate)
+				/* We have a 6-byte match at ip, and we need to emit bytes in
+				   [next_emit, ip). */
+
+				var insert int = int(base - next_emit)
+				ip += int(matched)
+				emitInsertLen(uint32(insert), commands)
+				copy(*literals, input[next_emit:][:uint(insert)])
+				*literals = (*literals)[insert:]
+				if distance == last_distance {
+					(*commands)[0] = 64
+					*commands = (*commands)[1:]
+				} else {
+					emitDistance(uint32(distance), commands)
+					last_distance = distance
+				}
+
+				emitCopyLenLastDistance(matched, commands)
+
+				next_emit = ip
+				if ip >= ip_limit {
+					goto emit_remainder
+				}
+				{
+					var input_bytes uint64
+					var cur_hash uint32
+					/* We could immediately start working at ip now, but to improve
+					   compression we first update "table" with the hashes of some
+					   positions within the last copy. */
+
+					var prev_hash uint32
+					if min_match == 4 {
+						input_bytes = binary.LittleEndian.Uint64(input[ip-3:])
+						cur_hash = hashBytesAtOffset(input_bytes, 3, shift, min_match)
+						prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match)
+						table[prev_hash] = int(ip - base_ip - 3)
+						prev_hash = hashBytesAtOffset(input_bytes, 1, shift, min_match)
+						table[prev_hash] = int(ip - base_ip - 2)
+						prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match)
+						table[prev_hash] = int(ip - base_ip - 1)
+					} else {
+						input_bytes = binary.LittleEndian.Uint64(input[ip-5:])
+						prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match)
+						table[prev_hash] = int(ip - base_ip - 5)
+						prev_hash = hashBytesAtOffset(input_bytes, 1, shift, min_match)
+						table[prev_hash] = int(ip - base_ip - 4)
+						prev_hash = hashBytesAtOffset(input_bytes, 2, shift, min_match)
+						table[prev_hash] = int(ip - base_ip - 3)
+						input_bytes = binary.LittleEndian.Uint64(input[ip-2:])
+						cur_hash = hashBytesAtOffset(input_bytes, 2, shift, min_match)
+						prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match)
+						table[prev_hash] = int(ip - base_ip - 2)
+						prev_hash = hashBytesAtOffset(input_bytes, 1, shift, min_match)
+						table[prev_hash] = int(ip - base_ip - 1)
+					}
+
+					candidate = base_ip + table[cur_hash]
+					table[cur_hash] = int(ip - base_ip)
+				}
+			}
+
+			for ip-candidate <= maxDistance_compress_fragment && isMatch1(input[ip:], base_ip_ptr[candidate-base_ip:], min_match) {
+				var base int = ip
+				/* We have a 6-byte match at ip, and no need to emit any
+				   literal bytes prior to ip. */
+
+				var matched uint = min_match + findMatchLengthWithLimit(base_ip_ptr[uint(candidate-base_ip)+min_match:], input[uint(ip)+min_match:], uint(ip_end-ip)-min_match)
+				ip += int(matched)
+				last_distance = int(base - candidate) /* > 0 */
+				emitCopyLen(matched, commands)
+				emitDistance(uint32(last_distance), commands)
+
+				next_emit = ip
+				if ip >= ip_limit {
+					goto emit_remainder
+				}
+				{
+					var input_bytes uint64
+					var cur_hash uint32
+					/* We could immediately start working at ip now, but to improve
+					   compression we first update "table" with the hashes of some
+					   positions within the last copy. */
+
+					var prev_hash uint32
+					if min_match == 4 {
+						input_bytes = binary.LittleEndian.Uint64(input[ip-3:])
+						cur_hash = hashBytesAtOffset(input_bytes, 3, shift, min_match)
+						prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match)
+						table[prev_hash] = int(ip - base_ip - 3)
+						prev_hash = hashBytesAtOffset(input_bytes, 1, shift, min_match)
+						table[prev_hash] = int(ip - base_ip - 2)
+						prev_hash = hashBytesAtOffset(input_bytes, 2, shift, min_match)
+						table[prev_hash] = int(ip - base_ip - 1)
+					} else {
+						input_bytes = binary.LittleEndian.Uint64(input[ip-5:])
+						prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match)
+						table[prev_hash] = int(ip - base_ip - 5)
+						prev_hash = hashBytesAtOffset(input_bytes, 1, shift, min_match)
+						table[prev_hash] = int(ip - base_ip - 4)
+						prev_hash = hashBytesAtOffset(input_bytes, 2, shift, min_match)
+						table[prev_hash] = int(ip - base_ip - 3)
+						input_bytes = binary.LittleEndian.Uint64(input[ip-2:])
+						cur_hash = hashBytesAtOffset(input_bytes, 2, shift, min_match)
+						prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match)
+						table[prev_hash] = int(ip - base_ip - 2)
+						prev_hash = hashBytesAtOffset(input_bytes, 1, shift, min_match)
+						table[prev_hash] = int(ip - base_ip - 1)
+					}
+
+					candidate = base_ip + table[cur_hash]
+					table[cur_hash] = int(ip - base_ip)
+				}
+			}
+
+			ip++
+			next_hash = hash1(input[ip:], shift, min_match)
+		}
+	}
+
+emit_remainder:
+	assert(next_emit <= ip_end)
+
+	/* Emit the remaining bytes as literals. */
+	if next_emit < ip_end {
+		var insert uint32 = uint32(ip_end - next_emit)
+		emitInsertLen(insert, commands)
+		copy(*literals, input[next_emit:][:insert])
+		*literals = (*literals)[insert:]
+	}
+}
+
+var storeCommands_kNumExtraBits = [128]uint32{
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	1,
+	1,
+	2,
+	2,
+	3,
+	3,
+	4,
+	4,
+	5,
+	5,
+	6,
+	7,
+	8,
+	9,
+	10,
+	12,
+	14,
+	24,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	1,
+	1,
+	2,
+	2,
+	3,
+	3,
+	4,
+	4,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	1,
+	1,
+	2,
+	2,
+	3,
+	3,
+	4,
+	4,
+	5,
+	5,
+	6,
+	7,
+	8,
+	9,
+	10,
+	24,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	1,
+	1,
+	2,
+	2,
+	3,
+	3,
+	4,
+	4,
+	5,
+	5,
+	6,
+	6,
+	7,
+	7,
+	8,
+	8,
+	9,
+	9,
+	10,
+	10,
+	11,
+	11,
+	12,
+	12,
+	13,
+	13,
+	14,
+	14,
+	15,
+	15,
+	16,
+	16,
+	17,
+	17,
+	18,
+	18,
+	19,
+	19,
+	20,
+	20,
+	21,
+	21,
+	22,
+	22,
+	23,
+	23,
+	24,
+	24,
+}
+var storeCommands_kInsertOffset = [24]uint32{
+	0,
+	1,
+	2,
+	3,
+	4,
+	5,
+	6,
+	8,
+	10,
+	14,
+	18,
+	26,
+	34,
+	50,
+	66,
+	98,
+	130,
+	194,
+	322,
+	578,
+	1090,
+	2114,
+	6210,
+	22594,
+}
+
+func storeCommands(literals []byte, num_literals uint, commands []uint32, num_commands uint, storage_ix *uint, storage []byte) {
+	var lit_depths [256]byte
+	var lit_bits [256]uint16
+	var lit_histo = [256]uint32{0}
+	var cmd_depths = [128]byte{0}
+	var cmd_bits = [128]uint16{0}
+	var cmd_histo = [128]uint32{0}
+	var i uint
+	for i = 0; i < num_literals; i++ {
+		lit_histo[literals[i]]++
+	}
+
+	buildAndStoreHuffmanTreeFast(lit_histo[:], num_literals, /* max_bits = */
+		8, lit_depths[:], lit_bits[:], storage_ix, storage)
+
+	for i = 0; i < num_commands; i++ {
+		var code uint32 = commands[i] & 0xFF
+		assert(code < 128)
+		cmd_histo[code]++
+	}
+
+	cmd_histo[1] += 1
+	cmd_histo[2] += 1
+	cmd_histo[64] += 1
+	cmd_histo[84] += 1
+	buildAndStoreCommandPrefixCode(cmd_histo[:], cmd_depths[:], cmd_bits[:], storage_ix, storage)
+
+	for i = 0; i < num_commands; i++ {
+		var cmd uint32 = commands[i]
+		var code uint32 = cmd & 0xFF
+		var extra uint32 = cmd >> 8
+		assert(code < 128)
+		writeBits(uint(cmd_depths[code]), uint64(cmd_bits[code]), storage_ix, storage)
+		writeBits(uint(storeCommands_kNumExtraBits[code]), uint64(extra), storage_ix, storage)
+		if code < 24 {
+			var insert uint32 = storeCommands_kInsertOffset[code] + extra
+			var j uint32
+			for j = 0; j < insert; j++ {
+				var lit byte = literals[0]
+				writeBits(uint(lit_depths[lit]), uint64(lit_bits[lit]), storage_ix, storage)
+				literals = literals[1:]
+			}
+		}
+	}
+}
+
+/* Acceptable loss for uncompressible speedup is 2% */
+const minRatio = 0.98
+
+const sampleRate = 43
+
+func shouldCompress(input []byte, input_size uint, num_literals uint) bool {
+	var corpus_size float64 = float64(input_size)
+	if float64(num_literals) < minRatio*corpus_size {
+		return true
+	} else {
+		var literal_histo = [256]uint32{0}
+		var max_total_bit_cost float64 = corpus_size * 8 * minRatio / sampleRate
+		var i uint
+		for i = 0; i < input_size; i += sampleRate {
+			literal_histo[input[i]]++
+		}
+
+		return bitsEntropy(literal_histo[:], 256) < max_total_bit_cost
+	}
+}
+
+func rewindBitPosition(new_storage_ix uint, storage_ix *uint, storage []byte) {
+	var bitpos uint = new_storage_ix & 7
+	var mask uint = (1 << bitpos) - 1
+	storage[new_storage_ix>>3] &= byte(mask)
+	*storage_ix = new_storage_ix
+}
+
+func emitUncompressedMetaBlock(input []byte, input_size uint, storage_ix *uint, storage []byte) {
+	storeMetaBlockHeader(input_size, true, storage_ix, storage)
+	*storage_ix = (*storage_ix + 7) &^ 7
+	copy(storage[*storage_ix>>3:], input[:input_size])
+	*storage_ix += input_size << 3
+	storage[*storage_ix>>3] = 0
+}
+
+func compressFragmentTwoPassImpl(input []byte, input_size uint, is_last bool, command_buf []uint32, literal_buf []byte, table []int, table_bits uint, min_match uint, storage_ix *uint, storage []byte) {
+	/* Save the start of the first block for position and distance computations.
+	 */
+	var base_ip []byte = input
+
+	for input_size > 0 {
+		var block_size uint = brotli_min_size_t(input_size, kCompressFragmentTwoPassBlockSize)
+		var commands []uint32 = command_buf
+		var literals []byte = literal_buf
+		var num_literals uint
+		createCommands(input, block_size, input_size, base_ip, table, table_bits, min_match, &literals, &commands)
+		num_literals = uint(-cap(literals) + cap(literal_buf))
+		if shouldCompress(input, block_size, num_literals) {
+			var num_commands uint = uint(-cap(commands) + cap(command_buf))
+			storeMetaBlockHeader(block_size, false, storage_ix, storage)
+
+			/* No block splits, no contexts. */
+			writeBits(13, 0, storage_ix, storage)
+
+			storeCommands(literal_buf, num_literals, command_buf, num_commands, storage_ix, storage)
+		} else {
+			/* Since we did not find many backward references and the entropy of
+			   the data is close to 8 bits, we can simply emit an uncompressed block.
+			   This makes compression speed of uncompressible data about 3x faster. */
+			emitUncompressedMetaBlock(input, block_size, storage_ix, storage)
+		}
+
+		input = input[block_size:]
+		input_size -= block_size
+	}
+}
+
+/* Compresses "input" string to the "*storage" buffer as one or more complete
+   meta-blocks, and updates the "*storage_ix" bit position.
+
+   If "is_last" is 1, emits an additional empty last meta-block.
+
+   REQUIRES: "input_size" is greater than zero, or "is_last" is 1.
+   REQUIRES: "input_size" is less or equal to maximal metablock size (1 << 24).
+   REQUIRES: "command_buf" and "literal_buf" point to at least
+              kCompressFragmentTwoPassBlockSize long arrays.
+   REQUIRES: All elements in "table[0..table_size-1]" are initialized to zero.
+   REQUIRES: "table_size" is a power of two
+   OUTPUT: maximal copy distance <= |input_size|
+   OUTPUT: maximal copy distance <= BROTLI_MAX_BACKWARD_LIMIT(18) */
+func compressFragmentTwoPass(input []byte, input_size uint, is_last bool, command_buf []uint32, literal_buf []byte, table []int, table_size uint, storage_ix *uint, storage []byte) {
+	var initial_storage_ix uint = *storage_ix
+	var table_bits uint = uint(log2FloorNonZero(table_size))
+	var min_match uint
+	if table_bits <= 15 {
+		min_match = 4
+	} else {
+		min_match = 6
+	}
+	compressFragmentTwoPassImpl(input, input_size, is_last, command_buf, literal_buf, table, table_bits, min_match, storage_ix, storage)
+
+	/* If output is larger than single uncompressed block, rewrite it. */
+	if *storage_ix-initial_storage_ix > 31+(input_size<<3) {
+		rewindBitPosition(initial_storage_ix, storage_ix, storage)
+		emitUncompressedMetaBlock(input, input_size, storage_ix, storage)
+	}
+
+	if is_last {
+		writeBits(1, 1, storage_ix, storage) /* islast */
+		writeBits(1, 1, storage_ix, storage) /* isempty */
+		*storage_ix = (*storage_ix + 7) &^ 7
+	}
+}

vendor/github.com/andybalholm/brotli/constants.go

@@ -0,0 +1,77 @@
+package brotli
+
+/* Copyright 2016 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* Specification: 7.3. Encoding of the context map */
+const contextMapMaxRle = 16
+
+/* Specification: 2. Compressed representation overview */
+const maxNumberOfBlockTypes = 256
+
+/* Specification: 3.3. Alphabet sizes: insert-and-copy length */
+const numLiteralSymbols = 256
+
+const numCommandSymbols = 704
+
+const numBlockLenSymbols = 26
+
+const maxContextMapSymbols = (maxNumberOfBlockTypes + contextMapMaxRle)
+
+const maxBlockTypeSymbols = (maxNumberOfBlockTypes + 2)
+
+/* Specification: 3.5. Complex prefix codes */
+const repeatPreviousCodeLength = 16
+
+const repeatZeroCodeLength = 17
+
+const codeLengthCodes = (repeatZeroCodeLength + 1)
+
+/* "code length of 8 is repeated" */
+const initialRepeatedCodeLength = 8
+
+/* "Large Window Brotli" */
+const largeMaxDistanceBits = 62
+
+const largeMinWbits = 10
+
+const largeMaxWbits = 30
+
+/* Specification: 4. Encoding of distances */
+const numDistanceShortCodes = 16
+
+const maxNpostfix = 3
+
+const maxNdirect = 120
+
+const maxDistanceBits = 24
+
+func distanceAlphabetSize(NPOSTFIX uint, NDIRECT uint, MAXNBITS uint) uint {
+	return numDistanceShortCodes + NDIRECT + uint(MAXNBITS<<(NPOSTFIX+1))
+}
+
+/* numDistanceSymbols == 1128 */
+const numDistanceSymbols = 1128
+
+const maxDistance = 0x3FFFFFC
+
+const maxAllowedDistance = 0x7FFFFFFC
+
+/* 7.1. Context modes and context ID lookup for literals */
+/* "context IDs for literals are in the range of 0..63" */
+const literalContextBits = 6
+
+/* 7.2. Context ID for distances */
+const distanceContextBits = 2
+
+/* 9.1. Format of the Stream Header */
+/* Number of slack bytes for window size. Don't confuse
+   with BROTLI_NUM_DISTANCE_SHORT_CODES. */
+const windowGap = 16
+
+func maxBackwardLimit(W uint) uint {
+	return (uint(1) << W) - windowGap
+}

vendor/github.com/andybalholm/brotli/encoder_dict.go

@@ -0,0 +1,22 @@
+package brotli
+
+/* Dictionary data (words and transforms) for 1 possible context */
+type encoderDictionary struct {
+	words                 *dictionary
+	cutoffTransformsCount uint32
+	cutoffTransforms      uint64
+	hash_table            []uint16
+	buckets               []uint16
+	dict_words            []dictWord
+}
+
+func initEncoderDictionary(dict *encoderDictionary) {
+	dict.words = getDictionary()
+
+	dict.hash_table = kStaticDictionaryHash[:]
+	dict.buckets = kStaticDictionaryBuckets[:]
+	dict.dict_words = kStaticDictionaryWords[:]
+
+	dict.cutoffTransformsCount = kCutoffTransformsCount
+	dict.cutoffTransforms = kCutoffTransforms
+}

vendor/github.com/andybalholm/brotli/entropy_encode.go

@@ -0,0 +1,593 @@
+package brotli
+
+import "math"
+
+/* Copyright 2010 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* Entropy encoding (Huffman) utilities. */
+
+/* A node of a Huffman tree. */
+type huffmanTree struct {
+	total_count_          uint32
+	index_left_           int16
+	index_right_or_value_ int16
+}
+
+func initHuffmanTree(self *huffmanTree, count uint32, left int16, right int16) {
+	self.total_count_ = count
+	self.index_left_ = left
+	self.index_right_or_value_ = right
+}
+
+/* Input size optimized Shell sort. */
+type huffmanTreeComparator func(*huffmanTree, *huffmanTree) bool
+
+var sortHuffmanTreeItems_gaps = []uint{132, 57, 23, 10, 4, 1}
+
+func sortHuffmanTreeItems(items []huffmanTree, n uint, comparator huffmanTreeComparator) {
+	if n < 13 {
+		/* Insertion sort. */
+		var i uint
+		for i = 1; i < n; i++ {
+			var tmp huffmanTree = items[i]
+			var k uint = i
+			var j uint = i - 1
+			for comparator(&tmp, &items[j]) {
+				items[k] = items[j]
+				k = j
+				tmp10 := j
+				j--
+				if tmp10 == 0 {
+					break
+				}
+			}
+
+			items[k] = tmp
+		}
+
+		return
+	} else {
+		var g int
+		if n < 57 {
+			g = 2
+		} else {
+			g = 0
+		}
+		for ; g < 6; g++ {
+			var gap uint = sortHuffmanTreeItems_gaps[g]
+			var i uint
+			for i = gap; i < n; i++ {
+				var j uint = i
+				var tmp huffmanTree = items[i]
+				for ; j >= gap && comparator(&tmp, &items[j-gap]); j -= gap {
+					items[j] = items[j-gap]
+				}
+
+				items[j] = tmp
+			}
+		}
+	}
+}
+
+/* Returns 1 if assignment of depths succeeded, otherwise 0. */
+func setDepth(p0 int, pool []huffmanTree, depth []byte, max_depth int) bool {
+	var stack [16]int
+	var level int = 0
+	var p int = p0
+	assert(max_depth <= 15)
+	stack[0] = -1
+	for {
+		if pool[p].index_left_ >= 0 {
+			level++
+			if level > max_depth {
+				return false
+			}
+			stack[level] = int(pool[p].index_right_or_value_)
+			p = int(pool[p].index_left_)
+			continue
+		} else {
+			depth[pool[p].index_right_or_value_] = byte(level)
+		}
+
+		for level >= 0 && stack[level] == -1 {
+			level--
+		}
+		if level < 0 {
+			return true
+		}
+		p = stack[level]
+		stack[level] = -1
+	}
+}
+
+/* Sort the root nodes, least popular first. */
+func sortHuffmanTree(v0 *huffmanTree, v1 *huffmanTree) bool {
+	if v0.total_count_ != v1.total_count_ {
+		return v0.total_count_ < v1.total_count_
+	}
+
+	return v0.index_right_or_value_ > v1.index_right_or_value_
+}
+
+/* This function will create a Huffman tree.
+
+   The catch here is that the tree cannot be arbitrarily deep.
+   Brotli specifies a maximum depth of 15 bits for "code trees"
+   and 7 bits for "code length code trees."
+
+   count_limit is the value that is to be faked as the minimum value
+   and this minimum value is raised until the tree matches the
+   maximum length requirement.
+
+   This algorithm is not of excellent performance for very long data blocks,
+   especially when population counts are longer than 2**tree_limit, but
+   we are not planning to use this with extremely long blocks.
+
+   See http://en.wikipedia.org/wiki/Huffman_coding */
+func createHuffmanTree(data []uint32, length uint, tree_limit int, tree []huffmanTree, depth []byte) {
+	var count_limit uint32
+	var sentinel huffmanTree
+	initHuffmanTree(&sentinel, math.MaxUint32, -1, -1)
+
+	/* For block sizes below 64 kB, we never need to do a second iteration
+	   of this loop. Probably all of our block sizes will be smaller than
+	   that, so this loop is mostly of academic interest. If we actually
+	   would need this, we would be better off with the Katajainen algorithm. */
+	for count_limit = 1; ; count_limit *= 2 {
+		var n uint = 0
+		var i uint
+		var j uint
+		var k uint
+		for i = length; i != 0; {
+			i--
+			if data[i] != 0 {
+				var count uint32 = brotli_max_uint32_t(data[i], count_limit)
+				initHuffmanTree(&tree[n], count, -1, int16(i))
+				n++
+			}
+		}
+
+		if n == 1 {
+			depth[tree[0].index_right_or_value_] = 1 /* Only one element. */
+			break
+		}
+
+		sortHuffmanTreeItems(tree, n, huffmanTreeComparator(sortHuffmanTree))
+
+		/* The nodes are:
+		   [0, n): the sorted leaf nodes that we start with.
+		   [n]: we add a sentinel here.
+		   [n + 1, 2n): new parent nodes are added here, starting from
+		                (n+1). These are naturally in ascending order.
+		   [2n]: we add a sentinel at the end as well.
+		   There will be (2n+1) elements at the end. */
+		tree[n] = sentinel
+
+		tree[n+1] = sentinel
+
+		i = 0     /* Points to the next leaf node. */
+		j = n + 1 /* Points to the next non-leaf node. */
+		for k = n - 1; k != 0; k-- {
+			var left uint
+			var right uint
+			if tree[i].total_count_ <= tree[j].total_count_ {
+				left = i
+				i++
+			} else {
+				left = j
+				j++
+			}
+
+			if tree[i].total_count_ <= tree[j].total_count_ {
+				right = i
+				i++
+			} else {
+				right = j
+				j++
+			}
+			{
+				/* The sentinel node becomes the parent node. */
+				var j_end uint = 2*n - k
+				tree[j_end].total_count_ = tree[left].total_count_ + tree[right].total_count_
+				tree[j_end].index_left_ = int16(left)
+				tree[j_end].index_right_or_value_ = int16(right)
+
+				/* Add back the last sentinel node. */
+				tree[j_end+1] = sentinel
+			}
+		}
+
+		if setDepth(int(2*n-1), tree[0:], depth, tree_limit) {
+			/* We need to pack the Huffman tree in tree_limit bits. If this was not
+			   successful, add fake entities to the lowest values and retry. */
+			break
+		}
+	}
+}
+
+func reverse(v []byte, start uint, end uint) {
+	end--
+	for start < end {
+		var tmp byte = v[start]
+		v[start] = v[end]
+		v[end] = tmp
+		start++
+		end--
+	}
+}
+
+func writeHuffmanTreeRepetitions(previous_value byte, value byte, repetitions uint, tree_size *uint, tree []byte, extra_bits_data []byte) {
+	assert(repetitions > 0)
+	if previous_value != value {
+		tree[*tree_size] = value
+		extra_bits_data[*tree_size] = 0
+		(*tree_size)++
+		repetitions--
+	}
+
+	if repetitions == 7 {
+		tree[*tree_size] = value
+		extra_bits_data[*tree_size] = 0
+		(*tree_size)++
+		repetitions--
+	}
+
+	if repetitions < 3 {
+		var i uint
+		for i = 0; i < repetitions; i++ {
+			tree[*tree_size] = value
+			extra_bits_data[*tree_size] = 0
+			(*tree_size)++
+		}
+	} else {
+		var start uint = *tree_size
+		repetitions -= 3
+		for {
+			tree[*tree_size] = repeatPreviousCodeLength
+			extra_bits_data[*tree_size] = byte(repetitions & 0x3)
+			(*tree_size)++
+			repetitions >>= 2
+			if repetitions == 0 {
+				break
+			}
+
+			repetitions--
+		}
+
+		reverse(tree, start, *tree_size)
+		reverse(extra_bits_data, start, *tree_size)
+	}
+}
+
+func writeHuffmanTreeRepetitionsZeros(repetitions uint, tree_size *uint, tree []byte, extra_bits_data []byte) {
+	if repetitions == 11 {
+		tree[*tree_size] = 0
+		extra_bits_data[*tree_size] = 0
+		(*tree_size)++
+		repetitions--
+	}
+
+	if repetitions < 3 {
+		var i uint
+		for i = 0; i < repetitions; i++ {
+			tree[*tree_size] = 0
+			extra_bits_data[*tree_size] = 0
+			(*tree_size)++
+		}
+	} else {
+		var start uint = *tree_size
+		repetitions -= 3
+		for {
+			tree[*tree_size] = repeatZeroCodeLength
+			extra_bits_data[*tree_size] = byte(repetitions & 0x7)
+			(*tree_size)++
+			repetitions >>= 3
+			if repetitions == 0 {
+				break
+			}
+
+			repetitions--
+		}
+
+		reverse(tree, start, *tree_size)
+		reverse(extra_bits_data, start, *tree_size)
+	}
+}
+
+/* Change the population counts in a way that the consequent
+   Huffman tree compression, especially its RLE-part will be more
+   likely to compress this data more efficiently.
+
+   length contains the size of the histogram.
+   counts contains the population counts.
+   good_for_rle is a buffer of at least length size */
+func optimizeHuffmanCountsForRLE(length uint, counts []uint32, good_for_rle []byte) {
+	var nonzero_count uint = 0
+	var stride uint
+	var limit uint
+	var sum uint
+	var streak_limit uint = 1240
+	var i uint
+	/* Let's make the Huffman code more compatible with RLE encoding. */
+	for i = 0; i < length; i++ {
+		if counts[i] != 0 {
+			nonzero_count++
+		}
+	}
+
+	if nonzero_count < 16 {
+		return
+	}
+
+	for length != 0 && counts[length-1] == 0 {
+		length--
+	}
+
+	if length == 0 {
+		return /* All zeros. */
+	}
+
+	/* Now counts[0..length - 1] does not have trailing zeros. */
+	{
+		var nonzeros uint = 0
+		var smallest_nonzero uint32 = 1 << 30
+		for i = 0; i < length; i++ {
+			if counts[i] != 0 {
+				nonzeros++
+				if smallest_nonzero > counts[i] {
+					smallest_nonzero = counts[i]
+				}
+			}
+		}
+
+		if nonzeros < 5 {
+			/* Small histogram will model it well. */
+			return
+		}
+
+		if smallest_nonzero < 4 {
+			var zeros uint = length - nonzeros
+			if zeros < 6 {
+				for i = 1; i < length-1; i++ {
+					if counts[i-1] != 0 && counts[i] == 0 && counts[i+1] != 0 {
+						counts[i] = 1
+					}
+				}
+			}
+		}
+
+		if nonzeros < 28 {
+			return
+		}
+	}
+
+	/* 2) Let's mark all population counts that already can be encoded
+	   with an RLE code. */
+	for i := 0; i < int(length); i++ {
+		good_for_rle[i] = 0
+	}
+	{
+		var symbol uint32 = counts[0]
+		/* Let's not spoil any of the existing good RLE codes.
+		   Mark any seq of 0's that is longer as 5 as a good_for_rle.
+		   Mark any seq of non-0's that is longer as 7 as a good_for_rle. */
+
+		var step uint = 0
+		for i = 0; i <= length; i++ {
+			if i == length || counts[i] != symbol {
+				if (symbol == 0 && step >= 5) || (symbol != 0 && step >= 7) {
+					var k uint
+					for k = 0; k < step; k++ {
+						good_for_rle[i-k-1] = 1
+					}
+				}
+
+				step = 1
+				if i != length {
+					symbol = counts[i]
+				}
+			} else {
+				step++
+			}
+		}
+	}
+
+	/* 3) Let's replace those population counts that lead to more RLE codes.
+	   Math here is in 24.8 fixed point representation. */
+	stride = 0
+
+	limit = uint(256*(counts[0]+counts[1]+counts[2])/3 + 420)
+	sum = 0
+	for i = 0; i <= length; i++ {
+		if i == length || good_for_rle[i] != 0 || (i != 0 && good_for_rle[i-1] != 0) || (256*counts[i]-uint32(limit)+uint32(streak_limit)) >= uint32(2*streak_limit) {
+			if stride >= 4 || (stride >= 3 && sum == 0) {
+				var k uint
+				var count uint = (sum + stride/2) / stride
+				/* The stride must end, collapse what we have, if we have enough (4). */
+				if count == 0 {
+					count = 1
+				}
+
+				if sum == 0 {
+					/* Don't make an all zeros stride to be upgraded to ones. */
+					count = 0
+				}
+
+				for k = 0; k < stride; k++ {
+					/* We don't want to change value at counts[i],
+					   that is already belonging to the next stride. Thus - 1. */
+					counts[i-k-1] = uint32(count)
+				}
+			}
+
+			stride = 0
+			sum = 0
+			if i < length-2 {
+				/* All interesting strides have a count of at least 4, */
+				/* at least when non-zeros. */
+				limit = uint(256*(counts[i]+counts[i+1]+counts[i+2])/3 + 420)
+			} else if i < length {
+				limit = uint(256 * counts[i])
+			} else {
+				limit = 0
+			}
+		}
+
+		stride++
+		if i != length {
+			sum += uint(counts[i])
+			if stride >= 4 {
+				limit = (256*sum + stride/2) / stride
+			}
+
+			if stride == 4 {
+				limit += 120
+			}
+		}
+	}
+}
+
+func decideOverRLEUse(depth []byte, length uint, use_rle_for_non_zero *bool, use_rle_for_zero *bool) {
+	var total_reps_zero uint = 0
+	var total_reps_non_zero uint = 0
+	var count_reps_zero uint = 1
+	var count_reps_non_zero uint = 1
+	var i uint
+	for i = 0; i < length; {
+		var value byte = depth[i]
+		var reps uint = 1
+		var k uint
+		for k = i + 1; k < length && depth[k] == value; k++ {
+			reps++
+		}
+
+		if reps >= 3 && value == 0 {
+			total_reps_zero += reps
+			count_reps_zero++
+		}
+
+		if reps >= 4 && value != 0 {
+			total_reps_non_zero += reps
+			count_reps_non_zero++
+		}
+
+		i += reps
+	}
+
+	*use_rle_for_non_zero = total_reps_non_zero > count_reps_non_zero*2
+	*use_rle_for_zero = total_reps_zero > count_reps_zero*2
+}
+
+/* Write a Huffman tree from bit depths into the bit-stream representation
+   of a Huffman tree. The generated Huffman tree is to be compressed once
+   more using a Huffman tree */
+func writeHuffmanTree(depth []byte, length uint, tree_size *uint, tree []byte, extra_bits_data []byte) {
+	var previous_value byte = initialRepeatedCodeLength
+	var i uint
+	var use_rle_for_non_zero bool = false
+	var use_rle_for_zero bool = false
+	var new_length uint = length
+	/* Throw away trailing zeros. */
+	for i = 0; i < length; i++ {
+		if depth[length-i-1] == 0 {
+			new_length--
+		} else {
+			break
+		}
+	}
+
+	/* First gather statistics on if it is a good idea to do RLE. */
+	if length > 50 {
+		/* Find RLE coding for longer codes.
+		   Shorter codes seem not to benefit from RLE. */
+		decideOverRLEUse(depth, new_length, &use_rle_for_non_zero, &use_rle_for_zero)
+	}
+
+	/* Actual RLE coding. */
+	for i = 0; i < new_length; {
+		var value byte = depth[i]
+		var reps uint = 1
+		if (value != 0 && use_rle_for_non_zero) || (value == 0 && use_rle_for_zero) {
+			var k uint
+			for k = i + 1; k < new_length && depth[k] == value; k++ {
+				reps++
+			}
+		}
+
+		if value == 0 {
+			writeHuffmanTreeRepetitionsZeros(reps, tree_size, tree, extra_bits_data)
+		} else {
+			writeHuffmanTreeRepetitions(previous_value, value, reps, tree_size, tree, extra_bits_data)
+			previous_value = value
+		}
+
+		i += reps
+	}
+}
+
+var reverseBits_kLut = [16]uint{
+	0x00,
+	0x08,
+	0x04,
+	0x0C,
+	0x02,
+	0x0A,
+	0x06,
+	0x0E,
+	0x01,
+	0x09,
+	0x05,
+	0x0D,
+	0x03,
+	0x0B,
+	0x07,
+	0x0F,
+}
+
+func reverseBits(num_bits uint, bits uint16) uint16 {
+	var retval uint = reverseBits_kLut[bits&0x0F]
+	var i uint
+	for i = 4; i < num_bits; i += 4 {
+		retval <<= 4
+		bits = uint16(bits >> 4)
+		retval |= reverseBits_kLut[bits&0x0F]
+	}
+
+	retval >>= ((0 - num_bits) & 0x03)
+	return uint16(retval)
+}
+
+/* 0..15 are values for bits */
+const maxHuffmanBits = 16
+
+/* Get the actual bit values for a tree of bit depths. */
+func convertBitDepthsToSymbols(depth []byte, len uint, bits []uint16) {
+	var bl_count = [maxHuffmanBits]uint16{0}
+	var next_code [maxHuffmanBits]uint16
+	var i uint
+	/* In Brotli, all bit depths are [1..15]
+	   0 bit depth means that the symbol does not exist. */
+
+	var code int = 0
+	for i = 0; i < len; i++ {
+		bl_count[depth[i]]++
+	}
+
+	bl_count[0] = 0
+	next_code[0] = 0
+	for i = 1; i < maxHuffmanBits; i++ {
+		code = (code + int(bl_count[i-1])) << 1
+		next_code[i] = uint16(code)
+	}
+
+	for i = 0; i < len; i++ {
+		if depth[i] != 0 {
+			bits[i] = reverseBits(uint(depth[i]), next_code[depth[i]])
+			next_code[depth[i]]++
+		}
+	}
+}

vendor/github.com/andybalholm/brotli/fast_log.go

@@ -0,0 +1,296 @@
+package brotli
+
+import "math"
+
+/* Copyright 2013 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* Utilities for fast computation of logarithms. */
+
+func log2FloorNonZero(n uint) uint32 {
+	/* TODO: generalize and move to platform.h */
+	var result uint32 = 0
+	for {
+		n >>= 1
+		if n == 0 {
+			break
+		}
+		result++
+	}
+	return result
+}
+
+/* A lookup table for small values of log2(int) to be used in entropy
+   computation.
+
+   ", ".join(["%.16ff" % x for x in [0.0]+[log2(x) for x in range(1, 256)]]) */
+var kLog2Table = []float32{
+	0.0000000000000000,
+	0.0000000000000000,
+	1.0000000000000000,
+	1.5849625007211563,
+	2.0000000000000000,
+	2.3219280948873622,
+	2.5849625007211561,
+	2.8073549220576042,
+	3.0000000000000000,
+	3.1699250014423126,
+	3.3219280948873626,
+	3.4594316186372978,
+	3.5849625007211565,
+	3.7004397181410922,
+	3.8073549220576037,
+	3.9068905956085187,
+	4.0000000000000000,
+	4.0874628412503400,
+	4.1699250014423122,
+	4.2479275134435852,
+	4.3219280948873626,
+	4.3923174227787607,
+	4.4594316186372973,
+	4.5235619560570131,
+	4.5849625007211570,
+	4.6438561897747244,
+	4.7004397181410926,
+	4.7548875021634691,
+	4.8073549220576037,
+	4.8579809951275728,
+	4.9068905956085187,
+	4.9541963103868758,
+	5.0000000000000000,
+	5.0443941193584534,
+	5.0874628412503400,
+	5.1292830169449664,
+	5.1699250014423122,
+	5.2094533656289501,
+	5.2479275134435852,
+	5.2854022188622487,
+	5.3219280948873626,
+	5.3575520046180838,
+	5.3923174227787607,
+	5.4262647547020979,
+	5.4594316186372973,
+	5.4918530963296748,
+	5.5235619560570131,
+	5.5545888516776376,
+	5.5849625007211570,
+	5.6147098441152083,
+	5.6438561897747244,
+	5.6724253419714961,
+	5.7004397181410926,
+	5.7279204545631996,
+	5.7548875021634691,
+	5.7813597135246599,
+	5.8073549220576046,
+	5.8328900141647422,
+	5.8579809951275719,
+	5.8826430493618416,
+	5.9068905956085187,
+	5.9307373375628867,
+	5.9541963103868758,
+	5.9772799234999168,
+	6.0000000000000000,
+	6.0223678130284544,
+	6.0443941193584534,
+	6.0660891904577721,
+	6.0874628412503400,
+	6.1085244567781700,
+	6.1292830169449672,
+	6.1497471195046822,
+	6.1699250014423122,
+	6.1898245588800176,
+	6.2094533656289510,
+	6.2288186904958804,
+	6.2479275134435861,
+	6.2667865406949019,
+	6.2854022188622487,
+	6.3037807481771031,
+	6.3219280948873617,
+	6.3398500028846252,
+	6.3575520046180847,
+	6.3750394313469254,
+	6.3923174227787598,
+	6.4093909361377026,
+	6.4262647547020979,
+	6.4429434958487288,
+	6.4594316186372982,
+	6.4757334309663976,
+	6.4918530963296748,
+	6.5077946401986964,
+	6.5235619560570131,
+	6.5391588111080319,
+	6.5545888516776376,
+	6.5698556083309478,
+	6.5849625007211561,
+	6.5999128421871278,
+	6.6147098441152092,
+	6.6293566200796095,
+	6.6438561897747253,
+	6.6582114827517955,
+	6.6724253419714952,
+	6.6865005271832185,
+	6.7004397181410917,
+	6.7142455176661224,
+	6.7279204545631988,
+	6.7414669864011465,
+	6.7548875021634691,
+	6.7681843247769260,
+	6.7813597135246599,
+	6.7944158663501062,
+	6.8073549220576037,
+	6.8201789624151887,
+	6.8328900141647422,
+	6.8454900509443757,
+	6.8579809951275719,
+	6.8703647195834048,
+	6.8826430493618416,
+	6.8948177633079437,
+	6.9068905956085187,
+	6.9188632372745955,
+	6.9307373375628867,
+	6.9425145053392399,
+	6.9541963103868758,
+	6.9657842846620879,
+	6.9772799234999168,
+	6.9886846867721664,
+	7.0000000000000000,
+	7.0112272554232540,
+	7.0223678130284544,
+	7.0334230015374501,
+	7.0443941193584534,
+	7.0552824355011898,
+	7.0660891904577721,
+	7.0768155970508317,
+	7.0874628412503400,
+	7.0980320829605272,
+	7.1085244567781700,
+	7.1189410727235076,
+	7.1292830169449664,
+	7.1395513523987937,
+	7.1497471195046822,
+	7.1598713367783891,
+	7.1699250014423130,
+	7.1799090900149345,
+	7.1898245588800176,
+	7.1996723448363644,
+	7.2094533656289492,
+	7.2191685204621621,
+	7.2288186904958804,
+	7.2384047393250794,
+	7.2479275134435861,
+	7.2573878426926521,
+	7.2667865406949019,
+	7.2761244052742384,
+	7.2854022188622487,
+	7.2946207488916270,
+	7.3037807481771031,
+	7.3128829552843557,
+	7.3219280948873617,
+	7.3309168781146177,
+	7.3398500028846243,
+	7.3487281542310781,
+	7.3575520046180847,
+	7.3663222142458151,
+	7.3750394313469254,
+	7.3837042924740528,
+	7.3923174227787607,
+	7.4008794362821844,
+	7.4093909361377026,
+	7.4178525148858991,
+	7.4262647547020979,
+	7.4346282276367255,
+	7.4429434958487288,
+	7.4512111118323299,
+	7.4594316186372973,
+	7.4676055500829976,
+	7.4757334309663976,
+	7.4838157772642564,
+	7.4918530963296748,
+	7.4998458870832057,
+	7.5077946401986964,
+	7.5156998382840436,
+	7.5235619560570131,
+	7.5313814605163119,
+	7.5391588111080319,
+	7.5468944598876373,
+	7.5545888516776376,
+	7.5622424242210728,
+	7.5698556083309478,
+	7.5774288280357487,
+	7.5849625007211561,
+	7.5924570372680806,
+	7.5999128421871278,
+	7.6073303137496113,
+	7.6147098441152075,
+	7.6220518194563764,
+	7.6293566200796095,
+	7.6366246205436488,
+	7.6438561897747244,
+	7.6510516911789290,
+	7.6582114827517955,
+	7.6653359171851765,
+	7.6724253419714952,
+	7.6794800995054464,
+	7.6865005271832185,
+	7.6934869574993252,
+	7.7004397181410926,
+	7.7073591320808825,
+	7.7142455176661224,
+	7.7210991887071856,
+	7.7279204545631996,
+	7.7347096202258392,
+	7.7414669864011465,
+	7.7481928495894596,
+	7.7548875021634691,
+	7.7615512324444795,
+	7.7681843247769260,
+	7.7747870596011737,
+	7.7813597135246608,
+	7.7879025593914317,
+	7.7944158663501062,
+	7.8008998999203047,
+	7.8073549220576037,
+	7.8137811912170374,
+	7.8201789624151887,
+	7.8265484872909159,
+	7.8328900141647422,
+	7.8392037880969445,
+	7.8454900509443757,
+	7.8517490414160571,
+	7.8579809951275719,
+	7.8641861446542798,
+	7.8703647195834048,
+	7.8765169465650002,
+	7.8826430493618425,
+	7.8887432488982601,
+	7.8948177633079446,
+	7.9008668079807496,
+	7.9068905956085187,
+	7.9128893362299619,
+	7.9188632372745955,
+	7.9248125036057813,
+	7.9307373375628867,
+	7.9366379390025719,
+	7.9425145053392399,
+	7.9483672315846778,
+	7.9541963103868758,
+	7.9600019320680806,
+	7.9657842846620870,
+	7.9715435539507720,
+	7.9772799234999168,
+	7.9829935746943104,
+	7.9886846867721664,
+	7.9943534368588578,
+}
+
+/* Faster logarithm for small integers, with the property of log2(0) == 0. */
+func fastLog2(v uint) float64 {
+	if v < uint(len(kLog2Table)) {
+		return float64(kLog2Table[v])
+	}
+
+	return math.Log2(float64(v))
+}

vendor/github.com/andybalholm/brotli/find_match_length.go

@@ -0,0 +1,16 @@
+package brotli
+
+/* Copyright 2010 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* Function to find maximal matching prefixes of strings. */
+func findMatchLengthWithLimit(s1 []byte, s2 []byte, limit uint) uint {
+	var matched uint = 0
+	for matched < limit && s1[matched] == s2[matched] {
+		matched++
+	}
+	return matched
+}

vendor/github.com/andybalholm/brotli/go.mod

@@ -0,0 +1,5 @@
+module github.com/andybalholm/brotli
+
+go 1.12
+
+require github.com/golang/gddo v0.0.0-20190419222130-af0f2af80721

vendor/github.com/andybalholm/brotli/go.sum

@@ -0,0 +1,2 @@
+github.com/golang/gddo v0.0.0-20190419222130-af0f2af80721 h1:KRMr9A3qfbVM7iV/WcLY/rL5LICqwMHLhwRXKu99fXw=
+github.com/golang/gddo v0.0.0-20190419222130-af0f2af80721/go.mod h1:xEhNfoBDX1hzLm2Nf80qUvZ2sVwoMZ8d6IE2SrsQfh4=

vendor/github.com/andybalholm/brotli/h10.go

@@ -0,0 +1,287 @@
+package brotli
+
+import "encoding/binary"
+
+/* Copyright 2016 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+func (*h10) HashTypeLength() uint {
+	return 4
+}
+
+func (*h10) StoreLookahead() uint {
+	return 128
+}
+
+func hashBytesH10(data []byte) uint32 {
+	var h uint32 = binary.LittleEndian.Uint32(data) * kHashMul32
+
+	/* The higher bits contain more mixture from the multiplication,
+	   so we take our results from there. */
+	return h >> (32 - 17)
+}
+
+/* A (forgetful) hash table where each hash bucket contains a binary tree of
+   sequences whose first 4 bytes share the same hash code.
+   Each sequence is 128 long and is identified by its starting
+   position in the input data. The binary tree is sorted by the lexicographic
+   order of the sequences, and it is also a max-heap with respect to the
+   starting positions. */
+type h10 struct {
+	hasherCommon
+	window_mask_ uint
+	buckets_     [1 << 17]uint32
+	invalid_pos_ uint32
+	forest       []uint32
+}
+
+func (h *h10) Initialize(params *encoderParams) {
+	h.window_mask_ = (1 << params.lgwin) - 1
+	h.invalid_pos_ = uint32(0 - h.window_mask_)
+	var num_nodes uint = uint(1) << params.lgwin
+	h.forest = make([]uint32, 2*num_nodes)
+}
+
+func (h *h10) Prepare(one_shot bool, input_size uint, data []byte) {
+	var invalid_pos uint32 = h.invalid_pos_
+	var i uint32
+	for i = 0; i < 1<<17; i++ {
+		h.buckets_[i] = invalid_pos
+	}
+}
+
+func leftChildIndexH10(self *h10, pos uint) uint {
+	return 2 * (pos & self.window_mask_)
+}
+
+func rightChildIndexH10(self *h10, pos uint) uint {
+	return 2*(pos&self.window_mask_) + 1
+}
+
+/* Stores the hash of the next 4 bytes and in a single tree-traversal, the
+   hash bucket's binary tree is searched for matches and is re-rooted at the
+   current position.
+
+   If less than 128 data is available, the hash bucket of the
+   current position is searched for matches, but the state of the hash table
+   is not changed, since we can not know the final sorting order of the
+   current (incomplete) sequence.
+
+   This function must be called with increasing cur_ix positions. */
+func storeAndFindMatchesH10(self *h10, data []byte, cur_ix uint, ring_buffer_mask uint, max_length uint, max_backward uint, best_len *uint, matches []backwardMatch) []backwardMatch {
+	var cur_ix_masked uint = cur_ix & ring_buffer_mask
+	var max_comp_len uint = brotli_min_size_t(max_length, 128)
+	var should_reroot_tree bool = (max_length >= 128)
+	var key uint32 = hashBytesH10(data[cur_ix_masked:])
+	var forest []uint32 = self.forest
+	var prev_ix uint = uint(self.buckets_[key])
+	var node_left uint = leftChildIndexH10(self, cur_ix)
+	var node_right uint = rightChildIndexH10(self, cur_ix)
+	var best_len_left uint = 0
+	var best_len_right uint = 0
+	var depth_remaining uint
+	/* The forest index of the rightmost node of the left subtree of the new
+	   root, updated as we traverse and re-root the tree of the hash bucket. */
+
+	/* The forest index of the leftmost node of the right subtree of the new
+	   root, updated as we traverse and re-root the tree of the hash bucket. */
+
+	/* The match length of the rightmost node of the left subtree of the new
+	   root, updated as we traverse and re-root the tree of the hash bucket. */
+
+	/* The match length of the leftmost node of the right subtree of the new
+	   root, updated as we traverse and re-root the tree of the hash bucket. */
+	if should_reroot_tree {
+		self.buckets_[key] = uint32(cur_ix)
+	}
+
+	for depth_remaining = 64; ; depth_remaining-- {
+		var backward uint = cur_ix - prev_ix
+		var prev_ix_masked uint = prev_ix & ring_buffer_mask
+		if backward == 0 || backward > max_backward || depth_remaining == 0 {
+			if should_reroot_tree {
+				forest[node_left] = self.invalid_pos_
+				forest[node_right] = self.invalid_pos_
+			}
+
+			break
+		}
+		{
+			var cur_len uint = brotli_min_size_t(best_len_left, best_len_right)
+			var len uint
+			assert(cur_len <= 128)
+			len = cur_len + findMatchLengthWithLimit(data[cur_ix_masked+cur_len:], data[prev_ix_masked+cur_len:], max_length-cur_len)
+			if matches != nil && len > *best_len {
+				*best_len = uint(len)
+				initBackwardMatch(&matches[0], backward, uint(len))
+				matches = matches[1:]
+			}
+
+			if len >= max_comp_len {
+				if should_reroot_tree {
+					forest[node_left] = forest[leftChildIndexH10(self, prev_ix)]
+					forest[node_right] = forest[rightChildIndexH10(self, prev_ix)]
+				}
+
+				break
+			}
+
+			if data[cur_ix_masked+len] > data[prev_ix_masked+len] {
+				best_len_left = uint(len)
+				if should_reroot_tree {
+					forest[node_left] = uint32(prev_ix)
+				}
+
+				node_left = rightChildIndexH10(self, prev_ix)
+				prev_ix = uint(forest[node_left])
+			} else {
+				best_len_right = uint(len)
+				if should_reroot_tree {
+					forest[node_right] = uint32(prev_ix)
+				}
+
+				node_right = leftChildIndexH10(self, prev_ix)
+				prev_ix = uint(forest[node_right])
+			}
+		}
+	}
+
+	return matches
+}
+
+/* Finds all backward matches of &data[cur_ix & ring_buffer_mask] up to the
+   length of max_length and stores the position cur_ix in the hash table.
+
+   Sets *num_matches to the number of matches found, and stores the found
+   matches in matches[0] to matches[*num_matches - 1]. The matches will be
+   sorted by strictly increasing length and (non-strictly) increasing
+   distance. */
+func findAllMatchesH10(handle *h10, dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, cur_ix uint, max_length uint, max_backward uint, gap uint, params *encoderParams, matches []backwardMatch) uint {
+	var orig_matches []backwardMatch = matches
+	var cur_ix_masked uint = cur_ix & ring_buffer_mask
+	var best_len uint = 1
+	var short_match_max_backward uint
+	if params.quality != hqZopflificationQuality {
+		short_match_max_backward = 16
+	} else {
+		short_match_max_backward = 64
+	}
+	var stop uint = cur_ix - short_match_max_backward
+	var dict_matches [maxStaticDictionaryMatchLen + 1]uint32
+	var i uint
+	if cur_ix < short_match_max_backward {
+		stop = 0
+	}
+	for i = cur_ix - 1; i > stop && best_len <= 2; i-- {
+		var prev_ix uint = i
+		var backward uint = cur_ix - prev_ix
+		if backward > max_backward {
+			break
+		}
+
+		prev_ix &= ring_buffer_mask
+		if data[cur_ix_masked] != data[prev_ix] || data[cur_ix_masked+1] != data[prev_ix+1] {
+			continue
+		}
+		{
+			var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
+			if len > best_len {
+				best_len = uint(len)
+				initBackwardMatch(&matches[0], backward, uint(len))
+				matches = matches[1:]
+			}
+		}
+	}
+
+	if best_len < max_length {
+		matches = storeAndFindMatchesH10(handle, data, cur_ix, ring_buffer_mask, max_length, max_backward, &best_len, matches)
+	}
+
+	for i = 0; i <= maxStaticDictionaryMatchLen; i++ {
+		dict_matches[i] = kInvalidMatch
+	}
+	{
+		var minlen uint = brotli_max_size_t(4, best_len+1)
+		if findAllStaticDictionaryMatches(dictionary, data[cur_ix_masked:], minlen, max_length, dict_matches[0:]) {
+			var maxlen uint = brotli_min_size_t(maxStaticDictionaryMatchLen, max_length)
+			var l uint
+			for l = minlen; l <= maxlen; l++ {
+				var dict_id uint32 = dict_matches[l]
+				if dict_id < kInvalidMatch {
+					var distance uint = max_backward + gap + uint(dict_id>>5) + 1
+					if distance <= params.dist.max_distance {
+						initDictionaryBackwardMatch(&matches[0], distance, l, uint(dict_id&31))
+						matches = matches[1:]
+					}
+				}
+			}
+		}
+	}
+
+	return uint(-cap(matches) + cap(orig_matches))
+}
+
+/* Stores the hash of the next 4 bytes and re-roots the binary tree at the
+   current sequence, without returning any matches.
+   REQUIRES: ix + 128 <= end-of-current-block */
+func (h *h10) Store(data []byte, mask uint, ix uint) {
+	var max_backward uint = h.window_mask_ - windowGap + 1
+	/* Maximum distance is window size - 16, see section 9.1. of the spec. */
+	storeAndFindMatchesH10(h, data, ix, mask, 128, max_backward, nil, nil)
+}
+
+func (h *h10) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) {
+	var i uint = ix_start
+	var j uint = ix_start
+	if ix_start+63 <= ix_end {
+		i = ix_end - 63
+	}
+
+	if ix_start+512 <= i {
+		for ; j < i; j += 8 {
+			h.Store(data, mask, j)
+		}
+	}
+
+	for ; i < ix_end; i++ {
+		h.Store(data, mask, i)
+	}
+}
+
+func (h *h10) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint) {
+	if num_bytes >= h.HashTypeLength()-1 && position >= 128 {
+		var i_start uint = position - 128 + 1
+		var i_end uint = brotli_min_size_t(position, i_start+num_bytes)
+		/* Store the last `128 - 1` positions in the hasher.
+		   These could not be calculated before, since they require knowledge
+		   of both the previous and the current block. */
+
+		var i uint
+		for i = i_start; i < i_end; i++ {
+			/* Maximum distance is window size - 16, see section 9.1. of the spec.
+			   Furthermore, we have to make sure that we don't look further back
+			   from the start of the next block than the window size, otherwise we
+			   could access already overwritten areas of the ring-buffer. */
+			var max_backward uint = h.window_mask_ - brotli_max_size_t(windowGap-1, position-i)
+
+			/* We know that i + 128 <= position + num_bytes, i.e. the
+			   end of the current block and that we have at least
+			   128 tail in the ring-buffer. */
+			storeAndFindMatchesH10(h, ringbuffer, i, ringbuffer_mask, 128, max_backward, nil, nil)
+		}
+	}
+}
+
+/* MAX_NUM_MATCHES == 64 + MAX_TREE_SEARCH_DEPTH */
+const maxNumMatchesH10 = 128
+
+func (*h10) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) {
+	panic("unimplemented")
+}
+
+func (*h10) PrepareDistanceCache(distance_cache []int) {
+	panic("unimplemented")
+}

vendor/github.com/andybalholm/brotli/h5.go

@@ -0,0 +1,214 @@
+package brotli
+
+import "encoding/binary"
+
+/* Copyright 2010 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* A (forgetful) hash table to the data seen by the compressor, to
+   help create backward references to previous data.
+
+   This is a hash map of fixed size (bucket_size_) to a ring buffer of
+   fixed size (block_size_). The ring buffer contains the last block_size_
+   index positions of the given hash key in the compressed data. */
+func (*h5) HashTypeLength() uint {
+	return 4
+}
+
+func (*h5) StoreLookahead() uint {
+	return 4
+}
+
+/* HashBytes is the function that chooses the bucket to place the address in. */
+func hashBytesH5(data []byte, shift int) uint32 {
+	var h uint32 = binary.LittleEndian.Uint32(data) * kHashMul32
+
+	/* The higher bits contain more mixture from the multiplication,
+	   so we take our results from there. */
+	return uint32(h >> uint(shift))
+}
+
+type h5 struct {
+	hasherCommon
+	bucket_size_ uint
+	block_size_  uint
+	hash_shift_  int
+	block_mask_  uint32
+	num          []uint16
+	buckets      []uint32
+}
+
+func (h *h5) Initialize(params *encoderParams) {
+	h.hash_shift_ = 32 - h.params.bucket_bits
+	h.bucket_size_ = uint(1) << uint(h.params.bucket_bits)
+	h.block_size_ = uint(1) << uint(h.params.block_bits)
+	h.block_mask_ = uint32(h.block_size_ - 1)
+	h.num = make([]uint16, h.bucket_size_)
+	h.buckets = make([]uint32, h.block_size_*h.bucket_size_)
+}
+
+func (h *h5) Prepare(one_shot bool, input_size uint, data []byte) {
+	var num []uint16 = h.num
+	var partial_prepare_threshold uint = h.bucket_size_ >> 6
+	/* Partial preparation is 100 times slower (per socket). */
+	if one_shot && input_size <= partial_prepare_threshold {
+		var i uint
+		for i = 0; i < input_size; i++ {
+			var key uint32 = hashBytesH5(data[i:], h.hash_shift_)
+			num[key] = 0
+		}
+	} else {
+		for i := 0; i < int(h.bucket_size_); i++ {
+			num[i] = 0
+		}
+	}
+}
+
+/* Look at 4 bytes at &data[ix & mask].
+   Compute a hash from these, and store the value of ix at that position. */
+func (h *h5) Store(data []byte, mask uint, ix uint) {
+	var num []uint16 = h.num
+	var key uint32 = hashBytesH5(data[ix&mask:], h.hash_shift_)
+	var minor_ix uint = uint(num[key]) & uint(h.block_mask_)
+	var offset uint = minor_ix + uint(key<<uint(h.params.block_bits))
+	h.buckets[offset] = uint32(ix)
+	num[key]++
+}
+
+func (h *h5) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) {
+	var i uint
+	for i = ix_start; i < ix_end; i++ {
+		h.Store(data, mask, i)
+	}
+}
+
+func (h *h5) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint) {
+	if num_bytes >= h.HashTypeLength()-1 && position >= 3 {
+		/* Prepare the hashes for three last bytes of the last write.
+		   These could not be calculated before, since they require knowledge
+		   of both the previous and the current block. */
+		h.Store(ringbuffer, ringbuffer_mask, position-3)
+		h.Store(ringbuffer, ringbuffer_mask, position-2)
+		h.Store(ringbuffer, ringbuffer_mask, position-1)
+	}
+}
+
+func (h *h5) PrepareDistanceCache(distance_cache []int) {
+	prepareDistanceCache(distance_cache, h.params.num_last_distances_to_check)
+}
+
+/* Find a longest backward match of &data[cur_ix] up to the length of
+   max_length and stores the position cur_ix in the hash table.
+
+   REQUIRES: PrepareDistanceCacheH5 must be invoked for current distance cache
+             values; if this method is invoked repeatedly with the same distance
+             cache values, it is enough to invoke PrepareDistanceCacheH5 once.
+
+   Does not look for matches longer than max_length.
+   Does not look for matches further away than max_backward.
+   Writes the best match into |out|.
+   |out|->score is updated only if a better match is found. */
+func (h *h5) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) {
+	var num []uint16 = h.num
+	var buckets []uint32 = h.buckets
+	var cur_ix_masked uint = cur_ix & ring_buffer_mask
+	var min_score uint = out.score
+	var best_score uint = out.score
+	var best_len uint = out.len
+	var i uint
+	var bucket []uint32
+	/* Don't accept a short copy from far away. */
+	out.len = 0
+
+	out.len_code_delta = 0
+
+	/* Try last distance first. */
+	for i = 0; i < uint(h.params.num_last_distances_to_check); i++ {
+		var backward uint = uint(distance_cache[i])
+		var prev_ix uint = uint(cur_ix - backward)
+		if prev_ix >= cur_ix {
+			continue
+		}
+
+		if backward > max_backward {
+			continue
+		}
+
+		prev_ix &= ring_buffer_mask
+
+		if cur_ix_masked+best_len > ring_buffer_mask || prev_ix+best_len > ring_buffer_mask || data[cur_ix_masked+best_len] != data[prev_ix+best_len] {
+			continue
+		}
+		{
+			var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
+			if len >= 3 || (len == 2 && i < 2) {
+				/* Comparing for >= 2 does not change the semantics, but just saves for
+				   a few unnecessary binary logarithms in backward reference score,
+				   since we are not interested in such short matches. */
+				var score uint = backwardReferenceScoreUsingLastDistance(uint(len))
+				if best_score < score {
+					if i != 0 {
+						score -= backwardReferencePenaltyUsingLastDistance(i)
+					}
+					if best_score < score {
+						best_score = score
+						best_len = uint(len)
+						out.len = best_len
+						out.distance = backward
+						out.score = best_score
+					}
+				}
+			}
+		}
+	}
+	{
+		var key uint32 = hashBytesH5(data[cur_ix_masked:], h.hash_shift_)
+		bucket = buckets[key<<uint(h.params.block_bits):]
+		var down uint
+		if uint(num[key]) > h.block_size_ {
+			down = uint(num[key]) - h.block_size_
+		} else {
+			down = 0
+		}
+		for i = uint(num[key]); i > down; {
+			var prev_ix uint
+			i--
+			prev_ix = uint(bucket[uint32(i)&h.block_mask_])
+			var backward uint = cur_ix - prev_ix
+			if backward > max_backward {
+				break
+			}
+
+			prev_ix &= ring_buffer_mask
+			if cur_ix_masked+best_len > ring_buffer_mask || prev_ix+best_len > ring_buffer_mask || data[cur_ix_masked+best_len] != data[prev_ix+best_len] {
+				continue
+			}
+			{
+				var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
+				if len >= 4 {
+					/* Comparing for >= 3 does not change the semantics, but just saves
+					   for a few unnecessary binary logarithms in backward reference
+					   score, since we are not interested in such short matches. */
+					var score uint = backwardReferenceScore(uint(len), backward)
+					if best_score < score {
+						best_score = score
+						best_len = uint(len)
+						out.len = best_len
+						out.distance = backward
+						out.score = best_score
+					}
+				}
+			}
+		}
+
+		bucket[uint32(num[key])&h.block_mask_] = uint32(cur_ix)
+		num[key]++
+	}
+
+	if min_score == out.score {
+		searchInStaticDictionary(dictionary, h, data[cur_ix_masked:], max_length, max_backward+gap, max_distance, out, false)
+	}
+}

vendor/github.com/andybalholm/brotli/h6.go

@@ -0,0 +1,216 @@
+package brotli
+
+import "encoding/binary"
+
+/* Copyright 2010 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* A (forgetful) hash table to the data seen by the compressor, to
+   help create backward references to previous data.
+
+   This is a hash map of fixed size (bucket_size_) to a ring buffer of
+   fixed size (block_size_). The ring buffer contains the last block_size_
+   index positions of the given hash key in the compressed data. */
+func (*h6) HashTypeLength() uint {
+	return 8
+}
+
+func (*h6) StoreLookahead() uint {
+	return 8
+}
+
+/* HashBytes is the function that chooses the bucket to place the address in. */
+func hashBytesH6(data []byte, mask uint64, shift int) uint32 {
+	var h uint64 = (binary.LittleEndian.Uint64(data) & mask) * kHashMul64Long
+
+	/* The higher bits contain more mixture from the multiplication,
+	   so we take our results from there. */
+	return uint32(h >> uint(shift))
+}
+
+type h6 struct {
+	hasherCommon
+	bucket_size_ uint
+	block_size_  uint
+	hash_shift_  int
+	hash_mask_   uint64
+	block_mask_  uint32
+	num          []uint16
+	buckets      []uint32
+}
+
+func (h *h6) Initialize(params *encoderParams) {
+	h.hash_shift_ = 64 - h.params.bucket_bits
+	h.hash_mask_ = (^(uint64(0))) >> uint(64-8*h.params.hash_len)
+	h.bucket_size_ = uint(1) << uint(h.params.bucket_bits)
+	h.block_size_ = uint(1) << uint(h.params.block_bits)
+	h.block_mask_ = uint32(h.block_size_ - 1)
+	h.num = make([]uint16, h.bucket_size_)
+	h.buckets = make([]uint32, h.block_size_*h.bucket_size_)
+}
+
+func (h *h6) Prepare(one_shot bool, input_size uint, data []byte) {
+	var num []uint16 = h.num
+	var partial_prepare_threshold uint = h.bucket_size_ >> 6
+	/* Partial preparation is 100 times slower (per socket). */
+	if one_shot && input_size <= partial_prepare_threshold {
+		var i uint
+		for i = 0; i < input_size; i++ {
+			var key uint32 = hashBytesH6(data[i:], h.hash_mask_, h.hash_shift_)
+			num[key] = 0
+		}
+	} else {
+		for i := 0; i < int(h.bucket_size_); i++ {
+			num[i] = 0
+		}
+	}
+}
+
+/* Look at 4 bytes at &data[ix & mask].
+   Compute a hash from these, and store the value of ix at that position. */
+func (h *h6) Store(data []byte, mask uint, ix uint) {
+	var num []uint16 = h.num
+	var key uint32 = hashBytesH6(data[ix&mask:], h.hash_mask_, h.hash_shift_)
+	var minor_ix uint = uint(num[key]) & uint(h.block_mask_)
+	var offset uint = minor_ix + uint(key<<uint(h.params.block_bits))
+	h.buckets[offset] = uint32(ix)
+	num[key]++
+}
+
+func (h *h6) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) {
+	var i uint
+	for i = ix_start; i < ix_end; i++ {
+		h.Store(data, mask, i)
+	}
+}
+
+func (h *h6) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint) {
+	if num_bytes >= h.HashTypeLength()-1 && position >= 3 {
+		/* Prepare the hashes for three last bytes of the last write.
+		   These could not be calculated before, since they require knowledge
+		   of both the previous and the current block. */
+		h.Store(ringbuffer, ringbuffer_mask, position-3)
+		h.Store(ringbuffer, ringbuffer_mask, position-2)
+		h.Store(ringbuffer, ringbuffer_mask, position-1)
+	}
+}
+
+func (h *h6) PrepareDistanceCache(distance_cache []int) {
+	prepareDistanceCache(distance_cache, h.params.num_last_distances_to_check)
+}
+
+/* Find a longest backward match of &data[cur_ix] up to the length of
+   max_length and stores the position cur_ix in the hash table.
+
+   REQUIRES: PrepareDistanceCacheH6 must be invoked for current distance cache
+             values; if this method is invoked repeatedly with the same distance
+             cache values, it is enough to invoke PrepareDistanceCacheH6 once.
+
+   Does not look for matches longer than max_length.
+   Does not look for matches further away than max_backward.
+   Writes the best match into |out|.
+   |out|->score is updated only if a better match is found. */
+func (h *h6) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) {
+	var num []uint16 = h.num
+	var buckets []uint32 = h.buckets
+	var cur_ix_masked uint = cur_ix & ring_buffer_mask
+	var min_score uint = out.score
+	var best_score uint = out.score
+	var best_len uint = out.len
+	var i uint
+	var bucket []uint32
+	/* Don't accept a short copy from far away. */
+	out.len = 0
+
+	out.len_code_delta = 0
+
+	/* Try last distance first. */
+	for i = 0; i < uint(h.params.num_last_distances_to_check); i++ {
+		var backward uint = uint(distance_cache[i])
+		var prev_ix uint = uint(cur_ix - backward)
+		if prev_ix >= cur_ix {
+			continue
+		}
+
+		if backward > max_backward {
+			continue
+		}
+
+		prev_ix &= ring_buffer_mask
+
+		if cur_ix_masked+best_len > ring_buffer_mask || prev_ix+best_len > ring_buffer_mask || data[cur_ix_masked+best_len] != data[prev_ix+best_len] {
+			continue
+		}
+		{
+			var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
+			if len >= 3 || (len == 2 && i < 2) {
+				/* Comparing for >= 2 does not change the semantics, but just saves for
+				   a few unnecessary binary logarithms in backward reference score,
+				   since we are not interested in such short matches. */
+				var score uint = backwardReferenceScoreUsingLastDistance(uint(len))
+				if best_score < score {
+					if i != 0 {
+						score -= backwardReferencePenaltyUsingLastDistance(i)
+					}
+					if best_score < score {
+						best_score = score
+						best_len = uint(len)
+						out.len = best_len
+						out.distance = backward
+						out.score = best_score
+					}
+				}
+			}
+		}
+	}
+	{
+		var key uint32 = hashBytesH6(data[cur_ix_masked:], h.hash_mask_, h.hash_shift_)
+		bucket = buckets[key<<uint(h.params.block_bits):]
+		var down uint
+		if uint(num[key]) > h.block_size_ {
+			down = uint(num[key]) - h.block_size_
+		} else {
+			down = 0
+		}
+		for i = uint(num[key]); i > down; {
+			var prev_ix uint
+			i--
+			prev_ix = uint(bucket[uint32(i)&h.block_mask_])
+			var backward uint = cur_ix - prev_ix
+			if backward > max_backward {
+				break
+			}
+
+			prev_ix &= ring_buffer_mask
+			if cur_ix_masked+best_len > ring_buffer_mask || prev_ix+best_len > ring_buffer_mask || data[cur_ix_masked+best_len] != data[prev_ix+best_len] {
+				continue
+			}
+			{
+				var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
+				if len >= 4 {
+					/* Comparing for >= 3 does not change the semantics, but just saves
+					   for a few unnecessary binary logarithms in backward reference
+					   score, since we are not interested in such short matches. */
+					var score uint = backwardReferenceScore(uint(len), backward)
+					if best_score < score {
+						best_score = score
+						best_len = uint(len)
+						out.len = best_len
+						out.distance = backward
+						out.score = best_score
+					}
+				}
+			}
+		}
+
+		bucket[uint32(num[key])&h.block_mask_] = uint32(cur_ix)
+		num[key]++
+	}
+
+	if min_score == out.score {
+		searchInStaticDictionary(dictionary, h, data[cur_ix_masked:], max_length, max_backward+gap, max_distance, out, false)
+	}
+}

vendor/github.com/andybalholm/brotli/hash.go

@@ -0,0 +1,344 @@
+package brotli
+
+import (
+	"encoding/binary"
+	"fmt"
+)
+
+type hasherCommon struct {
+	params           hasherParams
+	is_prepared_     bool
+	dict_num_lookups uint
+	dict_num_matches uint
+}
+
+func (h *hasherCommon) Common() *hasherCommon {
+	return h
+}
+
+type hasherHandle interface {
+	Common() *hasherCommon
+	Initialize(params *encoderParams)
+	Prepare(one_shot bool, input_size uint, data []byte)
+	StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint)
+	HashTypeLength() uint
+	StoreLookahead() uint
+	PrepareDistanceCache(distance_cache []int)
+	FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult)
+	StoreRange(data []byte, mask uint, ix_start uint, ix_end uint)
+	Store(data []byte, mask uint, ix uint)
+}
+
+type score_t uint
+
+const kCutoffTransformsCount uint32 = 10
+
+/*   0,  12,   27,    23,    42,    63,    56,    48,    59,    64 */
+/* 0+0, 4+8, 8+19, 12+11, 16+26, 20+43, 24+32, 28+20, 32+27, 36+28 */
+const kCutoffTransforms uint64 = 0x071B520ADA2D3200
+
+type hasherSearchResult struct {
+	len            uint
+	distance       uint
+	score          uint
+	len_code_delta int
+}
+
+/* kHashMul32 multiplier has these properties:
+   * The multiplier must be odd. Otherwise we may lose the highest bit.
+   * No long streaks of ones or zeros.
+   * There is no effort to ensure that it is a prime, the oddity is enough
+     for this use.
+   * The number has been tuned heuristically against compression benchmarks. */
+const kHashMul32 uint32 = 0x1E35A7BD
+
+const kHashMul64 uint64 = 0x1E35A7BD1E35A7BD
+
+const kHashMul64Long uint64 = 0x1FE35A7BD3579BD3
+
+func hash14(data []byte) uint32 {
+	var h uint32 = binary.LittleEndian.Uint32(data) * kHashMul32
+
+	/* The higher bits contain more mixture from the multiplication,
+	   so we take our results from there. */
+	return h >> (32 - 14)
+}
+
+func prepareDistanceCache(distance_cache []int, num_distances int) {
+	if num_distances > 4 {
+		var last_distance int = distance_cache[0]
+		distance_cache[4] = last_distance - 1
+		distance_cache[5] = last_distance + 1
+		distance_cache[6] = last_distance - 2
+		distance_cache[7] = last_distance + 2
+		distance_cache[8] = last_distance - 3
+		distance_cache[9] = last_distance + 3
+		if num_distances > 10 {
+			var next_last_distance int = distance_cache[1]
+			distance_cache[10] = next_last_distance - 1
+			distance_cache[11] = next_last_distance + 1
+			distance_cache[12] = next_last_distance - 2
+			distance_cache[13] = next_last_distance + 2
+			distance_cache[14] = next_last_distance - 3
+			distance_cache[15] = next_last_distance + 3
+		}
+	}
+}
+
+const literalByteScore = 135
+
+const distanceBitPenalty = 30
+
+/* Score must be positive after applying maximal penalty. */
+const scoreBase = (distanceBitPenalty * 8 * 8)
+
+/* Usually, we always choose the longest backward reference. This function
+   allows for the exception of that rule.
+
+   If we choose a backward reference that is further away, it will
+   usually be coded with more bits. We approximate this by assuming
+   log2(distance). If the distance can be expressed in terms of the
+   last four distances, we use some heuristic constants to estimate
+   the bits cost. For the first up to four literals we use the bit
+   cost of the literals from the literal cost model, after that we
+   use the average bit cost of the cost model.
+
+   This function is used to sometimes discard a longer backward reference
+   when it is not much longer and the bit cost for encoding it is more
+   than the saved literals.
+
+   backward_reference_offset MUST be positive. */
+func backwardReferenceScore(copy_length uint, backward_reference_offset uint) uint {
+	return scoreBase + literalByteScore*uint(copy_length) - distanceBitPenalty*uint(log2FloorNonZero(backward_reference_offset))
+}
+
+func backwardReferenceScoreUsingLastDistance(copy_length uint) uint {
+	return literalByteScore*uint(copy_length) + scoreBase + 15
+}
+
+func backwardReferencePenaltyUsingLastDistance(distance_short_code uint) uint {
+	return uint(39) + ((0x1CA10 >> (distance_short_code & 0xE)) & 0xE)
+}
+
+func testStaticDictionaryItem(dictionary *encoderDictionary, item uint, data []byte, max_length uint, max_backward uint, max_distance uint, out *hasherSearchResult) bool {
+	var len uint
+	var word_idx uint
+	var offset uint
+	var matchlen uint
+	var backward uint
+	var score uint
+	len = item & 0x1F
+	word_idx = item >> 5
+	offset = uint(dictionary.words.offsets_by_length[len]) + len*word_idx
+	if len > max_length {
+		return false
+	}
+
+	matchlen = findMatchLengthWithLimit(data, dictionary.words.data[offset:], uint(len))
+	if matchlen+uint(dictionary.cutoffTransformsCount) <= len || matchlen == 0 {
+		return false
+	}
+	{
+		var cut uint = len - matchlen
+		var transform_id uint = (cut << 2) + uint((dictionary.cutoffTransforms>>(cut*6))&0x3F)
+		backward = max_backward + 1 + word_idx + (transform_id << dictionary.words.size_bits_by_length[len])
+	}
+
+	if backward > max_distance {
+		return false
+	}
+
+	score = backwardReferenceScore(matchlen, backward)
+	if score < out.score {
+		return false
+	}
+
+	out.len = matchlen
+	out.len_code_delta = int(len) - int(matchlen)
+	out.distance = backward
+	out.score = score
+	return true
+}
+
+func searchInStaticDictionary(dictionary *encoderDictionary, handle hasherHandle, data []byte, max_length uint, max_backward uint, max_distance uint, out *hasherSearchResult, shallow bool) {
+	var key uint
+	var i uint
+	var self *hasherCommon = handle.Common()
+	if self.dict_num_matches < self.dict_num_lookups>>7 {
+		return
+	}
+
+	key = uint(hash14(data) << 1)
+	for i = 0; ; (func() { i++; key++ })() {
+		var tmp uint
+		if shallow {
+			tmp = 1
+		} else {
+			tmp = 2
+		}
+		if i >= tmp {
+			break
+		}
+		var item uint = uint(dictionary.hash_table[key])
+		self.dict_num_lookups++
+		if item != 0 {
+			var item_matches bool = testStaticDictionaryItem(dictionary, item, data, max_length, max_backward, max_distance, out)
+			if item_matches {
+				self.dict_num_matches++
+			}
+		}
+	}
+}
+
+type backwardMatch struct {
+	distance        uint32
+	length_and_code uint32
+}
+
+func initBackwardMatch(self *backwardMatch, dist uint, len uint) {
+	self.distance = uint32(dist)
+	self.length_and_code = uint32(len << 5)
+}
+
+func initDictionaryBackwardMatch(self *backwardMatch, dist uint, len uint, len_code uint) {
+	self.distance = uint32(dist)
+	var tmp uint
+	if len == len_code {
+		tmp = 0
+	} else {
+		tmp = len_code
+	}
+	self.length_and_code = uint32(len<<5 | tmp)
+}
+
+func backwardMatchLength(self *backwardMatch) uint {
+	return uint(self.length_and_code >> 5)
+}
+
+func backwardMatchLengthCode(self *backwardMatch) uint {
+	var code uint = uint(self.length_and_code) & 31
+	if code != 0 {
+		return code
+	} else {
+		return backwardMatchLength(self)
+	}
+}
+
+func hasherReset(handle hasherHandle) {
+	if handle == nil {
+		return
+	}
+	handle.Common().is_prepared_ = false
+}
+
+func newHasher(typ int) hasherHandle {
+	switch typ {
+	case 2:
+		return &hashLongestMatchQuickly{
+			bucketBits:    16,
+			bucketSweep:   1,
+			hashLen:       5,
+			useDictionary: true,
+		}
+	case 3:
+		return &hashLongestMatchQuickly{
+			bucketBits:    16,
+			bucketSweep:   2,
+			hashLen:       5,
+			useDictionary: false,
+		}
+	case 4:
+		return &hashLongestMatchQuickly{
+			bucketBits:    17,
+			bucketSweep:   4,
+			hashLen:       5,
+			useDictionary: true,
+		}
+	case 5:
+		return new(h5)
+	case 6:
+		return new(h6)
+	case 10:
+		return new(h10)
+	case 35:
+		return &hashComposite{
+			ha: newHasher(3),
+			hb: &hashRolling{jump: 4},
+		}
+	case 40:
+		return &hashForgetfulChain{
+			bucketBits:              15,
+			numBanks:                1,
+			bankBits:                16,
+			numLastDistancesToCheck: 4,
+		}
+	case 41:
+		return &hashForgetfulChain{
+			bucketBits:              15,
+			numBanks:                1,
+			bankBits:                16,
+			numLastDistancesToCheck: 10,
+		}
+	case 42:
+		return &hashForgetfulChain{
+			bucketBits:              15,
+			numBanks:                512,
+			bankBits:                9,
+			numLastDistancesToCheck: 16,
+		}
+	case 54:
+		return &hashLongestMatchQuickly{
+			bucketBits:    20,
+			bucketSweep:   4,
+			hashLen:       7,
+			useDictionary: false,
+		}
+	case 55:
+		return &hashComposite{
+			ha: newHasher(54),
+			hb: &hashRolling{jump: 4},
+		}
+	case 65:
+		return &hashComposite{
+			ha: newHasher(6),
+			hb: &hashRolling{jump: 1},
+		}
+	}
+
+	panic(fmt.Sprintf("unknown hasher type: %d", typ))
+}
+
+func hasherSetup(handle *hasherHandle, params *encoderParams, data []byte, position uint, input_size uint, is_last bool) {
+	var self hasherHandle = nil
+	var common *hasherCommon = nil
+	var one_shot bool = (position == 0 && is_last)
+	if *handle == nil {
+		chooseHasher(params, &params.hasher)
+		self = newHasher(params.hasher.type_)
+
+		*handle = self
+		common = self.Common()
+		common.params = params.hasher
+		self.Initialize(params)
+	}
+
+	self = *handle
+	common = self.Common()
+	if !common.is_prepared_ {
+		self.Prepare(one_shot, input_size, data)
+
+		if position == 0 {
+			common.dict_num_lookups = 0
+			common.dict_num_matches = 0
+		}
+
+		common.is_prepared_ = true
+	}
+}
+
+func initOrStitchToPreviousBlock(handle *hasherHandle, data []byte, mask uint, params *encoderParams, position uint, input_size uint, is_last bool) {
+	var self hasherHandle
+	hasherSetup(handle, params, data, position, input_size, is_last)
+	self = *handle
+	self.StitchToPreviousBlock(input_size, position, data, mask)
+}

vendor/github.com/andybalholm/brotli/hash_composite.go

@@ -0,0 +1,93 @@
+package brotli
+
+/* Copyright 2018 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+func (h *hashComposite) HashTypeLength() uint {
+	var a uint = h.ha.HashTypeLength()
+	var b uint = h.hb.HashTypeLength()
+	if a > b {
+		return a
+	} else {
+		return b
+	}
+}
+
+func (h *hashComposite) StoreLookahead() uint {
+	var a uint = h.ha.StoreLookahead()
+	var b uint = h.hb.StoreLookahead()
+	if a > b {
+		return a
+	} else {
+		return b
+	}
+}
+
+/* Composite hasher: This hasher allows to combine two other hashers, HASHER_A
+   and HASHER_B. */
+type hashComposite struct {
+	hasherCommon
+	ha     hasherHandle
+	hb     hasherHandle
+	params *encoderParams
+}
+
+func (h *hashComposite) Initialize(params *encoderParams) {
+	h.params = params
+}
+
+/* TODO: Initialize of the hashers is defered to Prepare (and params
+   remembered here) because we don't get the one_shot and input_size params
+   here that are needed to know the memory size of them. Instead provide
+   those params to all hashers InitializehashComposite */
+func (h *hashComposite) Prepare(one_shot bool, input_size uint, data []byte) {
+	if h.ha == nil {
+		var common_a *hasherCommon
+		var common_b *hasherCommon
+
+		common_a = h.ha.Common()
+		common_a.params = h.params.hasher
+		common_a.is_prepared_ = false
+		common_a.dict_num_lookups = 0
+		common_a.dict_num_matches = 0
+		h.ha.Initialize(h.params)
+
+		common_b = h.hb.Common()
+		common_b.params = h.params.hasher
+		common_b.is_prepared_ = false
+		common_b.dict_num_lookups = 0
+		common_b.dict_num_matches = 0
+		h.hb.Initialize(h.params)
+	}
+
+	h.ha.Prepare(one_shot, input_size, data)
+	h.hb.Prepare(one_shot, input_size, data)
+}
+
+func (h *hashComposite) Store(data []byte, mask uint, ix uint) {
+	h.ha.Store(data, mask, ix)
+	h.hb.Store(data, mask, ix)
+}
+
+func (h *hashComposite) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) {
+	h.ha.StoreRange(data, mask, ix_start, ix_end)
+	h.hb.StoreRange(data, mask, ix_start, ix_end)
+}
+
+func (h *hashComposite) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ring_buffer_mask uint) {
+	h.ha.StitchToPreviousBlock(num_bytes, position, ringbuffer, ring_buffer_mask)
+	h.hb.StitchToPreviousBlock(num_bytes, position, ringbuffer, ring_buffer_mask)
+}
+
+func (h *hashComposite) PrepareDistanceCache(distance_cache []int) {
+	h.ha.PrepareDistanceCache(distance_cache)
+	h.hb.PrepareDistanceCache(distance_cache)
+}
+
+func (h *hashComposite) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) {
+	h.ha.FindLongestMatch(dictionary, data, ring_buffer_mask, distance_cache, cur_ix, max_length, max_backward, gap, max_distance, out)
+	h.hb.FindLongestMatch(dictionary, data, ring_buffer_mask, distance_cache, cur_ix, max_length, max_backward, gap, max_distance, out)
+}

vendor/github.com/andybalholm/brotli/hash_forgetful_chain.go

@@ -0,0 +1,253 @@
+package brotli
+
+import "encoding/binary"
+
+/* Copyright 2016 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+func (*hashForgetfulChain) HashTypeLength() uint {
+	return 4
+}
+
+func (*hashForgetfulChain) StoreLookahead() uint {
+	return 4
+}
+
+/* HashBytes is the function that chooses the bucket to place the address in.*/
+func (h *hashForgetfulChain) HashBytes(data []byte) uint {
+	var hash uint32 = binary.LittleEndian.Uint32(data) * kHashMul32
+
+	/* The higher bits contain more mixture from the multiplication,
+	   so we take our results from there. */
+	return uint(hash >> (32 - h.bucketBits))
+}
+
+type slot struct {
+	delta uint16
+	next  uint16
+}
+
+/* A (forgetful) hash table to the data seen by the compressor, to
+   help create backward references to previous data.
+
+   Hashes are stored in chains which are bucketed to groups. Group of chains
+   share a storage "bank". When more than "bank size" chain nodes are added,
+   oldest nodes are replaced; this way several chains may share a tail. */
+type hashForgetfulChain struct {
+	hasherCommon
+
+	bucketBits              uint
+	numBanks                uint
+	bankBits                uint
+	numLastDistancesToCheck int
+
+	addr          []uint32
+	head          []uint16
+	tiny_hash     [65536]byte
+	banks         [][]slot
+	free_slot_idx []uint16
+	max_hops      uint
+}
+
+func (h *hashForgetfulChain) Initialize(params *encoderParams) {
+	var q uint
+	if params.quality > 6 {
+		q = 7
+	} else {
+		q = 8
+	}
+	h.max_hops = q << uint(params.quality-4)
+
+	bankSize := 1 << h.bankBits
+	bucketSize := 1 << h.bucketBits
+
+	h.addr = make([]uint32, bucketSize)
+	h.head = make([]uint16, bucketSize)
+	h.banks = make([][]slot, h.numBanks)
+	for i := range h.banks {
+		h.banks[i] = make([]slot, bankSize)
+	}
+	h.free_slot_idx = make([]uint16, h.numBanks)
+}
+
+func (h *hashForgetfulChain) Prepare(one_shot bool, input_size uint, data []byte) {
+	var partial_prepare_threshold uint = (1 << h.bucketBits) >> 6
+	/* Partial preparation is 100 times slower (per socket). */
+	if one_shot && input_size <= partial_prepare_threshold {
+		var i uint
+		for i = 0; i < input_size; i++ {
+			var bucket uint = h.HashBytes(data[i:])
+
+			/* See InitEmpty comment. */
+			h.addr[bucket] = 0xCCCCCCCC
+
+			h.head[bucket] = 0xCCCC
+		}
+	} else {
+		/* Fill |addr| array with 0xCCCCCCCC value. Because of wrapping, position
+		   processed by hasher never reaches 3GB + 64M; this makes all new chains
+		   to be terminated after the first node. */
+		for i := range h.addr {
+			h.addr[i] = 0xCCCCCCCC
+		}
+
+		for i := range h.head {
+			h.head[i] = 0
+		}
+	}
+
+	h.tiny_hash = [65536]byte{}
+	for i := range h.free_slot_idx {
+		h.free_slot_idx[i] = 0
+	}
+}
+
+/* Look at 4 bytes at &data[ix & mask]. Compute a hash from these, and prepend
+   node to corresponding chain; also update tiny_hash for current position. */
+func (h *hashForgetfulChain) Store(data []byte, mask uint, ix uint) {
+	var key uint = h.HashBytes(data[ix&mask:])
+	var bank uint = key & (h.numBanks - 1)
+	var idx uint
+	idx = uint(h.free_slot_idx[bank]) & ((1 << h.bankBits) - 1)
+	h.free_slot_idx[bank]++
+	var delta uint = ix - uint(h.addr[key])
+	h.tiny_hash[uint16(ix)] = byte(key)
+	if delta > 0xFFFF {
+		delta = 0xFFFF
+	}
+	h.banks[bank][idx].delta = uint16(delta)
+	h.banks[bank][idx].next = h.head[key]
+	h.addr[key] = uint32(ix)
+	h.head[key] = uint16(idx)
+}
+
+func (h *hashForgetfulChain) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) {
+	var i uint
+	for i = ix_start; i < ix_end; i++ {
+		h.Store(data, mask, i)
+	}
+}
+
+func (h *hashForgetfulChain) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ring_buffer_mask uint) {
+	if num_bytes >= h.HashTypeLength()-1 && position >= 3 {
+		/* Prepare the hashes for three last bytes of the last write.
+		   These could not be calculated before, since they require knowledge
+		   of both the previous and the current block. */
+		h.Store(ringbuffer, ring_buffer_mask, position-3)
+		h.Store(ringbuffer, ring_buffer_mask, position-2)
+		h.Store(ringbuffer, ring_buffer_mask, position-1)
+	}
+}
+
+func (h *hashForgetfulChain) PrepareDistanceCache(distance_cache []int) {
+	prepareDistanceCache(distance_cache, h.numLastDistancesToCheck)
+}
+
+/* Find a longest backward match of &data[cur_ix] up to the length of
+   max_length and stores the position cur_ix in the hash table.
+
+   REQUIRES: PrepareDistanceCachehashForgetfulChain must be invoked for current distance cache
+             values; if this method is invoked repeatedly with the same distance
+             cache values, it is enough to invoke PrepareDistanceCachehashForgetfulChain once.
+
+   Does not look for matches longer than max_length.
+   Does not look for matches further away than max_backward.
+   Writes the best match into |out|.
+   |out|->score is updated only if a better match is found. */
+func (h *hashForgetfulChain) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) {
+	var cur_ix_masked uint = cur_ix & ring_buffer_mask
+	var min_score uint = out.score
+	var best_score uint = out.score
+	var best_len uint = out.len
+	var key uint = h.HashBytes(data[cur_ix_masked:])
+	var tiny_hash byte = byte(key)
+	/* Don't accept a short copy from far away. */
+	out.len = 0
+
+	out.len_code_delta = 0
+
+	/* Try last distance first. */
+	for i := 0; i < h.numLastDistancesToCheck; i++ {
+		var backward uint = uint(distance_cache[i])
+		var prev_ix uint = (cur_ix - backward)
+
+		/* For distance code 0 we want to consider 2-byte matches. */
+		if i > 0 && h.tiny_hash[uint16(prev_ix)] != tiny_hash {
+			continue
+		}
+		if prev_ix >= cur_ix || backward > max_backward {
+			continue
+		}
+
+		prev_ix &= ring_buffer_mask
+		{
+			var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
+			if len >= 2 {
+				var score uint = backwardReferenceScoreUsingLastDistance(uint(len))
+				if best_score < score {
+					if i != 0 {
+						score -= backwardReferencePenaltyUsingLastDistance(uint(i))
+					}
+					if best_score < score {
+						best_score = score
+						best_len = uint(len)
+						out.len = best_len
+						out.distance = backward
+						out.score = best_score
+					}
+				}
+			}
+		}
+	}
+	{
+		var bank uint = key & (h.numBanks - 1)
+		var backward uint = 0
+		var hops uint = h.max_hops
+		var delta uint = cur_ix - uint(h.addr[key])
+		var slot uint = uint(h.head[key])
+		for {
+			tmp6 := hops
+			hops--
+			if tmp6 == 0 {
+				break
+			}
+			var prev_ix uint
+			var last uint = slot
+			backward += delta
+			if backward > max_backward {
+				break
+			}
+			prev_ix = (cur_ix - backward) & ring_buffer_mask
+			slot = uint(h.banks[bank][last].next)
+			delta = uint(h.banks[bank][last].delta)
+			if cur_ix_masked+best_len > ring_buffer_mask || prev_ix+best_len > ring_buffer_mask || data[cur_ix_masked+best_len] != data[prev_ix+best_len] {
+				continue
+			}
+			{
+				var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
+				if len >= 4 {
+					/* Comparing for >= 3 does not change the semantics, but just saves
+					   for a few unnecessary binary logarithms in backward reference
+					   score, since we are not interested in such short matches. */
+					var score uint = backwardReferenceScore(uint(len), backward)
+					if best_score < score {
+						best_score = score
+						best_len = uint(len)
+						out.len = best_len
+						out.distance = backward
+						out.score = best_score
+					}
+				}
+			}
+		}
+
+		h.Store(data, ring_buffer_mask, cur_ix)
+	}
+
+	if out.score == min_score {
+		searchInStaticDictionary(dictionary, h, data[cur_ix_masked:], max_length, max_backward+gap, max_distance, out, false)
+	}
+}