BACKWARDS INCOMPATIBLE: version update + json marshalling (#5)

* go modules support

* minor refactorings

* update binary format, fix text marshalling

* implement json marshalling/unmarshalling

* remove text marshaller interface

* remove debug, remove gob file

* Add .circleci/config.yml

* circle-ci config

* lint nitpicks

* readme: circleci badge

Author: holiman

.circleci/config.yml

@@ -0,0 +1,24 @@
+# Golang CircleCI 2.0 configuration file
+#
+# Check https://circleci.com/docs/2.0/language-go/ for more details
+version: 2
+jobs:
+  build:
+    docker:
+      # specify the version
+      - image: circleci/golang:1.14
+
+    working_directory: /go/src/github.com/holiman/bloomfilter
+    steps:
+      - checkout
+
+      # specify any bash command here prefixed with `run: `
+      - run: go get -v -t -d ./...
+      - run: go test -v ./...
+      - run:
+          name: "Install tools"
+          command: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.23.8
+      - run:
+          name: "Lint"
+          command: golangci-lint run
+

.travis.yml

@@ -1,28 +1,29 @@
 
-[![GoDoc](https://godoc.org/github.com/steakknife/bloomfilter?status.png)](https://godoc.org/github.com/steakknife/bloomfilter) [![travis](https://img.shields.io/travis/steakknife/bloomfilter.svg)](https://travis-ci.org/steakknife/bloomfilter)
+[![GoDoc](https://godoc.org/github.com/holiman/bloomfilter?status.png)](https://godoc.org/github.com/holiman/bloomfilter)
+[![CircleCI](https://circleci.com/gh/holiman/bloomfilter.svg?style=svg)](https://app.circleci.com/pipelines/github/holiman/bloomfilter)
 
 # History
 
 This bloom filter implementation is a fork from [steakknife/bloomfilter](https://github.com/steakknife/bloomfilter) by Barry Allard. 
 The upstream project is now archived, so this fork exists to fix some bugs and also
 make a few improvements. Below is the original description. 
 
-All recent changes are copyright © 2019 Martin Holst Swende. 
+The original implemenation is Copyright © 2014-2016,2018 Barry Allard
+[MIT license](MIT-LICENSE.txt)
+
+All recent changes are copyright © 2019-2020 Martin Holst Swende. 
 
 ## Installation 
 
 ```
 $ go get github.com/holiman/bloomfilter
 ```
 
-# Face-meltingly fast, thread-safe, marshalable, unionable, probability- and optimal-size-calculating Bloom filter in go
+## Face-meltingly fast, thread-safe, marshalable, unionable, probability- and optimal-size-calculating Bloom filter in go
 
-Copyright © 2014-2016,2018 Barry Allard
-[MIT license](MIT-LICENSE.txt)
+### WTF is a bloom filter
 
-## WTF is a bloom filter
-
-**TL;DR: **Probabilistic, extra lookup table to track a set of elements kept elsewhere to reduce expensive, unnecessary set element retrieval and/or iterator operations **when an element is not present in the set.** It's a classic time-storage tradeoff algoritm.
+**TL;DR:** Probabilistic, extra lookup table to track a set of elements kept elsewhere to reduce expensive, unnecessary set element retrieval and/or iterator operations **when an element is not present in the set.** It's a classic time-storage tradeoff algoritm.
 
 ### Properties
 
@@ -70,7 +71,7 @@ All values in Little-endian format
 
 ```go
 
-import "github.com/steakknife/bloomfilter"
+import "github.com/holiman/bloomfilter"
 
 const (
   maxElements = 100000
@@ -120,3 +121,5 @@ Where possible, branch-free operations are used to avoid deep pipeline / executi
 [MIT license](MIT-LICENSE.txt)
 
 Copyright © 2014-2016 Barry Allard
+Copyright © 2019-2020 Martin Holst Swende
+

README.md

@@ -17,6 +17,13 @@ import (
 	"io"
 )
 
+// headerMagic is used to disambiguate between this package and the original
+// steakknife implementation.
+// Since the key hashing algorithm has changed, the format is no longer
+// binary compatible
+var version = []byte("v02\n")
+var headerMagic = append([]byte{0, 0, 0, 0, 0, 0, 0, 0}, version...)
+
 // counter is a utility to count bytes written
 type counter struct {
 	bytes int
@@ -51,8 +58,10 @@ func (f *Filter) MarshallToWriter(out io.Writer) (int, [sha512.Size384]byte, err
 	)
 	f.lock.RLock()
 	defer f.lock.RUnlock()
-	debug("write bf k=%d n=%d m=%d\n", f.K(), f.n, f.m)
 
+	if _, err := mw.Write(headerMagic); err != nil {
+		return c.bytes, hash, err
+	}
 	if err := binary.Write(mw, binary.LittleEndian, f.K()); err != nil {
 		return c.bytes, hash, err
 	}
@@ -90,10 +99,6 @@ func (f *Filter) MarshallToWriter(out io.Writer) (int, [sha512.Size384]byte, err
 	hashbytes := hasher.Sum(nil)
 	copy(hash[:], hashbytes[:sha512.Size384])
 	err := binary.Write(out, binary.LittleEndian, hashbytes)
-	if err != nil {
-		debug("bloomfilter.MarshalBinary: Successfully wrote %d byte(s), sha384 %v",
-			c.bytes, hash)
-	}
 	return c.bytes + len(hashbytes), hash, err
 }
 

binarymarshaler.go

@@ -14,36 +14,41 @@ import (
 	"bytes"
 	"crypto/sha512"
 	"encoding/binary"
+	"fmt"
 	"hash"
 	"io"
 )
 
 func unmarshalBinaryHeader(r io.Reader) (k, n, m uint64, err error) {
+	magic := make([]byte, len(headerMagic))
+	if _, err := r.Read(magic); err != nil {
+		return 0, 0, 0, err
+	}
+	if !bytes.Equal(magic, headerMagic) {
+		return 0, 0, 0, fmt.Errorf("incompatible version (wrong magic), got %x", magic)
+	}
 	err = binary.Read(r, binary.LittleEndian, &k)
 	if err != nil {
-		return k, n, m, err
+		return 0, 0, 0, err
 	}
-
 	if k < KMin {
-		return k, n, m, errK()
+		return 0, 0, 0, fmt.Errorf("keys must have length %d or greater (was %d)", KMin, k)
 	}
 
 	err = binary.Read(r, binary.LittleEndian, &n)
 	if err != nil {
-		return k, n, m, err
+		return 0, 0, 0, err
 	}
 
 	err = binary.Read(r, binary.LittleEndian, &m)
 	if err != nil {
-		return k, n, m, err
+		return 0, 0, 0, err
 	}
 
 	if m < MMin {
-		return k, n, m, errM()
+		return 0, 0, 0, fmt.Errorf("number of bits in the filter must be >= %d (was %d)", MMin, m)
 	}
 
-	debug("read bf k=%d n=%d m=%d\n", k, n, m)
-
 	return k, n, m, err
 }
 
@@ -82,7 +87,7 @@ func (h *hashingReader) Read(p []byte) (n int, err error) {
 	if err != nil {
 		return n, err
 	}
-	h.hasher.Write(p)
+	_, _ = h.hasher.Write(p)
 	return n, err
 }
 
@@ -126,9 +131,7 @@ func (f *Filter) UnmarshalFromReader(input io.Reader) (n int64, err error) {
 		return buf.tot, err
 	}
 	if !bytes.Equal(gotHash, expHash) {
-		debug("bloomfilter.UnmarshalBinary() sha384 hash failed:"+
-			" actual %v  expected %v", gotHash, expHash)
-		return buf.tot, errHash()
+		return buf.tot, errHashMismatch
 	}
 	return buf.tot, nil
 }

binaryunmarshaler.go

@@ -13,10 +13,15 @@
 package bloomfilter
 
 import (
+	"errors"
 	"hash"
 	"sync"
 )
 
+var (
+	errHashMismatch = errors.New("hash mismatch, bloom filter corruption or wrong version")
+)
+
 // Filter is an opaque Bloom filter type
 type Filter struct {
 	lock sync.RWMutex
@@ -104,7 +109,7 @@ func (f *Filter) Copy() (*Filter, error) {
 // UnionInPlace merges Bloom filter f2 into f
 func (f *Filter) UnionInPlace(f2 *Filter) error {
 	if !f.IsCompatible(f2) {
-		return errIncompatibleBloomFilters()
+		return errors.New("incompatible bloom filters")
 	}
 
 	f.lock.Lock()
@@ -121,7 +126,7 @@ func (f *Filter) UnionInPlace(f2 *Filter) error {
 // Union merges f2 and f2 into a new Filter out
 func (f *Filter) Union(f2 *Filter) (out *Filter, err error) {
 	if !f.IsCompatible(f2) {
-		return nil, errIncompatibleBloomFilters()
+		return nil, errors.New("incompatible bloom filters")
 	}
 
 	f.lock.RLock()

bloomfilter.go

@@ -195,7 +195,7 @@ func BenchmarkUnionInPlace(b *testing.B) {
 	b.Run("union-8", func(b *testing.B) {
 		for i := 0; i < b.N; i++ {
 			for _, bx := range filters {
-				b1.UnionInPlace(bx)
+				_ = b1.UnionInPlace(bx)
 			}
 		}
 	})

bloomfilter_test.go

@@ -13,17 +13,18 @@ package bloomfilter
 import (
 	"encoding"
 	"encoding/gob"
+	"encoding/json"
 	"io"
 )
 
 // compile-time conformance tests
 var (
 	_ encoding.BinaryMarshaler   = (*Filter)(nil)
 	_ encoding.BinaryUnmarshaler = (*Filter)(nil)
-	_ encoding.TextMarshaler     = (*Filter)(nil)
-	_ encoding.TextUnmarshaler   = (*Filter)(nil)
 	_ io.ReaderFrom              = (*Filter)(nil)
 	_ io.WriterTo                = (*Filter)(nil)
 	_ gob.GobDecoder             = (*Filter)(nil)
 	_ gob.GobEncoder             = (*Filter)(nil)
+	_ json.Marshaler             = (*Filter)(nil)
+	_ json.Unmarshaler           = (*Filter)(nil)
 )