fix file IO flaw + CI fixes (#7)

There was a suble bug in reading files, due to not using io.ReadFull, but instead doing partial reads. This wasn't found in the tests, due to them only handling small files. The root issue has been fixed, aswell as better tests which hit the issue.

* ci: fixes for circle-ci config on v2
* fix error with file reading/writing, where non-full reads caused corruption

Author: holiman

.circleci/config.yml

@@ -14,7 +14,7 @@ jobs:
 
       # specify any bash command here prefixed with `run: `
       - run: go get -v -t -d ./...
-      - run: go test -v ./... -coverprofile=coverage.txt -covermode=count
+      - run: (cd v2 && go test -v ./... -coverprofile=coverage.txt -covermode=count )
       - run:
           name: "Codecov upload"
           command: bash <(curl -s https://codecov.io/bash)
@@ -23,5 +23,5 @@ jobs:
           command: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.23.8
       - run:
           name: "Lint"
-          command: golangci-lint run
+          command: (cd v2 && golangci-lint run)
 

v2/binarymarshaler.go

@@ -74,16 +74,16 @@ func (f *Filter) MarshallToWriter(out io.Writer) (int, [sha512.Size384]byte, err
 	if chunkSize < 512 {
 		chunkSize = 512 // Min 4K bytes (512 uint64s)
 	}
-	bs := make([]byte, chunkSize*8)
+	buf := make([]byte, chunkSize*8)
 	for start := 0; start < len(f.bits); {
 		end := start + chunkSize
 		if end > len(f.bits) {
 			end = len(f.bits)
 		}
 		for i, x := range f.bits[start:end] {
-			binary.LittleEndian.PutUint64(bs[8*i:], x)
+			binary.LittleEndian.PutUint64(buf[8*i:], x)
 		}
-		if _, err := mw.Write(bs[0 : (end-start)*8]); err != nil {
+		if _, err := mw.Write(buf[0 : (end-start)*8]); err != nil {
 			return c.bytes, hash, err
 		}
 		start = end

v2/binaryunmarshaler.go

@@ -21,7 +21,7 @@ import (
 
 func unmarshalBinaryHeader(r io.Reader) (k, n, m uint64, err error) {
 	magic := make([]byte, len(headerMagic))
-	if _, err := r.Read(magic); err != nil {
+	if _, err := io.ReadFull(r, magic); err != nil {
 		return 0, 0, 0, err
 	}
 	if !bytes.Equal(magic, headerMagic) {
@@ -41,7 +41,6 @@ func unmarshalBinaryHeader(r io.Reader) (k, n, m uint64, err error) {
 	if m < MMin {
 		return 0, 0, 0, fmt.Errorf("number of bits in the filter must be >= %d (was %d)", MMin, m)
 	}
-
 	return k, n, m, err
 }
 
@@ -52,7 +51,7 @@ func unmarshalBinaryBits(r io.Reader, m uint64) (bits []uint64, err error) {
 	}
 	bs := make([]byte, 8)
 	for i := 0; i < len(bits) && err == nil; i++ {
-		_, err = r.Read(bs)
+		_, err = io.ReadFull(r, bs)
 		bits[i] = binary.LittleEndian.Uint64(bs)
 	}
 	if err != nil {
@@ -81,7 +80,7 @@ func (h *hashingReader) Read(p []byte) (n int, err error) {
 	if err != nil {
 		return n, err
 	}
-	_, _ = h.hasher.Write(p)
+	_, _ = h.hasher.Write(p[:n])
 	return n, err
 }
 

v2/fileio.go

@@ -36,12 +36,12 @@ func (f *Filter) ReadFrom(r io.Reader) (n int64, err error) {
 
 // ReadFrom Reader r into a lossless-compressed Bloom filter f
 func ReadFrom(r io.Reader) (f *Filter, n int64, err error) {
+	f = new(Filter)
 	rawR, err := gzip.NewReader(r)
 	if err != nil {
 		return nil, -1, err
 	}
 	defer rawR.Close()
-	f = new(Filter)
 	n, err = f.UnmarshalFromReader(rawR)
 	if err != nil {
 		return nil, -1, err
@@ -67,11 +67,10 @@ func (f *Filter) WriteTo(w io.Writer) (n int64, err error) {
 	defer f.lock.RUnlock()
 
 	rawW := gzip.NewWriter(w)
-	defer func() {
-		err = rawW.Close()
-	}()
+	defer rawW.Close()
 
 	intN, _, err := f.MarshallToWriter(rawW)
+	//intN, _, err := f.MarshallToWriter(w)
 	n = int64(intN)
 	return n, err
 }
@@ -83,9 +82,7 @@ func (f *Filter) WriteFile(filename string) (n int64, err error) {
 	if err != nil {
 		return -1, err
 	}
-	defer func() {
-		err = w.Close()
-	}()
+	defer w.Close()
 
 	return f.WriteTo(w)
 }

v2/fileio_test.go

@@ -30,7 +30,7 @@ func (d devnull) Write(p []byte) (n int, err error) {
 
 func TestWriteRead(t *testing.T) {
 	// minimal filter
-	f, _ := New(8*32, 5)
+	f, _ := New(8*1024*100, 5)
 	// Add some content
 	var tests = make([]hashableUint64, 20)
 	for i := 0; i < 20; i++ {
@@ -89,7 +89,7 @@ func TestWriteRead(t *testing.T) {
 		verify(t, &f2)
 	})
 	t.Run("file", func(t *testing.T) {
-		fName := filepath.Join(os.TempDir(), "temp.deleteme")
+		fName := filepath.Join(os.TempDir(), "temp.deleteme.gz")
 		if _, err := f.WriteFile(fName); err != nil {
 			t.Fatal(err)
 		}
@@ -100,7 +100,6 @@ func TestWriteRead(t *testing.T) {
 			verify(t, f2)
 		}
 	})
-
 }
 
 func TestCorruption(t *testing.T) {