chore(release): v0.1.0

Merge pull request #7 from qri-io/v0.1.0

.circleci/config.yml

@@ -3,48 +3,32 @@ jobs:
   build:
     working_directory: /go/src/github.com/qri-io/qfs
     docker:
-      - image: circleci/golang:1.11
+      - image: circleci/golang:1.12
         environment:
           GOLANG_ENV: test
+          GO111MODULE: "on"
     environment:
       TEST_RESULTS: /tmp/test-results
     steps:
       - checkout
       - run: mkdir -p $TEST_RESULTS
+      - restore_cache:
+          key: dependency-cache-{{ checksum "go.sum" }}
       - run: 
           name: Install test deps
           command: >
-            go get -v
+            go get -v 
             github.com/jstemmer/go-junit-report 
-            github.com/golang/lint/golint
-            github.com/whyrusleeping/gx 
-            github.com/whyrusleeping/gx-go 
-      - run:
-          name: Install non-gx deps
-          command: >
-            go get -t -d -v 
-            github.com/ipfs/go-datastore
-            github.com/jbenet/goprocess
-            github.com/satori/go.uuid
-            github.com/jbenet/go-base58
-            github.com/multiformats/go-multihash
-            github.com/spaolacci/murmur3
-            golang.org/x/crypto/blake2b
-            github.com/ipfs/go-log
-      - restore_cache:
-          key: dependency-cache-{{ checksum "package.json" }}
-      - run:
-          name: Install gx deps
-          command: gx install
-      - save_cache:
-          key: dependency-cache-{{ checksum "package.json" }}
-          paths:
-            - /go/src/gx/
+            golang.org/x/lint/golint
       - run:
           name: Run Tests
           command: |
             trap "go-junit-report <${TEST_RESULTS}/go-test.out > ${TEST_RESULTS}/go-test-report.xml" EXIT
-            ./.circleci/cover.test.sh | tee ${TEST_RESULTS}/go-test.out
+            make test | tee ${TEST_RESULTS}/go-test.out
+      - save_cache:
+          key: dependency-cache-{{ checksum "go.sum" }}
+          paths:
+            - /go/pkg/mod
       - run:
           name: Publish coverage info to codecov.io
           command: bash <(curl -s https://codecov.io/bash)

CHANGELOG.md

@@ -0,0 +1,39 @@
+<a name="0.1.0"></a>
+#  (2019-06-03)
+
+This is the first proper release of `qfs`. In preparation for go 1.13, in which `go.mod` files and go modules are the primary way to handle go dependencies, we are going to do an official release of all our modules. This will be version v0.1.0 of `qfs`. We'll be working on adding details & documentation in the near future.
+
+### Bug Fixes
+
+* **adder:** because we now have VisConfig as a possible field in dataset, we need to make another channel available to the adder (from 8 to 9). Fixed in both ipfs and memfs ([aef9d08](https://github.com/qri-io/qfs/commit/aef9d08))
+* **Fetcher:** fix fetcher interface ([91fd46c](https://github.com/qri-io/qfs/commit/91fd46c))
+* **fsrepo:** close fsrepo on context cancellation ([ff18631](https://github.com/qri-io/qfs/commit/ff18631))
+* **ipfs:** coreapi not being reallocated when going online ([7394a3c](https://github.com/qri-io/qfs/commit/7394a3c))
+* **ipfs plugins:** export LoadPlugins call once for package tests ([e4205f8](https://github.com/qri-io/qfs/commit/e4205f8))
+
+
+### Code Refactoring
+
+* **cafs.Filestore:** work with strings instead of datastore.Key ([8f5ce11](https://github.com/qri-io/qfs/commit/8f5ce11))
+
+
+### Features
+
+* **cache, gcloud:** new cache interface, starting into gcloud cache ([1ed3e3a](https://github.com/qri-io/qfs/commit/1ed3e3a))
+* **destroyer:** add destroyer interface ([43a6e0f](https://github.com/qri-io/qfs/commit/43a6e0f))
+* **Filestore:** added PathPrefix to filestore interface ([a79eb74](https://github.com/qri-io/qfs/commit/a79eb74))
+* **GoOnline:** added new methods to take offline node online ([67ca85e](https://github.com/qri-io/qfs/commit/67ca85e))
+* **IPFS api:** add options to enable IPFS API, pubsub ([ed80762](https://github.com/qri-io/qfs/commit/ed80762))
+* **mapstore:** Improve Mapstore for better tests. ([3ebe881](https://github.com/qri-io/qfs/commit/3ebe881))
+* **mapstore.Print:** new method to print mapstore contents ([4055c2f](https://github.com/qri-io/qfs/commit/4055c2f))
+* **MemFS:** make MemFS actually do something ([840dd74](https://github.com/qri-io/qfs/commit/840dd74))
+* **migration error:** detect ipfs migration errors ([81556ca](https://github.com/qri-io/qfs/commit/81556ca))
+* bring file interface into package ([3eae520](https://github.com/qri-io/qfs/commit/3eae520))
+
+
+### BREAKING CHANGES
+
+* **cafs.Filestore:** cafs.Filestore interface methods accept and return string values instead of datastore.Key.
+
+
+

Makefile

@@ -0,0 +1,6 @@
+# Let's keep all our changelog commands the same across all our packages:
+update-changelog:
+	conventional-changelog -p angular -i CHANGELOG.md -s
+
+test:
+	go test ./... -v --coverprofile=coverage.txt --covermode=atomic

cafs/ipfs/config.go

@@ -4,8 +4,8 @@ import (
 	"context"
 	"fmt"
 
-	"gx/ipfs/QmUJYo4etAQqFfSS2rarFAE97eNGB8ej64YkRT2SmsYD4r/go-ipfs/core"
-	fsrepo "gx/ipfs/QmUJYo4etAQqFfSS2rarFAE97eNGB8ej64YkRT2SmsYD4r/go-ipfs/repo/fsrepo"
+	"github.com/ipfs/go-ipfs/core"
+	fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo"
 )
 
 var ErrIPFSRepoNeedsMigration = fmt.Errorf(`Your IPFS repo needs an update!

cafs/ipfs/coreunix/add.go

@@ -1,3 +1,23 @@
+// Package coreunix is a fork of github.com/ipfs/go-ipfs/core/coreunix
+//
+// gx paths used:
+// "gx/ipfs/QmUJYo4etAQqFfSS2rarFAE97eNGB8ej64YkRT2SmsYD4r/go-ipfs/pin"
+// ipld "gx/ipfs/QmR7TcHkR9nxkUorfi8XMTAMLUK7GiP64TWWBzY3aacc1o/go-ipld-format"
+// dag "gx/ipfs/QmSei8kFMfqdJq7Q68d2LMnHbTWKKg2daA29ezUYFAUNgc/go-merkledag"
+// posinfo "gx/ipfs/QmQyUyYcpKG1u53V7N25qRTGw5XwaAxTMKXbduqHotQztg/go-ipfs-posinfo"
+// core "gx/ipfs/QmUJYo4etAQqFfSS2rarFAE97eNGB8ej64YkRT2SmsYD4r/go-ipfs/core"
+// unixfs "gx/ipfs/QmfB3oNXGGq9S4B2a9YeCajoATms3Zw2VvDm8fK7VeLSV8/go-unixfs"
+// balanced "gx/ipfs/QmfB3oNXGGq9S4B2a9YeCajoATms3Zw2VvDm8fK7VeLSV8/go-unixfs/importer/balanced"
+// ihelper "gx/ipfs/QmfB3oNXGGq9S4B2a9YeCajoATms3Zw2VvDm8fK7VeLSV8/go-unixfs/importer/helpers"
+// trickle "gx/ipfs/QmfB3oNXGGq9S4B2a9YeCajoATms3Zw2VvDm8fK7VeLSV8/go-unixfs/importer/trickle"
+// coreiface "gx/ipfs/QmUJYo4etAQqFfSS2rarFAE97eNGB8ej64YkRT2SmsYD4r/go-ipfs/core/coreapi/interface"
+// posinfo "gx/ipfs/QmQyUyYcpKG1u53V7N25qRTGw5XwaAxTMKXbduqHotQztg/go-ipfs-posinfo"
+// cid "gx/ipfs/QmPSQnBKM9g7BaUcZCvswUJVscQ1ipjmwxN5PXCjkp9EQ7/go-cid"
+// mfs "gx/ipfs/QmUwXQs8aZ472DmXZ8uJNf7HJNKoMJQVa7RaCz7ujZ3ua9/go-mfs"
+// chunker "gx/ipfs/QmTUTG9Jg9ZRA1EzTPGTDvnwfcfKhDMnqANnP9fe4rSjMR/go-ipfs-chunker"
+// logging "gx/ipfs/QmZChCsSt8DctjceaL56Eibc29CVQq4dGKRXC5JRZ6Ppae/go-log"
+// bstore "gx/ipfs/QmcDDgAXDbpDUpadCJKLr49KYR4HuL7T8Z1dZTHt6ixsoR/go-ipfs-blockstore"
+// files "gx/ipfs/QmZMWMvWMVKCbHetJ4RgndbuEF1io2UpUxwQwtNjtYPzSC/go-ipfs-files"
 package coreunix
 
 import (
@@ -10,23 +30,21 @@ import (
 	"path/filepath"
 	"strconv"
 
-	core "gx/ipfs/QmUJYo4etAQqFfSS2rarFAE97eNGB8ej64YkRT2SmsYD4r/go-ipfs/core"
-	coreiface "gx/ipfs/QmUJYo4etAQqFfSS2rarFAE97eNGB8ej64YkRT2SmsYD4r/go-ipfs/core/coreapi/interface"
-	"gx/ipfs/QmUJYo4etAQqFfSS2rarFAE97eNGB8ej64YkRT2SmsYD4r/go-ipfs/pin"
-
-	cid "gx/ipfs/QmPSQnBKM9g7BaUcZCvswUJVscQ1ipjmwxN5PXCjkp9EQ7/go-cid"
-	posinfo "gx/ipfs/QmQyUyYcpKG1u53V7N25qRTGw5XwaAxTMKXbduqHotQztg/go-ipfs-posinfo"
-	ipld "gx/ipfs/QmR7TcHkR9nxkUorfi8XMTAMLUK7GiP64TWWBzY3aacc1o/go-ipld-format"
-	dag "gx/ipfs/QmSei8kFMfqdJq7Q68d2LMnHbTWKKg2daA29ezUYFAUNgc/go-merkledag"
-	chunker "gx/ipfs/QmTUTG9Jg9ZRA1EzTPGTDvnwfcfKhDMnqANnP9fe4rSjMR/go-ipfs-chunker"
-	mfs "gx/ipfs/QmUwXQs8aZ472DmXZ8uJNf7HJNKoMJQVa7RaCz7ujZ3ua9/go-mfs"
-	logging "gx/ipfs/QmZChCsSt8DctjceaL56Eibc29CVQq4dGKRXC5JRZ6Ppae/go-log"
-	files "gx/ipfs/QmZMWMvWMVKCbHetJ4RgndbuEF1io2UpUxwQwtNjtYPzSC/go-ipfs-files"
-	bstore "gx/ipfs/QmcDDgAXDbpDUpadCJKLr49KYR4HuL7T8Z1dZTHt6ixsoR/go-ipfs-blockstore"
-	unixfs "gx/ipfs/QmfB3oNXGGq9S4B2a9YeCajoATms3Zw2VvDm8fK7VeLSV8/go-unixfs"
-	balanced "gx/ipfs/QmfB3oNXGGq9S4B2a9YeCajoATms3Zw2VvDm8fK7VeLSV8/go-unixfs/importer/balanced"
-	ihelper "gx/ipfs/QmfB3oNXGGq9S4B2a9YeCajoATms3Zw2VvDm8fK7VeLSV8/go-unixfs/importer/helpers"
-	trickle "gx/ipfs/QmfB3oNXGGq9S4B2a9YeCajoATms3Zw2VvDm8fK7VeLSV8/go-unixfs/importer/trickle"
+	cid "github.com/ipfs/go-cid"
+	bstore "github.com/ipfs/go-ipfs-blockstore"
+	chunker "github.com/ipfs/go-ipfs-chunker"
+	posinfo "github.com/ipfs/go-ipfs-posinfo"
+	"github.com/ipfs/go-ipfs/core"
+	"github.com/ipfs/go-ipfs/pin"
+	ipld "github.com/ipfs/go-ipld-format"
+	logging "github.com/ipfs/go-log"
+	dag "github.com/ipfs/go-merkledag"
+	mfs "github.com/ipfs/go-mfs"
+	unixfs "github.com/ipfs/go-unixfs"
+	balanced "github.com/ipfs/go-unixfs/importer/balanced"
+	ihelper "github.com/ipfs/go-unixfs/importer/helpers"
+	trickle "github.com/ipfs/go-unixfs/importer/trickle"
+	files "github.com/qri-io/qfs/cafs/ipfs/go-ipfs-files"
 )
 
 var log = logging.Logger("coreunix")
@@ -47,6 +65,14 @@ type Object struct {
 	Size  string
 }
 
+// AddEvent is emitted when a unixfs file is completed & has an addressable hash
+type AddEvent struct {
+	Name  string
+	Hash  string `json:",omitempty"`
+	Bytes int64  `json:",omitempty"`
+	Size  string `json:",omitempty"`
+}
+
 // NewAdder Returns a new Adder used for a file add operation.
 func NewAdder(ctx context.Context, p pin.Pinner, bs bstore.GCBlockstore, ds ipld.DAGService) (*Adder, error) {
 	bufferedDS := ipld.NewBufferedDAG(ctx, ds)
@@ -129,11 +155,16 @@ func (adder *Adder) add(reader io.Reader) (ipld.Node, error) {
 		CidBuilder: adder.CidBuilder,
 	}
 
+	p, err := params.New(chnk)
+	if err != nil {
+		return nil, err
+	}
+
 	if adder.Trickle {
-		return trickle.Layout(params.New(chnk))
+		return trickle.Layout(p)
 	}
 
-	return balanced.Layout(params.New(chnk))
+	return balanced.Layout(p)
 }
 
 // RootNode returns the root node of the Added.
@@ -581,7 +612,7 @@ func outputDagnode(out chan<- interface{}, name string, dn ipld.Node) error {
 		return err
 	}
 
-	out <- &coreiface.AddEvent{
+	out <- &AddEvent{
 		Hash: o.Hash,
 		Name: name,
 		Size: o.Size,
@@ -627,7 +658,7 @@ func (i *progressReader) Read(p []byte) (int, error) {
 	i.bytes += int64(n)
 	if i.bytes-i.lastProgress >= progressReaderIncrement || err == io.EOF {
 		i.lastProgress = i.bytes
-		i.out <- &coreiface.AddEvent{
+		i.out <- &AddEvent{
 			Name:  i.file.FileName(),
 			Bytes: i.bytes,
 		}

cafs/ipfs/coreunix/add_test.go

@@ -10,21 +10,21 @@ import (
 	"testing"
 	"time"
 
-	"gx/ipfs/QmUJYo4etAQqFfSS2rarFAE97eNGB8ej64YkRT2SmsYD4r/go-ipfs/core"
-	coreiface "gx/ipfs/QmUJYo4etAQqFfSS2rarFAE97eNGB8ej64YkRT2SmsYD4r/go-ipfs/core/coreapi/interface"
-	"gx/ipfs/QmUJYo4etAQqFfSS2rarFAE97eNGB8ej64YkRT2SmsYD4r/go-ipfs/pin/gc"
-	"gx/ipfs/QmUJYo4etAQqFfSS2rarFAE97eNGB8ej64YkRT2SmsYD4r/go-ipfs/repo"
-
-	config "gx/ipfs/QmPEpj17FDRpc7K1aArKZp3RsHtzRMKykeK9GVgn4WQGPR/go-ipfs-config"
-	cid "gx/ipfs/QmPSQnBKM9g7BaUcZCvswUJVscQ1ipjmwxN5PXCjkp9EQ7/go-cid"
-	pi "gx/ipfs/QmQyUyYcpKG1u53V7N25qRTGw5XwaAxTMKXbduqHotQztg/go-ipfs-posinfo"
-	blocks "gx/ipfs/QmRcHuYzAyswytBuMF78rj3LTChYszomRFXNg4685ZN1WM/go-block-format"
-	dag "gx/ipfs/QmSei8kFMfqdJq7Q68d2LMnHbTWKKg2daA29ezUYFAUNgc/go-merkledag"
-	"gx/ipfs/QmWfhv1D18DRSiSm73r4QGcByspzPtxxRTcmHW3axFXZo8/go-blockservice"
-	files "gx/ipfs/QmZMWMvWMVKCbHetJ4RgndbuEF1io2UpUxwQwtNjtYPzSC/go-ipfs-files"
-	datastore "gx/ipfs/QmaRb5yNXKonhbkpNxNawoydk4N6es6b4fPj19sjEKsh5D/go-datastore"
-	syncds "gx/ipfs/QmaRb5yNXKonhbkpNxNawoydk4N6es6b4fPj19sjEKsh5D/go-datastore/sync"
-	blockstore "gx/ipfs/QmcDDgAXDbpDUpadCJKLr49KYR4HuL7T8Z1dZTHt6ixsoR/go-ipfs-blockstore"
+	"github.com/ipfs/go-ipfs/core"
+	"github.com/ipfs/go-ipfs/pin/gc"
+	"github.com/ipfs/go-ipfs/repo"
+
+	// config "gx/ipfs/QmPEpj17FDRpc7K1aArKZp3RsHtzRMKykeK9GVgn4WQGPR/go-ipfs-config"
+	config "github.com/ipfs/go-ipfs-config"
+	cid "github.com/ipfs/go-cid"
+	pi "github.com/ipfs/go-ipfs-posinfo"
+	blocks "github.com/ipfs/go-block-format"
+	dag "github.com/ipfs/go-merkledag"
+	"github.com/ipfs/go-blockservice"
+	files "github.com/qri-io/qfs/cafs/ipfs/go-ipfs-files"
+	datastore "github.com/ipfs/go-datastore"
+	syncds "github.com/ipfs/go-datastore/sync"
+	blockstore "github.com/ipfs/go-ipfs-blockstore"
 )
 
 const testPeerID = "QmTFauExutTsy4XP6JbMFcw2Wa9645HJt2bTqL6qYDCKfe"
@@ -97,7 +97,7 @@ func TestAddGCLive(t *testing.T) {
 	addedHashes := make(map[string]struct{})
 	select {
 	case o := <-out:
-		addedHashes[o.(*coreiface.AddEvent).Hash] = struct{}{}
+		addedHashes[o.(*AddEvent).Hash] = struct{}{}
 	case <-addDone:
 		t.Fatal("add shouldnt complete yet")
 	}
@@ -125,7 +125,7 @@ func TestAddGCLive(t *testing.T) {
 
 	// receive next object from adder
 	o := <-out
-	addedHashes[o.(*coreiface.AddEvent).Hash] = struct{}{}
+	addedHashes[o.(*AddEvent).Hash] = struct{}{}
 
 	<-gcstarted
 
@@ -141,7 +141,7 @@ func TestAddGCLive(t *testing.T) {
 	var last cid.Cid
 	for a := range out {
 		// wait for it to finish
-		c, err := cid.Decode(a.(*coreiface.AddEvent).Hash)
+		c, err := cid.Decode(a.(*AddEvent).Hash)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -159,6 +159,7 @@ func TestAddGCLive(t *testing.T) {
 }
 
 func testAddWPosInfo(t *testing.T, rawLeaves bool) {
+	t.Skip("TODO (b5): restore this test. low priority b/c qri doesn't need this feature at the moment")
 	r := &repo.Mock{
 		C: config.Config{
 			Identity: config.Identity{
@@ -221,6 +222,7 @@ func TestAddWPosInfo(t *testing.T) {
 }
 
 func TestAddWPosInfoAndRawLeafs(t *testing.T) {
+	t.Skip("TODO (b5): restore this test. low priority b/c qri doesn't need this feature at the moment")
 	testAddWPosInfo(t, true)
 }
 
@@ -271,4 +273,4 @@ func (fi *dummyFileInfo) Size() int64        { return fi.size }
 func (fi *dummyFileInfo) Mode() os.FileMode  { return 0 }
 func (fi *dummyFileInfo) ModTime() time.Time { return fi.modTime }
 func (fi *dummyFileInfo) IsDir() bool        { return false }
-func (fi *dummyFileInfo) Sys() interface{}   { return nil }
+func (fi *dummyFileInfo) Sys() interface{} { return nil }

cafs/ipfs/coreunix/metadata.go

@@ -1,10 +1,14 @@
 package coreunix
 
 import (
-	cid "gx/ipfs/QmPSQnBKM9g7BaUcZCvswUJVscQ1ipjmwxN5PXCjkp9EQ7/go-cid"
-	dag "gx/ipfs/QmSei8kFMfqdJq7Q68d2LMnHbTWKKg2daA29ezUYFAUNgc/go-merkledag"
-	core "gx/ipfs/QmUJYo4etAQqFfSS2rarFAE97eNGB8ej64YkRT2SmsYD4r/go-ipfs/core"
-	ft "gx/ipfs/QmfB3oNXGGq9S4B2a9YeCajoATms3Zw2VvDm8fK7VeLSV8/go-unixfs"
+	// cid "gx/ipfs/QmPSQnBKM9g7BaUcZCvswUJVscQ1ipjmwxN5PXCjkp9EQ7/go-cid"
+	cid "github.com/ipfs/go-cid"
+	// dag "gx/ipfs/QmSei8kFMfqdJq7Q68d2LMnHbTWKKg2daA29ezUYFAUNgc/go-merkledag"
+	dag "github.com/ipfs/go-merkledag"
+	// ft "gx/ipfs/QmfB3oNXGGq9S4B2a9YeCajoATms3Zw2VvDm8fK7VeLSV8/go-unixfs"
+	ft "github.com/ipfs/go-unixfs"
+
+	core "github.com/ipfs/go-ipfs/core"
 )
 
 func AddMetadataTo(n *core.IpfsNode, skey string, m *ft.Metadata) (string, error) {