[WIP] politeiad: Import legacy records to tstore

politeiad/backendv2/tstorebe/tstore/tstore.go

@@ -49,7 +49,7 @@ var (
 // trillian log (tlog) and a key-value store. When data is saved to a tstore
 // instance it is first saved to the key-value store then a digest of the data
 // is appended onto the tlog tree. Tlog trees are episodically timestamped onto
-// the decred blockchain. An inlcusion proof, i.e. the cryptographic proof that
+// the decred blockchain. An inclusion proof, i.e. the cryptographic proof that
 // the data was included in the decred timestamp, can be retrieved for any
 // individual piece of data saved to the tstore.
 //

politeiad/cmd/legacyimport/README.md

@@ -0,0 +1,102 @@
+## legacy
+
+`legacyimport` is a tool that will be used to import the legacy records from
+the git backend into tlog. It opens a connection with tstore and inserts the
+records and blobs manually, bypassing backend validations.
+
+The tool has two commands:
+
+- `dump`
+- `import`
+
+### Flags
+
+Application flags:
+
+`--path`        - Path to git record repository. This flag is mandatory.
+`--comments`    - Enable `comments.journal` parsing.
+  `default: false`
+`--ballot`      - Enable `ballot.journal` parsing.
+  `default: false`
+`--ballotCount` - Limits the number of votes to parse from `ballot.journal`.
+  `default: 0 (all)`
+`--userid`      - Replace userid data of blobs with a userid from the localdb.
+  `default: "" (maintain userid from gitbe payloads)`
+
+Connection to tstore/mysql flags:
+
+`--tloghost`    - Host for tlog.
+  `default: localhost:8090`
+`--tlogpass`    - Password for tlog host.
+  `default: tlogpass`
+`--dbhost`      - Host for mysql db.
+  `default: localhost:3306`
+`--dbpass`    - Password for tlog host.
+  `default: politeiadpass`
+
+
+Notes: 
+
+Limiting the ballot count is useful for testing since parsing the whole
+ballot journal takes a few hours. 
+
+A userid from a localdb must be set in order to test the legacy import locally.
+This is because `politeiawww` will throw a `user not found` error when it tries
+to locally fetch userid's coming from the git records.
+
+##### Usage
+
+Testing scenario importing comments journal and ten cast votes from the ballot 
+journal: 
+
+`legacyimport --path=/path/to/repo 
+  \ --comments 
+  \ --ballot 
+  \ --ballotCount=10 
+  \ --userid="<uuid>"`
+
+Production scenario fully importing all data from legacy records into tstore:
+
+`legacyimport --path=/path/to/repo 
+  \ --comments 
+  \ --ballot`
+
+### Considerations
+
+- Since the `recordmetadata.json` data struct was never signed in the first
+place, it was decided to replace the `Token` field from this struct for the
+newly created tstore token, instead of the legacy gitbe token, when importing
+the records. This solves a lot of functionality problems that arise by having
+the recordmetadata pointing to a legacy gitbe token.
+
+- The vote parameter options from vote details struct was also updated to
+reference the newly created tstore token, in order to preserve summary fetching
+functionality. This struct did not exist on gitbe and therefore we do not lose
+any signature verification capabilities by doing this.
+
+- It was decided to import only the latest version of each record into tstore, 
+and save it as a version 1 record. If one wishes to check further versions of
+a finished legacy record, the git repo will be available.
+
+- Legacy signatures cannot be verified using the current politeia public key.
+The key that should be used for verifying legacy record signatures is:
+
+`a70134196c3cdf3f85f8af6abaa38c15feb7bccf5e6d3db6212358363465e502`
+
+- Some signature verification capabilities were lost due to significant data
+changes and/or the data included on the signature's message. Therefore, below
+is a list of legacy structs we are able to verify on tstore backend, and the
+ones we are not:
+
+##### Verified structs:
+  - cast vote details
+  - auth details
+
+##### Not verified structs:
+  - record
+  - censorship record
+  - user metadata
+  - status change
+  - vote details 
+  - comment
+  - comment del

politeiad/cmd/legacyimport/blobs.go

@@ -0,0 +1,277 @@
+package main
+
+import (
+	"encoding/hex"
+	"encoding/json"
+
+	"github.com/decred/politeia/politeiad/backendv2/tstorebe/store"
+	"github.com/decred/politeia/politeiad/plugins/comments"
+	tv "github.com/decred/politeia/politeiad/plugins/ticketvote"
+	"github.com/decred/politeia/politeiawww/client"
+)
+
+// Ticketvote blobs
+func (l *legacy) saveVotesBlobs(votes []*tv.CastVoteDetails, newToken []byte) error {
+
+	for _, vote := range votes {
+		// Save cast vote details to tstore.
+		err := l.blobSaveCastVoteDetails(*vote, newToken)
+		if err != nil {
+			return err
+		}
+
+		// Save vote collider blob to tstore.
+		err = l.blobSaveVoteCollider(voteCollider{
+			Token:  hex.EncodeToString(newToken),
+			Ticket: vote.Ticket,
+		}, newToken)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (l *legacy) blobSaveCastVoteDetails(cdv tv.CastVoteDetails, newToken []byte) error {
+	// Verify cast vote details signature.
+	err := client.CastVoteDetailsVerify(convertCastVoteDetailsToV1(cdv), serverPubkey)
+	if err != nil {
+		return err
+	}
+
+	data, err := json.Marshal(cdv)
+	if err != nil {
+		return err
+	}
+	hint, err := json.Marshal(
+		store.DataDescriptor{
+			Type:       store.DataTypeStructure,
+			Descriptor: tv.PluginID + "-castvote-v1",
+		})
+	if err != nil {
+		return err
+	}
+	be := store.NewBlobEntry(hint, data)
+
+	err = l.tstore.BlobSave(newToken, be)
+	if err != nil && err.Error() == "duplicate payload" {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (l *legacy) blobSaveAuthDetails(authDetails tv.AuthDetails, newToken []byte) error {
+	// Verify auth details signature.
+	err := client.AuthDetailsVerify(convertAuthDetailsToV1(authDetails),
+		serverPubkey)
+	if err != nil {
+		return err
+	}
+
+	data, err := json.Marshal(authDetails)
+	if err != nil {
+		return err
+	}
+	hint, err := json.Marshal(
+		store.DataDescriptor{
+			Type:       store.DataTypeStructure,
+			Descriptor: tv.PluginID + "-auth-v1",
+		})
+	if err != nil {
+		return err
+	}
+
+	be := store.NewBlobEntry(hint, data)
+	err = l.tstore.BlobSave(newToken, be)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (l *legacy) blobSaveVoteDetails(voteDetails tv.VoteDetails, newToken []byte) error {
+	data, err := json.Marshal(voteDetails)
+	if err != nil {
+		return err
+	}
+	hint, err := json.Marshal(
+		store.DataDescriptor{
+			Type:       store.DataTypeStructure,
+			Descriptor: tv.PluginID + "-vote-v1",
+		})
+	if err != nil {
+		return err
+	}
+
+	be := store.NewBlobEntry(hint, data)
+	err = l.tstore.BlobSave(newToken, be)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (l *legacy) blobSaveVoteCollider(vc voteCollider, newToken []byte) error {
+	data, err := json.Marshal(vc)
+	if err != nil {
+		return err
+	}
+	hint, err := json.Marshal(
+		store.DataDescriptor{
+			Type:       store.DataTypeStructure,
+			Descriptor: tv.PluginID + "-vcollider-v1",
+		})
+	if err != nil {
+		return err
+	}
+
+	be := store.NewBlobEntry(hint, data)
+	err = l.tstore.BlobSave(newToken, be)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (l *legacy) blobSaveStartRunoff(srr startRunoffRecord, newToken []byte) error {
+	data, err := json.Marshal(srr)
+	if err != nil {
+		return err
+	}
+	hint, err := json.Marshal(
+		store.DataDescriptor{
+			Type:       store.DataTypeStructure,
+			Descriptor: tv.PluginID + "-startrunoff-v1",
+		})
+	if err != nil {
+		return err
+	}
+
+	be := store.NewBlobEntry(hint, data)
+	err = l.tstore.BlobSave(newToken, be)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Comments Blobs
+
+// saveCommentsBlobs ...
+func (l *legacy) saveCommentsBlobs(comments parsedComments, newToken []byte) error {
+	// Save add comments blob to tstore.
+	for _, add := range comments.Adds {
+		err := l.blobSaveCommentAdd(add, newToken)
+		if err != nil {
+			return err
+		}
+	}
+
+	// Save del comments blob to tstore.
+	for _, del := range comments.Dels {
+		err := l.blobSaveCommentDel(del, newToken)
+		if err != nil {
+			return err
+		}
+	}
+
+	// Save vote comments blob to tstore.
+	for _, vote := range comments.Votes {
+		err := l.blobSaveCommentVote(vote, newToken)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (l *legacy) blobSaveCommentAdd(add comments.CommentAdd, newToken []byte) error {
+	// Create blob entry.
+	data, err := json.Marshal(add)
+	if err != nil {
+		return err
+	}
+	hint, err := json.Marshal(
+		store.DataDescriptor{
+			Type:       store.DataTypeStructure,
+			Descriptor: comments.PluginID + "-add-v1",
+		})
+	if err != nil {
+		return err
+	}
+	be := store.NewBlobEntry(hint, data)
+
+	// Save to tstore.
+	err = l.tstore.BlobSave(newToken, be)
+	if err != nil && err.Error() == "duplicate payload" {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (l *legacy) blobSaveCommentDel(del comments.CommentDel, newToken []byte) error {
+	// Create blob entry.
+	data, err := json.Marshal(del)
+	if err != nil {
+		return err
+	}
+	hint, err := json.Marshal(
+		store.DataDescriptor{
+			Type:       store.DataTypeStructure,
+			Descriptor: comments.PluginID + "-del-v1",
+		})
+	if err != nil {
+		return err
+	}
+	be := store.NewBlobEntry(hint, data)
+
+	// Save to tstore.
+	err = l.tstore.BlobSave(newToken, be)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (l *legacy) blobSaveCommentVote(vote comments.CommentVote, newToken []byte) error {
+	// Create blob entry.
+	data, err := json.Marshal(vote)
+	if err != nil {
+		return err
+	}
+	hint, err := json.Marshal(
+		store.DataDescriptor{
+			Type:       store.DataTypeStructure,
+			Descriptor: comments.PluginID + "-vote-v1",
+		})
+	if err != nil {
+		return err
+	}
+	be := store.NewBlobEntry(hint, data)
+
+	// Save to tstore.
+	err = l.tstore.BlobSave(newToken, be)
+	if err != nil && err.Error() == "duplicate payload" {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+
+	return nil
+}