Merge pull request #196 from mit-dci/shortcutserve

Split proof building and serving functions

bridgenode/chainio.go

@@ -2,6 +2,7 @@ package bridgenode
 
 import (
 	"encoding/binary"
+	"fmt"
 	"os"
 
 	"github.com/btcsuite/btcd/chaincfg"
@@ -78,7 +79,6 @@ func restoreForest(
 
 // restoreHeight restores height from util.ForestLastSyncedBlockHeightFilePath
 func restoreHeight() (height int32, err error) {
-
 	// if there is a heightfile, get the height from that
 	// heightFile saves the last block that was written to ttldb
 	if util.HasAccess(util.ForestLastSyncedBlockHeightFilePath) {
@@ -92,6 +92,9 @@ func restoreHeight() (height int32, err error) {
 		if err != nil {
 			return 0, err
 		}
+	} else {
+		return 0, fmt.Errorf("can't read height at %s\n",
+			util.ForestLastSyncedBlockHeightFilePath)
 	}
 	return
 }

bridgenode/genproofs.go

@@ -151,16 +151,6 @@ func BuildProofs(
 
 	fmt.Println("Done writing")
 
-	if stop {
-		// genproofs was paused.
-		// Tell stopBuildProofs that it's ok to exit
-		haltAccept <- true
-		return nil
-	}
-
-	// should be a goroutine..?  isn't right now
-	blockServer(knownTipHeight, dataDir, haltRequest, haltAccept, lvdb)
-
 	// Tell stopBuildProofs that it's ok to exit
 	haltAccept <- true
 	return nil
@@ -297,7 +287,13 @@ func stopBuildProofs(
 	sig, offsetfinished, haltRequest, haltAccept chan bool) {
 
 	// Listen for SIGINT, SIGQUIT, SIGTERM
-	<-sig
+	// Also listen for an unrequested haltAccept which means upstream is finshed
+	// and to end this goroutine
+	select {
+	case <-haltAccept:
+		return
+	case <-sig:
+	}
 
 	trace.Stop()
 	pprof.StopCPUProfile()

bridgenode/server.go

@@ -5,15 +5,82 @@ import (
 	"fmt"
 	"math"
 	"net"
+	"os"
+	"time"
 
+	"github.com/btcsuite/btcd/chaincfg"
 	"github.com/mit-dci/utreexo/util"
 	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/opt"
 )
 
+func ArchiveServer(param chaincfg.Params, dataDir string, sig chan bool) error {
+
+	// Channel to alert the tell the main loop it's ok to exit
+	haltRequest := make(chan bool, 1)
+
+	// Channel for ServeBlock() to wait
+	haltAccept := make(chan bool, 1)
+
+	// Handle user interruptions
+	go stopServer(sig, haltRequest, haltAccept)
+
+	_, err := os.Stat(dataDir)
+	if os.IsNotExist(err) {
+		return fmt.Errorf("%s not found, can't serve blocks\n", dataDir)
+	}
+
+	// TODO ****** server shouldn't need levelDB access, fix this
+	ttlpath := "utree/" + param.Name + "ttldb"
+	// Open leveldb
+	o := opt.Options{CompactionTableSizeMultiplier: 8}
+	lvdb, err := leveldb.OpenFile(ttlpath, &o)
+	if err != nil {
+		fmt.Printf("initialization error.  If your .blk and .dat files are ")
+		fmt.Printf("not in %s, specify alternate path with -datadir\n.", dataDir)
+		return err
+	}
+	defer lvdb.Close()
+	// **********************************
+
+	// Init forest and variables. Resumes if the data directory exists
+	maxHeight, err := restoreHeight()
+	if err != nil {
+		return err
+	}
+
+	blockServer(maxHeight, dataDir, haltRequest, haltAccept, lvdb)
+
+	return nil
+}
+
+// stopServer listens for the signal from the OS and initiates an exit sequence
+func stopServer(sig, haltRequest, haltAccept chan bool) {
+
+	// Listen for SIGINT, SIGQUIT, SIGTERM
+	<-sig
+	haltRequest <- true
+	// Sometimes there are bugs that make the program run forever.
+	// Utreexo binary should never take more than 10 seconds to exit
+	go func() {
+		time.Sleep(2 * time.Second)
+		fmt.Println("Exit timed out. Force quitting.")
+		os.Exit(1)
+	}()
+
+	// Tell the user that the sig is received
+	fmt.Println("User exit signal received. Exiting...")
+
+	// Wait until server says it's ok to exit
+	<-haltAccept
+	os.Exit(0)
+}
+
 // blockServer listens on a TCP port for incoming connections, then gives
 // ublocks blocks over that connection
 func blockServer(endHeight int32, dataDir string, haltRequest,
 	haltAccept chan bool, lvdb *leveldb.DB) {
+	fmt.Printf("serving up to & including block height %d\n", endHeight)
 	listenAdr, err := net.ResolveTCPAddr("tcp", "0.0.0.0:8338")
 	if err != nil {
 		fmt.Printf(err.Error())
@@ -28,7 +95,6 @@ func blockServer(endHeight int32, dataDir string, haltRequest,
 
 	cons := make(chan net.Conn)
 	go acceptConnections(listener, cons)
-
 	for {
 		select {
 		case <-haltRequest:
@@ -43,6 +109,7 @@ func blockServer(endHeight int32, dataDir string, haltRequest,
 }
 
 func acceptConnections(listener *net.TCPListener, cons chan net.Conn) {
+	fmt.Printf("listening for connections on %s\n", listener.Addr().String())
 	for {
 		select {
 		case <-cons:
@@ -63,7 +130,8 @@ func acceptConnections(listener *net.TCPListener, cons chan net.Conn) {
 
 // serveBlocksWorker gets height requests from client and sends out the ublock
 // for that height
-func serveBlocksWorker(c net.Conn, endHeight int32, blockDir string, lvdb *leveldb.DB) {
+func serveBlocksWorker(
+	c net.Conn, endHeight int32, blockDir string, lvdb *leveldb.DB) {
 	defer c.Close()
 	fmt.Printf("start serving %s\n", c.RemoteAddr().String())
 	var fromHeight, toHeight int32
@@ -136,7 +204,8 @@ func serveBlocksWorker(c net.Conn, endHeight int32, blockDir string, lvdb *level
 			udb = ud.ToBytes()
 
 			// fmt.Printf("h %d read %d byte udb\n", curHeight, len(udb))
-			blkbytes, err := GetBlockBytesFromFile(curHeight, util.OffsetFilePath, blockDir)
+			blkbytes, err := GetBlockBytesFromFile(
+				curHeight, util.OffsetFilePath, blockDir)
 			if err != nil {
 				fmt.Printf("pushBlocks GetRawBlockFromFile %s\n", err.Error())
 				return

cmd/utreexoserver/bridgeserver.go

@@ -30,6 +30,7 @@ OPTIONS:
 
   -cpuprof                     configure whether to use use cpu profiling
   -memprof                     configure whether to use use heap profiling
+  -serve						 immediately serve whatever data is built
 
 `
 
@@ -44,6 +45,8 @@ var forestInRam = optionCmd.Bool("inram", false,
 	`keep forest in ram instead of disk.  Faster but needs lots of ram`)
 var forestCache = optionCmd.Bool("cache", false,
 	`use ram-cached forest.  Speed between on disk and fully in-ram`)
+var serve = optionCmd.Bool("serve", false,
+	`immediately start server without building or checking proof data`)
 var traceCmd = optionCmd.String("trace", "",
 	`Enable trace. Usage: 'trace='path/to/file'`)
 var cpuProfCmd = optionCmd.String("cpuprof", "",
@@ -114,13 +117,22 @@ func main() {
 	sig := make(chan bool, 1)
 	handleIntSig(sig, *cpuProfCmd, *traceCmd)
 
-	fmt.Printf("datadir is %s\n", dataDir)
-	err := bridge.BuildProofs(param, dataDir, *forestInRam, *forestCache, sig)
+	// only do buildProofs or serve; need to restart to serve after
+	// building proofs
+
+	if !*serve {
+		fmt.Printf("datadir is %s\n", dataDir)
+		err := bridge.BuildProofs(param, dataDir, *forestInRam, *forestCache, sig)
+		if err != nil {
+			fmt.Printf("Buildproofs error: %s\n", err.Error())
+			panic("proof build halting")
+		}
+	}
+	err := bridge.ArchiveServer(param, dataDir, sig)
 	if err != nil {
-		fmt.Printf("Buildproofs error: %s\n", err.Error())
+		fmt.Printf("ArchiveServer error: %s\n", err.Error())
 		panic("server halting")
 	}
-
 }
 
 func handleIntSig(sig chan bool, cpuProfCmd, traceCmd string) {

test/install_bitcoind.sh

@@ -5,17 +5,17 @@
 
 set -ev
 
-export BITCOIND_VERSION=0.20.0
+export BITCOIND_VERSION=0.20.1
 
 if sudo cp ~/bitcoin/bitcoin-$BITCOIND_VERSION/bin/bitcoind /usr/local/bin/bitcoind
 then
         echo "found cached bitcoind"
 else
         mkdir -p ~/bitcoin && \
         pushd ~/bitcoin && \
-        wget https://bitcoin.org/bin/bitcoin-core-$BITCOIND_VERSION/bitcoin-$BITCOIND_VERSION-x86_64-linux-gnu.tar.gz && \
+        wget -q https://adiabat.github.io/bitcoin-$BITCOIND_VERSION-x86_64-linux-gnu.tar.gz && \
         tar xvfz bitcoin-$BITCOIND_VERSION-x86_64-linux-gnu.tar.gz && \
         sudo cp ./bitcoin-$BITCOIND_VERSION/bin/bitcoind /usr/local/bin/bitcoind && \
         sudo cp ./bitcoin-$BITCOIND_VERSION/bin/bitcoin-cli /usr/local/bin/bitcoin-cli && \
         popd
-fi
+fi