diff --git a/lib/bitcoin/BitcoinProcessor.ts b/lib/bitcoin/BitcoinProcessor.ts index 52188be16..d39a5d2e2 100644 --- a/lib/bitcoin/BitcoinProcessor.ts +++ b/lib/bitcoin/BitcoinProcessor.ts @@ -159,8 +159,21 @@ export default class BitcoinProcessor { await this.normalizedFeeCalculator.initialize(); await this.mongoDbLockTransactionStore.initialize(); + // We always need to start the processing from the first block of a fee sampling group + // so that in-memory state for fee sampling will be repoppulated yielding correct fee calculation, + // so we trim the databases to make sure this condition is met. + // NOTE: We also initialize the `lastProcessedBlock`, this is an opional step currently, + // but will be required if Issue #692 is implemented. + this.lastProcessedBlock = await this.trimDatabasesToLastFeeSamplingGroupBoundary(); + console.debug('Synchronizing blocks for sidetree transactions...'); - const startingBlock = await this.getStartingBlockForInitialization(); + const startingBlock = await this.getStartingBlockForPeriodicPoll(); + + // Throw if bitcoin client is not synced up to the bitcoin service's known height. + // NOTE: Implementation for issue #692 can simplify this method and remove this check. + if (startingBlock === undefined) { + throw new SidetreeError(ErrorCode.BitcoinProcessorBitcoinClientCurrentHeightNotUpToDate); + } console.info(`Starting block: ${startingBlock.height} (${startingBlock.hash})`); await this.processTransactions(startingBlock); @@ -172,6 +185,27 @@ export default class BitcoinProcessor { void this.periodicPoll(); } + /** + * NOTE: Should be used ONLY during service initialization. + * @returns The last processed block after trimming. `undefined` if all data are deleted after trimming. + */ + private async trimDatabasesToLastFeeSamplingGroupBoundary (): Promise { + // Look in the transaction store to figure out the last block that we need to start from. + const lastSavedTransaction = await this.transactionStore.getLastTransaction(); + + // If there is no transaction saved in the DBs, we still need to perform trimming + // because there could be stale fee calculater data lingering, so we trim using the genesis block. + let lastValidBlock; + if (lastSavedTransaction === undefined) { + lastValidBlock = await this.trimDatabasesToFeeSamplingGroupBoundary(this.genesisBlockNumber); + } else { + // Else we trim DBs using the block number of the last saved transaction. + lastValidBlock = await this.trimDatabasesToFeeSamplingGroupBoundary(lastSavedTransaction.transactionTime); + } + + return lastValidBlock; + } + /** * Gets the blockchain time of the given time hash. * Gets the latest logical blockchain time if time hash is not given. @@ -435,45 +469,14 @@ export default class BitcoinProcessor { return this.lastProcessedBlock!; } - private async getStartingBlockForInitialization (): Promise { - - // Look in the transaction store to figure out the last block that we need to - // start from. - const lastSavedTransaction = (await this.transactionStore.getLastTransaction()); - - // If there's nothing saved in the DB then let's start from the genesis block - if (!lastSavedTransaction) { - // Put the system DBs in the correct state. We are going to ignore the return value - // from the following call as we already know the starting block is the - // genesis block. - await this.trimDatabasesToFeeSamplingGroupBoundary(this.genesisBlockNumber); + private async getStartingBlockForPeriodicPoll (): Promise { + // If last processed block is undefined, start processing from genesis block. + if (this.lastProcessedBlock === undefined) { + await this.trimDatabasesToLastFeeSamplingGroupBoundary(); return this.bitcoinClient.getBlockInfoFromHeight(this.genesisBlockNumber); } - // If we are here then it means that there is a potential starting point in the DB. - // Since we are initializing, it is quite possible that the last block that we processed - // (and saved in the db) has been forked. Check for the fork. - const lastSavedBlockIsValid = await this.verifyBlock(lastSavedTransaction.transactionTime, lastSavedTransaction.transactionTimeHash); - - let lastValidBlock: IBlockInfo | undefined; - if (lastSavedBlockIsValid) { - // There was no fork ... let's put the system DBs in the correct state. - lastValidBlock = await this.trimDatabasesToFeeSamplingGroupBoundary(lastSavedTransaction.transactionTime); - } else { - // There was a fork so we need to revert. The revert function peforms all the correct - // operations and puts the system in the correct state and returns the last valid block. - lastValidBlock = await this.revertDatabases(); - } - - // If there is a valid processed block, we will start processing the block following it, else start processing from the genesis block. - const startingBlockHeight = lastValidBlock ? lastValidBlock.height + 1 : this.genesisBlockNumber; - - return this.bitcoinClient.getBlockInfoFromHeight(startingBlockHeight); - } - - private async getStartingBlockForPeriodicPoll (): Promise { - - const lastProcessedBlockIsValid = await this.verifyBlock(this.lastProcessedBlock!.height, this.lastProcessedBlock!.hash); + const lastProcessedBlockIsValid = await this.verifyBlock(this.lastProcessedBlock.height, this.lastProcessedBlock.hash); // If the last processed block is not valid then that means that we need to // revert the DB back to a known valid block. diff --git a/lib/bitcoin/ErrorCode.ts b/lib/bitcoin/ErrorCode.ts index 4d7fbad72..e4dfb84ff 100644 --- a/lib/bitcoin/ErrorCode.ts +++ b/lib/bitcoin/ErrorCode.ts @@ -1,5 +1,6 @@ export default { BitcoinWalletIncorrectImportString: 'bitcoin_wallet_incorrect_import_string', + BitcoinProcessorBitcoinClientCurrentHeightNotUpToDate: 'bitcion_processor_bitcoin_client_current_height_not_up_to_date', BitcoinProcessorCannotProcessBlocksBeforeGenesis: 'bitcoin_processor_cannot_process_blocks_before_genesis', BitcoinProcessInvalidPreviousBlockHash: 'bitcoin_processor_invalid_previous_block_hash', LockIdentifierIncorrectFormat: 'lock_identifier_incorrect_format', diff --git a/tests/bitcoin/BitcoinProcessor.spec.ts b/tests/bitcoin/BitcoinProcessor.spec.ts index 784ec1e06..f81c0970e 100644 --- a/tests/bitcoin/BitcoinProcessor.spec.ts +++ b/tests/bitcoin/BitcoinProcessor.spec.ts @@ -58,11 +58,12 @@ describe('BitcoinProcessor', () => { let normalizedFeeCalculatorInitializeSpy: jasmine.Spy; let mongoQuantileStoreInitializeSpy: jasmine.Spy; let transactionStoreLatestTransactionSpy: jasmine.Spy; - let getStartingBlockForInitializationSpy: jasmine.Spy; + let getStartingBlockForPeriodicPollSpy: jasmine.Spy; let processTransactionsSpy: jasmine.Spy; let periodicPollSpy: jasmine.Spy; let mongoLockTxnStoreSpy: jasmine.Spy; let lockMonitorSpy: jasmine.Spy; + let trimDatabasesToLastFeeSamplingGroupBoundarySpy: jasmine.Spy; beforeEach(() => { bitcoinProcessor = new BitcoinProcessor(testConfig); @@ -74,8 +75,8 @@ describe('BitcoinProcessor', () => { transactionStoreLatestTransactionSpy = spyOn(bitcoinProcessor['transactionStore'], 'getLastTransaction'); transactionStoreLatestTransactionSpy.and.returnValue(Promise.resolve(undefined)); - getStartingBlockForInitializationSpy = spyOn(bitcoinProcessor as any, 'getStartingBlockForInitialization'); - getStartingBlockForInitializationSpy.and.returnValue(Promise.resolve(undefined)); + getStartingBlockForPeriodicPollSpy = spyOn(bitcoinProcessor as any, 'getStartingBlockForPeriodicPoll'); + getStartingBlockForPeriodicPollSpy.and.returnValue(Promise.resolve(undefined)); normalizedFeeCalculatorInitializeSpy = spyOn(bitcoinProcessor['normalizedFeeCalculator'], 'initialize'); normalizedFeeCalculatorInitializeSpy.and.returnValue(Promise.resolve()); @@ -85,7 +86,10 @@ describe('BitcoinProcessor', () => { processTransactionsSpy = spyOn(bitcoinProcessor, 'processTransactions' as any); processTransactionsSpy.and.returnValue(Promise.resolve({ hash: 'IamAHash', height: 54321 })); + periodicPollSpy = spyOn(bitcoinProcessor, 'periodicPoll' as any); + trimDatabasesToLastFeeSamplingGroupBoundarySpy = spyOn(bitcoinProcessor as any, 'trimDatabasesToLastFeeSamplingGroupBoundary'); + }); function createTransactions (count?: number, height?: number, incrementalHeight = false): TransactionModel[] { @@ -148,12 +152,12 @@ describe('BitcoinProcessor', () => { }); describe('initialize', () => { - beforeEach(async () => { bitcoinClientInitializeSpy.and.returnValue(Promise.resolve()); - getStartingBlockForInitializationSpy.and.returnValue(Promise.resolve({ height: 123, hash: 'hash' })); + getStartingBlockForPeriodicPollSpy.and.returnValue(Promise.resolve({ height: 123, hash: 'hash' })); mongoLockTxnStoreSpy.and.returnValue(Promise.resolve()); lockMonitorSpy.and.returnValue(Promise.resolve()); + trimDatabasesToLastFeeSamplingGroupBoundarySpy.and.returnValue(Promise.resolve('unused')); }); it('should initialize the internal objects', async (done) => { @@ -176,11 +180,22 @@ describe('BitcoinProcessor', () => { done(); }); + it('should throw error if unable to find a starting block.', async (done) => { + getStartingBlockForPeriodicPollSpy.and.returnValue(Promise.resolve(undefined)); + + await JasmineSidetreeErrorValidator.expectSidetreeErrorToBeThrownAsync( + () => bitcoinProcessor.initialize(), + ErrorCode.BitcoinProcessorBitcoinClientCurrentHeightNotUpToDate + ); + + done(); + }); + it('should process all the blocks since its last known', async (done) => { const fromNumber = randomNumber(); const fromHash = randomString(); - getStartingBlockForInitializationSpy.and.returnValue( + getStartingBlockForPeriodicPollSpy.and.returnValue( Promise.resolve({ height: fromNumber, hash: fromHash @@ -195,10 +210,10 @@ describe('BitcoinProcessor', () => { height: 12345 }); }); - expect(getStartingBlockForInitializationSpy).not.toHaveBeenCalled(); + expect(getStartingBlockForPeriodicPollSpy).not.toHaveBeenCalled(); expect(processTransactionsSpy).not.toHaveBeenCalled(); await bitcoinProcessor.initialize(); - expect(getStartingBlockForInitializationSpy).toHaveBeenCalled(); + expect(getStartingBlockForPeriodicPollSpy).toHaveBeenCalled(); expect(processTransactionsSpy).toHaveBeenCalled(); done(); }); @@ -684,7 +699,7 @@ describe('BitcoinProcessor', () => { }); }); - spyOn(bitcoinProcessor as any,'getStartingBlockForPeriodicPoll').and.returnValue(Promise.resolve(lastBlock)); + getStartingBlockForPeriodicPollSpy.and.returnValue(Promise.resolve(lastBlock)); /* tslint:disable-next-line */ await bitcoinProcessor['periodicPoll'](); // need to wait for the process call @@ -699,7 +714,7 @@ describe('BitcoinProcessor', () => { }); it('should not call process transaction if the starting block is undefined', async (done) => { - spyOn(bitcoinProcessor as any,'getStartingBlockForPeriodicPoll').and.returnValue(Promise.resolve(undefined)); + getStartingBlockForPeriodicPollSpy.and.returnValue(Promise.resolve(undefined)); await bitcoinProcessor['periodicPoll'](); // need to wait for the process call @@ -715,7 +730,7 @@ describe('BitcoinProcessor', () => { }); it('should not throw if the processing throws', async (done) => { - spyOn(bitcoinProcessor as any,'getStartingBlockForPeriodicPoll').and.throwError('Test error'); + getStartingBlockForPeriodicPollSpy.and.throwError('Test error'); try { await bitcoinProcessor['periodicPoll'](); @@ -740,7 +755,7 @@ describe('BitcoinProcessor', () => { height: randomNumber() })); - spyOn(bitcoinProcessor as any,'getStartingBlockForPeriodicPoll').and.returnValue(bitcoinProcessor['lastProcessedBlock']); + getStartingBlockForPeriodicPollSpy.and.returnValue(bitcoinProcessor['lastProcessedBlock']); /* tslint:disable-next-line */ bitcoinProcessor['periodicPoll'](); @@ -755,7 +770,7 @@ describe('BitcoinProcessor', () => { it('should clear the prevoius timeout if set', async (done) => { - spyOn(bitcoinProcessor as any,'getStartingBlockForPeriodicPoll').and.returnValue(Promise.resolve()); + getStartingBlockForPeriodicPollSpy.and.returnValue(Promise.resolve()); const clearTimeoutSpy = spyOn(global, 'clearTimeout').and.returnValue(); bitcoinProcessor['pollTimeoutId'] = 1234; @@ -860,157 +875,14 @@ describe('BitcoinProcessor', () => { }); }); - describe('getStartingBlockForInitialization', () => { - - beforeEach(() => { - getStartingBlockForInitializationSpy.and.callThrough(); - }); - - it('should return the genesis block if no transactions are saved in the DB', async (done) => { - - const mockBlock: IBlockInfo = { - hash: 'some_hash', - height: randomNumber(), - previousHash: 'some previous hash' - }; - - spyOn(bitcoinProcessor['bitcoinClient'], 'getBlockInfoFromHeight').and.callFake((inputBlockNumber) => { - expect(inputBlockNumber).toEqual(bitcoinProcessor['genesisBlockNumber']); - - return Promise.resolve(mockBlock); - }); - - transactionStoreLatestTransactionSpy.and.returnValue(Promise.resolve()); - const revertDbsSpy = spyOn(bitcoinProcessor as any, 'trimDatabasesToFeeSamplingGroupBoundary'); - - const startingBlock = await bitcoinProcessor['getStartingBlockForInitialization'](); - expect(startingBlock).toEqual(mockBlock); - expect(revertDbsSpy).toHaveBeenCalledWith(bitcoinProcessor['genesisBlockNumber']); - done(); - }); - - it('should revert the DBs to the last saved transaction block in the transaction store.', async (done) => { - const revertDbsSpy = spyOn(bitcoinProcessor as any, 'trimDatabasesToFeeSamplingGroupBoundary'); - const verifySpy = spyOn(bitcoinProcessor as any, 'verifyBlock'); - const revertChainSpy = spyOn(bitcoinProcessor as any, 'revertDatabases'); - - const mockTxnModel: TransactionModel = { - anchorString: 'anchor1', - transactionTimeHash: 'timehash1', - transactionTime: 100, - transactionNumber: 200, - transactionFeePaid: 300, - normalizedTransactionFee: 400, - writer: 'writer' - }; - - const mockBlock: IBlockInfo = { - hash: 'some_hash', - height: randomNumber(), - previousHash: 'some previous hash' - }; - - transactionStoreLatestTransactionSpy.and.returnValue(Promise.resolve(mockTxnModel)); - revertDbsSpy.and.returnValue(Promise.resolve(mockBlock)); - verifySpy.and.returnValue(Promise.resolve(true)); - - spyOn(bitcoinProcessor['bitcoinClient'], 'getBlockInfoFromHeight').and.callFake((inputBlockNumber) => { - expect(inputBlockNumber).toEqual(mockBlock.height + 1); - - return Promise.resolve(mockBlock); - }); - - const startingBlock = await bitcoinProcessor['getStartingBlockForInitialization'](); - expect(startingBlock).toEqual(mockBlock); - expect(revertDbsSpy).toHaveBeenCalledWith(mockTxnModel.transactionTime); - expect(verifySpy).toHaveBeenCalledWith(mockTxnModel.transactionTime, mockTxnModel.transactionTimeHash); - expect(revertChainSpy).not.toHaveBeenCalled(); - done(); - }); - - it('should revert the blockchain if the last saved transaction is invalid.', async (done) => { - const revertDbsSpy = spyOn(bitcoinProcessor as any, 'trimDatabasesToFeeSamplingGroupBoundary'); - const verifySpy = spyOn(bitcoinProcessor as any, 'verifyBlock'); - const revertChainSpy = spyOn(bitcoinProcessor as any, 'revertDatabases'); - - const mockTxnModel: TransactionModel = { - anchorString: 'anchor1', - transactionTimeHash: 'timehash1', - transactionTime: 100, - transactionNumber: 200, - transactionFeePaid: 300, - normalizedTransactionFee: 400, - writer: 'writer' - }; - - const mockBlock: IBlockInfo = { - hash: 'some_hash', - height: randomNumber(), - previousHash: 'some previous hash' - }; - - transactionStoreLatestTransactionSpy.and.returnValue(Promise.resolve(mockTxnModel)); - verifySpy.and.returnValue(Promise.resolve(false)); - revertChainSpy.and.returnValue(Promise.resolve(mockBlock)); - - spyOn(bitcoinProcessor['bitcoinClient'], 'getBlockInfoFromHeight').and.callFake((inputBlockNumber) => { - expect(inputBlockNumber).toEqual(mockBlock.height + 1); - - return Promise.resolve(mockBlock); - }); - - const startingBlock = await bitcoinProcessor['getStartingBlockForInitialization'](); - - expect(startingBlock).toEqual(mockBlock); - expect(revertDbsSpy).not.toHaveBeenCalled(); - expect(verifySpy).toHaveBeenCalledWith(mockTxnModel.transactionTime, mockTxnModel.transactionTimeHash); - expect(revertChainSpy).toHaveBeenCalled(); - done(); - }); - - it('should use genesis block as the starting block if no valid block is left after reverting.', async (done) => { - - const mockLastSavedTransaction: TransactionModel = { - anchorString: 'unused', - transactionTimeHash: 'unused', - transactionTime: 100, - transactionNumber: 200, - transactionFeePaid: 300, - normalizedTransactionFee: 400, - writer: 'unused' - }; - transactionStoreLatestTransactionSpy.and.returnValue(Promise.resolve(mockLastSavedTransaction)); - - const verifyBlockSpy = spyOn(bitcoinProcessor as any, 'verifyBlock'); - verifyBlockSpy.and.returnValue(Promise.resolve(false)); - - // Simulate no valid block left after reverting. - const revertDatabaseSpy = spyOn(bitcoinProcessor as any, 'revertDatabases'); - revertDatabaseSpy.and.returnValue(Promise.resolve(undefined)); - - const mockBlock: IBlockInfo = { - hash: 'unused', - height: randomNumber(), - previousHash: 'unused' - }; - const getBlockInfoFromHeightSpy = spyOn(bitcoinProcessor['bitcoinClient'], 'getBlockInfoFromHeight'); - getBlockInfoFromHeightSpy.and.returnValue(Promise.resolve(mockBlock)); - - // We don't care about the mocked return value, we only care that `getBlockInfoFromHeightSpy()` is invoked with the genesis block height. - await bitcoinProcessor['getStartingBlockForInitialization'](); - - expect(verifyBlockSpy).toHaveBeenCalledWith(mockLastSavedTransaction.transactionTime, mockLastSavedTransaction.transactionTimeHash); - expect(revertDatabaseSpy).toHaveBeenCalled(); - expect(getBlockInfoFromHeightSpy).toHaveBeenCalledWith(bitcoinProcessor.genesisBlockNumber); - done(); - }); - }); - describe('getStartingBlockForPeriodicPoll', () => { let actualLastProcessedBlock: IBlockInfo; let getBlockInfoFromHeightSpy: jasmine.Spy; beforeEach(() => { + // Revert the spy call in parent beforeEach(); + getStartingBlockForPeriodicPollSpy.and.callThrough(); + bitcoinProcessor['lastProcessedBlock'] = { height: randomNumber(), hash: randomString(), previousHash: randomString() }; actualLastProcessedBlock = bitcoinProcessor['lastProcessedBlock']; getBlockInfoFromHeightSpy = spyOn(bitcoinProcessor['bitcoinClient'], 'getBlockInfoFromHeight'); @@ -1056,6 +928,19 @@ describe('BitcoinProcessor', () => { expect(revertBlockchainSpy).toHaveBeenCalled(); }); + it('should use genesis block as the starting block if no last processed block is found.', async (done) => { + // Simulate no last processed block is found. + bitcoinProcessor['lastProcessedBlock'] = undefined; + + await bitcoinProcessor['getStartingBlockForPeriodicPoll'](); + + expect(trimDatabasesToLastFeeSamplingGroupBoundarySpy).toHaveBeenCalled(); + + // We don't care about the mocked return value, we only care that `getBlockInfoFromHeightSpy()` is invoked with the genesis block height. + expect(getBlockInfoFromHeightSpy).toHaveBeenCalledWith(bitcoinProcessor.genesisBlockNumber); + done(); + }); + it('should use genesis block as the starting block if no valid block is left after reverting.', async (done) => { const verifyBlockSpy = spyOn(bitcoinProcessor as any, 'verifyBlock'); verifyBlockSpy.and.returnValue(Promise.resolve(false)); @@ -1123,6 +1008,43 @@ describe('BitcoinProcessor', () => { }); }); + describe('trimDatabasesToLastFeeSamplingGroupBoundary', () => { + beforeEach(() => { + trimDatabasesToLastFeeSamplingGroupBoundarySpy.and.callThrough(); + }); + + it('should trim based on the blockchain time of last saved transaction.', async (done) => { + // Simulate a saved transaction found. + const mockLastSaveTransaction: TransactionModel = { + anchorString: 'unused', + transactionTimeHash: 'unused', + transactionTime: 100, + transactionNumber: 200, + transactionFeePaid: 300, + normalizedTransactionFee: 400, + writer: 'unused' + }; + transactionStoreLatestTransactionSpy.and.returnValue(Promise.resolve(mockLastSaveTransaction)); + + const trimDatabasesToFeeSamplingGroupBoundarySpy = spyOn(bitcoinProcessor,'trimDatabasesToFeeSamplingGroupBoundary' as any); + + await (bitcoinProcessor as any).trimDatabasesToLastFeeSamplingGroupBoundary(); + expect(trimDatabasesToFeeSamplingGroupBoundarySpy).toHaveBeenCalledWith(mockLastSaveTransaction.transactionTime); + done(); + }); + + it('should trim based on genesis block if cannot find a saved transaction.', async (done) => { + // Simulate that a saved transaction cannot be found. + transactionStoreLatestTransactionSpy.and.returnValue(Promise.resolve(undefined)); + + const trimDatabasesToFeeSamplingGroupBoundarySpy = spyOn(bitcoinProcessor,'trimDatabasesToFeeSamplingGroupBoundary' as any); + + await (bitcoinProcessor as any).trimDatabasesToLastFeeSamplingGroupBoundary(); + expect(trimDatabasesToFeeSamplingGroupBoundarySpy).toHaveBeenCalledWith(bitcoinProcessor.genesisBlockNumber); + done(); + }); + }); + describe('trimDatabasesToFeeSamplingGroupBoundary', () => { it('should revert the DBs to the correct values.', async () => { const mockFirstTxnOfGroup = 123456;