Add feature to simulate reorgs for debugging

docs/ENV-NOTES

@@ -82,3 +82,9 @@ UTXO_MB   - amount of UTXO and history cache, in MB, to retain before
             leveldb caching and Python GC effects.  However this may be
             very dependent on hardware and you may have different
             results.
+
+The following are for debugging purposes.
+
+FORCE_REORG - if set to a positive integer, it will simulate a reorg
+              of the blockchain for that number of blocks.  Do not set
+              to a value greater than REORG_LIMIT.

server/block_processor.py

@@ -58,6 +58,7 @@ class Prefetcher(LoggedClass):
                 self.queue.get_nowait()
             self.queue_size = 0
             self.fetched_height = height
+            self.caught_up = False
 
     async def get_blocks(self):
         '''Blocking function that returns prefetched blocks.
@@ -185,6 +186,13 @@ class BlockProcessor(server.db.DB):
         Safely flushes the DB on clean shutdown.
         '''
         self.futures.append(asyncio.ensure_future(self.prefetcher.main_loop()))
+
+        # Simulate a reorg if requested
+        if self.env.force_reorg > 0:
+            self.logger.info('DEBUG: simulating chain reorg of {:,d} blocks'
+                             .format(self.env.force_reorg))
+            await self.handle_chain_reorg(self.env.force_reorg)
+
         try:
             while True:
                 await self._wait_for_update()
@@ -223,7 +231,7 @@ class BlockProcessor(server.db.DB):
                 self.advance_block(block, self.caught_up)
                 await asyncio.sleep(0)  # Yield
         except ChainReorg:
-            await self.handle_chain_reorg()
+            await self.handle_chain_reorg(None)
 
         if self.caught_up:
             # Flush everything as queries are performed on the DB and
@@ -250,24 +258,26 @@ class BlockProcessor(server.db.DB):
         Only called for blocks found after first_caught_up is called.
         Intended to be overridden in derived classes.'''
 
-    async def handle_chain_reorg(self):
+    async def handle_chain_reorg(self, count):
-        # First get all state on disk
+        '''Handle a chain reorganisation.
+
+        Count is the number of blocks to simulate a reorg, or None for
+        a real reorg.'''
         self.logger.info('chain reorg detected')
         self.flush(True)
         self.logger.info('finding common height...')
 
-        hashes = await self.reorg_hashes()
+        hashes = await self.reorg_hashes(count)
         # Reverse and convert to hex strings.
         hashes = [hash_to_str(hash) for hash in reversed(hashes)]
         for hex_hashes in chunks(hashes, 50):
             blocks = await self.daemon.raw_blocks(hex_hashes)
             self.backup_blocks(blocks)
 
-        self.logger.info('backed up to height {:,d}'.format(self.height))
         await self.prefetcher.clear(self.height)
         self.logger.info('prefetcher reset')
 
-    async def reorg_hashes(self):
+    async def reorg_hashes(self, count):
         '''Return the list of hashes to back up beacuse of a reorg.
 
         The hashes are returned in order of increasing height.'''
@@ -278,24 +288,27 @@ class BlockProcessor(server.db.DB):
                     return n
             return -1
 
-        start = self.height - 1
+        if count is None:
-        count = 1
+            # A real reorg
-        while start > 0:
+            start = self.height - 1
-            hashes = self.fs_block_hashes(start, count)
+            count = 1
-            hex_hashes = [hash_to_str(hash) for hash in hashes]
+            while start > 0:
-            d_hex_hashes = await self.daemon.block_hex_hashes(start, count)
+                hashes = self.fs_block_hashes(start, count)
-            n = match_pos(hex_hashes, d_hex_hashes)
+                hex_hashes = [hash_to_str(hash) for hash in hashes]
-            if n >= 0:
+                d_hex_hashes = await self.daemon.block_hex_hashes(start, count)
-                start += n + 1
+                n = match_pos(hex_hashes, d_hex_hashes)
-                break
+                if n >= 0:
-            count = min(count * 2, start)
+                    start += n + 1
-            start -= count
+                    break
+                count = min(count * 2, start)
+                start -= count
 
-        # Hashes differ from height 'start'
+            count = (self.height - start) + 1
-        count = (self.height - start) + 1
+        else:
+            start = (self.height - count) + 1
 
-        self.logger.info('chain was reorganised for {:,d} blocks from '
+        self.logger.info('chain was reorganised for {:,d} blocks over '
-                         'height {:,d} to height {:,d}'
+                         'heights {:,d}-{:,d} inclusive'
                          .format(count, start, start + count - 1))
 
         return self.fs_block_hashes(start, count)
@@ -660,6 +673,9 @@ class BlockProcessor(server.db.DB):
         # Prevout values, in order down the block (coinbase first if present)
         # undo_info is in reverse block order
         undo_info = self.read_undo_info(self.height)
+        if not undo_info:
+            raise ChainError('no undo information found for height {:,d}'
+                             .format(self.height))
         n = len(undo_info)
 
         # Use local vars for speed in the loops

server/env.py

@@ -53,6 +53,8 @@ class Env(LoggedClass):
         self.report_host = self.default('REPORT_HOST', self.host)
         self.irc_nick = self.default('IRC_NICK', None)
         self.irc = self.default('IRC', False)
+        # Debugging
+        self.force_reorg = self.integer('FORCE_REORG', 0)
 
     def default(self, envvar, default):
         return environ.get(envvar, default)