Clean up DB open handling

Closes #89

server/block_processor.py

@@ -259,7 +259,7 @@ class BlockProcessor(server.db.DB):
                                  .format(VERSION, self.height))
                 self.first_sync = False
             self.flush_state(self.db)
-            self.reopen_db(False)
+            self.open_db(for_sync=False)
             self.caught_up_event.set()
 
     async def handle_chain_reorg(self, touched, count=None):

server/db.py

@@ -51,7 +51,7 @@ class DB(LoggedClass):
                          .format(self.env.reorg_limit))
 
         self.db = None
-        self.reopen_db(True)
+        self.open_db(for_sync=False)
 
         create = self.db_height == -1
         self.headers_file = self.open_file('headers', create)
@@ -69,30 +69,36 @@ class DB(LoggedClass):
             assert self.db_tx_count == 0
         self.clean_db()
 
-    def reopen_db(self, first_sync):
+    def open_db(self, for_sync):
         '''Open the database.  If the database is already open, it is
-        closed (implicitly via GC) and re-opened.
+        closed and re-opened.
 
+        If for_sync is True, it is opened for sync (high number of open
+        file, etc.)
         Re-open to set the maximum number of open files appropriately.
         '''
-        if self.db:
+        def log_reason(message, is_for_sync):
-            self.logger.info('closing DB to re-open')
+            reason = 'sync' if is_for_sync else 'serving'
-            self.db.close()
+            self.logger.info('{} for {}'.format(message, reason))
 
-        max_open_files = 1024 if first_sync else 256
+        if self.db:
+            if self.db.for_sync == for_sync:
+                return
+            log_reason('closing DB to re-open', for_sync)
+            self.db.close()
 
         # Open DB and metadata files.  Record some of its state.
         db_name = '{}-{}'.format(self.coin.NAME, self.coin.NET)
-        self.db = open_db(db_name, self.env.db_engine, max_open_files)
+        self.db = open_db(db_name, self.env.db_engine, for_sync)
         if self.db.is_new:
             self.logger.info('created new {} database {}'
                              .format(self.env.db_engine, db_name))
         else:
-            self.logger.info('successfully opened {} database {} for sync: {}'
+            log_reason('opened {} database {}'
-                             .format(self.env.db_engine, db_name, first_sync))
+                       .format(self.env.db_engine, db_name), self.db.for_sync)
-        self.read_state()
 
-        if self.first_sync == first_sync:
+        self.read_state()
+        if self.first_sync == self.db.for_sync:
             self.logger.info('software version: {}'.format(VERSION))
             self.logger.info('DB version: {:d}'.format(self.db_version))
             self.logger.info('coin: {}'.format(self.coin.NAME))
@@ -104,7 +110,7 @@ class DB(LoggedClass):
                 self.logger.info('sync time so far: {}'
                                  .format(formatted_time(self.wall_time)))
         else:
-            self.reopen_db(self.first_sync)
+            self.open_db(self.first_sync)
 
     def read_state(self):
         if self.db.is_new:

server/storage.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2016, the ElectrumX authors
+# Copyright (c) 2016-2017, the ElectrumX authors
 #
 # All rights reserved.
 #
@@ -31,7 +31,8 @@ class Storage(object):
 
     def __init__(self, name, for_sync):
         self.is_new = not os.path.exists(name)
-        self.open(name, create=self.is_new, for_sync=for_sync)
+        self.for_sync = for_sync or self.is_new
+        self.open(name, create=self.is_new)
 
     @classmethod
     def import_module(cls):
@@ -79,8 +80,8 @@ class LevelDB(Storage):
         import plyvel
         cls.module = plyvel
 
-    def open(self, name, create, for_sync):
+    def open(self, name, create):
-        mof = 1024 if for_sync else 256
+        mof = 1024 if self.for_sync else 256
         self.db = self.module.DB(name, create_if_missing=create,
                                  max_open_files=mof, compression=None)
         self.close = self.db.close
@@ -99,8 +100,8 @@ class RocksDB(Storage):
         import rocksdb
         cls.module = rocksdb
 
-    def open(self, name, create, for_sync):
+    def open(self, name, create):
-        mof = 1024 if for_sync else 256
+        mof = 1024 if self.for_sync else 256
         compression = "no"
         compression = getattr(self.module.CompressionType,
                               compression + "_compression")
@@ -172,7 +173,7 @@ class LMDB(Storage):
         import lmdb
         cls.module = lmdb
 
-    def open(self, name, create, for_sync):
+    def open(self, name, create):
         # I don't see anything equivalent to max_open_files for for_sync
         self.env = LMDB.module.Environment('.', subdir=True, create=create,
                                           max_dbs=32, map_size=5 * 10 ** 10)