Merge pull request libbitcoin#135 from evoskuil/version3

Backport master/v4 commits for a version3/v3.1 release.

README.md

@@ -20,4 +20,4 @@ libbitcoin-database is now installed in `/usr/local/`.
 
 **About Libbitcoin Database**
 
-Libbitcoin Database is a custom database build directly on the operating system's [memory-mapped file](https://en.wikipedia.org/wiki/Memory-mapped_file) system. All primary tables and indexes are built on in-memory hash tables, resulting in constant-time lookups. The database uses [sequence locking](https://en.wikipedia.org/wiki/Seqlock) to avoid blocking the writer. This is ideal for a high performance blockchain server as reads are significantly more frequent than writes and yet writes must proceed wtihout delay. The [libbitcoin-blockchain](https://github.com/libbitcoin/libbitcoin-blockchain) library uses the database as its blockchain store.
+Libbitcoin Database is a custom database build directly on the operating system's [memory-mapped file](https://en.wikipedia.org/wiki/Memory-mapped_file) system. All primary tables and indexes are built on in-memory hash tables, resulting in constant-time lookups. The database uses [sequence locking](https://en.wikipedia.org/wiki/Seqlock) to avoid blocking the writer. This is ideal for a high performance blockchain server as reads are significantly more frequent than writes and yet writes must proceed without delay. The [libbitcoin-blockchain](https://github.com/libbitcoin/libbitcoin-blockchain) library uses the database as its blockchain store.

configure.ac

@@ -13,7 +13,7 @@
 AC_PREREQ([2.65])
 
 # Process command-line arguments and perform initialization and verification.
-AC_INIT([libbitcoin-database], [3.0.0], [[email protected]])
+AC_INIT([libbitcoin-database], [3.1.0], [[email protected]])
 
 # Do compilation tests.
 AC_LANG(C++)
@@ -133,10 +133,10 @@ AS_CASE([${with_tests}], [yes],
      AC_MSG_NOTICE([boost_unit_test_framework_LIBS : ${boost_unit_test_framework_LIBS}])],
     [AC_SUBST([boost_unit_test_framework_LIBS], [])])
 
-# Require bitcoin of at least version 3.0.0 and output ${bitcoin_CPPFLAGS/LIBS/PKG}.
+# Require bitcoin of at least version 3.1.0 and output ${bitcoin_CPPFLAGS/LIBS/PKG}.
 #------------------------------------------------------------------------------
-PKG_CHECK_MODULES([bitcoin], [libbitcoin >= 3.0.0])
-AC_SUBST([bitcoin_PKG], ['libbitcoin >= 3.0.0'])
+PKG_CHECK_MODULES([bitcoin], [libbitcoin >= 3.1.0])
+AC_SUBST([bitcoin_PKG], ['libbitcoin >= 3.1.0'])
 AC_SUBST([bitcoin_CPPFLAGS], [${bitcoin_CFLAGS}])
 AC_MSG_NOTICE([bitcoin_CPPFLAGS : ${bitcoin_CPPFLAGS}])
 AC_MSG_NOTICE([bitcoin_LIBS : ${bitcoin_LIBS}])

include/bitcoin/database/version.hpp

@@ -12,9 +12,9 @@
  * For interpretation of the versioning scheme see: http://semver.org
  */
 
-#define LIBBITCOIN_DATABASE_VERSION "3.0.0"
+#define LIBBITCOIN_DATABASE_VERSION "3.1.0"
 #define LIBBITCOIN_DATABASE_MAJOR_VERSION 3
-#define LIBBITCOIN_DATABASE_MINOR_VERSION 0
+#define LIBBITCOIN_DATABASE_MINOR_VERSION 1
 #define LIBBITCOIN_DATABASE_PATCH_VERSION 0
 
 #endif

libbitcoin-database.pc.in

@@ -25,7 +25,7 @@ Version: @PACKAGE_VERSION@
 #==============================================================================
 # Dependencies that publish package configuration.
 #------------------------------------------------------------------------------
-Requires: libbitcoin >= 3.0.0
+Requires: libbitcoin >= 3.1.0
 
 # Include directory and any other required compiler flags.
 #------------------------------------------------------------------------------

src/data_base.cpp

@@ -204,6 +204,12 @@ bool data_base::flush() const
             history_->flush() &&
             stealth_->flush();
 
+    // Just for the log.
+    code ec(flushed ? error::success : error::operation_failed);
+
+    LOG_DEBUG(LOG_DATABASE)
+        << "Write flushed to disk: " << ec.message();
+
     return flushed;
 }
 
@@ -533,14 +539,15 @@ bool data_base::pop(block& out_block)
 
     for (size_t position = 0; position < count; ++position)
     {
-        const auto tx_hash = block.transaction_hash(position);
+        auto tx_hash = block.transaction_hash(position);
         const auto tx = transactions_->get(tx_hash, height, true);
 
         if (!tx || (tx.height() != height) || (tx.position() != position))
             return false;
 
         // Deserialize transaction and move it to the block.
-        transactions.emplace_back(tx.transaction());
+        // The tx move/copy constructors do not currently transfer cache.
+        transactions.emplace_back(tx.transaction(), std::move(tx_hash));
     }
 
     // Loop txs backwards, the reverse of how they were added.
@@ -685,6 +692,8 @@ void data_base::do_push(block_const_ptr block, size_t height,
 
     const auto threads = dispatch.size();
     const auto buckets = std::min(threads, block->transactions().size());
+    BITCOIN_ASSERT(buckets != 0);
+
     const auto join_handler = bc::synchronize(std::move(block_complete),
         buckets, NAME "_do_push");
 

src/databases/block_database.cpp

@@ -36,14 +36,11 @@ static constexpr auto index_header_size = 0u;
 static constexpr auto index_record_size = sizeof(file_offset);
 
 // Record format:
-// main:
-//  [ header:80      ]
-//  [ height:4       ]
-//  [ number_txs:1-8 ]
-// hashes:
-//  [ [    ...     ] ]
-//  [ [ tx_hash:32 ] ]
-//  [ [    ...     ] ]
+//  [ header:80 ]
+//  [ height:4 ]
+//  TODO: [ checksum:4 ] (store all zeros if not computed).
+//  [ tx_count:1-2 ]
+//  [ [ tx_hash:32 ]... ]
 
 // Blocks uses a hash table and an array index, both O(1).
 block_database::block_database(const path& map_filename,
@@ -216,7 +213,7 @@ bool block_database::unlink(size_t from_height)
 // This is necessary for parallel import, as gaps are created.
 void block_database::zeroize(array_index first, array_index count)
 {
-    for (array_index index = first; index < (first + count); ++index)
+    for (auto index = first; index < (first + count); ++index)
     {
         const auto memory = index_manager_.get(index);
         auto serial = make_unsafe_serializer(REMAP_ADDRESS(memory));

src/databases/history_database.cpp

@@ -33,7 +33,7 @@ using namespace bc::chain;
 static constexpr auto rows_header_size = 0u;
 
 static constexpr auto flag_size = sizeof(uint8_t);
-static constexpr auto point_size = hash_size + sizeof(uint32_t);
+static constexpr auto point_size = std::tuple_size<point>::value;
 static constexpr auto height_position = flag_size + point_size;
 static constexpr auto height_size = sizeof(uint32_t);
 static constexpr auto checksum_size = sizeof(uint64_t);
@@ -143,7 +143,7 @@ void history_database::add_output(const short_hash& key,
     const auto write = [&](serializer<uint8_t*>& serial)
     {
         serial.write_byte(static_cast<uint8_t>(point_kind::output));
-        outpoint.to_data(serial);
+        outpoint.to_data(serial, false);
         serial.write_4_bytes_little_endian(output_height32);
         serial.write_8_bytes_little_endian(value);
     };
@@ -160,7 +160,7 @@ void history_database::add_input(const short_hash& key,
     const auto write = [&](serializer<uint8_t*>& serial)
     {
         serial.write_byte(static_cast<uint8_t>(point_kind::spend));
-        inpoint.to_data(serial);
+        inpoint.to_data(serial, false);
         serial.write_4_bytes_little_endian(input_height32);
         serial.write_8_bytes_little_endian(previous.checksum());
     };
@@ -194,7 +194,7 @@ history_compact::list history_database::get(const short_hash& key,
             static_cast<point_kind>(deserial.read_byte()),
 
             // point
-            point::factory_from_data(deserial),
+            point::factory_from_data(deserial, false),
 
             // height
             deserial.read_4_bytes_little_endian(),

src/databases/spend_database.cpp

@@ -29,6 +29,7 @@ namespace database {
 
 using namespace bc::chain;
 
+// The spend database keys off of output point and has input point value.
 static constexpr auto value_size = std::tuple_size<point>::value;
 static BC_CONSTEXPR auto record_size = hash_table_record_size<point>(value_size);
 
@@ -107,25 +108,25 @@ bool spend_database::flush() const
 
 input_point spend_database::get(const output_point& outpoint) const
 {
-    input_point point;
+    input_point spend;
     const auto memory = lookup_map_.find(outpoint);
 
     if (!memory)
-        return point;
+        return spend;
 
     // The order of properties in this serialization was changed in v3.
     // Previously it was { index, hash }, which was inconsistent with wire.
     auto deserial = make_unsafe_deserializer(REMAP_ADDRESS(memory));
-    point.from_data(deserial);
-    return point;
+    spend.from_data(deserial, false);
+    return spend;
 }
 
 void spend_database::store(const chain::output_point& outpoint,
     const chain::input_point& spend)
 {
     const auto write = [&](serializer<uint8_t*>& serial)
     {
-        spend.to_data(serial);
+        spend.to_data(serial, false);
     };
 
     lookup_map_.store(outpoint, write);

src/databases/stealth_database.cpp

@@ -33,8 +33,12 @@ static constexpr auto rows_header_size = 0u;
 static constexpr auto height_size = sizeof(uint32_t);
 static constexpr auto prefix_size = sizeof(uint32_t);
 
+// [ prefix:4 ]
+// [ height:4 ]
+// [ ephemkey:32 ]
+// [ address:20 ]
+// [ tx_hash:32 ]
 // ephemkey is without sign byte and address is without version byte.
-// [ prefix_bitfield:4 ][ height:32 ][ ephemkey:32 ][ address:20 ][ tx_id:32 ]
 static constexpr auto row_size = prefix_size + height_size + hash_size +
     short_hash_size + hash_size;
 

src/databases/transaction_database.cpp

@@ -32,12 +32,11 @@ using namespace bc::machine;
 
 static constexpr auto value_size = sizeof(uint64_t);
 static constexpr auto height_size = sizeof(uint32_t);
-static constexpr auto version_size = sizeof(uint32_t);
-static constexpr auto locktime_size = sizeof(uint32_t);
-static constexpr auto position_size = sizeof(uint32_t);
-static constexpr auto version_lock_size = version_size + locktime_size;
+static constexpr auto position_size = sizeof(uint16_t);
+static constexpr auto height_position_size = height_size + position_size;
+static constexpr auto spender_height_value_size = height_size + value_size;
 
-const size_t transaction_database::unconfirmed = max_uint32;
+const size_t transaction_database::unconfirmed = max_uint16;
 
 // Transactions uses a hash table index, O(1).
 transaction_database::transaction_database(const path& map_filename,
@@ -136,7 +135,7 @@ memory_ptr transaction_database::find(const hash_digest& hash,
     // Read the height and position.
     // If position is unconfirmed then height is the forks used for validation.
     const size_t height = deserial.read_4_bytes_little_endian();
-    const size_t position = deserial.read_4_bytes_little_endian();
+    const size_t position = deserial.read_2_bytes_little_endian();
 
     return (height > fork_height) ||
         (require_confirmed && position == unconfirmed) ?
@@ -169,13 +168,24 @@ bool transaction_database::get_output(output& out_output, size_t& out_height,
     if (!slab)
         return false;
 
-    transaction_result result (slab, hash);
+    transaction_result result(slab, hash);
     out_height = result.height();
     out_coinbase = result.position() == 0;
     out_output = result.output(point.index());
     return true;
 }
 
+// [ height:4 ]
+// [ position:2 ]
+// ----------------------------------------------------------------------------
+// [ output_count:varint ]
+// [ [ spender_height:4 ][ value:8 ][ script:varint ]... ]
+// [ input_count:varint ]
+// [ [ hash:4 ][ index:2 ][ script:varint ][ sequence:4 ]... ]
+// [ locktime:varint ]
+// [ version:varint ]
+// ----------------------------------------------------------------------------
+
 void transaction_database::store(const chain::transaction& tx,
     size_t height, size_t position)
 {
@@ -199,27 +209,27 @@ void transaction_database::store(const chain::transaction& tx,
 
     // Create the transaction.
     BITCOIN_ASSERT(height <= max_uint32);
-    BITCOIN_ASSERT(position <= max_uint32);
+    BITCOIN_ASSERT(position <= max_uint16);
 
     // Unconfirmed txs: position is unconfirmed and height is validation forks.
     const auto write = [&](serializer<uint8_t*>& serial)
     {
-        serial.write_4_bytes_little_endian(static_cast<size_t>(height));
-        serial.write_4_bytes_little_endian(static_cast<size_t>(position));
+        serial.write_4_bytes_little_endian(static_cast<uint32_t>(height));
+        serial.write_2_bytes_little_endian(static_cast<uint16_t>(position));
 
         // WRITE THE TX
         tx.to_data(serial, false);
     };
 
     const auto tx_size = tx.serialized_size(false);
-    BITCOIN_ASSERT(tx_size <= max_size_t - version_lock_size);
-    const auto value_size = version_lock_size + static_cast<size_t>(tx_size);
+    BITCOIN_ASSERT(tx_size <= max_size_t - height_position_size);
+    const auto total_size = height_position_size + static_cast<size_t>(tx_size);
 
     // Create slab for the new tx instance.
-    lookup_map_.store(hash, write, value_size);
+    lookup_map_.store(hash, write, total_size);
     cache_.add(tx, height, position != unconfirmed);
 
-    // We report theis here because its a steady interval (block announce).
+    // We report this here because its a steady interval (block announce).
     if (!cache_.disabled() && position == 0)
     {
         LOG_DEBUG(LOG_DATABASE)
@@ -246,9 +256,8 @@ bool transaction_database::spend(const output_point& point,
         return false;
 
     const auto memory = REMAP_ADDRESS(slab);
-    const auto tx_start = memory + height_size + position_size;
+    const auto tx_start = memory + height_position_size;
     auto serial = make_unsafe_serializer(tx_start);
-    serial.skip(version_size + locktime_size);
     const auto outputs = serial.read_size_little_endian();
     BITCOIN_ASSERT(serial);
 
@@ -259,7 +268,7 @@ bool transaction_database::spend(const output_point& point,
     // Skip outputs until the target output.
     for (uint32_t output = 0; output < point.index(); ++output)
     {
-        serial.skip(height_size + value_size);
+        serial.skip(spender_height_value_size);
         serial.skip(serial.read_size_little_endian());
         BITCOIN_ASSERT(serial);
     }
@@ -284,12 +293,12 @@ bool transaction_database::confirm(const hash_digest& hash, size_t height,
         return false;
 
     BITCOIN_ASSERT(height <= max_uint32);
-    BITCOIN_ASSERT(position <= max_uint32);
+    BITCOIN_ASSERT(position <= max_uint16);
 
     const auto memory = REMAP_ADDRESS(slab);
     auto serial = make_unsafe_serializer(memory);
-    serial.write_4_bytes_little_endian(static_cast<size_t>(height));
-    serial.write_4_bytes_little_endian(static_cast<size_t>(position));
+    serial.write_4_bytes_little_endian(static_cast<uint32_t>(height));
+    serial.write_2_bytes_little_endian(static_cast<uint16_t>(position));
     return true;
 }