Merge pull request #121 from Chia-Network/alpha-1.5

Alpha 1.5

1. You can now provide an index to create_plots using the -i flag to create an arbitrary new plot derived from an existing plot key. Thanks @xorinox.
2. There is a new restart_harvester.sh in scripts/ to easily restart a harvester when you want to add a newly completed plot to the farm without restarting farmer, fullnode, timelord, etc.
3. Harvesters now log errors if they encounter a malformed or corrupted plot file. Again thanks @xorinox.
4. New AJAX based full node UI. To access go to http://127.0.0.1:8555/index.html with any modern web browser on the same machine as the full node.
5. VDF verification code is improved and is now more paranoid.
6. Timelords can now be run as a cluster of VDF client instances around a central Timelord instance.. Instructions are available in the Cluster Timelord section of the repo wiki.
7. If you want to benchmark your CPU as a VDF you can use vdf_bench square_asm 500000 for the assembly optimized test or just /vdf_bench square 500000 for the plain C++ code path. This tool is found in lib/chiavdf/fast_vdf/.
8. Improvements to shutting down services in all of the scripts in scripts/. Another @xorinox HT.
9. Thanks clean @dkacman for clean ups to the proof of space code.
10. Thanks to @davision for some typo fixes.

CONTRIBUTING.md

@@ -12,7 +12,7 @@ If you want to learn more about this project, read the [wiki](https://github.com
 We would be pleased to accept code contributions to this project.
 As we are in the alpha stage, the main priority is getting a robust blockchain up and running, with as many of the mainnet features as possible.
 You can visit our [Trello project board](https://trello.com/b/ZuNx7sET) to get a sense of what is in the backlog.
-Generally things to the left are in progess or done. Some things go through "Coming up soon" but some will come directly out of other columns. 
+Generally things to the left are in progress or done. Some things go through "Coming up soon" but some will come directly out of other columns. 
 Usually the things closer to the top of each column are the ones that will be worked on soonest.
 If you are interested in cryptography, math, or just like hacking in python, there are many interesting problems to work on.
 Contact any of the team members on keybase: https://keybase.io/team/chia_network.public, which we use as the main communication method and you can comment on any Trello card.

README.md

@@ -279,5 +279,6 @@ You can also use the [HTTP RPC](https://github.com/Chia-Network/chia-blockchain/
 
 
 ```bash
-curl -X POST  http://localhost:8555/get_blockchain_state
+curl -X POST http://localhost:8555/get_blockchain_state
+curl -d '{"header_hash":"afe223d75d40dd7bd19bf35846d0c9dce608bfc77ee5baa9f9cd6b98436e428b"}' -H "Content-Type: application/json" -X POST http://localhost:8555/get_header
 ```

config/config.yaml

@@ -36,27 +36,39 @@ farmer:
   propagate_threshold: 10000
   logging: *logging
 
+# Don't run this unless you want to run VDF clients on the local machine.
+timelord_launcher:
+  # The server where the VDF clients will connect to.
+  host: 127.0.0.1
+  port: 8000
+  # Number of VDF client processes to keep alive in the local machine.
+  process_count: 2
+  logging: *logging
+
+
 timelord:
   # The timelord server (if run) will run on this host and port
   host: 127.0.0.1
   port: 8446
   # How much recursion to use for the wesolowski VDF proof. This increases the size
   # of the proofs.
   n_wesolowski: 2
-  # VDF servers will be spawned at these ports, and closed after generating proofs
-  # Adding more vdf servers will increase memory usage and CPU usage.
-  vdf_server_ips:
-    - 127.0.0.1
-  vdf_server_ports:
-    - 8889
-  servers_ips_estimate:
+  # Provides a list of VDF clients expected to connect to this timelord.
+  # For each client, an IP is provided, together with the estimated iterations per second.
+  vdf_clients:
     ip:
       - 127.0.0.1
-    ips:
+    ips_estimate:
       - 100000
   full_node_peer:
       host: 127.0.0.1
       port: 8444
+  # Maximum number of seconds allowed for a client to reconnect to the server.
+  max_connection_time: 60
+  # The ip and port where the TCP clients will connect. 
+  vdf_server:
+    host: 127.0.0.1
+    port: 8000
   logging: *logging
 
 full_node:

lib/chiapos/src/bits.hpp

@@ -33,7 +33,7 @@
 struct SmallVector {
     typedef uint16_t size_type;
 
-    SmallVector() {
+    SmallVector() noexcept {
         count_ = 0;
     }
 
@@ -56,7 +56,7 @@ struct SmallVector {
         return (*this);
     }
 
-    size_type size() const {
+    size_type size() const noexcept {
         return count_;
     }
 
@@ -71,7 +71,7 @@ struct SmallVector {
 struct ParkVector {
     typedef uint32_t size_type;
 
-    ParkVector() {
+    ParkVector() noexcept {
         count_ = 0;
     }
 
@@ -94,7 +94,7 @@ struct ParkVector {
         return (*this);
     }
 
-    size_type size() const {
+    size_type size() const noexcept {
         return count_;
     }
 
@@ -120,7 +120,7 @@ template <class T> class BitsGeneric {
  public:
     template <class> friend class BitsGeneric;
 
-    BitsGeneric<T>()  {
+    BitsGeneric<T>() noexcept {
         this->last_size_ = 0;
     }
 
@@ -198,9 +198,10 @@ template <class T> class BitsGeneric {
         }
     }
 
-    BitsGeneric<T>(const BitsGeneric<T>& other) {
-        values_ = other.values_;
-        last_size_ = other.last_size_;
+    BitsGeneric<T>(const BitsGeneric<T>& other) noexcept :
+        values_(other.values_),
+        last_size_(other.last_size_)
+    {
     }
 
     BitsGeneric<T>& operator = (const BitsGeneric<T>& other) {
@@ -329,9 +330,7 @@ template <class T> class BitsGeneric {
         if (end_index > GetSize()) {
             end_index = GetSize();
         }
-        if (start_index < 0) {
-            start_index = 0;
-        }
+
         if (end_index == start_index) return BitsGeneric<T>();
         assert(end_index > start_index);
         uint32_t start_bucket = start_index / 128;
@@ -502,8 +501,8 @@ template <class T> class BitsGeneric {
     friend BitsGeneric<X> operator>>(BitsGeneric<X> lhs, uint32_t shift_amount);
 
  private:
-    void SplitNumberByPrefix(uint128_t number, uint8_t num_bits, uint8_t prefix_size, uint128_t* prefix,
-                             uint128_t* suffix) const {
+    static void SplitNumberByPrefix(uint128_t number, uint8_t num_bits, uint8_t prefix_size, uint128_t* prefix,
+                             uint128_t* suffix) {
         assert(num_bits >= prefix_size);
         if (prefix_size == 0) {
             *prefix = 0;

lib/chiapos/src/calculate_bucket.hpp

@@ -152,7 +152,7 @@ class F1Calculator {
     }
 
     // Returns an evaluation of F1(L), and the metadata (L) that must be stored to evaluate F2.
-    inline std::pair<Bits, Bits> CalculateBucket(const Bits& L) {
+    inline std::pair<Bits, Bits> CalculateBucket(const Bits& L) const {
         return std::make_pair(CalculateF(L), L);
     }
 
@@ -299,7 +299,7 @@ class FxCalculator {
     }
 
     // Composes two metadatas L and R, into a metadata for the next table.
-    inline Bits Compose(const Bits& L, const Bits& R) {
+    inline Bits Compose(const Bits& L, const Bits& R) const {
         switch (table_index_) {
             case 2:
             case 3:

lib/chiapos/src/prover_disk.hpp

@@ -111,19 +111,19 @@ class DiskProver {
         memcpy(buffer, memo, this->memo_size);
     }
 
-    uint32_t GetMemoSize() {
+    uint32_t GetMemoSize() const noexcept {
         return this->memo_size;
     }
 
     void GetId(uint8_t* buffer) {
         memcpy(buffer, id, kIdLen);
     }
 
-    std::string GetFilename() {
+    std::string GetFilename() const noexcept {
         return filename;
     }
 
-    uint8_t GetSize() {
+    uint8_t GetSize() const noexcept {
         return k;
     }
 
@@ -489,7 +489,7 @@ class DiskProver {
     //     C(x1, x2) < C(x3, x4)
     //     For all comparisons up to f7
     //     Where a < b is defined as:  max(b) > max(a) where a and b are lists of k bit elements
-    std::vector<LargeBits> ReorderProof(const std::vector<Bits>& xs_input) {
+    std::vector<LargeBits> ReorderProof(const std::vector<Bits>& xs_input) const {
         F1Calculator f1(k, id);
         std::vector<std::pair<Bits, Bits> > results;
         LargeBits xs;

lib/chiapos/src/sort_on_disk.hpp

@@ -90,21 +90,21 @@ class Disk {
 
 class FileDisk : public Disk {
  public:
-    inline explicit FileDisk(std::string filename) {
+    inline explicit FileDisk(const std::string& filename) {
         Initialize(filename);
     }
 
     inline void Close() {
         f_.close();
     }
 
-    inline void Read(uint64_t begin, uint8_t* memcache, uint32_t length) {
+    inline void Read(uint64_t begin, uint8_t* memcache, uint32_t length) override {
         // Seek, read, and replace into memcache
         f_.seekg(begin);
         f_.read(reinterpret_cast<char*>(memcache), length);
     }
 
-    inline void Write(uint64_t begin, uint8_t* memcache, uint32_t length) {
+    inline void Write(uint64_t begin, uint8_t* memcache, uint32_t length) override {
         // Seek and write from memcache
         f_.seekp(begin);
         f_.write(reinterpret_cast<char*>(memcache), length);
@@ -113,22 +113,22 @@ class FileDisk : public Disk {
     /**
      * Returns a read handle at the specified byte offset from the beginning
      */
-    inline std::iostream* ReadHandle(uint64_t begin) {
+    inline std::iostream* ReadHandle(uint64_t begin) override {
         f_.seekg(begin);
         return &f_;
     }
 
-    inline std::iostream* WriteHandle(uint64_t begin) {
+    inline std::iostream* WriteHandle(uint64_t begin) override {
         f_.seekp(begin);
         return &f_;
     }
 
-    inline std::string GetFileName() const {
+    inline std::string GetFileName() const noexcept {
         return filename_;
     }
 
  private:
-    void Initialize(std::string filename) {
+    void Initialize(const std::string& filename) {
         filename_ = filename;
 
         // Creates the file if it does not exist
@@ -204,17 +204,17 @@ class BucketStore {
         Util::IntToFourBytes(mem_ + i * seg_size_, v);
     }
 
-    inline uint64_t GetSegmentId(uint64_t i) {
+    inline uint64_t GetSegmentId(uint64_t i) const {
         return Util::FourBytesToInt(mem_ + i * seg_size_);
     }
 
     // Get the first empty position from the head segment of bucket b.
-    inline uint64_t GetEntryPos(uint64_t b) {
+    inline uint64_t GetEntryPos(uint64_t b) const {
         return bucket_head_ids_[b] * seg_size_ + 4
                + bucket_head_counts_[b] * entry_len_;
     }
 
-    inline void Audit() {
+    inline void Audit() const {
         uint64_t count = 0;
         uint64_t pos = first_empty_seg_id_;
 
@@ -231,19 +231,19 @@ class BucketStore {
         assert(count == length_);
     }
 
-    inline uint64_t NumFree() {
+    inline uint64_t NumFree() const {
         uint64_t used = GetSegmentId(first_empty_seg_id_);
         return (bucket_sizes_.size() - used) * entries_per_seg_;
     }
 
-    inline bool IsEmpty() {
+    inline bool IsEmpty() const noexcept {
         for (uint64_t s : bucket_sizes_) {
             if (s > 0) return false;
         }
         return true;
     }
 
-    inline bool IsFull() {
+    inline bool IsFull() const noexcept {
         return first_empty_seg_id_ == length_;
     }
 
@@ -275,7 +275,7 @@ class BucketStore {
         bucket_head_counts_[b] += 1;
     }
 
-    inline uint64_t MaxBucket() {
+    inline uint64_t MaxBucket() const {
         uint64_t max_bucket_size = bucket_sizes_[0];
         uint64_t max_index = 0;
         for (uint64_t i = 1; i < bucket_sizes_.size(); i++) {
@@ -287,7 +287,7 @@ class BucketStore {
         return max_index;
     }
 
-    inline std::vector<uint64_t> BucketsBySize() {
+    inline std::vector<uint64_t> BucketsBySize() const {
         // Lukasz Wiklendt (https://stackoverflow.com/questions/1577475/c-sorting-and-keeping-track-of-indexes)
         std::vector<uint64_t> idx(bucket_sizes_.size());
         iota(idx.begin(), idx.end(), 0);
@@ -298,7 +298,7 @@ class BucketStore {
 
     // Similar to how 'Bits' class works, appends an entry to the entries list, such as all entries are stored into 128-bit blocks.
     // Bits class was avoided since it consumes more time than a uint128_t array.
-    void AddBucketEntry(uint8_t* big_endian_bytes, uint64_t num_bytes, uint16_t size_bits, uint128_t* entries, uint64_t& cnt) {
+    static void AddBucketEntry(uint8_t* big_endian_bytes, uint64_t num_bytes, uint16_t size_bits, uint128_t* entries, uint64_t& cnt) {
         assert(size_bits / 8 >= num_bytes);
         uint16_t extra_space = size_bits - num_bytes * 8;
         uint64_t init_cnt = cnt;

lib/chiapos/src/stack_allocator.h

@@ -54,17 +54,17 @@ class stack_allocator
 
 	public:
 
-	explicit stack_allocator(const allocator_type& alloc = allocator_type()) 
+	explicit stack_allocator(const allocator_type& alloc = allocator_type()) noexcept
 		: m_allocator(alloc), m_begin(nullptr), m_end(nullptr), m_stack_pointer(nullptr)
 	{ }
 
-	explicit stack_allocator(pointer buffer, const allocator_type& alloc = allocator_type())
+	explicit stack_allocator(pointer buffer, const allocator_type& alloc = allocator_type()) noexcept
 		: m_allocator(alloc), m_begin(buffer), m_end(buffer + N), 
 			m_stack_pointer(buffer)
 	{ }
 
 	template <class U>
-	stack_allocator(const stack_allocator<U, N, Allocator>& other)
+	stack_allocator(const stack_allocator<U, N, Allocator>& other) noexcept
 		: m_allocator(other.m_allocator), m_begin(other.m_begin), m_end(other.m_end),
 			m_stack_pointer(other.m_stack_pointer)
 	{ }

lib/chiapos/src/util.hpp

@@ -45,9 +45,10 @@ std::ostream &operator<<(std::ostream & strm, uint128_t const & v) {
 
 class Timer {
  public:
-    Timer() {
-        this->wall_clock_time_start_ = std::chrono::steady_clock::now();
-        this->cpu_time_start_ = clock();
+    Timer() :
+        wall_clock_time_start_(std::chrono::steady_clock::now()),
+        cpu_time_start_(clock())
+    {
     }
 
     static char* GetNow()
@@ -57,7 +58,7 @@ class Timer {
         return ctime(&tt); // ctime includes newline
     }
 
-    void PrintElapsed(std::string name) {
+    void PrintElapsed(const std::string& name) const {
         auto end = std::chrono::steady_clock::now();
         auto wall_clock_ms = std::chrono::duration_cast<std::chrono::milliseconds>(
                              end - this->wall_clock_time_start_).count();

lib/chiapos/src/verifier.hpp

@@ -24,7 +24,7 @@ class Verifier {
     // Gets the quality string from a proof in proof ordering. The quality string is two
     // adjacent values, determined by the quality index (1-32), and the proof in plot
     // ordering.
-    LargeBits GetQualityString(uint8_t k, LargeBits proof, uint16_t quality_index) {
+    static LargeBits GetQualityString(uint8_t k, LargeBits proof, uint16_t quality_index) {
         // Converts the proof from proof ordering to plot ordering
         for (uint8_t table_index = 1; table_index < 7; table_index++) {
             LargeBits new_proof;
@@ -116,7 +116,7 @@ class Verifier {
  private:
     // Compares two lists of k values, a and b. a > b iff max(a) > max(b),
     // if there is a tie, the next largest value is compared.
-    bool CompareProofBits(LargeBits left, LargeBits right, uint8_t k) {
+    static bool CompareProofBits(LargeBits left, LargeBits right, uint8_t k) {
         uint16_t size = left.GetSize() / k;
         assert(left.GetSize() == right.GetSize());
         for (int16_t i = size - 1; i >= 0; i--) {

lib/chiavdf/fast_vdf/.gitignore

@@ -5,7 +5,9 @@
 /compile_asm
 /vdf
 /server
+/vdf_bench
 /vdf_server
+/vdf_client
 /verifier
 /verifier_test
 /build/*