Merge remote-tracking branch 'origin/dev' into danger

Reviewed-on: https://git.hush.is/hush/hush3/pulls/332

AUTHORS

@@ -2,7 +2,7 @@
 
 Duke Leto https://git.hush.is/duke https://github.com/leto
 Miodrag https://github.com/miodragpop
-jahway603 https://git.hush.is/jahway603 https://github.com/jahway603
+jahway603 https://git.hush.is/jahway603 https://codeberg.org/jahway603 https://github.com/jahway603
 
 # The SuperNET Developers
 

antispam

@@ -1,4 +1,4 @@
 #!/bin/sh
 
 echo "./src/hush-cli -ac_name=ANTISPAM $@"
-./src/hush-cli -ac_name=ANTISPAM $@
+./src/hush-cli -ac_name=ANTISPAM "$@"

depends/packages/libcurl.mk

@@ -5,14 +5,14 @@ $(package)_version=7.67.0
 $(package)_file_name=curl-$($(package)_version).tar.gz
 $(package)_sha256_hash=52af3361cf806330b88b4fe6f483b6844209d47ae196ac46da4de59bb361ab02
 else
-$(package)_version=7.77.0
+$(package)_version=8.4.0
 $(package)_file_name=curl-$($(package)_version).tar.gz
-$(package)_sha256_hash=b0a3428acb60fa59044c4d0baae4e4fc09ae9af1d8a3aa84b2e3fbcd99841f77
+$(package)_sha256_hash=816e41809c043ff285e8c0f06a75a1fa250211bbfb2dc0a037eeef39f1a9e427
 endif
 
 $(package)_dependencies=wolfssl
 $(package)_download_path=https://curl.haxx.se/download
-$(package)_config_opts_linux=--disable-shared --enable-static --with-wolfssl --without-ssl --prefix=$(host_prefix) --host=$(host)
+$(package)_config_opts_linux=--disable-shared --enable-static --without-ssl --prefix=$(host_prefix) --host=$(host)
 $(package)_config_opts_mingw32=--enable-mingw --disable-shared --enable-static --with-wolfssl --without-ssl --prefix=$(host_prefix) --host=x86_64-w64-mingw32
 $(package)_config_opts_darwin=--disable-shared --enable-static --with-wolfssl --without-ssl --prefix=$(host_prefix)
 $(package)_cflags_darwin=-mmacosx-version-min=10.9

depends/packages/rust.mk

@@ -7,6 +7,8 @@ $(package)_file_name_darwin=rust-$($(package)_version)-x86_64-apple-darwin.tar.g
 $(package)_sha256_hash_darwin=f0dfba507192f9b5c330b5984ba71d57d434475f3d62bd44a39201e36fa76304
 $(package)_file_name_mingw32=rust-$($(package)_version)-x86_64-pc-windows-gnu.tar.gz
 $(package)_sha256_hash_mingw32=358e1435347c67dbf33aa9cad6fe501a833d6633ed5d5aa1863d5dffa0349be9
+$(package)_file_name_aarch64_linux=rust-$($(package)_version)-aarch64-unknown-linux-gnu.tar.gz
+$(package)_sha256_hash_aarch64_linux=60def40961728212da4b3a9767d5a2ddb748400e150a5f8a6d5aa0e1b8ba1cee
 
 # Mapping from GCC canonical hosts to Rust targets
 # If a mapping is not present, we assume they are identical, unless $host_os is

src/Makefile.am

@@ -39,6 +39,7 @@ BITCOIN_INCLUDES += -I$(srcdir)/cryptoconditions/include
 BITCOIN_INCLUDES += -I$(srcdir)/cryptoconditions/src
 BITCOIN_INCLUDES += -I$(srcdir)/cryptoconditions/src/asn
 BITCOIN_INCLUDES += -I$(srcdir)/univalue/include
+BITCOIN_INCLUDES += -I$(srcdir)/leveldb/include
 
 if TARGET_WINDOWS
 LIBBITCOIN_SERVER=libbitcoin_server.a -lcurl

src/cc/Makefile

@@ -3,9 +3,9 @@ CC		= gcc
 CC_DARWIN	= g++-6
 CC_WIN		= x86_64-w64-mingw32-gcc-posix
 CFLAGS = -arch x86_64
-CXXFLAGS_DARWIN	= -std=c++11 -arch x86_64 -I/usr/local/Cellar/gcc\@6/6.4.0_2/include/c++/6.4.0/ -I../../depends/$(shell echo `../..//depends/config.guess`/include) -I../univalue/include -I../cryptoconditions/include -I../cryptoconditions/src -I../cryptoconditions/src/asn -I.. -I. -fPIC -c -Wl,-undefined -Wl,dynamic_lookup -dynamiclib
+CXXFLAGS_DARWIN	= -std=c++11 -arch x86_64 -I/usr/local/Cellar/gcc\@6/6.4.0_2/include/c++/6.4.0/ -I../../depends/$(shell echo `../..//depends/config.guess`/include) -I../univalue/include -I../cryptoconditions/include -I../cryptoconditions/src -I../cryptoconditions/src/asn -I -I../leveldb/include -I.. -I. -fPIC -c -Wl,-undefined -Wl,dynamic_lookup -dynamiclib
-CXXFLAGS		= -std=c++11 -I../../depends/$(shell echo `../..//depends/config.guess`/include) -I../univalue/include -I../cryptoconditions/include -I../cryptoconditions/src -I../cryptoconditions/src/asn -I.. -I. -fPIC -shared -c
+CXXFLAGS		= -std=c++11 -I../../depends/$(shell echo `../..//depends/config.guess`/include) -I../univalue/include -I../cryptoconditions/include -I../cryptoconditions/src -I../cryptoconditions/src/asn -I../leveldb/include -I.. -I. -fPIC -shared -c
-CXXFLAGS_WIN	= -std=c++11 -I../../depends/$(shell echo `../..//depends/config.guess`/include) -I../univalue/include -I../cryptoconditions/include -I../cryptoconditions/src -I../cryptoconditions/src/asn -I.. -I. -fPIC -shared -c
+CXXFLAGS_WIN	= -std=c++11 -I../../depends/$(shell echo `../..//depends/config.guess`/include) -I../univalue/include -I../cryptoconditions/include -I../cryptoconditions/src -I../cryptoconditions/src/asn -I../leveldb/include -I.. -I. -fPIC -shared -c
 DEBUGFLAGS	= -O0 -D _DEBUG
 RELEASEFLAGS	= -O2 -D NDEBUG -combine -fwhole-program
 $(info $(OS))

src/cc/Makefile_custom

@@ -3,9 +3,9 @@ CC		= gcc
 CC_DARWIN	= g++-8
 CC_WIN		= x86_64-w64-mingw32-gcc-posix
 CC_AARCH64	= aarch64-linux-gnu-g++
-CFLAGS_DARWIN	= -DBUILD_CUSTOMCC -std=c++11 -arch x86_64 -I../secp256k1/include -I../../depends/$(shell echo `../..//depends/config.guess`/include) -I../univalue/include -I../cryptoconditions/include -I../cryptoconditions/src -I../cryptoconditions/src/asn -I.. -I. -fPIC -Wl,-undefined -Wl,dynamic_lookup -Wno-write-strings -shared -dynamiclib
+CFLAGS_DARWIN	= -DBUILD_CUSTOMCC -std=c++11 -arch x86_64 -I../secp256k1/include -I../../depends/$(shell echo `../..//depends/config.guess`/include) -I../univalue/include -I../cryptoconditions/include -I../cryptoconditions/src -I../cryptoconditions/src/asn -I../leveldb/include -I.. -I. -fPIC -Wl,-undefined -Wl,dynamic_lookup -Wno-write-strings -shared -dynamiclib
-CFLAGS		= -Wno-write-strings -DBUILD_CUSTOMCC -std=c++11 -I../secp256k1/include -I../../depends/$(shell echo `../..//depends/config.guess`/include) -I../univalue/include -I../cryptoconditions/include -I../cryptoconditions/src -I../cryptoconditions/src/asn -I.. -I. -fPIC -shared
+CFLAGS		= -Wno-write-strings -DBUILD_CUSTOMCC -std=c++11 -I../secp256k1/include -I../../depends/$(shell echo `../..//depends/config.guess`/include) -I../univalue/include -I../cryptoconditions/include -I../cryptoconditions/src -I../cryptoconditions/src/asn -I../leveldb/include -I.. -I. -fPIC -shared
-CFLAGS_WIN	= -Wno-write-strings -DBUILD_CUSTOMCC -std=c++11 -I../secp256k1/include -I../../depends/x86_64-w64-mingw32/include -I../univalue/include -I../cryptoconditions/include -I../cryptoconditions/src -I../cryptoconditions/src/asn -I.. -I. -fPIC -shared
+CFLAGS_WIN	= -Wno-write-strings -DBUILD_CUSTOMCC -std=c++11 -I../secp256k1/include -I../../depends/x86_64-w64-mingw32/include -I../univalue/include -I../cryptoconditions/include -I../cryptoconditions/src -I../cryptoconditions/src/asn -I../leveldb/include -I.. -I. -fPIC -shared
 DEBUGFLAGS	= -O0 -D _DEBUG
 RELEASEFLAGS	= -O2 -D NDEBUG -combine -fwhole-program
 $(info $(OS))

src/chain.cpp

@@ -19,12 +19,56 @@
  ******************************************************************************/
 
 #include "chain.h"
+#include "main.h"
+#include "txdb.h"
 
 using namespace std;
 
 /**
  * CChain implementation
  */
+void CBlockIndex::TrimSolution()
+{
+    AssertLockHeld(cs_main);
+
+    // We can correctly trim a solution as soon as the block index entry has been added
+    // to leveldb. Updates to the block index entry (to update validity status) will be
+    // handled by re-reading the solution from the existing db entry. It does not help to
+    // try to avoid these reads by gating trimming on the validity status: the re-reads are
+    // efficient anyway because of caching in leveldb, and most of them are unavoidable.
+    if (HasSolution()) {
+        std::vector<unsigned char> empty;
+        nSolution.swap(empty);
+    }
+}
+
+CBlockHeader CBlockIndex::GetBlockHeader() const
+{
+    AssertLockHeld(cs_main);
+
+    CBlockHeader header;
+    header.nVersion             = nVersion;
+    if (pprev) {
+        header.hashPrevBlock    = pprev->GetBlockHash();
+    }
+    header.hashMerkleRoot       = hashMerkleRoot;
+    header.hashFinalSaplingRoot = hashFinalSaplingRoot;
+    header.nTime                = nTime;
+    header.nBits                = nBits;
+    header.nNonce               = nNonce;
+    if (HasSolution()) {
+        header.nSolution        = nSolution;
+    } else {
+        CDiskBlockIndex dbindex;
+        if (!pblocktree->ReadDiskBlockIndex(GetBlockHash(), dbindex)) {
+            LogPrintf("%s: ReadDiskBlockIndex failed to read index entry of block %s", __func__, GetBlockHash().ToString().c_str());
+            throw std::runtime_error("Failed to read index entry");
+        }
+        header.nSolution        = dbindex.GetSolution();
+    }
+    return header;
+}
+
 void CChain::SetTip(CBlockIndex *pindex) {
     lastTip = pindex;
     if (pindex == NULL) {

src/chain.h

@@ -27,6 +27,7 @@ class CChainPower;
 #include "pow.h"
 #include "tinyformat.h"
 #include "uint256.h"
+#include "util/strencodings.h"
 #include <vector>
 #include <boost/foreach.hpp>
 
@@ -384,8 +385,14 @@ public:
     unsigned int nTime;
     unsigned int nBits;
     uint256 nNonce;
+protected:
+    // The Equihash solution, if it is stored. Once we know that the block index
+    // entry is present in leveldb, this field can be cleared via the TrimSolution
+    // method to save memory.
     std::vector<unsigned char> nSolution;
 
+public:
+
     //! (memory only) Sequential id assigned to distinguish order in which blocks are received.
     uint32_t nSequenceId;
     
@@ -497,23 +504,15 @@ public:
         return ret;
     }
 
-    CBlockHeader GetBlockHeader() const
+    //! Get the block header for this block index. Requires cs_main.
-    {
+    CBlockHeader GetBlockHeader() const;
-        CBlockHeader block;
+
-        block.nVersion       = nVersion;
+    //! Clear the Equihash solution to save memory. Requires cs_main.
-        if (pprev)
+    void TrimSolution();
-            block.hashPrevBlock = pprev->GetBlockHash();
-        block.hashMerkleRoot = hashMerkleRoot;
-        block.hashFinalSaplingRoot   = hashFinalSaplingRoot;
-        block.nTime          = nTime;
-        block.nBits          = nBits;
-        block.nNonce         = nNonce;
-        block.nSolution      = nSolution;
-        return block;
-    }
 
     uint256 GetBlockHash() const
     {
+        assert(phashBlock);
         return *phashBlock;
     }
 
@@ -540,10 +539,11 @@ public:
 
     std::string ToString() const
     {
-        return strprintf("CBlockIndex(pprev=%p, nHeight=%d, merkle=%s, hashBlock=%s)",
+        return strprintf("CBlockIndex(pprev=%p, nHeight=%d, merkle=%s, hashBlock=%s, HasSolution=%s)",
             pprev, this->chainPower.nHeight,
             hashMerkleRoot.ToString(),
-            GetBlockHash().ToString());
+            phashBlock ? GetBlockHash().ToString() : "(nil)",
+            HasSolution());
     }
 
     //! Check whether this block index entry is valid up to the passed validity level.
@@ -555,6 +555,12 @@ public:
         return ((nStatus & BLOCK_VALID_MASK) >= nUpTo);
     }
 
+    //! Is the Equihash solution stored?
+    bool HasSolution() const
+    {
+        return !nSolution.empty();
+    }
+
     //! Raise the validity level of this block index entry.
     //! Returns true if the validity was changed.
     bool RaiseValidity(enum BlockStatus nUpTo)
@@ -588,8 +594,11 @@ public:
         hashPrev = uint256();
     }
 
-    explicit CDiskBlockIndex(const CBlockIndex* pindex) : CBlockIndex(*pindex) {
+    explicit CDiskBlockIndex(const CBlockIndex* pindex, std::function<std::vector<unsigned char>()> getSolution) : CBlockIndex(*pindex) {
         hashPrev = (pprev ? pprev->GetBlockHash() : uint256());
+        if (!HasSolution()) {
+            nSolution = getSolution();
+        }
     }
 
     ADD_SERIALIZE_METHODS;
@@ -670,18 +679,29 @@ public:
 
     uint256 GetBlockHash() const
     {
-        CBlockHeader block;
+        return GetBlockHeader().GetHash();
-        block.nVersion        = nVersion;
-        block.hashPrevBlock   = hashPrev;
-        block.hashMerkleRoot  = hashMerkleRoot;
-        block.hashFinalSaplingRoot    = hashFinalSaplingRoot;
-        block.nTime           = nTime;
-        block.nBits           = nBits;
-        block.nNonce          = nNonce;
-        block.nSolution       = nSolution;
-        return block.GetHash();
     }
 
+    //! Get the block header for this block index.
+    CBlockHeader GetBlockHeader() const
+    {
+        CBlockHeader header;
+        header.nVersion             = nVersion;
+        header.hashPrevBlock        = hashPrev;
+        header.hashMerkleRoot       = hashMerkleRoot;
+        header.hashFinalSaplingRoot = hashFinalSaplingRoot;
+        header.nTime                = nTime;
+        header.nBits                = nBits;
+        header.nNonce               = nNonce;
+        header.nSolution            = nSolution;
+        return header;
+    }
+
+    std::vector<unsigned char> GetSolution() const
+    {
+        assert(HasSolution());
+        return nSolution;
+    }
 
     std::string ToString() const
     {
@@ -692,6 +712,13 @@ public:
             hashPrev.ToString());
         return str;
     }
+
+private:
+    //! This method should not be called on a CDiskBlockIndex.
+    void TrimSolution()
+    {
+        assert(!"called CDiskBlockIndex::TrimSolution");
+    }
 };
 
 /** An in-memory indexed chain of blocks. */

src/dbwrapper.h

@@ -12,8 +12,8 @@
 #include "util.h"
 #include "version.h"
 #include <boost/filesystem/path.hpp>
-#include <leveldb/db.h>
+#include "leveldb/db.h"
-#include <leveldb/write_batch.h>
+#include "leveldb/write_batch.h"
 
 static const size_t DBWRAPPER_PREALLOC_KEY_SIZE = 64;
 static const size_t DBWRAPPER_PREALLOC_VALUE_SIZE = 1024;

src/dragonx-cli

@@ -10,4 +10,4 @@ cd $DIR
 DIR="$( cd "$( dirname "$( readlink "${BASH_SOURCE[0]}" )" )" && pwd )"
 cd $DIR
 
-./hush-cli -ac_name=DRAGONX $@
+./hush-cli -ac_name=DRAGONX "$@"

src/hush_nSPV_fullnode.h

@@ -124,7 +124,8 @@ int32_t NSPV_setequihdr(struct NSPV_equihdr *hdr,int32_t height)
         hdr->nTime = pindex->nTime;
         hdr->nBits = pindex->nBits;
         hdr->nNonce = pindex->nNonce;
-        memcpy(hdr->nSolution,&pindex->nSolution[0],sizeof(hdr->nSolution));
+
+        memcpy(hdr->nSolution,&pindex->GetBlockHeader().nSolution[0],sizeof(hdr->nSolution));
         return(sizeof(*hdr));
     }
     return(-1);

src/main.cpp

@@ -3254,7 +3254,8 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin
     uint256 hashPrevBlock = pindex->pprev == NULL ? uint256() : pindex->pprev->GetBlockHash();
     if ( hashPrevBlock != view.GetBestBlock() )
     {
-        fprintf(stderr,"ConnectBlock(): hashPrevBlock != view.GetBestBlock()\n");
+        fprintf(stderr,"ConnectBlock(): hashPrevBlock != view.GetBestBlock() %s != %s\n", hashPrevBlock.ToString().c_str(), view.GetBestBlock().ToString().c_str() );
+
         return state.DoS(1, error("ConnectBlock(): hashPrevBlock != view.GetBestBlock()"),
                          REJECT_INVALID, "hashPrevBlock-not-bestblock");
     }
@@ -3700,7 +3701,7 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) {
                         vFiles.push_back(make_pair(*it, &vinfoBlockFile[*it]));
                     setDirtyFileInfo.erase(it++);
                 }
-                std::vector<const CBlockIndex*> vBlocks;
+                std::vector<CBlockIndex*> vBlocks;
                 vBlocks.reserve(setDirtyBlockIndex.size());
                 for (set<CBlockIndex*>::iterator it = setDirtyBlockIndex.begin(); it != setDirtyBlockIndex.end(); ) {
                     vBlocks.push_back(*it);
@@ -3709,6 +3710,12 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) {
                 if (!pblocktree->WriteBatchSync(vFiles, nLastBlockFile, vBlocks)) {
                     return AbortNode(state, "Files to write to block index database");
                 }
+                // Now that we have written the block indices to the database, we do not
+                // need to store solutions for these CBlockIndex objects in memory.
+                // cs_main must be held here.
+                for (CBlockIndex *pblockindex : vBlocks) {
+                    pblockindex->TrimSolution();
+                }
             }
             // Finally remove any pruned files
             if (fFlushForPrune)
@@ -5135,7 +5142,7 @@ bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationState& sta
                     //    " " << block.GetHash().ToString() << " @ "       << block.GetBlockTime() <<  endl;
                     return state.DoS(100, error("%s: Incorrect diffbits at height %d: %lu vs %lu ", __func__, nHeight, nNextWork, block.nBits), REJECT_INVALID, "bad-diffbits");
                 } else {
-                    cout << "Ignoring nbits for height=" << nHeight << endl;
+                    // cout << "Ignoring nbits for height=" << nHeight << endl;
                 }
         }
     }
@@ -6591,7 +6598,11 @@ void static CheckBlockIndex()
                 }
             }
         }
+        // try {
         //     assert(pindex->GetBlockHash() == pindex->GetBlockHeader().GetHash()); // Perhaps too slow
+        // } catch (const runtime_error&) {
+        //     assert(!"Failed to read index entry");
+        // }
         // End: actual consistency checks.
 
         // Try descending into the first subnode.

src/main.h

@@ -39,6 +39,7 @@
 #include "spentindex.h"
 #include "sync.h"
 #include "tinyformat.h"
+#include "txdb.h"
 #include "txmempool.h"
 #include "uint256.h"
 

src/net.cpp

@@ -2468,7 +2468,7 @@ void RelayTransaction(const CTransaction& tx, const CDataStream& ss)
         fprintf(stderr, "%s: -testnode=1, no peers, not relaying\n", __func__ );
         return;
     } else {
-        fprintf(stderr, "%s: Relaying to %lu of %lu peers\n", __func__, newSize, vNodes.size() );
+        fprintf(stderr, "%s: Relaying %s to %lu of %lu peers\n", __func__, tx.GetHash().GetHex().c_str(), newSize, vNodes.size() );
     }
 
     // Only relay to randomly chosen 50% of peers

src/rest.cpp

@@ -151,6 +151,7 @@ static bool rest_headers(HTTPRequest* req,
 
     std::vector<const CBlockIndex *> headers;
     headers.reserve(count);
+    CDataStream ssHeader(SER_NETWORK, PROTOCOL_VERSION);
     {
         LOCK(cs_main);
         BlockMap::const_iterator it = mapBlockIndex.find(hash);
@@ -161,12 +162,17 @@ static bool rest_headers(HTTPRequest* req,
                 break;
             pindex = chainActive.Next(pindex);
         }
-    }
 
-    CDataStream ssHeader(SER_NETWORK, PROTOCOL_VERSION);
+        if (rf == RF_BINARY || rf == RF_HEX) {
-    BOOST_FOREACH(const CBlockIndex *pindex, headers) {
+            try {
+                for (const CBlockIndex *pindex : headers) {
                     ssHeader << pindex->GetBlockHeader();
                 }
+            } catch (const std::runtime_error&) {
+                return RESTERR(req, HTTP_INTERNAL_SERVER_ERROR, "Failed to read index entry");
+            }
+        }
+    }
 
     switch (rf) {
     case RF_BINARY: {

src/rpc/blockchain.cpp

@@ -147,7 +147,7 @@ UniValue blockheaderToJSON(const CBlockIndex* blockindex)
     result.push_back(Pair("finalsaplingroot", blockindex->hashFinalSaplingRoot.GetHex()));
     result.push_back(Pair("time", (int64_t)blockindex->nTime));
     result.push_back(Pair("nonce", blockindex->nNonce.GetHex()));
-    result.push_back(Pair("solution", HexStr(blockindex->nSolution)));
+    result.pushKV("solution", HexStr(blockindex->GetBlockHeader().nSolution));
     result.push_back(Pair("bits", strprintf("%08x", blockindex->nBits)));
     result.push_back(Pair("difficulty", GetDifficulty(blockindex)));
     result.push_back(Pair("chainwork", blockindex->chainPower.chainWork.GetHex()));
@@ -693,16 +693,19 @@ UniValue getblockheader(const UniValue& params, bool fHelp, const CPubKey& mypk)
 
     CBlockIndex* pblockindex = mapBlockIndex[hash];
 
-    if (!fVerbose)
+    try {
-    {
+        if (!fVerbose) {
             CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION);
             ssBlock << pblockindex->GetBlockHeader();
             std::string strHex = HexStr(ssBlock.begin(), ssBlock.end());
             return strHex;
-    }
+        } else {
-
             return blockheaderToJSON(pblockindex);
         }
+    } catch (const runtime_error&) {
+        throw JSONRPCError(RPC_DATABASE_ERROR, "Failed to read index entry");
+    }
+}
 
 UniValue getblock(const UniValue& params, bool fHelp, const CPubKey& mypk)
 {

src/transaction_builder.cpp

@@ -182,7 +182,7 @@ boost::optional<CTransaction> TransactionBuilder::Build()
 
     auto ctx = librustzcash_sapling_proving_ctx_init();
 
-    LogPrintf("%s: Creating Sapling SpendDescriptions\n", __FUNCTION__);
+    LogPrintf("%s: Creating Sapling SpendDescriptions size=%d\n", __FUNCTION__, spends.size());
     // Create Sapling SpendDescriptions
     for (auto spend : spends) {
         auto cm = spend.note.cm();
@@ -213,7 +213,7 @@ boost::optional<CTransaction> TransactionBuilder::Build()
                 sdesc.rk.begin(),
                 sdesc.zkproof.data())) {
             librustzcash_sapling_proving_ctx_free(ctx);
-            LogPrintf("%s: Invalid sapling spend proof!\n", __FUNCTION__);
+            LogPrintf("%s: Invalid sapling spend proof! note value=%d\n", __FUNCTION__, spend.note.value() );
             return boost::none;
         }
 

src/txdb.cpp

@@ -269,18 +269,33 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const {
     return true;
 }
 
-bool CBlockTreeDB::WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*> >& fileInfo, int nLastFile, const std::vector<const CBlockIndex*>& blockinfo) {
+bool CBlockTreeDB::WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*> >& fileInfo, int nLastFile, const std::vector<CBlockIndex*>& blockinfo) {
     CDBBatch batch(*this);
     if (fZdebug)
 	    fprintf(stderr, "%s: Writing block files\n", __FUNCTION__);
-    for (std::vector<std::pair<int, const CBlockFileInfo*> >::const_iterator it=fileInfo.begin(); it != fileInfo.end(); it++) {
+    for (const auto& it : fileInfo) {
-        batch.Write(make_pair(DB_BLOCK_FILES, it->first), *it->second);
+        batch.Write(make_pair(DB_BLOCK_FILES, it.first), *it.second);
     }
     batch.Write(DB_LAST_BLOCK, nLastFile);
     if (fZdebug)
 	    fprintf(stderr, "%s: Writing block index\n", __FUNCTION__);
-    for (std::vector<const CBlockIndex*>::const_iterator it=blockinfo.begin(); it != blockinfo.end(); it++) {
+    for (const auto& it : blockinfo) {
-        batch.Write(make_pair(DB_BLOCK_INDEX, (*it)->GetBlockHash()), CDiskBlockIndex(*it));
+        std::pair<char, uint256> key = make_pair(DB_BLOCK_INDEX, it->GetBlockHash());
+        try {
+            CDiskBlockIndex dbindex {it, [this, &key]() {
+                // It can happen that the index entry is written, then the Equihash solution is cleared from memory,
+                // then the index entry is rewritten. In that case we must read the solution from the old entry.
+                CDiskBlockIndex dbindex_old;
+                if (!Read(key, dbindex_old)) {
+                    LogPrintf("%s: Failed to read index entry", __func__);
+                    throw runtime_error("Failed to read index entry");
+                }
+                return dbindex_old.GetSolution();
+            }};
+            batch.Write(key, dbindex);
+        } catch (const runtime_error&) {
+            return false;
+        }
     }
     return WriteBatch(batch, true);
 }
@@ -293,6 +308,11 @@ bool CBlockTreeDB::EraseBatchSync(const std::vector<const CBlockIndex*>& blockin
     return WriteBatch(batch, true);
 }
 
+bool CBlockTreeDB::ReadDiskBlockIndex(const uint256 &blockhash, CDiskBlockIndex &dbindex) const {
+    return Read(make_pair(DB_BLOCK_INDEX, blockhash), dbindex);
+}
+
+
 bool CBlockTreeDB::ReadTxIndex(const uint256 &txid, CDiskTxPos &pos) const {
     return Read(make_pair(DB_TXINDEX, txid), pos);
 }
@@ -692,7 +712,8 @@ bool CBlockTreeDB::LoadBlockIndexGuts()
                 pindexNew->nTime                  = diskindex.nTime;
                 pindexNew->nBits                  = diskindex.nBits;
                 pindexNew->nNonce                 = diskindex.nNonce;
-                pindexNew->nSolution              = diskindex.nSolution;
+                // the Equihash solution will be loaded lazily from the dbindex entry
+                // pindexNew->nSolution              = diskindex.nSolution;
                 pindexNew->nStatus                = diskindex.nStatus;
                 pindexNew->nCachedBranchId        = diskindex.nCachedBranchId;
                 pindexNew->nTx                    = diskindex.nTx;
@@ -716,10 +737,24 @@ bool CBlockTreeDB::LoadBlockIndexGuts()
 
                 //fprintf(stderr,"loadguts ht.%d\n",pindexNew->GetHeight());
                 // Consistency checks
-                auto header = pindexNew->GetBlockHeader();
+                /*
+                CBlockHeader header;
+                {
+                    LOCK(cs_main);
+                    try {
+                        header = pindexNew->GetBlockHeader();
+                    } catch (const runtime_error&) {
+                        return error("LoadBlockIndex(): failed to read index entry: diskindex hash = %s",
+                            diskindex.GetBlockHash().ToString());
+                    }
+                }
+                if (header.GetHash() != diskindex.GetBlockHash())
+                    return error("LoadBlockIndex(): inconsistent header vs diskindex hash: header hash = %s, diskindex hash = %s",
+                        header.GetHash().ToString(), diskindex.GetBlockHash().ToString());
                 if (header.GetHash() != pindexNew->GetBlockHash())
                     return error("LoadBlockIndex(): block header inconsistency detected: on-disk = %s, in-memory = %s",
                                  diskindex.ToString(),  pindexNew->ToString());
+
                 if ( 0 ) // POW will be checked before any block is connected
                 {
                     uint8_t pubkey33[33];
@@ -727,6 +762,7 @@ bool CBlockTreeDB::LoadBlockIndexGuts()
                     if (!CheckProofOfWork(header,pubkey33,pindexNew->GetHeight(),Params().GetConsensus()))
                         return error("LoadBlockIndex(): CheckProofOfWork failed: %s", pindexNew->ToString());
                 }
+                */
                 pcursor->Next();
             } else {
                 return error("LoadBlockIndex() : failed to read value");

src/txdb.h

@@ -26,6 +26,7 @@
 
 #include "coins.h"
 #include "dbwrapper.h"
+#include "chain.h"
 #include <map>
 #include <string>
 #include <utility>
@@ -91,12 +92,13 @@ private:
     CBlockTreeDB(const CBlockTreeDB&);
     void operator=(const CBlockTreeDB&);
 public:
-    bool WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*> >& fileInfo, int nLastFile, const std::vector<const CBlockIndex*>& blockinfo);
+    bool WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*> >& fileInfo, int nLastFile, const std::vector<CBlockIndex*>& blockinfo);
     bool EraseBatchSync(const std::vector<const CBlockIndex*>& blockinfo);
     bool ReadBlockFileInfo(int nFile, CBlockFileInfo &fileinfo) const;
     bool ReadLastBlockFile(int &nFile) const;
     bool WriteReindexing(bool fReindex);
     bool ReadReindexing(bool &fReindex) const;
+    bool ReadDiskBlockIndex(const uint256 &blockhash, CDiskBlockIndex &dbindex) const;
     bool ReadTxIndex(const uint256 &txid, CDiskTxPos &pos) const;
     bool WriteTxIndex(const std::vector<std::pair<uint256, CDiskTxPos> > &list);
     bool ReadSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value) const;

src/wallet/wallet.cpp

@@ -1146,6 +1146,15 @@ void CWallet::BuildWitnessCache(const CBlockIndex* pindex, bool witnessOnly)
     LogPrintf("%s: height=%d, startHeight=%d\n", __func__, height, startHeight);
 
   while (pblockindex) {
+    if (ShutdownRequested()) {
+        LogPrintf("%s: shutdown requested, aborting building witnesses\n", __func__);
+        break;
+    }
+    if(pwalletMain->fAbortRescan) {
+        LogPrintf("%s: rescan aborted at block %d, stopping witness building\n", pwalletMain->rescanHeight);
+        pwalletMain->fRescanning = false;
+        return;
+    }
 
     if (pblockindex->GetHeight() % 100 == 0 && pblockindex->GetHeight() < height - 5) {
       LogPrintf("Building Witnesses for block %i %.4f complete, %d remaining\n", pblockindex->GetHeight(), pblockindex->GetHeight() / double(height), height - pblockindex->GetHeight() );

util/README.md

@@ -12,6 +12,11 @@ Compile Hush full node code.
 ## build-arm.sh
 
 Compile Hush full node code for ARM architecture.
+**has not worked for some time**
+
+## build-arm-xcompile.sh
+
+Cross-Compile Hush full node code for ARM architecture on an x86 build server.
 
 ## build-debian-package.sh
 

util/build-arm-xcompile.sh

@@ -0,0 +1,139 @@
+#!/usr/bin/env bash
+# Copyright (c) 2016-2023 The Hush developers
+# Distributed under the GPLv3 software license, see the accompanying
+# file COPYING or https://www.gnu.org/licenses/gpl-3.0.en.html
+
+# For use with https://git.hush.is/jahway603/hush-docs/src/branch/master/advanced/cross-compile-hush-full-node-to-aarch64-with-docker.md,
+# Please follow that
+set -eu -o pipefail
+
+# Check if cmake, a new dependency for randomx support, is installed on system and exits if it is not
+if ! [ -x "$(command -v cmake)" ]; then
+  echo 'Error: cmake is not installed. Install cmake and try again.' >&2
+  exit 1
+fi
+
+function cmd_pref() {
+    if type -p "$2" > /dev/null; then
+        eval "$1=$2"
+    else
+        eval "$1=$3"
+    fi
+}
+cat <<'EOF'
+ .~~~~~~~~~~~~~~~~.
+{{ Building Hush!! }}
+ `~~~~~~~~~~~~~~~~`
+        \   ^__^
+         \  (@@)\_______            
+            (__)\ HUSH  )\/\      $
+    z        zz ||----w |      z  |
+zz  zz  z       || z   ||xxx   z z|z zz
+zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
+zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
+EOF
+
+# If a g-prefixed version of the command exists, use it preferentially.
+function gprefix() {
+    cmd_pref "$1" "g$2" "$2"
+}
+
+gprefix READLINK readlink
+cd "$(dirname "$("$READLINK" -f "$0")")/.."
+
+# Allow user overrides to $MAKE. Typical usage for users who need it:
+#   MAKE=gmake ./util/build.sh -j$(nproc)
+if [[ -z "${MAKE-}" ]]; then
+    MAKE=make
+fi
+
+# Allow overrides to $BUILD and $HOST for porters. Most users will not need it.
+#   BUILD=i686-pc-linux-gnu ./util/build.sh
+if [[ -z "${BUILD-}" ]]; then
+    BUILD="$(./depends/config.guess)"
+fi
+if [[ -z "${HOST-}" ]]; then
+    HOST="$BUILD"
+fi
+
+# Allow users to set arbitrary compile flags. Most users will not need this.
+if [[ -z "${CONFIGURE_FLAGS-}" ]]; then
+    CONFIGURE_FLAGS=""
+fi
+
+if [ "x$*" = 'x--help' ]
+then
+    cat ./util/dragon.txt
+    cat <<EOF
+Welcome To The Hush Build System, Here Be Dragons!
+Usage:
+$0 --help
+  Show this help message and exit.
+$0 [ --enable-lcov || --disable-tests ] [ --disable-mining ] [ --disable-libs ] [ MAKEARGS... ]
+  Build Hush and most of its transitive dependencies from source. MAKEARGS are applied to both dependencies and Hush itself.
+  If --enable-lcov is passed, Hush is configured to add coverage instrumentation, thus enabling "make cov" to work.
+  If --disable-tests is passed instead, the Hush tests are not built.
+  If --disable-mining is passed, Hush is configured to not build any mining code. It must be passed after the test arguments, if present.
+  It must be passed after the test/mining arguments, if present.
+EOF
+    exit 0
+fi
+
+set -x
+
+# If --enable-lcov is the first argument, enable lcov coverage support:
+LCOV_ARG=''
+HARDENING_ARG='--enable-hardening'
+TEST_ARG=''
+if [ "x${1:-}" = 'x--enable-lcov' ]
+then
+    LCOV_ARG='--enable-lcov'
+    HARDENING_ARG='--disable-hardening'
+    shift
+elif [ "x${1:-}" = 'x--disable-tests' ]
+then
+    TEST_ARG='--enable-tests=no'
+    shift
+fi
+
+# If --disable-mining is the next argument, disable mining code:
+MINING_ARG=''
+if [ "x${1:-}" = 'x--disable-mining' ]
+then
+    MINING_ARG='--enable-mining=no'
+    shift
+fi
+
+# Just show the useful info
+eval "$MAKE" --version | head -n2
+as --version | head -n1
+as --version | tail -n1
+ld -v
+
+HOST="$HOST" BUILD="$BUILD" "$MAKE" "$@" -C ./depends/ V=1
+
+./autogen.sh
+
+CONFIG_SITE="$PWD/depends/$HOST/share/config.site" ./configure "$HARDENING_ARG" "$LCOV_ARG" "$TEST_ARG" "$MINING_ARG" $CONFIGURE_FLAGS CXXFLAGS='-g'
+
+# Build CryptoConditions stuff
+WD=$PWD
+cd src/cc
+echo $PWD
+./makecustom
+cd $WD
+
+# Build RandomX
+cd src/RandomX
+if [ -d "build" ]
+then
+    ls -la build/librandomx*
+else
+    mkdir build && cd build
+    cmake -DCMAKE_C_COMPILER=aarch64-linux-gnu-gcc -DCMAKE_CXX_COMPILER=aarch64-linux-gnu-g++ -DCMAKE_STRIP=/usr/bin/aarch64-linux-gnu-strip -DARCH_ID=aarch64 ..
+    make
+fi
+
+cd $WD
+
+"$MAKE" "$@" V=1