Release randomx dataset+cache when mining is interrupted or errors

Reviewed-on: https://git.hush.is/hush/hush3/pulls/322

antispam

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+echo "./src/hush-cli -ac_name=ANTISPAM $@"
+./src/hush-cli -ac_name=ANTISPAM $@

qa/pull-tester/rpc-tests.sh

@@ -14,6 +14,7 @@ export BITCOIND=${REAL_BITCOIND}
 #Run the tests
 
 testScripts=(
+    'antispam.py'
     'dpow.py'
     'dpowconfs.py'
     'ac_private.py'

qa/pull-tester/tests-config.sh.in

@@ -11,7 +11,6 @@ EXEEXT="@EXEEXT@"
 @ENABLE_WALLET_TRUE@ENABLE_WALLET=1
 @BUILD_BITCOIN_UTILS_TRUE@ENABLE_UTILS=1
 @BUILD_BITCOIND_TRUE@ENABLE_BITCOIND=1
-@ENABLE_PROTON_TRUE@ENABLE_PROTON=1
 
 REAL_BITCOIND="$BUILDDIR/src/hushd${EXEEXT}"
 REAL_BITCOINCLI="$BUILDDIR/src/hush-cli${EXEEXT}"

qa/rpc-tests/antispam.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python2
+# Copyright (c) 2016-2023 The Hush developers
+# Distributed under the GPLv3 software license, see the accompanying
+# file COPYING or https://www.gnu.org/licenses/gpl-3.0.en.html
+
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.authproxy import JSONRPCException
+from test_framework.util import (
+    assert_equal,
+    start_nodes,
+    wait_and_assert_operationid_status,
+)
+
+from decimal import Decimal
+
+class AntispamTest(BitcoinTestFramework):
+
+    def setup_nodes(self):
+        return start_nodes(2, self.options.tmpdir, [[ ]] * 2)
+
+    def run_test(self):
+        # Sanity-check the test harness
+        assert_equal(self.nodes[0].getblockcount(), 200)
+
+        # make sure we can mine a block
+        self.nodes[1].generate(1)
+        self.sync_all()
+
+        # make a new zaddr on each node
+        saplingAddr0 = self.nodes[0].z_getnewaddress()
+        saplingAddr1 = self.nodes[1].z_getnewaddress()
+
+        # Verify addresses
+        assert(saplingAddr0 in self.nodes[0].z_listaddresses())
+        assert(saplingAddr1 in self.nodes[1].z_listaddresses())
+        assert_equal(self.nodes[0].z_validateaddress(saplingAddr0)['type'], 'sapling')
+        assert_equal(self.nodes[0].z_validateaddress(saplingAddr1)['type'], 'sapling')
+
+        # Verify balance
+        assert_equal(self.nodes[0].z_getbalance(saplingAddr0), Decimal('0'))
+        assert_equal(self.nodes[1].z_getbalance(saplingAddr1), Decimal('0'))
+
+if __name__ == '__main__':
+    AntispamTest().main()

qa/rpc-tests/test_framework/util.py

@@ -104,10 +104,10 @@ def initialize_datadir(dirname, n):
         f.write("showmetrics=0\n");
         f.write("rpcuser=hush\n");
         f.write("rpcpassword=puppy\n");
-        #f.write("port="+str(p2p_port(n))+"\n");
-        #rpcport = str(rpc_port(n))
-        #f.write("rpcport="+rpcport+"\n");
-        #print "RPC port=" + rpcport
+        f.write("port="+str(p2p_port(n))+"\n");
+        rpcport = str(rpc_port(n))
+        f.write("rpcport="+rpcport+"\n");
+        print "RPC port=" + rpcport
         f.write("listenonion=0\n");
         # TODO: maybe make these optional, via arg to initialize_datadir, defaulted to on for now
         f.write("addressindex=1\n");
@@ -148,7 +148,7 @@ def initialize_chain(test_dir):
         rpcs = []
         for i in range(4):
             try:
-                url = "http://rt:rt@127.0.0.1:%d"%(rpc_port(i),)
+                url = "http://hush:puppy@127.0.0.1:%d"%(rpc_port(i),)
                 rpcs.append(AuthServiceProxy(url))
             except:
                 sys.stderr.write("Error connecting to "+url+"\n")
@@ -165,11 +165,13 @@ def initialize_chain(test_dir):
                 for j in range(25):
                     set_node_times(rpcs, block_time)
                     rpcs[peer].generate(1)
-                    block_time += 10*60
+                    # TODO: HUSH3 has 75s blocktime, other HSCs could be different
+                    block_time += 10*75
                 # Must sync before next peer starts generating blocks
                 sync_blocks(rpcs)
 
         # Shut them down, and clean up cache directories:
+        print("Stopping nodes")
         stop_nodes(rpcs)
         wait_bitcoinds()
         for i in range(4):
@@ -182,8 +184,9 @@ def initialize_chain(test_dir):
     for i in range(4):
         from_dir = os.path.join("cache", "node"+str(i))
         to_dir = os.path.join(test_dir,  "node"+str(i))
+        print("Copying " + from_dir + " to " + to_dir)
         shutil.copytree(from_dir, to_dir)
-        initialize_datadir(test_dir, i) # Overwrite port/rpcport in hush.conf
+        initialize_datadir(test_dir, i) # Overwrite port/rpcport in HUSH3.conf
 
 def initialize_chain_clean(test_dir, num_nodes):
     """
@@ -218,9 +221,10 @@ def start_node(i, dirname, extra_args=None, rpchost=None, timewait=None, binary=
     """
     Start a hushd and return RPC connection to it
     """
+    print("Starting node " + str(i))
     datadir = os.path.join(dirname, "node"+str(i))
     # creating special config in case of cryptocondition asset chain test
-    if extra_args[0] == '-ac_name=REGTEST':
+    if len(extra_args) > 0 and extra_args[0] == '-ac_name=REGTEST':
         configpath = datadir + "/REGTEST.conf"
         with open(configpath, "w+") as config:
             config.write("regtest=1\n")
@@ -259,7 +263,8 @@ def start_node(i, dirname, extra_args=None, rpchost=None, timewait=None, binary=
     if os.getenv("PYTHON_DEBUG", ""):
         print "start_node: calling hush-cli -rpcwait getblockcount returned"
     devnull.close()
-    port     = extra_args[3]
+    #port     = extra_args[3]
+    port     = rpc_port(i)
     username = rpc_username()
     password = rpc_password()
     url      = "http://%s:%s@%s:%d" % (username, password, rpchost or '127.0.0.1', int(port[9:]))
@@ -276,6 +281,7 @@ def start_nodes(num_nodes, dirname, extra_args=None, rpchost=None, binary=None):
     """
     Start multiple hushds, return RPC connections to them
     """
+    print("Starting " + str(num_nodes) + " nodes")
     if extra_args is None: extra_args = [ None for i in range(num_nodes) ]
     if binary is None: binary = [ None for i in range(num_nodes) ]
     return [ start_node(i, dirname, extra_args[i], rpchost, binary=binary[i]) for i in range(num_nodes) ]
@@ -288,6 +294,7 @@ def check_node(i):
     return bitcoind_processes[i].returncode
 
 def stop_node(node, i):
+    print("Stopping node " + i)
     node.stop()
     bitcoind_processes[i].wait()
     del bitcoind_processes[i]
@@ -298,11 +305,12 @@ def stop_nodes(nodes):
     del nodes[:] # Emptying array closes connections as a side effect
 
 def set_node_times(nodes, t):
+    print("Setting nodes time to " + t)
     for node in nodes:
         node.setmocktime(t)
 
 def wait_bitcoinds():
-    # Wait for all bitcoinds to cleanly exit
+    print("Waiting for all nodes to cleanly exit")
     for bitcoind in bitcoind_processes.values():
         bitcoind.wait()
     bitcoind_processes.clear()

qa/rpc-tests/wallet_sapling.py

@@ -19,9 +19,9 @@ class WalletSaplingTest(BitcoinTestFramework):
 
     def setup_nodes(self):
         return start_nodes(4, self.options.tmpdir, [[
-            '-nuparams=5ba81b19:201', # Overwinter
-            '-nuparams=76b809bb:203', # Sapling
-            '-experimentalfeatures', '-zmergetoaddress',
+            #'-nuparams=5ba81b19:201', # Overwinter
+            #'-nuparams=76b809bb:203', # Sapling
+            #'-experimentalfeatures', '-zmergetoaddress',
         ]] * 4)
 
     def run_test(self):

src/miner.cpp

@@ -279,11 +279,16 @@ CBlockTemplate* CreateNewBlock(CPubKey _pk,const CScript& _scriptPubKeyIn, int32
         vecPriority.reserve(mempool.mapTx.size() + 1);
 
         //fprintf(stderr,"%s: going to add txs from mempool\n", __func__);
-        // now add transactions from the mem pool
+        // now add transactions from the mempool
         int32_t Notarizations = 0; uint64_t txvalue;
+        uint32_t large_zins  = 0;                  // number of ztxs with large number of inputs in block
+        uint32_t large_zouts = 0;                  // number of ztxs with large number of outputs in block
+        const uint32_t LARGE_ZINS_MAX        = 1;  // max ztxs with large zins per block
+        const uint32_t LARGE_ZOUTS_MAX       = 1;  // max ztxs with large zouts per block
+        const uint32_t LARGE_ZINS_THRESHOLD  = 50; // min number of zins to be considered large
+        const uint32_t LARGE_ZOUTS_THRESHOLD = 10; // min number of zouts to be considered large
         for (CTxMemPool::indexed_transaction_set::iterator mi = mempool.mapTx.begin();
-             mi != mempool.mapTx.end(); ++mi)
-        {
+             mi != mempool.mapTx.end(); ++mi) {
             const CTransaction& tx = mi->GetTx();
 
             int64_t nLockTimeCutoff = (STANDARD_LOCKTIME_VERIFY_FLAGS & LOCKTIME_MEDIAN_TIME_PAST)
@@ -466,6 +471,18 @@ CBlockTemplate* CreateNewBlock(CPubKey _pk,const CScript& _scriptPubKeyIn, int32
             // fprintf(stderr,"%s: compared first tx from priority queue\n", __func__);
             vecPriority.pop_back();
 
+            if(tx.vShieldedSpend.size() >= LARGE_ZINS_THRESHOLD && large_zins >= LARGE_ZINS_MAX) {
+                LogPrintf("%s: skipping ztx %s with %d zins because there are already %d ztxs with large zins\n",
+                    __func__, tx.GetHash().ToString().c_str(), tx.vShieldedSpend.size(), LARGE_ZINS_MAX);
+                continue;
+            }
+
+            if(tx.vShieldedOutput.size() >= LARGE_ZOUTS_THRESHOLD && large_zouts >= LARGE_ZOUTS_MAX) {
+                LogPrintf("%s: skipping ztx %s with %d zouts because there are already %d ztxs with large zouts\n",
+                    __func__, tx.GetHash().ToString().c_str(), tx.vShieldedOutput.size(), LARGE_ZOUTS_MAX);
+                continue;
+            }
+
             // Size limits
             unsigned int nTxSize = ::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION);
             // fprintf(stderr,"%s: nTxSize = %u\n", __func__, nTxSize);
@@ -576,6 +593,18 @@ CBlockTemplate* CreateNewBlock(CPubKey _pk,const CScript& _scriptPubKeyIn, int32
             nBlockSigOps += nTxSigOps;
             nFees += nTxFees;
 
+            if(tx.vShieldedOutput.size() >= LARGE_ZOUTS_THRESHOLD) {
+                large_zouts++;
+                LogPrintf("%s: txid=%s has large zouts=%d (%d large zouts in block)\n", __func__, tx.GetHash().ToString().c_str(),
+                    tx.vShieldedOutput.size(), large_zouts );
+            }
+
+            if(tx.vShieldedSpend.size() >= LARGE_ZINS_THRESHOLD) {
+                large_zins++;
+                LogPrintf("%s: txid=%s has large zins=%d (%d large zins in block)\n", __func__, tx.GetHash().ToString().c_str(),
+                    tx.vShieldedSpend.size(), large_zins );
+            }
+
             if (fPrintPriority)
             {
                 LogPrintf("priority %.1f fee %s txid %s\n",dPriority, feeRate.ToString(), tx.GetHash().ToString());
@@ -1395,12 +1424,24 @@ void static RandomXMiner()
    } catch (const boost::thread_interrupted&) {
        miningTimer.stop();
        c.disconnect();
+
+       randomx_release_dataset(randomxDataset);
+       rxdebug("%s: released dataset\n");
+       randomx_release_cache(randomxCache);
+       rxdebug("%s: released cache\n");
+
        LogPrintf("HushRandomXMiner terminated\n");
        throw;
    } catch (const std::runtime_error &e) {
        miningTimer.stop();
        c.disconnect();
        fprintf(stderr,"RandomXMiner: runtime error: %s\n", e.what());
+
+       randomx_release_dataset(randomxDataset);
+       rxdebug("%s: released dataset\n");
+       randomx_release_cache(randomxCache);
+       rxdebug("%s: released cache\n");
+
        return;
    }
 

test_antispam

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+# any CLI args given to this script will be passed along
+# example: ./test_antispam -debug=blah
+#./src/hushd -ac_name=ANTISPAM -ac_private=1 -ac_blocktime=180 -ac_reward=500000000 -ac_supply=55555 -gen=1 -genproclimit=1 -testnode=1 $@
+
+./src/hushd -ac_name=ANTISPAM -ac_private=1 -ac_blocktime=180 -ac_reward=500000000 -ac_supply=55555 $@
+# to run via the debugger
+# type "run" when gdb prompt appears
+#gdb --args ./src/hushd -- -ac_algo=randomx -ac_name=ANTISPAM -ac_private=1 -ac_blocktime=180 -ac_reward=500000000 -ac_supply=55555 -gen=1 -genproclimit=1 -testnode=1