Consolidate tests and configuration

* Refactory
* Fix performance issues
* Make test utils configurable via config.ini

Author: everoddandeven

README.md

@@ -3,7 +3,7 @@
 [![Tests](https://github.com/everoddandeven/monero-python/actions/workflows/test.yml/badge.svg)](https://github.com/everoddandeven/monero-python/actions/workflows/test.yml)
 [![Codacy Badge](https://app.codacy.com/project/badge/Grade/aeff91a5b1d543ddb400f88ffce150a8)](https://app.codacy.com/gh/everoddandeven/monero-python/dashboard?utm_source=gh&utm_medium=referral&utm_content=&utm_campaign=Badge_grade)
 
-> [!NOTE]
+> [!WARNING]
 >
 > monero-python is currently under maintenance, expect bugs and breaking changes.
 > The maintenance of this project has been generously funded by the [Monero CCS](https://ccs.getmonero.org/proposals/everoddandeven-monero-python-maintenance.html).
@@ -35,7 +35,7 @@ from monero import *
 
 # connect to daemon
 daemon: MoneroDaemon = MoneroDaemonRpc("http://localhost:38081", "superuser", "abctesting123")
-height: int = daemon.get_height(); # 1523651
+height: int = daemon.get_height() # 1523651
 txsInPool: list[MoneroTx] = daemon.get_tx_pool() # get transactions in the pool
 
 # create wallet from mnemonic phrase using Python bindings to monero-project
@@ -170,7 +170,7 @@ For example: `export LD_PRELOAD=/path/to/libjemalloc.a` then run your app.
     ```bash
     # With PIP
 
-    pip3 install pytest --break-system-packages
+    pip3 install pytest pytest-rerunfailures --break-system-packages
     ```
     ```bash
     # System-wide installation Ubuntu/Debian
@@ -185,14 +185,22 @@ For example: `export LD_PRELOAD=/path/to/libjemalloc.a` then run your app.
 2. Clone the project repository: 
     ```bash
     git clone --recurse-submodules https://github.com/everoddandeven/monero-python.git
+
+    cd monero-python
     ```
-3. `cd monero-python`
-4. Start RPC servers:
+3. Setup docker test environment
+    ```bash
+    docker compose -f tests/docker-compose.yml up -d node_1 node_2 xmr_wallet_1 xmr_wallet_2
+    ```
+4. Or start RPC servers locally:
 	1. Download and install [Monero CLI](https://web.getmonero.org/downloads/).
 	2. Start monerod, e.g.: `./monerod --stagenet` (or use a remote daemon).
 	3. Start monero-wallet-rpc, e.g.: `./monero-wallet-rpc --daemon-address http://localhost:38081 --stagenet --rpc-bind-port 38083 --rpc-login rpc_user:abc123 --wallet-dir ./`
-5. Configure the appropriate RPC endpoints, authentication, and other settings in [monero_test_utils.py](tests/utils/monero_test_utils.py).
-6. Run all *.py files in tests folder with `pytest`.
+5. Configure the appropriate RPC endpoints, authentication, and other settings in [config.ini](tests/config/config.ini).
+6. Run all python tests with:
+    ```bash
+    pytest
+    ```
 
 
 ## Related projects

pytest.ini

@@ -1,6 +1,9 @@
 [pytest]
 minversion = 6.0
-addopts = -ra -q
+addopts = -v
+log_level = INFO
+log_cli = True
 log_cli_level = INFO
+console_output_style = progress
 testpaths =
     tests

src/cpp/daemon/py_monero_daemon_model.cpp

@@ -340,7 +340,6 @@ void PyMoneroBlock::from_property_tree(const boost::property_tree::ptree& node,
 void PyMoneroBlock::from_property_tree(const boost::property_tree::ptree& node, const std::vector<uint64_t>& heights, std::vector<std::shared_ptr<monero::monero_block>>& blocks) {
   const auto& rpc_blocks = node.get_child("blocks");
   const auto& rpc_txs    = node.get_child("txs");
-
   if (rpc_blocks.size() != rpc_txs.size()) {
     throw std::runtime_error("blocks and txs size mismatch");
   }
@@ -358,18 +357,14 @@ void PyMoneroBlock::from_property_tree(const boost::property_tree::ptree& node,
     PyMoneroBlock::from_property_tree(block_n, block);
     block->m_height = heights.at(idx);
     blocks.push_back(block);
-
     std::vector<std::string> tx_hashes;
     if (auto hashes = it_block->second.get_child_optional("tx_hashes")) {
-      for (const auto& h : *hashes) {
-        tx_hashes.push_back(h.second.get_value<std::string>());
-      }
+      for (const auto& h : *hashes) tx_hashes.push_back(h.second.get_value<std::string>());
     }
 
     // build transactions
     std::vector<std::shared_ptr<monero::monero_tx>> txs;
     size_t tx_idx = 0;
-
     for (const auto& tx_node : it_txs->second) {
       auto tx = std::make_shared<monero::monero_tx>();
       tx->m_hash = tx_hashes.at(tx_idx++);
@@ -390,9 +385,7 @@ void PyMoneroBlock::from_property_tree(const boost::property_tree::ptree& node,
     // merge into one block
     block->m_txs.clear();
     for (auto& tx : txs) {
-      if (tx->m_block != boost::none) {
-        block->merge(block, tx->m_block.get());
-      }
+      if (tx->m_block != boost::none) block->merge(block, tx->m_block.get());
       else {
         tx->m_block = block;
         block->m_txs.push_back(tx);
@@ -401,7 +394,6 @@ void PyMoneroBlock::from_property_tree(const boost::property_tree::ptree& node,
   }
 }
 
-
 void PyMoneroOutput::from_property_tree(const boost::property_tree::ptree& node, const std::shared_ptr<monero_output>& output) {
   for (boost::property_tree::ptree::const_iterator it = node.begin(); it != node.end(); ++it) {
     std::string key = it->first;
@@ -712,10 +704,15 @@ void PyMoneroMiningStatus::from_property_tree(const boost::property_tree::ptree&
     std::string key = it->first;
     if (key == std::string("active")) status->m_is_active = it->second.get_value<bool>();
     else if (key == std::string("is_background_mining_enabled")) status->m_is_background = it->second.get_value<bool>();
-    else if (key == std::string("address")) status->m_address = it->second.data();
+    else if (key == std::string("address") && !it->second.data().empty()) status->m_address = it->second.data();
     else if (key == std::string("speed")) status->m_speed = it->second.get_value<uint64_t>();
     else if (key == std::string("threads_count")) status->m_num_threads = it->second.get_value<int>();
   }
+
+  if (status->m_is_active != boost::none && *status->m_is_active == false) {
+    status->m_is_background = boost::none;
+    status->m_address = boost::none;
+  }
 }
 
 std::vector<std::string> PyMoneroTxHashes::from_property_tree(const boost::property_tree::ptree& node) {
@@ -973,15 +970,9 @@ void PyMoneroDaemonInfo::from_property_tree(const boost::property_tree::ptree& n
     else if (key == std::string("height_without_bootstrap")) info->m_height_without_bootstrap = it->second.get_value<uint64_t>();
     else if (key == std::string("nettype")) {
       std::string nettype = it->second.data();
-      if (nettype == std::string("mainnet") || nettype == std::string("fakechain")) {
-        info->m_network_type = monero::monero_network_type::MAINNET;
-      }
-      else if (nettype == std::string("testnet")) {
-        info->m_network_type = monero::monero_network_type::TESTNET;
-      }
-      else if (nettype == std::string("stagenet")) {
-        info->m_network_type = monero::monero_network_type::STAGENET;
-      }
+      if (nettype == std::string("mainnet") || nettype == std::string("fakechain")) info->m_network_type = monero::monero_network_type::MAINNET;
+      else if (nettype == std::string("testnet")) info->m_network_type = monero::monero_network_type::TESTNET;
+      else if (nettype == std::string("stagenet")) info->m_network_type = monero::monero_network_type::STAGENET;
     }
     else if (key == std::string("offline")) info->m_is_offline = it->second.get_value<bool>();
     else if (key == std::string("incoming_connections_count")) info->m_num_incoming_connections = it->second.get_value<int>();

src/cpp/wallet/py_monero_wallet_model.cpp

@@ -1215,19 +1215,18 @@ rapidjson::Value PyMoneroWalletRelayTxParams::to_rapidjson_val(rapidjson::Docume
   return root;
 }
 
-PyMoneroSweepParams::PyMoneroSweepParams(const monero_tx_config& config) {
-  m_address = config.m_address;
-  m_account_index = config.m_account_index;
-  m_subaddr_indices = config.m_subaddress_indices;
-  m_key_image = config.m_key_image;
-  m_relay = config.m_relay;
-  m_priority = config.m_priority;
-  m_payment_id = config.m_payment_id;
-  m_below_amount = config.m_below_amount;
-  m_get_tx_key = true;
-  m_get_tx_hex = true;
-  m_get_tx_metadata = true;
-}
+PyMoneroSweepParams::PyMoneroSweepParams(const monero_tx_config& config):
+  m_address(config.m_address),
+  m_account_index(config.m_account_index),
+  m_subaddr_indices(config.m_subaddress_indices),
+  m_key_image(config.m_key_image),
+  m_relay(config.m_relay),
+  m_priority(config.m_priority),
+  m_payment_id(config.m_payment_id),
+  m_below_amount(config.m_below_amount),
+  m_get_tx_key(true),
+  m_get_tx_hex(true),
+  m_get_tx_metadata(true) { }
 
 rapidjson::Value PyMoneroSweepParams::to_rapidjson_val(rapidjson::Document::AllocatorType& allocator) const { 
   rapidjson::Value root(rapidjson::kObjectType);
@@ -1301,37 +1300,30 @@ rapidjson::Value PyMoneroImportExportKeyImagesParams::to_rapidjson_val(rapidjson
   return root;
 }
 
-PyMoneroCreateOpenWalletParams::PyMoneroCreateOpenWalletParams(const boost::optional<std::string>& filename, const boost::optional<std::string> &password) {
-  m_filename = filename;
-  m_password = password;
-}
+PyMoneroCreateOpenWalletParams::PyMoneroCreateOpenWalletParams(const boost::optional<std::string>& filename, const boost::optional<std::string> &password):
+  m_filename(filename), m_password(password) { }
 
-PyMoneroCreateOpenWalletParams::PyMoneroCreateOpenWalletParams(const boost::optional<std::string>& filename, const boost::optional<std::string> &password, const boost::optional<std::string> &language) {
-  m_filename = filename;
-  m_password = password;
-  m_language = language;
-}
+PyMoneroCreateOpenWalletParams::PyMoneroCreateOpenWalletParams(const boost::optional<std::string>& filename, const boost::optional<std::string> &password, const boost::optional<std::string> &language):
+  m_filename(filename), m_password(password), m_language(language) { }
 
-PyMoneroCreateOpenWalletParams::PyMoneroCreateOpenWalletParams(const boost::optional<std::string>& filename, const boost::optional<std::string> &password, const boost::optional<std::string> &seed, const boost::optional<std::string> &seed_offset, const boost::optional<uint64_t> &restore_height, const boost::optional<std::string> &language, const boost::optional<bool> &autosave_current, const boost::optional<bool> &enable_multisig_experimental) {
-  m_filename = filename;
-  m_password = password;
-  m_seed = seed;
-  m_seed_offset = seed_offset;
-  m_restore_height = restore_height;
-  m_language = language;
-  m_autosave_current = autosave_current;
-  m_enable_multisig_experimental = enable_multisig_experimental;
-}
+PyMoneroCreateOpenWalletParams::PyMoneroCreateOpenWalletParams(const boost::optional<std::string>& filename, const boost::optional<std::string> &password, const boost::optional<std::string> &seed, const boost::optional<std::string> &seed_offset, const boost::optional<uint64_t> &restore_height, const boost::optional<std::string> &language, const boost::optional<bool> &autosave_current, const boost::optional<bool> &enable_multisig_experimental): 
+  m_filename(filename),
+  m_password(password),
+  m_seed(seed),
+  m_seed_offset(seed_offset),
+  m_restore_height(restore_height),
+  m_language(language),
+  m_autosave_current(autosave_current),
+  m_enable_multisig_experimental(enable_multisig_experimental) { }
 
-PyMoneroCreateOpenWalletParams::PyMoneroCreateOpenWalletParams(const boost::optional<std::string>& filename, const boost::optional<std::string> &password, const boost::optional<std::string> &address, const boost::optional<std::string> &view_key, const boost::optional<std::string> &spend_key, const boost::optional<uint64_t> &restore_height, const boost::optional<bool> &autosave_current) {
-  m_filename = filename;
-  m_password = password;
-  m_address = address;
-  m_view_key = view_key;
-  m_spend_key = spend_key;
-  m_restore_height = restore_height;
-  m_autosave_current = autosave_current;
-}
+PyMoneroCreateOpenWalletParams::PyMoneroCreateOpenWalletParams(const boost::optional<std::string>& filename, const boost::optional<std::string> &password, const boost::optional<std::string> &address, const boost::optional<std::string> &view_key, const boost::optional<std::string> &spend_key, const boost::optional<uint64_t> &restore_height, const boost::optional<bool> &autosave_current):
+  m_filename(filename),
+  m_password(password),
+  m_address(address),
+  m_view_key(view_key),
+  m_spend_key(spend_key),
+  m_restore_height(restore_height),
+  m_autosave_current(autosave_current) { }
 
 rapidjson::Value PyMoneroCreateOpenWalletParams::to_rapidjson_val(rapidjson::Document::AllocatorType& allocator) const { 
   rapidjson::Value root(rapidjson::kObjectType);
@@ -1351,34 +1343,20 @@ rapidjson::Value PyMoneroCreateOpenWalletParams::to_rapidjson_val(rapidjson::Doc
   return root;
 }
 
-PyMoneroReserveProofParams::PyMoneroReserveProofParams(const std::string &message, bool all) {
-  m_all = all;
-  m_message = message;
-}
+PyMoneroReserveProofParams::PyMoneroReserveProofParams(const std::string &message, bool all):
+  m_all(all), m_message(message) { }
 
-PyMoneroReserveProofParams::PyMoneroReserveProofParams(const std::string &address, const std::string &message, const std::string &signature) {
-  m_address = address;
-  m_message = message;
-  m_signature = signature;
-}
+PyMoneroReserveProofParams::PyMoneroReserveProofParams(const std::string &address, const std::string &message, const std::string &signature): 
+  m_address(address), m_message(message), m_signature(signature) { }
 
-PyMoneroReserveProofParams::PyMoneroReserveProofParams(const std::string &tx_hash, const std::string &address, const std::string &message, const std::string &signature) {
-  m_tx_hash = tx_hash;
-  m_address = address;
-  m_message = message;
-  m_signature = signature;
-}
+PyMoneroReserveProofParams::PyMoneroReserveProofParams(const std::string &tx_hash, const std::string &address, const std::string &message, const std::string &signature):
+  m_tx_hash(tx_hash), m_address(address), m_message(message), m_signature(signature) { }
 
-PyMoneroReserveProofParams::PyMoneroReserveProofParams(const std::string &tx_hash, const std::string &message) {
-  m_tx_hash = tx_hash;
-  m_message = message;
-}
+PyMoneroReserveProofParams::PyMoneroReserveProofParams(const std::string &tx_hash, const std::string &message):
+  m_tx_hash(tx_hash), m_message(message) { }
 
-PyMoneroReserveProofParams::PyMoneroReserveProofParams(uint32_t account_index, uint64_t amount, const std::string &message) {
-  m_account_index = account_index;
-  m_amount = amount;
-  m_message = message;
-}
+PyMoneroReserveProofParams::PyMoneroReserveProofParams(uint32_t account_index, uint64_t amount, const std::string &message):
+  m_account_index(account_index), m_amount(amount), m_message(message) { }
 
 rapidjson::Value PyMoneroReserveProofParams::to_rapidjson_val(rapidjson::Document::AllocatorType& allocator) const { 
   rapidjson::Value root(rapidjson::kObjectType);

src/cpp/wallet/py_monero_wallet_rpc.cpp

@@ -602,8 +602,10 @@ monero_account PyMoneroWalletRpc::get_account(const uint32_t account_idx, bool i
 monero_account PyMoneroWalletRpc::get_account(const uint32_t account_idx, bool include_subaddresses, bool skip_balances) const {
   for(auto& account : monero::monero_wallet::get_accounts()) {
     if (account.m_index.get() == account_idx) {
-      std::vector<uint32_t> empty_indices;
-      if (include_subaddresses) account.m_subaddresses = get_subaddresses(account_idx, empty_indices, skip_balances);
+      if (include_subaddresses) {
+        std::vector<uint32_t> empty_indices;
+        account.m_subaddresses = get_subaddresses(account_idx, empty_indices, skip_balances);
+      }
       return account;
     }
   }
@@ -701,13 +703,11 @@ std::vector<monero_subaddress> PyMoneroWalletRpc::get_subaddresses(const uint32_
   auto response = m_rpc->send_json_request(request);
   if (response->m_result == boost::none) throw std::runtime_error("Invalid Monero JSONRPC response");
   auto node = response->m_result.get();
-
   std::vector<monero_subaddress> subaddresses;
 
   // initialize subaddresses
   for (auto it = node.begin(); it != node.end(); ++it) {
     std::string key = it->first;
-
     if (key == std::string("addresses")) {
       auto node2 = it->second;
       for (auto it2 = node2.begin(); it2 != node2.end(); ++it2) {
@@ -718,12 +718,10 @@ std::vector<monero_subaddress> PyMoneroWalletRpc::get_subaddresses(const uint32_
       }
       break;
     }
-
   }
 
   // fetch and initialize subaddress balances
   if (!skip_balances) {
-    
     // these fields are not initialized if subaddress is unused and therefore not returned from `get_balance`
     for (auto &subaddress : subaddresses) {
       subaddress.m_balance = 0;
@@ -737,7 +735,6 @@ std::vector<monero_subaddress> PyMoneroWalletRpc::get_subaddresses(const uint32_
     auto response2 = m_rpc->send_json_request(request);
     if (response2->m_result == boost::none) throw std::runtime_error("Invalid Monero JSONRPC response");
     auto node2 = response2->m_result.get();
-
     std::vector<std::shared_ptr<monero::monero_subaddress>> subaddresses2;
     PyMoneroSubaddress::from_rpc_property_tree(node2, subaddresses2);
 

tests/config/config.ini

@@ -0,0 +1,34 @@
+[general]
+test_non_relays=True
+lite_mode=False
+test_notifications=True
+network_type=mainnet
+auto_connect_timeout_ms=3000
+
+[daemon]
+rpc_uri=127.0.0.1:18081
+rpc_username=
+rpc_password=
+
+[wallet]
+name=test_wallet_1
+password=supersecretpassword123
+dir=./test_wallets
+language=English
+address=48W9YHwPzRz9aPTeXCA6kmSpW6HsvmWx578jj3of2gT3JwZzwTf33amESBoNDkL6SVK34Q2HTKqgYbGyE1hBws3wCrcBDR2
+private_view_key=e8c2288181bad9ec410d7322efd65f663c6da57bd1d1198636278a039743a600
+private_spend_key=be7a2f71097f146bdf0fb5bb8edfe2240a9767e15adee74d95af1b5a64f29a0c
+public_view_key=42e465bdcd00de50516f1c7049bbe26bd3c11195e8dae5cceb38bad92d484269
+public_spend_key=b58d33a1dac23d334539cbed3657b69a5c967d6860357e24ab4d11899a312a6b
+seed=vortex degrees outbreak teeming gimmick school rounded tonic observant injury leech ought problems ahead upcoming ledge textbook cigar atrium trash dunes eavesdrop dullness evolved vortex
+first_receive_height=171
+rpc_port_start=18082
+rpc_username=rpc_user
+rpc_password=abc123
+rpc_access_control_origins="http:#localhost:8080"
+rpc_domain=localhost
+rpc_zmq_enabled=False
+rpc_zmq_port_start=48083
+rpc_zmq_bind_port_start=48083
+rpc_zmq_domain=127.0.0.1
+sync_period_in_ms=5000

tests/config/test_monero_utils.ini

@@ -53,4 +53,5 @@ public_spend_key=3e48df9e9d8038dbf6f5382fac2becd8686273cda5bd87187e45dca7ec5af37
 invalid_private_view_key=5B8s3obCY2ETeQB3GNAGPK2zRGen5UeW1WzegSizVsmf6z5NvM2GLoN6zzk1vHyzGAAfA8pGhuYAeCFZjHAp59jRVQkunGS
 invalid_public_view_key=z86cf351d10894769feba29b9e201e12fb100b85bb52fc5825c864eef55c5840d
 invalid_private_spend_key=z86cf351d10894769feba29b9e201e12fb100b85bb52fc5825c864eef55c5840d
-invalid_public_spend_key=z86cf351d10894769feba29b9e201e12fb100b85bb52fc5825c864eef55c5840d
+invalid_public_spend_key=z86cf351d10894769feba29b9e201e12fb100b85bb52fc5825c864eef55c5840d
+seed=vortex degrees outbreak teeming gimmick school rounded tonic observant injury leech ought problems ahead upcoming ledge textbook cigar atrium trash dunes eavesdrop dullness evolved vortex

tests/test_monero_connection_manager.py

@@ -4,7 +4,7 @@
 from monero import (
     MoneroWallet, MoneroConnectionManager, MoneroRpcConnection, MoneroConnectionPollType
 )
-from utils import ConnectionChangeCollector, MoneroTestUtils as Utils
+from utils import ConnectionChangeCollector, TestUtils as Utils
 
 
 # TODO enable connection manager tests