diff --git a/.gitignore b/.gitignore index c1793cdcc..808248844 100644 --- a/.gitignore +++ b/.gitignore @@ -29,13 +29,6 @@ user_data/plot/ # Ignore Logo Folder Logo/ -# Ignore GitHub Secrets -GitHub-Secrets.txt - -# Ignore fthypt and legacy pickle Files -*.fthypt -*.pickle - # Installer logs pip-log.txt pip-delete-this-directory.txt @@ -59,25 +52,3 @@ test-results*.xml # Ignore pytest cache files .pytest_cache/ - -# Ignore freqtrade installation -.env/ -build_helpers/ -config_examples/ -docker/ -!docker/Dockerfile.MoniGoMani -docs/ -freqtrade/ -junit/ -scripts/ -tests/junit/ - -freqtrade.service -freqtrade.service.watchdog -freqtrade.egg-info -MANIFEST.in -pyproject.toml -pyproject.yml -setup.cfg -setup.py -setup.sh diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..cb3e9a71f --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "freqtrade"] + path = freqtrade + url = git@github.com:freqtrade/freqtrade.git diff --git a/.hurry b/.hurry index dc2162d6d..bf1cd880a 100644 --- a/.hurry +++ b/.hurry @@ -1,16 +1,11 @@ config: username: MoniGoMani Community exchange: binance - ft_binary: source ./.env/bin/activate; freqtrade + install_type: source hyperopt: epochs: 1000 loss: MGM_WinRatioAndProfitRatioHyperOptLoss stake_currency: USDT spaces: buy sell strategy: MoniGoManiHyperStrategy - install_type: source - mgm_config_names: - mgm-config: mgm-config.json - mgm-config-hyperopt: mgm-config-hyperopt.json - mgm-config-private: mgm-config-private.json timerange: 20210501-20210616 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..9d60d2bd8 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,30 @@ +--- +version: '3' +services: + freqtrade: + image: fq:mgm-recommended + # image: freqtradeorg/freqtrade:stable + # image: freqtradeorg/freqtrade:develop + # Use plotting image + # image: freqtradeorg/freqtrade:develop_plot + # Build step - only needed when additional dependencies are needed + # build: + # context: . + # dockerfile: "./docker/Dockerfile.custom" + restart: unless-stopped + container_name: freqtrade + volumes: + - "./user_data:/freqtrade/user_data" + # Expose api on port 8080 (localhost only) + # Please read the https://www.freqtrade.io/en/stable/rest-api/ documentation + # before enabling this. + ports: + # - "127.0.0.1:8080:8080" + - "127.0.0.1:8080:8080" # Unsafe + # Default command used when running `docker compose up` + command: > + trade + --logfile /freqtrade/user_data/logs/freqtrade.log + --db-url sqlite:////freqtrade/user_data/tradesv3.sqlite + --config /freqtrade/user_data/config.json + --strategy SampleStrategy diff --git a/freqtrade b/freqtrade new file mode 160000 index 000000000..3503fdb4e --- /dev/null +++ b/freqtrade @@ -0,0 +1 @@ +Subproject commit 3503fdb4ec31be99f433fdce039543e0911964d6 diff --git a/mgm-hurry b/mgm-hurry index f0b36ab9b..33fcb7a7c 100755 --- a/mgm-hurry +++ b/mgm-hurry @@ -20,6 +20,7 @@ import glob import json import logging import os +import shutil import sys from datetime import datetime from string import Template @@ -69,6 +70,7 @@ class MGMHurry: self.freqtrade_cli = FreqtradeCli(self.basedir) self.monigomani_cli = MoniGoManiCli(self.basedir) + # TODO redo. def version(self): if self.freqtrade_cli.install_type == 'source': if self.freqtrade_cli.installation_exists(silent=True): @@ -82,7 +84,6 @@ class MGMHurry: self.monigomani_cli.run_command('git log -1; echo "";') else: self.logger.warning(Color.yellow('"No Freqtrade installation detected!')) - else: self.logger.warning(Color.yellow('"version" command currently only supported on "source" installations.')) @@ -577,8 +578,9 @@ class MGMHurry: self.logger.info(f'👉 Downloading candle data ({tickers}) for timerange ({timerange})') - command = (f'{self.monigomani_config.config["ft_binary"]} download-data --timerange {timerange} ' - f'-t {tickers} {self.monigomani_config.command_configs()}') + command = f'{self.monigomani_config.get_freqtrade_cmd()} download-data' + command += f' --timerange {timerange}' + command += f' -t {tickers} {self.monigomani_config.command_configs()}' if self.monigomani_config.config['exchange'] == 'kraken': command += ' --dl-trades' @@ -698,30 +700,24 @@ class MGMHurry: if spaces is None: spaces = self.monigomani_config.config['hyperopt']['spaces'] - command = (f'$ft_binary hyperopt -s $ho_strategy {self.monigomani_config.command_configs()}' - f'--hyperopt-loss $ho_loss --spaces $ho_spaces -e $ho_epochs --timerange $timerange ') + command = f'{self.monigomani_config.get_freqtrade_cmd()} hyperopt' + command += f' {self.monigomani_config.command_configs()}' + command += f' --hyperopt-loss {strategy}' + command += f' --spaces {spaces}' + command += f' -e {epochs}' + command += f' --timerange {timerange}' if enable_protections is True: - command = f'{command.strip()} --enable-protections ' + command = f' {command.strip()} --enable-protections' if random_state is not None: - command = f'{command.strip()} --random-state $random_state ' + command = f' {command.strip()} --random-state {random_state}' if jobs is not None: - command = f'{command.strip()} -j $jobs ' + command = f' {command.strip()} -j {jobs}' if min_trades is not None: - command = f'{command.strip()} --min-trades $min_trades ' - - command = Template(command).substitute( - ft_binary=self.monigomani_config.config['ft_binary'], - ho_strategy=strategy, - ho_loss=loss, - ho_spaces=spaces, - ho_epochs=epochs, - timerange=timerange, - random_state=random_state, - jobs=jobs, - min_trades=min_trades) + command = f' {command.strip()} --min-trades {min_trades}' self.logger.debug(command) + if output_file_name is None: output_file_name = f'{strategy}-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}' hyperopt_file_name = f'HyperOptResults-{output_file_name}' @@ -794,7 +790,7 @@ class MGMHurry: best = '--best' if only_best is True else '' profit = '--profitable' if only_profitable is True else '' - command = (f'{self.monigomani_config.config["ft_binary"]} hyperopt-list ' + command = (f'{self.monigomani_config.get_freqtrade_cmd()} hyperopt-list ' f'--hyperopt-filename "{choice}" {best} {profit}') self.logger.debug(command) @@ -823,7 +819,7 @@ class MGMHurry: self.logger.info(f'👉 Showing {"" if apply is False else "and applying "}HyperOpt results for epoch #{epoch}') - command = (f'{self.monigomani_config.config["ft_binary"]} hyperopt-show -n {epoch} ' + command = (f'{self.monigomani_config.get_freqtrade_cmd()} hyperopt-show -n {epoch} ' f'{self.monigomani_config.command_configs()}') if fthypt_name is not None: @@ -857,7 +853,7 @@ class MGMHurry: self.logger.info(Color.title('👉 Start BackTesting. Lets see how it all turns out!')) timerange = self.monigomani_config.get_preset_timerange(timerange) - command = (f'$ft_binary backtesting -s $ho_strategy {self.monigomani_config.command_configs()}' + command = (f'{self.monigomani_config.get_freqtrade_cmd()} backtesting -s $ho_strategy {self.monigomani_config.command_configs()}' f'--timerange $timerange') if timerange is None: @@ -950,7 +946,7 @@ class MGMHurry: output_file_path = f'{self.basedir}/user_data/plot/{plot_profit_file_name}.html' self.monigomani_cli.run_command( - f'{self.monigomani_config.config["ft_binary"]} plot-profit {self.monigomani_config.command_configs()}' + f'{self.monigomani_config.get_freqtrade_cmd()} plot-profit {self.monigomani_config.command_configs()}' f'--export-filename {backtest_results_path} --timerange {timerange} --timeframe {timeframe} && ' f'mv {self.basedir}/user_data/plot/freqtrade-profit-plot.html {output_file_path}') @@ -1098,6 +1094,44 @@ class MGMHurry: message=f'🚀 Fresh **{strategy}** SpreadSheet (.csv) Results ⬇️', results_paths=[output_file_path]) + def use_configuration(self, config_ho: str = None, config: str = None, config_private: str = None, override: bool = False) -> None: + """ + Configure the bot with the given configuration. Use this to quickly move between different configurations + stored in your filesystem. + + :param config_ho: (str) + :param config: (str, Optional) + :param config_private: (str, Optional) + :param override: (bool) + """ + self.logger.info(Color.title('👉 Configuring the bot with strategy:')) + + if not override: + self.logger.error('Impossible to use provided configuration without overridden the present one.' + ' Include the `--override` flag if you wish to override the current configuration.') + return + + if os.path.isfile(config_ho): + self.logger.info(f'Using config-hyperopt from: {config_ho}') + + shutil.copy(config_ho, f'{self.basedir}/user_data/mgm-config-hyperopt.json') + elif config_ho is not None: + self.logger.error(f'Can\'t locate config-hyperopt file from: {config_ho}') + + if os.path.isfile(config): + self.logger.info(f'Using config from: {config}') + + shutil.copy(config, f'{self.basedir}/user_data/mgm-config.json') + elif config is not None: + self.logger.error(f'Can\'t locate config file from: {config}') + + if os.path.isfile(config_private): + self.logger.info(f'Using config-private from: {config_private}') + + shutil.copy(config_private, f'{self.basedir}/user_data/mgm-config-private.json') + elif config_private is not None: + self.logger.error(f'Can\'t locate config-private file from: {config_private}') + def start_trader(self, dry_run: bool = True): """ Start the trader. Your ultimate goal! @@ -1107,8 +1141,9 @@ class MGMHurry: self.logger.info(Color.title(f'👉 Starting the trader, ' f'{"in Dry-Run Mode!" if dry_run else "in Live-Run Mode, fingers crossed! 🤞"}')) - command = (f'{self.monigomani_config.config["ft_binary"]} trade {self.monigomani_config.command_configs()}' - f'--strategy {self.monigomani_config.config["hyperopt"]["strategy"]}') + command = f'{self.monigomani_config.get_freqtrade_cmd()} trade' + command += f' {self.monigomani_config.command_configs()}' + command += f' --strategy {self.monigomani_config.config["hyperopt"]["strategy"]}' if dry_run is True: command += ' --dry-run' diff --git a/requirements-mgm-dev.txt b/requirements-mgm-dev.txt new file mode 100644 index 000000000..0cdaaf228 --- /dev/null +++ b/requirements-mgm-dev.txt @@ -0,0 +1,2 @@ +-r requirements-mgm.txt +pytest==6.2.5 diff --git a/user_data/mgm-config-private.example.json b/strategy-config/mgm-config-private.example.json similarity index 100% rename from user_data/mgm-config-private.example.json rename to strategy-config/mgm-config-private.example.json diff --git a/user_data/mgm-config.example.json b/strategy-config/mgm-config.example.json similarity index 100% rename from user_data/mgm-config.example.json rename to strategy-config/mgm-config.example.json diff --git a/user_data/mgm_tools/mgm_hurry/FreqtradeCli.py b/user_data/mgm_tools/mgm_hurry/FreqtradeCli.py index 86e0d9384..ef2ddf9ce 100644 --- a/user_data/mgm_tools/mgm_hurry/FreqtradeCli.py +++ b/user_data/mgm_tools/mgm_hurry/FreqtradeCli.py @@ -13,6 +13,7 @@ # \_| |_| \___| \__, | \__||_| \__,_| \__,_| \___| \____/|_||_| # | | # |_| +import shutil import distro import glob @@ -129,27 +130,36 @@ def installation_exists(self, silent: bool = False) -> bool: return False # Well if install_type is docker, we return True because we don't verify if docker is installed - if self.install_type == 'docker': + if self.install_type == 'docker-compose': if silent is False: self.cli_logger.debug('FreqtradeCli - installation_exists() succeeded because ' 'install_type is set to docker.') return True - if self.freqtrade_binary is None: - if silent is False: - self.cli_logger.warning(Color.yellow('FreqtradeCli - installation_exists() failed. ' - 'No freqtrade_binary.')) - return False - if self.install_type == 'source': if silent is False: self.cli_logger.debug('FreqtradeCli - installation_exists() install_type is "source".') - if os.path.exists(f'{self.basedir}/.env/bin/freqtrade'): + + if os.path.isfile('freqtrade/.env/bin/freqtrade'): return True if silent is False: self.cli_logger.warning(Color.yellow(f'FreqtradeCli - installation_exists() failed. Freqtrade binary ' - f'not found in {self.basedir}/.env/bin/freqtrade.')) + f'not found.')) + + if self.install_type == 'custom' and self.freqtrade_binary: + if os.path.isfile(self.freqtrade_binary): + if silent is False: + self.cli_logger.debug('FreqtradeCli - installation_exists() succeeded because ' + 'install_type is set to custom.') + else: + if silent is False: + self.cli_logger.warning(Color.yellow(f'FreqtradeCli - installation_exists() failed. Freqtrade binary ' + f'not found.')) + + if silent is False: + self.cli_logger.warning(Color.yellow('FreqtradeCli - installation_exists() failed. ' + 'No freqtrade_binary.')) return False diff --git a/user_data/mgm_tools/mgm_hurry/MoniGoManiConfig.py b/user_data/mgm_tools/mgm_hurry/MoniGoManiConfig.py index 1bd3f7c46..187f46765 100644 --- a/user_data/mgm_tools/mgm_hurry/MoniGoManiConfig.py +++ b/user_data/mgm_tools/mgm_hurry/MoniGoManiConfig.py @@ -206,6 +206,18 @@ def read_hurry_config(self) -> dict: return hurry_config + def get_freqtrade_cmd(self): + if self.config['install_type'] == 'docker-compose': + cmd = 'docker-compose run --rm freqtrade' + + return cmd + elif self.config['install_type'] == 'source': + cmd = f'{self.basedir}/freqtrade/.env/bin/freqtrade' + + return cmd + elif self.config['install_type'] == 'custom': + raise "TODO install_type 'custom' unsupported" + def get_config_filepath(self, cfg_key: str) -> str: """ Transforms given cfg_key into the corresponding absolute config filepath. @@ -476,13 +488,12 @@ def save_weak_strong_signal_overrides(self) -> bool: def command_configs(self) -> str: """ - Returns a string with the 'mgm-config' & 'mgm-config-private' names loaded from '.hurry' + Returns a string with the 'mgm-config' & 'mgm-config-private' file paths ready to implement in a freqtrade command. :return str: String with 'mgm-config' & 'mgm-config-private' for a freqtrade command """ - mgm_json_name = self.config['mgm_config_names']['mgm-config'] - mgm_private_json_name = self.config['mgm_config_names']['mgm-config-private'] - return f'-c ./user_data/{mgm_json_name} -c ./user_data/{mgm_private_json_name} ' + + return '-c ./user_data/mgm-config.json -c ./user_data/mgm-config-private.json' def get_preset_timerange(self, timerange: str) -> str: """ diff --git a/user_data/strategies/MasterMoniGoManiHyperStrategy.py b/user_data/strategies/MasterMoniGoManiHyperStrategy.py index 32b65d266..2d70fc10c 100644 --- a/user_data/strategies/MasterMoniGoManiHyperStrategy.py +++ b/user_data/strategies/MasterMoniGoManiHyperStrategy.py @@ -9,6 +9,7 @@ from abc import ABC from datetime import datetime, timedelta from functools import reduce +from pprint import pprint from typing import Any, Dict, List, Optional, Union import numpy as np # noqa @@ -32,7 +33,6 @@ # --- ↑ Do not remove these libs ↑ ------------------------------------------------------------------------------------- - class MasterMoniGoManiHyperStrategy(IStrategy, ABC): """ #################################################################################### @@ -67,30 +67,15 @@ class MasterMoniGoManiHyperStrategy(IStrategy, ABC): buy_params = {} sell_params = {} - # Load the MoniGoMani config names from '.hurry' - mgm_config_name = mgm_config_hyperopt_name = None - hurry_config_path = f'{os.getcwd()}/.hurry' - if os.path.isfile(hurry_config_path) is True: - with open(hurry_config_path, 'r') as yml_file: - config = full_load(yml_file) or {} - - if 'config' in config: - hurry_config = config['config'] - mgm_config_name = hurry_config['mgm_config_names']['mgm-config'] - mgm_config_hyperopt_name = hurry_config['mgm_config_names']['mgm-config-hyperopt'] - - if (mgm_config_name is None) or (mgm_config_hyperopt_name is None): - sys.exit('MoniGoManiHyperStrategy - ERROR - The MoniGoMani Configuration filenames could not be loaded from' - '".hurry"... Please run "python3 ./mgm-hurry setup" to create your ".hurry" file') + mgm_config_path = './user_data/mgm-config.json' + mgm_config_hyperopt_path = './user_data/mgm-config-hyperopt.json' # Load the MoniGoMani settings - mgm_config_path = f'{os.getcwd()}/user_data/{mgm_config_name}' - if os.path.isfile(mgm_config_path) is True: - # Load the 'mgm-config.json' file as an object and parse it as a dictionary - file_object = open(mgm_config_path, ) - json_data = json.load(file_object) - mgm_config = json_data['monigomani_settings'] - else: + try: + with open(mgm_config_path, 'r') as f: + mgm_config_object = json.load(f) + mgm_config = mgm_config_object['monigomani_settings'] + except IOError: sys.exit(f'MoniGoManiHyperStrategy - ERROR - The main MoniGoMani configuration file "mgm-config" can\'t ' f'be found at: {mgm_config_path}... Please provide the correct file and/or alter "mgm_config_name" in ' f'".hurry"') @@ -153,45 +138,41 @@ class MasterMoniGoManiHyperStrategy(IStrategy, ABC): f': \nhttps://github.com/Rikj000/MoniGoMani/blob/development/user_data/mgm-config.example.json') # If results from a previous HyperOpt Run are found then continue the next HyperOpt Run upon them - mgm_config_hyperopt_path = f'{os.getcwd()}/user_data/{mgm_config_hyperopt_name}' - if os.path.isfile(mgm_config_hyperopt_path) is True: - # Try to load the previous 'mgm-config-hyperopt' file as an object and parse it as a dictionary - # if the parse fails, warn and continue as if it didn't exist. - try: - file_object = open(mgm_config_hyperopt_path, ) - mgm_config_hyperopt = json.load(file_object) - except ValueError as e: - mgm_config_hyperopt = {} - logger.warning(f'MoniGoManiHyperStrategy - WARN - {mgm_config_hyperopt_path} is inaccessible or is ' - f'not valid JSON, disregarding existing "mgm-config-hyperopt" file and ' - f'treating as first hyperopt run!') - - # Convert the loaded 'mgm-config-hyperopt' data to the needed HyperOpt Results format if it's found - # Default stub values from 'mgm-config' are used otherwise. - if mgm_config_hyperopt != {}: - for space in mgm_config_hyperopt['params']: - if space in ['buy', 'sell']: - for param, param_value in mgm_config_hyperopt['params'][space].items(): - if param.startswith('buy'): - buy_params[param] = param_value - else: - sell_params[param] = param_value - - if space == 'roi': - minimal_roi = mgm_config_hyperopt['params'][space] - - if space == 'stoploss': - stoploss = mgm_config_hyperopt['params'][space][space] - - if space == 'trailing': - trailing_stop = mgm_config_hyperopt['params'][space]['trailing_stop'] - trailing_stop_positive = mgm_config_hyperopt['params'][space]['trailing_stop_positive'] - trailing_stop_positive_offset = mgm_config_hyperopt[ - 'params'][space]['trailing_stop_positive_offset'] - trailing_only_offset_is_reached = mgm_config_hyperopt[ - 'params'][space]['trailing_only_offset_is_reached'] - else: - mgm_config_hyperopt = {} + try: + with open(mgm_config_hyperopt_path) as f: + mgm_config_hyperopt = json.load(f) + except ValueError: + mgm_config_hyperopt = None + logger.warning(f'MoniGoManiHyperStrategy - WARN - {mgm_config_hyperopt_path} is inaccessible or is ' + f'not valid JSON, disregarding existing "mgm-config-hyperopt" file and ' + f'treating as first hyperopt run!') + except IOError: + mgm_config_hyperopt = None + + # Convert the loaded 'mgm-config-hyperopt' data to the needed HyperOpt Results format if it's found + # Default stub values from 'mgm-config' are used otherwise. + if mgm_config_hyperopt is not None: + for space in mgm_config_hyperopt['params']: + if space in ['buy', 'sell']: + for param, param_value in mgm_config_hyperopt['params'][space].items(): + if param.startswith('buy'): + buy_params[param] = param_value + else: + sell_params[param] = param_value + + if space == 'roi': + minimal_roi = mgm_config_hyperopt['params'][space] + + if space == 'stoploss': + stoploss = mgm_config_hyperopt['params'][space][space] + + if space == 'trailing': + trailing_stop = mgm_config_hyperopt['params'][space]['trailing_stop'] + trailing_stop_positive = mgm_config_hyperopt['params'][space]['trailing_stop_positive'] + trailing_stop_positive_offset = mgm_config_hyperopt[ + 'params'][space]['trailing_stop_positive_offset'] + trailing_only_offset_is_reached = mgm_config_hyperopt[ + 'params'][space]['trailing_only_offset_is_reached'] # Create dictionary to store custom information MoniGoMani will be using in RAM initial_custom_info: dict = {'open_trades': {}, 'unclogger_cooldown_pairs': {}}