From 1e4920059650cba27d91999dfc311de0a21af07b Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Sun, 7 Jan 2024 20:16:44 +0900 Subject: [PATCH 01/43] early stage of marketcapfilter --- freqtrade/constants.py | 7 +- freqtrade/plugins/pairlist/MarketCapFilter.py | 119 ++++++++++++++++++ 2 files changed, 123 insertions(+), 3 deletions(-) create mode 100644 freqtrade/plugins/pairlist/MarketCapFilter.py diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 71081433e..d7767dc2e 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -33,9 +33,10 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss', 'MaxDrawDownHyperOptLoss', 'MaxDrawDownRelativeHyperOptLoss', 'ProfitDrawDownHyperOptLoss'] AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'ProducerPairList', 'RemotePairList', - 'AgeFilter', "FullTradesFilter", 'OffsetFilter', 'PerformanceFilter', - 'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter', - 'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter'] + 'AgeFilter', "FullTradesFilter", 'MarketCapFilter', 'OffsetFilter', + 'PerformanceFilter', 'PrecisionFilter', 'PriceFilter', + 'RangeStabilityFilter', 'ShuffleFilter', 'SpreadFilter', + 'VolatilityFilter'] AVAILABLE_PROTECTIONS = ['CooldownPeriod', 'LowProfitPairs', 'MaxDrawdown', 'StoplossGuard'] AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5', 'feather', 'parquet'] diff --git a/freqtrade/plugins/pairlist/MarketCapFilter.py b/freqtrade/plugins/pairlist/MarketCapFilter.py new file mode 100644 index 000000000..6238b6211 --- /dev/null +++ b/freqtrade/plugins/pairlist/MarketCapFilter.py @@ -0,0 +1,119 @@ +""" +Market Cap PairList provider + +Provides dynamic pair list based on Market Cap +""" +import logging +from datetime import timedelta +from typing import Any, Dict, List, Literal + +from cachetools import TTLCache + +from freqtrade.constants import Config, ListPairsWithTimeframes +from freqtrade.exceptions import OperationalException +from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date +from freqtrade.exchange.types import Tickers +from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter +from freqtrade.util import dt_now, format_ms_time + +from pycoingecko import CoinGeckoAPI + +logger = logging.getLogger(__name__) + + +SORT_VALUES = ['quoteVolume'] + + +class MarketCapFilter(IPairList): + + is_pairlist_generator = True + + def __init__(self, exchange, pairlistmanager, + config: Config, pairlistconfig: Dict[str, Any], + pairlist_pos: int) -> None: + super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) + + if 'max_rank' not in self._pairlistconfig: + raise OperationalException( + '`max_rank` not specified. Please check your configuration ' + 'for "pairlist.config.max_rank"') + + self._stake_currency = config['stake_currency'] + self._max_rank = self._pairlistconfig['max_rank'] + self._refresh_period = self._pairlistconfig.get('refresh_period', 86400) + self._marketcap_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period) + self._def_candletype = self._config['candle_type_def'] + self._coingekko: CoinGeckoAPI = CoinGeckoAPI() + + if self._max_rank > 250: + raise OperationalException( + "This filter only support up to rank 250." + ) + + + @property + def needstickers(self) -> bool: + """ + Boolean property defining if tickers are necessary. + If no Pairlist requires tickers, an empty Dict is passed + as tickers argument to filter_pairlist + """ + return False + + def _validate_keys(self, key): + return key in SORT_VALUES + + def short_desc(self) -> str: + """ + Short whitelist method description - used for startup-messages + """ + return f"{self.name} - Only use top {self._pairlistconfig['max_rank']} market cap pairs." + + @staticmethod + def description() -> str: + return "Filter pair list based on market cap." + + @staticmethod + def available_parameters() -> Dict[str, PairlistParameter]: + return { + "max_rank": { + "type": "number", + "default": 30, + "description": "Max market cap rank", + "help": "Only use assets that ranked within top max_rank market cap", + }, + "refresh_period": { + "type": "number", + "default": 86400, + "description": "Refresh period", + "help": "Refresh period in seconds", + } + } + + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: + """ + Filters and sorts pairlist and returns the whitelist again. + Called on each bot iteration - please use internal caching if necessary + :param pairlist: pairlist to filter or sort + :param tickers: Tickers (from exchange.get_tickers). May be cached. + :return: new whitelist + """ + marketcap_list = self._marketcap_cache.get('marketcap') + can_filter = False + + if marketcap_list: + can_filter = True + else: + data = self._coingekko.get_coins_markets(vs_currencies='usd', order='market_cap_desc', + per_page='250', page='1', sparkline='false', + locale='en') + if data: + pairs_data = [] + for row in data: + pairs_data.append(row['symbol']) + + if len(pairs_data) > 0: + logger.info(pairs_data) + self._marketcap_cache['marketcap'] = pairs_data + + return pairlist From 70cc2942d7632d1ee1af02affba44b862850a8c4 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Sun, 7 Jan 2024 23:10:18 +0900 Subject: [PATCH 02/43] wrong arg name --- freqtrade/plugins/pairlist/MarketCapFilter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/plugins/pairlist/MarketCapFilter.py b/freqtrade/plugins/pairlist/MarketCapFilter.py index 6238b6211..a98f435bb 100644 --- a/freqtrade/plugins/pairlist/MarketCapFilter.py +++ b/freqtrade/plugins/pairlist/MarketCapFilter.py @@ -104,7 +104,7 @@ class MarketCapFilter(IPairList): if marketcap_list: can_filter = True else: - data = self._coingekko.get_coins_markets(vs_currencies='usd', order='market_cap_desc', + data = self._coingekko.get_coins_markets(vs_currency='usd', order='market_cap_desc', per_page='250', page='1', sparkline='false', locale='en') if data: From 2eb3b73a934527c04f7ce68ff2f604cced24d594 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Sun, 7 Jan 2024 23:22:58 +0900 Subject: [PATCH 03/43] cut the list to top x --- freqtrade/plugins/pairlist/MarketCapFilter.py | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/freqtrade/plugins/pairlist/MarketCapFilter.py b/freqtrade/plugins/pairlist/MarketCapFilter.py index a98f435bb..4650a8c35 100644 --- a/freqtrade/plugins/pairlist/MarketCapFilter.py +++ b/freqtrade/plugins/pairlist/MarketCapFilter.py @@ -108,12 +108,22 @@ class MarketCapFilter(IPairList): per_page='250', page='1', sparkline='false', locale='en') if data: - pairs_data = [] + marketcap_list = [] for row in data: - pairs_data.append(row['symbol']) + marketcap_list.append(row['symbol']) + + if len(marketcap_list) > 0: + self._marketcap_cache['marketcap'] = marketcap_list + can_filter = True + + + if can_filter: + filtered_pairlist = [] + top_marketcap = marketcap_list[:self._max_rank:] + logger.info(top_marketcap) + logger.info(len(top_marketcap)) + + # for pair in pairlist: - if len(pairs_data) > 0: - logger.info(pairs_data) - self._marketcap_cache['marketcap'] = pairs_data return pairlist From 1cd08c96a1030bf870e875d80695acca405c1245 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Sun, 7 Jan 2024 23:30:11 +0900 Subject: [PATCH 04/43] filter the pairlist --- freqtrade/plugins/pairlist/MarketCapFilter.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/freqtrade/plugins/pairlist/MarketCapFilter.py b/freqtrade/plugins/pairlist/MarketCapFilter.py index 4650a8c35..c922402cd 100644 --- a/freqtrade/plugins/pairlist/MarketCapFilter.py +++ b/freqtrade/plugins/pairlist/MarketCapFilter.py @@ -120,10 +120,14 @@ class MarketCapFilter(IPairList): if can_filter: filtered_pairlist = [] top_marketcap = marketcap_list[:self._max_rank:] - logger.info(top_marketcap) - logger.info(len(top_marketcap)) - # for pair in pairlist: + for pair in pairlist: + base = pair.split('/')[0] + if base.lower() in top_marketcap: + filtered_pairlist.append(pair) + + if len(filtered_pairlist) > 0: + return filtered_pairlist return pairlist From a5862dcc4a47d38ad0222374895c8507fd6598d8 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Sun, 7 Jan 2024 23:33:34 +0900 Subject: [PATCH 05/43] add log message --- freqtrade/plugins/pairlist/MarketCapFilter.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/plugins/pairlist/MarketCapFilter.py b/freqtrade/plugins/pairlist/MarketCapFilter.py index c922402cd..0d28671ef 100644 --- a/freqtrade/plugins/pairlist/MarketCapFilter.py +++ b/freqtrade/plugins/pairlist/MarketCapFilter.py @@ -125,6 +125,9 @@ class MarketCapFilter(IPairList): base = pair.split('/')[0] if base.lower() in top_marketcap: filtered_pairlist.append(pair) + else: + logger.info(f"Remove {pair} from whitelist because it's not in " + f"top {self._max_rank} market cap") if len(filtered_pairlist) > 0: return filtered_pairlist From e10ab8c543afa905879a74356a0e993d39eda325 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Mon, 8 Jan 2024 00:04:58 +0900 Subject: [PATCH 06/43] test gen_pairlist --- freqtrade/plugins/pairlist/MarketCapFilter.py | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/freqtrade/plugins/pairlist/MarketCapFilter.py b/freqtrade/plugins/pairlist/MarketCapFilter.py index 0d28671ef..5b48ed2f5 100644 --- a/freqtrade/plugins/pairlist/MarketCapFilter.py +++ b/freqtrade/plugins/pairlist/MarketCapFilter.py @@ -90,6 +90,41 @@ class MarketCapFilter(IPairList): } } + def gen_pairlist(self, tickers: Tickers) -> List[str]: + """ + Generate the pairlist + :param tickers: Tickers (from exchange.get_tickers). May be cached. + :return: List of pairs + """ + # Generate dynamic whitelist + # Must always run if this pairlist is not the first in the list. + pairlist = self._marketcap_cache.get('pairlist_mc') + if pairlist: + # Item found - no refresh necessary + return pairlist.copy() + else: + # Use fresh pairlist + # Check if pair quote currency equals to the stake currency. + _pairlist = [k for k in self._exchange.get_markets( + quote_currencies=[self._stake_currency], + tradable_only=True, active_only=True).keys()] + # No point in testing for blacklisted pairs... + _pairlist = self.verify_blacklist(_pairlist, logger.info) + # if not self._use_range: + # filtered_tickers = [ + # v for k, v in tickers.items() + # if (self._exchange.get_pair_quote_currency(k) == self._stake_currency + # and (self._use_range or v.get(self._sort_key) is not None) + # and v['symbol'] in _pairlist)] + # pairlist = [s['symbol'] for s in filtered_tickers] + # else: + # pairlist = _pairlist + + pairlist = self.filter_pairlist(_pairlist, tickers) + self._marketcap_cache['pairlist_mc'] = pairlist.copy() + + return pairlist + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ Filters and sorts pairlist and returns the whitelist again. From ef528fa69c3e9eb4387dac55eea3c620ae243efa Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Mon, 8 Jan 2024 00:14:55 +0900 Subject: [PATCH 07/43] option B --- freqtrade/plugins/pairlist/MarketCapFilter.py | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/freqtrade/plugins/pairlist/MarketCapFilter.py b/freqtrade/plugins/pairlist/MarketCapFilter.py index 5b48ed2f5..bd58a7a11 100644 --- a/freqtrade/plugins/pairlist/MarketCapFilter.py +++ b/freqtrade/plugins/pairlist/MarketCapFilter.py @@ -154,15 +154,25 @@ class MarketCapFilter(IPairList): if can_filter: filtered_pairlist = [] - top_marketcap = marketcap_list[:self._max_rank:] - - for pair in pairlist: - base = pair.split('/')[0] - if base.lower() in top_marketcap: - filtered_pairlist.append(pair) - else: - logger.info(f"Remove {pair} from whitelist because it's not in " - f"top {self._max_rank} market cap") + + # option A + # top_marketcap = marketcap_list[:self._max_rank:] + + # for pair in pairlist: + # base = pair.split('/')[0] + # if base.lower() in top_marketcap: + # filtered_pairlist.append(pair) + # else: + # logger.info(f"Remove {pair} from whitelist because it's not in " + # f"top {self._max_rank} market cap") + + # option B + for mc_pair in marketcap_list: + test_pair = f"{mc_pair.upper()}/{self._stake_currency.upper}" + if test_pair in pairlist: + filtered_pairlist.append(test_pair) + if len(filtered_pairlist) == self._max_rank: + break if len(filtered_pairlist) > 0: return filtered_pairlist From adf07dd3e133476aa255403ab96e66cb29bf3429 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Mon, 8 Jan 2024 00:16:16 +0900 Subject: [PATCH 08/43] remove cache for noe --- freqtrade/plugins/pairlist/MarketCapFilter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/plugins/pairlist/MarketCapFilter.py b/freqtrade/plugins/pairlist/MarketCapFilter.py index bd58a7a11..a3b794c6f 100644 --- a/freqtrade/plugins/pairlist/MarketCapFilter.py +++ b/freqtrade/plugins/pairlist/MarketCapFilter.py @@ -98,7 +98,7 @@ class MarketCapFilter(IPairList): """ # Generate dynamic whitelist # Must always run if this pairlist is not the first in the list. - pairlist = self._marketcap_cache.get('pairlist_mc') + # pairlist = self._marketcap_cache.get('pairlist_mc') if pairlist: # Item found - no refresh necessary return pairlist.copy() @@ -121,7 +121,7 @@ class MarketCapFilter(IPairList): # pairlist = _pairlist pairlist = self.filter_pairlist(_pairlist, tickers) - self._marketcap_cache['pairlist_mc'] = pairlist.copy() + # self._marketcap_cache['pairlist_mc'] = pairlist.copy() return pairlist From bb684dd6137ed9001b0cc3e5774c6fed9c33fb9e Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Mon, 8 Jan 2024 00:16:49 +0900 Subject: [PATCH 09/43] empty pairlist --- freqtrade/plugins/pairlist/MarketCapFilter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/plugins/pairlist/MarketCapFilter.py b/freqtrade/plugins/pairlist/MarketCapFilter.py index a3b794c6f..9775d50c9 100644 --- a/freqtrade/plugins/pairlist/MarketCapFilter.py +++ b/freqtrade/plugins/pairlist/MarketCapFilter.py @@ -99,6 +99,7 @@ class MarketCapFilter(IPairList): # Generate dynamic whitelist # Must always run if this pairlist is not the first in the list. # pairlist = self._marketcap_cache.get('pairlist_mc') + pairlist=[] if pairlist: # Item found - no refresh necessary return pairlist.copy() From 35467619c45d953fbb4fc5c9004d130ca17de26b Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Mon, 8 Jan 2024 00:20:09 +0900 Subject: [PATCH 10/43] add logger --- freqtrade/plugins/pairlist/MarketCapFilter.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/plugins/pairlist/MarketCapFilter.py b/freqtrade/plugins/pairlist/MarketCapFilter.py index 9775d50c9..c2df8921c 100644 --- a/freqtrade/plugins/pairlist/MarketCapFilter.py +++ b/freqtrade/plugins/pairlist/MarketCapFilter.py @@ -170,8 +170,11 @@ class MarketCapFilter(IPairList): # option B for mc_pair in marketcap_list: test_pair = f"{mc_pair.upper()}/{self._stake_currency.upper}" + logger.info(f"Check test pair name {test_pair}") if test_pair in pairlist: + logger.info(f"{test_pair} is in pairlist") filtered_pairlist.append(test_pair) + logger.info(f"{len(filtered_pairlist)} pairs in filtered pairlist") if len(filtered_pairlist) == self._max_rank: break From 1f0a2ab242fab892159e3577b06440171103cae5 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Mon, 8 Jan 2024 00:21:07 +0900 Subject: [PATCH 11/43] forgot () --- freqtrade/plugins/pairlist/MarketCapFilter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/plugins/pairlist/MarketCapFilter.py b/freqtrade/plugins/pairlist/MarketCapFilter.py index c2df8921c..dca8c911e 100644 --- a/freqtrade/plugins/pairlist/MarketCapFilter.py +++ b/freqtrade/plugins/pairlist/MarketCapFilter.py @@ -169,7 +169,7 @@ class MarketCapFilter(IPairList): # option B for mc_pair in marketcap_list: - test_pair = f"{mc_pair.upper()}/{self._stake_currency.upper}" + test_pair = f"{mc_pair.upper()}/{self._stake_currency.upper()}" logger.info(f"Check test pair name {test_pair}") if test_pair in pairlist: logger.info(f"{test_pair} is in pairlist") From 09cb6cb02d632d731ffa98bf82acb1bbffde5699 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Mon, 8 Jan 2024 00:21:58 +0900 Subject: [PATCH 12/43] remove debug --- freqtrade/plugins/pairlist/MarketCapFilter.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/freqtrade/plugins/pairlist/MarketCapFilter.py b/freqtrade/plugins/pairlist/MarketCapFilter.py index dca8c911e..c9dfa7769 100644 --- a/freqtrade/plugins/pairlist/MarketCapFilter.py +++ b/freqtrade/plugins/pairlist/MarketCapFilter.py @@ -170,11 +170,8 @@ class MarketCapFilter(IPairList): # option B for mc_pair in marketcap_list: test_pair = f"{mc_pair.upper()}/{self._stake_currency.upper()}" - logger.info(f"Check test pair name {test_pair}") if test_pair in pairlist: - logger.info(f"{test_pair} is in pairlist") filtered_pairlist.append(test_pair) - logger.info(f"{len(filtered_pairlist)} pairs in filtered pairlist") if len(filtered_pairlist) == self._max_rank: break From 96a6c00acc80adf3a9aa01b5c8acbef87d2c325c Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Mon, 8 Jan 2024 00:42:33 +0900 Subject: [PATCH 13/43] add mode --- freqtrade/plugins/pairlist/MarketCapFilter.py | 68 +++++++++++-------- 1 file changed, 40 insertions(+), 28 deletions(-) diff --git a/freqtrade/plugins/pairlist/MarketCapFilter.py b/freqtrade/plugins/pairlist/MarketCapFilter.py index c9dfa7769..1231d7752 100644 --- a/freqtrade/plugins/pairlist/MarketCapFilter.py +++ b/freqtrade/plugins/pairlist/MarketCapFilter.py @@ -21,7 +21,7 @@ from pycoingecko import CoinGeckoAPI logger = logging.getLogger(__name__) -SORT_VALUES = ['quoteVolume'] +MODE_VALUES = ['top_rank', 'total_assets'] class MarketCapFilter(IPairList): @@ -33,23 +33,28 @@ class MarketCapFilter(IPairList): pairlist_pos: int) -> None: super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) - if 'max_rank' not in self._pairlistconfig: + if 'limit' not in self._pairlistconfig: raise OperationalException( - '`max_rank` not specified. Please check your configuration ' - 'for "pairlist.config.max_rank"') + '`limit` not specified. Please check your configuration ' + 'for "pairlist.config.limit"') self._stake_currency = config['stake_currency'] - self._max_rank = self._pairlistconfig['max_rank'] + self._mode = self._pairlistconfig.get('mode', 'top_rank') + self._limit = self._pairlistconfig['limit'] self._refresh_period = self._pairlistconfig.get('refresh_period', 86400) self._marketcap_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period) self._def_candletype = self._config['candle_type_def'] self._coingekko: CoinGeckoAPI = CoinGeckoAPI() - if self._max_rank > 250: + if self._limit > 250: raise OperationalException( "This filter only support up to rank 250." ) + if not self._validate_keys(self._mode): + raise OperationalException( + f'key {self._mode} not in {MODE_VALUES}') + @property def needstickers(self) -> bool: @@ -61,13 +66,13 @@ class MarketCapFilter(IPairList): return False def _validate_keys(self, key): - return key in SORT_VALUES + return key in MODE_VALUES def short_desc(self) -> str: """ Short whitelist method description - used for startup-messages """ - return f"{self.name} - Only use top {self._pairlistconfig['max_rank']} market cap pairs." + return f"{self.name} - Only use top {self._pairlistconfig['limit']} market cap pairs." @staticmethod def description() -> str: @@ -76,11 +81,18 @@ class MarketCapFilter(IPairList): @staticmethod def available_parameters() -> Dict[str, PairlistParameter]: return { - "max_rank": { + "limit": { "type": "number", "default": 30, "description": "Max market cap rank", - "help": "Only use assets that ranked within top max_rank market cap", + "help": "Only use assets with high market cap rank", + }, + "mode": { + "type": "option", + "default": "top_rank", + "options": MODE_VALUES, + "description": "Mode of number", + "help": "How to interpret the number", }, "refresh_period": { "type": "number", @@ -156,24 +168,24 @@ class MarketCapFilter(IPairList): if can_filter: filtered_pairlist = [] - # option A - # top_marketcap = marketcap_list[:self._max_rank:] - - # for pair in pairlist: - # base = pair.split('/')[0] - # if base.lower() in top_marketcap: - # filtered_pairlist.append(pair) - # else: - # logger.info(f"Remove {pair} from whitelist because it's not in " - # f"top {self._max_rank} market cap") - - # option B - for mc_pair in marketcap_list: - test_pair = f"{mc_pair.upper()}/{self._stake_currency.upper()}" - if test_pair in pairlist: - filtered_pairlist.append(test_pair) - if len(filtered_pairlist) == self._max_rank: - break + if self._mode == 'top_rank': + top_marketcap = marketcap_list[:self._limit:] + + for pair in pairlist: + base = pair.split('/')[0] + if base.lower() in top_marketcap: + filtered_pairlist.append(pair) + else: + logger.info(f"Remove {pair} from whitelist because it's not ranked " + f"within top {self._limit} market cap") + + else: + for mc_pair in marketcap_list: + test_pair = f"{mc_pair.upper()}/{self._stake_currency.upper()}" + if test_pair in pairlist: + filtered_pairlist.append(test_pair) + if len(filtered_pairlist) == self._limit: + break if len(filtered_pairlist) > 0: return filtered_pairlist From 8055140e665313478c7a760a5662971df8b9df14 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Mon, 8 Jan 2024 00:53:37 +0900 Subject: [PATCH 14/43] simplify code --- freqtrade/plugins/pairlist/MarketCapFilter.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/freqtrade/plugins/pairlist/MarketCapFilter.py b/freqtrade/plugins/pairlist/MarketCapFilter.py index 1231d7752..0553dcc68 100644 --- a/freqtrade/plugins/pairlist/MarketCapFilter.py +++ b/freqtrade/plugins/pairlist/MarketCapFilter.py @@ -156,9 +156,7 @@ class MarketCapFilter(IPairList): per_page='250', page='1', sparkline='false', locale='en') if data: - marketcap_list = [] - for row in data: - marketcap_list.append(row['symbol']) + marketcap_list = [row['symbol'] for row in data] if len(marketcap_list) > 0: self._marketcap_cache['marketcap'] = marketcap_list From f29fcb31e03b74720a8f8d273e62d520c26cbb58 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Mon, 8 Jan 2024 00:54:21 +0900 Subject: [PATCH 15/43] debug --- freqtrade/plugins/pairlist/MarketCapFilter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/plugins/pairlist/MarketCapFilter.py b/freqtrade/plugins/pairlist/MarketCapFilter.py index 0553dcc68..7632117ab 100644 --- a/freqtrade/plugins/pairlist/MarketCapFilter.py +++ b/freqtrade/plugins/pairlist/MarketCapFilter.py @@ -146,7 +146,8 @@ class MarketCapFilter(IPairList): :param tickers: Tickers (from exchange.get_tickers). May be cached. :return: new whitelist """ - marketcap_list = self._marketcap_cache.get('marketcap') + # marketcap_list = self._marketcap_cache.get('marketcap') + marketcap_list = [] can_filter = False if marketcap_list: From 5bb4824aaefff30cd48d1532e9d5973eed6cc864 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Mon, 8 Jan 2024 00:58:39 +0900 Subject: [PATCH 16/43] add log print to see the time --- freqtrade/plugins/pairlist/MarketCapFilter.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/freqtrade/plugins/pairlist/MarketCapFilter.py b/freqtrade/plugins/pairlist/MarketCapFilter.py index 7632117ab..cae971762 100644 --- a/freqtrade/plugins/pairlist/MarketCapFilter.py +++ b/freqtrade/plugins/pairlist/MarketCapFilter.py @@ -118,6 +118,7 @@ class MarketCapFilter(IPairList): else: # Use fresh pairlist # Check if pair quote currency equals to the stake currency. + logger.info("Get active pairs for the market") _pairlist = [k for k in self._exchange.get_markets( quote_currencies=[self._stake_currency], tradable_only=True, active_only=True).keys()] @@ -153,6 +154,7 @@ class MarketCapFilter(IPairList): if marketcap_list: can_filter = True else: + logger.info("Get top 250 marketcap coins from coingecko") data = self._coingekko.get_coins_markets(vs_currency='usd', order='market_cap_desc', per_page='250', page='1', sparkline='false', locale='en') @@ -167,6 +169,8 @@ class MarketCapFilter(IPairList): if can_filter: filtered_pairlist = [] + logger.info("Create filtered pairlist") + if self._mode == 'top_rank': top_marketcap = marketcap_list[:self._limit:] From d578c910bccb769bb3fd654ef1a2a6cfe13db59c Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Mon, 8 Jan 2024 15:09:42 +0900 Subject: [PATCH 17/43] remove unused lines --- freqtrade/plugins/pairlist/MarketCapFilter.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/freqtrade/plugins/pairlist/MarketCapFilter.py b/freqtrade/plugins/pairlist/MarketCapFilter.py index cae971762..b9b9eb067 100644 --- a/freqtrade/plugins/pairlist/MarketCapFilter.py +++ b/freqtrade/plugins/pairlist/MarketCapFilter.py @@ -48,7 +48,7 @@ class MarketCapFilter(IPairList): if self._limit > 250: raise OperationalException( - "This filter only support up to rank 250." + "This filter only support limit value up to 250." ) if not self._validate_keys(self._mode): @@ -124,15 +124,6 @@ class MarketCapFilter(IPairList): tradable_only=True, active_only=True).keys()] # No point in testing for blacklisted pairs... _pairlist = self.verify_blacklist(_pairlist, logger.info) - # if not self._use_range: - # filtered_tickers = [ - # v for k, v in tickers.items() - # if (self._exchange.get_pair_quote_currency(k) == self._stake_currency - # and (self._use_range or v.get(self._sort_key) is not None) - # and v['symbol'] in _pairlist)] - # pairlist = [s['symbol'] for s in filtered_tickers] - # else: - # pairlist = _pairlist pairlist = self.filter_pairlist(_pairlist, tickers) # self._marketcap_cache['pairlist_mc'] = pairlist.copy() From 442ca86d1508b65552d0b18de34a3445b459d357 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Mon, 8 Jan 2024 15:33:43 +0900 Subject: [PATCH 18/43] add market check --- freqtrade/plugins/pairlist/MarketCapFilter.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/freqtrade/plugins/pairlist/MarketCapFilter.py b/freqtrade/plugins/pairlist/MarketCapFilter.py index b9b9eb067..663eac0e3 100644 --- a/freqtrade/plugins/pairlist/MarketCapFilter.py +++ b/freqtrade/plugins/pairlist/MarketCapFilter.py @@ -174,8 +174,10 @@ class MarketCapFilter(IPairList): f"within top {self._limit} market cap") else: + market = self._config['trading_mode'] + pair_format = f"{self._stake_currency.upper()}" if (market == 'spot') else f"{self._stake_currency.upper()}:{self._stake_currency.upper()}" for mc_pair in marketcap_list: - test_pair = f"{mc_pair.upper()}/{self._stake_currency.upper()}" + test_pair = f"{mc_pair.upper()}/{pair_format}" if test_pair in pairlist: filtered_pairlist.append(test_pair) if len(filtered_pairlist) == self._limit: From fdf9cc1b76bcee8f3f184f70942fc2019be3392d Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Mon, 8 Jan 2024 16:02:31 +0900 Subject: [PATCH 19/43] tidy up codes --- freqtrade/plugins/pairlist/MarketCapFilter.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/freqtrade/plugins/pairlist/MarketCapFilter.py b/freqtrade/plugins/pairlist/MarketCapFilter.py index 663eac0e3..1cf8b7b7c 100644 --- a/freqtrade/plugins/pairlist/MarketCapFilter.py +++ b/freqtrade/plugins/pairlist/MarketCapFilter.py @@ -11,7 +11,6 @@ from cachetools import TTLCache from freqtrade.constants import Config, ListPairsWithTimeframes from freqtrade.exceptions import OperationalException -from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date from freqtrade.exchange.types import Tickers from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter from freqtrade.util import dt_now, format_ms_time @@ -110,8 +109,7 @@ class MarketCapFilter(IPairList): """ # Generate dynamic whitelist # Must always run if this pairlist is not the first in the list. - # pairlist = self._marketcap_cache.get('pairlist_mc') - pairlist=[] + pairlist = self._marketcap_cache.get('pairlist_mc') if pairlist: # Item found - no refresh necessary return pairlist.copy() @@ -126,7 +124,7 @@ class MarketCapFilter(IPairList): _pairlist = self.verify_blacklist(_pairlist, logger.info) pairlist = self.filter_pairlist(_pairlist, tickers) - # self._marketcap_cache['pairlist_mc'] = pairlist.copy() + self._marketcap_cache['pairlist_mc'] = pairlist.copy() return pairlist @@ -138,8 +136,7 @@ class MarketCapFilter(IPairList): :param tickers: Tickers (from exchange.get_tickers). May be cached. :return: new whitelist """ - # marketcap_list = self._marketcap_cache.get('marketcap') - marketcap_list = [] + marketcap_list = self._marketcap_cache.get('marketcap') can_filter = False if marketcap_list: @@ -175,7 +172,8 @@ class MarketCapFilter(IPairList): else: market = self._config['trading_mode'] - pair_format = f"{self._stake_currency.upper()}" if (market == 'spot') else f"{self._stake_currency.upper()}:{self._stake_currency.upper()}" + pair_format = f"{self._stake_currency.upper()}" if (market == 'spot') + else f"{self._stake_currency.upper()}:{self._stake_currency.upper()}" for mc_pair in marketcap_list: test_pair = f"{mc_pair.upper()}/{pair_format}" if test_pair in pairlist: @@ -186,5 +184,4 @@ class MarketCapFilter(IPairList): if len(filtered_pairlist) > 0: return filtered_pairlist - return pairlist From ef8712f4d967a5d3f71dafc432a34b4569dd694c Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Mon, 8 Jan 2024 16:04:30 +0900 Subject: [PATCH 20/43] tidy up --- freqtrade/plugins/pairlist/MarketCapFilter.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/plugins/pairlist/MarketCapFilter.py b/freqtrade/plugins/pairlist/MarketCapFilter.py index 1cf8b7b7c..a8c1c2080 100644 --- a/freqtrade/plugins/pairlist/MarketCapFilter.py +++ b/freqtrade/plugins/pairlist/MarketCapFilter.py @@ -172,8 +172,9 @@ class MarketCapFilter(IPairList): else: market = self._config['trading_mode'] - pair_format = f"{self._stake_currency.upper()}" if (market == 'spot') - else f"{self._stake_currency.upper()}:{self._stake_currency.upper()}" + pair_format = f"{self._stake_currency.upper()}" + if (market == 'futures'): + pair_format += f":{self._stake_currency.upper()}" for mc_pair in marketcap_list: test_pair = f"{mc_pair.upper()}/{pair_format}" if test_pair in pairlist: From bc8c5e743fb1295a4a59e5bb0512f161bfdece49 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Mon, 8 Jan 2024 16:38:30 +0900 Subject: [PATCH 21/43] rename to PairList --- freqtrade/constants.py | 2 +- .../pairlist/{MarketCapFilter.py => MarketCapPairList.py} | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) rename freqtrade/plugins/pairlist/{MarketCapFilter.py => MarketCapPairList.py} (96%) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index d7767dc2e..37e2d849c 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -33,7 +33,7 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss', 'MaxDrawDownHyperOptLoss', 'MaxDrawDownRelativeHyperOptLoss', 'ProfitDrawDownHyperOptLoss'] AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'ProducerPairList', 'RemotePairList', - 'AgeFilter', "FullTradesFilter", 'MarketCapFilter', 'OffsetFilter', + 'MarketCapPairList', 'AgeFilter', "FullTradesFilter", 'OffsetFilter', 'PerformanceFilter', 'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter', 'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter'] diff --git a/freqtrade/plugins/pairlist/MarketCapFilter.py b/freqtrade/plugins/pairlist/MarketCapPairList.py similarity index 96% rename from freqtrade/plugins/pairlist/MarketCapFilter.py rename to freqtrade/plugins/pairlist/MarketCapPairList.py index a8c1c2080..1f5474303 100644 --- a/freqtrade/plugins/pairlist/MarketCapFilter.py +++ b/freqtrade/plugins/pairlist/MarketCapPairList.py @@ -23,7 +23,7 @@ logger = logging.getLogger(__name__) MODE_VALUES = ['top_rank', 'total_assets'] -class MarketCapFilter(IPairList): +class MarketCapPairList(IPairList): is_pairlist_generator = True @@ -116,7 +116,6 @@ class MarketCapFilter(IPairList): else: # Use fresh pairlist # Check if pair quote currency equals to the stake currency. - logger.info("Get active pairs for the market") _pairlist = [k for k in self._exchange.get_markets( quote_currencies=[self._stake_currency], tradable_only=True, active_only=True).keys()] @@ -142,7 +141,6 @@ class MarketCapFilter(IPairList): if marketcap_list: can_filter = True else: - logger.info("Get top 250 marketcap coins from coingecko") data = self._coingekko.get_coins_markets(vs_currency='usd', order='market_cap_desc', per_page='250', page='1', sparkline='false', locale='en') @@ -157,7 +155,6 @@ class MarketCapFilter(IPairList): if can_filter: filtered_pairlist = [] - logger.info("Create filtered pairlist") if self._mode == 'top_rank': top_marketcap = marketcap_list[:self._limit:] From d3506c249d2fd5d087fee62d2a9a4b0f0e36c904 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Tue, 9 Jan 2024 14:08:15 +0900 Subject: [PATCH 22/43] update docs --- docs/includes/pairlists.md | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 8e4b43178..c5ea43382 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -6,7 +6,7 @@ In your configuration, you can use Static Pairlist (defined by the [`StaticPairL Additionally, [`AgeFilter`](#agefilter), [`PrecisionFilter`](#precisionfilter), [`PriceFilter`](#pricefilter), [`ShuffleFilter`](#shufflefilter), [`SpreadFilter`](#spreadfilter) and [`VolatilityFilter`](#volatilityfilter) act as Pairlist Filters, removing certain pairs and/or moving their positions in the pairlist. -If multiple Pairlist Handlers are used, they are chained and a combination of all Pairlist Handlers forms the resulting pairlist the bot uses for trading and backtesting. Pairlist Handlers are executed in the sequence they are configured. You should always configure either `StaticPairList` or `VolumePairList` as the starting Pairlist Handler. +If multiple Pairlist Handlers are used, they are chained and a combination of all Pairlist Handlers forms the resulting pairlist the bot uses for trading and backtesting. Pairlist Handlers are executed in the sequence they are configured. You can define either `StaticPairList`, `VolumePairList`, `ProducerPairList`, `RemotePairList` or `MarketCapPairList` as the starting Pairlist Handler. Inactive markets are always removed from the resulting pairlist. Explicitly blacklisted pairs (those in the `pair_blacklist` configuration setting) are also always removed from the resulting pairlist. @@ -24,6 +24,7 @@ You may also use something like `.*DOWN/BTC` or `.*UP/BTC` to exclude leveraged * [`VolumePairList`](#volume-pair-list) * [`ProducerPairList`](#producerpairlist) * [`RemotePairList`](#remotepairlist) +* [`MarketCapPairList`](#marketcappairlist) * [`AgeFilter`](#agefilter) * [`FullTradesFilter`](#fulltradesfilter) * [`OffsetFilter`](#offsetfilter) @@ -227,6 +228,29 @@ The optional `bearer_token` will be included in the requests Authorization Heade !!! Note In case of a server error the last received pairlist will be kept if `keep_pairlist_on_failure` is set to true, when set to false a empty pairlist is returned. +#### MarketCapPairList + +`MarketCapPairList` employs sorting/filtering of pairs by their marketcap rank based of CoinGecko. It will only recognize coins up to the coin placed at rank 250. The number of pairs in the resulted pairlist will be slightly different depends on the `mode` defined in the config (available mode are `top_rank` and `total_assets`). The marketcap data from + +```json +"pairlists": [ + { + "method": "MarketCapPairList", + "mode": "top_rank", + "limit": 20, + "refresh_period": 86400 + } +] +``` + +##### `top_rank` mode +In this mode, it will return pairlist consist of active and not-blaclisted pairs that are placed at the top `limit` rank of the marketcap. + +##### `total_assets` mode +In this mode, it will return pairlist consist of `limit` number of active and not-blaclisted pairs sorted by their marketcap rank. + +The refresh_period setting allows to define the period (in seconds), at which the marketcap rank data will be refreshed. Defaults to 86,400s (1 day). The pairlist cache (refresh_period) is applicable on both generating pairlists (first position in the list) and filtering instances (not the first position in the list). + #### AgeFilter Removes pairs that have been listed on the exchange for less than `min_days_listed` days (defaults to `10`) or more than `max_days_listed` days (defaults `None` mean infinity). From 95af462e80f333eb221b99345dc0437146743a5f Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Tue, 9 Jan 2024 14:21:07 +0900 Subject: [PATCH 23/43] fix pre-commit --- freqtrade/plugins/pairlist/MarketCapPairList.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/plugins/pairlist/MarketCapPairList.py b/freqtrade/plugins/pairlist/MarketCapPairList.py index 1f5474303..465374edb 100644 --- a/freqtrade/plugins/pairlist/MarketCapPairList.py +++ b/freqtrade/plugins/pairlist/MarketCapPairList.py @@ -8,6 +8,7 @@ from datetime import timedelta from typing import Any, Dict, List, Literal from cachetools import TTLCache +from pycoingecko import CoinGeckoAPI from freqtrade.constants import Config, ListPairsWithTimeframes from freqtrade.exceptions import OperationalException @@ -15,7 +16,6 @@ from freqtrade.exchange.types import Tickers from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter from freqtrade.util import dt_now, format_ms_time -from pycoingecko import CoinGeckoAPI logger = logging.getLogger(__name__) From e8fcac491a0c5b70f47a53e509e44a78f3cd55f5 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Tue, 9 Jan 2024 14:32:27 +0900 Subject: [PATCH 24/43] use number_assets instead of limit --- docs/includes/pairlists.md | 6 ++-- .../plugins/pairlist/MarketCapPairList.py | 28 +++++++++++-------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index c5ea43382..07d672039 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -237,17 +237,17 @@ The optional `bearer_token` will be included in the requests Authorization Heade { "method": "MarketCapPairList", "mode": "top_rank", - "limit": 20, + "number_assets": 20, "refresh_period": 86400 } ] ``` ##### `top_rank` mode -In this mode, it will return pairlist consist of active and not-blaclisted pairs that are placed at the top `limit` rank of the marketcap. +In this mode, it will return pairlist consist of active and not-blaclisted pairs that are placed at the top `number_assets` rank of the marketcap. ##### `total_assets` mode -In this mode, it will return pairlist consist of `limit` number of active and not-blaclisted pairs sorted by their marketcap rank. +In this mode, it will return pairlist consist of `number_assets` number of active and not-blaclisted pairs sorted by their marketcap rank. The refresh_period setting allows to define the period (in seconds), at which the marketcap rank data will be refreshed. Defaults to 86,400s (1 day). The pairlist cache (refresh_period) is applicable on both generating pairlists (first position in the list) and filtering instances (not the first position in the list). diff --git a/freqtrade/plugins/pairlist/MarketCapPairList.py b/freqtrade/plugins/pairlist/MarketCapPairList.py index 465374edb..fb026759a 100644 --- a/freqtrade/plugins/pairlist/MarketCapPairList.py +++ b/freqtrade/plugins/pairlist/MarketCapPairList.py @@ -32,22 +32,22 @@ class MarketCapPairList(IPairList): pairlist_pos: int) -> None: super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) - if 'limit' not in self._pairlistconfig: + if 'number_assets' not in self._pairlistconfig: raise OperationalException( - '`limit` not specified. Please check your configuration ' - 'for "pairlist.config.limit"') + '`number_assets` not specified. Please check your configuration ' + 'for "pairlist.config.number_assets"') self._stake_currency = config['stake_currency'] self._mode = self._pairlistconfig.get('mode', 'top_rank') - self._limit = self._pairlistconfig['limit'] + self._number_assets = self._pairlistconfig['number_assets'] self._refresh_period = self._pairlistconfig.get('refresh_period', 86400) self._marketcap_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period) self._def_candletype = self._config['candle_type_def'] self._coingekko: CoinGeckoAPI = CoinGeckoAPI() - if self._limit > 250: + if self._number_assets > 250: raise OperationalException( - "This filter only support limit value up to 250." + "This filter only support number_assets value up to 250." ) if not self._validate_keys(self._mode): @@ -71,16 +71,20 @@ class MarketCapPairList(IPairList): """ Short whitelist method description - used for startup-messages """ - return f"{self.name} - Only use top {self._pairlistconfig['limit']} market cap pairs." + num = self._pairlistconfig['number_assets'] + msg = f"{self.name} - Only include pairs ranked within top {num} market cap." + if self._mode == "total_assets": + msg = f"{self.name} - top {num} pairs sorted by market cap." + return msg @staticmethod def description() -> str: - return "Filter pair list based on market cap." + return "Provides pair list based on CoinGecko's market cap rank." @staticmethod def available_parameters() -> Dict[str, PairlistParameter]: return { - "limit": { + "number_assets": { "type": "number", "default": 30, "description": "Max market cap rank", @@ -157,7 +161,7 @@ class MarketCapPairList(IPairList): if self._mode == 'top_rank': - top_marketcap = marketcap_list[:self._limit:] + top_marketcap = marketcap_list[:self._number_assets:] for pair in pairlist: base = pair.split('/')[0] @@ -165,7 +169,7 @@ class MarketCapPairList(IPairList): filtered_pairlist.append(pair) else: logger.info(f"Remove {pair} from whitelist because it's not ranked " - f"within top {self._limit} market cap") + f"within top {self._number_assets} market cap") else: market = self._config['trading_mode'] @@ -176,7 +180,7 @@ class MarketCapPairList(IPairList): test_pair = f"{mc_pair.upper()}/{pair_format}" if test_pair in pairlist: filtered_pairlist.append(test_pair) - if len(filtered_pairlist) == self._limit: + if len(filtered_pairlist) == self._number_assets: break if len(filtered_pairlist) > 0: From de91261f732357f59bfd867791a039345aa22e33 Mon Sep 17 00:00:00 2001 From: Stefano Date: Tue, 9 Jan 2024 15:03:10 +0900 Subject: [PATCH 25/43] fix pre-commit --- freqtrade/plugins/pairlist/MarketCapPairList.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/freqtrade/plugins/pairlist/MarketCapPairList.py b/freqtrade/plugins/pairlist/MarketCapPairList.py index fb026759a..7faf971f3 100644 --- a/freqtrade/plugins/pairlist/MarketCapPairList.py +++ b/freqtrade/plugins/pairlist/MarketCapPairList.py @@ -4,17 +4,15 @@ Market Cap PairList provider Provides dynamic pair list based on Market Cap """ import logging -from datetime import timedelta -from typing import Any, Dict, List, Literal +from typing import Any, Dict, List from cachetools import TTLCache from pycoingecko import CoinGeckoAPI -from freqtrade.constants import Config, ListPairsWithTimeframes +from freqtrade.constants import Config from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Tickers from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter -from freqtrade.util import dt_now, format_ms_time logger = logging.getLogger(__name__) @@ -54,7 +52,6 @@ class MarketCapPairList(IPairList): raise OperationalException( f'key {self._mode} not in {MODE_VALUES}') - @property def needstickers(self) -> bool: """ @@ -150,16 +147,12 @@ class MarketCapPairList(IPairList): locale='en') if data: marketcap_list = [row['symbol'] for row in data] - - if len(marketcap_list) > 0: - self._marketcap_cache['marketcap'] = marketcap_list - can_filter = True - + self._marketcap_cache['marketcap'] = marketcap_list + can_filter = True if can_filter: filtered_pairlist = [] - if self._mode == 'top_rank': top_marketcap = marketcap_list[:self._number_assets:] From bbec51685d8217d52c80eec52e792e1a1dd4dbf5 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Sun, 21 Jan 2024 13:28:50 +0900 Subject: [PATCH 26/43] remove can_filter, and use log_once --- freqtrade/plugins/pairlist/MarketCapPairList.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/freqtrade/plugins/pairlist/MarketCapPairList.py b/freqtrade/plugins/pairlist/MarketCapPairList.py index 7faf971f3..7e75d2223 100644 --- a/freqtrade/plugins/pairlist/MarketCapPairList.py +++ b/freqtrade/plugins/pairlist/MarketCapPairList.py @@ -137,20 +137,16 @@ class MarketCapPairList(IPairList): :return: new whitelist """ marketcap_list = self._marketcap_cache.get('marketcap') - can_filter = False - if marketcap_list: - can_filter = True - else: + if marketcap_list is None: data = self._coingekko.get_coins_markets(vs_currency='usd', order='market_cap_desc', per_page='250', page='1', sparkline='false', locale='en') if data: marketcap_list = [row['symbol'] for row in data] self._marketcap_cache['marketcap'] = marketcap_list - can_filter = True - if can_filter: + if marketcap_list: filtered_pairlist = [] if self._mode == 'top_rank': @@ -161,8 +157,8 @@ class MarketCapPairList(IPairList): if base.lower() in top_marketcap: filtered_pairlist.append(pair) else: - logger.info(f"Remove {pair} from whitelist because it's not ranked " - f"within top {self._number_assets} market cap") + self.log_once(f"Remove {pair} from whitelist because it's not ranked " + f"within top {self._number_assets} market cap") else: market = self._config['trading_mode'] From 60812983e3d46f487cb5e3b6fd249ed1104ba346 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Sun, 21 Jan 2024 13:33:38 +0900 Subject: [PATCH 27/43] fix error --- freqtrade/plugins/pairlist/MarketCapPairList.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/plugins/pairlist/MarketCapPairList.py b/freqtrade/plugins/pairlist/MarketCapPairList.py index 7e75d2223..0cddcd05a 100644 --- a/freqtrade/plugins/pairlist/MarketCapPairList.py +++ b/freqtrade/plugins/pairlist/MarketCapPairList.py @@ -158,7 +158,7 @@ class MarketCapPairList(IPairList): filtered_pairlist.append(pair) else: self.log_once(f"Remove {pair} from whitelist because it's not ranked " - f"within top {self._number_assets} market cap") + f"within top {self._number_assets} market cap", logger.info) else: market = self._config['trading_mode'] From bf17236e9baea090465065e17cd5a1430b9a8e5d Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Sat, 27 Jan 2024 16:25:54 +0900 Subject: [PATCH 28/43] fix typo in the docs --- docs/includes/pairlists.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 07d672039..85ce6e143 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -230,7 +230,7 @@ The optional `bearer_token` will be included in the requests Authorization Heade #### MarketCapPairList -`MarketCapPairList` employs sorting/filtering of pairs by their marketcap rank based of CoinGecko. It will only recognize coins up to the coin placed at rank 250. The number of pairs in the resulted pairlist will be slightly different depends on the `mode` defined in the config (available mode are `top_rank` and `total_assets`). The marketcap data from +`MarketCapPairList` employs sorting/filtering of pairs by their marketcap rank based of CoinGecko. It will only recognize coins up to the coin placed at rank 250. The number of pairs in the resulted pairlist will be slightly different depends on the `mode` defined in the config (available mode are `top_rank` and `total_assets`). ```json "pairlists": [ @@ -244,12 +244,12 @@ The optional `bearer_token` will be included in the requests Authorization Heade ``` ##### `top_rank` mode -In this mode, it will return pairlist consist of active and not-blaclisted pairs that are placed at the top `number_assets` rank of the marketcap. +In this mode, it will return pairlist consist of active and not-blacklisted pairs that are placed at the top `number_assets` rank of the marketcap. ##### `total_assets` mode -In this mode, it will return pairlist consist of `number_assets` number of active and not-blaclisted pairs sorted by their marketcap rank. +In this mode, it will return pairlist consist of `number_assets` number of active and not-blacklisted pairs sorted by their marketcap rank. -The refresh_period setting allows to define the period (in seconds), at which the marketcap rank data will be refreshed. Defaults to 86,400s (1 day). The pairlist cache (refresh_period) is applicable on both generating pairlists (first position in the list) and filtering instances (not the first position in the list). +The `refresh_period` setting define the period (in seconds) at which the marketcap rank data will be cached. Defaults to 86,400s (1 day). The pairlist cache (refresh_period) is applicable on both generating pairlists (first position in the list) and filtering instances (not the first position in the list). #### AgeFilter From 006639820bbddb68c02a02b8a1d379baf6f9650a Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Sat, 27 Jan 2024 18:10:10 +0900 Subject: [PATCH 29/43] add simple test --- tests/plugins/test_pairlist.py | 77 ++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index d66a47aa6..b2aa2b8f0 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -1513,3 +1513,80 @@ def test_FullTradesFilter(mocker, default_conf_usdt, fee, caplog) -> None: pm.refresh_pairlist() assert pm.whitelist == [] assert log_has_re(r'Whitelist with 0 pairs: \[]', caplog) + + + +def test_MarketCapPairList_filter(mocker, default_conf_usdt): + mock_response = MagicMock() + + mock_response.json.return_value = [ + { + "symbol": "btc", + }, + { + "symbol": "eth", + }, + { + "symbol": "usdt", + }, + { + "symbol": "bnb", + }, + { + "symbol": "sol", + }, + { + "symbol": "xrp", + }, + { + "symbol": "usdc", + }, + { + "symbol": "steth", + }, + { + "symbol": "ada", + }, + { + "symbol": "avax", + } + ] + + mock_response.headers = { + "content-type": "application/json" + } + + # Test top 2 mc + default_conf_usdt['exchange']['pair_whitelist'].extend(['BTC/USDT', 'ETC/USDT']) + default_conf_usdt['pairlists'] = [ + {"method": "StaticPairList"}, + {"method": "MarketCapPairList", "mode": "top_rank", "number_assets": 2} + ] + mocker.patch(f'{EXMS}.exchange_has', MagicMock(return_value=True)) + + mocker.patch("freqtrade.plugins.pairlist.MarketCapPairList._coingekko.get_coins_markets", + return_value=mock_response) + + exchange = get_patched_exchange(mocker, default_conf_usdt) + + pm = PairListManager(exchange, default_conf_usdt) + + pm.refresh_pairlist() + + whitelist = ['ETH/USDT', 'BTC/USDT'] + + assert set(whitelist) == set(pm.whitelist) + + # Test top 6 mc + default_conf_usdt['pairlists'] = [ + {"method": "StaticPairList"}, + {"method": "MarketCapPairList", "mode": "top_rank", "number_assets": 6} + ] + + pm = PairListManager(exchange, default_conf_usdt) + + pm.refresh_pairlist() + + whitelist = ['ETH/USDT', 'XRP/USDT', 'BTC/USDT'] + + assert set(whitelist) == set(pm.whitelist) From c854cef3138acfe79dc838a4a274a5e31f6a1239 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Sat, 27 Jan 2024 18:12:19 +0900 Subject: [PATCH 30/43] fix precommit --- tests/plugins/test_pairlist.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index b2aa2b8f0..0f50ec165 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -1515,40 +1515,39 @@ def test_FullTradesFilter(mocker, default_conf_usdt, fee, caplog) -> None: assert log_has_re(r'Whitelist with 0 pairs: \[]', caplog) - def test_MarketCapPairList_filter(mocker, default_conf_usdt): mock_response = MagicMock() mock_response.json.return_value = [ { - "symbol": "btc", + "symbol": "btc", }, { - "symbol": "eth", + "symbol": "eth", }, { - "symbol": "usdt", + "symbol": "usdt", }, { - "symbol": "bnb", + "symbol": "bnb", }, { - "symbol": "sol", + "symbol": "sol", }, { - "symbol": "xrp", + "symbol": "xrp", }, { - "symbol": "usdc", + "symbol": "usdc", }, { - "symbol": "steth", + "symbol": "steth", }, { - "symbol": "ada", + "symbol": "ada", }, { - "symbol": "avax", + "symbol": "avax", } ] From e0f7b62b542bfb740ecf606a64d860528b872513 Mon Sep 17 00:00:00 2001 From: Stefano Date: Sat, 27 Jan 2024 19:12:50 +0900 Subject: [PATCH 31/43] fix initial tests --- tests/plugins/test_pairlist.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 0f50ec165..1582cfcfb 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -1516,9 +1516,7 @@ def test_FullTradesFilter(mocker, default_conf_usdt, fee, caplog) -> None: def test_MarketCapPairList_filter(mocker, default_conf_usdt): - mock_response = MagicMock() - - mock_response.json.return_value = [ + test_value = [ { "symbol": "btc", }, @@ -1551,20 +1549,16 @@ def test_MarketCapPairList_filter(mocker, default_conf_usdt): } ] - mock_response.headers = { - "content-type": "application/json" - } - # Test top 2 mc default_conf_usdt['exchange']['pair_whitelist'].extend(['BTC/USDT', 'ETC/USDT']) default_conf_usdt['pairlists'] = [ - {"method": "StaticPairList"}, + {"method": "StaticPairList", "allow_inactive": True}, {"method": "MarketCapPairList", "mode": "top_rank", "number_assets": 2} ] mocker.patch(f'{EXMS}.exchange_has', MagicMock(return_value=True)) - mocker.patch("freqtrade.plugins.pairlist.MarketCapPairList._coingekko.get_coins_markets", - return_value=mock_response) + mocker.patch("freqtrade.plugins.pairlist.MarketCapPairList.CoinGeckoAPI.get_coins_markets", + return_value=test_value) exchange = get_patched_exchange(mocker, default_conf_usdt) @@ -1578,7 +1572,7 @@ def test_MarketCapPairList_filter(mocker, default_conf_usdt): # Test top 6 mc default_conf_usdt['pairlists'] = [ - {"method": "StaticPairList"}, + {"method": "StaticPairList", "allow_inactive": True}, {"method": "MarketCapPairList", "mode": "top_rank", "number_assets": 6} ] From f8dc161ab45174ccad85e88ef113930a82659c6a Mon Sep 17 00:00:00 2001 From: Stefano Date: Sat, 27 Jan 2024 19:33:12 +0900 Subject: [PATCH 32/43] add 2 more final tests --- tests/plugins/test_pairlist.py | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 1582cfcfb..6bef0456f 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -1551,6 +1551,7 @@ def test_MarketCapPairList_filter(mocker, default_conf_usdt): # Test top 2 mc default_conf_usdt['exchange']['pair_whitelist'].extend(['BTC/USDT', 'ETC/USDT']) + default_conf_usdt['trading_mode'] = 'spot' default_conf_usdt['pairlists'] = [ {"method": "StaticPairList", "allow_inactive": True}, {"method": "MarketCapPairList", "mode": "top_rank", "number_assets": 2} @@ -1568,7 +1569,7 @@ def test_MarketCapPairList_filter(mocker, default_conf_usdt): whitelist = ['ETH/USDT', 'BTC/USDT'] - assert set(whitelist) == set(pm.whitelist) + assert whitelist == pm.whitelist # Test top 6 mc default_conf_usdt['pairlists'] = [ @@ -1582,4 +1583,32 @@ def test_MarketCapPairList_filter(mocker, default_conf_usdt): whitelist = ['ETH/USDT', 'XRP/USDT', 'BTC/USDT'] - assert set(whitelist) == set(pm.whitelist) + assert whitelist == pm.whitelist + + # Test total assets mode, 2 assets + default_conf_usdt['pairlists'] = [ + {"method": "StaticPairList", "allow_inactive": True}, + {"method": "MarketCapPairList", "mode": "total_assets", "number_assets": 2} + ] + + pm = PairListManager(exchange, default_conf_usdt) + + pm.refresh_pairlist() + + whitelist = ['BTC/USDT', 'ETH/USDT'] + + assert whitelist == pm.whitelist + + # Test total assets mode, 5 assets + default_conf_usdt['pairlists'] = [ + {"method": "StaticPairList", "allow_inactive": True}, + {"method": "MarketCapPairList", "mode": "total_assets", "number_assets": 5} + ] + + pm = PairListManager(exchange, default_conf_usdt) + + pm.refresh_pairlist() + + whitelist = ['BTC/USDT', 'ETH/USDT', 'XRP/USDT'] + + assert whitelist == pm.whitelist From 132e143b9ad25c342769919e73c660e59bcafd44 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 27 Jan 2024 14:49:50 +0100 Subject: [PATCH 33/43] Minor comment fix --- docs/includes/pairlists.md | 4 +++- freqtrade/plugins/pairlist/MarketCapPairList.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 85ce6e143..482de3782 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -244,12 +244,14 @@ The optional `bearer_token` will be included in the requests Authorization Heade ``` ##### `top_rank` mode + In this mode, it will return pairlist consist of active and not-blacklisted pairs that are placed at the top `number_assets` rank of the marketcap. ##### `total_assets` mode + In this mode, it will return pairlist consist of `number_assets` number of active and not-blacklisted pairs sorted by their marketcap rank. -The `refresh_period` setting define the period (in seconds) at which the marketcap rank data will be cached. Defaults to 86,400s (1 day). The pairlist cache (refresh_period) is applicable on both generating pairlists (first position in the list) and filtering instances (not the first position in the list). +The `refresh_period` setting define the period (in seconds) at which the marketcap rank data will be cached. Defaults to 86,400s (1 day). The pairlist cache (`refresh_period`) is applicable on both generating pairlists (first position in the list) and filtering instances (not the first position in the list). #### AgeFilter diff --git a/freqtrade/plugins/pairlist/MarketCapPairList.py b/freqtrade/plugins/pairlist/MarketCapPairList.py index 0cddcd05a..c0232e0d0 100644 --- a/freqtrade/plugins/pairlist/MarketCapPairList.py +++ b/freqtrade/plugins/pairlist/MarketCapPairList.py @@ -109,7 +109,7 @@ class MarketCapPairList(IPairList): :return: List of pairs """ # Generate dynamic whitelist - # Must always run if this pairlist is not the first in the list. + # Must always run if this pairlist is the first in the list. pairlist = self._marketcap_cache.get('pairlist_mc') if pairlist: # Item found - no refresh necessary From 076ca7520096c7533408a83151bbd5eef5f7b6a5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 27 Jan 2024 14:52:05 +0100 Subject: [PATCH 34/43] Slightly refactor pairlist test --- tests/plugins/test_pairlist.py | 82 +++++++++++++--------------------- 1 file changed, 30 insertions(+), 52 deletions(-) diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 6bef0456f..e19968482 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -1515,7 +1515,34 @@ def test_FullTradesFilter(mocker, default_conf_usdt, fee, caplog) -> None: assert log_has_re(r'Whitelist with 0 pairs: \[]', caplog) -def test_MarketCapPairList_filter(mocker, default_conf_usdt): +@pytest.mark.parametrize('pairlists,result', [ + ([ + # Test top 2 mc + {"method": "StaticPairList", "allow_inactive": True}, + {"method": "MarketCapPairList", "mode": "top_rank", "number_assets": 2} + ], ['ETH/USDT', 'BTC/USDT']), + ([ + # Test top 6 mc + {"method": "StaticPairList", "allow_inactive": True}, + {"method": "MarketCapPairList", "mode": "top_rank", "number_assets": 6} + ], ['ETH/USDT', 'XRP/USDT', 'BTC/USDT']), + ([ + # Test total assets mode, 2 assets + {"method": "StaticPairList", "allow_inactive": True}, + {"method": "MarketCapPairList", "mode": "total_assets", "number_assets": 2} + ], ['BTC/USDT', 'ETH/USDT']), + + ([ + # Test total assets mode, 5 assets + {"method": "StaticPairList", "allow_inactive": True}, + {"method": "MarketCapPairList", "mode": "total_assets", "number_assets": 5} + ], ['BTC/USDT', 'ETH/USDT', 'XRP/USDT']), + ([ + # MarketCapPairList as generator + {"method": "MarketCapPairList", "mode": "total_assets", "number_assets": 5} + ], ['ETH/USDT', 'XRP/USDT']) +]) +def test_MarketCapPairList_filter(mocker, default_conf_usdt, pairlists, result): test_value = [ { "symbol": "btc", @@ -1549,13 +1576,9 @@ def test_MarketCapPairList_filter(mocker, default_conf_usdt): } ] - # Test top 2 mc default_conf_usdt['exchange']['pair_whitelist'].extend(['BTC/USDT', 'ETC/USDT']) default_conf_usdt['trading_mode'] = 'spot' - default_conf_usdt['pairlists'] = [ - {"method": "StaticPairList", "allow_inactive": True}, - {"method": "MarketCapPairList", "mode": "top_rank", "number_assets": 2} - ] + default_conf_usdt['pairlists'] = pairlists mocker.patch(f'{EXMS}.exchange_has', MagicMock(return_value=True)) mocker.patch("freqtrade.plugins.pairlist.MarketCapPairList.CoinGeckoAPI.get_coins_markets", @@ -1564,51 +1587,6 @@ def test_MarketCapPairList_filter(mocker, default_conf_usdt): exchange = get_patched_exchange(mocker, default_conf_usdt) pm = PairListManager(exchange, default_conf_usdt) - - pm.refresh_pairlist() - - whitelist = ['ETH/USDT', 'BTC/USDT'] - - assert whitelist == pm.whitelist - - # Test top 6 mc - default_conf_usdt['pairlists'] = [ - {"method": "StaticPairList", "allow_inactive": True}, - {"method": "MarketCapPairList", "mode": "top_rank", "number_assets": 6} - ] - - pm = PairListManager(exchange, default_conf_usdt) - - pm.refresh_pairlist() - - whitelist = ['ETH/USDT', 'XRP/USDT', 'BTC/USDT'] - - assert whitelist == pm.whitelist - - # Test total assets mode, 2 assets - default_conf_usdt['pairlists'] = [ - {"method": "StaticPairList", "allow_inactive": True}, - {"method": "MarketCapPairList", "mode": "total_assets", "number_assets": 2} - ] - - pm = PairListManager(exchange, default_conf_usdt) - pm.refresh_pairlist() - whitelist = ['BTC/USDT', 'ETH/USDT'] - - assert whitelist == pm.whitelist - - # Test total assets mode, 5 assets - default_conf_usdt['pairlists'] = [ - {"method": "StaticPairList", "allow_inactive": True}, - {"method": "MarketCapPairList", "mode": "total_assets", "number_assets": 5} - ] - - pm = PairListManager(exchange, default_conf_usdt) - - pm.refresh_pairlist() - - whitelist = ['BTC/USDT', 'ETH/USDT', 'XRP/USDT'] - - assert whitelist == pm.whitelist + assert pm.whitelist == result From b72078e76eacb50733158ca78c891ae7b955bcaa Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 27 Jan 2024 16:07:55 +0100 Subject: [PATCH 35/43] use get_pair_base_currency --- freqtrade/plugins/pairlist/MarketCapPairList.py | 2 +- tests/plugins/test_pairlist.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/freqtrade/plugins/pairlist/MarketCapPairList.py b/freqtrade/plugins/pairlist/MarketCapPairList.py index c0232e0d0..afe47bdce 100644 --- a/freqtrade/plugins/pairlist/MarketCapPairList.py +++ b/freqtrade/plugins/pairlist/MarketCapPairList.py @@ -153,7 +153,7 @@ class MarketCapPairList(IPairList): top_marketcap = marketcap_list[:self._number_assets:] for pair in pairlist: - base = pair.split('/')[0] + base = self._exchange.get_pair_base_currency(pair) if base.lower() in top_marketcap: filtered_pairlist.append(pair) else: diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index e19968482..1f3f0d16a 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -1542,7 +1542,7 @@ def test_FullTradesFilter(mocker, default_conf_usdt, fee, caplog) -> None: {"method": "MarketCapPairList", "mode": "total_assets", "number_assets": 5} ], ['ETH/USDT', 'XRP/USDT']) ]) -def test_MarketCapPairList_filter(mocker, default_conf_usdt, pairlists, result): +def test_MarketCapPairList_filter(mocker, default_conf_usdt, markets, pairlists, result): test_value = [ { "symbol": "btc", @@ -1579,7 +1579,10 @@ def test_MarketCapPairList_filter(mocker, default_conf_usdt, pairlists, result): default_conf_usdt['exchange']['pair_whitelist'].extend(['BTC/USDT', 'ETC/USDT']) default_conf_usdt['trading_mode'] = 'spot' default_conf_usdt['pairlists'] = pairlists - mocker.patch(f'{EXMS}.exchange_has', MagicMock(return_value=True)) + mocker.patch.multiple(EXMS, + markets=PropertyMock(return_value=markets), + exchange_has=MagicMock(return_value=True), + ) mocker.patch("freqtrade.plugins.pairlist.MarketCapPairList.CoinGeckoAPI.get_coins_markets", return_value=test_value) From 23ac9e145a91acd91000ec95a675c22b07d4334f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 27 Jan 2024 16:25:00 +0100 Subject: [PATCH 36/43] Fix generator test --- tests/plugins/test_pairlist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 1f3f0d16a..3d26979e1 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -1540,7 +1540,7 @@ def test_FullTradesFilter(mocker, default_conf_usdt, fee, caplog) -> None: ([ # MarketCapPairList as generator {"method": "MarketCapPairList", "mode": "total_assets", "number_assets": 5} - ], ['ETH/USDT', 'XRP/USDT']) + ], ['BTC/USDT', 'ETH/USDT', 'XRP/USDT']) ]) def test_MarketCapPairList_filter(mocker, default_conf_usdt, markets, pairlists, result): test_value = [ From 17af69435f9a1fc06aa7bfad11d357f54a189bf0 Mon Sep 17 00:00:00 2001 From: Stefano Date: Mon, 29 Jan 2024 17:57:14 +0900 Subject: [PATCH 37/43] remove mode, add max_rank, modify test --- .../plugins/pairlist/MarketCapPairList.py | 73 +++++++------------ tests/plugins/test_pairlist.py | 26 +++---- 2 files changed, 39 insertions(+), 60 deletions(-) diff --git a/freqtrade/plugins/pairlist/MarketCapPairList.py b/freqtrade/plugins/pairlist/MarketCapPairList.py index afe47bdce..a618f72d2 100644 --- a/freqtrade/plugins/pairlist/MarketCapPairList.py +++ b/freqtrade/plugins/pairlist/MarketCapPairList.py @@ -18,9 +18,6 @@ from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter logger = logging.getLogger(__name__) -MODE_VALUES = ['top_rank', 'total_assets'] - - class MarketCapPairList(IPairList): is_pairlist_generator = True @@ -36,22 +33,18 @@ class MarketCapPairList(IPairList): 'for "pairlist.config.number_assets"') self._stake_currency = config['stake_currency'] - self._mode = self._pairlistconfig.get('mode', 'top_rank') self._number_assets = self._pairlistconfig['number_assets'] + self._max_rank = self._pairlistconfig.get('max_rank', 30) self._refresh_period = self._pairlistconfig.get('refresh_period', 86400) self._marketcap_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period) self._def_candletype = self._config['candle_type_def'] self._coingekko: CoinGeckoAPI = CoinGeckoAPI() - if self._number_assets > 250: + if self._max_rank > 250: raise OperationalException( - "This filter only support number_assets value up to 250." + "This filter only support marketcap rank up to 250." ) - if not self._validate_keys(self._mode): - raise OperationalException( - f'key {self._mode} not in {MODE_VALUES}') - @property def needstickers(self) -> bool: """ @@ -61,17 +54,13 @@ class MarketCapPairList(IPairList): """ return False - def _validate_keys(self, key): - return key in MODE_VALUES - def short_desc(self) -> str: """ Short whitelist method description - used for startup-messages """ - num = self._pairlistconfig['number_assets'] - msg = f"{self.name} - Only include pairs ranked within top {num} market cap." - if self._mode == "total_assets": - msg = f"{self.name} - top {num} pairs sorted by market cap." + num = self._number_assets + rank = self._max_rank + msg = f"{self.name} - {num} pairs placed within top {rank} market cap." return msg @staticmethod @@ -84,15 +73,14 @@ class MarketCapPairList(IPairList): "number_assets": { "type": "number", "default": 30, - "description": "Max market cap rank", - "help": "Only use assets with high market cap rank", + "description": "Number of assets", + "help": "Number of assets to use from the pairlist", }, - "mode": { - "type": "option", - "default": "top_rank", - "options": MODE_VALUES, - "description": "Mode of number", - "help": "How to interpret the number", + "max_rank": { + "type": "number", + "default": 30, + "description": "Max rank of assets", + "help": "Maximum rank of assets to use from the pairlist", }, "refresh_period": { "type": "number", @@ -149,28 +137,19 @@ class MarketCapPairList(IPairList): if marketcap_list: filtered_pairlist = [] - if self._mode == 'top_rank': - top_marketcap = marketcap_list[:self._number_assets:] - - for pair in pairlist: - base = self._exchange.get_pair_base_currency(pair) - if base.lower() in top_marketcap: - filtered_pairlist.append(pair) - else: - self.log_once(f"Remove {pair} from whitelist because it's not ranked " - f"within top {self._number_assets} market cap", logger.info) - - else: - market = self._config['trading_mode'] - pair_format = f"{self._stake_currency.upper()}" - if (market == 'futures'): - pair_format += f":{self._stake_currency.upper()}" - for mc_pair in marketcap_list: - test_pair = f"{mc_pair.upper()}/{pair_format}" - if test_pair in pairlist: - filtered_pairlist.append(test_pair) - if len(filtered_pairlist) == self._number_assets: - break + market = self._config['trading_mode'] + pair_format = f"{self._stake_currency.upper()}" + if (market == 'futures'): + pair_format += f":{self._stake_currency.upper()}" + + top_marketcap = marketcap_list[:self._max_rank:] + + for mc_pair in top_marketcap: + test_pair = f"{mc_pair.upper()}/{pair_format}" + if test_pair in pairlist: + filtered_pairlist.append(test_pair) + if len(filtered_pairlist) == self._number_assets: + break if len(filtered_pairlist) > 0: return filtered_pairlist diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 3d26979e1..72fce666d 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -1517,29 +1517,29 @@ def test_FullTradesFilter(mocker, default_conf_usdt, fee, caplog) -> None: @pytest.mark.parametrize('pairlists,result', [ ([ - # Test top 2 mc + # Get 2 pairs {"method": "StaticPairList", "allow_inactive": True}, - {"method": "MarketCapPairList", "mode": "top_rank", "number_assets": 2} - ], ['ETH/USDT', 'BTC/USDT']), + {"method": "MarketCapPairList", "number_assets": 2} + ], ['BTC/USDT', 'ETH/USDT']), ([ - # Test top 6 mc + # Get 6 pairs {"method": "StaticPairList", "allow_inactive": True}, - {"method": "MarketCapPairList", "mode": "top_rank", "number_assets": 6} - ], ['ETH/USDT', 'XRP/USDT', 'BTC/USDT']), + {"method": "MarketCapPairList", "number_assets": 6} + ], ['BTC/USDT', 'ETH/USDT', 'XRP/USDT', 'ADA/USDT']), ([ - # Test total assets mode, 2 assets + # Get 3 pairs within top 6 ranks {"method": "StaticPairList", "allow_inactive": True}, - {"method": "MarketCapPairList", "mode": "total_assets", "number_assets": 2} - ], ['BTC/USDT', 'ETH/USDT']), + {"method": "MarketCapPairList", "max_rank": 6, "number_assets": 3} + ], ['BTC/USDT', 'ETH/USDT', 'XRP/USDT']), ([ - # Test total assets mode, 5 assets + # Get 4 pairs within top 8 ranks {"method": "StaticPairList", "allow_inactive": True}, - {"method": "MarketCapPairList", "mode": "total_assets", "number_assets": 5} + {"method": "MarketCapPairList", "max_rank": 8, "number_assets": 4} ], ['BTC/USDT', 'ETH/USDT', 'XRP/USDT']), ([ # MarketCapPairList as generator - {"method": "MarketCapPairList", "mode": "total_assets", "number_assets": 5} + {"method": "MarketCapPairList", "number_assets": 5} ], ['BTC/USDT', 'ETH/USDT', 'XRP/USDT']) ]) def test_MarketCapPairList_filter(mocker, default_conf_usdt, markets, pairlists, result): @@ -1576,7 +1576,7 @@ def test_MarketCapPairList_filter(mocker, default_conf_usdt, markets, pairlists, } ] - default_conf_usdt['exchange']['pair_whitelist'].extend(['BTC/USDT', 'ETC/USDT']) + default_conf_usdt['exchange']['pair_whitelist'].extend(['BTC/USDT', 'ETC/USDT', 'ADA/USDT']) default_conf_usdt['trading_mode'] = 'spot' default_conf_usdt['pairlists'] = pairlists mocker.patch.multiple(EXMS, From 63aac1a2c99ef7f8156f3181c7ef1190154f484e Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Mon, 29 Jan 2024 19:50:19 +0900 Subject: [PATCH 38/43] update docs --- docs/includes/pairlists.md | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 12613adaf..b9074ab6b 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -267,28 +267,22 @@ The optional `bearer_token` will be included in the requests Authorization Heade #### MarketCapPairList -`MarketCapPairList` employs sorting/filtering of pairs by their marketcap rank based of CoinGecko. It will only recognize coins up to the coin placed at rank 250. The number of pairs in the resulted pairlist will be slightly different depends on the `mode` defined in the config (available mode are `top_rank` and `total_assets`). +`MarketCapPairList` employs sorting/filtering of pairs by their marketcap rank based of CoinGecko. It will only recognize coins up to the coin placed at rank 250. The returned pairlist will be sorted based of their marketcap ranks. ```json "pairlists": [ { "method": "MarketCapPairList", - "mode": "top_rank", "number_assets": 20, + "max_rank": 50, "refresh_period": 86400 } ] ``` -##### `top_rank` mode +`number_assets` will defines the maximum number of pairs returned by the pairlist. `max_rank` will defines the maximum rank used in creating/filtering the pairlist. Please note that it's normal that some coins that placed within `max_rank` might not included in the resulted pairlist. It's because some coins might not have active trading pairs in your preferred market/stake/exchange. -In this mode, it will return pairlist consist of active and not-blacklisted pairs that are placed at the top `number_assets` rank of the marketcap. - -##### `total_assets` mode - -In this mode, it will return pairlist consist of `number_assets` number of active and not-blacklisted pairs sorted by their marketcap rank. - -The `refresh_period` setting define the period (in seconds) at which the marketcap rank data will be cached. Defaults to 86,400s (1 day). The pairlist cache (`refresh_period`) is applicable on both generating pairlists (first position in the list) and filtering instances (not the first position in the list). +`refresh_period` setting define the period (in seconds) at which the marketcap rank data will be cached. Defaults to 86,400s (1 day). The pairlist cache (`refresh_period`) is applicable on both generating pairlists (first position in the list) and filtering instances (not the first position in the list). #### AgeFilter From eb7fbb00965168c9bd86b975bdeca5b5f034a3c2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 29 Jan 2024 19:39:02 +0100 Subject: [PATCH 39/43] Improve doc wording --- docs/includes/pairlists.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index b9074ab6b..9781edf10 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -280,9 +280,9 @@ The optional `bearer_token` will be included in the requests Authorization Heade ] ``` -`number_assets` will defines the maximum number of pairs returned by the pairlist. `max_rank` will defines the maximum rank used in creating/filtering the pairlist. Please note that it's normal that some coins that placed within `max_rank` might not included in the resulted pairlist. It's because some coins might not have active trading pairs in your preferred market/stake/exchange. +`number_assets` defines the maximum number of pairs returned by the pairlist. `max_rank` will determine the maximum rank used in creating/filtering the pairlist. It's expected that some coins within the top `max_rank` marketcap will not be included in the resulting pairlist since not all pairs will have active trading pairs in your preferred market/stake/exchange combination. -`refresh_period` setting define the period (in seconds) at which the marketcap rank data will be cached. Defaults to 86,400s (1 day). The pairlist cache (`refresh_period`) is applicable on both generating pairlists (first position in the list) and filtering instances (not the first position in the list). +`refresh_period` setting defines the period (in seconds) at which the marketcap rank data will be refreshed. Defaults to 86,400s (1 day). The pairlist cache (`refresh_period`) is applicable on both generating pairlists (first position in the list) and filtering instances (not the first position in the list). #### AgeFilter From d691ffe9db80f36f67a5bff911914e6535652d25 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 29 Jan 2024 19:46:21 +0100 Subject: [PATCH 40/43] Test more diff. scenario --- tests/plugins/test_pairlist.py | 71 +++++++++++++++------------------- 1 file changed, 32 insertions(+), 39 deletions(-) diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 72fce666d..26227a09f 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -1515,69 +1515,62 @@ def test_FullTradesFilter(mocker, default_conf_usdt, fee, caplog) -> None: assert log_has_re(r'Whitelist with 0 pairs: \[]', caplog) -@pytest.mark.parametrize('pairlists,result', [ +@pytest.mark.parametrize('pairlists,trade_mode,result', [ ([ # Get 2 pairs {"method": "StaticPairList", "allow_inactive": True}, {"method": "MarketCapPairList", "number_assets": 2} - ], ['BTC/USDT', 'ETH/USDT']), + ], 'spot', ['BTC/USDT', 'ETH/USDT']), ([ # Get 6 pairs {"method": "StaticPairList", "allow_inactive": True}, {"method": "MarketCapPairList", "number_assets": 6} - ], ['BTC/USDT', 'ETH/USDT', 'XRP/USDT', 'ADA/USDT']), + ], 'spot', ['BTC/USDT', 'ETH/USDT', 'XRP/USDT', 'ADA/USDT']), ([ # Get 3 pairs within top 6 ranks {"method": "StaticPairList", "allow_inactive": True}, {"method": "MarketCapPairList", "max_rank": 6, "number_assets": 3} - ], ['BTC/USDT', 'ETH/USDT', 'XRP/USDT']), + ], 'spot', ['BTC/USDT', 'ETH/USDT', 'XRP/USDT']), ([ # Get 4 pairs within top 8 ranks {"method": "StaticPairList", "allow_inactive": True}, {"method": "MarketCapPairList", "max_rank": 8, "number_assets": 4} - ], ['BTC/USDT', 'ETH/USDT', 'XRP/USDT']), + ], 'spot', ['BTC/USDT', 'ETH/USDT', 'XRP/USDT']), ([ # MarketCapPairList as generator {"method": "MarketCapPairList", "number_assets": 5} - ], ['BTC/USDT', 'ETH/USDT', 'XRP/USDT']) + ], 'spot', ['BTC/USDT', 'ETH/USDT', 'XRP/USDT']), + ([ + # MarketCapPairList as generator - low max_rank + {"method": "MarketCapPairList", "max_rank": 2, "number_assets": 5} + ], 'spot', ['BTC/USDT', 'ETH/USDT']), + ([ + # MarketCapPairList as generator - futures - low max_rank + {"method": "MarketCapPairList", "max_rank": 2, "number_assets": 5} + ], 'futures', ['ETH/USDT:USDT']), + ([ + # MarketCapPairList as generator - futures - low number_assets + {"method": "MarketCapPairList", "number_assets": 2} + ], 'futures', ['ETH/USDT:USDT', 'ADA/USDT:USDT']), ]) -def test_MarketCapPairList_filter(mocker, default_conf_usdt, markets, pairlists, result): +def test_MarketCapPairList_filter(mocker, default_conf_usdt, trade_mode, markets, pairlists, result): test_value = [ - { - "symbol": "btc", - }, - { - "symbol": "eth", - }, - { - "symbol": "usdt", - }, - { - "symbol": "bnb", - }, - { - "symbol": "sol", - }, - { - "symbol": "xrp", - }, - { - "symbol": "usdc", - }, - { - "symbol": "steth", - }, - { - "symbol": "ada", - }, - { - "symbol": "avax", - } + {"symbol": "btc"}, + {"symbol": "eth"}, + {"symbol": "usdt"}, + {"symbol": "bnb"}, + {"symbol": "sol"}, + {"symbol": "xrp"}, + {"symbol": "usdc"}, + {"symbol": "steth"}, + {"symbol": "ada"}, + {"symbol": "avax"}, ] - default_conf_usdt['exchange']['pair_whitelist'].extend(['BTC/USDT', 'ETC/USDT', 'ADA/USDT']) - default_conf_usdt['trading_mode'] = 'spot' + default_conf_usdt['trading_mode'] = trade_mode + if trade_mode == 'spot': + default_conf_usdt['exchange']['pair_whitelist'].extend(['BTC/USDT', 'ETC/USDT', 'ADA/USDT']) default_conf_usdt['pairlists'] = pairlists mocker.patch.multiple(EXMS, markets=PropertyMock(return_value=markets), From 4a580fc72f9f6e28de0c37eade3b721ad9a45e4d Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 29 Jan 2024 20:01:21 +0100 Subject: [PATCH 41/43] Add test validating cache --- tests/plugins/test_pairlist.py | 51 ++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 26227a09f..c48347f2d 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -18,6 +18,7 @@ from freqtrade.persistence import LocalTrade, Trade from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist, expand_pairlist from freqtrade.plugins.pairlistmanager import PairListManager from freqtrade.resolvers import PairListResolver +from freqtrade.util.datetime_helpers import dt_now from tests.conftest import (EXMS, create_mock_trades_usdt, get_patched_exchange, get_patched_freqtradebot, log_has, log_has_re, num_log_has) @@ -1586,3 +1587,53 @@ def test_MarketCapPairList_filter(mocker, default_conf_usdt, trade_mode, markets pm.refresh_pairlist() assert pm.whitelist == result + + +def test_MarketCapPairList_timing(mocker, default_conf_usdt, markets, time_machine): + test_value = [ + {"symbol": "btc"}, + {"symbol": "eth"}, + {"symbol": "usdt"}, + {"symbol": "bnb"}, + {"symbol": "sol"}, + {"symbol": "xrp"}, + {"symbol": "usdc"}, + {"symbol": "steth"}, + {"symbol": "ada"}, + {"symbol": "avax"}, + ] + + default_conf_usdt['trading_mode'] = 'spot' + default_conf_usdt['exchange']['pair_whitelist'].extend(['BTC/USDT', 'ETC/USDT', 'ADA/USDT']) + default_conf_usdt['pairlists'] = [{"method": "MarketCapPairList", "number_assets": 2}] + + markets_mock = MagicMock(return_value=markets) + mocker.patch.multiple(EXMS, + get_markets=markets_mock, + exchange_has=MagicMock(return_value=True), + ) + + mocker.patch("freqtrade.plugins.pairlist.MarketCapPairList.CoinGeckoAPI.get_coins_markets", + return_value=test_value) + + start_dt = dt_now() + + exchange = get_patched_exchange(mocker, default_conf_usdt) + time_machine.move_to(start_dt) + + pm = PairListManager(exchange, default_conf_usdt) + markets_mock.reset_mock() + pm.refresh_pairlist() + assert markets_mock.call_count == 3 + markets_mock.reset_mock() + + time_machine.move_to(start_dt + timedelta(hours=20)) + pm.refresh_pairlist() + # Cached pairlist ... + assert markets_mock.call_count == 1 + + markets_mock.reset_mock() + time_machine.move_to(start_dt + timedelta(days=2)) + pm.refresh_pairlist() + # No longer cached pairlist ... + assert markets_mock.call_count == 3 From 01baea8aab84abdfbe32d4677dd5b2e4fdedcb7d Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 29 Jan 2024 20:08:25 +0100 Subject: [PATCH 42/43] Test exceptions / errors --- tests/plugins/test_pairlist.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index c48347f2d..c3a7e4c66 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -1637,3 +1637,19 @@ def test_MarketCapPairList_timing(mocker, default_conf_usdt, markets, time_machi pm.refresh_pairlist() # No longer cached pairlist ... assert markets_mock.call_count == 3 + + +def test_MarketCapPairList_exceptions(mocker, default_conf_usdt, markets, time_machine): + + exchange = get_patched_exchange(mocker, default_conf_usdt) + default_conf_usdt['pairlists'] = [{"method": "MarketCapPairList"}] + with pytest.raises(OperationalException, match=r"`number_assets` not specified.*"): + # No number_assets + PairListManager(exchange, default_conf_usdt) + + default_conf_usdt['pairlists'] = [{ + "method": "MarketCapPairList", 'number_assets': 20, 'max_rank': 260 + }] + with pytest.raises(OperationalException, + match="This filter only support marketcap rank up to 250."): + PairListManager(exchange, default_conf_usdt) From 470a239e8279be842f998062815183e52c982ac8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 29 Jan 2024 20:10:50 +0100 Subject: [PATCH 43/43] Formatting ... --- tests/plugins/test_pairlist.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index c3a7e4c66..09dcd0af3 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -1555,7 +1555,9 @@ def test_FullTradesFilter(mocker, default_conf_usdt, fee, caplog) -> None: {"method": "MarketCapPairList", "number_assets": 2} ], 'futures', ['ETH/USDT:USDT', 'ADA/USDT:USDT']), ]) -def test_MarketCapPairList_filter(mocker, default_conf_usdt, trade_mode, markets, pairlists, result): +def test_MarketCapPairList_filter( + mocker, default_conf_usdt, trade_mode, markets, pairlists, result +): test_value = [ {"symbol": "btc"}, {"symbol": "eth"},