add more tests for datakitchen functionalities, add regression tests for freqai_interface train/backtest
robcaulk
4 years ago
parent
9c051958a6
commit
d43c146676
@ -1,95 +0,0 @@
|
||||
# from unittest.mock import MagicMock
|
||||
# from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_edge
|
||||
import copy
|
||||
|
||||
import pytest
|
||||
|
||||
from freqtrade.configuration import TimeRange
|
||||
from freqtrade.data.dataprovider import DataProvider
|
||||
# from freqtrade.freqai.data_drawer import FreqaiDataDrawer
|
||||
from freqtrade.exceptions import OperationalException
|
||||
from freqtrade.freqai.data_kitchen import FreqaiDataKitchen
|
||||
from tests.conftest import get_patched_exchange
|
||||
from tests.freqai.conftest import freqai_conf, get_patched_data_kitchen, get_patched_strategy
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"timerange, train_period_days, expected_result",
|
||||
[
|
||||
("20220101-20220201", 30, "20211202-20220201"),
|
||||
("20220301-20220401", 15, "20220214-20220401"),
|
||||
],
|
||||
)
|
||||
def test_create_fulltimerange(
|
||||
timerange, train_period_days, expected_result, default_conf, mocker, caplog
|
||||
):
|
||||
dk = get_patched_data_kitchen(mocker, freqai_conf(copy.deepcopy(default_conf)))
|
||||
assert dk.create_fulltimerange(timerange, train_period_days) == expected_result
|
||||
|
||||
|
||||
def test_create_fulltimerange_incorrect_backtest_period(mocker, default_conf):
|
||||
dk = get_patched_data_kitchen(mocker, freqai_conf(copy.deepcopy(default_conf)))
|
||||
with pytest.raises(OperationalException, match=r"backtest_period_days must be an integer"):
|
||||
dk.create_fulltimerange("20220101-20220201", 0.5)
|
||||
with pytest.raises(OperationalException, match=r"backtest_period_days must be positive"):
|
||||
dk.create_fulltimerange("20220101-20220201", -1)
|
||||
|
||||
|
||||
def test_split_timerange(mocker, default_conf):
|
||||
freqaiconf = freqai_conf(copy.deepcopy(default_conf))
|
||||
freqaiconf.update({"timerange": "20220101-20220401"})
|
||||
dk = get_patched_data_kitchen(mocker, freqaiconf)
|
||||
tr_list, bt_list = dk.split_timerange("20220101-20220201", 30, 7)
|
||||
assert len(tr_list) == len(bt_list) == 9
|
||||
|
||||
tr_list, bt_list = dk.split_timerange("20220101-20220201", 30, 0.5)
|
||||
assert len(tr_list) == len(bt_list) == 120
|
||||
|
||||
tr_list, bt_list = dk.split_timerange("20220101-20220201", 10, 1)
|
||||
assert len(tr_list) == len(bt_list) == 80
|
||||
|
||||
with pytest.raises(
|
||||
OperationalException, match=r"train_period_days must be an integer greater than 0."
|
||||
):
|
||||
dk.split_timerange("20220101-20220201", -1, 0.5)
|
||||
|
||||
|
||||
def test_update_historic_data(mocker, default_conf):
|
||||
freqaiconf = freqai_conf(copy.deepcopy(default_conf))
|
||||
strategy = get_patched_strategy(mocker, freqaiconf)
|
||||
exchange = get_patched_exchange(mocker, freqaiconf)
|
||||
strategy.dp = DataProvider(freqaiconf, exchange)
|
||||
freqai = strategy.model.bridge
|
||||
freqai.live = True
|
||||
freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd)
|
||||
timerange = TimeRange.parse_timerange("20180110-20180114")
|
||||
|
||||
freqai.dk.load_all_pair_histories(timerange)
|
||||
historic_candles = len(freqai.dd.historic_data["ADA/BTC"]["5m"])
|
||||
dp_candles = len(strategy.dp.get_pair_dataframe("ADA/BTC", "5m"))
|
||||
candle_difference = dp_candles - historic_candles
|
||||
freqai.dk.update_historic_data(strategy)
|
||||
|
||||
updated_historic_candles = len(freqai.dd.historic_data["ADA/BTC"]["5m"])
|
||||
|
||||
assert updated_historic_candles - historic_candles == candle_difference
|
||||
|
||||
|
||||
# def generate_test_data(timeframe: str, size: int, start: str = '2020-07-05'):
|
||||
# np.random.seed(42)
|
||||
# tf_mins = timeframe_to_minutes(timeframe)
|
||||
|
||||
# base = np.random.normal(20, 2, size=size)
|
||||
|
||||
# date = pd.date_range(start, periods=size, freq=f'{tf_mins}min', tz='UTC')
|
||||
# df = pd.DataFrame({
|
||||
# 'date': date,
|
||||
# 'open': base,
|
||||
# 'high': base + np.random.normal(2, 1, size=size),
|
||||
# 'low': base - np.random.normal(2, 1, size=size),
|
||||
# 'close': base + np.random.normal(0, 1, size=size),
|
||||
# 'volume': np.random.normal(200, size=size)
|
||||
# }
|
||||
# )
|
||||
# df = df.dropna()
|
||||
# return df
|
||||
@ -0,0 +1,167 @@
|
||||
# from unittest.mock import MagicMock
|
||||
# from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_edge
|
||||
import copy
|
||||
import datetime
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from freqtrade.configuration import TimeRange
|
||||
from freqtrade.data.dataprovider import DataProvider
|
||||
# from freqtrade.freqai.data_drawer import FreqaiDataDrawer
|
||||
from freqtrade.exceptions import OperationalException
|
||||
from freqtrade.freqai.data_kitchen import FreqaiDataKitchen
|
||||
from tests.conftest import get_patched_exchange
|
||||
from tests.freqai.conftest import freqai_conf, get_patched_data_kitchen, get_patched_freqai_strategy
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"timerange, train_period_days, expected_result",
|
||||
[
|
||||
("20220101-20220201", 30, "20211202-20220201"),
|
||||
("20220301-20220401", 15, "20220214-20220401"),
|
||||
],
|
||||
)
|
||||
def test_create_fulltimerange(
|
||||
timerange, train_period_days, expected_result, default_conf, mocker, caplog
|
||||
):
|
||||
dk = get_patched_data_kitchen(mocker, freqai_conf(copy.deepcopy(default_conf)))
|
||||
assert dk.create_fulltimerange(timerange, train_period_days) == expected_result
|
||||
shutil.rmtree(Path(dk.full_path))
|
||||
|
||||
|
||||
def test_create_fulltimerange_incorrect_backtest_period(mocker, default_conf):
|
||||
dk = get_patched_data_kitchen(mocker, freqai_conf(copy.deepcopy(default_conf)))
|
||||
with pytest.raises(OperationalException, match=r"backtest_period_days must be an integer"):
|
||||
dk.create_fulltimerange("20220101-20220201", 0.5)
|
||||
with pytest.raises(OperationalException, match=r"backtest_period_days must be positive"):
|
||||
dk.create_fulltimerange("20220101-20220201", -1)
|
||||
shutil.rmtree(Path(dk.full_path))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"timerange, train_period_days, backtest_period_days, expected_result",
|
||||
[
|
||||
("20220101-20220201", 30, 7, 9),
|
||||
("20220101-20220201", 30, 0.5, 120),
|
||||
("20220101-20220201", 10, 1, 80),
|
||||
],
|
||||
)
|
||||
def test_split_timerange(
|
||||
mocker, default_conf, timerange, train_period_days, backtest_period_days, expected_result
|
||||
):
|
||||
freqaiconf = freqai_conf(copy.deepcopy(default_conf))
|
||||
freqaiconf.update({"timerange": "20220101-20220401"})
|
||||
dk = get_patched_data_kitchen(mocker, freqaiconf)
|
||||
tr_list, bt_list = dk.split_timerange(timerange, train_period_days, backtest_period_days)
|
||||
assert len(tr_list) == len(bt_list) == expected_result
|
||||
|
||||
with pytest.raises(
|
||||
OperationalException, match=r"train_period_days must be an integer greater than 0."
|
||||
):
|
||||
dk.split_timerange("20220101-20220201", -1, 0.5)
|
||||
shutil.rmtree(Path(dk.full_path))
|
||||
|
||||
|
||||
def test_update_historic_data(mocker, default_conf):
|
||||
freqaiconf = freqai_conf(copy.deepcopy(default_conf))
|
||||
strategy = get_patched_freqai_strategy(mocker, freqaiconf)
|
||||
exchange = get_patched_exchange(mocker, freqaiconf)
|
||||
strategy.dp = DataProvider(freqaiconf, exchange)
|
||||
freqai = strategy.model.bridge
|
||||
freqai.live = True
|
||||
freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd)
|
||||
timerange = TimeRange.parse_timerange("20180110-20180114")
|
||||
|
||||
freqai.dk.load_all_pair_histories(timerange)
|
||||
historic_candles = len(freqai.dd.historic_data["ADA/BTC"]["5m"])
|
||||
dp_candles = len(strategy.dp.get_pair_dataframe("ADA/BTC", "5m"))
|
||||
candle_difference = dp_candles - historic_candles
|
||||
freqai.dk.update_historic_data(strategy)
|
||||
|
||||
updated_historic_candles = len(freqai.dd.historic_data["ADA/BTC"]["5m"])
|
||||
|
||||
assert updated_historic_candles - historic_candles == candle_difference
|
||||
shutil.rmtree(Path(freqai.dk.full_path))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"timestamp, expected",
|
||||
[
|
||||
(datetime.datetime.now(tz=datetime.timezone.utc).timestamp() - 7200, True),
|
||||
(datetime.datetime.now(tz=datetime.timezone.utc).timestamp(), False),
|
||||
],
|
||||
)
|
||||
def test_check_if_model_expired(mocker, default_conf, timestamp, expected):
|
||||
freqaiconf = freqai_conf(copy.deepcopy(default_conf))
|
||||
dk = get_patched_data_kitchen(mocker, freqaiconf)
|
||||
assert dk.check_if_model_expired(timestamp) == expected
|
||||
shutil.rmtree(Path(dk.full_path))
|
||||
|
||||
|
||||
def test_load_all_pairs_histories(mocker, default_conf):
|
||||
freqaiconf = freqai_conf(copy.deepcopy(default_conf))
|
||||
strategy = get_patched_freqai_strategy(mocker, freqaiconf)
|
||||
exchange = get_patched_exchange(mocker, freqaiconf)
|
||||
strategy.dp = DataProvider(freqaiconf, exchange)
|
||||
freqai = strategy.model.bridge
|
||||
freqai.live = True
|
||||
freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd)
|
||||
timerange = TimeRange.parse_timerange("20180110-20180114")
|
||||
freqai.dk.load_all_pair_histories(timerange)
|
||||
|
||||
assert len(freqai.dd.historic_data.keys()) == len(
|
||||
freqaiconf.get("exchange", {}).get("pair_whitelist")
|
||||
)
|
||||
assert len(freqai.dd.historic_data["ADA/BTC"]) == len(
|
||||
freqaiconf.get("freqai", {}).get("feature_parameters", {}).get("include_timeframes")
|
||||
)
|
||||
shutil.rmtree(Path(freqai.dk.full_path))
|
||||
|
||||
|
||||
def test_get_base_and_corr_dataframes(mocker, default_conf):
|
||||
freqaiconf = freqai_conf(copy.deepcopy(default_conf))
|
||||
strategy = get_patched_freqai_strategy(mocker, freqaiconf)
|
||||
exchange = get_patched_exchange(mocker, freqaiconf)
|
||||
strategy.dp = DataProvider(freqaiconf, exchange)
|
||||
freqai = strategy.model.bridge
|
||||
freqai.live = True
|
||||
freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd)
|
||||
timerange = TimeRange.parse_timerange("20180110-20180114")
|
||||
freqai.dk.load_all_pair_histories(timerange)
|
||||
sub_timerange = TimeRange.parse_timerange("20180111-20180114")
|
||||
corr_df, base_df = freqai.dk.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC")
|
||||
|
||||
num_tfs = len(
|
||||
freqaiconf.get("freqai", {}).get("feature_parameters", {}).get("include_timeframes")
|
||||
)
|
||||
|
||||
assert len(base_df.keys()) == num_tfs
|
||||
|
||||
assert len(corr_df.keys()) == len(
|
||||
freqaiconf.get("freqai", {}).get("feature_parameters", {}).get("include_corr_pairlist")
|
||||
)
|
||||
|
||||
assert len(corr_df["ADA/BTC"].keys()) == num_tfs
|
||||
shutil.rmtree(Path(freqai.dk.full_path))
|
||||
|
||||
|
||||
def test_use_strategy_to_populate_indicators(mocker, default_conf):
|
||||
freqaiconf = freqai_conf(copy.deepcopy(default_conf))
|
||||
strategy = get_patched_freqai_strategy(mocker, freqaiconf)
|
||||
exchange = get_patched_exchange(mocker, freqaiconf)
|
||||
strategy.dp = DataProvider(freqaiconf, exchange)
|
||||
strategy.freqai_info = freqaiconf.get("freqai", {})
|
||||
freqai = strategy.model.bridge
|
||||
freqai.live = True
|
||||
freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd)
|
||||
timerange = TimeRange.parse_timerange("20180110-20180114")
|
||||
freqai.dk.load_all_pair_histories(timerange)
|
||||
sub_timerange = TimeRange.parse_timerange("20180111-20180114")
|
||||
corr_df, base_df = freqai.dk.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC")
|
||||
|
||||
df = freqai.dk.use_strategy_to_populate_indicators(strategy, corr_df, base_df, 'LTC/BTC')
|
||||
|
||||
assert len(df.columns) == 90
|
||||
shutil.rmtree(Path(freqai.dk.full_path))
|
||||
@ -0,0 +1,183 @@
|
||||
# from unittest.mock import MagicMock
|
||||
# from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_edge
|
||||
import copy
|
||||
import platform
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from freqtrade.configuration import TimeRange
|
||||
from freqtrade.data.dataprovider import DataProvider
|
||||
# from freqtrade.freqai.data_drawer import FreqaiDataDrawer
|
||||
from freqtrade.freqai.data_kitchen import FreqaiDataKitchen
|
||||
from tests.conftest import get_patched_exchange, log_has
|
||||
from tests.freqai.conftest import freqai_conf, get_patched_freqai_strategy
|
||||
|
||||
|
||||
def test_train_model_in_series_LightGBM(mocker, default_conf):
|
||||
freqaiconf = freqai_conf(copy.deepcopy(default_conf))
|
||||
freqaiconf.update({"timerange": "20180110-20180130"})
|
||||
|
||||
strategy = get_patched_freqai_strategy(mocker, freqaiconf)
|
||||
exchange = get_patched_exchange(mocker, freqaiconf)
|
||||
strategy.dp = DataProvider(freqaiconf, exchange)
|
||||
strategy.freqai_info = freqaiconf.get("freqai", {})
|
||||
freqai = strategy.model.bridge
|
||||
freqai.live = True
|
||||
freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd)
|
||||
timerange = TimeRange.parse_timerange("20180110-20180130")
|
||||
freqai.dk.load_all_pair_histories(timerange)
|
||||
|
||||
freqai.dd.pair_dict = MagicMock()
|
||||
|
||||
data_load_timerange = TimeRange.parse_timerange("20180110-20180130")
|
||||
new_timerange = TimeRange.parse_timerange("20180120-20180130")
|
||||
|
||||
freqai.train_model_in_series(new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange)
|
||||
|
||||
assert (
|
||||
Path(freqai.dk.data_path / str(freqai.dk.model_filename + "_model.joblib"))
|
||||
.resolve()
|
||||
.exists()
|
||||
)
|
||||
assert (
|
||||
Path(freqai.dk.data_path / str(freqai.dk.model_filename + "_metadata.json"))
|
||||
.resolve()
|
||||
.exists()
|
||||
)
|
||||
assert (
|
||||
Path(freqai.dk.data_path / str(freqai.dk.model_filename + "_trained_df.pkl"))
|
||||
.resolve()
|
||||
.exists()
|
||||
)
|
||||
assert (
|
||||
Path(freqai.dk.data_path / str(freqai.dk.model_filename + "_svm_model.joblib"))
|
||||
.resolve()
|
||||
.exists()
|
||||
)
|
||||
|
||||
shutil.rmtree(Path(freqai.dk.full_path))
|
||||
|
||||
|
||||
# Catboost not available for ARM architecture. using platform lib to check processor type
|
||||
if "arm" not in platform.uname()[-1]:
|
||||
|
||||
def test_train_model_in_series_Catboost(mocker, default_conf):
|
||||
freqaiconf = freqai_conf(copy.deepcopy(default_conf))
|
||||
freqaiconf.update({"timerange": "20180110-20180130"})
|
||||
freqaiconf.update({"freqaimodel": "CatboostPredictionModel"})
|
||||
strategy = get_patched_freqai_strategy(mocker, freqaiconf)
|
||||
exchange = get_patched_exchange(mocker, freqaiconf)
|
||||
strategy.dp = DataProvider(freqaiconf, exchange)
|
||||
strategy.freqai_info = freqaiconf.get("freqai", {})
|
||||
freqai = strategy.model.bridge
|
||||
freqai.live = True
|
||||
freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd)
|
||||
timerange = TimeRange.parse_timerange("20180110-20180130")
|
||||
freqai.dk.load_all_pair_histories(timerange)
|
||||
|
||||
freqai.dd.pair_dict = MagicMock()
|
||||
|
||||
data_load_timerange = TimeRange.parse_timerange("20180110-20180130")
|
||||
new_timerange = TimeRange.parse_timerange("20180120-20180130")
|
||||
|
||||
freqai.train_model_in_series(
|
||||
new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange
|
||||
)
|
||||
|
||||
assert (
|
||||
Path(freqai.dk.data_path / str(freqai.dk.model_filename + "_model.joblib"))
|
||||
.resolve()
|
||||
.exists()
|
||||
)
|
||||
assert (
|
||||
Path(freqai.dk.data_path / str(freqai.dk.model_filename + "_metadata.json"))
|
||||
.resolve()
|
||||
.exists()
|
||||
)
|
||||
assert (
|
||||
Path(freqai.dk.data_path / str(freqai.dk.model_filename + "_trained_df.pkl"))
|
||||
.resolve()
|
||||
.exists()
|
||||
)
|
||||
assert (
|
||||
Path(freqai.dk.data_path / str(freqai.dk.model_filename + "_svm_model.joblib"))
|
||||
.resolve()
|
||||
.exists()
|
||||
)
|
||||
|
||||
shutil.rmtree(Path(freqai.dk.full_path))
|
||||
|
||||
|
||||
def test_start_backtesting(mocker, default_conf):
|
||||
freqaiconf = freqai_conf(copy.deepcopy(default_conf))
|
||||
freqaiconf.update({"timerange": "20180120-20180130"})
|
||||
strategy = get_patched_freqai_strategy(mocker, freqaiconf)
|
||||
exchange = get_patched_exchange(mocker, freqaiconf)
|
||||
strategy.dp = DataProvider(freqaiconf, exchange)
|
||||
strategy.freqai_info = freqaiconf.get("freqai", {})
|
||||
freqai = strategy.model.bridge
|
||||
freqai.live = False
|
||||
freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd)
|
||||
timerange = TimeRange.parse_timerange("20180110-20180130")
|
||||
freqai.dk.load_all_pair_histories(timerange)
|
||||
sub_timerange = TimeRange.parse_timerange("20180110-20180130")
|
||||
corr_df, base_df = freqai.dk.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC")
|
||||
|
||||
df = freqai.dk.use_strategy_to_populate_indicators(strategy, corr_df, base_df, "LTC/BTC")
|
||||
|
||||
metadata = {"pair": "ADA/BTC"}
|
||||
freqai.start_backtesting(df, metadata, freqai.dk)
|
||||
model_folders = [x for x in freqai.dd.full_path.iterdir() if x.is_dir()]
|
||||
|
||||
assert len(model_folders) == 5
|
||||
|
||||
shutil.rmtree(Path(freqai.dk.full_path))
|
||||
|
||||
|
||||
def test_start_backtesting_from_existing_folder(mocker, default_conf, caplog):
|
||||
freqaiconf = freqai_conf(copy.deepcopy(default_conf))
|
||||
freqaiconf.update({"timerange": "20180120-20180130"})
|
||||
strategy = get_patched_freqai_strategy(mocker, freqaiconf)
|
||||
exchange = get_patched_exchange(mocker, freqaiconf)
|
||||
strategy.dp = DataProvider(freqaiconf, exchange)
|
||||
strategy.freqai_info = freqaiconf.get("freqai", {})
|
||||
freqai = strategy.model.bridge
|
||||
freqai.live = False
|
||||
freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd)
|
||||
timerange = TimeRange.parse_timerange("20180110-20180130")
|
||||
freqai.dk.load_all_pair_histories(timerange)
|
||||
sub_timerange = TimeRange.parse_timerange("20180110-20180130")
|
||||
corr_df, base_df = freqai.dk.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC")
|
||||
|
||||
df = freqai.dk.use_strategy_to_populate_indicators(strategy, corr_df, base_df, "LTC/BTC")
|
||||
|
||||
metadata = {"pair": "ADA/BTC"}
|
||||
freqai.start_backtesting(df, metadata, freqai.dk)
|
||||
model_folders = [x for x in freqai.dd.full_path.iterdir() if x.is_dir()]
|
||||
|
||||
assert len(model_folders) == 5
|
||||
|
||||
# without deleting the exiting folder structure, re-run
|
||||
|
||||
freqaiconf.update({"timerange": "20180120-20180130"})
|
||||
strategy = get_patched_freqai_strategy(mocker, freqaiconf)
|
||||
exchange = get_patched_exchange(mocker, freqaiconf)
|
||||
strategy.dp = DataProvider(freqaiconf, exchange)
|
||||
strategy.freqai_info = freqaiconf.get("freqai", {})
|
||||
freqai = strategy.model.bridge
|
||||
freqai.live = False
|
||||
freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd)
|
||||
timerange = TimeRange.parse_timerange("20180110-20180130")
|
||||
freqai.dk.load_all_pair_histories(timerange)
|
||||
sub_timerange = TimeRange.parse_timerange("20180110-20180130")
|
||||
corr_df, base_df = freqai.dk.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC")
|
||||
|
||||
df = freqai.dk.use_strategy_to_populate_indicators(strategy, corr_df, base_df, "LTC/BTC")
|
||||
freqai.start_backtesting(df, metadata, freqai.dk)
|
||||
assert log_has(
|
||||
"Found model at user_data/models/uniqe-id100/sub-train-ADA1517097600/cb_ada_1517097600",
|
||||
caplog,
|
||||
)
|
||||
|
||||
shutil.rmtree(Path(freqai.dk.full_path))
|
||||
Loading…
Reference in new issue