diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index bddf311c6..3c9e78132 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -264,7 +264,7 @@ def convert_trades_format(config: Config, convert_from: str, convert_to: str, er logger.info(f"Converting trades for {config['pairs']}") for pair in config['pairs']: - data = src.trades_load_aslist(pair=pair) + data = src.trades_load(pair=pair) logger.info(f"Converting {len(data)} trades for {pair}") trg.trades_store(pair, data) if erase and convert_from != convert_to: diff --git a/freqtrade/data/history/featherdatahandler.py b/freqtrade/data/history/featherdatahandler.py index b8efe6d02..b70bb5720 100644 --- a/freqtrade/data/history/featherdatahandler.py +++ b/freqtrade/data/history/featherdatahandler.py @@ -4,7 +4,7 @@ from typing import Optional from pandas import DataFrame, read_feather, to_datetime from freqtrade.configuration import TimeRange -from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TradeList +from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS from freqtrade.enums import CandleType from .idatahandler import IDataHandler @@ -82,18 +82,17 @@ class FeatherDataHandler(IDataHandler): """ raise NotImplementedError() - def trades_store(self, pair: str, data: TradeList) -> None: + def trades_store(self, pair: str, data: DataFrame) -> None: """ Store trades data (list of Dicts) to file :param pair: Pair - used for filename - :param data: List of Lists containing trade data, + :param data: Dataframe containing trades column sequence as in DEFAULT_TRADES_COLUMNS """ filename = self._pair_trades_filename(self._datadir, pair) self.create_dir_if_needed(filename) - tradesdata = DataFrame(data, columns=DEFAULT_TRADES_COLUMNS) - tradesdata.to_feather(filename, compression_level=9, compression='lz4') + data.to_feather(filename, compression_level=9, compression='lz4') def trades_append(self, pair: str, data: DataFrame): """ diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index 38747ed5f..0ddbc0b3b 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -5,7 +5,7 @@ import numpy as np import pandas as pd from freqtrade.configuration import TimeRange -from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TradeList +from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS from freqtrade.enums import CandleType from .idatahandler import IDataHandler @@ -100,16 +100,16 @@ class HDF5DataHandler(IDataHandler): """ raise NotImplementedError() - def trades_store(self, pair: str, data: TradeList) -> None: + def trades_store(self, pair: str, data: pd.DataFrame) -> None: """ Store trades data (list of Dicts) to file :param pair: Pair - used for filename - :param data: List of Lists containing trade data, + :param data: Dataframe containing trades column sequence as in DEFAULT_TRADES_COLUMNS """ key = self._pair_trades_key(pair) - pd.DataFrame(data, columns=DEFAULT_TRADES_COLUMNS).to_hdf( + data.to_hdf( self._pair_trades_filename(self._datadir, pair), key, mode='a', complevel=9, complib='blosc', format='table', data_columns=['timestamp'] diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index fe2bb1427..5e940134c 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -171,11 +171,11 @@ class IDataHandler(ABC): return [cls.rebuild_pair_from_filename(match[0]) for match in _tmp if match] @abstractmethod - def trades_store(self, pair: str, data: TradeList) -> None: + def trades_store(self, pair: str, data: DataFrame) -> None: """ Store trades data (list of Dicts) to file :param pair: Pair - used for filename - :param data: List of Lists containing trade data, + :param data: Dataframe containing trades column sequence as in DEFAULT_TRADES_COLUMNS """ diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 4e0a5dc1a..b6f5a5131 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -6,7 +6,7 @@ from pandas import DataFrame, read_json, to_datetime from freqtrade import misc from freqtrade.configuration import TimeRange -from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TradeList +from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS from freqtrade.data.converter import trades_dict_to_list from freqtrade.enums import CandleType @@ -94,15 +94,17 @@ class JsonDataHandler(IDataHandler): """ raise NotImplementedError() - def trades_store(self, pair: str, data: TradeList) -> None: + def trades_store(self, pair: str, data: DataFrame) -> None: """ Store trades data (list of Dicts) to file :param pair: Pair - used for filename - :param data: List of Lists containing trade data, + :param data: Dataframe containing trades column sequence as in DEFAULT_TRADES_COLUMNS """ filename = self._pair_trades_filename(self._datadir, pair) - misc.file_dump_json(filename, data, is_zip=self._use_zip) + data.loc[:, 'timestamp'] = data.loc[:, 'timestamp'].view(np.int64) // 1000 // 1000 + trades = data[DEFAULT_TRADES_COLUMNS].values.tolist() + misc.file_dump_json(filename, trades, is_zip=self._use_zip) def trades_append(self, pair: str, data: DataFrame): """ diff --git a/freqtrade/data/history/parquetdatahandler.py b/freqtrade/data/history/parquetdatahandler.py index 085ff8be6..254fa0df4 100644 --- a/freqtrade/data/history/parquetdatahandler.py +++ b/freqtrade/data/history/parquetdatahandler.py @@ -81,11 +81,11 @@ class ParquetDataHandler(IDataHandler): """ raise NotImplementedError() - def trades_store(self, pair: str, data: TradeList) -> None: + def trades_store(self, pair: str, data: DataFrame) -> None: """ Store trades data (list of Dicts) to file :param pair: Pair - used for filename - :param data: List of Lists containing trade data, + :param data: Dataframe containing trades column sequence as in DEFAULT_TRADES_COLUMNS """ # filename = self._pair_trades_filename(self._datadir, pair) diff --git a/tests/data/test_datahandler.py b/tests/data/test_datahandler.py index b0c2149d4..f0c367efe 100644 --- a/tests/data/test_datahandler.py +++ b/tests/data/test_datahandler.py @@ -7,6 +7,7 @@ from unittest.mock import MagicMock import pytest from pandas import DataFrame, Timestamp +from pandas.testing import assert_frame_equal from freqtrade.configuration import TimeRange from freqtrade.constants import AVAILABLE_DATAHANDLERS @@ -331,30 +332,18 @@ def test_hdf5datahandler_trades_load(testdatadir): def test_hdf5datahandler_trades_store(testdatadir, tmpdir): tmpdir1 = Path(tmpdir) dh = get_datahandler(testdatadir, 'hdf5') - trades = dh.trades_load_aslist('XRP/ETH') + trades = dh.trades_load('XRP/ETH') dh1 = get_datahandler(tmpdir1, 'hdf5') dh1.trades_store('XRP/NEW', trades) file = tmpdir1 / 'XRP_NEW-trades.h5' assert file.is_file() # Load trades back - trades_new = dh1.trades_load_aslist('XRP/NEW') + trades_new = dh1.trades_load('XRP/NEW') assert len(trades_new) == len(trades) - assert trades[0][0] == trades_new[0][0] - assert trades[0][1] == trades_new[0][1] - # assert trades[0][2] == trades_new[0][2] # This is nan - so comparison does not make sense - assert trades[0][3] == trades_new[0][3] - assert trades[0][4] == trades_new[0][4] - assert trades[0][5] == trades_new[0][5] - assert trades[0][6] == trades_new[0][6] - assert trades[-1][0] == trades_new[-1][0] - assert trades[-1][1] == trades_new[-1][1] - # assert trades[-1][2] == trades_new[-1][2] # This is nan - so comparison does not make sense - assert trades[-1][3] == trades_new[-1][3] - assert trades[-1][4] == trades_new[-1][4] - assert trades[-1][5] == trades_new[-1][5] - assert trades[-1][6] == trades_new[-1][6] + assert_frame_equal(trades, trades_new, check_exact=True) + assert len(trades_new) == len(trades) def test_hdf5datahandler_trades_purge(mocker, testdatadir): @@ -508,30 +497,16 @@ def test_featherdatahandler_trades_load(testdatadir): def test_featherdatahandler_trades_store(testdatadir, tmpdir): tmpdir1 = Path(tmpdir) dh = get_datahandler(testdatadir, 'feather') - trades = dh.trades_load_aslist('XRP/ETH') + trades = dh.trades_load('XRP/ETH') dh1 = get_datahandler(tmpdir1, 'feather') dh1.trades_store('XRP/NEW', trades) file = tmpdir1 / 'XRP_NEW-trades.feather' assert file.is_file() # Load trades back - trades_new = dh1.trades_load_aslist('XRP/NEW') - + trades_new = dh1.trades_load('XRP/NEW') + assert_frame_equal(trades, trades_new, check_exact=True) assert len(trades_new) == len(trades) - assert trades[0][0] == trades_new[0][0] - assert trades[0][1] == trades_new[0][1] - # assert trades[0][2] == trades_new[0][2] # This is nan - so comparison does not make sense - assert trades[0][3] == trades_new[0][3] - assert trades[0][4] == trades_new[0][4] - assert trades[0][5] == trades_new[0][5] - assert trades[0][6] == trades_new[0][6] - assert trades[-1][0] == trades_new[-1][0] - assert trades[-1][1] == trades_new[-1][1] - # assert trades[-1][2] == trades_new[-1][2] # This is nan - so comparison does not make sense - assert trades[-1][3] == trades_new[-1][3] - assert trades[-1][4] == trades_new[-1][4] - assert trades[-1][5] == trades_new[-1][5] - assert trades[-1][6] == trades_new[-1][6] def test_featherdatahandler_trades_purge(mocker, testdatadir):