Add conversion function from fetch_trades to list

freqtrade/constants.py

@@ -24,7 +24,9 @@ AVAILABLE_DATAHANDLERS = ['json', 'jsongz']
 DRY_RUN_WALLET = 1000
 MATH_CLOSE_PREC = 1e-14  # Precision used for float comparisons
 DEFAULT_DATAFRAME_COLUMNS = ['date', 'open', 'high', 'low', 'close', 'volume']
-DEFAULT_TRADES_COLUMNS = ['timestamp', 'type', 'side', 'price', 'amount', 'cost']
+# Don't modify sequence of DEFAULT_TRADES_COLUMNS
+# it has wide consequences for stored trades files
+DEFAULT_TRADES_COLUMNS = ['timestamp', 'id', 'type', 'side', 'price', 'amount', 'cost']
 
 USERPATH_HYPEROPTS = 'hyperopts'
 USERPATH_STRATEGIES = 'strategies'

freqtrade/data/converter.py

@@ -3,12 +3,13 @@ Functions to convert data from one format to another
 """
 import logging
 from datetime import datetime, timezone
-from typing import Any, Dict
+from typing import Any, Dict, List
 
 import pandas as pd
 from pandas import DataFrame, to_datetime
 
-from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS
+from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS
+
 
 logger = logging.getLogger(__name__)
 
@@ -154,7 +155,16 @@ def order_book_to_dataframe(bids: list, asks: list) -> DataFrame:
     return frame
 
 
-def trades_to_ohlcv(trades: list, timeframe: str) -> DataFrame:
+def trades_dict_to_list(trades: List[Dict]) -> List[List]:
+    """
+    Convert fetch_trades result into a List (to be more memory efficient).
+    :param trades: List of trades, as returned by ccxt.fetch_trades.
+    :return: List of Lists, with constants.DEFAULT_TRADES_COLUMNS as columns
+    """
+    return [[t[col] for col in DEFAULT_TRADES_COLUMNS]for t in trades]
+
+
+def trades_to_ohlcv(trades: List, timeframe: str) -> DataFrame:
     """
     Converts trades list to OHLCV list
     TODO: This should get a dedicated test

tests/conftest.py

@@ -1391,6 +1391,50 @@ def buy_order_fee():
     }
 
 
+@pytest.fixture
+def fetch_trades_result():
+    return [{'info': {'a': 153942751,
+                      'p': '0.02059000',
+                      'q': '6.95500000',
+                      'f': 170811204,
+                      'l': 170811204,
+                      'T': 1585670882666,
+                      'm': True,
+                      'M': True},
+             'timestamp': 1585670882666,
+             'datetime': '2020-03-31T16:08:02.666Z',
+             'symbol': 'ETH/BTC',
+             'id': '153942751',
+             'order': None,
+             'type': None,
+             'takerOrMaker': None,
+             'side': 'sell',
+             'price': 0.02059,
+             'amount': 6.955,
+             'cost': 0.14320345,
+             'fee': None},
+            {'info': {'a': 153942752,
+                      'p': '0.02059100',
+                      'q': '0.04900000',
+                      'f': 170811205,
+                      'l': 170811205,
+                      'T': 1585670889154,
+                      'm': False,
+                      'M': True},
+             'timestamp': 1585670889154,
+             'datetime': '2020-03-31T16:08:09.154Z',
+             'symbol': 'ETH/BTC',
+             'id': '153942752',
+             'order': None,
+             'type': None,
+             'takerOrMaker': None,
+             'side': 'buy',
+             'price': 0.020591,
+             'amount': 0.049,
+             'cost': 0.001008959,
+             'fee': None}]
+
+
 @pytest.fixture(scope="function")
 def edge_conf(default_conf):
     conf = deepcopy(default_conf)

tests/data/test_converter.py

@@ -5,12 +5,10 @@ from freqtrade.configuration.timerange import TimeRange
 from freqtrade.data.converter import (convert_ohlcv_format,
                                       convert_trades_format,
                                       ohlcv_fill_up_missing_data,
-                                      ohlcv_to_dataframe,
+                                      ohlcv_to_dataframe, trades_dict_to_list,
                                       trim_dataframe)
-from freqtrade.data.history import (get_timerange,
-                                    load_data,
-                                    load_pair_history,
-                                    validate_backtest_data)
+from freqtrade.data.history import (get_timerange, load_data,
+                                    load_pair_history, validate_backtest_data)
 from tests.conftest import log_has
 from tests.data.test_history import _backup_file, _clean_test_file
 
@@ -197,6 +195,21 @@ def test_trim_dataframe(testdatadir) -> None:
     assert all(data_modify.iloc[0] == data.iloc[25])
 
 
+def test_trades_dict_to_list(mocker, fetch_trades_result):
+    res = trades_dict_to_list(fetch_trades_result)
+    assert isinstance(res, list)
+    assert isinstance(res[0], list)
+    for i, t in enumerate(res):
+        assert t[0] == fetch_trades_result[i]['timestamp']
+        assert t[1] == fetch_trades_result[i]['id']
+        assert t[2] == fetch_trades_result[i]['type']
+        assert t[3] == fetch_trades_result[i]['side']
+        assert t[4] == fetch_trades_result[i]['price']
+        assert t[5] == fetch_trades_result[i]['amount']
+        assert t[6] == fetch_trades_result[i]['cost']
+
+
+
 def test_convert_trades_format(mocker, default_conf, testdatadir):
     file = testdatadir / "XRP_ETH-trades.json.gz"
     file_new = testdatadir / "XRP_ETH-trades.json"