From 9b3085a429660dd442176ba63de196a476edc4e0 Mon Sep 17 00:00:00 2001 From: Christian Gruber Date: Mon, 27 Jan 2020 22:38:26 +0100 Subject: [PATCH 01/17] Add test cases to check for exact token matching --- libgnucash/engine/test/gtest-import-map.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/libgnucash/engine/test/gtest-import-map.cpp b/libgnucash/engine/test/gtest-import-map.cpp index f2e4509e61..33d64ab18e 100644 --- a/libgnucash/engine/test/gtest-import-map.cpp +++ b/libgnucash/engine/test/gtest-import-map.cpp @@ -271,6 +271,14 @@ TEST_F(ImapBayesTest, FindAccountBayes) EXPECT_EQ(t_expense_account2, account); account = gnc_account_imap_find_account_bayes(t_imap, t_list5); EXPECT_EQ(nullptr, account); + + // only imap entries with exact token matching should be considered + root->set_path({std::string{IMAP_FRAME_BAYES} + "/" + pepper + waldo + "/" + acct2_guid}, new KvpValue{*value}); + account = gnc_account_imap_find_account_bayes(t_imap, t_list3); + EXPECT_EQ(t_expense_account1, account); + root->set_path({std::string{IMAP_FRAME_BAYES} + "/" + pepper + "/" + waldo + "/" + acct2_guid}, new KvpValue{*value}); + account = gnc_account_imap_find_account_bayes(t_imap, t_list3); + EXPECT_EQ(t_expense_account1, account); } TEST_F(ImapBayesTest, AddAccountBayes) From da60560ac4ad1994a79185eaaafaaa0363a21076 Mon Sep 17 00:00:00 2001 From: Christian Gruber Date: Mon, 27 Jan 2020 22:43:09 +0100 Subject: [PATCH 02/17] Change behaviour of KvpFrame::for_each_slot_prefix() Provided function func is now called with key suffix only instead of full key (prefix is omitted). This is neccessary for fixing function build_token_info() in the next commit. --- libgnucash/engine/Account.cpp | 24 +++++++++++------------- libgnucash/engine/kvp-frame.hpp | 4 ++-- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/libgnucash/engine/Account.cpp b/libgnucash/engine/Account.cpp index 194976dbd6..fee1187bc8 100644 --- a/libgnucash/engine/Account.cpp +++ b/libgnucash/engine/Account.cpp @@ -5279,11 +5279,11 @@ struct AccountInfo }; static void -build_token_info(char const * key, KvpValue * value, TokenAccountsInfo & tokenInfo) +build_token_info(char const * suffix, KvpValue * value, TokenAccountsInfo & tokenInfo) { tokenInfo.total_count += value->get(); AccountTokenCount this_account; - std::string account_guid {key}; + std::string account_guid {suffix}; /*By convention, the key ends with the account GUID.*/ this_account.account_guid = account_guid.substr(account_guid.size() - GUID_ENCODING_LENGTH); this_account.token_count = value->get(); @@ -5665,38 +5665,36 @@ build_non_bayes (const char *key, const GValue *value, gpointer user_data) g_free (guid_string); } -static std::tuple +static std::tuple parse_bayes_imap_info (std::string const & imap_bayes_entry) { - auto header_length = strlen (IMAP_FRAME_BAYES); - std::string header {imap_bayes_entry.substr (0, header_length)}; auto guid_start = imap_bayes_entry.size() - GUID_ENCODING_LENGTH; - std::string keyword {imap_bayes_entry.substr (header_length + 1, guid_start - header_length - 2)}; + std::string keyword {imap_bayes_entry.substr (1, guid_start - 2)}; std::string account_guid {imap_bayes_entry.substr (guid_start)}; - return std::tuple {header, keyword, account_guid}; + return std::tuple {keyword, account_guid}; } static void -build_bayes (const char *key, KvpValue * value, GncImapInfo & imapInfo) +build_bayes (const char *suffix, KvpValue * value, GncImapInfo & imapInfo) { - auto parsed_key = parse_bayes_imap_info (key); + auto parsed_key = parse_bayes_imap_info (suffix); GncGUID guid; try { - auto temp_guid = gnc::GUID::from_string (std::get <2> (parsed_key)); + auto temp_guid = gnc::GUID::from_string (std::get <1> (parsed_key)); guid = temp_guid; } catch (const gnc::guid_syntax_exception& err) { - PWARN("Invalid GUID string from %s", key); + PWARN("Invalid GUID string from %s%s", IMAP_FRAME_BAYES, suffix); } auto map_account = xaccAccountLookup (&guid, gnc_account_get_book (imapInfo.source_account)); auto imap_node = static_cast (g_malloc (sizeof (GncImapInfo))); auto count = value->get (); imap_node->source_account = imapInfo.source_account; imap_node->map_account = map_account; - imap_node->head = g_strdup (key); - imap_node->match_string = g_strdup (std::get <1> (parsed_key).c_str ()); + imap_node->head = g_strdup_printf ("%s%s", IMAP_FRAME_BAYES, suffix); + imap_node->match_string = g_strdup (std::get <0> (parsed_key).c_str ()); imap_node->category = g_strdup(" "); imap_node->count = g_strdup_printf ("%" G_GINT64_FORMAT, count); imapInfo.list = g_list_prepend (imapInfo.list, imap_node); diff --git a/libgnucash/engine/kvp-frame.hpp b/libgnucash/engine/kvp-frame.hpp index 23eb0c6153..1419e626e5 100644 --- a/libgnucash/engine/kvp-frame.hpp +++ b/libgnucash/engine/kvp-frame.hpp @@ -247,7 +247,7 @@ void KvpFrame::for_each_slot_prefix(std::string const & prefix, return; /* Testing for prefix matching */ if (std::mismatch(prefix.begin(), prefix.end(), temp_key.begin()).first == prefix.end()) - func (a.first, a.second); + func (&a.first[prefix.size()], a.second); } ); } @@ -264,7 +264,7 @@ void KvpFrame::for_each_slot_prefix(std::string const & prefix, return; /* Testing for prefix matching */ if (std::mismatch(prefix.begin(), prefix.end(), temp_key.begin()).first == prefix.end()) - func (a.first, a.second, data); + func (&a.first[prefix.size()], a.second, data); } ); } From a13184978a07e4c6735e5d5893076ffd8117c6a1 Mon Sep 17 00:00:00 2001 From: Christian Gruber Date: Mon, 27 Jan 2020 23:01:25 +0100 Subject: [PATCH 03/17] Fix calculation of token info to find exactly matching tokens only In get_first_pass_probabilities() function qof_instance_foreach_slot_prefix() is called with a prefix path including closing slash after token now. This avoids, that also entries with token as a substring are included in token info, where key only starts with token. Finally function build_token_info() checks, if the key suffix after the token consists only of the GUID. This avoids, that also entries with the same prefix and slashes are included in token info. --- libgnucash/engine/Account.cpp | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/libgnucash/engine/Account.cpp b/libgnucash/engine/Account.cpp index fee1187bc8..805daeea4e 100644 --- a/libgnucash/engine/Account.cpp +++ b/libgnucash/engine/Account.cpp @@ -5281,13 +5281,15 @@ struct AccountInfo static void build_token_info(char const * suffix, KvpValue * value, TokenAccountsInfo & tokenInfo) { - tokenInfo.total_count += value->get(); - AccountTokenCount this_account; - std::string account_guid {suffix}; - /*By convention, the key ends with the account GUID.*/ - this_account.account_guid = account_guid.substr(account_guid.size() - GUID_ENCODING_LENGTH); - this_account.token_count = value->get(); - tokenInfo.accounts.push_back(this_account); + if (strlen(suffix) == GUID_ENCODING_LENGTH) + { + tokenInfo.total_count += value->get(); + AccountTokenCount this_account; + /*By convention, the key ends with the account GUID.*/ + this_account.account_guid = std::string{suffix}; + this_account.token_count = value->get(); + tokenInfo.accounts.push_back(this_account); + } } /** We scale the probability values by probability_factor. @@ -5332,7 +5334,7 @@ get_first_pass_probabilities(GncImportMatchMap * imap, GList * tokens) for (auto current_token = tokens; current_token; current_token = current_token->next) { TokenAccountsInfo tokenInfo{}; - auto path = std::string{IMAP_FRAME_BAYES "/"} + static_cast (current_token->data); + auto path = std::string{IMAP_FRAME_BAYES "/"} + static_cast (current_token->data) + "/"; qof_instance_foreach_slot_prefix (QOF_INSTANCE (imap->acc), path, &build_token_info, tokenInfo); for (auto const & current_account_token : tokenInfo.accounts) { From 322f2d99de89849b4d193afcb779520aabcafa4f Mon Sep 17 00:00:00 2001 From: Christian Gruber Date: Sun, 2 Feb 2020 17:55:30 +0100 Subject: [PATCH 04/17] Rework key prefix matching Use C string comparison instead of C++ function std::mismatch to increase performance. --- libgnucash/engine/kvp-frame.hpp | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/libgnucash/engine/kvp-frame.hpp b/libgnucash/engine/kvp-frame.hpp index 1419e626e5..c84cf18a4d 100644 --- a/libgnucash/engine/kvp-frame.hpp +++ b/libgnucash/engine/kvp-frame.hpp @@ -242,11 +242,8 @@ void KvpFrame::for_each_slot_prefix(std::string const & prefix, std::for_each (m_valuemap.begin(), m_valuemap.end(), [&prefix,&func](const KvpFrameImpl::map_type::value_type & a) { - std::string temp_key {a.first}; - if (temp_key.size() < prefix.size()) - return; /* Testing for prefix matching */ - if (std::mismatch(prefix.begin(), prefix.end(), temp_key.begin()).first == prefix.end()) + if (strncmp(a.first, prefix.c_str(), prefix.size()) == 0) func (&a.first[prefix.size()], a.second); } ); @@ -259,11 +256,8 @@ void KvpFrame::for_each_slot_prefix(std::string const & prefix, std::for_each (m_valuemap.begin(), m_valuemap.end(), [&prefix,&func,&data](const KvpFrameImpl::map_type::value_type & a) { - std::string temp_key {a.first}; - if (temp_key.size() < prefix.size()) - return; /* Testing for prefix matching */ - if (std::mismatch(prefix.begin(), prefix.end(), temp_key.begin()).first == prefix.end()) + if (strncmp(a.first, prefix.c_str(), prefix.size()) == 0) func (&a.first[prefix.size()], a.second, data); } ); From d07d4b962fc89ef332626b984b8fb3149033113d Mon Sep 17 00:00:00 2001 From: Christian Gruber Date: Sun, 2 Feb 2020 22:38:44 +0100 Subject: [PATCH 05/17] Fix tokenize_string() This fix prevents empty strings as tokens and removes duplicated tokens. Function tokenize_string() is used for bayesian import matching, where empty token strings or duplicated tokens lead to wrong results within probability calculation for matching of a transaction to an account. Empty token strings can occur if (see function g_strsplit()) * two or more spaces occur directly after another * the string begins or ends with spaces --- gnucash/import-export/import-backend.c | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/gnucash/import-export/import-backend.c b/gnucash/import-export/import-backend.c index ebc37c3a7b..888b913b2f 100644 --- a/gnucash/import-export/import-backend.c +++ b/gnucash/import-export/import-backend.c @@ -387,8 +387,24 @@ tokenize_string(GList* existing_tokens, const char *string) /* add each token to the token GList */ while (stringpos && *stringpos) { - /* prepend the char* to the token GList */ - existing_tokens = g_list_prepend(existing_tokens, g_strdup(*stringpos)); + if (strlen(*stringpos) > 0) + { + /* check for duplicated tokens */ + gboolean duplicated = FALSE; + for (GList* token = existing_tokens; token != NULL; token = token->next) + { + if (g_strcmp0(token->data, *stringpos) == 0) + { + duplicated = TRUE; + break; + } + } + if (duplicated == FALSE) + { + /* prepend the char* to the token GList */ + existing_tokens = g_list_prepend(existing_tokens, g_strdup(*stringpos)); + } + } /* then move to the next string */ stringpos++; From 41863be9c7251098764c5a36e25f401fe536162d Mon Sep 17 00:00:00 2001 From: Christian Gruber Date: Mon, 17 Feb 2020 22:56:15 +0100 Subject: [PATCH 06/17] Avoid copying local instance of AccountTokenCount --- libgnucash/engine/Account.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/libgnucash/engine/Account.cpp b/libgnucash/engine/Account.cpp index 805daeea4e..83d49a7480 100644 --- a/libgnucash/engine/Account.cpp +++ b/libgnucash/engine/Account.cpp @@ -5284,11 +5284,8 @@ build_token_info(char const * suffix, KvpValue * value, TokenAccountsInfo & toke if (strlen(suffix) == GUID_ENCODING_LENGTH) { tokenInfo.total_count += value->get(); - AccountTokenCount this_account; /*By convention, the key ends with the account GUID.*/ - this_account.account_guid = std::string{suffix}; - this_account.token_count = value->get(); - tokenInfo.accounts.push_back(this_account); + tokenInfo.accounts.emplace_back(AccountTokenCount{std::string{suffix}, value->get()}); } } From 01c76e23913ea870fb331fc6c747d67f9bdd7f85 Mon Sep 17 00:00:00 2001 From: Christian Gruber Date: Mon, 17 Feb 2020 23:29:28 +0100 Subject: [PATCH 07/17] Remove unused template of function for_each_slot_prefix() for_each_slot_prefix() is not used anywhere with two arguments --- libgnucash/engine/kvp-frame.hpp | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/libgnucash/engine/kvp-frame.hpp b/libgnucash/engine/kvp-frame.hpp index c84cf18a4d..253eec3187 100644 --- a/libgnucash/engine/kvp-frame.hpp +++ b/libgnucash/engine/kvp-frame.hpp @@ -235,20 +235,6 @@ struct KvpFrameImpl KvpValue * set_impl (std::string const &, KvpValue *) noexcept; }; -template -void KvpFrame::for_each_slot_prefix(std::string const & prefix, - func_type const & func) const noexcept -{ - std::for_each (m_valuemap.begin(), m_valuemap.end(), - [&prefix,&func](const KvpFrameImpl::map_type::value_type & a) - { - /* Testing for prefix matching */ - if (strncmp(a.first, prefix.c_str(), prefix.size()) == 0) - func (&a.first[prefix.size()], a.second); - } - ); -} - template void KvpFrame::for_each_slot_prefix(std::string const & prefix, func_type const & func, data_type & data) const noexcept From 7509b542da67aeb42bff0b98a20ffc2989f6e146 Mon Sep 17 00:00:00 2001 From: Christian Gruber Date: Mon, 17 Feb 2020 23:31:12 +0100 Subject: [PATCH 08/17] Simplify function build_bayes() Inline function parse_bayes_imap_info() into build_bayes() and remove temp_guid. --- libgnucash/engine/Account.cpp | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/libgnucash/engine/Account.cpp b/libgnucash/engine/Account.cpp index 83d49a7480..ea976be491 100644 --- a/libgnucash/engine/Account.cpp +++ b/libgnucash/engine/Account.cpp @@ -5664,24 +5664,15 @@ build_non_bayes (const char *key, const GValue *value, gpointer user_data) g_free (guid_string); } -static std::tuple -parse_bayes_imap_info (std::string const & imap_bayes_entry) -{ - auto guid_start = imap_bayes_entry.size() - GUID_ENCODING_LENGTH; - std::string keyword {imap_bayes_entry.substr (1, guid_start - 2)}; - std::string account_guid {imap_bayes_entry.substr (guid_start)}; - return std::tuple {keyword, account_guid}; -} - static void build_bayes (const char *suffix, KvpValue * value, GncImapInfo & imapInfo) { - auto parsed_key = parse_bayes_imap_info (suffix); + size_t guid_start = strlen(suffix) - GUID_ENCODING_LENGTH; + std::string account_guid {&suffix[guid_start]}; GncGUID guid; try { - auto temp_guid = gnc::GUID::from_string (std::get <1> (parsed_key)); - guid = temp_guid; + guid = gnc::GUID::from_string (account_guid); } catch (const gnc::guid_syntax_exception& err) { @@ -5693,7 +5684,7 @@ build_bayes (const char *suffix, KvpValue * value, GncImapInfo & imapInfo) imap_node->source_account = imapInfo.source_account; imap_node->map_account = map_account; imap_node->head = g_strdup_printf ("%s%s", IMAP_FRAME_BAYES, suffix); - imap_node->match_string = g_strdup (std::get <0> (parsed_key).c_str ()); + imap_node->match_string = g_strndup (&suffix[1], guid_start - 2); imap_node->category = g_strdup(" "); imap_node->count = g_strdup_printf ("%" G_GINT64_FORMAT, count); imapInfo.list = g_list_prepend (imapInfo.list, imap_node); From b5dfef628eda8f91bb33ece2b11a1f23b64e9e34 Mon Sep 17 00:00:00 2001 From: Geert Janssens Date: Sat, 29 Feb 2020 10:51:49 +0100 Subject: [PATCH 09/17] Travis - use ctest built-in feature to verbosely log test failures Setting CTEST_OUTPUT_ON_FAILURE will cause ctest to log all output from failing tests (and only from failing tests). This will reduce our ci output in case of failures as our homebrew script 'afterfailure' would output all test logs in case of failure not only the output of failed tests. --- util/ci/afterfailure | 12 ------------ util/ci/commonbuild | 5 +++-- 2 files changed, 3 insertions(+), 14 deletions(-) delete mode 100755 util/ci/afterfailure diff --git a/util/ci/afterfailure b/util/ci/afterfailure deleted file mode 100755 index dd065c1939..0000000000 --- a/util/ci/afterfailure +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -if [[ "$BUILDTYPE" == "cmake-make" ]] || [[ "$BUILDTYPE" == "cmake-ninja" ]]; then - echo "########################"; - echo "##### LastTest.log #####"; - echo "########################"; - cat /build/Testing/Temporary/LastTest.log; -else - echo "Unknown BUILDTYPE: \"$BUILDTYPE\", cannot create failure information."; -fi -# This script should fail so that the entire script will fail. -exit 1; diff --git a/util/ci/commonbuild b/util/ci/commonbuild index 9b1875d55e..cf27b1286a 100644 --- a/util/ci/commonbuild +++ b/util/ci/commonbuild @@ -5,15 +5,16 @@ mkdir -p "$HOME"/.local/share mkdir build cd build export TZ="America/Los_Angeles" +export CTEST_OUTPUT_ON_FAILURE=On if [[ "$BUILDTYPE" == "cmake-make" ]]; then cmake ../gnucash $PLATFORM_CMAKE_OPTS make -j 4 - make check || ../afterfailure + make check elif [[ "$BUILDTYPE" == "cmake-ninja" ]]; then cmake ../gnucash -DWITH_PYTHON=ON -DCMAKE_BUILD_TYPE=debug -G Ninja $PLATFORM_CMAKE_OPTS ninja - ninja check || ../afterfailure; + ninja check else echo "Unknown buildtype: \"$BUILDTYPE\". Not building." fi From d7eb24f205ec09a1bf3bf6fb4b4e00b7f6a8fcd0 Mon Sep 17 00:00:00 2001 From: Geert Janssens Date: Sat, 29 Feb 2020 13:43:33 +0100 Subject: [PATCH 10/17] Travis - don't try to copy the removed afterfailure file to the test container --- util/ci/arch-docker | 4 ++-- util/ci/ubuntu-14.04-docker | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/util/ci/arch-docker b/util/ci/arch-docker index 47f0c5e984..b6e0703b38 100644 --- a/util/ci/arch-docker +++ b/util/ci/arch-docker @@ -4,6 +4,6 @@ run echo en_US.UTF-8 UTF-8 >> /etc/locale.gen run echo en_GB.UTF-8 UTF-8 >> /etc/locale.gen run echo fr_FR.UTF-8 UTF-8 >> /etc/locale.gen run locale-gen -copy arch-testscript afterfailure commonbuild / -run chmod +x /arch-testscript /afterfailure /commonbuild +copy arch-testscript commonbuild / +run chmod +x /arch-testscript /commonbuild entrypoint /arch-testscript diff --git a/util/ci/ubuntu-14.04-docker b/util/ci/ubuntu-14.04-docker index f788331854..0f055141c6 100644 --- a/util/ci/ubuntu-14.04-docker +++ b/util/ci/ubuntu-14.04-docker @@ -5,6 +5,6 @@ run apt-get build-dep -qq gnucash > /dev/null run apt-get install -qq git bash-completion cmake3 make swig xsltproc libdbd-sqlite3 texinfo ninja-build libboost-all-dev libgtk-3-dev libwebkit2gtk-3.0-dev > /dev/null run apt-get --reinstall install -qq language-pack-en language-pack-fr run git clone https://github.com/google/googletest -b release-1.8.0 gtest -copy ubuntu-14.04-testscript afterfailure commonbuild / -run chmod +x /ubuntu-14.04-testscript /afterfailure /commonbuild +copy ubuntu-14.04-testscript commonbuild / +run chmod +x /ubuntu-14.04-testscript /commonbuild entrypoint /ubuntu-14.04-testscript From 7845f818467b08121695716f4aa3562a21414d27 Mon Sep 17 00:00:00 2001 From: Rob Laan Date: Sat, 29 Feb 2020 11:19:20 +0100 Subject: [PATCH 11/17] Bug 797613 Due Invoices Reminder shows Job Name instead of Company Name In the invoices due dialog, the column 'Company' should show the name of the customer/vendor of an invoice/bill. Currently the content is set to the owner name, but if a job is assigned to the invoice, that will show the job name. This change changes the content to the name of the owner parent, which is always the customer/vendor name. --- gnucash/gnome/dialog-invoice.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gnucash/gnome/dialog-invoice.c b/gnucash/gnome/dialog-invoice.c index e38d00e2eb..c5224dbca8 100644 --- a/gnucash/gnome/dialog-invoice.c +++ b/gnucash/gnome/dialog-invoice.c @@ -3387,7 +3387,7 @@ gnc_invoice_show_docs_due (GtkWindow *parent, QofBook *book, double days_in_adva param_list = gnc_search_param_prepend (param_list, _("Amount"), NULL, type, INVOICE_POST_LOT, LOT_BALANCE, NULL); param_list = gnc_search_param_prepend (param_list, _("Company"), NULL, type, - INVOICE_OWNER, OWNER_NAME, NULL); + INVOICE_OWNER, OWNER_PARENT, OWNER_NAME, NULL); param_list = gnc_search_param_prepend (param_list, _("Due"), NULL, type, INVOICE_DUE, NULL); } From 890c96ce239f5a8c8e41517f134872df3e976021 Mon Sep 17 00:00:00 2001 From: Christopher Lam Date: Sat, 29 Feb 2020 21:28:33 +0800 Subject: [PATCH 12/17] [qif-merge-groups] Move gnc:account-tree-get-transactions to qif-to-gnc.scm This function is only used by qif-to-gnc.scm --- .../import-export/qif-imp/qif-merge-groups.scm | 15 --------------- gnucash/import-export/qif-imp/qif-to-gnc.scm | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/gnucash/import-export/qif-imp/qif-merge-groups.scm b/gnucash/import-export/qif-imp/qif-merge-groups.scm index e576f48a94..c8bdd12159 100644 --- a/gnucash/import-export/qif-imp/qif-merge-groups.scm +++ b/gnucash/import-export/qif-imp/qif-merge-groups.scm @@ -24,21 +24,6 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; gnc:account-tree-get-transactions -;; -;; Given an account tree, this procedure returns a list of all -;; transactions whose splits only use accounts in the tree. -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -(define (gnc:account-tree-get-transactions root) - (let ((accounts (gnc-account-get-descendants-sorted root))) - (let ((q (qof-query-create-for-splits))) - (qof-query-set-book q (gnc-account-get-book root)) - (xaccQueryAddAccountMatch q accounts QOF-GUID-MATCH-ANY QOF-QUERY-AND) - (let ((xtns (xaccQueryGetTransactions q QUERY-TXN-MATCH-ALL))) - (qof-query-destroy q) - xtns)))) - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; gnc:account-tree-find-duplicates ;; diff --git a/gnucash/import-export/qif-imp/qif-to-gnc.scm b/gnucash/import-export/qif-imp/qif-to-gnc.scm index df74b48dc7..10ac1780b1 100644 --- a/gnucash/import-export/qif-imp/qif-to-gnc.scm +++ b/gnucash/import-export/qif-imp/qif-to-gnc.scm @@ -1230,6 +1230,20 @@ (if all-marked (qif-xtn:set-mark! xtn #t)))) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; gnc:account-tree-get-transactions +;; +;; Given an account tree, this procedure returns a list of all +;; transactions whose splits only use accounts in the tree. +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +(define (gnc:account-tree-get-transactions root) + (let ((accounts (gnc-account-get-descendants-sorted root))) + (let ((q (qof-query-create-for-splits))) + (qof-query-set-book q (gnc-account-get-book root)) + (xaccQueryAddAccountMatch q accounts QOF-GUID-MATCH-ANY QOF-QUERY-AND) + (let ((xtns (xaccQueryGetTransactions q QUERY-TXN-MATCH-ALL))) + (qof-query-destroy q) + xtns)))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; qif-import:qif-to-gnc-undo From 11689e2a4daaf70c3f6ba9b8345db01090603237 Mon Sep 17 00:00:00 2001 From: Christopher Lam Date: Sat, 29 Feb 2020 21:22:11 +0800 Subject: [PATCH 13/17] [qif-guess-map] use (ice-9 match), and compact functions --- .../import-export/qif-imp/qif-guess-map.scm | 53 ++++++++----------- 1 file changed, 22 insertions(+), 31 deletions(-) diff --git a/gnucash/import-export/qif-imp/qif-guess-map.scm b/gnucash/import-export/qif-imp/qif-guess-map.scm index 067013dee0..da8ff87dfb 100644 --- a/gnucash/import-export/qif-imp/qif-guess-map.scm +++ b/gnucash/import-export/qif-imp/qif-guess-map.scm @@ -25,6 +25,7 @@ (use-modules (srfi srfi-13)) +(use-modules (ice-9 match)) (define GNC-BANK-TYPE 0) (define GNC-CASH-TYPE 1) @@ -227,24 +228,21 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (define (qif-import:read-securities security-list) - (let ((table (make-hash-table 20))) + (let ((table (make-hash-table)) + (comm-table (gnc-commodity-table-get-table (gnc-get-current-book)))) (for-each - (lambda (entry) - (if (and (list? entry) - (= 3 (length entry))) - ;; The saved information about each security mapping is a - ;; list of three items: the QIF name, and the GnuCash - ;; namespace and mnemonic (symbol) to which it maps. - ;; Example: ("McDonald's" "NYSE" "MCD") - (let ((commodity (gnc-commodity-table-lookup - (gnc-commodity-table-get-table - (gnc-get-current-book)) - (cadr entry) - (caddr entry)))) - (if (and commodity (not (null? commodity))) - ;; There is an existing GnuCash commodity for this - ;; combination of namespace and symbol. - (hash-set! table (car entry) commodity))))) + (match-lambda + ((name ns mnemonic) + ;; The saved information about each security mapping is a + ;; list of three items: the QIF name, and the GnuCash + ;; namespace and mnemonic (symbol) to which it maps. + ;; Example: ("McDonald's" "NYSE" "MCD") + (let ((commodity (gnc-commodity-table-lookup comm-table ns mnemonic))) + (if (and commodity (not (null? commodity))) + ;; There is an existing GnuCash commodity for this + ;; combination of namespace and symbol. + (hash-set! table name commodity)))) + (_ #f)) security-list) table)) @@ -462,18 +460,11 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (define (qif-import:find-new-acct qif-acct allowed-types gnc-acct-info) - (cond ((and (string? qif-acct) - (string=? qif-acct (default-equity-account))) - (let ((existing-equity - (qif-import:find-similar-acct (default-equity-account) - (list GNC-EQUITY-TYPE) - gnc-acct-info))) - (if existing-equity - existing-equity - (list (default-equity-account) (list GNC-EQUITY-TYPE))))) - ((and (string? qif-acct) - (not (string=? qif-acct ""))) - (list qif-acct allowed-types)) - (#t - (list (default-unspec-acct) allowed-types)))) + (cond + ((equal? qif-acct (default-equity-account)) + (or (qif-import:find-similar-acct + (default-equity-account) (list GNC-EQUITY-TYPE) gnc-acct-info) + (list (default-equity-account) (list GNC-EQUITY-TYPE)))) + ((equal? qif-acct "") (list (default-unspec-acct) allowed-types)) + (else (list qif-acct allowed-types)))) From ec99c007149d050c4e69cb2f819f79bbf64fb534 Mon Sep 17 00:00:00 2001 From: Christopher Lam Date: Sat, 29 Feb 2020 21:29:08 +0800 Subject: [PATCH 14/17] [qif-to-gnc] use (ice-9 match), and compact functions --- gnucash/import-export/qif-imp/qif-to-gnc.scm | 39 +++++++------------- 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/gnucash/import-export/qif-imp/qif-to-gnc.scm b/gnucash/import-export/qif-imp/qif-to-gnc.scm index 10ac1780b1..4ab978dcb9 100644 --- a/gnucash/import-export/qif-imp/qif-to-gnc.scm +++ b/gnucash/import-export/qif-imp/qif-to-gnc.scm @@ -26,6 +26,7 @@ (use-modules (srfi srfi-13)) +(use-modules (ice-9 match)) (use-modules (gnucash import-export string)) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -513,12 +514,10 @@ ;; Look for the transaction status (QIF "C" line). When it exists, apply ;; the cleared (c) or reconciled (y) status to the split. Otherwise, apply ;; user preference. - (if (eq? qif-cleared 'cleared) - (xaccSplitSetReconcile gnc-near-split #\c) - (if (eq? qif-cleared 'reconciled) - (xaccSplitSetReconcile gnc-near-split #\y) - ;; Apply user preference by default. - (xaccSplitSetReconcile gnc-near-split transaction-status-pref))) + (case qif-cleared + ((cleared) (xaccSplitSetReconcile gnc-near-split #\c)) + ((reconciled) (xaccSplitSetReconcile gnc-near-split #\y)) + (else (xaccSplitSetReconcile gnc-near-split transaction-status-pref))) (if (not qif-security) (begin @@ -595,11 +594,9 @@ (set! far-acct (hash-ref gnc-acct-hash far-acct-name)) ;; set the reconcile status. - (let ((cleared (qif-split:matching-cleared qif-split))) - (if (eq? 'cleared cleared) - (xaccSplitSetReconcile gnc-far-split #\c)) - (if (eq? 'reconciled cleared) - (xaccSplitSetReconcile gnc-far-split #\y))) + (case (qif-split:matching-cleared qif-split) + ((cleared) (xaccSplitSetReconcile gnc-far-split #\c)) + ((reconciled) (xaccSplitSetReconcile gnc-far-split #\y))) ;; finally, plug the split into the account (xaccSplitSetAccount gnc-far-split far-acct) @@ -762,12 +759,9 @@ (xaccSplitSetValue gnc-near-split (n- split-amt)) (xaccSplitSetValue gnc-far-split split-amt)))) - (let ((cleared (qif-split:matching-cleared - (car (qif-xtn:splits qif-xtn))))) - (if (eq? 'cleared cleared) - (xaccSplitSetReconcile gnc-far-split #\c)) - (if (eq? 'reconciled cleared) - (xaccSplitSetReconcile gnc-far-split #\y))) + (case (qif-split:matching-cleared (car (qif-xtn:splits qif-xtn))) + ((cleared) (xaccSplitSetReconcile gnc-far-split #\c)) + ((reconciled) (xaccSplitSetReconcile gnc-far-split #\y))) (if qif-commission-acct (let* ((commission-acct-info @@ -957,14 +951,9 @@ (this-group-amt (gnc-numeric-zero)) (how #f) (date-matches - (let ((self-date (qif-xtn:date xtn))) - (and (pair? self-date) - (pair? date) - (eq? (length self-date) 3) - (eq? (length date) 3) - (= (car self-date) (car date)) - (= (cadr self-date) (cadr date)) - (= (caddr self-date) (caddr date))))) + (match (cons date (qif-xtn:date xtn)) + (((a b c) . (a b c)) #t) + (_ #f))) (n- (lambda (n) (gnc-numeric-neg n))) (nsub (lambda (a b) (gnc-numeric-sub a b 0 GNC-DENOM-LCD))) (n+ (lambda (a b) (gnc-numeric-add a b 0 GNC-DENOM-LCD))) From 1d9c51d8e46f487d976b3f637016ee15a3283899 Mon Sep 17 00:00:00 2001 From: Christopher Lam Date: Sat, 29 Feb 2020 22:32:14 +0800 Subject: [PATCH 15/17] [qif-to-gnc] Bug 796579 - Cannot go forward with empty duplicates screen better fix for bug 796579. far-acct-info is a record with acct-info far-acct-name is a string far-acct is a gnucash account object (default-unspec-acct) always returns a string "Unspecified". 983c7ce0b was trying to find, from qif-memo-map the mapping for "Unspecified" which isn't typically part of qif-memo-map. Therefore far-acct-info would remain #f and fail the qif-map-entry:gnc-name. Better fix: if far-acct-info cannot be derived, use unspecified account. --- gnucash/import-export/qif-imp/qif-to-gnc.scm | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/gnucash/import-export/qif-imp/qif-to-gnc.scm b/gnucash/import-export/qif-imp/qif-to-gnc.scm index 4ab978dcb9..75c21ceeac 100644 --- a/gnucash/import-export/qif-imp/qif-to-gnc.scm +++ b/gnucash/import-export/qif-imp/qif-to-gnc.scm @@ -584,13 +584,11 @@ (hash-ref qif-memo-map qif-memo))) (and (string? memo) (not (string=? memo "")) - (hash-ref qif-memo-map memo)))) - (if (not far-acct-info) - (set! far-acct-info - (hash-ref qif-memo-map - (default-unspec-acct)))))) + (hash-ref qif-memo-map memo)))))) - (set! far-acct-name (qif-map-entry:gnc-name far-acct-info)) + (set! far-acct-name (if far-acct-info + (qif-map-entry:gnc-name far-acct-info) + (default-unspec-acct))) (set! far-acct (hash-ref gnc-acct-hash far-acct-name)) ;; set the reconcile status. From 742b97229c0afd588730d11dac537626fb7b17e2 Mon Sep 17 00:00:00 2001 From: Christopher Lam Date: Sun, 1 Mar 2020 10:36:25 +0800 Subject: [PATCH 16/17] [trep-engine] when Running Balance is shown, print "Balance b/f" --- gnucash/report/report-system/trep-engine.scm | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/gnucash/report/report-system/trep-engine.scm b/gnucash/report/report-system/trep-engine.scm index 0a10504de9..8e12fca7cc 100644 --- a/gnucash/report/report-system/trep-engine.scm +++ b/gnucash/report/report-system/trep-engine.scm @@ -1401,6 +1401,11 @@ be excluded from periodic reporting.") (case level ((primary) optname-prime-sortkey) ((secondary) optname-sec-sortkey)))) + (data (if (and (any (lambda (c) (eq? 'bal-bf (vector-ref c 5))) + calculated-cells) + (memq sortkey ACCOUNT-SORTING-TYPES)) + (string-append data ": " (_ "Balance b/f")) + data)) (renderer-fn (keylist-get-info (sortkey-list BOOK-SPLIT-ACTION) sortkey 'renderer-fn)) From 2c1ce30d7de02e1791689f53ed5a13b61cce54e2 Mon Sep 17 00:00:00 2001 From: Christopher Lam Date: Sun, 1 Mar 2020 13:40:12 +0800 Subject: [PATCH 17/17] [qif-to-gnc] centralize arithmetic functions --- gnucash/import-export/qif-imp/qif-to-gnc.scm | 27 +++++++------------- 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/gnucash/import-export/qif-imp/qif-to-gnc.scm b/gnucash/import-export/qif-imp/qif-to-gnc.scm index 75c21ceeac..08bc0b31ed 100644 --- a/gnucash/import-export/qif-imp/qif-to-gnc.scm +++ b/gnucash/import-export/qif-imp/qif-to-gnc.scm @@ -29,6 +29,12 @@ (use-modules (ice-9 match)) (use-modules (gnucash import-export string)) +(define (n- n) (gnc-numeric-neg n)) +(define (nsub a b) (gnc-numeric-sub a b 0 GNC-DENOM-LCD)) +(define (n+ a b) (gnc-numeric-add a b 0 GNC-DENOM-LCD)) +(define (n* a b) (gnc-numeric-mul a b 0 GNC-DENOM-REDUCE)) +(define (n/ a b) (gnc-numeric-div a b 0 GNC-DENOM-REDUCE)) + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; qif-import:find-or-make-acct ;; @@ -465,12 +471,7 @@ (qif-memo #f) (qif-date (qif-xtn:date qif-xtn)) (qif-from-acct (qif-xtn:from-acct qif-xtn)) - (qif-cleared (qif-xtn:cleared qif-xtn)) - (n- (lambda (n) (gnc-numeric-neg n))) - (nsub (lambda (a b) (gnc-numeric-sub a b 0 GNC-DENOM-LCD))) - (n+ (lambda (a b) (gnc-numeric-add a b 0 GNC-DENOM-LCD))) - (n* (lambda (a b) (gnc-numeric-mul a b 0 GNC-DENOM-REDUCE))) - (n/ (lambda (a b) (gnc-numeric-div a b 0 GNC-DENOM-REDUCE)))) + (qif-cleared (qif-xtn:cleared qif-xtn))) ;; Set properties of the whole transaction. @@ -834,12 +835,7 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (define (qif-import:mark-some-splits splits xtn candidate-xtns errorproc) - (let* ((n- (lambda (n) (gnc-numeric-neg n))) - (nsub (lambda (a b) (gnc-numeric-sub a b 0 GNC-DENOM-LCD))) - (n+ (lambda (a b) (gnc-numeric-add a b 0 GNC-DENOM-LCD))) - (n* (lambda (a b) (gnc-numeric-mul a b 0 GNC-DENOM-REDUCE))) - (n/ (lambda (a b) (gnc-numeric-div a b 0 GNC-DENOM-REDUCE))) - (split (car splits)) + (let* ((split (car splits)) (near-acct-name #f) (far-acct-name #f) (date (qif-xtn:date xtn)) @@ -951,12 +947,7 @@ (date-matches (match (cons date (qif-xtn:date xtn)) (((a b c) . (a b c)) #t) - (_ #f))) - (n- (lambda (n) (gnc-numeric-neg n))) - (nsub (lambda (a b) (gnc-numeric-sub a b 0 GNC-DENOM-LCD))) - (n+ (lambda (a b) (gnc-numeric-add a b 0 GNC-DENOM-LCD))) - (n* (lambda (a b) (gnc-numeric-mul a b 0 GNC-DENOM-REDUCE))) - (n/ (lambda (a b) (gnc-numeric-div a b 0 GNC-DENOM-REDUCE)))) + (_ #f)))) (if date-matches (begin