diff --git a/README.md b/README.md index 2bb9f45..bea47b0 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ The third stage is executed by the `bb_archive.py` script. ### Configuration -Each financial institution from which data will be imported must have a dedicated YAML configuration file. +Each financial statement from which data will be imported must have a dedicated YAML configuration file. The configuration file is used by the import scripts to determine the CSV file structure and other information, including which rules to apply. ### Structure of a configuration file @@ -112,6 +112,8 @@ Note that the first index starts from `0`. | amount_in | Some financial institutions, use separate indexes for debit and credit. In this case, it is possible to specify the index for the index corresponding to the credited amount | | | narration | The index corresponding to the narration or reference field of the transaction | | +In the context of a financial transaction, a "counterparty" simply refers to the other party involved in the transaction. It can be an individual, a company, or any other entity that you are transacting with. For example, if you purchase groceries at a store, the store is the counterparty in that transaction. Similarly, if you receive a salary from your employer, the employer is the counterparty to that transaction. + #### rules @@ -146,7 +148,6 @@ indexes: rules: beancount_file: 'main-ledger.ldg' - rules_file: well-fargo.rules account: 565444499 currency: USD ruleset: @@ -198,20 +199,55 @@ Fresh Food Inc.;equals;FRESH FOOD #### Replace_Expense -This rule is used to assign an Account to a transaction based on the value of the `counterparty` index of the CSV file. This rule requires a look-up file named `account.rules` located in the directory defined by the `rules.rules_folder` option of the config file. +The `Replace_Expense` rule is designed to automatically assign a specific account to a Beancount transaction, based on the counterparty field in the CSV file. This rule simplifies the categorization of transactions by utilizing a look-up file, `account.rules`, which is located in the directory specified by the `rules.rules_folder` configuration. + +##### Example Usage -For example: we want to add this transaction to the ledger and we want to assign the Account `Expenses:Grocery` to the transaction. +- To assign `Expenses:Grocery` to transactions from "Fresh Food Inc.", add the following to your `account.rules` file: ``` -04.11.2020;04.11.2020;Direct Debit;"Fresh Food Inc.";-21,30;EUR;0000001;UK0000001444555 +Fresh Food Inc.;equals;Expenses:Groceries ``` + Add the `Replace_Expense` rule to the list of rules in the configuration file for the target financial institution and add this entry to the `account.rules` file: ``` Fresh Food Inc.;equals;Expenses:Groceries ``` +##### Advanced Categorization: + +- In scenarios where the same counterparty should be categorized differently based on the account (e.g., business vs. personal expenses), the rule can distinguish based on the `rules.account` property of the yaml configuration file. +- For example, transactions with "Apple" in a business account could be categorized as "Expenses:Business:TaxClaim," while in a personal savings account as "Expenses:Gadgets". + +`account.rules`` File Example: + +``` +apple;contains_ic;Expenses:Gadgets +apple;contains_ic;Expenses:Business:TaxClaim;biz +``` + +The value `biz` must match the value of the `rules.account` property from the financial statement Yaml configuration file. + +Configuration File Snippet: + +``` +--- !Config +csv: + ... + +indexes: + ... + +rules: + account: biz + currency: USD + ruleset: + - Replace_Asset + - Replace_Expense +``` + #### Replace_Asset Assigns an "origin" account to a transaction, based on value of the `account` index of a CSV file row. diff --git a/beanborg/rule_engine/decision_tables.py b/beanborg/rule_engine/decision_tables.py index d638d0f..a3990df 100644 --- a/beanborg/rule_engine/decision_tables.py +++ b/beanborg/rule_engine/decision_tables.py @@ -2,9 +2,10 @@ import csv import os +from typing import Dict, List def init_decision_table(file, debug = False): - table = {} + table: Dict[str, List[Tuple]] = {} tablefile = os.path.join(os.getcwd(), file) if not os.path.isfile(tablefile) or os.stat(file).st_size == 0: if debug: print("The decision table file: " + file + " is missing or empty.") @@ -14,8 +15,11 @@ def init_decision_table(file, debug = False): next(csv_reader) # skip first line for row in csv_reader: if any(row): - if len(row) == 3: - table[row[0]] = (row[1], row[2]) + key = row[0] + if len(row) == 3 or len(row) == 4: + if key not in table: + table[key] = [] + table[key].append(tuple(row[1:])) else: print("invalid rule: " + ", ".join(row)) return table @@ -25,8 +29,7 @@ def decomment(csvfile): raw = row.split('#')[0].strip() if raw: yield row -def resolve_from_decision_table(table, string, default): - +def resolve_from_decision_table(table, string, default, account=None): eq_check_func = { "equals": _equals, "equals_ic": _equals_ignore_case, @@ -40,16 +43,24 @@ def resolve_from_decision_table(table, string, default): "ew": _endsWith, "co": _contains } - for k in table.keys(): - t = table[k] - eq_check_type = t[0] - ## TODO: do not fail if string (equals, contains, etc does not match) - if eq_check_func.get(eq_check_type)(string, k): - return t[1] + + for val in table.keys(): + # Sort the list of tuples by the number of elements in each tuple (in descending order) + t = sorted(table[val], key=lambda x: len(x), reverse=True) + + for value in t: + eq_check_type = value[0] + if len(value) == 3 and account is not None: + if eq_check_func.get(eq_check_type)(string, val) and account == value[2]: + return value[1] + elif len(value) == 2: + if eq_check_func.get(eq_check_type)(string, val): + return value[1] + else: + print("ignore row from rule file: " + str(value)) return default - def _equals(string_a, string_b): return string_a == string_b diff --git a/beanborg/rule_engine/rules.py b/beanborg/rule_engine/rules.py index 55cbbc5..66842ac 100644 --- a/beanborg/rule_engine/rules.py +++ b/beanborg/rule_engine/rules.py @@ -183,7 +183,7 @@ class Replace_Expense(Rule): Categorizes a transaction by assigning the account extracted from a look-up table based on the 'payee_pos' index of a CSV file row. - The rule is based on the 'payee.rules' look-up file. + The rule is based on the 'account.rules' look-up file. """ def __init__(self, name, context): Rule.__init__(self, name, context) @@ -199,6 +199,7 @@ def execute(self, csv_line, tx=None, ruleDef=None): LookUpCache.init_decision_table("account", table), csv_line[self.context.payee_pos], self.context.default_expense, + self.context.account ) if expense: posting = Posting(expense, None, None, None, None, None) diff --git a/beanborg/rule_engine/rules_engine.py b/beanborg/rule_engine/rules_engine.py index 4c83fa0..56f2692 100755 --- a/beanborg/rule_engine/rules_engine.py +++ b/beanborg/rule_engine/rules_engine.py @@ -30,10 +30,6 @@ class RuleDef: def get(self, key): return self.attributes[key] - - - - class Rule_Init(Rule): def __init__(self, name, context): Rule.__init__(self, name, context) @@ -79,7 +75,6 @@ def handle(self, cr): return cr def __init__(self, ctx: Context): - self._ctx = ctx self.rules = {} diff --git a/tests/files/account.rules b/tests/files/account.rules index adb3368..f413637 100644 --- a/tests/files/account.rules +++ b/tests/files/account.rules @@ -1,2 +1,5 @@ value;expression;result -freshfood;sw;Expenses:Groceries \ No newline at end of file +freshfood;sw;Expenses:Groceries +books;contains_ic;Expenses:John:Books +books;contains_ic;Expenses:Multimedia:Books;visa + diff --git a/tests/files/bank1.yaml b/tests/files/bank1.yaml index 13796a0..1193b96 100644 --- a/tests/files/bank1.yaml +++ b/tests/files/bank1.yaml @@ -21,7 +21,6 @@ indexes: rules: beancount_file: 'main1.ldg' - #rules_file: luciano.amex.rules account: '1234567' currency: GBP default_expense: 'Expense:Magic' diff --git a/tests/files/bank1_custom_rule.yaml b/tests/files/bank1_custom_rule.yaml index aed1ed2..c4b03e3 100644 --- a/tests/files/bank1_custom_rule.yaml +++ b/tests/files/bank1_custom_rule.yaml @@ -21,7 +21,6 @@ indexes: rules: beancount_file: 'main1.ldg' - #rules_file: luciano.amex.rules account: '1234567' currency: GBP default_expense: 'Expense:Magic' diff --git a/tests/files/bank1_ignore_at_pos.yaml b/tests/files/bank1_ignore_at_pos.yaml index 852cf36..be88323 100644 --- a/tests/files/bank1_ignore_at_pos.yaml +++ b/tests/files/bank1_ignore_at_pos.yaml @@ -21,7 +21,6 @@ indexes: rules: beancount_file: 'main1.ldg' - #rules_file: luciano.amex.rules account: '1234567' currency: GBP default_expense: 'Expense:Magic' diff --git a/tests/files/bank1_ignore_by_counterparty.yaml b/tests/files/bank1_ignore_by_counterparty.yaml index 7d992c3..09b8c2e 100644 --- a/tests/files/bank1_ignore_by_counterparty.yaml +++ b/tests/files/bank1_ignore_by_counterparty.yaml @@ -21,7 +21,6 @@ indexes: rules: beancount_file: 'main1.ldg' - #rules_file: luciano.amex.rules account: '1234567' currency: GBP default_expense: 'Expense:Magic' diff --git a/tests/files/bank1_ignore_contains_string_at_pos.yaml b/tests/files/bank1_ignore_contains_string_at_pos.yaml index f48a97e..2f0b7ce 100644 --- a/tests/files/bank1_ignore_contains_string_at_pos.yaml +++ b/tests/files/bank1_ignore_contains_string_at_pos.yaml @@ -21,7 +21,6 @@ indexes: rules: beancount_file: 'main1.ldg' - #rules_file: luciano.amex.rules account: '1234567' currency: GBP default_expense: 'Expense:Magic' diff --git a/tests/files/bank1_replace_asset.yaml b/tests/files/bank1_replace_asset.yaml index 1cf630f..85c1df3 100644 --- a/tests/files/bank1_replace_asset.yaml +++ b/tests/files/bank1_replace_asset.yaml @@ -21,8 +21,6 @@ indexes: rules: beancount_file: 'main1.ldg' - #rules_file: luciano.amex.rules - account: '1234567' currency: GBP default_expense: 'Expense:Magic' force_negative: true diff --git a/tests/files/bank1_replace_counterparty.yaml b/tests/files/bank1_replace_counterparty.yaml index 0969e23..a7e3777 100644 --- a/tests/files/bank1_replace_counterparty.yaml +++ b/tests/files/bank1_replace_counterparty.yaml @@ -21,7 +21,6 @@ indexes: rules: beancount_file: 'main1.ldg' - #rules_file: luciano.amex.rules account: '1234567' currency: GBP default_expense: 'Expense:Magic' diff --git a/tests/files/bank1_replace_expense.yaml b/tests/files/bank1_replace_expense.yaml index e7daafc..b77b1fc 100644 --- a/tests/files/bank1_replace_expense.yaml +++ b/tests/files/bank1_replace_expense.yaml @@ -21,7 +21,6 @@ indexes: rules: beancount_file: 'main1.ldg' - #rules_file: luciano.amex.rules account: '1234567' currency: GBP default_expense: 'Expense:Magic' diff --git a/tests/files/bank1_replace_expense2.yaml b/tests/files/bank1_replace_expense2.yaml new file mode 100644 index 0000000..b89a5bd --- /dev/null +++ b/tests/files/bank1_replace_expense2.yaml @@ -0,0 +1,30 @@ +--- !Config +csv: + download_path: "/Users/luciano/Desktop" + name: bbk_statement + bank_ref: bbk + target: tmp2 + archive_path: archive2 + separator: '|' + date_format: "%d/%m/%Y" + currency_sep: "," + skip: 3 + +indexes: + date: 8 + counterparty: 9 + amount: 10 + account: 11 + currency: 12 + tx_type: 13 + amount_in: 14 + +rules: + beancount_file: 'main1.ldg' + account: 'visa' + currency: GBP + default_expense: 'Expense:Magic' + force_negative: true + invert_negative: true + ruleset: + - name: Replace_Expense diff --git a/tests/test_decision_tables.py b/tests/test_decision_tables.py index 7408c9b..5f6a89b 100644 --- a/tests/test_decision_tables.py +++ b/tests/test_decision_tables.py @@ -1,42 +1,37 @@ from beanborg.rule_engine.decision_tables import * - +from typing import Dict, List def test_equal_value(): - table = {} - table["superman"] = ("equals", "batman") + table: Dict[str, List[Tuple]] = {"superman": [("equals", "batman")]} assert "batman" == resolve_from_decision_table(table, "superman", "mini") def test_equal_value_different_case(): - table = {} - table["superman"] = ("equals", "batman") + table: Dict[str, List[Tuple]] = {"superman": [("equals", "batman")]} assert "Batman" != resolve_from_decision_table(table, "superman", "mini") def test_equal_value_ignore_different_case(): - table = {} - table["rewe"] = ("equals_ic", "Expenses:Groceries") + table: Dict[str, List[Tuple]] = {"rewe": [("equals_ic", "Expenses:Groceries")]} + + # table: Dict[str, List[Tuple]] = {} + # table["rewe"] = ("equals_ic", "Expenses:Groceries") assert "Expenses:Groceries" == resolve_from_decision_table(table, "rewe", "Expenses:Unknown") def test_startsWith_value(): - table = {} - table["superman"] = ("startsWith", "batman") + table: Dict[str, List[Tuple]] = {"superman": [("startsWith", "batman")]} assert "batman" == resolve_from_decision_table(table, "superman_is_cool", "mini") def test_endsWith_value(): - table = {} - table["superman"] = ("endsWith", "batman") + table: Dict[str, List[Tuple]] = {"superman": [("endsWith", "batman")]} assert "batman" == resolve_from_decision_table(table, "hello_superman", "mini") def test_contains_value(): - table = {} - table["superman"] = ("contains", "batman") + table: Dict[str, List[Tuple]] = {"superman": [("contains", "batman")]} assert "batman" == resolve_from_decision_table( table, "hello_superman_hello", "mini" ) def test_contains_value_ignore_case(): - table = {} - table["rewe"] = ("contains_ic", "Expenses:Groceries") - + table: Dict[str, List[Tuple]] = {"rewe": [("contains_ic", "Expenses:Groceries")]} assert "Expenses:Groceries" == resolve_from_decision_table( table, "card transaction - supermarket REWE", "Expenses:Unknown" ) @@ -44,6 +39,6 @@ def test_contains_value_ignore_case(): def test_loadfile(): table = init_decision_table("tests/files/payee_with_comments.rules") assert table["ford"] != None - assert table["ford"][0] == "contains" - assert table["ford"][1] == "Ford Auto" + assert table["ford"][0][0] == "contains" + assert table["ford"][0][1] == "Ford Auto" \ No newline at end of file diff --git a/tests/test_rules_engine.py b/tests/test_rules_engine.py index 2587e3e..93336b1 100644 --- a/tests/test_rules_engine.py +++ b/tests/test_rules_engine.py @@ -27,6 +27,15 @@ def test_expense_replacement(): tx = rule_engine.execute(entries) assert tx.postings[1].account == "Expenses:Groceries" +def test_expense_replacement2(): + + rule_engine = make_rule_engine('tests/files/bank1_replace_expense2.yaml') + entries = "31.10.2019,b,auszahlung,books,x,ZZ03100400000608903100".split( + "," + ) + tx = rule_engine.execute(entries) + assert tx.postings[1].account == "Expenses:Multimedia:Books" + def test_ignore(): rule_engine = make_rule_engine('tests/files/bank1_ignore_by_counterparty.yaml') @@ -89,12 +98,11 @@ def test_no_rulefile(): def make_rule_engine(config_file): config = init_config(config_file, False) - return RuleEngine( Context( ruleset=config.rules.ruleset, rules_dir="tests/files", - account=None, + account=config.rules.account, date_fomat="%d.%m.%Y", default_expense="Expenses:Unknown", date_pos=0,