From d82f67397cf39ff4263528a99caf12966c13b4c6 Mon Sep 17 00:00:00 2001 From: Matthew Burke Date: Sun, 20 Mar 2022 13:30:30 -0500 Subject: [PATCH 1/3] [CHORE] - Update Accounts Endpoint --- mintapi/api.py | 79 ++++++++++++++++++++------------------------------ mintapi/cli.py | 72 +++++++++++---------------------------------- 2 files changed, 48 insertions(+), 103 deletions(-) diff --git a/mintapi/api.py b/mintapi/api.py index ef463392..59ef91fa 100644 --- a/mintapi/api.py +++ b/mintapi/api.py @@ -299,6 +299,28 @@ def get_investment_data(self): raise MintException("Cannot find investment data") return investments["Investment"] + def get_account_data(self, get_detail=False): + accounts = self.__call_accounts_endpoint() + #for account in accounts: + # convert_account_dates_to_datetime(account) + + #if get_detail: + # accounts = self.populate_extended_account_detail(accounts) + if "Account" in accounts.keys(): + for i in accounts["Account"]: + i["createdDate"] = i["metaData"]["createdDate"] + i["lastUpdatedDate"] = i["metaData"]["lastUpdatedDate"] + i.pop("metaData", None) + else: + raise MintException("Cannot find account data") + return accounts["Account"] + + def __call_accounts_endpoint(self, get_detail=False): + return self.get( + "{}/pfm/v1/accounts".format(MINT_ROOT_URL), + headers=self._get_api_key_header(), + ).json() + def __call_investments_endpoint(self): return self.get( "{}/pfm/v1/investments".format(MINT_ROOT_URL), @@ -310,51 +332,12 @@ def get_categories(self): "{}/pfm/v1/categories".format(MINT_ROOT_URL), headers=self._get_api_key_header(), ).json()["Category"] - - def get_accounts(self, get_detail=False): # {{{ - # Issue service request. - req_id = self.get_request_id_str() - - input = { - "args": { - "types": [ - "BANK", - "CREDIT", - "INVESTMENT", - "LOAN", - "MORTGAGE", - "OTHER_PROPERTY", - "REAL_ESTATE", - "VEHICLE", - "UNCLASSIFIED", - ] - }, - "id": req_id, - "service": "MintAccountService", - "task": "getAccountsSorted" - # 'task': 'getAccountsSortedByBalanceDescending' - } - - data = {"input": json.dumps([input])} - response = self.make_post_request( - url=self.build_bundledServiceController_url(), - data=data, - convert_to_text=True, - ) - if req_id not in response: - raise MintException("Could not parse account data: " + response) - - # Parse the request - response = json.loads(response) - accounts = response["response"][req_id]["response"] - - for account in accounts: - convert_account_dates_to_datetime(account) - - if get_detail: - accounts = self.populate_extended_account_detail(accounts) - - return accounts + + def __call_accounts_endpoint(self): + return self.get( + "{}/pfm/v1/accounts".format(MINT_ROOT_URL), + headers=self._get_api_key_header(), + ).json() def set_user_property(self, name, value): req_id = self.get_request_id_str() @@ -526,7 +509,7 @@ def get_transactions_csv( def get_net_worth(self, account_data=None): if account_data is None: - account_data = self.get_accounts() + account_data = self.get_account_data() # account types in this list will be subtracted invert = set(["loan", "loans", "credit"]) @@ -836,12 +819,12 @@ def _include_investments_with_transactions(self, id, include_investment): def get_accounts(email, password, get_detail=False): mint = Mint(email, password) - return mint.get_accounts(get_detail=get_detail) + return mint.get_account_data(get_detail=get_detail) def get_net_worth(email, password): mint = Mint(email, password) - account_data = mint.get_accounts() + account_data = mint.get_account_data() return mint.get_net_worth(account_data) diff --git a/mintapi/cli.py b/mintapi/cli.py index 6d012a35..005871f8 100644 --- a/mintapi/cli.py +++ b/mintapi/cli.py @@ -114,7 +114,7 @@ def parse_arguments(args): { "nargs": "?", "default": None, - "help": "Latest date for transactions to be retrieved from. Used with --transactions or --extended-transactions. Format: mm/dd/yy", + "help": "Latest date for transactions to be retrieved from. Used with --transactions. Format: mm/dd/yy", }, ), ( @@ -150,14 +150,6 @@ def parse_arguments(args): "help": "When accessing credit report details, exclude data related to credit utilization. Used with --credit-report.", }, ), - ( - ("--extended-transactions",), - { - "action": "store_true", - "default": False, - "help": "Retrieve transactions with extra information and arguments", - }, - ), ( ("--filename", "-f"), { @@ -192,7 +184,7 @@ def parse_arguments(args): { "action": "store_true", "default": False, - "help": "Used with --extended-transactions", + "help": "Used with --transactions", }, ), ( @@ -245,15 +237,7 @@ def parse_arguments(args): { "action": "store_false", "default": True, - "help": "Exclude pending transactions from being retrieved. Used with --extended-transactions", - }, - ), - ( - ("--skip-duplicates",), - { - "action": "store_true", - "default": False, - "help": "Used with --extended-transactions", + "help": "Exclude pending transactions from being retrieved. Used with --transactions", }, ), ( @@ -261,7 +245,7 @@ def parse_arguments(args): { "nargs": "?", "default": None, - "help": "Earliest date for transactions to be retrieved from. Used with --transactions or --extended-transactions. Format: mm/dd/yy", + "help": "Earliest date for transactions to be retrieved from. Used with --transactions. Format: mm/dd/yy", }, ), ( @@ -332,7 +316,6 @@ def validate_file_extensions(options): if any( [ options.transactions, - options.extended_transactions, options.investments, ] ): @@ -350,28 +333,19 @@ def validate_file_extensions(options): def output_data(options, data, attention_msg=None): - # output the data - if options.transactions or options.extended_transactions: - if options.filename is None: - print(data.to_json(orient="records")) - elif options.filename.endswith(".csv"): - data.to_csv(options.filename, index=False) - elif options.filename.endswith(".json"): - data.to_json(options.filename, orient="records") - else: - if options.filename is None: - print(json.dumps(data, indent=2)) + if options.filename is None: + print(json.dumps(data, indent=2)) # NOTE: While this logic is here, unless validate_file_extensions # allows for other data types to export to CSV, this will # only include investment data. - elif options.filename.endswith(".csv"): - # NOTE: Currently, investment_data, which is a flat JSON, is the only - # type of data that uses this section. So, if we open this up to - # other non-flat JSON data, we will need to revisit this. - json_normalize(data).to_csv(options.filename, index=False) - elif options.filename.endswith(".json"): - with open(options.filename, "w+") as f: - json.dump(data, f, indent=2) + elif options.filename.endswith(".csv"): + # NOTE: Currently, investment_data, which is a flat JSON, is the only + # type of data that uses this section. So, if we open this up to + # other non-flat JSON data, we will need to revisit this. + json_normalize(data).to_csv(options.filename, index=False) + elif options.filename.endswith(".json"): + with open(options.filename, "w+") as f: + json.dump(data, f, indent=2) if options.attention: if attention_msg is None or attention_msg == "": @@ -420,7 +394,6 @@ def main(): options.accounts, options.budgets, options.transactions, - options.extended_transactions, options.net_worth, options.credit_score, options.credit_report, @@ -468,9 +441,7 @@ def main(): data = None if options.accounts and options.budgets: try: - accounts = make_accounts_presentable( - mint.get_accounts(get_detail=options.accounts_ext) - ) + accounts = mint.get_account_data(get_detail=options.accounts_ext) except Exception: accounts = None @@ -492,24 +463,15 @@ def main(): data = None elif options.accounts: try: - data = make_accounts_presentable( - mint.get_accounts(get_detail=options.accounts_ext) - ) + data = mint.get_account_data(get_detail=False) except Exception: data = None elif options.transactions: - data = mint.get_transactions( - start_date=options.start_date, - end_date=options.end_date, - include_investment=options.include_investment, - ) - elif options.extended_transactions: - data = mint.get_detailed_transactions( + data = mint.get_transaction_data( start_date=options.start_date, end_date=options.end_date, include_investment=options.include_investment, remove_pending=options.show_pending, - skip_duplicates=options.skip_duplicates, ) elif options.categories: data = mint.get_categories() From 369e489b6dea79d1eb26e654bea8a198b78df1fb Mon Sep 17 00:00:00 2001 From: Matthew Burke Date: Sun, 27 Mar 2022 18:45:23 -0500 Subject: [PATCH 2/3] [CHORE] - Part 2 of Accounts Endpoint --- mintapi/api.py | 93 ++----------------------------------------- mintapi/cli.py | 69 +++++++++++++++++++------------- tests/test_driver.py | 94 ++++++++++++++++++++++++++++++-------------- 3 files changed, 109 insertions(+), 147 deletions(-) diff --git a/mintapi/api.py b/mintapi/api.py index 19bf3210..384902e5 100644 --- a/mintapi/api.py +++ b/mintapi/api.py @@ -57,27 +57,6 @@ def parse_float(str_number): return None -DATE_FIELDS = [ - "addAccountDate", - "closeDate", - "fiLastUpdated", - "lastUpdated", -] - - -def convert_account_dates_to_datetime(account): - for df in DATE_FIELDS: - if df in account: - # Convert from javascript timestamp to unix timestamp - # http://stackoverflow.com/a/9744811/5026 - try: - ts = account[df] / 1e3 - except TypeError: - # returned data is not a number, don't parse - continue - account[df + "InDate"] = datetime.fromtimestamp(ts) - - MINT_ROOT_URL = "https://mint.intuit.com" MINT_ACCOUNTS_URL = "https://accounts.intuit.com" MINT_CREDIT_URL = "https://credit.finance.intuit.com" @@ -299,13 +278,8 @@ def get_investment_data(self): raise MintException("Cannot find investment data") return investments["Investment"] - def get_account_data(self, get_detail=False): + def get_account_data(self): accounts = self.__call_accounts_endpoint() - #for account in accounts: - # convert_account_dates_to_datetime(account) - - #if get_detail: - # accounts = self.populate_extended_account_detail(accounts) if "Account" in accounts.keys(): for i in accounts["Account"]: i["createdDate"] = i["metaData"]["createdDate"] @@ -315,7 +289,7 @@ def get_account_data(self, get_detail=False): raise MintException("Cannot find account data") return accounts["Account"] - def __call_accounts_endpoint(self, get_detail=False): + def __call_accounts_endpoint(self): return self.get( "{}/pfm/v1/accounts".format(MINT_ROOT_URL), headers=self._get_api_key_header(), @@ -332,12 +306,12 @@ def get_categories(self): "{}/pfm/v1/categories".format(MINT_ROOT_URL), headers=self._get_api_key_header(), ).json()["Category"] - + def __call_accounts_endpoint(self): return self.get( "{}/pfm/v1/accounts".format(MINT_ROOT_URL), headers=self._get_api_key_header(), - ).json() + ).json() def get_transactions_json( self, @@ -505,65 +479,6 @@ def get_transactions( df.category = df.category.str.lower().replace("uncategorized", pd.NA) return df - def populate_extended_account_detail(self, accounts): # {{{ - # I can't find any way to retrieve this information other than by - # doing this stupid one-call-per-account to listTransactions.xevent - # and parsing the HTML snippet :( - for account in accounts: - headers = dict(JSON_HEADER) - headers["Referer"] = "{}/transaction.event?accountId={}".format( - MINT_ROOT_URL, account["id"] - ) - - list_txn_url = "{}/listTransaction.xevent".format(MINT_ROOT_URL) - params = { - "accountId": str(account["id"]), - "queryNew": "", - "offset": 0, - "comparableType": 8, - "acctChanged": "T", - "rnd": Mint.get_rnd(), - } - - response = json.loads( - self.get(list_txn_url, params=params, headers=headers).text - ) - xml = "
" + response["accountHeader"] + "
" - xml = xml.replace("–", "-") - xml = xmltodict.parse(xml) - - account["availableMoney"] = None - account["totalFees"] = None - account["totalCredit"] = None - account["nextPaymentAmount"] = None - account["nextPaymentDate"] = None - - xml = xml["div"]["div"][1]["table"] - if "tbody" not in xml: - continue - xml = xml["tbody"] - table_type = xml["@id"] - xml = xml["tr"][1]["td"] - - if table_type == "account-table-bank": - account["availableMoney"] = parse_float(xml[1]["#text"]) - account["totalFees"] = parse_float(xml[3]["a"]["#text"]) - if account["interestRate"] is None: - account["interestRate"] = parse_float(xml[2]["#text"]) / 100.0 - elif table_type == "account-table-credit": - account["availableMoney"] = parse_float(xml[1]["#text"]) - account["totalCredit"] = parse_float(xml[2]["#text"]) - account["totalFees"] = parse_float(xml[4]["a"]["#text"]) - if account["interestRate"] is None: - account["interestRate"] = parse_float(xml[3]["#text"]) / 100.0 - elif table_type == "account-table-loan": - account["nextPaymentAmount"] = parse_float(xml[1]["#text"]) - account["nextPaymentDate"] = xml[2].get("#text", None) - elif table_type == "account-type-investment": - account["totalFees"] = parse_float(xml[2]["a"]["#text"]) - - return accounts - def get_budgets(self): budgets = self.__call_budgets_endpoint() if "Budget" in budgets.keys(): diff --git a/mintapi/cli.py b/mintapi/cli.py index d47ac5f3..5611df4c 100644 --- a/mintapi/cli.py +++ b/mintapi/cli.py @@ -114,16 +114,7 @@ def parse_arguments(args): { "nargs": "?", "default": None, - "help": "Latest date for transactions to be retrieved from. Used with --transactions. Format: mm/dd/yy", - }, - ), - ( - ("--extended-accounts",), - { - "action": "store_true", - "dest": "accounts_ext", - "default": False, - "help": "Retrieve extended account information (slower, implies --accounts)", + "help": "Latest date for transactions to be retrieved from. Used with --transactions or --extended-transactions. Format: mm/dd/yy", }, ), ( @@ -150,6 +141,14 @@ def parse_arguments(args): "help": "When accessing credit report details, exclude data related to credit utilization. Used with --credit-report.", }, ), + ( + ("--extended-transactions",), + { + "action": "store_true", + "default": False, + "help": "Retrieve transactions with extra information and arguments", + }, + ), ( ("--filename", "-f"), { @@ -184,7 +183,7 @@ def parse_arguments(args): { "action": "store_true", "default": False, - "help": "Used with --transactions", + "help": "Used with --extended-transactions", }, ), ( @@ -245,7 +244,7 @@ def parse_arguments(args): { "nargs": "?", "default": None, - "help": "Earliest date for transactions to be retrieved from. Used with --transactions. Format: mm/dd/yy", + "help": "Earliest date for transactions to be retrieved from. Used with --transactions or --extended-transactions. Format: mm/dd/yy", }, ), ( @@ -316,6 +315,7 @@ def validate_file_extensions(options): if any( [ options.transactions, + options.extended_transactions, options.investments, ] ): @@ -333,19 +333,28 @@ def validate_file_extensions(options): def output_data(options, data, attention_msg=None): - if options.filename is None: - print(json.dumps(data, indent=2)) + # output the data + if options.transactions or options.extended_transactions: + if options.filename is None: + print(data.to_json(orient="records")) + elif options.filename.endswith(".csv"): + data.to_csv(options.filename, index=False) + elif options.filename.endswith(".json"): + data.to_json(options.filename, orient="records") + else: + if options.filename is None: + print(json.dumps(data, indent=2)) # NOTE: While this logic is here, unless validate_file_extensions # allows for other data types to export to CSV, this will # only include investment data. - elif options.filename.endswith(".csv"): - # NOTE: Currently, investment_data, which is a flat JSON, is the only - # type of data that uses this section. So, if we open this up to - # other non-flat JSON data, we will need to revisit this. - json_normalize(data).to_csv(options.filename, index=False) - elif options.filename.endswith(".json"): - with open(options.filename, "w+") as f: - json.dump(data, f, indent=2) + elif options.filename.endswith(".csv"): + # NOTE: Currently, investment_data, which is a flat JSON, is the only + # type of data that uses this section. So, if we open this up to + # other non-flat JSON data, we will need to revisit this. + json_normalize(data).to_csv(options.filename, index=False) + elif options.filename.endswith(".json"): + with open(options.filename, "w+") as f: + json.dump(data, f, indent=2) if options.attention: if attention_msg is None or attention_msg == "": @@ -386,14 +395,12 @@ def main(): options.keyring, ) - if options.accounts_ext: - options.accounts = True - if not any( [ options.accounts, options.budgets, options.transactions, + options.extended_transactions, options.net_worth, options.credit_score, options.credit_report, @@ -441,7 +448,7 @@ def main(): data = None if options.accounts and options.budgets: try: - accounts = mint.get_account_data(get_detail=options.accounts_ext) + data = mint.get_account_data() except Exception: accounts = None @@ -463,11 +470,17 @@ def main(): data = None elif options.accounts: try: - data = mint.get_account_data(get_detail=False) + data = mint.get_account_data() except Exception: data = None elif options.transactions: - data = mint.get_transaction_data( + data = mint.get_transactions( + start_date=options.start_date, + end_date=options.end_date, + include_investment=options.include_investment, + ) + elif options.extended_transactions: + data = mint.get_detailed_transactions( start_date=options.start_date, end_date=options.end_date, include_investment=options.include_investment, diff --git a/tests/test_driver.py b/tests/test_driver.py index 113ab205..ab0ce46b 100644 --- a/tests/test_driver.py +++ b/tests/test_driver.py @@ -14,15 +14,62 @@ from unittest.mock import patch, DEFAULT -accounts_example = [ - { - "accountName": "Chase Checking", - "lastUpdated": 1401201492000, - "lastUpdatedInString": "25 minutes", - "accountType": "bank", - "currentBalance": 100.12, - } -] +accounts_example = { + "Account": [ + { + "type": "CreditAccount", + "userCardType": "UNKNOWN", + "creditAccountType": "CREDIT_CARD", + "creditLimit": 2222.0, + "availableCredit": 1111.0, + "interestRate": 0.444, + "minPayment": 111.0, + "absoluteMinPayment": 111.0, + "statementMinPayment": 22.0, + "statementDueDate": "2022-04-19T07:00:00Z", + "statementDueAmount": 0.0, + "metaData": { + "createdDate": "2017-01-05T17:12:15Z", + "lastUpdatedDate": "2022-03-27T16:46:41Z", + "link": [ + { + "otherAttributes": {}, + "href": "/v1/accounts/id", + "rel": "self", + } + ], + }, + "id": "id", + "name": "name", + "value": -555.55, + "isVisible": True, + "isDeleted": False, + "planningTrendsVisible": True, + "accountStatus": "ACTIVE", + "systemStatus": "ACTIVE", + "currency": "USD", + "fiLoginId": "fiLoginId", + "fiLoginStatus": "OK", + "currentBalance": 555.55, + "cpId": "cpId", + "cpAccountName": "cpAccountName", + "cpAccountNumberLast4": "cpAccountNumberLast4", + "hostAccount": False, + "fiName": "fiName", + "accountTypeInt": 0, + "isAccountClosedByMint": False, + "isAccountNotFound": False, + "isActive": True, + "isClosed": False, + "isError": False, + "isHiddenFromPlanningTrends": True, + "isTerminal": True, + "credentialSetId": "credentialSetId", + "ccAggrStatus": "0", + } + ] +} + category_example = [ { @@ -204,27 +251,6 @@ def request(a, b, **c): class MintApiTests(unittest.TestCase): - @patch.object(mintapi.api, "sign_in") - @patch.object(mintapi.api, "_create_web_driver_at_mint_com") - def test_accounts(self, mock_driver, mock_sign_in): - mock_driver.return_value = TestMock() - mock_sign_in.return_value = ("test", "token") - accounts = mintapi.get_accounts("foo", "bar") - - self.assertFalse("lastUpdatedInDate" in accounts) - self.assertNotEqual(accounts, accounts_example) - - accounts_annotated = copy.deepcopy(accounts_example) - for account in accounts_annotated: - account["lastUpdatedInDate"] = datetime.datetime.fromtimestamp( - account["lastUpdated"] / 1000 - ) - self.assertEqual(accounts, accounts_annotated) - - # ensure everything is json serializable as this is the command-line - # behavior. - mintapi.cli.print_accounts(accounts) - def test_chrome_driver_links(self): latest_version = mintapi.signIn.get_latest_chrome_driver_version() for platform in mintapi.signIn.CHROME_ZIP_TYPES: @@ -320,6 +346,14 @@ def test_build_bundledServiceController_url(self, mock_driver): url = mintapi.Mint.build_bundledServiceController_url(mock_driver) self.assertTrue(mintapi.api.MINT_ROOT_URL in url) + @patch.object(mintapi.Mint, "_Mint__call_accounts_endpoint") + def test_get_investment_data_new(self, mock_call_accounts_endpoint): + mock_call_accounts_endpoint.return_value = accounts_example + account_data = mintapi.Mint().get_account_data()[0] + self.assertFalse("metaData" in account_data) + self.assertTrue("createdDate" in account_data) + self.assertTrue("lastUpdatedDate" in account_data) + @patch.object(mintapi.Mint, "_Mint__call_investments_endpoint") def test_get_investment_data_new(self, mock_call_investments_endpoint): mock_call_investments_endpoint.return_value = investments_example From bbfc1cf5a1424e967c80acc7e859291eae428e7e Mon Sep 17 00:00:00 2001 From: Matthew Burke Date: Mon, 28 Mar 2022 13:48:12 -0500 Subject: [PATCH 3/3] Final Cleanup --- mintapi/api.py | 15 ++++----------- mintapi/cli.py | 19 ------------------- 2 files changed, 4 insertions(+), 30 deletions(-) diff --git a/mintapi/api.py b/mintapi/api.py index 6fa940a1..a93134af 100644 --- a/mintapi/api.py +++ b/mintapi/api.py @@ -1,7 +1,5 @@ -from datetime import date, datetime, timedelta +from datetime import date, datetime from dateutil.relativedelta import relativedelta -import io -import json import logging import os import random @@ -10,9 +8,6 @@ import time import warnings -import xmltodict -import pandas as pd - from mintapi.signIn import sign_in, _create_web_driver_at_mint_com @@ -344,7 +339,7 @@ def __call_transactions_endpoint( if include_investment: id = 0 if start_date is None: - start_date = self.x_months_ago(2) + start_date = self.__x_months_ago(2) else: start_date = convert_mmddyy_to_datetime(start_date) if end_date is None: @@ -381,12 +376,10 @@ def get_net_worth(self, account_data=None): account_data = self.get_account_data() # account types in this list will be subtracted - invert = set(["loan", "loans", "credit"]) + invert = set(["LoanAccount", "CreditAccount"]) return sum( [ - -a["currentBalance"] - if a["accountType"] in invert - else a["currentBalance"] + -a["currentBalance"] if a["type"] in invert else a["currentBalance"] for a in account_data if a["isActive"] ] diff --git a/mintapi/cli.py b/mintapi/cli.py index d0922a10..ed245083 100644 --- a/mintapi/cli.py +++ b/mintapi/cli.py @@ -3,7 +3,6 @@ import os import sys import json -from datetime import datetime import getpass import keyring @@ -269,24 +268,6 @@ def parse_arguments(args): return cmdline.parse_args(args) -def make_accounts_presentable(accounts, presentable_format="EXCEL"): - formatter = { - "DATE": "%Y-%m-%d", - "ISO8601": "%Y-%m-%dT%H:%M:%SZ", - "EXCEL": "%Y-%m-%d %H:%M:%S", - }[presentable_format] - - for account in accounts: - for k, v in account.items(): - if isinstance(v, datetime): - account[k] = v.strftime(formatter) - return accounts - - -def print_accounts(accounts): - print(json.dumps(make_accounts_presentable(accounts), indent=2)) - - def handle_password(type, prompt, email, password, use_keyring=False): if use_keyring and not password: # If we don't yet have a password, try prompting for it