Skip to content

Commit

Permalink
🔨 - Update Transactions Endpoint (#429)
Browse files Browse the repository at this point in the history 
* [FEATURE] - Update Transactions Endpoint

* Round 2 of changes related to Transactions

* Round 3 of changes related to Transactions
  • Loading branch information
@burkematthew
burkematthew committed Mar 28, 2022
1 parent 4acb4dc commit 3d875fe
Show file tree
Hide file tree
Showing 4 changed files with 159 additions and 309 deletions.
26 changes: 10 additions & 16 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ an MFA prompt, you'll be prompted on the command line for your code, which by de
goes to SMS unless you specify `--mfa-method=email`. This will also persist a browser
session in $HOME/.mintapi/session to avoid an MFA in the future, unless you specify `--session-path=None`.

If you wish to simplify the number of arguments passed in the command line, you can use a configuration file by specifying `--config-file`. For arguments such as `--extended-transactions`, you can add a line in your config file that says `extended-transactions`. For other arguments that have input, such as `--start-date`, you would add a line such as `start-date=10/01/21`. There are two exceptions to what you can add to the config file: email and password. Since these arguments do not include `--`, you cannot add them to the config file.
If you wish to simplify the number of arguments passed in the command line, you can use a configuration file by specifying `--config-file`. For arguments such as `--transactions`, you can add a line in your config file that says `transactions`. For other arguments that have input, such as `--start-date`, you would add a line such as `start-date=10/01/21`. There are two exceptions to what you can add to the config file: email and password. Since these arguments do not include `--`, you cannot add them to the config file.

### Linux Distributions (including Raspberry Pi OS)

Expand Down Expand Up @@ -130,15 +130,12 @@ make calls to retrieve account/budget information. We recommend using the
mint.get_budgets()

# Get transactions
mint.get_transactions() # as pandas dataframe
mint.get_transactions_csv(include_investment=False) # as raw csv data
mint.get_transactions_json(include_investment=False)
mint.get_transaction_data() # as pandas dataframe

# Get transactions for a specific account
accounts = mint.get_accounts(True)
for account in accounts:
mint.get_transactions_csv(id=account["id"])
mint.get_transactions_json(id=account["id"])
mint.get_transaction_data(id=account["id"])

# Get net worth
mint.get_net_worth()
Expand Down Expand Up @@ -175,16 +172,16 @@ make calls to retrieve account/budget information. We recommend using the
)
# now you can do all the normal api calls
# ex:
mint.get_transactions()
mint.get_transaction_data()
```

---
Run it as a sub-process from your favorite language; `pip install mintapi` creates a binary in your $PATH. From the command-line, the output is JSON:

```shell
usage: mintapi [-h] [--session-path [SESSION_PATH]] [--accounts] [--investment]
[--budgets | --budget_hist] [--net-worth] [--extended-accounts] [--transactions]
[--extended-transactions] [--credit-score] [--credit-report]
[--budgets | --budget_hist] [--net-worth] [--extended-accounts]
[--transactions] [--credit-score] [--credit-report]
[--exclude-inquiries] [--exclude-accounts] [--exclude-utilization]
[--start-date [START_DATE]] [--end-date [END_DATE]]
[--include-investment] [--show-pending]
Expand Down Expand Up @@ -217,19 +214,16 @@ Run it as a sub-process from your favorite language; `pip install mintapi` creat
--net-worth Retrieve net worth information
--extended-accounts Retrieve extended account information (slower, implies --accounts)
--transactions, -t Retrieve transactions
--extended-transactions
Retrieve transactions with extra information and
arguments
--start-date [START_DATE]
Earliest date for which to retrieve transactions.
Used with --transactions or --extended-transactions. Format: mm/dd/yy
Used with --transactions. Format: mm/dd/yy
--end-date [END_DATE]
Latest date for which to retrieve transactions.
Used with --transactions or --extended-transactions. Format: mm/dd/yy
Used with --transactions. Format: mm/dd/yy
--investments Retrieve data related to your investments, whether they be retirement or personal stock purchases
--include-investment Used with --extended-transactions
--include-investment Used with --transactions
--show-pending Exclude pending transactions from being retrieved.
Used with --extended-transactions
Used with --transactions
--filename FILENAME, -f FILENAME
write results to file. can be {csv,json} format.
default is to write to stdout.
Expand Down
251 changes: 68 additions & 183 deletions mintapi/api.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
from datetime import date, datetime, timedelta
from dateutil.relativedelta import relativedelta
import io
import json
import logging
Expand Down Expand Up @@ -299,6 +300,68 @@ def get_investment_data(self):
raise MintException("Cannot find investment data")
return investments["Investment"]

def get_transaction_data(
self,
include_investment=False,
start_date=None,
end_date=None,
remove_pending=True,
id=0,
):
"""
Note: start_date and end_date must be in format mm/dd/yy.
If pulls take too long, consider a narrower range of start and end
date. See json explanation of include_investment.
Also note: Mint includes pending transactions, however these sometimes
change dates/amounts after the transactions post. They have been
removed by default in this pull, but can be included by changing
remove_pending to False
"""

result = self.__call_transactions_endpoint(
include_investment, start_date, end_date, id
)
if "Transaction" in result.keys():
if remove_pending:
filtered = filter(
lambda transaction: transaction["isPending"] == False,
result["Transaction"],
)
transactions = list(filtered)
else:
transactions = result["Transaction"]
for i in transactions:
i["lastUpdatedDate"] = i["metaData"]["lastUpdatedDate"]
i.pop("metaData", None)

else:
raise MintException("Cannot find transaction data")
return transactions

def __call_transactions_endpoint(
self, include_investment=False, start_date=None, end_date=None, id=0
):
# Specifying accountId=0 causes Mint to return investment
# transactions as well. Otherwise they are skipped by
# default.
if include_investment:
id = 0
if start_date is None:
start_date = self.x_months_ago(2)
else:
start_date = convert_mmddyy_to_datetime(start_date)
if end_date is None:
end_date = date.today()
else:
end_date = convert_mmddyy_to_datetime(end_date)
return self.get(
"{}/pfm/v1/transactions?id={}&fromDate={}&toDate={}".format(
MINT_ROOT_URL, id, start_date, end_date
),
headers=self._get_api_key_header(),
).json()

def __call_investments_endpoint(self):
return self.get(
"{}/pfm/v1/investments".format(MINT_ROOT_URL),
Expand Down Expand Up @@ -356,139 +419,6 @@ def get_accounts(self, get_detail=False): # {{{

return accounts

def get_transactions_json(
self,
include_investment=False,
start_date=None,
end_date=None,
id=0,
):
"""Returns the raw JSON transaction data as downloaded from Mint. The JSON
transaction data includes some additional information missing from the
CSV data, such as whether the transaction is pending or completed, but
leaves off the year for current year transactions.
"""

# Converts the start date into datetime format - input must be mm/dd/yy
start_date = convert_mmddyy_to_datetime(start_date)
# Converts the end date into datetime format - input must be mm/dd/yy
end_date = convert_mmddyy_to_datetime(end_date)

all_txns = []
offset = 0
# Mint only returns some of the transactions at once. To get all of
# them, we have to keep asking for more until we reach the end.
while 1:
url = MINT_ROOT_URL + "/getJsonData.xevent"
params = {
"queryNew": "",
"offset": offset,
"comparableType": "8",
"startDate": convert_date_to_string(start_date),
"endDate": convert_date_to_string(end_date),
"rnd": Mint.get_rnd(),
}
# Specifying accountId=0 causes Mint to return investment
# transactions as well. Otherwise they are skipped by
# default.
if self._include_investments_with_transactions(id, include_investment):
params["accountId"] = id
if include_investment:
params["task"] = "transactions"
else:
params["task"] = "transactions,txnfilters"
params["filterType"] = "cash"
result = self.request_and_check(
url,
headers=JSON_HEADER,
params=params,
expected_content_type="text/json|application/json",
)
data = json.loads(result.text)
txns = data["set"][0].get("data", [])
if not txns:
break
all_txns.extend(txns)
offset += len(txns)
return all_txns

def get_detailed_transactions(
self,
include_investment=False,
remove_pending=True,
start_date=None,
end_date=None,
):
"""Returns the JSON transaction data as a DataFrame, and converts
current year dates and prior year dates into consistent datetime
format, and reverses credit activity.
Note: start_date and end_date must be in format mm/dd/yy.
If pulls take too long, consider a narrower range of start and end
date. See json explanation of include_investment.
Also note: Mint includes pending transactions, however these sometimes
change dates/amounts after the transactions post. They have been
removed by default in this pull, but can be included by changing
remove_pending to False
"""
result = self.get_transactions_json(include_investment, start_date, end_date)

df = pd.DataFrame(self.add_parent_category_to_result(result))
df["odate"] = df["odate"].apply(json_date_to_datetime)

if remove_pending:
df = df[~df.isPending]
df.reset_index(drop=True, inplace=True)

df.amount = df.apply(reverse_credit_amount, axis=1)

return df

def add_parent_category_to_result(self, result):
# Finds the parent category name from the categories object based on
# the transaction category ID
categories = self.get_categories()
for transaction in result:
category = self.get_category_object_from_id(
transaction["categoryId"], categories
)
parent = self._find_parent_from_category(category, categories)
transaction["parentCategoryId"] = self.__format_category_id(parent["id"])
transaction["parentCategoryName"] = parent["name"]

return result

def get_transactions_csv(
self, include_investment=False, start_date=None, end_date=None, acct=0
):
"""Returns the raw CSV transaction data as downloaded from Mint.
If include_investment == True, also includes transactions that Mint
classifies as investment-related. You may find that the investment
transaction data is not sufficiently detailed to actually be useful,
however.
"""

# Specifying accountId=0 causes Mint to return investment
# transactions as well. Otherwise they are skipped by
# default.

params = {
"accountId": acct
if self._include_investments_with_transactions(acct, include_investment)
else None,
"startDate": convert_date_to_string(convert_mmddyy_to_datetime(start_date)),
"endDate": convert_date_to_string(convert_mmddyy_to_datetime(end_date)),
}
result = self.request_and_check(
"{}/transactionDownload.event".format(MINT_ROOT_URL),
params=params,
expected_content_type="text/csv",
)
return result.content

def get_net_worth(self, account_data=None):
if account_data is None:
account_data = self.get_accounts()
Expand All @@ -505,23 +435,6 @@ def get_net_worth(self, account_data=None):
]
)

def get_transactions(
self, include_investment=False, start_date=None, end_date=None
):
"""Returns the transaction data as a Pandas DataFrame."""
s = io.BytesIO(
self.get_transactions_csv(
start_date=start_date,
end_date=end_date,
include_investment=include_investment,
)
)
s.seek(0)
df = pd.read_csv(s, parse_dates=["Date"])
df.columns = [c.lower().replace(" ", "_") for c in df.columns]
df.category = df.category.str.lower().replace("uncategorized", pd.NA)
return df

def populate_extended_account_detail(self, accounts): # {{{
# I can't find any way to retrieve this information other than by
# doing this stupid one-call-per-account to listTransactions.xevent
Expand Down Expand Up @@ -594,38 +507,11 @@ def get_budgets(self):
def __call_budgets_endpoint(self):
return self.get(
"{}/pfm/v1/budgets?startDate={}&endDate={}".format(
MINT_ROOT_URL, self.__eleven_months_ago(), self.__first_of_this_month()
MINT_ROOT_URL, self.__x_months_ago(11), self.__first_of_this_month()
),
headers=self._get_api_key_header(),
).json()

def get_category_object_from_id(self, cid, categories):
if cid == 0:
return {"parent": "Uncategorized", "depth": 1, "name": "Uncategorized"}

result = filter(
lambda category: self.__format_category_id(category["id"]) == str(cid),
categories,
)
category = list(result)
return (
category[0]
if len(category) > 0
else {"parent": "Unknown", "depth": 1, "name": "Unknown"}
)

def __format_category_id(self, cid):
return cid if str(cid).find("_") == "-1" else str(cid)[str(cid).find("_") + 1 :]

def _find_parent_from_category(self, category, categories):
if category["depth"] == 1:
return {"id": "", "name": ""}

parent = self.get_category_object_from_id(
self.__format_category_id(category["parentId"]), categories
)
return {"id": parent["id"], "name": parent["name"]}

def initiate_account_refresh(self):
self.make_post_request(url="{}/refreshFILogins.xevent".format(MINT_ROOT_URL))

Expand Down Expand Up @@ -743,14 +629,13 @@ def _flatten_utilization(self, data):
)
return utilization

def _include_investments_with_transactions(self, id, include_investment):
return id > 0 or include_investment

def __first_of_this_month(self):
return date.today().replace(day=1)

def __eleven_months_ago(self):
return (self.__first_of_this_month() - timedelta(days=330)).replace(day=1)
def __x_months_ago(self, months=2):
return (self.__first_of_this_month() - relativedelta(months=months)).replace(
day=1
)


def get_accounts(email, password, get_detail=False):
Expand Down

0 comments on commit 3d875fe

Please sign in to comment.