Skip to content

Commit

Permalink
Fixing broken pd.merge due to Pandas upgrade
Browse files Browse the repository at this point in the history
We can only pass argument "on" OR "left_index" and "right_index", not a combination of both in pd.merge()
Fix for the issue:
e-mission/e-mission-docs#856 (comment)
  • Loading branch information
swastis10 committed Mar 6, 2023
1 parent d2ada64 commit a760b55
Show file tree
Hide file tree
Showing 2 changed files with 5 additions and 3 deletions.
3 changes: 2 additions & 1 deletion emission/analysis/intake/cleaning/filter_accuracy.py
Expand Up @@ -100,7 +100,8 @@ def filter_accuracy(user_id):
logging.debug("Partial filtered points %d found" % len(filtered_points_df))
filtered_points_df = filtered_points_df[SEL_FIELDS_FOR_DUP]
filtered_points_df["msts"] = filtered_points_df.ts.apply(lambda x: int(x * 10**3))
matched_points_df = filtered_from_unfiltered_df.merge(filtered_points_df, on="msts", right_index=True)
matched_points_df = filtered_from_unfiltered_df.reset_index().merge(filtered_points_df, on="msts")
matched_points_df.set_index('index', inplace=True)
to_insert_df = filtered_from_unfiltered_df.drop(index=matched_points_df.index)
for idx, entry in to_insert_df.iterrows():
unfiltered_entry = unfiltered_points_list[idx]
Expand Down
Expand Up @@ -165,8 +165,9 @@ def testPandasMergeBehavior(self):
import pandas as pd
df_a = pd.DataFrame({"ts": [1,2,3,4]})
df_b = pd.DataFrame({"ts": [1,3]})
merged_left_idx = df_a.merge(df_b, on="ts", how="inner", left_index=True)
merged_right_idx = df_a.merge(df_b, on="ts", how="inner", right_index=True)
merged_left_idx = df_a.merge(df_b, on="ts", how="inner")
merged_right_idx = df_a.reset_index().merge(df_b, on="ts", how="inner")
merged_right_idx.set_index('index', inplace=True)
self.assertEqual(merged_left_idx.index.to_list(), [0,1])
self.assertEqual(merged_right_idx.index.to_list(), [0,2])

Expand Down

0 comments on commit a760b55

Please sign in to comment.