primary data visualisation.ipynb

{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**TASKS**\n\n1. Always normalize data. - use Z scaling. \n2. eliminate rows with greater than 5 sig deviation\n3. compare the statistics for train, val and test\n4. One hot encoding for coupling type\n5. Check if Electronegativity is a necessary measure - Khan Academy\n5. Check if size and type of atom influence the value of coupling constant - Khan Academy and others\n6. Test to see if separating the prediction coupling types is a good idea\n7. Investigate corrolations\n8. HOW DO NEARBY ATOMS AFFECT THE DISTRIBUTION? ELECTRONEGATIVITY?\n\n**NOTES:**\n\n1. No corrolation between mulliken charge and scalar coupling constant\n2. No corrolation between mulliken charge and coupling type\n3. May need to isolate certain types of coupling based on distribution of scalar coupling. Refer violin plot of distribution of scalar coupling constant\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"#libraries\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.metrics import mean_absolute_error\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom mpl_toolkits.mplot3d import Axes3D","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def normalise (x):\n    temp = pd.Series()\n    temp = (x - x.mean())/x.std()\n    return temp, x.mean(), x.std()\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X.atom_x.unique(), X.atom_y.unique(), X.type.unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def nega (x):\n    nega_dict = {'C':2.55, 'H':2.2, 'N':3.04, 'O':3.44, 'F':3.98}\n    return [nega_dict[var] for var in x]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def radii (x):\n    radi_dict = {'C':2.55, 'H':2.2, 'N':3.04, 'O':3.44, 'F':3.98}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"structure.atom.unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# nega_dict = {'C':2.55, 'H':2.2, 'N':3.04, 'O':3.44, 'F':3.98}\ntemp = ['C','C','H','N','O','F']\nnega(temp)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"structure.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\" LOG MAE score calculator\n    taken from the public kernel - https://www.kaggle.com/marcelotamashiro/lgb-public-kernels-plus-more-features @marcelotomashiro\n\"\"\"\n\ndef comp_score (y_true, y_pred, jtype):\n    df = pd.DataFrame()\n    df['y_true'] , df['y_pred'], df['jtype'] = y_true , y_pred, jtype\n    score = 0 \n    for t in jtype.unique():\n        score_jtype = np.log(mean_absolute_error(df[df.jtype==t]['y_true'],df[df.jtype==t]['y_pred']))\n        score += score_jtype\n        print(f'{t} : {score_jtype}')\n    score /= len(jtype.unique())\n    return score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/train.csv\")\nstructure = pd.read_csv(\"../input/structures.csv\")\n\nX = train[train.columns[1:-1]]\ny = train[train.columns[-1]]\n\nX = pd.merge(X,structure, left_on=['molecule_name','atom_index_0'], right_on=['molecule_name','atom_index'], how='left')\nX = pd.merge(X,structure, left_on=['molecule_name','atom_index_1'], right_on=['molecule_name','atom_index'], how='left')\nX = X.drop(['atom_index_x','atom_index_y'], axis =1)\n\nX_train, X_valid, y_train, y_valid = train_test_split(X,y,test_size=0.3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### import additional stuff\nmulliken_charges = pd.read_csv(\"../input/mulliken_charges.csv\")\nmulliken_charges.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv(\"../input/test.csv\")\nX_test, y_test = test[test.columns[1:-1]],test[test.columns[-1]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.type.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_m = pd.DataFrame()\nX_m = pd.merge(X, mulliken_charges, left_on=['molecule_name','atom_index_0'], right_on=['molecule_name','atom_index'], how='left')\nX_m = pd.merge(X_m, mulliken_charges, left_on=['molecule_name','atom_index_1'], right_on=['molecule_name','atom_index'], how='left')\nX_m = X_m.drop(['atom_index_x','atom_index_y'], axis =1)\n\nX_m.head()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"structure.atom.unique()\n# X.atom_y.unique()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_m.type.unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_m_temp = X_m.loc[X_m.type=='2JHN']\nfig = plt.figure()\nax = fig.add_subplot(111, projection='3d')\nax.scatter(X_m_temp.x_x, X_m_temp.y_x, y[X_m_temp.index])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_m.mulliken_charge_x.max() - X_m.mulliken_charge_x.min()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.pyplot.scatter(np.abs([X_m.mulliken_charge_y[::500] - X_m.mulliken_charge_x[::500]]), y[::500])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.violinplot(X_m.type[::500], np.abs(X_m.mulliken_charge_x[::500] - X_m.mulliken_charge_y[::500]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"structure.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# bond length calculation\ndef bondlength (X, X_structures):\n# calculate cartesian distance    \n    X  \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"[x for x in range(5)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lista = [x for x in range(len(train))]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"temp = train.assign(atom_0 = lista)\ntemp.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"structure.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"structure.x.loc[(structure.molecule_name == 'dsgdb9nsd_000001') & (structure.atom == 'C')]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"var = 'x'\nstructure[var].loc[(structure.molecule_name == 'dsgdb9nsd_000001') & (structure.atom == 'C')]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def newfeature(X, struct, target, filter_1, filter_2, filter_3):\n    temp = list()\n    for index, molecule in X.iterrows():\n        temp.append(struct[target].loc((struct[filter_1] == molecule[filter_1]) & (struct[filter_3] == molecule[filter_2])))\n    \n    return temp\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# atom_0_x, atom_0_y, atom_0 = some function\n\ntrain_compd = train.assign(atom_0_x = newfeature(train,structure,'x','molecule_name','atom_index_0','atom_index'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# IGNORE\ncompendium = pd.DataFrame()\nfor molecule in train.molecule_name:\n    train.loc[train.molecule_name==molecule].insert({'bondlength':bondlength(train.loc[train.molecule_name==molecule], \n                                                                            structures.loc[structures.molecule_name==molecule])})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"temp,scc_mean, scc_std = normalise(train.scalar_coupling_constant)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import seaborn as sns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# log transformation\nlog_temp = np.log(temp - temp.min())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"temp.loc[temp>1.5].count()/temp.count()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**RESIDUAL CODE AFTER THIS POINT**"},{"metadata":{"trusted":true},"cell_type":"code","source":"train.loc[train.molecule_name=='dsgdb9nsd_000001']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.loc[train.type=='2JHH'].index.max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.sqrt(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X.x_x.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def bond_length(var):\n    return np.sqrt((var.x_x - var.x_y)**2 + (var.y_x - var.y_y)**2 + (var.z_x - var.z_y)**2)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = X.assign(bond_length = bond_length(X))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_sub = train.loc[(train.type!='1JHC') & (train.type!='1JHN')]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_sub_norm, sub_mean, sub_std = normalise(train_sub.scalar_coupling_constant)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_sub_norm.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_sub_norm_red = y_sub_norm.loc[(y_sub_norm >= -4) & (y_sub_norm <= 4)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_sub_norm_red.shape[0] - y_sub_norm.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_sub.type.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_sub.loc[y_sub_norm_red.index].type.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.violinplot(train.type,train.scalar_coupling_constant)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X.loc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X.loc[X.type=='3JHN']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"var = 7\nsns.scatterplot(X.bond_length.loc[X.type == X.type.unique()[var]], y.iloc[X.loc[X.type == X.type.unique()[var]].index] )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.violinplot(X.type, X.bond_length)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"Train set minus three types - 1JHC, 2JHH, 1JHN\n\n\"\"\"\ntrain_minus_three = train.loc[ (train.type!= '1JHC') & (train.type!= '2JHH') & (train.type!= '1JHN')]\ny_minus, minus_mean, minus_std = normalise(train_minus_three.scalar_coupling_constant)\ntrain_minus_red = train_minus_three.loc[y_minus.loc[(y_minus >= -5) & (y_minus<= 5)].index]\n\ntrain_plus_three = train.loc[ (train.type == '1JHC') | (train.type == '2JHH') | (train.type == '1JHN')]\ny_plus, plus_mean, plus_std = normalise(train_plus_three.scalar_coupling_constant)\ntrain_plus_red = train_plus_three.loc[y_plus.loc[(y_plus >= -5) & (y_plus <= 5)].index]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_plus_red.type.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_plus_three.type.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.distplot(y_plus, kde=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.distplot(y_minus.loc[(y_minus >= -5) & (y_minus<= 5)], kde=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.distplot(y_minus, kde=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.distplot(y_sub_norm_red, kde= True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.distplot(y_sub_norm, kde=True)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}