
# coding: utf-8

# In[1]:


##This kernel draws heavily from Other Kaggle Kernels on Embdeddings Adjusted to match this Comp.... So thanks to them!!!


# In[2]:


from numba import jit
import numpy as np 
import pandas as pd 
from datetime import datetime as dt
import os
import seaborn as sns 
import matplotlib.pyplot as plt
plt.style.use('ggplot')
import lightgbm as lgb
import xgboost as xgb
import time
import datetime
from tqdm import tqdm_notebook as tqdm
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import StratifiedKFold, KFold, TimeSeriesSplit
from sklearn.metrics import mean_squared_error, roc_auc_score
from sklearn import metrics
from itertools import combinations
import gc

import warnings
warnings.filterwarnings("ignore")
import os
from functools import wraps
from timeit import default_timer as timer

import os
print(os.listdir("./"))

dtypes = {
        'MachineIdentifier':                                    'category',
        'ProductName':                                          'category',
        'EngineVersion':                                        'category',
        'AppVersion':                                           'category',
        'AvSigVersion':                                         'category',
        'IsBeta':                                               'int8',
        'RtpStateBitfield':                                     'float16',
        'IsSxsPassiveMode':                                     'int8',
        'DefaultBrowsersIdentifier':                            'float16',
        'AVProductStatesIdentifier':                            'float32',
        'AVProductsInstalled':                                  'float16',
        'AVProductsEnabled':                                    'float16',
        'HasTpm':                                               'int8',
        'CountryIdentifier':                                    'int16',
        'CityIdentifier':                                       'float32',
        'OrganizationIdentifier':                               'float16',
        'GeoNameIdentifier':                                    'float16',
        'LocaleEnglishNameIdentifier':                          'int8',
        'Platform':                                             'category',
        'Processor':                                            'category',
        'OsVer':                                                'category',
        'OsBuild':                                              'int16',
        'OsSuite':                                              'int16',
        'OsPlatformSubRelease':                                 'category',
        'OsBuildLab':                                           'category',
        'SkuEdition':                                           'category',
        'IsProtected':                                          'float16',
        'AutoSampleOptIn':                                      'int8',
        'PuaMode':                                              'category',
        'SMode':                                                'float16',
        'IeVerIdentifier':                                      'float16',
        'SmartScreen':                                          'category',
        'Firewall':                                             'float16',
        'UacLuaenable':                                         'float32',
        'Census_MDC2FormFactor':                                'category',
        'Census_DeviceFamily':                                  'category',
        'Census_OEMNameIdentifier':                             'float16',
        'Census_OEMModelIdentifier':                            'float32',
        'Census_ProcessorCoreCount':                            'float16',
        'Census_ProcessorManufacturerIdentifier':               'float16',
        'Census_ProcessorModelIdentifier':                      'float16',
        'Census_ProcessorClass':                                'category',
        'Census_PrimaryDiskTotalCapacity':                      'float32',
        'Census_PrimaryDiskTypeName':                           'category',
        'Census_SystemVolumeTotalCapacity':                     'float32',
        'Census_HasOpticalDiskDrive':                           'int8',
        'Census_TotalPhysicalRAM':                              'float32',
        'Census_ChassisTypeName':                               'category',
        'Census_InternalPrimaryDiagonalDisplaySizeInInches':    'float16',
        'Census_InternalPrimaryDisplayResolutionHorizontal':    'float16',
        'Census_InternalPrimaryDisplayResolutionVertical':      'float16',
        'Census_PowerPlatformRoleName':                         'category',
        'Census_InternalBatteryType':                           'category',
        'Census_InternalBatteryNumberOfCharges':                'float32',
        'Census_OSVersion':                                     'category',
        'Census_OSArchitecture':                                'category',
        'Census_OSBranch':                                      'category',
        'Census_OSBuildNumber':                                 'int16',
        'Census_OSBuildRevision':                               'int32',
        'Census_OSEdition':                                     'category',
        'Census_OSSkuName':                                     'category',
        'Census_OSInstallTypeName':                             'category',
        'Census_OSInstallLanguageIdentifier':                   'float16',
        'Census_OSUILocaleIdentifier':                          'int16',
        'Census_OSWUAutoUpdateOptionsName':                     'category',
        'Census_IsPortableOperatingSystem':                     'int8',
        'Census_GenuineStateName':                              'category',
        'Census_ActivationChannel':                             'category',
        'Census_IsFlightingInternal':                           'float16',
        'Census_IsFlightsDisabled':                             'float16',
        'Census_FlightRing':                                    'category',
        'Census_ThresholdOptIn':                                'float16',
        'Census_FirmwareManufacturerIdentifier':                'float16',
        'Census_FirmwareVersionIdentifier':                     'float32',
        'Census_IsSecureBootEnabled':                           'int8',
        'Census_IsWIMBootEnabled':                              'float16',
        'Census_IsVirtualDevice':                               'float16',
        'Census_IsTouchEnabled':                                'int8',
        'Census_IsPenCapable':                                  'int8',
        'Census_IsAlwaysOnAlwaysConnectedCapable':              'float16',
        'Wdft_IsGamer':                                         'float16',
        'Wdft_RegionIdentifier':                                'float16',
        'HasDetections':                                        'int8'
        }

def reduce_mem_usage(df, verbose=True):
    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
    start_mem = df.memory_usage(deep=True).sum() / 1024**2    
    for col in df.columns:
        col_type = df[col].dtypes
        if col_type in numerics:
            c_min = df[col].min()
            c_max = df[col].max()
            if str(col_type)[:3] == 'int':
                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:
                    df[col] = df[col].astype(np.int8)
                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:
                    df[col] = df[col].astype(np.int16)
                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:
                    df[col] = df[col].astype(np.int32)
                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:
                    df[col] = df[col].astype(np.int64)  
            else:
                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:
                    df[col] = df[col].astype(np.float16)
                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:
                    df[col] = df[col].astype(np.float32)
                else:
                    df[col] = df[col].astype(np.float64)    
    end_mem = df.memory_usage(deep=True).sum() / 1024**2
    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))
    print('Thanks!!')
    return df

def add_count(df, c):
    new_col = 'fe_count_'+ c
    df[new_col] = df.groupby(c)[c].transform('count')

# In[ ]:


numerics = ['int8', 'int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numerical_columns = [c for c,v in dtypes.items() if v in numerics]
categorical_columns = [c for c,v in dtypes.items() if v not in numerics]


# In[3]:


train = pd.read_csv('../input/train.csv', dtype=dtypes)

train_y = train['HasDetections']
test = pd.read_csv('../input/test.csv', dtype=dtypes)

#display section
train['fe_aspect_ratio'] = train['Census_InternalPrimaryDisplayResolutionHorizontal']/ train['Census_InternalPrimaryDisplayResolutionVertical']
test['fe_aspect_ratio']  = test['Census_InternalPrimaryDisplayResolutionHorizontal']/ train['Census_InternalPrimaryDisplayResolutionVertical']

train['Census_InternalPrimaryDisplayResolutionHorizontal'] = train['Census_InternalPrimaryDisplayResolutionHorizontal'].astype(np.float64)
test['Census_InternalPrimaryDisplayResolutionHorizontal']  = test['Census_InternalPrimaryDisplayResolutionHorizontal'].astype(np.float64)

train['Census_InternalPrimaryDisplayResolutionVertical'] = train['Census_InternalPrimaryDisplayResolutionVertical'].astype(np.float64)
test['Census_InternalPrimaryDisplayResolutionVertical']  = test['Census_InternalPrimaryDisplayResolutionVertical'].astype(np.float64)

train['fe_dpi'] = ((train['Census_InternalPrimaryDisplayResolutionHorizontal']**2 + train['Census_InternalPrimaryDisplayResolutionVertical']**2)**.5)/(train['Census_InternalPrimaryDiagonalDisplaySizeInInches'])
test['fe_dpi']  = ((test['Census_InternalPrimaryDisplayResolutionHorizontal']**2 + test['Census_InternalPrimaryDisplayResolutionVertical']**2)**.5)/(test['Census_InternalPrimaryDiagonalDisplaySizeInInches'])

train['fe_MegaPixels'] = (train['Census_InternalPrimaryDisplayResolutionHorizontal'] * train['Census_InternalPrimaryDisplayResolutionVertical'])/1e6
test['fe_MegaPixels']  = (test['Census_InternalPrimaryDisplayResolutionHorizontal'] * test['Census_InternalPrimaryDisplayResolutionVertical'])/1e6

print('Done Display Features\n')

def encode_categorical_columns(x_train, x_test, columns, sort=True):
    train_length = x_train.shape[0]
    for col in tqdm(columns):
        if col == 'MachineIdentifier' or col == 'HasDetections':
            continue
            
        combined_data = pd.concat([x_train[col], x_test[col]])
        combined_data, _ = pd.factorize(combined_data, sort=sort)
        combined_data = pd.Series(combined_data).astype('int32')
        x_train[col] = combined_data.iloc[:train_length].values
        x_test[col] = combined_data.iloc[train_length:].values
        x_train[col] = x_train[col].fillna(0)
        x_test[col] = x_test[col].fillna(0)
        del combined_data
        gc.collect()
        
    return x_train, x_test


# In[4]:


def fe(df):

    print(gc.collect())
    print('Cooking Pointless Things....')

    ##grouby stats
    gp = df[['CountryIdentifier','OrganizationIdentifier', 'Census_OSInstallTypeName']].groupby(by=['CountryIdentifier','OrganizationIdentifier'], sort=False)[['Census_OSInstallTypeName']].count().reset_index().rename(columns={'Census_OSInstallTypeName':'cnt_cnt_org_os'})
    df = df.merge(gp, on=['CountryIdentifier','OrganizationIdentifier'], how='left')
    del gp
    gc.collect()

    gp = df[['CountryIdentifier','OrganizationIdentifier','CityIdentifier', 'Census_OSInstallTypeName']].groupby(['CountryIdentifier','OrganizationIdentifier', 'CityIdentifier'])[['Census_OSInstallTypeName']].count().reset_index().rename(columns={'Census_OSInstallTypeName':'cnt_cnt_org_city_os'})
    df = df.merge(gp, on=['CountryIdentifier','OrganizationIdentifier','CityIdentifier'], how='left')
    del gp
    gc.collect()

    gp = df[['CountryIdentifier','OrganizationIdentifier','Census_OSBuildNumber', 'Census_OSInstallTypeName']].groupby(['CountryIdentifier','OrganizationIdentifier', 'Census_OSBuildNumber'], sort=False)[['Census_OSInstallTypeName']].count().reset_index().rename(columns={'Census_OSInstallTypeName':'cnt_cnt_org_build_type'})
    df = df.merge(gp, on=['CountryIdentifier','OrganizationIdentifier', 'Census_OSBuildNumber'], how='left')
    del gp
    gc.collect()

    ###################
    gp = df[['CountryIdentifier','OrganizationIdentifier','CityIdentifier', 'Census_FirmwareManufacturerIdentifier']].groupby(['CountryIdentifier','OrganizationIdentifier', 'CityIdentifier'])[['Census_FirmwareManufacturerIdentifier']].count().reset_index().rename(columns={'Census_FirmwareManufacturerIdentifier':'cnt_cnt_org_city_frwar'})
    df = df.merge(gp, on=['CountryIdentifier','OrganizationIdentifier','CityIdentifier'], how='left')
    del gp

    gp = df[['CountryIdentifier','OrganizationIdentifier', 'Census_FirmwareManufacturerIdentifier']].groupby(['CountryIdentifier','OrganizationIdentifier'])[['Census_FirmwareManufacturerIdentifier']].count().reset_index().rename(columns={'Census_FirmwareManufacturerIdentifier':'cnt_cnt_org_frwar'})
    df = df.merge(gp, on=['CountryIdentifier','OrganizationIdentifier'], how='left')
    del gp

    gp = df[['CountryIdentifier','OrganizationIdentifier','AvSigVersion', 'Census_FirmwareManufacturerIdentifier']].groupby(['CountryIdentifier','OrganizationIdentifier', 'AvSigversion'])[['Census_FirmwareManufacturerIdentifier']].count().reset_index().rename(columns={'Census_FirmwareManufacturerIdentifier':'cnt_cnt_org_avsig_frwar'})
    df = df.merge(gp, on=['CountryIdentifier','OrganizationIdentifier','AvSigversion'], how='left')
    del gp
    ###################

    df['fe_EngineVersion_2'] = df['EngineVersion'].apply(lambda x: x.split('.')[2]).astype('category')
    df['fe_one_less_AVproductInstalled'] = df['AVProductsInstalled'] - 1
    df['fe_AvSigVersion_minor'] = df['AvSigVersion'].apply(lambda x: x.split('.')[1]).astype('category')
    df['fe_AvSigVersion_build'] = df['AvSigVersion'].apply(lambda x: x.split('.')[2]).astype('category')
    
    df['fe_AvSigVersion_minor_build'] = df['AvSigVersion'].str.replace('1.2&#x17;3.1144.0','1.273.1144.0').apply(lambda x: float((x.split('.')[1]) +'.'+(x.split('.')[2]))).astype('float32')
    df['fe_AvSigVersion_sum'] = df['AvSigVersion'].str.replace('1.2&#x17;3.1144.0','1.273.1144.0').apply(lambda x: float(x.split('.')[1]) + float(x.split('.')[2])).astype(int).values
    df['AvSigVersion'] = df['AvSigVersion'].astype('category')

    top_20 = df['AVProductStatesIdentifier'].value_counts(dropna=False, normalize=True).cumsum().index[:20]
    df['fe_magic_4'] = 0
    df.loc[df['AVProductStatesIdentifier'].isin(top_20) == True, 'fe_magic_4'] = 1
    del top_20
    df = reduce_mem_usage(df)

    gc.collect()
    
    df['fe_primary_drive_c_ratio'] = df['Census_SystemVolumeTotalCapacity']/ df['Census_PrimaryDiskTotalCapacity']
    df['fe_Census_SystemVolumeTotalCapacity_GB'] = df['Census_SystemVolumeTotalCapacity']/1024.
    df['fe_non_primary_drive_MB'] = df['Census_PrimaryDiskTotalCapacity'] - df['Census_SystemVolumeTotalCapacity']
    df['fe_non_primary_drive_GB'] = df['fe_non_primary_drive_MB']//1024
    df['fe_ram_per_processor'] = df['Census_TotalPhysicalRAM']/ df['Census_ProcessorCoreCount']
    df['fe_physical_cores'] = df['Census_ProcessorCoreCount'] / 2
        
    print("Preparing ratios")
    ##### testing feats

    nrows = df.shape[0]
    df['fe_avsig_gamer_freq'] = df.groupby(['AvSigVersion','Wdft_IsGamer'])['OsBuild'].transform('count') / nrows
    df['fe_avsig_region_freq'] = df.groupby(['AvSigversion', 'Wdft_RegionIdentifier'])['OsBuild'].transform('count') / nrows
    df['fe_cpucores_region_freq'] = df.groupby(['Census_ProcessorCoreCount','Wdft_RegionIdentifier'])['OsBuild'].transform('count') / nrows
    df['fe_cpucores_oemname_freq'] = df.groupby(['Census_ProcessorCoreCount','Census_OEMNameIdentifier'])['OsBuild'].transform('count') / nrows
    df['fe_geoname_oemname_freq'] = df.groupby(['GeoNameIdentifier','Census_OEMNameIdentifier'])['OsBuild'].transform('count') / nrows
    df['fe_cntiden_oemname_freq'] = df.groupby(['CountryIdentifier','Census_OEMNameIdentifier'])['OsBuild'].transform('count') / nrows
    df = reduce_mem_usage(df)


    df['fe_hghdec_cnt1'] = 0
    df.loc[df['CountryIdentifier'].isin([214,89,195,4,141,158,43,201,41,9,29,203,171,60,93,142,66,149,207,97,107,68,5,35,160]) == True, 'fe_hghdec_cnt1'] = 1

    df['fe_hghdec_cnt_3'] = 0
    df.loc[df['AppVersion'].isin(['4.18.1807.18075', '4.18.1806.18062', '4.12.16299.15', '4.10.209.0', '4.13.17134.1', '4.16.17656.18052']) == True, 'fe_hghdec_cnt_3'] = 1

    df['fe_hghdec_cnt_5'] = 0
    df.loc[df['CityIdentifier'].isin([130775.0,16668,82373.0,10222.0,61668.0,143782.0,66202.0,58607.0,66953.0]) == True, 'fe_hghdec_cnt_5'] = 1

    df = reduce_mem_usage(df)

    df['fe_hghdec_cnt_8'] = 0
    df.loc[df['Census_ProcessorModelIdentifier'].isin([2696.,1998.,2660.,2372.,1992.,2382.,2640.,2524.,1985.,2096.]) == True, 'fe_hghdec_cnt_8'] = 1

    df['fe_hghdec_cnt_10'] = 0
    df.loc[df['Census_FirmwareManufacturerIdentifier'].isin([142.,628.,554.,355.,556.,500.,93.,807.,513.]) == True, 'fe_hghdec_cnt_10'] = 1

    df = reduce_mem_usage(df)
    gc.collect()
    
    return df


# In[5]:


train = fe(train)
test = fe(test)s
binary_variables = [c for c in train.columns if train[c].nunique() == 2];

# In[6]:


numerical_columns = list(train.select_dtypes(include=numerics).columns)
categorical_columns = categorical_columns + ['fe_EngineVersion_2', 'fe_AvSigVersion_minor', 'fe_AvSigVersion_build']
train, test = encode_categorical_columns(train, test, categorical_columns)
print(train.shape, test.shape)


# In[7]:


train = reduce_mem_usage(train)
test  = reduce_mem_usage(test)


# In[8]:


numerical_columns.remove('HasDetections')

remove = ['MachineIdentifier','Census_OSEdition','Census_OSArchitecture',
          'OsPlatformSubRelease','OsVer','PuaMode', 'Census_DeviceFamily','ProductName']

for col in remove:categorical_columns.remove(col)

dont_scale_cols = ['IsSxsPassiveMode',
                   'HasTpm','IsProtected', 'Firewall','UacLuaenable', 'Census_HasOpticalDiskDrive', 
                   'Census_IsPortableOperatingSystem','Census_IsSecureBootEnabled', 'Census_IsVirtualDevice', 'Census_IsTouchEnabled',
                   'Census_IsPenCapable', 'Census_IsAlwaysOnAlwaysConnectedCapable','Wdft_IsGamer', 
                   'fe_magic_4', 'fe_hghdec_cnt1', 'fe_hghdec_cnt_3', 'fe_hghdec_cnt_5', 'fe_hghdec_cnt_8','fe_hghdec_cnt_10',
                   'fe_avsig_gamer_freq','fe_cpucores_region_freq','fe_cpucores_oemname_freq','fe_geoname_oemname_freq',
                   'fe_cntiden_oemname_freq','fe_primary_drive_c_ratio'
                  ] 
for col in remove+dont_scale_cols:
    try: 
        numerical_columns.remove(col)
    except:
        print(col, 'Already Removed.....!')
            
train = train[numerical_columns + categorical_columns + dont_scale_cols]
test  = test[numerical_columns  + categorical_columns + dont_scale_cols]

print(train.shape, test.shape)


# In[9]:


col_vals_dict = {c: list(train[c].unique()) for c in categorical_columns }


# In[10]:


nb_numeric   = len(train.columns) - len(col_vals_dict)
nb_categoric = len(col_vals_dict)
print('Number of Numerical features:', nb_numeric)
print('Number of Categorical features:', nb_categoric)
print('Number of Don\'t Scale features:', len(dont_scale_cols))


# ### Create the network
# In order to create our embedding model we need to have a look at the spatiality of the cat features. We choose here to use Embedding only on cat features that present more than 2 outcomes otherwise it is count as a numeric value (0 or 1).

# In[11]:


embed_cols = []
len_embed_cols = []
dont_embed = []
for c in col_vals_dict:
    if len(col_vals_dict[c])>2:
        embed_cols.append(c)
        len_embed_cols.append((c, len(col_vals_dict[c])))
        print(c + ': %d values' % len(col_vals_dict[c])) #look at value counts to know the embedding dimensions
    else:
        dont_embed.append((c, len(col_vals_dict[c])))

print('\n Number of embed features :', len(embed_cols))
print('\n Number of embed features Didn\'t Embed:', len(dont_embed))


# - We are including 24 features out of 24 categorical features into our Embedding.
# - **embedding size = min(50, number of categories/2)**

# In[13]:


len(numerical_columns), len(embed_cols), dont_embed, train.shape, test.shape


# In[28]:


def build_embedding_network(len_embed_cols):
    
    model_out = []
    model_in  = []
    
    for name, dim in len_embed_cols:
        
        input_dim = Input(shape=(1,), dtype='int32')
        embed_dim = Embedding(dim, min(50, (dim+1)//2) + 1, input_length=1, name= name)(input_dim) #1 for unknowns
        embed_dim = Reshape(target_shape=(min(50, (dim+1)//2) + 1,))(embed_dim)
        model_out.append(embed_dim)
        model_in.append(input_dim)
    
    input_num = Input(shape=(len(numerical_columns)+ len(dont_embed),), name='numerical_feats',dtype='float32')
    x2 = Dense(256, activation='relu')(input_num)
    x2 = Dropout(.25)(x2)
    x2 = BatchNormalization()(x2)
    
    input_num_2 = Input(shape=(len(dont_scale_cols),), name='non_scaled_numerical_feats',dtype='float32')
    x3 = Dense(256, activation='relu')(input_num_2)
    x3 = Dropout(.25)(x3)
    x3 = BatchNormalization()(x3)
    
    outputs = Concatenate(axis=1)([*model_out, x2, x3])
    outputs = (Dense(128))(outputs)
    outputs = (Activation('relu'))(outputs)
    outputs = (Dropout(.35))(outputs)
    outputs = (Dense(16))(outputs) 
    outputs = (BatchNormalization())(outputs)
    outputs = (Activation('relu'))(outputs)
    outputs = (Dropout(.2))(outputs)
    outputs = Dense(1,activation='sigmoid')(outputs)
    
    model = Model([*model_in, input_num, input_num_2], outputs)
    
#     batch_size = 2**11
#     n_epochs = 10
#     exp_decay = lambda init, fin, steps: (init/fin)**(1/(steps-1)) - 1
#     steps = int(len(train) / batch_size) * n_epochs
    
#     lr_init, lr_fin = 0.01, 0.0001
#     lr_decay = exp_decay(lr_init, lr_fin, steps)
#     optimizer_adam = Adam(lr=0.01, decay=lr_decay)

    sgd = SGD(lr=0.01, decay=1e-5, momentum=0.9, nesterov=True)
    model.compile(optimizer=sgd, loss="binary_crossentropy", metrics=["accuracy"])
    
    from keras.utils import plot_model
    plot_model(model, to_file='model.png')
    gc.collect()
    return model


# In[15]:


def preproc(X_train, X_val, X_test):

    input_list_train = []
    input_list_val = []
    input_list_test = []
    
    #the cols to be embedded: rescaling to range [0, # values)
    print('Embedding cols\n', embed_cols)
    for c in embed_cols:
        raw_vals = np.unique(X_train[c])
        val_map = {}
        for i in range(len(raw_vals)):
            val_map[raw_vals[i]] = i
            
        input_list_train.append(X_train[c].map(val_map).values)
        input_list_val.append(X_val[c].map(val_map).fillna(0).values)
        input_list_test.append(X_test[c].map(val_map).fillna(0).values)
        
    #the rest of the columns (numerical)
    other_cols = numerical_columns + dont_embed
    print('Scaled cols\n', other_cols)

    input_list_train.append(X_train[other_cols].values)
    input_list_val.append(X_val[other_cols].values)
    input_list_test.append(X_test[other_cols].values)
    
    #the rest of the columns (non-scaling ones)
    other_cols = dont_scale_cols
    print('Non Scaled cols\n', other_cols)

    input_list_train.append(X_train[other_cols].values)
    input_list_val.append(X_val[other_cols].values)
    input_list_test.append(X_test[other_cols].values)
    
    gc.collect()
    return input_list_train, input_list_val, input_list_test


# In[18]:


# Impute missing values in order to scale
from sklearn.preprocessing import StandardScaler

train[numerical_columns + dont_scale_cols] = train[numerical_columns + dont_scale_cols].fillna(value = 0)
test[numerical_columns + dont_scale_cols]  = test[numerical_columns + dont_scale_cols].fillna(value = 0)

# Fit the scaler only on train data
scaler = StandardScaler().fit(train[numerical_columns])
train.loc[:,numerical_columns] = scaler.transform(train[numerical_columns])
test.loc[:,numerical_columns]  = scaler.transform(test[numerical_columns])


# In[19]:


print ('neural network....')
from keras.layers import Input, Embedding, Dense, Flatten, Dropout, concatenate, Reshape
from keras.layers import BatchNormalization, SpatialDropout1D, Concatenate, Activation, concatenate
from keras.callbacks import Callback, EarlyStopping
from keras.models import Model
from keras.optimizers import Adam


# In[20]:


K = 5
runs_per_fold = 1 #increase this or nb epochs
n_epochs = 20
batch_size = 2**11
patience = 5
models = []
cv_aucs   = []
full_val_preds = np.zeros(np.shape(train)[0])
y_preds = np.zeros((np.shape(test)[0],K))

kfold = StratifiedKFold(n_splits = K, shuffle = True, random_state=2**10)

for i, (f_ind, outf_ind) in enumerate(kfold.split(train, train_y)):

    train_f, X_val_f = train.loc[f_ind].copy(), train.loc[outf_ind].copy()
    train_y_f, y_val_f = train_y[f_ind], train_y[outf_ind]
    
    print('Shapes Are', train_f.shape, X_val_f.shape)
    
    test_f = test.copy()
    # Shuffle data
    idx = np.arange(len(train_f))
    np.random.shuffle(idx)
    train_f = train_f.iloc[idx]
    train_y_f = train_y_f.iloc[idx]

    #preprocessing
    print('Preprocessing........!!!')
    proc_train_f, proc_X_val_f, proc_test_f = preproc(train_f, X_val_f, test_f)
    
    #track oof prediction for cv scores
    val_preds = 0
    
    for j in range(runs_per_fold):
        
        NN = build_embedding_network(len_embed_cols)
        
        # Set callback functions to early stop training and save the best model so far
        callbacks = [EarlyStopping(monitor='val_loss', patience=patience)]
        print(len(proc_train_f))
        
        NN.fit(proc_train_f, train_y_f.values, 
               epochs=n_epochs, batch_size= batch_size, 
               verbose=1,callbacks=callbacks,
               validation_data=(proc_X_val_f, y_val_f))
        
        del proc_train_f, train_y_f, callbacks
        gc.collect()
        print('Val Preds OOF......!!!')
        val_preds += NN.predict(proc_X_val_f)[:,0] / runs_per_fold
        del proc_X_val_f
        gc.collect()
        #need to fix the y_preds over batches...
        print('Test Preds......!!!!')
        y_preds[:,i] += NN.predict(proc_test_f)[:,0] / runs_per_fold
        
        models.append(NN)
        
    full_val_preds[outf_ind] += val_preds
        
    cv_auc  = roc_auc_score(y_val_f.values, val_preds)
    cv_aucs.append(cv_auc)
    #########################################
    #thanks to @tksasagi for her tweet#######
    #########################################
    print('|￣￣￣￣￣￣￣￣|')
    print('|    VALIDATION    |') 
    print('|     Fold %i prediction cv AUC: %.5f  |' %(i+1, cv_auc))
    print('| ＿＿＿_＿＿＿＿|') 
    print(' (\__/) ||') 
    print(' (•ㅅ•) || ')
    print(' / 　 づ')
    print('#'*100)
    
print('Mean out of fold AUC: %.5f' % np.mean(cv_auc))
print('Full validation AUC: %.5f' % roc_auc_score(train_y.values, full_val_preds))

print('Saving OOF Train.......')
np.save('train_oof_nn_with_embedddings_V2.npy', full_val_preds)
print('Saved\n')

print('Saving Predictions (Don\'t Take mean now.......')
np.save('test_nn_with_embedddings_V2.npy', np.mean(y_preds, axis=1))
print('Saved Test Preds to be submitted...\n')

# In[27]:


from keras.utils import plot_model
plot_model(models[0], to_file='model_V2.png', dpi=256)


# In[21]:


# from IPython.display import SVG
# from keras.utils import model_to_dot

# SVG(model_to_dot(models[0]).create(prog='dot', format='svg'))


# In[ ]:


import pickle
save_embeddings = True
saved_embeddings_fname = "embeddings_V2.pickle"
all_emb = []
if save_embeddings:
    model_ = models[4]    
    for cols in embed_cols:
        emb = model_.get_layer(cols).get_weights()[0]
        all_emb.append(emb)
    with open(saved_embeddings_fname , 'wb') as f:
        pickle.dump(all_emb, f, -1)


# In[ ]:


from keras.models import load_model

models[0].save('my_model_0.h5')
models[1].save('my_model_1.h5')
models[3].save('my_model_3.h5')

del models  # deletes the existing models


# In[ ]:


print(embed_cols)


# In[ ]:


for idx, col in enumerate(train.columns):
    if col in embed_cols:
        print(idx, col)
##mapping is left for the User to Give it a Go On Purpose:)


# In[ ]:


## working on this thing

# del df_train
# del X_train, X_val, Y_train, Y_val
# x = gc.collect()

# # LOAD BEST SAVED NET
# from keras.models import load_model
# model = load_model('bestNet.h5')

# pred = np.zeros((7853253,1))
# id = 1
# chunksize = 2000000
# for df_test in pd.read_csv('../input/test.csv', 
#             chunksize = chunksize, usecols=list(dtypes.keys())[0:-1], dtype=dtypes):
#     print ('Loaded',len(df_test),'rows of TEST.CSV!')
#     # ENCODE TEST
#     cols = []
#     for x in FE:
#         cols += encode_FE(df_test,x,verbose=0)
#     for x in range(len(OHE)):
#         cols += encode_OHE_test(df_test,OHE[x],dd[x])
#     # PREDICT TEST
#     end = (id)*chunksize
#     if end>7853253: end = 7853253
#     pred[(id-1)*chunksize:end] = model.predict_proba(df_test[cols])
#     print('  encoded and predicted part',id)
#     id += 1