
# coding: utf-8

# In[1]:


from numba import jit
import numpy as np 
import pandas as pd 
from datetime import datetime as dt
import os
import seaborn as sns 
import matplotlib.pyplot as plt
plt.style.use('ggplot')
import lightgbm as lgb
import xgboost as xgb
import time
import datetime
from tqdm import tqdm_notebook as tqdm
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import StratifiedKFold, KFold, TimeSeriesSplit
from sklearn.metrics import mean_squared_error, roc_auc_score
from sklearn import metrics
from itertools import combinations
import gc
import pickle
import warnings
warnings.filterwarnings("ignore")
import os
from functools import wraps
from timeit import default_timer as timer

import os
print(os.listdir("./"))

dtypes = {
        'MachineIdentifier':                                    'category',
        'ProductName':                                          'category',
        'EngineVersion':                                        'category',
        'AppVersion':                                           'category',
        'AvSigVersion':                                         'category',
        'IsBeta':                                               'int8',
        'RtpStateBitfield':                                     'float16',
        'IsSxsPassiveMode':                                     'int8',
        'DefaultBrowsersIdentifier':                            'float16',
        'AVProductStatesIdentifier':                            'float32',
        'AVProductsInstalled':                                  'float16',
        'AVProductsEnabled':                                    'float16',
        'HasTpm':                                               'int8',
        'CountryIdentifier':                                    'int16',
        'CityIdentifier':                                       'float32',
        'OrganizationIdentifier':                               'float16',
        'GeoNameIdentifier':                                    'float16',
        'LocaleEnglishNameIdentifier':                          'int8',
        'Platform':                                             'category',
        'Processor':                                            'category',
        'OsVer':                                                'category',
        'OsBuild':                                              'int16',
        'OsSuite':                                              'int16',
        'OsPlatformSubRelease':                                 'category',
        'OsBuildLab':                                           'category',
        'SkuEdition':                                           'category',
        'IsProtected':                                          'float16',
        'AutoSampleOptIn':                                      'int8',
        'PuaMode':                                              'category',
        'SMode':                                                'float16',
        'IeVerIdentifier':                                      'float16',
        'SmartScreen':                                          'category',
        'Firewall':                                             'float16',
        'UacLuaenable':                                         'float32',
        'Census_MDC2FormFactor':                                'category',
        'Census_DeviceFamily':                                  'category',
        'Census_OEMNameIdentifier':                             'float16',
        'Census_OEMModelIdentifier':                            'float32',
        'Census_ProcessorCoreCount':                            'float16',
        'Census_ProcessorManufacturerIdentifier':               'float16',
        'Census_ProcessorModelIdentifier':                      'float16',
        'Census_ProcessorClass':                                'category',
        'Census_PrimaryDiskTotalCapacity':                      'float32',
        'Census_PrimaryDiskTypeName':                           'category',
        'Census_SystemVolumeTotalCapacity':                     'float32',
        'Census_HasOpticalDiskDrive':                           'int8',
        'Census_TotalPhysicalRAM':                              'float32',
        'Census_ChassisTypeName':                               'category',
        'Census_InternalPrimaryDiagonalDisplaySizeInInches':    'float16',
        'Census_InternalPrimaryDisplayResolutionHorizontal':    'float16',
        'Census_InternalPrimaryDisplayResolutionVertical':      'float16',
        'Census_PowerPlatformRoleName':                         'category',
        'Census_InternalBatteryType':                           'category',
        'Census_InternalBatteryNumberOfCharges':                'float32',
        'Census_OSVersion':                                     'category',
        'Census_OSArchitecture':                                'category',
        'Census_OSBranch':                                      'category',
        'Census_OSBuildNumber':                                 'int16',
        'Census_OSBuildRevision':                               'int32',
        'Census_OSEdition':                                     'category',
        'Census_OSSkuName':                                     'category',
        'Census_OSInstallTypeName':                             'category',
        'Census_OSInstallLanguageIdentifier':                   'float16',
        'Census_OSUILocaleIdentifier':                          'int16',
        'Census_OSWUAutoUpdateOptionsName':                     'category',
        'Census_IsPortableOperatingSystem':                     'int8',
        'Census_GenuineStateName':                              'category',
        'Census_ActivationChannel':                             'category',
        'Census_IsFlightingInternal':                           'float16',
        'Census_IsFlightsDisabled':                             'float16',
        'Census_FlightRing':                                    'category',
        'Census_ThresholdOptIn':                                'float16',
        'Census_FirmwareManufacturerIdentifier':                'float16',
        'Census_FirmwareVersionIdentifier':                     'float32',
        'Census_IsSecureBootEnabled':                           'int8',
        'Census_IsWIMBootEnabled':                              'float16',
        'Census_IsVirtualDevice':                               'float16',
        'Census_IsTouchEnabled':                                'int8',
        'Census_IsPenCapable':                                  'int8',
        'Census_IsAlwaysOnAlwaysConnectedCapable':              'float16',
        'Wdft_IsGamer':                                         'float16',
        'Wdft_RegionIdentifier':                                'float16',
        'HasDetections':                                        'int8'
        }

def reduce_mem_usage(df, verbose=True):
    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
    start_mem = df.memory_usage(deep=True).sum() / 1024**2    
    for col in df.columns:
        col_type = df[col].dtypes
        if col_type in numerics:
            c_min = df[col].min()
            c_max = df[col].max()
            if str(col_type)[:3] == 'int':
                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:
                    df[col] = df[col].astype(np.int8)
                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:
                    df[col] = df[col].astype(np.int16)
                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:
                    df[col] = df[col].astype(np.int32)
                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:
                    df[col] = df[col].astype(np.int64)  
            else:
                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:
                    df[col] = df[col].astype(np.float16)
                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:
                    df[col] = df[col].astype(np.float32)
                else:
                    df[col] = df[col].astype(np.float64)    
    end_mem = df.memory_usage(deep=True).sum() / 1024**2
    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))
    print('Thanks!!')
    return df

def add_count(df, c):
    new_col = 'fe_count_'+ c
    df[new_col] = df.groupby(c)[c].transform('count')

# In[ ]:


numerics = ['int8', 'int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numerical_columns = [c for c,v in dtypes.items() if v in numerics]
categorical_columns = [c for c,v in dtypes.items() if v not in numerics]


# In[ ]:


train = pd.read_csv('../input/train.csv', dtype=dtypes)

train_y = train['HasDetections']
test = pd.read_csv('../input/test.csv', dtype=dtypes)

#display section
train['fe_aspect_ratio'] = train['Census_InternalPrimaryDisplayResolutionHorizontal']/ train['Census_InternalPrimaryDisplayResolutionVertical']
test['fe_aspect_ratio']  = test['Census_InternalPrimaryDisplayResolutionHorizontal']/ train['Census_InternalPrimaryDisplayResolutionVertical']

train['Census_InternalPrimaryDisplayResolutionHorizontal'] = train['Census_InternalPrimaryDisplayResolutionHorizontal'].astype(np.float64)
test['Census_InternalPrimaryDisplayResolutionHorizontal']  = test['Census_InternalPrimaryDisplayResolutionHorizontal'].astype(np.float64)

train['Census_InternalPrimaryDisplayResolutionVertical'] = train['Census_InternalPrimaryDisplayResolutionVertical'].astype(np.float64)
test['Census_InternalPrimaryDisplayResolutionVertical']  = test['Census_InternalPrimaryDisplayResolutionVertical'].astype(np.float64)

train['fe_dpi'] = ((train['Census_InternalPrimaryDisplayResolutionHorizontal']**2 + train['Census_InternalPrimaryDisplayResolutionVertical']**2)**.5)/(train['Census_InternalPrimaryDiagonalDisplaySizeInInches'])
test['fe_dpi']  = ((test['Census_InternalPrimaryDisplayResolutionHorizontal']**2 + test['Census_InternalPrimaryDisplayResolutionVertical']**2)**.5)/(test['Census_InternalPrimaryDiagonalDisplaySizeInInches'])

train['fe_MegaPixels'] = (train['Census_InternalPrimaryDisplayResolutionHorizontal'] * train['Census_InternalPrimaryDisplayResolutionVertical'])/1e6
test['fe_MegaPixels']  = (test['Census_InternalPrimaryDisplayResolutionHorizontal'] * test['Census_InternalPrimaryDisplayResolutionVertical'])/1e6

print('Done Display Features\n')

def encode_categorical_columns(x_train, x_test, columns, sort=True):
    train_length = x_train.shape[0]
    for col in tqdm(columns):
        if col == 'MachineIdentifier' or col == 'HasDetections':
            continue
            
        combined_data = pd.concat([x_train[col], x_test[col]])
        combined_data, _ = pd.factorize(combined_data, sort=sort)
        combined_data = pd.Series(combined_data).astype('int32')
        x_train[col] = combined_data.iloc[:train_length].values
        x_test[col] = combined_data.iloc[train_length:].values
        x_train[col] = x_train[col].fillna(0)
        x_test[col] = x_test[col].fillna(0)
        del combined_data
        gc.collect()
        
    return x_train, x_test


# In[ ]:


def fe(df):

    print(gc.collect())
    print('Cooking Pointless Things....')

    df['fe_EngineVersion_2'] = df['EngineVersion'].apply(lambda x: x.split('.')[2]).astype('category')
    df['fe_one_less_AVproductInstalled'] = df['AVProductsInstalled'] - 1

    df['fe_AvSigVersion_minor'] = df['AvSigVersion'].apply(lambda x: x.split('.')[1]).astype('category')
    df['fe_AvSigVersion_build'] = df['AvSigVersion'].apply(lambda x: x.split('.')[2]).astype('category')
    df['fe_AvSigVersion_minor_build'] = df['AvSigVersion'].str.replace('1.2&#x17;3.1144.0','1.273.1144.0').apply(lambda x: float((x.split('.')[1]) +'.'+(x.split('.')[2]))).astype('float32')
    df['fe_AvSigVersion_sum'] = df['AvSigVersion'].str.replace('1.2&#x17;3.1144.0','1.273.1144.0').apply(lambda x: float(x.split('.')[1]) + float(x.split('.')[2])).astype(int).values
    df['AvSigVersion'] = df['AvSigVersion'].astype('category')
        
    top_20 = df['AVProductStatesIdentifier'].value_counts(dropna=False, normalize=True).cumsum().index[:20]
    df['fe_magic_4'] = 0
    df.loc[df['AVProductStatesIdentifier'].isin(top_20) == True, 'fe_magic_4'] = 1
    del top_20

    gc.collect()
    
    df['fe_primary_drive_c_ratio'] = df['Census_SystemVolumeTotalCapacity']/ df['Census_PrimaryDiskTotalCapacity']
    df['fe_Census_SystemVolumeTotalCapacity_GB'] = df['Census_SystemVolumeTotalCapacity']/1024.
    df['fe_non_primary_drive_MB'] = df['Census_PrimaryDiskTotalCapacity'] - df['Census_SystemVolumeTotalCapacity']
    df['fe_ram_per_processor'] = df['Census_TotalPhysicalRAM']/ df['Census_ProcessorCoreCount']
    df['fe_physical_cores'] = df['Census_ProcessorCoreCount'] / 2
        
    print("Preparing ratios")
    
    #faster thanks to cpmp
    nrows = df.shape[0]
    df['fe_avsig_gamer_freq'] = df.groupby(['AvSigVersion','Wdft_IsGamer'])['OsBuild'].transform('count') / nrows
    df['fe_cpucores_region_freq'] = df.groupby(['Census_ProcessorCoreCount','Wdft_RegionIdentifier'])['OsBuild'].transform('count') / nrows
    df['fe_cpucores_oemname_freq'] = df.groupby(['Census_ProcessorCoreCount','Census_OEMNameIdentifier'])['OsBuild'].transform('count') / nrows
    df['fe_geoname_oemname_freq'] = df.groupby(['GeoNameIdentifier','Census_OEMNameIdentifier'])['OsBuild'].transform('count') / nrows
    df['fe_cntiden_oemname_freq'] = df.groupby(['CountryIdentifier','Census_OEMNameIdentifier'])['OsBuild'].transform('count') / nrows
    
    ##### testing feats
    df['fe_hghdec_cnt1'] = 0
    df.loc[df['CountryIdentifier'].isin([214,89,195,4,141,158,43,201,41,9,29,203,171,60,93,142,66,149,207,97,107,68,5,35,160]) == True, 'fe_hghdec_cnt1'] = 1

    df['fe_hghdec_cnt_2'] = 0
    df.loc[df['EngineVersion'].isin(['1.1.15100.1', '1.1.15200.1', '1.1.14600.4', '1.1.15000.2', '1.1.14901.4']) == True, 'fe_hghdec_cnt_2'] = 1

    df['fe_hghdec_cnt_3'] = 0
    df.loc[df['AppVersion'].isin(['4.18.1807.18075', '4.18.1806.18062', '4.12.16299.15', '4.10.209.0', '4.13.17134.1', '4.16.17656.18052']) == True, 'fe_hghdec_cnt_3'] = 1

    df['fe_hghdec_cnt_8'] = 0
    df.loc[df['Census_ProcessorModelIdentifier'].isin([2696.,1998.,2660.,2372.,1992.,2382.,2640.,2524.,1985.,2096.]) == True, 'fe_hghdec_cnt_8'] = 1

    df['fe_hghdec_cnt_9'] = 0
    df.loc[df['Census_OSInstallTypeName'].isin(['UUPUgrade','IBSClean', 'Update', 'Upgrade', 'Other','Reset']) == True, 'fe_hghdec_cnt_9'] = 1

    df['fe_hghdec_cnt_10'] = 0
    df.loc[df['Census_FirmwareManufacturerIdentifier'].isin([142.,628.,554.,355.,556.,500.,93.,807.,513.]) == True, 'fe_hghdec_cnt_10'] = 1
    
    df = reduce_mem_usage(df)
    gc.collect()
    
    return df

# In[ ]:


# In[ ]:


train = fe(train)
test = fe(test)


# In[ ]:


numerical_columns = list(train.select_dtypes(include=numerics).columns)
categorical_columns = categorical_columns + ['fe_EngineVersion_2', 'fe_AvSigVersion_minor', 'fe_AvSigVersion_build']
train, test = encode_categorical_columns(train, test, categorical_columns)
print(train.shape, test.shape)


# In[ ]:


train = reduce_mem_usage(train)
test = reduce_mem_usage(test)


# In[ ]:


numerical_columns.remove('HasDetections')
numerical_columns.remove('PuaMode')
numerical_columns.remove('DefaultBrowsersIdentifier')
remove = ['MachineIdentifier','Census_ChassisTypeName','Census_OSEdition','Census_OSArchitecture',
          'OsPlatformSubRelease','OsVer', 'Census_DeviceFamily']
for col in remove:categorical_columns.remove(col)

train = train[numerical_columns+categorical_columns]
test  = test[numerical_columns+categorical_columns]


# In[ ]:


col_vals_dict = {c: list(train[c].unique()) for c in categorical_columns if c not in ['MachineIdentifier', 'ProductName']}


# In[ ]:


nb_numeric   = len(train.columns) - len(col_vals_dict)
nb_categoric = len(col_vals_dict)
print('Number of Numerical features:', nb_numeric)
print('Number of Categorical features:', nb_categoric)


# ### Create the network
# In order to create our embedding model we need to have a look at the spatiality of the cat features. We choose here to use Embedding only on cat features that present more than 2 outcomes otherwise it is count as a numeric value (0 or 1).

# In[ ]:


embed_cols = []
len_embed_cols = []
for c in col_vals_dict:
    if len(col_vals_dict[c])>2:
        embed_cols.append(c)
        len_embed_cols.append((c, len(col_vals_dict[c])))
        print(c + ': %d values' % len(col_vals_dict[c])) #look at value counts to know the embedding dimensions
        
print('\n Number of embed features :', len(embed_cols))


# - We are including 24 features out of 33 categorical features into our Embedding.
# - **embedding size = min(50, number of categories/2)**

# In[ ]:


print(len_embed_cols)



# In[ ]:


def build_embedding_network(len_embed_cols):
    
    model_out = []
    model_in  = []
    
    for name, dim in len_embed_cols:
        input_dim = Input(shape=(1,), dtype='int32')
        embed_dim = Embedding(dim, dim//2 + 1, input_length=1, name= name)(input_dim) #1 for unknowns
        embed_dim = Dropout(0.2)(embed_dim)
        embed_dim = Reshape((dim//2 + 1,))(embed_dim)
        model_out.append(embed_dim)
        model_in.append(input_dim)
    
    input_num = Input(shape=(len(numerical_columns)+ len(categorical_columns) - len(len_embed_cols),) , dtype='float32')
    outputs = Concatenate(axis=1)([*model_out, input_num])
    
    outputs = (Dense(128))(outputs) 
    outputs = (Activation('relu'))(outputs)
    outputs = (Dropout(.2))(outputs)
    outputs = (Dense(32))(outputs) 
    outputs = (Activation('relu'))(outputs)
    outputs = (Dropout(.15))(outputs)
    outputs = (Dense(1))(outputs)
    outputs = (Activation('sigmoid'))(outputs)
    
    model = Model([*model_in, input_num], outputs)

    model.compile(loss='binary_crossentropy', optimizer='adam')
    
    return model


# In[ ]:


def preproc(X_train, X_val, X_test):

    input_list_train = []
    input_list_val = []
    input_list_test = []
    
    #the cols to be embedded: rescaling to range [0, # values)
    for c in embed_cols:
        raw_vals = np.unique(X_train[c])
        val_map = {}
        for i in range(len(raw_vals)):
            val_map[raw_vals[i]] = i       
        input_list_train.append(X_train[c].map(val_map).values)
        input_list_val.append(X_val[c].map(val_map).fillna(0).values)
        input_list_test.append(X_test[c].map(val_map).fillna(0).values)
        
    #the rest of the columns
    other_cols = [c for c in X_train.columns if (not c in embed_cols+['HasDetections', 'MachineIdentifier'])]
    input_list_train.append(X_train[other_cols].values)
    input_list_val.append(X_val[other_cols].values)
    input_list_test.append(X_test[other_cols].values)
    
    return input_list_train, input_list_val, input_list_test


# In[ ]:


# Impute missing values in order to scale
from sklearn.preprocessing import MinMaxScaler

train[numerical_columns] = train[numerical_columns].fillna(value = 0)
test[numerical_columns] = test[numerical_columns].fillna(value = 0)

# Fit the scaler only on train data
scaler = MinMaxScaler().fit(train[numerical_columns])
train.loc[:,numerical_columns] = scaler.transform(train[numerical_columns])
test.loc[:,numerical_columns] = scaler.transform(test[numerical_columns])


# In[ ]:


print ('neural network....')
from keras.layers import Input, Embedding, Dense, Flatten, Dropout, concatenate, Reshape
from keras.layers import BatchNormalization, SpatialDropout1D, Concatenate, Activation
from keras.callbacks import Callback, EarlyStopping
from keras.models import Model
from keras.optimizers import Adam


# In[ ]:


K = 5
runs_per_fold = 1
n_epochs = 10
patience = 5
models = []
cv_aucs   = []
full_val_preds = np.zeros(np.shape(train)[0])
y_preds = np.zeros((np.shape(test)[0], K))

kfold = StratifiedKFold(n_splits = K, shuffle = True, random_state=2**10)

for i, (f_ind, outf_ind) in enumerate(kfold.split(train, train_y)):

    train_f, X_val_f = train.loc[f_ind].copy(), train.loc[outf_ind].copy()
    train_y_f, y_val_f = train_y[f_ind], train_y[outf_ind]
    
    print('Shapes Are', train_f.shape, X_val_f.shape)
    
    test_f = test.copy()
    # Shuffle data
    idx = np.arange(len(train_f))
    np.random.shuffle(idx)
    train_f = train_f.iloc[idx]
    train_y_f = train_y_f.iloc[idx]

    #preprocessing

    print('Preprocessing........!!!')
    proc_train_f, proc_X_val_f, proc_test_f = preproc(train_f, X_val_f, test_f)
    
    #track oof prediction for cv scores
    val_preds = 0
    
    for j in range(runs_per_fold):
        
        print('Build_embedding_network........!!!')
        NN = build_embedding_network(len_embed_cols)
        
        # Set callback functions to early stop training and save the best model so far
        callbacks = [EarlyStopping(monitor='val_loss', patience=patience)]

        print(len(proc_train_f))
        
        NN.fit(proc_train_f, train_y_f.values, 
               epochs=n_epochs, batch_size= 2**15, 
               verbose=1,callbacks=callbacks,
               validation_data=(proc_X_val_f, y_val_f))
        
        del proc_train_f, train_y_f, callbacks
        gc.collect()
        print('OOF Val Predictions........!!!')
        val_preds += NN.predict(proc_X_val_f)[:,0] / runs_per_fold
        del proc_X_val_f
        gc.collect()

        print('Test Predictions........!!!')
        #need to fix the y_preds over batches...
        y_preds[:,i] += NN.predict(proc_test_f)[:,0] / runs_per_fold
        
        models.append(NN)
        
    full_val_preds[outf_ind] += val_preds
        
    cv_auc  = roc_auc_score(y_val_f.values, val_preds)
    cv_aucs.append(cv_auc)
    print ('\nFold %i prediction cv AUC: %.5f\n' %(i,cv_auc))
    
print('Mean out of fold AUC: %.5f' % np.mean(cv_auc))
print('Full validation AUC: %.5f' % roc_auc_score(train_y.values, full_val_preds))

print('Saving OOF Train.......')
np.save('train_oof_nn_with_embedddings.npy', full_val_preds)
print('Saved\n')

print('Saving Predictions.......')
np.save('test_nn_with_embedddings.npy', y_preds)
print('Saved\n')


# In[ ]:


print(models[0].summary())


# In[ ]:
print('Saving Embeddings....')
save_embeddings = True
saved_embeddings_fname = "embeddings.pickle"
all_emb = []
if save_embeddings:
    model_ = models[0]    
    for cols in embed_cols:
        emb = model_.get_layer(cols).get_weights()[0]
        all_emb.append(emb)
    
    with open(saved_embeddings_fname , 'wb') as f:
        pickle.dump(all_emb, f, -1)


# In[ ]:


from keras.models import load_model

print('Saving Models...')
models[0].save('my_model_0.h5')
models[1].save('my_model_1.h5')
models[3].save('my_model_3.h5')

del models  # deletes the existing models


# In[ ]:


print(embed_cols)


# In[ ]:


for idx, col in enumerate(train.columns):
    if col in embed_cols:
        print(idx, col)
        
##mapping is left for the User to Give it a Go On Purpose:)