- Add more feats like how many people updated to a particular AvSig, OSBuild, Version on a particulara day, month, etc...
- interactions feats
- which iare insider builds, which are not... 
- regional malware statistics 
- company wise statistsics of malware, how updted, verdions of avsig, os, 
- insider build or not etc..

# coding: utf-8

##libraries

from numba import jit
import numpy as np 
import pandas as pd 
from datetime import datetime as dt
import os
import seaborn as sns 
import matplotlib.pyplot as plt
plt.style.use('ggplot')
import lightgbm as lgb
import xgboost as xgb
import time
import datetime
from tqdm import tqdm_notebook as tqdm
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import StratifiedKFold, KFold, TimeSeriesSplit
from sklearn.metrics import mean_squared_error, roc_auc_score
from sklearn import metrics
from itertools import combinations
import gc

import warnings
warnings.filterwarnings("ignore")

import logging
import os
from functools import wraps
from timeit import default_timer as timer
logger = logging.getLogger("ms-malware")

def setup_logs(save_file):
  ## initialize logger
  logger = logging.getLogger("ms-malware")
  logger.setLevel(logging.INFO)

  ## create the logging file handler
  fh = logging.FileHandler(save_file)

  ## create the logging console handler
  ch = logging.StreamHandler()

  ## format
  formatter = logging.Formatter("%(asctime)s - %(message)s")
  fh.setFormatter(formatter)
  ch.setFormatter(formatter)

  ## add handlers to logger object
  logger.addHandler(fh)
  logger.addHandler(ch)

  return logger

def logspeed(f):
  @wraps(f)
  def wrapper(*args, **kwargs):
    logger = logging.getLogger("ms-malware")
    start = timer()
    result = f(*args, **kwargs)
    end = timer()
    logger.info(f'{f.__name__} - elapsed time: {end-start:.4f} seconds')
    return result
  return wrapper


pd.set_option('max_colwidth', 500)
pd.set_option('max_columns', 400)
pd.set_option('max_rows', 100)

import os
print(os.listdir("./"))

# datedict = np.load(r'./AvSigVersionTimestamps.npy')
# datedict = datedict[()]

# In[ ]:


#https://www.kaggle.com/theoviel/load-the-totality-of-the-data
dtypes = {
        'MachineIdentifier':                                    'category',
        'ProductName':                                          'category',
        'EngineVersion':                                        'category',
        'AppVersion':                                           'category',
        'AvSigVersion':                                         'category',
        'IsBeta':                                               'int8',
        'RtpStateBitfield':                                     'float16',
        'IsSxsPassiveMode':                                     'int8',
        'DefaultBrowsersIdentifier':                            'float16',
        'AVProductStatesIdentifier':                            'float32',
        'AVProductsInstalled':                                  'float16',
        'AVProductsEnabled':                                    'float16',
        'HasTpm':                                               'int8',
        'CountryIdentifier':                                    'int16',
        'CityIdentifier':                                       'float32',
        'OrganizationIdentifier':                               'float16',
        'GeoNameIdentifier':                                    'float16',
        'LocaleEnglishNameIdentifier':                          'int8',
        'Platform':                                             'category',
        'Processor':                                            'category',
        'OsVer':                                                'category',
        'OsBuild':                                              'int16',
        'OsSuite':                                              'int16',
        'OsPlatformSubRelease':                                 'category',
        'OsBuildLab':                                           'category',
        'SkuEdition':                                           'category',
        'IsProtected':                                          'float16',
        'AutoSampleOptIn':                                      'int8',
        'PuaMode':                                              'category',
        'SMode':                                                'float16',
        'IeVerIdentifier':                                      'float16',
        'SmartScreen':                                          'category',
        'Firewall':                                             'float16',
        'UacLuaenable':                                         'float32',
        'Census_MDC2FormFactor':                                'category',
        'Census_DeviceFamily':                                  'category',
        'Census_OEMNameIdentifier':                             'float16',
        'Census_OEMModelIdentifier':                            'float32',
        'Census_ProcessorCoreCount':                            'float16',
        'Census_ProcessorManufacturerIdentifier':               'float16',
        'Census_ProcessorModelIdentifier':                      'float16',
        'Census_ProcessorClass':                                'category',
        'Census_PrimaryDiskTotalCapacity':                      'float32',
        'Census_PrimaryDiskTypeName':                           'category',
        'Census_SystemVolumeTotalCapacity':                     'float32',
        'Census_HasOpticalDiskDrive':                           'int8',
        'Census_TotalPhysicalRAM':                              'float32',
        'Census_ChassisTypeName':                               'category',
        'Census_InternalPrimaryDiagonalDisplaySizeInInches':    'float16',
        'Census_InternalPrimaryDisplayResolutionHorizontal':    'float16',
        'Census_InternalPrimaryDisplayResolutionVertical':      'float16',
        'Census_PowerPlatformRoleName':                         'category',
        'Census_InternalBatteryType':                           'category',
        'Census_InternalBatteryNumberOfCharges':                'float32',
        'Census_OSVersion':                                     'category',
        'Census_OSArchitecture':                                'category',
        'Census_OSBranch':                                      'category',
        'Census_OSBuildNumber':                                 'int16',
        'Census_OSBuildRevision':                               'int32',
        'Census_OSEdition':                                     'category',
        'Census_OSSkuName':                                     'category',
        'Census_OSInstallTypeName':                             'category',
        'Census_OSInstallLanguageIdentifier':                   'float16',
        'Census_OSUILocaleIdentifier':                          'int16',
        'Census_OSWUAutoUpdateOptionsName':                     'category',
        'Census_IsPortableOperatingSystem':                     'int8',
        'Census_GenuineStateName':                              'category',
        'Census_ActivationChannel':                             'category',
        'Census_IsFlightingInternal':                           'float16',
        'Census_IsFlightsDisabled':                             'float16',
        'Census_FlightRing':                                    'category',
        'Census_ThresholdOptIn':                                'float16',
        'Census_FirmwareManufacturerIdentifier':                'float16',
        'Census_FirmwareVersionIdentifier':                     'float32',
        'Census_IsSecureBootEnabled':                           'int8',
        'Census_IsWIMBootEnabled':                              'float16',
        'Census_IsVirtualDevice':                               'float16',
        'Census_IsTouchEnabled':                                'int8',
        'Census_IsPenCapable':                                  'int8',
        'Census_IsAlwaysOnAlwaysConnectedCapable':              'float16',
        'Wdft_IsGamer':                                         'float16',
        'Wdft_RegionIdentifier':                                'float16',
        'HasDetections':                                        'int8'
        }

def reduce_mem_usage(df, verbose=True):
    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
    start_mem = df.memory_usage(deep=True).sum() / 1024**2    
    for col in df.columns:
        col_type = df[col].dtypes
        if col_type in numerics:
            c_min = df[col].min()
            c_max = df[col].max()
            if str(col_type)[:3] == 'int':
                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:
                    df[col] = df[col].astype(np.int8)
                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:
                    df[col] = df[col].astype(np.int16)
                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:
                    df[col] = df[col].astype(np.int32)
                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:
                    df[col] = df[col].astype(np.int64)  
            else:
                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:
                    df[col] = df[col].astype(np.float16)
                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:
                    df[col] = df[col].astype(np.float32)
                else:
                    df[col] = df[col].astype(np.float64)    
    end_mem = df.memory_usage(deep=True).sum() / 1024**2
    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))
    print('Thanks!!')
    return df

def add_count(df, c):
    new_col = 'fe_count_'+ c
    df[new_col] = df.groupby(c)[c].transform('count')

# In[ ]:


numerics = ['int8', 'int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numerical_columns = [c for c,v in dtypes.items() if v in numerics]
categorical_columns = [c for c,v in dtypes.items() if v not in numerics]

cnt_feats = [
 'AvSigVersion',
 'EngineVersion',
 'AppVersion',
 'Processor',
 'IsProtected',
 'Census_ProcessorCoreCount',
 'Census_TotalPhysicalRAM',
 'Census_OSVersion',
 'Census_OSBranch',
 'Census_OSBuildRevision',
 'Census_OEMModelIdentifier',
 'Wdft_IsGamer',
]


# In[ ]:


train = pd.read_csv('../input/train.csv', dtype=dtypes)


# In[ ]:

random_state = 15
number_of_folds = 5
stop_after_fold_number = 4 ##CHANGE THIS ACCORDINGLY
shuffle = True
nthread = 12 #CHANGE THIS ACCORDINGLY
num_rounds = 4500 # CHANGE THIS AS WELL


train_y = train['HasDetections']

test = pd.read_csv('../input/test.csv', dtype=dtypes)

# ##need to fix these
# train['SmartScreen'] = train['SmartScreen'].cat.add_categories(['unknown'])
# train['SmartScreen'].fillna('unknown',inplace=True)
# train['SmartScreen_bucket'] = train['SmartScreen'].replace({'&#x01;':'unknown', '&#x02;':'unknown', '&#x03;':'unknown', '0':'off', '00000000':'off', 'Block':'unknown', 'BLOCK':'unknown', 'Deny':'off', 'Enabled':'on',
# 'ExistsNotSet':'off', 'of':'off', 'Off':'off', 'off':'off', 'OFF':'off', 'On':'on', 'on':'on', 'ON':'on', 'Promprt':'unsure',
# 'Prompt':'unsure', 'prompt':'unsure', 'Promt':'unsure', 'RequireAdmin':'unsure', 'requireadmin':'unsure', 'requireAdmin':'unsure', 'RequiredAdmin':'unsure',
# 'Warn':'unsure', 'warn':'unsure'})

# test['SmartScreen'] = test['SmartScreen'].cat.add_categories(['unknown'])
# test['SmartScreen'].fillna('unknown',inplace=True)
# test['SmartScreen_bucket'] = test['SmartScreen'].replace({'&#x01;':'unknown', '&#x02;':'unknown', '&#x03;':'unknown', '0':'off', '00000000':'off', 'Block':'unknown', 'BLOCK':'unknown', 'Deny':'off', 'Enabled':'on',
# 'ExistsNotSet':'off', 'of':'off', 'Off':'off', 'off':'off', 'OFF':'off', 'On':'on', 'on':'on', 'ON':'on', 'Promprt':'unsure',
# 'Prompt':'unsure', 'prompt':'unsure', 'Promt':'unsure', 'RequireAdmin':'unsure', 'requireadmin':'unsure', 'requireAdmin':'unsure', 'RequiredAdmin':'unsure',
# 'Warn':'unsure', 'warn':'unsure'})

# train['SmartScreen_bucket'] = train['SmartScreen']

#trivial groupby feats....

print('grouping combination...')
gp = train[['CountryIdentifier','OrganizationIdentifier', 'Census_OSInstallTypeName']].groupby(by=['CountryIdentifier','OrganizationIdentifier'], sort=False)[['Census_OSInstallTypeName']].count().reset_index().rename(columns={'Census_OSInstallTypeName':'cnt_cnt_org_os'})
train = train.merge(gp, on=['CountryIdentifier','OrganizationIdentifier'], how='left')
del gp
gc.collect()
print('grouping combination...')
gp = test[['CountryIdentifier','OrganizationIdentifier', 'Census_OSInstallTypeName']].groupby(by=['CountryIdentifier','OrganizationIdentifier'], sort=False)[['Census_OSInstallTypeName']].count().reset_index().rename(columns={'Census_OSInstallTypeName':'cnt_cnt_org_os'})
test = test.merge(gp, on=['CountryIdentifier','OrganizationIdentifier'], how='left')
del gp
train = reduce_mem_usage(train)
test = reduce_mem_usage(test)
gc.collect()

print('grouping combination...')
gp = train[['CountryIdentifier','OrganizationIdentifier','CityIdentifier', 'Census_OSInstallTypeName']].groupby(['CountryIdentifier','OrganizationIdentifier', 'CityIdentifier'])[['Census_OSInstallTypeName']].count().reset_index().rename(columns={'Census_OSInstallTypeName':'cnt_cnt_org_city_os'})
train = train.merge(gp, on=['CountryIdentifier','OrganizationIdentifier','CityIdentifier'], how='left')
del gp
gc.collect()
print('grouping combination...')
gp = test[['CountryIdentifier','OrganizationIdentifier','CityIdentifier', 'Census_OSInstallTypeName']].groupby(['CountryIdentifier','OrganizationIdentifier', 'CityIdentifier'])[['Census_OSInstallTypeName']].count().reset_index().rename(columns={'Census_OSInstallTypeName':'cnt_cnt_org_city_os'})
test = test.merge(gp, on=['CountryIdentifier','OrganizationIdentifier', 'CityIdentifier'], how='left')
del gp
train = reduce_mem_usage(train)
test = reduce_mem_usage(test)
gc.collect()

print('grouping combination...')
gp = train[['CountryIdentifier','OrganizationIdentifier','Census_OSBuildNumber', 'Census_OSInstallTypeName']].groupby(['CountryIdentifier','OrganizationIdentifier', 'Census_OSBuildNumber'], sort=False)[['Census_OSInstallTypeName']].count().reset_index().rename(columns={'Census_OSInstallTypeName':'cnt_cnt_org_build_type'})
train = train.merge(gp, on=['CountryIdentifier','OrganizationIdentifier', 'Census_OSBuildNumber'], how='left')
del gp
gc.collect()
print('grouping combination...')
gp = test[['CountryIdentifier','OrganizationIdentifier','Census_OSBuildNumber', 'Census_OSInstallTypeName']].groupby(['CountryIdentifier','OrganizationIdentifier', 'Census_OSBuildNumber'], sort=False)[['Census_OSInstallTypeName']].count().reset_index().rename(columns={'Census_OSInstallTypeName':'cnt_cnt_org_build_type'})
test = test.merge(gp, on=['CountryIdentifier','OrganizationIdentifier', 'Census_OSBuildNumber'], how='left')
del gp
train = reduce_mem_usage(train)
test = reduce_mem_usage(test)
gc.collect()


#display section
train['fe_aspect_ratio'] = train['Census_InternalPrimaryDisplayResolutionHorizontal']/ train['Census_InternalPrimaryDisplayResolutionVertical']
test['fe_aspect_ratio']  = test['Census_InternalPrimaryDisplayResolutionHorizontal']/ train['Census_InternalPrimaryDisplayResolutionVertical']

train['fe_monitor_dims'] = train['Census_InternalPrimaryDisplayResolutionHorizontal'].astype(str) + '*' + train['Census_InternalPrimaryDisplayResolutionVertical'].astype('str')
test['fe_monitor_dims']  = test['Census_InternalPrimaryDisplayResolutionHorizontal'].astype(str)  + '*' + test['Census_InternalPrimaryDisplayResolutionVertical'].astype('str')

train['fe_monitor_dims'] = train['fe_monitor_dims'].astype('category')
test['fe_monitor_dims']  = test['fe_monitor_dims'].astype('category')

train['Census_InternalPrimaryDisplayResolutionHorizontal'] = train['Census_InternalPrimaryDisplayResolutionHorizontal'].astype(np.float64)
test['Census_InternalPrimaryDisplayResolutionHorizontal']  = test['Census_InternalPrimaryDisplayResolutionHorizontal'].astype(np.float64)

train['Census_InternalPrimaryDisplayResolutionVertical'] = train['Census_InternalPrimaryDisplayResolutionVertical'].astype(np.float64)
test['Census_InternalPrimaryDisplayResolutionVertical']  = test['Census_InternalPrimaryDisplayResolutionVertical'].astype(np.float64)

train['fe_dpi'] = ((train['Census_InternalPrimaryDisplayResolutionHorizontal']**2 + train['Census_InternalPrimaryDisplayResolutionVertical']**2)**.5)/(train['Census_InternalPrimaryDiagonalDisplaySizeInInches'])
test['fe_dpi']  = ((test['Census_InternalPrimaryDisplayResolutionHorizontal']**2 + test['Census_InternalPrimaryDisplayResolutionVertical']**2)**.5)/(test['Census_InternalPrimaryDiagonalDisplaySizeInInches'])

train['fe_MegaPixels'] = (train['Census_InternalPrimaryDisplayResolutionHorizontal'] * train['Census_InternalPrimaryDisplayResolutionVertical'])/1e6
test['fe_MegaPixels']  = (test['Census_InternalPrimaryDisplayResolutionHorizontal'] * test['Census_InternalPrimaryDisplayResolutionVertical'])/1e6

print('Done Display Features\n')

@jit
def fast_auc(y_true, y_prob):
    y_true = np.asarray(y_true)
    y_true = y_true[np.argsort(y_prob)]
    nfalse = 0
    auc = 0
    n = len(y_true)
    for i in range(n):
        y_i = y_true[i]
        nfalse += (1 - y_i)
        auc += y_i * nfalse
    auc /= (nfalse * (n - nfalse))
    return auc

def eval_auc(preds, dtrain):
    labels = dtrain.get_label()
    return 'auc', fast_auc(labels, preds), True

def encode_categorical_columns(x_train, x_test, columns, sort=True):
    train_length = x_train.shape[0]
    for col in tqdm(columns):
        if col == 'MachineIdentifier' or col == 'HasDetections':
            continue
            
        combined_data = pd.concat([x_train[col], x_test[col]])
        combined_data, _ = pd.factorize(combined_data, sort=sort)
        combined_data = pd.Series(combined_data).astype('int32')
        x_train[col] = combined_data.iloc[:train_length].values
        x_test[col] = combined_data.iloc[train_length:].values
        x_train[col] = x_train[col].fillna(0)
        x_test[col] = x_test[col].fillna(0)
        del combined_data
        gc.collect()
        
    return x_train, x_test

def rename_edition(x):
    x = x.lower()
    if 'core' in x:
        return 'Core'
    elif 'pro' in x:
        return 'pro'
    elif 'enterprise' in x:
        return 'Enterprise'
    elif 'server' in x:
        return 'Server'
    elif 'home' in x:
        return 'Home'
    elif 'education' in x:
        return 'Education'
    elif 'cloud' in x:
        return 'Cloud'
    else:
        return x

def fe(df):

    print(gc.collect())
    print('Cooking Pointless Things....')

    df['fe_EngineVersion_2'] = df['EngineVersion'].apply(lambda x: x.split('.')[2]).astype('category')
    df['fe_one_less_AVproductInstalled'] = df['AVProductsInstalled'] - 1
     
    df['fe_OsBuild_exact'] = df['OsBuildLab'].apply(lambda x: x.split('.')[0] +'.'+ x.split('.')[1])
    df['fe_OsBuild_exact'] = df['fe_OsBuild_exact'].astype('category')

    df['fe_AvSigVersion_minor'] = df['AvSigVersion'].apply(lambda x: x.split('.')[1]).astype('category')
    df['fe_AvSigVersion_build'] = df['AvSigVersion'].apply(lambda x: x.split('.')[2]).astype('category')
    df['fe_AvSigVersion_minor_build'] = df['AvSigVersion'].str.replace('1.2&#x17;3.1144.0','1.273.1144.0').apply(lambda x: float((x.split('.')[1]) +'.'+(x.split('.')[2]))).astype('float32')
    df['fe_AvSigVersion_sum'] = df['AvSigVersion'].str.replace('1.2&#x17;3.1144.0','1.273.1144.0').apply(lambda x: float(x.split('.')[1]) + float(x.split('.')[2])).astype(int).values
    df['AvSigVersion'] = df['AvSigVersion'].astype('category')
    
    df = reduce_mem_usage(df)
    
    top_20 = df['AVProductStatesIdentifier'].value_counts(dropna=False, normalize=True).cumsum().index[:20]
    df['fe_magic_4'] = 0
    df.loc[df['AVProductStatesIdentifier'].isin(top_20) == True, 'fe_magic_4'] = 1
    del top_20
    
    gc.collect()
    
    df['fe_primary_drive_c_ratio'] = df['Census_SystemVolumeTotalCapacity']/ df['Census_PrimaryDiskTotalCapacity']
    df['fe_Census_SystemVolumeTotalCapacity_GB'] = df['Census_SystemVolumeTotalCapacity']/1024.
    df['fe_non_primary_drive_MB'] = df['Census_PrimaryDiskTotalCapacity'] - df['Census_SystemVolumeTotalCapacity']
    df['fe_ram_per_processor'] = df['Census_TotalPhysicalRAM']/ df['Census_ProcessorCoreCount']
    df['fe_physical_cores'] = df['Census_ProcessorCoreCount'] / 2
        
    print("Preparing ratios")
    
    #faster thanks to cpmp
    nrows = df.shape[0]
    df['fe_avsig_gamer_freq'] = df.groupby(['AvSigVersion','Wdft_IsGamer'])['OsBuild'].transform('count') / nrows
    df['fe_cpucores_region_freq'] = df.groupby(['Census_ProcessorCoreCount','Wdft_RegionIdentifier'])['OsBuild'].transform('count') / nrows
    df['fe_cpucores_oemname_freq'] = df.groupby(['Census_ProcessorCoreCount','Census_OEMNameIdentifier'])['OsBuild'].transform('count') / nrows
    df['fe_geoname_oemname_freq'] = df.groupby(['GeoNameIdentifier','Census_OEMNameIdentifier'])['OsBuild'].transform('count') / nrows
    df['fe_cntiden_oemname_freq'] = df.groupby(['CountryIdentifier','Census_OEMNameIdentifier'])['OsBuild'].transform('count') / nrows
    df = reduce_mem_usage(df)

    ##### testing feats
    df['fe_hghdec_cnt1'] = 0
    df.loc[df['CountryIdentifier'].isin([214,89,195,4,141,158,43,201,41,9,29,203,171,60,93,142,66,149,207,97,107,68,5,35,160]) == True, 'fe_hghdec_cnt1'] = 1

    df['fe_hghdec_cnt_3'] = 0
    df.loc[df['AppVersion'].isin(['4.18.1807.18075', '4.18.1806.18062', '4.12.16299.15', '4.10.209.0', '4.13.17134.1', '4.16.17656.18052']) == True, 'fe_hghdec_cnt_3'] = 1

    df['fe_hghdec_cnt_5'] = 0
    df.loc[df['CityIdentifier'].isin([130775.0,16668,82373.0,10222.0,61668.0,143782.0,66202.0,58607.0,66953.0]) == True, 'fe_hghdec_cnt_5'] = 1

    df = reduce_mem_usage(df)

    df['fe_hghdec_cnt_8'] = 0
    df.loc[df['Census_ProcessorModelIdentifier'].isin([2696.,1998.,2660.,2372.,1992.,2382.,2640.,2524.,1985.,2096.]) == True, 'fe_hghdec_cnt_8'] = 1

    df['fe_hghdec_cnt_10'] = 0
    df.loc[df['Census_FirmwareManufacturerIdentifier'].isin([142.,628.,554.,355.,556.,500.,93.,807.,513.]) == True, 'fe_hghdec_cnt_10'] = 1

    del nrows

    # adding count features
    for col in cnt_feats:
    	add_count(df, col)

    df = reduce_mem_usage(df)

    ###fixing processing temp now; need to revisit the datasets few cols here...


    df['Census_OSEdition'] = df['Census_OSEdition'].astype(str)
    df['Census_OSEdition'] = df['Census_OSEdition'].apply(rename_edition)
    df['Census_OSEdition'] = df['Census_OSEdition'].astype('category')

    df['Census_OSSkuName'] = df['Census_OSSkuName'].astype(str)
    df['Census_OSSkuName'] = df['Census_OSSkuName'].apply(rename_edition)
    df['Census_OSSkuName'] = df['Census_OSSkuName'].astype('category')

    print('Done...!!!')

    gc.collect()
    return df

# In[ ]:

train = fe(train)
test = fe(test)

# In[ ]:

categorical_columns = categorical_columns + ['fe_EngineVersion_2', 'fe_OsBuild_exact', 'fe_AvSigVersion_minor', 'fe_AvSigVersion_build', 'fe_monitor_dims']
train, test = encode_categorical_columns(train, test, categorical_columns)
print(train.shape, test.shape)
train = reduce_mem_usage(train)
test = reduce_mem_usage(test)


# # FEATURE ENGINEER - WEEK
# first = datetime.datetime(2016,1,1); datedict2 = {}
# for x in datedict: datedict2[x] = (datedict[x]-first).days//7
# train['Week_2016'] = train['AvSigVersion'].map(datedict2)
# test['Week_2016'] = test['AvSigVersion'].map(datedict2)

# # FEATURE ENGINEER - WEEK
# first = datetime.datetime(2017,1,1); datedict2 = {}
# for x in datedict: datedict2[x] = (datedict[x]-first).days//7
# train['Week_2017'] = train['AvSigVersion'].map(datedict2)
# test['Week_2017'] = test['AvSigVersion'].map(datedict2)

# # FEATURE ENGINEER - WEEK
# first = datetime.datetime(2018,1,1); datedict2 = {}
# for x in datedict: datedict2[x] = (datedict[x]-first).days//7
# train['Week_2018'] = train['AvSigVersion'].map(datedict2)
# test['Week_2018'] = test['AvSigVersion'].map(datedict2)

# del first, datedict, datedict2
# gc.collect()

categorical_columns = [col for col in train.columns if col not in ['MachineIdentifier', 'Census_SystemVolumeTotalCapacity', 'HasDetections'] and str(train[col].dtype) == 'category']
print(len(categorical_columns))

# In[ ]:


# idea from this kernel: https://www.kaggle.com/fabiendaniel/detecting-malwares-with-lgbm

def predict_chunk(model, test):
    initial_idx = 0
    chunk_size = 1000000
    current_pred = np.zeros(len(test))
    while initial_idx < test.shape[0]:
        final_idx = min(initial_idx + chunk_size, test.shape[0])
        idx = range(initial_idx, final_idx)
        current_pred[idx] = model.predict(test.iloc[idx], num_iteration=model.best_iteration)
        initial_idx = final_idx
    return current_pred


# In[ ]:

def train_model(x, y, lgb_params, 
                number_of_folds=5, 
                evaluation_metric='auc', 
                save_feature_importances=False, 
                early_stopping_rounds=50, 
                num_round = 50,
                identifier_columns=['MachineIdentifier'],
                stop_after_fold_number=None):


    cross_validator = StratifiedKFold(n_splits=number_of_folds,
                                  random_state=random_state,
                                  shuffle=shuffle)
    
    validation_scores = []
    classifier_models = []

    feature_importance_df = pd.DataFrame()
    
    for fold_index, (train_index, validation_index) in enumerate(cross_validator.split(x, y)):
        x_train, x_validation = x.iloc[train_index], x.iloc[validation_index]
        y_train, y_validation = y.iloc[train_index], y.iloc[validation_index]
    
        x_train.drop(identifier_columns, axis=1, inplace=True)
        validation_identifier_data = x_validation[identifier_columns]
        x_validation.drop(identifier_columns, axis=1, inplace=True)
        x_train_columns = x_train.columns
        trn_data = lgb.Dataset(x_train,
                       label=y_train,
                       # categorical_feature=categorical_columns
                       )
        del x_train
        del y_train

        gc.collect()

        val_data = lgb.Dataset(x_validation,
                               label=y_validation,
                               # categorical_feature=categorical_columns
                               )
        classifier_model = lgb.train(lgb_params,
                                     trn_data,
                                     num_round,
                                     valid_sets=[trn_data, val_data],
                                     verbose_eval=100,
                                     early_stopping_rounds=early_stopping_rounds,
                                     feval=eval_auc
                                     )

        classifier_models.append(classifier_model)
        
        predictions = classifier_model.predict(x_validation, num_iteration=classifier_model.best_iteration)
        false_positive_rate, recall, thresholds = metrics.roc_curve(y_validation, predictions)
        score = metrics.auc(false_positive_rate, recall)
        validation_scores.append(score)
        
        fold_importance_df = pd.DataFrame()
        fold_importance_df["feature"] = x_train_columns
        fold_importance_df["importance"] = classifier_model.feature_importance(importance_type='gain')
        fold_importance_df["fold"] = fold_index + 1
        feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)

        if stop_after_fold_number == fold_index + 1:
            break

    if save_feature_importances:
        cols = (feature_importance_df[["feature", "importance"]]
                .groupby("feature")
                .mean()
                .sort_values(by="importance", ascending=False)[:1000].index)

        best_features = feature_importance_df.loc[feature_importance_df.feature.isin(cols)]

        plt.figure(figsize=(14, 25))
        sns.barplot(x="importance",
                    y="feature",
                    data=best_features.sort_values(by="importance",
                                                   ascending=False))
        plt.title('LightGBM Features (avg over folds)')
        plt.tight_layout()
        plt.savefig('lgbm_importances.png')

        best_features.sort_values(by="importance", ascending=False).groupby("feature").mean().sort_values(by="importance", ascending=False)             .to_csv('feature_importances_new.csv', index=True)

    score = sum(validation_scores) / len(validation_scores)
    return classifier_models, score


# In[ ]:

base_params = {   
        'boosting_type': 'gbdt',
        'objective': 'binary',
        'nthread': nthread,
        'learning_rate': 0.1,
        'max_depth': 10,
        'num_leaves': 100,
        'min_data_in_leaf': 60,
        'sub_feature': 0.9,
        'sub_row':0.9,
        'bagging_freq': 5,
        'lambda_l1': 0.1,
        'lambda_l2': 0.1,
        'random_state': random_state
        }

###

models, validation_score = train_model(train.drop('HasDetections', axis=1),
                                      train_y, base_params,
                                      num_round= num_rounds,
                                      stop_after_fold_number= stop_after_fold_number,
                                      save_feature_importances= True)

print(validation_score)

del train
gc.collect()

# In[ ]:


submission_data = test[['MachineIdentifier']]
predictions = np.zeros(len(test))
test = test.drop('MachineIdentifier', axis=1)
chunk_size = 2000000
for classifier_model in tqdm(models):
    current_pred = np.zeros(len(test))
    initial_idx = 0
    while initial_idx < test.shape[0]:
        final_idx = min(initial_idx + chunk_size, test.shape[0])
        idx = range(initial_idx, final_idx)
        current_pred[idx] = classifier_model.predict(test.iloc[idx],
                                                     num_iteration=classifier_model.best_iteration)
        initial_idx = final_idx

    predictions += current_pred / len(models)

del test

print('Thanks Sanyam and Neutrino !!!!!')

submission_data['HasDetections'] = predictions
filename = 'submission_{:.6f}_{}_folds_{}_data.csv'.format(validation_score,
                                                              dt.now().strftime('%Y-%m-%d-%H-%M'),
                                                              len(models))
submission_data.to_csv('single_{}'.format(filename), index=False)