# - Add more feats like how many people updated to a particular AvSig, OSBuild, Version on a particulara day, month, etc...
# - interactions feats
# - which iare insider builds, which are not... 
# - regional malware statistics 
# - company wise statistsics of malware, how updted, verdions of avsig, os, 
# - insider build or not etc..

# coding: utf-8

##libraries

from numba import jit
import numpy as np 
import pandas as pd 
from datetime import datetime as dt
import os
import seaborn as sns 
import matplotlib.pyplot as plt
plt.style.use('ggplot')
import lightgbm as lgb
import xgboost as xgb
import time
import datetime
from tqdm import tqdm_notebook as tqdm
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import StratifiedKFold, KFold, TimeSeriesSplit
from sklearn.metrics import mean_squared_error, roc_auc_score
from sklearn import metrics
from itertools import combinations
import gc

import warnings
warnings.filterwarnings("ignore")

import logging
import os
from functools import wraps
from timeit import default_timer as timer
logger = logging.getLogger("ms-malware")

def setup_logs(save_file):
  ## initialize logger
  logger = logging.getLogger("ms-malware")
  logger.setLevel(logging.INFO)

  ## create the logging file handler
  fh = logging.FileHandler(save_file)

  ## create the logging console handler
  ch = logging.StreamHandler()

  ## format
  formatter = logging.Formatter("%(asctime)s - %(message)s")
  fh.setFormatter(formatter)
  ch.setFormatter(formatter)

  ## add handlers to logger object
  logger.addHandler(fh)
  logger.addHandler(ch)

  return logger

def logspeed(f):
  @wraps(f)
  def wrapper(*args, **kwargs):
    logger = logging.getLogger("ms-malware")
    start = timer()
    result = f(*args, **kwargs)
    end = timer()
    logger.info(f'{f.__name__} - elapsed time: {end-start:.4f} seconds')
    return result
  return wrapper


pd.set_option('max_colwidth', 500)
pd.set_option('max_columns', 400)
pd.set_option('max_rows', 100)

import os
print(os.listdir("./"))

datedictAS = np.load('./AvSigVersionTimestamps.npy')[()]

# In[ ]:


#https://www.kaggle.com/theoviel/load-the-totality-of-the-data
dtypes = {
        'MachineIdentifier':                                    'category',
        'ProductName':                                          'category',
        'EngineVersion':                                        'category',
        'AppVersion':                                           'category',
        'AvSigVersion':                                         'category',
        'IsBeta':                                               'int8',
        'RtpStateBitfield':                                     'float16',
        'IsSxsPassiveMode':                                     'int8',
        'DefaultBrowsersIdentifier':                            'float16',
        'AVProductStatesIdentifier':                            'float32',
        'AVProductsInstalled':                                  'float16',
        'AVProductsEnabled':                                    'float16',
        'HasTpm':                                               'int8',
        'CountryIdentifier':                                    'int16',
        'CityIdentifier':                                       'float32',
        'OrganizationIdentifier':                               'float16',
        'GeoNameIdentifier':                                    'float16',
        'LocaleEnglishNameIdentifier':                          'int8',
        'Platform':                                             'category',
        'Processor':                                            'category',
        'OsVer':                                                'category',
        'OsBuild':                                              'int16',
        'OsSuite':                                              'int16',
        'OsPlatformSubRelease':                                 'category',
        'OsBuildLab':                                           'category',
        'SkuEdition':                                           'category',
        'IsProtected':                                          'float16',
        'AutoSampleOptIn':                                      'int8',
        'PuaMode':                                              'category',
        'SMode':                                                'float16',
        'IeVerIdentifier':                                      'float16',
        'SmartScreen':                                          'category',
        'Firewall':                                             'float16',
        'UacLuaenable':                                         'float32',
        'Census_MDC2FormFactor':                                'category',
        'Census_DeviceFamily':                                  'category',
        'Census_OEMNameIdentifier':                             'float16',
        'Census_OEMModelIdentifier':                            'float32',
        'Census_ProcessorCoreCount':                            'float16',
        'Census_ProcessorManufacturerIdentifier':               'float16',
        'Census_ProcessorModelIdentifier':                      'float16',
        'Census_ProcessorClass':                                'category',
        'Census_PrimaryDiskTotalCapacity':                      'float32',
        'Census_PrimaryDiskTypeName':                           'category',
        'Census_SystemVolumeTotalCapacity':                     'float32',
        'Census_HasOpticalDiskDrive':                           'int8',
        'Census_TotalPhysicalRAM':                              'float32',
        'Census_ChassisTypeName':                               'category',
        'Census_InternalPrimaryDiagonalDisplaySizeInInches':    'float16',
        'Census_InternalPrimaryDisplayResolutionHorizontal':    'float16',
        'Census_InternalPrimaryDisplayResolutionVertical':      'float16',
        'Census_PowerPlatformRoleName':                         'category',
        'Census_InternalBatteryType':                           'category',
        'Census_InternalBatteryNumberOfCharges':                'float32',
        'Census_OSVersion':                                     'category',
        'Census_OSArchitecture':                                'category',
        'Census_OSBranch':                                      'category',
        'Census_OSBuildNumber':                                 'int16',
        'Census_OSBuildRevision':                               'int32',
        'Census_OSEdition':                                     'category',
        'Census_OSSkuName':                                     'category',
        'Census_OSInstallTypeName':                             'category',
        'Census_OSInstallLanguageIdentifier':                   'float16',
        'Census_OSUILocaleIdentifier':                          'int16',
        'Census_OSWUAutoUpdateOptionsName':                     'category',
        'Census_IsPortableOperatingSystem':                     'int8',
        'Census_GenuineStateName':                              'category',
        'Census_ActivationChannel':                             'category',
        'Census_IsFlightingInternal':                           'float16',
        'Census_IsFlightsDisabled':                             'float16',
        'Census_FlightRing':                                    'category',
        'Census_ThresholdOptIn':                                'float16',
        'Census_FirmwareManufacturerIdentifier':                'float16',
        'Census_FirmwareVersionIdentifier':                     'float32',
        'Census_IsSecureBootEnabled':                           'int8',
        'Census_IsWIMBootEnabled':                              'float16',
        'Census_IsVirtualDevice':                               'float16',
        'Census_IsTouchEnabled':                                'int8',
        'Census_IsPenCapable':                                  'int8',
        'Census_IsAlwaysOnAlwaysConnectedCapable':              'float16',
        'Wdft_IsGamer':                                         'float16',
        'Wdft_RegionIdentifier':                                'float16',
        'HasDetections':                                        'int8'
        }

def reduce_mem_usage(df, verbose=True):
    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
    start_mem = df.memory_usage(deep=True).sum() / 1024**2    
    for col in df.columns:
        col_type = df[col].dtypes
        if col_type in numerics:
            c_min = df[col].min()
            c_max = df[col].max()
            if str(col_type)[:3] == 'int':
                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:
                    df[col] = df[col].astype(np.int8)
                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:
                    df[col] = df[col].astype(np.int16)
                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:
                    df[col] = df[col].astype(np.int32)
                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:
                    df[col] = df[col].astype(np.int64)  
            else:
                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:
                    df[col] = df[col].astype(np.float16)
                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:
                    df[col] = df[col].astype(np.float32)
                else:
                    df[col] = df[col].astype(np.float64)    
    end_mem = df.memory_usage(deep=True).sum() / 1024**2
    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))
    print('Thanks!!')
    return df

def add_count(df, c):
    new_col = 'fe_count_'+ c
    df[new_col] = df.groupby(c)[c].transform('count')

# In[ ]:


numerics = ['int8', 'int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numerical_columns = [c for c,v in dtypes.items() if v in numerics]
categorical_columns = [c for c,v in dtypes.items() if v not in numerics]

cnt_feats = [
 'AvSigVersion',
 'EngineVersion',
 'AppVersion',
 'Processor',
 'IsProtected',
 'Census_FirmwareManufacturerIdentifier',
 'Census_FirmwareVersionIdentifier',
 'Census_ProcessorCoreCount',
 'Census_TotalPhysicalRAM',
 'Census_OSVersion',
 'Census_OSArchitecture',
 'Census_OSBranch',
 'Census_OSBuildNumber',
 'Census_OSBuildRevision',
 'Census_IsTouchEnabled',
 'Census_OEMModelIdentifier',
 'Wdft_IsGamer',
 'Firewall',
 'CountryIdentifier',
 'OrganizationIdentifier',
 'GeoNameIdentifier',
]


# In[ ]:


train = pd.read_csv('../input/train.csv', dtype=dtypes)


# In[ ]:

random_state = 15
number_of_folds = 5
stop_after_fold_number = 4 ##CHANGE THIS ACCORDINGLY
shuffle = True
nthread = 6 #CHANGE THIS ACCORDINGLY
num_rounds = 3000 # CHANGE THIS AS WELL


train_y = train['HasDetections']

test = pd.read_csv('../input/test.csv', dtype=dtypes)

# ##need to fix these
# train['SmartScreen'] = train['SmartScreen'].cat.add_categories(['unknown'])
# train['SmartScreen'].fillna('unknown',inplace=True)
# train['SmartScreen_bucket'] = train['SmartScreen'].replace({'&#x01;':'unknown', '&#x02;':'unknown', '&#x03;':'unknown', '0':'off', '00000000':'off', 'Block':'unknown', 'BLOCK':'unknown', 'Deny':'off', 'Enabled':'on',
# 'ExistsNotSet':'off', 'of':'off', 'Off':'off', 'off':'off', 'OFF':'off', 'On':'on', 'on':'on', 'ON':'on', 'Promprt':'unsure',
# 'Prompt':'unsure', 'prompt':'unsure', 'Promt':'unsure', 'RequireAdmin':'unsure', 'requireadmin':'unsure', 'requireAdmin':'unsure', 'RequiredAdmin':'unsure',
# 'Warn':'unsure', 'warn':'unsure'})

# test['SmartScreen'] = test['SmartScreen'].cat.add_categories(['unknown'])
# test['SmartScreen'].fillna('unknown',inplace=True)
# test['SmartScreen_bucket'] = test['SmartScreen'].replace({'&#x01;':'unknown', '&#x02;':'unknown', '&#x03;':'unknown', '0':'off', '00000000':'off', 'Block':'unknown', 'BLOCK':'unknown', 'Deny':'off', 'Enabled':'on',
# 'ExistsNotSet':'off', 'of':'off', 'Off':'off', 'off':'off', 'OFF':'off', 'On':'on', 'on':'on', 'ON':'on', 'Promprt':'unsure',
# 'Prompt':'unsure', 'prompt':'unsure', 'Promt':'unsure', 'RequireAdmin':'unsure', 'requireadmin':'unsure', 'requireAdmin':'unsure', 'RequiredAdmin':'unsure',
# 'Warn':'unsure', 'warn':'unsure'})

# train['SmartScreen_bucket'] = train['SmartScreen']

#trivial groupby feats....

train = reduce_mem_usage(train)
test = reduce_mem_usage(test)
gc.collect()

# BL timestamp
def convert(x):
    try:
        d = datetime.strptime(x.split('.')[4],'%y%m%d-%H%M')
    except:
        d = np.nan
    return d

def add_datepart(df, fldname, drop=True, time=False):
    "Helper function that adds columns relevant to a date in the column `fldname` of `df`."
    import re
    fld = df[fldname]
    fld_dtype = fld.dtype
    if isinstance(fld_dtype, pd.core.dtypes.dtypes.DatetimeTZDtype):
        fld_dtype = np.datetime64

    if not np.issubdtype(fld_dtype, np.datetime64):
        df[fldname] = fld = pd.to_datetime(fld, infer_datetime_format=True)
    targ_pre = re.sub('[Dd]ate$', '', fldname)
    attr = ['Year', 'Month', 'Week', 'Day', 'Dayofweek', 'Dayofyear',
            'Is_month_end', 'Is_month_start', 'Is_quarter_end', 'Is_quarter_start', 'Is_year_end', 'Is_year_start']
    if time: attr = attr + ['Hour']
    for n in attr: 
        df[targ_pre + n] = getattr(fld.dt, n.lower())
        df[targ_pre + n] = df[targ_pre + n]*1
    df = reduce_mem_usage(df,False)
    if drop: df.drop(fldname, axis=1, inplace=True)

@jit
def fast_auc(y_true, y_prob):
    y_true = np.asarray(y_true)
    y_true = y_true[np.argsort(y_prob)]
    nfalse = 0
    auc = 0
    n = len(y_true)
    for i in range(n):
        y_i = y_true[i]
        nfalse += (1 - y_i)
        auc += y_i * nfalse
    auc /= (nfalse * (n - nfalse))
    return auc

def eval_auc(preds, dtrain):
    labels = dtrain.get_label()
    return 'auc', fast_auc(labels, preds), True

def encode_categorical_columns(x_train, x_test, columns, sort=True):
    train_length = x_train.shape[0]
    for col in tqdm(columns):
        if col == 'MachineIdentifier' or col == 'HasDetections':
            continue
            
        combined_data = pd.concat([x_train[col], x_test[col]])
        combined_data, _ = pd.factorize(combined_data, sort=sort)
        combined_data = pd.Series(combined_data).astype('int32')
        x_train[col] = combined_data.iloc[:train_length].values
        x_test[col] = combined_data.iloc[train_length:].values
        x_train[col] = x_train[col].fillna(0)
        x_test[col] = x_test[col].fillna(0)
        del combined_data
        gc.collect()
        
    return x_train, x_test

def rename_edition(x):
    x = x.lower()
    if 'core' in x:
        return 'Core'
    elif 'pro' in x:
        return 'pro'
    elif 'enterprise' in x:
        return 'Enterprise'
    elif 'server' in x:
        return 'Server'
    elif 'home' in x:
        return 'Home'
    elif 'education' in x:
        return 'Education'
    elif 'cloud' in x:
        return 'Cloud'
    else:
        return x

def fe(df):

    gc.collect()
    print('Cooking Pointless Things....')

    ##grouby stats
    print('grouping combination...')

    #############################################################################################
    gp = df[['CountryIdentifier','OrganizationIdentifier', 'Census_OSInstallTypeName']].groupby(by=['CountryIdentifier','OrganizationIdentifier'], sort=False)[['Census_OSInstallTypeName']].count().reset_index().rename(columns={'Census_OSInstallTypeName':'cnt_cnt_org_os'})
    df = df.merge(gp, on=['CountryIdentifier','OrganizationIdentifier'], how='left')
    del gp
    gc.collect()

    gp = df[['CountryIdentifier','OrganizationIdentifier','CityIdentifier', 'Census_OSInstallTypeName']].groupby(['CountryIdentifier','OrganizationIdentifier', 'CityIdentifier'])[['Census_OSInstallTypeName']].count().reset_index().rename(columns={'Census_OSInstallTypeName':'cnt_cnt_org_city_os'})
    df = df.merge(gp, on=['CountryIdentifier','OrganizationIdentifier','CityIdentifier'], how='left')
    del gp
    gc.collect()


    gp = df[['CountryIdentifier','OrganizationIdentifier','CityIdentifier', 'Census_OSInstallTypeName']].groupby(['CountryIdentifier','OrganizationIdentifier', 'CityIdentifier'])[['Census_OSInstallTypeName']].count().reset_index().rename(columns={'Census_OSInstallTypeName':'cnt_cnt_org_city_os'})
    df = df.merge(gp, on=['CountryIdentifier','OrganizationIdentifier','CityIdentifier'], how='left')
    del gp
    gc.collect()

    gp = df[['CountryIdentifier','OrganizationIdentifier','Census_OSBuildNumber','OsBuildLab', 'Census_OSInstallTypeName']].groupby(['CountryIdentifier','OrganizationIdentifier', 'Census_OSBuildNumber', 'OsBuildLab'], sort=False)[['Census_OSInstallTypeName']].count().reset_index().rename(columns={'Census_OSInstallTypeName':'cnt_cnt_org_build_oslab_type'})
    df = df.merge(gp, on=['CountryIdentifier','OrganizationIdentifier', 'Census_OSBuildNumber','OsBuildLab'], how='left')
    del gp
    gc.collect()
    print(df.shape)
    ##########################################
    
    gp = df[['CountryIdentifier','OrganizationIdentifier','CityIdentifier', 'Census_FirmwareManufacturerIdentifier']].groupby(['CountryIdentifier','OrganizationIdentifier', 'CityIdentifier'])[['Census_FirmwareManufacturerIdentifier']].count().reset_index().rename(columns={'Census_FirmwareManufacturerIdentifier':'cnt_cnt_org_city_frwar'})
    df = df.merge(gp, on=['CountryIdentifier','OrganizationIdentifier','CityIdentifier'], how='left')
    del gp

    gp = df[['CountryIdentifier','OrganizationIdentifier', 'Census_FirmwareManufacturerIdentifier']].groupby(['CountryIdentifier','OrganizationIdentifier'])[['Census_FirmwareManufacturerIdentifier']].count().reset_index().rename(columns={'Census_FirmwareManufacturerIdentifier':'cnt_cnt_org_frwar'})
    df = df.merge(gp, on=['CountryIdentifier','OrganizationIdentifier'], how='left')
    del gp

    gp = df[['CountryIdentifier','OrganizationIdentifier','AvSigVersion', 'Census_FirmwareManufacturerIdentifier']].groupby(['CountryIdentifier','OrganizationIdentifier', 'AvSigversion'])[['Census_FirmwareManufacturerIdentifier']].count().reset_index().rename(columns={'Census_FirmwareManufacturerIdentifier':'cnt_cnt_org_avsig_frwar'})
    df = df.merge(gp, on=['CountryIdentifier','OrganizationIdentifier','AvSigversion'], how='left')
    del gp
    gc.collect()

    print(df.shape)
    #########################################################################################
    gp = df[['CountryIdentifier','OrganizationIdentifier','CityIdentifier', 'Census_FirmwareVersionIdentifier']].groupby(['CountryIdentifier','OrganizationIdentifier', 'CityIdentifier'])[['Census_FirmwareVersionIdentifier']].count().reset_index().rename(columns={'Census_FirmwareVersionIdentifier':'cnt_cnt_org_city_frwar_ver'})
    df = df.merge(gp, on=['CountryIdentifier','OrganizationIdentifier','CityIdentifier'], how='left')
    del gp

    gp = df[['CountryIdentifier','OrganizationIdentifier', 'Census_FirmwareVersionIdentifier']].groupby(['CountryIdentifier','OrganizationIdentifier'])[['Census_FirmwareVersionIdentifier']].count().reset_index().rename(columns={'Census_FirmwareVersionIdentifier':'cnt_cnt_org_frwar_ver'})
    df = df.merge(gp, on=['CountryIdentifier','OrganizationIdentifier'], how='left')
    del gp

    gp = df[['CountryIdentifier','OrganizationIdentifier','AvSigVersion', 'Census_FirmwareVersionIdentifier']].groupby(['CountryIdentifier','OrganizationIdentifier', 'AvSigversion'])[['Census_FirmwareVersionIdentifier']].count().reset_index().rename(columns={'Census_FirmwareVersionIdentifier':'cnt_cnt_org_avsig_frwar_ver'})
    df = df.merge(gp, on=['CountryIdentifier','OrganizationIdentifier','AvSigversion'], how='left')
    del gp
    gc.collect()

    ############################################

    gp = df[['CountryIdentifier','OrganizationIdentifier','CityIdentifier', 'OsBuildLab']].groupby(['CountryIdentifier','OrganizationIdentifier', 'CityIdentifier'])[['OsBuildLab']].count().reset_index().rename(columns={'OsBuildLab':'cnt_cnt_org_city_osbuild'})
    df = df.merge(gp, on=['CountryIdentifier','OrganizationIdentifier','CityIdentifier'], how='left')
    del gp
    gc.collect()
    print(df.shape)
    ##############################################################3

    df['DateBL'] = df['OsBuildLab'].map(convert)
    df['DateAS'] = df['AvSigVersion'].map(datedictAS)

    df['diff_dates'] = (df['DateAS'] - df['DateBL']).dt.days
    df['old_sig'] = (df['diff_dates']<0)*1
    print(df.shape)

    add_datepart(df,'DateAS', time=True)
    print(df.shape)

    df.drop('DateBL', inplace=True, axis=1)

    print('Added TS......')

    prnt('Display Sections..')

    df['fe_aspect_ratio'] = df['Census_InternalPrimaryDisplayResolutionHorizontal']/ df['Census_InternalPrimaryDisplayResolutionVertical']
    df['ResolutionRatio'] = df['Census_InternalPrimaryDisplayResolutionVertical'] / df['Census_InternalPrimaryDisplayResolutionHorizontal']
    df['fe_monitor_dims'] = df['Census_InternalPrimaryDisplayResolutionHorizontal'].astype(str) + '*' + df['Census_InternalPrimaryDisplayResolutionVertical'].astype('str')
	df['fe_monitor_dims'] = df['fe_monitor_dims'].astype('category')
	
	df['Census_InternalPrimaryDisplayResolutionHorizontal'] = df['Census_InternalPrimaryDisplayResolutionHorizontal'].astype(np.float64)
	df['Census_InternalPrimaryDisplayResolutionVertical'] = df['Census_InternalPrimaryDisplayResolutionVertical'].astype(np.float64)
	
	df['fe_dpi'] = ((df['Census_InternalPrimaryDisplayResolutionHorizontal']**2 + df['Census_InternalPrimaryDisplayResolutionVertical']**2)**.5)/(df['Census_InternalPrimaryDiagonalDisplaySizeInInches'])
	df['fe_MegaPixels'] = (train['Census_InternalPrimaryDisplayResolutionHorizontal'] * df['Census_InternalPrimaryDisplayResolutionVertical'])/1e6

	print(df.shape)

	#public
	print('Screen quality...')

	df['SD'] = (df['Census_InternalPrimaryDisplayResolutionVertical'] < 720).astype('uint8')
    df['HD'] = (df['Census_InternalPrimaryDisplayResolutionVertical'].isin(range(720,1080))).astype('int8')
    df['FullHD'] = (df['Census_InternalPrimaryDisplayResolutionVertical'].isin(range(1080,2160))).astype('int8')
    df['4k'] = (df['Census_InternalPrimaryDisplayResolutionVertical'] >= 2160).astype('uint8')

    print('Random feats..')

    df['fe_EngineVersion_2'] = df['EngineVersion'].apply(lambda x: x.split('.')[2]).astype('category')
    df['fe_one_less_AVproductInstalled'] = df['AVProductsInstalled'] - 1
     
    df['fe_OsBuild_exact'] = df['OsBuildLab'].apply(lambda x: x.split('.')[0] +'.'+ x.split('.')[1])
    df['fe_OsBuild_exact'] = df['fe_OsBuild_exact'].astype('category')

    df['fe_AvSigVersion_minor'] = df['AvSigVersion'].apply(lambda x: x.split('.')[1]).astype('category')
    df['fe_AvSigVersion_build'] = df['AvSigVersion'].apply(lambda x: x.split('.')[2]).astype('category')
    df['fe_AvSigVersion_minor_build'] = df['AvSigVersion'].str.replace('1.2&#x17;3.1144.0','1.273.1144.0').apply(lambda x: float((x.split('.')[1]) +'.'+(x.split('.')[2]))).astype('float32')
    df['fe_AvSigVersion_sum'] = df['AvSigVersion'].str.replace('1.2&#x17;3.1144.0','1.273.1144.0').apply(lambda x: float(x.split('.')[1]) + float(x.split('.')[2])).astype(int).values
    df['AvSigVersion'] = df['AvSigVersion'].astype('category')
    
    print(df.shape)

    df = reduce_mem_usage(df)
    
    top_20 = df['AVProductStatesIdentifier'].value_counts(dropna=False, normalize=True).cumsum().index[:20]
    df['fe_magic_4'] = 0
    df.loc[df['AVProductStatesIdentifier'].isin(top_20) == True, 'fe_magic_4'] = 1
    del top_20
    gc.collect()

    print("Preparing ratios")

    df['fe_primary_drive_c_ratio'] = df['Census_SystemVolumeTotalCapacity']/ df['Census_PrimaryDiskTotalCapacity']
    df['fe_Census_SystemVolumeTotalCapacity_GB'] = df['Census_SystemVolumeTotalCapacity']/1024.
    df['fe_non_primary_drive_MB'] = df['Census_PrimaryDiskTotalCapacity'] - df['Census_SystemVolumeTotalCapacity']
    df['fe_ram_per_processor'] = df['Census_TotalPhysicalRAM']/ df['Census_ProcessorCoreCount']
    df['fe_physical_cores'] = df['Census_ProcessorCoreCount'] / 2
    
    #faster thanks to cpmp
    nrows = df.shape[0]
    df['fe_avsig_gamer_freq'] = df.groupby(['AvSigVersion','Wdft_IsGamer'])['OsBuild'].transform('count') / nrows
    df['fe_cpucores_region_freq'] = df.groupby(['Census_ProcessorCoreCount','Wdft_RegionIdentifier'])['OsBuild'].transform('count') / nrows
    df['fe_cpucores_oemname_freq'] = df.groupby(['Census_ProcessorCoreCount','Census_OEMNameIdentifier'])['OsBuild'].transform('count') / nrows
    df['fe_geoname_oemname_freq'] = df.groupby(['GeoNameIdentifier','Census_OEMNameIdentifier'])['OsBuild'].transform('count') / nrows
    df['fe_cntiden_oemname_freq'] = df.groupby(['CountryIdentifier','Census_OEMNameIdentifier'])['OsBuild'].transform('count') / nrows
    df = reduce_mem_usage(df)

    print(df.shape)
    ##### testing feats

    print('Binary Feats')

    df['fe_hghdec_cnt1'] = 0
    df.loc[df['CountryIdentifier'].isin([214,89,195,4,141,158,43,201,41,9,29,203,171,60,93,142,66,149,207,97,107,68,5,35,160]) == True, 'fe_hghdec_cnt1'] = 1

    # df['fe_hghdec_cnt_2'] = 0
    # df.loc[df['EngineVersion'].isin(['1.1.15100.1', '1.1.15200.1', '1.1.14600.4', '1.1.15000.2', '1.1.14901.4']) == True, 'fe_hghdec_cnt_2'] = 1

    df['fe_hghdec_cnt_3'] = 0
    df.loc[df['AppVersion'].isin(['4.18.1807.18075', '4.18.1806.18062', '4.12.16299.15', '4.10.209.0', '4.13.17134.1', '4.16.17656.18052']) == True, 'fe_hghdec_cnt_3'] = 1

    # df['fe_hghdec_cnt_4'] = 0
    # df.loc[df['EngineVersion'].isin(['1.1.15100.1', '1.1.15200.1', '1.1.14600.4', '1.1.15000.2', '1.1.14901.4']) == True, 'fe_hghdec_cnt_4'] = 1

    df['fe_hghdec_cnt_5'] = 0
    df.loc[df['CityIdentifier'].isin([130775.0,16668,82373.0,10222.0,61668.0,143782.0,66202.0,58607.0,66953.0]) == True, 'fe_hghdec_cnt_5'] = 1

    # df['fe_hghdec_cnt_6'] = 0
    # df.loc[df['OrganizationIdentifier'].isin([np.nan, 27.0,18.0,48.0,50.,11.,37.,49.,46.,14.]) == True, 'fe_hghdec_cnt_6'] = 1
    df = reduce_mem_usage(df)

    # df['fe_hghdec_cnt_7'] = 0
    # df.loc[df['Census_OEMModelIdentifier'].isin([313586.,242491.,317701.,317708.,228975.,188345.,24582.,241876.,24755.,np.nan]) == True, 'fe_hghdec_cnt_7'] = 1

    df['fe_hghdec_cnt_8'] = 0
    df.loc[df['Census_ProcessorModelIdentifier'].isin([2696.,1998.,2660.,2372.,1992.,2382.,2640.,2524.,1985.,2096.]) == True, 'fe_hghdec_cnt_8'] = 1

    df['fe_hghdec_cnt_9'] = 0
    df.loc[df['Census_OSInstallTypeName'].isin(['UUPUgrade','IBSClean', 'Update', 'Upgrade', 'Other','Reset']) == True, 'fe_hghdec_cnt_9'] = 1

    df['fe_hghdec_cnt_10'] = 0
    df.loc[df['Census_FirmwareManufacturerIdentifier'].isin([142.,628.,554.,355.,556.,500.,93.,807.,513.]) == True, 'fe_hghdec_cnt_10'] = 1

    del nrows

    # adding count features
    for col in cnt_feats:
    	add_count(df, col)

    df = reduce_mem_usage(df)

    ###fixing processing temp now; need to revisit the datasets few cols here...

    print(df.shape)

    df['Census_OSEdition'] = df['Census_OSEdition'].astype(str)
    df['Census_OSEdition'] = df['Census_OSEdition'].apply(rename_edition)
    df['Census_OSEdition'] = df['Census_OSEdition'].astype('category')

    df['Census_OSSkuName'] = df['Census_OSSkuName'].astype(str)
    df['Census_OSSkuName'] = df['Census_OSSkuName'].apply(rename_edition)
    df['Census_OSSkuName'] = df['Census_OSSkuName'].astype('category')

    print('Done...!!!')
    gc.collect()

    bad_cols = ['ProductName','IsBeta','RtpStateBitfield','IsSxsPassiveMode','DefaultBrowsersIdentifier',
	'AVProductsEnabled','HasTpm',
	'Platform','OsVer',	'AutoSampleOptIn',
	'PuaMode','Firewall','UacLuaenable','Census_DeviceFamily',
	'Census_ProcessorClass','Census_IsPortableOperatingSystem',
	'Census_IsFlightsDisabled','Census_IsVirtualDevice','Census_IsPenCapable'
	]
    return df

# In[ ]:

train = fe(train)
test = fe(test)

# In[ ]:

categorical_columns = categorical_columns + ['fe_EngineVersion_2', 'fe_OsBuild_exact', 'fe_AvSigVersion_minor', 'fe_AvSigVersion_build', 'fe_monitor_dims']
train, test = encode_categorical_columns(train, test, categorical_columns)
print(train.shape, test.shape)
train = reduce_mem_usage(train)
test = reduce_mem_usage(test)


# # FEATURE ENGINEER - WEEK
# first = datetime.datetime(2016,1,1); datedict2 = {}
# for x in datedict: datedict2[x] = (datedict[x]-first).days//7
# train['Week_2016'] = train['AvSigVersion'].map(datedict2)
# test['Week_2016'] = test['AvSigVersion'].map(datedict2)

# # FEATURE ENGINEER - WEEK
# first = datetime.datetime(2017,1,1); datedict2 = {}
# for x in datedict: datedict2[x] = (datedict[x]-first).days//7
# train['Week_2017'] = train['AvSigVersion'].map(datedict2)
# test['Week_2017'] = test['AvSigVersion'].map(datedict2)

# # FEATURE ENGINEER - WEEK
# first = datetime.datetime(2018,1,1); datedict2 = {}
# for x in datedict: datedict2[x] = (datedict[x]-first).days//7
# train['Week_2018'] = train['AvSigVersion'].map(datedict2)
# test['Week_2018'] = test['AvSigVersion'].map(datedict2)

# del first, datedict, datedict2
# gc.collect()

categorical_columns = [col for col in train.columns if col not in ['MachineIdentifier', 'Census_SystemVolumeTotalCapacity', 'HasDetections'] and str(train[col].dtype) == 'category']
print(len(categorical_columns))

# In[ ]:


# idea from this kernel: https://www.kaggle.com/fabiendaniel/detecting-malwares-with-lgbm

def predict_chunk(model, test):
    initial_idx = 0
    chunk_size = 3000000
    current_pred = np.zeros(len(test))
    while initial_idx < test.shape[0]:
        final_idx = min(initial_idx + chunk_size, test.shape[0])
        idx = range(initial_idx, final_idx)
        current_pred[idx] = model.predict(test.iloc[idx], num_iteration=model.best_iteration)
        initial_idx = final_idx
    return current_pred


# In[ ]:

def train_model(x, y, lgb_params, 
                number_of_folds=5, 
                evaluation_metric='auc', 
                save_feature_importances=True, 
                early_stopping_rounds=100, 
                num_round = 50,
                identifier_columns=['MachineIdentifier'],
                stop_after_fold_number=None):


    cross_validator = StratifiedKFold(n_splits=number_of_folds,
                                  random_state=random_state,
                                  shuffle=shuffle)
    
    validation_scores = []
    classifier_models = []

    feature_importance_df = pd.DataFrame()
    
    for fold_index, (train_index, validation_index) in enumerate(cross_validator.split(x, y)):
        x_train, x_validation = x.iloc[train_index], x.iloc[validation_index]
        y_train, y_validation = y.iloc[train_index], y.iloc[validation_index]
    
        x_train.drop(identifier_columns, axis=1, inplace=True)
        validation_identifier_data = x_validation[identifier_columns]
        x_validation.drop(identifier_columns, axis=1, inplace=True)
        x_train_columns = x_train.columns
        trn_data = lgb.Dataset(x_train,
                       label=y_train,
                       # categorical_feature=categorical_columns
                       )
        del x_train
        del y_train

        gc.collect()

        val_data = lgb.Dataset(x_validation,
                               label=y_validation,
                               # categorical_feature=categorical_columns
                               )
        classifier_model = lgb.train(lgb_params,
                                     trn_data,
                                     num_round,
                                     valid_sets=[trn_data, val_data],
                                     verbose_eval=100,
                                     early_stopping_rounds=early_stopping_rounds,
                                     feval=eval_auc
                                     )

        classifier_models.append(classifier_model)
        
        predictions = classifier_model.predict(x_validation, num_iteration=classifier_model.best_iteration)
        false_positive_rate, recall, thresholds = metrics.roc_curve(y_validation, predictions)
        score = metrics.auc(false_positive_rate, recall)
        validation_scores.append(score)
        
        fold_importance_df = pd.DataFrame()
        fold_importance_df["feature"] = x_train_columns
        fold_importance_df["importance"] = classifier_model.feature_importance(importance_type='gain')
        fold_importance_df["fold"] = fold_index + 1
        feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)

        if stop_after_fold_number == fold_index + 1:
            break

    if save_feature_importances:
        cols = (feature_importance_df[["feature", "importance"]]
                .groupby("feature")
                .mean()
                .sort_values(by="importance", ascending=False)[:1000].index)

        best_features = feature_importance_df.loc[feature_importance_df.feature.isin(cols)]

        plt.figure(figsize=(14, 25))
        sns.barplot(x="importance",
                    y="feature",
                    data=best_features.sort_values(by="importance",
                                                   ascending=False))
        plt.title('LightGBM Features (avg over folds)')
        plt.tight_layout()
        plt.savefig('lgbm_importances.png')

        best_features.sort_values(by="importance", ascending=False).groupby("feature").mean().sort_values(by="importance", ascending=False)             .to_csv('feature_importances_new.csv', index=True)

    score = sum(validation_scores) / len(validation_scores)
    return classifier_models, score


# In[ ]:

base_params = {   
        'boosting_type': 'gbdt',
        'objective': 'binary',
        'nthread': nthread,
        'learning_rate': 0.1,
        'max_depth': 10,
        'num_leaves': 100,
        'min_data_in_leaf': 60,
        'sub_feature': 0.8,
        'sub_row':0.8,
        'bagging_freq': 5,
        'lambda_l1': 2.,
        'lambda_l2': 2.,
        'random_state': random_state
        }

########################################################################

models, validation_score = train_model(train.drop('HasDetections', axis=1),
                                      train_y, base_params,
                                      num_round= num_rounds,
                                      stop_after_fold_number= stop_after_fold_number,
                                      save_feature_importances= True)

print(validation_score)

del train
gc.collect()

# In[ ]:


submission_data = test[['MachineIdentifier']]
predictions = np.zeros(len(test))
test = test.drop('MachineIdentifier', axis=1)
chunk_size = 2000000
for classifier_model in tqdm(models):
    current_pred = np.zeros(len(test))
    initial_idx = 0
    while initial_idx < test.shape[0]:
        final_idx = min(initial_idx + chunk_size, test.shape[0])
        idx = range(initial_idx, final_idx)
        current_pred[idx] = classifier_model.predict(test.iloc[idx],
                                                     num_iteration=classifier_model.best_iteration)
        initial_idx = final_idx

    predictions += current_pred / len(models)

del test

print('Thanks Sanyam, Rishi and Neutrino !!!!!')
# In[ ]:

submission_data['HasDetections'] = predictions
filename = 'submission_{:.6f}_{}_folds_{}_data.csv'.format(validation_score,
                                                              dt.now().strftime('%Y-%m-%d-%H-%M'),
                                                              len(models))
submission_data.to_csv('single_{}'.format(filename), index=False)