17134.1.amd64fre.rs4_release.180410-1804

import os
from collections import defaultdict
from csv import DictReader
import math, time as t
from datetime import datetime
from datetime import datetime
from csv import DictReader

'''
# Binary Feats
```
    117 -----> 'fe_guy_didnt_update',
    118 -----> 'fst_public_ver_still_users',
    137 -----> 'fe_EngineVersion_2_less_13701',
    145 -----> 'fe_AvSigVersion_more_275',
    156 -----> 'fe_hghdec_cnt1',
    157 -----> 'fe_hghdec_cnt_3',
    158 -----> 'fe_hghdec_cnt_5',
    159 -----> 'fe_hghdec_cnt_8',
    160 -----> 'fe_hghdec_cnt_9',
    161 -----> 'fe_hghdec_cnt_10',
```
#    # Numerical Feats Name
    ```
    83 -----> 'fe_EngineVersion_factor',
    84 -----> 'fe_AppVersion_factor',
    114 -----> 'temp_x1',
    115 -----> 'temp_x2',
    116 -----> 'temp_diffs',
    120 -----> 'DateASYear',
    121 -----> 'DateASMonth',
    122 -----> 'DateASWeek',
    123 -----> 'DateASDay',
    124 -----> 'DateASDayofweek',
    125 -----> 'DateASDayofyear',
    126 -----> 'DateOSYear',
    127 -----> 'DateOSMonth',
    128 -----> 'DateOSWeek',
    129 -----> 'DateOSDay',
    130 -----> 'DateOSDayofweek',
    131 -----> 'DateOSDayofyear',
    132 -----> 'fe_aspect_ratio',
    133 -----> 'fe_ResolutionRatio',
    134 -----> 'fe_dpi',
    135 -----> 'fe_MegaPixels',
    138 -----> 'fe_one_less_AVproductInstalled',
    139 -----> 'diff_AV_Products',
    140 -----> 'sun_AV_Products',
    147 -----> 'fe_AvSigVersion_sum',
    148 -----> 'fe_AppVersion_sum',
    149 -----> 'fe_EngineVersion_sum',
    150 -----> 'fe_magic_4',
    151 -----> 'fe_primary_drive_c_ratio',
    152 -----> 'fe_Census_SystemVolumeTotalCapacity_GB',
    153 -----> 'fe_non_primary_drive_MB',
    154 -----> 'fe_ram_per_processor',
    155 -----> 'fe_physical_cores',
```
# GroupBy Feats
```
    90 -----> 'fe_avsig_org_freq',
    91 -----> 'fe_avsig_cty_freq',
    92 -----> 'fe_avsig_gamer_freq',
    93 -----> 'fe_cpucores_region_freq',
    94 -----> 'fe_cpucores_oemname_freq',
    95 -----> 'fe_geoname_oemname_freq',
    96 -----> 'fe_cntiden_oemname_freq',
    97 -----> 'fe_cty_oemname_freq',
    98 -----> 'fe_orgs_oemname_freq',
    99 -----> 'fe_avsig_org_touch_freq',
    100 -----> 'fe_avsig_cty_touch_freq',
    101 -----> 'fe_avsig_gamer_touch_freq',
    102 -----> 'fe_cpucores_region_touch_freq',
    103 -----> 'fe_cpucores_oemname_touch_freq',
    104 -----> 'fe_geoname_oemname_touch_freq',
    105 -----> 'fe_cntiden_oemname_touch_freq',
    106 -----> 'fe_cty_oemname_touch_freq',
    107 -----> 'fe_orgs_oemname_touch_freq',
    108 -----> 'fe_cpucores_oemname_bridenti_freq',
    109 -----> 'fe_geoname_oemname_bridenti_freq',
    110 -----> 'fe_cntiden_oemname_bridenti_freq',
    111 -----> 'fe_cty_oemname_bridenti_freq',
    112 -----> 'fe_orgs_oemname_bridenti_freq',
    119 -----> 'fe_orgs_cnt_cty_ring_freq',
```
# Count Feats
  ```
    85 -----> cnt_cnt_org_os
    86 -----> cnt_cnt_org_city_os_x
    87 -----> cnt_cnt_org_city_os_y
    88 -----> cnt_cnt_org_build_osbld_type
    89 -----> cnt_cnt_org_city_frwar
    162 -----> fe_count_Census_FirmwareManufacturerIdentifier
    163 -----> fe_count_Census_OSBuildNumber
    164 -----> fe_count_Census_OSBuildRevision
    165 -----> fe_count_Census_OEMModelIdentifier
    166 -----> fe_count_Wdft_IsGamer
    167 -----> fe_count_CountryIdentifier
    168 -----> fe_count_OrganizationIdentifier
    169 -----> fe_count_Census_FirmwareVersionIdentifier
    170 -----> fe_count_Wdft_RegionIdentifier
  ```
'''

binary_feats = ['fe_guy_didnt_update','fst_public_ver_still_users','fe_EngineVersion_2_less_13701','fe_AvSigVersion_more_275',
'fe_hghdec_cnt1','fe_hghdec_cnt_3','fe_hghdec_cnt_5','fe_hghdec_cnt_8','fe_hghdec_cnt_9','fe_hghdec_cnt_10',]

num_feats = ['fe_EngineVersion_factor','fe_AppVersion_factor','temp_x1','temp_x2','temp_diffs','DateASYear','DateASMonth','DateASWeek',
'DateASDay','DateASDayofweek','DateASDayofyear','DateOSYear','DateOSMonth','DateOSWeek','DateOSDay','DateOSDayofweek','DateOSDayofyear',
'fe_aspect_ratio','fe_ResolutionRatio','fe_dpi','fe_MegaPixels','fe_one_less_AVproductInstalled','diff_AV_Products','sun_AV_Products','fe_AvSigVersion_sum',
'fe_AppVersion_sum','fe_EngineVersion_sum','fe_magic_4','fe_primary_drive_c_ratio','fe_Census_SystemVolumeTotalCapacity_GB',
'fe_non_primary_drive_MB','fe_ram_per_processor','fe_physical_cores',]


grp_feats=['fe_avsig_org_freq','fe_avsig_cty_freq','fe_avsig_gamer_freq','fe_cpucores_region_freq','fe_cpucores_oemname_freq',
'fe_geoname_oemname_freq','fe_cntiden_oemname_freq','fe_cty_oemname_freq','fe_orgs_oemname_freq','fe_avsig_org_touch_freq',
'fe_avsig_cty_touch_freq','fe_avsig_gamer_touch_freq','fe_cpucores_region_touch_freq','fe_cpucores_oemname_touch_freq',
'fe_geoname_oemname_touch_freq','fe_cntiden_oemname_touch_freq','fe_cty_oemname_touch_freq','fe_orgs_oemname_touch_freq',
'fe_cpucores_oemname_bridenti_freq','fe_geoname_oemname_bridenti_freq','fe_cntiden_oemname_bridenti_freq',
'fe_cty_oemname_bridenti_freq','fe_orgs_oemname_bridenti_freq','fe_orgs_cnt_cty_ring_freq',]

cnt_feats = [
'cnt_cnt_org_os','cnt_cnt_org_city_os_x',
'cnt_cnt_org_city_os_y','cnt_cnt_org_build_osbld_type','cnt_cnt_org_city_frwar','fe_count_Census_FirmwareManufacturerIdentifier',
'fe_count_Census_OSBuildNumber','fe_count_Census_OSBuildRevision','fe_count_Census_OEMModelIdentifier','fe_count_Wdft_IsGamer',
'fe_count_CountryIdentifier','fe_count_OrganizationIdentifier','fe_count_Census_FirmwareVersionIdentifier','fe_count_Wdft_RegionIdentifier',]

dtypes = {
        'MachineIdentifier':                                    'category',
        'ProductName':                                          'category',
        'EngineVersion':                                        'category',
        'AppVersion':                                           'category',
        'AvSigVersion':                                         'category',
        'IsBeta':                                               'int8',
        'RtpStateBitfield':                                     'float16',
        'IsSxsPassiveMode':                                     'int8',
        'DefaultBrowsersIdentifier':                            'float16',
        'AVProductStatesIdentifier':                            'float32',
        'AVProductsInstalled':                                  'float16',
        'AVProductsEnabled':                                    'float16',
        'HasTpm':                                               'int8',
        'CountryIdentifier':                                    'int16',
        'CityIdentifier':                                       'float32',
        'OrganizationIdentifier':                               'float16',
        'GeoNameIdentifier':                                    'float16',
        'LocaleEnglishNameIdentifier':                          'int8',
        'Platform':                                             'category',
        'Processor':                                            'category',
        'OsVer':                                                'category',
        'OsBuild':                                              'int16',
        'OsSuite':                                              'int16',
        'OsPlatformSubRelease':                                 'category',
        'OsBuildLab':                                           'category',
        'SkuEdition':                                           'category',
        'IsProtected':                                          'float16',
        'AutoSampleOptIn':                                      'int8',
        'PuaMode':                                              'category',
        'SMode':                                                'float16',
        'IeVerIdentifier':                                      'float16',
        'SmartScreen':                                          'category',
        'Firewall':                                             'float16',
        'UacLuaenable':                                         'float32',
        'Census_MDC2FormFactor':                                'category',
        'Census_DeviceFamily':                                  'category',
        'Census_OEMNameIdentifier':                             'float16',
        'Census_OEMModelIdentifier':                            'float32',
        'Census_ProcessorCoreCount':                            'float16',
        'Census_ProcessorManufacturerIdentifier':               'float16',
        'Census_ProcessorModelIdentifier':                      'float16',
        'Census_ProcessorClass':                                'category',
        'Census_PrimaryDiskTotalCapacity':                      'float32',
        'Census_PrimaryDiskTypeName':                           'category',
        'Census_SystemVolumeTotalCapacity':                     'float32',
        'Census_HasOpticalDiskDrive':                           'int8',
        'Census_TotalPhysicalRAM':                              'float32',
        'Census_ChassisTypeName':                               'category',
        'Census_InternalPrimaryDiagonalDisplaySizeInInches':    'float16',
        'Census_InternalPrimaryDisplayResolutionHorizontal':    'float16',
        'Census_InternalPrimaryDisplayResolutionVertical':      'float16',
        'Census_PowerPlatformRoleName':                         'category',
        'Census_InternalBatteryType':                           'category',
        'Census_InternalBatteryNumberOfCharges':                'float32',
        'Census_OSVersion':                                     'category',
        'Census_OSArchitecture':                                'category',
        'Census_OSBranch':                                      'category',
        'Census_OSBuildNumber':                                 'int16',
        'Census_OSBuildRevision':                               'int32',
        'Census_OSEdition':                                     'category',
        'Census_OSSkuName':                                     'category',
        'Census_OSInstallTypeName':                             'category',
        'Census_OSInstallLanguageIdentifier':                   'float16',
        'Census_OSUILocaleIdentifier':                          'int16',
        'Census_OSWUAutoUpdateOptionsName':                     'category',
        'Census_IsPortableOperatingSystem':                     'int8',
        'Census_GenuineStateName':                              'category',
        'Census_ActivationChannel':                             'category',
        'Census_IsFlightingInternal':                           'float16',
        'Census_IsFlightsDisabled':                             'float16',
        'Census_FlightRing':                                    'category',
        'Census_ThresholdOptIn':                                'float16',
        'Census_FirmwareManufacturerIdentifier':                'float16',
        'Census_FirmwareVersionIdentifier':                     'float32',
        'Census_IsSecureBootEnabled':                           'int8',
        'Census_IsWIMBootEnabled':                              'float16',
        'Census_IsVirtualDevice':                               'float16',
        'Census_IsTouchEnabled':                                'int8',
        'Census_IsPenCapable':                                  'int8',
        'Census_IsAlwaysOnAlwaysConnectedCapable':              'float16',
        'Wdft_IsGamer':                                         'float16',
        'Wdft_RegionIdentifier':                                'float16'
        }

dont_use = ['MachineIdentifier', 'Census_FirmwareVersionIdentifier', 'Census_OEMModelIdentifier', 'CityIdentifier']

num_cols = [
    'Census_ProcessorCoreCount',
    'Census_PrimaryDiskTotalCapacity',
    'Census_SystemVolumeTotalCapacity',
    'Census_TotalPhysicalRAM',
    'Census_InternalPrimaryDiagonalDisplaySizeInInches',
    'Census_InternalPrimaryDisplayResolutionHorizontal',
    'Census_InternalPrimaryDisplayResolutionVertical',
    'Census_InternalBatteryNumberOfCharges']

too_many_vals = ["Census_PrimaryDiskTotalCapacity","Census_SystemVolumeTotalCapacity",
                "Census_TotalPhysicalRAM", 
                "Census_InternalPrimaryDiagonalDisplaySizeInInches",
                 "Census_InternalPrimaryDisplayResolutionHorizontal", 
                 "Census_InternalPrimaryDisplayResolutionVertical",
                 "Census_InternalBatteryNumberOfCharges"
                 ]

categories = [k for k, v in dtypes.items() if k not in dont_use]
categories_index = dict(zip(categories, range(len(categories))))
field_features = defaultdict()

def csv_to_vw(loc_csv, file_name, train=True):

    start = datetime.now()
    print('Converting .csv to .vw')
    
    cnt, imp = 0, 1
    print("\nTurning %s into %s Is_train_set? %s"%(loc_csv, file_name, train))

    with open(file_name, 'a') as the_file:

        for t, row in enumerate(DictReader(open(train_path))):

            if t % 1e5 == 0:
              print("%s\t%s"%(t, str(datetime.now() - start)))

            numerical_feats = ""
            cnt_feats = ""
            boolean_feats = ""
            categorical_feats = ""

            for field in categories:
                if field == 'HasDetections':
                    continue
                feature = row[field].strip()
                if feature == '':
                    feature = 'unk'
                if field not in num_cols:
                    feature = feature.replace(':','').replace( "|", "" )
                    categorical_feats += " %s" % (feature)
                else:
                    if feature == "unk" or float(feature) == -1:
                        categorical_feats += " %s" % (str(0))
                    else:
                        if field in too_many_vals:
                            numerical_feats += " %s:%s" % (field, (int(round(math.log(1 + float(feature))))))
                        else:
                            numerical_feats += " %s:%s" % (field, (int(float(feature))))

            if train:
                if row['HasDetections'] == "1":
                    label = 1
                    imp = 1
                elif row['HasDetections'] == "0":
                    label = -1
                    imp = 1 #str imp

                the_file.write( "%s %s |num %s |cat %s\n" % (label, imp, numerical_feats, categorical_feats))
            else:
                #test set
                label = "1"
                the_file.write( "%s  |num %s |cat %s\n" % (label, numerical_feats, categorical_feats))

#csv_to_vw(train_path, train=True)
train_path = './train/train.csv'
print(train_path)
csv_to_vw(train_path, file_name='train_2.vw', train=True)

train_path = './test/test.csv'
print(train_path)
csv_to_vw(train_path, file_name='test_2.vw', train=False)

'''
from kaggle Discussion's
vw  --compressed x0_f0_v6.vw.gz -c -k -l 0.1 -b 22 -q cc  -q nn -q cn --loss_function logistic --passes 8 --holdout_off -f model_f0.vw
'''

###################
def to_vw(loc_csv, file_name, categ_column_ids, binary_feats_ids, num_column_ids, cnt_columns_ids, grpby_column_ids,  column_names, train=True):
    """
    Converts a string to VW format.

    :param line: a string with comma-separated feature values, str
    :param categ_column_ids: ids of categorical features, list
    :param num_column_ids: ids of numeric features, list
    :param column_names: column (or feature) names to use (both categorical and numeric), list
    :param train: whether the line belongs to a training set
    :return: processed line, str
    """

    with open(file_name, 'a') as the_file:
        for t, line in enumerate(DictReader(open(train_path))):
            if t % 1e5 == 0:
              print("%s\t%s"%(t, str(datetime.now() - start)))

    values = line.strip().split(',')
    # VW treats '|' and ':' as special symbols, so jnust in case we'll replace them

    for i in range(len(values)):
        values[i] = values[i].replace('|', '').replace(':', '')
    label = '-1'

    if train:
        label, values = values[-1], values[:-1] #last , rest all row vals
        # in case of binary classification, VW eats labels 1 and -1, so 1 -> 1, 0 -> -1
        label = str(2 * int(label) - 1)
    
    # for categorical features, we fill in missing values with 'unk'
    for i in categ_column_ids:
        if not values[i]:
            values[i] = 'unk'
            
    # for numeric features, we fill in missing values with '-1'
    for i in num_column_ids:
        if values[i] == '':
            values[i] = '-1'

    for i in cnt_columns_ids:
        if values[i] == '':
            values[i] = '-1'

    for i in grpby_column_ids:
        if values[i] == '':
            values[i] = '-1'

    categ_vw = ' '.join(['{}={}'.format(column_names[i], values[i]) for i in categ_column_ids])
    # we apply log1p transformation to numeric features
    numeric_vw = ' '.join(['{}:{}'.format(column_names[i], round(math.log(1 + float(values[i]) + 1e-7))) for i in num_column_ids])
    
    binary_vw  = ' '.join(['{}:{}'.format(column_names[i], values[i]) for i in binary_feats_ids])
    cnt_vw     = ' '.join(['{}:{}'.format(column_names[i], values[i]) for i in cnt_columns_ids])
    grp_vw     = ' '.join(['{}:{}'.format(column_names[i], values[i]) for i in grpby_column_ids])

    new_line   = label + ' |num ' + numeric_vw + ' |cat ' + categ_vw + ' |bi ' + binary_vw + ' |cnt ' + cnt_vw + ' |grp '+ grp_vw
    return new_line

# def csv_to_vw(loc_csv, loc_output, train=True):
#   """
#   Munges a CSV file (loc_csv) to a VW file (loc_output). Set "train"
#   to False when munging a test set.
#   TODO: Too slow for a daily cron job. Try optimize, Pandas or Go.
#   """
#   start = datetime.now()
#   print("\nTurning %s into %s. Is_train_set? %s"%(loc_csv,loc_output,train))
#   cnt = 0
#   with open(loc_output,"wb") as outfile:
#     for e, row in enumerate( DictReader(open(loc_csv)) ):
    
#       #Creating the features
#       numerical_features = ""
#       categorical_features = ""
#       for k,v in row.items():
#         if k not in ["HasDetections","MachineIdentifier"]:
#           if "I" in k: # numerical feature, example: I5
#             if len(str(v)) > 0: #check for empty values
#               numerical_features += " %s:%s" % (k,(float(v)))
#           if "C" in k: # categorical feature, example: C2
#             if len(str(v)) > 0:
#               categorical_features += " %s" % v
              
#       #Creating the labels        
#       if train: #we care about labels
#         if row['HasDetections'] == "1":
#           label = 1
#         else:
#           label = -1 #we set negative label to -1
#         outfile.write( "%s '%s |i%s |c%s\n" % (label,row['MachineIdentifier'],numerical_features,categorical_features) )
        
#       else: #we dont care about labels
#         outfile.write( "1 '%s |i%s |c%s\n" % (row['MachineIdentifier'],numerical_features,categorical_features) )
      
#       #Reporting progress
#       if e % 1e6 == 0:
#         print("%s\t%s"%(e, str(datetime.now() - start)))
#         cnt += 1
#         if cnt %2 ==0:
#             exit()

#   print("\n %s Task execution time:\n\t%s"%(e, str(datetime.now() - start)))

# csv_to_vw(train_path, "click.train.vw",train=True)
# csv_to_vw('./test/test.csv', "click.test.vw",train=False)
