import sys
import re
import os
import codecs
import unicodedata

# This script will clean up the author transliterations
# the synonym file generated by the /batch/command=dump-authors
# process; it removes duplicates and also deals with some 
# obvious data problems
#
# usage:
# python clean_transliterations.py synonyms.input synonyms.output

blacklist = [unicode(x) for x in ['cern', 'heidelberg', 'hamburg', 'desy', 'muenchen', 'footnote',
             'present', 'institute', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9']]
def admissible_entry(key):
    k = key
    for x in blacklist:
        if x in k:
            return False
    parts = filter(lambda x: len(x) > 2, k.replace(u',', '').split())
    if k.startswith(u','):
        return False
    if len(parts) == 0:
        return False
    return True

def run(input, output):
    
    out = {}
    
    fi = codecs.open(input, 'r', 'UTF-8')
    c = 0
    for line in fi: # line will be unicode
        c += 1
        parts = line.strip().split('=>')
        key = parts[0].lower()
        
        key = unicodedata.normalize('NFKD', key).encode('ascii','ignore')
        
        if (not admissible_entry(key)):
            continue
        
        if (u',' not in key):
            key = key + u','
        
        if key not in out:
            out[key] = set()
        
        pointer = out[key]
        
        for x in parts[1].split(u';'):
            if u',' not in x:
                x = x + u','
            pointer.add(x.lower())
    
    result = []    
    for k,v in out.items():
        result.append(u'%s=>%s' % (k.replace(u',', u'\\,').replace(u' ', u'\\ '), 
                                   u','.join([x.replace(u',', u'\\,').replace(u' ', u'\\ ') for x in v])))
    
    result.sort()
    
    r = 0
    fo = codecs.open(output, 'w', 'UTF-8')
    for x in result:
        r += 1
        fo.write(x)
        fo.write('\n')
    fo.close()
    fi.close()
    
    print 'Done: input was %s lines, output was: %s lines' % (c, r)



if __name__ == "__main__":
    if len(sys.argv) < 3:
        print 'usage: python %s synonyms.input synonyms.ouput' % os.path.basename(sys.argv[0])
        exit(0)
    run(sys.argv[1], sys.argv[2])